drivers/gpu/drm/tegra/drm.c

   1 /*
   2  * Copyright (C) 2012 Avionic Design GmbH
   3  * Copyright (C) 2012-2016 NVIDIA CORPORATION.  All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  */
   9
  10 #include <linux/bitops.h>
  11 #include <linux/host1x.h>
  12 #include <linux/idr.h>
  13 #include <linux/iommu.h>
  14
  15 #include <drm/drm_atomic.h>
  16 #include <drm/drm_atomic_helper.h>
  17
  18 #include "drm.h"
  19 #include "gem.h"
  20
  21 #define DRIVER_NAME "tegra"
  22 #define DRIVER_DESC "NVIDIA Tegra graphics"
  23 #define DRIVER_DATE "20120330"
  24 #define DRIVER_MAJOR 0
  25 #define DRIVER_MINOR 0
  26 #define DRIVER_PATCHLEVEL 0
  27
  28 #define CARVEOUT_SZ SZ_64M
  29 #define CDMA_GATHER_FETCHES_MAX_NB 16383
  30
  31 struct tegra_drm_file {
  32         struct idr contexts;
  33         struct mutex lock;
  34 };
  35
  36 static int tegra_atomic_check(struct drm_device *drm,
  37                               struct drm_atomic_state *state)
  38 {
  39         int err;
  40
  41         err = drm_atomic_helper_check_modeset(drm, state);
  42         if (err < 0)
  43                 return err;
  44
  45         err = tegra_display_hub_atomic_check(drm, state);
  46         if (err < 0)
  47                 return err;
  48
  49         err = drm_atomic_normalize_zpos(drm, state);
  50         if (err < 0)
  51                 return err;
  52
  53         err = drm_atomic_helper_check_planes(drm, state);
  54         if (err < 0)
  55                 return err;
  56
  57         if (state->legacy_cursor_update)
  58                 state->async_update = !drm_atomic_helper_async_check(drm, state);
  59
  60         return 0;
  61 }
  62
  63 static const struct drm_mode_config_funcs tegra_drm_mode_config_funcs = {
  64         .fb_create = tegra_fb_create,
  65 #ifdef CONFIG_DRM_FBDEV_EMULATION
  66         .output_poll_changed = drm_fb_helper_output_poll_changed,
  67 #endif
  68         .atomic_check = tegra_atomic_check,
  69         .atomic_commit = drm_atomic_helper_commit,
  70 };
  71
  72 static void tegra_atomic_commit_tail(struct drm_atomic_state *old_state)
  73 {
  74         struct drm_device *drm = old_state->dev;
  75         struct tegra_drm *tegra = drm->dev_private;
  76
  77         if (tegra->hub) {
  78                 drm_atomic_helper_commit_modeset_disables(drm, old_state);
  79                 tegra_display_hub_atomic_commit(drm, old_state);
  80                 drm_atomic_helper_commit_planes(drm, old_state, 0);
  81                 drm_atomic_helper_commit_modeset_enables(drm, old_state);
  82                 drm_atomic_helper_commit_hw_done(old_state);
  83                 drm_atomic_helper_wait_for_vblanks(drm, old_state);
  84                 drm_atomic_helper_cleanup_planes(drm, old_state);
  85         } else {
  86                 drm_atomic_helper_commit_tail_rpm(old_state);
  87         }
  88 }
  89
  90 static const struct drm_mode_config_helper_funcs
  91 tegra_drm_mode_config_helpers = {
  92         .atomic_commit_tail = tegra_atomic_commit_tail,
  93 };
  94
  95 static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
  96 {
  97         struct host1x_device *device = to_host1x_device(drm->dev);
  98         struct tegra_drm *tegra;
  99         int err;
 100
 101         tegra = kzalloc(sizeof(*tegra), GFP_KERNEL);
 102         if (!tegra)
 103                 return -ENOMEM;
 104
 105         if (iommu_present(&platform_bus_type)) {
 106                 u64 carveout_start, carveout_end, gem_start, gem_end;
 107                 struct iommu_domain_geometry *geometry;
 108                 unsigned long order;
 109
 110                 tegra->domain = iommu_domain_alloc(&platform_bus_type);
 111                 if (!tegra->domain) {
 112                         err = -ENOMEM;
 113                         goto free;
 114                 }
 115
 116                 geometry = &tegra->domain->geometry;
 117                 gem_start = geometry->aperture_start;
 118                 gem_end = geometry->aperture_end - CARVEOUT_SZ;
 119                 carveout_start = gem_end + 1;
 120                 carveout_end = geometry->aperture_end;
 121
 122                 order = __ffs(tegra->domain->pgsize_bitmap);
 123                 init_iova_domain(&tegra->carveout.domain, 1UL << order,
 124                                  carveout_start >> order);
 125
 126                 tegra->carveout.shift = iova_shift(&tegra->carveout.domain);
 127                 tegra->carveout.limit = carveout_end >> tegra->carveout.shift;
 128
 129                 drm_mm_init(&tegra->mm, gem_start, gem_end - gem_start + 1);
 130                 mutex_init(&tegra->mm_lock);
 131
 132                 DRM_DEBUG("IOMMU apertures:\n");
 133                 DRM_DEBUG("  GEM: %#llx-%#llx\n", gem_start, gem_end);
 134                 DRM_DEBUG("  Carveout: %#llx-%#llx\n", carveout_start,
 135                           carveout_end);
 136         }
 137
 138         mutex_init(&tegra->clients_lock);
 139         INIT_LIST_HEAD(&tegra->clients);
 140
 141         drm->dev_private = tegra;
 142         tegra->drm = drm;
 143
 144         drm_mode_config_init(drm);
 145
 146         drm->mode_config.min_width = 0;
 147         drm->mode_config.min_height = 0;
 148
 149         drm->mode_config.max_width = 4096;
 150         drm->mode_config.max_height = 4096;
 151
 152         drm->mode_config.allow_fb_modifiers = true;
 153
 154         drm->mode_config.funcs = &tegra_drm_mode_config_funcs;
 155         drm->mode_config.helper_private = &tegra_drm_mode_config_helpers;
 156
 157         err = tegra_drm_fb_prepare(drm);
 158         if (err < 0)
 159                 goto config;
 160
 161         drm_kms_helper_poll_init(drm);
 162
 163         err = host1x_device_init(device);
 164         if (err < 0)
 165                 goto fbdev;
 166
 167         if (tegra->hub) {
 168                 err = tegra_display_hub_prepare(tegra->hub);
 169                 if (err < 0)
 170                         goto device;
 171         }
 172
 173         /*
 174          * We don't use the drm_irq_install() helpers provided by the DRM
 175          * core, so we need to set this manually in order to allow the
 176          * DRM_IOCTL_WAIT_VBLANK to operate correctly.
 177          */
 178         drm->irq_enabled = true;
 179
 180         /* syncpoints are used for full 32-bit hardware VBLANK counters */
 181         drm->max_vblank_count = 0xffffffff;
 182
 183         err = drm_vblank_init(drm, drm->mode_config.num_crtc);
 184         if (err < 0)
 185                 goto hub;
 186
 187         drm_mode_config_reset(drm);
 188
 189         err = tegra_drm_fb_init(drm);
 190         if (err < 0)
 191                 goto hub;
 192
 193         return 0;
 194
 195 hub:
 196         if (tegra->hub)
 197                 tegra_display_hub_cleanup(tegra->hub);
 198 device:
 199         host1x_device_exit(device);
 200 fbdev:
 201         drm_kms_helper_poll_fini(drm);
 202         tegra_drm_fb_free(drm);
 203 config:
 204         drm_mode_config_cleanup(drm);
 205
 206         if (tegra->domain) {
 207                 iommu_domain_free(tegra->domain);
 208                 drm_mm_takedown(&tegra->mm);
 209                 mutex_destroy(&tegra->mm_lock);
 210                 put_iova_domain(&tegra->carveout.domain);
 211         }
 212 free:
 213         kfree(tegra);
 214         return err;
 215 }
 216
 217 static void tegra_drm_unload(struct drm_device *drm)
 218 {
 219         struct host1x_device *device = to_host1x_device(drm->dev);
 220         struct tegra_drm *tegra = drm->dev_private;
 221         int err;
 222
 223         drm_kms_helper_poll_fini(drm);
 224         tegra_drm_fb_exit(drm);
 225         drm_atomic_helper_shutdown(drm);
 226         drm_mode_config_cleanup(drm);
 227
 228         err = host1x_device_exit(device);
 229         if (err < 0)
 230                 return;
 231
 232         if (tegra->domain) {
 233                 iommu_domain_free(tegra->domain);
 234                 drm_mm_takedown(&tegra->mm);
 235                 mutex_destroy(&tegra->mm_lock);
 236                 put_iova_domain(&tegra->carveout.domain);
 237         }
 238
 239         kfree(tegra);
 240 }
 241
 242 static int tegra_drm_open(struct drm_device *drm, struct drm_file *filp)
 243 {
 244         struct tegra_drm_file *fpriv;
 245
 246         fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
 247         if (!fpriv)
 248                 return -ENOMEM;
 249
 250         idr_init(&fpriv->contexts);
 251         mutex_init(&fpriv->lock);
 252         filp->driver_priv = fpriv;
 253
 254         return 0;
 255 }
 256
 257 static void tegra_drm_context_free(struct tegra_drm_context *context)
 258 {
 259         context->client->ops->close_channel(context);
 260         kfree(context);
 261 }
 262
 263 static struct host1x_bo *
 264 host1x_bo_lookup(struct drm_file *file, u32 handle)
 265 {
 266         struct drm_gem_object *gem;
 267         struct tegra_bo *bo;
 268
 269         gem = drm_gem_object_lookup(file, handle);
 270         if (!gem)
 271                 return NULL;
 272
 273         bo = to_tegra_bo(gem);
 274         return &bo->base;
 275 }
 276
 277 static int host1x_reloc_copy_from_user(struct host1x_reloc *dest,
 278                                        struct drm_tegra_reloc __user *src,
 279                                        struct drm_device *drm,
 280                                        struct drm_file *file)
 281 {
 282         u32 cmdbuf, target;
 283         int err;
 284
 285         err = get_user(cmdbuf, &src->cmdbuf.handle);
 286         if (err < 0)
 287                 return err;
 288
 289         err = get_user(dest->cmdbuf.offset, &src->cmdbuf.offset);
 290         if (err < 0)
 291                 return err;
 292
 293         err = get_user(target, &src->target.handle);
 294         if (err < 0)
 295                 return err;
 296
 297         err = get_user(dest->target.offset, &src->target.offset);
 298         if (err < 0)
 299                 return err;
 300
 301         err = get_user(dest->shift, &src->shift);
 302         if (err < 0)
 303                 return err;
 304
 305         dest->cmdbuf.bo = host1x_bo_lookup(file, cmdbuf);
 306         if (!dest->cmdbuf.bo)
 307                 return -ENOENT;
 308
 309         dest->target.bo = host1x_bo_lookup(file, target);
 310         if (!dest->target.bo)
 311                 return -ENOENT;
 312
 313         return 0;
 314 }
 315
 316 static int host1x_waitchk_copy_from_user(struct host1x_waitchk *dest,
 317                                          struct drm_tegra_waitchk __user *src,
 318                                          struct drm_file *file)
 319 {
 320         u32 cmdbuf;
 321         int err;
 322
 323         err = get_user(cmdbuf, &src->handle);
 324         if (err < 0)
 325                 return err;
 326
 327         err = get_user(dest->offset, &src->offset);
 328         if (err < 0)
 329                 return err;
 330
 331         err = get_user(dest->syncpt_id, &src->syncpt);
 332         if (err < 0)
 333                 return err;
 334
 335         err = get_user(dest->thresh, &src->thresh);
 336         if (err < 0)
 337                 return err;
 338
 339         dest->bo = host1x_bo_lookup(file, cmdbuf);
 340         if (!dest->bo)
 341                 return -ENOENT;
 342
 343         return 0;
 344 }
 345
 346 int tegra_drm_submit(struct tegra_drm_context *context,
 347                      struct drm_tegra_submit *args, struct drm_device *drm,
 348                      struct drm_file *file)
 349 {
 350         unsigned int num_cmdbufs = args->num_cmdbufs;
 351         unsigned int num_relocs = args->num_relocs;
 352         unsigned int num_waitchks = args->num_waitchks;
 353         struct drm_tegra_cmdbuf __user *user_cmdbufs;
 354         struct drm_tegra_reloc __user *user_relocs;
 355         struct drm_tegra_waitchk __user *user_waitchks;
 356         struct drm_tegra_syncpt __user *user_syncpt;
 357         struct drm_tegra_syncpt syncpt;
 358         struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
 359         struct drm_gem_object **refs;
 360         struct host1x_syncpt *sp;
 361         struct host1x_job *job;
 362         unsigned int num_refs;
 363         int err;
 364
 365         user_cmdbufs = u64_to_user_ptr(args->cmdbufs);
 366         user_relocs = u64_to_user_ptr(args->relocs);
 367         user_waitchks = u64_to_user_ptr(args->waitchks);
 368         user_syncpt = u64_to_user_ptr(args->syncpts);
 369
 370         /* We don't yet support other than one syncpt_incr struct per submit */
 371         if (args->num_syncpts != 1)
 372                 return -EINVAL;
 373
 374         /* We don't yet support waitchks */
 375         if (args->num_waitchks != 0)
 376                 return -EINVAL;
 377
 378         job = host1x_job_alloc(context->channel, args->num_cmdbufs,
 379                                args->num_relocs, args->num_waitchks);
 380         if (!job)
 381                 return -ENOMEM;
 382
 383         job->num_relocs = args->num_relocs;
 384         job->num_waitchk = args->num_waitchks;
 385         job->client = (u32)args->context;
 386         job->class = context->client->base.class;
 387         job->serialize = true;
 388
 389         /*
 390          * Track referenced BOs so that they can be unreferenced after the
 391          * submission is complete.
 392          */
 393         num_refs = num_cmdbufs + num_relocs * 2 + num_waitchks;
 394
 395         refs = kmalloc_array(num_refs, sizeof(*refs), GFP_KERNEL);
 396         if (!refs) {
 397                 err = -ENOMEM;
 398                 goto put;
 399         }
 400
 401         /* reuse as an iterator later */
 402         num_refs = 0;
 403
 404         while (num_cmdbufs) {
 405                 struct drm_tegra_cmdbuf cmdbuf;
 406                 struct host1x_bo *bo;
 407                 struct tegra_bo *obj;
 408                 u64 offset;
 409
 410                 if (copy_from_user(&cmdbuf, user_cmdbufs, sizeof(cmdbuf))) {
 411                         err = -EFAULT;
 412                         goto fail;
 413                 }
 414
 415                 /*
 416                  * The maximum number of CDMA gather fetches is 16383, a higher
 417                  * value means the words count is malformed.
 418                  */
 419                 if (cmdbuf.words > CDMA_GATHER_FETCHES_MAX_NB) {
 420                         err = -EINVAL;
 421                         goto fail;
 422                 }
 423
 424                 bo = host1x_bo_lookup(file, cmdbuf.handle);
 425                 if (!bo) {
 426                         err = -ENOENT;
 427                         goto fail;
 428                 }
 429
 430                 offset = (u64)cmdbuf.offset + (u64)cmdbuf.words * sizeof(u32);
 431                 obj = host1x_to_tegra_bo(bo);
 432                 refs[num_refs++] = &obj->gem;
 433
 434                 /*
 435                  * Gather buffer base address must be 4-bytes aligned,
 436                  * unaligned offset is malformed and cause commands stream
 437                  * corruption on the buffer address relocation.
 438                  */
 439                 if (offset & 3 || offset >= obj->gem.size) {
 440                         err = -EINVAL;
 441                         goto fail;
 442                 }
 443
 444                 host1x_job_add_gather(job, bo, cmdbuf.words, cmdbuf.offset);
 445                 num_cmdbufs--;
 446                 user_cmdbufs++;
 447         }
 448
 449         /* copy and resolve relocations from submit */
 450         while (num_relocs--) {
 451                 struct host1x_reloc *reloc;
 452                 struct tegra_bo *obj;
 453
 454                 err = host1x_reloc_copy_from_user(&job->relocarray[num_relocs],
 455                                                   &user_relocs[num_relocs], drm,
 456                                                   file);
 457                 if (err < 0)
 458                         goto fail;
 459
 460                 reloc = &job->relocarray[num_relocs];
 461                 obj = host1x_to_tegra_bo(reloc->cmdbuf.bo);
 462                 refs[num_refs++] = &obj->gem;
 463
 464                 /*
 465                  * The unaligned cmdbuf offset will cause an unaligned write
 466                  * during of the relocations patching, corrupting the commands
 467                  * stream.
 468                  */
 469                 if (reloc->cmdbuf.offset & 3 ||
 470                     reloc->cmdbuf.offset >= obj->gem.size) {
 471                         err = -EINVAL;
 472                         goto fail;
 473                 }
 474
 475                 obj = host1x_to_tegra_bo(reloc->target.bo);
 476                 refs[num_refs++] = &obj->gem;
 477
 478                 if (reloc->target.offset >= obj->gem.size) {
 479                         err = -EINVAL;
 480                         goto fail;
 481                 }
 482         }
 483
 484         /* copy and resolve waitchks from submit */
 485         while (num_waitchks--) {
 486                 struct host1x_waitchk *wait = &job->waitchk[num_waitchks];
 487                 struct tegra_bo *obj;
 488
 489                 err = host1x_waitchk_copy_from_user(
 490                         wait, &user_waitchks[num_waitchks], file);
 491                 if (err < 0)
 492                         goto fail;
 493
 494                 obj = host1x_to_tegra_bo(wait->bo);
 495                 refs[num_refs++] = &obj->gem;
 496
 497                 /*
 498                  * The unaligned offset will cause an unaligned write during
 499                  * of the waitchks patching, corrupting the commands stream.
 500                  */
 501                 if (wait->offset & 3 ||
 502                     wait->offset >= obj->gem.size) {
 503                         err = -EINVAL;
 504                         goto fail;
 505                 }
 506         }
 507
 508         if (copy_from_user(&syncpt, user_syncpt, sizeof(syncpt))) {
 509                 err = -EFAULT;
 510                 goto fail;
 511         }
 512
 513         /* check whether syncpoint ID is valid */
 514         sp = host1x_syncpt_get(host1x, syncpt.id);
 515         if (!sp) {
 516                 err = -ENOENT;
 517                 goto fail;
 518         }
 519
 520         job->is_addr_reg = context->client->ops->is_addr_reg;
 521         job->is_valid_class = context->client->ops->is_valid_class;
 522         job->syncpt_incrs = syncpt.incrs;
 523         job->syncpt_id = syncpt.id;
 524         job->timeout = 10000;
 525
 526         if (args->timeout && args->timeout < 10000)
 527                 job->timeout = args->timeout;
 528
 529         err = host1x_job_pin(job, context->client->base.dev);
 530         if (err)
 531                 goto fail;
 532
 533         err = host1x_job_submit(job);
 534         if (err) {
 535                 host1x_job_unpin(job);
 536                 goto fail;
 537         }
 538
 539         args->fence = job->syncpt_end;
 540
 541 fail:
 542         while (num_refs--)
 543                 drm_gem_object_put_unlocked(refs[num_refs]);
 544
 545         kfree(refs);
 546
 547 put:
 548         host1x_job_put(job);
 549         return err;
 550 }
 551
 552
 553 #ifdef CONFIG_DRM_TEGRA_STAGING
 554 static int tegra_gem_create(struct drm_device *drm, void *data,
 555                             struct drm_file *file)
 556 {
 557         struct drm_tegra_gem_create *args = data;
 558         struct tegra_bo *bo;
 559
 560         bo = tegra_bo_create_with_handle(file, drm, args->size, args->flags,
 561                                          &args->handle);
 562         if (IS_ERR(bo))
 563                 return PTR_ERR(bo);
 564
 565         return 0;
 566 }
 567
 568 static int tegra_gem_mmap(struct drm_device *drm, void *data,
 569                           struct drm_file *file)
 570 {
 571         struct drm_tegra_gem_mmap *args = data;
 572         struct drm_gem_object *gem;
 573         struct tegra_bo *bo;
 574
 575         gem = drm_gem_object_lookup(file, args->handle);
 576         if (!gem)
 577                 return -EINVAL;
 578
 579         bo = to_tegra_bo(gem);
 580
 581         args->offset = drm_vma_node_offset_addr(&bo->gem.vma_node);
 582
 583         drm_gem_object_put_unlocked(gem);
 584
 585         return 0;
 586 }
 587
 588 static int tegra_syncpt_read(struct drm_device *drm, void *data,
 589                              struct drm_file *file)
 590 {
 591         struct host1x *host = dev_get_drvdata(drm->dev->parent);
 592         struct drm_tegra_syncpt_read *args = data;
 593         struct host1x_syncpt *sp;
 594
 595         sp = host1x_syncpt_get(host, args->id);
 596         if (!sp)
 597                 return -EINVAL;
 598
 599         args->value = host1x_syncpt_read_min(sp);
 600         return 0;
 601 }
 602
 603 static int tegra_syncpt_incr(struct drm_device *drm, void *data,
 604                              struct drm_file *file)
 605 {
 606         struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
 607         struct drm_tegra_syncpt_incr *args = data;
 608         struct host1x_syncpt *sp;
 609
 610         sp = host1x_syncpt_get(host1x, args->id);
 611         if (!sp)
 612                 return -EINVAL;
 613
 614         return host1x_syncpt_incr(sp);
 615 }
 616
 617 static int tegra_syncpt_wait(struct drm_device *drm, void *data,
 618                              struct drm_file *file)
 619 {
 620         struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
 621         struct drm_tegra_syncpt_wait *args = data;
 622         struct host1x_syncpt *sp;
 623
 624         sp = host1x_syncpt_get(host1x, args->id);
 625         if (!sp)
 626                 return -EINVAL;
 627
 628         return host1x_syncpt_wait(sp, args->thresh,
 629                                   msecs_to_jiffies(args->timeout),
 630                                   &args->value);
 631 }
 632
 633 static int tegra_client_open(struct tegra_drm_file *fpriv,
 634                              struct tegra_drm_client *client,
 635                              struct tegra_drm_context *context)
 636 {
 637         int err;
 638
 639         err = client->ops->open_channel(client, context);
 640         if (err < 0)
 641                 return err;
 642
 643         err = idr_alloc(&fpriv->contexts, context, 1, 0, GFP_KERNEL);
 644         if (err < 0) {
 645                 client->ops->close_channel(context);
 646                 return err;
 647         }
 648
 649         context->client = client;
 650         context->id = err;
 651
 652         return 0;
 653 }
 654
 655 static int tegra_open_channel(struct drm_device *drm, void *data,
 656                               struct drm_file *file)
 657 {
 658         struct tegra_drm_file *fpriv = file->driver_priv;
 659         struct tegra_drm *tegra = drm->dev_private;
 660         struct drm_tegra_open_channel *args = data;
 661         struct tegra_drm_context *context;
 662         struct tegra_drm_client *client;
 663         int err = -ENODEV;
 664
 665         context = kzalloc(sizeof(*context), GFP_KERNEL);
 666         if (!context)
 667                 return -ENOMEM;
 668
 669         mutex_lock(&fpriv->lock);
 670
 671         list_for_each_entry(client, &tegra->clients, list)
 672                 if (client->base.class == args->client) {
 673                         err = tegra_client_open(fpriv, client, context);
 674                         if (err < 0)
 675                                 break;
 676
 677                         args->context = context->id;
 678                         break;
 679                 }
 680
 681         if (err < 0)
 682                 kfree(context);
 683
 684         mutex_unlock(&fpriv->lock);
 685         return err;
 686 }
 687
 688 static int tegra_close_channel(struct drm_device *drm, void *data,
 689                                struct drm_file *file)
 690 {
 691         struct tegra_drm_file *fpriv = file->driver_priv;
 692         struct drm_tegra_close_channel *args = data;
 693         struct tegra_drm_context *context;
 694         int err = 0;
 695
 696         mutex_lock(&fpriv->lock);
 697
 698         context = idr_find(&fpriv->contexts, args->context);
 699         if (!context) {
 700                 err = -EINVAL;
 701                 goto unlock;
 702         }
 703
 704         idr_remove(&fpriv->contexts, context->id);
 705         tegra_drm_context_free(context);
 706
 707 unlock:
 708         mutex_unlock(&fpriv->lock);
 709         return err;
 710 }
 711
 712 static int tegra_get_syncpt(struct drm_device *drm, void *data,
 713                             struct drm_file *file)
 714 {
 715         struct tegra_drm_file *fpriv = file->driver_priv;
 716         struct drm_tegra_get_syncpt *args = data;
 717         struct tegra_drm_context *context;
 718         struct host1x_syncpt *syncpt;
 719         int err = 0;
 720
 721         mutex_lock(&fpriv->lock);
 722
 723         context = idr_find(&fpriv->contexts, args->context);
 724         if (!context) {
 725                 err = -ENODEV;
 726                 goto unlock;
 727         }
 728
 729         if (args->index >= context->client->base.num_syncpts) {
 730                 err = -EINVAL;
 731                 goto unlock;
 732         }
 733
 734         syncpt = context->client->base.syncpts[args->index];
 735         args->id = host1x_syncpt_id(syncpt);
 736
 737 unlock:
 738         mutex_unlock(&fpriv->lock);
 739         return err;
 740 }
 741
 742 static int tegra_submit(struct drm_device *drm, void *data,
 743                         struct drm_file *file)
 744 {
 745         struct tegra_drm_file *fpriv = file->driver_priv;
 746         struct drm_tegra_submit *args = data;
 747         struct tegra_drm_context *context;
 748         int err;
 749
 750         mutex_lock(&fpriv->lock);
 751
 752         context = idr_find(&fpriv->contexts, args->context);
 753         if (!context) {
 754                 err = -ENODEV;
 755                 goto unlock;
 756         }
 757
 758         err = context->client->ops->submit(context, args, drm, file);
 759
 760 unlock:
 761         mutex_unlock(&fpriv->lock);
 762         return err;
 763 }
 764
 765 static int tegra_get_syncpt_base(struct drm_device *drm, void *data,
 766                                  struct drm_file *file)
 767 {
 768         struct tegra_drm_file *fpriv = file->driver_priv;
 769         struct drm_tegra_get_syncpt_base *args = data;
 770         struct tegra_drm_context *context;
 771         struct host1x_syncpt_base *base;
 772         struct host1x_syncpt *syncpt;
 773         int err = 0;
 774
 775         mutex_lock(&fpriv->lock);
 776
 777         context = idr_find(&fpriv->contexts, args->context);
 778         if (!context) {
 779                 err = -ENODEV;
 780                 goto unlock;
 781         }
 782
 783         if (args->syncpt >= context->client->base.num_syncpts) {
 784                 err = -EINVAL;
 785                 goto unlock;
 786         }
 787
 788         syncpt = context->client->base.syncpts[args->syncpt];
 789
 790         base = host1x_syncpt_get_base(syncpt);
 791         if (!base) {
 792                 err = -ENXIO;
 793                 goto unlock;
 794         }
 795
 796         args->id = host1x_syncpt_base_id(base);
 797
 798 unlock:
 799         mutex_unlock(&fpriv->lock);
 800         return err;
 801 }
 802
 803 static int tegra_gem_set_tiling(struct drm_device *drm, void *data,
 804                                 struct drm_file *file)
 805 {
 806         struct drm_tegra_gem_set_tiling *args = data;
 807         enum tegra_bo_tiling_mode mode;
 808         struct drm_gem_object *gem;
 809         unsigned long value = 0;
 810         struct tegra_bo *bo;
 811
 812         switch (args->mode) {
 813         case DRM_TEGRA_GEM_TILING_MODE_PITCH:
 814                 mode = TEGRA_BO_TILING_MODE_PITCH;
 815
 816                 if (args->value != 0)
 817                         return -EINVAL;
 818
 819                 break;
 820
 821         case DRM_TEGRA_GEM_TILING_MODE_TILED:
 822                 mode = TEGRA_BO_TILING_MODE_TILED;
 823
 824                 if (args->value != 0)
 825                         return -EINVAL;
 826
 827                 break;
 828
 829         case DRM_TEGRA_GEM_TILING_MODE_BLOCK:
 830                 mode = TEGRA_BO_TILING_MODE_BLOCK;
 831
 832                 if (args->value > 5)
 833                         return -EINVAL;
 834
 835                 value = args->value;
 836                 break;
 837
 838         default:
 839                 return -EINVAL;
 840         }
 841
 842         gem = drm_gem_object_lookup(file, args->handle);
 843         if (!gem)
 844                 return -ENOENT;
 845
 846         bo = to_tegra_bo(gem);
 847
 848         bo->tiling.mode = mode;
 849         bo->tiling.value = value;
 850
 851         drm_gem_object_put_unlocked(gem);
 852
 853         return 0;
 854 }
 855
 856 static int tegra_gem_get_tiling(struct drm_device *drm, void *data,
 857                                 struct drm_file *file)
 858 {
 859         struct drm_tegra_gem_get_tiling *args = data;
 860         struct drm_gem_object *gem;
 861         struct tegra_bo *bo;
 862         int err = 0;
 863
 864         gem = drm_gem_object_lookup(file, args->handle);
 865         if (!gem)
 866                 return -ENOENT;
 867
 868         bo = to_tegra_bo(gem);
 869
 870         switch (bo->tiling.mode) {
 871         case TEGRA_BO_TILING_MODE_PITCH:
 872                 args->mode = DRM_TEGRA_GEM_TILING_MODE_PITCH;
 873                 args->value = 0;
 874                 break;
 875
 876         case TEGRA_BO_TILING_MODE_TILED:
 877                 args->mode = DRM_TEGRA_GEM_TILING_MODE_TILED;
 878                 args->value = 0;
 879                 break;
 880
 881         case TEGRA_BO_TILING_MODE_BLOCK:
 882                 args->mode = DRM_TEGRA_GEM_TILING_MODE_BLOCK;
 883                 args->value = bo->tiling.value;
 884                 break;
 885
 886         default:
 887                 err = -EINVAL;
 888                 break;
 889         }
 890
 891         drm_gem_object_put_unlocked(gem);
 892
 893         return err;
 894 }
 895
 896 static int tegra_gem_set_flags(struct drm_device *drm, void *data,
 897                                struct drm_file *file)
 898 {
 899         struct drm_tegra_gem_set_flags *args = data;
 900         struct drm_gem_object *gem;
 901         struct tegra_bo *bo;
 902
 903         if (args->flags & ~DRM_TEGRA_GEM_FLAGS)
 904                 return -EINVAL;
 905
 906         gem = drm_gem_object_lookup(file, args->handle);
 907         if (!gem)
 908                 return -ENOENT;
 909
 910         bo = to_tegra_bo(gem);
 911         bo->flags = 0;
 912
 913         if (args->flags & DRM_TEGRA_GEM_BOTTOM_UP)
 914                 bo->flags |= TEGRA_BO_BOTTOM_UP;
 915
 916         drm_gem_object_put_unlocked(gem);
 917
 918         return 0;
 919 }
 920
 921 static int tegra_gem_get_flags(struct drm_device *drm, void *data,
 922                                struct drm_file *file)
 923 {
 924         struct drm_tegra_gem_get_flags *args = data;
 925         struct drm_gem_object *gem;
 926         struct tegra_bo *bo;
 927
 928         gem = drm_gem_object_lookup(file, args->handle);
 929         if (!gem)
 930                 return -ENOENT;
 931
 932         bo = to_tegra_bo(gem);
 933         args->flags = 0;
 934
 935         if (bo->flags & TEGRA_BO_BOTTOM_UP)
 936                 args->flags |= DRM_TEGRA_GEM_BOTTOM_UP;
 937
 938         drm_gem_object_put_unlocked(gem);
 939
 940         return 0;
 941 }
 942 #endif
 943
 944 static const struct drm_ioctl_desc tegra_drm_ioctls[] = {
 945 #ifdef CONFIG_DRM_TEGRA_STAGING
 946         DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_gem_create,
 947                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 948         DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_gem_mmap,
 949                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 950         DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_READ, tegra_syncpt_read,
 951                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 952         DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_INCR, tegra_syncpt_incr,
 953                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 954         DRM_IOCTL_DEF_DRV(TEGRA_SYNCPT_WAIT, tegra_syncpt_wait,
 955                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 956         DRM_IOCTL_DEF_DRV(TEGRA_OPEN_CHANNEL, tegra_open_channel,
 957                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 958         DRM_IOCTL_DEF_DRV(TEGRA_CLOSE_CHANNEL, tegra_close_channel,
 959                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 960         DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT, tegra_get_syncpt,
 961                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 962         DRM_IOCTL_DEF_DRV(TEGRA_SUBMIT, tegra_submit,
 963                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 964         DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT_BASE, tegra_get_syncpt_base,
 965                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 966         DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_TILING, tegra_gem_set_tiling,
 967                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 968         DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_TILING, tegra_gem_get_tiling,
 969                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 970         DRM_IOCTL_DEF_DRV(TEGRA_GEM_SET_FLAGS, tegra_gem_set_flags,
 971                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 972         DRM_IOCTL_DEF_DRV(TEGRA_GEM_GET_FLAGS, tegra_gem_get_flags,
 973                           DRM_UNLOCKED | DRM_RENDER_ALLOW),
 974 #endif
 975 };
 976
 977 static const struct file_operations tegra_drm_fops = {
 978         .owner = THIS_MODULE,
 979         .open = drm_open,
 980         .release = drm_release,
 981         .unlocked_ioctl = drm_ioctl,
 982         .mmap = tegra_drm_mmap,
 983         .poll = drm_poll,
 984         .read = drm_read,
 985         .compat_ioctl = drm_compat_ioctl,
 986         .llseek = noop_llseek,
 987 };
 988
 989 static int tegra_drm_context_cleanup(int id, void *p, void *data)
 990 {
 991         struct tegra_drm_context *context = p;
 992
 993         tegra_drm_context_free(context);
 994
 995         return 0;
 996 }
 997
 998 static void tegra_drm_postclose(struct drm_device *drm, struct drm_file *file)
 999 {
1000         struct tegra_drm_file *fpriv = file->driver_priv;
1001
1002         mutex_lock(&fpriv->lock);
1003         idr_for_each(&fpriv->contexts, tegra_drm_context_cleanup, NULL);
1004         mutex_unlock(&fpriv->lock);
1005
1006         idr_destroy(&fpriv->contexts);
1007         mutex_destroy(&fpriv->lock);
1008         kfree(fpriv);
1009 }
1010
1011 #ifdef CONFIG_DEBUG_FS
1012 static int tegra_debugfs_framebuffers(struct seq_file *s, void *data)
1013 {
1014         struct drm_info_node *node = (struct drm_info_node *)s->private;
1015         struct drm_device *drm = node->minor->dev;
1016         struct drm_framebuffer *fb;
1017
1018         mutex_lock(&drm->mode_config.fb_lock);
1019
1020         list_for_each_entry(fb, &drm->mode_config.fb_list, head) {
1021                 seq_printf(s, "%3d: user size: %d x %d, depth %d, %d bpp, refcount %d\n",
1022                            fb->base.id, fb->width, fb->height,
1023                            fb->format->depth,
1024                            fb->format->cpp[0] * 8,
1025                            drm_framebuffer_read_refcount(fb));
1026         }
1027
1028         mutex_unlock(&drm->mode_config.fb_lock);
1029
1030         return 0;
1031 }
1032
1033 static int tegra_debugfs_iova(struct seq_file *s, void *data)
1034 {
1035         struct drm_info_node *node = (struct drm_info_node *)s->private;
1036         struct drm_device *drm = node->minor->dev;
1037         struct tegra_drm *tegra = drm->dev_private;
1038         struct drm_printer p = drm_seq_file_printer(s);
1039
1040         if (tegra->domain) {
1041                 mutex_lock(&tegra->mm_lock);
1042                 drm_mm_print(&tegra->mm, &p);
1043                 mutex_unlock(&tegra->mm_lock);
1044         }
1045
1046         return 0;
1047 }
1048
1049 static struct drm_info_list tegra_debugfs_list[] = {
1050         { "framebuffers", tegra_debugfs_framebuffers, 0 },
1051         { "iova", tegra_debugfs_iova, 0 },
1052 };
1053
1054 static int tegra_debugfs_init(struct drm_minor *minor)
1055 {
1056         return drm_debugfs_create_files(tegra_debugfs_list,
1057                                         ARRAY_SIZE(tegra_debugfs_list),
1058                                         minor->debugfs_root, minor);
1059 }
1060 #endif
1061
1062 static struct drm_driver tegra_drm_driver = {
1063         .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
1064                            DRIVER_ATOMIC | DRIVER_RENDER,
1065         .load = tegra_drm_load,
1066         .unload = tegra_drm_unload,
1067         .open = tegra_drm_open,
1068         .postclose = tegra_drm_postclose,
1069         .lastclose = drm_fb_helper_lastclose,
1070
1071 #if defined(CONFIG_DEBUG_FS)
1072         .debugfs_init = tegra_debugfs_init,
1073 #endif
1074
1075         .gem_free_object_unlocked = tegra_bo_free_object,
1076         .gem_vm_ops = &tegra_bo_vm_ops,
1077
1078         .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
1079         .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
1080         .gem_prime_export = tegra_gem_prime_export,
1081         .gem_prime_import = tegra_gem_prime_import,
1082
1083         .dumb_create = tegra_bo_dumb_create,
1084
1085         .ioctls = tegra_drm_ioctls,
1086         .num_ioctls = ARRAY_SIZE(tegra_drm_ioctls),
1087         .fops = &tegra_drm_fops,
1088
1089         .name = DRIVER_NAME,
1090         .desc = DRIVER_DESC,
1091         .date = DRIVER_DATE,
1092         .major = DRIVER_MAJOR,
1093         .minor = DRIVER_MINOR,
1094         .patchlevel = DRIVER_PATCHLEVEL,
1095 };
1096
1097 int tegra_drm_register_client(struct tegra_drm *tegra,
1098                               struct tegra_drm_client *client)
1099 {
1100         mutex_lock(&tegra->clients_lock);
1101         list_add_tail(&client->list, &tegra->clients);
1102         mutex_unlock(&tegra->clients_lock);
1103
1104         return 0;
1105 }
1106
1107 int tegra_drm_unregister_client(struct tegra_drm *tegra,
1108                                 struct tegra_drm_client *client)
1109 {
1110         mutex_lock(&tegra->clients_lock);
1111         list_del_init(&client->list);
1112         mutex_unlock(&tegra->clients_lock);
1113
1114         return 0;
1115 }
1116
1117 void *tegra_drm_alloc(struct tegra_drm *tegra, size_t size, dma_addr_t *dma)
1118 {
1119         struct iova *alloc;
1120         void *virt;
1121         gfp_t gfp;
1122         int err;
1123
1124         if (tegra->domain)
1125                 size = iova_align(&tegra->carveout.domain, size);
1126         else
1127                 size = PAGE_ALIGN(size);
1128
1129         gfp = GFP_KERNEL | __GFP_ZERO;
1130         if (!tegra->domain) {
1131                 /*
1132                  * Many units only support 32-bit addresses, even on 64-bit
1133                  * SoCs. If there is no IOMMU to translate into a 32-bit IO
1134                  * virtual address space, force allocations to be in the
1135                  * lower 32-bit range.
1136                  */
1137                 gfp |= GFP_DMA;
1138         }
1139
1140         virt = (void *)__get_free_pages(gfp, get_order(size));
1141         if (!virt)
1142                 return ERR_PTR(-ENOMEM);
1143
1144         if (!tegra->domain) {
1145                 /*
1146                  * If IOMMU is disabled, devices address physical memory
1147                  * directly.
1148                  */
1149                 *dma = virt_to_phys(virt);
1150                 return virt;
1151         }
1152
1153         alloc = alloc_iova(&tegra->carveout.domain,
1154                            size >> tegra->carveout.shift,
1155                            tegra->carveout.limit, true);
1156         if (!alloc) {
1157                 err = -EBUSY;
1158                 goto free_pages;
1159         }
1160
1161         *dma = iova_dma_addr(&tegra->carveout.domain, alloc);
1162         err = iommu_map(tegra->domain, *dma, virt_to_phys(virt),
1163                         size, IOMMU_READ | IOMMU_WRITE);
1164         if (err < 0)
1165                 goto free_iova;
1166
1167         return virt;
1168
1169 free_iova:
1170         __free_iova(&tegra->carveout.domain, alloc);
1171 free_pages:
1172         free_pages((unsigned long)virt, get_order(size));
1173
1174         return ERR_PTR(err);
1175 }
1176
1177 void tegra_drm_free(struct tegra_drm *tegra, size_t size, void *virt,
1178                     dma_addr_t dma)
1179 {
1180         if (tegra->domain)
1181                 size = iova_align(&tegra->carveout.domain, size);
1182         else
1183                 size = PAGE_ALIGN(size);
1184
1185         if (tegra->domain) {
1186                 iommu_unmap(tegra->domain, dma, size);
1187                 free_iova(&tegra->carveout.domain,
1188                           iova_pfn(&tegra->carveout.domain, dma));
1189         }
1190
1191         free_pages((unsigned long)virt, get_order(size));
1192 }
1193
1194 static int host1x_drm_probe(struct host1x_device *dev)
1195 {
1196         struct drm_driver *driver = &tegra_drm_driver;
1197         struct drm_device *drm;
1198         int err;
1199
1200         drm = drm_dev_alloc(driver, &dev->dev);
1201         if (IS_ERR(drm))
1202                 return PTR_ERR(drm);
1203
1204         dev_set_drvdata(&dev->dev, drm);
1205
1206         err = drm_dev_register(drm, 0);
1207         if (err < 0)
1208                 goto unref;
1209
1210         return 0;
1211
1212 unref:
1213         drm_dev_unref(drm);
1214         return err;
1215 }
1216
1217 static int host1x_drm_remove(struct host1x_device *dev)
1218 {
1219         struct drm_device *drm = dev_get_drvdata(&dev->dev);
1220
1221         drm_dev_unregister(drm);
1222         drm_dev_unref(drm);
1223
1224         return 0;
1225 }
1226
1227 #ifdef CONFIG_PM_SLEEP
1228 static int host1x_drm_suspend(struct device *dev)
1229 {
1230         struct drm_device *drm = dev_get_drvdata(dev);
1231         struct tegra_drm *tegra = drm->dev_private;
1232
1233         drm_kms_helper_poll_disable(drm);
1234         tegra_drm_fb_suspend(drm);
1235
1236         tegra->state = drm_atomic_helper_suspend(drm);
1237         if (IS_ERR(tegra->state)) {
1238                 tegra_drm_fb_resume(drm);
1239                 drm_kms_helper_poll_enable(drm);
1240                 return PTR_ERR(tegra->state);
1241         }
1242
1243         return 0;
1244 }
1245
1246 static int host1x_drm_resume(struct device *dev)
1247 {
1248         struct drm_device *drm = dev_get_drvdata(dev);
1249         struct tegra_drm *tegra = drm->dev_private;
1250
1251         drm_atomic_helper_resume(drm, tegra->state);
1252         tegra_drm_fb_resume(drm);
1253         drm_kms_helper_poll_enable(drm);
1254
1255         return 0;
1256 }
1257 #endif
1258
1259 static SIMPLE_DEV_PM_OPS(host1x_drm_pm_ops, host1x_drm_suspend,
1260                          host1x_drm_resume);
1261
1262 static const struct of_device_id host1x_drm_subdevs[] = {
1263         { .compatible = "nvidia,tegra20-dc", },
1264         { .compatible = "nvidia,tegra20-hdmi", },
1265         { .compatible = "nvidia,tegra20-gr2d", },
1266         { .compatible = "nvidia,tegra20-gr3d", },
1267         { .compatible = "nvidia,tegra30-dc", },
1268         { .compatible = "nvidia,tegra30-hdmi", },
1269         { .compatible = "nvidia,tegra30-gr2d", },
1270         { .compatible = "nvidia,tegra30-gr3d", },
1271         { .compatible = "nvidia,tegra114-dsi", },
1272         { .compatible = "nvidia,tegra114-hdmi", },
1273         { .compatible = "nvidia,tegra114-gr3d", },
1274         { .compatible = "nvidia,tegra124-dc", },
1275         { .compatible = "nvidia,tegra124-sor", },
1276         { .compatible = "nvidia,tegra124-hdmi", },
1277         { .compatible = "nvidia,tegra124-dsi", },
1278         { .compatible = "nvidia,tegra124-vic", },
1279         { .compatible = "nvidia,tegra132-dsi", },
1280         { .compatible = "nvidia,tegra210-dc", },
1281         { .compatible = "nvidia,tegra210-dsi", },
1282         { .compatible = "nvidia,tegra210-sor", },
1283         { .compatible = "nvidia,tegra210-sor1", },
1284         { .compatible = "nvidia,tegra210-vic", },
1285         { .compatible = "nvidia,tegra186-display", },
1286         { .compatible = "nvidia,tegra186-dc", },
1287         { .compatible = "nvidia,tegra186-sor", },
1288         { .compatible = "nvidia,tegra186-sor1", },
1289         { .compatible = "nvidia,tegra186-vic", },
1290         { /* sentinel */ }
1291 };
1292
1293 static struct host1x_driver host1x_drm_driver = {
1294         .driver = {
1295                 .name = "drm",
1296                 .pm = &host1x_drm_pm_ops,
1297         },
1298         .probe = host1x_drm_probe,
1299         .remove = host1x_drm_remove,
1300         .subdevs = host1x_drm_subdevs,
1301 };
1302
1303 static struct platform_driver * const drivers[] = {
1304         &tegra_display_hub_driver,
1305         &tegra_dc_driver,
1306         &tegra_hdmi_driver,
1307         &tegra_dsi_driver,
1308         &tegra_dpaux_driver,
1309         &tegra_sor_driver,
1310         &tegra_gr2d_driver,
1311         &tegra_gr3d_driver,
1312         &tegra_vic_driver,
1313 };
1314
1315 static int __init host1x_drm_init(void)
1316 {
1317         int err;
1318
1319         err = host1x_driver_register(&host1x_drm_driver);
1320         if (err < 0)
1321                 return err;
1322
1323         err = platform_register_drivers(drivers, ARRAY_SIZE(drivers));
1324         if (err < 0)
1325                 goto unregister_host1x;
1326
1327         return 0;
1328
1329 unregister_host1x:
1330         host1x_driver_unregister(&host1x_drm_driver);
1331         return err;
1332 }
1333 module_init(host1x_drm_init);
1334
1335 static void __exit host1x_drm_exit(void)
1336 {
1337         platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
1338         host1x_driver_unregister(&host1x_drm_driver);
1339 }
1340 module_exit(host1x_drm_exit);
1341
1342 MODULE_AUTHOR("Thierry Reding <thierry.reding@avionic-design.de>");
1343 MODULE_DESCRIPTION("NVIDIA Tegra DRM driver");
1344 MODULE_LICENSE("GPL v2");