drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drm_vma_manager.h>
  29 #include <linux/dma-fence-array.h>
  30 #include <linux/kthread.h>
  31 #include <linux/dma-resv.h>
  32 #include <linux/shmem_fs.h>
  33 #include <linux/slab.h>
  34 #include <linux/stop_machine.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/dma-buf.h>
  38 #include <linux/mman.h>
  39
  40 #include "display/intel_display.h"
  41 #include "display/intel_frontbuffer.h"
  42
  43 #include "gem/i915_gem_clflush.h"
  44 #include "gem/i915_gem_context.h"
  45 #include "gem/i915_gem_ioctls.h"
  46 #include "gem/i915_gem_mman.h"
  47 #include "gem/i915_gem_region.h"
  48 #include "gt/intel_engine_user.h"
  49 #include "gt/intel_gt.h"
  50 #include "gt/intel_gt_pm.h"
  51 #include "gt/intel_workarounds.h"
  52
  53 #include "i915_drv.h"
  54 #include "i915_trace.h"
  55 #include "i915_vgpu.h"
  56
  57 #include "intel_pm.h"
  58
  59 static int
  60 insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
  61 {
  62         int err;
  63
  64         err = mutex_lock_interruptible(&ggtt->vm.mutex);
  65         if (err)
  66                 return err;
  67
  68         memset(node, 0, sizeof(*node));
  69         err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
  70                                           size, 0, I915_COLOR_UNEVICTABLE,
  71                                           0, ggtt->mappable_end,
  72                                           DRM_MM_INSERT_LOW);
  73
  74         mutex_unlock(&ggtt->vm.mutex);
  75
  76         return err;
  77 }
  78
  79 static void
  80 remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
  81 {
  82         mutex_lock(&ggtt->vm.mutex);
  83         drm_mm_remove_node(node);
  84         mutex_unlock(&ggtt->vm.mutex);
  85 }
  86
  87 int
  88 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  89                             struct drm_file *file)
  90 {
  91         struct drm_i915_private *i915 = to_i915(dev);
  92         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
  93         struct drm_i915_gem_get_aperture *args = data;
  94         struct i915_vma *vma;
  95         u64 pinned;
  96
  97         if (mutex_lock_interruptible(&ggtt->vm.mutex))
  98                 return -EINTR;
  99
 100         pinned = ggtt->vm.reserved;
 101         list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 102                 if (i915_vma_is_pinned(vma))
 103                         pinned += vma->node.size;
 104
 105         mutex_unlock(&ggtt->vm.mutex);
 106
 107         args->aper_size = ggtt->vm.total;
 108         args->aper_available_size = args->aper_size - pinned;
 109
 110         return 0;
 111 }
 112
 113 int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
 114                            unsigned long flags)
 115 {
 116         struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
 117         LIST_HEAD(still_in_list);
 118         intel_wakeref_t wakeref;
 119         struct i915_vma *vma;
 120         int ret;
 121
 122         assert_object_held(obj);
 123
 124         if (list_empty(&obj->vma.list))
 125                 return 0;
 126
 127         /*
 128          * As some machines use ACPI to handle runtime-resume callbacks, and
 129          * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
 130          * as they are required by the shrinker. Ergo, we wake the device up
 131          * first just in case.
 132          */
 133         wakeref = intel_runtime_pm_get(rpm);
 134
 135 try_again:
 136         ret = 0;
 137         spin_lock(&obj->vma.lock);
 138         while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
 139                                                        struct i915_vma,
 140                                                        obj_link))) {
 141                 struct i915_address_space *vm = vma->vm;
 142
 143                 list_move_tail(&vma->obj_link, &still_in_list);
 144                 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
 145                         continue;
 146
 147                 if (flags & I915_GEM_OBJECT_UNBIND_TEST) {
 148                         ret = -EBUSY;
 149                         break;
 150                 }
 151
 152                 ret = -EAGAIN;
 153                 if (!i915_vm_tryopen(vm))
 154                         break;
 155
 156                 /* Prevent vma being freed by i915_vma_parked as we unbind */
 157                 vma = __i915_vma_get(vma);
 158                 spin_unlock(&obj->vma.lock);
 159
 160                 if (vma) {
 161                         bool vm_trylock = !!(flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK);
 162                         ret = -EBUSY;
 163                         if (flags & I915_GEM_OBJECT_UNBIND_ASYNC) {
 164                                 assert_object_held(vma->obj);
 165                                 ret = i915_vma_unbind_async(vma, vm_trylock);
 166                         }
 167
 168                         if (ret == -EBUSY && (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
 169                                               !i915_vma_is_active(vma))) {
 170                                 if (vm_trylock) {
 171                                         if (mutex_trylock(&vma->vm->mutex)) {
 172                                                 ret = __i915_vma_unbind(vma);
 173                                                 mutex_unlock(&vma->vm->mutex);
 174                                         } else {
 175                                                 ret = -EBUSY;
 176                                         }
 177                                 } else {
 178                                         ret = i915_vma_unbind(vma);
 179                                 }
 180                         }
 181
 182                         __i915_vma_put(vma);
 183                 }
 184
 185                 i915_vm_close(vm);
 186                 spin_lock(&obj->vma.lock);
 187         }
 188         list_splice_init(&still_in_list, &obj->vma.list);
 189         spin_unlock(&obj->vma.lock);
 190
 191         if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
 192                 rcu_barrier(); /* flush the i915_vm_release() */
 193                 goto try_again;
 194         }
 195
 196         intel_runtime_pm_put(rpm, wakeref);
 197
 198         return ret;
 199 }
 200
 201 static int
 202 shmem_pread(struct page *page, int offset, int len, char __user *user_data,
 203             bool needs_clflush)
 204 {
 205         char *vaddr;
 206         int ret;
 207
 208         vaddr = kmap(page);
 209
 210         if (needs_clflush)
 211                 drm_clflush_virt_range(vaddr + offset, len);
 212
 213         ret = __copy_to_user(user_data, vaddr + offset, len);
 214
 215         kunmap(page);
 216
 217         return ret ? -EFAULT : 0;
 218 }
 219
 220 static int
 221 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 222                      struct drm_i915_gem_pread *args)
 223 {
 224         unsigned int needs_clflush;
 225         unsigned int idx, offset;
 226         char __user *user_data;
 227         u64 remain;
 228         int ret;
 229
 230         ret = i915_gem_object_lock_interruptible(obj, NULL);
 231         if (ret)
 232                 return ret;
 233
 234         ret = i915_gem_object_pin_pages(obj);
 235         if (ret)
 236                 goto err_unlock;
 237
 238         ret = i915_gem_object_prepare_read(obj, &needs_clflush);
 239         if (ret)
 240                 goto err_unpin;
 241
 242         i915_gem_object_finish_access(obj);
 243         i915_gem_object_unlock(obj);
 244
 245         remain = args->size;
 246         user_data = u64_to_user_ptr(args->data_ptr);
 247         offset = offset_in_page(args->offset);
 248         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 249                 struct page *page = i915_gem_object_get_page(obj, idx);
 250                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 251
 252                 ret = shmem_pread(page, offset, length, user_data,
 253                                   needs_clflush);
 254                 if (ret)
 255                         break;
 256
 257                 remain -= length;
 258                 user_data += length;
 259                 offset = 0;
 260         }
 261
 262         i915_gem_object_unpin_pages(obj);
 263         return ret;
 264
 265 err_unpin:
 266         i915_gem_object_unpin_pages(obj);
 267 err_unlock:
 268         i915_gem_object_unlock(obj);
 269         return ret;
 270 }
 271
 272 static inline bool
 273 gtt_user_read(struct io_mapping *mapping,
 274               loff_t base, int offset,
 275               char __user *user_data, int length)
 276 {
 277         void __iomem *vaddr;
 278         unsigned long unwritten;
 279
 280         /* We can use the cpu mem copy function because this is X86. */
 281         vaddr = io_mapping_map_atomic_wc(mapping, base);
 282         unwritten = __copy_to_user_inatomic(user_data,
 283                                             (void __force *)vaddr + offset,
 284                                             length);
 285         io_mapping_unmap_atomic(vaddr);
 286         if (unwritten) {
 287                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 288                 unwritten = copy_to_user(user_data,
 289                                          (void __force *)vaddr + offset,
 290                                          length);
 291                 io_mapping_unmap(vaddr);
 292         }
 293         return unwritten;
 294 }
 295
 296 static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
 297                                              struct drm_mm_node *node,
 298                                              bool write)
 299 {
 300         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 301         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 302         struct i915_vma *vma;
 303         struct i915_gem_ww_ctx ww;
 304         int ret;
 305
 306         i915_gem_ww_ctx_init(&ww, true);
 307 retry:
 308         vma = ERR_PTR(-ENODEV);
 309         ret = i915_gem_object_lock(obj, &ww);
 310         if (ret)
 311                 goto err_ww;
 312
 313         ret = i915_gem_object_set_to_gtt_domain(obj, write);
 314         if (ret)
 315                 goto err_ww;
 316
 317         if (!i915_gem_object_is_tiled(obj))
 318                 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
 319                                                   PIN_MAPPABLE |
 320                                                   PIN_NONBLOCK /* NOWARN */ |
 321                                                   PIN_NOEVICT);
 322         if (vma == ERR_PTR(-EDEADLK)) {
 323                 ret = -EDEADLK;
 324                 goto err_ww;
 325         } else if (!IS_ERR(vma)) {
 326                 node->start = i915_ggtt_offset(vma);
 327                 node->flags = 0;
 328         } else {
 329                 ret = insert_mappable_node(ggtt, node, PAGE_SIZE);
 330                 if (ret)
 331                         goto err_ww;
 332                 GEM_BUG_ON(!drm_mm_node_allocated(node));
 333                 vma = NULL;
 334         }
 335
 336         ret = i915_gem_object_pin_pages(obj);
 337         if (ret) {
 338                 if (drm_mm_node_allocated(node)) {
 339                         ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 340                         remove_mappable_node(ggtt, node);
 341                 } else {
 342                         i915_vma_unpin(vma);
 343                 }
 344         }
 345
 346 err_ww:
 347         if (ret == -EDEADLK) {
 348                 ret = i915_gem_ww_ctx_backoff(&ww);
 349                 if (!ret)
 350                         goto retry;
 351         }
 352         i915_gem_ww_ctx_fini(&ww);
 353
 354         return ret ? ERR_PTR(ret) : vma;
 355 }
 356
 357 static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
 358                                  struct drm_mm_node *node,
 359                                  struct i915_vma *vma)
 360 {
 361         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 362         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 363
 364         i915_gem_object_unpin_pages(obj);
 365         if (drm_mm_node_allocated(node)) {
 366                 ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 367                 remove_mappable_node(ggtt, node);
 368         } else {
 369                 i915_vma_unpin(vma);
 370         }
 371 }
 372
 373 static int
 374 i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 375                    const struct drm_i915_gem_pread *args)
 376 {
 377         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 378         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 379         intel_wakeref_t wakeref;
 380         struct drm_mm_node node;
 381         void __user *user_data;
 382         struct i915_vma *vma;
 383         u64 remain, offset;
 384         int ret = 0;
 385
 386         wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 387
 388         vma = i915_gem_gtt_prepare(obj, &node, false);
 389         if (IS_ERR(vma)) {
 390                 ret = PTR_ERR(vma);
 391                 goto out_rpm;
 392         }
 393
 394         user_data = u64_to_user_ptr(args->data_ptr);
 395         remain = args->size;
 396         offset = args->offset;
 397
 398         while (remain > 0) {
 399                 /* Operation in this page
 400                  *
 401                  * page_base = page offset within aperture
 402                  * page_offset = offset within page
 403                  * page_length = bytes to copy for this page
 404                  */
 405                 u32 page_base = node.start;
 406                 unsigned page_offset = offset_in_page(offset);
 407                 unsigned page_length = PAGE_SIZE - page_offset;
 408                 page_length = remain < page_length ? remain : page_length;
 409                 if (drm_mm_node_allocated(&node)) {
 410                         ggtt->vm.insert_page(&ggtt->vm,
 411                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 412                                              node.start, I915_CACHE_NONE, 0);
 413                 } else {
 414                         page_base += offset & PAGE_MASK;
 415                 }
 416
 417                 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 418                                   user_data, page_length)) {
 419                         ret = -EFAULT;
 420                         break;
 421                 }
 422
 423                 remain -= page_length;
 424                 user_data += page_length;
 425                 offset += page_length;
 426         }
 427
 428         i915_gem_gtt_cleanup(obj, &node, vma);
 429 out_rpm:
 430         intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 431         return ret;
 432 }
 433
 434 /**
 435  * Reads data from the object referenced by handle.
 436  * @dev: drm device pointer
 437  * @data: ioctl data blob
 438  * @file: drm file pointer
 439  *
 440  * On error, the contents of *data are undefined.
 441  */
 442 int
 443 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 444                      struct drm_file *file)
 445 {
 446         struct drm_i915_private *i915 = to_i915(dev);
 447         struct drm_i915_gem_pread *args = data;
 448         struct drm_i915_gem_object *obj;
 449         int ret;
 450
 451         /* PREAD is disallowed for all platforms after TGL-LP.  This also
 452          * covers all platforms with local memory.
 453          */
 454         if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 455                 return -EOPNOTSUPP;
 456
 457         if (args->size == 0)
 458                 return 0;
 459
 460         if (!access_ok(u64_to_user_ptr(args->data_ptr),
 461                        args->size))
 462                 return -EFAULT;
 463
 464         obj = i915_gem_object_lookup(file, args->handle);
 465         if (!obj)
 466                 return -ENOENT;
 467
 468         /* Bounds check source.  */
 469         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 470                 ret = -EINVAL;
 471                 goto out;
 472         }
 473
 474         trace_i915_gem_object_pread(obj, args->offset, args->size);
 475         ret = -ENODEV;
 476         if (obj->ops->pread)
 477                 ret = obj->ops->pread(obj, args);
 478         if (ret != -ENODEV)
 479                 goto out;
 480
 481         ret = i915_gem_object_wait(obj,
 482                                    I915_WAIT_INTERRUPTIBLE,
 483                                    MAX_SCHEDULE_TIMEOUT);
 484         if (ret)
 485                 goto out;
 486
 487         ret = i915_gem_shmem_pread(obj, args);
 488         if (ret == -EFAULT || ret == -ENODEV)
 489                 ret = i915_gem_gtt_pread(obj, args);
 490
 491 out:
 492         i915_gem_object_put(obj);
 493         return ret;
 494 }
 495
 496 /* This is the fast write path which cannot handle
 497  * page faults in the source data
 498  */
 499
 500 static inline bool
 501 ggtt_write(struct io_mapping *mapping,
 502            loff_t base, int offset,
 503            char __user *user_data, int length)
 504 {
 505         void __iomem *vaddr;
 506         unsigned long unwritten;
 507
 508         /* We can use the cpu mem copy function because this is X86. */
 509         vaddr = io_mapping_map_atomic_wc(mapping, base);
 510         unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
 511                                                       user_data, length);
 512         io_mapping_unmap_atomic(vaddr);
 513         if (unwritten) {
 514                 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 515                 unwritten = copy_from_user((void __force *)vaddr + offset,
 516                                            user_data, length);
 517                 io_mapping_unmap(vaddr);
 518         }
 519
 520         return unwritten;
 521 }
 522
 523 /**
 524  * This is the fast pwrite path, where we copy the data directly from the
 525  * user into the GTT, uncached.
 526  * @obj: i915 GEM object
 527  * @args: pwrite arguments structure
 528  */
 529 static int
 530 i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 531                          const struct drm_i915_gem_pwrite *args)
 532 {
 533         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 534         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 535         struct intel_runtime_pm *rpm = &i915->runtime_pm;
 536         intel_wakeref_t wakeref;
 537         struct drm_mm_node node;
 538         struct i915_vma *vma;
 539         u64 remain, offset;
 540         void __user *user_data;
 541         int ret = 0;
 542
 543         if (i915_gem_object_has_struct_page(obj)) {
 544                 /*
 545                  * Avoid waking the device up if we can fallback, as
 546                  * waking/resuming is very slow (worst-case 10-100 ms
 547                  * depending on PCI sleeps and our own resume time).
 548                  * This easily dwarfs any performance advantage from
 549                  * using the cache bypass of indirect GGTT access.
 550                  */
 551                 wakeref = intel_runtime_pm_get_if_in_use(rpm);
 552                 if (!wakeref)
 553                         return -EFAULT;
 554         } else {
 555                 /* No backing pages, no fallback, we must force GGTT access */
 556                 wakeref = intel_runtime_pm_get(rpm);
 557         }
 558
 559         vma = i915_gem_gtt_prepare(obj, &node, true);
 560         if (IS_ERR(vma)) {
 561                 ret = PTR_ERR(vma);
 562                 goto out_rpm;
 563         }
 564
 565         i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 566
 567         user_data = u64_to_user_ptr(args->data_ptr);
 568         offset = args->offset;
 569         remain = args->size;
 570         while (remain) {
 571                 /* Operation in this page
 572                  *
 573                  * page_base = page offset within aperture
 574                  * page_offset = offset within page
 575                  * page_length = bytes to copy for this page
 576                  */
 577                 u32 page_base = node.start;
 578                 unsigned int page_offset = offset_in_page(offset);
 579                 unsigned int page_length = PAGE_SIZE - page_offset;
 580                 page_length = remain < page_length ? remain : page_length;
 581                 if (drm_mm_node_allocated(&node)) {
 582                         /* flush the write before we modify the GGTT */
 583                         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 584                         ggtt->vm.insert_page(&ggtt->vm,
 585                                              i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 586                                              node.start, I915_CACHE_NONE, 0);
 587                         wmb(); /* flush modifications to the GGTT (insert_page) */
 588                 } else {
 589                         page_base += offset & PAGE_MASK;
 590                 }
 591                 /* If we get a fault while copying data, then (presumably) our
 592                  * source page isn't available.  Return the error and we'll
 593                  * retry in the slow path.
 594                  * If the object is non-shmem backed, we retry again with the
 595                  * path that handles page fault.
 596                  */
 597                 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 598                                user_data, page_length)) {
 599                         ret = -EFAULT;
 600                         break;
 601                 }
 602
 603                 remain -= page_length;
 604                 user_data += page_length;
 605                 offset += page_length;
 606         }
 607
 608         intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 609         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 610
 611         i915_gem_gtt_cleanup(obj, &node, vma);
 612 out_rpm:
 613         intel_runtime_pm_put(rpm, wakeref);
 614         return ret;
 615 }
 616
 617 /* Per-page copy function for the shmem pwrite fastpath.
 618  * Flushes invalid cachelines before writing to the target if
 619  * needs_clflush_before is set and flushes out any written cachelines after
 620  * writing if needs_clflush is set.
 621  */
 622 static int
 623 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
 624              bool needs_clflush_before,
 625              bool needs_clflush_after)
 626 {
 627         char *vaddr;
 628         int ret;
 629
 630         vaddr = kmap(page);
 631
 632         if (needs_clflush_before)
 633                 drm_clflush_virt_range(vaddr + offset, len);
 634
 635         ret = __copy_from_user(vaddr + offset, user_data, len);
 636         if (!ret && needs_clflush_after)
 637                 drm_clflush_virt_range(vaddr + offset, len);
 638
 639         kunmap(page);
 640
 641         return ret ? -EFAULT : 0;
 642 }
 643
 644 static int
 645 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 646                       const struct drm_i915_gem_pwrite *args)
 647 {
 648         unsigned int partial_cacheline_write;
 649         unsigned int needs_clflush;
 650         unsigned int offset, idx;
 651         void __user *user_data;
 652         u64 remain;
 653         int ret;
 654
 655         ret = i915_gem_object_lock_interruptible(obj, NULL);
 656         if (ret)
 657                 return ret;
 658
 659         ret = i915_gem_object_pin_pages(obj);
 660         if (ret)
 661                 goto err_unlock;
 662
 663         ret = i915_gem_object_prepare_write(obj, &needs_clflush);
 664         if (ret)
 665                 goto err_unpin;
 666
 667         i915_gem_object_finish_access(obj);
 668         i915_gem_object_unlock(obj);
 669
 670         /* If we don't overwrite a cacheline completely we need to be
 671          * careful to have up-to-date data by first clflushing. Don't
 672          * overcomplicate things and flush the entire patch.
 673          */
 674         partial_cacheline_write = 0;
 675         if (needs_clflush & CLFLUSH_BEFORE)
 676                 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 677
 678         user_data = u64_to_user_ptr(args->data_ptr);
 679         remain = args->size;
 680         offset = offset_in_page(args->offset);
 681         for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 682                 struct page *page = i915_gem_object_get_page(obj, idx);
 683                 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 684
 685                 ret = shmem_pwrite(page, offset, length, user_data,
 686                                    (offset | length) & partial_cacheline_write,
 687                                    needs_clflush & CLFLUSH_AFTER);
 688                 if (ret)
 689                         break;
 690
 691                 remain -= length;
 692                 user_data += length;
 693                 offset = 0;
 694         }
 695
 696         i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 697
 698         i915_gem_object_unpin_pages(obj);
 699         return ret;
 700
 701 err_unpin:
 702         i915_gem_object_unpin_pages(obj);
 703 err_unlock:
 704         i915_gem_object_unlock(obj);
 705         return ret;
 706 }
 707
 708 /**
 709  * Writes data to the object referenced by handle.
 710  * @dev: drm device
 711  * @data: ioctl data blob
 712  * @file: drm file
 713  *
 714  * On error, the contents of the buffer that were to be modified are undefined.
 715  */
 716 int
 717 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 718                       struct drm_file *file)
 719 {
 720         struct drm_i915_private *i915 = to_i915(dev);
 721         struct drm_i915_gem_pwrite *args = data;
 722         struct drm_i915_gem_object *obj;
 723         int ret;
 724
 725         /* PWRITE is disallowed for all platforms after TGL-LP.  This also
 726          * covers all platforms with local memory.
 727          */
 728         if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 729                 return -EOPNOTSUPP;
 730
 731         if (args->size == 0)
 732                 return 0;
 733
 734         if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
 735                 return -EFAULT;
 736
 737         obj = i915_gem_object_lookup(file, args->handle);
 738         if (!obj)
 739                 return -ENOENT;
 740
 741         /* Bounds check destination. */
 742         if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 743                 ret = -EINVAL;
 744                 goto err;
 745         }
 746
 747         /* Writes not allowed into this read-only object */
 748         if (i915_gem_object_is_readonly(obj)) {
 749                 ret = -EINVAL;
 750                 goto err;
 751         }
 752
 753         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 754
 755         ret = -ENODEV;
 756         if (obj->ops->pwrite)
 757                 ret = obj->ops->pwrite(obj, args);
 758         if (ret != -ENODEV)
 759                 goto err;
 760
 761         ret = i915_gem_object_wait(obj,
 762                                    I915_WAIT_INTERRUPTIBLE |
 763                                    I915_WAIT_ALL,
 764                                    MAX_SCHEDULE_TIMEOUT);
 765         if (ret)
 766                 goto err;
 767
 768         ret = -EFAULT;
 769         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 770          * it would end up going through the fenced access, and we'll get
 771          * different detiling behavior between reading and writing.
 772          * pread/pwrite currently are reading and writing from the CPU
 773          * perspective, requiring manual detiling by the client.
 774          */
 775         if (!i915_gem_object_has_struct_page(obj) ||
 776             i915_gem_cpu_write_needs_clflush(obj))
 777                 /* Note that the gtt paths might fail with non-page-backed user
 778                  * pointers (e.g. gtt mappings when moving data between
 779                  * textures). Fallback to the shmem path in that case.
 780                  */
 781                 ret = i915_gem_gtt_pwrite_fast(obj, args);
 782
 783         if (ret == -EFAULT || ret == -ENOSPC) {
 784                 if (i915_gem_object_has_struct_page(obj))
 785                         ret = i915_gem_shmem_pwrite(obj, args);
 786         }
 787
 788 err:
 789         i915_gem_object_put(obj);
 790         return ret;
 791 }
 792
 793 /**
 794  * Called when user space has done writes to this buffer
 795  * @dev: drm device
 796  * @data: ioctl data blob
 797  * @file: drm file
 798  */
 799 int
 800 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 801                          struct drm_file *file)
 802 {
 803         struct drm_i915_gem_sw_finish *args = data;
 804         struct drm_i915_gem_object *obj;
 805
 806         obj = i915_gem_object_lookup(file, args->handle);
 807         if (!obj)
 808                 return -ENOENT;
 809
 810         /*
 811          * Proxy objects are barred from CPU access, so there is no
 812          * need to ban sw_finish as it is a nop.
 813          */
 814
 815         /* Pinned buffers may be scanout, so flush the cache */
 816         i915_gem_object_flush_if_display(obj);
 817         i915_gem_object_put(obj);
 818
 819         return 0;
 820 }
 821
 822 void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 823 {
 824         struct drm_i915_gem_object *obj, *on;
 825         int i;
 826
 827         /*
 828          * Only called during RPM suspend. All users of the userfault_list
 829          * must be holding an RPM wakeref to ensure that this can not
 830          * run concurrently with themselves (and use the struct_mutex for
 831          * protection between themselves).
 832          */
 833
 834         list_for_each_entry_safe(obj, on,
 835                                  &to_gt(i915)->ggtt->userfault_list, userfault_link)
 836                 __i915_gem_object_release_mmap_gtt(obj);
 837
 838         /*
 839          * The fence will be lost when the device powers down. If any were
 840          * in use by hardware (i.e. they are pinned), we should not be powering
 841          * down! All other fences will be reacquired by the user upon waking.
 842          */
 843         for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) {
 844                 struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i];
 845
 846                 /*
 847                  * Ideally we want to assert that the fence register is not
 848                  * live at this point (i.e. that no piece of code will be
 849                  * trying to write through fence + GTT, as that both violates
 850                  * our tracking of activity and associated locking/barriers,
 851                  * but also is illegal given that the hw is powered down).
 852                  *
 853                  * Previously we used reg->pin_count as a "liveness" indicator.
 854                  * That is not sufficient, and we need a more fine-grained
 855                  * tool if we want to have a sanity check here.
 856                  */
 857
 858                 if (!reg->vma)
 859                         continue;
 860
 861                 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
 862                 reg->dirty = true;
 863         }
 864 }
 865
 866 static void discard_ggtt_vma(struct i915_vma *vma)
 867 {
 868         struct drm_i915_gem_object *obj = vma->obj;
 869
 870         spin_lock(&obj->vma.lock);
 871         if (!RB_EMPTY_NODE(&vma->obj_node)) {
 872                 rb_erase(&vma->obj_node, &obj->vma.tree);
 873                 RB_CLEAR_NODE(&vma->obj_node);
 874         }
 875         spin_unlock(&obj->vma.lock);
 876 }
 877
 878 struct i915_vma *
 879 i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 880                             struct i915_gem_ww_ctx *ww,
 881                             const struct i915_ggtt_view *view,
 882                             u64 size, u64 alignment, u64 flags)
 883 {
 884         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 885         struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 886         struct i915_vma *vma;
 887         int ret;
 888
 889         GEM_WARN_ON(!ww);
 890
 891         if (flags & PIN_MAPPABLE &&
 892             (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
 893                 /*
 894                  * If the required space is larger than the available
 895                  * aperture, we will not able to find a slot for the
 896                  * object and unbinding the object now will be in
 897                  * vain. Worse, doing so may cause us to ping-pong
 898                  * the object in and out of the Global GTT and
 899                  * waste a lot of cycles under the mutex.
 900                  */
 901                 if (obj->base.size > ggtt->mappable_end)
 902                         return ERR_PTR(-E2BIG);
 903
 904                 /*
 905                  * If NONBLOCK is set the caller is optimistically
 906                  * trying to cache the full object within the mappable
 907                  * aperture, and *must* have a fallback in place for
 908                  * situations where we cannot bind the object. We
 909                  * can be a little more lax here and use the fallback
 910                  * more often to avoid costly migrations of ourselves
 911                  * and other objects within the aperture.
 912                  *
 913                  * Half-the-aperture is used as a simple heuristic.
 914                  * More interesting would to do search for a free
 915                  * block prior to making the commitment to unbind.
 916                  * That caters for the self-harm case, and with a
 917                  * little more heuristics (e.g. NOFAULT, NOEVICT)
 918                  * we could try to minimise harm to others.
 919                  */
 920                 if (flags & PIN_NONBLOCK &&
 921                     obj->base.size > ggtt->mappable_end / 2)
 922                         return ERR_PTR(-ENOSPC);
 923         }
 924
 925 new_vma:
 926         vma = i915_vma_instance(obj, &ggtt->vm, view);
 927         if (IS_ERR(vma))
 928                 return vma;
 929
 930         if (i915_vma_misplaced(vma, size, alignment, flags)) {
 931                 if (flags & PIN_NONBLOCK) {
 932                         if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
 933                                 return ERR_PTR(-ENOSPC);
 934
 935                         if (flags & PIN_MAPPABLE &&
 936                             vma->fence_size > ggtt->mappable_end / 2)
 937                                 return ERR_PTR(-ENOSPC);
 938                 }
 939
 940                 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
 941                         discard_ggtt_vma(vma);
 942                         goto new_vma;
 943                 }
 944
 945                 ret = i915_vma_unbind(vma);
 946                 if (ret)
 947                         return ERR_PTR(ret);
 948         }
 949
 950         ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
 951
 952         if (ret)
 953                 return ERR_PTR(ret);
 954
 955         if (vma->fence && !i915_gem_object_is_tiled(obj)) {
 956                 mutex_lock(&ggtt->vm.mutex);
 957                 i915_vma_revoke_fence(vma);
 958                 mutex_unlock(&ggtt->vm.mutex);
 959         }
 960
 961         ret = i915_vma_wait_for_bind(vma);
 962         if (ret) {
 963                 i915_vma_unpin(vma);
 964                 return ERR_PTR(ret);
 965         }
 966
 967         return vma;
 968 }
 969
 970 struct i915_vma * __must_check
 971 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 972                          const struct i915_ggtt_view *view,
 973                          u64 size, u64 alignment, u64 flags)
 974 {
 975         struct i915_gem_ww_ctx ww;
 976         struct i915_vma *ret;
 977         int err;
 978
 979         for_i915_gem_ww(&ww, err, true) {
 980                 err = i915_gem_object_lock(obj, &ww);
 981                 if (err)
 982                         continue;
 983
 984                 ret = i915_gem_object_ggtt_pin_ww(obj, &ww, view, size,
 985                                                   alignment, flags);
 986                 if (IS_ERR(ret))
 987                         err = PTR_ERR(ret);
 988         }
 989
 990         return err ? ERR_PTR(err) : ret;
 991 }
 992
 993 int
 994 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 995                        struct drm_file *file_priv)
 996 {
 997         struct drm_i915_private *i915 = to_i915(dev);
 998         struct drm_i915_gem_madvise *args = data;
 999         struct drm_i915_gem_object *obj;
1000         int err;
1001
1002         switch (args->madv) {
1003         case I915_MADV_DONTNEED:
1004         case I915_MADV_WILLNEED:
1005             break;
1006         default:
1007             return -EINVAL;
1008         }
1009
1010         obj = i915_gem_object_lookup(file_priv, args->handle);
1011         if (!obj)
1012                 return -ENOENT;
1013
1014         err = i915_gem_object_lock_interruptible(obj, NULL);
1015         if (err)
1016                 goto out;
1017
1018         if (i915_gem_object_has_pages(obj) &&
1019             i915_gem_object_is_tiled(obj) &&
1020             i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
1021                 if (obj->mm.madv == I915_MADV_WILLNEED) {
1022                         GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
1023                         i915_gem_object_clear_tiling_quirk(obj);
1024                         i915_gem_object_make_shrinkable(obj);
1025                 }
1026                 if (args->madv == I915_MADV_WILLNEED) {
1027                         GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
1028                         i915_gem_object_make_unshrinkable(obj);
1029                         i915_gem_object_set_tiling_quirk(obj);
1030                 }
1031         }
1032
1033         if (obj->mm.madv != __I915_MADV_PURGED) {
1034                 obj->mm.madv = args->madv;
1035                 if (obj->ops->adjust_lru)
1036                         obj->ops->adjust_lru(obj);
1037         }
1038
1039         if (i915_gem_object_has_pages(obj) ||
1040             i915_gem_object_has_self_managed_shrink_list(obj)) {
1041                 unsigned long flags;
1042
1043                 spin_lock_irqsave(&i915->mm.obj_lock, flags);
1044                 if (!list_empty(&obj->mm.link)) {
1045                         struct list_head *list;
1046
1047                         if (obj->mm.madv != I915_MADV_WILLNEED)
1048                                 list = &i915->mm.purge_list;
1049                         else
1050                                 list = &i915->mm.shrink_list;
1051                         list_move_tail(&obj->mm.link, list);
1052
1053                 }
1054                 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1055         }
1056
1057         /* if the object is no longer attached, discard its backing storage */
1058         if (obj->mm.madv == I915_MADV_DONTNEED &&
1059             !i915_gem_object_has_pages(obj))
1060                 i915_gem_object_truncate(obj);
1061
1062         args->retained = obj->mm.madv != __I915_MADV_PURGED;
1063
1064         i915_gem_object_unlock(obj);
1065 out:
1066         i915_gem_object_put(obj);
1067         return err;
1068 }
1069
1070 int i915_gem_init(struct drm_i915_private *dev_priv)
1071 {
1072         int ret;
1073
1074         /* We need to fallback to 4K pages if host doesn't support huge gtt. */
1075         if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1076                 mkwrite_device_info(dev_priv)->page_sizes =
1077                         I915_GTT_PAGE_SIZE_4K;
1078
1079         ret = i915_gem_init_userptr(dev_priv);
1080         if (ret)
1081                 return ret;
1082
1083         intel_uc_fetch_firmwares(&to_gt(dev_priv)->uc);
1084         intel_wopcm_init(&dev_priv->wopcm);
1085
1086         ret = i915_init_ggtt(dev_priv);
1087         if (ret) {
1088                 GEM_BUG_ON(ret == -EIO);
1089                 goto err_unlock;
1090         }
1091
1092         /*
1093          * Despite its name intel_init_clock_gating applies both display
1094          * clock gating workarounds; GT mmio workarounds and the occasional
1095          * GT power context workaround. Worse, sometimes it includes a context
1096          * register workaround which we need to apply before we record the
1097          * default HW state for all contexts.
1098          *
1099          * FIXME: break up the workarounds and apply them at the right time!
1100          */
1101         intel_init_clock_gating(dev_priv);
1102
1103         ret = intel_gt_init(to_gt(dev_priv));
1104         if (ret)
1105                 goto err_unlock;
1106
1107         return 0;
1108
1109         /*
1110          * Unwinding is complicated by that we want to handle -EIO to mean
1111          * disable GPU submission but keep KMS alive. We want to mark the
1112          * HW as irrevisibly wedged, but keep enough state around that the
1113          * driver doesn't explode during runtime.
1114          */
1115 err_unlock:
1116         i915_gem_drain_workqueue(dev_priv);
1117
1118         if (ret != -EIO)
1119                 intel_uc_cleanup_firmwares(&to_gt(dev_priv)->uc);
1120
1121         if (ret == -EIO) {
1122                 /*
1123                  * Allow engines or uC initialisation to fail by marking the GPU
1124                  * as wedged. But we only want to do this when the GPU is angry,
1125                  * for all other failure, such as an allocation failure, bail.
1126                  */
1127                 if (!intel_gt_is_wedged(to_gt(dev_priv))) {
1128                         i915_probe_error(dev_priv,
1129                                          "Failed to initialize GPU, declaring it wedged!\n");
1130                         intel_gt_set_wedged(to_gt(dev_priv));
1131                 }
1132
1133                 /* Minimal basic recovery for KMS */
1134                 ret = i915_ggtt_enable_hw(dev_priv);
1135                 i915_ggtt_resume(to_gt(dev_priv)->ggtt);
1136                 intel_init_clock_gating(dev_priv);
1137         }
1138
1139         i915_gem_drain_freed_objects(dev_priv);
1140
1141         return ret;
1142 }
1143
1144 void i915_gem_driver_register(struct drm_i915_private *i915)
1145 {
1146         i915_gem_driver_register__shrinker(i915);
1147
1148         intel_engines_driver_register(i915);
1149 }
1150
1151 void i915_gem_driver_unregister(struct drm_i915_private *i915)
1152 {
1153         i915_gem_driver_unregister__shrinker(i915);
1154 }
1155
1156 void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1157 {
1158         intel_wakeref_auto_fini(&to_gt(dev_priv)->ggtt->userfault_wakeref);
1159
1160         i915_gem_suspend_late(dev_priv);
1161         intel_gt_driver_remove(to_gt(dev_priv));
1162         dev_priv->uabi_engines = RB_ROOT;
1163
1164         /* Flush any outstanding unpin_work. */
1165         i915_gem_drain_workqueue(dev_priv);
1166
1167         i915_gem_drain_freed_objects(dev_priv);
1168 }
1169
1170 void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1171 {
1172         intel_gt_driver_release(to_gt(dev_priv));
1173
1174         intel_uc_cleanup_firmwares(&to_gt(dev_priv)->uc);
1175
1176         i915_gem_drain_freed_objects(dev_priv);
1177
1178         drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));
1179 }
1180
1181 static void i915_gem_init__mm(struct drm_i915_private *i915)
1182 {
1183         spin_lock_init(&i915->mm.obj_lock);
1184
1185         init_llist_head(&i915->mm.free_list);
1186
1187         INIT_LIST_HEAD(&i915->mm.purge_list);
1188         INIT_LIST_HEAD(&i915->mm.shrink_list);
1189
1190         i915_gem_init__objects(i915);
1191 }
1192
1193 void i915_gem_init_early(struct drm_i915_private *dev_priv)
1194 {
1195         i915_gem_init__mm(dev_priv);
1196         i915_gem_init__contexts(dev_priv);
1197
1198         spin_lock_init(&dev_priv->fb_tracking.lock);
1199 }
1200
1201 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1202 {
1203         i915_gem_drain_freed_objects(dev_priv);
1204         GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1205         GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1206         drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);
1207 }
1208
1209 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1210 {
1211         struct drm_i915_file_private *file_priv;
1212         int ret;
1213
1214         DRM_DEBUG("\n");
1215
1216         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1217         if (!file_priv)
1218                 return -ENOMEM;
1219
1220         file->driver_priv = file_priv;
1221         file_priv->dev_priv = i915;
1222         file_priv->file = file;
1223
1224         file_priv->bsd_engine = -1;
1225         file_priv->hang_timestamp = jiffies;
1226
1227         ret = i915_gem_context_open(i915, file);
1228         if (ret)
1229                 kfree(file_priv);
1230
1231         return ret;
1232 }
1233
1234 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1235 #include "selftests/mock_gem_device.c"
1236 #include "selftests/i915_gem.c"
1237 #endif