drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_gem_dmabuf.h"
  33 #include "i915_vgpu.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36 #include "intel_mocs.h"
  37 #include <linux/reservation.h>
  38 #include <linux/shmem_fs.h>
  39 #include <linux/slab.h>
  40 #include <linux/swap.h>
  41 #include <linux/pci.h>
  42 #include <linux/dma-buf.h>
  43
  44 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  45 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  46 static void
  47 i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
  48 static void
  49 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int engine);
  50
  51 static bool cpu_cache_is_coherent(struct drm_device *dev,
  52                                   enum i915_cache_level level)
  53 {
  54         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  55 }
  56
  57 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  58 {
  59         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  60                 return false;
  61
  62         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  63                 return true;
  64
  65         return obj->pin_display;
  66 }
  67
  68 static int
  69 insert_mappable_node(struct drm_i915_private *i915,
  70                      struct drm_mm_node *node, u32 size)
  71 {
  72         memset(node, 0, sizeof(*node));
  73         return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
  74                                                    size, 0, 0, 0,
  75                                                    i915->ggtt.mappable_end,
  76                                                    DRM_MM_SEARCH_DEFAULT,
  77                                                    DRM_MM_CREATE_DEFAULT);
  78 }
  79
  80 static void
  81 remove_mappable_node(struct drm_mm_node *node)
  82 {
  83         drm_mm_remove_node(node);
  84 }
  85
  86 /* some bookkeeping */
  87 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  88                                   size_t size)
  89 {
  90         spin_lock(&dev_priv->mm.object_stat_lock);
  91         dev_priv->mm.object_count++;
  92         dev_priv->mm.object_memory += size;
  93         spin_unlock(&dev_priv->mm.object_stat_lock);
  94 }
  95
  96 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  97                                      size_t size)
  98 {
  99         spin_lock(&dev_priv->mm.object_stat_lock);
 100         dev_priv->mm.object_count--;
 101         dev_priv->mm.object_memory -= size;
 102         spin_unlock(&dev_priv->mm.object_stat_lock);
 103 }
 104
 105 static int
 106 i915_gem_wait_for_error(struct i915_gpu_error *error)
 107 {
 108         int ret;
 109
 110         if (!i915_reset_in_progress(error))
 111                 return 0;
 112
 113         /*
 114          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 115          * userspace. If it takes that long something really bad is going on and
 116          * we should simply try to bail out and fail as gracefully as possible.
 117          */
 118         ret = wait_event_interruptible_timeout(error->reset_queue,
 119                                                !i915_reset_in_progress(error),
 120                                                10*HZ);
 121         if (ret == 0) {
 122                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 123                 return -EIO;
 124         } else if (ret < 0) {
 125                 return ret;
 126         } else {
 127                 return 0;
 128         }
 129 }
 130
 131 int i915_mutex_lock_interruptible(struct drm_device *dev)
 132 {
 133         struct drm_i915_private *dev_priv = to_i915(dev);
 134         int ret;
 135
 136         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 137         if (ret)
 138                 return ret;
 139
 140         ret = mutex_lock_interruptible(&dev->struct_mutex);
 141         if (ret)
 142                 return ret;
 143
 144         WARN_ON(i915_verify_lists(dev));
 145         return 0;
 146 }
 147
 148 int
 149 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 150                             struct drm_file *file)
 151 {
 152         struct drm_i915_private *dev_priv = to_i915(dev);
 153         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 154         struct drm_i915_gem_get_aperture *args = data;
 155         struct i915_vma *vma;
 156         size_t pinned;
 157
 158         pinned = 0;
 159         mutex_lock(&dev->struct_mutex);
 160         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
 161                 if (vma->pin_count)
 162                         pinned += vma->node.size;
 163         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
 164                 if (vma->pin_count)
 165                         pinned += vma->node.size;
 166         mutex_unlock(&dev->struct_mutex);
 167
 168         args->aper_size = ggtt->base.total;
 169         args->aper_available_size = args->aper_size - pinned;
 170
 171         return 0;
 172 }
 173
 174 static int
 175 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 176 {
 177         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 178         char *vaddr = obj->phys_handle->vaddr;
 179         struct sg_table *st;
 180         struct scatterlist *sg;
 181         int i;
 182
 183         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 184                 return -EINVAL;
 185
 186         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 187                 struct page *page;
 188                 char *src;
 189
 190                 page = shmem_read_mapping_page(mapping, i);
 191                 if (IS_ERR(page))
 192                         return PTR_ERR(page);
 193
 194                 src = kmap_atomic(page);
 195                 memcpy(vaddr, src, PAGE_SIZE);
 196                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 197                 kunmap_atomic(src);
 198
 199                 put_page(page);
 200                 vaddr += PAGE_SIZE;
 201         }
 202
 203         i915_gem_chipset_flush(to_i915(obj->base.dev));
 204
 205         st = kmalloc(sizeof(*st), GFP_KERNEL);
 206         if (st == NULL)
 207                 return -ENOMEM;
 208
 209         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 210                 kfree(st);
 211                 return -ENOMEM;
 212         }
 213
 214         sg = st->sgl;
 215         sg->offset = 0;
 216         sg->length = obj->base.size;
 217
 218         sg_dma_address(sg) = obj->phys_handle->busaddr;
 219         sg_dma_len(sg) = obj->base.size;
 220
 221         obj->pages = st;
 222         return 0;
 223 }
 224
 225 static void
 226 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 227 {
 228         int ret;
 229
 230         BUG_ON(obj->madv == __I915_MADV_PURGED);
 231
 232         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 233         if (WARN_ON(ret)) {
 234                 /* In the event of a disaster, abandon all caches and
 235                  * hope for the best.
 236                  */
 237                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 238         }
 239
 240         if (obj->madv == I915_MADV_DONTNEED)
 241                 obj->dirty = 0;
 242
 243         if (obj->dirty) {
 244                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 245                 char *vaddr = obj->phys_handle->vaddr;
 246                 int i;
 247
 248                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 249                         struct page *page;
 250                         char *dst;
 251
 252                         page = shmem_read_mapping_page(mapping, i);
 253                         if (IS_ERR(page))
 254                                 continue;
 255
 256                         dst = kmap_atomic(page);
 257                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 258                         memcpy(dst, vaddr, PAGE_SIZE);
 259                         kunmap_atomic(dst);
 260
 261                         set_page_dirty(page);
 262                         if (obj->madv == I915_MADV_WILLNEED)
 263                                 mark_page_accessed(page);
 264                         put_page(page);
 265                         vaddr += PAGE_SIZE;
 266                 }
 267                 obj->dirty = 0;
 268         }
 269
 270         sg_free_table(obj->pages);
 271         kfree(obj->pages);
 272 }
 273
 274 static void
 275 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 276 {
 277         drm_pci_free(obj->base.dev, obj->phys_handle);
 278 }
 279
 280 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 281         .get_pages = i915_gem_object_get_pages_phys,
 282         .put_pages = i915_gem_object_put_pages_phys,
 283         .release = i915_gem_object_release_phys,
 284 };
 285
 286 static int
 287 drop_pages(struct drm_i915_gem_object *obj)
 288 {
 289         struct i915_vma *vma, *next;
 290         int ret;
 291
 292         i915_gem_object_get(obj);
 293         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
 294                 if (i915_vma_unbind(vma))
 295                         break;
 296
 297         ret = i915_gem_object_put_pages(obj);
 298         i915_gem_object_put(obj);
 299
 300         return ret;
 301 }
 302
 303 int
 304 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 305                             int align)
 306 {
 307         drm_dma_handle_t *phys;
 308         int ret;
 309
 310         if (obj->phys_handle) {
 311                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 312                         return -EBUSY;
 313
 314                 return 0;
 315         }
 316
 317         if (obj->madv != I915_MADV_WILLNEED)
 318                 return -EFAULT;
 319
 320         if (obj->base.filp == NULL)
 321                 return -EINVAL;
 322
 323         ret = drop_pages(obj);
 324         if (ret)
 325                 return ret;
 326
 327         /* create a new object */
 328         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 329         if (!phys)
 330                 return -ENOMEM;
 331
 332         obj->phys_handle = phys;
 333         obj->ops = &i915_gem_phys_ops;
 334
 335         return i915_gem_object_get_pages(obj);
 336 }
 337
 338 static int
 339 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 340                      struct drm_i915_gem_pwrite *args,
 341                      struct drm_file *file_priv)
 342 {
 343         struct drm_device *dev = obj->base.dev;
 344         void *vaddr = obj->phys_handle->vaddr + args->offset;
 345         char __user *user_data = u64_to_user_ptr(args->data_ptr);
 346         int ret = 0;
 347
 348         /* We manually control the domain here and pretend that it
 349          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 350          */
 351         ret = i915_gem_object_wait_rendering(obj, false);
 352         if (ret)
 353                 return ret;
 354
 355         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 356         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 357                 unsigned long unwritten;
 358
 359                 /* The physical object once assigned is fixed for the lifetime
 360                  * of the obj, so we can safely drop the lock and continue
 361                  * to access vaddr.
 362                  */
 363                 mutex_unlock(&dev->struct_mutex);
 364                 unwritten = copy_from_user(vaddr, user_data, args->size);
 365                 mutex_lock(&dev->struct_mutex);
 366                 if (unwritten) {
 367                         ret = -EFAULT;
 368                         goto out;
 369                 }
 370         }
 371
 372         drm_clflush_virt_range(vaddr, args->size);
 373         i915_gem_chipset_flush(to_i915(dev));
 374
 375 out:
 376         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 377         return ret;
 378 }
 379
 380 void *i915_gem_object_alloc(struct drm_device *dev)
 381 {
 382         struct drm_i915_private *dev_priv = to_i915(dev);
 383         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 384 }
 385
 386 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 387 {
 388         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 389         kmem_cache_free(dev_priv->objects, obj);
 390 }
 391
 392 static int
 393 i915_gem_create(struct drm_file *file,
 394                 struct drm_device *dev,
 395                 uint64_t size,
 396                 uint32_t *handle_p)
 397 {
 398         struct drm_i915_gem_object *obj;
 399         int ret;
 400         u32 handle;
 401
 402         size = roundup(size, PAGE_SIZE);
 403         if (size == 0)
 404                 return -EINVAL;
 405
 406         /* Allocate the new object */
 407         obj = i915_gem_object_create(dev, size);
 408         if (IS_ERR(obj))
 409                 return PTR_ERR(obj);
 410
 411         ret = drm_gem_handle_create(file, &obj->base, &handle);
 412         /* drop reference from allocate - handle holds it now */
 413         i915_gem_object_put_unlocked(obj);
 414         if (ret)
 415                 return ret;
 416
 417         *handle_p = handle;
 418         return 0;
 419 }
 420
 421 int
 422 i915_gem_dumb_create(struct drm_file *file,
 423                      struct drm_device *dev,
 424                      struct drm_mode_create_dumb *args)
 425 {
 426         /* have to work out size/pitch and return them */
 427         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 428         args->size = args->pitch * args->height;
 429         return i915_gem_create(file, dev,
 430                                args->size, &args->handle);
 431 }
 432
 433 /**
 434  * Creates a new mm object and returns a handle to it.
 435  * @dev: drm device pointer
 436  * @data: ioctl data blob
 437  * @file: drm file pointer
 438  */
 439 int
 440 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 441                       struct drm_file *file)
 442 {
 443         struct drm_i915_gem_create *args = data;
 444
 445         return i915_gem_create(file, dev,
 446                                args->size, &args->handle);
 447 }
 448
 449 static inline int
 450 __copy_to_user_swizzled(char __user *cpu_vaddr,
 451                         const char *gpu_vaddr, int gpu_offset,
 452                         int length)
 453 {
 454         int ret, cpu_offset = 0;
 455
 456         while (length > 0) {
 457                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 458                 int this_length = min(cacheline_end - gpu_offset, length);
 459                 int swizzled_gpu_offset = gpu_offset ^ 64;
 460
 461                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 462                                      gpu_vaddr + swizzled_gpu_offset,
 463                                      this_length);
 464                 if (ret)
 465                         return ret + length;
 466
 467                 cpu_offset += this_length;
 468                 gpu_offset += this_length;
 469                 length -= this_length;
 470         }
 471
 472         return 0;
 473 }
 474
 475 static inline int
 476 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 477                           const char __user *cpu_vaddr,
 478                           int length)
 479 {
 480         int ret, cpu_offset = 0;
 481
 482         while (length > 0) {
 483                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 484                 int this_length = min(cacheline_end - gpu_offset, length);
 485                 int swizzled_gpu_offset = gpu_offset ^ 64;
 486
 487                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 488                                        cpu_vaddr + cpu_offset,
 489                                        this_length);
 490                 if (ret)
 491                         return ret + length;
 492
 493                 cpu_offset += this_length;
 494                 gpu_offset += this_length;
 495                 length -= this_length;
 496         }
 497
 498         return 0;
 499 }
 500
 501 /*
 502  * Pins the specified object's pages and synchronizes the object with
 503  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 504  * flush the object from the CPU cache.
 505  */
 506 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 507                                     int *needs_clflush)
 508 {
 509         int ret;
 510
 511         *needs_clflush = 0;
 512
 513         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
 514                 return -EINVAL;
 515
 516         ret = i915_gem_object_wait_rendering(obj, true);
 517         if (ret)
 518                 return ret;
 519
 520         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 521                 /* If we're not in the cpu read domain, set ourself into the gtt
 522                  * read domain and manually flush cachelines (if required). This
 523                  * optimizes for the case when the gpu will dirty the data
 524                  * anyway again before the next pread happens. */
 525                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 526                                                         obj->cache_level);
 527         }
 528
 529         ret = i915_gem_object_get_pages(obj);
 530         if (ret)
 531                 return ret;
 532
 533         i915_gem_object_pin_pages(obj);
 534
 535         return ret;
 536 }
 537
 538 /* Per-page copy function for the shmem pread fastpath.
 539  * Flushes invalid cachelines before reading the target if
 540  * needs_clflush is set. */
 541 static int
 542 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 543                  char __user *user_data,
 544                  bool page_do_bit17_swizzling, bool needs_clflush)
 545 {
 546         char *vaddr;
 547         int ret;
 548
 549         if (unlikely(page_do_bit17_swizzling))
 550                 return -EINVAL;
 551
 552         vaddr = kmap_atomic(page);
 553         if (needs_clflush)
 554                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 555                                        page_length);
 556         ret = __copy_to_user_inatomic(user_data,
 557                                       vaddr + shmem_page_offset,
 558                                       page_length);
 559         kunmap_atomic(vaddr);
 560
 561         return ret ? -EFAULT : 0;
 562 }
 563
 564 static void
 565 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 566                              bool swizzled)
 567 {
 568         if (unlikely(swizzled)) {
 569                 unsigned long start = (unsigned long) addr;
 570                 unsigned long end = (unsigned long) addr + length;
 571
 572                 /* For swizzling simply ensure that we always flush both
 573                  * channels. Lame, but simple and it works. Swizzled
 574                  * pwrite/pread is far from a hotpath - current userspace
 575                  * doesn't use it at all. */
 576                 start = round_down(start, 128);
 577                 end = round_up(end, 128);
 578
 579                 drm_clflush_virt_range((void *)start, end - start);
 580         } else {
 581                 drm_clflush_virt_range(addr, length);
 582         }
 583
 584 }
 585
 586 /* Only difference to the fast-path function is that this can handle bit17
 587  * and uses non-atomic copy and kmap functions. */
 588 static int
 589 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 590                  char __user *user_data,
 591                  bool page_do_bit17_swizzling, bool needs_clflush)
 592 {
 593         char *vaddr;
 594         int ret;
 595
 596         vaddr = kmap(page);
 597         if (needs_clflush)
 598                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 599                                              page_length,
 600                                              page_do_bit17_swizzling);
 601
 602         if (page_do_bit17_swizzling)
 603                 ret = __copy_to_user_swizzled(user_data,
 604                                               vaddr, shmem_page_offset,
 605                                               page_length);
 606         else
 607                 ret = __copy_to_user(user_data,
 608                                      vaddr + shmem_page_offset,
 609                                      page_length);
 610         kunmap(page);
 611
 612         return ret ? - EFAULT : 0;
 613 }
 614
 615 static inline unsigned long
 616 slow_user_access(struct io_mapping *mapping,
 617                  uint64_t page_base, int page_offset,
 618                  char __user *user_data,
 619                  unsigned long length, bool pwrite)
 620 {
 621         void __iomem *ioaddr;
 622         void *vaddr;
 623         uint64_t unwritten;
 624
 625         ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
 626         /* We can use the cpu mem copy function because this is X86. */
 627         vaddr = (void __force *)ioaddr + page_offset;
 628         if (pwrite)
 629                 unwritten = __copy_from_user(vaddr, user_data, length);
 630         else
 631                 unwritten = __copy_to_user(user_data, vaddr, length);
 632
 633         io_mapping_unmap(ioaddr);
 634         return unwritten;
 635 }
 636
 637 static int
 638 i915_gem_gtt_pread(struct drm_device *dev,
 639                    struct drm_i915_gem_object *obj, uint64_t size,
 640                    uint64_t data_offset, uint64_t data_ptr)
 641 {
 642         struct drm_i915_private *dev_priv = to_i915(dev);
 643         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 644         struct drm_mm_node node;
 645         char __user *user_data;
 646         uint64_t remain;
 647         uint64_t offset;
 648         int ret;
 649
 650         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
 651         if (ret) {
 652                 ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
 653                 if (ret)
 654                         goto out;
 655
 656                 ret = i915_gem_object_get_pages(obj);
 657                 if (ret) {
 658                         remove_mappable_node(&node);
 659                         goto out;
 660                 }
 661
 662                 i915_gem_object_pin_pages(obj);
 663         } else {
 664                 node.start = i915_gem_obj_ggtt_offset(obj);
 665                 node.allocated = false;
 666                 ret = i915_gem_object_put_fence(obj);
 667                 if (ret)
 668                         goto out_unpin;
 669         }
 670
 671         ret = i915_gem_object_set_to_gtt_domain(obj, false);
 672         if (ret)
 673                 goto out_unpin;
 674
 675         user_data = u64_to_user_ptr(data_ptr);
 676         remain = size;
 677         offset = data_offset;
 678
 679         mutex_unlock(&dev->struct_mutex);
 680         if (likely(!i915.prefault_disable)) {
 681                 ret = fault_in_multipages_writeable(user_data, remain);
 682                 if (ret) {
 683                         mutex_lock(&dev->struct_mutex);
 684                         goto out_unpin;
 685                 }
 686         }
 687
 688         while (remain > 0) {
 689                 /* Operation in this page
 690                  *
 691                  * page_base = page offset within aperture
 692                  * page_offset = offset within page
 693                  * page_length = bytes to copy for this page
 694                  */
 695                 u32 page_base = node.start;
 696                 unsigned page_offset = offset_in_page(offset);
 697                 unsigned page_length = PAGE_SIZE - page_offset;
 698                 page_length = remain < page_length ? remain : page_length;
 699                 if (node.allocated) {
 700                         wmb();
 701                         ggtt->base.insert_page(&ggtt->base,
 702                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 703                                                node.start,
 704                                                I915_CACHE_NONE, 0);
 705                         wmb();
 706                 } else {
 707                         page_base += offset & PAGE_MASK;
 708                 }
 709                 /* This is a slow read/write as it tries to read from
 710                  * and write to user memory which may result into page
 711                  * faults, and so we cannot perform this under struct_mutex.
 712                  */
 713                 if (slow_user_access(ggtt->mappable, page_base,
 714                                      page_offset, user_data,
 715                                      page_length, false)) {
 716                         ret = -EFAULT;
 717                         break;
 718                 }
 719
 720                 remain -= page_length;
 721                 user_data += page_length;
 722                 offset += page_length;
 723         }
 724
 725         mutex_lock(&dev->struct_mutex);
 726         if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
 727                 /* The user has modified the object whilst we tried
 728                  * reading from it, and we now have no idea what domain
 729                  * the pages should be in. As we have just been touching
 730                  * them directly, flush everything back to the GTT
 731                  * domain.
 732                  */
 733                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 734         }
 735
 736 out_unpin:
 737         if (node.allocated) {
 738                 wmb();
 739                 ggtt->base.clear_range(&ggtt->base,
 740                                        node.start, node.size,
 741                                        true);
 742                 i915_gem_object_unpin_pages(obj);
 743                 remove_mappable_node(&node);
 744         } else {
 745                 i915_gem_object_ggtt_unpin(obj);
 746         }
 747 out:
 748         return ret;
 749 }
 750
 751 static int
 752 i915_gem_shmem_pread(struct drm_device *dev,
 753                      struct drm_i915_gem_object *obj,
 754                      struct drm_i915_gem_pread *args,
 755                      struct drm_file *file)
 756 {
 757         char __user *user_data;
 758         ssize_t remain;
 759         loff_t offset;
 760         int shmem_page_offset, page_length, ret = 0;
 761         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 762         int prefaulted = 0;
 763         int needs_clflush = 0;
 764         struct sg_page_iter sg_iter;
 765
 766         if (!i915_gem_object_has_struct_page(obj))
 767                 return -ENODEV;
 768
 769         user_data = u64_to_user_ptr(args->data_ptr);
 770         remain = args->size;
 771
 772         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 773
 774         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 775         if (ret)
 776                 return ret;
 777
 778         offset = args->offset;
 779
 780         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 781                          offset >> PAGE_SHIFT) {
 782                 struct page *page = sg_page_iter_page(&sg_iter);
 783
 784                 if (remain <= 0)
 785                         break;
 786
 787                 /* Operation in this page
 788                  *
 789                  * shmem_page_offset = offset within page in shmem file
 790                  * page_length = bytes to copy for this page
 791                  */
 792                 shmem_page_offset = offset_in_page(offset);
 793                 page_length = remain;
 794                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 795                         page_length = PAGE_SIZE - shmem_page_offset;
 796
 797                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 798                         (page_to_phys(page) & (1 << 17)) != 0;
 799
 800                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 801                                        user_data, page_do_bit17_swizzling,
 802                                        needs_clflush);
 803                 if (ret == 0)
 804                         goto next_page;
 805
 806                 mutex_unlock(&dev->struct_mutex);
 807
 808                 if (likely(!i915.prefault_disable) && !prefaulted) {
 809                         ret = fault_in_multipages_writeable(user_data, remain);
 810                         /* Userspace is tricking us, but we've already clobbered
 811                          * its pages with the prefault and promised to write the
 812                          * data up to the first fault. Hence ignore any errors
 813                          * and just continue. */
 814                         (void)ret;
 815                         prefaulted = 1;
 816                 }
 817
 818                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 819                                        user_data, page_do_bit17_swizzling,
 820                                        needs_clflush);
 821
 822                 mutex_lock(&dev->struct_mutex);
 823
 824                 if (ret)
 825                         goto out;
 826
 827 next_page:
 828                 remain -= page_length;
 829                 user_data += page_length;
 830                 offset += page_length;
 831         }
 832
 833 out:
 834         i915_gem_object_unpin_pages(obj);
 835
 836         return ret;
 837 }
 838
 839 /**
 840  * Reads data from the object referenced by handle.
 841  * @dev: drm device pointer
 842  * @data: ioctl data blob
 843  * @file: drm file pointer
 844  *
 845  * On error, the contents of *data are undefined.
 846  */
 847 int
 848 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 849                      struct drm_file *file)
 850 {
 851         struct drm_i915_gem_pread *args = data;
 852         struct drm_i915_gem_object *obj;
 853         int ret = 0;
 854
 855         if (args->size == 0)
 856                 return 0;
 857
 858         if (!access_ok(VERIFY_WRITE,
 859                        u64_to_user_ptr(args->data_ptr),
 860                        args->size))
 861                 return -EFAULT;
 862
 863         ret = i915_mutex_lock_interruptible(dev);
 864         if (ret)
 865                 return ret;
 866
 867         obj = i915_gem_object_lookup(file, args->handle);
 868         if (!obj) {
 869                 ret = -ENOENT;
 870                 goto unlock;
 871         }
 872
 873         /* Bounds check source.  */
 874         if (args->offset > obj->base.size ||
 875             args->size > obj->base.size - args->offset) {
 876                 ret = -EINVAL;
 877                 goto out;
 878         }
 879
 880         trace_i915_gem_object_pread(obj, args->offset, args->size);
 881
 882         ret = i915_gem_shmem_pread(dev, obj, args, file);
 883
 884         /* pread for non shmem backed objects */
 885         if (ret == -EFAULT || ret == -ENODEV)
 886                 ret = i915_gem_gtt_pread(dev, obj, args->size,
 887                                         args->offset, args->data_ptr);
 888
 889 out:
 890         i915_gem_object_put(obj);
 891 unlock:
 892         mutex_unlock(&dev->struct_mutex);
 893         return ret;
 894 }
 895
 896 /* This is the fast write path which cannot handle
 897  * page faults in the source data
 898  */
 899
 900 static inline int
 901 fast_user_write(struct io_mapping *mapping,
 902                 loff_t page_base, int page_offset,
 903                 char __user *user_data,
 904                 int length)
 905 {
 906         void __iomem *vaddr_atomic;
 907         void *vaddr;
 908         unsigned long unwritten;
 909
 910         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 911         /* We can use the cpu mem copy function because this is X86. */
 912         vaddr = (void __force*)vaddr_atomic + page_offset;
 913         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 914                                                       user_data, length);
 915         io_mapping_unmap_atomic(vaddr_atomic);
 916         return unwritten;
 917 }
 918
 919 /**
 920  * This is the fast pwrite path, where we copy the data directly from the
 921  * user into the GTT, uncached.
 922  * @i915: i915 device private data
 923  * @obj: i915 gem object
 924  * @args: pwrite arguments structure
 925  * @file: drm file pointer
 926  */
 927 static int
 928 i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 929                          struct drm_i915_gem_object *obj,
 930                          struct drm_i915_gem_pwrite *args,
 931                          struct drm_file *file)
 932 {
 933         struct i915_ggtt *ggtt = &i915->ggtt;
 934         struct drm_device *dev = obj->base.dev;
 935         struct drm_mm_node node;
 936         uint64_t remain, offset;
 937         char __user *user_data;
 938         int ret;
 939         bool hit_slow_path = false;
 940
 941         if (obj->tiling_mode != I915_TILING_NONE)
 942                 return -EFAULT;
 943
 944         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
 945         if (ret) {
 946                 ret = insert_mappable_node(i915, &node, PAGE_SIZE);
 947                 if (ret)
 948                         goto out;
 949
 950                 ret = i915_gem_object_get_pages(obj);
 951                 if (ret) {
 952                         remove_mappable_node(&node);
 953                         goto out;
 954                 }
 955
 956                 i915_gem_object_pin_pages(obj);
 957         } else {
 958                 node.start = i915_gem_obj_ggtt_offset(obj);
 959                 node.allocated = false;
 960                 ret = i915_gem_object_put_fence(obj);
 961                 if (ret)
 962                         goto out_unpin;
 963         }
 964
 965         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 966         if (ret)
 967                 goto out_unpin;
 968
 969         intel_fb_obj_invalidate(obj, ORIGIN_GTT);
 970         obj->dirty = true;
 971
 972         user_data = u64_to_user_ptr(args->data_ptr);
 973         offset = args->offset;
 974         remain = args->size;
 975         while (remain) {
 976                 /* Operation in this page
 977                  *
 978                  * page_base = page offset within aperture
 979                  * page_offset = offset within page
 980                  * page_length = bytes to copy for this page
 981                  */
 982                 u32 page_base = node.start;
 983                 unsigned page_offset = offset_in_page(offset);
 984                 unsigned page_length = PAGE_SIZE - page_offset;
 985                 page_length = remain < page_length ? remain : page_length;
 986                 if (node.allocated) {
 987                         wmb(); /* flush the write before we modify the GGTT */
 988                         ggtt->base.insert_page(&ggtt->base,
 989                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 990                                                node.start, I915_CACHE_NONE, 0);
 991                         wmb(); /* flush modifications to the GGTT (insert_page) */
 992                 } else {
 993                         page_base += offset & PAGE_MASK;
 994                 }
 995                 /* If we get a fault while copying data, then (presumably) our
 996                  * source page isn't available.  Return the error and we'll
 997                  * retry in the slow path.
 998                  * If the object is non-shmem backed, we retry again with the
 999                  * path that handles page fault.
1000                  */
1001                 if (fast_user_write(ggtt->mappable, page_base,
1002                                     page_offset, user_data, page_length)) {
1003                         hit_slow_path = true;
1004                         mutex_unlock(&dev->struct_mutex);
1005                         if (slow_user_access(ggtt->mappable,
1006                                              page_base,
1007                                              page_offset, user_data,
1008                                              page_length, true)) {
1009                                 ret = -EFAULT;
1010                                 mutex_lock(&dev->struct_mutex);
1011                                 goto out_flush;
1012                         }
1013
1014                         mutex_lock(&dev->struct_mutex);
1015                 }
1016
1017                 remain -= page_length;
1018                 user_data += page_length;
1019                 offset += page_length;
1020         }
1021
1022 out_flush:
1023         if (hit_slow_path) {
1024                 if (ret == 0 &&
1025                     (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
1026                         /* The user has modified the object whilst we tried
1027                          * reading from it, and we now have no idea what domain
1028                          * the pages should be in. As we have just been touching
1029                          * them directly, flush everything back to the GTT
1030                          * domain.
1031                          */
1032                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
1033                 }
1034         }
1035
1036         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
1037 out_unpin:
1038         if (node.allocated) {
1039                 wmb();
1040                 ggtt->base.clear_range(&ggtt->base,
1041                                        node.start, node.size,
1042                                        true);
1043                 i915_gem_object_unpin_pages(obj);
1044                 remove_mappable_node(&node);
1045         } else {
1046                 i915_gem_object_ggtt_unpin(obj);
1047         }
1048 out:
1049         return ret;
1050 }
1051
1052 /* Per-page copy function for the shmem pwrite fastpath.
1053  * Flushes invalid cachelines before writing to the target if
1054  * needs_clflush_before is set and flushes out any written cachelines after
1055  * writing if needs_clflush is set. */
1056 static int
1057 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
1058                   char __user *user_data,
1059                   bool page_do_bit17_swizzling,
1060                   bool needs_clflush_before,
1061                   bool needs_clflush_after)
1062 {
1063         char *vaddr;
1064         int ret;
1065
1066         if (unlikely(page_do_bit17_swizzling))
1067                 return -EINVAL;
1068
1069         vaddr = kmap_atomic(page);
1070         if (needs_clflush_before)
1071                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1072                                        page_length);
1073         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
1074                                         user_data, page_length);
1075         if (needs_clflush_after)
1076                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1077                                        page_length);
1078         kunmap_atomic(vaddr);
1079
1080         return ret ? -EFAULT : 0;
1081 }
1082
1083 /* Only difference to the fast-path function is that this can handle bit17
1084  * and uses non-atomic copy and kmap functions. */
1085 static int
1086 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
1087                   char __user *user_data,
1088                   bool page_do_bit17_swizzling,
1089                   bool needs_clflush_before,
1090                   bool needs_clflush_after)
1091 {
1092         char *vaddr;
1093         int ret;
1094
1095         vaddr = kmap(page);
1096         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1097                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1098                                              page_length,
1099                                              page_do_bit17_swizzling);
1100         if (page_do_bit17_swizzling)
1101                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1102                                                 user_data,
1103                                                 page_length);
1104         else
1105                 ret = __copy_from_user(vaddr + shmem_page_offset,
1106                                        user_data,
1107                                        page_length);
1108         if (needs_clflush_after)
1109                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1110                                              page_length,
1111                                              page_do_bit17_swizzling);
1112         kunmap(page);
1113
1114         return ret ? -EFAULT : 0;
1115 }
1116
1117 static int
1118 i915_gem_shmem_pwrite(struct drm_device *dev,
1119                       struct drm_i915_gem_object *obj,
1120                       struct drm_i915_gem_pwrite *args,
1121                       struct drm_file *file)
1122 {
1123         ssize_t remain;
1124         loff_t offset;
1125         char __user *user_data;
1126         int shmem_page_offset, page_length, ret = 0;
1127         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1128         int hit_slowpath = 0;
1129         int needs_clflush_after = 0;
1130         int needs_clflush_before = 0;
1131         struct sg_page_iter sg_iter;
1132
1133         user_data = u64_to_user_ptr(args->data_ptr);
1134         remain = args->size;
1135
1136         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1137
1138         ret = i915_gem_object_wait_rendering(obj, false);
1139         if (ret)
1140                 return ret;
1141
1142         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1143                 /* If we're not in the cpu write domain, set ourself into the gtt
1144                  * write domain and manually flush cachelines (if required). This
1145                  * optimizes for the case when the gpu will use the data
1146                  * right away and we therefore have to clflush anyway. */
1147                 needs_clflush_after = cpu_write_needs_clflush(obj);
1148         }
1149         /* Same trick applies to invalidate partially written cachelines read
1150          * before writing. */
1151         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1152                 needs_clflush_before =
1153                         !cpu_cache_is_coherent(dev, obj->cache_level);
1154
1155         ret = i915_gem_object_get_pages(obj);
1156         if (ret)
1157                 return ret;
1158
1159         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1160
1161         i915_gem_object_pin_pages(obj);
1162
1163         offset = args->offset;
1164         obj->dirty = 1;
1165
1166         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
1167                          offset >> PAGE_SHIFT) {
1168                 struct page *page = sg_page_iter_page(&sg_iter);
1169                 int partial_cacheline_write;
1170
1171                 if (remain <= 0)
1172                         break;
1173
1174                 /* Operation in this page
1175                  *
1176                  * shmem_page_offset = offset within page in shmem file
1177                  * page_length = bytes to copy for this page
1178                  */
1179                 shmem_page_offset = offset_in_page(offset);
1180
1181                 page_length = remain;
1182                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1183                         page_length = PAGE_SIZE - shmem_page_offset;
1184
1185                 /* If we don't overwrite a cacheline completely we need to be
1186                  * careful to have up-to-date data by first clflushing. Don't
1187                  * overcomplicate things and flush the entire patch. */
1188                 partial_cacheline_write = needs_clflush_before &&
1189                         ((shmem_page_offset | page_length)
1190                                 & (boot_cpu_data.x86_clflush_size - 1));
1191
1192                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1193                         (page_to_phys(page) & (1 << 17)) != 0;
1194
1195                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
1196                                         user_data, page_do_bit17_swizzling,
1197                                         partial_cacheline_write,
1198                                         needs_clflush_after);
1199                 if (ret == 0)
1200                         goto next_page;
1201
1202                 hit_slowpath = 1;
1203                 mutex_unlock(&dev->struct_mutex);
1204                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1205                                         user_data, page_do_bit17_swizzling,
1206                                         partial_cacheline_write,
1207                                         needs_clflush_after);
1208
1209                 mutex_lock(&dev->struct_mutex);
1210
1211                 if (ret)
1212                         goto out;
1213
1214 next_page:
1215                 remain -= page_length;
1216                 user_data += page_length;
1217                 offset += page_length;
1218         }
1219
1220 out:
1221         i915_gem_object_unpin_pages(obj);
1222
1223         if (hit_slowpath) {
1224                 /*
1225                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1226                  * cachelines in-line while writing and the object moved
1227                  * out of the cpu write domain while we've dropped the lock.
1228                  */
1229                 if (!needs_clflush_after &&
1230                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1231                         if (i915_gem_clflush_object(obj, obj->pin_display))
1232                                 needs_clflush_after = true;
1233                 }
1234         }
1235
1236         if (needs_clflush_after)
1237                 i915_gem_chipset_flush(to_i915(dev));
1238         else
1239                 obj->cache_dirty = true;
1240
1241         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1242         return ret;
1243 }
1244
1245 /**
1246  * Writes data to the object referenced by handle.
1247  * @dev: drm device
1248  * @data: ioctl data blob
1249  * @file: drm file
1250  *
1251  * On error, the contents of the buffer that were to be modified are undefined.
1252  */
1253 int
1254 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1255                       struct drm_file *file)
1256 {
1257         struct drm_i915_private *dev_priv = to_i915(dev);
1258         struct drm_i915_gem_pwrite *args = data;
1259         struct drm_i915_gem_object *obj;
1260         int ret;
1261
1262         if (args->size == 0)
1263                 return 0;
1264
1265         if (!access_ok(VERIFY_READ,
1266                        u64_to_user_ptr(args->data_ptr),
1267                        args->size))
1268                 return -EFAULT;
1269
1270         if (likely(!i915.prefault_disable)) {
1271                 ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
1272                                                    args->size);
1273                 if (ret)
1274                         return -EFAULT;
1275         }
1276
1277         intel_runtime_pm_get(dev_priv);
1278
1279         ret = i915_mutex_lock_interruptible(dev);
1280         if (ret)
1281                 goto put_rpm;
1282
1283         obj = i915_gem_object_lookup(file, args->handle);
1284         if (!obj) {
1285                 ret = -ENOENT;
1286                 goto unlock;
1287         }
1288
1289         /* Bounds check destination. */
1290         if (args->offset > obj->base.size ||
1291             args->size > obj->base.size - args->offset) {
1292                 ret = -EINVAL;
1293                 goto out;
1294         }
1295
1296         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1297
1298         ret = -EFAULT;
1299         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1300          * it would end up going through the fenced access, and we'll get
1301          * different detiling behavior between reading and writing.
1302          * pread/pwrite currently are reading and writing from the CPU
1303          * perspective, requiring manual detiling by the client.
1304          */
1305         if (!i915_gem_object_has_struct_page(obj) ||
1306             cpu_write_needs_clflush(obj)) {
1307                 ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
1308                 /* Note that the gtt paths might fail with non-page-backed user
1309                  * pointers (e.g. gtt mappings when moving data between
1310                  * textures). Fallback to the shmem path in that case. */
1311         }
1312
1313         if (ret == -EFAULT || ret == -ENOSPC) {
1314                 if (obj->phys_handle)
1315                         ret = i915_gem_phys_pwrite(obj, args, file);
1316                 else if (i915_gem_object_has_struct_page(obj))
1317                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1318                 else
1319                         ret = -ENODEV;
1320         }
1321
1322 out:
1323         i915_gem_object_put(obj);
1324 unlock:
1325         mutex_unlock(&dev->struct_mutex);
1326 put_rpm:
1327         intel_runtime_pm_put(dev_priv);
1328
1329         return ret;
1330 }
1331
1332 /**
1333  * Ensures that all rendering to the object has completed and the object is
1334  * safe to unbind from the GTT or access from the CPU.
1335  * @obj: i915 gem object
1336  * @readonly: waiting for read access or write
1337  */
1338 int
1339 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1340                                bool readonly)
1341 {
1342         struct reservation_object *resv;
1343         int ret, i;
1344
1345         if (readonly) {
1346                 if (obj->last_write_req != NULL) {
1347                         ret = i915_wait_request(obj->last_write_req);
1348                         if (ret)
1349                                 return ret;
1350
1351                         i = obj->last_write_req->engine->id;
1352                         if (obj->last_read_req[i] == obj->last_write_req)
1353                                 i915_gem_object_retire__read(obj, i);
1354                         else
1355                                 i915_gem_object_retire__write(obj);
1356                 }
1357         } else {
1358                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1359                         if (obj->last_read_req[i] == NULL)
1360                                 continue;
1361
1362                         ret = i915_wait_request(obj->last_read_req[i]);
1363                         if (ret)
1364                                 return ret;
1365
1366                         i915_gem_object_retire__read(obj, i);
1367                 }
1368                 GEM_BUG_ON(obj->active);
1369         }
1370
1371         resv = i915_gem_object_get_dmabuf_resv(obj);
1372         if (resv) {
1373                 long err;
1374
1375                 err = reservation_object_wait_timeout_rcu(resv, !readonly, true,
1376                                                           MAX_SCHEDULE_TIMEOUT);
1377                 if (err < 0)
1378                         return err;
1379         }
1380
1381         return 0;
1382 }
1383
1384 static void
1385 i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1386                                struct drm_i915_gem_request *req)
1387 {
1388         int idx = req->engine->id;
1389
1390         if (obj->last_read_req[idx] == req)
1391                 i915_gem_object_retire__read(obj, idx);
1392         else if (obj->last_write_req == req)
1393                 i915_gem_object_retire__write(obj);
1394
1395         if (!i915_reset_in_progress(&req->i915->gpu_error))
1396                 i915_gem_request_retire_upto(req);
1397 }
1398
1399 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1400  * as the object state may change during this call.
1401  */
1402 static __must_check int
1403 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1404                                             struct intel_rps_client *rps,
1405                                             bool readonly)
1406 {
1407         struct drm_device *dev = obj->base.dev;
1408         struct drm_i915_private *dev_priv = to_i915(dev);
1409         struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
1410         int ret, i, n = 0;
1411
1412         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1413         BUG_ON(!dev_priv->mm.interruptible);
1414
1415         if (!obj->active)
1416                 return 0;
1417
1418         if (readonly) {
1419                 struct drm_i915_gem_request *req;
1420
1421                 req = obj->last_write_req;
1422                 if (req == NULL)
1423                         return 0;
1424
1425                 requests[n++] = i915_gem_request_get(req);
1426         } else {
1427                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1428                         struct drm_i915_gem_request *req;
1429
1430                         req = obj->last_read_req[i];
1431                         if (req == NULL)
1432                                 continue;
1433
1434                         requests[n++] = i915_gem_request_get(req);
1435                 }
1436         }
1437
1438         mutex_unlock(&dev->struct_mutex);
1439         ret = 0;
1440         for (i = 0; ret == 0 && i < n; i++)
1441                 ret = __i915_wait_request(requests[i], true, NULL, rps);
1442         mutex_lock(&dev->struct_mutex);
1443
1444         for (i = 0; i < n; i++) {
1445                 if (ret == 0)
1446                         i915_gem_object_retire_request(obj, requests[i]);
1447                 i915_gem_request_put(requests[i]);
1448         }
1449
1450         return ret;
1451 }
1452
1453 static struct intel_rps_client *to_rps_client(struct drm_file *file)
1454 {
1455         struct drm_i915_file_private *fpriv = file->driver_priv;
1456         return &fpriv->rps;
1457 }
1458
1459 static enum fb_op_origin
1460 write_origin(struct drm_i915_gem_object *obj, unsigned domain)
1461 {
1462         return domain == I915_GEM_DOMAIN_GTT && !obj->has_wc_mmap ?
1463                ORIGIN_GTT : ORIGIN_CPU;
1464 }
1465
1466 /**
1467  * Called when user space prepares to use an object with the CPU, either
1468  * through the mmap ioctl's mapping or a GTT mapping.
1469  * @dev: drm device
1470  * @data: ioctl data blob
1471  * @file: drm file
1472  */
1473 int
1474 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1475                           struct drm_file *file)
1476 {
1477         struct drm_i915_gem_set_domain *args = data;
1478         struct drm_i915_gem_object *obj;
1479         uint32_t read_domains = args->read_domains;
1480         uint32_t write_domain = args->write_domain;
1481         int ret;
1482
1483         /* Only handle setting domains to types used by the CPU. */
1484         if (write_domain & I915_GEM_GPU_DOMAINS)
1485                 return -EINVAL;
1486
1487         if (read_domains & I915_GEM_GPU_DOMAINS)
1488                 return -EINVAL;
1489
1490         /* Having something in the write domain implies it's in the read
1491          * domain, and only that read domain.  Enforce that in the request.
1492          */
1493         if (write_domain != 0 && read_domains != write_domain)
1494                 return -EINVAL;
1495
1496         ret = i915_mutex_lock_interruptible(dev);
1497         if (ret)
1498                 return ret;
1499
1500         obj = i915_gem_object_lookup(file, args->handle);
1501         if (!obj) {
1502                 ret = -ENOENT;
1503                 goto unlock;
1504         }
1505
1506         /* Try to flush the object off the GPU without holding the lock.
1507          * We will repeat the flush holding the lock in the normal manner
1508          * to catch cases where we are gazumped.
1509          */
1510         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1511                                                           to_rps_client(file),
1512                                                           !write_domain);
1513         if (ret)
1514                 goto unref;
1515
1516         if (read_domains & I915_GEM_DOMAIN_GTT)
1517                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1518         else
1519                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1520
1521         if (write_domain != 0)
1522                 intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
1523
1524 unref:
1525         i915_gem_object_put(obj);
1526 unlock:
1527         mutex_unlock(&dev->struct_mutex);
1528         return ret;
1529 }
1530
1531 /**
1532  * Called when user space has done writes to this buffer
1533  * @dev: drm device
1534  * @data: ioctl data blob
1535  * @file: drm file
1536  */
1537 int
1538 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1539                          struct drm_file *file)
1540 {
1541         struct drm_i915_gem_sw_finish *args = data;
1542         struct drm_i915_gem_object *obj;
1543         int ret = 0;
1544
1545         ret = i915_mutex_lock_interruptible(dev);
1546         if (ret)
1547                 return ret;
1548
1549         obj = i915_gem_object_lookup(file, args->handle);
1550         if (!obj) {
1551                 ret = -ENOENT;
1552                 goto unlock;
1553         }
1554
1555         /* Pinned buffers may be scanout, so flush the cache */
1556         if (obj->pin_display)
1557                 i915_gem_object_flush_cpu_write_domain(obj);
1558
1559         i915_gem_object_put(obj);
1560 unlock:
1561         mutex_unlock(&dev->struct_mutex);
1562         return ret;
1563 }
1564
1565 /**
1566  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1567  *                       it is mapped to.
1568  * @dev: drm device
1569  * @data: ioctl data blob
1570  * @file: drm file
1571  *
1572  * While the mapping holds a reference on the contents of the object, it doesn't
1573  * imply a ref on the object itself.
1574  *
1575  * IMPORTANT:
1576  *
1577  * DRM driver writers who look a this function as an example for how to do GEM
1578  * mmap support, please don't implement mmap support like here. The modern way
1579  * to implement DRM mmap support is with an mmap offset ioctl (like
1580  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1581  * That way debug tooling like valgrind will understand what's going on, hiding
1582  * the mmap call in a driver private ioctl will break that. The i915 driver only
1583  * does cpu mmaps this way because we didn't know better.
1584  */
1585 int
1586 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1587                     struct drm_file *file)
1588 {
1589         struct drm_i915_gem_mmap *args = data;
1590         struct drm_i915_gem_object *obj;
1591         unsigned long addr;
1592
1593         if (args->flags & ~(I915_MMAP_WC))
1594                 return -EINVAL;
1595
1596         if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1597                 return -ENODEV;
1598
1599         obj = i915_gem_object_lookup(file, args->handle);
1600         if (!obj)
1601                 return -ENOENT;
1602
1603         /* prime objects have no backing filp to GEM mmap
1604          * pages from.
1605          */
1606         if (!obj->base.filp) {
1607                 i915_gem_object_put_unlocked(obj);
1608                 return -EINVAL;
1609         }
1610
1611         addr = vm_mmap(obj->base.filp, 0, args->size,
1612                        PROT_READ | PROT_WRITE, MAP_SHARED,
1613                        args->offset);
1614         if (args->flags & I915_MMAP_WC) {
1615                 struct mm_struct *mm = current->mm;
1616                 struct vm_area_struct *vma;
1617
1618                 if (down_write_killable(&mm->mmap_sem)) {
1619                         i915_gem_object_put_unlocked(obj);
1620                         return -EINTR;
1621                 }
1622                 vma = find_vma(mm, addr);
1623                 if (vma)
1624                         vma->vm_page_prot =
1625                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1626                 else
1627                         addr = -ENOMEM;
1628                 up_write(&mm->mmap_sem);
1629
1630                 /* This may race, but that's ok, it only gets set */
1631                 WRITE_ONCE(obj->has_wc_mmap, true);
1632         }
1633         i915_gem_object_put_unlocked(obj);
1634         if (IS_ERR((void *)addr))
1635                 return addr;
1636
1637         args->addr_ptr = (uint64_t) addr;
1638
1639         return 0;
1640 }
1641
1642 /**
1643  * i915_gem_fault - fault a page into the GTT
1644  * @vma: VMA in question
1645  * @vmf: fault info
1646  *
1647  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1648  * from userspace.  The fault handler takes care of binding the object to
1649  * the GTT (if needed), allocating and programming a fence register (again,
1650  * only if needed based on whether the old reg is still valid or the object
1651  * is tiled) and inserting a new PTE into the faulting process.
1652  *
1653  * Note that the faulting process may involve evicting existing objects
1654  * from the GTT and/or fence registers to make room.  So performance may
1655  * suffer if the GTT working set is large or there are few fence registers
1656  * left.
1657  */
1658 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1659 {
1660         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1661         struct drm_device *dev = obj->base.dev;
1662         struct drm_i915_private *dev_priv = to_i915(dev);
1663         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1664         struct i915_ggtt_view view = i915_ggtt_view_normal;
1665         pgoff_t page_offset;
1666         unsigned long pfn;
1667         int ret = 0;
1668         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1669
1670         intel_runtime_pm_get(dev_priv);
1671
1672         /* We don't use vmf->pgoff since that has the fake offset */
1673         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1674                 PAGE_SHIFT;
1675
1676         ret = i915_mutex_lock_interruptible(dev);
1677         if (ret)
1678                 goto out;
1679
1680         trace_i915_gem_object_fault(obj, page_offset, true, write);
1681
1682         /* Try to flush the object off the GPU first without holding the lock.
1683          * Upon reacquiring the lock, we will perform our sanity checks and then
1684          * repeat the flush holding the lock in the normal manner to catch cases
1685          * where we are gazumped.
1686          */
1687         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1688         if (ret)
1689                 goto unlock;
1690
1691         /* Access to snoopable pages through the GTT is incoherent. */
1692         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1693                 ret = -EFAULT;
1694                 goto unlock;
1695         }
1696
1697         /* Use a partial view if the object is bigger than the aperture. */
1698         if (obj->base.size >= ggtt->mappable_end &&
1699             obj->tiling_mode == I915_TILING_NONE) {
1700                 static const unsigned int chunk_size = 256; // 1 MiB
1701
1702                 memset(&view, 0, sizeof(view));
1703                 view.type = I915_GGTT_VIEW_PARTIAL;
1704                 view.params.partial.offset = rounddown(page_offset, chunk_size);
1705                 view.params.partial.size =
1706                         min_t(unsigned int,
1707                               chunk_size,
1708                               (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1709                               view.params.partial.offset);
1710         }
1711
1712         /* Now pin it into the GTT if needed */
1713         ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
1714         if (ret)
1715                 goto unlock;
1716
1717         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1718         if (ret)
1719                 goto unpin;
1720
1721         ret = i915_gem_object_get_fence(obj);
1722         if (ret)
1723                 goto unpin;
1724
1725         /* Finally, remap it using the new GTT offset */
1726         pfn = ggtt->mappable_base +
1727                 i915_gem_obj_ggtt_offset_view(obj, &view);
1728         pfn >>= PAGE_SHIFT;
1729
1730         if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1731                 /* Overriding existing pages in partial view does not cause
1732                  * us any trouble as TLBs are still valid because the fault
1733                  * is due to userspace losing part of the mapping or never
1734                  * having accessed it before (at this partials' range).
1735                  */
1736                 unsigned long base = vma->vm_start +
1737                                      (view.params.partial.offset << PAGE_SHIFT);
1738                 unsigned int i;
1739
1740                 for (i = 0; i < view.params.partial.size; i++) {
1741                         ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1742                         if (ret)
1743                                 break;
1744                 }
1745
1746                 obj->fault_mappable = true;
1747         } else {
1748                 if (!obj->fault_mappable) {
1749                         unsigned long size = min_t(unsigned long,
1750                                                    vma->vm_end - vma->vm_start,
1751                                                    obj->base.size);
1752                         int i;
1753
1754                         for (i = 0; i < size >> PAGE_SHIFT; i++) {
1755                                 ret = vm_insert_pfn(vma,
1756                                                     (unsigned long)vma->vm_start + i * PAGE_SIZE,
1757                                                     pfn + i);
1758                                 if (ret)
1759                                         break;
1760                         }
1761
1762                         obj->fault_mappable = true;
1763                 } else
1764                         ret = vm_insert_pfn(vma,
1765                                             (unsigned long)vmf->virtual_address,
1766                                             pfn + page_offset);
1767         }
1768 unpin:
1769         i915_gem_object_ggtt_unpin_view(obj, &view);
1770 unlock:
1771         mutex_unlock(&dev->struct_mutex);
1772 out:
1773         switch (ret) {
1774         case -EIO:
1775                 /*
1776                  * We eat errors when the gpu is terminally wedged to avoid
1777                  * userspace unduly crashing (gl has no provisions for mmaps to
1778                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1779                  * and so needs to be reported.
1780                  */
1781                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1782                         ret = VM_FAULT_SIGBUS;
1783                         break;
1784                 }
1785         case -EAGAIN:
1786                 /*
1787                  * EAGAIN means the gpu is hung and we'll wait for the error
1788                  * handler to reset everything when re-faulting in
1789                  * i915_mutex_lock_interruptible.
1790                  */
1791         case 0:
1792         case -ERESTARTSYS:
1793         case -EINTR:
1794         case -EBUSY:
1795                 /*
1796                  * EBUSY is ok: this just means that another thread
1797                  * already did the job.
1798                  */
1799                 ret = VM_FAULT_NOPAGE;
1800                 break;
1801         case -ENOMEM:
1802                 ret = VM_FAULT_OOM;
1803                 break;
1804         case -ENOSPC:
1805         case -EFAULT:
1806                 ret = VM_FAULT_SIGBUS;
1807                 break;
1808         default:
1809                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1810                 ret = VM_FAULT_SIGBUS;
1811                 break;
1812         }
1813
1814         intel_runtime_pm_put(dev_priv);
1815         return ret;
1816 }
1817
1818 /**
1819  * i915_gem_release_mmap - remove physical page mappings
1820  * @obj: obj in question
1821  *
1822  * Preserve the reservation of the mmapping with the DRM core code, but
1823  * relinquish ownership of the pages back to the system.
1824  *
1825  * It is vital that we remove the page mapping if we have mapped a tiled
1826  * object through the GTT and then lose the fence register due to
1827  * resource pressure. Similarly if the object has been moved out of the
1828  * aperture, than pages mapped into userspace must be revoked. Removing the
1829  * mapping will then trigger a page fault on the next user access, allowing
1830  * fixup by i915_gem_fault().
1831  */
1832 void
1833 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1834 {
1835         /* Serialisation between user GTT access and our code depends upon
1836          * revoking the CPU's PTE whilst the mutex is held. The next user
1837          * pagefault then has to wait until we release the mutex.
1838          */
1839         lockdep_assert_held(&obj->base.dev->struct_mutex);
1840
1841         if (!obj->fault_mappable)
1842                 return;
1843
1844         drm_vma_node_unmap(&obj->base.vma_node,
1845                            obj->base.dev->anon_inode->i_mapping);
1846
1847         /* Ensure that the CPU's PTE are revoked and there are not outstanding
1848          * memory transactions from userspace before we return. The TLB
1849          * flushing implied above by changing the PTE above *should* be
1850          * sufficient, an extra barrier here just provides us with a bit
1851          * of paranoid documentation about our requirement to serialise
1852          * memory writes before touching registers / GSM.
1853          */
1854         wmb();
1855
1856         obj->fault_mappable = false;
1857 }
1858
1859 void
1860 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1861 {
1862         struct drm_i915_gem_object *obj;
1863
1864         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1865                 i915_gem_release_mmap(obj);
1866 }
1867
1868 uint32_t
1869 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1870 {
1871         uint32_t gtt_size;
1872
1873         if (INTEL_INFO(dev)->gen >= 4 ||
1874             tiling_mode == I915_TILING_NONE)
1875                 return size;
1876
1877         /* Previous chips need a power-of-two fence region when tiling */
1878         if (IS_GEN3(dev))
1879                 gtt_size = 1024*1024;
1880         else
1881                 gtt_size = 512*1024;
1882
1883         while (gtt_size < size)
1884                 gtt_size <<= 1;
1885
1886         return gtt_size;
1887 }
1888
1889 /**
1890  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1891  * @dev: drm device
1892  * @size: object size
1893  * @tiling_mode: tiling mode
1894  * @fenced: is fenced alignemned required or not
1895  *
1896  * Return the required GTT alignment for an object, taking into account
1897  * potential fence register mapping.
1898  */
1899 uint32_t
1900 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1901                            int tiling_mode, bool fenced)
1902 {
1903         /*
1904          * Minimum alignment is 4k (GTT page size), but might be greater
1905          * if a fence register is needed for the object.
1906          */
1907         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1908             tiling_mode == I915_TILING_NONE)
1909                 return 4096;
1910
1911         /*
1912          * Previous chips need to be aligned to the size of the smallest
1913          * fence register that can contain the object.
1914          */
1915         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1916 }
1917
1918 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1919 {
1920         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
1921         int ret;
1922
1923         dev_priv->mm.shrinker_no_lock_stealing = true;
1924
1925         ret = drm_gem_create_mmap_offset(&obj->base);
1926         if (ret != -ENOSPC)
1927                 goto out;
1928
1929         /* Badly fragmented mmap space? The only way we can recover
1930          * space is by destroying unwanted objects. We can't randomly release
1931          * mmap_offsets as userspace expects them to be persistent for the
1932          * lifetime of the objects. The closest we can is to release the
1933          * offsets on purgeable objects by truncating it and marking it purged,
1934          * which prevents userspace from ever using that object again.
1935          */
1936         i915_gem_shrink(dev_priv,
1937                         obj->base.size >> PAGE_SHIFT,
1938                         I915_SHRINK_BOUND |
1939                         I915_SHRINK_UNBOUND |
1940                         I915_SHRINK_PURGEABLE);
1941         ret = drm_gem_create_mmap_offset(&obj->base);
1942         if (ret != -ENOSPC)
1943                 goto out;
1944
1945         i915_gem_shrink_all(dev_priv);
1946         ret = drm_gem_create_mmap_offset(&obj->base);
1947 out:
1948         dev_priv->mm.shrinker_no_lock_stealing = false;
1949
1950         return ret;
1951 }
1952
1953 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1954 {
1955         drm_gem_free_mmap_offset(&obj->base);
1956 }
1957
1958 int
1959 i915_gem_mmap_gtt(struct drm_file *file,
1960                   struct drm_device *dev,
1961                   uint32_t handle,
1962                   uint64_t *offset)
1963 {
1964         struct drm_i915_gem_object *obj;
1965         int ret;
1966
1967         ret = i915_mutex_lock_interruptible(dev);
1968         if (ret)
1969                 return ret;
1970
1971         obj = i915_gem_object_lookup(file, handle);
1972         if (!obj) {
1973                 ret = -ENOENT;
1974                 goto unlock;
1975         }
1976
1977         if (obj->madv != I915_MADV_WILLNEED) {
1978                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
1979                 ret = -EFAULT;
1980                 goto out;
1981         }
1982
1983         ret = i915_gem_object_create_mmap_offset(obj);
1984         if (ret)
1985                 goto out;
1986
1987         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1988
1989 out:
1990         i915_gem_object_put(obj);
1991 unlock:
1992         mutex_unlock(&dev->struct_mutex);
1993         return ret;
1994 }
1995
1996 /**
1997  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1998  * @dev: DRM device
1999  * @data: GTT mapping ioctl data
2000  * @file: GEM object info
2001  *
2002  * Simply returns the fake offset to userspace so it can mmap it.
2003  * The mmap call will end up in drm_gem_mmap(), which will set things
2004  * up so we can get faults in the handler above.
2005  *
2006  * The fault handler will take care of binding the object into the GTT
2007  * (since it may have been evicted to make room for something), allocating
2008  * a fence register, and mapping the appropriate aperture address into
2009  * userspace.
2010  */
2011 int
2012 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2013                         struct drm_file *file)
2014 {
2015         struct drm_i915_gem_mmap_gtt *args = data;
2016
2017         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2018 }
2019
2020 /* Immediately discard the backing storage */
2021 static void
2022 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2023 {
2024         i915_gem_object_free_mmap_offset(obj);
2025
2026         if (obj->base.filp == NULL)
2027                 return;
2028
2029         /* Our goal here is to return as much of the memory as
2030          * is possible back to the system as we are called from OOM.
2031          * To do this we must instruct the shmfs to drop all of its
2032          * backing pages, *now*.
2033          */
2034         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2035         obj->madv = __I915_MADV_PURGED;
2036 }
2037
2038 /* Try to discard unwanted pages */
2039 static void
2040 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2041 {
2042         struct address_space *mapping;
2043
2044         switch (obj->madv) {
2045         case I915_MADV_DONTNEED:
2046                 i915_gem_object_truncate(obj);
2047         case __I915_MADV_PURGED:
2048                 return;
2049         }
2050
2051         if (obj->base.filp == NULL)
2052                 return;
2053
2054         mapping = file_inode(obj->base.filp)->i_mapping,
2055         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2056 }
2057
2058 static void
2059 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2060 {
2061         struct sgt_iter sgt_iter;
2062         struct page *page;
2063         int ret;
2064
2065         BUG_ON(obj->madv == __I915_MADV_PURGED);
2066
2067         ret = i915_gem_object_set_to_cpu_domain(obj, true);
2068         if (WARN_ON(ret)) {
2069                 /* In the event of a disaster, abandon all caches and
2070                  * hope for the best.
2071                  */
2072                 i915_gem_clflush_object(obj, true);
2073                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2074         }
2075
2076         i915_gem_gtt_finish_object(obj);
2077
2078         if (i915_gem_object_needs_bit17_swizzle(obj))
2079                 i915_gem_object_save_bit_17_swizzle(obj);
2080
2081         if (obj->madv == I915_MADV_DONTNEED)
2082                 obj->dirty = 0;
2083
2084         for_each_sgt_page(page, sgt_iter, obj->pages) {
2085                 if (obj->dirty)
2086                         set_page_dirty(page);
2087
2088                 if (obj->madv == I915_MADV_WILLNEED)
2089                         mark_page_accessed(page);
2090
2091                 put_page(page);
2092         }
2093         obj->dirty = 0;
2094
2095         sg_free_table(obj->pages);
2096         kfree(obj->pages);
2097 }
2098
2099 int
2100 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2101 {
2102         const struct drm_i915_gem_object_ops *ops = obj->ops;
2103
2104         if (obj->pages == NULL)
2105                 return 0;
2106
2107         if (obj->pages_pin_count)
2108                 return -EBUSY;
2109
2110         BUG_ON(i915_gem_obj_bound_any(obj));
2111
2112         /* ->put_pages might need to allocate memory for the bit17 swizzle
2113          * array, hence protect them from being reaped by removing them from gtt
2114          * lists early. */
2115         list_del(&obj->global_list);
2116
2117         if (obj->mapping) {
2118                 if (is_vmalloc_addr(obj->mapping))
2119                         vunmap(obj->mapping);
2120                 else
2121                         kunmap(kmap_to_page(obj->mapping));
2122                 obj->mapping = NULL;
2123         }
2124
2125         ops->put_pages(obj);
2126         obj->pages = NULL;
2127
2128         i915_gem_object_invalidate(obj);
2129
2130         return 0;
2131 }
2132
2133 static int
2134 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2135 {
2136         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2137         int page_count, i;
2138         struct address_space *mapping;
2139         struct sg_table *st;
2140         struct scatterlist *sg;
2141         struct sgt_iter sgt_iter;
2142         struct page *page;
2143         unsigned long last_pfn = 0;     /* suppress gcc warning */
2144         int ret;
2145         gfp_t gfp;
2146
2147         /* Assert that the object is not currently in any GPU domain. As it
2148          * wasn't in the GTT, there shouldn't be any way it could have been in
2149          * a GPU cache
2150          */
2151         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2152         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2153
2154         st = kmalloc(sizeof(*st), GFP_KERNEL);
2155         if (st == NULL)
2156                 return -ENOMEM;
2157
2158         page_count = obj->base.size / PAGE_SIZE;
2159         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2160                 kfree(st);
2161                 return -ENOMEM;
2162         }
2163
2164         /* Get the list of pages out of our struct file.  They'll be pinned
2165          * at this point until we release them.
2166          *
2167          * Fail silently without starting the shrinker
2168          */
2169         mapping = file_inode(obj->base.filp)->i_mapping;
2170         gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2171         gfp |= __GFP_NORETRY | __GFP_NOWARN;
2172         sg = st->sgl;
2173         st->nents = 0;
2174         for (i = 0; i < page_count; i++) {
2175                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2176                 if (IS_ERR(page)) {
2177                         i915_gem_shrink(dev_priv,
2178                                         page_count,
2179                                         I915_SHRINK_BOUND |
2180                                         I915_SHRINK_UNBOUND |
2181                                         I915_SHRINK_PURGEABLE);
2182                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2183                 }
2184                 if (IS_ERR(page)) {
2185                         /* We've tried hard to allocate the memory by reaping
2186                          * our own buffer, now let the real VM do its job and
2187                          * go down in flames if truly OOM.
2188                          */
2189                         i915_gem_shrink_all(dev_priv);
2190                         page = shmem_read_mapping_page(mapping, i);
2191                         if (IS_ERR(page)) {
2192                                 ret = PTR_ERR(page);
2193                                 goto err_pages;
2194                         }
2195                 }
2196 #ifdef CONFIG_SWIOTLB
2197                 if (swiotlb_nr_tbl()) {
2198                         st->nents++;
2199                         sg_set_page(sg, page, PAGE_SIZE, 0);
2200                         sg = sg_next(sg);
2201                         continue;
2202                 }
2203 #endif
2204                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2205                         if (i)
2206                                 sg = sg_next(sg);
2207                         st->nents++;
2208                         sg_set_page(sg, page, PAGE_SIZE, 0);
2209                 } else {
2210                         sg->length += PAGE_SIZE;
2211                 }
2212                 last_pfn = page_to_pfn(page);
2213
2214                 /* Check that the i965g/gm workaround works. */
2215                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2216         }
2217 #ifdef CONFIG_SWIOTLB
2218         if (!swiotlb_nr_tbl())
2219 #endif
2220                 sg_mark_end(sg);
2221         obj->pages = st;
2222
2223         ret = i915_gem_gtt_prepare_object(obj);
2224         if (ret)
2225                 goto err_pages;
2226
2227         if (i915_gem_object_needs_bit17_swizzle(obj))
2228                 i915_gem_object_do_bit_17_swizzle(obj);
2229
2230         if (obj->tiling_mode != I915_TILING_NONE &&
2231             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2232                 i915_gem_object_pin_pages(obj);
2233
2234         return 0;
2235
2236 err_pages:
2237         sg_mark_end(sg);
2238         for_each_sgt_page(page, sgt_iter, st)
2239                 put_page(page);
2240         sg_free_table(st);
2241         kfree(st);
2242
2243         /* shmemfs first checks if there is enough memory to allocate the page
2244          * and reports ENOSPC should there be insufficient, along with the usual
2245          * ENOMEM for a genuine allocation failure.
2246          *
2247          * We use ENOSPC in our driver to mean that we have run out of aperture
2248          * space and so want to translate the error from shmemfs back to our
2249          * usual understanding of ENOMEM.
2250          */
2251         if (ret == -ENOSPC)
2252                 ret = -ENOMEM;
2253
2254         return ret;
2255 }
2256
2257 /* Ensure that the associated pages are gathered from the backing storage
2258  * and pinned into our object. i915_gem_object_get_pages() may be called
2259  * multiple times before they are released by a single call to
2260  * i915_gem_object_put_pages() - once the pages are no longer referenced
2261  * either as a result of memory pressure (reaping pages under the shrinker)
2262  * or as the object is itself released.
2263  */
2264 int
2265 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2266 {
2267         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2268         const struct drm_i915_gem_object_ops *ops = obj->ops;
2269         int ret;
2270
2271         if (obj->pages)
2272                 return 0;
2273
2274         if (obj->madv != I915_MADV_WILLNEED) {
2275                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2276                 return -EFAULT;
2277         }
2278
2279         BUG_ON(obj->pages_pin_count);
2280
2281         ret = ops->get_pages(obj);
2282         if (ret)
2283                 return ret;
2284
2285         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2286
2287         obj->get_page.sg = obj->pages->sgl;
2288         obj->get_page.last = 0;
2289
2290         return 0;
2291 }
2292
2293 /* The 'mapping' part of i915_gem_object_pin_map() below */
2294 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
2295 {
2296         unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2297         struct sg_table *sgt = obj->pages;
2298         struct sgt_iter sgt_iter;
2299         struct page *page;
2300         struct page *stack_pages[32];
2301         struct page **pages = stack_pages;
2302         unsigned long i = 0;
2303         void *addr;
2304
2305         /* A single page can always be kmapped */
2306         if (n_pages == 1)
2307                 return kmap(sg_page(sgt->sgl));
2308
2309         if (n_pages > ARRAY_SIZE(stack_pages)) {
2310                 /* Too big for stack -- allocate temporary array instead */
2311                 pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
2312                 if (!pages)
2313                         return NULL;
2314         }
2315
2316         for_each_sgt_page(page, sgt_iter, sgt)
2317                 pages[i++] = page;
2318
2319         /* Check that we have the expected number of pages */
2320         GEM_BUG_ON(i != n_pages);
2321
2322         addr = vmap(pages, n_pages, 0, PAGE_KERNEL);
2323
2324         if (pages != stack_pages)
2325                 drm_free_large(pages);
2326
2327         return addr;
2328 }
2329
2330 /* get, pin, and map the pages of the object into kernel space */
2331 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
2332 {
2333         int ret;
2334
2335         lockdep_assert_held(&obj->base.dev->struct_mutex);
2336
2337         ret = i915_gem_object_get_pages(obj);
2338         if (ret)
2339                 return ERR_PTR(ret);
2340
2341         i915_gem_object_pin_pages(obj);
2342
2343         if (!obj->mapping) {
2344                 obj->mapping = i915_gem_object_map(obj);
2345                 if (!obj->mapping) {
2346                         i915_gem_object_unpin_pages(obj);
2347                         return ERR_PTR(-ENOMEM);
2348                 }
2349         }
2350
2351         return obj->mapping;
2352 }
2353
2354 void i915_vma_move_to_active(struct i915_vma *vma,
2355                              struct drm_i915_gem_request *req)
2356 {
2357         struct drm_i915_gem_object *obj = vma->obj;
2358         struct intel_engine_cs *engine;
2359
2360         engine = i915_gem_request_get_engine(req);
2361
2362         /* Add a reference if we're newly entering the active list. */
2363         if (obj->active == 0)
2364                 i915_gem_object_get(obj);
2365         obj->active |= intel_engine_flag(engine);
2366
2367         list_move_tail(&obj->engine_list[engine->id], &engine->active_list);
2368         i915_gem_request_assign(&obj->last_read_req[engine->id], req);
2369
2370         list_move_tail(&vma->vm_link, &vma->vm->active_list);
2371 }
2372
2373 static void
2374 i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
2375 {
2376         GEM_BUG_ON(obj->last_write_req == NULL);
2377         GEM_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine)));
2378
2379         i915_gem_request_assign(&obj->last_write_req, NULL);
2380         intel_fb_obj_flush(obj, true, ORIGIN_CS);
2381 }
2382
2383 static void
2384 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int idx)
2385 {
2386         struct i915_vma *vma;
2387
2388         GEM_BUG_ON(obj->last_read_req[idx] == NULL);
2389         GEM_BUG_ON(!(obj->active & (1 << idx)));
2390
2391         list_del_init(&obj->engine_list[idx]);
2392         i915_gem_request_assign(&obj->last_read_req[idx], NULL);
2393
2394         if (obj->last_write_req && obj->last_write_req->engine->id == idx)
2395                 i915_gem_object_retire__write(obj);
2396
2397         obj->active &= ~(1 << idx);
2398         if (obj->active)
2399                 return;
2400
2401         /* Bump our place on the bound list to keep it roughly in LRU order
2402          * so that we don't steal from recently used but inactive objects
2403          * (unless we are forced to ofc!)
2404          */
2405         list_move_tail(&obj->global_list,
2406                        &to_i915(obj->base.dev)->mm.bound_list);
2407
2408         list_for_each_entry(vma, &obj->vma_list, obj_link) {
2409                 if (!list_empty(&vma->vm_link))
2410                         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
2411         }
2412
2413         i915_gem_request_assign(&obj->last_fenced_req, NULL);
2414         i915_gem_object_put(obj);
2415 }
2416
2417 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
2418 {
2419         unsigned long elapsed;
2420
2421         if (ctx->hang_stats.banned)
2422                 return true;
2423
2424         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2425         if (ctx->hang_stats.ban_period_seconds &&
2426             elapsed <= ctx->hang_stats.ban_period_seconds) {
2427                 DRM_DEBUG("context hanging too fast, banning!\n");
2428                 return true;
2429         }
2430
2431         return false;
2432 }
2433
2434 static void i915_set_reset_status(struct i915_gem_context *ctx,
2435                                   const bool guilty)
2436 {
2437         struct i915_ctx_hang_stats *hs = &ctx->hang_stats;
2438
2439         if (guilty) {
2440                 hs->banned = i915_context_is_banned(ctx);
2441                 hs->batch_active++;
2442                 hs->guilty_ts = get_seconds();
2443         } else {
2444                 hs->batch_pending++;
2445         }
2446 }
2447
2448 struct drm_i915_gem_request *
2449 i915_gem_find_active_request(struct intel_engine_cs *engine)
2450 {
2451         struct drm_i915_gem_request *request;
2452
2453         /* We are called by the error capture and reset at a random
2454          * point in time. In particular, note that neither is crucially
2455          * ordered with an interrupt. After a hang, the GPU is dead and we
2456          * assume that no more writes can happen (we waited long enough for
2457          * all writes that were in transaction to be flushed) - adding an
2458          * extra delay for a recent interrupt is pointless. Hence, we do
2459          * not need an engine->irq_seqno_barrier() before the seqno reads.
2460          */
2461         list_for_each_entry(request, &engine->request_list, list) {
2462                 if (i915_gem_request_completed(request))
2463                         continue;
2464
2465                 return request;
2466         }
2467
2468         return NULL;
2469 }
2470
2471 static void i915_gem_reset_engine_status(struct intel_engine_cs *engine)
2472 {
2473         struct drm_i915_gem_request *request;
2474         bool ring_hung;
2475
2476         request = i915_gem_find_active_request(engine);
2477         if (request == NULL)
2478                 return;
2479
2480         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2481
2482         i915_set_reset_status(request->ctx, ring_hung);
2483         list_for_each_entry_continue(request, &engine->request_list, list)
2484                 i915_set_reset_status(request->ctx, false);
2485 }
2486
2487 static void i915_gem_reset_engine_cleanup(struct intel_engine_cs *engine)
2488 {
2489         struct intel_ring *ring;
2490
2491         while (!list_empty(&engine->active_list)) {
2492                 struct drm_i915_gem_object *obj;
2493
2494                 obj = list_first_entry(&engine->active_list,
2495                                        struct drm_i915_gem_object,
2496                                        engine_list[engine->id]);
2497
2498                 i915_gem_object_retire__read(obj, engine->id);
2499         }
2500
2501         /* Mark all pending requests as complete so that any concurrent
2502          * (lockless) lookup doesn't try and wait upon the request as we
2503          * reset it.
2504          */
2505         intel_engine_init_seqno(engine, engine->last_submitted_seqno);
2506
2507         /*
2508          * Clear the execlists queue up before freeing the requests, as those
2509          * are the ones that keep the context and ringbuffer backing objects
2510          * pinned in place.
2511          */
2512
2513         if (i915.enable_execlists) {
2514                 /* Ensure irq handler finishes or is cancelled. */
2515                 tasklet_kill(&engine->irq_tasklet);
2516
2517                 intel_execlists_cancel_requests(engine);
2518         }
2519
2520         /*
2521          * We must free the requests after all the corresponding objects have
2522          * been moved off active lists. Which is the same order as the normal
2523          * retire_requests function does. This is important if object hold
2524          * implicit references on things like e.g. ppgtt address spaces through
2525          * the request.
2526          */
2527         if (!list_empty(&engine->request_list)) {
2528                 struct drm_i915_gem_request *request;
2529
2530                 request = list_last_entry(&engine->request_list,
2531                                           struct drm_i915_gem_request,
2532                                           list);
2533
2534                 i915_gem_request_retire_upto(request);
2535         }
2536
2537         /* Having flushed all requests from all queues, we know that all
2538          * ringbuffers must now be empty. However, since we do not reclaim
2539          * all space when retiring the request (to prevent HEADs colliding
2540          * with rapid ringbuffer wraparound) the amount of available space
2541          * upon reset is less than when we start. Do one more pass over
2542          * all the ringbuffers to reset last_retired_head.
2543          */
2544         list_for_each_entry(ring, &engine->buffers, link) {
2545                 ring->last_retired_head = ring->tail;
2546                 intel_ring_update_space(ring);
2547         }
2548
2549         engine->i915->gt.active_engines &= ~intel_engine_flag(engine);
2550 }
2551
2552 void i915_gem_reset(struct drm_device *dev)
2553 {
2554         struct drm_i915_private *dev_priv = to_i915(dev);
2555         struct intel_engine_cs *engine;
2556
2557         /*
2558          * Before we free the objects from the requests, we need to inspect
2559          * them for finding the guilty party. As the requests only borrow
2560          * their reference to the objects, the inspection must be done first.
2561          */
2562         for_each_engine(engine, dev_priv)
2563                 i915_gem_reset_engine_status(engine);
2564
2565         for_each_engine(engine, dev_priv)
2566                 i915_gem_reset_engine_cleanup(engine);
2567         mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
2568
2569         i915_gem_context_reset(dev);
2570
2571         i915_gem_restore_fences(dev);
2572
2573         WARN_ON(i915_verify_lists(dev));
2574 }
2575
2576 /**
2577  * This function clears the request list as sequence numbers are passed.
2578  * @engine: engine to retire requests on
2579  */
2580 void
2581 i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
2582 {
2583         WARN_ON(i915_verify_lists(engine->dev));
2584
2585         /* Retire requests first as we use it above for the early return.
2586          * If we retire requests last, we may use a later seqno and so clear
2587          * the requests lists without clearing the active list, leading to
2588          * confusion.
2589          */
2590         while (!list_empty(&engine->request_list)) {
2591                 struct drm_i915_gem_request *request;
2592
2593                 request = list_first_entry(&engine->request_list,
2594                                            struct drm_i915_gem_request,
2595                                            list);
2596
2597                 if (!i915_gem_request_completed(request))
2598                         break;
2599
2600                 i915_gem_request_retire_upto(request);
2601         }
2602
2603         /* Move any buffers on the active list that are no longer referenced
2604          * by the ringbuffer to the flushing/inactive lists as appropriate,
2605          * before we free the context associated with the requests.
2606          */
2607         while (!list_empty(&engine->active_list)) {
2608                 struct drm_i915_gem_object *obj;
2609
2610                 obj = list_first_entry(&engine->active_list,
2611                                        struct drm_i915_gem_object,
2612                                        engine_list[engine->id]);
2613
2614                 if (!list_empty(&obj->last_read_req[engine->id]->list))
2615                         break;
2616
2617                 i915_gem_object_retire__read(obj, engine->id);
2618         }
2619
2620         WARN_ON(i915_verify_lists(engine->dev));
2621 }
2622
2623 void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
2624 {
2625         struct intel_engine_cs *engine;
2626
2627         lockdep_assert_held(&dev_priv->drm.struct_mutex);
2628
2629         if (dev_priv->gt.active_engines == 0)
2630                 return;
2631
2632         GEM_BUG_ON(!dev_priv->gt.awake);
2633
2634         for_each_engine(engine, dev_priv) {
2635                 i915_gem_retire_requests_ring(engine);
2636                 if (list_empty(&engine->request_list))
2637                         dev_priv->gt.active_engines &= ~intel_engine_flag(engine);
2638         }
2639
2640         if (dev_priv->gt.active_engines == 0)
2641                 queue_delayed_work(dev_priv->wq,
2642                                    &dev_priv->gt.idle_work,
2643                                    msecs_to_jiffies(100));
2644 }
2645
2646 static void
2647 i915_gem_retire_work_handler(struct work_struct *work)
2648 {
2649         struct drm_i915_private *dev_priv =
2650                 container_of(work, typeof(*dev_priv), gt.retire_work.work);
2651         struct drm_device *dev = &dev_priv->drm;
2652
2653         /* Come back later if the device is busy... */
2654         if (mutex_trylock(&dev->struct_mutex)) {
2655                 i915_gem_retire_requests(dev_priv);
2656                 mutex_unlock(&dev->struct_mutex);
2657         }
2658
2659         /* Keep the retire handler running until we are finally idle.
2660          * We do not need to do this test under locking as in the worst-case
2661          * we queue the retire worker once too often.
2662          */
2663         if (READ_ONCE(dev_priv->gt.awake)) {
2664                 i915_queue_hangcheck(dev_priv);
2665                 queue_delayed_work(dev_priv->wq,
2666                                    &dev_priv->gt.retire_work,
2667                                    round_jiffies_up_relative(HZ));
2668         }
2669 }
2670
2671 static void
2672 i915_gem_idle_work_handler(struct work_struct *work)
2673 {
2674         struct drm_i915_private *dev_priv =
2675                 container_of(work, typeof(*dev_priv), gt.idle_work.work);
2676         struct drm_device *dev = &dev_priv->drm;
2677         struct intel_engine_cs *engine;
2678         unsigned int stuck_engines;
2679         bool rearm_hangcheck;
2680
2681         if (!READ_ONCE(dev_priv->gt.awake))
2682                 return;
2683
2684         if (READ_ONCE(dev_priv->gt.active_engines))
2685                 return;
2686
2687         rearm_hangcheck =
2688                 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
2689
2690         if (!mutex_trylock(&dev->struct_mutex)) {
2691                 /* Currently busy, come back later */
2692                 mod_delayed_work(dev_priv->wq,
2693                                  &dev_priv->gt.idle_work,
2694                                  msecs_to_jiffies(50));
2695                 goto out_rearm;
2696         }
2697
2698         if (dev_priv->gt.active_engines)
2699                 goto out_unlock;
2700
2701         for_each_engine(engine, dev_priv)
2702                 i915_gem_batch_pool_fini(&engine->batch_pool);
2703
2704         GEM_BUG_ON(!dev_priv->gt.awake);
2705         dev_priv->gt.awake = false;
2706         rearm_hangcheck = false;
2707
2708         /* As we have disabled hangcheck, we need to unstick any waiters still
2709          * hanging around. However, as we may be racing against the interrupt
2710          * handler or the waiters themselves, we skip enabling the fake-irq.
2711          */
2712         stuck_engines = intel_kick_waiters(dev_priv);
2713         if (unlikely(stuck_engines))
2714                 DRM_DEBUG_DRIVER("kicked stuck waiters (%x)...missed irq?\n",
2715                                  stuck_engines);
2716
2717         if (INTEL_GEN(dev_priv) >= 6)
2718                 gen6_rps_idle(dev_priv);
2719         intel_runtime_pm_put(dev_priv);
2720 out_unlock:
2721         mutex_unlock(&dev->struct_mutex);
2722
2723 out_rearm:
2724         if (rearm_hangcheck) {
2725                 GEM_BUG_ON(!dev_priv->gt.awake);
2726                 i915_queue_hangcheck(dev_priv);
2727         }
2728 }
2729
2730 /**
2731  * Ensures that an object will eventually get non-busy by flushing any required
2732  * write domains, emitting any outstanding lazy request and retiring and
2733  * completed requests.
2734  * @obj: object to flush
2735  */
2736 static int
2737 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2738 {
2739         int i;
2740
2741         if (!obj->active)
2742                 return 0;
2743
2744         for (i = 0; i < I915_NUM_ENGINES; i++) {
2745                 struct drm_i915_gem_request *req;
2746
2747                 req = obj->last_read_req[i];
2748                 if (req == NULL)
2749                         continue;
2750
2751                 if (i915_gem_request_completed(req))
2752                         i915_gem_object_retire__read(obj, i);
2753         }
2754
2755         return 0;
2756 }
2757
2758 /**
2759  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2760  * @dev: drm device pointer
2761  * @data: ioctl data blob
2762  * @file: drm file pointer
2763  *
2764  * Returns 0 if successful, else an error is returned with the remaining time in
2765  * the timeout parameter.
2766  *  -ETIME: object is still busy after timeout
2767  *  -ERESTARTSYS: signal interrupted the wait
2768  *  -ENONENT: object doesn't exist
2769  * Also possible, but rare:
2770  *  -EAGAIN: GPU wedged
2771  *  -ENOMEM: damn
2772  *  -ENODEV: Internal IRQ fail
2773  *  -E?: The add request failed
2774  *
2775  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2776  * non-zero timeout parameter the wait ioctl will wait for the given number of
2777  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2778  * without holding struct_mutex the object may become re-busied before this
2779  * function completes. A similar but shorter * race condition exists in the busy
2780  * ioctl
2781  */
2782 int
2783 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2784 {
2785         struct drm_i915_gem_wait *args = data;
2786         struct drm_i915_gem_object *obj;
2787         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
2788         int i, n = 0;
2789         int ret;
2790
2791         if (args->flags != 0)
2792                 return -EINVAL;
2793
2794         ret = i915_mutex_lock_interruptible(dev);
2795         if (ret)
2796                 return ret;
2797
2798         obj = i915_gem_object_lookup(file, args->bo_handle);
2799         if (!obj) {
2800                 mutex_unlock(&dev->struct_mutex);
2801                 return -ENOENT;
2802         }
2803
2804         /* Need to make sure the object gets inactive eventually. */
2805         ret = i915_gem_object_flush_active(obj);
2806         if (ret)
2807                 goto out;
2808
2809         if (!obj->active)
2810                 goto out;
2811
2812         /* Do this after OLR check to make sure we make forward progress polling
2813          * on this IOCTL with a timeout == 0 (like busy ioctl)
2814          */
2815         if (args->timeout_ns == 0) {
2816                 ret = -ETIME;
2817                 goto out;
2818         }
2819
2820         i915_gem_object_put(obj);
2821
2822         for (i = 0; i < I915_NUM_ENGINES; i++) {
2823                 if (obj->last_read_req[i] == NULL)
2824                         continue;
2825
2826                 req[n++] = i915_gem_request_get(obj->last_read_req[i]);
2827         }
2828
2829         mutex_unlock(&dev->struct_mutex);
2830
2831         for (i = 0; i < n; i++) {
2832                 if (ret == 0)
2833                         ret = __i915_wait_request(req[i], true,
2834                                                   args->timeout_ns > 0 ? &args->timeout_ns : NULL,
2835                                                   to_rps_client(file));
2836                 i915_gem_request_put(req[i]);
2837         }
2838         return ret;
2839
2840 out:
2841         i915_gem_object_put(obj);
2842         mutex_unlock(&dev->struct_mutex);
2843         return ret;
2844 }
2845
2846 static int
2847 __i915_gem_object_sync(struct drm_i915_gem_object *obj,
2848                        struct intel_engine_cs *to,
2849                        struct drm_i915_gem_request *from_req,
2850                        struct drm_i915_gem_request **to_req)
2851 {
2852         struct intel_engine_cs *from;
2853         int ret;
2854
2855         from = i915_gem_request_get_engine(from_req);
2856         if (to == from)
2857                 return 0;
2858
2859         if (i915_gem_request_completed(from_req))
2860                 return 0;
2861
2862         if (!i915.semaphores) {
2863                 struct drm_i915_private *i915 = to_i915(obj->base.dev);
2864                 ret = __i915_wait_request(from_req,
2865                                           i915->mm.interruptible,
2866                                           NULL,
2867                                           NO_WAITBOOST);
2868                 if (ret)
2869                         return ret;
2870
2871                 i915_gem_object_retire_request(obj, from_req);
2872         } else {
2873                 int idx = intel_engine_sync_index(from, to);
2874                 u32 seqno = i915_gem_request_get_seqno(from_req);
2875
2876                 WARN_ON(!to_req);
2877
2878                 if (seqno <= from->semaphore.sync_seqno[idx])
2879                         return 0;
2880
2881                 if (*to_req == NULL) {
2882                         struct drm_i915_gem_request *req;
2883
2884                         req = i915_gem_request_alloc(to, NULL);
2885                         if (IS_ERR(req))
2886                                 return PTR_ERR(req);
2887
2888                         *to_req = req;
2889                 }
2890
2891                 trace_i915_gem_ring_sync_to(*to_req, from, from_req);
2892                 ret = to->semaphore.sync_to(*to_req, from, seqno);
2893                 if (ret)
2894                         return ret;
2895
2896                 /* We use last_read_req because sync_to()
2897                  * might have just caused seqno wrap under
2898                  * the radar.
2899                  */
2900                 from->semaphore.sync_seqno[idx] =
2901                         i915_gem_request_get_seqno(obj->last_read_req[from->id]);
2902         }
2903
2904         return 0;
2905 }
2906
2907 /**
2908  * i915_gem_object_sync - sync an object to a ring.
2909  *
2910  * @obj: object which may be in use on another ring.
2911  * @to: ring we wish to use the object on. May be NULL.
2912  * @to_req: request we wish to use the object for. See below.
2913  *          This will be allocated and returned if a request is
2914  *          required but not passed in.
2915  *
2916  * This code is meant to abstract object synchronization with the GPU.
2917  * Calling with NULL implies synchronizing the object with the CPU
2918  * rather than a particular GPU ring. Conceptually we serialise writes
2919  * between engines inside the GPU. We only allow one engine to write
2920  * into a buffer at any time, but multiple readers. To ensure each has
2921  * a coherent view of memory, we must:
2922  *
2923  * - If there is an outstanding write request to the object, the new
2924  *   request must wait for it to complete (either CPU or in hw, requests
2925  *   on the same ring will be naturally ordered).
2926  *
2927  * - If we are a write request (pending_write_domain is set), the new
2928  *   request must wait for outstanding read requests to complete.
2929  *
2930  * For CPU synchronisation (NULL to) no request is required. For syncing with
2931  * rings to_req must be non-NULL. However, a request does not have to be
2932  * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
2933  * request will be allocated automatically and returned through *to_req. Note
2934  * that it is not guaranteed that commands will be emitted (because the system
2935  * might already be idle). Hence there is no need to create a request that
2936  * might never have any work submitted. Note further that if a request is
2937  * returned in *to_req, it is the responsibility of the caller to submit
2938  * that request (after potentially adding more work to it).
2939  *
2940  * Returns 0 if successful, else propagates up the lower layer error.
2941  */
2942 int
2943 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2944                      struct intel_engine_cs *to,
2945                      struct drm_i915_gem_request **to_req)
2946 {
2947         const bool readonly = obj->base.pending_write_domain == 0;
2948         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
2949         int ret, i, n;
2950
2951         if (!obj->active)
2952                 return 0;
2953
2954         if (to == NULL)
2955                 return i915_gem_object_wait_rendering(obj, readonly);
2956
2957         n = 0;
2958         if (readonly) {
2959                 if (obj->last_write_req)
2960                         req[n++] = obj->last_write_req;
2961         } else {
2962                 for (i = 0; i < I915_NUM_ENGINES; i++)
2963                         if (obj->last_read_req[i])
2964                                 req[n++] = obj->last_read_req[i];
2965         }
2966         for (i = 0; i < n; i++) {
2967                 ret = __i915_gem_object_sync(obj, to, req[i], to_req);
2968                 if (ret)
2969                         return ret;
2970         }
2971
2972         return 0;
2973 }
2974
2975 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2976 {
2977         u32 old_write_domain, old_read_domains;
2978
2979         /* Force a pagefault for domain tracking on next user access */
2980         i915_gem_release_mmap(obj);
2981
2982         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2983                 return;
2984
2985         old_read_domains = obj->base.read_domains;
2986         old_write_domain = obj->base.write_domain;
2987
2988         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2989         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2990
2991         trace_i915_gem_object_change_domain(obj,
2992                                             old_read_domains,
2993                                             old_write_domain);
2994 }
2995
2996 static void __i915_vma_iounmap(struct i915_vma *vma)
2997 {
2998         GEM_BUG_ON(vma->pin_count);
2999
3000         if (vma->iomap == NULL)
3001                 return;
3002
3003         io_mapping_unmap(vma->iomap);
3004         vma->iomap = NULL;
3005 }
3006
3007 static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
3008 {
3009         struct drm_i915_gem_object *obj = vma->obj;
3010         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3011         int ret;
3012
3013         if (list_empty(&vma->obj_link))
3014                 return 0;
3015
3016         if (!drm_mm_node_allocated(&vma->node)) {
3017                 i915_gem_vma_destroy(vma);
3018                 return 0;
3019         }
3020
3021         if (vma->pin_count)
3022                 return -EBUSY;
3023
3024         BUG_ON(obj->pages == NULL);
3025
3026         if (wait) {
3027                 ret = i915_gem_object_wait_rendering(obj, false);
3028                 if (ret)
3029                         return ret;
3030         }
3031
3032         if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3033                 i915_gem_object_finish_gtt(obj);
3034
3035                 /* release the fence reg _after_ flushing */
3036                 ret = i915_gem_object_put_fence(obj);
3037                 if (ret)
3038                         return ret;
3039
3040                 __i915_vma_iounmap(vma);
3041         }
3042
3043         trace_i915_vma_unbind(vma);
3044
3045         vma->vm->unbind_vma(vma);
3046         vma->bound = 0;
3047
3048         list_del_init(&vma->vm_link);
3049         if (vma->is_ggtt) {
3050                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3051                         obj->map_and_fenceable = false;
3052                 } else if (vma->ggtt_view.pages) {
3053                         sg_free_table(vma->ggtt_view.pages);
3054                         kfree(vma->ggtt_view.pages);
3055                 }
3056                 vma->ggtt_view.pages = NULL;
3057         }
3058
3059         drm_mm_remove_node(&vma->node);
3060         i915_gem_vma_destroy(vma);
3061
3062         /* Since the unbound list is global, only move to that list if
3063          * no more VMAs exist. */
3064         if (list_empty(&obj->vma_list))
3065                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3066
3067         /* And finally now the object is completely decoupled from this vma,
3068          * we can drop its hold on the backing storage and allow it to be
3069          * reaped by the shrinker.
3070          */
3071         i915_gem_object_unpin_pages(obj);
3072
3073         return 0;
3074 }
3075
3076 int i915_vma_unbind(struct i915_vma *vma)
3077 {
3078         return __i915_vma_unbind(vma, true);
3079 }
3080
3081 int __i915_vma_unbind_no_wait(struct i915_vma *vma)
3082 {
3083         return __i915_vma_unbind(vma, false);
3084 }
3085
3086 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv)
3087 {
3088         struct intel_engine_cs *engine;
3089         int ret;
3090
3091         lockdep_assert_held(&dev_priv->drm.struct_mutex);
3092
3093         for_each_engine(engine, dev_priv) {
3094                 if (engine->last_context == NULL)
3095                         continue;
3096
3097                 ret = intel_engine_idle(engine);
3098                 if (ret)
3099                         return ret;
3100         }
3101
3102         WARN_ON(i915_verify_lists(dev));
3103         return 0;
3104 }
3105
3106 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3107                                      unsigned long cache_level)
3108 {
3109         struct drm_mm_node *gtt_space = &vma->node;
3110         struct drm_mm_node *other;
3111
3112         /*
3113          * On some machines we have to be careful when putting differing types
3114          * of snoopable memory together to avoid the prefetcher crossing memory
3115          * domains and dying. During vm initialisation, we decide whether or not
3116          * these constraints apply and set the drm_mm.color_adjust
3117          * appropriately.
3118          */
3119         if (vma->vm->mm.color_adjust == NULL)
3120                 return true;
3121
3122         if (!drm_mm_node_allocated(gtt_space))
3123                 return true;
3124
3125         if (list_empty(&gtt_space->node_list))
3126                 return true;
3127
3128         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3129         if (other->allocated && !other->hole_follows && other->color != cache_level)
3130                 return false;
3131
3132         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3133         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3134                 return false;
3135
3136         return true;
3137 }
3138
3139 /**
3140  * Finds free space in the GTT aperture and binds the object or a view of it
3141  * there.
3142  * @obj: object to bind
3143  * @vm: address space to bind into
3144  * @ggtt_view: global gtt view if applicable
3145  * @alignment: requested alignment
3146  * @flags: mask of PIN_* flags to use
3147  */
3148 static struct i915_vma *
3149 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3150                            struct i915_address_space *vm,
3151                            const struct i915_ggtt_view *ggtt_view,
3152                            unsigned alignment,
3153                            uint64_t flags)
3154 {
3155         struct drm_device *dev = obj->base.dev;
3156         struct drm_i915_private *dev_priv = to_i915(dev);
3157         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3158         u32 fence_alignment, unfenced_alignment;
3159         u32 search_flag, alloc_flag;
3160         u64 start, end;
3161         u64 size, fence_size;
3162         struct i915_vma *vma;
3163         int ret;
3164
3165         if (i915_is_ggtt(vm)) {
3166                 u32 view_size;
3167
3168                 if (WARN_ON(!ggtt_view))
3169                         return ERR_PTR(-EINVAL);
3170
3171                 view_size = i915_ggtt_view_size(obj, ggtt_view);
3172
3173                 fence_size = i915_gem_get_gtt_size(dev,
3174                                                    view_size,
3175                                                    obj->tiling_mode);
3176                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3177                                                              view_size,
3178                                                              obj->tiling_mode,
3179                                                              true);
3180                 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
3181                                                                 view_size,
3182                                                                 obj->tiling_mode,
3183                                                                 false);
3184                 size = flags & PIN_MAPPABLE ? fence_size : view_size;
3185         } else {
3186                 fence_size = i915_gem_get_gtt_size(dev,
3187                                                    obj->base.size,
3188                                                    obj->tiling_mode);
3189                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3190                                                              obj->base.size,
3191                                                              obj->tiling_mode,
3192                                                              true);
3193                 unfenced_alignment =
3194                         i915_gem_get_gtt_alignment(dev,
3195                                                    obj->base.size,
3196                                                    obj->tiling_mode,
3197                                                    false);
3198                 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3199         }
3200
3201         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3202         end = vm->total;
3203         if (flags & PIN_MAPPABLE)
3204                 end = min_t(u64, end, ggtt->mappable_end);
3205         if (flags & PIN_ZONE_4G)
3206                 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
3207
3208         if (alignment == 0)
3209                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3210                                                 unfenced_alignment;
3211         if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3212                 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
3213                           ggtt_view ? ggtt_view->type : 0,
3214                           alignment);
3215                 return ERR_PTR(-EINVAL);
3216         }
3217
3218         /* If binding the object/GGTT view requires more space than the entire
3219          * aperture has, reject it early before evicting everything in a vain
3220          * attempt to find space.
3221          */
3222         if (size > end) {
3223                 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
3224                           ggtt_view ? ggtt_view->type : 0,
3225                           size,
3226                           flags & PIN_MAPPABLE ? "mappable" : "total",
3227                           end);
3228                 return ERR_PTR(-E2BIG);
3229         }
3230
3231         ret = i915_gem_object_get_pages(obj);
3232         if (ret)
3233                 return ERR_PTR(ret);
3234
3235         i915_gem_object_pin_pages(obj);
3236
3237         vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
3238                           i915_gem_obj_lookup_or_create_vma(obj, vm);
3239
3240         if (IS_ERR(vma))
3241                 goto err_unpin;
3242
3243         if (flags & PIN_OFFSET_FIXED) {
3244                 uint64_t offset = flags & PIN_OFFSET_MASK;
3245
3246                 if (offset & (alignment - 1) || offset + size > end) {
3247                         ret = -EINVAL;
3248                         goto err_free_vma;
3249                 }
3250                 vma->node.start = offset;
3251                 vma->node.size = size;
3252                 vma->node.color = obj->cache_level;
3253                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3254                 if (ret) {
3255                         ret = i915_gem_evict_for_vma(vma);
3256                         if (ret == 0)
3257                                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3258                 }
3259                 if (ret)
3260                         goto err_free_vma;
3261         } else {
3262                 if (flags & PIN_HIGH) {
3263                         search_flag = DRM_MM_SEARCH_BELOW;
3264                         alloc_flag = DRM_MM_CREATE_TOP;
3265                 } else {
3266                         search_flag = DRM_MM_SEARCH_DEFAULT;
3267                         alloc_flag = DRM_MM_CREATE_DEFAULT;
3268                 }
3269
3270 search_free:
3271                 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3272                                                           size, alignment,
3273                                                           obj->cache_level,
3274                                                           start, end,
3275                                                           search_flag,
3276                                                           alloc_flag);
3277                 if (ret) {
3278                         ret = i915_gem_evict_something(dev, vm, size, alignment,
3279                                                        obj->cache_level,
3280                                                        start, end,
3281                                                        flags);
3282                         if (ret == 0)
3283                                 goto search_free;
3284
3285                         goto err_free_vma;
3286                 }
3287         }
3288         if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3289                 ret = -EINVAL;
3290                 goto err_remove_node;
3291         }
3292
3293         trace_i915_vma_bind(vma, flags);
3294         ret = i915_vma_bind(vma, obj->cache_level, flags);
3295         if (ret)
3296                 goto err_remove_node;
3297
3298         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3299         list_add_tail(&vma->vm_link, &vm->inactive_list);
3300
3301         return vma;
3302
3303 err_remove_node:
3304         drm_mm_remove_node(&vma->node);
3305 err_free_vma:
3306         i915_gem_vma_destroy(vma);
3307         vma = ERR_PTR(ret);
3308 err_unpin:
3309         i915_gem_object_unpin_pages(obj);
3310         return vma;
3311 }
3312
3313 bool
3314 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3315                         bool force)
3316 {
3317         /* If we don't have a page list set up, then we're not pinned
3318          * to GPU, and we can ignore the cache flush because it'll happen
3319          * again at bind time.
3320          */
3321         if (obj->pages == NULL)
3322                 return false;
3323
3324         /*
3325          * Stolen memory is always coherent with the GPU as it is explicitly
3326          * marked as wc by the system, or the system is cache-coherent.
3327          */
3328         if (obj->stolen || obj->phys_handle)
3329                 return false;
3330
3331         /* If the GPU is snooping the contents of the CPU cache,
3332          * we do not need to manually clear the CPU cache lines.  However,
3333          * the caches are only snooped when the render cache is
3334          * flushed/invalidated.  As we always have to emit invalidations
3335          * and flushes when moving into and out of the RENDER domain, correct
3336          * snooping behaviour occurs naturally as the result of our domain
3337          * tracking.
3338          */
3339         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3340                 obj->cache_dirty = true;
3341                 return false;
3342         }
3343
3344         trace_i915_gem_object_clflush(obj);
3345         drm_clflush_sg(obj->pages);
3346         obj->cache_dirty = false;
3347
3348         return true;
3349 }
3350
3351 /** Flushes the GTT write domain for the object if it's dirty. */
3352 static void
3353 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3354 {
3355         uint32_t old_write_domain;
3356
3357         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3358                 return;
3359
3360         /* No actual flushing is required for the GTT write domain.  Writes
3361          * to it immediately go to main memory as far as we know, so there's
3362          * no chipset flush.  It also doesn't land in render cache.
3363          *
3364          * However, we do have to enforce the order so that all writes through
3365          * the GTT land before any writes to the device, such as updates to
3366          * the GATT itself.
3367          */
3368         wmb();
3369
3370         old_write_domain = obj->base.write_domain;
3371         obj->base.write_domain = 0;
3372
3373         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3374
3375         trace_i915_gem_object_change_domain(obj,
3376                                             obj->base.read_domains,
3377                                             old_write_domain);
3378 }
3379
3380 /** Flushes the CPU write domain for the object if it's dirty. */
3381 static void
3382 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3383 {
3384         uint32_t old_write_domain;
3385
3386         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3387                 return;
3388
3389         if (i915_gem_clflush_object(obj, obj->pin_display))
3390                 i915_gem_chipset_flush(to_i915(obj->base.dev));
3391
3392         old_write_domain = obj->base.write_domain;
3393         obj->base.write_domain = 0;
3394
3395         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3396
3397         trace_i915_gem_object_change_domain(obj,
3398                                             obj->base.read_domains,
3399                                             old_write_domain);
3400 }
3401
3402 /**
3403  * Moves a single object to the GTT read, and possibly write domain.
3404  * @obj: object to act on
3405  * @write: ask for write access or read only
3406  *
3407  * This function returns when the move is complete, including waiting on
3408  * flushes to occur.
3409  */
3410 int
3411 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3412 {
3413         struct drm_device *dev = obj->base.dev;
3414         struct drm_i915_private *dev_priv = to_i915(dev);
3415         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3416         uint32_t old_write_domain, old_read_domains;
3417         struct i915_vma *vma;
3418         int ret;
3419
3420         ret = i915_gem_object_wait_rendering(obj, !write);
3421         if (ret)
3422                 return ret;
3423
3424         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3425                 return 0;
3426
3427         /* Flush and acquire obj->pages so that we are coherent through
3428          * direct access in memory with previous cached writes through
3429          * shmemfs and that our cache domain tracking remains valid.
3430          * For example, if the obj->filp was moved to swap without us
3431          * being notified and releasing the pages, we would mistakenly
3432          * continue to assume that the obj remained out of the CPU cached
3433          * domain.
3434          */
3435         ret = i915_gem_object_get_pages(obj);
3436         if (ret)
3437                 return ret;
3438
3439         i915_gem_object_flush_cpu_write_domain(obj);
3440
3441         /* Serialise direct access to this object with the barriers for
3442          * coherent writes from the GPU, by effectively invalidating the
3443          * GTT domain upon first access.
3444          */
3445         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3446                 mb();
3447
3448         old_write_domain = obj->base.write_domain;
3449         old_read_domains = obj->base.read_domains;
3450
3451         /* It should now be out of any other write domains, and we can update
3452          * the domain values for our changes.
3453          */
3454         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3455         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3456         if (write) {
3457                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3458                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3459                 obj->dirty = 1;
3460         }
3461
3462         trace_i915_gem_object_change_domain(obj,
3463                                             old_read_domains,
3464                                             old_write_domain);
3465
3466         /* And bump the LRU for this access */
3467         vma = i915_gem_obj_to_ggtt(obj);
3468         if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
3469                 list_move_tail(&vma->vm_link,
3470                                &ggtt->base.inactive_list);
3471
3472         return 0;
3473 }
3474
3475 /**
3476  * Changes the cache-level of an object across all VMA.
3477  * @obj: object to act on
3478  * @cache_level: new cache level to set for the object
3479  *
3480  * After this function returns, the object will be in the new cache-level
3481  * across all GTT and the contents of the backing storage will be coherent,
3482  * with respect to the new cache-level. In order to keep the backing storage
3483  * coherent for all users, we only allow a single cache level to be set
3484  * globally on the object and prevent it from being changed whilst the
3485  * hardware is reading from the object. That is if the object is currently
3486  * on the scanout it will be set to uncached (or equivalent display
3487  * cache coherency) and all non-MOCS GPU access will also be uncached so
3488  * that all direct access to the scanout remains coherent.
3489  */
3490 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3491                                     enum i915_cache_level cache_level)
3492 {
3493         struct drm_device *dev = obj->base.dev;
3494         struct i915_vma *vma, *next;
3495         bool bound = false;
3496         int ret = 0;
3497
3498         if (obj->cache_level == cache_level)
3499                 goto out;
3500
3501         /* Inspect the list of currently bound VMA and unbind any that would
3502          * be invalid given the new cache-level. This is principally to
3503          * catch the issue of the CS prefetch crossing page boundaries and
3504          * reading an invalid PTE on older architectures.
3505          */
3506         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
3507                 if (!drm_mm_node_allocated(&vma->node))
3508                         continue;
3509
3510                 if (vma->pin_count) {
3511                         DRM_DEBUG("can not change the cache level of pinned objects\n");
3512                         return -EBUSY;
3513                 }
3514
3515                 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
3516                         ret = i915_vma_unbind(vma);
3517                         if (ret)
3518                                 return ret;
3519                 } else
3520                         bound = true;
3521         }
3522
3523         /* We can reuse the existing drm_mm nodes but need to change the
3524          * cache-level on the PTE. We could simply unbind them all and
3525          * rebind with the correct cache-level on next use. However since
3526          * we already have a valid slot, dma mapping, pages etc, we may as
3527          * rewrite the PTE in the belief that doing so tramples upon less
3528          * state and so involves less work.
3529          */
3530         if (bound) {
3531                 /* Before we change the PTE, the GPU must not be accessing it.
3532                  * If we wait upon the object, we know that all the bound
3533                  * VMA are no longer active.
3534                  */
3535                 ret = i915_gem_object_wait_rendering(obj, false);
3536                 if (ret)
3537                         return ret;
3538
3539                 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
3540                         /* Access to snoopable pages through the GTT is
3541                          * incoherent and on some machines causes a hard
3542                          * lockup. Relinquish the CPU mmaping to force
3543                          * userspace to refault in the pages and we can
3544                          * then double check if the GTT mapping is still
3545                          * valid for that pointer access.
3546                          */
3547                         i915_gem_release_mmap(obj);
3548
3549                         /* As we no longer need a fence for GTT access,
3550                          * we can relinquish it now (and so prevent having
3551                          * to steal a fence from someone else on the next
3552                          * fence request). Note GPU activity would have
3553                          * dropped the fence as all snoopable access is
3554                          * supposed to be linear.
3555                          */
3556                         ret = i915_gem_object_put_fence(obj);
3557                         if (ret)
3558                                 return ret;
3559                 } else {
3560                         /* We either have incoherent backing store and
3561                          * so no GTT access or the architecture is fully
3562                          * coherent. In such cases, existing GTT mmaps
3563                          * ignore the cache bit in the PTE and we can
3564                          * rewrite it without confusing the GPU or having
3565                          * to force userspace to fault back in its mmaps.
3566                          */
3567                 }
3568
3569                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3570                         if (!drm_mm_node_allocated(&vma->node))
3571                                 continue;
3572
3573                         ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3574                         if (ret)
3575                                 return ret;
3576                 }
3577         }
3578
3579         list_for_each_entry(vma, &obj->vma_list, obj_link)
3580                 vma->node.color = cache_level;
3581         obj->cache_level = cache_level;
3582
3583 out:
3584         /* Flush the dirty CPU caches to the backing storage so that the
3585          * object is now coherent at its new cache level (with respect
3586          * to the access domain).
3587          */
3588         if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
3589                 if (i915_gem_clflush_object(obj, true))
3590                         i915_gem_chipset_flush(to_i915(obj->base.dev));
3591         }
3592
3593         return 0;
3594 }
3595
3596 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3597                                struct drm_file *file)
3598 {
3599         struct drm_i915_gem_caching *args = data;
3600         struct drm_i915_gem_object *obj;
3601
3602         obj = i915_gem_object_lookup(file, args->handle);
3603         if (!obj)
3604                 return -ENOENT;
3605
3606         switch (obj->cache_level) {
3607         case I915_CACHE_LLC:
3608         case I915_CACHE_L3_LLC:
3609                 args->caching = I915_CACHING_CACHED;
3610                 break;
3611
3612         case I915_CACHE_WT:
3613                 args->caching = I915_CACHING_DISPLAY;
3614                 break;
3615
3616         default:
3617                 args->caching = I915_CACHING_NONE;
3618                 break;
3619         }
3620
3621         i915_gem_object_put_unlocked(obj);
3622         return 0;
3623 }
3624
3625 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3626                                struct drm_file *file)
3627 {
3628         struct drm_i915_private *dev_priv = to_i915(dev);
3629         struct drm_i915_gem_caching *args = data;
3630         struct drm_i915_gem_object *obj;
3631         enum i915_cache_level level;
3632         int ret;
3633
3634         switch (args->caching) {
3635         case I915_CACHING_NONE:
3636                 level = I915_CACHE_NONE;
3637                 break;
3638         case I915_CACHING_CACHED:
3639                 /*
3640                  * Due to a HW issue on BXT A stepping, GPU stores via a
3641                  * snooped mapping may leave stale data in a corresponding CPU
3642                  * cacheline, whereas normally such cachelines would get
3643                  * invalidated.
3644                  */
3645                 if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
3646                         return -ENODEV;
3647
3648                 level = I915_CACHE_LLC;
3649                 break;
3650         case I915_CACHING_DISPLAY:
3651                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3652                 break;
3653         default:
3654                 return -EINVAL;
3655         }
3656
3657         intel_runtime_pm_get(dev_priv);
3658
3659         ret = i915_mutex_lock_interruptible(dev);
3660         if (ret)
3661                 goto rpm_put;
3662
3663         obj = i915_gem_object_lookup(file, args->handle);
3664         if (!obj) {
3665                 ret = -ENOENT;
3666                 goto unlock;
3667         }
3668
3669         ret = i915_gem_object_set_cache_level(obj, level);
3670
3671         i915_gem_object_put(obj);
3672 unlock:
3673         mutex_unlock(&dev->struct_mutex);
3674 rpm_put:
3675         intel_runtime_pm_put(dev_priv);
3676
3677         return ret;
3678 }
3679
3680 /*
3681  * Prepare buffer for display plane (scanout, cursors, etc).
3682  * Can be called from an uninterruptible phase (modesetting) and allows
3683  * any flushes to be pipelined (for pageflips).
3684  */
3685 int
3686 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3687                                      u32 alignment,
3688                                      const struct i915_ggtt_view *view)
3689 {
3690         u32 old_read_domains, old_write_domain;
3691         int ret;
3692
3693         /* Mark the pin_display early so that we account for the
3694          * display coherency whilst setting up the cache domains.
3695          */
3696         obj->pin_display++;
3697
3698         /* The display engine is not coherent with the LLC cache on gen6.  As
3699          * a result, we make sure that the pinning that is about to occur is
3700          * done with uncached PTEs. This is lowest common denominator for all
3701          * chipsets.
3702          *
3703          * However for gen6+, we could do better by using the GFDT bit instead
3704          * of uncaching, which would allow us to flush all the LLC-cached data
3705          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3706          */
3707         ret = i915_gem_object_set_cache_level(obj,
3708                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3709         if (ret)
3710                 goto err_unpin_display;
3711
3712         /* As the user may map the buffer once pinned in the display plane
3713          * (e.g. libkms for the bootup splash), we have to ensure that we
3714          * always use map_and_fenceable for all scanout buffers.
3715          */
3716         ret = i915_gem_object_ggtt_pin(obj, view, alignment,
3717                                        view->type == I915_GGTT_VIEW_NORMAL ?
3718                                        PIN_MAPPABLE : 0);
3719         if (ret)
3720                 goto err_unpin_display;
3721
3722         i915_gem_object_flush_cpu_write_domain(obj);
3723
3724         old_write_domain = obj->base.write_domain;
3725         old_read_domains = obj->base.read_domains;
3726
3727         /* It should now be out of any other write domains, and we can update
3728          * the domain values for our changes.
3729          */
3730         obj->base.write_domain = 0;
3731         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3732
3733         trace_i915_gem_object_change_domain(obj,
3734                                             old_read_domains,
3735                                             old_write_domain);
3736
3737         return 0;
3738
3739 err_unpin_display:
3740         obj->pin_display--;
3741         return ret;
3742 }
3743
3744 void
3745 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
3746                                          const struct i915_ggtt_view *view)
3747 {
3748         if (WARN_ON(obj->pin_display == 0))
3749                 return;
3750
3751         i915_gem_object_ggtt_unpin_view(obj, view);
3752
3753         obj->pin_display--;
3754 }
3755
3756 /**
3757  * Moves a single object to the CPU read, and possibly write domain.
3758  * @obj: object to act on
3759  * @write: requesting write or read-only access
3760  *
3761  * This function returns when the move is complete, including waiting on
3762  * flushes to occur.
3763  */
3764 int
3765 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3766 {
3767         uint32_t old_write_domain, old_read_domains;
3768         int ret;
3769
3770         ret = i915_gem_object_wait_rendering(obj, !write);
3771         if (ret)
3772                 return ret;
3773
3774         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3775                 return 0;
3776
3777         i915_gem_object_flush_gtt_write_domain(obj);
3778
3779         old_write_domain = obj->base.write_domain;
3780         old_read_domains = obj->base.read_domains;
3781
3782         /* Flush the CPU cache if it's still invalid. */
3783         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3784                 i915_gem_clflush_object(obj, false);
3785
3786                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3787         }
3788
3789         /* It should now be out of any other write domains, and we can update
3790          * the domain values for our changes.
3791          */
3792         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3793
3794         /* If we're writing through the CPU, then the GPU read domains will
3795          * need to be invalidated at next use.
3796          */
3797         if (write) {
3798                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3799                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3800         }
3801
3802         trace_i915_gem_object_change_domain(obj,
3803                                             old_read_domains,
3804                                             old_write_domain);
3805
3806         return 0;
3807 }
3808
3809 /* Throttle our rendering by waiting until the ring has completed our requests
3810  * emitted over 20 msec ago.
3811  *
3812  * Note that if we were to use the current jiffies each time around the loop,
3813  * we wouldn't escape the function with any frames outstanding if the time to
3814  * render a frame was over 20ms.
3815  *
3816  * This should get us reasonable parallelism between CPU and GPU but also
3817  * relatively low latency when blocking on a particular request to finish.
3818  */
3819 static int
3820 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3821 {
3822         struct drm_i915_private *dev_priv = to_i915(dev);
3823         struct drm_i915_file_private *file_priv = file->driver_priv;
3824         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3825         struct drm_i915_gem_request *request, *target = NULL;
3826         int ret;
3827
3828         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3829         if (ret)
3830                 return ret;
3831
3832         /* ABI: return -EIO if already wedged */
3833         if (i915_terminally_wedged(&dev_priv->gpu_error))
3834                 return -EIO;
3835
3836         spin_lock(&file_priv->mm.lock);
3837         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3838                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3839                         break;
3840
3841                 /*
3842                  * Note that the request might not have been submitted yet.
3843                  * In which case emitted_jiffies will be zero.
3844                  */
3845                 if (!request->emitted_jiffies)
3846                         continue;
3847
3848                 target = request;
3849         }
3850         if (target)
3851                 i915_gem_request_get(target);
3852         spin_unlock(&file_priv->mm.lock);
3853
3854         if (target == NULL)
3855                 return 0;
3856
3857         ret = __i915_wait_request(target, true, NULL, NULL);
3858         i915_gem_request_put(target);
3859
3860         return ret;
3861 }
3862
3863 static bool
3864 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
3865 {
3866         struct drm_i915_gem_object *obj = vma->obj;
3867
3868         if (alignment &&
3869             vma->node.start & (alignment - 1))
3870                 return true;
3871
3872         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
3873                 return true;
3874
3875         if (flags & PIN_OFFSET_BIAS &&
3876             vma->node.start < (flags & PIN_OFFSET_MASK))
3877                 return true;
3878
3879         if (flags & PIN_OFFSET_FIXED &&
3880             vma->node.start != (flags & PIN_OFFSET_MASK))
3881                 return true;
3882
3883         return false;
3884 }
3885
3886 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
3887 {
3888         struct drm_i915_gem_object *obj = vma->obj;
3889         bool mappable, fenceable;
3890         u32 fence_size, fence_alignment;
3891
3892         fence_size = i915_gem_get_gtt_size(obj->base.dev,
3893                                            obj->base.size,
3894                                            obj->tiling_mode);
3895         fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
3896                                                      obj->base.size,
3897                                                      obj->tiling_mode,
3898                                                      true);
3899
3900         fenceable = (vma->node.size == fence_size &&
3901                      (vma->node.start & (fence_alignment - 1)) == 0);
3902
3903         mappable = (vma->node.start + fence_size <=
3904                     to_i915(obj->base.dev)->ggtt.mappable_end);
3905
3906         obj->map_and_fenceable = mappable && fenceable;
3907 }
3908
3909 static int
3910 i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
3911                        struct i915_address_space *vm,
3912                        const struct i915_ggtt_view *ggtt_view,
3913                        uint32_t alignment,
3914                        uint64_t flags)
3915 {
3916         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3917         struct i915_vma *vma;
3918         unsigned bound;
3919         int ret;
3920
3921         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
3922                 return -ENODEV;
3923
3924         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
3925                 return -EINVAL;
3926
3927         if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
3928                 return -EINVAL;
3929
3930         if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
3931                 return -EINVAL;
3932
3933         vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
3934                           i915_gem_obj_to_vma(obj, vm);
3935
3936         if (vma) {
3937                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3938                         return -EBUSY;
3939
3940                 if (i915_vma_misplaced(vma, alignment, flags)) {
3941                         WARN(vma->pin_count,
3942                              "bo is already pinned in %s with incorrect alignment:"
3943                              " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
3944                              " obj->map_and_fenceable=%d\n",
3945                              ggtt_view ? "ggtt" : "ppgtt",
3946                              upper_32_bits(vma->node.start),
3947                              lower_32_bits(vma->node.start),
3948                              alignment,
3949                              !!(flags & PIN_MAPPABLE),
3950                              obj->map_and_fenceable);
3951                         ret = i915_vma_unbind(vma);
3952                         if (ret)
3953                                 return ret;
3954
3955                         vma = NULL;
3956                 }
3957         }
3958
3959         bound = vma ? vma->bound : 0;
3960         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
3961                 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
3962                                                  flags);
3963                 if (IS_ERR(vma))
3964                         return PTR_ERR(vma);
3965         } else {
3966                 ret = i915_vma_bind(vma, obj->cache_level, flags);
3967                 if (ret)
3968                         return ret;
3969         }
3970
3971         if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
3972             (bound ^ vma->bound) & GLOBAL_BIND) {
3973                 __i915_vma_set_map_and_fenceable(vma);
3974                 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
3975         }
3976
3977         vma->pin_count++;
3978         return 0;
3979 }
3980
3981 int
3982 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3983                     struct i915_address_space *vm,
3984                     uint32_t alignment,
3985                     uint64_t flags)
3986 {
3987         return i915_gem_object_do_pin(obj, vm,
3988                                       i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
3989                                       alignment, flags);
3990 }
3991
3992 int
3993 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3994                          const struct i915_ggtt_view *view,
3995                          uint32_t alignment,
3996                          uint64_t flags)
3997 {
3998         struct drm_device *dev = obj->base.dev;
3999         struct drm_i915_private *dev_priv = to_i915(dev);
4000         struct i915_ggtt *ggtt = &dev_priv->ggtt;
4001
4002         BUG_ON(!view);
4003
4004         return i915_gem_object_do_pin(obj, &ggtt->base, view,
4005                                       alignment, flags | PIN_GLOBAL);
4006 }
4007
4008 void
4009 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
4010                                 const struct i915_ggtt_view *view)
4011 {
4012         struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
4013
4014         WARN_ON(vma->pin_count == 0);
4015         WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
4016
4017         --vma->pin_count;
4018 }
4019
4020 int
4021 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4022                     struct drm_file *file)
4023 {
4024         struct drm_i915_gem_busy *args = data;
4025         struct drm_i915_gem_object *obj;
4026         int ret;
4027
4028         ret = i915_mutex_lock_interruptible(dev);
4029         if (ret)
4030                 return ret;
4031
4032         obj = i915_gem_object_lookup(file, args->handle);
4033         if (!obj) {
4034                 ret = -ENOENT;
4035                 goto unlock;
4036         }
4037
4038         /* Count all active objects as busy, even if they are currently not used
4039          * by the gpu. Users of this interface expect objects to eventually
4040          * become non-busy without any further actions, therefore emit any
4041          * necessary flushes here.
4042          */
4043         ret = i915_gem_object_flush_active(obj);
4044         if (ret)
4045                 goto unref;
4046
4047         args->busy = 0;
4048         if (obj->active) {
4049                 int i;
4050
4051                 for (i = 0; i < I915_NUM_ENGINES; i++) {
4052                         struct drm_i915_gem_request *req;
4053
4054                         req = obj->last_read_req[i];
4055                         if (req)
4056                                 args->busy |= 1 << (16 + req->engine->exec_id);
4057                 }
4058                 if (obj->last_write_req)
4059                         args->busy |= obj->last_write_req->engine->exec_id;
4060         }
4061
4062 unref:
4063         i915_gem_object_put(obj);
4064 unlock:
4065         mutex_unlock(&dev->struct_mutex);
4066         return ret;
4067 }
4068
4069 int
4070 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4071                         struct drm_file *file_priv)
4072 {
4073         return i915_gem_ring_throttle(dev, file_priv);
4074 }
4075
4076 int
4077 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4078                        struct drm_file *file_priv)
4079 {
4080         struct drm_i915_private *dev_priv = to_i915(dev);
4081         struct drm_i915_gem_madvise *args = data;
4082         struct drm_i915_gem_object *obj;
4083         int ret;
4084
4085         switch (args->madv) {
4086         case I915_MADV_DONTNEED:
4087         case I915_MADV_WILLNEED:
4088             break;
4089         default:
4090             return -EINVAL;
4091         }
4092
4093         ret = i915_mutex_lock_interruptible(dev);
4094         if (ret)
4095                 return ret;
4096
4097         obj = i915_gem_object_lookup(file_priv, args->handle);
4098         if (!obj) {
4099                 ret = -ENOENT;
4100                 goto unlock;
4101         }
4102
4103         if (i915_gem_obj_is_pinned(obj)) {
4104                 ret = -EINVAL;
4105                 goto out;
4106         }
4107
4108         if (obj->pages &&
4109             obj->tiling_mode != I915_TILING_NONE &&
4110             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4111                 if (obj->madv == I915_MADV_WILLNEED)
4112                         i915_gem_object_unpin_pages(obj);
4113                 if (args->madv == I915_MADV_WILLNEED)
4114                         i915_gem_object_pin_pages(obj);
4115         }
4116
4117         if (obj->madv != __I915_MADV_PURGED)
4118                 obj->madv = args->madv;
4119
4120         /* if the object is no longer attached, discard its backing storage */
4121         if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4122                 i915_gem_object_truncate(obj);
4123
4124         args->retained = obj->madv != __I915_MADV_PURGED;
4125
4126 out:
4127         i915_gem_object_put(obj);
4128 unlock:
4129         mutex_unlock(&dev->struct_mutex);
4130         return ret;
4131 }
4132
4133 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4134                           const struct drm_i915_gem_object_ops *ops)
4135 {
4136         int i;
4137
4138         INIT_LIST_HEAD(&obj->global_list);
4139         for (i = 0; i < I915_NUM_ENGINES; i++)
4140                 INIT_LIST_HEAD(&obj->engine_list[i]);
4141         INIT_LIST_HEAD(&obj->obj_exec_link);
4142         INIT_LIST_HEAD(&obj->vma_list);
4143         INIT_LIST_HEAD(&obj->batch_pool_link);
4144
4145         obj->ops = ops;
4146
4147         obj->fence_reg = I915_FENCE_REG_NONE;
4148         obj->madv = I915_MADV_WILLNEED;
4149
4150         i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
4151 }
4152
4153 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4154         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
4155         .get_pages = i915_gem_object_get_pages_gtt,
4156         .put_pages = i915_gem_object_put_pages_gtt,
4157 };
4158
4159 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
4160                                                   size_t size)
4161 {
4162         struct drm_i915_gem_object *obj;
4163         struct address_space *mapping;
4164         gfp_t mask;
4165         int ret;
4166
4167         obj = i915_gem_object_alloc(dev);
4168         if (obj == NULL)
4169                 return ERR_PTR(-ENOMEM);
4170
4171         ret = drm_gem_object_init(dev, &obj->base, size);
4172         if (ret)
4173                 goto fail;
4174
4175         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4176         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4177                 /* 965gm cannot relocate objects above 4GiB. */
4178                 mask &= ~__GFP_HIGHMEM;
4179                 mask |= __GFP_DMA32;
4180         }
4181
4182         mapping = file_inode(obj->base.filp)->i_mapping;
4183         mapping_set_gfp_mask(mapping, mask);
4184
4185         i915_gem_object_init(obj, &i915_gem_object_ops);
4186
4187         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4188         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4189
4190         if (HAS_LLC(dev)) {
4191                 /* On some devices, we can have the GPU use the LLC (the CPU
4192                  * cache) for about a 10% performance improvement
4193                  * compared to uncached.  Graphics requests other than
4194                  * display scanout are coherent with the CPU in
4195                  * accessing this cache.  This means in this mode we
4196                  * don't need to clflush on the CPU side, and on the
4197                  * GPU side we only need to flush internal caches to
4198                  * get data visible to the CPU.
4199                  *
4200                  * However, we maintain the display planes as UC, and so
4201                  * need to rebind when first used as such.
4202                  */
4203                 obj->cache_level = I915_CACHE_LLC;
4204         } else
4205                 obj->cache_level = I915_CACHE_NONE;
4206
4207         trace_i915_gem_object_create(obj);
4208
4209         return obj;
4210
4211 fail:
4212         i915_gem_object_free(obj);
4213
4214         return ERR_PTR(ret);
4215 }
4216
4217 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4218 {
4219         /* If we are the last user of the backing storage (be it shmemfs
4220          * pages or stolen etc), we know that the pages are going to be
4221          * immediately released. In this case, we can then skip copying
4222          * back the contents from the GPU.
4223          */
4224
4225         if (obj->madv != I915_MADV_WILLNEED)
4226                 return false;
4227
4228         if (obj->base.filp == NULL)
4229                 return true;
4230
4231         /* At first glance, this looks racy, but then again so would be
4232          * userspace racing mmap against close. However, the first external
4233          * reference to the filp can only be obtained through the
4234          * i915_gem_mmap_ioctl() which safeguards us against the user
4235          * acquiring such a reference whilst we are in the middle of
4236          * freeing the object.
4237          */
4238         return atomic_long_read(&obj->base.filp->f_count) == 1;
4239 }
4240
4241 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4242 {
4243         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4244         struct drm_device *dev = obj->base.dev;
4245         struct drm_i915_private *dev_priv = to_i915(dev);
4246         struct i915_vma *vma, *next;
4247
4248         intel_runtime_pm_get(dev_priv);
4249
4250         trace_i915_gem_object_destroy(obj);
4251
4252         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4253                 int ret;
4254
4255                 vma->pin_count = 0;
4256                 ret = __i915_vma_unbind_no_wait(vma);
4257                 if (WARN_ON(ret == -ERESTARTSYS)) {
4258                         bool was_interruptible;
4259
4260                         was_interruptible = dev_priv->mm.interruptible;
4261                         dev_priv->mm.interruptible = false;
4262
4263                         WARN_ON(i915_vma_unbind(vma));
4264
4265                         dev_priv->mm.interruptible = was_interruptible;
4266                 }
4267         }
4268
4269         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4270          * before progressing. */
4271         if (obj->stolen)
4272                 i915_gem_object_unpin_pages(obj);
4273
4274         WARN_ON(obj->frontbuffer_bits);
4275
4276         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4277             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4278             obj->tiling_mode != I915_TILING_NONE)
4279                 i915_gem_object_unpin_pages(obj);
4280
4281         if (WARN_ON(obj->pages_pin_count))
4282                 obj->pages_pin_count = 0;
4283         if (discard_backing_storage(obj))
4284                 obj->madv = I915_MADV_DONTNEED;
4285         i915_gem_object_put_pages(obj);
4286
4287         BUG_ON(obj->pages);
4288
4289         if (obj->base.import_attach)
4290                 drm_prime_gem_destroy(&obj->base, NULL);
4291
4292         if (obj->ops->release)
4293                 obj->ops->release(obj);
4294
4295         drm_gem_object_release(&obj->base);
4296         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4297
4298         kfree(obj->bit_17);
4299         i915_gem_object_free(obj);
4300
4301         intel_runtime_pm_put(dev_priv);
4302 }
4303
4304 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4305                                      struct i915_address_space *vm)
4306 {
4307         struct i915_vma *vma;
4308         list_for_each_entry(vma, &obj->vma_list, obj_link) {
4309                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4310                     vma->vm == vm)
4311                         return vma;
4312         }
4313         return NULL;
4314 }
4315
4316 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4317                                            const struct i915_ggtt_view *view)
4318 {
4319         struct i915_vma *vma;
4320
4321         GEM_BUG_ON(!view);
4322
4323         list_for_each_entry(vma, &obj->vma_list, obj_link)
4324                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
4325                         return vma;
4326         return NULL;
4327 }
4328
4329 void i915_gem_vma_destroy(struct i915_vma *vma)
4330 {
4331         WARN_ON(vma->node.allocated);
4332
4333         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4334         if (!list_empty(&vma->exec_list))
4335                 return;
4336
4337         if (!vma->is_ggtt)
4338                 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
4339
4340         list_del(&vma->obj_link);
4341
4342         kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
4343 }
4344
4345 static void
4346 i915_gem_stop_engines(struct drm_device *dev)
4347 {
4348         struct drm_i915_private *dev_priv = to_i915(dev);
4349         struct intel_engine_cs *engine;
4350
4351         for_each_engine(engine, dev_priv)
4352                 dev_priv->gt.stop_engine(engine);
4353 }
4354
4355 int
4356 i915_gem_suspend(struct drm_device *dev)
4357 {
4358         struct drm_i915_private *dev_priv = to_i915(dev);
4359         int ret = 0;
4360
4361         intel_suspend_gt_powersave(dev_priv);
4362
4363         mutex_lock(&dev->struct_mutex);
4364
4365         /* We have to flush all the executing contexts to main memory so
4366          * that they can saved in the hibernation image. To ensure the last
4367          * context image is coherent, we have to switch away from it. That
4368          * leaves the dev_priv->kernel_context still active when
4369          * we actually suspend, and its image in memory may not match the GPU
4370          * state. Fortunately, the kernel_context is disposable and we do
4371          * not rely on its state.
4372          */
4373         ret = i915_gem_switch_to_kernel_context(dev_priv);
4374         if (ret)
4375                 goto err;
4376
4377         ret = i915_gem_wait_for_idle(dev_priv);
4378         if (ret)
4379                 goto err;
4380
4381         i915_gem_retire_requests(dev_priv);
4382
4383         /* Note that rather than stopping the engines, all we have to do
4384          * is assert that every RING_HEAD == RING_TAIL (all execution complete)
4385          * and similar for all logical context images (to ensure they are
4386          * all ready for hibernation).
4387          */
4388         i915_gem_stop_engines(dev);
4389         i915_gem_context_lost(dev_priv);
4390         mutex_unlock(&dev->struct_mutex);
4391
4392         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4393         cancel_delayed_work_sync(&dev_priv->gt.retire_work);
4394         flush_delayed_work(&dev_priv->gt.idle_work);
4395
4396         /* Assert that we sucessfully flushed all the work and
4397          * reset the GPU back to its idle, low power state.
4398          */
4399         WARN_ON(dev_priv->gt.awake);
4400
4401         return 0;
4402
4403 err:
4404         mutex_unlock(&dev->struct_mutex);
4405         return ret;
4406 }
4407
4408 void i915_gem_resume(struct drm_device *dev)
4409 {
4410         struct drm_i915_private *dev_priv = to_i915(dev);
4411
4412         mutex_lock(&dev->struct_mutex);
4413         i915_gem_restore_gtt_mappings(dev);
4414
4415         /* As we didn't flush the kernel context before suspend, we cannot
4416          * guarantee that the context image is complete. So let's just reset
4417          * it and start again.
4418          */
4419         if (i915.enable_execlists)
4420                 intel_lr_context_reset(dev_priv, dev_priv->kernel_context);
4421
4422         mutex_unlock(&dev->struct_mutex);
4423 }
4424
4425 void i915_gem_init_swizzling(struct drm_device *dev)
4426 {
4427         struct drm_i915_private *dev_priv = to_i915(dev);
4428
4429         if (INTEL_INFO(dev)->gen < 5 ||
4430             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4431                 return;
4432
4433         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4434                                  DISP_TILE_SURFACE_SWIZZLING);
4435
4436         if (IS_GEN5(dev))
4437                 return;
4438
4439         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4440         if (IS_GEN6(dev))
4441                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4442         else if (IS_GEN7(dev))
4443                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4444         else if (IS_GEN8(dev))
4445                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4446         else
4447                 BUG();
4448 }
4449
4450 static void init_unused_ring(struct drm_device *dev, u32 base)
4451 {
4452         struct drm_i915_private *dev_priv = to_i915(dev);
4453
4454         I915_WRITE(RING_CTL(base), 0);
4455         I915_WRITE(RING_HEAD(base), 0);
4456         I915_WRITE(RING_TAIL(base), 0);
4457         I915_WRITE(RING_START(base), 0);
4458 }
4459
4460 static void init_unused_rings(struct drm_device *dev)
4461 {
4462         if (IS_I830(dev)) {
4463                 init_unused_ring(dev, PRB1_BASE);
4464                 init_unused_ring(dev, SRB0_BASE);
4465                 init_unused_ring(dev, SRB1_BASE);
4466                 init_unused_ring(dev, SRB2_BASE);
4467                 init_unused_ring(dev, SRB3_BASE);
4468         } else if (IS_GEN2(dev)) {
4469                 init_unused_ring(dev, SRB0_BASE);
4470                 init_unused_ring(dev, SRB1_BASE);
4471         } else if (IS_GEN3(dev)) {
4472                 init_unused_ring(dev, PRB1_BASE);
4473                 init_unused_ring(dev, PRB2_BASE);
4474         }
4475 }
4476
4477 int
4478 i915_gem_init_hw(struct drm_device *dev)
4479 {
4480         struct drm_i915_private *dev_priv = to_i915(dev);
4481         struct intel_engine_cs *engine;
4482         int ret;
4483
4484         /* Double layer security blanket, see i915_gem_init() */
4485         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4486
4487         if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
4488                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4489
4490         if (IS_HASWELL(dev))
4491                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4492                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4493
4494         if (HAS_PCH_NOP(dev)) {
4495                 if (IS_IVYBRIDGE(dev)) {
4496                         u32 temp = I915_READ(GEN7_MSG_CTL);
4497                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4498                         I915_WRITE(GEN7_MSG_CTL, temp);
4499                 } else if (INTEL_INFO(dev)->gen >= 7) {
4500                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4501                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4502                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4503                 }
4504         }
4505
4506         i915_gem_init_swizzling(dev);
4507
4508         /*
4509          * At least 830 can leave some of the unused rings
4510          * "active" (ie. head != tail) after resume which
4511          * will prevent c3 entry. Makes sure all unused rings
4512          * are totally idle.
4513          */
4514         init_unused_rings(dev);
4515
4516         BUG_ON(!dev_priv->kernel_context);
4517
4518         ret = i915_ppgtt_init_hw(dev);
4519         if (ret) {
4520                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4521                 goto out;
4522         }
4523
4524         /* Need to do basic initialisation of all rings first: */
4525         for_each_engine(engine, dev_priv) {
4526                 ret = engine->init_hw(engine);
4527                 if (ret)
4528                         goto out;
4529         }
4530
4531         intel_mocs_init_l3cc_table(dev);
4532
4533         /* We can't enable contexts until all firmware is loaded */
4534         ret = intel_guc_setup(dev);
4535         if (ret)
4536                 goto out;
4537
4538 out:
4539         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4540         return ret;
4541 }
4542
4543 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
4544 {
4545         if (INTEL_INFO(dev_priv)->gen < 6)
4546                 return false;
4547
4548         /* TODO: make semaphores and Execlists play nicely together */
4549         if (i915.enable_execlists)
4550                 return false;
4551
4552         if (value >= 0)
4553                 return value;
4554
4555 #ifdef CONFIG_INTEL_IOMMU
4556         /* Enable semaphores on SNB when IO remapping is off */
4557         if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
4558                 return false;
4559 #endif
4560
4561         return true;
4562 }
4563
4564 int i915_gem_init(struct drm_device *dev)
4565 {
4566         struct drm_i915_private *dev_priv = to_i915(dev);
4567         int ret;
4568
4569         mutex_lock(&dev->struct_mutex);
4570
4571         if (!i915.enable_execlists) {
4572                 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
4573                 dev_priv->gt.cleanup_engine = intel_engine_cleanup;
4574                 dev_priv->gt.stop_engine = intel_engine_stop;
4575         } else {
4576                 dev_priv->gt.execbuf_submit = intel_execlists_submission;
4577                 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
4578                 dev_priv->gt.stop_engine = intel_logical_ring_stop;
4579         }
4580
4581         /* This is just a security blanket to placate dragons.
4582          * On some systems, we very sporadically observe that the first TLBs
4583          * used by the CS may be stale, despite us poking the TLB reset. If
4584          * we hold the forcewake during initialisation these problems
4585          * just magically go away.
4586          */
4587         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4588
4589         i915_gem_init_userptr(dev_priv);
4590         i915_gem_init_ggtt(dev);
4591
4592         ret = i915_gem_context_init(dev);
4593         if (ret)
4594                 goto out_unlock;
4595
4596         ret = intel_engines_init(dev);
4597         if (ret)
4598                 goto out_unlock;
4599
4600         ret = i915_gem_init_hw(dev);
4601         if (ret == -EIO) {
4602                 /* Allow engine initialisation to fail by marking the GPU as
4603                  * wedged. But we only want to do this where the GPU is angry,
4604                  * for all other failure, such as an allocation failure, bail.
4605                  */
4606                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4607                 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4608                 ret = 0;
4609         }
4610
4611 out_unlock:
4612         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4613         mutex_unlock(&dev->struct_mutex);
4614
4615         return ret;
4616 }
4617
4618 void
4619 i915_gem_cleanup_engines(struct drm_device *dev)
4620 {
4621         struct drm_i915_private *dev_priv = to_i915(dev);
4622         struct intel_engine_cs *engine;
4623
4624         for_each_engine(engine, dev_priv)
4625                 dev_priv->gt.cleanup_engine(engine);
4626 }
4627
4628 static void
4629 init_engine_lists(struct intel_engine_cs *engine)
4630 {
4631         INIT_LIST_HEAD(&engine->active_list);
4632         INIT_LIST_HEAD(&engine->request_list);
4633 }
4634
4635 void
4636 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4637 {
4638         struct drm_device *dev = &dev_priv->drm;
4639
4640         if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
4641             !IS_CHERRYVIEW(dev_priv))
4642                 dev_priv->num_fence_regs = 32;
4643         else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
4644                  IS_I945GM(dev_priv) || IS_G33(dev_priv))
4645                 dev_priv->num_fence_regs = 16;
4646         else
4647                 dev_priv->num_fence_regs = 8;
4648
4649         if (intel_vgpu_active(dev_priv))
4650                 dev_priv->num_fence_regs =
4651                                 I915_READ(vgtif_reg(avail_rs.fence_num));
4652
4653         /* Initialize fence registers to zero */
4654         i915_gem_restore_fences(dev);
4655
4656         i915_gem_detect_bit_6_swizzle(dev);
4657 }
4658
4659 void
4660 i915_gem_load_init(struct drm_device *dev)
4661 {
4662         struct drm_i915_private *dev_priv = to_i915(dev);
4663         int i;
4664
4665         dev_priv->objects =
4666                 kmem_cache_create("i915_gem_object",
4667                                   sizeof(struct drm_i915_gem_object), 0,
4668                                   SLAB_HWCACHE_ALIGN,
4669                                   NULL);
4670         dev_priv->vmas =
4671                 kmem_cache_create("i915_gem_vma",
4672                                   sizeof(struct i915_vma), 0,
4673                                   SLAB_HWCACHE_ALIGN,
4674                                   NULL);
4675         dev_priv->requests =
4676                 kmem_cache_create("i915_gem_request",
4677                                   sizeof(struct drm_i915_gem_request), 0,
4678                                   SLAB_HWCACHE_ALIGN,
4679                                   NULL);
4680
4681         INIT_LIST_HEAD(&dev_priv->vm_list);
4682         INIT_LIST_HEAD(&dev_priv->context_list);
4683         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4684         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4685         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4686         for (i = 0; i < I915_NUM_ENGINES; i++)
4687                 init_engine_lists(&dev_priv->engine[i]);
4688         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4689                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4690         INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
4691                           i915_gem_retire_work_handler);
4692         INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
4693                           i915_gem_idle_work_handler);
4694         init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
4695         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4696
4697         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4698
4699         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4700
4701         init_waitqueue_head(&dev_priv->pending_flip_queue);
4702
4703         dev_priv->mm.interruptible = true;
4704
4705         mutex_init(&dev_priv->fb_tracking.lock);
4706 }
4707
4708 void i915_gem_load_cleanup(struct drm_device *dev)
4709 {
4710         struct drm_i915_private *dev_priv = to_i915(dev);
4711
4712         kmem_cache_destroy(dev_priv->requests);
4713         kmem_cache_destroy(dev_priv->vmas);
4714         kmem_cache_destroy(dev_priv->objects);
4715 }
4716
4717 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
4718 {
4719         struct drm_i915_gem_object *obj;
4720
4721         /* Called just before we write the hibernation image.
4722          *
4723          * We need to update the domain tracking to reflect that the CPU
4724          * will be accessing all the pages to create and restore from the
4725          * hibernation, and so upon restoration those pages will be in the
4726          * CPU domain.
4727          *
4728          * To make sure the hibernation image contains the latest state,
4729          * we update that state just before writing out the image.
4730          */
4731
4732         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
4733                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4734                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4735         }
4736
4737         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
4738                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4739                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4740         }
4741
4742         return 0;
4743 }
4744
4745 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4746 {
4747         struct drm_i915_file_private *file_priv = file->driver_priv;
4748         struct drm_i915_gem_request *request;
4749
4750         /* Clean up our request list when the client is going away, so that
4751          * later retire_requests won't dereference our soon-to-be-gone
4752          * file_priv.
4753          */
4754         spin_lock(&file_priv->mm.lock);
4755         list_for_each_entry(request, &file_priv->mm.request_list, client_list)
4756                 request->file_priv = NULL;
4757         spin_unlock(&file_priv->mm.lock);
4758
4759         if (!list_empty(&file_priv->rps.link)) {
4760                 spin_lock(&to_i915(dev)->rps.client_lock);
4761                 list_del(&file_priv->rps.link);
4762                 spin_unlock(&to_i915(dev)->rps.client_lock);
4763         }
4764 }
4765
4766 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4767 {
4768         struct drm_i915_file_private *file_priv;
4769         int ret;
4770
4771         DRM_DEBUG_DRIVER("\n");
4772
4773         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4774         if (!file_priv)
4775                 return -ENOMEM;
4776
4777         file->driver_priv = file_priv;
4778         file_priv->dev_priv = to_i915(dev);
4779         file_priv->file = file;
4780         INIT_LIST_HEAD(&file_priv->rps.link);
4781
4782         spin_lock_init(&file_priv->mm.lock);
4783         INIT_LIST_HEAD(&file_priv->mm.request_list);
4784
4785         file_priv->bsd_engine = -1;
4786
4787         ret = i915_gem_context_open(dev, file);
4788         if (ret)
4789                 kfree(file_priv);
4790
4791         return ret;
4792 }
4793
4794 /**
4795  * i915_gem_track_fb - update frontbuffer tracking
4796  * @old: current GEM buffer for the frontbuffer slots
4797  * @new: new GEM buffer for the frontbuffer slots
4798  * @frontbuffer_bits: bitmask of frontbuffer slots
4799  *
4800  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
4801  * from @old and setting them in @new. Both @old and @new can be NULL.
4802  */
4803 void i915_gem_track_fb(struct drm_i915_gem_object *old,
4804                        struct drm_i915_gem_object *new,
4805                        unsigned frontbuffer_bits)
4806 {
4807         if (old) {
4808                 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
4809                 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
4810                 old->frontbuffer_bits &= ~frontbuffer_bits;
4811         }
4812
4813         if (new) {
4814                 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
4815                 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
4816                 new->frontbuffer_bits |= frontbuffer_bits;
4817         }
4818 }
4819
4820 /* All the new VM stuff */
4821 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
4822                         struct i915_address_space *vm)
4823 {
4824         struct drm_i915_private *dev_priv = to_i915(o->base.dev);
4825         struct i915_vma *vma;
4826
4827         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4828
4829         list_for_each_entry(vma, &o->vma_list, obj_link) {
4830                 if (vma->is_ggtt &&
4831                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4832                         continue;
4833                 if (vma->vm == vm)
4834                         return vma->node.start;
4835         }
4836
4837         WARN(1, "%s vma for this object not found.\n",
4838              i915_is_ggtt(vm) ? "global" : "ppgtt");
4839         return -1;
4840 }
4841
4842 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
4843                                   const struct i915_ggtt_view *view)
4844 {
4845         struct i915_vma *vma;
4846
4847         list_for_each_entry(vma, &o->vma_list, obj_link)
4848                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
4849                         return vma->node.start;
4850
4851         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
4852         return -1;
4853 }
4854
4855 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
4856                         struct i915_address_space *vm)
4857 {
4858         struct i915_vma *vma;
4859
4860         list_for_each_entry(vma, &o->vma_list, obj_link) {
4861                 if (vma->is_ggtt &&
4862                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4863                         continue;
4864                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
4865                         return true;
4866         }
4867
4868         return false;
4869 }
4870
4871 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
4872                                   const struct i915_ggtt_view *view)
4873 {
4874         struct i915_vma *vma;
4875
4876         list_for_each_entry(vma, &o->vma_list, obj_link)
4877                 if (vma->is_ggtt &&
4878                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
4879                     drm_mm_node_allocated(&vma->node))
4880                         return true;
4881
4882         return false;
4883 }
4884
4885 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
4886 {
4887         struct i915_vma *vma;
4888
4889         list_for_each_entry(vma, &o->vma_list, obj_link)
4890                 if (drm_mm_node_allocated(&vma->node))
4891                         return true;
4892
4893         return false;
4894 }
4895
4896 unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o)
4897 {
4898         struct i915_vma *vma;
4899
4900         GEM_BUG_ON(list_empty(&o->vma_list));
4901
4902         list_for_each_entry(vma, &o->vma_list, obj_link) {
4903                 if (vma->is_ggtt &&
4904                     vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
4905                         return vma->node.size;
4906         }
4907
4908         return 0;
4909 }
4910
4911 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
4912 {
4913         struct i915_vma *vma;
4914         list_for_each_entry(vma, &obj->vma_list, obj_link)
4915                 if (vma->pin_count > 0)
4916                         return true;
4917
4918         return false;
4919 }
4920
4921 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
4922 struct page *
4923 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
4924 {
4925         struct page *page;
4926
4927         /* Only default objects have per-page dirty tracking */
4928         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
4929                 return NULL;
4930
4931         page = i915_gem_object_get_page(obj, n);
4932         set_page_dirty(page);
4933         return page;
4934 }
4935
4936 /* Allocate a new GEM object and fill it with the supplied data */
4937 struct drm_i915_gem_object *
4938 i915_gem_object_create_from_data(struct drm_device *dev,
4939                                  const void *data, size_t size)
4940 {
4941         struct drm_i915_gem_object *obj;
4942         struct sg_table *sg;
4943         size_t bytes;
4944         int ret;
4945
4946         obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
4947         if (IS_ERR(obj))
4948                 return obj;
4949
4950         ret = i915_gem_object_set_to_cpu_domain(obj, true);
4951         if (ret)
4952                 goto fail;
4953
4954         ret = i915_gem_object_get_pages(obj);
4955         if (ret)
4956                 goto fail;
4957
4958         i915_gem_object_pin_pages(obj);
4959         sg = obj->pages;
4960         bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
4961         obj->dirty = 1;         /* Backing store is now out of date */
4962         i915_gem_object_unpin_pages(obj);
4963
4964         if (WARN_ON(bytes != size)) {
4965                 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
4966                 ret = -EFAULT;
4967                 goto fail;
4968         }
4969
4970         return obj;
4971
4972 fail:
4973         i915_gem_object_put(obj);
4974         return ERR_PTR(ret);
4975 }