drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_gem_dmabuf.h"
  33 #include "i915_vgpu.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36 #include "intel_mocs.h"
  37 #include <linux/reservation.h>
  38 #include <linux/shmem_fs.h>
  39 #include <linux/slab.h>
  40 #include <linux/swap.h>
  41 #include <linux/pci.h>
  42 #include <linux/dma-buf.h>
  43
  44 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  45 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  46 static void
  47 i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
  48 static void
  49 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int engine);
  50
  51 static bool cpu_cache_is_coherent(struct drm_device *dev,
  52                                   enum i915_cache_level level)
  53 {
  54         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  55 }
  56
  57 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  58 {
  59         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  60                 return false;
  61
  62         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  63                 return true;
  64
  65         return obj->pin_display;
  66 }
  67
  68 static int
  69 insert_mappable_node(struct drm_i915_private *i915,
  70                      struct drm_mm_node *node, u32 size)
  71 {
  72         memset(node, 0, sizeof(*node));
  73         return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
  74                                                    size, 0, 0, 0,
  75                                                    i915->ggtt.mappable_end,
  76                                                    DRM_MM_SEARCH_DEFAULT,
  77                                                    DRM_MM_CREATE_DEFAULT);
  78 }
  79
  80 static void
  81 remove_mappable_node(struct drm_mm_node *node)
  82 {
  83         drm_mm_remove_node(node);
  84 }
  85
  86 /* some bookkeeping */
  87 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  88                                   size_t size)
  89 {
  90         spin_lock(&dev_priv->mm.object_stat_lock);
  91         dev_priv->mm.object_count++;
  92         dev_priv->mm.object_memory += size;
  93         spin_unlock(&dev_priv->mm.object_stat_lock);
  94 }
  95
  96 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  97                                      size_t size)
  98 {
  99         spin_lock(&dev_priv->mm.object_stat_lock);
 100         dev_priv->mm.object_count--;
 101         dev_priv->mm.object_memory -= size;
 102         spin_unlock(&dev_priv->mm.object_stat_lock);
 103 }
 104
 105 static int
 106 i915_gem_wait_for_error(struct i915_gpu_error *error)
 107 {
 108         int ret;
 109
 110         if (!i915_reset_in_progress(error))
 111                 return 0;
 112
 113         /*
 114          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 115          * userspace. If it takes that long something really bad is going on and
 116          * we should simply try to bail out and fail as gracefully as possible.
 117          */
 118         ret = wait_event_interruptible_timeout(error->reset_queue,
 119                                                !i915_reset_in_progress(error),
 120                                                10*HZ);
 121         if (ret == 0) {
 122                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 123                 return -EIO;
 124         } else if (ret < 0) {
 125                 return ret;
 126         } else {
 127                 return 0;
 128         }
 129 }
 130
 131 int i915_mutex_lock_interruptible(struct drm_device *dev)
 132 {
 133         struct drm_i915_private *dev_priv = to_i915(dev);
 134         int ret;
 135
 136         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 137         if (ret)
 138                 return ret;
 139
 140         ret = mutex_lock_interruptible(&dev->struct_mutex);
 141         if (ret)
 142                 return ret;
 143
 144         WARN_ON(i915_verify_lists(dev));
 145         return 0;
 146 }
 147
 148 int
 149 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 150                             struct drm_file *file)
 151 {
 152         struct drm_i915_private *dev_priv = to_i915(dev);
 153         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 154         struct drm_i915_gem_get_aperture *args = data;
 155         struct i915_vma *vma;
 156         size_t pinned;
 157
 158         pinned = 0;
 159         mutex_lock(&dev->struct_mutex);
 160         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
 161                 if (vma->pin_count)
 162                         pinned += vma->node.size;
 163         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
 164                 if (vma->pin_count)
 165                         pinned += vma->node.size;
 166         mutex_unlock(&dev->struct_mutex);
 167
 168         args->aper_size = ggtt->base.total;
 169         args->aper_available_size = args->aper_size - pinned;
 170
 171         return 0;
 172 }
 173
 174 static int
 175 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 176 {
 177         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 178         char *vaddr = obj->phys_handle->vaddr;
 179         struct sg_table *st;
 180         struct scatterlist *sg;
 181         int i;
 182
 183         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 184                 return -EINVAL;
 185
 186         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 187                 struct page *page;
 188                 char *src;
 189
 190                 page = shmem_read_mapping_page(mapping, i);
 191                 if (IS_ERR(page))
 192                         return PTR_ERR(page);
 193
 194                 src = kmap_atomic(page);
 195                 memcpy(vaddr, src, PAGE_SIZE);
 196                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 197                 kunmap_atomic(src);
 198
 199                 put_page(page);
 200                 vaddr += PAGE_SIZE;
 201         }
 202
 203         i915_gem_chipset_flush(to_i915(obj->base.dev));
 204
 205         st = kmalloc(sizeof(*st), GFP_KERNEL);
 206         if (st == NULL)
 207                 return -ENOMEM;
 208
 209         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 210                 kfree(st);
 211                 return -ENOMEM;
 212         }
 213
 214         sg = st->sgl;
 215         sg->offset = 0;
 216         sg->length = obj->base.size;
 217
 218         sg_dma_address(sg) = obj->phys_handle->busaddr;
 219         sg_dma_len(sg) = obj->base.size;
 220
 221         obj->pages = st;
 222         return 0;
 223 }
 224
 225 static void
 226 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 227 {
 228         int ret;
 229
 230         BUG_ON(obj->madv == __I915_MADV_PURGED);
 231
 232         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 233         if (WARN_ON(ret)) {
 234                 /* In the event of a disaster, abandon all caches and
 235                  * hope for the best.
 236                  */
 237                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 238         }
 239
 240         if (obj->madv == I915_MADV_DONTNEED)
 241                 obj->dirty = 0;
 242
 243         if (obj->dirty) {
 244                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 245                 char *vaddr = obj->phys_handle->vaddr;
 246                 int i;
 247
 248                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 249                         struct page *page;
 250                         char *dst;
 251
 252                         page = shmem_read_mapping_page(mapping, i);
 253                         if (IS_ERR(page))
 254                                 continue;
 255
 256                         dst = kmap_atomic(page);
 257                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 258                         memcpy(dst, vaddr, PAGE_SIZE);
 259                         kunmap_atomic(dst);
 260
 261                         set_page_dirty(page);
 262                         if (obj->madv == I915_MADV_WILLNEED)
 263                                 mark_page_accessed(page);
 264                         put_page(page);
 265                         vaddr += PAGE_SIZE;
 266                 }
 267                 obj->dirty = 0;
 268         }
 269
 270         sg_free_table(obj->pages);
 271         kfree(obj->pages);
 272 }
 273
 274 static void
 275 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 276 {
 277         drm_pci_free(obj->base.dev, obj->phys_handle);
 278 }
 279
 280 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 281         .get_pages = i915_gem_object_get_pages_phys,
 282         .put_pages = i915_gem_object_put_pages_phys,
 283         .release = i915_gem_object_release_phys,
 284 };
 285
 286 static int
 287 drop_pages(struct drm_i915_gem_object *obj)
 288 {
 289         struct i915_vma *vma, *next;
 290         int ret;
 291
 292         i915_gem_object_get(obj);
 293         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
 294                 if (i915_vma_unbind(vma))
 295                         break;
 296
 297         ret = i915_gem_object_put_pages(obj);
 298         i915_gem_object_put(obj);
 299
 300         return ret;
 301 }
 302
 303 int
 304 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 305                             int align)
 306 {
 307         drm_dma_handle_t *phys;
 308         int ret;
 309
 310         if (obj->phys_handle) {
 311                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 312                         return -EBUSY;
 313
 314                 return 0;
 315         }
 316
 317         if (obj->madv != I915_MADV_WILLNEED)
 318                 return -EFAULT;
 319
 320         if (obj->base.filp == NULL)
 321                 return -EINVAL;
 322
 323         ret = drop_pages(obj);
 324         if (ret)
 325                 return ret;
 326
 327         /* create a new object */
 328         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 329         if (!phys)
 330                 return -ENOMEM;
 331
 332         obj->phys_handle = phys;
 333         obj->ops = &i915_gem_phys_ops;
 334
 335         return i915_gem_object_get_pages(obj);
 336 }
 337
 338 static int
 339 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 340                      struct drm_i915_gem_pwrite *args,
 341                      struct drm_file *file_priv)
 342 {
 343         struct drm_device *dev = obj->base.dev;
 344         void *vaddr = obj->phys_handle->vaddr + args->offset;
 345         char __user *user_data = u64_to_user_ptr(args->data_ptr);
 346         int ret = 0;
 347
 348         /* We manually control the domain here and pretend that it
 349          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 350          */
 351         ret = i915_gem_object_wait_rendering(obj, false);
 352         if (ret)
 353                 return ret;
 354
 355         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 356         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 357                 unsigned long unwritten;
 358
 359                 /* The physical object once assigned is fixed for the lifetime
 360                  * of the obj, so we can safely drop the lock and continue
 361                  * to access vaddr.
 362                  */
 363                 mutex_unlock(&dev->struct_mutex);
 364                 unwritten = copy_from_user(vaddr, user_data, args->size);
 365                 mutex_lock(&dev->struct_mutex);
 366                 if (unwritten) {
 367                         ret = -EFAULT;
 368                         goto out;
 369                 }
 370         }
 371
 372         drm_clflush_virt_range(vaddr, args->size);
 373         i915_gem_chipset_flush(to_i915(dev));
 374
 375 out:
 376         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 377         return ret;
 378 }
 379
 380 void *i915_gem_object_alloc(struct drm_device *dev)
 381 {
 382         struct drm_i915_private *dev_priv = to_i915(dev);
 383         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 384 }
 385
 386 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 387 {
 388         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 389         kmem_cache_free(dev_priv->objects, obj);
 390 }
 391
 392 static int
 393 i915_gem_create(struct drm_file *file,
 394                 struct drm_device *dev,
 395                 uint64_t size,
 396                 uint32_t *handle_p)
 397 {
 398         struct drm_i915_gem_object *obj;
 399         int ret;
 400         u32 handle;
 401
 402         size = roundup(size, PAGE_SIZE);
 403         if (size == 0)
 404                 return -EINVAL;
 405
 406         /* Allocate the new object */
 407         obj = i915_gem_object_create(dev, size);
 408         if (IS_ERR(obj))
 409                 return PTR_ERR(obj);
 410
 411         ret = drm_gem_handle_create(file, &obj->base, &handle);
 412         /* drop reference from allocate - handle holds it now */
 413         i915_gem_object_put_unlocked(obj);
 414         if (ret)
 415                 return ret;
 416
 417         *handle_p = handle;
 418         return 0;
 419 }
 420
 421 int
 422 i915_gem_dumb_create(struct drm_file *file,
 423                      struct drm_device *dev,
 424                      struct drm_mode_create_dumb *args)
 425 {
 426         /* have to work out size/pitch and return them */
 427         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 428         args->size = args->pitch * args->height;
 429         return i915_gem_create(file, dev,
 430                                args->size, &args->handle);
 431 }
 432
 433 /**
 434  * Creates a new mm object and returns a handle to it.
 435  * @dev: drm device pointer
 436  * @data: ioctl data blob
 437  * @file: drm file pointer
 438  */
 439 int
 440 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 441                       struct drm_file *file)
 442 {
 443         struct drm_i915_gem_create *args = data;
 444
 445         return i915_gem_create(file, dev,
 446                                args->size, &args->handle);
 447 }
 448
 449 static inline int
 450 __copy_to_user_swizzled(char __user *cpu_vaddr,
 451                         const char *gpu_vaddr, int gpu_offset,
 452                         int length)
 453 {
 454         int ret, cpu_offset = 0;
 455
 456         while (length > 0) {
 457                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 458                 int this_length = min(cacheline_end - gpu_offset, length);
 459                 int swizzled_gpu_offset = gpu_offset ^ 64;
 460
 461                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 462                                      gpu_vaddr + swizzled_gpu_offset,
 463                                      this_length);
 464                 if (ret)
 465                         return ret + length;
 466
 467                 cpu_offset += this_length;
 468                 gpu_offset += this_length;
 469                 length -= this_length;
 470         }
 471
 472         return 0;
 473 }
 474
 475 static inline int
 476 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 477                           const char __user *cpu_vaddr,
 478                           int length)
 479 {
 480         int ret, cpu_offset = 0;
 481
 482         while (length > 0) {
 483                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 484                 int this_length = min(cacheline_end - gpu_offset, length);
 485                 int swizzled_gpu_offset = gpu_offset ^ 64;
 486
 487                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 488                                        cpu_vaddr + cpu_offset,
 489                                        this_length);
 490                 if (ret)
 491                         return ret + length;
 492
 493                 cpu_offset += this_length;
 494                 gpu_offset += this_length;
 495                 length -= this_length;
 496         }
 497
 498         return 0;
 499 }
 500
 501 /*
 502  * Pins the specified object's pages and synchronizes the object with
 503  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 504  * flush the object from the CPU cache.
 505  */
 506 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 507                                     int *needs_clflush)
 508 {
 509         int ret;
 510
 511         *needs_clflush = 0;
 512
 513         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
 514                 return -EINVAL;
 515
 516         ret = i915_gem_object_wait_rendering(obj, true);
 517         if (ret)
 518                 return ret;
 519
 520         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 521                 /* If we're not in the cpu read domain, set ourself into the gtt
 522                  * read domain and manually flush cachelines (if required). This
 523                  * optimizes for the case when the gpu will dirty the data
 524                  * anyway again before the next pread happens. */
 525                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 526                                                         obj->cache_level);
 527         }
 528
 529         ret = i915_gem_object_get_pages(obj);
 530         if (ret)
 531                 return ret;
 532
 533         i915_gem_object_pin_pages(obj);
 534
 535         return ret;
 536 }
 537
 538 /* Per-page copy function for the shmem pread fastpath.
 539  * Flushes invalid cachelines before reading the target if
 540  * needs_clflush is set. */
 541 static int
 542 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 543                  char __user *user_data,
 544                  bool page_do_bit17_swizzling, bool needs_clflush)
 545 {
 546         char *vaddr;
 547         int ret;
 548
 549         if (unlikely(page_do_bit17_swizzling))
 550                 return -EINVAL;
 551
 552         vaddr = kmap_atomic(page);
 553         if (needs_clflush)
 554                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 555                                        page_length);
 556         ret = __copy_to_user_inatomic(user_data,
 557                                       vaddr + shmem_page_offset,
 558                                       page_length);
 559         kunmap_atomic(vaddr);
 560
 561         return ret ? -EFAULT : 0;
 562 }
 563
 564 static void
 565 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 566                              bool swizzled)
 567 {
 568         if (unlikely(swizzled)) {
 569                 unsigned long start = (unsigned long) addr;
 570                 unsigned long end = (unsigned long) addr + length;
 571
 572                 /* For swizzling simply ensure that we always flush both
 573                  * channels. Lame, but simple and it works. Swizzled
 574                  * pwrite/pread is far from a hotpath - current userspace
 575                  * doesn't use it at all. */
 576                 start = round_down(start, 128);
 577                 end = round_up(end, 128);
 578
 579                 drm_clflush_virt_range((void *)start, end - start);
 580         } else {
 581                 drm_clflush_virt_range(addr, length);
 582         }
 583
 584 }
 585
 586 /* Only difference to the fast-path function is that this can handle bit17
 587  * and uses non-atomic copy and kmap functions. */
 588 static int
 589 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 590                  char __user *user_data,
 591                  bool page_do_bit17_swizzling, bool needs_clflush)
 592 {
 593         char *vaddr;
 594         int ret;
 595
 596         vaddr = kmap(page);
 597         if (needs_clflush)
 598                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 599                                              page_length,
 600                                              page_do_bit17_swizzling);
 601
 602         if (page_do_bit17_swizzling)
 603                 ret = __copy_to_user_swizzled(user_data,
 604                                               vaddr, shmem_page_offset,
 605                                               page_length);
 606         else
 607                 ret = __copy_to_user(user_data,
 608                                      vaddr + shmem_page_offset,
 609                                      page_length);
 610         kunmap(page);
 611
 612         return ret ? - EFAULT : 0;
 613 }
 614
 615 static inline unsigned long
 616 slow_user_access(struct io_mapping *mapping,
 617                  uint64_t page_base, int page_offset,
 618                  char __user *user_data,
 619                  unsigned long length, bool pwrite)
 620 {
 621         void __iomem *ioaddr;
 622         void *vaddr;
 623         uint64_t unwritten;
 624
 625         ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
 626         /* We can use the cpu mem copy function because this is X86. */
 627         vaddr = (void __force *)ioaddr + page_offset;
 628         if (pwrite)
 629                 unwritten = __copy_from_user(vaddr, user_data, length);
 630         else
 631                 unwritten = __copy_to_user(user_data, vaddr, length);
 632
 633         io_mapping_unmap(ioaddr);
 634         return unwritten;
 635 }
 636
 637 static int
 638 i915_gem_gtt_pread(struct drm_device *dev,
 639                    struct drm_i915_gem_object *obj, uint64_t size,
 640                    uint64_t data_offset, uint64_t data_ptr)
 641 {
 642         struct drm_i915_private *dev_priv = to_i915(dev);
 643         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 644         struct drm_mm_node node;
 645         char __user *user_data;
 646         uint64_t remain;
 647         uint64_t offset;
 648         int ret;
 649
 650         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
 651         if (ret) {
 652                 ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
 653                 if (ret)
 654                         goto out;
 655
 656                 ret = i915_gem_object_get_pages(obj);
 657                 if (ret) {
 658                         remove_mappable_node(&node);
 659                         goto out;
 660                 }
 661
 662                 i915_gem_object_pin_pages(obj);
 663         } else {
 664                 node.start = i915_gem_obj_ggtt_offset(obj);
 665                 node.allocated = false;
 666                 ret = i915_gem_object_put_fence(obj);
 667                 if (ret)
 668                         goto out_unpin;
 669         }
 670
 671         ret = i915_gem_object_set_to_gtt_domain(obj, false);
 672         if (ret)
 673                 goto out_unpin;
 674
 675         user_data = u64_to_user_ptr(data_ptr);
 676         remain = size;
 677         offset = data_offset;
 678
 679         mutex_unlock(&dev->struct_mutex);
 680         if (likely(!i915.prefault_disable)) {
 681                 ret = fault_in_multipages_writeable(user_data, remain);
 682                 if (ret) {
 683                         mutex_lock(&dev->struct_mutex);
 684                         goto out_unpin;
 685                 }
 686         }
 687
 688         while (remain > 0) {
 689                 /* Operation in this page
 690                  *
 691                  * page_base = page offset within aperture
 692                  * page_offset = offset within page
 693                  * page_length = bytes to copy for this page
 694                  */
 695                 u32 page_base = node.start;
 696                 unsigned page_offset = offset_in_page(offset);
 697                 unsigned page_length = PAGE_SIZE - page_offset;
 698                 page_length = remain < page_length ? remain : page_length;
 699                 if (node.allocated) {
 700                         wmb();
 701                         ggtt->base.insert_page(&ggtt->base,
 702                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 703                                                node.start,
 704                                                I915_CACHE_NONE, 0);
 705                         wmb();
 706                 } else {
 707                         page_base += offset & PAGE_MASK;
 708                 }
 709                 /* This is a slow read/write as it tries to read from
 710                  * and write to user memory which may result into page
 711                  * faults, and so we cannot perform this under struct_mutex.
 712                  */
 713                 if (slow_user_access(ggtt->mappable, page_base,
 714                                      page_offset, user_data,
 715                                      page_length, false)) {
 716                         ret = -EFAULT;
 717                         break;
 718                 }
 719
 720                 remain -= page_length;
 721                 user_data += page_length;
 722                 offset += page_length;
 723         }
 724
 725         mutex_lock(&dev->struct_mutex);
 726         if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
 727                 /* The user has modified the object whilst we tried
 728                  * reading from it, and we now have no idea what domain
 729                  * the pages should be in. As we have just been touching
 730                  * them directly, flush everything back to the GTT
 731                  * domain.
 732                  */
 733                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 734         }
 735
 736 out_unpin:
 737         if (node.allocated) {
 738                 wmb();
 739                 ggtt->base.clear_range(&ggtt->base,
 740                                        node.start, node.size,
 741                                        true);
 742                 i915_gem_object_unpin_pages(obj);
 743                 remove_mappable_node(&node);
 744         } else {
 745                 i915_gem_object_ggtt_unpin(obj);
 746         }
 747 out:
 748         return ret;
 749 }
 750
 751 static int
 752 i915_gem_shmem_pread(struct drm_device *dev,
 753                      struct drm_i915_gem_object *obj,
 754                      struct drm_i915_gem_pread *args,
 755                      struct drm_file *file)
 756 {
 757         char __user *user_data;
 758         ssize_t remain;
 759         loff_t offset;
 760         int shmem_page_offset, page_length, ret = 0;
 761         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 762         int prefaulted = 0;
 763         int needs_clflush = 0;
 764         struct sg_page_iter sg_iter;
 765
 766         if (!i915_gem_object_has_struct_page(obj))
 767                 return -ENODEV;
 768
 769         user_data = u64_to_user_ptr(args->data_ptr);
 770         remain = args->size;
 771
 772         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 773
 774         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 775         if (ret)
 776                 return ret;
 777
 778         offset = args->offset;
 779
 780         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 781                          offset >> PAGE_SHIFT) {
 782                 struct page *page = sg_page_iter_page(&sg_iter);
 783
 784                 if (remain <= 0)
 785                         break;
 786
 787                 /* Operation in this page
 788                  *
 789                  * shmem_page_offset = offset within page in shmem file
 790                  * page_length = bytes to copy for this page
 791                  */
 792                 shmem_page_offset = offset_in_page(offset);
 793                 page_length = remain;
 794                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 795                         page_length = PAGE_SIZE - shmem_page_offset;
 796
 797                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 798                         (page_to_phys(page) & (1 << 17)) != 0;
 799
 800                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 801                                        user_data, page_do_bit17_swizzling,
 802                                        needs_clflush);
 803                 if (ret == 0)
 804                         goto next_page;
 805
 806                 mutex_unlock(&dev->struct_mutex);
 807
 808                 if (likely(!i915.prefault_disable) && !prefaulted) {
 809                         ret = fault_in_multipages_writeable(user_data, remain);
 810                         /* Userspace is tricking us, but we've already clobbered
 811                          * its pages with the prefault and promised to write the
 812                          * data up to the first fault. Hence ignore any errors
 813                          * and just continue. */
 814                         (void)ret;
 815                         prefaulted = 1;
 816                 }
 817
 818                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 819                                        user_data, page_do_bit17_swizzling,
 820                                        needs_clflush);
 821
 822                 mutex_lock(&dev->struct_mutex);
 823
 824                 if (ret)
 825                         goto out;
 826
 827 next_page:
 828                 remain -= page_length;
 829                 user_data += page_length;
 830                 offset += page_length;
 831         }
 832
 833 out:
 834         i915_gem_object_unpin_pages(obj);
 835
 836         return ret;
 837 }
 838
 839 /**
 840  * Reads data from the object referenced by handle.
 841  * @dev: drm device pointer
 842  * @data: ioctl data blob
 843  * @file: drm file pointer
 844  *
 845  * On error, the contents of *data are undefined.
 846  */
 847 int
 848 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 849                      struct drm_file *file)
 850 {
 851         struct drm_i915_gem_pread *args = data;
 852         struct drm_i915_gem_object *obj;
 853         int ret = 0;
 854
 855         if (args->size == 0)
 856                 return 0;
 857
 858         if (!access_ok(VERIFY_WRITE,
 859                        u64_to_user_ptr(args->data_ptr),
 860                        args->size))
 861                 return -EFAULT;
 862
 863         ret = i915_mutex_lock_interruptible(dev);
 864         if (ret)
 865                 return ret;
 866
 867         obj = i915_gem_object_lookup(file, args->handle);
 868         if (!obj) {
 869                 ret = -ENOENT;
 870                 goto unlock;
 871         }
 872
 873         /* Bounds check source.  */
 874         if (args->offset > obj->base.size ||
 875             args->size > obj->base.size - args->offset) {
 876                 ret = -EINVAL;
 877                 goto out;
 878         }
 879
 880         trace_i915_gem_object_pread(obj, args->offset, args->size);
 881
 882         ret = i915_gem_shmem_pread(dev, obj, args, file);
 883
 884         /* pread for non shmem backed objects */
 885         if (ret == -EFAULT || ret == -ENODEV)
 886                 ret = i915_gem_gtt_pread(dev, obj, args->size,
 887                                         args->offset, args->data_ptr);
 888
 889 out:
 890         i915_gem_object_put(obj);
 891 unlock:
 892         mutex_unlock(&dev->struct_mutex);
 893         return ret;
 894 }
 895
 896 /* This is the fast write path which cannot handle
 897  * page faults in the source data
 898  */
 899
 900 static inline int
 901 fast_user_write(struct io_mapping *mapping,
 902                 loff_t page_base, int page_offset,
 903                 char __user *user_data,
 904                 int length)
 905 {
 906         void __iomem *vaddr_atomic;
 907         void *vaddr;
 908         unsigned long unwritten;
 909
 910         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 911         /* We can use the cpu mem copy function because this is X86. */
 912         vaddr = (void __force*)vaddr_atomic + page_offset;
 913         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 914                                                       user_data, length);
 915         io_mapping_unmap_atomic(vaddr_atomic);
 916         return unwritten;
 917 }
 918
 919 /**
 920  * This is the fast pwrite path, where we copy the data directly from the
 921  * user into the GTT, uncached.
 922  * @i915: i915 device private data
 923  * @obj: i915 gem object
 924  * @args: pwrite arguments structure
 925  * @file: drm file pointer
 926  */
 927 static int
 928 i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 929                          struct drm_i915_gem_object *obj,
 930                          struct drm_i915_gem_pwrite *args,
 931                          struct drm_file *file)
 932 {
 933         struct i915_ggtt *ggtt = &i915->ggtt;
 934         struct drm_device *dev = obj->base.dev;
 935         struct drm_mm_node node;
 936         uint64_t remain, offset;
 937         char __user *user_data;
 938         int ret;
 939         bool hit_slow_path = false;
 940
 941         if (obj->tiling_mode != I915_TILING_NONE)
 942                 return -EFAULT;
 943
 944         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
 945         if (ret) {
 946                 ret = insert_mappable_node(i915, &node, PAGE_SIZE);
 947                 if (ret)
 948                         goto out;
 949
 950                 ret = i915_gem_object_get_pages(obj);
 951                 if (ret) {
 952                         remove_mappable_node(&node);
 953                         goto out;
 954                 }
 955
 956                 i915_gem_object_pin_pages(obj);
 957         } else {
 958                 node.start = i915_gem_obj_ggtt_offset(obj);
 959                 node.allocated = false;
 960                 ret = i915_gem_object_put_fence(obj);
 961                 if (ret)
 962                         goto out_unpin;
 963         }
 964
 965         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 966         if (ret)
 967                 goto out_unpin;
 968
 969         intel_fb_obj_invalidate(obj, ORIGIN_GTT);
 970         obj->dirty = true;
 971
 972         user_data = u64_to_user_ptr(args->data_ptr);
 973         offset = args->offset;
 974         remain = args->size;
 975         while (remain) {
 976                 /* Operation in this page
 977                  *
 978                  * page_base = page offset within aperture
 979                  * page_offset = offset within page
 980                  * page_length = bytes to copy for this page
 981                  */
 982                 u32 page_base = node.start;
 983                 unsigned page_offset = offset_in_page(offset);
 984                 unsigned page_length = PAGE_SIZE - page_offset;
 985                 page_length = remain < page_length ? remain : page_length;
 986                 if (node.allocated) {
 987                         wmb(); /* flush the write before we modify the GGTT */
 988                         ggtt->base.insert_page(&ggtt->base,
 989                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 990                                                node.start, I915_CACHE_NONE, 0);
 991                         wmb(); /* flush modifications to the GGTT (insert_page) */
 992                 } else {
 993                         page_base += offset & PAGE_MASK;
 994                 }
 995                 /* If we get a fault while copying data, then (presumably) our
 996                  * source page isn't available.  Return the error and we'll
 997                  * retry in the slow path.
 998                  * If the object is non-shmem backed, we retry again with the
 999                  * path that handles page fault.
1000                  */
1001                 if (fast_user_write(ggtt->mappable, page_base,
1002                                     page_offset, user_data, page_length)) {
1003                         hit_slow_path = true;
1004                         mutex_unlock(&dev->struct_mutex);
1005                         if (slow_user_access(ggtt->mappable,
1006                                              page_base,
1007                                              page_offset, user_data,
1008                                              page_length, true)) {
1009                                 ret = -EFAULT;
1010                                 mutex_lock(&dev->struct_mutex);
1011                                 goto out_flush;
1012                         }
1013
1014                         mutex_lock(&dev->struct_mutex);
1015                 }
1016
1017                 remain -= page_length;
1018                 user_data += page_length;
1019                 offset += page_length;
1020         }
1021
1022 out_flush:
1023         if (hit_slow_path) {
1024                 if (ret == 0 &&
1025                     (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
1026                         /* The user has modified the object whilst we tried
1027                          * reading from it, and we now have no idea what domain
1028                          * the pages should be in. As we have just been touching
1029                          * them directly, flush everything back to the GTT
1030                          * domain.
1031                          */
1032                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
1033                 }
1034         }
1035
1036         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
1037 out_unpin:
1038         if (node.allocated) {
1039                 wmb();
1040                 ggtt->base.clear_range(&ggtt->base,
1041                                        node.start, node.size,
1042                                        true);
1043                 i915_gem_object_unpin_pages(obj);
1044                 remove_mappable_node(&node);
1045         } else {
1046                 i915_gem_object_ggtt_unpin(obj);
1047         }
1048 out:
1049         return ret;
1050 }
1051
1052 /* Per-page copy function for the shmem pwrite fastpath.
1053  * Flushes invalid cachelines before writing to the target if
1054  * needs_clflush_before is set and flushes out any written cachelines after
1055  * writing if needs_clflush is set. */
1056 static int
1057 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
1058                   char __user *user_data,
1059                   bool page_do_bit17_swizzling,
1060                   bool needs_clflush_before,
1061                   bool needs_clflush_after)
1062 {
1063         char *vaddr;
1064         int ret;
1065
1066         if (unlikely(page_do_bit17_swizzling))
1067                 return -EINVAL;
1068
1069         vaddr = kmap_atomic(page);
1070         if (needs_clflush_before)
1071                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1072                                        page_length);
1073         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
1074                                         user_data, page_length);
1075         if (needs_clflush_after)
1076                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1077                                        page_length);
1078         kunmap_atomic(vaddr);
1079
1080         return ret ? -EFAULT : 0;
1081 }
1082
1083 /* Only difference to the fast-path function is that this can handle bit17
1084  * and uses non-atomic copy and kmap functions. */
1085 static int
1086 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
1087                   char __user *user_data,
1088                   bool page_do_bit17_swizzling,
1089                   bool needs_clflush_before,
1090                   bool needs_clflush_after)
1091 {
1092         char *vaddr;
1093         int ret;
1094
1095         vaddr = kmap(page);
1096         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1097                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1098                                              page_length,
1099                                              page_do_bit17_swizzling);
1100         if (page_do_bit17_swizzling)
1101                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1102                                                 user_data,
1103                                                 page_length);
1104         else
1105                 ret = __copy_from_user(vaddr + shmem_page_offset,
1106                                        user_data,
1107                                        page_length);
1108         if (needs_clflush_after)
1109                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1110                                              page_length,
1111                                              page_do_bit17_swizzling);
1112         kunmap(page);
1113
1114         return ret ? -EFAULT : 0;
1115 }
1116
1117 static int
1118 i915_gem_shmem_pwrite(struct drm_device *dev,
1119                       struct drm_i915_gem_object *obj,
1120                       struct drm_i915_gem_pwrite *args,
1121                       struct drm_file *file)
1122 {
1123         ssize_t remain;
1124         loff_t offset;
1125         char __user *user_data;
1126         int shmem_page_offset, page_length, ret = 0;
1127         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1128         int hit_slowpath = 0;
1129         int needs_clflush_after = 0;
1130         int needs_clflush_before = 0;
1131         struct sg_page_iter sg_iter;
1132
1133         user_data = u64_to_user_ptr(args->data_ptr);
1134         remain = args->size;
1135
1136         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1137
1138         ret = i915_gem_object_wait_rendering(obj, false);
1139         if (ret)
1140                 return ret;
1141
1142         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1143                 /* If we're not in the cpu write domain, set ourself into the gtt
1144                  * write domain and manually flush cachelines (if required). This
1145                  * optimizes for the case when the gpu will use the data
1146                  * right away and we therefore have to clflush anyway. */
1147                 needs_clflush_after = cpu_write_needs_clflush(obj);
1148         }
1149         /* Same trick applies to invalidate partially written cachelines read
1150          * before writing. */
1151         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1152                 needs_clflush_before =
1153                         !cpu_cache_is_coherent(dev, obj->cache_level);
1154
1155         ret = i915_gem_object_get_pages(obj);
1156         if (ret)
1157                 return ret;
1158
1159         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1160
1161         i915_gem_object_pin_pages(obj);
1162
1163         offset = args->offset;
1164         obj->dirty = 1;
1165
1166         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
1167                          offset >> PAGE_SHIFT) {
1168                 struct page *page = sg_page_iter_page(&sg_iter);
1169                 int partial_cacheline_write;
1170
1171                 if (remain <= 0)
1172                         break;
1173
1174                 /* Operation in this page
1175                  *
1176                  * shmem_page_offset = offset within page in shmem file
1177                  * page_length = bytes to copy for this page
1178                  */
1179                 shmem_page_offset = offset_in_page(offset);
1180
1181                 page_length = remain;
1182                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1183                         page_length = PAGE_SIZE - shmem_page_offset;
1184
1185                 /* If we don't overwrite a cacheline completely we need to be
1186                  * careful to have up-to-date data by first clflushing. Don't
1187                  * overcomplicate things and flush the entire patch. */
1188                 partial_cacheline_write = needs_clflush_before &&
1189                         ((shmem_page_offset | page_length)
1190                                 & (boot_cpu_data.x86_clflush_size - 1));
1191
1192                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1193                         (page_to_phys(page) & (1 << 17)) != 0;
1194
1195                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
1196                                         user_data, page_do_bit17_swizzling,
1197                                         partial_cacheline_write,
1198                                         needs_clflush_after);
1199                 if (ret == 0)
1200                         goto next_page;
1201
1202                 hit_slowpath = 1;
1203                 mutex_unlock(&dev->struct_mutex);
1204                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1205                                         user_data, page_do_bit17_swizzling,
1206                                         partial_cacheline_write,
1207                                         needs_clflush_after);
1208
1209                 mutex_lock(&dev->struct_mutex);
1210
1211                 if (ret)
1212                         goto out;
1213
1214 next_page:
1215                 remain -= page_length;
1216                 user_data += page_length;
1217                 offset += page_length;
1218         }
1219
1220 out:
1221         i915_gem_object_unpin_pages(obj);
1222
1223         if (hit_slowpath) {
1224                 /*
1225                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1226                  * cachelines in-line while writing and the object moved
1227                  * out of the cpu write domain while we've dropped the lock.
1228                  */
1229                 if (!needs_clflush_after &&
1230                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1231                         if (i915_gem_clflush_object(obj, obj->pin_display))
1232                                 needs_clflush_after = true;
1233                 }
1234         }
1235
1236         if (needs_clflush_after)
1237                 i915_gem_chipset_flush(to_i915(dev));
1238         else
1239                 obj->cache_dirty = true;
1240
1241         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1242         return ret;
1243 }
1244
1245 /**
1246  * Writes data to the object referenced by handle.
1247  * @dev: drm device
1248  * @data: ioctl data blob
1249  * @file: drm file
1250  *
1251  * On error, the contents of the buffer that were to be modified are undefined.
1252  */
1253 int
1254 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1255                       struct drm_file *file)
1256 {
1257         struct drm_i915_private *dev_priv = to_i915(dev);
1258         struct drm_i915_gem_pwrite *args = data;
1259         struct drm_i915_gem_object *obj;
1260         int ret;
1261
1262         if (args->size == 0)
1263                 return 0;
1264
1265         if (!access_ok(VERIFY_READ,
1266                        u64_to_user_ptr(args->data_ptr),
1267                        args->size))
1268                 return -EFAULT;
1269
1270         if (likely(!i915.prefault_disable)) {
1271                 ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
1272                                                    args->size);
1273                 if (ret)
1274                         return -EFAULT;
1275         }
1276
1277         intel_runtime_pm_get(dev_priv);
1278
1279         ret = i915_mutex_lock_interruptible(dev);
1280         if (ret)
1281                 goto put_rpm;
1282
1283         obj = i915_gem_object_lookup(file, args->handle);
1284         if (!obj) {
1285                 ret = -ENOENT;
1286                 goto unlock;
1287         }
1288
1289         /* Bounds check destination. */
1290         if (args->offset > obj->base.size ||
1291             args->size > obj->base.size - args->offset) {
1292                 ret = -EINVAL;
1293                 goto out;
1294         }
1295
1296         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1297
1298         ret = -EFAULT;
1299         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1300          * it would end up going through the fenced access, and we'll get
1301          * different detiling behavior between reading and writing.
1302          * pread/pwrite currently are reading and writing from the CPU
1303          * perspective, requiring manual detiling by the client.
1304          */
1305         if (!i915_gem_object_has_struct_page(obj) ||
1306             cpu_write_needs_clflush(obj)) {
1307                 ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
1308                 /* Note that the gtt paths might fail with non-page-backed user
1309                  * pointers (e.g. gtt mappings when moving data between
1310                  * textures). Fallback to the shmem path in that case. */
1311         }
1312
1313         if (ret == -EFAULT || ret == -ENOSPC) {
1314                 if (obj->phys_handle)
1315                         ret = i915_gem_phys_pwrite(obj, args, file);
1316                 else if (i915_gem_object_has_struct_page(obj))
1317                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1318                 else
1319                         ret = -ENODEV;
1320         }
1321
1322 out:
1323         i915_gem_object_put(obj);
1324 unlock:
1325         mutex_unlock(&dev->struct_mutex);
1326 put_rpm:
1327         intel_runtime_pm_put(dev_priv);
1328
1329         return ret;
1330 }
1331
1332 /**
1333  * Ensures that all rendering to the object has completed and the object is
1334  * safe to unbind from the GTT or access from the CPU.
1335  * @obj: i915 gem object
1336  * @readonly: waiting for read access or write
1337  */
1338 int
1339 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1340                                bool readonly)
1341 {
1342         struct reservation_object *resv;
1343         int ret, i;
1344
1345         if (readonly) {
1346                 if (obj->last_write_req != NULL) {
1347                         ret = i915_wait_request(obj->last_write_req);
1348                         if (ret)
1349                                 return ret;
1350
1351                         i = obj->last_write_req->engine->id;
1352                         if (obj->last_read_req[i] == obj->last_write_req)
1353                                 i915_gem_object_retire__read(obj, i);
1354                         else
1355                                 i915_gem_object_retire__write(obj);
1356                 }
1357         } else {
1358                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1359                         if (obj->last_read_req[i] == NULL)
1360                                 continue;
1361
1362                         ret = i915_wait_request(obj->last_read_req[i]);
1363                         if (ret)
1364                                 return ret;
1365
1366                         i915_gem_object_retire__read(obj, i);
1367                 }
1368                 GEM_BUG_ON(obj->active);
1369         }
1370
1371         resv = i915_gem_object_get_dmabuf_resv(obj);
1372         if (resv) {
1373                 long err;
1374
1375                 err = reservation_object_wait_timeout_rcu(resv, !readonly, true,
1376                                                           MAX_SCHEDULE_TIMEOUT);
1377                 if (err < 0)
1378                         return err;
1379         }
1380
1381         return 0;
1382 }
1383
1384 static void
1385 i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1386                                struct drm_i915_gem_request *req)
1387 {
1388         int idx = req->engine->id;
1389
1390         if (obj->last_read_req[idx] == req)
1391                 i915_gem_object_retire__read(obj, idx);
1392         else if (obj->last_write_req == req)
1393                 i915_gem_object_retire__write(obj);
1394
1395         if (!i915_reset_in_progress(&req->i915->gpu_error))
1396                 i915_gem_request_retire_upto(req);
1397 }
1398
1399 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1400  * as the object state may change during this call.
1401  */
1402 static __must_check int
1403 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1404                                             struct intel_rps_client *rps,
1405                                             bool readonly)
1406 {
1407         struct drm_device *dev = obj->base.dev;
1408         struct drm_i915_private *dev_priv = to_i915(dev);
1409         struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
1410         int ret, i, n = 0;
1411
1412         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1413         BUG_ON(!dev_priv->mm.interruptible);
1414
1415         if (!obj->active)
1416                 return 0;
1417
1418         if (readonly) {
1419                 struct drm_i915_gem_request *req;
1420
1421                 req = obj->last_write_req;
1422                 if (req == NULL)
1423                         return 0;
1424
1425                 requests[n++] = i915_gem_request_get(req);
1426         } else {
1427                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1428                         struct drm_i915_gem_request *req;
1429
1430                         req = obj->last_read_req[i];
1431                         if (req == NULL)
1432                                 continue;
1433
1434                         requests[n++] = i915_gem_request_get(req);
1435                 }
1436         }
1437
1438         mutex_unlock(&dev->struct_mutex);
1439         ret = 0;
1440         for (i = 0; ret == 0 && i < n; i++)
1441                 ret = __i915_wait_request(requests[i], true, NULL, rps);
1442         mutex_lock(&dev->struct_mutex);
1443
1444         for (i = 0; i < n; i++) {
1445                 if (ret == 0)
1446                         i915_gem_object_retire_request(obj, requests[i]);
1447                 i915_gem_request_put(requests[i]);
1448         }
1449
1450         return ret;
1451 }
1452
1453 static struct intel_rps_client *to_rps_client(struct drm_file *file)
1454 {
1455         struct drm_i915_file_private *fpriv = file->driver_priv;
1456         return &fpriv->rps;
1457 }
1458
1459 static enum fb_op_origin
1460 write_origin(struct drm_i915_gem_object *obj, unsigned domain)
1461 {
1462         return domain == I915_GEM_DOMAIN_GTT && !obj->has_wc_mmap ?
1463                ORIGIN_GTT : ORIGIN_CPU;
1464 }
1465
1466 /**
1467  * Called when user space prepares to use an object with the CPU, either
1468  * through the mmap ioctl's mapping or a GTT mapping.
1469  * @dev: drm device
1470  * @data: ioctl data blob
1471  * @file: drm file
1472  */
1473 int
1474 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1475                           struct drm_file *file)
1476 {
1477         struct drm_i915_gem_set_domain *args = data;
1478         struct drm_i915_gem_object *obj;
1479         uint32_t read_domains = args->read_domains;
1480         uint32_t write_domain = args->write_domain;
1481         int ret;
1482
1483         /* Only handle setting domains to types used by the CPU. */
1484         if (write_domain & I915_GEM_GPU_DOMAINS)
1485                 return -EINVAL;
1486
1487         if (read_domains & I915_GEM_GPU_DOMAINS)
1488                 return -EINVAL;
1489
1490         /* Having something in the write domain implies it's in the read
1491          * domain, and only that read domain.  Enforce that in the request.
1492          */
1493         if (write_domain != 0 && read_domains != write_domain)
1494                 return -EINVAL;
1495
1496         ret = i915_mutex_lock_interruptible(dev);
1497         if (ret)
1498                 return ret;
1499
1500         obj = i915_gem_object_lookup(file, args->handle);
1501         if (!obj) {
1502                 ret = -ENOENT;
1503                 goto unlock;
1504         }
1505
1506         /* Try to flush the object off the GPU without holding the lock.
1507          * We will repeat the flush holding the lock in the normal manner
1508          * to catch cases where we are gazumped.
1509          */
1510         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1511                                                           to_rps_client(file),
1512                                                           !write_domain);
1513         if (ret)
1514                 goto unref;
1515
1516         if (read_domains & I915_GEM_DOMAIN_GTT)
1517                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1518         else
1519                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1520
1521         if (write_domain != 0)
1522                 intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
1523
1524 unref:
1525         i915_gem_object_put(obj);
1526 unlock:
1527         mutex_unlock(&dev->struct_mutex);
1528         return ret;
1529 }
1530
1531 /**
1532  * Called when user space has done writes to this buffer
1533  * @dev: drm device
1534  * @data: ioctl data blob
1535  * @file: drm file
1536  */
1537 int
1538 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1539                          struct drm_file *file)
1540 {
1541         struct drm_i915_gem_sw_finish *args = data;
1542         struct drm_i915_gem_object *obj;
1543         int ret = 0;
1544
1545         ret = i915_mutex_lock_interruptible(dev);
1546         if (ret)
1547                 return ret;
1548
1549         obj = i915_gem_object_lookup(file, args->handle);
1550         if (!obj) {
1551                 ret = -ENOENT;
1552                 goto unlock;
1553         }
1554
1555         /* Pinned buffers may be scanout, so flush the cache */
1556         if (obj->pin_display)
1557                 i915_gem_object_flush_cpu_write_domain(obj);
1558
1559         i915_gem_object_put(obj);
1560 unlock:
1561         mutex_unlock(&dev->struct_mutex);
1562         return ret;
1563 }
1564
1565 /**
1566  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1567  *                       it is mapped to.
1568  * @dev: drm device
1569  * @data: ioctl data blob
1570  * @file: drm file
1571  *
1572  * While the mapping holds a reference on the contents of the object, it doesn't
1573  * imply a ref on the object itself.
1574  *
1575  * IMPORTANT:
1576  *
1577  * DRM driver writers who look a this function as an example for how to do GEM
1578  * mmap support, please don't implement mmap support like here. The modern way
1579  * to implement DRM mmap support is with an mmap offset ioctl (like
1580  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1581  * That way debug tooling like valgrind will understand what's going on, hiding
1582  * the mmap call in a driver private ioctl will break that. The i915 driver only
1583  * does cpu mmaps this way because we didn't know better.
1584  */
1585 int
1586 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1587                     struct drm_file *file)
1588 {
1589         struct drm_i915_gem_mmap *args = data;
1590         struct drm_i915_gem_object *obj;
1591         unsigned long addr;
1592
1593         if (args->flags & ~(I915_MMAP_WC))
1594                 return -EINVAL;
1595
1596         if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1597                 return -ENODEV;
1598
1599         obj = i915_gem_object_lookup(file, args->handle);
1600         if (!obj)
1601                 return -ENOENT;
1602
1603         /* prime objects have no backing filp to GEM mmap
1604          * pages from.
1605          */
1606         if (!obj->base.filp) {
1607                 i915_gem_object_put_unlocked(obj);
1608                 return -EINVAL;
1609         }
1610
1611         addr = vm_mmap(obj->base.filp, 0, args->size,
1612                        PROT_READ | PROT_WRITE, MAP_SHARED,
1613                        args->offset);
1614         if (args->flags & I915_MMAP_WC) {
1615                 struct mm_struct *mm = current->mm;
1616                 struct vm_area_struct *vma;
1617
1618                 if (down_write_killable(&mm->mmap_sem)) {
1619                         i915_gem_object_put_unlocked(obj);
1620                         return -EINTR;
1621                 }
1622                 vma = find_vma(mm, addr);
1623                 if (vma)
1624                         vma->vm_page_prot =
1625                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1626                 else
1627                         addr = -ENOMEM;
1628                 up_write(&mm->mmap_sem);
1629
1630                 /* This may race, but that's ok, it only gets set */
1631                 WRITE_ONCE(obj->has_wc_mmap, true);
1632         }
1633         i915_gem_object_put_unlocked(obj);
1634         if (IS_ERR((void *)addr))
1635                 return addr;
1636
1637         args->addr_ptr = (uint64_t) addr;
1638
1639         return 0;
1640 }
1641
1642 /**
1643  * i915_gem_fault - fault a page into the GTT
1644  * @vma: VMA in question
1645  * @vmf: fault info
1646  *
1647  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1648  * from userspace.  The fault handler takes care of binding the object to
1649  * the GTT (if needed), allocating and programming a fence register (again,
1650  * only if needed based on whether the old reg is still valid or the object
1651  * is tiled) and inserting a new PTE into the faulting process.
1652  *
1653  * Note that the faulting process may involve evicting existing objects
1654  * from the GTT and/or fence registers to make room.  So performance may
1655  * suffer if the GTT working set is large or there are few fence registers
1656  * left.
1657  */
1658 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1659 {
1660         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1661         struct drm_device *dev = obj->base.dev;
1662         struct drm_i915_private *dev_priv = to_i915(dev);
1663         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1664         struct i915_ggtt_view view = i915_ggtt_view_normal;
1665         pgoff_t page_offset;
1666         unsigned long pfn;
1667         int ret = 0;
1668         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1669
1670         intel_runtime_pm_get(dev_priv);
1671
1672         /* We don't use vmf->pgoff since that has the fake offset */
1673         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1674                 PAGE_SHIFT;
1675
1676         ret = i915_mutex_lock_interruptible(dev);
1677         if (ret)
1678                 goto out;
1679
1680         trace_i915_gem_object_fault(obj, page_offset, true, write);
1681
1682         /* Try to flush the object off the GPU first without holding the lock.
1683          * Upon reacquiring the lock, we will perform our sanity checks and then
1684          * repeat the flush holding the lock in the normal manner to catch cases
1685          * where we are gazumped.
1686          */
1687         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1688         if (ret)
1689                 goto unlock;
1690
1691         /* Access to snoopable pages through the GTT is incoherent. */
1692         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1693                 ret = -EFAULT;
1694                 goto unlock;
1695         }
1696
1697         /* Use a partial view if the object is bigger than the aperture. */
1698         if (obj->base.size >= ggtt->mappable_end &&
1699             obj->tiling_mode == I915_TILING_NONE) {
1700                 static const unsigned int chunk_size = 256; // 1 MiB
1701
1702                 memset(&view, 0, sizeof(view));
1703                 view.type = I915_GGTT_VIEW_PARTIAL;
1704                 view.params.partial.offset = rounddown(page_offset, chunk_size);
1705                 view.params.partial.size =
1706                         min_t(unsigned int,
1707                               chunk_size,
1708                               (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1709                               view.params.partial.offset);
1710         }
1711
1712         /* Now pin it into the GTT if needed */
1713         ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
1714         if (ret)
1715                 goto unlock;
1716
1717         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1718         if (ret)
1719                 goto unpin;
1720
1721         ret = i915_gem_object_get_fence(obj);
1722         if (ret)
1723                 goto unpin;
1724
1725         /* Finally, remap it using the new GTT offset */
1726         pfn = ggtt->mappable_base +
1727                 i915_gem_obj_ggtt_offset_view(obj, &view);
1728         pfn >>= PAGE_SHIFT;
1729
1730         if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1731                 /* Overriding existing pages in partial view does not cause
1732                  * us any trouble as TLBs are still valid because the fault
1733                  * is due to userspace losing part of the mapping or never
1734                  * having accessed it before (at this partials' range).
1735                  */
1736                 unsigned long base = vma->vm_start +
1737                                      (view.params.partial.offset << PAGE_SHIFT);
1738                 unsigned int i;
1739
1740                 for (i = 0; i < view.params.partial.size; i++) {
1741                         ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1742                         if (ret)
1743                                 break;
1744                 }
1745
1746                 obj->fault_mappable = true;
1747         } else {
1748                 if (!obj->fault_mappable) {
1749                         unsigned long size = min_t(unsigned long,
1750                                                    vma->vm_end - vma->vm_start,
1751                                                    obj->base.size);
1752                         int i;
1753
1754                         for (i = 0; i < size >> PAGE_SHIFT; i++) {
1755                                 ret = vm_insert_pfn(vma,
1756                                                     (unsigned long)vma->vm_start + i * PAGE_SIZE,
1757                                                     pfn + i);
1758                                 if (ret)
1759                                         break;
1760                         }
1761
1762                         obj->fault_mappable = true;
1763                 } else
1764                         ret = vm_insert_pfn(vma,
1765                                             (unsigned long)vmf->virtual_address,
1766                                             pfn + page_offset);
1767         }
1768 unpin:
1769         i915_gem_object_ggtt_unpin_view(obj, &view);
1770 unlock:
1771         mutex_unlock(&dev->struct_mutex);
1772 out:
1773         switch (ret) {
1774         case -EIO:
1775                 /*
1776                  * We eat errors when the gpu is terminally wedged to avoid
1777                  * userspace unduly crashing (gl has no provisions for mmaps to
1778                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1779                  * and so needs to be reported.
1780                  */
1781                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1782                         ret = VM_FAULT_SIGBUS;
1783                         break;
1784                 }
1785         case -EAGAIN:
1786                 /*
1787                  * EAGAIN means the gpu is hung and we'll wait for the error
1788                  * handler to reset everything when re-faulting in
1789                  * i915_mutex_lock_interruptible.
1790                  */
1791         case 0:
1792         case -ERESTARTSYS:
1793         case -EINTR:
1794         case -EBUSY:
1795                 /*
1796                  * EBUSY is ok: this just means that another thread
1797                  * already did the job.
1798                  */
1799                 ret = VM_FAULT_NOPAGE;
1800                 break;
1801         case -ENOMEM:
1802                 ret = VM_FAULT_OOM;
1803                 break;
1804         case -ENOSPC:
1805         case -EFAULT:
1806                 ret = VM_FAULT_SIGBUS;
1807                 break;
1808         default:
1809                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1810                 ret = VM_FAULT_SIGBUS;
1811                 break;
1812         }
1813
1814         intel_runtime_pm_put(dev_priv);
1815         return ret;
1816 }
1817
1818 /**
1819  * i915_gem_release_mmap - remove physical page mappings
1820  * @obj: obj in question
1821  *
1822  * Preserve the reservation of the mmapping with the DRM core code, but
1823  * relinquish ownership of the pages back to the system.
1824  *
1825  * It is vital that we remove the page mapping if we have mapped a tiled
1826  * object through the GTT and then lose the fence register due to
1827  * resource pressure. Similarly if the object has been moved out of the
1828  * aperture, than pages mapped into userspace must be revoked. Removing the
1829  * mapping will then trigger a page fault on the next user access, allowing
1830  * fixup by i915_gem_fault().
1831  */
1832 void
1833 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1834 {
1835         /* Serialisation between user GTT access and our code depends upon
1836          * revoking the CPU's PTE whilst the mutex is held. The next user
1837          * pagefault then has to wait until we release the mutex.
1838          */
1839         lockdep_assert_held(&obj->base.dev->struct_mutex);
1840
1841         if (!obj->fault_mappable)
1842                 return;
1843
1844         drm_vma_node_unmap(&obj->base.vma_node,
1845                            obj->base.dev->anon_inode->i_mapping);
1846
1847         /* Ensure that the CPU's PTE are revoked and there are not outstanding
1848          * memory transactions from userspace before we return. The TLB
1849          * flushing implied above by changing the PTE above *should* be
1850          * sufficient, an extra barrier here just provides us with a bit
1851          * of paranoid documentation about our requirement to serialise
1852          * memory writes before touching registers / GSM.
1853          */
1854         wmb();
1855
1856         obj->fault_mappable = false;
1857 }
1858
1859 void
1860 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1861 {
1862         struct drm_i915_gem_object *obj;
1863
1864         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1865                 i915_gem_release_mmap(obj);
1866 }
1867
1868 uint32_t
1869 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1870 {
1871         uint32_t gtt_size;
1872
1873         if (INTEL_INFO(dev)->gen >= 4 ||
1874             tiling_mode == I915_TILING_NONE)
1875                 return size;
1876
1877         /* Previous chips need a power-of-two fence region when tiling */
1878         if (IS_GEN3(dev))
1879                 gtt_size = 1024*1024;
1880         else
1881                 gtt_size = 512*1024;
1882
1883         while (gtt_size < size)
1884                 gtt_size <<= 1;
1885
1886         return gtt_size;
1887 }
1888
1889 /**
1890  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1891  * @dev: drm device
1892  * @size: object size
1893  * @tiling_mode: tiling mode
1894  * @fenced: is fenced alignemned required or not
1895  *
1896  * Return the required GTT alignment for an object, taking into account
1897  * potential fence register mapping.
1898  */
1899 uint32_t
1900 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1901                            int tiling_mode, bool fenced)
1902 {
1903         /*
1904          * Minimum alignment is 4k (GTT page size), but might be greater
1905          * if a fence register is needed for the object.
1906          */
1907         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1908             tiling_mode == I915_TILING_NONE)
1909                 return 4096;
1910
1911         /*
1912          * Previous chips need to be aligned to the size of the smallest
1913          * fence register that can contain the object.
1914          */
1915         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1916 }
1917
1918 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1919 {
1920         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
1921         int ret;
1922
1923         dev_priv->mm.shrinker_no_lock_stealing = true;
1924
1925         ret = drm_gem_create_mmap_offset(&obj->base);
1926         if (ret != -ENOSPC)
1927                 goto out;
1928
1929         /* Badly fragmented mmap space? The only way we can recover
1930          * space is by destroying unwanted objects. We can't randomly release
1931          * mmap_offsets as userspace expects them to be persistent for the
1932          * lifetime of the objects. The closest we can is to release the
1933          * offsets on purgeable objects by truncating it and marking it purged,
1934          * which prevents userspace from ever using that object again.
1935          */
1936         i915_gem_shrink(dev_priv,
1937                         obj->base.size >> PAGE_SHIFT,
1938                         I915_SHRINK_BOUND |
1939                         I915_SHRINK_UNBOUND |
1940                         I915_SHRINK_PURGEABLE);
1941         ret = drm_gem_create_mmap_offset(&obj->base);
1942         if (ret != -ENOSPC)
1943                 goto out;
1944
1945         i915_gem_shrink_all(dev_priv);
1946         ret = drm_gem_create_mmap_offset(&obj->base);
1947 out:
1948         dev_priv->mm.shrinker_no_lock_stealing = false;
1949
1950         return ret;
1951 }
1952
1953 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1954 {
1955         drm_gem_free_mmap_offset(&obj->base);
1956 }
1957
1958 int
1959 i915_gem_mmap_gtt(struct drm_file *file,
1960                   struct drm_device *dev,
1961                   uint32_t handle,
1962                   uint64_t *offset)
1963 {
1964         struct drm_i915_gem_object *obj;
1965         int ret;
1966
1967         ret = i915_mutex_lock_interruptible(dev);
1968         if (ret)
1969                 return ret;
1970
1971         obj = i915_gem_object_lookup(file, handle);
1972         if (!obj) {
1973                 ret = -ENOENT;
1974                 goto unlock;
1975         }
1976
1977         if (obj->madv != I915_MADV_WILLNEED) {
1978                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
1979                 ret = -EFAULT;
1980                 goto out;
1981         }
1982
1983         ret = i915_gem_object_create_mmap_offset(obj);
1984         if (ret)
1985                 goto out;
1986
1987         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1988
1989 out:
1990         i915_gem_object_put(obj);
1991 unlock:
1992         mutex_unlock(&dev->struct_mutex);
1993         return ret;
1994 }
1995
1996 /**
1997  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1998  * @dev: DRM device
1999  * @data: GTT mapping ioctl data
2000  * @file: GEM object info
2001  *
2002  * Simply returns the fake offset to userspace so it can mmap it.
2003  * The mmap call will end up in drm_gem_mmap(), which will set things
2004  * up so we can get faults in the handler above.
2005  *
2006  * The fault handler will take care of binding the object into the GTT
2007  * (since it may have been evicted to make room for something), allocating
2008  * a fence register, and mapping the appropriate aperture address into
2009  * userspace.
2010  */
2011 int
2012 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2013                         struct drm_file *file)
2014 {
2015         struct drm_i915_gem_mmap_gtt *args = data;
2016
2017         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2018 }
2019
2020 /* Immediately discard the backing storage */
2021 static void
2022 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2023 {
2024         i915_gem_object_free_mmap_offset(obj);
2025
2026         if (obj->base.filp == NULL)
2027                 return;
2028
2029         /* Our goal here is to return as much of the memory as
2030          * is possible back to the system as we are called from OOM.
2031          * To do this we must instruct the shmfs to drop all of its
2032          * backing pages, *now*.
2033          */
2034         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2035         obj->madv = __I915_MADV_PURGED;
2036 }
2037
2038 /* Try to discard unwanted pages */
2039 static void
2040 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2041 {
2042         struct address_space *mapping;
2043
2044         switch (obj->madv) {
2045         case I915_MADV_DONTNEED:
2046                 i915_gem_object_truncate(obj);
2047         case __I915_MADV_PURGED:
2048                 return;
2049         }
2050
2051         if (obj->base.filp == NULL)
2052                 return;
2053
2054         mapping = file_inode(obj->base.filp)->i_mapping,
2055         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2056 }
2057
2058 static void
2059 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2060 {
2061         struct sgt_iter sgt_iter;
2062         struct page *page;
2063         int ret;
2064
2065         BUG_ON(obj->madv == __I915_MADV_PURGED);
2066
2067         ret = i915_gem_object_set_to_cpu_domain(obj, true);
2068         if (WARN_ON(ret)) {
2069                 /* In the event of a disaster, abandon all caches and
2070                  * hope for the best.
2071                  */
2072                 i915_gem_clflush_object(obj, true);
2073                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2074         }
2075
2076         i915_gem_gtt_finish_object(obj);
2077
2078         if (i915_gem_object_needs_bit17_swizzle(obj))
2079                 i915_gem_object_save_bit_17_swizzle(obj);
2080
2081         if (obj->madv == I915_MADV_DONTNEED)
2082                 obj->dirty = 0;
2083
2084         for_each_sgt_page(page, sgt_iter, obj->pages) {
2085                 if (obj->dirty)
2086                         set_page_dirty(page);
2087
2088                 if (obj->madv == I915_MADV_WILLNEED)
2089                         mark_page_accessed(page);
2090
2091                 put_page(page);
2092         }
2093         obj->dirty = 0;
2094
2095         sg_free_table(obj->pages);
2096         kfree(obj->pages);
2097 }
2098
2099 int
2100 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2101 {
2102         const struct drm_i915_gem_object_ops *ops = obj->ops;
2103
2104         if (obj->pages == NULL)
2105                 return 0;
2106
2107         if (obj->pages_pin_count)
2108                 return -EBUSY;
2109
2110         BUG_ON(i915_gem_obj_bound_any(obj));
2111
2112         /* ->put_pages might need to allocate memory for the bit17 swizzle
2113          * array, hence protect them from being reaped by removing them from gtt
2114          * lists early. */
2115         list_del(&obj->global_list);
2116
2117         if (obj->mapping) {
2118                 if (is_vmalloc_addr(obj->mapping))
2119                         vunmap(obj->mapping);
2120                 else
2121                         kunmap(kmap_to_page(obj->mapping));
2122                 obj->mapping = NULL;
2123         }
2124
2125         ops->put_pages(obj);
2126         obj->pages = NULL;
2127
2128         i915_gem_object_invalidate(obj);
2129
2130         return 0;
2131 }
2132
2133 static int
2134 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2135 {
2136         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2137         int page_count, i;
2138         struct address_space *mapping;
2139         struct sg_table *st;
2140         struct scatterlist *sg;
2141         struct sgt_iter sgt_iter;
2142         struct page *page;
2143         unsigned long last_pfn = 0;     /* suppress gcc warning */
2144         int ret;
2145         gfp_t gfp;
2146
2147         /* Assert that the object is not currently in any GPU domain. As it
2148          * wasn't in the GTT, there shouldn't be any way it could have been in
2149          * a GPU cache
2150          */
2151         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2152         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2153
2154         st = kmalloc(sizeof(*st), GFP_KERNEL);
2155         if (st == NULL)
2156                 return -ENOMEM;
2157
2158         page_count = obj->base.size / PAGE_SIZE;
2159         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2160                 kfree(st);
2161                 return -ENOMEM;
2162         }
2163
2164         /* Get the list of pages out of our struct file.  They'll be pinned
2165          * at this point until we release them.
2166          *
2167          * Fail silently without starting the shrinker
2168          */
2169         mapping = file_inode(obj->base.filp)->i_mapping;
2170         gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2171         gfp |= __GFP_NORETRY | __GFP_NOWARN;
2172         sg = st->sgl;
2173         st->nents = 0;
2174         for (i = 0; i < page_count; i++) {
2175                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2176                 if (IS_ERR(page)) {
2177                         i915_gem_shrink(dev_priv,
2178                                         page_count,
2179                                         I915_SHRINK_BOUND |
2180                                         I915_SHRINK_UNBOUND |
2181                                         I915_SHRINK_PURGEABLE);
2182                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2183                 }
2184                 if (IS_ERR(page)) {
2185                         /* We've tried hard to allocate the memory by reaping
2186                          * our own buffer, now let the real VM do its job and
2187                          * go down in flames if truly OOM.
2188                          */
2189                         i915_gem_shrink_all(dev_priv);
2190                         page = shmem_read_mapping_page(mapping, i);
2191                         if (IS_ERR(page)) {
2192                                 ret = PTR_ERR(page);
2193                                 goto err_pages;
2194                         }
2195                 }
2196 #ifdef CONFIG_SWIOTLB
2197                 if (swiotlb_nr_tbl()) {
2198                         st->nents++;
2199                         sg_set_page(sg, page, PAGE_SIZE, 0);
2200                         sg = sg_next(sg);
2201                         continue;
2202                 }
2203 #endif
2204                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2205                         if (i)
2206                                 sg = sg_next(sg);
2207                         st->nents++;
2208                         sg_set_page(sg, page, PAGE_SIZE, 0);
2209                 } else {
2210                         sg->length += PAGE_SIZE;
2211                 }
2212                 last_pfn = page_to_pfn(page);
2213
2214                 /* Check that the i965g/gm workaround works. */
2215                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2216         }
2217 #ifdef CONFIG_SWIOTLB
2218         if (!swiotlb_nr_tbl())
2219 #endif
2220                 sg_mark_end(sg);
2221         obj->pages = st;
2222
2223         ret = i915_gem_gtt_prepare_object(obj);
2224         if (ret)
2225                 goto err_pages;
2226
2227         if (i915_gem_object_needs_bit17_swizzle(obj))
2228                 i915_gem_object_do_bit_17_swizzle(obj);
2229
2230         if (obj->tiling_mode != I915_TILING_NONE &&
2231             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2232                 i915_gem_object_pin_pages(obj);
2233
2234         return 0;
2235
2236 err_pages:
2237         sg_mark_end(sg);
2238         for_each_sgt_page(page, sgt_iter, st)
2239                 put_page(page);
2240         sg_free_table(st);
2241         kfree(st);
2242
2243         /* shmemfs first checks if there is enough memory to allocate the page
2244          * and reports ENOSPC should there be insufficient, along with the usual
2245          * ENOMEM for a genuine allocation failure.
2246          *
2247          * We use ENOSPC in our driver to mean that we have run out of aperture
2248          * space and so want to translate the error from shmemfs back to our
2249          * usual understanding of ENOMEM.
2250          */
2251         if (ret == -ENOSPC)
2252                 ret = -ENOMEM;
2253
2254         return ret;
2255 }
2256
2257 /* Ensure that the associated pages are gathered from the backing storage
2258  * and pinned into our object. i915_gem_object_get_pages() may be called
2259  * multiple times before they are released by a single call to
2260  * i915_gem_object_put_pages() - once the pages are no longer referenced
2261  * either as a result of memory pressure (reaping pages under the shrinker)
2262  * or as the object is itself released.
2263  */
2264 int
2265 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2266 {
2267         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2268         const struct drm_i915_gem_object_ops *ops = obj->ops;
2269         int ret;
2270
2271         if (obj->pages)
2272                 return 0;
2273
2274         if (obj->madv != I915_MADV_WILLNEED) {
2275                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2276                 return -EFAULT;
2277         }
2278
2279         BUG_ON(obj->pages_pin_count);
2280
2281         ret = ops->get_pages(obj);
2282         if (ret)
2283                 return ret;
2284
2285         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2286
2287         obj->get_page.sg = obj->pages->sgl;
2288         obj->get_page.last = 0;
2289
2290         return 0;
2291 }
2292
2293 /* The 'mapping' part of i915_gem_object_pin_map() below */
2294 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
2295 {
2296         unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2297         struct sg_table *sgt = obj->pages;
2298         struct sgt_iter sgt_iter;
2299         struct page *page;
2300         struct page *stack_pages[32];
2301         struct page **pages = stack_pages;
2302         unsigned long i = 0;
2303         void *addr;
2304
2305         /* A single page can always be kmapped */
2306         if (n_pages == 1)
2307                 return kmap(sg_page(sgt->sgl));
2308
2309         if (n_pages > ARRAY_SIZE(stack_pages)) {
2310                 /* Too big for stack -- allocate temporary array instead */
2311                 pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
2312                 if (!pages)
2313                         return NULL;
2314         }
2315
2316         for_each_sgt_page(page, sgt_iter, sgt)
2317                 pages[i++] = page;
2318
2319         /* Check that we have the expected number of pages */
2320         GEM_BUG_ON(i != n_pages);
2321
2322         addr = vmap(pages, n_pages, 0, PAGE_KERNEL);
2323
2324         if (pages != stack_pages)
2325                 drm_free_large(pages);
2326
2327         return addr;
2328 }
2329
2330 /* get, pin, and map the pages of the object into kernel space */
2331 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
2332 {
2333         int ret;
2334
2335         lockdep_assert_held(&obj->base.dev->struct_mutex);
2336
2337         ret = i915_gem_object_get_pages(obj);
2338         if (ret)
2339                 return ERR_PTR(ret);
2340
2341         i915_gem_object_pin_pages(obj);
2342
2343         if (!obj->mapping) {
2344                 obj->mapping = i915_gem_object_map(obj);
2345                 if (!obj->mapping) {
2346                         i915_gem_object_unpin_pages(obj);
2347                         return ERR_PTR(-ENOMEM);
2348                 }
2349         }
2350
2351         return obj->mapping;
2352 }
2353
2354 void i915_vma_move_to_active(struct i915_vma *vma,
2355                              struct drm_i915_gem_request *req)
2356 {
2357         struct drm_i915_gem_object *obj = vma->obj;
2358         struct intel_engine_cs *engine;
2359
2360         engine = i915_gem_request_get_engine(req);
2361
2362         /* Add a reference if we're newly entering the active list. */
2363         if (obj->active == 0)
2364                 i915_gem_object_get(obj);
2365         obj->active |= intel_engine_flag(engine);
2366
2367         list_move_tail(&obj->engine_list[engine->id], &engine->active_list);
2368         i915_gem_request_assign(&obj->last_read_req[engine->id], req);
2369
2370         list_move_tail(&vma->vm_link, &vma->vm->active_list);
2371 }
2372
2373 static void
2374 i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
2375 {
2376         GEM_BUG_ON(obj->last_write_req == NULL);
2377         GEM_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine)));
2378
2379         i915_gem_request_assign(&obj->last_write_req, NULL);
2380         intel_fb_obj_flush(obj, true, ORIGIN_CS);
2381 }
2382
2383 static void
2384 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int idx)
2385 {
2386         struct i915_vma *vma;
2387
2388         GEM_BUG_ON(obj->last_read_req[idx] == NULL);
2389         GEM_BUG_ON(!(obj->active & (1 << idx)));
2390
2391         list_del_init(&obj->engine_list[idx]);
2392         i915_gem_request_assign(&obj->last_read_req[idx], NULL);
2393
2394         if (obj->last_write_req && obj->last_write_req->engine->id == idx)
2395                 i915_gem_object_retire__write(obj);
2396
2397         obj->active &= ~(1 << idx);
2398         if (obj->active)
2399                 return;
2400
2401         /* Bump our place on the bound list to keep it roughly in LRU order
2402          * so that we don't steal from recently used but inactive objects
2403          * (unless we are forced to ofc!)
2404          */
2405         list_move_tail(&obj->global_list,
2406                        &to_i915(obj->base.dev)->mm.bound_list);
2407
2408         list_for_each_entry(vma, &obj->vma_list, obj_link) {
2409                 if (!list_empty(&vma->vm_link))
2410                         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
2411         }
2412
2413         i915_gem_request_assign(&obj->last_fenced_req, NULL);
2414         i915_gem_object_put(obj);
2415 }
2416
2417 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
2418 {
2419         unsigned long elapsed;
2420
2421         if (ctx->hang_stats.banned)
2422                 return true;
2423
2424         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2425         if (ctx->hang_stats.ban_period_seconds &&
2426             elapsed <= ctx->hang_stats.ban_period_seconds) {
2427                 DRM_DEBUG("context hanging too fast, banning!\n");
2428                 return true;
2429         }
2430
2431         return false;
2432 }
2433
2434 static void i915_set_reset_status(struct i915_gem_context *ctx,
2435                                   const bool guilty)
2436 {
2437         struct i915_ctx_hang_stats *hs = &ctx->hang_stats;
2438
2439         if (guilty) {
2440                 hs->banned = i915_context_is_banned(ctx);
2441                 hs->batch_active++;
2442                 hs->guilty_ts = get_seconds();
2443         } else {
2444                 hs->batch_pending++;
2445         }
2446 }
2447
2448 struct drm_i915_gem_request *
2449 i915_gem_find_active_request(struct intel_engine_cs *engine)
2450 {
2451         struct drm_i915_gem_request *request;
2452
2453         /* We are called by the error capture and reset at a random
2454          * point in time. In particular, note that neither is crucially
2455          * ordered with an interrupt. After a hang, the GPU is dead and we
2456          * assume that no more writes can happen (we waited long enough for
2457          * all writes that were in transaction to be flushed) - adding an
2458          * extra delay for a recent interrupt is pointless. Hence, we do
2459          * not need an engine->irq_seqno_barrier() before the seqno reads.
2460          */
2461         list_for_each_entry(request, &engine->request_list, list) {
2462                 if (i915_gem_request_completed(request))
2463                         continue;
2464
2465                 return request;
2466         }
2467
2468         return NULL;
2469 }
2470
2471 static void i915_gem_reset_engine_status(struct intel_engine_cs *engine)
2472 {
2473         struct drm_i915_gem_request *request;
2474         bool ring_hung;
2475
2476         request = i915_gem_find_active_request(engine);
2477         if (request == NULL)
2478                 return;
2479
2480         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2481
2482         i915_set_reset_status(request->ctx, ring_hung);
2483         list_for_each_entry_continue(request, &engine->request_list, list)
2484                 i915_set_reset_status(request->ctx, false);
2485 }
2486
2487 static void i915_gem_reset_engine_cleanup(struct intel_engine_cs *engine)
2488 {
2489         struct intel_ring *ring;
2490
2491         while (!list_empty(&engine->active_list)) {
2492                 struct drm_i915_gem_object *obj;
2493
2494                 obj = list_first_entry(&engine->active_list,
2495                                        struct drm_i915_gem_object,
2496                                        engine_list[engine->id]);
2497
2498                 i915_gem_object_retire__read(obj, engine->id);
2499         }
2500
2501         /* Mark all pending requests as complete so that any concurrent
2502          * (lockless) lookup doesn't try and wait upon the request as we
2503          * reset it.
2504          */
2505         intel_engine_init_seqno(engine, engine->last_submitted_seqno);
2506
2507         /*
2508          * Clear the execlists queue up before freeing the requests, as those
2509          * are the ones that keep the context and ringbuffer backing objects
2510          * pinned in place.
2511          */
2512
2513         if (i915.enable_execlists) {
2514                 /* Ensure irq handler finishes or is cancelled. */
2515                 tasklet_kill(&engine->irq_tasklet);
2516
2517                 intel_execlists_cancel_requests(engine);
2518         }
2519
2520         /*
2521          * We must free the requests after all the corresponding objects have
2522          * been moved off active lists. Which is the same order as the normal
2523          * retire_requests function does. This is important if object hold
2524          * implicit references on things like e.g. ppgtt address spaces through
2525          * the request.
2526          */
2527         if (!list_empty(&engine->request_list)) {
2528                 struct drm_i915_gem_request *request;
2529
2530                 request = list_last_entry(&engine->request_list,
2531                                           struct drm_i915_gem_request,
2532                                           list);
2533
2534                 i915_gem_request_retire_upto(request);
2535         }
2536
2537         /* Having flushed all requests from all queues, we know that all
2538          * ringbuffers must now be empty. However, since we do not reclaim
2539          * all space when retiring the request (to prevent HEADs colliding
2540          * with rapid ringbuffer wraparound) the amount of available space
2541          * upon reset is less than when we start. Do one more pass over
2542          * all the ringbuffers to reset last_retired_head.
2543          */
2544         list_for_each_entry(ring, &engine->buffers, link) {
2545                 ring->last_retired_head = ring->tail;
2546                 intel_ring_update_space(ring);
2547         }
2548
2549         engine->i915->gt.active_engines &= ~intel_engine_flag(engine);
2550 }
2551
2552 void i915_gem_reset(struct drm_device *dev)
2553 {
2554         struct drm_i915_private *dev_priv = to_i915(dev);
2555         struct intel_engine_cs *engine;
2556
2557         /*
2558          * Before we free the objects from the requests, we need to inspect
2559          * them for finding the guilty party. As the requests only borrow
2560          * their reference to the objects, the inspection must be done first.
2561          */
2562         for_each_engine(engine, dev_priv)
2563                 i915_gem_reset_engine_status(engine);
2564
2565         for_each_engine(engine, dev_priv)
2566                 i915_gem_reset_engine_cleanup(engine);
2567         mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
2568
2569         i915_gem_context_reset(dev);
2570
2571         i915_gem_restore_fences(dev);
2572
2573         WARN_ON(i915_verify_lists(dev));
2574 }
2575
2576 /**
2577  * This function clears the request list as sequence numbers are passed.
2578  * @engine: engine to retire requests on
2579  */
2580 void
2581 i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
2582 {
2583         WARN_ON(i915_verify_lists(engine->dev));
2584
2585         /* Retire requests first as we use it above for the early return.
2586          * If we retire requests last, we may use a later seqno and so clear
2587          * the requests lists without clearing the active list, leading to
2588          * confusion.
2589          */
2590         while (!list_empty(&engine->request_list)) {
2591                 struct drm_i915_gem_request *request;
2592
2593                 request = list_first_entry(&engine->request_list,
2594                                            struct drm_i915_gem_request,
2595                                            list);
2596
2597                 if (!i915_gem_request_completed(request))
2598                         break;
2599
2600                 i915_gem_request_retire_upto(request);
2601         }
2602
2603         /* Move any buffers on the active list that are no longer referenced
2604          * by the ringbuffer to the flushing/inactive lists as appropriate,
2605          * before we free the context associated with the requests.
2606          */
2607         while (!list_empty(&engine->active_list)) {
2608                 struct drm_i915_gem_object *obj;
2609
2610                 obj = list_first_entry(&engine->active_list,
2611                                        struct drm_i915_gem_object,
2612                                        engine_list[engine->id]);
2613
2614                 if (!list_empty(&obj->last_read_req[engine->id]->list))
2615                         break;
2616
2617                 i915_gem_object_retire__read(obj, engine->id);
2618         }
2619
2620         WARN_ON(i915_verify_lists(engine->dev));
2621 }
2622
2623 void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
2624 {
2625         struct intel_engine_cs *engine;
2626
2627         lockdep_assert_held(&dev_priv->drm.struct_mutex);
2628
2629         if (dev_priv->gt.active_engines == 0)
2630                 return;
2631
2632         GEM_BUG_ON(!dev_priv->gt.awake);
2633
2634         for_each_engine(engine, dev_priv) {
2635                 i915_gem_retire_requests_ring(engine);
2636                 if (list_empty(&engine->request_list))
2637                         dev_priv->gt.active_engines &= ~intel_engine_flag(engine);
2638         }
2639
2640         if (dev_priv->gt.active_engines == 0)
2641                 queue_delayed_work(dev_priv->wq,
2642                                    &dev_priv->gt.idle_work,
2643                                    msecs_to_jiffies(100));
2644 }
2645
2646 static void
2647 i915_gem_retire_work_handler(struct work_struct *work)
2648 {
2649         struct drm_i915_private *dev_priv =
2650                 container_of(work, typeof(*dev_priv), gt.retire_work.work);
2651         struct drm_device *dev = &dev_priv->drm;
2652
2653         /* Come back later if the device is busy... */
2654         if (mutex_trylock(&dev->struct_mutex)) {
2655                 i915_gem_retire_requests(dev_priv);
2656                 mutex_unlock(&dev->struct_mutex);
2657         }
2658
2659         /* Keep the retire handler running until we are finally idle.
2660          * We do not need to do this test under locking as in the worst-case
2661          * we queue the retire worker once too often.
2662          */
2663         if (READ_ONCE(dev_priv->gt.awake)) {
2664                 i915_queue_hangcheck(dev_priv);
2665                 queue_delayed_work(dev_priv->wq,
2666                                    &dev_priv->gt.retire_work,
2667                                    round_jiffies_up_relative(HZ));
2668         }
2669 }
2670
2671 static void
2672 i915_gem_idle_work_handler(struct work_struct *work)
2673 {
2674         struct drm_i915_private *dev_priv =
2675                 container_of(work, typeof(*dev_priv), gt.idle_work.work);
2676         struct drm_device *dev = &dev_priv->drm;
2677         struct intel_engine_cs *engine;
2678         unsigned int stuck_engines;
2679         bool rearm_hangcheck;
2680
2681         if (!READ_ONCE(dev_priv->gt.awake))
2682                 return;
2683
2684         if (READ_ONCE(dev_priv->gt.active_engines))
2685                 return;
2686
2687         rearm_hangcheck =
2688                 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
2689
2690         if (!mutex_trylock(&dev->struct_mutex)) {
2691                 /* Currently busy, come back later */
2692                 mod_delayed_work(dev_priv->wq,
2693                                  &dev_priv->gt.idle_work,
2694                                  msecs_to_jiffies(50));
2695                 goto out_rearm;
2696         }
2697
2698         if (dev_priv->gt.active_engines)
2699                 goto out_unlock;
2700
2701         for_each_engine(engine, dev_priv)
2702                 i915_gem_batch_pool_fini(&engine->batch_pool);
2703
2704         GEM_BUG_ON(!dev_priv->gt.awake);
2705         dev_priv->gt.awake = false;
2706         rearm_hangcheck = false;
2707
2708         /* As we have disabled hangcheck, we need to unstick any waiters still
2709          * hanging around. However, as we may be racing against the interrupt
2710          * handler or the waiters themselves, we skip enabling the fake-irq.
2711          */
2712         stuck_engines = intel_kick_waiters(dev_priv);
2713         if (unlikely(stuck_engines))
2714                 DRM_DEBUG_DRIVER("kicked stuck waiters (%x)...missed irq?\n",
2715                                  stuck_engines);
2716
2717         if (INTEL_GEN(dev_priv) >= 6)
2718                 gen6_rps_idle(dev_priv);
2719         intel_runtime_pm_put(dev_priv);
2720 out_unlock:
2721         mutex_unlock(&dev->struct_mutex);
2722
2723 out_rearm:
2724         if (rearm_hangcheck) {
2725                 GEM_BUG_ON(!dev_priv->gt.awake);
2726                 i915_queue_hangcheck(dev_priv);
2727         }
2728 }
2729
2730 /**
2731  * Ensures that an object will eventually get non-busy by flushing any required
2732  * write domains, emitting any outstanding lazy request and retiring and
2733  * completed requests.
2734  * @obj: object to flush
2735  */
2736 static int
2737 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2738 {
2739         int i;
2740
2741         if (!obj->active)
2742                 return 0;
2743
2744         for (i = 0; i < I915_NUM_ENGINES; i++) {
2745                 struct drm_i915_gem_request *req;
2746
2747                 req = obj->last_read_req[i];
2748                 if (req == NULL)
2749                         continue;
2750
2751                 if (i915_gem_request_completed(req))
2752                         i915_gem_object_retire__read(obj, i);
2753         }
2754
2755         return 0;
2756 }
2757
2758 /**
2759  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2760  * @dev: drm device pointer
2761  * @data: ioctl data blob
2762  * @file: drm file pointer
2763  *
2764  * Returns 0 if successful, else an error is returned with the remaining time in
2765  * the timeout parameter.
2766  *  -ETIME: object is still busy after timeout
2767  *  -ERESTARTSYS: signal interrupted the wait
2768  *  -ENONENT: object doesn't exist
2769  * Also possible, but rare:
2770  *  -EAGAIN: GPU wedged
2771  *  -ENOMEM: damn
2772  *  -ENODEV: Internal IRQ fail
2773  *  -E?: The add request failed
2774  *
2775  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2776  * non-zero timeout parameter the wait ioctl will wait for the given number of
2777  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2778  * without holding struct_mutex the object may become re-busied before this
2779  * function completes. A similar but shorter * race condition exists in the busy
2780  * ioctl
2781  */
2782 int
2783 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2784 {
2785         struct drm_i915_gem_wait *args = data;
2786         struct drm_i915_gem_object *obj;
2787         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
2788         int i, n = 0;
2789         int ret;
2790
2791         if (args->flags != 0)
2792                 return -EINVAL;
2793
2794         ret = i915_mutex_lock_interruptible(dev);
2795         if (ret)
2796                 return ret;
2797
2798         obj = i915_gem_object_lookup(file, args->bo_handle);
2799         if (!obj) {
2800                 mutex_unlock(&dev->struct_mutex);
2801                 return -ENOENT;
2802         }
2803
2804         /* Need to make sure the object gets inactive eventually. */
2805         ret = i915_gem_object_flush_active(obj);
2806         if (ret)
2807                 goto out;
2808
2809         if (!obj->active)
2810                 goto out;
2811
2812         /* Do this after OLR check to make sure we make forward progress polling
2813          * on this IOCTL with a timeout == 0 (like busy ioctl)
2814          */
2815         if (args->timeout_ns == 0) {
2816                 ret = -ETIME;
2817                 goto out;
2818         }
2819
2820         i915_gem_object_put(obj);
2821
2822         for (i = 0; i < I915_NUM_ENGINES; i++) {
2823                 if (obj->last_read_req[i] == NULL)
2824                         continue;
2825
2826                 req[n++] = i915_gem_request_get(obj->last_read_req[i]);
2827         }
2828
2829         mutex_unlock(&dev->struct_mutex);
2830
2831         for (i = 0; i < n; i++) {
2832                 if (ret == 0)
2833                         ret = __i915_wait_request(req[i], true,
2834                                                   args->timeout_ns > 0 ? &args->timeout_ns : NULL,
2835                                                   to_rps_client(file));
2836                 i915_gem_request_put(req[i]);
2837         }
2838         return ret;
2839
2840 out:
2841         i915_gem_object_put(obj);
2842         mutex_unlock(&dev->struct_mutex);
2843         return ret;
2844 }
2845
2846 static int
2847 __i915_gem_object_sync(struct drm_i915_gem_object *obj,
2848                        struct drm_i915_gem_request *to,
2849                        struct drm_i915_gem_request *from)
2850 {
2851         int ret;
2852
2853         if (to->engine == from->engine)
2854                 return 0;
2855
2856         if (i915_gem_request_completed(from))
2857                 return 0;
2858
2859         if (!i915.semaphores) {
2860                 ret = __i915_wait_request(from,
2861                                           from->i915->mm.interruptible,
2862                                           NULL,
2863                                           NO_WAITBOOST);
2864                 if (ret)
2865                         return ret;
2866
2867                 i915_gem_object_retire_request(obj, from);
2868         } else {
2869                 int idx = intel_engine_sync_index(from->engine, to->engine);
2870                 u32 seqno = i915_gem_request_get_seqno(from);
2871
2872                 if (seqno <= from->engine->semaphore.sync_seqno[idx])
2873                         return 0;
2874
2875                 trace_i915_gem_ring_sync_to(to, from);
2876                 ret = to->engine->semaphore.sync_to(to, from->engine, seqno);
2877                 if (ret)
2878                         return ret;
2879
2880                 /* We use last_read_req because sync_to()
2881                  * might have just caused seqno wrap under
2882                  * the radar.
2883                  */
2884                 from->engine->semaphore.sync_seqno[idx] =
2885                         i915_gem_request_get_seqno(obj->last_read_req[from->engine->id]);
2886         }
2887
2888         return 0;
2889 }
2890
2891 /**
2892  * i915_gem_object_sync - sync an object to a ring.
2893  *
2894  * @obj: object which may be in use on another ring.
2895  * @to: request we are wishing to use
2896  *
2897  * This code is meant to abstract object synchronization with the GPU.
2898  * Conceptually we serialise writes between engines inside the GPU.
2899  * We only allow one engine to write into a buffer at any time, but
2900  * multiple readers. To ensure each has a coherent view of memory, we must:
2901  *
2902  * - If there is an outstanding write request to the object, the new
2903  *   request must wait for it to complete (either CPU or in hw, requests
2904  *   on the same ring will be naturally ordered).
2905  *
2906  * - If we are a write request (pending_write_domain is set), the new
2907  *   request must wait for outstanding read requests to complete.
2908  *
2909  * Returns 0 if successful, else propagates up the lower layer error.
2910  */
2911 int
2912 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2913                      struct drm_i915_gem_request *to)
2914 {
2915         const bool readonly = obj->base.pending_write_domain == 0;
2916         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
2917         int ret, i, n;
2918
2919         if (!obj->active)
2920                 return 0;
2921
2922         n = 0;
2923         if (readonly) {
2924                 if (obj->last_write_req)
2925                         req[n++] = obj->last_write_req;
2926         } else {
2927                 for (i = 0; i < I915_NUM_ENGINES; i++)
2928                         if (obj->last_read_req[i])
2929                                 req[n++] = obj->last_read_req[i];
2930         }
2931         for (i = 0; i < n; i++) {
2932                 ret = __i915_gem_object_sync(obj, to, req[i]);
2933                 if (ret)
2934                         return ret;
2935         }
2936
2937         return 0;
2938 }
2939
2940 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2941 {
2942         u32 old_write_domain, old_read_domains;
2943
2944         /* Force a pagefault for domain tracking on next user access */
2945         i915_gem_release_mmap(obj);
2946
2947         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2948                 return;
2949
2950         old_read_domains = obj->base.read_domains;
2951         old_write_domain = obj->base.write_domain;
2952
2953         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2954         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2955
2956         trace_i915_gem_object_change_domain(obj,
2957                                             old_read_domains,
2958                                             old_write_domain);
2959 }
2960
2961 static void __i915_vma_iounmap(struct i915_vma *vma)
2962 {
2963         GEM_BUG_ON(vma->pin_count);
2964
2965         if (vma->iomap == NULL)
2966                 return;
2967
2968         io_mapping_unmap(vma->iomap);
2969         vma->iomap = NULL;
2970 }
2971
2972 static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
2973 {
2974         struct drm_i915_gem_object *obj = vma->obj;
2975         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2976         int ret;
2977
2978         if (list_empty(&vma->obj_link))
2979                 return 0;
2980
2981         if (!drm_mm_node_allocated(&vma->node)) {
2982                 i915_gem_vma_destroy(vma);
2983                 return 0;
2984         }
2985
2986         if (vma->pin_count)
2987                 return -EBUSY;
2988
2989         BUG_ON(obj->pages == NULL);
2990
2991         if (wait) {
2992                 ret = i915_gem_object_wait_rendering(obj, false);
2993                 if (ret)
2994                         return ret;
2995         }
2996
2997         if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
2998                 i915_gem_object_finish_gtt(obj);
2999
3000                 /* release the fence reg _after_ flushing */
3001                 ret = i915_gem_object_put_fence(obj);
3002                 if (ret)
3003                         return ret;
3004
3005                 __i915_vma_iounmap(vma);
3006         }
3007
3008         trace_i915_vma_unbind(vma);
3009
3010         vma->vm->unbind_vma(vma);
3011         vma->bound = 0;
3012
3013         list_del_init(&vma->vm_link);
3014         if (vma->is_ggtt) {
3015                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3016                         obj->map_and_fenceable = false;
3017                 } else if (vma->ggtt_view.pages) {
3018                         sg_free_table(vma->ggtt_view.pages);
3019                         kfree(vma->ggtt_view.pages);
3020                 }
3021                 vma->ggtt_view.pages = NULL;
3022         }
3023
3024         drm_mm_remove_node(&vma->node);
3025         i915_gem_vma_destroy(vma);
3026
3027         /* Since the unbound list is global, only move to that list if
3028          * no more VMAs exist. */
3029         if (list_empty(&obj->vma_list))
3030                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3031
3032         /* And finally now the object is completely decoupled from this vma,
3033          * we can drop its hold on the backing storage and allow it to be
3034          * reaped by the shrinker.
3035          */
3036         i915_gem_object_unpin_pages(obj);
3037
3038         return 0;
3039 }
3040
3041 int i915_vma_unbind(struct i915_vma *vma)
3042 {
3043         return __i915_vma_unbind(vma, true);
3044 }
3045
3046 int __i915_vma_unbind_no_wait(struct i915_vma *vma)
3047 {
3048         return __i915_vma_unbind(vma, false);
3049 }
3050
3051 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv)
3052 {
3053         struct intel_engine_cs *engine;
3054         int ret;
3055
3056         lockdep_assert_held(&dev_priv->drm.struct_mutex);
3057
3058         for_each_engine(engine, dev_priv) {
3059                 if (engine->last_context == NULL)
3060                         continue;
3061
3062                 ret = intel_engine_idle(engine);
3063                 if (ret)
3064                         return ret;
3065         }
3066
3067         WARN_ON(i915_verify_lists(dev));
3068         return 0;
3069 }
3070
3071 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3072                                      unsigned long cache_level)
3073 {
3074         struct drm_mm_node *gtt_space = &vma->node;
3075         struct drm_mm_node *other;
3076
3077         /*
3078          * On some machines we have to be careful when putting differing types
3079          * of snoopable memory together to avoid the prefetcher crossing memory
3080          * domains and dying. During vm initialisation, we decide whether or not
3081          * these constraints apply and set the drm_mm.color_adjust
3082          * appropriately.
3083          */
3084         if (vma->vm->mm.color_adjust == NULL)
3085                 return true;
3086
3087         if (!drm_mm_node_allocated(gtt_space))
3088                 return true;
3089
3090         if (list_empty(&gtt_space->node_list))
3091                 return true;
3092
3093         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3094         if (other->allocated && !other->hole_follows && other->color != cache_level)
3095                 return false;
3096
3097         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3098         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3099                 return false;
3100
3101         return true;
3102 }
3103
3104 /**
3105  * Finds free space in the GTT aperture and binds the object or a view of it
3106  * there.
3107  * @obj: object to bind
3108  * @vm: address space to bind into
3109  * @ggtt_view: global gtt view if applicable
3110  * @alignment: requested alignment
3111  * @flags: mask of PIN_* flags to use
3112  */
3113 static struct i915_vma *
3114 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3115                            struct i915_address_space *vm,
3116                            const struct i915_ggtt_view *ggtt_view,
3117                            unsigned alignment,
3118                            uint64_t flags)
3119 {
3120         struct drm_device *dev = obj->base.dev;
3121         struct drm_i915_private *dev_priv = to_i915(dev);
3122         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3123         u32 fence_alignment, unfenced_alignment;
3124         u32 search_flag, alloc_flag;
3125         u64 start, end;
3126         u64 size, fence_size;
3127         struct i915_vma *vma;
3128         int ret;
3129
3130         if (i915_is_ggtt(vm)) {
3131                 u32 view_size;
3132
3133                 if (WARN_ON(!ggtt_view))
3134                         return ERR_PTR(-EINVAL);
3135
3136                 view_size = i915_ggtt_view_size(obj, ggtt_view);
3137
3138                 fence_size = i915_gem_get_gtt_size(dev,
3139                                                    view_size,
3140                                                    obj->tiling_mode);
3141                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3142                                                              view_size,
3143                                                              obj->tiling_mode,
3144                                                              true);
3145                 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
3146                                                                 view_size,
3147                                                                 obj->tiling_mode,
3148                                                                 false);
3149                 size = flags & PIN_MAPPABLE ? fence_size : view_size;
3150         } else {
3151                 fence_size = i915_gem_get_gtt_size(dev,
3152                                                    obj->base.size,
3153                                                    obj->tiling_mode);
3154                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3155                                                              obj->base.size,
3156                                                              obj->tiling_mode,
3157                                                              true);
3158                 unfenced_alignment =
3159                         i915_gem_get_gtt_alignment(dev,
3160                                                    obj->base.size,
3161                                                    obj->tiling_mode,
3162                                                    false);
3163                 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3164         }
3165
3166         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3167         end = vm->total;
3168         if (flags & PIN_MAPPABLE)
3169                 end = min_t(u64, end, ggtt->mappable_end);
3170         if (flags & PIN_ZONE_4G)
3171                 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
3172
3173         if (alignment == 0)
3174                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3175                                                 unfenced_alignment;
3176         if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3177                 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
3178                           ggtt_view ? ggtt_view->type : 0,
3179                           alignment);
3180                 return ERR_PTR(-EINVAL);
3181         }
3182
3183         /* If binding the object/GGTT view requires more space than the entire
3184          * aperture has, reject it early before evicting everything in a vain
3185          * attempt to find space.
3186          */
3187         if (size > end) {
3188                 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
3189                           ggtt_view ? ggtt_view->type : 0,
3190                           size,
3191                           flags & PIN_MAPPABLE ? "mappable" : "total",
3192                           end);
3193                 return ERR_PTR(-E2BIG);
3194         }
3195
3196         ret = i915_gem_object_get_pages(obj);
3197         if (ret)
3198                 return ERR_PTR(ret);
3199
3200         i915_gem_object_pin_pages(obj);
3201
3202         vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
3203                           i915_gem_obj_lookup_or_create_vma(obj, vm);
3204
3205         if (IS_ERR(vma))
3206                 goto err_unpin;
3207
3208         if (flags & PIN_OFFSET_FIXED) {
3209                 uint64_t offset = flags & PIN_OFFSET_MASK;
3210
3211                 if (offset & (alignment - 1) || offset + size > end) {
3212                         ret = -EINVAL;
3213                         goto err_free_vma;
3214                 }
3215                 vma->node.start = offset;
3216                 vma->node.size = size;
3217                 vma->node.color = obj->cache_level;
3218                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3219                 if (ret) {
3220                         ret = i915_gem_evict_for_vma(vma);
3221                         if (ret == 0)
3222                                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3223                 }
3224                 if (ret)
3225                         goto err_free_vma;
3226         } else {
3227                 if (flags & PIN_HIGH) {
3228                         search_flag = DRM_MM_SEARCH_BELOW;
3229                         alloc_flag = DRM_MM_CREATE_TOP;
3230                 } else {
3231                         search_flag = DRM_MM_SEARCH_DEFAULT;
3232                         alloc_flag = DRM_MM_CREATE_DEFAULT;
3233                 }
3234
3235 search_free:
3236                 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3237                                                           size, alignment,
3238                                                           obj->cache_level,
3239                                                           start, end,
3240                                                           search_flag,
3241                                                           alloc_flag);
3242                 if (ret) {
3243                         ret = i915_gem_evict_something(dev, vm, size, alignment,
3244                                                        obj->cache_level,
3245                                                        start, end,
3246                                                        flags);
3247                         if (ret == 0)
3248                                 goto search_free;
3249
3250                         goto err_free_vma;
3251                 }
3252         }
3253         if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3254                 ret = -EINVAL;
3255                 goto err_remove_node;
3256         }
3257
3258         trace_i915_vma_bind(vma, flags);
3259         ret = i915_vma_bind(vma, obj->cache_level, flags);
3260         if (ret)
3261                 goto err_remove_node;
3262
3263         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3264         list_add_tail(&vma->vm_link, &vm->inactive_list);
3265
3266         return vma;
3267
3268 err_remove_node:
3269         drm_mm_remove_node(&vma->node);
3270 err_free_vma:
3271         i915_gem_vma_destroy(vma);
3272         vma = ERR_PTR(ret);
3273 err_unpin:
3274         i915_gem_object_unpin_pages(obj);
3275         return vma;
3276 }
3277
3278 bool
3279 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3280                         bool force)
3281 {
3282         /* If we don't have a page list set up, then we're not pinned
3283          * to GPU, and we can ignore the cache flush because it'll happen
3284          * again at bind time.
3285          */
3286         if (obj->pages == NULL)
3287                 return false;
3288
3289         /*
3290          * Stolen memory is always coherent with the GPU as it is explicitly
3291          * marked as wc by the system, or the system is cache-coherent.
3292          */
3293         if (obj->stolen || obj->phys_handle)
3294                 return false;
3295
3296         /* If the GPU is snooping the contents of the CPU cache,
3297          * we do not need to manually clear the CPU cache lines.  However,
3298          * the caches are only snooped when the render cache is
3299          * flushed/invalidated.  As we always have to emit invalidations
3300          * and flushes when moving into and out of the RENDER domain, correct
3301          * snooping behaviour occurs naturally as the result of our domain
3302          * tracking.
3303          */
3304         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3305                 obj->cache_dirty = true;
3306                 return false;
3307         }
3308
3309         trace_i915_gem_object_clflush(obj);
3310         drm_clflush_sg(obj->pages);
3311         obj->cache_dirty = false;
3312
3313         return true;
3314 }
3315
3316 /** Flushes the GTT write domain for the object if it's dirty. */
3317 static void
3318 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3319 {
3320         uint32_t old_write_domain;
3321
3322         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3323                 return;
3324
3325         /* No actual flushing is required for the GTT write domain.  Writes
3326          * to it immediately go to main memory as far as we know, so there's
3327          * no chipset flush.  It also doesn't land in render cache.
3328          *
3329          * However, we do have to enforce the order so that all writes through
3330          * the GTT land before any writes to the device, such as updates to
3331          * the GATT itself.
3332          */
3333         wmb();
3334
3335         old_write_domain = obj->base.write_domain;
3336         obj->base.write_domain = 0;
3337
3338         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3339
3340         trace_i915_gem_object_change_domain(obj,
3341                                             obj->base.read_domains,
3342                                             old_write_domain);
3343 }
3344
3345 /** Flushes the CPU write domain for the object if it's dirty. */
3346 static void
3347 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3348 {
3349         uint32_t old_write_domain;
3350
3351         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3352                 return;
3353
3354         if (i915_gem_clflush_object(obj, obj->pin_display))
3355                 i915_gem_chipset_flush(to_i915(obj->base.dev));
3356
3357         old_write_domain = obj->base.write_domain;
3358         obj->base.write_domain = 0;
3359
3360         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3361
3362         trace_i915_gem_object_change_domain(obj,
3363                                             obj->base.read_domains,
3364                                             old_write_domain);
3365 }
3366
3367 /**
3368  * Moves a single object to the GTT read, and possibly write domain.
3369  * @obj: object to act on
3370  * @write: ask for write access or read only
3371  *
3372  * This function returns when the move is complete, including waiting on
3373  * flushes to occur.
3374  */
3375 int
3376 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3377 {
3378         struct drm_device *dev = obj->base.dev;
3379         struct drm_i915_private *dev_priv = to_i915(dev);
3380         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3381         uint32_t old_write_domain, old_read_domains;
3382         struct i915_vma *vma;
3383         int ret;
3384
3385         ret = i915_gem_object_wait_rendering(obj, !write);
3386         if (ret)
3387                 return ret;
3388
3389         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3390                 return 0;
3391
3392         /* Flush and acquire obj->pages so that we are coherent through
3393          * direct access in memory with previous cached writes through
3394          * shmemfs and that our cache domain tracking remains valid.
3395          * For example, if the obj->filp was moved to swap without us
3396          * being notified and releasing the pages, we would mistakenly
3397          * continue to assume that the obj remained out of the CPU cached
3398          * domain.
3399          */
3400         ret = i915_gem_object_get_pages(obj);
3401         if (ret)
3402                 return ret;
3403
3404         i915_gem_object_flush_cpu_write_domain(obj);
3405
3406         /* Serialise direct access to this object with the barriers for
3407          * coherent writes from the GPU, by effectively invalidating the
3408          * GTT domain upon first access.
3409          */
3410         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3411                 mb();
3412
3413         old_write_domain = obj->base.write_domain;
3414         old_read_domains = obj->base.read_domains;
3415
3416         /* It should now be out of any other write domains, and we can update
3417          * the domain values for our changes.
3418          */
3419         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3420         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3421         if (write) {
3422                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3423                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3424                 obj->dirty = 1;
3425         }
3426
3427         trace_i915_gem_object_change_domain(obj,
3428                                             old_read_domains,
3429                                             old_write_domain);
3430
3431         /* And bump the LRU for this access */
3432         vma = i915_gem_obj_to_ggtt(obj);
3433         if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
3434                 list_move_tail(&vma->vm_link,
3435                                &ggtt->base.inactive_list);
3436
3437         return 0;
3438 }
3439
3440 /**
3441  * Changes the cache-level of an object across all VMA.
3442  * @obj: object to act on
3443  * @cache_level: new cache level to set for the object
3444  *
3445  * After this function returns, the object will be in the new cache-level
3446  * across all GTT and the contents of the backing storage will be coherent,
3447  * with respect to the new cache-level. In order to keep the backing storage
3448  * coherent for all users, we only allow a single cache level to be set
3449  * globally on the object and prevent it from being changed whilst the
3450  * hardware is reading from the object. That is if the object is currently
3451  * on the scanout it will be set to uncached (or equivalent display
3452  * cache coherency) and all non-MOCS GPU access will also be uncached so
3453  * that all direct access to the scanout remains coherent.
3454  */
3455 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3456                                     enum i915_cache_level cache_level)
3457 {
3458         struct drm_device *dev = obj->base.dev;
3459         struct i915_vma *vma, *next;
3460         bool bound = false;
3461         int ret = 0;
3462
3463         if (obj->cache_level == cache_level)
3464                 goto out;
3465
3466         /* Inspect the list of currently bound VMA and unbind any that would
3467          * be invalid given the new cache-level. This is principally to
3468          * catch the issue of the CS prefetch crossing page boundaries and
3469          * reading an invalid PTE on older architectures.
3470          */
3471         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
3472                 if (!drm_mm_node_allocated(&vma->node))
3473                         continue;
3474
3475                 if (vma->pin_count) {
3476                         DRM_DEBUG("can not change the cache level of pinned objects\n");
3477                         return -EBUSY;
3478                 }
3479
3480                 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
3481                         ret = i915_vma_unbind(vma);
3482                         if (ret)
3483                                 return ret;
3484                 } else
3485                         bound = true;
3486         }
3487
3488         /* We can reuse the existing drm_mm nodes but need to change the
3489          * cache-level on the PTE. We could simply unbind them all and
3490          * rebind with the correct cache-level on next use. However since
3491          * we already have a valid slot, dma mapping, pages etc, we may as
3492          * rewrite the PTE in the belief that doing so tramples upon less
3493          * state and so involves less work.
3494          */
3495         if (bound) {
3496                 /* Before we change the PTE, the GPU must not be accessing it.
3497                  * If we wait upon the object, we know that all the bound
3498                  * VMA are no longer active.
3499                  */
3500                 ret = i915_gem_object_wait_rendering(obj, false);
3501                 if (ret)
3502                         return ret;
3503
3504                 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
3505                         /* Access to snoopable pages through the GTT is
3506                          * incoherent and on some machines causes a hard
3507                          * lockup. Relinquish the CPU mmaping to force
3508                          * userspace to refault in the pages and we can
3509                          * then double check if the GTT mapping is still
3510                          * valid for that pointer access.
3511                          */
3512                         i915_gem_release_mmap(obj);
3513
3514                         /* As we no longer need a fence for GTT access,
3515                          * we can relinquish it now (and so prevent having
3516                          * to steal a fence from someone else on the next
3517                          * fence request). Note GPU activity would have
3518                          * dropped the fence as all snoopable access is
3519                          * supposed to be linear.
3520                          */
3521                         ret = i915_gem_object_put_fence(obj);
3522                         if (ret)
3523                                 return ret;
3524                 } else {
3525                         /* We either have incoherent backing store and
3526                          * so no GTT access or the architecture is fully
3527                          * coherent. In such cases, existing GTT mmaps
3528                          * ignore the cache bit in the PTE and we can
3529                          * rewrite it without confusing the GPU or having
3530                          * to force userspace to fault back in its mmaps.
3531                          */
3532                 }
3533
3534                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3535                         if (!drm_mm_node_allocated(&vma->node))
3536                                 continue;
3537
3538                         ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3539                         if (ret)
3540                                 return ret;
3541                 }
3542         }
3543
3544         list_for_each_entry(vma, &obj->vma_list, obj_link)
3545                 vma->node.color = cache_level;
3546         obj->cache_level = cache_level;
3547
3548 out:
3549         /* Flush the dirty CPU caches to the backing storage so that the
3550          * object is now coherent at its new cache level (with respect
3551          * to the access domain).
3552          */
3553         if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
3554                 if (i915_gem_clflush_object(obj, true))
3555                         i915_gem_chipset_flush(to_i915(obj->base.dev));
3556         }
3557
3558         return 0;
3559 }
3560
3561 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3562                                struct drm_file *file)
3563 {
3564         struct drm_i915_gem_caching *args = data;
3565         struct drm_i915_gem_object *obj;
3566
3567         obj = i915_gem_object_lookup(file, args->handle);
3568         if (!obj)
3569                 return -ENOENT;
3570
3571         switch (obj->cache_level) {
3572         case I915_CACHE_LLC:
3573         case I915_CACHE_L3_LLC:
3574                 args->caching = I915_CACHING_CACHED;
3575                 break;
3576
3577         case I915_CACHE_WT:
3578                 args->caching = I915_CACHING_DISPLAY;
3579                 break;
3580
3581         default:
3582                 args->caching = I915_CACHING_NONE;
3583                 break;
3584         }
3585
3586         i915_gem_object_put_unlocked(obj);
3587         return 0;
3588 }
3589
3590 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3591                                struct drm_file *file)
3592 {
3593         struct drm_i915_private *dev_priv = to_i915(dev);
3594         struct drm_i915_gem_caching *args = data;
3595         struct drm_i915_gem_object *obj;
3596         enum i915_cache_level level;
3597         int ret;
3598
3599         switch (args->caching) {
3600         case I915_CACHING_NONE:
3601                 level = I915_CACHE_NONE;
3602                 break;
3603         case I915_CACHING_CACHED:
3604                 /*
3605                  * Due to a HW issue on BXT A stepping, GPU stores via a
3606                  * snooped mapping may leave stale data in a corresponding CPU
3607                  * cacheline, whereas normally such cachelines would get
3608                  * invalidated.
3609                  */
3610                 if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
3611                         return -ENODEV;
3612
3613                 level = I915_CACHE_LLC;
3614                 break;
3615         case I915_CACHING_DISPLAY:
3616                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3617                 break;
3618         default:
3619                 return -EINVAL;
3620         }
3621
3622         intel_runtime_pm_get(dev_priv);
3623
3624         ret = i915_mutex_lock_interruptible(dev);
3625         if (ret)
3626                 goto rpm_put;
3627
3628         obj = i915_gem_object_lookup(file, args->handle);
3629         if (!obj) {
3630                 ret = -ENOENT;
3631                 goto unlock;
3632         }
3633
3634         ret = i915_gem_object_set_cache_level(obj, level);
3635
3636         i915_gem_object_put(obj);
3637 unlock:
3638         mutex_unlock(&dev->struct_mutex);
3639 rpm_put:
3640         intel_runtime_pm_put(dev_priv);
3641
3642         return ret;
3643 }
3644
3645 /*
3646  * Prepare buffer for display plane (scanout, cursors, etc).
3647  * Can be called from an uninterruptible phase (modesetting) and allows
3648  * any flushes to be pipelined (for pageflips).
3649  */
3650 int
3651 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3652                                      u32 alignment,
3653                                      const struct i915_ggtt_view *view)
3654 {
3655         u32 old_read_domains, old_write_domain;
3656         int ret;
3657
3658         /* Mark the pin_display early so that we account for the
3659          * display coherency whilst setting up the cache domains.
3660          */
3661         obj->pin_display++;
3662
3663         /* The display engine is not coherent with the LLC cache on gen6.  As
3664          * a result, we make sure that the pinning that is about to occur is
3665          * done with uncached PTEs. This is lowest common denominator for all
3666          * chipsets.
3667          *
3668          * However for gen6+, we could do better by using the GFDT bit instead
3669          * of uncaching, which would allow us to flush all the LLC-cached data
3670          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3671          */
3672         ret = i915_gem_object_set_cache_level(obj,
3673                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3674         if (ret)
3675                 goto err_unpin_display;
3676
3677         /* As the user may map the buffer once pinned in the display plane
3678          * (e.g. libkms for the bootup splash), we have to ensure that we
3679          * always use map_and_fenceable for all scanout buffers.
3680          */
3681         ret = i915_gem_object_ggtt_pin(obj, view, alignment,
3682                                        view->type == I915_GGTT_VIEW_NORMAL ?
3683                                        PIN_MAPPABLE : 0);
3684         if (ret)
3685                 goto err_unpin_display;
3686
3687         i915_gem_object_flush_cpu_write_domain(obj);
3688
3689         old_write_domain = obj->base.write_domain;
3690         old_read_domains = obj->base.read_domains;
3691
3692         /* It should now be out of any other write domains, and we can update
3693          * the domain values for our changes.
3694          */
3695         obj->base.write_domain = 0;
3696         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3697
3698         trace_i915_gem_object_change_domain(obj,
3699                                             old_read_domains,
3700                                             old_write_domain);
3701
3702         return 0;
3703
3704 err_unpin_display:
3705         obj->pin_display--;
3706         return ret;
3707 }
3708
3709 void
3710 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
3711                                          const struct i915_ggtt_view *view)
3712 {
3713         if (WARN_ON(obj->pin_display == 0))
3714                 return;
3715
3716         i915_gem_object_ggtt_unpin_view(obj, view);
3717
3718         obj->pin_display--;
3719 }
3720
3721 /**
3722  * Moves a single object to the CPU read, and possibly write domain.
3723  * @obj: object to act on
3724  * @write: requesting write or read-only access
3725  *
3726  * This function returns when the move is complete, including waiting on
3727  * flushes to occur.
3728  */
3729 int
3730 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3731 {
3732         uint32_t old_write_domain, old_read_domains;
3733         int ret;
3734
3735         ret = i915_gem_object_wait_rendering(obj, !write);
3736         if (ret)
3737                 return ret;
3738
3739         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3740                 return 0;
3741
3742         i915_gem_object_flush_gtt_write_domain(obj);
3743
3744         old_write_domain = obj->base.write_domain;
3745         old_read_domains = obj->base.read_domains;
3746
3747         /* Flush the CPU cache if it's still invalid. */
3748         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3749                 i915_gem_clflush_object(obj, false);
3750
3751                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3752         }
3753
3754         /* It should now be out of any other write domains, and we can update
3755          * the domain values for our changes.
3756          */
3757         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3758
3759         /* If we're writing through the CPU, then the GPU read domains will
3760          * need to be invalidated at next use.
3761          */
3762         if (write) {
3763                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3764                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3765         }
3766
3767         trace_i915_gem_object_change_domain(obj,
3768                                             old_read_domains,
3769                                             old_write_domain);
3770
3771         return 0;
3772 }
3773
3774 /* Throttle our rendering by waiting until the ring has completed our requests
3775  * emitted over 20 msec ago.
3776  *
3777  * Note that if we were to use the current jiffies each time around the loop,
3778  * we wouldn't escape the function with any frames outstanding if the time to
3779  * render a frame was over 20ms.
3780  *
3781  * This should get us reasonable parallelism between CPU and GPU but also
3782  * relatively low latency when blocking on a particular request to finish.
3783  */
3784 static int
3785 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3786 {
3787         struct drm_i915_private *dev_priv = to_i915(dev);
3788         struct drm_i915_file_private *file_priv = file->driver_priv;
3789         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3790         struct drm_i915_gem_request *request, *target = NULL;
3791         int ret;
3792
3793         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3794         if (ret)
3795                 return ret;
3796
3797         /* ABI: return -EIO if already wedged */
3798         if (i915_terminally_wedged(&dev_priv->gpu_error))
3799                 return -EIO;
3800
3801         spin_lock(&file_priv->mm.lock);
3802         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3803                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3804                         break;
3805
3806                 /*
3807                  * Note that the request might not have been submitted yet.
3808                  * In which case emitted_jiffies will be zero.
3809                  */
3810                 if (!request->emitted_jiffies)
3811                         continue;
3812
3813                 target = request;
3814         }
3815         if (target)
3816                 i915_gem_request_get(target);
3817         spin_unlock(&file_priv->mm.lock);
3818
3819         if (target == NULL)
3820                 return 0;
3821
3822         ret = __i915_wait_request(target, true, NULL, NULL);
3823         i915_gem_request_put(target);
3824
3825         return ret;
3826 }
3827
3828 static bool
3829 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
3830 {
3831         struct drm_i915_gem_object *obj = vma->obj;
3832
3833         if (alignment &&
3834             vma->node.start & (alignment - 1))
3835                 return true;
3836
3837         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
3838                 return true;
3839
3840         if (flags & PIN_OFFSET_BIAS &&
3841             vma->node.start < (flags & PIN_OFFSET_MASK))
3842                 return true;
3843
3844         if (flags & PIN_OFFSET_FIXED &&
3845             vma->node.start != (flags & PIN_OFFSET_MASK))
3846                 return true;
3847
3848         return false;
3849 }
3850
3851 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
3852 {
3853         struct drm_i915_gem_object *obj = vma->obj;
3854         bool mappable, fenceable;
3855         u32 fence_size, fence_alignment;
3856
3857         fence_size = i915_gem_get_gtt_size(obj->base.dev,
3858                                            obj->base.size,
3859                                            obj->tiling_mode);
3860         fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
3861                                                      obj->base.size,
3862                                                      obj->tiling_mode,
3863                                                      true);
3864
3865         fenceable = (vma->node.size == fence_size &&
3866                      (vma->node.start & (fence_alignment - 1)) == 0);
3867
3868         mappable = (vma->node.start + fence_size <=
3869                     to_i915(obj->base.dev)->ggtt.mappable_end);
3870
3871         obj->map_and_fenceable = mappable && fenceable;
3872 }
3873
3874 static int
3875 i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
3876                        struct i915_address_space *vm,
3877                        const struct i915_ggtt_view *ggtt_view,
3878                        uint32_t alignment,
3879                        uint64_t flags)
3880 {
3881         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3882         struct i915_vma *vma;
3883         unsigned bound;
3884         int ret;
3885
3886         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
3887                 return -ENODEV;
3888
3889         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
3890                 return -EINVAL;
3891
3892         if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
3893                 return -EINVAL;
3894
3895         if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
3896                 return -EINVAL;
3897
3898         vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
3899                           i915_gem_obj_to_vma(obj, vm);
3900
3901         if (vma) {
3902                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3903                         return -EBUSY;
3904
3905                 if (i915_vma_misplaced(vma, alignment, flags)) {
3906                         WARN(vma->pin_count,
3907                              "bo is already pinned in %s with incorrect alignment:"
3908                              " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
3909                              " obj->map_and_fenceable=%d\n",
3910                              ggtt_view ? "ggtt" : "ppgtt",
3911                              upper_32_bits(vma->node.start),
3912                              lower_32_bits(vma->node.start),
3913                              alignment,
3914                              !!(flags & PIN_MAPPABLE),
3915                              obj->map_and_fenceable);
3916                         ret = i915_vma_unbind(vma);
3917                         if (ret)
3918                                 return ret;
3919
3920                         vma = NULL;
3921                 }
3922         }
3923
3924         bound = vma ? vma->bound : 0;
3925         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
3926                 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
3927                                                  flags);
3928                 if (IS_ERR(vma))
3929                         return PTR_ERR(vma);
3930         } else {
3931                 ret = i915_vma_bind(vma, obj->cache_level, flags);
3932                 if (ret)
3933                         return ret;
3934         }
3935
3936         if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
3937             (bound ^ vma->bound) & GLOBAL_BIND) {
3938                 __i915_vma_set_map_and_fenceable(vma);
3939                 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
3940         }
3941
3942         vma->pin_count++;
3943         return 0;
3944 }
3945
3946 int
3947 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3948                     struct i915_address_space *vm,
3949                     uint32_t alignment,
3950                     uint64_t flags)
3951 {
3952         return i915_gem_object_do_pin(obj, vm,
3953                                       i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
3954                                       alignment, flags);
3955 }
3956
3957 int
3958 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3959                          const struct i915_ggtt_view *view,
3960                          uint32_t alignment,
3961                          uint64_t flags)
3962 {
3963         struct drm_device *dev = obj->base.dev;
3964         struct drm_i915_private *dev_priv = to_i915(dev);
3965         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3966
3967         BUG_ON(!view);
3968
3969         return i915_gem_object_do_pin(obj, &ggtt->base, view,
3970                                       alignment, flags | PIN_GLOBAL);
3971 }
3972
3973 void
3974 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
3975                                 const struct i915_ggtt_view *view)
3976 {
3977         struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
3978
3979         WARN_ON(vma->pin_count == 0);
3980         WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
3981
3982         --vma->pin_count;
3983 }
3984
3985 int
3986 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3987                     struct drm_file *file)
3988 {
3989         struct drm_i915_gem_busy *args = data;
3990         struct drm_i915_gem_object *obj;
3991         int ret;
3992
3993         ret = i915_mutex_lock_interruptible(dev);
3994         if (ret)
3995                 return ret;
3996
3997         obj = i915_gem_object_lookup(file, args->handle);
3998         if (!obj) {
3999                 ret = -ENOENT;
4000                 goto unlock;
4001         }
4002
4003         /* Count all active objects as busy, even if they are currently not used
4004          * by the gpu. Users of this interface expect objects to eventually
4005          * become non-busy without any further actions, therefore emit any
4006          * necessary flushes here.
4007          */
4008         ret = i915_gem_object_flush_active(obj);
4009         if (ret)
4010                 goto unref;
4011
4012         args->busy = 0;
4013         if (obj->active) {
4014                 int i;
4015
4016                 for (i = 0; i < I915_NUM_ENGINES; i++) {
4017                         struct drm_i915_gem_request *req;
4018
4019                         req = obj->last_read_req[i];
4020                         if (req)
4021                                 args->busy |= 1 << (16 + req->engine->exec_id);
4022                 }
4023                 if (obj->last_write_req)
4024                         args->busy |= obj->last_write_req->engine->exec_id;
4025         }
4026
4027 unref:
4028         i915_gem_object_put(obj);
4029 unlock:
4030         mutex_unlock(&dev->struct_mutex);
4031         return ret;
4032 }
4033
4034 int
4035 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4036                         struct drm_file *file_priv)
4037 {
4038         return i915_gem_ring_throttle(dev, file_priv);
4039 }
4040
4041 int
4042 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4043                        struct drm_file *file_priv)
4044 {
4045         struct drm_i915_private *dev_priv = to_i915(dev);
4046         struct drm_i915_gem_madvise *args = data;
4047         struct drm_i915_gem_object *obj;
4048         int ret;
4049
4050         switch (args->madv) {
4051         case I915_MADV_DONTNEED:
4052         case I915_MADV_WILLNEED:
4053             break;
4054         default:
4055             return -EINVAL;
4056         }
4057
4058         ret = i915_mutex_lock_interruptible(dev);
4059         if (ret)
4060                 return ret;
4061
4062         obj = i915_gem_object_lookup(file_priv, args->handle);
4063         if (!obj) {
4064                 ret = -ENOENT;
4065                 goto unlock;
4066         }
4067
4068         if (i915_gem_obj_is_pinned(obj)) {
4069                 ret = -EINVAL;
4070                 goto out;
4071         }
4072
4073         if (obj->pages &&
4074             obj->tiling_mode != I915_TILING_NONE &&
4075             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4076                 if (obj->madv == I915_MADV_WILLNEED)
4077                         i915_gem_object_unpin_pages(obj);
4078                 if (args->madv == I915_MADV_WILLNEED)
4079                         i915_gem_object_pin_pages(obj);
4080         }
4081
4082         if (obj->madv != __I915_MADV_PURGED)
4083                 obj->madv = args->madv;
4084
4085         /* if the object is no longer attached, discard its backing storage */
4086         if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4087                 i915_gem_object_truncate(obj);
4088
4089         args->retained = obj->madv != __I915_MADV_PURGED;
4090
4091 out:
4092         i915_gem_object_put(obj);
4093 unlock:
4094         mutex_unlock(&dev->struct_mutex);
4095         return ret;
4096 }
4097
4098 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4099                           const struct drm_i915_gem_object_ops *ops)
4100 {
4101         int i;
4102
4103         INIT_LIST_HEAD(&obj->global_list);
4104         for (i = 0; i < I915_NUM_ENGINES; i++)
4105                 INIT_LIST_HEAD(&obj->engine_list[i]);
4106         INIT_LIST_HEAD(&obj->obj_exec_link);
4107         INIT_LIST_HEAD(&obj->vma_list);
4108         INIT_LIST_HEAD(&obj->batch_pool_link);
4109
4110         obj->ops = ops;
4111
4112         obj->fence_reg = I915_FENCE_REG_NONE;
4113         obj->madv = I915_MADV_WILLNEED;
4114
4115         i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
4116 }
4117
4118 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4119         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
4120         .get_pages = i915_gem_object_get_pages_gtt,
4121         .put_pages = i915_gem_object_put_pages_gtt,
4122 };
4123
4124 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
4125                                                   size_t size)
4126 {
4127         struct drm_i915_gem_object *obj;
4128         struct address_space *mapping;
4129         gfp_t mask;
4130         int ret;
4131
4132         obj = i915_gem_object_alloc(dev);
4133         if (obj == NULL)
4134                 return ERR_PTR(-ENOMEM);
4135
4136         ret = drm_gem_object_init(dev, &obj->base, size);
4137         if (ret)
4138                 goto fail;
4139
4140         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4141         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4142                 /* 965gm cannot relocate objects above 4GiB. */
4143                 mask &= ~__GFP_HIGHMEM;
4144                 mask |= __GFP_DMA32;
4145         }
4146
4147         mapping = file_inode(obj->base.filp)->i_mapping;
4148         mapping_set_gfp_mask(mapping, mask);
4149
4150         i915_gem_object_init(obj, &i915_gem_object_ops);
4151
4152         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4153         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4154
4155         if (HAS_LLC(dev)) {
4156                 /* On some devices, we can have the GPU use the LLC (the CPU
4157                  * cache) for about a 10% performance improvement
4158                  * compared to uncached.  Graphics requests other than
4159                  * display scanout are coherent with the CPU in
4160                  * accessing this cache.  This means in this mode we
4161                  * don't need to clflush on the CPU side, and on the
4162                  * GPU side we only need to flush internal caches to
4163                  * get data visible to the CPU.
4164                  *
4165                  * However, we maintain the display planes as UC, and so
4166                  * need to rebind when first used as such.
4167                  */
4168                 obj->cache_level = I915_CACHE_LLC;
4169         } else
4170                 obj->cache_level = I915_CACHE_NONE;
4171
4172         trace_i915_gem_object_create(obj);
4173
4174         return obj;
4175
4176 fail:
4177         i915_gem_object_free(obj);
4178
4179         return ERR_PTR(ret);
4180 }
4181
4182 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4183 {
4184         /* If we are the last user of the backing storage (be it shmemfs
4185          * pages or stolen etc), we know that the pages are going to be
4186          * immediately released. In this case, we can then skip copying
4187          * back the contents from the GPU.
4188          */
4189
4190         if (obj->madv != I915_MADV_WILLNEED)
4191                 return false;
4192
4193         if (obj->base.filp == NULL)
4194                 return true;
4195
4196         /* At first glance, this looks racy, but then again so would be
4197          * userspace racing mmap against close. However, the first external
4198          * reference to the filp can only be obtained through the
4199          * i915_gem_mmap_ioctl() which safeguards us against the user
4200          * acquiring such a reference whilst we are in the middle of
4201          * freeing the object.
4202          */
4203         return atomic_long_read(&obj->base.filp->f_count) == 1;
4204 }
4205
4206 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4207 {
4208         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4209         struct drm_device *dev = obj->base.dev;
4210         struct drm_i915_private *dev_priv = to_i915(dev);
4211         struct i915_vma *vma, *next;
4212
4213         intel_runtime_pm_get(dev_priv);
4214
4215         trace_i915_gem_object_destroy(obj);
4216
4217         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4218                 int ret;
4219
4220                 vma->pin_count = 0;
4221                 ret = __i915_vma_unbind_no_wait(vma);
4222                 if (WARN_ON(ret == -ERESTARTSYS)) {
4223                         bool was_interruptible;
4224
4225                         was_interruptible = dev_priv->mm.interruptible;
4226                         dev_priv->mm.interruptible = false;
4227
4228                         WARN_ON(i915_vma_unbind(vma));
4229
4230                         dev_priv->mm.interruptible = was_interruptible;
4231                 }
4232         }
4233
4234         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4235          * before progressing. */
4236         if (obj->stolen)
4237                 i915_gem_object_unpin_pages(obj);
4238
4239         WARN_ON(obj->frontbuffer_bits);
4240
4241         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4242             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4243             obj->tiling_mode != I915_TILING_NONE)
4244                 i915_gem_object_unpin_pages(obj);
4245
4246         if (WARN_ON(obj->pages_pin_count))
4247                 obj->pages_pin_count = 0;
4248         if (discard_backing_storage(obj))
4249                 obj->madv = I915_MADV_DONTNEED;
4250         i915_gem_object_put_pages(obj);
4251
4252         BUG_ON(obj->pages);
4253
4254         if (obj->base.import_attach)
4255                 drm_prime_gem_destroy(&obj->base, NULL);
4256
4257         if (obj->ops->release)
4258                 obj->ops->release(obj);
4259
4260         drm_gem_object_release(&obj->base);
4261         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4262
4263         kfree(obj->bit_17);
4264         i915_gem_object_free(obj);
4265
4266         intel_runtime_pm_put(dev_priv);
4267 }
4268
4269 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4270                                      struct i915_address_space *vm)
4271 {
4272         struct i915_vma *vma;
4273         list_for_each_entry(vma, &obj->vma_list, obj_link) {
4274                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4275                     vma->vm == vm)
4276                         return vma;
4277         }
4278         return NULL;
4279 }
4280
4281 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4282                                            const struct i915_ggtt_view *view)
4283 {
4284         struct i915_vma *vma;
4285
4286         GEM_BUG_ON(!view);
4287
4288         list_for_each_entry(vma, &obj->vma_list, obj_link)
4289                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
4290                         return vma;
4291         return NULL;
4292 }
4293
4294 void i915_gem_vma_destroy(struct i915_vma *vma)
4295 {
4296         WARN_ON(vma->node.allocated);
4297
4298         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4299         if (!list_empty(&vma->exec_list))
4300                 return;
4301
4302         if (!vma->is_ggtt)
4303                 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
4304
4305         list_del(&vma->obj_link);
4306
4307         kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
4308 }
4309
4310 static void
4311 i915_gem_stop_engines(struct drm_device *dev)
4312 {
4313         struct drm_i915_private *dev_priv = to_i915(dev);
4314         struct intel_engine_cs *engine;
4315
4316         for_each_engine(engine, dev_priv)
4317                 dev_priv->gt.stop_engine(engine);
4318 }
4319
4320 int
4321 i915_gem_suspend(struct drm_device *dev)
4322 {
4323         struct drm_i915_private *dev_priv = to_i915(dev);
4324         int ret = 0;
4325
4326         intel_suspend_gt_powersave(dev_priv);
4327
4328         mutex_lock(&dev->struct_mutex);
4329
4330         /* We have to flush all the executing contexts to main memory so
4331          * that they can saved in the hibernation image. To ensure the last
4332          * context image is coherent, we have to switch away from it. That
4333          * leaves the dev_priv->kernel_context still active when
4334          * we actually suspend, and its image in memory may not match the GPU
4335          * state. Fortunately, the kernel_context is disposable and we do
4336          * not rely on its state.
4337          */
4338         ret = i915_gem_switch_to_kernel_context(dev_priv);
4339         if (ret)
4340                 goto err;
4341
4342         ret = i915_gem_wait_for_idle(dev_priv);
4343         if (ret)
4344                 goto err;
4345
4346         i915_gem_retire_requests(dev_priv);
4347
4348         /* Note that rather than stopping the engines, all we have to do
4349          * is assert that every RING_HEAD == RING_TAIL (all execution complete)
4350          * and similar for all logical context images (to ensure they are
4351          * all ready for hibernation).
4352          */
4353         i915_gem_stop_engines(dev);
4354         i915_gem_context_lost(dev_priv);
4355         mutex_unlock(&dev->struct_mutex);
4356
4357         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4358         cancel_delayed_work_sync(&dev_priv->gt.retire_work);
4359         flush_delayed_work(&dev_priv->gt.idle_work);
4360
4361         /* Assert that we sucessfully flushed all the work and
4362          * reset the GPU back to its idle, low power state.
4363          */
4364         WARN_ON(dev_priv->gt.awake);
4365
4366         return 0;
4367
4368 err:
4369         mutex_unlock(&dev->struct_mutex);
4370         return ret;
4371 }
4372
4373 void i915_gem_resume(struct drm_device *dev)
4374 {
4375         struct drm_i915_private *dev_priv = to_i915(dev);
4376
4377         mutex_lock(&dev->struct_mutex);
4378         i915_gem_restore_gtt_mappings(dev);
4379
4380         /* As we didn't flush the kernel context before suspend, we cannot
4381          * guarantee that the context image is complete. So let's just reset
4382          * it and start again.
4383          */
4384         if (i915.enable_execlists)
4385                 intel_lr_context_reset(dev_priv, dev_priv->kernel_context);
4386
4387         mutex_unlock(&dev->struct_mutex);
4388 }
4389
4390 void i915_gem_init_swizzling(struct drm_device *dev)
4391 {
4392         struct drm_i915_private *dev_priv = to_i915(dev);
4393
4394         if (INTEL_INFO(dev)->gen < 5 ||
4395             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4396                 return;
4397
4398         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4399                                  DISP_TILE_SURFACE_SWIZZLING);
4400
4401         if (IS_GEN5(dev))
4402                 return;
4403
4404         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4405         if (IS_GEN6(dev))
4406                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4407         else if (IS_GEN7(dev))
4408                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4409         else if (IS_GEN8(dev))
4410                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4411         else
4412                 BUG();
4413 }
4414
4415 static void init_unused_ring(struct drm_device *dev, u32 base)
4416 {
4417         struct drm_i915_private *dev_priv = to_i915(dev);
4418
4419         I915_WRITE(RING_CTL(base), 0);
4420         I915_WRITE(RING_HEAD(base), 0);
4421         I915_WRITE(RING_TAIL(base), 0);
4422         I915_WRITE(RING_START(base), 0);
4423 }
4424
4425 static void init_unused_rings(struct drm_device *dev)
4426 {
4427         if (IS_I830(dev)) {
4428                 init_unused_ring(dev, PRB1_BASE);
4429                 init_unused_ring(dev, SRB0_BASE);
4430                 init_unused_ring(dev, SRB1_BASE);
4431                 init_unused_ring(dev, SRB2_BASE);
4432                 init_unused_ring(dev, SRB3_BASE);
4433         } else if (IS_GEN2(dev)) {
4434                 init_unused_ring(dev, SRB0_BASE);
4435                 init_unused_ring(dev, SRB1_BASE);
4436         } else if (IS_GEN3(dev)) {
4437                 init_unused_ring(dev, PRB1_BASE);
4438                 init_unused_ring(dev, PRB2_BASE);
4439         }
4440 }
4441
4442 int
4443 i915_gem_init_hw(struct drm_device *dev)
4444 {
4445         struct drm_i915_private *dev_priv = to_i915(dev);
4446         struct intel_engine_cs *engine;
4447         int ret;
4448
4449         /* Double layer security blanket, see i915_gem_init() */
4450         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4451
4452         if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
4453                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4454
4455         if (IS_HASWELL(dev))
4456                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4457                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4458
4459         if (HAS_PCH_NOP(dev)) {
4460                 if (IS_IVYBRIDGE(dev)) {
4461                         u32 temp = I915_READ(GEN7_MSG_CTL);
4462                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4463                         I915_WRITE(GEN7_MSG_CTL, temp);
4464                 } else if (INTEL_INFO(dev)->gen >= 7) {
4465                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4466                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4467                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4468                 }
4469         }
4470
4471         i915_gem_init_swizzling(dev);
4472
4473         /*
4474          * At least 830 can leave some of the unused rings
4475          * "active" (ie. head != tail) after resume which
4476          * will prevent c3 entry. Makes sure all unused rings
4477          * are totally idle.
4478          */
4479         init_unused_rings(dev);
4480
4481         BUG_ON(!dev_priv->kernel_context);
4482
4483         ret = i915_ppgtt_init_hw(dev);
4484         if (ret) {
4485                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4486                 goto out;
4487         }
4488
4489         /* Need to do basic initialisation of all rings first: */
4490         for_each_engine(engine, dev_priv) {
4491                 ret = engine->init_hw(engine);
4492                 if (ret)
4493                         goto out;
4494         }
4495
4496         intel_mocs_init_l3cc_table(dev);
4497
4498         /* We can't enable contexts until all firmware is loaded */
4499         ret = intel_guc_setup(dev);
4500         if (ret)
4501                 goto out;
4502
4503 out:
4504         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4505         return ret;
4506 }
4507
4508 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
4509 {
4510         if (INTEL_INFO(dev_priv)->gen < 6)
4511                 return false;
4512
4513         /* TODO: make semaphores and Execlists play nicely together */
4514         if (i915.enable_execlists)
4515                 return false;
4516
4517         if (value >= 0)
4518                 return value;
4519
4520 #ifdef CONFIG_INTEL_IOMMU
4521         /* Enable semaphores on SNB when IO remapping is off */
4522         if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
4523                 return false;
4524 #endif
4525
4526         return true;
4527 }
4528
4529 int i915_gem_init(struct drm_device *dev)
4530 {
4531         struct drm_i915_private *dev_priv = to_i915(dev);
4532         int ret;
4533
4534         mutex_lock(&dev->struct_mutex);
4535
4536         if (!i915.enable_execlists) {
4537                 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
4538                 dev_priv->gt.cleanup_engine = intel_engine_cleanup;
4539                 dev_priv->gt.stop_engine = intel_engine_stop;
4540         } else {
4541                 dev_priv->gt.execbuf_submit = intel_execlists_submission;
4542                 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
4543                 dev_priv->gt.stop_engine = intel_logical_ring_stop;
4544         }
4545
4546         /* This is just a security blanket to placate dragons.
4547          * On some systems, we very sporadically observe that the first TLBs
4548          * used by the CS may be stale, despite us poking the TLB reset. If
4549          * we hold the forcewake during initialisation these problems
4550          * just magically go away.
4551          */
4552         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4553
4554         i915_gem_init_userptr(dev_priv);
4555         i915_gem_init_ggtt(dev);
4556
4557         ret = i915_gem_context_init(dev);
4558         if (ret)
4559                 goto out_unlock;
4560
4561         ret = intel_engines_init(dev);
4562         if (ret)
4563                 goto out_unlock;
4564
4565         ret = i915_gem_init_hw(dev);
4566         if (ret == -EIO) {
4567                 /* Allow engine initialisation to fail by marking the GPU as
4568                  * wedged. But we only want to do this where the GPU is angry,
4569                  * for all other failure, such as an allocation failure, bail.
4570                  */
4571                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4572                 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4573                 ret = 0;
4574         }
4575
4576 out_unlock:
4577         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4578         mutex_unlock(&dev->struct_mutex);
4579
4580         return ret;
4581 }
4582
4583 void
4584 i915_gem_cleanup_engines(struct drm_device *dev)
4585 {
4586         struct drm_i915_private *dev_priv = to_i915(dev);
4587         struct intel_engine_cs *engine;
4588
4589         for_each_engine(engine, dev_priv)
4590                 dev_priv->gt.cleanup_engine(engine);
4591 }
4592
4593 static void
4594 init_engine_lists(struct intel_engine_cs *engine)
4595 {
4596         INIT_LIST_HEAD(&engine->active_list);
4597         INIT_LIST_HEAD(&engine->request_list);
4598 }
4599
4600 void
4601 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4602 {
4603         struct drm_device *dev = &dev_priv->drm;
4604
4605         if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
4606             !IS_CHERRYVIEW(dev_priv))
4607                 dev_priv->num_fence_regs = 32;
4608         else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
4609                  IS_I945GM(dev_priv) || IS_G33(dev_priv))
4610                 dev_priv->num_fence_regs = 16;
4611         else
4612                 dev_priv->num_fence_regs = 8;
4613
4614         if (intel_vgpu_active(dev_priv))
4615                 dev_priv->num_fence_regs =
4616                                 I915_READ(vgtif_reg(avail_rs.fence_num));
4617
4618         /* Initialize fence registers to zero */
4619         i915_gem_restore_fences(dev);
4620
4621         i915_gem_detect_bit_6_swizzle(dev);
4622 }
4623
4624 void
4625 i915_gem_load_init(struct drm_device *dev)
4626 {
4627         struct drm_i915_private *dev_priv = to_i915(dev);
4628         int i;
4629
4630         dev_priv->objects =
4631                 kmem_cache_create("i915_gem_object",
4632                                   sizeof(struct drm_i915_gem_object), 0,
4633                                   SLAB_HWCACHE_ALIGN,
4634                                   NULL);
4635         dev_priv->vmas =
4636                 kmem_cache_create("i915_gem_vma",
4637                                   sizeof(struct i915_vma), 0,
4638                                   SLAB_HWCACHE_ALIGN,
4639                                   NULL);
4640         dev_priv->requests =
4641                 kmem_cache_create("i915_gem_request",
4642                                   sizeof(struct drm_i915_gem_request), 0,
4643                                   SLAB_HWCACHE_ALIGN,
4644                                   NULL);
4645
4646         INIT_LIST_HEAD(&dev_priv->vm_list);
4647         INIT_LIST_HEAD(&dev_priv->context_list);
4648         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4649         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4650         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4651         for (i = 0; i < I915_NUM_ENGINES; i++)
4652                 init_engine_lists(&dev_priv->engine[i]);
4653         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4654                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4655         INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
4656                           i915_gem_retire_work_handler);
4657         INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
4658                           i915_gem_idle_work_handler);
4659         init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
4660         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4661
4662         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4663
4664         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4665
4666         init_waitqueue_head(&dev_priv->pending_flip_queue);
4667
4668         dev_priv->mm.interruptible = true;
4669
4670         mutex_init(&dev_priv->fb_tracking.lock);
4671 }
4672
4673 void i915_gem_load_cleanup(struct drm_device *dev)
4674 {
4675         struct drm_i915_private *dev_priv = to_i915(dev);
4676
4677         kmem_cache_destroy(dev_priv->requests);
4678         kmem_cache_destroy(dev_priv->vmas);
4679         kmem_cache_destroy(dev_priv->objects);
4680 }
4681
4682 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
4683 {
4684         struct drm_i915_gem_object *obj;
4685
4686         /* Called just before we write the hibernation image.
4687          *
4688          * We need to update the domain tracking to reflect that the CPU
4689          * will be accessing all the pages to create and restore from the
4690          * hibernation, and so upon restoration those pages will be in the
4691          * CPU domain.
4692          *
4693          * To make sure the hibernation image contains the latest state,
4694          * we update that state just before writing out the image.
4695          */
4696
4697         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
4698                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4699                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4700         }
4701
4702         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
4703                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4704                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4705         }
4706
4707         return 0;
4708 }
4709
4710 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4711 {
4712         struct drm_i915_file_private *file_priv = file->driver_priv;
4713         struct drm_i915_gem_request *request;
4714
4715         /* Clean up our request list when the client is going away, so that
4716          * later retire_requests won't dereference our soon-to-be-gone
4717          * file_priv.
4718          */
4719         spin_lock(&file_priv->mm.lock);
4720         list_for_each_entry(request, &file_priv->mm.request_list, client_list)
4721                 request->file_priv = NULL;
4722         spin_unlock(&file_priv->mm.lock);
4723
4724         if (!list_empty(&file_priv->rps.link)) {
4725                 spin_lock(&to_i915(dev)->rps.client_lock);
4726                 list_del(&file_priv->rps.link);
4727                 spin_unlock(&to_i915(dev)->rps.client_lock);
4728         }
4729 }
4730
4731 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4732 {
4733         struct drm_i915_file_private *file_priv;
4734         int ret;
4735
4736         DRM_DEBUG_DRIVER("\n");
4737
4738         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4739         if (!file_priv)
4740                 return -ENOMEM;
4741
4742         file->driver_priv = file_priv;
4743         file_priv->dev_priv = to_i915(dev);
4744         file_priv->file = file;
4745         INIT_LIST_HEAD(&file_priv->rps.link);
4746
4747         spin_lock_init(&file_priv->mm.lock);
4748         INIT_LIST_HEAD(&file_priv->mm.request_list);
4749
4750         file_priv->bsd_engine = -1;
4751
4752         ret = i915_gem_context_open(dev, file);
4753         if (ret)
4754                 kfree(file_priv);
4755
4756         return ret;
4757 }
4758
4759 /**
4760  * i915_gem_track_fb - update frontbuffer tracking
4761  * @old: current GEM buffer for the frontbuffer slots
4762  * @new: new GEM buffer for the frontbuffer slots
4763  * @frontbuffer_bits: bitmask of frontbuffer slots
4764  *
4765  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
4766  * from @old and setting them in @new. Both @old and @new can be NULL.
4767  */
4768 void i915_gem_track_fb(struct drm_i915_gem_object *old,
4769                        struct drm_i915_gem_object *new,
4770                        unsigned frontbuffer_bits)
4771 {
4772         if (old) {
4773                 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
4774                 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
4775                 old->frontbuffer_bits &= ~frontbuffer_bits;
4776         }
4777
4778         if (new) {
4779                 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
4780                 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
4781                 new->frontbuffer_bits |= frontbuffer_bits;
4782         }
4783 }
4784
4785 /* All the new VM stuff */
4786 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
4787                         struct i915_address_space *vm)
4788 {
4789         struct drm_i915_private *dev_priv = to_i915(o->base.dev);
4790         struct i915_vma *vma;
4791
4792         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4793
4794         list_for_each_entry(vma, &o->vma_list, obj_link) {
4795                 if (vma->is_ggtt &&
4796                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4797                         continue;
4798                 if (vma->vm == vm)
4799                         return vma->node.start;
4800         }
4801
4802         WARN(1, "%s vma for this object not found.\n",
4803              i915_is_ggtt(vm) ? "global" : "ppgtt");
4804         return -1;
4805 }
4806
4807 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
4808                                   const struct i915_ggtt_view *view)
4809 {
4810         struct i915_vma *vma;
4811
4812         list_for_each_entry(vma, &o->vma_list, obj_link)
4813                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
4814                         return vma->node.start;
4815
4816         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
4817         return -1;
4818 }
4819
4820 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
4821                         struct i915_address_space *vm)
4822 {
4823         struct i915_vma *vma;
4824
4825         list_for_each_entry(vma, &o->vma_list, obj_link) {
4826                 if (vma->is_ggtt &&
4827                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4828                         continue;
4829                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
4830                         return true;
4831         }
4832
4833         return false;
4834 }
4835
4836 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
4837                                   const struct i915_ggtt_view *view)
4838 {
4839         struct i915_vma *vma;
4840
4841         list_for_each_entry(vma, &o->vma_list, obj_link)
4842                 if (vma->is_ggtt &&
4843                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
4844                     drm_mm_node_allocated(&vma->node))
4845                         return true;
4846
4847         return false;
4848 }
4849
4850 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
4851 {
4852         struct i915_vma *vma;
4853
4854         list_for_each_entry(vma, &o->vma_list, obj_link)
4855                 if (drm_mm_node_allocated(&vma->node))
4856                         return true;
4857
4858         return false;
4859 }
4860
4861 unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o)
4862 {
4863         struct i915_vma *vma;
4864
4865         GEM_BUG_ON(list_empty(&o->vma_list));
4866
4867         list_for_each_entry(vma, &o->vma_list, obj_link) {
4868                 if (vma->is_ggtt &&
4869                     vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
4870                         return vma->node.size;
4871         }
4872
4873         return 0;
4874 }
4875
4876 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
4877 {
4878         struct i915_vma *vma;
4879         list_for_each_entry(vma, &obj->vma_list, obj_link)
4880                 if (vma->pin_count > 0)
4881                         return true;
4882
4883         return false;
4884 }
4885
4886 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
4887 struct page *
4888 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
4889 {
4890         struct page *page;
4891
4892         /* Only default objects have per-page dirty tracking */
4893         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
4894                 return NULL;
4895
4896         page = i915_gem_object_get_page(obj, n);
4897         set_page_dirty(page);
4898         return page;
4899 }
4900
4901 /* Allocate a new GEM object and fill it with the supplied data */
4902 struct drm_i915_gem_object *
4903 i915_gem_object_create_from_data(struct drm_device *dev,
4904                                  const void *data, size_t size)
4905 {
4906         struct drm_i915_gem_object *obj;
4907         struct sg_table *sg;
4908         size_t bytes;
4909         int ret;
4910
4911         obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
4912         if (IS_ERR(obj))
4913                 return obj;
4914
4915         ret = i915_gem_object_set_to_cpu_domain(obj, true);
4916         if (ret)
4917                 goto fail;
4918
4919         ret = i915_gem_object_get_pages(obj);
4920         if (ret)
4921                 goto fail;
4922
4923         i915_gem_object_pin_pages(obj);
4924         sg = obj->pages;
4925         bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
4926         obj->dirty = 1;         /* Backing store is now out of date */
4927         i915_gem_object_unpin_pages(obj);
4928
4929         if (WARN_ON(bytes != size)) {
4930                 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
4931                 ret = -EFAULT;
4932                 goto fail;
4933         }
4934
4935         return obj;
4936
4937 fail:
4938         i915_gem_object_put(obj);
4939         return ERR_PTR(ret);
4940 }