drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_vgpu.h"
  33 #include "i915_trace.h"
  34 #include "intel_drv.h"
  35 #include "intel_mocs.h"
  36 #include <linux/shmem_fs.h>
  37 #include <linux/slab.h>
  38 #include <linux/swap.h>
  39 #include <linux/pci.h>
  40 #include <linux/dma-buf.h>
  41
  42 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  43 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  44 static void
  45 i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
  46 static void
  47 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
  48
  49 static bool cpu_cache_is_coherent(struct drm_device *dev,
  50                                   enum i915_cache_level level)
  51 {
  52         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  53 }
  54
  55 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  56 {
  57         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  58                 return false;
  59
  60         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  61                 return true;
  62
  63         return obj->pin_display;
  64 }
  65
  66 static int
  67 insert_mappable_node(struct drm_i915_private *i915,
  68                      struct drm_mm_node *node, u32 size)
  69 {
  70         memset(node, 0, sizeof(*node));
  71         return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
  72                                                    size, 0, 0, 0,
  73                                                    i915->ggtt.mappable_end,
  74                                                    DRM_MM_SEARCH_DEFAULT,
  75                                                    DRM_MM_CREATE_DEFAULT);
  76 }
  77
  78 static void
  79 remove_mappable_node(struct drm_mm_node *node)
  80 {
  81         drm_mm_remove_node(node);
  82 }
  83
  84 /* some bookkeeping */
  85 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  86                                   size_t size)
  87 {
  88         spin_lock(&dev_priv->mm.object_stat_lock);
  89         dev_priv->mm.object_count++;
  90         dev_priv->mm.object_memory += size;
  91         spin_unlock(&dev_priv->mm.object_stat_lock);
  92 }
  93
  94 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  95                                      size_t size)
  96 {
  97         spin_lock(&dev_priv->mm.object_stat_lock);
  98         dev_priv->mm.object_count--;
  99         dev_priv->mm.object_memory -= size;
 100         spin_unlock(&dev_priv->mm.object_stat_lock);
 101 }
 102
 103 static int
 104 i915_gem_wait_for_error(struct i915_gpu_error *error)
 105 {
 106         int ret;
 107
 108         if (!i915_reset_in_progress(error))
 109                 return 0;
 110
 111         /*
 112          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 113          * userspace. If it takes that long something really bad is going on and
 114          * we should simply try to bail out and fail as gracefully as possible.
 115          */
 116         ret = wait_event_interruptible_timeout(error->reset_queue,
 117                                                !i915_reset_in_progress(error),
 118                                                10*HZ);
 119         if (ret == 0) {
 120                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 121                 return -EIO;
 122         } else if (ret < 0) {
 123                 return ret;
 124         } else {
 125                 return 0;
 126         }
 127 }
 128
 129 int i915_mutex_lock_interruptible(struct drm_device *dev)
 130 {
 131         struct drm_i915_private *dev_priv = dev->dev_private;
 132         int ret;
 133
 134         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 135         if (ret)
 136                 return ret;
 137
 138         ret = mutex_lock_interruptible(&dev->struct_mutex);
 139         if (ret)
 140                 return ret;
 141
 142         WARN_ON(i915_verify_lists(dev));
 143         return 0;
 144 }
 145
 146 int
 147 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 148                             struct drm_file *file)
 149 {
 150         struct drm_i915_private *dev_priv = to_i915(dev);
 151         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 152         struct drm_i915_gem_get_aperture *args = data;
 153         struct i915_vma *vma;
 154         size_t pinned;
 155
 156         pinned = 0;
 157         mutex_lock(&dev->struct_mutex);
 158         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
 159                 if (vma->pin_count)
 160                         pinned += vma->node.size;
 161         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
 162                 if (vma->pin_count)
 163                         pinned += vma->node.size;
 164         mutex_unlock(&dev->struct_mutex);
 165
 166         args->aper_size = ggtt->base.total;
 167         args->aper_available_size = args->aper_size - pinned;
 168
 169         return 0;
 170 }
 171
 172 static int
 173 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 174 {
 175         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 176         char *vaddr = obj->phys_handle->vaddr;
 177         struct sg_table *st;
 178         struct scatterlist *sg;
 179         int i;
 180
 181         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 182                 return -EINVAL;
 183
 184         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 185                 struct page *page;
 186                 char *src;
 187
 188                 page = shmem_read_mapping_page(mapping, i);
 189                 if (IS_ERR(page))
 190                         return PTR_ERR(page);
 191
 192                 src = kmap_atomic(page);
 193                 memcpy(vaddr, src, PAGE_SIZE);
 194                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 195                 kunmap_atomic(src);
 196
 197                 put_page(page);
 198                 vaddr += PAGE_SIZE;
 199         }
 200
 201         i915_gem_chipset_flush(to_i915(obj->base.dev));
 202
 203         st = kmalloc(sizeof(*st), GFP_KERNEL);
 204         if (st == NULL)
 205                 return -ENOMEM;
 206
 207         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 208                 kfree(st);
 209                 return -ENOMEM;
 210         }
 211
 212         sg = st->sgl;
 213         sg->offset = 0;
 214         sg->length = obj->base.size;
 215
 216         sg_dma_address(sg) = obj->phys_handle->busaddr;
 217         sg_dma_len(sg) = obj->base.size;
 218
 219         obj->pages = st;
 220         return 0;
 221 }
 222
 223 static void
 224 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 225 {
 226         int ret;
 227
 228         BUG_ON(obj->madv == __I915_MADV_PURGED);
 229
 230         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 231         if (WARN_ON(ret)) {
 232                 /* In the event of a disaster, abandon all caches and
 233                  * hope for the best.
 234                  */
 235                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 236         }
 237
 238         if (obj->madv == I915_MADV_DONTNEED)
 239                 obj->dirty = 0;
 240
 241         if (obj->dirty) {
 242                 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
 243                 char *vaddr = obj->phys_handle->vaddr;
 244                 int i;
 245
 246                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 247                         struct page *page;
 248                         char *dst;
 249
 250                         page = shmem_read_mapping_page(mapping, i);
 251                         if (IS_ERR(page))
 252                                 continue;
 253
 254                         dst = kmap_atomic(page);
 255                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 256                         memcpy(dst, vaddr, PAGE_SIZE);
 257                         kunmap_atomic(dst);
 258
 259                         set_page_dirty(page);
 260                         if (obj->madv == I915_MADV_WILLNEED)
 261                                 mark_page_accessed(page);
 262                         put_page(page);
 263                         vaddr += PAGE_SIZE;
 264                 }
 265                 obj->dirty = 0;
 266         }
 267
 268         sg_free_table(obj->pages);
 269         kfree(obj->pages);
 270 }
 271
 272 static void
 273 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 274 {
 275         drm_pci_free(obj->base.dev, obj->phys_handle);
 276 }
 277
 278 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 279         .get_pages = i915_gem_object_get_pages_phys,
 280         .put_pages = i915_gem_object_put_pages_phys,
 281         .release = i915_gem_object_release_phys,
 282 };
 283
 284 static int
 285 drop_pages(struct drm_i915_gem_object *obj)
 286 {
 287         struct i915_vma *vma, *next;
 288         int ret;
 289
 290         drm_gem_object_reference(&obj->base);
 291         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
 292                 if (i915_vma_unbind(vma))
 293                         break;
 294
 295         ret = i915_gem_object_put_pages(obj);
 296         drm_gem_object_unreference(&obj->base);
 297
 298         return ret;
 299 }
 300
 301 int
 302 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 303                             int align)
 304 {
 305         drm_dma_handle_t *phys;
 306         int ret;
 307
 308         if (obj->phys_handle) {
 309                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 310                         return -EBUSY;
 311
 312                 return 0;
 313         }
 314
 315         if (obj->madv != I915_MADV_WILLNEED)
 316                 return -EFAULT;
 317
 318         if (obj->base.filp == NULL)
 319                 return -EINVAL;
 320
 321         ret = drop_pages(obj);
 322         if (ret)
 323                 return ret;
 324
 325         /* create a new object */
 326         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 327         if (!phys)
 328                 return -ENOMEM;
 329
 330         obj->phys_handle = phys;
 331         obj->ops = &i915_gem_phys_ops;
 332
 333         return i915_gem_object_get_pages(obj);
 334 }
 335
 336 static int
 337 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 338                      struct drm_i915_gem_pwrite *args,
 339                      struct drm_file *file_priv)
 340 {
 341         struct drm_device *dev = obj->base.dev;
 342         void *vaddr = obj->phys_handle->vaddr + args->offset;
 343         char __user *user_data = u64_to_user_ptr(args->data_ptr);
 344         int ret = 0;
 345
 346         /* We manually control the domain here and pretend that it
 347          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 348          */
 349         ret = i915_gem_object_wait_rendering(obj, false);
 350         if (ret)
 351                 return ret;
 352
 353         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 354         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 355                 unsigned long unwritten;
 356
 357                 /* The physical object once assigned is fixed for the lifetime
 358                  * of the obj, so we can safely drop the lock and continue
 359                  * to access vaddr.
 360                  */
 361                 mutex_unlock(&dev->struct_mutex);
 362                 unwritten = copy_from_user(vaddr, user_data, args->size);
 363                 mutex_lock(&dev->struct_mutex);
 364                 if (unwritten) {
 365                         ret = -EFAULT;
 366                         goto out;
 367                 }
 368         }
 369
 370         drm_clflush_virt_range(vaddr, args->size);
 371         i915_gem_chipset_flush(to_i915(dev));
 372
 373 out:
 374         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 375         return ret;
 376 }
 377
 378 void *i915_gem_object_alloc(struct drm_device *dev)
 379 {
 380         struct drm_i915_private *dev_priv = dev->dev_private;
 381         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 382 }
 383
 384 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 385 {
 386         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 387         kmem_cache_free(dev_priv->objects, obj);
 388 }
 389
 390 static int
 391 i915_gem_create(struct drm_file *file,
 392                 struct drm_device *dev,
 393                 uint64_t size,
 394                 uint32_t *handle_p)
 395 {
 396         struct drm_i915_gem_object *obj;
 397         int ret;
 398         u32 handle;
 399
 400         size = roundup(size, PAGE_SIZE);
 401         if (size == 0)
 402                 return -EINVAL;
 403
 404         /* Allocate the new object */
 405         obj = i915_gem_object_create(dev, size);
 406         if (IS_ERR(obj))
 407                 return PTR_ERR(obj);
 408
 409         ret = drm_gem_handle_create(file, &obj->base, &handle);
 410         /* drop reference from allocate - handle holds it now */
 411         drm_gem_object_unreference_unlocked(&obj->base);
 412         if (ret)
 413                 return ret;
 414
 415         *handle_p = handle;
 416         return 0;
 417 }
 418
 419 int
 420 i915_gem_dumb_create(struct drm_file *file,
 421                      struct drm_device *dev,
 422                      struct drm_mode_create_dumb *args)
 423 {
 424         /* have to work out size/pitch and return them */
 425         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 426         args->size = args->pitch * args->height;
 427         return i915_gem_create(file, dev,
 428                                args->size, &args->handle);
 429 }
 430
 431 /**
 432  * Creates a new mm object and returns a handle to it.
 433  * @dev: drm device pointer
 434  * @data: ioctl data blob
 435  * @file: drm file pointer
 436  */
 437 int
 438 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 439                       struct drm_file *file)
 440 {
 441         struct drm_i915_gem_create *args = data;
 442
 443         return i915_gem_create(file, dev,
 444                                args->size, &args->handle);
 445 }
 446
 447 static inline int
 448 __copy_to_user_swizzled(char __user *cpu_vaddr,
 449                         const char *gpu_vaddr, int gpu_offset,
 450                         int length)
 451 {
 452         int ret, cpu_offset = 0;
 453
 454         while (length > 0) {
 455                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 456                 int this_length = min(cacheline_end - gpu_offset, length);
 457                 int swizzled_gpu_offset = gpu_offset ^ 64;
 458
 459                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 460                                      gpu_vaddr + swizzled_gpu_offset,
 461                                      this_length);
 462                 if (ret)
 463                         return ret + length;
 464
 465                 cpu_offset += this_length;
 466                 gpu_offset += this_length;
 467                 length -= this_length;
 468         }
 469
 470         return 0;
 471 }
 472
 473 static inline int
 474 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 475                           const char __user *cpu_vaddr,
 476                           int length)
 477 {
 478         int ret, cpu_offset = 0;
 479
 480         while (length > 0) {
 481                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 482                 int this_length = min(cacheline_end - gpu_offset, length);
 483                 int swizzled_gpu_offset = gpu_offset ^ 64;
 484
 485                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 486                                        cpu_vaddr + cpu_offset,
 487                                        this_length);
 488                 if (ret)
 489                         return ret + length;
 490
 491                 cpu_offset += this_length;
 492                 gpu_offset += this_length;
 493                 length -= this_length;
 494         }
 495
 496         return 0;
 497 }
 498
 499 /*
 500  * Pins the specified object's pages and synchronizes the object with
 501  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 502  * flush the object from the CPU cache.
 503  */
 504 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 505                                     int *needs_clflush)
 506 {
 507         int ret;
 508
 509         *needs_clflush = 0;
 510
 511         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
 512                 return -EINVAL;
 513
 514         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 515                 /* If we're not in the cpu read domain, set ourself into the gtt
 516                  * read domain and manually flush cachelines (if required). This
 517                  * optimizes for the case when the gpu will dirty the data
 518                  * anyway again before the next pread happens. */
 519                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 520                                                         obj->cache_level);
 521                 ret = i915_gem_object_wait_rendering(obj, true);
 522                 if (ret)
 523                         return ret;
 524         }
 525
 526         ret = i915_gem_object_get_pages(obj);
 527         if (ret)
 528                 return ret;
 529
 530         i915_gem_object_pin_pages(obj);
 531
 532         return ret;
 533 }
 534
 535 /* Per-page copy function for the shmem pread fastpath.
 536  * Flushes invalid cachelines before reading the target if
 537  * needs_clflush is set. */
 538 static int
 539 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 540                  char __user *user_data,
 541                  bool page_do_bit17_swizzling, bool needs_clflush)
 542 {
 543         char *vaddr;
 544         int ret;
 545
 546         if (unlikely(page_do_bit17_swizzling))
 547                 return -EINVAL;
 548
 549         vaddr = kmap_atomic(page);
 550         if (needs_clflush)
 551                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 552                                        page_length);
 553         ret = __copy_to_user_inatomic(user_data,
 554                                       vaddr + shmem_page_offset,
 555                                       page_length);
 556         kunmap_atomic(vaddr);
 557
 558         return ret ? -EFAULT : 0;
 559 }
 560
 561 static void
 562 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 563                              bool swizzled)
 564 {
 565         if (unlikely(swizzled)) {
 566                 unsigned long start = (unsigned long) addr;
 567                 unsigned long end = (unsigned long) addr + length;
 568
 569                 /* For swizzling simply ensure that we always flush both
 570                  * channels. Lame, but simple and it works. Swizzled
 571                  * pwrite/pread is far from a hotpath - current userspace
 572                  * doesn't use it at all. */
 573                 start = round_down(start, 128);
 574                 end = round_up(end, 128);
 575
 576                 drm_clflush_virt_range((void *)start, end - start);
 577         } else {
 578                 drm_clflush_virt_range(addr, length);
 579         }
 580
 581 }
 582
 583 /* Only difference to the fast-path function is that this can handle bit17
 584  * and uses non-atomic copy and kmap functions. */
 585 static int
 586 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 587                  char __user *user_data,
 588                  bool page_do_bit17_swizzling, bool needs_clflush)
 589 {
 590         char *vaddr;
 591         int ret;
 592
 593         vaddr = kmap(page);
 594         if (needs_clflush)
 595                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 596                                              page_length,
 597                                              page_do_bit17_swizzling);
 598
 599         if (page_do_bit17_swizzling)
 600                 ret = __copy_to_user_swizzled(user_data,
 601                                               vaddr, shmem_page_offset,
 602                                               page_length);
 603         else
 604                 ret = __copy_to_user(user_data,
 605                                      vaddr + shmem_page_offset,
 606                                      page_length);
 607         kunmap(page);
 608
 609         return ret ? - EFAULT : 0;
 610 }
 611
 612 static inline unsigned long
 613 slow_user_access(struct io_mapping *mapping,
 614                  uint64_t page_base, int page_offset,
 615                  char __user *user_data,
 616                  unsigned long length, bool pwrite)
 617 {
 618         void __iomem *ioaddr;
 619         void *vaddr;
 620         uint64_t unwritten;
 621
 622         ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
 623         /* We can use the cpu mem copy function because this is X86. */
 624         vaddr = (void __force *)ioaddr + page_offset;
 625         if (pwrite)
 626                 unwritten = __copy_from_user(vaddr, user_data, length);
 627         else
 628                 unwritten = __copy_to_user(user_data, vaddr, length);
 629
 630         io_mapping_unmap(ioaddr);
 631         return unwritten;
 632 }
 633
 634 static int
 635 i915_gem_gtt_pread(struct drm_device *dev,
 636                    struct drm_i915_gem_object *obj, uint64_t size,
 637                    uint64_t data_offset, uint64_t data_ptr)
 638 {
 639         struct drm_i915_private *dev_priv = dev->dev_private;
 640         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 641         struct drm_mm_node node;
 642         char __user *user_data;
 643         uint64_t remain;
 644         uint64_t offset;
 645         int ret;
 646
 647         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
 648         if (ret) {
 649                 ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
 650                 if (ret)
 651                         goto out;
 652
 653                 ret = i915_gem_object_get_pages(obj);
 654                 if (ret) {
 655                         remove_mappable_node(&node);
 656                         goto out;
 657                 }
 658
 659                 i915_gem_object_pin_pages(obj);
 660         } else {
 661                 node.start = i915_gem_obj_ggtt_offset(obj);
 662                 node.allocated = false;
 663                 ret = i915_gem_object_put_fence(obj);
 664                 if (ret)
 665                         goto out_unpin;
 666         }
 667
 668         ret = i915_gem_object_set_to_gtt_domain(obj, false);
 669         if (ret)
 670                 goto out_unpin;
 671
 672         user_data = u64_to_user_ptr(data_ptr);
 673         remain = size;
 674         offset = data_offset;
 675
 676         mutex_unlock(&dev->struct_mutex);
 677         if (likely(!i915.prefault_disable)) {
 678                 ret = fault_in_multipages_writeable(user_data, remain);
 679                 if (ret) {
 680                         mutex_lock(&dev->struct_mutex);
 681                         goto out_unpin;
 682                 }
 683         }
 684
 685         while (remain > 0) {
 686                 /* Operation in this page
 687                  *
 688                  * page_base = page offset within aperture
 689                  * page_offset = offset within page
 690                  * page_length = bytes to copy for this page
 691                  */
 692                 u32 page_base = node.start;
 693                 unsigned page_offset = offset_in_page(offset);
 694                 unsigned page_length = PAGE_SIZE - page_offset;
 695                 page_length = remain < page_length ? remain : page_length;
 696                 if (node.allocated) {
 697                         wmb();
 698                         ggtt->base.insert_page(&ggtt->base,
 699                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 700                                                node.start,
 701                                                I915_CACHE_NONE, 0);
 702                         wmb();
 703                 } else {
 704                         page_base += offset & PAGE_MASK;
 705                 }
 706                 /* This is a slow read/write as it tries to read from
 707                  * and write to user memory which may result into page
 708                  * faults, and so we cannot perform this under struct_mutex.
 709                  */
 710                 if (slow_user_access(ggtt->mappable, page_base,
 711                                      page_offset, user_data,
 712                                      page_length, false)) {
 713                         ret = -EFAULT;
 714                         break;
 715                 }
 716
 717                 remain -= page_length;
 718                 user_data += page_length;
 719                 offset += page_length;
 720         }
 721
 722         mutex_lock(&dev->struct_mutex);
 723         if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
 724                 /* The user has modified the object whilst we tried
 725                  * reading from it, and we now have no idea what domain
 726                  * the pages should be in. As we have just been touching
 727                  * them directly, flush everything back to the GTT
 728                  * domain.
 729                  */
 730                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 731         }
 732
 733 out_unpin:
 734         if (node.allocated) {
 735                 wmb();
 736                 ggtt->base.clear_range(&ggtt->base,
 737                                        node.start, node.size,
 738                                        true);
 739                 i915_gem_object_unpin_pages(obj);
 740                 remove_mappable_node(&node);
 741         } else {
 742                 i915_gem_object_ggtt_unpin(obj);
 743         }
 744 out:
 745         return ret;
 746 }
 747
 748 static int
 749 i915_gem_shmem_pread(struct drm_device *dev,
 750                      struct drm_i915_gem_object *obj,
 751                      struct drm_i915_gem_pread *args,
 752                      struct drm_file *file)
 753 {
 754         char __user *user_data;
 755         ssize_t remain;
 756         loff_t offset;
 757         int shmem_page_offset, page_length, ret = 0;
 758         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 759         int prefaulted = 0;
 760         int needs_clflush = 0;
 761         struct sg_page_iter sg_iter;
 762
 763         if (!i915_gem_object_has_struct_page(obj))
 764                 return -ENODEV;
 765
 766         user_data = u64_to_user_ptr(args->data_ptr);
 767         remain = args->size;
 768
 769         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 770
 771         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 772         if (ret)
 773                 return ret;
 774
 775         offset = args->offset;
 776
 777         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 778                          offset >> PAGE_SHIFT) {
 779                 struct page *page = sg_page_iter_page(&sg_iter);
 780
 781                 if (remain <= 0)
 782                         break;
 783
 784                 /* Operation in this page
 785                  *
 786                  * shmem_page_offset = offset within page in shmem file
 787                  * page_length = bytes to copy for this page
 788                  */
 789                 shmem_page_offset = offset_in_page(offset);
 790                 page_length = remain;
 791                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 792                         page_length = PAGE_SIZE - shmem_page_offset;
 793
 794                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 795                         (page_to_phys(page) & (1 << 17)) != 0;
 796
 797                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 798                                        user_data, page_do_bit17_swizzling,
 799                                        needs_clflush);
 800                 if (ret == 0)
 801                         goto next_page;
 802
 803                 mutex_unlock(&dev->struct_mutex);
 804
 805                 if (likely(!i915.prefault_disable) && !prefaulted) {
 806                         ret = fault_in_multipages_writeable(user_data, remain);
 807                         /* Userspace is tricking us, but we've already clobbered
 808                          * its pages with the prefault and promised to write the
 809                          * data up to the first fault. Hence ignore any errors
 810                          * and just continue. */
 811                         (void)ret;
 812                         prefaulted = 1;
 813                 }
 814
 815                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 816                                        user_data, page_do_bit17_swizzling,
 817                                        needs_clflush);
 818
 819                 mutex_lock(&dev->struct_mutex);
 820
 821                 if (ret)
 822                         goto out;
 823
 824 next_page:
 825                 remain -= page_length;
 826                 user_data += page_length;
 827                 offset += page_length;
 828         }
 829
 830 out:
 831         i915_gem_object_unpin_pages(obj);
 832
 833         return ret;
 834 }
 835
 836 /**
 837  * Reads data from the object referenced by handle.
 838  * @dev: drm device pointer
 839  * @data: ioctl data blob
 840  * @file: drm file pointer
 841  *
 842  * On error, the contents of *data are undefined.
 843  */
 844 int
 845 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 846                      struct drm_file *file)
 847 {
 848         struct drm_i915_gem_pread *args = data;
 849         struct drm_i915_gem_object *obj;
 850         int ret = 0;
 851
 852         if (args->size == 0)
 853                 return 0;
 854
 855         if (!access_ok(VERIFY_WRITE,
 856                        u64_to_user_ptr(args->data_ptr),
 857                        args->size))
 858                 return -EFAULT;
 859
 860         ret = i915_mutex_lock_interruptible(dev);
 861         if (ret)
 862                 return ret;
 863
 864         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
 865         if (&obj->base == NULL) {
 866                 ret = -ENOENT;
 867                 goto unlock;
 868         }
 869
 870         /* Bounds check source.  */
 871         if (args->offset > obj->base.size ||
 872             args->size > obj->base.size - args->offset) {
 873                 ret = -EINVAL;
 874                 goto out;
 875         }
 876
 877         trace_i915_gem_object_pread(obj, args->offset, args->size);
 878
 879         ret = i915_gem_shmem_pread(dev, obj, args, file);
 880
 881         /* pread for non shmem backed objects */
 882         if (ret == -EFAULT || ret == -ENODEV)
 883                 ret = i915_gem_gtt_pread(dev, obj, args->size,
 884                                         args->offset, args->data_ptr);
 885
 886 out:
 887         drm_gem_object_unreference(&obj->base);
 888 unlock:
 889         mutex_unlock(&dev->struct_mutex);
 890         return ret;
 891 }
 892
 893 /* This is the fast write path which cannot handle
 894  * page faults in the source data
 895  */
 896
 897 static inline int
 898 fast_user_write(struct io_mapping *mapping,
 899                 loff_t page_base, int page_offset,
 900                 char __user *user_data,
 901                 int length)
 902 {
 903         void __iomem *vaddr_atomic;
 904         void *vaddr;
 905         unsigned long unwritten;
 906
 907         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 908         /* We can use the cpu mem copy function because this is X86. */
 909         vaddr = (void __force*)vaddr_atomic + page_offset;
 910         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 911                                                       user_data, length);
 912         io_mapping_unmap_atomic(vaddr_atomic);
 913         return unwritten;
 914 }
 915
 916 /**
 917  * This is the fast pwrite path, where we copy the data directly from the
 918  * user into the GTT, uncached.
 919  * @dev: drm device pointer
 920  * @obj: i915 gem object
 921  * @args: pwrite arguments structure
 922  * @file: drm file pointer
 923  */
 924 static int
 925 i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 926                          struct drm_i915_gem_object *obj,
 927                          struct drm_i915_gem_pwrite *args,
 928                          struct drm_file *file)
 929 {
 930         struct i915_ggtt *ggtt = &i915->ggtt;
 931         struct drm_device *dev = obj->base.dev;
 932         struct drm_mm_node node;
 933         uint64_t remain, offset;
 934         char __user *user_data;
 935         int ret;
 936         bool hit_slow_path = false;
 937
 938         if (obj->tiling_mode != I915_TILING_NONE)
 939                 return -EFAULT;
 940
 941         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
 942         if (ret) {
 943                 ret = insert_mappable_node(i915, &node, PAGE_SIZE);
 944                 if (ret)
 945                         goto out;
 946
 947                 ret = i915_gem_object_get_pages(obj);
 948                 if (ret) {
 949                         remove_mappable_node(&node);
 950                         goto out;
 951                 }
 952
 953                 i915_gem_object_pin_pages(obj);
 954         } else {
 955                 node.start = i915_gem_obj_ggtt_offset(obj);
 956                 node.allocated = false;
 957                 ret = i915_gem_object_put_fence(obj);
 958                 if (ret)
 959                         goto out_unpin;
 960         }
 961
 962         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 963         if (ret)
 964                 goto out_unpin;
 965
 966         intel_fb_obj_invalidate(obj, ORIGIN_GTT);
 967         obj->dirty = true;
 968
 969         user_data = u64_to_user_ptr(args->data_ptr);
 970         offset = args->offset;
 971         remain = args->size;
 972         while (remain) {
 973                 /* Operation in this page
 974                  *
 975                  * page_base = page offset within aperture
 976                  * page_offset = offset within page
 977                  * page_length = bytes to copy for this page
 978                  */
 979                 u32 page_base = node.start;
 980                 unsigned page_offset = offset_in_page(offset);
 981                 unsigned page_length = PAGE_SIZE - page_offset;
 982                 page_length = remain < page_length ? remain : page_length;
 983                 if (node.allocated) {
 984                         wmb(); /* flush the write before we modify the GGTT */
 985                         ggtt->base.insert_page(&ggtt->base,
 986                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 987                                                node.start, I915_CACHE_NONE, 0);
 988                         wmb(); /* flush modifications to the GGTT (insert_page) */
 989                 } else {
 990                         page_base += offset & PAGE_MASK;
 991                 }
 992                 /* If we get a fault while copying data, then (presumably) our
 993                  * source page isn't available.  Return the error and we'll
 994                  * retry in the slow path.
 995                  * If the object is non-shmem backed, we retry again with the
 996                  * path that handles page fault.
 997                  */
 998                 if (fast_user_write(ggtt->mappable, page_base,
 999                                     page_offset, user_data, page_length)) {
1000                         hit_slow_path = true;
1001                         mutex_unlock(&dev->struct_mutex);
1002                         if (slow_user_access(ggtt->mappable,
1003                                              page_base,
1004                                              page_offset, user_data,
1005                                              page_length, true)) {
1006                                 ret = -EFAULT;
1007                                 mutex_lock(&dev->struct_mutex);
1008                                 goto out_flush;
1009                         }
1010
1011                         mutex_lock(&dev->struct_mutex);
1012                 }
1013
1014                 remain -= page_length;
1015                 user_data += page_length;
1016                 offset += page_length;
1017         }
1018
1019 out_flush:
1020         if (hit_slow_path) {
1021                 if (ret == 0 &&
1022                     (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
1023                         /* The user has modified the object whilst we tried
1024                          * reading from it, and we now have no idea what domain
1025                          * the pages should be in. As we have just been touching
1026                          * them directly, flush everything back to the GTT
1027                          * domain.
1028                          */
1029                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
1030                 }
1031         }
1032
1033         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
1034 out_unpin:
1035         if (node.allocated) {
1036                 wmb();
1037                 ggtt->base.clear_range(&ggtt->base,
1038                                        node.start, node.size,
1039                                        true);
1040                 i915_gem_object_unpin_pages(obj);
1041                 remove_mappable_node(&node);
1042         } else {
1043                 i915_gem_object_ggtt_unpin(obj);
1044         }
1045 out:
1046         return ret;
1047 }
1048
1049 /* Per-page copy function for the shmem pwrite fastpath.
1050  * Flushes invalid cachelines before writing to the target if
1051  * needs_clflush_before is set and flushes out any written cachelines after
1052  * writing if needs_clflush is set. */
1053 static int
1054 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
1055                   char __user *user_data,
1056                   bool page_do_bit17_swizzling,
1057                   bool needs_clflush_before,
1058                   bool needs_clflush_after)
1059 {
1060         char *vaddr;
1061         int ret;
1062
1063         if (unlikely(page_do_bit17_swizzling))
1064                 return -EINVAL;
1065
1066         vaddr = kmap_atomic(page);
1067         if (needs_clflush_before)
1068                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1069                                        page_length);
1070         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
1071                                         user_data, page_length);
1072         if (needs_clflush_after)
1073                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1074                                        page_length);
1075         kunmap_atomic(vaddr);
1076
1077         return ret ? -EFAULT : 0;
1078 }
1079
1080 /* Only difference to the fast-path function is that this can handle bit17
1081  * and uses non-atomic copy and kmap functions. */
1082 static int
1083 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
1084                   char __user *user_data,
1085                   bool page_do_bit17_swizzling,
1086                   bool needs_clflush_before,
1087                   bool needs_clflush_after)
1088 {
1089         char *vaddr;
1090         int ret;
1091
1092         vaddr = kmap(page);
1093         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1094                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1095                                              page_length,
1096                                              page_do_bit17_swizzling);
1097         if (page_do_bit17_swizzling)
1098                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1099                                                 user_data,
1100                                                 page_length);
1101         else
1102                 ret = __copy_from_user(vaddr + shmem_page_offset,
1103                                        user_data,
1104                                        page_length);
1105         if (needs_clflush_after)
1106                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1107                                              page_length,
1108                                              page_do_bit17_swizzling);
1109         kunmap(page);
1110
1111         return ret ? -EFAULT : 0;
1112 }
1113
1114 static int
1115 i915_gem_shmem_pwrite(struct drm_device *dev,
1116                       struct drm_i915_gem_object *obj,
1117                       struct drm_i915_gem_pwrite *args,
1118                       struct drm_file *file)
1119 {
1120         ssize_t remain;
1121         loff_t offset;
1122         char __user *user_data;
1123         int shmem_page_offset, page_length, ret = 0;
1124         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1125         int hit_slowpath = 0;
1126         int needs_clflush_after = 0;
1127         int needs_clflush_before = 0;
1128         struct sg_page_iter sg_iter;
1129
1130         user_data = u64_to_user_ptr(args->data_ptr);
1131         remain = args->size;
1132
1133         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1134
1135         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1136                 /* If we're not in the cpu write domain, set ourself into the gtt
1137                  * write domain and manually flush cachelines (if required). This
1138                  * optimizes for the case when the gpu will use the data
1139                  * right away and we therefore have to clflush anyway. */
1140                 needs_clflush_after = cpu_write_needs_clflush(obj);
1141                 ret = i915_gem_object_wait_rendering(obj, false);
1142                 if (ret)
1143                         return ret;
1144         }
1145         /* Same trick applies to invalidate partially written cachelines read
1146          * before writing. */
1147         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1148                 needs_clflush_before =
1149                         !cpu_cache_is_coherent(dev, obj->cache_level);
1150
1151         ret = i915_gem_object_get_pages(obj);
1152         if (ret)
1153                 return ret;
1154
1155         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1156
1157         i915_gem_object_pin_pages(obj);
1158
1159         offset = args->offset;
1160         obj->dirty = 1;
1161
1162         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
1163                          offset >> PAGE_SHIFT) {
1164                 struct page *page = sg_page_iter_page(&sg_iter);
1165                 int partial_cacheline_write;
1166
1167                 if (remain <= 0)
1168                         break;
1169
1170                 /* Operation in this page
1171                  *
1172                  * shmem_page_offset = offset within page in shmem file
1173                  * page_length = bytes to copy for this page
1174                  */
1175                 shmem_page_offset = offset_in_page(offset);
1176
1177                 page_length = remain;
1178                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1179                         page_length = PAGE_SIZE - shmem_page_offset;
1180
1181                 /* If we don't overwrite a cacheline completely we need to be
1182                  * careful to have up-to-date data by first clflushing. Don't
1183                  * overcomplicate things and flush the entire patch. */
1184                 partial_cacheline_write = needs_clflush_before &&
1185                         ((shmem_page_offset | page_length)
1186                                 & (boot_cpu_data.x86_clflush_size - 1));
1187
1188                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1189                         (page_to_phys(page) & (1 << 17)) != 0;
1190
1191                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
1192                                         user_data, page_do_bit17_swizzling,
1193                                         partial_cacheline_write,
1194                                         needs_clflush_after);
1195                 if (ret == 0)
1196                         goto next_page;
1197
1198                 hit_slowpath = 1;
1199                 mutex_unlock(&dev->struct_mutex);
1200                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1201                                         user_data, page_do_bit17_swizzling,
1202                                         partial_cacheline_write,
1203                                         needs_clflush_after);
1204
1205                 mutex_lock(&dev->struct_mutex);
1206
1207                 if (ret)
1208                         goto out;
1209
1210 next_page:
1211                 remain -= page_length;
1212                 user_data += page_length;
1213                 offset += page_length;
1214         }
1215
1216 out:
1217         i915_gem_object_unpin_pages(obj);
1218
1219         if (hit_slowpath) {
1220                 /*
1221                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1222                  * cachelines in-line while writing and the object moved
1223                  * out of the cpu write domain while we've dropped the lock.
1224                  */
1225                 if (!needs_clflush_after &&
1226                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1227                         if (i915_gem_clflush_object(obj, obj->pin_display))
1228                                 needs_clflush_after = true;
1229                 }
1230         }
1231
1232         if (needs_clflush_after)
1233                 i915_gem_chipset_flush(to_i915(dev));
1234         else
1235                 obj->cache_dirty = true;
1236
1237         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1238         return ret;
1239 }
1240
1241 /**
1242  * Writes data to the object referenced by handle.
1243  * @dev: drm device
1244  * @data: ioctl data blob
1245  * @file: drm file
1246  *
1247  * On error, the contents of the buffer that were to be modified are undefined.
1248  */
1249 int
1250 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1251                       struct drm_file *file)
1252 {
1253         struct drm_i915_private *dev_priv = dev->dev_private;
1254         struct drm_i915_gem_pwrite *args = data;
1255         struct drm_i915_gem_object *obj;
1256         int ret;
1257
1258         if (args->size == 0)
1259                 return 0;
1260
1261         if (!access_ok(VERIFY_READ,
1262                        u64_to_user_ptr(args->data_ptr),
1263                        args->size))
1264                 return -EFAULT;
1265
1266         if (likely(!i915.prefault_disable)) {
1267                 ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
1268                                                    args->size);
1269                 if (ret)
1270                         return -EFAULT;
1271         }
1272
1273         intel_runtime_pm_get(dev_priv);
1274
1275         ret = i915_mutex_lock_interruptible(dev);
1276         if (ret)
1277                 goto put_rpm;
1278
1279         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
1280         if (&obj->base == NULL) {
1281                 ret = -ENOENT;
1282                 goto unlock;
1283         }
1284
1285         /* Bounds check destination. */
1286         if (args->offset > obj->base.size ||
1287             args->size > obj->base.size - args->offset) {
1288                 ret = -EINVAL;
1289                 goto out;
1290         }
1291
1292         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1293
1294         ret = -EFAULT;
1295         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1296          * it would end up going through the fenced access, and we'll get
1297          * different detiling behavior between reading and writing.
1298          * pread/pwrite currently are reading and writing from the CPU
1299          * perspective, requiring manual detiling by the client.
1300          */
1301         if (!i915_gem_object_has_struct_page(obj) ||
1302             cpu_write_needs_clflush(obj)) {
1303                 ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
1304                 /* Note that the gtt paths might fail with non-page-backed user
1305                  * pointers (e.g. gtt mappings when moving data between
1306                  * textures). Fallback to the shmem path in that case. */
1307         }
1308
1309         if (ret == -EFAULT) {
1310                 if (obj->phys_handle)
1311                         ret = i915_gem_phys_pwrite(obj, args, file);
1312                 else if (i915_gem_object_has_struct_page(obj))
1313                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1314                 else
1315                         ret = -ENODEV;
1316         }
1317
1318 out:
1319         drm_gem_object_unreference(&obj->base);
1320 unlock:
1321         mutex_unlock(&dev->struct_mutex);
1322 put_rpm:
1323         intel_runtime_pm_put(dev_priv);
1324
1325         return ret;
1326 }
1327
1328 static int
1329 i915_gem_check_wedge(unsigned reset_counter, bool interruptible)
1330 {
1331         if (__i915_terminally_wedged(reset_counter))
1332                 return -EIO;
1333
1334         if (__i915_reset_in_progress(reset_counter)) {
1335                 /* Non-interruptible callers can't handle -EAGAIN, hence return
1336                  * -EIO unconditionally for these. */
1337                 if (!interruptible)
1338                         return -EIO;
1339
1340                 return -EAGAIN;
1341         }
1342
1343         return 0;
1344 }
1345
1346 static void fake_irq(unsigned long data)
1347 {
1348         wake_up_process((struct task_struct *)data);
1349 }
1350
1351 static bool missed_irq(struct drm_i915_private *dev_priv,
1352                        struct intel_engine_cs *engine)
1353 {
1354         return test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings);
1355 }
1356
1357 static unsigned long local_clock_us(unsigned *cpu)
1358 {
1359         unsigned long t;
1360
1361         /* Cheaply and approximately convert from nanoseconds to microseconds.
1362          * The result and subsequent calculations are also defined in the same
1363          * approximate microseconds units. The principal source of timing
1364          * error here is from the simple truncation.
1365          *
1366          * Note that local_clock() is only defined wrt to the current CPU;
1367          * the comparisons are no longer valid if we switch CPUs. Instead of
1368          * blocking preemption for the entire busywait, we can detect the CPU
1369          * switch and use that as indicator of system load and a reason to
1370          * stop busywaiting, see busywait_stop().
1371          */
1372         *cpu = get_cpu();
1373         t = local_clock() >> 10;
1374         put_cpu();
1375
1376         return t;
1377 }
1378
1379 static bool busywait_stop(unsigned long timeout, unsigned cpu)
1380 {
1381         unsigned this_cpu;
1382
1383         if (time_after(local_clock_us(&this_cpu), timeout))
1384                 return true;
1385
1386         return this_cpu != cpu;
1387 }
1388
1389 static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
1390 {
1391         unsigned long timeout;
1392         unsigned cpu;
1393
1394         /* When waiting for high frequency requests, e.g. during synchronous
1395          * rendering split between the CPU and GPU, the finite amount of time
1396          * required to set up the irq and wait upon it limits the response
1397          * rate. By busywaiting on the request completion for a short while we
1398          * can service the high frequency waits as quick as possible. However,
1399          * if it is a slow request, we want to sleep as quickly as possible.
1400          * The tradeoff between waiting and sleeping is roughly the time it
1401          * takes to sleep on a request, on the order of a microsecond.
1402          */
1403
1404         if (req->engine->irq_refcount)
1405                 return -EBUSY;
1406
1407         /* Only spin if we know the GPU is processing this request */
1408         if (!i915_gem_request_started(req, true))
1409                 return -EAGAIN;
1410
1411         timeout = local_clock_us(&cpu) + 5;
1412         while (!need_resched()) {
1413                 if (i915_gem_request_completed(req, true))
1414                         return 0;
1415
1416                 if (signal_pending_state(state, current))
1417                         break;
1418
1419                 if (busywait_stop(timeout, cpu))
1420                         break;
1421
1422                 cpu_relax_lowlatency();
1423         }
1424
1425         if (i915_gem_request_completed(req, false))
1426                 return 0;
1427
1428         return -EAGAIN;
1429 }
1430
1431 /**
1432  * __i915_wait_request - wait until execution of request has finished
1433  * @req: duh!
1434  * @interruptible: do an interruptible wait (normally yes)
1435  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1436  * @rps: RPS client
1437  *
1438  * Note: It is of utmost importance that the passed in seqno and reset_counter
1439  * values have been read by the caller in an smp safe manner. Where read-side
1440  * locks are involved, it is sufficient to read the reset_counter before
1441  * unlocking the lock that protects the seqno. For lockless tricks, the
1442  * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1443  * inserted.
1444  *
1445  * Returns 0 if the request was found within the alloted time. Else returns the
1446  * errno with remaining time filled in timeout argument.
1447  */
1448 int __i915_wait_request(struct drm_i915_gem_request *req,
1449                         bool interruptible,
1450                         s64 *timeout,
1451                         struct intel_rps_client *rps)
1452 {
1453         struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
1454         struct drm_i915_private *dev_priv = req->i915;
1455         const bool irq_test_in_progress =
1456                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_engine_flag(engine);
1457         int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1458         DEFINE_WAIT(wait);
1459         unsigned long timeout_expire;
1460         s64 before = 0; /* Only to silence a compiler warning. */
1461         int ret;
1462
1463         WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1464
1465         if (list_empty(&req->list))
1466                 return 0;
1467
1468         if (i915_gem_request_completed(req, true))
1469                 return 0;
1470
1471         timeout_expire = 0;
1472         if (timeout) {
1473                 if (WARN_ON(*timeout < 0))
1474                         return -EINVAL;
1475
1476                 if (*timeout == 0)
1477                         return -ETIME;
1478
1479                 timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
1480
1481                 /*
1482                  * Record current time in case interrupted by signal, or wedged.
1483                  */
1484                 before = ktime_get_raw_ns();
1485         }
1486
1487         if (INTEL_INFO(dev_priv)->gen >= 6)
1488                 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
1489
1490         trace_i915_gem_request_wait_begin(req);
1491
1492         /* Optimistic spin for the next jiffie before touching IRQs */
1493         ret = __i915_spin_request(req, state);
1494         if (ret == 0)
1495                 goto out;
1496
1497         if (!irq_test_in_progress && WARN_ON(!engine->irq_get(engine))) {
1498                 ret = -ENODEV;
1499                 goto out;
1500         }
1501
1502         for (;;) {
1503                 struct timer_list timer;
1504
1505                 prepare_to_wait(&engine->irq_queue, &wait, state);
1506
1507                 /* We need to check whether any gpu reset happened in between
1508                  * the request being submitted and now. If a reset has occurred,
1509                  * the request is effectively complete (we either are in the
1510                  * process of or have discarded the rendering and completely
1511                  * reset the GPU. The results of the request are lost and we
1512                  * are free to continue on with the original operation.
1513                  */
1514                 if (req->reset_counter != i915_reset_counter(&dev_priv->gpu_error)) {
1515                         ret = 0;
1516                         break;
1517                 }
1518
1519                 if (i915_gem_request_completed(req, false)) {
1520                         ret = 0;
1521                         break;
1522                 }
1523
1524                 if (signal_pending_state(state, current)) {
1525                         ret = -ERESTARTSYS;
1526                         break;
1527                 }
1528
1529                 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1530                         ret = -ETIME;
1531                         break;
1532                 }
1533
1534                 timer.function = NULL;
1535                 if (timeout || missed_irq(dev_priv, engine)) {
1536                         unsigned long expire;
1537
1538                         setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1539                         expire = missed_irq(dev_priv, engine) ? jiffies + 1 : timeout_expire;
1540                         mod_timer(&timer, expire);
1541                 }
1542
1543                 io_schedule();
1544
1545                 if (timer.function) {
1546                         del_singleshot_timer_sync(&timer);
1547                         destroy_timer_on_stack(&timer);
1548                 }
1549         }
1550         if (!irq_test_in_progress)
1551                 engine->irq_put(engine);
1552
1553         finish_wait(&engine->irq_queue, &wait);
1554
1555 out:
1556         trace_i915_gem_request_wait_end(req);
1557
1558         if (timeout) {
1559                 s64 tres = *timeout - (ktime_get_raw_ns() - before);
1560
1561                 *timeout = tres < 0 ? 0 : tres;
1562
1563                 /*
1564                  * Apparently ktime isn't accurate enough and occasionally has a
1565                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1566                  * things up to make the test happy. We allow up to 1 jiffy.
1567                  *
1568                  * This is a regrssion from the timespec->ktime conversion.
1569                  */
1570                 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1571                         *timeout = 0;
1572         }
1573
1574         return ret;
1575 }
1576
1577 int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
1578                                    struct drm_file *file)
1579 {
1580         struct drm_i915_file_private *file_priv;
1581
1582         WARN_ON(!req || !file || req->file_priv);
1583
1584         if (!req || !file)
1585                 return -EINVAL;
1586
1587         if (req->file_priv)
1588                 return -EINVAL;
1589
1590         file_priv = file->driver_priv;
1591
1592         spin_lock(&file_priv->mm.lock);
1593         req->file_priv = file_priv;
1594         list_add_tail(&req->client_list, &file_priv->mm.request_list);
1595         spin_unlock(&file_priv->mm.lock);
1596
1597         req->pid = get_pid(task_pid(current));
1598
1599         return 0;
1600 }
1601
1602 static inline void
1603 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1604 {
1605         struct drm_i915_file_private *file_priv = request->file_priv;
1606
1607         if (!file_priv)
1608                 return;
1609
1610         spin_lock(&file_priv->mm.lock);
1611         list_del(&request->client_list);
1612         request->file_priv = NULL;
1613         spin_unlock(&file_priv->mm.lock);
1614
1615         put_pid(request->pid);
1616         request->pid = NULL;
1617 }
1618
1619 static void i915_gem_request_retire(struct drm_i915_gem_request *request)
1620 {
1621         trace_i915_gem_request_retire(request);
1622
1623         /* We know the GPU must have read the request to have
1624          * sent us the seqno + interrupt, so use the position
1625          * of tail of the request to update the last known position
1626          * of the GPU head.
1627          *
1628          * Note this requires that we are always called in request
1629          * completion order.
1630          */
1631         request->ringbuf->last_retired_head = request->postfix;
1632
1633         list_del_init(&request->list);
1634         i915_gem_request_remove_from_client(request);
1635
1636         if (request->previous_context) {
1637                 if (i915.enable_execlists)
1638                         intel_lr_context_unpin(request->previous_context,
1639                                                request->engine);
1640         }
1641
1642         i915_gem_context_unreference(request->ctx);
1643         i915_gem_request_unreference(request);
1644 }
1645
1646 static void
1647 __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
1648 {
1649         struct intel_engine_cs *engine = req->engine;
1650         struct drm_i915_gem_request *tmp;
1651
1652         lockdep_assert_held(&engine->i915->dev->struct_mutex);
1653
1654         if (list_empty(&req->list))
1655                 return;
1656
1657         do {
1658                 tmp = list_first_entry(&engine->request_list,
1659                                        typeof(*tmp), list);
1660
1661                 i915_gem_request_retire(tmp);
1662         } while (tmp != req);
1663
1664         WARN_ON(i915_verify_lists(engine->dev));
1665 }
1666
1667 /**
1668  * Waits for a request to be signaled, and cleans up the
1669  * request and object lists appropriately for that event.
1670  * @req: request to wait on
1671  */
1672 int
1673 i915_wait_request(struct drm_i915_gem_request *req)
1674 {
1675         struct drm_i915_private *dev_priv = req->i915;
1676         bool interruptible;
1677         int ret;
1678
1679         interruptible = dev_priv->mm.interruptible;
1680
1681         BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
1682
1683         ret = __i915_wait_request(req, interruptible, NULL, NULL);
1684         if (ret)
1685                 return ret;
1686
1687         /* If the GPU hung, we want to keep the requests to find the guilty. */
1688         if (req->reset_counter == i915_reset_counter(&dev_priv->gpu_error))
1689                 __i915_gem_request_retire__upto(req);
1690
1691         return 0;
1692 }
1693
1694 /**
1695  * Ensures that all rendering to the object has completed and the object is
1696  * safe to unbind from the GTT or access from the CPU.
1697  * @obj: i915 gem object
1698  * @readonly: waiting for read access or write
1699  */
1700 int
1701 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1702                                bool readonly)
1703 {
1704         int ret, i;
1705
1706         if (!obj->active)
1707                 return 0;
1708
1709         if (readonly) {
1710                 if (obj->last_write_req != NULL) {
1711                         ret = i915_wait_request(obj->last_write_req);
1712                         if (ret)
1713                                 return ret;
1714
1715                         i = obj->last_write_req->engine->id;
1716                         if (obj->last_read_req[i] == obj->last_write_req)
1717                                 i915_gem_object_retire__read(obj, i);
1718                         else
1719                                 i915_gem_object_retire__write(obj);
1720                 }
1721         } else {
1722                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1723                         if (obj->last_read_req[i] == NULL)
1724                                 continue;
1725
1726                         ret = i915_wait_request(obj->last_read_req[i]);
1727                         if (ret)
1728                                 return ret;
1729
1730                         i915_gem_object_retire__read(obj, i);
1731                 }
1732                 GEM_BUG_ON(obj->active);
1733         }
1734
1735         return 0;
1736 }
1737
1738 static void
1739 i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1740                                struct drm_i915_gem_request *req)
1741 {
1742         int ring = req->engine->id;
1743
1744         if (obj->last_read_req[ring] == req)
1745                 i915_gem_object_retire__read(obj, ring);
1746         else if (obj->last_write_req == req)
1747                 i915_gem_object_retire__write(obj);
1748
1749         if (req->reset_counter == i915_reset_counter(&req->i915->gpu_error))
1750                 __i915_gem_request_retire__upto(req);
1751 }
1752
1753 /* A nonblocking variant of the above wait. This is a highly dangerous routine
1754  * as the object state may change during this call.
1755  */
1756 static __must_check int
1757 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1758                                             struct intel_rps_client *rps,
1759                                             bool readonly)
1760 {
1761         struct drm_device *dev = obj->base.dev;
1762         struct drm_i915_private *dev_priv = dev->dev_private;
1763         struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
1764         int ret, i, n = 0;
1765
1766         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1767         BUG_ON(!dev_priv->mm.interruptible);
1768
1769         if (!obj->active)
1770                 return 0;
1771
1772         if (readonly) {
1773                 struct drm_i915_gem_request *req;
1774
1775                 req = obj->last_write_req;
1776                 if (req == NULL)
1777                         return 0;
1778
1779                 requests[n++] = i915_gem_request_reference(req);
1780         } else {
1781                 for (i = 0; i < I915_NUM_ENGINES; i++) {
1782                         struct drm_i915_gem_request *req;
1783
1784                         req = obj->last_read_req[i];
1785                         if (req == NULL)
1786                                 continue;
1787
1788                         requests[n++] = i915_gem_request_reference(req);
1789                 }
1790         }
1791
1792         mutex_unlock(&dev->struct_mutex);
1793         ret = 0;
1794         for (i = 0; ret == 0 && i < n; i++)
1795                 ret = __i915_wait_request(requests[i], true, NULL, rps);
1796         mutex_lock(&dev->struct_mutex);
1797
1798         for (i = 0; i < n; i++) {
1799                 if (ret == 0)
1800                         i915_gem_object_retire_request(obj, requests[i]);
1801                 i915_gem_request_unreference(requests[i]);
1802         }
1803
1804         return ret;
1805 }
1806
1807 static struct intel_rps_client *to_rps_client(struct drm_file *file)
1808 {
1809         struct drm_i915_file_private *fpriv = file->driver_priv;
1810         return &fpriv->rps;
1811 }
1812
1813 /**
1814  * Called when user space prepares to use an object with the CPU, either
1815  * through the mmap ioctl's mapping or a GTT mapping.
1816  * @dev: drm device
1817  * @data: ioctl data blob
1818  * @file: drm file
1819  */
1820 int
1821 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1822                           struct drm_file *file)
1823 {
1824         struct drm_i915_gem_set_domain *args = data;
1825         struct drm_i915_gem_object *obj;
1826         uint32_t read_domains = args->read_domains;
1827         uint32_t write_domain = args->write_domain;
1828         int ret;
1829
1830         /* Only handle setting domains to types used by the CPU. */
1831         if (write_domain & I915_GEM_GPU_DOMAINS)
1832                 return -EINVAL;
1833
1834         if (read_domains & I915_GEM_GPU_DOMAINS)
1835                 return -EINVAL;
1836
1837         /* Having something in the write domain implies it's in the read
1838          * domain, and only that read domain.  Enforce that in the request.
1839          */
1840         if (write_domain != 0 && read_domains != write_domain)
1841                 return -EINVAL;
1842
1843         ret = i915_mutex_lock_interruptible(dev);
1844         if (ret)
1845                 return ret;
1846
1847         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
1848         if (&obj->base == NULL) {
1849                 ret = -ENOENT;
1850                 goto unlock;
1851         }
1852
1853         /* Try to flush the object off the GPU without holding the lock.
1854          * We will repeat the flush holding the lock in the normal manner
1855          * to catch cases where we are gazumped.
1856          */
1857         ret = i915_gem_object_wait_rendering__nonblocking(obj,
1858                                                           to_rps_client(file),
1859                                                           !write_domain);
1860         if (ret)
1861                 goto unref;
1862
1863         if (read_domains & I915_GEM_DOMAIN_GTT)
1864                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1865         else
1866                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1867
1868         if (write_domain != 0)
1869                 intel_fb_obj_invalidate(obj,
1870                                         write_domain == I915_GEM_DOMAIN_GTT ?
1871                                         ORIGIN_GTT : ORIGIN_CPU);
1872
1873 unref:
1874         drm_gem_object_unreference(&obj->base);
1875 unlock:
1876         mutex_unlock(&dev->struct_mutex);
1877         return ret;
1878 }
1879
1880 /**
1881  * Called when user space has done writes to this buffer
1882  * @dev: drm device
1883  * @data: ioctl data blob
1884  * @file: drm file
1885  */
1886 int
1887 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1888                          struct drm_file *file)
1889 {
1890         struct drm_i915_gem_sw_finish *args = data;
1891         struct drm_i915_gem_object *obj;
1892         int ret = 0;
1893
1894         ret = i915_mutex_lock_interruptible(dev);
1895         if (ret)
1896                 return ret;
1897
1898         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
1899         if (&obj->base == NULL) {
1900                 ret = -ENOENT;
1901                 goto unlock;
1902         }
1903
1904         /* Pinned buffers may be scanout, so flush the cache */
1905         if (obj->pin_display)
1906                 i915_gem_object_flush_cpu_write_domain(obj);
1907
1908         drm_gem_object_unreference(&obj->base);
1909 unlock:
1910         mutex_unlock(&dev->struct_mutex);
1911         return ret;
1912 }
1913
1914 /**
1915  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1916  *                       it is mapped to.
1917  * @dev: drm device
1918  * @data: ioctl data blob
1919  * @file: drm file
1920  *
1921  * While the mapping holds a reference on the contents of the object, it doesn't
1922  * imply a ref on the object itself.
1923  *
1924  * IMPORTANT:
1925  *
1926  * DRM driver writers who look a this function as an example for how to do GEM
1927  * mmap support, please don't implement mmap support like here. The modern way
1928  * to implement DRM mmap support is with an mmap offset ioctl (like
1929  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1930  * That way debug tooling like valgrind will understand what's going on, hiding
1931  * the mmap call in a driver private ioctl will break that. The i915 driver only
1932  * does cpu mmaps this way because we didn't know better.
1933  */
1934 int
1935 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1936                     struct drm_file *file)
1937 {
1938         struct drm_i915_gem_mmap *args = data;
1939         struct drm_gem_object *obj;
1940         unsigned long addr;
1941
1942         if (args->flags & ~(I915_MMAP_WC))
1943                 return -EINVAL;
1944
1945         if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1946                 return -ENODEV;
1947
1948         obj = drm_gem_object_lookup(file, args->handle);
1949         if (obj == NULL)
1950                 return -ENOENT;
1951
1952         /* prime objects have no backing filp to GEM mmap
1953          * pages from.
1954          */
1955         if (!obj->filp) {
1956                 drm_gem_object_unreference_unlocked(obj);
1957                 return -EINVAL;
1958         }
1959
1960         addr = vm_mmap(obj->filp, 0, args->size,
1961                        PROT_READ | PROT_WRITE, MAP_SHARED,
1962                        args->offset);
1963         if (args->flags & I915_MMAP_WC) {
1964                 struct mm_struct *mm = current->mm;
1965                 struct vm_area_struct *vma;
1966
1967                 if (down_write_killable(&mm->mmap_sem)) {
1968                         drm_gem_object_unreference_unlocked(obj);
1969                         return -EINTR;
1970                 }
1971                 vma = find_vma(mm, addr);
1972                 if (vma)
1973                         vma->vm_page_prot =
1974                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1975                 else
1976                         addr = -ENOMEM;
1977                 up_write(&mm->mmap_sem);
1978         }
1979         drm_gem_object_unreference_unlocked(obj);
1980         if (IS_ERR((void *)addr))
1981                 return addr;
1982
1983         args->addr_ptr = (uint64_t) addr;
1984
1985         return 0;
1986 }
1987
1988 /**
1989  * i915_gem_fault - fault a page into the GTT
1990  * @vma: VMA in question
1991  * @vmf: fault info
1992  *
1993  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1994  * from userspace.  The fault handler takes care of binding the object to
1995  * the GTT (if needed), allocating and programming a fence register (again,
1996  * only if needed based on whether the old reg is still valid or the object
1997  * is tiled) and inserting a new PTE into the faulting process.
1998  *
1999  * Note that the faulting process may involve evicting existing objects
2000  * from the GTT and/or fence registers to make room.  So performance may
2001  * suffer if the GTT working set is large or there are few fence registers
2002  * left.
2003  */
2004 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2005 {
2006         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
2007         struct drm_device *dev = obj->base.dev;
2008         struct drm_i915_private *dev_priv = to_i915(dev);
2009         struct i915_ggtt *ggtt = &dev_priv->ggtt;
2010         struct i915_ggtt_view view = i915_ggtt_view_normal;
2011         pgoff_t page_offset;
2012         unsigned long pfn;
2013         int ret = 0;
2014         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
2015
2016         intel_runtime_pm_get(dev_priv);
2017
2018         /* We don't use vmf->pgoff since that has the fake offset */
2019         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
2020                 PAGE_SHIFT;
2021
2022         ret = i915_mutex_lock_interruptible(dev);
2023         if (ret)
2024                 goto out;
2025
2026         trace_i915_gem_object_fault(obj, page_offset, true, write);
2027
2028         /* Try to flush the object off the GPU first without holding the lock.
2029          * Upon reacquiring the lock, we will perform our sanity checks and then
2030          * repeat the flush holding the lock in the normal manner to catch cases
2031          * where we are gazumped.
2032          */
2033         ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
2034         if (ret)
2035                 goto unlock;
2036
2037         /* Access to snoopable pages through the GTT is incoherent. */
2038         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
2039                 ret = -EFAULT;
2040                 goto unlock;
2041         }
2042
2043         /* Use a partial view if the object is bigger than the aperture. */
2044         if (obj->base.size >= ggtt->mappable_end &&
2045             obj->tiling_mode == I915_TILING_NONE) {
2046                 static const unsigned int chunk_size = 256; // 1 MiB
2047
2048                 memset(&view, 0, sizeof(view));
2049                 view.type = I915_GGTT_VIEW_PARTIAL;
2050                 view.params.partial.offset = rounddown(page_offset, chunk_size);
2051                 view.params.partial.size =
2052                         min_t(unsigned int,
2053                               chunk_size,
2054                               (vma->vm_end - vma->vm_start)/PAGE_SIZE -
2055                               view.params.partial.offset);
2056         }
2057
2058         /* Now pin it into the GTT if needed */
2059         ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
2060         if (ret)
2061                 goto unlock;
2062
2063         ret = i915_gem_object_set_to_gtt_domain(obj, write);
2064         if (ret)
2065                 goto unpin;
2066
2067         ret = i915_gem_object_get_fence(obj);
2068         if (ret)
2069                 goto unpin;
2070
2071         /* Finally, remap it using the new GTT offset */
2072         pfn = ggtt->mappable_base +
2073                 i915_gem_obj_ggtt_offset_view(obj, &view);
2074         pfn >>= PAGE_SHIFT;
2075
2076         if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
2077                 /* Overriding existing pages in partial view does not cause
2078                  * us any trouble as TLBs are still valid because the fault
2079                  * is due to userspace losing part of the mapping or never
2080                  * having accessed it before (at this partials' range).
2081                  */
2082                 unsigned long base = vma->vm_start +
2083                                      (view.params.partial.offset << PAGE_SHIFT);
2084                 unsigned int i;
2085
2086                 for (i = 0; i < view.params.partial.size; i++) {
2087                         ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
2088                         if (ret)
2089                                 break;
2090                 }
2091
2092                 obj->fault_mappable = true;
2093         } else {
2094                 if (!obj->fault_mappable) {
2095                         unsigned long size = min_t(unsigned long,
2096                                                    vma->vm_end - vma->vm_start,
2097                                                    obj->base.size);
2098                         int i;
2099
2100                         for (i = 0; i < size >> PAGE_SHIFT; i++) {
2101                                 ret = vm_insert_pfn(vma,
2102                                                     (unsigned long)vma->vm_start + i * PAGE_SIZE,
2103                                                     pfn + i);
2104                                 if (ret)
2105                                         break;
2106                         }
2107
2108                         obj->fault_mappable = true;
2109                 } else
2110                         ret = vm_insert_pfn(vma,
2111                                             (unsigned long)vmf->virtual_address,
2112                                             pfn + page_offset);
2113         }
2114 unpin:
2115         i915_gem_object_ggtt_unpin_view(obj, &view);
2116 unlock:
2117         mutex_unlock(&dev->struct_mutex);
2118 out:
2119         switch (ret) {
2120         case -EIO:
2121                 /*
2122                  * We eat errors when the gpu is terminally wedged to avoid
2123                  * userspace unduly crashing (gl has no provisions for mmaps to
2124                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
2125                  * and so needs to be reported.
2126                  */
2127                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
2128                         ret = VM_FAULT_SIGBUS;
2129                         break;
2130                 }
2131         case -EAGAIN:
2132                 /*
2133                  * EAGAIN means the gpu is hung and we'll wait for the error
2134                  * handler to reset everything when re-faulting in
2135                  * i915_mutex_lock_interruptible.
2136                  */
2137         case 0:
2138         case -ERESTARTSYS:
2139         case -EINTR:
2140         case -EBUSY:
2141                 /*
2142                  * EBUSY is ok: this just means that another thread
2143                  * already did the job.
2144                  */
2145                 ret = VM_FAULT_NOPAGE;
2146                 break;
2147         case -ENOMEM:
2148                 ret = VM_FAULT_OOM;
2149                 break;
2150         case -ENOSPC:
2151         case -EFAULT:
2152                 ret = VM_FAULT_SIGBUS;
2153                 break;
2154         default:
2155                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
2156                 ret = VM_FAULT_SIGBUS;
2157                 break;
2158         }
2159
2160         intel_runtime_pm_put(dev_priv);
2161         return ret;
2162 }
2163
2164 /**
2165  * i915_gem_release_mmap - remove physical page mappings
2166  * @obj: obj in question
2167  *
2168  * Preserve the reservation of the mmapping with the DRM core code, but
2169  * relinquish ownership of the pages back to the system.
2170  *
2171  * It is vital that we remove the page mapping if we have mapped a tiled
2172  * object through the GTT and then lose the fence register due to
2173  * resource pressure. Similarly if the object has been moved out of the
2174  * aperture, than pages mapped into userspace must be revoked. Removing the
2175  * mapping will then trigger a page fault on the next user access, allowing
2176  * fixup by i915_gem_fault().
2177  */
2178 void
2179 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
2180 {
2181         /* Serialisation between user GTT access and our code depends upon
2182          * revoking the CPU's PTE whilst the mutex is held. The next user
2183          * pagefault then has to wait until we release the mutex.
2184          */
2185         lockdep_assert_held(&obj->base.dev->struct_mutex);
2186
2187         if (!obj->fault_mappable)
2188                 return;
2189
2190         drm_vma_node_unmap(&obj->base.vma_node,
2191                            obj->base.dev->anon_inode->i_mapping);
2192
2193         /* Ensure that the CPU's PTE are revoked and there are not outstanding
2194          * memory transactions from userspace before we return. The TLB
2195          * flushing implied above by changing the PTE above *should* be
2196          * sufficient, an extra barrier here just provides us with a bit
2197          * of paranoid documentation about our requirement to serialise
2198          * memory writes before touching registers / GSM.
2199          */
2200         wmb();
2201
2202         obj->fault_mappable = false;
2203 }
2204
2205 void
2206 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
2207 {
2208         struct drm_i915_gem_object *obj;
2209
2210         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
2211                 i915_gem_release_mmap(obj);
2212 }
2213
2214 uint32_t
2215 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
2216 {
2217         uint32_t gtt_size;
2218
2219         if (INTEL_INFO(dev)->gen >= 4 ||
2220             tiling_mode == I915_TILING_NONE)
2221                 return size;
2222
2223         /* Previous chips need a power-of-two fence region when tiling */
2224         if (IS_GEN3(dev))
2225                 gtt_size = 1024*1024;
2226         else
2227                 gtt_size = 512*1024;
2228
2229         while (gtt_size < size)
2230                 gtt_size <<= 1;
2231
2232         return gtt_size;
2233 }
2234
2235 /**
2236  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
2237  * @dev: drm device
2238  * @size: object size
2239  * @tiling_mode: tiling mode
2240  * @fenced: is fenced alignemned required or not
2241  *
2242  * Return the required GTT alignment for an object, taking into account
2243  * potential fence register mapping.
2244  */
2245 uint32_t
2246 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
2247                            int tiling_mode, bool fenced)
2248 {
2249         /*
2250          * Minimum alignment is 4k (GTT page size), but might be greater
2251          * if a fence register is needed for the object.
2252          */
2253         if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
2254             tiling_mode == I915_TILING_NONE)
2255                 return 4096;
2256
2257         /*
2258          * Previous chips need to be aligned to the size of the smallest
2259          * fence register that can contain the object.
2260          */
2261         return i915_gem_get_gtt_size(dev, size, tiling_mode);
2262 }
2263
2264 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2265 {
2266         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2267         int ret;
2268
2269         dev_priv->mm.shrinker_no_lock_stealing = true;
2270
2271         ret = drm_gem_create_mmap_offset(&obj->base);
2272         if (ret != -ENOSPC)
2273                 goto out;
2274
2275         /* Badly fragmented mmap space? The only way we can recover
2276          * space is by destroying unwanted objects. We can't randomly release
2277          * mmap_offsets as userspace expects them to be persistent for the
2278          * lifetime of the objects. The closest we can is to release the
2279          * offsets on purgeable objects by truncating it and marking it purged,
2280          * which prevents userspace from ever using that object again.
2281          */
2282         i915_gem_shrink(dev_priv,
2283                         obj->base.size >> PAGE_SHIFT,
2284                         I915_SHRINK_BOUND |
2285                         I915_SHRINK_UNBOUND |
2286                         I915_SHRINK_PURGEABLE);
2287         ret = drm_gem_create_mmap_offset(&obj->base);
2288         if (ret != -ENOSPC)
2289                 goto out;
2290
2291         i915_gem_shrink_all(dev_priv);
2292         ret = drm_gem_create_mmap_offset(&obj->base);
2293 out:
2294         dev_priv->mm.shrinker_no_lock_stealing = false;
2295
2296         return ret;
2297 }
2298
2299 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2300 {
2301         drm_gem_free_mmap_offset(&obj->base);
2302 }
2303
2304 int
2305 i915_gem_mmap_gtt(struct drm_file *file,
2306                   struct drm_device *dev,
2307                   uint32_t handle,
2308                   uint64_t *offset)
2309 {
2310         struct drm_i915_gem_object *obj;
2311         int ret;
2312
2313         ret = i915_mutex_lock_interruptible(dev);
2314         if (ret)
2315                 return ret;
2316
2317         obj = to_intel_bo(drm_gem_object_lookup(file, handle));
2318         if (&obj->base == NULL) {
2319                 ret = -ENOENT;
2320                 goto unlock;
2321         }
2322
2323         if (obj->madv != I915_MADV_WILLNEED) {
2324                 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
2325                 ret = -EFAULT;
2326                 goto out;
2327         }
2328
2329         ret = i915_gem_object_create_mmap_offset(obj);
2330         if (ret)
2331                 goto out;
2332
2333         *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2334
2335 out:
2336         drm_gem_object_unreference(&obj->base);
2337 unlock:
2338         mutex_unlock(&dev->struct_mutex);
2339         return ret;
2340 }
2341
2342 /**
2343  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2344  * @dev: DRM device
2345  * @data: GTT mapping ioctl data
2346  * @file: GEM object info
2347  *
2348  * Simply returns the fake offset to userspace so it can mmap it.
2349  * The mmap call will end up in drm_gem_mmap(), which will set things
2350  * up so we can get faults in the handler above.
2351  *
2352  * The fault handler will take care of binding the object into the GTT
2353  * (since it may have been evicted to make room for something), allocating
2354  * a fence register, and mapping the appropriate aperture address into
2355  * userspace.
2356  */
2357 int
2358 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2359                         struct drm_file *file)
2360 {
2361         struct drm_i915_gem_mmap_gtt *args = data;
2362
2363         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2364 }
2365
2366 /* Immediately discard the backing storage */
2367 static void
2368 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2369 {
2370         i915_gem_object_free_mmap_offset(obj);
2371
2372         if (obj->base.filp == NULL)
2373                 return;
2374
2375         /* Our goal here is to return as much of the memory as
2376          * is possible back to the system as we are called from OOM.
2377          * To do this we must instruct the shmfs to drop all of its
2378          * backing pages, *now*.
2379          */
2380         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2381         obj->madv = __I915_MADV_PURGED;
2382 }
2383
2384 /* Try to discard unwanted pages */
2385 static void
2386 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2387 {
2388         struct address_space *mapping;
2389
2390         switch (obj->madv) {
2391         case I915_MADV_DONTNEED:
2392                 i915_gem_object_truncate(obj);
2393         case __I915_MADV_PURGED:
2394                 return;
2395         }
2396
2397         if (obj->base.filp == NULL)
2398                 return;
2399
2400         mapping = file_inode(obj->base.filp)->i_mapping,
2401         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2402 }
2403
2404 static void
2405 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2406 {
2407         struct sgt_iter sgt_iter;
2408         struct page *page;
2409         int ret;
2410
2411         BUG_ON(obj->madv == __I915_MADV_PURGED);
2412
2413         ret = i915_gem_object_set_to_cpu_domain(obj, true);
2414         if (WARN_ON(ret)) {
2415                 /* In the event of a disaster, abandon all caches and
2416                  * hope for the best.
2417                  */
2418                 i915_gem_clflush_object(obj, true);
2419                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2420         }
2421
2422         i915_gem_gtt_finish_object(obj);
2423
2424         if (i915_gem_object_needs_bit17_swizzle(obj))
2425                 i915_gem_object_save_bit_17_swizzle(obj);
2426
2427         if (obj->madv == I915_MADV_DONTNEED)
2428                 obj->dirty = 0;
2429
2430         for_each_sgt_page(page, sgt_iter, obj->pages) {
2431                 if (obj->dirty)
2432                         set_page_dirty(page);
2433
2434                 if (obj->madv == I915_MADV_WILLNEED)
2435                         mark_page_accessed(page);
2436
2437                 put_page(page);
2438         }
2439         obj->dirty = 0;
2440
2441         sg_free_table(obj->pages);
2442         kfree(obj->pages);
2443 }
2444
2445 int
2446 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2447 {
2448         const struct drm_i915_gem_object_ops *ops = obj->ops;
2449
2450         if (obj->pages == NULL)
2451                 return 0;
2452
2453         if (obj->pages_pin_count)
2454                 return -EBUSY;
2455
2456         BUG_ON(i915_gem_obj_bound_any(obj));
2457
2458         /* ->put_pages might need to allocate memory for the bit17 swizzle
2459          * array, hence protect them from being reaped by removing them from gtt
2460          * lists early. */
2461         list_del(&obj->global_list);
2462
2463         if (obj->mapping) {
2464                 if (is_vmalloc_addr(obj->mapping))
2465                         vunmap(obj->mapping);
2466                 else
2467                         kunmap(kmap_to_page(obj->mapping));
2468                 obj->mapping = NULL;
2469         }
2470
2471         ops->put_pages(obj);
2472         obj->pages = NULL;
2473
2474         i915_gem_object_invalidate(obj);
2475
2476         return 0;
2477 }
2478
2479 static int
2480 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2481 {
2482         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2483         int page_count, i;
2484         struct address_space *mapping;
2485         struct sg_table *st;
2486         struct scatterlist *sg;
2487         struct sgt_iter sgt_iter;
2488         struct page *page;
2489         unsigned long last_pfn = 0;     /* suppress gcc warning */
2490         int ret;
2491         gfp_t gfp;
2492
2493         /* Assert that the object is not currently in any GPU domain. As it
2494          * wasn't in the GTT, there shouldn't be any way it could have been in
2495          * a GPU cache
2496          */
2497         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2498         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2499
2500         st = kmalloc(sizeof(*st), GFP_KERNEL);
2501         if (st == NULL)
2502                 return -ENOMEM;
2503
2504         page_count = obj->base.size / PAGE_SIZE;
2505         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2506                 kfree(st);
2507                 return -ENOMEM;
2508         }
2509
2510         /* Get the list of pages out of our struct file.  They'll be pinned
2511          * at this point until we release them.
2512          *
2513          * Fail silently without starting the shrinker
2514          */
2515         mapping = file_inode(obj->base.filp)->i_mapping;
2516         gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2517         gfp |= __GFP_NORETRY | __GFP_NOWARN;
2518         sg = st->sgl;
2519         st->nents = 0;
2520         for (i = 0; i < page_count; i++) {
2521                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2522                 if (IS_ERR(page)) {
2523                         i915_gem_shrink(dev_priv,
2524                                         page_count,
2525                                         I915_SHRINK_BOUND |
2526                                         I915_SHRINK_UNBOUND |
2527                                         I915_SHRINK_PURGEABLE);
2528                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2529                 }
2530                 if (IS_ERR(page)) {
2531                         /* We've tried hard to allocate the memory by reaping
2532                          * our own buffer, now let the real VM do its job and
2533                          * go down in flames if truly OOM.
2534                          */
2535                         i915_gem_shrink_all(dev_priv);
2536                         page = shmem_read_mapping_page(mapping, i);
2537                         if (IS_ERR(page)) {
2538                                 ret = PTR_ERR(page);
2539                                 goto err_pages;
2540                         }
2541                 }
2542 #ifdef CONFIG_SWIOTLB
2543                 if (swiotlb_nr_tbl()) {
2544                         st->nents++;
2545                         sg_set_page(sg, page, PAGE_SIZE, 0);
2546                         sg = sg_next(sg);
2547                         continue;
2548                 }
2549 #endif
2550                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2551                         if (i)
2552                                 sg = sg_next(sg);
2553                         st->nents++;
2554                         sg_set_page(sg, page, PAGE_SIZE, 0);
2555                 } else {
2556                         sg->length += PAGE_SIZE;
2557                 }
2558                 last_pfn = page_to_pfn(page);
2559
2560                 /* Check that the i965g/gm workaround works. */
2561                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2562         }
2563 #ifdef CONFIG_SWIOTLB
2564         if (!swiotlb_nr_tbl())
2565 #endif
2566                 sg_mark_end(sg);
2567         obj->pages = st;
2568
2569         ret = i915_gem_gtt_prepare_object(obj);
2570         if (ret)
2571                 goto err_pages;
2572
2573         if (i915_gem_object_needs_bit17_swizzle(obj))
2574                 i915_gem_object_do_bit_17_swizzle(obj);
2575
2576         if (obj->tiling_mode != I915_TILING_NONE &&
2577             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2578                 i915_gem_object_pin_pages(obj);
2579
2580         return 0;
2581
2582 err_pages:
2583         sg_mark_end(sg);
2584         for_each_sgt_page(page, sgt_iter, st)
2585                 put_page(page);
2586         sg_free_table(st);
2587         kfree(st);
2588
2589         /* shmemfs first checks if there is enough memory to allocate the page
2590          * and reports ENOSPC should there be insufficient, along with the usual
2591          * ENOMEM for a genuine allocation failure.
2592          *
2593          * We use ENOSPC in our driver to mean that we have run out of aperture
2594          * space and so want to translate the error from shmemfs back to our
2595          * usual understanding of ENOMEM.
2596          */
2597         if (ret == -ENOSPC)
2598                 ret = -ENOMEM;
2599
2600         return ret;
2601 }
2602
2603 /* Ensure that the associated pages are gathered from the backing storage
2604  * and pinned into our object. i915_gem_object_get_pages() may be called
2605  * multiple times before they are released by a single call to
2606  * i915_gem_object_put_pages() - once the pages are no longer referenced
2607  * either as a result of memory pressure (reaping pages under the shrinker)
2608  * or as the object is itself released.
2609  */
2610 int
2611 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2612 {
2613         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2614         const struct drm_i915_gem_object_ops *ops = obj->ops;
2615         int ret;
2616
2617         if (obj->pages)
2618                 return 0;
2619
2620         if (obj->madv != I915_MADV_WILLNEED) {
2621                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2622                 return -EFAULT;
2623         }
2624
2625         BUG_ON(obj->pages_pin_count);
2626
2627         ret = ops->get_pages(obj);
2628         if (ret)
2629                 return ret;
2630
2631         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2632
2633         obj->get_page.sg = obj->pages->sgl;
2634         obj->get_page.last = 0;
2635
2636         return 0;
2637 }
2638
2639 /* The 'mapping' part of i915_gem_object_pin_map() below */
2640 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
2641 {
2642         unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2643         struct sg_table *sgt = obj->pages;
2644         struct sgt_iter sgt_iter;
2645         struct page *page;
2646         struct page *stack_pages[32];
2647         struct page **pages = stack_pages;
2648         unsigned long i = 0;
2649         void *addr;
2650
2651         /* A single page can always be kmapped */
2652         if (n_pages == 1)
2653                 return kmap(sg_page(sgt->sgl));
2654
2655         if (n_pages > ARRAY_SIZE(stack_pages)) {
2656                 /* Too big for stack -- allocate temporary array instead */
2657                 pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
2658                 if (!pages)
2659                         return NULL;
2660         }
2661
2662         for_each_sgt_page(page, sgt_iter, sgt)
2663                 pages[i++] = page;
2664
2665         /* Check that we have the expected number of pages */
2666         GEM_BUG_ON(i != n_pages);
2667
2668         addr = vmap(pages, n_pages, 0, PAGE_KERNEL);
2669
2670         if (pages != stack_pages)
2671                 drm_free_large(pages);
2672
2673         return addr;
2674 }
2675
2676 /* get, pin, and map the pages of the object into kernel space */
2677 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
2678 {
2679         int ret;
2680
2681         lockdep_assert_held(&obj->base.dev->struct_mutex);
2682
2683         ret = i915_gem_object_get_pages(obj);
2684         if (ret)
2685                 return ERR_PTR(ret);
2686
2687         i915_gem_object_pin_pages(obj);
2688
2689         if (!obj->mapping) {
2690                 obj->mapping = i915_gem_object_map(obj);
2691                 if (!obj->mapping) {
2692                         i915_gem_object_unpin_pages(obj);
2693                         return ERR_PTR(-ENOMEM);
2694                 }
2695         }
2696
2697         return obj->mapping;
2698 }
2699
2700 void i915_vma_move_to_active(struct i915_vma *vma,
2701                              struct drm_i915_gem_request *req)
2702 {
2703         struct drm_i915_gem_object *obj = vma->obj;
2704         struct intel_engine_cs *engine;
2705
2706         engine = i915_gem_request_get_engine(req);
2707
2708         /* Add a reference if we're newly entering the active list. */
2709         if (obj->active == 0)
2710                 drm_gem_object_reference(&obj->base);
2711         obj->active |= intel_engine_flag(engine);
2712
2713         list_move_tail(&obj->engine_list[engine->id], &engine->active_list);
2714         i915_gem_request_assign(&obj->last_read_req[engine->id], req);
2715
2716         list_move_tail(&vma->vm_link, &vma->vm->active_list);
2717 }
2718
2719 static void
2720 i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
2721 {
2722         GEM_BUG_ON(obj->last_write_req == NULL);
2723         GEM_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine)));
2724
2725         i915_gem_request_assign(&obj->last_write_req, NULL);
2726         intel_fb_obj_flush(obj, true, ORIGIN_CS);
2727 }
2728
2729 static void
2730 i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
2731 {
2732         struct i915_vma *vma;
2733
2734         GEM_BUG_ON(obj->last_read_req[ring] == NULL);
2735         GEM_BUG_ON(!(obj->active & (1 << ring)));
2736
2737         list_del_init(&obj->engine_list[ring]);
2738         i915_gem_request_assign(&obj->last_read_req[ring], NULL);
2739
2740         if (obj->last_write_req && obj->last_write_req->engine->id == ring)
2741                 i915_gem_object_retire__write(obj);
2742
2743         obj->active &= ~(1 << ring);
2744         if (obj->active)
2745                 return;
2746
2747         /* Bump our place on the bound list to keep it roughly in LRU order
2748          * so that we don't steal from recently used but inactive objects
2749          * (unless we are forced to ofc!)
2750          */
2751         list_move_tail(&obj->global_list,
2752                        &to_i915(obj->base.dev)->mm.bound_list);
2753
2754         list_for_each_entry(vma, &obj->vma_list, obj_link) {
2755                 if (!list_empty(&vma->vm_link))
2756                         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
2757         }
2758
2759         i915_gem_request_assign(&obj->last_fenced_req, NULL);
2760         drm_gem_object_unreference(&obj->base);
2761 }
2762
2763 static int
2764 i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
2765 {
2766         struct intel_engine_cs *engine;
2767         int ret;
2768
2769         /* Carefully retire all requests without writing to the rings */
2770         for_each_engine(engine, dev_priv) {
2771                 ret = intel_engine_idle(engine);
2772                 if (ret)
2773                         return ret;
2774         }
2775         i915_gem_retire_requests(dev_priv);
2776
2777         /* Finally reset hw state */
2778         for_each_engine(engine, dev_priv)
2779                 intel_ring_init_seqno(engine, seqno);
2780
2781         return 0;
2782 }
2783
2784 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2785 {
2786         struct drm_i915_private *dev_priv = dev->dev_private;
2787         int ret;
2788
2789         if (seqno == 0)
2790                 return -EINVAL;
2791
2792         /* HWS page needs to be set less than what we
2793          * will inject to ring
2794          */
2795         ret = i915_gem_init_seqno(dev_priv, seqno - 1);
2796         if (ret)
2797                 return ret;
2798
2799         /* Carefully set the last_seqno value so that wrap
2800          * detection still works
2801          */
2802         dev_priv->next_seqno = seqno;
2803         dev_priv->last_seqno = seqno - 1;
2804         if (dev_priv->last_seqno == 0)
2805                 dev_priv->last_seqno--;
2806
2807         return 0;
2808 }
2809
2810 int
2811 i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
2812 {
2813         /* reserve 0 for non-seqno */
2814         if (dev_priv->next_seqno == 0) {
2815                 int ret = i915_gem_init_seqno(dev_priv, 0);
2816                 if (ret)
2817                         return ret;
2818
2819                 dev_priv->next_seqno = 1;
2820         }
2821
2822         *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2823         return 0;
2824 }
2825
2826 /*
2827  * NB: This function is not allowed to fail. Doing so would mean the the
2828  * request is not being tracked for completion but the work itself is
2829  * going to happen on the hardware. This would be a Bad Thing(tm).
2830  */
2831 void __i915_add_request(struct drm_i915_gem_request *request,
2832                         struct drm_i915_gem_object *obj,
2833                         bool flush_caches)
2834 {
2835         struct intel_engine_cs *engine;
2836         struct drm_i915_private *dev_priv;
2837         struct intel_ringbuffer *ringbuf;
2838         u32 request_start;
2839         u32 reserved_tail;
2840         int ret;
2841
2842         if (WARN_ON(request == NULL))
2843                 return;
2844
2845         engine = request->engine;
2846         dev_priv = request->i915;
2847         ringbuf = request->ringbuf;
2848
2849         /*
2850          * To ensure that this call will not fail, space for its emissions
2851          * should already have been reserved in the ring buffer. Let the ring
2852          * know that it is time to use that space up.
2853          */
2854         request_start = intel_ring_get_tail(ringbuf);
2855         reserved_tail = request->reserved_space;
2856         request->reserved_space = 0;
2857
2858         /*
2859          * Emit any outstanding flushes - execbuf can fail to emit the flush
2860          * after having emitted the batchbuffer command. Hence we need to fix
2861          * things up similar to emitting the lazy request. The difference here
2862          * is that the flush _must_ happen before the next request, no matter
2863          * what.
2864          */
2865         if (flush_caches) {
2866                 if (i915.enable_execlists)
2867                         ret = logical_ring_flush_all_caches(request);
2868                 else
2869                         ret = intel_ring_flush_all_caches(request);
2870                 /* Not allowed to fail! */
2871                 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
2872         }
2873
2874         trace_i915_gem_request_add(request);
2875
2876         request->head = request_start;
2877
2878         /* Whilst this request exists, batch_obj will be on the
2879          * active_list, and so will hold the active reference. Only when this
2880          * request is retired will the the batch_obj be moved onto the
2881          * inactive_list and lose its active reference. Hence we do not need
2882          * to explicitly hold another reference here.
2883          */
2884         request->batch_obj = obj;
2885
2886         /* Seal the request and mark it as pending execution. Note that
2887          * we may inspect this state, without holding any locks, during
2888          * hangcheck. Hence we apply the barrier to ensure that we do not
2889          * see a more recent value in the hws than we are tracking.
2890          */
2891         request->emitted_jiffies = jiffies;
2892         request->previous_seqno = engine->last_submitted_seqno;
2893         smp_store_mb(engine->last_submitted_seqno, request->seqno);
2894         list_add_tail(&request->list, &engine->request_list);
2895
2896         /* Record the position of the start of the request so that
2897          * should we detect the updated seqno part-way through the
2898          * GPU processing the request, we never over-estimate the
2899          * position of the head.
2900          */
2901         request->postfix = intel_ring_get_tail(ringbuf);
2902
2903         if (i915.enable_execlists)
2904                 ret = engine->emit_request(request);
2905         else {
2906                 ret = engine->add_request(request);
2907
2908                 request->tail = intel_ring_get_tail(ringbuf);
2909         }
2910         /* Not allowed to fail! */
2911         WARN(ret, "emit|add_request failed: %d!\n", ret);
2912
2913         i915_queue_hangcheck(engine->i915);
2914
2915         queue_delayed_work(dev_priv->wq,
2916                            &dev_priv->mm.retire_work,
2917                            round_jiffies_up_relative(HZ));
2918         intel_mark_busy(dev_priv);
2919
2920         /* Sanity check that the reserved size was large enough. */
2921         ret = intel_ring_get_tail(ringbuf) - request_start;
2922         if (ret < 0)
2923                 ret += ringbuf->size;
2924         WARN_ONCE(ret > reserved_tail,
2925                   "Not enough space reserved (%d bytes) "
2926                   "for adding the request (%d bytes)\n",
2927                   reserved_tail, ret);
2928 }
2929
2930 static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2931                                    const struct i915_gem_context *ctx)
2932 {
2933         unsigned long elapsed;
2934
2935         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2936
2937         if (ctx->hang_stats.banned)
2938                 return true;
2939
2940         if (ctx->hang_stats.ban_period_seconds &&
2941             elapsed <= ctx->hang_stats.ban_period_seconds) {
2942                 if (!i915_gem_context_is_default(ctx)) {
2943                         DRM_DEBUG("context hanging too fast, banning!\n");
2944                         return true;
2945                 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2946                         if (i915_stop_ring_allow_warn(dev_priv))
2947                                 DRM_ERROR("gpu hanging too fast, banning!\n");
2948                         return true;
2949                 }
2950         }
2951
2952         return false;
2953 }
2954
2955 static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2956                                   struct i915_gem_context *ctx,
2957                                   const bool guilty)
2958 {
2959         struct i915_ctx_hang_stats *hs;
2960
2961         if (WARN_ON(!ctx))
2962                 return;
2963
2964         hs = &ctx->hang_stats;
2965
2966         if (guilty) {
2967                 hs->banned = i915_context_is_banned(dev_priv, ctx);
2968                 hs->batch_active++;
2969                 hs->guilty_ts = get_seconds();
2970         } else {
2971                 hs->batch_pending++;
2972         }
2973 }
2974
2975 void i915_gem_request_free(struct kref *req_ref)
2976 {
2977         struct drm_i915_gem_request *req = container_of(req_ref,
2978                                                  typeof(*req), ref);
2979         kmem_cache_free(req->i915->requests, req);
2980 }
2981
2982 static inline int
2983 __i915_gem_request_alloc(struct intel_engine_cs *engine,
2984                          struct i915_gem_context *ctx,
2985                          struct drm_i915_gem_request **req_out)
2986 {
2987         struct drm_i915_private *dev_priv = engine->i915;
2988         unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error);
2989         struct drm_i915_gem_request *req;
2990         int ret;
2991
2992         if (!req_out)
2993                 return -EINVAL;
2994
2995         *req_out = NULL;
2996
2997         /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
2998          * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
2999          * and restart.
3000          */
3001         ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible);
3002         if (ret)
3003                 return ret;
3004
3005         req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
3006         if (req == NULL)
3007                 return -ENOMEM;
3008
3009         ret = i915_gem_get_seqno(engine->i915, &req->seqno);
3010         if (ret)
3011                 goto err;
3012
3013         kref_init(&req->ref);
3014         req->i915 = dev_priv;
3015         req->engine = engine;
3016         req->reset_counter = reset_counter;
3017         req->ctx  = ctx;
3018         i915_gem_context_reference(req->ctx);
3019
3020         /*
3021          * Reserve space in the ring buffer for all the commands required to
3022          * eventually emit this request. This is to guarantee that the
3023          * i915_add_request() call can't fail. Note that the reserve may need
3024          * to be redone if the request is not actually submitted straight
3025          * away, e.g. because a GPU scheduler has deferred it.
3026          */
3027         req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
3028
3029         if (i915.enable_execlists)
3030                 ret = intel_logical_ring_alloc_request_extras(req);
3031         else
3032                 ret = intel_ring_alloc_request_extras(req);
3033         if (ret)
3034                 goto err_ctx;
3035
3036         *req_out = req;
3037         return 0;
3038
3039 err_ctx:
3040         i915_gem_context_unreference(ctx);
3041 err:
3042         kmem_cache_free(dev_priv->requests, req);
3043         return ret;
3044 }
3045
3046 /**
3047  * i915_gem_request_alloc - allocate a request structure
3048  *
3049  * @engine: engine that we wish to issue the request on.
3050  * @ctx: context that the request will be associated with.
3051  *       This can be NULL if the request is not directly related to
3052  *       any specific user context, in which case this function will
3053  *       choose an appropriate context to use.
3054  *
3055  * Returns a pointer to the allocated request if successful,
3056  * or an error code if not.
3057  */
3058 struct drm_i915_gem_request *
3059 i915_gem_request_alloc(struct intel_engine_cs *engine,
3060                        struct i915_gem_context *ctx)
3061 {
3062         struct drm_i915_gem_request *req;
3063         int err;
3064
3065         if (ctx == NULL)
3066                 ctx = engine->i915->kernel_context;
3067         err = __i915_gem_request_alloc(engine, ctx, &req);
3068         return err ? ERR_PTR(err) : req;
3069 }
3070
3071 struct drm_i915_gem_request *
3072 i915_gem_find_active_request(struct intel_engine_cs *engine)
3073 {
3074         struct drm_i915_gem_request *request;
3075
3076         list_for_each_entry(request, &engine->request_list, list) {
3077                 if (i915_gem_request_completed(request, false))
3078                         continue;
3079
3080                 return request;
3081         }
3082
3083         return NULL;
3084 }
3085
3086 static void i915_gem_reset_engine_status(struct drm_i915_private *dev_priv,
3087                                        struct intel_engine_cs *engine)
3088 {
3089         struct drm_i915_gem_request *request;
3090         bool ring_hung;
3091
3092         request = i915_gem_find_active_request(engine);
3093
3094         if (request == NULL)
3095                 return;
3096
3097         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
3098
3099         i915_set_reset_status(dev_priv, request->ctx, ring_hung);
3100
3101         list_for_each_entry_continue(request, &engine->request_list, list)
3102                 i915_set_reset_status(dev_priv, request->ctx, false);
3103 }
3104
3105 static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv,
3106                                         struct intel_engine_cs *engine)
3107 {
3108         struct intel_ringbuffer *buffer;
3109
3110         while (!list_empty(&engine->active_list)) {
3111                 struct drm_i915_gem_object *obj;
3112
3113                 obj = list_first_entry(&engine->active_list,
3114                                        struct drm_i915_gem_object,
3115                                        engine_list[engine->id]);
3116
3117                 i915_gem_object_retire__read(obj, engine->id);
3118         }
3119
3120         /*
3121          * Clear the execlists queue up before freeing the requests, as those
3122          * are the ones that keep the context and ringbuffer backing objects
3123          * pinned in place.
3124          */
3125
3126         if (i915.enable_execlists) {
3127                 /* Ensure irq handler finishes or is cancelled. */
3128                 tasklet_kill(&engine->irq_tasklet);
3129
3130                 intel_execlists_cancel_requests(engine);
3131         }
3132
3133         /*
3134          * We must free the requests after all the corresponding objects have
3135          * been moved off active lists. Which is the same order as the normal
3136          * retire_requests function does. This is important if object hold
3137          * implicit references on things like e.g. ppgtt address spaces through
3138          * the request.
3139          */
3140         while (!list_empty(&engine->request_list)) {
3141                 struct drm_i915_gem_request *request;
3142
3143                 request = list_first_entry(&engine->request_list,
3144                                            struct drm_i915_gem_request,
3145                                            list);
3146
3147                 i915_gem_request_retire(request);
3148         }
3149
3150         /* Having flushed all requests from all queues, we know that all
3151          * ringbuffers must now be empty. However, since we do not reclaim
3152          * all space when retiring the request (to prevent HEADs colliding
3153          * with rapid ringbuffer wraparound) the amount of available space
3154          * upon reset is less than when we start. Do one more pass over
3155          * all the ringbuffers to reset last_retired_head.
3156          */
3157         list_for_each_entry(buffer, &engine->buffers, link) {
3158                 buffer->last_retired_head = buffer->tail;
3159                 intel_ring_update_space(buffer);
3160         }
3161
3162         intel_ring_init_seqno(engine, engine->last_submitted_seqno);
3163 }
3164
3165 void i915_gem_reset(struct drm_device *dev)
3166 {
3167         struct drm_i915_private *dev_priv = dev->dev_private;
3168         struct intel_engine_cs *engine;
3169
3170         /*
3171          * Before we free the objects from the requests, we need to inspect
3172          * them for finding the guilty party. As the requests only borrow
3173          * their reference to the objects, the inspection must be done first.
3174          */
3175         for_each_engine(engine, dev_priv)
3176                 i915_gem_reset_engine_status(dev_priv, engine);
3177
3178         for_each_engine(engine, dev_priv)
3179                 i915_gem_reset_engine_cleanup(dev_priv, engine);
3180
3181         i915_gem_context_reset(dev);
3182
3183         i915_gem_restore_fences(dev);
3184
3185         WARN_ON(i915_verify_lists(dev));
3186 }
3187
3188 /**
3189  * This function clears the request list as sequence numbers are passed.
3190  * @engine: engine to retire requests on
3191  */
3192 void
3193 i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
3194 {
3195         WARN_ON(i915_verify_lists(engine->dev));
3196
3197         /* Retire requests first as we use it above for the early return.
3198          * If we retire requests last, we may use a later seqno and so clear
3199          * the requests lists without clearing the active list, leading to
3200          * confusion.
3201          */
3202         while (!list_empty(&engine->request_list)) {
3203                 struct drm_i915_gem_request *request;
3204
3205                 request = list_first_entry(&engine->request_list,
3206                                            struct drm_i915_gem_request,
3207                                            list);
3208
3209                 if (!i915_gem_request_completed(request, true))
3210                         break;
3211
3212                 i915_gem_request_retire(request);
3213         }
3214
3215         /* Move any buffers on the active list that are no longer referenced
3216          * by the ringbuffer to the flushing/inactive lists as appropriate,
3217          * before we free the context associated with the requests.
3218          */
3219         while (!list_empty(&engine->active_list)) {
3220                 struct drm_i915_gem_object *obj;
3221
3222                 obj = list_first_entry(&engine->active_list,
3223                                        struct drm_i915_gem_object,
3224                                        engine_list[engine->id]);
3225
3226                 if (!list_empty(&obj->last_read_req[engine->id]->list))
3227                         break;
3228
3229                 i915_gem_object_retire__read(obj, engine->id);
3230         }
3231
3232         if (unlikely(engine->trace_irq_req &&
3233                      i915_gem_request_completed(engine->trace_irq_req, true))) {
3234                 engine->irq_put(engine);
3235                 i915_gem_request_assign(&engine->trace_irq_req, NULL);
3236         }
3237
3238         WARN_ON(i915_verify_lists(engine->dev));
3239 }
3240
3241 bool
3242 i915_gem_retire_requests(struct drm_i915_private *dev_priv)
3243 {
3244         struct intel_engine_cs *engine;
3245         bool idle = true;
3246
3247         for_each_engine(engine, dev_priv) {
3248                 i915_gem_retire_requests_ring(engine);
3249                 idle &= list_empty(&engine->request_list);
3250                 if (i915.enable_execlists) {
3251                         spin_lock_bh(&engine->execlist_lock);
3252                         idle &= list_empty(&engine->execlist_queue);
3253                         spin_unlock_bh(&engine->execlist_lock);
3254                 }
3255         }
3256
3257         if (idle)
3258                 mod_delayed_work(dev_priv->wq,
3259                                    &dev_priv->mm.idle_work,
3260                                    msecs_to_jiffies(100));
3261
3262         return idle;
3263 }
3264
3265 static void
3266 i915_gem_retire_work_handler(struct work_struct *work)
3267 {
3268         struct drm_i915_private *dev_priv =
3269                 container_of(work, typeof(*dev_priv), mm.retire_work.work);
3270         struct drm_device *dev = dev_priv->dev;
3271         bool idle;
3272
3273         /* Come back later if the device is busy... */
3274         idle = false;
3275         if (mutex_trylock(&dev->struct_mutex)) {
3276                 idle = i915_gem_retire_requests(dev_priv);
3277                 mutex_unlock(&dev->struct_mutex);
3278         }
3279         if (!idle)
3280                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3281                                    round_jiffies_up_relative(HZ));
3282 }
3283
3284 static void
3285 i915_gem_idle_work_handler(struct work_struct *work)
3286 {
3287         struct drm_i915_private *dev_priv =
3288                 container_of(work, typeof(*dev_priv), mm.idle_work.work);
3289         struct drm_device *dev = dev_priv->dev;
3290         struct intel_engine_cs *engine;
3291
3292         for_each_engine(engine, dev_priv)
3293                 if (!list_empty(&engine->request_list))
3294                         return;
3295
3296         /* we probably should sync with hangcheck here, using cancel_work_sync.
3297          * Also locking seems to be fubar here, engine->request_list is protected
3298          * by dev->struct_mutex. */
3299
3300         intel_mark_idle(dev_priv);
3301
3302         if (mutex_trylock(&dev->struct_mutex)) {
3303                 for_each_engine(engine, dev_priv)
3304                         i915_gem_batch_pool_fini(&engine->batch_pool);
3305
3306                 mutex_unlock(&dev->struct_mutex);
3307         }
3308 }
3309
3310 /**
3311  * Ensures that an object will eventually get non-busy by flushing any required
3312  * write domains, emitting any outstanding lazy request and retiring and
3313  * completed requests.
3314  * @obj: object to flush
3315  */
3316 static int
3317 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
3318 {
3319         int i;
3320
3321         if (!obj->active)
3322                 return 0;
3323
3324         for (i = 0; i < I915_NUM_ENGINES; i++) {
3325                 struct drm_i915_gem_request *req;
3326
3327                 req = obj->last_read_req[i];
3328                 if (req == NULL)
3329                         continue;
3330
3331                 if (i915_gem_request_completed(req, true))
3332                         i915_gem_object_retire__read(obj, i);
3333         }
3334
3335         return 0;
3336 }
3337
3338 /**
3339  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3340  * @dev: drm device pointer
3341  * @data: ioctl data blob
3342  * @file: drm file pointer
3343  *
3344  * Returns 0 if successful, else an error is returned with the remaining time in
3345  * the timeout parameter.
3346  *  -ETIME: object is still busy after timeout
3347  *  -ERESTARTSYS: signal interrupted the wait
3348  *  -ENONENT: object doesn't exist
3349  * Also possible, but rare:
3350  *  -EAGAIN: GPU wedged
3351  *  -ENOMEM: damn
3352  *  -ENODEV: Internal IRQ fail
3353  *  -E?: The add request failed
3354  *
3355  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3356  * non-zero timeout parameter the wait ioctl will wait for the given number of
3357  * nanoseconds on an object becoming unbusy. Since the wait itself does so
3358  * without holding struct_mutex the object may become re-busied before this
3359  * function completes. A similar but shorter * race condition exists in the busy
3360  * ioctl
3361  */
3362 int
3363 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3364 {
3365         struct drm_i915_gem_wait *args = data;
3366         struct drm_i915_gem_object *obj;
3367         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
3368         int i, n = 0;
3369         int ret;
3370
3371         if (args->flags != 0)
3372                 return -EINVAL;
3373
3374         ret = i915_mutex_lock_interruptible(dev);
3375         if (ret)
3376                 return ret;
3377
3378         obj = to_intel_bo(drm_gem_object_lookup(file, args->bo_handle));
3379         if (&obj->base == NULL) {
3380                 mutex_unlock(&dev->struct_mutex);
3381                 return -ENOENT;
3382         }
3383
3384         /* Need to make sure the object gets inactive eventually. */
3385         ret = i915_gem_object_flush_active(obj);
3386         if (ret)
3387                 goto out;
3388
3389         if (!obj->active)
3390                 goto out;
3391
3392         /* Do this after OLR check to make sure we make forward progress polling
3393          * on this IOCTL with a timeout == 0 (like busy ioctl)
3394          */
3395         if (args->timeout_ns == 0) {
3396                 ret = -ETIME;
3397                 goto out;
3398         }
3399
3400         drm_gem_object_unreference(&obj->base);
3401
3402         for (i = 0; i < I915_NUM_ENGINES; i++) {
3403                 if (obj->last_read_req[i] == NULL)
3404                         continue;
3405
3406                 req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
3407         }
3408
3409         mutex_unlock(&dev->struct_mutex);
3410
3411         for (i = 0; i < n; i++) {
3412                 if (ret == 0)
3413                         ret = __i915_wait_request(req[i], true,
3414                                                   args->timeout_ns > 0 ? &args->timeout_ns : NULL,
3415                                                   to_rps_client(file));
3416                 i915_gem_request_unreference(req[i]);
3417         }
3418         return ret;
3419
3420 out:
3421         drm_gem_object_unreference(&obj->base);
3422         mutex_unlock(&dev->struct_mutex);
3423         return ret;
3424 }
3425
3426 static int
3427 __i915_gem_object_sync(struct drm_i915_gem_object *obj,
3428                        struct intel_engine_cs *to,
3429                        struct drm_i915_gem_request *from_req,
3430                        struct drm_i915_gem_request **to_req)
3431 {
3432         struct intel_engine_cs *from;
3433         int ret;
3434
3435         from = i915_gem_request_get_engine(from_req);
3436         if (to == from)
3437                 return 0;
3438
3439         if (i915_gem_request_completed(from_req, true))
3440                 return 0;
3441
3442         if (!i915_semaphore_is_enabled(to_i915(obj->base.dev))) {
3443                 struct drm_i915_private *i915 = to_i915(obj->base.dev);
3444                 ret = __i915_wait_request(from_req,
3445                                           i915->mm.interruptible,
3446                                           NULL,
3447                                           &i915->rps.semaphores);
3448                 if (ret)
3449                         return ret;
3450
3451                 i915_gem_object_retire_request(obj, from_req);
3452         } else {
3453                 int idx = intel_ring_sync_index(from, to);
3454                 u32 seqno = i915_gem_request_get_seqno(from_req);
3455
3456                 WARN_ON(!to_req);
3457
3458                 if (seqno <= from->semaphore.sync_seqno[idx])
3459                         return 0;
3460
3461                 if (*to_req == NULL) {
3462                         struct drm_i915_gem_request *req;
3463
3464                         req = i915_gem_request_alloc(to, NULL);
3465                         if (IS_ERR(req))
3466                                 return PTR_ERR(req);
3467
3468                         *to_req = req;
3469                 }
3470
3471                 trace_i915_gem_ring_sync_to(*to_req, from, from_req);
3472                 ret = to->semaphore.sync_to(*to_req, from, seqno);
3473                 if (ret)
3474                         return ret;
3475
3476                 /* We use last_read_req because sync_to()
3477                  * might have just caused seqno wrap under
3478                  * the radar.
3479                  */
3480                 from->semaphore.sync_seqno[idx] =
3481                         i915_gem_request_get_seqno(obj->last_read_req[from->id]);
3482         }
3483
3484         return 0;
3485 }
3486
3487 /**
3488  * i915_gem_object_sync - sync an object to a ring.
3489  *
3490  * @obj: object which may be in use on another ring.
3491  * @to: ring we wish to use the object on. May be NULL.
3492  * @to_req: request we wish to use the object for. See below.
3493  *          This will be allocated and returned if a request is
3494  *          required but not passed in.
3495  *
3496  * This code is meant to abstract object synchronization with the GPU.
3497  * Calling with NULL implies synchronizing the object with the CPU
3498  * rather than a particular GPU ring. Conceptually we serialise writes
3499  * between engines inside the GPU. We only allow one engine to write
3500  * into a buffer at any time, but multiple readers. To ensure each has
3501  * a coherent view of memory, we must:
3502  *
3503  * - If there is an outstanding write request to the object, the new
3504  *   request must wait for it to complete (either CPU or in hw, requests
3505  *   on the same ring will be naturally ordered).
3506  *
3507  * - If we are a write request (pending_write_domain is set), the new
3508  *   request must wait for outstanding read requests to complete.
3509  *
3510  * For CPU synchronisation (NULL to) no request is required. For syncing with
3511  * rings to_req must be non-NULL. However, a request does not have to be
3512  * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
3513  * request will be allocated automatically and returned through *to_req. Note
3514  * that it is not guaranteed that commands will be emitted (because the system
3515  * might already be idle). Hence there is no need to create a request that
3516  * might never have any work submitted. Note further that if a request is
3517  * returned in *to_req, it is the responsibility of the caller to submit
3518  * that request (after potentially adding more work to it).
3519  *
3520  * Returns 0 if successful, else propagates up the lower layer error.
3521  */
3522 int
3523 i915_gem_object_sync(struct drm_i915_gem_object *obj,
3524                      struct intel_engine_cs *to,
3525                      struct drm_i915_gem_request **to_req)
3526 {
3527         const bool readonly = obj->base.pending_write_domain == 0;
3528         struct drm_i915_gem_request *req[I915_NUM_ENGINES];
3529         int ret, i, n;
3530
3531         if (!obj->active)
3532                 return 0;
3533
3534         if (to == NULL)
3535                 return i915_gem_object_wait_rendering(obj, readonly);
3536
3537         n = 0;
3538         if (readonly) {
3539                 if (obj->last_write_req)
3540                         req[n++] = obj->last_write_req;
3541         } else {
3542                 for (i = 0; i < I915_NUM_ENGINES; i++)
3543                         if (obj->last_read_req[i])
3544                                 req[n++] = obj->last_read_req[i];
3545         }
3546         for (i = 0; i < n; i++) {
3547                 ret = __i915_gem_object_sync(obj, to, req[i], to_req);
3548                 if (ret)
3549                         return ret;
3550         }
3551
3552         return 0;
3553 }
3554
3555 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3556 {
3557         u32 old_write_domain, old_read_domains;
3558
3559         /* Force a pagefault for domain tracking on next user access */
3560         i915_gem_release_mmap(obj);
3561
3562         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3563                 return;
3564
3565         old_read_domains = obj->base.read_domains;
3566         old_write_domain = obj->base.write_domain;
3567
3568         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3569         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3570
3571         trace_i915_gem_object_change_domain(obj,
3572                                             old_read_domains,
3573                                             old_write_domain);
3574 }
3575
3576 static void __i915_vma_iounmap(struct i915_vma *vma)
3577 {
3578         GEM_BUG_ON(vma->pin_count);
3579
3580         if (vma->iomap == NULL)
3581                 return;
3582
3583         io_mapping_unmap(vma->iomap);
3584         vma->iomap = NULL;
3585 }
3586
3587 static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
3588 {
3589         struct drm_i915_gem_object *obj = vma->obj;
3590         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3591         int ret;
3592
3593         if (list_empty(&vma->obj_link))
3594                 return 0;
3595
3596         if (!drm_mm_node_allocated(&vma->node)) {
3597                 i915_gem_vma_destroy(vma);
3598                 return 0;
3599         }
3600
3601         if (vma->pin_count)
3602                 return -EBUSY;
3603
3604         BUG_ON(obj->pages == NULL);
3605
3606         if (wait) {
3607                 ret = i915_gem_object_wait_rendering(obj, false);
3608                 if (ret)
3609                         return ret;
3610         }
3611
3612         if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3613                 i915_gem_object_finish_gtt(obj);
3614
3615                 /* release the fence reg _after_ flushing */
3616                 ret = i915_gem_object_put_fence(obj);
3617                 if (ret)
3618                         return ret;
3619
3620                 __i915_vma_iounmap(vma);
3621         }
3622
3623         trace_i915_vma_unbind(vma);
3624
3625         vma->vm->unbind_vma(vma);
3626         vma->bound = 0;
3627
3628         list_del_init(&vma->vm_link);
3629         if (vma->is_ggtt) {
3630                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3631                         obj->map_and_fenceable = false;
3632                 } else if (vma->ggtt_view.pages) {
3633                         sg_free_table(vma->ggtt_view.pages);
3634                         kfree(vma->ggtt_view.pages);
3635                 }
3636                 vma->ggtt_view.pages = NULL;
3637         }
3638
3639         drm_mm_remove_node(&vma->node);
3640         i915_gem_vma_destroy(vma);
3641
3642         /* Since the unbound list is global, only move to that list if
3643          * no more VMAs exist. */
3644         if (list_empty(&obj->vma_list))
3645                 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3646
3647         /* And finally now the object is completely decoupled from this vma,
3648          * we can drop its hold on the backing storage and allow it to be
3649          * reaped by the shrinker.
3650          */
3651         i915_gem_object_unpin_pages(obj);
3652
3653         return 0;
3654 }
3655
3656 int i915_vma_unbind(struct i915_vma *vma)
3657 {
3658         return __i915_vma_unbind(vma, true);
3659 }
3660
3661 int __i915_vma_unbind_no_wait(struct i915_vma *vma)
3662 {
3663         return __i915_vma_unbind(vma, false);
3664 }
3665
3666 int i915_gpu_idle(struct drm_device *dev)
3667 {
3668         struct drm_i915_private *dev_priv = dev->dev_private;
3669         struct intel_engine_cs *engine;
3670         int ret;
3671
3672         /* Flush everything onto the inactive list. */
3673         for_each_engine(engine, dev_priv) {
3674                 if (!i915.enable_execlists) {
3675                         struct drm_i915_gem_request *req;
3676
3677                         req = i915_gem_request_alloc(engine, NULL);
3678                         if (IS_ERR(req))
3679                                 return PTR_ERR(req);
3680
3681                         ret = i915_switch_context(req);
3682                         i915_add_request_no_flush(req);
3683                         if (ret)
3684                                 return ret;
3685                 }
3686
3687                 ret = intel_engine_idle(engine);
3688                 if (ret)
3689                         return ret;
3690         }
3691
3692         WARN_ON(i915_verify_lists(dev));
3693         return 0;
3694 }
3695
3696 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3697                                      unsigned long cache_level)
3698 {
3699         struct drm_mm_node *gtt_space = &vma->node;
3700         struct drm_mm_node *other;
3701
3702         /*
3703          * On some machines we have to be careful when putting differing types
3704          * of snoopable memory together to avoid the prefetcher crossing memory
3705          * domains and dying. During vm initialisation, we decide whether or not
3706          * these constraints apply and set the drm_mm.color_adjust
3707          * appropriately.
3708          */
3709         if (vma->vm->mm.color_adjust == NULL)
3710                 return true;
3711
3712         if (!drm_mm_node_allocated(gtt_space))
3713                 return true;
3714
3715         if (list_empty(&gtt_space->node_list))
3716                 return true;
3717
3718         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3719         if (other->allocated && !other->hole_follows && other->color != cache_level)
3720                 return false;
3721
3722         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3723         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3724                 return false;
3725
3726         return true;
3727 }
3728
3729 /**
3730  * Finds free space in the GTT aperture and binds the object or a view of it
3731  * there.
3732  * @obj: object to bind
3733  * @vm: address space to bind into
3734  * @ggtt_view: global gtt view if applicable
3735  * @alignment: requested alignment
3736  * @flags: mask of PIN_* flags to use
3737  */
3738 static struct i915_vma *
3739 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3740                            struct i915_address_space *vm,
3741                            const struct i915_ggtt_view *ggtt_view,
3742                            unsigned alignment,
3743                            uint64_t flags)
3744 {
3745         struct drm_device *dev = obj->base.dev;
3746         struct drm_i915_private *dev_priv = to_i915(dev);
3747         struct i915_ggtt *ggtt = &dev_priv->ggtt;
3748         u32 fence_alignment, unfenced_alignment;
3749         u32 search_flag, alloc_flag;
3750         u64 start, end;
3751         u64 size, fence_size;
3752         struct i915_vma *vma;
3753         int ret;
3754
3755         if (i915_is_ggtt(vm)) {
3756                 u32 view_size;
3757
3758                 if (WARN_ON(!ggtt_view))
3759                         return ERR_PTR(-EINVAL);
3760
3761                 view_size = i915_ggtt_view_size(obj, ggtt_view);
3762
3763                 fence_size = i915_gem_get_gtt_size(dev,
3764                                                    view_size,
3765                                                    obj->tiling_mode);
3766                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3767                                                              view_size,
3768                                                              obj->tiling_mode,
3769                                                              true);
3770                 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
3771                                                                 view_size,
3772                                                                 obj->tiling_mode,
3773                                                                 false);
3774                 size = flags & PIN_MAPPABLE ? fence_size : view_size;
3775         } else {
3776                 fence_size = i915_gem_get_gtt_size(dev,
3777                                                    obj->base.size,
3778                                                    obj->tiling_mode);
3779                 fence_alignment = i915_gem_get_gtt_alignment(dev,
3780                                                              obj->base.size,
3781                                                              obj->tiling_mode,
3782                                                              true);
3783                 unfenced_alignment =
3784                         i915_gem_get_gtt_alignment(dev,
3785                                                    obj->base.size,
3786                                                    obj->tiling_mode,
3787                                                    false);
3788                 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3789         }
3790
3791         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3792         end = vm->total;
3793         if (flags & PIN_MAPPABLE)
3794                 end = min_t(u64, end, ggtt->mappable_end);
3795         if (flags & PIN_ZONE_4G)
3796                 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
3797
3798         if (alignment == 0)
3799                 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3800                                                 unfenced_alignment;
3801         if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3802                 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
3803                           ggtt_view ? ggtt_view->type : 0,
3804                           alignment);
3805                 return ERR_PTR(-EINVAL);
3806         }
3807
3808         /* If binding the object/GGTT view requires more space than the entire
3809          * aperture has, reject it early before evicting everything in a vain
3810          * attempt to find space.
3811          */
3812         if (size > end) {
3813                 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
3814                           ggtt_view ? ggtt_view->type : 0,
3815                           size,
3816                           flags & PIN_MAPPABLE ? "mappable" : "total",
3817                           end);
3818                 return ERR_PTR(-E2BIG);
3819         }
3820
3821         ret = i915_gem_object_get_pages(obj);
3822         if (ret)
3823                 return ERR_PTR(ret);
3824
3825         i915_gem_object_pin_pages(obj);
3826
3827         vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
3828                           i915_gem_obj_lookup_or_create_vma(obj, vm);
3829
3830         if (IS_ERR(vma))
3831                 goto err_unpin;
3832
3833         if (flags & PIN_OFFSET_FIXED) {
3834                 uint64_t offset = flags & PIN_OFFSET_MASK;
3835
3836                 if (offset & (alignment - 1) || offset + size > end) {
3837                         ret = -EINVAL;
3838                         goto err_free_vma;
3839                 }
3840                 vma->node.start = offset;
3841                 vma->node.size = size;
3842                 vma->node.color = obj->cache_level;
3843                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3844                 if (ret) {
3845                         ret = i915_gem_evict_for_vma(vma);
3846                         if (ret == 0)
3847                                 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3848                 }
3849                 if (ret)
3850                         goto err_free_vma;
3851         } else {
3852                 if (flags & PIN_HIGH) {
3853                         search_flag = DRM_MM_SEARCH_BELOW;
3854                         alloc_flag = DRM_MM_CREATE_TOP;
3855                 } else {
3856                         search_flag = DRM_MM_SEARCH_DEFAULT;
3857                         alloc_flag = DRM_MM_CREATE_DEFAULT;
3858                 }
3859
3860 search_free:
3861                 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3862                                                           size, alignment,
3863                                                           obj->cache_level,
3864                                                           start, end,
3865                                                           search_flag,
3866                                                           alloc_flag);
3867                 if (ret) {
3868                         ret = i915_gem_evict_something(dev, vm, size, alignment,
3869                                                        obj->cache_level,
3870                                                        start, end,
3871                                                        flags);
3872                         if (ret == 0)
3873                                 goto search_free;
3874
3875                         goto err_free_vma;
3876                 }
3877         }
3878         if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3879                 ret = -EINVAL;
3880                 goto err_remove_node;
3881         }
3882
3883         trace_i915_vma_bind(vma, flags);
3884         ret = i915_vma_bind(vma, obj->cache_level, flags);
3885         if (ret)
3886                 goto err_remove_node;
3887
3888         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3889         list_add_tail(&vma->vm_link, &vm->inactive_list);
3890
3891         return vma;
3892
3893 err_remove_node:
3894         drm_mm_remove_node(&vma->node);
3895 err_free_vma:
3896         i915_gem_vma_destroy(vma);
3897         vma = ERR_PTR(ret);
3898 err_unpin:
3899         i915_gem_object_unpin_pages(obj);
3900         return vma;
3901 }
3902
3903 bool
3904 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3905                         bool force)
3906 {
3907         /* If we don't have a page list set up, then we're not pinned
3908          * to GPU, and we can ignore the cache flush because it'll happen
3909          * again at bind time.
3910          */
3911         if (obj->pages == NULL)
3912                 return false;
3913
3914         /*
3915          * Stolen memory is always coherent with the GPU as it is explicitly
3916          * marked as wc by the system, or the system is cache-coherent.
3917          */
3918         if (obj->stolen || obj->phys_handle)
3919                 return false;
3920
3921         /* If the GPU is snooping the contents of the CPU cache,
3922          * we do not need to manually clear the CPU cache lines.  However,
3923          * the caches are only snooped when the render cache is
3924          * flushed/invalidated.  As we always have to emit invalidations
3925          * and flushes when moving into and out of the RENDER domain, correct
3926          * snooping behaviour occurs naturally as the result of our domain
3927          * tracking.
3928          */
3929         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3930                 obj->cache_dirty = true;
3931                 return false;
3932         }
3933
3934         trace_i915_gem_object_clflush(obj);
3935         drm_clflush_sg(obj->pages);
3936         obj->cache_dirty = false;
3937
3938         return true;
3939 }
3940
3941 /** Flushes the GTT write domain for the object if it's dirty. */
3942 static void
3943 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3944 {
3945         uint32_t old_write_domain;
3946
3947         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3948                 return;
3949
3950         /* No actual flushing is required for the GTT write domain.  Writes
3951          * to it immediately go to main memory as far as we know, so there's
3952          * no chipset flush.  It also doesn't land in render cache.
3953          *
3954          * However, we do have to enforce the order so that all writes through
3955          * the GTT land before any writes to the device, such as updates to
3956          * the GATT itself.
3957          */
3958         wmb();
3959
3960         old_write_domain = obj->base.write_domain;
3961         obj->base.write_domain = 0;
3962
3963         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3964
3965         trace_i915_gem_object_change_domain(obj,
3966                                             obj->base.read_domains,
3967                                             old_write_domain);
3968 }
3969
3970 /** Flushes the CPU write domain for the object if it's dirty. */
3971 static void
3972 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3973 {
3974         uint32_t old_write_domain;
3975
3976         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3977                 return;
3978
3979         if (i915_gem_clflush_object(obj, obj->pin_display))
3980                 i915_gem_chipset_flush(to_i915(obj->base.dev));
3981
3982         old_write_domain = obj->base.write_domain;
3983         obj->base.write_domain = 0;
3984
3985         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3986
3987         trace_i915_gem_object_change_domain(obj,
3988                                             obj->base.read_domains,
3989                                             old_write_domain);
3990 }
3991
3992 /**
3993  * Moves a single object to the GTT read, and possibly write domain.
3994  * @obj: object to act on
3995  * @write: ask for write access or read only
3996  *
3997  * This function returns when the move is complete, including waiting on
3998  * flushes to occur.
3999  */
4000 int
4001 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
4002 {
4003         struct drm_device *dev = obj->base.dev;
4004         struct drm_i915_private *dev_priv = to_i915(dev);
4005         struct i915_ggtt *ggtt = &dev_priv->ggtt;
4006         uint32_t old_write_domain, old_read_domains;
4007         struct i915_vma *vma;
4008         int ret;
4009
4010         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
4011                 return 0;
4012
4013         ret = i915_gem_object_wait_rendering(obj, !write);
4014         if (ret)
4015                 return ret;
4016
4017         /* Flush and acquire obj->pages so that we are coherent through
4018          * direct access in memory with previous cached writes through
4019          * shmemfs and that our cache domain tracking remains valid.
4020          * For example, if the obj->filp was moved to swap without us
4021          * being notified and releasing the pages, we would mistakenly
4022          * continue to assume that the obj remained out of the CPU cached
4023          * domain.
4024          */
4025         ret = i915_gem_object_get_pages(obj);
4026         if (ret)
4027                 return ret;
4028
4029         i915_gem_object_flush_cpu_write_domain(obj);
4030
4031         /* Serialise direct access to this object with the barriers for
4032          * coherent writes from the GPU, by effectively invalidating the
4033          * GTT domain upon first access.
4034          */
4035         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
4036                 mb();
4037
4038         old_write_domain = obj->base.write_domain;
4039         old_read_domains = obj->base.read_domains;
4040
4041         /* It should now be out of any other write domains, and we can update
4042          * the domain values for our changes.
4043          */
4044         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
4045         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
4046         if (write) {
4047                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
4048                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
4049                 obj->dirty = 1;
4050         }
4051
4052         trace_i915_gem_object_change_domain(obj,
4053                                             old_read_domains,
4054                                             old_write_domain);
4055
4056         /* And bump the LRU for this access */
4057         vma = i915_gem_obj_to_ggtt(obj);
4058         if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
4059                 list_move_tail(&vma->vm_link,
4060                                &ggtt->base.inactive_list);
4061
4062         return 0;
4063 }
4064
4065 /**
4066  * Changes the cache-level of an object across all VMA.
4067  * @obj: object to act on
4068  * @cache_level: new cache level to set for the object
4069  *
4070  * After this function returns, the object will be in the new cache-level
4071  * across all GTT and the contents of the backing storage will be coherent,
4072  * with respect to the new cache-level. In order to keep the backing storage
4073  * coherent for all users, we only allow a single cache level to be set
4074  * globally on the object and prevent it from being changed whilst the
4075  * hardware is reading from the object. That is if the object is currently
4076  * on the scanout it will be set to uncached (or equivalent display
4077  * cache coherency) and all non-MOCS GPU access will also be uncached so
4078  * that all direct access to the scanout remains coherent.
4079  */
4080 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
4081                                     enum i915_cache_level cache_level)
4082 {
4083         struct drm_device *dev = obj->base.dev;
4084         struct i915_vma *vma, *next;
4085         bool bound = false;
4086         int ret = 0;
4087
4088         if (obj->cache_level == cache_level)
4089                 goto out;
4090
4091         /* Inspect the list of currently bound VMA and unbind any that would
4092          * be invalid given the new cache-level. This is principally to
4093          * catch the issue of the CS prefetch crossing page boundaries and
4094          * reading an invalid PTE on older architectures.
4095          */
4096         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4097                 if (!drm_mm_node_allocated(&vma->node))
4098                         continue;
4099
4100                 if (vma->pin_count) {
4101                         DRM_DEBUG("can not change the cache level of pinned objects\n");
4102                         return -EBUSY;
4103                 }
4104
4105                 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
4106                         ret = i915_vma_unbind(vma);
4107                         if (ret)
4108                                 return ret;
4109                 } else
4110                         bound = true;
4111         }
4112
4113         /* We can reuse the existing drm_mm nodes but need to change the
4114          * cache-level on the PTE. We could simply unbind them all and
4115          * rebind with the correct cache-level on next use. However since
4116          * we already have a valid slot, dma mapping, pages etc, we may as
4117          * rewrite the PTE in the belief that doing so tramples upon less
4118          * state and so involves less work.
4119          */
4120         if (bound) {
4121                 /* Before we change the PTE, the GPU must not be accessing it.
4122                  * If we wait upon the object, we know that all the bound
4123                  * VMA are no longer active.
4124                  */
4125                 ret = i915_gem_object_wait_rendering(obj, false);
4126                 if (ret)
4127                         return ret;
4128
4129                 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
4130                         /* Access to snoopable pages through the GTT is
4131                          * incoherent and on some machines causes a hard
4132                          * lockup. Relinquish the CPU mmaping to force
4133                          * userspace to refault in the pages and we can
4134                          * then double check if the GTT mapping is still
4135                          * valid for that pointer access.
4136                          */
4137                         i915_gem_release_mmap(obj);
4138
4139                         /* As we no longer need a fence for GTT access,
4140                          * we can relinquish it now (and so prevent having
4141                          * to steal a fence from someone else on the next
4142                          * fence request). Note GPU activity would have
4143                          * dropped the fence as all snoopable access is
4144                          * supposed to be linear.
4145                          */
4146                         ret = i915_gem_object_put_fence(obj);
4147                         if (ret)
4148                                 return ret;
4149                 } else {
4150                         /* We either have incoherent backing store and
4151                          * so no GTT access or the architecture is fully
4152                          * coherent. In such cases, existing GTT mmaps
4153                          * ignore the cache bit in the PTE and we can
4154                          * rewrite it without confusing the GPU or having
4155                          * to force userspace to fault back in its mmaps.
4156                          */
4157                 }
4158
4159                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
4160                         if (!drm_mm_node_allocated(&vma->node))
4161                                 continue;
4162
4163                         ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
4164                         if (ret)
4165                                 return ret;
4166                 }
4167         }
4168
4169         list_for_each_entry(vma, &obj->vma_list, obj_link)
4170                 vma->node.color = cache_level;
4171         obj->cache_level = cache_level;
4172
4173 out:
4174         /* Flush the dirty CPU caches to the backing storage so that the
4175          * object is now coherent at its new cache level (with respect
4176          * to the access domain).
4177          */
4178         if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
4179                 if (i915_gem_clflush_object(obj, true))
4180                         i915_gem_chipset_flush(to_i915(obj->base.dev));
4181         }
4182
4183         return 0;
4184 }
4185
4186 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
4187                                struct drm_file *file)
4188 {
4189         struct drm_i915_gem_caching *args = data;
4190         struct drm_i915_gem_object *obj;
4191
4192         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
4193         if (&obj->base == NULL)
4194                 return -ENOENT;
4195
4196         switch (obj->cache_level) {
4197         case I915_CACHE_LLC:
4198         case I915_CACHE_L3_LLC:
4199                 args->caching = I915_CACHING_CACHED;
4200                 break;
4201
4202         case I915_CACHE_WT:
4203                 args->caching = I915_CACHING_DISPLAY;
4204                 break;
4205
4206         default:
4207                 args->caching = I915_CACHING_NONE;
4208                 break;
4209         }
4210
4211         drm_gem_object_unreference_unlocked(&obj->base);
4212         return 0;
4213 }
4214
4215 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
4216                                struct drm_file *file)
4217 {
4218         struct drm_i915_private *dev_priv = dev->dev_private;
4219         struct drm_i915_gem_caching *args = data;
4220         struct drm_i915_gem_object *obj;
4221         enum i915_cache_level level;
4222         int ret;
4223
4224         switch (args->caching) {
4225         case I915_CACHING_NONE:
4226                 level = I915_CACHE_NONE;
4227                 break;
4228         case I915_CACHING_CACHED:
4229                 /*
4230                  * Due to a HW issue on BXT A stepping, GPU stores via a
4231                  * snooped mapping may leave stale data in a corresponding CPU
4232                  * cacheline, whereas normally such cachelines would get
4233                  * invalidated.
4234                  */
4235                 if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
4236                         return -ENODEV;
4237
4238                 level = I915_CACHE_LLC;
4239                 break;
4240         case I915_CACHING_DISPLAY:
4241                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
4242                 break;
4243         default:
4244                 return -EINVAL;
4245         }
4246
4247         intel_runtime_pm_get(dev_priv);
4248
4249         ret = i915_mutex_lock_interruptible(dev);
4250         if (ret)
4251                 goto rpm_put;
4252
4253         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
4254         if (&obj->base == NULL) {
4255                 ret = -ENOENT;
4256                 goto unlock;
4257         }
4258
4259         ret = i915_gem_object_set_cache_level(obj, level);
4260
4261         drm_gem_object_unreference(&obj->base);
4262 unlock:
4263         mutex_unlock(&dev->struct_mutex);
4264 rpm_put:
4265         intel_runtime_pm_put(dev_priv);
4266
4267         return ret;
4268 }
4269
4270 /*
4271  * Prepare buffer for display plane (scanout, cursors, etc).
4272  * Can be called from an uninterruptible phase (modesetting) and allows
4273  * any flushes to be pipelined (for pageflips).
4274  */
4275 int
4276 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
4277                                      u32 alignment,
4278                                      const struct i915_ggtt_view *view)
4279 {
4280         u32 old_read_domains, old_write_domain;
4281         int ret;
4282
4283         /* Mark the pin_display early so that we account for the
4284          * display coherency whilst setting up the cache domains.
4285          */
4286         obj->pin_display++;
4287
4288         /* The display engine is not coherent with the LLC cache on gen6.  As
4289          * a result, we make sure that the pinning that is about to occur is
4290          * done with uncached PTEs. This is lowest common denominator for all
4291          * chipsets.
4292          *
4293          * However for gen6+, we could do better by using the GFDT bit instead
4294          * of uncaching, which would allow us to flush all the LLC-cached data
4295          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
4296          */
4297         ret = i915_gem_object_set_cache_level(obj,
4298                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
4299         if (ret)
4300                 goto err_unpin_display;
4301
4302         /* As the user may map the buffer once pinned in the display plane
4303          * (e.g. libkms for the bootup splash), we have to ensure that we
4304          * always use map_and_fenceable for all scanout buffers.
4305          */
4306         ret = i915_gem_object_ggtt_pin(obj, view, alignment,
4307                                        view->type == I915_GGTT_VIEW_NORMAL ?
4308                                        PIN_MAPPABLE : 0);
4309         if (ret)
4310                 goto err_unpin_display;
4311
4312         i915_gem_object_flush_cpu_write_domain(obj);
4313
4314         old_write_domain = obj->base.write_domain;
4315         old_read_domains = obj->base.read_domains;
4316
4317         /* It should now be out of any other write domains, and we can update
4318          * the domain values for our changes.
4319          */
4320         obj->base.write_domain = 0;
4321         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
4322
4323         trace_i915_gem_object_change_domain(obj,
4324                                             old_read_domains,
4325                                             old_write_domain);
4326
4327         return 0;
4328
4329 err_unpin_display:
4330         obj->pin_display--;
4331         return ret;
4332 }
4333
4334 void
4335 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
4336                                          const struct i915_ggtt_view *view)
4337 {
4338         if (WARN_ON(obj->pin_display == 0))
4339                 return;
4340
4341         i915_gem_object_ggtt_unpin_view(obj, view);
4342
4343         obj->pin_display--;
4344 }
4345
4346 /**
4347  * Moves a single object to the CPU read, and possibly write domain.
4348  * @obj: object to act on
4349  * @write: requesting write or read-only access
4350  *
4351  * This function returns when the move is complete, including waiting on
4352  * flushes to occur.
4353  */
4354 int
4355 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4356 {
4357         uint32_t old_write_domain, old_read_domains;
4358         int ret;
4359
4360         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
4361                 return 0;
4362
4363         ret = i915_gem_object_wait_rendering(obj, !write);
4364         if (ret)
4365                 return ret;
4366
4367         i915_gem_object_flush_gtt_write_domain(obj);
4368
4369         old_write_domain = obj->base.write_domain;
4370         old_read_domains = obj->base.read_domains;
4371
4372         /* Flush the CPU cache if it's still invalid. */
4373         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4374                 i915_gem_clflush_object(obj, false);
4375
4376                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
4377         }
4378
4379         /* It should now be out of any other write domains, and we can update
4380          * the domain values for our changes.
4381          */
4382         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
4383
4384         /* If we're writing through the CPU, then the GPU read domains will
4385          * need to be invalidated at next use.
4386          */
4387         if (write) {
4388                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4389                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4390         }
4391
4392         trace_i915_gem_object_change_domain(obj,
4393                                             old_read_domains,
4394                                             old_write_domain);
4395
4396         return 0;
4397 }
4398
4399 /* Throttle our rendering by waiting until the ring has completed our requests
4400  * emitted over 20 msec ago.
4401  *
4402  * Note that if we were to use the current jiffies each time around the loop,
4403  * we wouldn't escape the function with any frames outstanding if the time to
4404  * render a frame was over 20ms.
4405  *
4406  * This should get us reasonable parallelism between CPU and GPU but also
4407  * relatively low latency when blocking on a particular request to finish.
4408  */
4409 static int
4410 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4411 {
4412         struct drm_i915_private *dev_priv = dev->dev_private;
4413         struct drm_i915_file_private *file_priv = file->driver_priv;
4414         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
4415         struct drm_i915_gem_request *request, *target = NULL;
4416         int ret;
4417
4418         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
4419         if (ret)
4420                 return ret;
4421
4422         /* ABI: return -EIO if already wedged */
4423         if (i915_terminally_wedged(&dev_priv->gpu_error))
4424                 return -EIO;
4425
4426         spin_lock(&file_priv->mm.lock);
4427         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
4428                 if (time_after_eq(request->emitted_jiffies, recent_enough))
4429                         break;
4430
4431                 /*
4432                  * Note that the request might not have been submitted yet.
4433                  * In which case emitted_jiffies will be zero.
4434                  */
4435                 if (!request->emitted_jiffies)
4436                         continue;
4437
4438                 target = request;
4439         }
4440         if (target)
4441                 i915_gem_request_reference(target);
4442         spin_unlock(&file_priv->mm.lock);
4443
4444         if (target == NULL)
4445                 return 0;
4446
4447         ret = __i915_wait_request(target, true, NULL, NULL);
4448         if (ret == 0)
4449                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4450
4451         i915_gem_request_unreference(target);
4452
4453         return ret;
4454 }
4455
4456 static bool
4457 i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
4458 {
4459         struct drm_i915_gem_object *obj = vma->obj;
4460
4461         if (alignment &&
4462             vma->node.start & (alignment - 1))
4463                 return true;
4464
4465         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
4466                 return true;
4467
4468         if (flags & PIN_OFFSET_BIAS &&
4469             vma->node.start < (flags & PIN_OFFSET_MASK))
4470                 return true;
4471
4472         if (flags & PIN_OFFSET_FIXED &&
4473             vma->node.start != (flags & PIN_OFFSET_MASK))
4474                 return true;
4475
4476         return false;
4477 }
4478
4479 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
4480 {
4481         struct drm_i915_gem_object *obj = vma->obj;
4482         bool mappable, fenceable;
4483         u32 fence_size, fence_alignment;
4484
4485         fence_size = i915_gem_get_gtt_size(obj->base.dev,
4486                                            obj->base.size,
4487                                            obj->tiling_mode);
4488         fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
4489                                                      obj->base.size,
4490                                                      obj->tiling_mode,
4491                                                      true);
4492
4493         fenceable = (vma->node.size == fence_size &&
4494                      (vma->node.start & (fence_alignment - 1)) == 0);
4495
4496         mappable = (vma->node.start + fence_size <=
4497                     to_i915(obj->base.dev)->ggtt.mappable_end);
4498
4499         obj->map_and_fenceable = mappable && fenceable;
4500 }
4501
4502 static int
4503 i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
4504                        struct i915_address_space *vm,
4505                        const struct i915_ggtt_view *ggtt_view,
4506                        uint32_t alignment,
4507                        uint64_t flags)
4508 {
4509         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4510         struct i915_vma *vma;
4511         unsigned bound;
4512         int ret;
4513
4514         if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
4515                 return -ENODEV;
4516
4517         if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
4518                 return -EINVAL;
4519
4520         if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
4521                 return -EINVAL;
4522
4523         if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
4524                 return -EINVAL;
4525
4526         vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
4527                           i915_gem_obj_to_vma(obj, vm);
4528
4529         if (vma) {
4530                 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
4531                         return -EBUSY;
4532
4533                 if (i915_vma_misplaced(vma, alignment, flags)) {
4534                         WARN(vma->pin_count,
4535                              "bo is already pinned in %s with incorrect alignment:"
4536                              " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
4537                              " obj->map_and_fenceable=%d\n",
4538                              ggtt_view ? "ggtt" : "ppgtt",
4539                              upper_32_bits(vma->node.start),
4540                              lower_32_bits(vma->node.start),
4541                              alignment,
4542                              !!(flags & PIN_MAPPABLE),
4543                              obj->map_and_fenceable);
4544                         ret = i915_vma_unbind(vma);
4545                         if (ret)
4546                                 return ret;
4547
4548                         vma = NULL;
4549                 }
4550         }
4551
4552         bound = vma ? vma->bound : 0;
4553         if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
4554                 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
4555                                                  flags);
4556                 if (IS_ERR(vma))
4557                         return PTR_ERR(vma);
4558         } else {
4559                 ret = i915_vma_bind(vma, obj->cache_level, flags);
4560                 if (ret)
4561                         return ret;
4562         }
4563
4564         if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
4565             (bound ^ vma->bound) & GLOBAL_BIND) {
4566                 __i915_vma_set_map_and_fenceable(vma);
4567                 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4568         }
4569
4570         vma->pin_count++;
4571         return 0;
4572 }
4573
4574 int
4575 i915_gem_object_pin(struct drm_i915_gem_object *obj,
4576                     struct i915_address_space *vm,
4577                     uint32_t alignment,
4578                     uint64_t flags)
4579 {
4580         return i915_gem_object_do_pin(obj, vm,
4581                                       i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
4582                                       alignment, flags);
4583 }
4584
4585 int
4586 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4587                          const struct i915_ggtt_view *view,
4588                          uint32_t alignment,
4589                          uint64_t flags)
4590 {
4591         struct drm_device *dev = obj->base.dev;
4592         struct drm_i915_private *dev_priv = to_i915(dev);
4593         struct i915_ggtt *ggtt = &dev_priv->ggtt;
4594
4595         BUG_ON(!view);
4596
4597         return i915_gem_object_do_pin(obj, &ggtt->base, view,
4598                                       alignment, flags | PIN_GLOBAL);
4599 }
4600
4601 void
4602 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
4603                                 const struct i915_ggtt_view *view)
4604 {
4605         struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
4606
4607         WARN_ON(vma->pin_count == 0);
4608         WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
4609
4610         --vma->pin_count;
4611 }
4612
4613 int
4614 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4615                     struct drm_file *file)
4616 {
4617         struct drm_i915_gem_busy *args = data;
4618         struct drm_i915_gem_object *obj;
4619         int ret;
4620
4621         ret = i915_mutex_lock_interruptible(dev);
4622         if (ret)
4623                 return ret;
4624
4625         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
4626         if (&obj->base == NULL) {
4627                 ret = -ENOENT;
4628                 goto unlock;
4629         }
4630
4631         /* Count all active objects as busy, even if they are currently not used
4632          * by the gpu. Users of this interface expect objects to eventually
4633          * become non-busy without any further actions, therefore emit any
4634          * necessary flushes here.
4635          */
4636         ret = i915_gem_object_flush_active(obj);
4637         if (ret)
4638                 goto unref;
4639
4640         args->busy = 0;
4641         if (obj->active) {
4642                 int i;
4643
4644                 for (i = 0; i < I915_NUM_ENGINES; i++) {
4645                         struct drm_i915_gem_request *req;
4646
4647                         req = obj->last_read_req[i];
4648                         if (req)
4649                                 args->busy |= 1 << (16 + req->engine->exec_id);
4650                 }
4651                 if (obj->last_write_req)
4652                         args->busy |= obj->last_write_req->engine->exec_id;
4653         }
4654
4655 unref:
4656         drm_gem_object_unreference(&obj->base);
4657 unlock:
4658         mutex_unlock(&dev->struct_mutex);
4659         return ret;
4660 }
4661
4662 int
4663 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4664                         struct drm_file *file_priv)
4665 {
4666         return i915_gem_ring_throttle(dev, file_priv);
4667 }
4668
4669 int
4670 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4671                        struct drm_file *file_priv)
4672 {
4673         struct drm_i915_private *dev_priv = dev->dev_private;
4674         struct drm_i915_gem_madvise *args = data;
4675         struct drm_i915_gem_object *obj;
4676         int ret;
4677
4678         switch (args->madv) {
4679         case I915_MADV_DONTNEED:
4680         case I915_MADV_WILLNEED:
4681             break;
4682         default:
4683             return -EINVAL;
4684         }
4685
4686         ret = i915_mutex_lock_interruptible(dev);
4687         if (ret)
4688                 return ret;
4689
4690         obj = to_intel_bo(drm_gem_object_lookup(file_priv, args->handle));
4691         if (&obj->base == NULL) {
4692                 ret = -ENOENT;
4693                 goto unlock;
4694         }
4695
4696         if (i915_gem_obj_is_pinned(obj)) {
4697                 ret = -EINVAL;
4698                 goto out;
4699         }
4700
4701         if (obj->pages &&
4702             obj->tiling_mode != I915_TILING_NONE &&
4703             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4704                 if (obj->madv == I915_MADV_WILLNEED)
4705                         i915_gem_object_unpin_pages(obj);
4706                 if (args->madv == I915_MADV_WILLNEED)
4707                         i915_gem_object_pin_pages(obj);
4708         }
4709
4710         if (obj->madv != __I915_MADV_PURGED)
4711                 obj->madv = args->madv;
4712
4713         /* if the object is no longer attached, discard its backing storage */
4714         if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4715                 i915_gem_object_truncate(obj);
4716
4717         args->retained = obj->madv != __I915_MADV_PURGED;
4718
4719 out:
4720         drm_gem_object_unreference(&obj->base);
4721 unlock:
4722         mutex_unlock(&dev->struct_mutex);
4723         return ret;
4724 }
4725
4726 void i915_gem_object_init(struct drm_i915_gem_object *obj,
4727                           const struct drm_i915_gem_object_ops *ops)
4728 {
4729         int i;
4730
4731         INIT_LIST_HEAD(&obj->global_list);
4732         for (i = 0; i < I915_NUM_ENGINES; i++)
4733                 INIT_LIST_HEAD(&obj->engine_list[i]);
4734         INIT_LIST_HEAD(&obj->obj_exec_link);
4735         INIT_LIST_HEAD(&obj->vma_list);
4736         INIT_LIST_HEAD(&obj->batch_pool_link);
4737
4738         obj->ops = ops;
4739
4740         obj->fence_reg = I915_FENCE_REG_NONE;
4741         obj->madv = I915_MADV_WILLNEED;
4742
4743         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4744 }
4745
4746 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4747         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
4748         .get_pages = i915_gem_object_get_pages_gtt,
4749         .put_pages = i915_gem_object_put_pages_gtt,
4750 };
4751
4752 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
4753                                                   size_t size)
4754 {
4755         struct drm_i915_gem_object *obj;
4756         struct address_space *mapping;
4757         gfp_t mask;
4758         int ret;
4759
4760         obj = i915_gem_object_alloc(dev);
4761         if (obj == NULL)
4762                 return ERR_PTR(-ENOMEM);
4763
4764         ret = drm_gem_object_init(dev, &obj->base, size);
4765         if (ret)
4766                 goto fail;
4767
4768         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4769         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4770                 /* 965gm cannot relocate objects above 4GiB. */
4771                 mask &= ~__GFP_HIGHMEM;
4772                 mask |= __GFP_DMA32;
4773         }
4774
4775         mapping = file_inode(obj->base.filp)->i_mapping;
4776         mapping_set_gfp_mask(mapping, mask);
4777
4778         i915_gem_object_init(obj, &i915_gem_object_ops);
4779
4780         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4781         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4782
4783         if (HAS_LLC(dev)) {
4784                 /* On some devices, we can have the GPU use the LLC (the CPU
4785                  * cache) for about a 10% performance improvement
4786                  * compared to uncached.  Graphics requests other than
4787                  * display scanout are coherent with the CPU in
4788                  * accessing this cache.  This means in this mode we
4789                  * don't need to clflush on the CPU side, and on the
4790                  * GPU side we only need to flush internal caches to
4791                  * get data visible to the CPU.
4792                  *
4793                  * However, we maintain the display planes as UC, and so
4794                  * need to rebind when first used as such.
4795                  */
4796                 obj->cache_level = I915_CACHE_LLC;
4797         } else
4798                 obj->cache_level = I915_CACHE_NONE;
4799
4800         trace_i915_gem_object_create(obj);
4801
4802         return obj;
4803
4804 fail:
4805         i915_gem_object_free(obj);
4806
4807         return ERR_PTR(ret);
4808 }
4809
4810 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4811 {
4812         /* If we are the last user of the backing storage (be it shmemfs
4813          * pages or stolen etc), we know that the pages are going to be
4814          * immediately released. In this case, we can then skip copying
4815          * back the contents from the GPU.
4816          */
4817
4818         if (obj->madv != I915_MADV_WILLNEED)
4819                 return false;
4820
4821         if (obj->base.filp == NULL)
4822                 return true;
4823
4824         /* At first glance, this looks racy, but then again so would be
4825          * userspace racing mmap against close. However, the first external
4826          * reference to the filp can only be obtained through the
4827          * i915_gem_mmap_ioctl() which safeguards us against the user
4828          * acquiring such a reference whilst we are in the middle of
4829          * freeing the object.
4830          */
4831         return atomic_long_read(&obj->base.filp->f_count) == 1;
4832 }
4833
4834 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4835 {
4836         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4837         struct drm_device *dev = obj->base.dev;
4838         struct drm_i915_private *dev_priv = dev->dev_private;
4839         struct i915_vma *vma, *next;
4840
4841         intel_runtime_pm_get(dev_priv);
4842
4843         trace_i915_gem_object_destroy(obj);
4844
4845         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4846                 int ret;
4847
4848                 vma->pin_count = 0;
4849                 ret = i915_vma_unbind(vma);
4850                 if (WARN_ON(ret == -ERESTARTSYS)) {
4851                         bool was_interruptible;
4852
4853                         was_interruptible = dev_priv->mm.interruptible;
4854                         dev_priv->mm.interruptible = false;
4855
4856                         WARN_ON(i915_vma_unbind(vma));
4857
4858                         dev_priv->mm.interruptible = was_interruptible;
4859                 }
4860         }
4861
4862         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4863          * before progressing. */
4864         if (obj->stolen)
4865                 i915_gem_object_unpin_pages(obj);
4866
4867         WARN_ON(obj->frontbuffer_bits);
4868
4869         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4870             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4871             obj->tiling_mode != I915_TILING_NONE)
4872                 i915_gem_object_unpin_pages(obj);
4873
4874         if (WARN_ON(obj->pages_pin_count))
4875                 obj->pages_pin_count = 0;
4876         if (discard_backing_storage(obj))
4877                 obj->madv = I915_MADV_DONTNEED;
4878         i915_gem_object_put_pages(obj);
4879         i915_gem_object_free_mmap_offset(obj);
4880
4881         BUG_ON(obj->pages);
4882
4883         if (obj->base.import_attach)
4884                 drm_prime_gem_destroy(&obj->base, NULL);
4885
4886         if (obj->ops->release)
4887                 obj->ops->release(obj);
4888
4889         drm_gem_object_release(&obj->base);
4890         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4891
4892         kfree(obj->bit_17);
4893         i915_gem_object_free(obj);
4894
4895         intel_runtime_pm_put(dev_priv);
4896 }
4897
4898 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4899                                      struct i915_address_space *vm)
4900 {
4901         struct i915_vma *vma;
4902         list_for_each_entry(vma, &obj->vma_list, obj_link) {
4903                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4904                     vma->vm == vm)
4905                         return vma;
4906         }
4907         return NULL;
4908 }
4909
4910 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4911                                            const struct i915_ggtt_view *view)
4912 {
4913         struct i915_vma *vma;
4914
4915         GEM_BUG_ON(!view);
4916
4917         list_for_each_entry(vma, &obj->vma_list, obj_link)
4918                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
4919                         return vma;
4920         return NULL;
4921 }
4922
4923 void i915_gem_vma_destroy(struct i915_vma *vma)
4924 {
4925         WARN_ON(vma->node.allocated);
4926
4927         /* Keep the vma as a placeholder in the execbuffer reservation lists */
4928         if (!list_empty(&vma->exec_list))
4929                 return;
4930
4931         if (!vma->is_ggtt)
4932                 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
4933
4934         list_del(&vma->obj_link);
4935
4936         kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
4937 }
4938
4939 static void
4940 i915_gem_stop_engines(struct drm_device *dev)
4941 {
4942         struct drm_i915_private *dev_priv = dev->dev_private;
4943         struct intel_engine_cs *engine;
4944
4945         for_each_engine(engine, dev_priv)
4946                 dev_priv->gt.stop_engine(engine);
4947 }
4948
4949 int
4950 i915_gem_suspend(struct drm_device *dev)
4951 {
4952         struct drm_i915_private *dev_priv = dev->dev_private;
4953         int ret = 0;
4954
4955         mutex_lock(&dev->struct_mutex);
4956         ret = i915_gpu_idle(dev);
4957         if (ret)
4958                 goto err;
4959
4960         i915_gem_retire_requests(dev_priv);
4961
4962         i915_gem_stop_engines(dev);
4963         i915_gem_context_lost(dev_priv);
4964         mutex_unlock(&dev->struct_mutex);
4965
4966         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4967         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4968         flush_delayed_work(&dev_priv->mm.idle_work);
4969
4970         /* Assert that we sucessfully flushed all the work and
4971          * reset the GPU back to its idle, low power state.
4972          */
4973         WARN_ON(dev_priv->mm.busy);
4974
4975         return 0;
4976
4977 err:
4978         mutex_unlock(&dev->struct_mutex);
4979         return ret;
4980 }
4981
4982 void i915_gem_init_swizzling(struct drm_device *dev)
4983 {
4984         struct drm_i915_private *dev_priv = dev->dev_private;
4985
4986         if (INTEL_INFO(dev)->gen < 5 ||
4987             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4988                 return;
4989
4990         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4991                                  DISP_TILE_SURFACE_SWIZZLING);
4992
4993         if (IS_GEN5(dev))
4994                 return;
4995
4996         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4997         if (IS_GEN6(dev))
4998                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4999         else if (IS_GEN7(dev))
5000                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
5001         else if (IS_GEN8(dev))
5002                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
5003         else
5004                 BUG();
5005 }
5006
5007 static void init_unused_ring(struct drm_device *dev, u32 base)
5008 {
5009         struct drm_i915_private *dev_priv = dev->dev_private;
5010
5011         I915_WRITE(RING_CTL(base), 0);
5012         I915_WRITE(RING_HEAD(base), 0);
5013         I915_WRITE(RING_TAIL(base), 0);
5014         I915_WRITE(RING_START(base), 0);
5015 }
5016
5017 static void init_unused_rings(struct drm_device *dev)
5018 {
5019         if (IS_I830(dev)) {
5020                 init_unused_ring(dev, PRB1_BASE);
5021                 init_unused_ring(dev, SRB0_BASE);
5022                 init_unused_ring(dev, SRB1_BASE);
5023                 init_unused_ring(dev, SRB2_BASE);
5024                 init_unused_ring(dev, SRB3_BASE);
5025         } else if (IS_GEN2(dev)) {
5026                 init_unused_ring(dev, SRB0_BASE);
5027                 init_unused_ring(dev, SRB1_BASE);
5028         } else if (IS_GEN3(dev)) {
5029                 init_unused_ring(dev, PRB1_BASE);
5030                 init_unused_ring(dev, PRB2_BASE);
5031         }
5032 }
5033
5034 int i915_gem_init_engines(struct drm_device *dev)
5035 {
5036         struct drm_i915_private *dev_priv = dev->dev_private;
5037         int ret;
5038
5039         ret = intel_init_render_ring_buffer(dev);
5040         if (ret)
5041                 return ret;
5042
5043         if (HAS_BSD(dev)) {
5044                 ret = intel_init_bsd_ring_buffer(dev);
5045                 if (ret)
5046                         goto cleanup_render_ring;
5047         }
5048
5049         if (HAS_BLT(dev)) {
5050                 ret = intel_init_blt_ring_buffer(dev);
5051                 if (ret)
5052                         goto cleanup_bsd_ring;
5053         }
5054
5055         if (HAS_VEBOX(dev)) {
5056                 ret = intel_init_vebox_ring_buffer(dev);
5057                 if (ret)
5058                         goto cleanup_blt_ring;
5059         }
5060
5061         if (HAS_BSD2(dev)) {
5062                 ret = intel_init_bsd2_ring_buffer(dev);
5063                 if (ret)
5064                         goto cleanup_vebox_ring;
5065         }
5066
5067         return 0;
5068
5069 cleanup_vebox_ring:
5070         intel_cleanup_engine(&dev_priv->engine[VECS]);
5071 cleanup_blt_ring:
5072         intel_cleanup_engine(&dev_priv->engine[BCS]);
5073 cleanup_bsd_ring:
5074         intel_cleanup_engine(&dev_priv->engine[VCS]);
5075 cleanup_render_ring:
5076         intel_cleanup_engine(&dev_priv->engine[RCS]);
5077
5078         return ret;
5079 }
5080
5081 int
5082 i915_gem_init_hw(struct drm_device *dev)
5083 {
5084         struct drm_i915_private *dev_priv = dev->dev_private;
5085         struct intel_engine_cs *engine;
5086         int ret;
5087
5088         /* Double layer security blanket, see i915_gem_init() */
5089         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5090
5091         if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
5092                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
5093
5094         if (IS_HASWELL(dev))
5095                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
5096                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
5097
5098         if (HAS_PCH_NOP(dev)) {
5099                 if (IS_IVYBRIDGE(dev)) {
5100                         u32 temp = I915_READ(GEN7_MSG_CTL);
5101                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
5102                         I915_WRITE(GEN7_MSG_CTL, temp);
5103                 } else if (INTEL_INFO(dev)->gen >= 7) {
5104                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
5105                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
5106                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
5107                 }
5108         }
5109
5110         i915_gem_init_swizzling(dev);
5111
5112         /*
5113          * At least 830 can leave some of the unused rings
5114          * "active" (ie. head != tail) after resume which
5115          * will prevent c3 entry. Makes sure all unused rings
5116          * are totally idle.
5117          */
5118         init_unused_rings(dev);
5119
5120         BUG_ON(!dev_priv->kernel_context);
5121
5122         ret = i915_ppgtt_init_hw(dev);
5123         if (ret) {
5124                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
5125                 goto out;
5126         }
5127
5128         /* Need to do basic initialisation of all rings first: */
5129         for_each_engine(engine, dev_priv) {
5130                 ret = engine->init_hw(engine);
5131                 if (ret)
5132                         goto out;
5133         }
5134
5135         intel_mocs_init_l3cc_table(dev);
5136
5137         /* We can't enable contexts until all firmware is loaded */
5138         ret = intel_guc_setup(dev);
5139         if (ret)
5140                 goto out;
5141
5142         /*
5143          * Increment the next seqno by 0x100 so we have a visible break
5144          * on re-initialisation
5145          */
5146         ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
5147
5148 out:
5149         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5150         return ret;
5151 }
5152
5153 int i915_gem_init(struct drm_device *dev)
5154 {
5155         struct drm_i915_private *dev_priv = dev->dev_private;
5156         int ret;
5157
5158         mutex_lock(&dev->struct_mutex);
5159
5160         if (!i915.enable_execlists) {
5161                 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
5162                 dev_priv->gt.init_engines = i915_gem_init_engines;
5163                 dev_priv->gt.cleanup_engine = intel_cleanup_engine;
5164                 dev_priv->gt.stop_engine = intel_stop_engine;
5165         } else {
5166                 dev_priv->gt.execbuf_submit = intel_execlists_submission;
5167                 dev_priv->gt.init_engines = intel_logical_rings_init;
5168                 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
5169                 dev_priv->gt.stop_engine = intel_logical_ring_stop;
5170         }
5171
5172         /* This is just a security blanket to placate dragons.
5173          * On some systems, we very sporadically observe that the first TLBs
5174          * used by the CS may be stale, despite us poking the TLB reset. If
5175          * we hold the forcewake during initialisation these problems
5176          * just magically go away.
5177          */
5178         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5179
5180         i915_gem_init_userptr(dev_priv);
5181         i915_gem_init_ggtt(dev);
5182
5183         ret = i915_gem_context_init(dev);
5184         if (ret)
5185                 goto out_unlock;
5186
5187         ret = dev_priv->gt.init_engines(dev);
5188         if (ret)
5189                 goto out_unlock;
5190
5191         ret = i915_gem_init_hw(dev);
5192         if (ret == -EIO) {
5193                 /* Allow ring initialisation to fail by marking the GPU as
5194                  * wedged. But we only want to do this where the GPU is angry,
5195                  * for all other failure, such as an allocation failure, bail.
5196                  */
5197                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
5198                 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
5199                 ret = 0;
5200         }
5201
5202 out_unlock:
5203         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5204         mutex_unlock(&dev->struct_mutex);
5205
5206         return ret;
5207 }
5208
5209 void
5210 i915_gem_cleanup_engines(struct drm_device *dev)
5211 {
5212         struct drm_i915_private *dev_priv = dev->dev_private;
5213         struct intel_engine_cs *engine;
5214
5215         for_each_engine(engine, dev_priv)
5216                 dev_priv->gt.cleanup_engine(engine);
5217 }
5218
5219 static void
5220 init_engine_lists(struct intel_engine_cs *engine)
5221 {
5222         INIT_LIST_HEAD(&engine->active_list);
5223         INIT_LIST_HEAD(&engine->request_list);
5224 }
5225
5226 void
5227 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
5228 {
5229         struct drm_device *dev = dev_priv->dev;
5230
5231         if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
5232             !IS_CHERRYVIEW(dev_priv))
5233                 dev_priv->num_fence_regs = 32;
5234         else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
5235                  IS_I945GM(dev_priv) || IS_G33(dev_priv))
5236                 dev_priv->num_fence_regs = 16;
5237         else
5238                 dev_priv->num_fence_regs = 8;
5239
5240         if (intel_vgpu_active(dev_priv))
5241                 dev_priv->num_fence_regs =
5242                                 I915_READ(vgtif_reg(avail_rs.fence_num));
5243
5244         /* Initialize fence registers to zero */
5245         i915_gem_restore_fences(dev);
5246
5247         i915_gem_detect_bit_6_swizzle(dev);
5248 }
5249
5250 void
5251 i915_gem_load_init(struct drm_device *dev)
5252 {
5253         struct drm_i915_private *dev_priv = dev->dev_private;
5254         int i;
5255
5256         dev_priv->objects =
5257                 kmem_cache_create("i915_gem_object",
5258                                   sizeof(struct drm_i915_gem_object), 0,
5259                                   SLAB_HWCACHE_ALIGN,
5260                                   NULL);
5261         dev_priv->vmas =
5262                 kmem_cache_create("i915_gem_vma",
5263                                   sizeof(struct i915_vma), 0,
5264                                   SLAB_HWCACHE_ALIGN,
5265                                   NULL);
5266         dev_priv->requests =
5267                 kmem_cache_create("i915_gem_request",
5268                                   sizeof(struct drm_i915_gem_request), 0,
5269                                   SLAB_HWCACHE_ALIGN,
5270                                   NULL);
5271
5272         INIT_LIST_HEAD(&dev_priv->vm_list);
5273         INIT_LIST_HEAD(&dev_priv->context_list);
5274         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
5275         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
5276         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5277         for (i = 0; i < I915_NUM_ENGINES; i++)
5278                 init_engine_lists(&dev_priv->engine[i]);
5279         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
5280                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
5281         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
5282                           i915_gem_retire_work_handler);
5283         INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
5284                           i915_gem_idle_work_handler);
5285         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5286
5287         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
5288
5289         /*
5290          * Set initial sequence number for requests.
5291          * Using this number allows the wraparound to happen early,
5292          * catching any obvious problems.
5293          */
5294         dev_priv->next_seqno = ((u32)~0 - 0x1100);
5295         dev_priv->last_seqno = ((u32)~0 - 0x1101);
5296
5297         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5298
5299         init_waitqueue_head(&dev_priv->pending_flip_queue);
5300
5301         dev_priv->mm.interruptible = true;
5302
5303         mutex_init(&dev_priv->fb_tracking.lock);
5304 }
5305
5306 void i915_gem_load_cleanup(struct drm_device *dev)
5307 {
5308         struct drm_i915_private *dev_priv = to_i915(dev);
5309
5310         kmem_cache_destroy(dev_priv->requests);
5311         kmem_cache_destroy(dev_priv->vmas);
5312         kmem_cache_destroy(dev_priv->objects);
5313 }
5314
5315 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
5316 {
5317         struct drm_i915_gem_object *obj;
5318
5319         /* Called just before we write the hibernation image.
5320          *
5321          * We need to update the domain tracking to reflect that the CPU
5322          * will be accessing all the pages to create and restore from the
5323          * hibernation, and so upon restoration those pages will be in the
5324          * CPU domain.
5325          *
5326          * To make sure the hibernation image contains the latest state,
5327          * we update that state just before writing out the image.
5328          */
5329
5330         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
5331                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
5332                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
5333         }
5334
5335         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
5336                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
5337                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
5338         }
5339
5340         return 0;
5341 }
5342
5343 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5344 {
5345         struct drm_i915_file_private *file_priv = file->driver_priv;
5346
5347         /* Clean up our request list when the client is going away, so that
5348          * later retire_requests won't dereference our soon-to-be-gone
5349          * file_priv.
5350          */
5351         spin_lock(&file_priv->mm.lock);
5352         while (!list_empty(&file_priv->mm.request_list)) {
5353                 struct drm_i915_gem_request *request;
5354
5355                 request = list_first_entry(&file_priv->mm.request_list,
5356                                            struct drm_i915_gem_request,
5357                                            client_list);
5358                 list_del(&request->client_list);
5359                 request->file_priv = NULL;
5360         }
5361         spin_unlock(&file_priv->mm.lock);
5362
5363         if (!list_empty(&file_priv->rps.link)) {
5364                 spin_lock(&to_i915(dev)->rps.client_lock);
5365                 list_del(&file_priv->rps.link);
5366                 spin_unlock(&to_i915(dev)->rps.client_lock);
5367         }
5368 }
5369
5370 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
5371 {
5372         struct drm_i915_file_private *file_priv;
5373         int ret;
5374
5375         DRM_DEBUG_DRIVER("\n");
5376
5377         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5378         if (!file_priv)
5379                 return -ENOMEM;
5380
5381         file->driver_priv = file_priv;
5382         file_priv->dev_priv = dev->dev_private;
5383         file_priv->file = file;
5384         INIT_LIST_HEAD(&file_priv->rps.link);
5385
5386         spin_lock_init(&file_priv->mm.lock);
5387         INIT_LIST_HEAD(&file_priv->mm.request_list);
5388
5389         file_priv->bsd_ring = -1;
5390
5391         ret = i915_gem_context_open(dev, file);
5392         if (ret)
5393                 kfree(file_priv);
5394
5395         return ret;
5396 }
5397
5398 /**
5399  * i915_gem_track_fb - update frontbuffer tracking
5400  * @old: current GEM buffer for the frontbuffer slots
5401  * @new: new GEM buffer for the frontbuffer slots
5402  * @frontbuffer_bits: bitmask of frontbuffer slots
5403  *
5404  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5405  * from @old and setting them in @new. Both @old and @new can be NULL.
5406  */
5407 void i915_gem_track_fb(struct drm_i915_gem_object *old,
5408                        struct drm_i915_gem_object *new,
5409                        unsigned frontbuffer_bits)
5410 {
5411         if (old) {
5412                 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
5413                 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
5414                 old->frontbuffer_bits &= ~frontbuffer_bits;
5415         }
5416
5417         if (new) {
5418                 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
5419                 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
5420                 new->frontbuffer_bits |= frontbuffer_bits;
5421         }
5422 }
5423
5424 /* All the new VM stuff */
5425 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
5426                         struct i915_address_space *vm)
5427 {
5428         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5429         struct i915_vma *vma;
5430
5431         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5432
5433         list_for_each_entry(vma, &o->vma_list, obj_link) {
5434                 if (vma->is_ggtt &&
5435                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5436                         continue;
5437                 if (vma->vm == vm)
5438                         return vma->node.start;
5439         }
5440
5441         WARN(1, "%s vma for this object not found.\n",
5442              i915_is_ggtt(vm) ? "global" : "ppgtt");
5443         return -1;
5444 }
5445
5446 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
5447                                   const struct i915_ggtt_view *view)
5448 {
5449         struct i915_vma *vma;
5450
5451         list_for_each_entry(vma, &o->vma_list, obj_link)
5452                 if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
5453                         return vma->node.start;
5454
5455         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
5456         return -1;
5457 }
5458
5459 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
5460                         struct i915_address_space *vm)
5461 {
5462         struct i915_vma *vma;
5463
5464         list_for_each_entry(vma, &o->vma_list, obj_link) {
5465                 if (vma->is_ggtt &&
5466                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5467                         continue;
5468                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5469                         return true;
5470         }
5471
5472         return false;
5473 }
5474
5475 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
5476                                   const struct i915_ggtt_view *view)
5477 {
5478         struct i915_vma *vma;
5479
5480         list_for_each_entry(vma, &o->vma_list, obj_link)
5481                 if (vma->is_ggtt &&
5482                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
5483                     drm_mm_node_allocated(&vma->node))
5484                         return true;
5485
5486         return false;
5487 }
5488
5489 bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5490 {
5491         struct i915_vma *vma;
5492
5493         list_for_each_entry(vma, &o->vma_list, obj_link)
5494                 if (drm_mm_node_allocated(&vma->node))
5495                         return true;
5496
5497         return false;
5498 }
5499
5500 unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o)
5501 {
5502         struct i915_vma *vma;
5503
5504         GEM_BUG_ON(list_empty(&o->vma_list));
5505
5506         list_for_each_entry(vma, &o->vma_list, obj_link) {
5507                 if (vma->is_ggtt &&
5508                     vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
5509                         return vma->node.size;
5510         }
5511
5512         return 0;
5513 }
5514
5515 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
5516 {
5517         struct i915_vma *vma;
5518         list_for_each_entry(vma, &obj->vma_list, obj_link)
5519                 if (vma->pin_count > 0)
5520                         return true;
5521
5522         return false;
5523 }
5524
5525 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
5526 struct page *
5527 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
5528 {
5529         struct page *page;
5530
5531         /* Only default objects have per-page dirty tracking */
5532         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
5533                 return NULL;
5534
5535         page = i915_gem_object_get_page(obj, n);
5536         set_page_dirty(page);
5537         return page;
5538 }
5539
5540 /* Allocate a new GEM object and fill it with the supplied data */
5541 struct drm_i915_gem_object *
5542 i915_gem_object_create_from_data(struct drm_device *dev,
5543                                  const void *data, size_t size)
5544 {
5545         struct drm_i915_gem_object *obj;
5546         struct sg_table *sg;
5547         size_t bytes;
5548         int ret;
5549
5550         obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
5551         if (IS_ERR(obj))
5552                 return obj;
5553
5554         ret = i915_gem_object_set_to_cpu_domain(obj, true);
5555         if (ret)
5556                 goto fail;
5557
5558         ret = i915_gem_object_get_pages(obj);
5559         if (ret)
5560                 goto fail;
5561
5562         i915_gem_object_pin_pages(obj);
5563         sg = obj->pages;
5564         bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
5565         obj->dirty = 1;         /* Backing store is now out of date */
5566         i915_gem_object_unpin_pages(obj);
5567
5568         if (WARN_ON(bytes != size)) {
5569                 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
5570                 ret = -EFAULT;
5571                 goto fail;
5572         }
5573
5574         return obj;
5575
5576 fail:
5577         drm_gem_object_unreference(&obj->base);
5578         return ERR_PTR(ret);
5579 }