drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008-2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include <drm/drmP.h>
  29 #include <drm/drm_vma_manager.h>
  30 #include <drm/i915_drm.h>
  31 #include "i915_drv.h"
  32 #include "i915_gem_dmabuf.h"
  33 #include "i915_vgpu.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36 #include "intel_frontbuffer.h"
  37 #include "intel_mocs.h"
  38 #include <linux/reservation.h>
  39 #include <linux/shmem_fs.h>
  40 #include <linux/slab.h>
  41 #include <linux/swap.h>
  42 #include <linux/pci.h>
  43 #include <linux/dma-buf.h>
  44
  45 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  46 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  47
  48 static bool cpu_cache_is_coherent(struct drm_device *dev,
  49                                   enum i915_cache_level level)
  50 {
  51         return HAS_LLC(dev) || level != I915_CACHE_NONE;
  52 }
  53
  54 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
  55 {
  56         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
  57                 return false;
  58
  59         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
  60                 return true;
  61
  62         return obj->pin_display;
  63 }
  64
  65 static int
  66 insert_mappable_node(struct drm_i915_private *i915,
  67                      struct drm_mm_node *node, u32 size)
  68 {
  69         memset(node, 0, sizeof(*node));
  70         return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
  71                                                    size, 0, 0, 0,
  72                                                    i915->ggtt.mappable_end,
  73                                                    DRM_MM_SEARCH_DEFAULT,
  74                                                    DRM_MM_CREATE_DEFAULT);
  75 }
  76
  77 static void
  78 remove_mappable_node(struct drm_mm_node *node)
  79 {
  80         drm_mm_remove_node(node);
  81 }
  82
  83 /* some bookkeeping */
  84 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  85                                   size_t size)
  86 {
  87         spin_lock(&dev_priv->mm.object_stat_lock);
  88         dev_priv->mm.object_count++;
  89         dev_priv->mm.object_memory += size;
  90         spin_unlock(&dev_priv->mm.object_stat_lock);
  91 }
  92
  93 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  94                                      size_t size)
  95 {
  96         spin_lock(&dev_priv->mm.object_stat_lock);
  97         dev_priv->mm.object_count--;
  98         dev_priv->mm.object_memory -= size;
  99         spin_unlock(&dev_priv->mm.object_stat_lock);
 100 }
 101
 102 static int
 103 i915_gem_wait_for_error(struct i915_gpu_error *error)
 104 {
 105         int ret;
 106
 107         if (!i915_reset_in_progress(error))
 108                 return 0;
 109
 110         /*
 111          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 112          * userspace. If it takes that long something really bad is going on and
 113          * we should simply try to bail out and fail as gracefully as possible.
 114          */
 115         ret = wait_event_interruptible_timeout(error->reset_queue,
 116                                                !i915_reset_in_progress(error),
 117                                                10*HZ);
 118         if (ret == 0) {
 119                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 120                 return -EIO;
 121         } else if (ret < 0) {
 122                 return ret;
 123         } else {
 124                 return 0;
 125         }
 126 }
 127
 128 int i915_mutex_lock_interruptible(struct drm_device *dev)
 129 {
 130         struct drm_i915_private *dev_priv = to_i915(dev);
 131         int ret;
 132
 133         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
 134         if (ret)
 135                 return ret;
 136
 137         ret = mutex_lock_interruptible(&dev->struct_mutex);
 138         if (ret)
 139                 return ret;
 140
 141         return 0;
 142 }
 143
 144 int
 145 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 146                             struct drm_file *file)
 147 {
 148         struct drm_i915_private *dev_priv = to_i915(dev);
 149         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 150         struct drm_i915_gem_get_aperture *args = data;
 151         struct i915_vma *vma;
 152         size_t pinned;
 153
 154         pinned = 0;
 155         mutex_lock(&dev->struct_mutex);
 156         list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
 157                 if (i915_vma_is_pinned(vma))
 158                         pinned += vma->node.size;
 159         list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
 160                 if (i915_vma_is_pinned(vma))
 161                         pinned += vma->node.size;
 162         mutex_unlock(&dev->struct_mutex);
 163
 164         args->aper_size = ggtt->base.total;
 165         args->aper_available_size = args->aper_size - pinned;
 166
 167         return 0;
 168 }
 169
 170 static int
 171 i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 172 {
 173         struct address_space *mapping = obj->base.filp->f_mapping;
 174         char *vaddr = obj->phys_handle->vaddr;
 175         struct sg_table *st;
 176         struct scatterlist *sg;
 177         int i;
 178
 179         if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
 180                 return -EINVAL;
 181
 182         for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 183                 struct page *page;
 184                 char *src;
 185
 186                 page = shmem_read_mapping_page(mapping, i);
 187                 if (IS_ERR(page))
 188                         return PTR_ERR(page);
 189
 190                 src = kmap_atomic(page);
 191                 memcpy(vaddr, src, PAGE_SIZE);
 192                 drm_clflush_virt_range(vaddr, PAGE_SIZE);
 193                 kunmap_atomic(src);
 194
 195                 put_page(page);
 196                 vaddr += PAGE_SIZE;
 197         }
 198
 199         i915_gem_chipset_flush(to_i915(obj->base.dev));
 200
 201         st = kmalloc(sizeof(*st), GFP_KERNEL);
 202         if (st == NULL)
 203                 return -ENOMEM;
 204
 205         if (sg_alloc_table(st, 1, GFP_KERNEL)) {
 206                 kfree(st);
 207                 return -ENOMEM;
 208         }
 209
 210         sg = st->sgl;
 211         sg->offset = 0;
 212         sg->length = obj->base.size;
 213
 214         sg_dma_address(sg) = obj->phys_handle->busaddr;
 215         sg_dma_len(sg) = obj->base.size;
 216
 217         obj->pages = st;
 218         return 0;
 219 }
 220
 221 static void
 222 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
 223 {
 224         int ret;
 225
 226         BUG_ON(obj->madv == __I915_MADV_PURGED);
 227
 228         ret = i915_gem_object_set_to_cpu_domain(obj, true);
 229         if (WARN_ON(ret)) {
 230                 /* In the event of a disaster, abandon all caches and
 231                  * hope for the best.
 232                  */
 233                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 234         }
 235
 236         if (obj->madv == I915_MADV_DONTNEED)
 237                 obj->dirty = 0;
 238
 239         if (obj->dirty) {
 240                 struct address_space *mapping = obj->base.filp->f_mapping;
 241                 char *vaddr = obj->phys_handle->vaddr;
 242                 int i;
 243
 244                 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
 245                         struct page *page;
 246                         char *dst;
 247
 248                         page = shmem_read_mapping_page(mapping, i);
 249                         if (IS_ERR(page))
 250                                 continue;
 251
 252                         dst = kmap_atomic(page);
 253                         drm_clflush_virt_range(vaddr, PAGE_SIZE);
 254                         memcpy(dst, vaddr, PAGE_SIZE);
 255                         kunmap_atomic(dst);
 256
 257                         set_page_dirty(page);
 258                         if (obj->madv == I915_MADV_WILLNEED)
 259                                 mark_page_accessed(page);
 260                         put_page(page);
 261                         vaddr += PAGE_SIZE;
 262                 }
 263                 obj->dirty = 0;
 264         }
 265
 266         sg_free_table(obj->pages);
 267         kfree(obj->pages);
 268 }
 269
 270 static void
 271 i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
 272 {
 273         drm_pci_free(obj->base.dev, obj->phys_handle);
 274 }
 275
 276 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
 277         .get_pages = i915_gem_object_get_pages_phys,
 278         .put_pages = i915_gem_object_put_pages_phys,
 279         .release = i915_gem_object_release_phys,
 280 };
 281
 282 int
 283 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 284 {
 285         struct i915_vma *vma;
 286         LIST_HEAD(still_in_list);
 287         int ret;
 288
 289         /* The vma will only be freed if it is marked as closed, and if we wait
 290          * upon rendering to the vma, we may unbind anything in the list.
 291          */
 292         while ((vma = list_first_entry_or_null(&obj->vma_list,
 293                                                struct i915_vma,
 294                                                obj_link))) {
 295                 list_move_tail(&vma->obj_link, &still_in_list);
 296                 ret = i915_vma_unbind(vma);
 297                 if (ret)
 298                         break;
 299         }
 300         list_splice(&still_in_list, &obj->vma_list);
 301
 302         return ret;
 303 }
 304
 305 /**
 306  * Ensures that all rendering to the object has completed and the object is
 307  * safe to unbind from the GTT or access from the CPU.
 308  * @obj: i915 gem object
 309  * @readonly: waiting for just read access or read-write access
 310  */
 311 int
 312 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
 313                                bool readonly)
 314 {
 315         struct reservation_object *resv;
 316         struct i915_gem_active *active;
 317         unsigned long active_mask;
 318         int idx;
 319
 320         lockdep_assert_held(&obj->base.dev->struct_mutex);
 321
 322         if (!readonly) {
 323                 active = obj->last_read;
 324                 active_mask = i915_gem_object_get_active(obj);
 325         } else {
 326                 active_mask = 1;
 327                 active = &obj->last_write;
 328         }
 329
 330         for_each_active(active_mask, idx) {
 331                 int ret;
 332
 333                 ret = i915_gem_active_wait(&active[idx],
 334                                            &obj->base.dev->struct_mutex);
 335                 if (ret)
 336                         return ret;
 337         }
 338
 339         resv = i915_gem_object_get_dmabuf_resv(obj);
 340         if (resv) {
 341                 long err;
 342
 343                 err = reservation_object_wait_timeout_rcu(resv, !readonly, true,
 344                                                           MAX_SCHEDULE_TIMEOUT);
 345                 if (err < 0)
 346                         return err;
 347         }
 348
 349         return 0;
 350 }
 351
 352 /* A nonblocking variant of the above wait. Must be called prior to
 353  * acquiring the mutex for the object, as the object state may change
 354  * during this call. A reference must be held by the caller for the object.
 355  */
 356 static __must_check int
 357 __unsafe_wait_rendering(struct drm_i915_gem_object *obj,
 358                         struct intel_rps_client *rps,
 359                         bool readonly)
 360 {
 361         struct i915_gem_active *active;
 362         unsigned long active_mask;
 363         int idx;
 364
 365         active_mask = __I915_BO_ACTIVE(obj);
 366         if (!active_mask)
 367                 return 0;
 368
 369         if (!readonly) {
 370                 active = obj->last_read;
 371         } else {
 372                 active_mask = 1;
 373                 active = &obj->last_write;
 374         }
 375
 376         for_each_active(active_mask, idx) {
 377                 int ret;
 378
 379                 ret = i915_gem_active_wait_unlocked(&active[idx],
 380                                                     true, NULL, rps);
 381                 if (ret)
 382                         return ret;
 383         }
 384
 385         return 0;
 386 }
 387
 388 static struct intel_rps_client *to_rps_client(struct drm_file *file)
 389 {
 390         struct drm_i915_file_private *fpriv = file->driver_priv;
 391
 392         return &fpriv->rps;
 393 }
 394
 395 int
 396 i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
 397                             int align)
 398 {
 399         drm_dma_handle_t *phys;
 400         int ret;
 401
 402         if (obj->phys_handle) {
 403                 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
 404                         return -EBUSY;
 405
 406                 return 0;
 407         }
 408
 409         if (obj->madv != I915_MADV_WILLNEED)
 410                 return -EFAULT;
 411
 412         if (obj->base.filp == NULL)
 413                 return -EINVAL;
 414
 415         ret = i915_gem_object_unbind(obj);
 416         if (ret)
 417                 return ret;
 418
 419         ret = i915_gem_object_put_pages(obj);
 420         if (ret)
 421                 return ret;
 422
 423         /* create a new object */
 424         phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
 425         if (!phys)
 426                 return -ENOMEM;
 427
 428         obj->phys_handle = phys;
 429         obj->ops = &i915_gem_phys_ops;
 430
 431         return i915_gem_object_get_pages(obj);
 432 }
 433
 434 static int
 435 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 436                      struct drm_i915_gem_pwrite *args,
 437                      struct drm_file *file_priv)
 438 {
 439         struct drm_device *dev = obj->base.dev;
 440         void *vaddr = obj->phys_handle->vaddr + args->offset;
 441         char __user *user_data = u64_to_user_ptr(args->data_ptr);
 442         int ret = 0;
 443
 444         /* We manually control the domain here and pretend that it
 445          * remains coherent i.e. in the GTT domain, like shmem_pwrite.
 446          */
 447         ret = i915_gem_object_wait_rendering(obj, false);
 448         if (ret)
 449                 return ret;
 450
 451         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
 452         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
 453                 unsigned long unwritten;
 454
 455                 /* The physical object once assigned is fixed for the lifetime
 456                  * of the obj, so we can safely drop the lock and continue
 457                  * to access vaddr.
 458                  */
 459                 mutex_unlock(&dev->struct_mutex);
 460                 unwritten = copy_from_user(vaddr, user_data, args->size);
 461                 mutex_lock(&dev->struct_mutex);
 462                 if (unwritten) {
 463                         ret = -EFAULT;
 464                         goto out;
 465                 }
 466         }
 467
 468         drm_clflush_virt_range(vaddr, args->size);
 469         i915_gem_chipset_flush(to_i915(dev));
 470
 471 out:
 472         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
 473         return ret;
 474 }
 475
 476 void *i915_gem_object_alloc(struct drm_device *dev)
 477 {
 478         struct drm_i915_private *dev_priv = to_i915(dev);
 479         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 480 }
 481
 482 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 483 {
 484         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 485         kmem_cache_free(dev_priv->objects, obj);
 486 }
 487
 488 static int
 489 i915_gem_create(struct drm_file *file,
 490                 struct drm_device *dev,
 491                 uint64_t size,
 492                 uint32_t *handle_p)
 493 {
 494         struct drm_i915_gem_object *obj;
 495         int ret;
 496         u32 handle;
 497
 498         size = roundup(size, PAGE_SIZE);
 499         if (size == 0)
 500                 return -EINVAL;
 501
 502         /* Allocate the new object */
 503         obj = i915_gem_object_create(dev, size);
 504         if (IS_ERR(obj))
 505                 return PTR_ERR(obj);
 506
 507         ret = drm_gem_handle_create(file, &obj->base, &handle);
 508         /* drop reference from allocate - handle holds it now */
 509         i915_gem_object_put_unlocked(obj);
 510         if (ret)
 511                 return ret;
 512
 513         *handle_p = handle;
 514         return 0;
 515 }
 516
 517 int
 518 i915_gem_dumb_create(struct drm_file *file,
 519                      struct drm_device *dev,
 520                      struct drm_mode_create_dumb *args)
 521 {
 522         /* have to work out size/pitch and return them */
 523         args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 524         args->size = args->pitch * args->height;
 525         return i915_gem_create(file, dev,
 526                                args->size, &args->handle);
 527 }
 528
 529 /**
 530  * Creates a new mm object and returns a handle to it.
 531  * @dev: drm device pointer
 532  * @data: ioctl data blob
 533  * @file: drm file pointer
 534  */
 535 int
 536 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 537                       struct drm_file *file)
 538 {
 539         struct drm_i915_gem_create *args = data;
 540
 541         return i915_gem_create(file, dev,
 542                                args->size, &args->handle);
 543 }
 544
 545 static inline int
 546 __copy_to_user_swizzled(char __user *cpu_vaddr,
 547                         const char *gpu_vaddr, int gpu_offset,
 548                         int length)
 549 {
 550         int ret, cpu_offset = 0;
 551
 552         while (length > 0) {
 553                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 554                 int this_length = min(cacheline_end - gpu_offset, length);
 555                 int swizzled_gpu_offset = gpu_offset ^ 64;
 556
 557                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 558                                      gpu_vaddr + swizzled_gpu_offset,
 559                                      this_length);
 560                 if (ret)
 561                         return ret + length;
 562
 563                 cpu_offset += this_length;
 564                 gpu_offset += this_length;
 565                 length -= this_length;
 566         }
 567
 568         return 0;
 569 }
 570
 571 static inline int
 572 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 573                           const char __user *cpu_vaddr,
 574                           int length)
 575 {
 576         int ret, cpu_offset = 0;
 577
 578         while (length > 0) {
 579                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 580                 int this_length = min(cacheline_end - gpu_offset, length);
 581                 int swizzled_gpu_offset = gpu_offset ^ 64;
 582
 583                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 584                                        cpu_vaddr + cpu_offset,
 585                                        this_length);
 586                 if (ret)
 587                         return ret + length;
 588
 589                 cpu_offset += this_length;
 590                 gpu_offset += this_length;
 591                 length -= this_length;
 592         }
 593
 594         return 0;
 595 }
 596
 597 /*
 598  * Pins the specified object's pages and synchronizes the object with
 599  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 600  * flush the object from the CPU cache.
 601  */
 602 int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 603                                     int *needs_clflush)
 604 {
 605         int ret;
 606
 607         *needs_clflush = 0;
 608
 609         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
 610                 return -EINVAL;
 611
 612         ret = i915_gem_object_wait_rendering(obj, true);
 613         if (ret)
 614                 return ret;
 615
 616         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 617                 /* If we're not in the cpu read domain, set ourself into the gtt
 618                  * read domain and manually flush cachelines (if required). This
 619                  * optimizes for the case when the gpu will dirty the data
 620                  * anyway again before the next pread happens. */
 621                 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
 622                                                         obj->cache_level);
 623         }
 624
 625         ret = i915_gem_object_get_pages(obj);
 626         if (ret)
 627                 return ret;
 628
 629         i915_gem_object_pin_pages(obj);
 630
 631         return ret;
 632 }
 633
 634 /* Per-page copy function for the shmem pread fastpath.
 635  * Flushes invalid cachelines before reading the target if
 636  * needs_clflush is set. */
 637 static int
 638 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 639                  char __user *user_data,
 640                  bool page_do_bit17_swizzling, bool needs_clflush)
 641 {
 642         char *vaddr;
 643         int ret;
 644
 645         if (unlikely(page_do_bit17_swizzling))
 646                 return -EINVAL;
 647
 648         vaddr = kmap_atomic(page);
 649         if (needs_clflush)
 650                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 651                                        page_length);
 652         ret = __copy_to_user_inatomic(user_data,
 653                                       vaddr + shmem_page_offset,
 654                                       page_length);
 655         kunmap_atomic(vaddr);
 656
 657         return ret ? -EFAULT : 0;
 658 }
 659
 660 static void
 661 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 662                              bool swizzled)
 663 {
 664         if (unlikely(swizzled)) {
 665                 unsigned long start = (unsigned long) addr;
 666                 unsigned long end = (unsigned long) addr + length;
 667
 668                 /* For swizzling simply ensure that we always flush both
 669                  * channels. Lame, but simple and it works. Swizzled
 670                  * pwrite/pread is far from a hotpath - current userspace
 671                  * doesn't use it at all. */
 672                 start = round_down(start, 128);
 673                 end = round_up(end, 128);
 674
 675                 drm_clflush_virt_range((void *)start, end - start);
 676         } else {
 677                 drm_clflush_virt_range(addr, length);
 678         }
 679
 680 }
 681
 682 /* Only difference to the fast-path function is that this can handle bit17
 683  * and uses non-atomic copy and kmap functions. */
 684 static int
 685 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 686                  char __user *user_data,
 687                  bool page_do_bit17_swizzling, bool needs_clflush)
 688 {
 689         char *vaddr;
 690         int ret;
 691
 692         vaddr = kmap(page);
 693         if (needs_clflush)
 694                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 695                                              page_length,
 696                                              page_do_bit17_swizzling);
 697
 698         if (page_do_bit17_swizzling)
 699                 ret = __copy_to_user_swizzled(user_data,
 700                                               vaddr, shmem_page_offset,
 701                                               page_length);
 702         else
 703                 ret = __copy_to_user(user_data,
 704                                      vaddr + shmem_page_offset,
 705                                      page_length);
 706         kunmap(page);
 707
 708         return ret ? - EFAULT : 0;
 709 }
 710
 711 static inline unsigned long
 712 slow_user_access(struct io_mapping *mapping,
 713                  uint64_t page_base, int page_offset,
 714                  char __user *user_data,
 715                  unsigned long length, bool pwrite)
 716 {
 717         void __iomem *ioaddr;
 718         void *vaddr;
 719         uint64_t unwritten;
 720
 721         ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
 722         /* We can use the cpu mem copy function because this is X86. */
 723         vaddr = (void __force *)ioaddr + page_offset;
 724         if (pwrite)
 725                 unwritten = __copy_from_user(vaddr, user_data, length);
 726         else
 727                 unwritten = __copy_to_user(user_data, vaddr, length);
 728
 729         io_mapping_unmap(ioaddr);
 730         return unwritten;
 731 }
 732
 733 static int
 734 i915_gem_gtt_pread(struct drm_device *dev,
 735                    struct drm_i915_gem_object *obj, uint64_t size,
 736                    uint64_t data_offset, uint64_t data_ptr)
 737 {
 738         struct drm_i915_private *dev_priv = to_i915(dev);
 739         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 740         struct drm_mm_node node;
 741         char __user *user_data;
 742         uint64_t remain;
 743         uint64_t offset;
 744         int ret;
 745
 746         ret = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
 747         if (ret) {
 748                 ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
 749                 if (ret)
 750                         goto out;
 751
 752                 ret = i915_gem_object_get_pages(obj);
 753                 if (ret) {
 754                         remove_mappable_node(&node);
 755                         goto out;
 756                 }
 757
 758                 i915_gem_object_pin_pages(obj);
 759         } else {
 760                 node.start = i915_gem_obj_ggtt_offset(obj);
 761                 node.allocated = false;
 762                 ret = i915_gem_object_put_fence(obj);
 763                 if (ret)
 764                         goto out_unpin;
 765         }
 766
 767         ret = i915_gem_object_set_to_gtt_domain(obj, false);
 768         if (ret)
 769                 goto out_unpin;
 770
 771         user_data = u64_to_user_ptr(data_ptr);
 772         remain = size;
 773         offset = data_offset;
 774
 775         mutex_unlock(&dev->struct_mutex);
 776         if (likely(!i915.prefault_disable)) {
 777                 ret = fault_in_multipages_writeable(user_data, remain);
 778                 if (ret) {
 779                         mutex_lock(&dev->struct_mutex);
 780                         goto out_unpin;
 781                 }
 782         }
 783
 784         while (remain > 0) {
 785                 /* Operation in this page
 786                  *
 787                  * page_base = page offset within aperture
 788                  * page_offset = offset within page
 789                  * page_length = bytes to copy for this page
 790                  */
 791                 u32 page_base = node.start;
 792                 unsigned page_offset = offset_in_page(offset);
 793                 unsigned page_length = PAGE_SIZE - page_offset;
 794                 page_length = remain < page_length ? remain : page_length;
 795                 if (node.allocated) {
 796                         wmb();
 797                         ggtt->base.insert_page(&ggtt->base,
 798                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 799                                                node.start,
 800                                                I915_CACHE_NONE, 0);
 801                         wmb();
 802                 } else {
 803                         page_base += offset & PAGE_MASK;
 804                 }
 805                 /* This is a slow read/write as it tries to read from
 806                  * and write to user memory which may result into page
 807                  * faults, and so we cannot perform this under struct_mutex.
 808                  */
 809                 if (slow_user_access(ggtt->mappable, page_base,
 810                                      page_offset, user_data,
 811                                      page_length, false)) {
 812                         ret = -EFAULT;
 813                         break;
 814                 }
 815
 816                 remain -= page_length;
 817                 user_data += page_length;
 818                 offset += page_length;
 819         }
 820
 821         mutex_lock(&dev->struct_mutex);
 822         if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
 823                 /* The user has modified the object whilst we tried
 824                  * reading from it, and we now have no idea what domain
 825                  * the pages should be in. As we have just been touching
 826                  * them directly, flush everything back to the GTT
 827                  * domain.
 828                  */
 829                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 830         }
 831
 832 out_unpin:
 833         if (node.allocated) {
 834                 wmb();
 835                 ggtt->base.clear_range(&ggtt->base,
 836                                        node.start, node.size,
 837                                        true);
 838                 i915_gem_object_unpin_pages(obj);
 839                 remove_mappable_node(&node);
 840         } else {
 841                 i915_gem_object_ggtt_unpin(obj);
 842         }
 843 out:
 844         return ret;
 845 }
 846
 847 static int
 848 i915_gem_shmem_pread(struct drm_device *dev,
 849                      struct drm_i915_gem_object *obj,
 850                      struct drm_i915_gem_pread *args,
 851                      struct drm_file *file)
 852 {
 853         char __user *user_data;
 854         ssize_t remain;
 855         loff_t offset;
 856         int shmem_page_offset, page_length, ret = 0;
 857         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 858         int prefaulted = 0;
 859         int needs_clflush = 0;
 860         struct sg_page_iter sg_iter;
 861
 862         if (!i915_gem_object_has_struct_page(obj))
 863                 return -ENODEV;
 864
 865         user_data = u64_to_user_ptr(args->data_ptr);
 866         remain = args->size;
 867
 868         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 869
 870         ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
 871         if (ret)
 872                 return ret;
 873
 874         offset = args->offset;
 875
 876         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
 877                          offset >> PAGE_SHIFT) {
 878                 struct page *page = sg_page_iter_page(&sg_iter);
 879
 880                 if (remain <= 0)
 881                         break;
 882
 883                 /* Operation in this page
 884                  *
 885                  * shmem_page_offset = offset within page in shmem file
 886                  * page_length = bytes to copy for this page
 887                  */
 888                 shmem_page_offset = offset_in_page(offset);
 889                 page_length = remain;
 890                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 891                         page_length = PAGE_SIZE - shmem_page_offset;
 892
 893                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 894                         (page_to_phys(page) & (1 << 17)) != 0;
 895
 896                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 897                                        user_data, page_do_bit17_swizzling,
 898                                        needs_clflush);
 899                 if (ret == 0)
 900                         goto next_page;
 901
 902                 mutex_unlock(&dev->struct_mutex);
 903
 904                 if (likely(!i915.prefault_disable) && !prefaulted) {
 905                         ret = fault_in_multipages_writeable(user_data, remain);
 906                         /* Userspace is tricking us, but we've already clobbered
 907                          * its pages with the prefault and promised to write the
 908                          * data up to the first fault. Hence ignore any errors
 909                          * and just continue. */
 910                         (void)ret;
 911                         prefaulted = 1;
 912                 }
 913
 914                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 915                                        user_data, page_do_bit17_swizzling,
 916                                        needs_clflush);
 917
 918                 mutex_lock(&dev->struct_mutex);
 919
 920                 if (ret)
 921                         goto out;
 922
 923 next_page:
 924                 remain -= page_length;
 925                 user_data += page_length;
 926                 offset += page_length;
 927         }
 928
 929 out:
 930         i915_gem_object_unpin_pages(obj);
 931
 932         return ret;
 933 }
 934
 935 /**
 936  * Reads data from the object referenced by handle.
 937  * @dev: drm device pointer
 938  * @data: ioctl data blob
 939  * @file: drm file pointer
 940  *
 941  * On error, the contents of *data are undefined.
 942  */
 943 int
 944 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 945                      struct drm_file *file)
 946 {
 947         struct drm_i915_gem_pread *args = data;
 948         struct drm_i915_gem_object *obj;
 949         int ret = 0;
 950
 951         if (args->size == 0)
 952                 return 0;
 953
 954         if (!access_ok(VERIFY_WRITE,
 955                        u64_to_user_ptr(args->data_ptr),
 956                        args->size))
 957                 return -EFAULT;
 958
 959         obj = i915_gem_object_lookup(file, args->handle);
 960         if (!obj)
 961                 return -ENOENT;
 962
 963         /* Bounds check source.  */
 964         if (args->offset > obj->base.size ||
 965             args->size > obj->base.size - args->offset) {
 966                 ret = -EINVAL;
 967                 goto err;
 968         }
 969
 970         trace_i915_gem_object_pread(obj, args->offset, args->size);
 971
 972         ret = __unsafe_wait_rendering(obj, to_rps_client(file), true);
 973         if (ret)
 974                 goto err;
 975
 976         ret = i915_mutex_lock_interruptible(dev);
 977         if (ret)
 978                 goto err;
 979
 980         ret = i915_gem_shmem_pread(dev, obj, args, file);
 981
 982         /* pread for non shmem backed objects */
 983         if (ret == -EFAULT || ret == -ENODEV) {
 984                 intel_runtime_pm_get(to_i915(dev));
 985                 ret = i915_gem_gtt_pread(dev, obj, args->size,
 986                                         args->offset, args->data_ptr);
 987                 intel_runtime_pm_put(to_i915(dev));
 988         }
 989
 990         i915_gem_object_put(obj);
 991         mutex_unlock(&dev->struct_mutex);
 992
 993         return ret;
 994
 995 err:
 996         i915_gem_object_put_unlocked(obj);
 997         return ret;
 998 }
 999
1000 /* This is the fast write path which cannot handle
1001  * page faults in the source data
1002  */
1003
1004 static inline int
1005 fast_user_write(struct io_mapping *mapping,
1006                 loff_t page_base, int page_offset,
1007                 char __user *user_data,
1008                 int length)
1009 {
1010         void __iomem *vaddr_atomic;
1011         void *vaddr;
1012         unsigned long unwritten;
1013
1014         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
1015         /* We can use the cpu mem copy function because this is X86. */
1016         vaddr = (void __force*)vaddr_atomic + page_offset;
1017         unwritten = __copy_from_user_inatomic_nocache(vaddr,
1018                                                       user_data, length);
1019         io_mapping_unmap_atomic(vaddr_atomic);
1020         return unwritten;
1021 }
1022
1023 /**
1024  * This is the fast pwrite path, where we copy the data directly from the
1025  * user into the GTT, uncached.
1026  * @i915: i915 device private data
1027  * @obj: i915 gem object
1028  * @args: pwrite arguments structure
1029  * @file: drm file pointer
1030  */
1031 static int
1032 i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
1033                          struct drm_i915_gem_object *obj,
1034                          struct drm_i915_gem_pwrite *args,
1035                          struct drm_file *file)
1036 {
1037         struct i915_ggtt *ggtt = &i915->ggtt;
1038         struct drm_device *dev = obj->base.dev;
1039         struct drm_mm_node node;
1040         uint64_t remain, offset;
1041         char __user *user_data;
1042         int ret;
1043         bool hit_slow_path = false;
1044
1045         if (i915_gem_object_is_tiled(obj))
1046                 return -EFAULT;
1047
1048         ret = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1049                                        PIN_MAPPABLE | PIN_NONBLOCK);
1050         if (ret) {
1051                 ret = insert_mappable_node(i915, &node, PAGE_SIZE);
1052                 if (ret)
1053                         goto out;
1054
1055                 ret = i915_gem_object_get_pages(obj);
1056                 if (ret) {
1057                         remove_mappable_node(&node);
1058                         goto out;
1059                 }
1060
1061                 i915_gem_object_pin_pages(obj);
1062         } else {
1063                 node.start = i915_gem_obj_ggtt_offset(obj);
1064                 node.allocated = false;
1065                 ret = i915_gem_object_put_fence(obj);
1066                 if (ret)
1067                         goto out_unpin;
1068         }
1069
1070         ret = i915_gem_object_set_to_gtt_domain(obj, true);
1071         if (ret)
1072                 goto out_unpin;
1073
1074         intel_fb_obj_invalidate(obj, ORIGIN_GTT);
1075         obj->dirty = true;
1076
1077         user_data = u64_to_user_ptr(args->data_ptr);
1078         offset = args->offset;
1079         remain = args->size;
1080         while (remain) {
1081                 /* Operation in this page
1082                  *
1083                  * page_base = page offset within aperture
1084                  * page_offset = offset within page
1085                  * page_length = bytes to copy for this page
1086                  */
1087                 u32 page_base = node.start;
1088                 unsigned page_offset = offset_in_page(offset);
1089                 unsigned page_length = PAGE_SIZE - page_offset;
1090                 page_length = remain < page_length ? remain : page_length;
1091                 if (node.allocated) {
1092                         wmb(); /* flush the write before we modify the GGTT */
1093                         ggtt->base.insert_page(&ggtt->base,
1094                                                i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1095                                                node.start, I915_CACHE_NONE, 0);
1096                         wmb(); /* flush modifications to the GGTT (insert_page) */
1097                 } else {
1098                         page_base += offset & PAGE_MASK;
1099                 }
1100                 /* If we get a fault while copying data, then (presumably) our
1101                  * source page isn't available.  Return the error and we'll
1102                  * retry in the slow path.
1103                  * If the object is non-shmem backed, we retry again with the
1104                  * path that handles page fault.
1105                  */
1106                 if (fast_user_write(ggtt->mappable, page_base,
1107                                     page_offset, user_data, page_length)) {
1108                         hit_slow_path = true;
1109                         mutex_unlock(&dev->struct_mutex);
1110                         if (slow_user_access(ggtt->mappable,
1111                                              page_base,
1112                                              page_offset, user_data,
1113                                              page_length, true)) {
1114                                 ret = -EFAULT;
1115                                 mutex_lock(&dev->struct_mutex);
1116                                 goto out_flush;
1117                         }
1118
1119                         mutex_lock(&dev->struct_mutex);
1120                 }
1121
1122                 remain -= page_length;
1123                 user_data += page_length;
1124                 offset += page_length;
1125         }
1126
1127 out_flush:
1128         if (hit_slow_path) {
1129                 if (ret == 0 &&
1130                     (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
1131                         /* The user has modified the object whilst we tried
1132                          * reading from it, and we now have no idea what domain
1133                          * the pages should be in. As we have just been touching
1134                          * them directly, flush everything back to the GTT
1135                          * domain.
1136                          */
1137                         ret = i915_gem_object_set_to_gtt_domain(obj, false);
1138                 }
1139         }
1140
1141         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
1142 out_unpin:
1143         if (node.allocated) {
1144                 wmb();
1145                 ggtt->base.clear_range(&ggtt->base,
1146                                        node.start, node.size,
1147                                        true);
1148                 i915_gem_object_unpin_pages(obj);
1149                 remove_mappable_node(&node);
1150         } else {
1151                 i915_gem_object_ggtt_unpin(obj);
1152         }
1153 out:
1154         return ret;
1155 }
1156
1157 /* Per-page copy function for the shmem pwrite fastpath.
1158  * Flushes invalid cachelines before writing to the target if
1159  * needs_clflush_before is set and flushes out any written cachelines after
1160  * writing if needs_clflush is set. */
1161 static int
1162 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
1163                   char __user *user_data,
1164                   bool page_do_bit17_swizzling,
1165                   bool needs_clflush_before,
1166                   bool needs_clflush_after)
1167 {
1168         char *vaddr;
1169         int ret;
1170
1171         if (unlikely(page_do_bit17_swizzling))
1172                 return -EINVAL;
1173
1174         vaddr = kmap_atomic(page);
1175         if (needs_clflush_before)
1176                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1177                                        page_length);
1178         ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
1179                                         user_data, page_length);
1180         if (needs_clflush_after)
1181                 drm_clflush_virt_range(vaddr + shmem_page_offset,
1182                                        page_length);
1183         kunmap_atomic(vaddr);
1184
1185         return ret ? -EFAULT : 0;
1186 }
1187
1188 /* Only difference to the fast-path function is that this can handle bit17
1189  * and uses non-atomic copy and kmap functions. */
1190 static int
1191 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
1192                   char __user *user_data,
1193                   bool page_do_bit17_swizzling,
1194                   bool needs_clflush_before,
1195                   bool needs_clflush_after)
1196 {
1197         char *vaddr;
1198         int ret;
1199
1200         vaddr = kmap(page);
1201         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1202                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1203                                              page_length,
1204                                              page_do_bit17_swizzling);
1205         if (page_do_bit17_swizzling)
1206                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1207                                                 user_data,
1208                                                 page_length);
1209         else
1210                 ret = __copy_from_user(vaddr + shmem_page_offset,
1211                                        user_data,
1212                                        page_length);
1213         if (needs_clflush_after)
1214                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1215                                              page_length,
1216                                              page_do_bit17_swizzling);
1217         kunmap(page);
1218
1219         return ret ? -EFAULT : 0;
1220 }
1221
1222 static int
1223 i915_gem_shmem_pwrite(struct drm_device *dev,
1224                       struct drm_i915_gem_object *obj,
1225                       struct drm_i915_gem_pwrite *args,
1226                       struct drm_file *file)
1227 {
1228         ssize_t remain;
1229         loff_t offset;
1230         char __user *user_data;
1231         int shmem_page_offset, page_length, ret = 0;
1232         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1233         int hit_slowpath = 0;
1234         int needs_clflush_after = 0;
1235         int needs_clflush_before = 0;
1236         struct sg_page_iter sg_iter;
1237
1238         user_data = u64_to_user_ptr(args->data_ptr);
1239         remain = args->size;
1240
1241         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1242
1243         ret = i915_gem_object_wait_rendering(obj, false);
1244         if (ret)
1245                 return ret;
1246
1247         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1248                 /* If we're not in the cpu write domain, set ourself into the gtt
1249                  * write domain and manually flush cachelines (if required). This
1250                  * optimizes for the case when the gpu will use the data
1251                  * right away and we therefore have to clflush anyway. */
1252                 needs_clflush_after = cpu_write_needs_clflush(obj);
1253         }
1254         /* Same trick applies to invalidate partially written cachelines read
1255          * before writing. */
1256         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1257                 needs_clflush_before =
1258                         !cpu_cache_is_coherent(dev, obj->cache_level);
1259
1260         ret = i915_gem_object_get_pages(obj);
1261         if (ret)
1262                 return ret;
1263
1264         intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1265
1266         i915_gem_object_pin_pages(obj);
1267
1268         offset = args->offset;
1269         obj->dirty = 1;
1270
1271         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
1272                          offset >> PAGE_SHIFT) {
1273                 struct page *page = sg_page_iter_page(&sg_iter);
1274                 int partial_cacheline_write;
1275
1276                 if (remain <= 0)
1277                         break;
1278
1279                 /* Operation in this page
1280                  *
1281                  * shmem_page_offset = offset within page in shmem file
1282                  * page_length = bytes to copy for this page
1283                  */
1284                 shmem_page_offset = offset_in_page(offset);
1285
1286                 page_length = remain;
1287                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1288                         page_length = PAGE_SIZE - shmem_page_offset;
1289
1290                 /* If we don't overwrite a cacheline completely we need to be
1291                  * careful to have up-to-date data by first clflushing. Don't
1292                  * overcomplicate things and flush the entire patch. */
1293                 partial_cacheline_write = needs_clflush_before &&
1294                         ((shmem_page_offset | page_length)
1295                                 & (boot_cpu_data.x86_clflush_size - 1));
1296
1297                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1298                         (page_to_phys(page) & (1 << 17)) != 0;
1299
1300                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
1301                                         user_data, page_do_bit17_swizzling,
1302                                         partial_cacheline_write,
1303                                         needs_clflush_after);
1304                 if (ret == 0)
1305                         goto next_page;
1306
1307                 hit_slowpath = 1;
1308                 mutex_unlock(&dev->struct_mutex);
1309                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1310                                         user_data, page_do_bit17_swizzling,
1311                                         partial_cacheline_write,
1312                                         needs_clflush_after);
1313
1314                 mutex_lock(&dev->struct_mutex);
1315
1316                 if (ret)
1317                         goto out;
1318
1319 next_page:
1320                 remain -= page_length;
1321                 user_data += page_length;
1322                 offset += page_length;
1323         }
1324
1325 out:
1326         i915_gem_object_unpin_pages(obj);
1327
1328         if (hit_slowpath) {
1329                 /*
1330                  * Fixup: Flush cpu caches in case we didn't flush the dirty
1331                  * cachelines in-line while writing and the object moved
1332                  * out of the cpu write domain while we've dropped the lock.
1333                  */
1334                 if (!needs_clflush_after &&
1335                     obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1336                         if (i915_gem_clflush_object(obj, obj->pin_display))
1337                                 needs_clflush_after = true;
1338                 }
1339         }
1340
1341         if (needs_clflush_after)
1342                 i915_gem_chipset_flush(to_i915(dev));
1343         else
1344                 obj->cache_dirty = true;
1345
1346         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1347         return ret;
1348 }
1349
1350 /**
1351  * Writes data to the object referenced by handle.
1352  * @dev: drm device
1353  * @data: ioctl data blob
1354  * @file: drm file
1355  *
1356  * On error, the contents of the buffer that were to be modified are undefined.
1357  */
1358 int
1359 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1360                       struct drm_file *file)
1361 {
1362         struct drm_i915_private *dev_priv = to_i915(dev);
1363         struct drm_i915_gem_pwrite *args = data;
1364         struct drm_i915_gem_object *obj;
1365         int ret;
1366
1367         if (args->size == 0)
1368                 return 0;
1369
1370         if (!access_ok(VERIFY_READ,
1371                        u64_to_user_ptr(args->data_ptr),
1372                        args->size))
1373                 return -EFAULT;
1374
1375         if (likely(!i915.prefault_disable)) {
1376                 ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
1377                                                    args->size);
1378                 if (ret)
1379                         return -EFAULT;
1380         }
1381
1382         obj = i915_gem_object_lookup(file, args->handle);
1383         if (!obj)
1384                 return -ENOENT;
1385
1386         /* Bounds check destination. */
1387         if (args->offset > obj->base.size ||
1388             args->size > obj->base.size - args->offset) {
1389                 ret = -EINVAL;
1390                 goto err;
1391         }
1392
1393         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1394
1395         ret = __unsafe_wait_rendering(obj, to_rps_client(file), false);
1396         if (ret)
1397                 goto err;
1398
1399         intel_runtime_pm_get(dev_priv);
1400
1401         ret = i915_mutex_lock_interruptible(dev);
1402         if (ret)
1403                 goto err_rpm;
1404
1405         ret = -EFAULT;
1406         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1407          * it would end up going through the fenced access, and we'll get
1408          * different detiling behavior between reading and writing.
1409          * pread/pwrite currently are reading and writing from the CPU
1410          * perspective, requiring manual detiling by the client.
1411          */
1412         if (!i915_gem_object_has_struct_page(obj) ||
1413             cpu_write_needs_clflush(obj)) {
1414                 ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
1415                 /* Note that the gtt paths might fail with non-page-backed user
1416                  * pointers (e.g. gtt mappings when moving data between
1417                  * textures). Fallback to the shmem path in that case. */
1418         }
1419
1420         if (ret == -EFAULT || ret == -ENOSPC) {
1421                 if (obj->phys_handle)
1422                         ret = i915_gem_phys_pwrite(obj, args, file);
1423                 else if (i915_gem_object_has_struct_page(obj))
1424                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1425                 else
1426                         ret = -ENODEV;
1427         }
1428
1429         i915_gem_object_put(obj);
1430         mutex_unlock(&dev->struct_mutex);
1431         intel_runtime_pm_put(dev_priv);
1432
1433         return ret;
1434
1435 err_rpm:
1436         intel_runtime_pm_put(dev_priv);
1437 err:
1438         i915_gem_object_put_unlocked(obj);
1439         return ret;
1440 }
1441
1442 static enum fb_op_origin
1443 write_origin(struct drm_i915_gem_object *obj, unsigned domain)
1444 {
1445         return domain == I915_GEM_DOMAIN_GTT && !obj->has_wc_mmap ?
1446                ORIGIN_GTT : ORIGIN_CPU;
1447 }
1448
1449 /**
1450  * Called when user space prepares to use an object with the CPU, either
1451  * through the mmap ioctl's mapping or a GTT mapping.
1452  * @dev: drm device
1453  * @data: ioctl data blob
1454  * @file: drm file
1455  */
1456 int
1457 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1458                           struct drm_file *file)
1459 {
1460         struct drm_i915_gem_set_domain *args = data;
1461         struct drm_i915_gem_object *obj;
1462         uint32_t read_domains = args->read_domains;
1463         uint32_t write_domain = args->write_domain;
1464         int ret;
1465
1466         /* Only handle setting domains to types used by the CPU. */
1467         if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
1468                 return -EINVAL;
1469
1470         /* Having something in the write domain implies it's in the read
1471          * domain, and only that read domain.  Enforce that in the request.
1472          */
1473         if (write_domain != 0 && read_domains != write_domain)
1474                 return -EINVAL;
1475
1476         obj = i915_gem_object_lookup(file, args->handle);
1477         if (!obj)
1478                 return -ENOENT;
1479
1480         /* Try to flush the object off the GPU without holding the lock.
1481          * We will repeat the flush holding the lock in the normal manner
1482          * to catch cases where we are gazumped.
1483          */
1484         ret = __unsafe_wait_rendering(obj, to_rps_client(file), !write_domain);
1485         if (ret)
1486                 goto err;
1487
1488         ret = i915_mutex_lock_interruptible(dev);
1489         if (ret)
1490                 goto err;
1491
1492         if (read_domains & I915_GEM_DOMAIN_GTT)
1493                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1494         else
1495                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1496
1497         if (write_domain != 0)
1498                 intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
1499
1500         i915_gem_object_put(obj);
1501         mutex_unlock(&dev->struct_mutex);
1502         return ret;
1503
1504 err:
1505         i915_gem_object_put_unlocked(obj);
1506         return ret;
1507 }
1508
1509 /**
1510  * Called when user space has done writes to this buffer
1511  * @dev: drm device
1512  * @data: ioctl data blob
1513  * @file: drm file
1514  */
1515 int
1516 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1517                          struct drm_file *file)
1518 {
1519         struct drm_i915_gem_sw_finish *args = data;
1520         struct drm_i915_gem_object *obj;
1521         int err = 0;
1522
1523         obj = i915_gem_object_lookup(file, args->handle);
1524         if (!obj)
1525                 return -ENOENT;
1526
1527         /* Pinned buffers may be scanout, so flush the cache */
1528         if (READ_ONCE(obj->pin_display)) {
1529                 err = i915_mutex_lock_interruptible(dev);
1530                 if (!err) {
1531                         i915_gem_object_flush_cpu_write_domain(obj);
1532                         mutex_unlock(&dev->struct_mutex);
1533                 }
1534         }
1535
1536         i915_gem_object_put_unlocked(obj);
1537         return err;
1538 }
1539
1540 /**
1541  * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1542  *                       it is mapped to.
1543  * @dev: drm device
1544  * @data: ioctl data blob
1545  * @file: drm file
1546  *
1547  * While the mapping holds a reference on the contents of the object, it doesn't
1548  * imply a ref on the object itself.
1549  *
1550  * IMPORTANT:
1551  *
1552  * DRM driver writers who look a this function as an example for how to do GEM
1553  * mmap support, please don't implement mmap support like here. The modern way
1554  * to implement DRM mmap support is with an mmap offset ioctl (like
1555  * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1556  * That way debug tooling like valgrind will understand what's going on, hiding
1557  * the mmap call in a driver private ioctl will break that. The i915 driver only
1558  * does cpu mmaps this way because we didn't know better.
1559  */
1560 int
1561 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1562                     struct drm_file *file)
1563 {
1564         struct drm_i915_gem_mmap *args = data;
1565         struct drm_i915_gem_object *obj;
1566         unsigned long addr;
1567
1568         if (args->flags & ~(I915_MMAP_WC))
1569                 return -EINVAL;
1570
1571         if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1572                 return -ENODEV;
1573
1574         obj = i915_gem_object_lookup(file, args->handle);
1575         if (!obj)
1576                 return -ENOENT;
1577
1578         /* prime objects have no backing filp to GEM mmap
1579          * pages from.
1580          */
1581         if (!obj->base.filp) {
1582                 i915_gem_object_put_unlocked(obj);
1583                 return -EINVAL;
1584         }
1585
1586         addr = vm_mmap(obj->base.filp, 0, args->size,
1587                        PROT_READ | PROT_WRITE, MAP_SHARED,
1588                        args->offset);
1589         if (args->flags & I915_MMAP_WC) {
1590                 struct mm_struct *mm = current->mm;
1591                 struct vm_area_struct *vma;
1592
1593                 if (down_write_killable(&mm->mmap_sem)) {
1594                         i915_gem_object_put_unlocked(obj);
1595                         return -EINTR;
1596                 }
1597                 vma = find_vma(mm, addr);
1598                 if (vma)
1599                         vma->vm_page_prot =
1600                                 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1601                 else
1602                         addr = -ENOMEM;
1603                 up_write(&mm->mmap_sem);
1604
1605                 /* This may race, but that's ok, it only gets set */
1606                 WRITE_ONCE(obj->has_wc_mmap, true);
1607         }
1608         i915_gem_object_put_unlocked(obj);
1609         if (IS_ERR((void *)addr))
1610                 return addr;
1611
1612         args->addr_ptr = (uint64_t) addr;
1613
1614         return 0;
1615 }
1616
1617 /**
1618  * i915_gem_fault - fault a page into the GTT
1619  * @vma: VMA in question
1620  * @vmf: fault info
1621  *
1622  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1623  * from userspace.  The fault handler takes care of binding the object to
1624  * the GTT (if needed), allocating and programming a fence register (again,
1625  * only if needed based on whether the old reg is still valid or the object
1626  * is tiled) and inserting a new PTE into the faulting process.
1627  *
1628  * Note that the faulting process may involve evicting existing objects
1629  * from the GTT and/or fence registers to make room.  So performance may
1630  * suffer if the GTT working set is large or there are few fence registers
1631  * left.
1632  */
1633 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1634 {
1635         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1636         struct drm_device *dev = obj->base.dev;
1637         struct drm_i915_private *dev_priv = to_i915(dev);
1638         struct i915_ggtt *ggtt = &dev_priv->ggtt;
1639         struct i915_ggtt_view view = i915_ggtt_view_normal;
1640         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1641         pgoff_t page_offset;
1642         unsigned long pfn;
1643         int ret;
1644
1645         /* We don't use vmf->pgoff since that has the fake offset */
1646         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1647                 PAGE_SHIFT;
1648
1649         trace_i915_gem_object_fault(obj, page_offset, true, write);
1650
1651         /* Try to flush the object off the GPU first without holding the lock.
1652          * Upon acquiring the lock, we will perform our sanity checks and then
1653          * repeat the flush holding the lock in the normal manner to catch cases
1654          * where we are gazumped.
1655          */
1656         ret = __unsafe_wait_rendering(obj, NULL, !write);
1657         if (ret)
1658                 goto err;
1659
1660         intel_runtime_pm_get(dev_priv);
1661
1662         ret = i915_mutex_lock_interruptible(dev);
1663         if (ret)
1664                 goto err_rpm;
1665
1666         /* Access to snoopable pages through the GTT is incoherent. */
1667         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1668                 ret = -EFAULT;
1669                 goto err_unlock;
1670         }
1671
1672         /* Use a partial view if the object is bigger than the aperture. */
1673         if (obj->base.size >= ggtt->mappable_end &&
1674             !i915_gem_object_is_tiled(obj)) {
1675                 static const unsigned int chunk_size = 256; // 1 MiB
1676
1677                 memset(&view, 0, sizeof(view));
1678                 view.type = I915_GGTT_VIEW_PARTIAL;
1679                 view.params.partial.offset = rounddown(page_offset, chunk_size);
1680                 view.params.partial.size =
1681                         min_t(unsigned int,
1682                               chunk_size,
1683                               (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1684                               view.params.partial.offset);
1685         }
1686
1687         /* Now pin it into the GTT if needed */
1688         ret = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
1689         if (ret)
1690                 goto err_unlock;
1691
1692         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1693         if (ret)
1694                 goto err_unpin;
1695
1696         ret = i915_gem_object_get_fence(obj);
1697         if (ret)
1698                 goto err_unpin;
1699
1700         /* Finally, remap it using the new GTT offset */
1701         pfn = ggtt->mappable_base +
1702                 i915_gem_obj_ggtt_offset_view(obj, &view);
1703         pfn >>= PAGE_SHIFT;
1704
1705         if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1706                 /* Overriding existing pages in partial view does not cause
1707                  * us any trouble as TLBs are still valid because the fault
1708                  * is due to userspace losing part of the mapping or never
1709                  * having accessed it before (at this partials' range).
1710                  */
1711                 unsigned long base = vma->vm_start +
1712                                      (view.params.partial.offset << PAGE_SHIFT);
1713                 unsigned int i;
1714
1715                 for (i = 0; i < view.params.partial.size; i++) {
1716                         ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1717                         if (ret)
1718                                 break;
1719                 }
1720
1721                 obj->fault_mappable = true;
1722         } else {
1723                 if (!obj->fault_mappable) {
1724                         unsigned long size = min_t(unsigned long,
1725                                                    vma->vm_end - vma->vm_start,
1726                                                    obj->base.size);
1727                         int i;
1728
1729                         for (i = 0; i < size >> PAGE_SHIFT; i++) {
1730                                 ret = vm_insert_pfn(vma,
1731                                                     (unsigned long)vma->vm_start + i * PAGE_SIZE,
1732                                                     pfn + i);
1733                                 if (ret)
1734                                         break;
1735                         }
1736
1737                         obj->fault_mappable = true;
1738                 } else
1739                         ret = vm_insert_pfn(vma,
1740                                             (unsigned long)vmf->virtual_address,
1741                                             pfn + page_offset);
1742         }
1743 err_unpin:
1744         i915_gem_object_ggtt_unpin_view(obj, &view);
1745 err_unlock:
1746         mutex_unlock(&dev->struct_mutex);
1747 err_rpm:
1748         intel_runtime_pm_put(dev_priv);
1749 err:
1750         switch (ret) {
1751         case -EIO:
1752                 /*
1753                  * We eat errors when the gpu is terminally wedged to avoid
1754                  * userspace unduly crashing (gl has no provisions for mmaps to
1755                  * fail). But any other -EIO isn't ours (e.g. swap in failure)
1756                  * and so needs to be reported.
1757                  */
1758                 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1759                         ret = VM_FAULT_SIGBUS;
1760                         break;
1761                 }
1762         case -EAGAIN:
1763                 /*
1764                  * EAGAIN means the gpu is hung and we'll wait for the error
1765                  * handler to reset everything when re-faulting in
1766                  * i915_mutex_lock_interruptible.
1767                  */
1768         case 0:
1769         case -ERESTARTSYS:
1770         case -EINTR:
1771         case -EBUSY:
1772                 /*
1773                  * EBUSY is ok: this just means that another thread
1774                  * already did the job.
1775                  */
1776                 ret = VM_FAULT_NOPAGE;
1777                 break;
1778         case -ENOMEM:
1779                 ret = VM_FAULT_OOM;
1780                 break;
1781         case -ENOSPC:
1782         case -EFAULT:
1783                 ret = VM_FAULT_SIGBUS;
1784                 break;
1785         default:
1786                 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1787                 ret = VM_FAULT_SIGBUS;
1788                 break;
1789         }
1790         return ret;
1791 }
1792
1793 /**
1794  * i915_gem_release_mmap - remove physical page mappings
1795  * @obj: obj in question
1796  *
1797  * Preserve the reservation of the mmapping with the DRM core code, but
1798  * relinquish ownership of the pages back to the system.
1799  *
1800  * It is vital that we remove the page mapping if we have mapped a tiled
1801  * object through the GTT and then lose the fence register due to
1802  * resource pressure. Similarly if the object has been moved out of the
1803  * aperture, than pages mapped into userspace must be revoked. Removing the
1804  * mapping will then trigger a page fault on the next user access, allowing
1805  * fixup by i915_gem_fault().
1806  */
1807 void
1808 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1809 {
1810         /* Serialisation between user GTT access and our code depends upon
1811          * revoking the CPU's PTE whilst the mutex is held. The next user
1812          * pagefault then has to wait until we release the mutex.
1813          */
1814         lockdep_assert_held(&obj->base.dev->struct_mutex);
1815
1816         if (!obj->fault_mappable)
1817                 return;
1818
1819         drm_vma_node_unmap(&obj->base.vma_node,
1820                            obj->base.dev->anon_inode->i_mapping);
1821
1822         /* Ensure that the CPU's PTE are revoked and there are not outstanding
1823          * memory transactions from userspace before we return. The TLB
1824          * flushing implied above by changing the PTE above *should* be
1825          * sufficient, an extra barrier here just provides us with a bit
1826          * of paranoid documentation about our requirement to serialise
1827          * memory writes before touching registers / GSM.
1828          */
1829         wmb();
1830
1831         obj->fault_mappable = false;
1832 }
1833
1834 void
1835 i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1836 {
1837         struct drm_i915_gem_object *obj;
1838
1839         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1840                 i915_gem_release_mmap(obj);
1841 }
1842
1843 /**
1844  * i915_gem_get_ggtt_size - return required global GTT size for an object
1845  * @dev_priv: i915 device
1846  * @size: object size
1847  * @tiling_mode: tiling mode
1848  *
1849  * Return the required global GTT size for an object, taking into account
1850  * potential fence register mapping.
1851  */
1852 u64 i915_gem_get_ggtt_size(struct drm_i915_private *dev_priv,
1853                            u64 size, int tiling_mode)
1854 {
1855         u64 ggtt_size;
1856
1857         GEM_BUG_ON(size == 0);
1858
1859         if (INTEL_GEN(dev_priv) >= 4 ||
1860             tiling_mode == I915_TILING_NONE)
1861                 return size;
1862
1863         /* Previous chips need a power-of-two fence region when tiling */
1864         if (IS_GEN3(dev_priv))
1865                 ggtt_size = 1024*1024;
1866         else
1867                 ggtt_size = 512*1024;
1868
1869         while (ggtt_size < size)
1870                 ggtt_size <<= 1;
1871
1872         return ggtt_size;
1873 }
1874
1875 /**
1876  * i915_gem_get_ggtt_alignment - return required global GTT alignment
1877  * @dev_priv: i915 device
1878  * @size: object size
1879  * @tiling_mode: tiling mode
1880  * @fenced: is fenced alignment required or not
1881  *
1882  * Return the required global GTT alignment for an object, taking into account
1883  * potential fence register mapping.
1884  */
1885 u64 i915_gem_get_ggtt_alignment(struct drm_i915_private *dev_priv, u64 size,
1886                                 int tiling_mode, bool fenced)
1887 {
1888         GEM_BUG_ON(size == 0);
1889
1890         /*
1891          * Minimum alignment is 4k (GTT page size), but might be greater
1892          * if a fence register is needed for the object.
1893          */
1894         if (INTEL_GEN(dev_priv) >= 4 || (!fenced && IS_G33(dev_priv)) ||
1895             tiling_mode == I915_TILING_NONE)
1896                 return 4096;
1897
1898         /*
1899          * Previous chips need to be aligned to the size of the smallest
1900          * fence register that can contain the object.
1901          */
1902         return i915_gem_get_ggtt_size(dev_priv, size, tiling_mode);
1903 }
1904
1905 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1906 {
1907         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
1908         int err;
1909
1910         err = drm_gem_create_mmap_offset(&obj->base);
1911         if (!err)
1912                 return 0;
1913
1914         /* We can idle the GPU locklessly to flush stale objects, but in order
1915          * to claim that space for ourselves, we need to take the big
1916          * struct_mutex to free the requests+objects and allocate our slot.
1917          */
1918         err = i915_gem_wait_for_idle(dev_priv, true);
1919         if (err)
1920                 return err;
1921
1922         err = i915_mutex_lock_interruptible(&dev_priv->drm);
1923         if (!err) {
1924                 i915_gem_retire_requests(dev_priv);
1925                 err = drm_gem_create_mmap_offset(&obj->base);
1926                 mutex_unlock(&dev_priv->drm.struct_mutex);
1927         }
1928
1929         return err;
1930 }
1931
1932 static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1933 {
1934         drm_gem_free_mmap_offset(&obj->base);
1935 }
1936
1937 int
1938 i915_gem_mmap_gtt(struct drm_file *file,
1939                   struct drm_device *dev,
1940                   uint32_t handle,
1941                   uint64_t *offset)
1942 {
1943         struct drm_i915_gem_object *obj;
1944         int ret;
1945
1946         obj = i915_gem_object_lookup(file, handle);
1947         if (!obj)
1948                 return -ENOENT;
1949
1950         ret = i915_gem_object_create_mmap_offset(obj);
1951         if (ret == 0)
1952                 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1953
1954         i915_gem_object_put_unlocked(obj);
1955         return ret;
1956 }
1957
1958 /**
1959  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1960  * @dev: DRM device
1961  * @data: GTT mapping ioctl data
1962  * @file: GEM object info
1963  *
1964  * Simply returns the fake offset to userspace so it can mmap it.
1965  * The mmap call will end up in drm_gem_mmap(), which will set things
1966  * up so we can get faults in the handler above.
1967  *
1968  * The fault handler will take care of binding the object into the GTT
1969  * (since it may have been evicted to make room for something), allocating
1970  * a fence register, and mapping the appropriate aperture address into
1971  * userspace.
1972  */
1973 int
1974 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1975                         struct drm_file *file)
1976 {
1977         struct drm_i915_gem_mmap_gtt *args = data;
1978
1979         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1980 }
1981
1982 /* Immediately discard the backing storage */
1983 static void
1984 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1985 {
1986         i915_gem_object_free_mmap_offset(obj);
1987
1988         if (obj->base.filp == NULL)
1989                 return;
1990
1991         /* Our goal here is to return as much of the memory as
1992          * is possible back to the system as we are called from OOM.
1993          * To do this we must instruct the shmfs to drop all of its
1994          * backing pages, *now*.
1995          */
1996         shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
1997         obj->madv = __I915_MADV_PURGED;
1998 }
1999
2000 /* Try to discard unwanted pages */
2001 static void
2002 i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2003 {
2004         struct address_space *mapping;
2005
2006         switch (obj->madv) {
2007         case I915_MADV_DONTNEED:
2008                 i915_gem_object_truncate(obj);
2009         case __I915_MADV_PURGED:
2010                 return;
2011         }
2012
2013         if (obj->base.filp == NULL)
2014                 return;
2015
2016         mapping = obj->base.filp->f_mapping,
2017         invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2018 }
2019
2020 static void
2021 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2022 {
2023         struct sgt_iter sgt_iter;
2024         struct page *page;
2025         int ret;
2026
2027         BUG_ON(obj->madv == __I915_MADV_PURGED);
2028
2029         ret = i915_gem_object_set_to_cpu_domain(obj, true);
2030         if (WARN_ON(ret)) {
2031                 /* In the event of a disaster, abandon all caches and
2032                  * hope for the best.
2033                  */
2034                 i915_gem_clflush_object(obj, true);
2035                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2036         }
2037
2038         i915_gem_gtt_finish_object(obj);
2039
2040         if (i915_gem_object_needs_bit17_swizzle(obj))
2041                 i915_gem_object_save_bit_17_swizzle(obj);
2042
2043         if (obj->madv == I915_MADV_DONTNEED)
2044                 obj->dirty = 0;
2045
2046         for_each_sgt_page(page, sgt_iter, obj->pages) {
2047                 if (obj->dirty)
2048                         set_page_dirty(page);
2049
2050                 if (obj->madv == I915_MADV_WILLNEED)
2051                         mark_page_accessed(page);
2052
2053                 put_page(page);
2054         }
2055         obj->dirty = 0;
2056
2057         sg_free_table(obj->pages);
2058         kfree(obj->pages);
2059 }
2060
2061 int
2062 i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2063 {
2064         const struct drm_i915_gem_object_ops *ops = obj->ops;
2065
2066         if (obj->pages == NULL)
2067                 return 0;
2068
2069         if (obj->pages_pin_count)
2070                 return -EBUSY;
2071
2072         GEM_BUG_ON(obj->bind_count);
2073
2074         /* ->put_pages might need to allocate memory for the bit17 swizzle
2075          * array, hence protect them from being reaped by removing them from gtt
2076          * lists early. */
2077         list_del(&obj->global_list);
2078
2079         if (obj->mapping) {
2080                 if (is_vmalloc_addr(obj->mapping))
2081                         vunmap(obj->mapping);
2082                 else
2083                         kunmap(kmap_to_page(obj->mapping));
2084                 obj->mapping = NULL;
2085         }
2086
2087         ops->put_pages(obj);
2088         obj->pages = NULL;
2089
2090         i915_gem_object_invalidate(obj);
2091
2092         return 0;
2093 }
2094
2095 static int
2096 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2097 {
2098         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2099         int page_count, i;
2100         struct address_space *mapping;
2101         struct sg_table *st;
2102         struct scatterlist *sg;
2103         struct sgt_iter sgt_iter;
2104         struct page *page;
2105         unsigned long last_pfn = 0;     /* suppress gcc warning */
2106         int ret;
2107         gfp_t gfp;
2108
2109         /* Assert that the object is not currently in any GPU domain. As it
2110          * wasn't in the GTT, there shouldn't be any way it could have been in
2111          * a GPU cache
2112          */
2113         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2114         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2115
2116         st = kmalloc(sizeof(*st), GFP_KERNEL);
2117         if (st == NULL)
2118                 return -ENOMEM;
2119
2120         page_count = obj->base.size / PAGE_SIZE;
2121         if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2122                 kfree(st);
2123                 return -ENOMEM;
2124         }
2125
2126         /* Get the list of pages out of our struct file.  They'll be pinned
2127          * at this point until we release them.
2128          *
2129          * Fail silently without starting the shrinker
2130          */
2131         mapping = obj->base.filp->f_mapping;
2132         gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2133         gfp |= __GFP_NORETRY | __GFP_NOWARN;
2134         sg = st->sgl;
2135         st->nents = 0;
2136         for (i = 0; i < page_count; i++) {
2137                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2138                 if (IS_ERR(page)) {
2139                         i915_gem_shrink(dev_priv,
2140                                         page_count,
2141                                         I915_SHRINK_BOUND |
2142                                         I915_SHRINK_UNBOUND |
2143                                         I915_SHRINK_PURGEABLE);
2144                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2145                 }
2146                 if (IS_ERR(page)) {
2147                         /* We've tried hard to allocate the memory by reaping
2148                          * our own buffer, now let the real VM do its job and
2149                          * go down in flames if truly OOM.
2150                          */
2151                         i915_gem_shrink_all(dev_priv);
2152                         page = shmem_read_mapping_page(mapping, i);
2153                         if (IS_ERR(page)) {
2154                                 ret = PTR_ERR(page);
2155                                 goto err_pages;
2156                         }
2157                 }
2158 #ifdef CONFIG_SWIOTLB
2159                 if (swiotlb_nr_tbl()) {
2160                         st->nents++;
2161                         sg_set_page(sg, page, PAGE_SIZE, 0);
2162                         sg = sg_next(sg);
2163                         continue;
2164                 }
2165 #endif
2166                 if (!i || page_to_pfn(page) != last_pfn + 1) {
2167                         if (i)
2168                                 sg = sg_next(sg);
2169                         st->nents++;
2170                         sg_set_page(sg, page, PAGE_SIZE, 0);
2171                 } else {
2172                         sg->length += PAGE_SIZE;
2173                 }
2174                 last_pfn = page_to_pfn(page);
2175
2176                 /* Check that the i965g/gm workaround works. */
2177                 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2178         }
2179 #ifdef CONFIG_SWIOTLB
2180         if (!swiotlb_nr_tbl())
2181 #endif
2182                 sg_mark_end(sg);
2183         obj->pages = st;
2184
2185         ret = i915_gem_gtt_prepare_object(obj);
2186         if (ret)
2187                 goto err_pages;
2188
2189         if (i915_gem_object_needs_bit17_swizzle(obj))
2190                 i915_gem_object_do_bit_17_swizzle(obj);
2191
2192         if (i915_gem_object_is_tiled(obj) &&
2193             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2194                 i915_gem_object_pin_pages(obj);
2195
2196         return 0;
2197
2198 err_pages:
2199         sg_mark_end(sg);
2200         for_each_sgt_page(page, sgt_iter, st)
2201                 put_page(page);
2202         sg_free_table(st);
2203         kfree(st);
2204
2205         /* shmemfs first checks if there is enough memory to allocate the page
2206          * and reports ENOSPC should there be insufficient, along with the usual
2207          * ENOMEM for a genuine allocation failure.
2208          *
2209          * We use ENOSPC in our driver to mean that we have run out of aperture
2210          * space and so want to translate the error from shmemfs back to our
2211          * usual understanding of ENOMEM.
2212          */
2213         if (ret == -ENOSPC)
2214                 ret = -ENOMEM;
2215
2216         return ret;
2217 }
2218
2219 /* Ensure that the associated pages are gathered from the backing storage
2220  * and pinned into our object. i915_gem_object_get_pages() may be called
2221  * multiple times before they are released by a single call to
2222  * i915_gem_object_put_pages() - once the pages are no longer referenced
2223  * either as a result of memory pressure (reaping pages under the shrinker)
2224  * or as the object is itself released.
2225  */
2226 int
2227 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2228 {
2229         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2230         const struct drm_i915_gem_object_ops *ops = obj->ops;
2231         int ret;
2232
2233         if (obj->pages)
2234                 return 0;
2235
2236         if (obj->madv != I915_MADV_WILLNEED) {
2237                 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2238                 return -EFAULT;
2239         }
2240
2241         BUG_ON(obj->pages_pin_count);
2242
2243         ret = ops->get_pages(obj);
2244         if (ret)
2245                 return ret;
2246
2247         list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2248
2249         obj->get_page.sg = obj->pages->sgl;
2250         obj->get_page.last = 0;
2251
2252         return 0;
2253 }
2254
2255 /* The 'mapping' part of i915_gem_object_pin_map() below */
2256 static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
2257 {
2258         unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2259         struct sg_table *sgt = obj->pages;
2260         struct sgt_iter sgt_iter;
2261         struct page *page;
2262         struct page *stack_pages[32];
2263         struct page **pages = stack_pages;
2264         unsigned long i = 0;
2265         void *addr;
2266
2267         /* A single page can always be kmapped */
2268         if (n_pages == 1)
2269                 return kmap(sg_page(sgt->sgl));
2270
2271         if (n_pages > ARRAY_SIZE(stack_pages)) {
2272                 /* Too big for stack -- allocate temporary array instead */
2273                 pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
2274                 if (!pages)
2275                         return NULL;
2276         }
2277
2278         for_each_sgt_page(page, sgt_iter, sgt)
2279                 pages[i++] = page;
2280
2281         /* Check that we have the expected number of pages */
2282         GEM_BUG_ON(i != n_pages);
2283
2284         addr = vmap(pages, n_pages, 0, PAGE_KERNEL);
2285
2286         if (pages != stack_pages)
2287                 drm_free_large(pages);
2288
2289         return addr;
2290 }
2291
2292 /* get, pin, and map the pages of the object into kernel space */
2293 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
2294 {
2295         int ret;
2296
2297         lockdep_assert_held(&obj->base.dev->struct_mutex);
2298
2299         ret = i915_gem_object_get_pages(obj);
2300         if (ret)
2301                 return ERR_PTR(ret);
2302
2303         i915_gem_object_pin_pages(obj);
2304
2305         if (!obj->mapping) {
2306                 obj->mapping = i915_gem_object_map(obj);
2307                 if (!obj->mapping) {
2308                         i915_gem_object_unpin_pages(obj);
2309                         return ERR_PTR(-ENOMEM);
2310                 }
2311         }
2312
2313         return obj->mapping;
2314 }
2315
2316 static void
2317 i915_gem_object_retire__write(struct i915_gem_active *active,
2318                               struct drm_i915_gem_request *request)
2319 {
2320         struct drm_i915_gem_object *obj =
2321                 container_of(active, struct drm_i915_gem_object, last_write);
2322
2323         intel_fb_obj_flush(obj, true, ORIGIN_CS);
2324 }
2325
2326 static void
2327 i915_gem_object_retire__read(struct i915_gem_active *active,
2328                              struct drm_i915_gem_request *request)
2329 {
2330         int idx = request->engine->id;
2331         struct drm_i915_gem_object *obj =
2332                 container_of(active, struct drm_i915_gem_object, last_read[idx]);
2333
2334         GEM_BUG_ON(!i915_gem_object_has_active_engine(obj, idx));
2335
2336         i915_gem_object_clear_active(obj, idx);
2337         if (i915_gem_object_is_active(obj))
2338                 return;
2339
2340         /* Bump our place on the bound list to keep it roughly in LRU order
2341          * so that we don't steal from recently used but inactive objects
2342          * (unless we are forced to ofc!)
2343          */
2344         if (obj->bind_count)
2345                 list_move_tail(&obj->global_list,
2346                                &request->i915->mm.bound_list);
2347
2348         i915_gem_object_put(obj);
2349 }
2350
2351 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
2352 {
2353         unsigned long elapsed;
2354
2355         if (ctx->hang_stats.banned)
2356                 return true;
2357
2358         elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2359         if (ctx->hang_stats.ban_period_seconds &&
2360             elapsed <= ctx->hang_stats.ban_period_seconds) {
2361                 DRM_DEBUG("context hanging too fast, banning!\n");
2362                 return true;
2363         }
2364
2365         return false;
2366 }
2367
2368 static void i915_set_reset_status(struct i915_gem_context *ctx,
2369                                   const bool guilty)
2370 {
2371         struct i915_ctx_hang_stats *hs = &ctx->hang_stats;
2372
2373         if (guilty) {
2374                 hs->banned = i915_context_is_banned(ctx);
2375                 hs->batch_active++;
2376                 hs->guilty_ts = get_seconds();
2377         } else {
2378                 hs->batch_pending++;
2379         }
2380 }
2381
2382 struct drm_i915_gem_request *
2383 i915_gem_find_active_request(struct intel_engine_cs *engine)
2384 {
2385         struct drm_i915_gem_request *request;
2386
2387         /* We are called by the error capture and reset at a random
2388          * point in time. In particular, note that neither is crucially
2389          * ordered with an interrupt. After a hang, the GPU is dead and we
2390          * assume that no more writes can happen (we waited long enough for
2391          * all writes that were in transaction to be flushed) - adding an
2392          * extra delay for a recent interrupt is pointless. Hence, we do
2393          * not need an engine->irq_seqno_barrier() before the seqno reads.
2394          */
2395         list_for_each_entry(request, &engine->request_list, link) {
2396                 if (i915_gem_request_completed(request))
2397                         continue;
2398
2399                 return request;
2400         }
2401
2402         return NULL;
2403 }
2404
2405 static void i915_gem_reset_engine_status(struct intel_engine_cs *engine)
2406 {
2407         struct drm_i915_gem_request *request;
2408         bool ring_hung;
2409
2410         request = i915_gem_find_active_request(engine);
2411         if (request == NULL)
2412                 return;
2413
2414         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2415
2416         i915_set_reset_status(request->ctx, ring_hung);
2417         list_for_each_entry_continue(request, &engine->request_list, link)
2418                 i915_set_reset_status(request->ctx, false);
2419 }
2420
2421 static void i915_gem_reset_engine_cleanup(struct intel_engine_cs *engine)
2422 {
2423         struct drm_i915_gem_request *request;
2424         struct intel_ring *ring;
2425
2426         request = i915_gem_active_peek(&engine->last_request,
2427                                        &engine->i915->drm.struct_mutex);
2428
2429         /* Mark all pending requests as complete so that any concurrent
2430          * (lockless) lookup doesn't try and wait upon the request as we
2431          * reset it.
2432          */
2433         if (request)
2434                 intel_engine_init_seqno(engine, request->fence.seqno);
2435
2436         /*
2437          * Clear the execlists queue up before freeing the requests, as those
2438          * are the ones that keep the context and ringbuffer backing objects
2439          * pinned in place.
2440          */
2441
2442         if (i915.enable_execlists) {
2443                 /* Ensure irq handler finishes or is cancelled. */
2444                 tasklet_kill(&engine->irq_tasklet);
2445
2446                 intel_execlists_cancel_requests(engine);
2447         }
2448
2449         /*
2450          * We must free the requests after all the corresponding objects have
2451          * been moved off active lists. Which is the same order as the normal
2452          * retire_requests function does. This is important if object hold
2453          * implicit references on things like e.g. ppgtt address spaces through
2454          * the request.
2455          */
2456         if (request)
2457                 i915_gem_request_retire_upto(request);
2458         GEM_BUG_ON(intel_engine_is_active(engine));
2459
2460         /* Having flushed all requests from all queues, we know that all
2461          * ringbuffers must now be empty. However, since we do not reclaim
2462          * all space when retiring the request (to prevent HEADs colliding
2463          * with rapid ringbuffer wraparound) the amount of available space
2464          * upon reset is less than when we start. Do one more pass over
2465          * all the ringbuffers to reset last_retired_head.
2466          */
2467         list_for_each_entry(ring, &engine->buffers, link) {
2468                 ring->last_retired_head = ring->tail;
2469                 intel_ring_update_space(ring);
2470         }
2471
2472         engine->i915->gt.active_engines &= ~intel_engine_flag(engine);
2473 }
2474
2475 void i915_gem_reset(struct drm_device *dev)
2476 {
2477         struct drm_i915_private *dev_priv = to_i915(dev);
2478         struct intel_engine_cs *engine;
2479
2480         /*
2481          * Before we free the objects from the requests, we need to inspect
2482          * them for finding the guilty party. As the requests only borrow
2483          * their reference to the objects, the inspection must be done first.
2484          */
2485         for_each_engine(engine, dev_priv)
2486                 i915_gem_reset_engine_status(engine);
2487
2488         for_each_engine(engine, dev_priv)
2489                 i915_gem_reset_engine_cleanup(engine);
2490         mod_delayed_work(dev_priv->wq, &dev_priv->gt.idle_work, 0);
2491
2492         i915_gem_context_reset(dev);
2493
2494         i915_gem_restore_fences(dev);
2495 }
2496
2497 static void
2498 i915_gem_retire_work_handler(struct work_struct *work)
2499 {
2500         struct drm_i915_private *dev_priv =
2501                 container_of(work, typeof(*dev_priv), gt.retire_work.work);
2502         struct drm_device *dev = &dev_priv->drm;
2503
2504         /* Come back later if the device is busy... */
2505         if (mutex_trylock(&dev->struct_mutex)) {
2506                 i915_gem_retire_requests(dev_priv);
2507                 mutex_unlock(&dev->struct_mutex);
2508         }
2509
2510         /* Keep the retire handler running until we are finally idle.
2511          * We do not need to do this test under locking as in the worst-case
2512          * we queue the retire worker once too often.
2513          */
2514         if (READ_ONCE(dev_priv->gt.awake)) {
2515                 i915_queue_hangcheck(dev_priv);
2516                 queue_delayed_work(dev_priv->wq,
2517                                    &dev_priv->gt.retire_work,
2518                                    round_jiffies_up_relative(HZ));
2519         }
2520 }
2521
2522 static void
2523 i915_gem_idle_work_handler(struct work_struct *work)
2524 {
2525         struct drm_i915_private *dev_priv =
2526                 container_of(work, typeof(*dev_priv), gt.idle_work.work);
2527         struct drm_device *dev = &dev_priv->drm;
2528         struct intel_engine_cs *engine;
2529         unsigned int stuck_engines;
2530         bool rearm_hangcheck;
2531
2532         if (!READ_ONCE(dev_priv->gt.awake))
2533                 return;
2534
2535         if (READ_ONCE(dev_priv->gt.active_engines))
2536                 return;
2537
2538         rearm_hangcheck =
2539                 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
2540
2541         if (!mutex_trylock(&dev->struct_mutex)) {
2542                 /* Currently busy, come back later */
2543                 mod_delayed_work(dev_priv->wq,
2544                                  &dev_priv->gt.idle_work,
2545                                  msecs_to_jiffies(50));
2546                 goto out_rearm;
2547         }
2548
2549         if (dev_priv->gt.active_engines)
2550                 goto out_unlock;
2551
2552         for_each_engine(engine, dev_priv)
2553                 i915_gem_batch_pool_fini(&engine->batch_pool);
2554
2555         GEM_BUG_ON(!dev_priv->gt.awake);
2556         dev_priv->gt.awake = false;
2557         rearm_hangcheck = false;
2558
2559         /* As we have disabled hangcheck, we need to unstick any waiters still
2560          * hanging around. However, as we may be racing against the interrupt
2561          * handler or the waiters themselves, we skip enabling the fake-irq.
2562          */
2563         stuck_engines = intel_kick_waiters(dev_priv);
2564         if (unlikely(stuck_engines))
2565                 DRM_DEBUG_DRIVER("kicked stuck waiters (%x)...missed irq?\n",
2566                                  stuck_engines);
2567
2568         if (INTEL_GEN(dev_priv) >= 6)
2569                 gen6_rps_idle(dev_priv);
2570         intel_runtime_pm_put(dev_priv);
2571 out_unlock:
2572         mutex_unlock(&dev->struct_mutex);
2573
2574 out_rearm:
2575         if (rearm_hangcheck) {
2576                 GEM_BUG_ON(!dev_priv->gt.awake);
2577                 i915_queue_hangcheck(dev_priv);
2578         }
2579 }
2580
2581 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
2582 {
2583         struct drm_i915_gem_object *obj = to_intel_bo(gem);
2584         struct drm_i915_file_private *fpriv = file->driver_priv;
2585         struct i915_vma *vma, *vn;
2586
2587         mutex_lock(&obj->base.dev->struct_mutex);
2588         list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
2589                 if (vma->vm->file == fpriv)
2590                         i915_vma_close(vma);
2591         mutex_unlock(&obj->base.dev->struct_mutex);
2592 }
2593
2594 /**
2595  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2596  * @dev: drm device pointer
2597  * @data: ioctl data blob
2598  * @file: drm file pointer
2599  *
2600  * Returns 0 if successful, else an error is returned with the remaining time in
2601  * the timeout parameter.
2602  *  -ETIME: object is still busy after timeout
2603  *  -ERESTARTSYS: signal interrupted the wait
2604  *  -ENONENT: object doesn't exist
2605  * Also possible, but rare:
2606  *  -EAGAIN: GPU wedged
2607  *  -ENOMEM: damn
2608  *  -ENODEV: Internal IRQ fail
2609  *  -E?: The add request failed
2610  *
2611  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2612  * non-zero timeout parameter the wait ioctl will wait for the given number of
2613  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2614  * without holding struct_mutex the object may become re-busied before this
2615  * function completes. A similar but shorter * race condition exists in the busy
2616  * ioctl
2617  */
2618 int
2619 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2620 {
2621         struct drm_i915_gem_wait *args = data;
2622         struct intel_rps_client *rps = to_rps_client(file);
2623         struct drm_i915_gem_object *obj;
2624         unsigned long active;
2625         int idx, ret = 0;
2626
2627         if (args->flags != 0)
2628                 return -EINVAL;
2629
2630         obj = i915_gem_object_lookup(file, args->bo_handle);
2631         if (!obj)
2632                 return -ENOENT;
2633
2634         active = __I915_BO_ACTIVE(obj);
2635         for_each_active(active, idx) {
2636                 s64 *timeout = args->timeout_ns >= 0 ? &args->timeout_ns : NULL;
2637                 ret = i915_gem_active_wait_unlocked(&obj->last_read[idx], true,
2638                                                     timeout, rps);
2639                 if (ret)
2640                         break;
2641         }
2642
2643         i915_gem_object_put_unlocked(obj);
2644         return ret;
2645 }
2646
2647 static int
2648 __i915_gem_object_sync(struct drm_i915_gem_request *to,
2649                        struct drm_i915_gem_request *from)
2650 {
2651         int ret;
2652
2653         if (to->engine == from->engine)
2654                 return 0;
2655
2656         if (!i915.semaphores) {
2657                 ret = i915_wait_request(from,
2658                                         from->i915->mm.interruptible,
2659                                         NULL,
2660                                         NO_WAITBOOST);
2661                 if (ret)
2662                         return ret;
2663         } else {
2664                 int idx = intel_engine_sync_index(from->engine, to->engine);
2665                 if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx])
2666                         return 0;
2667
2668                 trace_i915_gem_ring_sync_to(to, from);
2669                 ret = to->engine->semaphore.sync_to(to, from);
2670                 if (ret)
2671                         return ret;
2672
2673                 from->engine->semaphore.sync_seqno[idx] = from->fence.seqno;
2674         }
2675
2676         return 0;
2677 }
2678
2679 /**
2680  * i915_gem_object_sync - sync an object to a ring.
2681  *
2682  * @obj: object which may be in use on another ring.
2683  * @to: request we are wishing to use
2684  *
2685  * This code is meant to abstract object synchronization with the GPU.
2686  * Conceptually we serialise writes between engines inside the GPU.
2687  * We only allow one engine to write into a buffer at any time, but
2688  * multiple readers. To ensure each has a coherent view of memory, we must:
2689  *
2690  * - If there is an outstanding write request to the object, the new
2691  *   request must wait for it to complete (either CPU or in hw, requests
2692  *   on the same ring will be naturally ordered).
2693  *
2694  * - If we are a write request (pending_write_domain is set), the new
2695  *   request must wait for outstanding read requests to complete.
2696  *
2697  * Returns 0 if successful, else propagates up the lower layer error.
2698  */
2699 int
2700 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2701                      struct drm_i915_gem_request *to)
2702 {
2703         struct i915_gem_active *active;
2704         unsigned long active_mask;
2705         int idx;
2706
2707         lockdep_assert_held(&obj->base.dev->struct_mutex);
2708
2709         active_mask = i915_gem_object_get_active(obj);
2710         if (!active_mask)
2711                 return 0;
2712
2713         if (obj->base.pending_write_domain) {
2714                 active = obj->last_read;
2715         } else {
2716                 active_mask = 1;
2717                 active = &obj->last_write;
2718         }
2719
2720         for_each_active(active_mask, idx) {
2721                 struct drm_i915_gem_request *request;
2722                 int ret;
2723
2724                 request = i915_gem_active_peek(&active[idx],
2725                                                &obj->base.dev->struct_mutex);
2726                 if (!request)
2727                         continue;
2728
2729                 ret = __i915_gem_object_sync(to, request);
2730                 if (ret)
2731                         return ret;
2732         }
2733
2734         return 0;
2735 }
2736
2737 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2738 {
2739         u32 old_write_domain, old_read_domains;
2740
2741         /* Force a pagefault for domain tracking on next user access */
2742         i915_gem_release_mmap(obj);
2743
2744         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2745                 return;
2746
2747         old_read_domains = obj->base.read_domains;
2748         old_write_domain = obj->base.write_domain;
2749
2750         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2751         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2752
2753         trace_i915_gem_object_change_domain(obj,
2754                                             old_read_domains,
2755                                             old_write_domain);
2756 }
2757
2758 static void __i915_vma_iounmap(struct i915_vma *vma)
2759 {
2760         GEM_BUG_ON(i915_vma_is_pinned(vma));
2761
2762         if (vma->iomap == NULL)
2763                 return;
2764
2765         io_mapping_unmap(vma->iomap);
2766         vma->iomap = NULL;
2767 }
2768
2769 int i915_vma_unbind(struct i915_vma *vma)
2770 {
2771         struct drm_i915_gem_object *obj = vma->obj;
2772         unsigned long active;
2773         int ret;
2774
2775         /* First wait upon any activity as retiring the request may
2776          * have side-effects such as unpinning or even unbinding this vma.
2777          */
2778         active = i915_vma_get_active(vma);
2779         if (active) {
2780                 int idx;
2781
2782                 /* When a closed VMA is retired, it is unbound - eek.
2783                  * In order to prevent it from being recursively closed,
2784                  * take a pin on the vma so that the second unbind is
2785                  * aborted.
2786                  */
2787                 __i915_vma_pin(vma);
2788
2789                 for_each_active(active, idx) {
2790                         ret = i915_gem_active_retire(&vma->last_read[idx],
2791                                                    &vma->vm->dev->struct_mutex);
2792                         if (ret)
2793                                 break;
2794                 }
2795
2796                 __i915_vma_unpin(vma);
2797                 if (ret)
2798                         return ret;
2799
2800                 GEM_BUG_ON(i915_vma_is_active(vma));
2801         }
2802
2803         if (i915_vma_is_pinned(vma))
2804                 return -EBUSY;
2805
2806         if (!drm_mm_node_allocated(&vma->node))
2807                 goto destroy;
2808
2809         GEM_BUG_ON(obj->bind_count == 0);
2810         GEM_BUG_ON(!obj->pages);
2811
2812         if (i915_vma_is_ggtt(vma) &&
2813             vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
2814                 i915_gem_object_finish_gtt(obj);
2815
2816                 /* release the fence reg _after_ flushing */
2817                 ret = i915_gem_object_put_fence(obj);
2818                 if (ret)
2819                         return ret;
2820
2821                 __i915_vma_iounmap(vma);
2822         }
2823
2824         if (likely(!vma->vm->closed)) {
2825                 trace_i915_vma_unbind(vma);
2826                 vma->vm->unbind_vma(vma);
2827         }
2828         vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
2829
2830         drm_mm_remove_node(&vma->node);
2831         list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
2832
2833         if (i915_vma_is_ggtt(vma)) {
2834                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
2835                         obj->map_and_fenceable = false;
2836                 } else if (vma->ggtt_view.pages) {
2837                         sg_free_table(vma->ggtt_view.pages);
2838                         kfree(vma->ggtt_view.pages);
2839                 }
2840                 vma->ggtt_view.pages = NULL;
2841         }
2842
2843         /* Since the unbound list is global, only move to that list if
2844          * no more VMAs exist. */
2845         if (--obj->bind_count == 0)
2846                 list_move_tail(&obj->global_list,
2847                                &to_i915(obj->base.dev)->mm.unbound_list);
2848
2849         /* And finally now the object is completely decoupled from this vma,
2850          * we can drop its hold on the backing storage and allow it to be
2851          * reaped by the shrinker.
2852          */
2853         i915_gem_object_unpin_pages(obj);
2854
2855 destroy:
2856         if (unlikely(i915_vma_is_closed(vma)))
2857                 i915_vma_destroy(vma);
2858
2859         return 0;
2860 }
2861
2862 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
2863                            bool interruptible)
2864 {
2865         struct intel_engine_cs *engine;
2866         int ret;
2867
2868         for_each_engine(engine, dev_priv) {
2869                 if (engine->last_context == NULL)
2870                         continue;
2871
2872                 ret = intel_engine_idle(engine, interruptible);
2873                 if (ret)
2874                         return ret;
2875         }
2876
2877         return 0;
2878 }
2879
2880 static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
2881                                      unsigned long cache_level)
2882 {
2883         struct drm_mm_node *gtt_space = &vma->node;
2884         struct drm_mm_node *other;
2885
2886         /*
2887          * On some machines we have to be careful when putting differing types
2888          * of snoopable memory together to avoid the prefetcher crossing memory
2889          * domains and dying. During vm initialisation, we decide whether or not
2890          * these constraints apply and set the drm_mm.color_adjust
2891          * appropriately.
2892          */
2893         if (vma->vm->mm.color_adjust == NULL)
2894                 return true;
2895
2896         if (!drm_mm_node_allocated(gtt_space))
2897                 return true;
2898
2899         if (list_empty(&gtt_space->node_list))
2900                 return true;
2901
2902         other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
2903         if (other->allocated && !other->hole_follows && other->color != cache_level)
2904                 return false;
2905
2906         other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
2907         if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
2908                 return false;
2909
2910         return true;
2911 }
2912
2913 /**
2914  * i915_vma_insert - finds a slot for the vma in its address space
2915  * @vma: the vma
2916  * @size: requested size in bytes (can be larger than the VMA)
2917  * @alignment: required alignment
2918  * @flags: mask of PIN_* flags to use
2919  *
2920  * First we try to allocate some free space that meets the requirements for
2921  * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
2922  * preferrably the oldest idle entry to make room for the new VMA.
2923  *
2924  * Returns:
2925  * 0 on success, negative error code otherwise.
2926  */
2927 static int
2928 i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
2929 {
2930         struct drm_i915_private *dev_priv = to_i915(vma->vm->dev);
2931         struct drm_i915_gem_object *obj = vma->obj;
2932         u64 start, end;
2933         u64 min_alignment;
2934         int ret;
2935
2936         GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
2937         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
2938
2939         size = max(size, vma->size);
2940         if (flags & PIN_MAPPABLE)
2941                 size = i915_gem_get_ggtt_size(dev_priv, size,
2942                                               i915_gem_object_get_tiling(obj));
2943
2944         min_alignment =
2945                 i915_gem_get_ggtt_alignment(dev_priv, size,
2946                                             i915_gem_object_get_tiling(obj),
2947                                             flags & PIN_MAPPABLE);
2948         if (alignment == 0)
2949                 alignment = min_alignment;
2950         if (alignment & (min_alignment - 1)) {
2951                 DRM_DEBUG("Invalid object alignment requested %llu, minimum %llu\n",
2952                           alignment, min_alignment);
2953                 return -EINVAL;
2954         }
2955
2956         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
2957
2958         end = vma->vm->total;
2959         if (flags & PIN_MAPPABLE)
2960                 end = min_t(u64, end, dev_priv->ggtt.mappable_end);
2961         if (flags & PIN_ZONE_4G)
2962                 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
2963
2964         /* If binding the object/GGTT view requires more space than the entire
2965          * aperture has, reject it early before evicting everything in a vain
2966          * attempt to find space.
2967          */
2968         if (size > end) {
2969                 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",
2970                           size, obj->base.size,
2971                           flags & PIN_MAPPABLE ? "mappable" : "total",
2972                           end);
2973                 return -E2BIG;
2974         }
2975
2976         ret = i915_gem_object_get_pages(obj);
2977         if (ret)
2978                 return ret;
2979
2980         i915_gem_object_pin_pages(obj);
2981
2982         if (flags & PIN_OFFSET_FIXED) {
2983                 u64 offset = flags & PIN_OFFSET_MASK;
2984                 if (offset & (alignment - 1) || offset > end - size) {
2985                         ret = -EINVAL;
2986                         goto err_unpin;
2987                 }
2988
2989                 vma->node.start = offset;
2990                 vma->node.size = size;
2991                 vma->node.color = obj->cache_level;
2992                 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
2993                 if (ret) {
2994                         ret = i915_gem_evict_for_vma(vma);
2995                         if (ret == 0)
2996                                 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node);
2997                         if (ret)
2998                                 goto err_unpin;
2999                 }
3000         } else {
3001                 u32 search_flag, alloc_flag;
3002
3003                 if (flags & PIN_HIGH) {
3004                         search_flag = DRM_MM_SEARCH_BELOW;
3005                         alloc_flag = DRM_MM_CREATE_TOP;
3006                 } else {
3007                         search_flag = DRM_MM_SEARCH_DEFAULT;
3008                         alloc_flag = DRM_MM_CREATE_DEFAULT;
3009                 }
3010
3011                 /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
3012                  * so we know that we always have a minimum alignment of 4096.
3013                  * The drm_mm range manager is optimised to return results
3014                  * with zero alignment, so where possible use the optimal
3015                  * path.
3016                  */
3017                 if (alignment <= 4096)
3018                         alignment = 0;
3019
3020 search_free:
3021                 ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm,
3022                                                           &vma->node,
3023                                                           size, alignment,
3024                                                           obj->cache_level,
3025                                                           start, end,
3026                                                           search_flag,
3027                                                           alloc_flag);
3028                 if (ret) {
3029                         ret = i915_gem_evict_something(vma->vm, size, alignment,
3030                                                        obj->cache_level,
3031                                                        start, end,
3032                                                        flags);
3033                         if (ret == 0)
3034                                 goto search_free;
3035
3036                         goto err_unpin;
3037                 }
3038         }
3039         GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
3040
3041         list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3042         list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3043         obj->bind_count++;
3044
3045         return 0;
3046
3047 err_unpin:
3048         i915_gem_object_unpin_pages(obj);
3049         return ret;
3050 }
3051
3052 bool
3053 i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3054                         bool force)
3055 {
3056         /* If we don't have a page list set up, then we're not pinned
3057          * to GPU, and we can ignore the cache flush because it'll happen
3058          * again at bind time.
3059          */
3060         if (obj->pages == NULL)
3061                 return false;
3062
3063         /*
3064          * Stolen memory is always coherent with the GPU as it is explicitly
3065          * marked as wc by the system, or the system is cache-coherent.
3066          */
3067         if (obj->stolen || obj->phys_handle)
3068                 return false;
3069
3070         /* If the GPU is snooping the contents of the CPU cache,
3071          * we do not need to manually clear the CPU cache lines.  However,
3072          * the caches are only snooped when the render cache is
3073          * flushed/invalidated.  As we always have to emit invalidations
3074          * and flushes when moving into and out of the RENDER domain, correct
3075          * snooping behaviour occurs naturally as the result of our domain
3076          * tracking.
3077          */
3078         if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3079                 obj->cache_dirty = true;
3080                 return false;
3081         }
3082
3083         trace_i915_gem_object_clflush(obj);
3084         drm_clflush_sg(obj->pages);
3085         obj->cache_dirty = false;
3086
3087         return true;
3088 }
3089
3090 /** Flushes the GTT write domain for the object if it's dirty. */
3091 static void
3092 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3093 {
3094         uint32_t old_write_domain;
3095
3096         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3097                 return;
3098
3099         /* No actual flushing is required for the GTT write domain.  Writes
3100          * to it immediately go to main memory as far as we know, so there's
3101          * no chipset flush.  It also doesn't land in render cache.
3102          *
3103          * However, we do have to enforce the order so that all writes through
3104          * the GTT land before any writes to the device, such as updates to
3105          * the GATT itself.
3106          */
3107         wmb();
3108
3109         old_write_domain = obj->base.write_domain;
3110         obj->base.write_domain = 0;
3111
3112         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3113
3114         trace_i915_gem_object_change_domain(obj,
3115                                             obj->base.read_domains,
3116                                             old_write_domain);
3117 }
3118
3119 /** Flushes the CPU write domain for the object if it's dirty. */
3120 static void
3121 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3122 {
3123         uint32_t old_write_domain;
3124
3125         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3126                 return;
3127
3128         if (i915_gem_clflush_object(obj, obj->pin_display))
3129                 i915_gem_chipset_flush(to_i915(obj->base.dev));
3130
3131         old_write_domain = obj->base.write_domain;
3132         obj->base.write_domain = 0;
3133
3134         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3135
3136         trace_i915_gem_object_change_domain(obj,
3137                                             obj->base.read_domains,
3138                                             old_write_domain);
3139 }
3140
3141 /**
3142  * Moves a single object to the GTT read, and possibly write domain.
3143  * @obj: object to act on
3144  * @write: ask for write access or read only
3145  *
3146  * This function returns when the move is complete, including waiting on
3147  * flushes to occur.
3148  */
3149 int
3150 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3151 {
3152         uint32_t old_write_domain, old_read_domains;
3153         struct i915_vma *vma;
3154         int ret;
3155
3156         ret = i915_gem_object_wait_rendering(obj, !write);
3157         if (ret)
3158                 return ret;
3159
3160         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3161                 return 0;
3162
3163         /* Flush and acquire obj->pages so that we are coherent through
3164          * direct access in memory with previous cached writes through
3165          * shmemfs and that our cache domain tracking remains valid.
3166          * For example, if the obj->filp was moved to swap without us
3167          * being notified and releasing the pages, we would mistakenly
3168          * continue to assume that the obj remained out of the CPU cached
3169          * domain.
3170          */
3171         ret = i915_gem_object_get_pages(obj);
3172         if (ret)
3173                 return ret;
3174
3175         i915_gem_object_flush_cpu_write_domain(obj);
3176
3177         /* Serialise direct access to this object with the barriers for
3178          * coherent writes from the GPU, by effectively invalidating the
3179          * GTT domain upon first access.
3180          */
3181         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3182                 mb();
3183
3184         old_write_domain = obj->base.write_domain;
3185         old_read_domains = obj->base.read_domains;
3186
3187         /* It should now be out of any other write domains, and we can update
3188          * the domain values for our changes.
3189          */
3190         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3191         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3192         if (write) {
3193                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3194                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3195                 obj->dirty = 1;
3196         }
3197
3198         trace_i915_gem_object_change_domain(obj,
3199                                             old_read_domains,
3200                                             old_write_domain);
3201
3202         /* And bump the LRU for this access */
3203         vma = i915_gem_obj_to_ggtt(obj);
3204         if (vma &&
3205             drm_mm_node_allocated(&vma->node) &&
3206             !i915_vma_is_active(vma))
3207                 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
3208
3209         return 0;
3210 }
3211
3212 /**
3213  * Changes the cache-level of an object across all VMA.
3214  * @obj: object to act on
3215  * @cache_level: new cache level to set for the object
3216  *
3217  * After this function returns, the object will be in the new cache-level
3218  * across all GTT and the contents of the backing storage will be coherent,
3219  * with respect to the new cache-level. In order to keep the backing storage
3220  * coherent for all users, we only allow a single cache level to be set
3221  * globally on the object and prevent it from being changed whilst the
3222  * hardware is reading from the object. That is if the object is currently
3223  * on the scanout it will be set to uncached (or equivalent display
3224  * cache coherency) and all non-MOCS GPU access will also be uncached so
3225  * that all direct access to the scanout remains coherent.
3226  */
3227 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3228                                     enum i915_cache_level cache_level)
3229 {
3230         struct i915_vma *vma;
3231         int ret = 0;
3232
3233         if (obj->cache_level == cache_level)
3234                 goto out;
3235
3236         /* Inspect the list of currently bound VMA and unbind any that would
3237          * be invalid given the new cache-level. This is principally to
3238          * catch the issue of the CS prefetch crossing page boundaries and
3239          * reading an invalid PTE on older architectures.
3240          */
3241 restart:
3242         list_for_each_entry(vma, &obj->vma_list, obj_link) {
3243                 if (!drm_mm_node_allocated(&vma->node))
3244                         continue;
3245
3246                 if (i915_vma_is_pinned(vma)) {
3247                         DRM_DEBUG("can not change the cache level of pinned objects\n");
3248                         return -EBUSY;
3249                 }
3250
3251                 if (i915_gem_valid_gtt_space(vma, cache_level))
3252                         continue;
3253
3254                 ret = i915_vma_unbind(vma);
3255                 if (ret)
3256                         return ret;
3257
3258                 /* As unbinding may affect other elements in the
3259                  * obj->vma_list (due to side-effects from retiring
3260                  * an active vma), play safe and restart the iterator.
3261                  */
3262                 goto restart;
3263         }
3264
3265         /* We can reuse the existing drm_mm nodes but need to change the
3266          * cache-level on the PTE. We could simply unbind them all and
3267          * rebind with the correct cache-level on next use. However since
3268          * we already have a valid slot, dma mapping, pages etc, we may as
3269          * rewrite the PTE in the belief that doing so tramples upon less
3270          * state and so involves less work.
3271          */
3272         if (obj->bind_count) {
3273                 /* Before we change the PTE, the GPU must not be accessing it.
3274                  * If we wait upon the object, we know that all the bound
3275                  * VMA are no longer active.
3276                  */
3277                 ret = i915_gem_object_wait_rendering(obj, false);
3278                 if (ret)
3279                         return ret;
3280
3281                 if (!HAS_LLC(obj->base.dev) && cache_level != I915_CACHE_NONE) {
3282                         /* Access to snoopable pages through the GTT is
3283                          * incoherent and on some machines causes a hard
3284                          * lockup. Relinquish the CPU mmaping to force
3285                          * userspace to refault in the pages and we can
3286                          * then double check if the GTT mapping is still
3287                          * valid for that pointer access.
3288                          */
3289                         i915_gem_release_mmap(obj);
3290
3291                         /* As we no longer need a fence for GTT access,
3292                          * we can relinquish it now (and so prevent having
3293                          * to steal a fence from someone else on the next
3294                          * fence request). Note GPU activity would have
3295                          * dropped the fence as all snoopable access is
3296                          * supposed to be linear.
3297                          */
3298                         ret = i915_gem_object_put_fence(obj);
3299                         if (ret)
3300                                 return ret;
3301                 } else {
3302                         /* We either have incoherent backing store and
3303                          * so no GTT access or the architecture is fully
3304                          * coherent. In such cases, existing GTT mmaps
3305                          * ignore the cache bit in the PTE and we can
3306                          * rewrite it without confusing the GPU or having
3307                          * to force userspace to fault back in its mmaps.
3308                          */
3309                 }
3310
3311                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3312                         if (!drm_mm_node_allocated(&vma->node))
3313                                 continue;
3314
3315                         ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3316                         if (ret)
3317                                 return ret;
3318                 }
3319         }
3320
3321         list_for_each_entry(vma, &obj->vma_list, obj_link)
3322                 vma->node.color = cache_level;
3323         obj->cache_level = cache_level;
3324
3325 out:
3326         /* Flush the dirty CPU caches to the backing storage so that the
3327          * object is now coherent at its new cache level (with respect
3328          * to the access domain).
3329          */
3330         if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
3331                 if (i915_gem_clflush_object(obj, true))
3332                         i915_gem_chipset_flush(to_i915(obj->base.dev));
3333         }
3334
3335         return 0;
3336 }
3337
3338 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3339                                struct drm_file *file)
3340 {
3341         struct drm_i915_gem_caching *args = data;
3342         struct drm_i915_gem_object *obj;
3343
3344         obj = i915_gem_object_lookup(file, args->handle);
3345         if (!obj)
3346                 return -ENOENT;
3347
3348         switch (obj->cache_level) {
3349         case I915_CACHE_LLC:
3350         case I915_CACHE_L3_LLC:
3351                 args->caching = I915_CACHING_CACHED;
3352                 break;
3353
3354         case I915_CACHE_WT:
3355                 args->caching = I915_CACHING_DISPLAY;
3356                 break;
3357
3358         default:
3359                 args->caching = I915_CACHING_NONE;
3360                 break;
3361         }
3362
3363         i915_gem_object_put_unlocked(obj);
3364         return 0;
3365 }
3366
3367 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3368                                struct drm_file *file)
3369 {
3370         struct drm_i915_private *dev_priv = to_i915(dev);
3371         struct drm_i915_gem_caching *args = data;
3372         struct drm_i915_gem_object *obj;
3373         enum i915_cache_level level;
3374         int ret;
3375
3376         switch (args->caching) {
3377         case I915_CACHING_NONE:
3378                 level = I915_CACHE_NONE;
3379                 break;
3380         case I915_CACHING_CACHED:
3381                 /*
3382                  * Due to a HW issue on BXT A stepping, GPU stores via a
3383                  * snooped mapping may leave stale data in a corresponding CPU
3384                  * cacheline, whereas normally such cachelines would get
3385                  * invalidated.
3386                  */
3387                 if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
3388                         return -ENODEV;
3389
3390                 level = I915_CACHE_LLC;
3391                 break;
3392         case I915_CACHING_DISPLAY:
3393                 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3394                 break;
3395         default:
3396                 return -EINVAL;
3397         }
3398
3399         intel_runtime_pm_get(dev_priv);
3400
3401         ret = i915_mutex_lock_interruptible(dev);
3402         if (ret)
3403                 goto rpm_put;
3404
3405         obj = i915_gem_object_lookup(file, args->handle);
3406         if (!obj) {
3407                 ret = -ENOENT;
3408                 goto unlock;
3409         }
3410
3411         ret = i915_gem_object_set_cache_level(obj, level);
3412
3413         i915_gem_object_put(obj);
3414 unlock:
3415         mutex_unlock(&dev->struct_mutex);
3416 rpm_put:
3417         intel_runtime_pm_put(dev_priv);
3418
3419         return ret;
3420 }
3421
3422 /*
3423  * Prepare buffer for display plane (scanout, cursors, etc).
3424  * Can be called from an uninterruptible phase (modesetting) and allows
3425  * any flushes to be pipelined (for pageflips).
3426  */
3427 int
3428 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3429                                      u32 alignment,
3430                                      const struct i915_ggtt_view *view)
3431 {
3432         u32 old_read_domains, old_write_domain;
3433         int ret;
3434
3435         /* Mark the pin_display early so that we account for the
3436          * display coherency whilst setting up the cache domains.
3437          */
3438         obj->pin_display++;
3439
3440         /* The display engine is not coherent with the LLC cache on gen6.  As
3441          * a result, we make sure that the pinning that is about to occur is
3442          * done with uncached PTEs. This is lowest common denominator for all
3443          * chipsets.
3444          *
3445          * However for gen6+, we could do better by using the GFDT bit instead
3446          * of uncaching, which would allow us to flush all the LLC-cached data
3447          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3448          */
3449         ret = i915_gem_object_set_cache_level(obj,
3450                                               HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3451         if (ret)
3452                 goto err_unpin_display;
3453
3454         /* As the user may map the buffer once pinned in the display plane
3455          * (e.g. libkms for the bootup splash), we have to ensure that we
3456          * always use map_and_fenceable for all scanout buffers.
3457          */
3458         ret = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
3459                                        view->type == I915_GGTT_VIEW_NORMAL ?
3460                                        PIN_MAPPABLE : 0);
3461         if (ret)
3462                 goto err_unpin_display;
3463
3464         i915_gem_object_flush_cpu_write_domain(obj);
3465
3466         old_write_domain = obj->base.write_domain;
3467         old_read_domains = obj->base.read_domains;
3468
3469         /* It should now be out of any other write domains, and we can update
3470          * the domain values for our changes.
3471          */
3472         obj->base.write_domain = 0;
3473         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3474
3475         trace_i915_gem_object_change_domain(obj,
3476                                             old_read_domains,
3477                                             old_write_domain);
3478
3479         return 0;
3480
3481 err_unpin_display:
3482         obj->pin_display--;
3483         return ret;
3484 }
3485
3486 void
3487 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
3488                                          const struct i915_ggtt_view *view)
3489 {
3490         if (WARN_ON(obj->pin_display == 0))
3491                 return;
3492
3493         i915_gem_object_ggtt_unpin_view(obj, view);
3494
3495         obj->pin_display--;
3496 }
3497
3498 /**
3499  * Moves a single object to the CPU read, and possibly write domain.
3500  * @obj: object to act on
3501  * @write: requesting write or read-only access
3502  *
3503  * This function returns when the move is complete, including waiting on
3504  * flushes to occur.
3505  */
3506 int
3507 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3508 {
3509         uint32_t old_write_domain, old_read_domains;
3510         int ret;
3511
3512         ret = i915_gem_object_wait_rendering(obj, !write);
3513         if (ret)
3514                 return ret;
3515
3516         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3517                 return 0;
3518
3519         i915_gem_object_flush_gtt_write_domain(obj);
3520
3521         old_write_domain = obj->base.write_domain;
3522         old_read_domains = obj->base.read_domains;
3523
3524         /* Flush the CPU cache if it's still invalid. */
3525         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3526                 i915_gem_clflush_object(obj, false);
3527
3528                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3529         }
3530
3531         /* It should now be out of any other write domains, and we can update
3532          * the domain values for our changes.
3533          */
3534         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3535
3536         /* If we're writing through the CPU, then the GPU read domains will
3537          * need to be invalidated at next use.
3538          */
3539         if (write) {
3540                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3541                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3542         }
3543
3544         trace_i915_gem_object_change_domain(obj,
3545                                             old_read_domains,
3546                                             old_write_domain);
3547
3548         return 0;
3549 }
3550
3551 /* Throttle our rendering by waiting until the ring has completed our requests
3552  * emitted over 20 msec ago.
3553  *
3554  * Note that if we were to use the current jiffies each time around the loop,
3555  * we wouldn't escape the function with any frames outstanding if the time to
3556  * render a frame was over 20ms.
3557  *
3558  * This should get us reasonable parallelism between CPU and GPU but also
3559  * relatively low latency when blocking on a particular request to finish.
3560  */
3561 static int
3562 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3563 {
3564         struct drm_i915_private *dev_priv = to_i915(dev);
3565         struct drm_i915_file_private *file_priv = file->driver_priv;
3566         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3567         struct drm_i915_gem_request *request, *target = NULL;
3568         int ret;
3569
3570         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3571         if (ret)
3572                 return ret;
3573
3574         /* ABI: return -EIO if already wedged */
3575         if (i915_terminally_wedged(&dev_priv->gpu_error))
3576                 return -EIO;
3577
3578         spin_lock(&file_priv->mm.lock);
3579         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3580                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3581                         break;
3582
3583                 /*
3584                  * Note that the request might not have been submitted yet.
3585                  * In which case emitted_jiffies will be zero.
3586                  */
3587                 if (!request->emitted_jiffies)
3588                         continue;
3589
3590                 target = request;
3591         }
3592         if (target)
3593                 i915_gem_request_get(target);
3594         spin_unlock(&file_priv->mm.lock);
3595
3596         if (target == NULL)
3597                 return 0;
3598
3599         ret = i915_wait_request(target, true, NULL, NULL);
3600         i915_gem_request_put(target);
3601
3602         return ret;
3603 }
3604
3605 static bool
3606 i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
3607 {
3608         struct drm_i915_gem_object *obj = vma->obj;
3609
3610         if (!drm_mm_node_allocated(&vma->node))
3611                 return false;
3612
3613         if (vma->node.size < size)
3614                 return true;
3615
3616         if (alignment && vma->node.start & (alignment - 1))
3617                 return true;
3618
3619         if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
3620                 return true;
3621
3622         if (flags & PIN_OFFSET_BIAS &&
3623             vma->node.start < (flags & PIN_OFFSET_MASK))
3624                 return true;
3625
3626         if (flags & PIN_OFFSET_FIXED &&
3627             vma->node.start != (flags & PIN_OFFSET_MASK))
3628                 return true;
3629
3630         return false;
3631 }
3632
3633 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
3634 {
3635         struct drm_i915_gem_object *obj = vma->obj;
3636         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3637         bool mappable, fenceable;
3638         u32 fence_size, fence_alignment;
3639
3640         fence_size = i915_gem_get_ggtt_size(dev_priv,
3641                                             obj->base.size,
3642                                             i915_gem_object_get_tiling(obj));
3643         fence_alignment = i915_gem_get_ggtt_alignment(dev_priv,
3644                                                       obj->base.size,
3645                                                       i915_gem_object_get_tiling(obj),
3646                                                       true);
3647
3648         fenceable = (vma->node.size == fence_size &&
3649                      (vma->node.start & (fence_alignment - 1)) == 0);
3650
3651         mappable = (vma->node.start + fence_size <=
3652                     dev_priv->ggtt.mappable_end);
3653
3654         obj->map_and_fenceable = mappable && fenceable;
3655 }
3656
3657 int __i915_vma_do_pin(struct i915_vma *vma,
3658                       u64 size, u64 alignment, u64 flags)
3659 {
3660         unsigned int bound = vma->flags;
3661         int ret;
3662
3663         GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
3664         GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
3665
3666         if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
3667                 ret = -EBUSY;
3668                 goto err;
3669         }
3670
3671         if ((bound & I915_VMA_BIND_MASK) == 0) {
3672                 ret = i915_vma_insert(vma, size, alignment, flags);
3673                 if (ret)
3674                         goto err;
3675         }
3676
3677         ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
3678         if (ret)
3679                 goto err;
3680
3681         if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
3682                 __i915_vma_set_map_and_fenceable(vma);
3683
3684         GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
3685         return 0;
3686
3687 err:
3688         __i915_vma_unpin(vma);
3689         return ret;
3690 }
3691
3692 int
3693 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
3694                          const struct i915_ggtt_view *view,
3695                          u64 size,
3696                          u64 alignment,
3697                          u64 flags)
3698 {
3699         struct i915_vma *vma;
3700         int ret;
3701
3702         if (!view)
3703                 view = &i915_ggtt_view_normal;
3704
3705         vma = i915_gem_obj_lookup_or_create_ggtt_vma(obj, view);
3706         if (IS_ERR(vma))
3707                 return PTR_ERR(vma);
3708
3709         if (i915_vma_misplaced(vma, size, alignment, flags)) {
3710                 if (flags & PIN_NONBLOCK &&
3711                     (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)))
3712                         return -ENOSPC;
3713
3714                 WARN(i915_vma_is_pinned(vma),
3715                      "bo is already pinned in ggtt with incorrect alignment:"
3716                      " offset=%08x %08x, req.alignment=%llx, req.map_and_fenceable=%d,"
3717                      " obj->map_and_fenceable=%d\n",
3718                      upper_32_bits(vma->node.start),
3719                      lower_32_bits(vma->node.start),
3720                      alignment,
3721                      !!(flags & PIN_MAPPABLE),
3722                      obj->map_and_fenceable);
3723                 ret = i915_vma_unbind(vma);
3724                 if (ret)
3725                         return ret;
3726         }
3727
3728         return i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
3729 }
3730
3731 void
3732 i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
3733                                 const struct i915_ggtt_view *view)
3734 {
3735         i915_vma_unpin(i915_gem_obj_to_ggtt_view(obj, view));
3736 }
3737
3738 static __always_inline unsigned __busy_read_flag(unsigned int id)
3739 {
3740         /* Note that we could alias engines in the execbuf API, but
3741          * that would be very unwise as it prevents userspace from
3742          * fine control over engine selection. Ahem.
3743          *
3744          * This should be something like EXEC_MAX_ENGINE instead of
3745          * I915_NUM_ENGINES.
3746          */
3747         BUILD_BUG_ON(I915_NUM_ENGINES > 16);
3748         return 0x10000 << id;
3749 }
3750
3751 static __always_inline unsigned int __busy_write_id(unsigned int id)
3752 {
3753         return id;
3754 }
3755
3756 static __always_inline unsigned
3757 __busy_set_if_active(const struct i915_gem_active *active,
3758                      unsigned int (*flag)(unsigned int id))
3759 {
3760         /* For more discussion about the barriers and locking concerns,
3761          * see __i915_gem_active_get_rcu().
3762          */
3763         do {
3764                 struct drm_i915_gem_request *request;
3765                 unsigned int id;
3766
3767                 request = rcu_dereference(active->request);
3768                 if (!request || i915_gem_request_completed(request))
3769                         return 0;
3770
3771                 id = request->engine->exec_id;
3772
3773                 /* Check that the pointer wasn't reassigned and overwritten. */
3774                 if (request == rcu_access_pointer(active->request))
3775                         return flag(id);
3776         } while (1);
3777 }
3778
3779 static inline unsigned
3780 busy_check_reader(const struct i915_gem_active *active)
3781 {
3782         return __busy_set_if_active(active, __busy_read_flag);
3783 }
3784
3785 static inline unsigned
3786 busy_check_writer(const struct i915_gem_active *active)
3787 {
3788         return __busy_set_if_active(active, __busy_write_id);
3789 }
3790
3791 int
3792 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3793                     struct drm_file *file)
3794 {
3795         struct drm_i915_gem_busy *args = data;
3796         struct drm_i915_gem_object *obj;
3797         unsigned long active;
3798
3799         obj = i915_gem_object_lookup(file, args->handle);
3800         if (!obj)
3801                 return -ENOENT;
3802
3803         args->busy = 0;
3804         active = __I915_BO_ACTIVE(obj);
3805         if (active) {
3806                 int idx;
3807
3808                 /* Yes, the lookups are intentionally racy.
3809                  *
3810                  * First, we cannot simply rely on __I915_BO_ACTIVE. We have
3811                  * to regard the value as stale and as our ABI guarantees
3812                  * forward progress, we confirm the status of each active
3813                  * request with the hardware.
3814                  *
3815                  * Even though we guard the pointer lookup by RCU, that only
3816                  * guarantees that the pointer and its contents remain
3817                  * dereferencable and does *not* mean that the request we
3818                  * have is the same as the one being tracked by the object.
3819                  *
3820                  * Consider that we lookup the request just as it is being
3821                  * retired and freed. We take a local copy of the pointer,
3822                  * but before we add its engine into the busy set, the other
3823                  * thread reallocates it and assigns it to a task on another
3824                  * engine with a fresh and incomplete seqno.
3825                  *
3826                  * So after we lookup the engine's id, we double check that
3827                  * the active request is the same and only then do we add it
3828                  * into the busy set.
3829                  */
3830                 rcu_read_lock();
3831
3832                 for_each_active(active, idx)
3833                         args->busy |= busy_check_reader(&obj->last_read[idx]);
3834
3835                 /* For ABI sanity, we only care that the write engine is in
3836                  * the set of read engines. This is ensured by the ordering
3837                  * of setting last_read/last_write in i915_vma_move_to_active,
3838                  * and then in reverse in retire.
3839                  *
3840                  * We don't care that the set of active read/write engines
3841                  * may change during construction of the result, as it is
3842                  * equally liable to change before userspace can inspect
3843                  * the result.
3844                  */
3845                 args->busy |= busy_check_writer(&obj->last_write);
3846
3847                 rcu_read_unlock();
3848         }
3849
3850         i915_gem_object_put_unlocked(obj);
3851         return 0;
3852 }
3853
3854 int
3855 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3856                         struct drm_file *file_priv)
3857 {
3858         return i915_gem_ring_throttle(dev, file_priv);
3859 }
3860
3861 int
3862 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3863                        struct drm_file *file_priv)
3864 {
3865         struct drm_i915_private *dev_priv = to_i915(dev);
3866         struct drm_i915_gem_madvise *args = data;
3867         struct drm_i915_gem_object *obj;
3868         int ret;
3869
3870         switch (args->madv) {
3871         case I915_MADV_DONTNEED:
3872         case I915_MADV_WILLNEED:
3873             break;
3874         default:
3875             return -EINVAL;
3876         }
3877
3878         ret = i915_mutex_lock_interruptible(dev);
3879         if (ret)
3880                 return ret;
3881
3882         obj = i915_gem_object_lookup(file_priv, args->handle);
3883         if (!obj) {
3884                 ret = -ENOENT;
3885                 goto unlock;
3886         }
3887
3888         if (obj->pages &&
3889             i915_gem_object_is_tiled(obj) &&
3890             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
3891                 if (obj->madv == I915_MADV_WILLNEED)
3892                         i915_gem_object_unpin_pages(obj);
3893                 if (args->madv == I915_MADV_WILLNEED)
3894                         i915_gem_object_pin_pages(obj);
3895         }
3896
3897         if (obj->madv != __I915_MADV_PURGED)
3898                 obj->madv = args->madv;
3899
3900         /* if the object is no longer attached, discard its backing storage */
3901         if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
3902                 i915_gem_object_truncate(obj);
3903
3904         args->retained = obj->madv != __I915_MADV_PURGED;
3905
3906         i915_gem_object_put(obj);
3907 unlock:
3908         mutex_unlock(&dev->struct_mutex);
3909         return ret;
3910 }
3911
3912 void i915_gem_object_init(struct drm_i915_gem_object *obj,
3913                           const struct drm_i915_gem_object_ops *ops)
3914 {
3915         int i;
3916
3917         INIT_LIST_HEAD(&obj->global_list);
3918         for (i = 0; i < I915_NUM_ENGINES; i++)
3919                 init_request_active(&obj->last_read[i],
3920                                     i915_gem_object_retire__read);
3921         init_request_active(&obj->last_write,
3922                             i915_gem_object_retire__write);
3923         init_request_active(&obj->last_fence, NULL);
3924         INIT_LIST_HEAD(&obj->obj_exec_link);
3925         INIT_LIST_HEAD(&obj->vma_list);
3926         INIT_LIST_HEAD(&obj->batch_pool_link);
3927
3928         obj->ops = ops;
3929
3930         obj->fence_reg = I915_FENCE_REG_NONE;
3931         obj->madv = I915_MADV_WILLNEED;
3932
3933         i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
3934 }
3935
3936 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
3937         .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
3938         .get_pages = i915_gem_object_get_pages_gtt,
3939         .put_pages = i915_gem_object_put_pages_gtt,
3940 };
3941
3942 struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
3943                                                   size_t size)
3944 {
3945         struct drm_i915_gem_object *obj;
3946         struct address_space *mapping;
3947         gfp_t mask;
3948         int ret;
3949
3950         obj = i915_gem_object_alloc(dev);
3951         if (obj == NULL)
3952                 return ERR_PTR(-ENOMEM);
3953
3954         ret = drm_gem_object_init(dev, &obj->base, size);
3955         if (ret)
3956                 goto fail;
3957
3958         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
3959         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
3960                 /* 965gm cannot relocate objects above 4GiB. */
3961                 mask &= ~__GFP_HIGHMEM;
3962                 mask |= __GFP_DMA32;
3963         }
3964
3965         mapping = obj->base.filp->f_mapping;
3966         mapping_set_gfp_mask(mapping, mask);
3967
3968         i915_gem_object_init(obj, &i915_gem_object_ops);
3969
3970         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3971         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3972
3973         if (HAS_LLC(dev)) {
3974                 /* On some devices, we can have the GPU use the LLC (the CPU
3975                  * cache) for about a 10% performance improvement
3976                  * compared to uncached.  Graphics requests other than
3977                  * display scanout are coherent with the CPU in
3978                  * accessing this cache.  This means in this mode we
3979                  * don't need to clflush on the CPU side, and on the
3980                  * GPU side we only need to flush internal caches to
3981                  * get data visible to the CPU.
3982                  *
3983                  * However, we maintain the display planes as UC, and so
3984                  * need to rebind when first used as such.
3985                  */
3986                 obj->cache_level = I915_CACHE_LLC;
3987         } else
3988                 obj->cache_level = I915_CACHE_NONE;
3989
3990         trace_i915_gem_object_create(obj);
3991
3992         return obj;
3993
3994 fail:
3995         i915_gem_object_free(obj);
3996
3997         return ERR_PTR(ret);
3998 }
3999
4000 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4001 {
4002         /* If we are the last user of the backing storage (be it shmemfs
4003          * pages or stolen etc), we know that the pages are going to be
4004          * immediately released. In this case, we can then skip copying
4005          * back the contents from the GPU.
4006          */
4007
4008         if (obj->madv != I915_MADV_WILLNEED)
4009                 return false;
4010
4011         if (obj->base.filp == NULL)
4012                 return true;
4013
4014         /* At first glance, this looks racy, but then again so would be
4015          * userspace racing mmap against close. However, the first external
4016          * reference to the filp can only be obtained through the
4017          * i915_gem_mmap_ioctl() which safeguards us against the user
4018          * acquiring such a reference whilst we are in the middle of
4019          * freeing the object.
4020          */
4021         return atomic_long_read(&obj->base.filp->f_count) == 1;
4022 }
4023
4024 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4025 {
4026         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4027         struct drm_device *dev = obj->base.dev;
4028         struct drm_i915_private *dev_priv = to_i915(dev);
4029         struct i915_vma *vma, *next;
4030
4031         intel_runtime_pm_get(dev_priv);
4032
4033         trace_i915_gem_object_destroy(obj);
4034
4035         /* All file-owned VMA should have been released by this point through
4036          * i915_gem_close_object(), or earlier by i915_gem_context_close().
4037          * However, the object may also be bound into the global GTT (e.g.
4038          * older GPUs without per-process support, or for direct access through
4039          * the GTT either for the user or for scanout). Those VMA still need to
4040          * unbound now.
4041          */
4042         list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4043                 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
4044                 GEM_BUG_ON(i915_vma_is_active(vma));
4045                 vma->flags &= ~I915_VMA_PIN_MASK;
4046                 i915_vma_close(vma);
4047         }
4048         GEM_BUG_ON(obj->bind_count);
4049
4050         /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4051          * before progressing. */
4052         if (obj->stolen)
4053                 i915_gem_object_unpin_pages(obj);
4054
4055         WARN_ON(atomic_read(&obj->frontbuffer_bits));
4056
4057         if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4058             dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4059             i915_gem_object_is_tiled(obj))
4060                 i915_gem_object_unpin_pages(obj);
4061
4062         if (WARN_ON(obj->pages_pin_count))
4063                 obj->pages_pin_count = 0;
4064         if (discard_backing_storage(obj))
4065                 obj->madv = I915_MADV_DONTNEED;
4066         i915_gem_object_put_pages(obj);
4067
4068         BUG_ON(obj->pages);
4069
4070         if (obj->base.import_attach)
4071                 drm_prime_gem_destroy(&obj->base, NULL);
4072
4073         if (obj->ops->release)
4074                 obj->ops->release(obj);
4075
4076         drm_gem_object_release(&obj->base);
4077         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4078
4079         kfree(obj->bit_17);
4080         i915_gem_object_free(obj);
4081
4082         intel_runtime_pm_put(dev_priv);
4083 }
4084
4085 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4086                                      struct i915_address_space *vm)
4087 {
4088         struct i915_vma *vma;
4089         list_for_each_entry(vma, &obj->vma_list, obj_link) {
4090                 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4091                     vma->vm == vm)
4092                         return vma;
4093         }
4094         return NULL;
4095 }
4096
4097 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4098                                            const struct i915_ggtt_view *view)
4099 {
4100         struct i915_vma *vma;
4101
4102         GEM_BUG_ON(!view);
4103
4104         list_for_each_entry(vma, &obj->vma_list, obj_link)
4105                 if (i915_vma_is_ggtt(vma) &&
4106                     i915_ggtt_view_equal(&vma->ggtt_view, view))
4107                         return vma;
4108         return NULL;
4109 }
4110
4111 int i915_gem_suspend(struct drm_device *dev)
4112 {
4113         struct drm_i915_private *dev_priv = to_i915(dev);
4114         int ret;
4115
4116         intel_suspend_gt_powersave(dev_priv);
4117
4118         mutex_lock(&dev->struct_mutex);
4119
4120         /* We have to flush all the executing contexts to main memory so
4121          * that they can saved in the hibernation image. To ensure the last
4122          * context image is coherent, we have to switch away from it. That
4123          * leaves the dev_priv->kernel_context still active when
4124          * we actually suspend, and its image in memory may not match the GPU
4125          * state. Fortunately, the kernel_context is disposable and we do
4126          * not rely on its state.
4127          */
4128         ret = i915_gem_switch_to_kernel_context(dev_priv);
4129         if (ret)
4130                 goto err;
4131
4132         ret = i915_gem_wait_for_idle(dev_priv, true);
4133         if (ret)
4134                 goto err;
4135
4136         i915_gem_retire_requests(dev_priv);
4137
4138         i915_gem_context_lost(dev_priv);
4139         mutex_unlock(&dev->struct_mutex);
4140
4141         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4142         cancel_delayed_work_sync(&dev_priv->gt.retire_work);
4143         flush_delayed_work(&dev_priv->gt.idle_work);
4144
4145         /* Assert that we sucessfully flushed all the work and
4146          * reset the GPU back to its idle, low power state.
4147          */
4148         WARN_ON(dev_priv->gt.awake);
4149
4150         return 0;
4151
4152 err:
4153         mutex_unlock(&dev->struct_mutex);
4154         return ret;
4155 }
4156
4157 void i915_gem_resume(struct drm_device *dev)
4158 {
4159         struct drm_i915_private *dev_priv = to_i915(dev);
4160
4161         mutex_lock(&dev->struct_mutex);
4162         i915_gem_restore_gtt_mappings(dev);
4163
4164         /* As we didn't flush the kernel context before suspend, we cannot
4165          * guarantee that the context image is complete. So let's just reset
4166          * it and start again.
4167          */
4168         if (i915.enable_execlists)
4169                 intel_lr_context_reset(dev_priv, dev_priv->kernel_context);
4170
4171         mutex_unlock(&dev->struct_mutex);
4172 }
4173
4174 void i915_gem_init_swizzling(struct drm_device *dev)
4175 {
4176         struct drm_i915_private *dev_priv = to_i915(dev);
4177
4178         if (INTEL_INFO(dev)->gen < 5 ||
4179             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4180                 return;
4181
4182         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4183                                  DISP_TILE_SURFACE_SWIZZLING);
4184
4185         if (IS_GEN5(dev))
4186                 return;
4187
4188         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4189         if (IS_GEN6(dev))
4190                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4191         else if (IS_GEN7(dev))
4192                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4193         else if (IS_GEN8(dev))
4194                 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4195         else
4196                 BUG();
4197 }
4198
4199 static void init_unused_ring(struct drm_device *dev, u32 base)
4200 {
4201         struct drm_i915_private *dev_priv = to_i915(dev);
4202
4203         I915_WRITE(RING_CTL(base), 0);
4204         I915_WRITE(RING_HEAD(base), 0);
4205         I915_WRITE(RING_TAIL(base), 0);
4206         I915_WRITE(RING_START(base), 0);
4207 }
4208
4209 static void init_unused_rings(struct drm_device *dev)
4210 {
4211         if (IS_I830(dev)) {
4212                 init_unused_ring(dev, PRB1_BASE);
4213                 init_unused_ring(dev, SRB0_BASE);
4214                 init_unused_ring(dev, SRB1_BASE);
4215                 init_unused_ring(dev, SRB2_BASE);
4216                 init_unused_ring(dev, SRB3_BASE);
4217         } else if (IS_GEN2(dev)) {
4218                 init_unused_ring(dev, SRB0_BASE);
4219                 init_unused_ring(dev, SRB1_BASE);
4220         } else if (IS_GEN3(dev)) {
4221                 init_unused_ring(dev, PRB1_BASE);
4222                 init_unused_ring(dev, PRB2_BASE);
4223         }
4224 }
4225
4226 int
4227 i915_gem_init_hw(struct drm_device *dev)
4228 {
4229         struct drm_i915_private *dev_priv = to_i915(dev);
4230         struct intel_engine_cs *engine;
4231         int ret;
4232
4233         /* Double layer security blanket, see i915_gem_init() */
4234         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4235
4236         if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
4237                 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4238
4239         if (IS_HASWELL(dev))
4240                 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4241                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4242
4243         if (HAS_PCH_NOP(dev)) {
4244                 if (IS_IVYBRIDGE(dev)) {
4245                         u32 temp = I915_READ(GEN7_MSG_CTL);
4246                         temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4247                         I915_WRITE(GEN7_MSG_CTL, temp);
4248                 } else if (INTEL_INFO(dev)->gen >= 7) {
4249                         u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4250                         temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4251                         I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4252                 }
4253         }
4254
4255         i915_gem_init_swizzling(dev);
4256
4257         /*
4258          * At least 830 can leave some of the unused rings
4259          * "active" (ie. head != tail) after resume which
4260          * will prevent c3 entry. Makes sure all unused rings
4261          * are totally idle.
4262          */
4263         init_unused_rings(dev);
4264
4265         BUG_ON(!dev_priv->kernel_context);
4266
4267         ret = i915_ppgtt_init_hw(dev);
4268         if (ret) {
4269                 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4270                 goto out;
4271         }
4272
4273         /* Need to do basic initialisation of all rings first: */
4274         for_each_engine(engine, dev_priv) {
4275                 ret = engine->init_hw(engine);
4276                 if (ret)
4277                         goto out;
4278         }
4279
4280         intel_mocs_init_l3cc_table(dev);
4281
4282         /* We can't enable contexts until all firmware is loaded */
4283         ret = intel_guc_setup(dev);
4284         if (ret)
4285                 goto out;
4286
4287 out:
4288         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4289         return ret;
4290 }
4291
4292 bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
4293 {
4294         if (INTEL_INFO(dev_priv)->gen < 6)
4295                 return false;
4296
4297         /* TODO: make semaphores and Execlists play nicely together */
4298         if (i915.enable_execlists)
4299                 return false;
4300
4301         if (value >= 0)
4302                 return value;
4303
4304 #ifdef CONFIG_INTEL_IOMMU
4305         /* Enable semaphores on SNB when IO remapping is off */
4306         if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
4307                 return false;
4308 #endif
4309
4310         return true;
4311 }
4312
4313 int i915_gem_init(struct drm_device *dev)
4314 {
4315         struct drm_i915_private *dev_priv = to_i915(dev);
4316         int ret;
4317
4318         mutex_lock(&dev->struct_mutex);
4319
4320         if (!i915.enable_execlists) {
4321                 dev_priv->gt.cleanup_engine = intel_engine_cleanup;
4322         } else {
4323                 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
4324         }
4325
4326         /* This is just a security blanket to placate dragons.
4327          * On some systems, we very sporadically observe that the first TLBs
4328          * used by the CS may be stale, despite us poking the TLB reset. If
4329          * we hold the forcewake during initialisation these problems
4330          * just magically go away.
4331          */
4332         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4333
4334         i915_gem_init_userptr(dev_priv);
4335
4336         ret = i915_gem_init_ggtt(dev_priv);
4337         if (ret)
4338                 goto out_unlock;
4339
4340         ret = i915_gem_context_init(dev);
4341         if (ret)
4342                 goto out_unlock;
4343
4344         ret = intel_engines_init(dev);
4345         if (ret)
4346                 goto out_unlock;
4347
4348         ret = i915_gem_init_hw(dev);
4349         if (ret == -EIO) {
4350                 /* Allow engine initialisation to fail by marking the GPU as
4351                  * wedged. But we only want to do this where the GPU is angry,
4352                  * for all other failure, such as an allocation failure, bail.
4353                  */
4354                 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4355                 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4356                 ret = 0;
4357         }
4358
4359 out_unlock:
4360         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4361         mutex_unlock(&dev->struct_mutex);
4362
4363         return ret;
4364 }
4365
4366 void
4367 i915_gem_cleanup_engines(struct drm_device *dev)
4368 {
4369         struct drm_i915_private *dev_priv = to_i915(dev);
4370         struct intel_engine_cs *engine;
4371
4372         for_each_engine(engine, dev_priv)
4373                 dev_priv->gt.cleanup_engine(engine);
4374 }
4375
4376 static void
4377 init_engine_lists(struct intel_engine_cs *engine)
4378 {
4379         INIT_LIST_HEAD(&engine->request_list);
4380 }
4381
4382 void
4383 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
4384 {
4385         struct drm_device *dev = &dev_priv->drm;
4386
4387         if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
4388             !IS_CHERRYVIEW(dev_priv))
4389                 dev_priv->num_fence_regs = 32;
4390         else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
4391                  IS_I945GM(dev_priv) || IS_G33(dev_priv))
4392                 dev_priv->num_fence_regs = 16;
4393         else
4394                 dev_priv->num_fence_regs = 8;
4395
4396         if (intel_vgpu_active(dev_priv))
4397                 dev_priv->num_fence_regs =
4398                                 I915_READ(vgtif_reg(avail_rs.fence_num));
4399
4400         /* Initialize fence registers to zero */
4401         i915_gem_restore_fences(dev);
4402
4403         i915_gem_detect_bit_6_swizzle(dev);
4404 }
4405
4406 void
4407 i915_gem_load_init(struct drm_device *dev)
4408 {
4409         struct drm_i915_private *dev_priv = to_i915(dev);
4410         int i;
4411
4412         dev_priv->objects =
4413                 kmem_cache_create("i915_gem_object",
4414                                   sizeof(struct drm_i915_gem_object), 0,
4415                                   SLAB_HWCACHE_ALIGN,
4416                                   NULL);
4417         dev_priv->vmas =
4418                 kmem_cache_create("i915_gem_vma",
4419                                   sizeof(struct i915_vma), 0,
4420                                   SLAB_HWCACHE_ALIGN,
4421                                   NULL);
4422         dev_priv->requests =
4423                 kmem_cache_create("i915_gem_request",
4424                                   sizeof(struct drm_i915_gem_request), 0,
4425                                   SLAB_HWCACHE_ALIGN |
4426                                   SLAB_RECLAIM_ACCOUNT |
4427                                   SLAB_DESTROY_BY_RCU,
4428                                   NULL);
4429
4430         INIT_LIST_HEAD(&dev_priv->context_list);
4431         INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4432         INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4433         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4434         for (i = 0; i < I915_NUM_ENGINES; i++)
4435                 init_engine_lists(&dev_priv->engine[i]);
4436         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4437                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4438         INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
4439                           i915_gem_retire_work_handler);
4440         INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
4441                           i915_gem_idle_work_handler);
4442         init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
4443         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4444
4445         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4446
4447         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4448
4449         init_waitqueue_head(&dev_priv->pending_flip_queue);
4450
4451         dev_priv->mm.interruptible = true;
4452
4453         spin_lock_init(&dev_priv->fb_tracking.lock);
4454 }
4455
4456 void i915_gem_load_cleanup(struct drm_device *dev)
4457 {
4458         struct drm_i915_private *dev_priv = to_i915(dev);
4459
4460         kmem_cache_destroy(dev_priv->requests);
4461         kmem_cache_destroy(dev_priv->vmas);
4462         kmem_cache_destroy(dev_priv->objects);
4463
4464         /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
4465         rcu_barrier();
4466 }
4467
4468 int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
4469 {
4470         struct drm_i915_gem_object *obj;
4471
4472         /* Called just before we write the hibernation image.
4473          *
4474          * We need to update the domain tracking to reflect that the CPU
4475          * will be accessing all the pages to create and restore from the
4476          * hibernation, and so upon restoration those pages will be in the
4477          * CPU domain.
4478          *
4479          * To make sure the hibernation image contains the latest state,
4480          * we update that state just before writing out the image.
4481          */
4482
4483         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
4484                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4485                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4486         }
4487
4488         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
4489                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4490                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4491         }
4492
4493         return 0;
4494 }
4495
4496 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4497 {
4498         struct drm_i915_file_private *file_priv = file->driver_priv;
4499         struct drm_i915_gem_request *request;
4500
4501         /* Clean up our request list when the client is going away, so that
4502          * later retire_requests won't dereference our soon-to-be-gone
4503          * file_priv.
4504          */
4505         spin_lock(&file_priv->mm.lock);
4506         list_for_each_entry(request, &file_priv->mm.request_list, client_list)
4507                 request->file_priv = NULL;
4508         spin_unlock(&file_priv->mm.lock);
4509
4510         if (!list_empty(&file_priv->rps.link)) {
4511                 spin_lock(&to_i915(dev)->rps.client_lock);
4512                 list_del(&file_priv->rps.link);
4513                 spin_unlock(&to_i915(dev)->rps.client_lock);
4514         }
4515 }
4516
4517 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
4518 {
4519         struct drm_i915_file_private *file_priv;
4520         int ret;
4521
4522         DRM_DEBUG_DRIVER("\n");
4523
4524         file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
4525         if (!file_priv)
4526                 return -ENOMEM;
4527
4528         file->driver_priv = file_priv;
4529         file_priv->dev_priv = to_i915(dev);
4530         file_priv->file = file;
4531         INIT_LIST_HEAD(&file_priv->rps.link);
4532
4533         spin_lock_init(&file_priv->mm.lock);
4534         INIT_LIST_HEAD(&file_priv->mm.request_list);
4535
4536         file_priv->bsd_engine = -1;
4537
4538         ret = i915_gem_context_open(dev, file);
4539         if (ret)
4540                 kfree(file_priv);
4541
4542         return ret;
4543 }
4544
4545 /**
4546  * i915_gem_track_fb - update frontbuffer tracking
4547  * @old: current GEM buffer for the frontbuffer slots
4548  * @new: new GEM buffer for the frontbuffer slots
4549  * @frontbuffer_bits: bitmask of frontbuffer slots
4550  *
4551  * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
4552  * from @old and setting them in @new. Both @old and @new can be NULL.
4553  */
4554 void i915_gem_track_fb(struct drm_i915_gem_object *old,
4555                        struct drm_i915_gem_object *new,
4556                        unsigned frontbuffer_bits)
4557 {
4558         /* Control of individual bits within the mask are guarded by
4559          * the owning plane->mutex, i.e. we can never see concurrent
4560          * manipulation of individual bits. But since the bitfield as a whole
4561          * is updated using RMW, we need to use atomics in order to update
4562          * the bits.
4563          */
4564         BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
4565                      sizeof(atomic_t) * BITS_PER_BYTE);
4566
4567         if (old) {
4568                 WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
4569                 atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
4570         }
4571
4572         if (new) {
4573                 WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
4574                 atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
4575         }
4576 }
4577
4578 /* All the new VM stuff */
4579 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
4580                         struct i915_address_space *vm)
4581 {
4582         struct drm_i915_private *dev_priv = to_i915(o->base.dev);
4583         struct i915_vma *vma;
4584
4585         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
4586
4587         list_for_each_entry(vma, &o->vma_list, obj_link) {
4588                 if (i915_vma_is_ggtt(vma) &&
4589                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4590                         continue;
4591                 if (vma->vm == vm)
4592                         return vma->node.start;
4593         }
4594
4595         WARN(1, "%s vma for this object not found.\n",
4596              i915_is_ggtt(vm) ? "global" : "ppgtt");
4597         return -1;
4598 }
4599
4600 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
4601                                   const struct i915_ggtt_view *view)
4602 {
4603         struct i915_vma *vma;
4604
4605         list_for_each_entry(vma, &o->vma_list, obj_link)
4606                 if (i915_vma_is_ggtt(vma) &&
4607                     i915_ggtt_view_equal(&vma->ggtt_view, view))
4608                         return vma->node.start;
4609
4610         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
4611         return -1;
4612 }
4613
4614 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
4615                         struct i915_address_space *vm)
4616 {
4617         struct i915_vma *vma;
4618
4619         list_for_each_entry(vma, &o->vma_list, obj_link) {
4620                 if (i915_vma_is_ggtt(vma) &&
4621                     vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
4622                         continue;
4623                 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
4624                         return true;
4625         }
4626
4627         return false;
4628 }
4629
4630 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
4631                                   const struct i915_ggtt_view *view)
4632 {
4633         struct i915_vma *vma;
4634
4635         list_for_each_entry(vma, &o->vma_list, obj_link)
4636                 if (i915_vma_is_ggtt(vma) &&
4637                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
4638                     drm_mm_node_allocated(&vma->node))
4639                         return true;
4640
4641         return false;
4642 }
4643
4644 unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o)
4645 {
4646         struct i915_vma *vma;
4647
4648         GEM_BUG_ON(list_empty(&o->vma_list));
4649
4650         list_for_each_entry(vma, &o->vma_list, obj_link) {
4651                 if (i915_vma_is_ggtt(vma) &&
4652                     vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
4653                         return vma->node.size;
4654         }
4655
4656         return 0;
4657 }
4658
4659 bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
4660 {
4661         struct i915_vma *vma;
4662         list_for_each_entry(vma, &obj->vma_list, obj_link)
4663                 if (i915_vma_is_pinned(vma))
4664                         return true;
4665
4666         return false;
4667 }
4668
4669 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
4670 struct page *
4671 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
4672 {
4673         struct page *page;
4674
4675         /* Only default objects have per-page dirty tracking */
4676         if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
4677                 return NULL;
4678
4679         page = i915_gem_object_get_page(obj, n);
4680         set_page_dirty(page);
4681         return page;
4682 }
4683
4684 /* Allocate a new GEM object and fill it with the supplied data */
4685 struct drm_i915_gem_object *
4686 i915_gem_object_create_from_data(struct drm_device *dev,
4687                                  const void *data, size_t size)
4688 {
4689         struct drm_i915_gem_object *obj;
4690         struct sg_table *sg;
4691         size_t bytes;
4692         int ret;
4693
4694         obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
4695         if (IS_ERR(obj))
4696                 return obj;
4697
4698         ret = i915_gem_object_set_to_cpu_domain(obj, true);
4699         if (ret)
4700                 goto fail;
4701
4702         ret = i915_gem_object_get_pages(obj);
4703         if (ret)
4704                 goto fail;
4705
4706         i915_gem_object_pin_pages(obj);
4707         sg = obj->pages;
4708         bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
4709         obj->dirty = 1;         /* Backing store is now out of date */
4710         i915_gem_object_unpin_pages(obj);
4711
4712         if (WARN_ON(bytes != size)) {
4713                 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
4714                 ret = -EFAULT;
4715                 goto fail;
4716         }
4717
4718         return obj;
4719
4720 fail:
4721         i915_gem_object_put(obj);
4722         return ERR_PTR(ret);
4723 }