drivers/gpu/drm/i915/i915_vma.c

   1 /*
   2  * Copyright © 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/sched/mm.h>
  26 #include <drm/drm_gem.h>
  27
  28 #include "display/intel_frontbuffer.h"
  29
  30 #include "gem/i915_gem_lmem.h"
  31 #include "gt/intel_engine.h"
  32 #include "gt/intel_engine_heartbeat.h"
  33 #include "gt/intel_gt.h"
  34 #include "gt/intel_gt_requests.h"
  35
  36 #include "i915_drv.h"
  37 #include "i915_sw_fence_work.h"
  38 #include "i915_trace.h"
  39 #include "i915_vma.h"
  40 #include "i915_vma_resource.h"
  41
  42 static struct kmem_cache *slab_vmas;
  43
  44 static struct i915_vma *i915_vma_alloc(void)
  45 {
  46         return kmem_cache_zalloc(slab_vmas, GFP_KERNEL);
  47 }
  48
  49 static void i915_vma_free(struct i915_vma *vma)
  50 {
  51         return kmem_cache_free(slab_vmas, vma);
  52 }
  53
  54 #if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
  55
  56 #include <linux/stackdepot.h>
  57
  58 static void vma_print_allocator(struct i915_vma *vma, const char *reason)
  59 {
  60         char buf[512];
  61
  62         if (!vma->node.stack) {
  63                 DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n",
  64                                  vma->node.start, vma->node.size, reason);
  65                 return;
  66         }
  67
  68         stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0);
  69         DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n",
  70                          vma->node.start, vma->node.size, reason, buf);
  71 }
  72
  73 #else
  74
  75 static void vma_print_allocator(struct i915_vma *vma, const char *reason)
  76 {
  77 }
  78
  79 #endif
  80
  81 static inline struct i915_vma *active_to_vma(struct i915_active *ref)
  82 {
  83         return container_of(ref, typeof(struct i915_vma), active);
  84 }
  85
  86 static int __i915_vma_active(struct i915_active *ref)
  87 {
  88         return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT;
  89 }
  90
  91 static void __i915_vma_retire(struct i915_active *ref)
  92 {
  93         i915_vma_put(active_to_vma(ref));
  94 }
  95
  96 static struct i915_vma *
  97 vma_create(struct drm_i915_gem_object *obj,
  98            struct i915_address_space *vm,
  99            const struct i915_ggtt_view *view)
 100 {
 101         struct i915_vma *pos = ERR_PTR(-E2BIG);
 102         struct i915_vma *vma;
 103         struct rb_node *rb, **p;
 104
 105         /* The aliasing_ppgtt should never be used directly! */
 106         GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm);
 107
 108         vma = i915_vma_alloc();
 109         if (vma == NULL)
 110                 return ERR_PTR(-ENOMEM);
 111
 112         kref_init(&vma->ref);
 113         vma->vm = i915_vm_get(vm);
 114         vma->ops = &vm->vma_ops;
 115         vma->obj = obj;
 116         vma->size = obj->base.size;
 117         vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
 118
 119         i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0);
 120
 121         /* Declare ourselves safe for use inside shrinkers */
 122         if (IS_ENABLED(CONFIG_LOCKDEP)) {
 123                 fs_reclaim_acquire(GFP_KERNEL);
 124                 might_lock(&vma->active.mutex);
 125                 fs_reclaim_release(GFP_KERNEL);
 126         }
 127
 128         INIT_LIST_HEAD(&vma->closed_link);
 129
 130         if (view && view->type != I915_GGTT_VIEW_NORMAL) {
 131                 vma->ggtt_view = *view;
 132                 if (view->type == I915_GGTT_VIEW_PARTIAL) {
 133                         GEM_BUG_ON(range_overflows_t(u64,
 134                                                      view->partial.offset,
 135                                                      view->partial.size,
 136                                                      obj->base.size >> PAGE_SHIFT));
 137                         vma->size = view->partial.size;
 138                         vma->size <<= PAGE_SHIFT;
 139                         GEM_BUG_ON(vma->size > obj->base.size);
 140                 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
 141                         vma->size = intel_rotation_info_size(&view->rotated);
 142                         vma->size <<= PAGE_SHIFT;
 143                 } else if (view->type == I915_GGTT_VIEW_REMAPPED) {
 144                         vma->size = intel_remapped_info_size(&view->remapped);
 145                         vma->size <<= PAGE_SHIFT;
 146                 }
 147         }
 148
 149         if (unlikely(vma->size > vm->total))
 150                 goto err_vma;
 151
 152         GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE));
 153
 154         spin_lock(&obj->vma.lock);
 155
 156         if (i915_is_ggtt(vm)) {
 157                 if (unlikely(overflows_type(vma->size, u32)))
 158                         goto err_unlock;
 159
 160                 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size,
 161                                                       i915_gem_object_get_tiling(obj),
 162                                                       i915_gem_object_get_stride(obj));
 163                 if (unlikely(vma->fence_size < vma->size || /* overflow */
 164                              vma->fence_size > vm->total))
 165                         goto err_unlock;
 166
 167                 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT));
 168
 169                 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size,
 170                                                                 i915_gem_object_get_tiling(obj),
 171                                                                 i915_gem_object_get_stride(obj));
 172                 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment));
 173
 174                 __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma));
 175         }
 176
 177         rb = NULL;
 178         p = &obj->vma.tree.rb_node;
 179         while (*p) {
 180                 long cmp;
 181
 182                 rb = *p;
 183                 pos = rb_entry(rb, struct i915_vma, obj_node);
 184
 185                 /*
 186                  * If the view already exists in the tree, another thread
 187                  * already created a matching vma, so return the older instance
 188                  * and dispose of ours.
 189                  */
 190                 cmp = i915_vma_compare(pos, vm, view);
 191                 if (cmp < 0)
 192                         p = &rb->rb_right;
 193                 else if (cmp > 0)
 194                         p = &rb->rb_left;
 195                 else
 196                         goto err_unlock;
 197         }
 198         rb_link_node(&vma->obj_node, rb, p);
 199         rb_insert_color(&vma->obj_node, &obj->vma.tree);
 200
 201         if (i915_vma_is_ggtt(vma))
 202                 /*
 203                  * We put the GGTT vma at the start of the vma-list, followed
 204                  * by the ppGGTT vma. This allows us to break early when
 205                  * iterating over only the GGTT vma for an object, see
 206                  * for_each_ggtt_vma()
 207                  */
 208                 list_add(&vma->obj_link, &obj->vma.list);
 209         else
 210                 list_add_tail(&vma->obj_link, &obj->vma.list);
 211
 212         spin_unlock(&obj->vma.lock);
 213
 214         return vma;
 215
 216 err_unlock:
 217         spin_unlock(&obj->vma.lock);
 218 err_vma:
 219         i915_vm_put(vm);
 220         i915_vma_free(vma);
 221         return pos;
 222 }
 223
 224 static struct i915_vma *
 225 i915_vma_lookup(struct drm_i915_gem_object *obj,
 226            struct i915_address_space *vm,
 227            const struct i915_ggtt_view *view)
 228 {
 229         struct rb_node *rb;
 230
 231         rb = obj->vma.tree.rb_node;
 232         while (rb) {
 233                 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
 234                 long cmp;
 235
 236                 cmp = i915_vma_compare(vma, vm, view);
 237                 if (cmp == 0)
 238                         return vma;
 239
 240                 if (cmp < 0)
 241                         rb = rb->rb_right;
 242                 else
 243                         rb = rb->rb_left;
 244         }
 245
 246         return NULL;
 247 }
 248
 249 /**
 250  * i915_vma_instance - return the singleton instance of the VMA
 251  * @obj: parent &struct drm_i915_gem_object to be mapped
 252  * @vm: address space in which the mapping is located
 253  * @view: additional mapping requirements
 254  *
 255  * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with
 256  * the same @view characteristics. If a match is not found, one is created.
 257  * Once created, the VMA is kept until either the object is freed, or the
 258  * address space is closed.
 259  *
 260  * Returns the vma, or an error pointer.
 261  */
 262 struct i915_vma *
 263 i915_vma_instance(struct drm_i915_gem_object *obj,
 264                   struct i915_address_space *vm,
 265                   const struct i915_ggtt_view *view)
 266 {
 267         struct i915_vma *vma;
 268
 269         GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm));
 270         GEM_BUG_ON(!atomic_read(&vm->open));
 271
 272         spin_lock(&obj->vma.lock);
 273         vma = i915_vma_lookup(obj, vm, view);
 274         spin_unlock(&obj->vma.lock);
 275
 276         /* vma_create() will resolve the race if another creates the vma */
 277         if (unlikely(!vma))
 278                 vma = vma_create(obj, vm, view);
 279
 280         GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view));
 281         return vma;
 282 }
 283
 284 struct i915_vma_work {
 285         struct dma_fence_work base;
 286         struct i915_address_space *vm;
 287         struct i915_vm_pt_stash stash;
 288         struct i915_vma_resource *vma_res;
 289         struct drm_i915_gem_object *pinned;
 290         struct i915_sw_dma_fence_cb cb;
 291         enum i915_cache_level cache_level;
 292         unsigned int flags;
 293 };
 294
 295 static void __vma_bind(struct dma_fence_work *work)
 296 {
 297         struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
 298         struct i915_vma_resource *vma_res = vw->vma_res;
 299
 300         vma_res->ops->bind_vma(vma_res->vm, &vw->stash,
 301                                vma_res, vw->cache_level, vw->flags);
 302
 303 }
 304
 305 static void __vma_release(struct dma_fence_work *work)
 306 {
 307         struct i915_vma_work *vw = container_of(work, typeof(*vw), base);
 308
 309         if (vw->pinned)
 310                 i915_gem_object_put(vw->pinned);
 311
 312         i915_vm_free_pt_stash(vw->vm, &vw->stash);
 313         i915_vm_put(vw->vm);
 314         if (vw->vma_res)
 315                 i915_vma_resource_put(vw->vma_res);
 316 }
 317
 318 static const struct dma_fence_work_ops bind_ops = {
 319         .name = "bind",
 320         .work = __vma_bind,
 321         .release = __vma_release,
 322 };
 323
 324 struct i915_vma_work *i915_vma_work(void)
 325 {
 326         struct i915_vma_work *vw;
 327
 328         vw = kzalloc(sizeof(*vw), GFP_KERNEL);
 329         if (!vw)
 330                 return NULL;
 331
 332         dma_fence_work_init(&vw->base, &bind_ops);
 333         vw->base.dma.error = -EAGAIN; /* disable the worker by default */
 334
 335         return vw;
 336 }
 337
 338 int i915_vma_wait_for_bind(struct i915_vma *vma)
 339 {
 340         int err = 0;
 341
 342         if (rcu_access_pointer(vma->active.excl.fence)) {
 343                 struct dma_fence *fence;
 344
 345                 rcu_read_lock();
 346                 fence = dma_fence_get_rcu_safe(&vma->active.excl.fence);
 347                 rcu_read_unlock();
 348                 if (fence) {
 349                         err = dma_fence_wait(fence, true);
 350                         dma_fence_put(fence);
 351                 }
 352         }
 353
 354         return err;
 355 }
 356
 357 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
 358 static int i915_vma_verify_bind_complete(struct i915_vma *vma)
 359 {
 360         struct dma_fence *fence = i915_active_fence_get(&vma->active.excl);
 361         int err;
 362
 363         if (!fence)
 364                 return 0;
 365
 366         if (dma_fence_is_signaled(fence))
 367                 err = fence->error;
 368         else
 369                 err = -EBUSY;
 370
 371         dma_fence_put(fence);
 372
 373         return err;
 374 }
 375 #else
 376 #define i915_vma_verify_bind_complete(_vma) 0
 377 #endif
 378
 379 I915_SELFTEST_EXPORT void
 380 i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res,
 381                                 struct i915_vma *vma)
 382 {
 383         struct drm_i915_gem_object *obj = vma->obj;
 384
 385         i915_vma_resource_init(vma_res, vma->vm, vma->pages, &vma->page_sizes,
 386                                obj->mm.rsgt, i915_gem_object_is_readonly(obj),
 387                                i915_gem_object_is_lmem(obj), obj->mm.region,
 388                                vma->ops, vma->private, vma->node.start,
 389                                vma->node.size, vma->size);
 390 }
 391
 392 /**
 393  * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
 394  * @vma: VMA to map
 395  * @cache_level: mapping cache level
 396  * @flags: flags like global or local mapping
 397  * @work: preallocated worker for allocating and binding the PTE
 398  * @vma_res: pointer to a preallocated vma resource. The resource is either
 399  * consumed or freed.
 400  *
 401  * DMA addresses are taken from the scatter-gather table of this object (or of
 402  * this VMA in case of non-default GGTT views) and PTE entries set up.
 403  * Note that DMA addresses are also the only part of the SG table we care about.
 404  */
 405 int i915_vma_bind(struct i915_vma *vma,
 406                   enum i915_cache_level cache_level,
 407                   u32 flags,
 408                   struct i915_vma_work *work,
 409                   struct i915_vma_resource *vma_res)
 410 {
 411         u32 bind_flags;
 412         u32 vma_flags;
 413         int ret;
 414
 415         lockdep_assert_held(&vma->vm->mutex);
 416         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 417         GEM_BUG_ON(vma->size > vma->node.size);
 418
 419         if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start,
 420                                               vma->node.size,
 421                                               vma->vm->total))) {
 422                 i915_vma_resource_free(vma_res);
 423                 return -ENODEV;
 424         }
 425
 426         if (GEM_DEBUG_WARN_ON(!flags)) {
 427                 i915_vma_resource_free(vma_res);
 428                 return -EINVAL;
 429         }
 430
 431         bind_flags = flags;
 432         bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 433
 434         vma_flags = atomic_read(&vma->flags);
 435         vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
 436
 437         bind_flags &= ~vma_flags;
 438         if (bind_flags == 0) {
 439                 i915_vma_resource_free(vma_res);
 440                 return 0;
 441         }
 442
 443         GEM_BUG_ON(!atomic_read(&vma->pages_count));
 444
 445         /* Wait for or await async unbinds touching our range */
 446         if (work && bind_flags & vma->vm->bind_async_flags)
 447                 ret = i915_vma_resource_bind_dep_await(vma->vm,
 448                                                        &work->base.chain,
 449                                                        vma->node.start,
 450                                                        vma->node.size,
 451                                                        true,
 452                                                        GFP_NOWAIT |
 453                                                        __GFP_RETRY_MAYFAIL |
 454                                                        __GFP_NOWARN);
 455         else
 456                 ret = i915_vma_resource_bind_dep_sync(vma->vm, vma->node.start,
 457                                                       vma->node.size, true);
 458         if (ret) {
 459                 i915_vma_resource_free(vma_res);
 460                 return ret;
 461         }
 462
 463         if (vma->resource || !vma_res) {
 464                 /* Rebinding with an additional I915_VMA_*_BIND */
 465                 GEM_WARN_ON(!vma_flags);
 466                 kfree(vma_res);
 467         } else {
 468                 i915_vma_resource_init_from_vma(vma_res, vma);
 469                 vma->resource = vma_res;
 470         }
 471         trace_i915_vma_bind(vma, bind_flags);
 472         if (work && bind_flags & vma->vm->bind_async_flags) {
 473                 struct dma_fence *prev;
 474
 475                 work->vma_res = i915_vma_resource_get(vma->resource);
 476                 work->cache_level = cache_level;
 477                 work->flags = bind_flags;
 478
 479                 /*
 480                  * Note we only want to chain up to the migration fence on
 481                  * the pages (not the object itself). As we don't track that,
 482                  * yet, we have to use the exclusive fence instead.
 483                  *
 484                  * Also note that we do not want to track the async vma as
 485                  * part of the obj->resv->excl_fence as it only affects
 486                  * execution and not content or object's backing store lifetime.
 487                  */
 488                 prev = i915_active_set_exclusive(&vma->active, &work->base.dma);
 489                 if (prev) {
 490                         __i915_sw_fence_await_dma_fence(&work->base.chain,
 491                                                         prev,
 492                                                         &work->cb);
 493                         dma_fence_put(prev);
 494                 }
 495
 496                 work->base.dma.error = 0; /* enable the queue_work() */
 497
 498                 /*
 499                  * If we don't have the refcounted pages list, keep a reference
 500                  * on the object to avoid waiting for the async bind to
 501                  * complete in the object destruction path.
 502                  */
 503                 if (!work->vma_res->bi.pages_rsgt)
 504                         work->pinned = i915_gem_object_get(vma->obj);
 505         } else {
 506                 if (vma->obj) {
 507                         int ret;
 508
 509                         ret = i915_gem_object_wait_moving_fence(vma->obj, true);
 510                         if (ret) {
 511                                 i915_vma_resource_free(vma->resource);
 512                                 vma->resource = NULL;
 513
 514                                 return ret;
 515                         }
 516                 }
 517                 vma->ops->bind_vma(vma->vm, NULL, vma->resource, cache_level,
 518                                    bind_flags);
 519         }
 520
 521         atomic_or(bind_flags, &vma->flags);
 522         return 0;
 523 }
 524
 525 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
 526 {
 527         void __iomem *ptr;
 528         int err;
 529
 530         if (!i915_gem_object_is_lmem(vma->obj)) {
 531                 if (GEM_WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
 532                         err = -ENODEV;
 533                         goto err;
 534                 }
 535         }
 536
 537         GEM_BUG_ON(!i915_vma_is_ggtt(vma));
 538         GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND));
 539         GEM_BUG_ON(i915_vma_verify_bind_complete(vma));
 540
 541         ptr = READ_ONCE(vma->iomap);
 542         if (ptr == NULL) {
 543                 /*
 544                  * TODO: consider just using i915_gem_object_pin_map() for lmem
 545                  * instead, which already supports mapping non-contiguous chunks
 546                  * of pages, that way we can also drop the
 547                  * I915_BO_ALLOC_CONTIGUOUS when allocating the object.
 548                  */
 549                 if (i915_gem_object_is_lmem(vma->obj))
 550                         ptr = i915_gem_object_lmem_io_map(vma->obj, 0,
 551                                                           vma->obj->base.size);
 552                 else
 553                         ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap,
 554                                                 vma->node.start,
 555                                                 vma->node.size);
 556                 if (ptr == NULL) {
 557                         err = -ENOMEM;
 558                         goto err;
 559                 }
 560
 561                 if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))) {
 562                         io_mapping_unmap(ptr);
 563                         ptr = vma->iomap;
 564                 }
 565         }
 566
 567         __i915_vma_pin(vma);
 568
 569         err = i915_vma_pin_fence(vma);
 570         if (err)
 571                 goto err_unpin;
 572
 573         i915_vma_set_ggtt_write(vma);
 574
 575         /* NB Access through the GTT requires the device to be awake. */
 576         return ptr;
 577
 578 err_unpin:
 579         __i915_vma_unpin(vma);
 580 err:
 581         return IO_ERR_PTR(err);
 582 }
 583
 584 void i915_vma_flush_writes(struct i915_vma *vma)
 585 {
 586         if (i915_vma_unset_ggtt_write(vma))
 587                 intel_gt_flush_ggtt_writes(vma->vm->gt);
 588 }
 589
 590 void i915_vma_unpin_iomap(struct i915_vma *vma)
 591 {
 592         GEM_BUG_ON(vma->iomap == NULL);
 593
 594         i915_vma_flush_writes(vma);
 595
 596         i915_vma_unpin_fence(vma);
 597         i915_vma_unpin(vma);
 598 }
 599
 600 void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags)
 601 {
 602         struct i915_vma *vma;
 603         struct drm_i915_gem_object *obj;
 604
 605         vma = fetch_and_zero(p_vma);
 606         if (!vma)
 607                 return;
 608
 609         obj = vma->obj;
 610         GEM_BUG_ON(!obj);
 611
 612         i915_vma_unpin(vma);
 613
 614         if (flags & I915_VMA_RELEASE_MAP)
 615                 i915_gem_object_unpin_map(obj);
 616
 617         i915_gem_object_put(obj);
 618 }
 619
 620 bool i915_vma_misplaced(const struct i915_vma *vma,
 621                         u64 size, u64 alignment, u64 flags)
 622 {
 623         if (!drm_mm_node_allocated(&vma->node))
 624                 return false;
 625
 626         if (test_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma)))
 627                 return true;
 628
 629         if (vma->node.size < size)
 630                 return true;
 631
 632         GEM_BUG_ON(alignment && !is_power_of_2(alignment));
 633         if (alignment && !IS_ALIGNED(vma->node.start, alignment))
 634                 return true;
 635
 636         if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
 637                 return true;
 638
 639         if (flags & PIN_OFFSET_BIAS &&
 640             vma->node.start < (flags & PIN_OFFSET_MASK))
 641                 return true;
 642
 643         if (flags & PIN_OFFSET_FIXED &&
 644             vma->node.start != (flags & PIN_OFFSET_MASK))
 645                 return true;
 646
 647         return false;
 648 }
 649
 650 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
 651 {
 652         bool mappable, fenceable;
 653
 654         GEM_BUG_ON(!i915_vma_is_ggtt(vma));
 655         GEM_BUG_ON(!vma->fence_size);
 656
 657         fenceable = (vma->node.size >= vma->fence_size &&
 658                      IS_ALIGNED(vma->node.start, vma->fence_alignment));
 659
 660         mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end;
 661
 662         if (mappable && fenceable)
 663                 set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
 664         else
 665                 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
 666 }
 667
 668 bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color)
 669 {
 670         struct drm_mm_node *node = &vma->node;
 671         struct drm_mm_node *other;
 672
 673         /*
 674          * On some machines we have to be careful when putting differing types
 675          * of snoopable memory together to avoid the prefetcher crossing memory
 676          * domains and dying. During vm initialisation, we decide whether or not
 677          * these constraints apply and set the drm_mm.color_adjust
 678          * appropriately.
 679          */
 680         if (!i915_vm_has_cache_coloring(vma->vm))
 681                 return true;
 682
 683         /* Only valid to be called on an already inserted vma */
 684         GEM_BUG_ON(!drm_mm_node_allocated(node));
 685         GEM_BUG_ON(list_empty(&node->node_list));
 686
 687         other = list_prev_entry(node, node_list);
 688         if (i915_node_color_differs(other, color) &&
 689             !drm_mm_hole_follows(other))
 690                 return false;
 691
 692         other = list_next_entry(node, node_list);
 693         if (i915_node_color_differs(other, color) &&
 694             !drm_mm_hole_follows(node))
 695                 return false;
 696
 697         return true;
 698 }
 699
 700 /**
 701  * i915_vma_insert - finds a slot for the vma in its address space
 702  * @vma: the vma
 703  * @size: requested size in bytes (can be larger than the VMA)
 704  * @alignment: required alignment
 705  * @flags: mask of PIN_* flags to use
 706  *
 707  * First we try to allocate some free space that meets the requirements for
 708  * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
 709  * preferrably the oldest idle entry to make room for the new VMA.
 710  *
 711  * Returns:
 712  * 0 on success, negative error code otherwise.
 713  */
 714 static int
 715 i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 716 {
 717         unsigned long color;
 718         u64 start, end;
 719         int ret;
 720
 721         GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
 722         GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 723
 724         size = max(size, vma->size);
 725         alignment = max(alignment, vma->display_alignment);
 726         if (flags & PIN_MAPPABLE) {
 727                 size = max_t(typeof(size), size, vma->fence_size);
 728                 alignment = max_t(typeof(alignment),
 729                                   alignment, vma->fence_alignment);
 730         }
 731
 732         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 733         GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
 734         GEM_BUG_ON(!is_power_of_2(alignment));
 735
 736         start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
 737         GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
 738
 739         end = vma->vm->total;
 740         if (flags & PIN_MAPPABLE)
 741                 end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end);
 742         if (flags & PIN_ZONE_4G)
 743                 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
 744         GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
 745
 746         /* If binding the object/GGTT view requires more space than the entire
 747          * aperture has, reject it early before evicting everything in a vain
 748          * attempt to find space.
 749          */
 750         if (size > end) {
 751                 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n",
 752                           size, flags & PIN_MAPPABLE ? "mappable" : "total",
 753                           end);
 754                 return -ENOSPC;
 755         }
 756
 757         color = 0;
 758         if (i915_vm_has_cache_coloring(vma->vm))
 759                 color = vma->obj->cache_level;
 760
 761         if (flags & PIN_OFFSET_FIXED) {
 762                 u64 offset = flags & PIN_OFFSET_MASK;
 763                 if (!IS_ALIGNED(offset, alignment) ||
 764                     range_overflows(offset, size, end))
 765                         return -EINVAL;
 766
 767                 ret = i915_gem_gtt_reserve(vma->vm, &vma->node,
 768                                            size, offset, color,
 769                                            flags);
 770                 if (ret)
 771                         return ret;
 772         } else {
 773                 /*
 774                  * We only support huge gtt pages through the 48b PPGTT,
 775                  * however we also don't want to force any alignment for
 776                  * objects which need to be tightly packed into the low 32bits.
 777                  *
 778                  * Note that we assume that GGTT are limited to 4GiB for the
 779                  * forseeable future. See also i915_ggtt_offset().
 780                  */
 781                 if (upper_32_bits(end - 1) &&
 782                     vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
 783                         /*
 784                          * We can't mix 64K and 4K PTEs in the same page-table
 785                          * (2M block), and so to avoid the ugliness and
 786                          * complexity of coloring we opt for just aligning 64K
 787                          * objects to 2M.
 788                          */
 789                         u64 page_alignment =
 790                                 rounddown_pow_of_two(vma->page_sizes.sg |
 791                                                      I915_GTT_PAGE_SIZE_2M);
 792
 793                         /*
 794                          * Check we don't expand for the limited Global GTT
 795                          * (mappable aperture is even more precious!). This
 796                          * also checks that we exclude the aliasing-ppgtt.
 797                          */
 798                         GEM_BUG_ON(i915_vma_is_ggtt(vma));
 799
 800                         alignment = max(alignment, page_alignment);
 801
 802                         if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
 803                                 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
 804                 }
 805
 806                 ret = i915_gem_gtt_insert(vma->vm, &vma->node,
 807                                           size, alignment, color,
 808                                           start, end, flags);
 809                 if (ret)
 810                         return ret;
 811
 812                 GEM_BUG_ON(vma->node.start < start);
 813                 GEM_BUG_ON(vma->node.start + vma->node.size > end);
 814         }
 815         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 816         GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color));
 817
 818         list_add_tail(&vma->vm_link, &vma->vm->bound_list);
 819
 820         return 0;
 821 }
 822
 823 static void
 824 i915_vma_detach(struct i915_vma *vma)
 825 {
 826         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 827         GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
 828
 829         /*
 830          * And finally now the object is completely decoupled from this
 831          * vma, we can drop its hold on the backing storage and allow
 832          * it to be reaped by the shrinker.
 833          */
 834         list_del(&vma->vm_link);
 835 }
 836
 837 static bool try_qad_pin(struct i915_vma *vma, unsigned int flags)
 838 {
 839         unsigned int bound;
 840         bool pinned = true;
 841
 842         bound = atomic_read(&vma->flags);
 843         do {
 844                 if (unlikely(flags & ~bound))
 845                         return false;
 846
 847                 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR)))
 848                         return false;
 849
 850                 if (!(bound & I915_VMA_PIN_MASK))
 851                         goto unpinned;
 852
 853                 GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0);
 854         } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1));
 855
 856         return true;
 857
 858 unpinned:
 859         /*
 860          * If pin_count==0, but we are bound, check under the lock to avoid
 861          * racing with a concurrent i915_vma_unbind().
 862          */
 863         mutex_lock(&vma->vm->mutex);
 864         do {
 865                 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))) {
 866                         pinned = false;
 867                         break;
 868                 }
 869
 870                 if (unlikely(flags & ~bound)) {
 871                         pinned = false;
 872                         break;
 873                 }
 874         } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1));
 875         mutex_unlock(&vma->vm->mutex);
 876
 877         return pinned;
 878 }
 879
 880 static struct scatterlist *
 881 rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
 882              unsigned int width, unsigned int height,
 883              unsigned int src_stride, unsigned int dst_stride,
 884              struct sg_table *st, struct scatterlist *sg)
 885 {
 886         unsigned int column, row;
 887         unsigned int src_idx;
 888
 889         for (column = 0; column < width; column++) {
 890                 unsigned int left;
 891
 892                 src_idx = src_stride * (height - 1) + column + offset;
 893                 for (row = 0; row < height; row++) {
 894                         st->nents++;
 895                         /*
 896                          * We don't need the pages, but need to initialize
 897                          * the entries so the sg list can be happily traversed.
 898                          * The only thing we need are DMA addresses.
 899                          */
 900                         sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0);
 901                         sg_dma_address(sg) =
 902                                 i915_gem_object_get_dma_address(obj, src_idx);
 903                         sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
 904                         sg = sg_next(sg);
 905                         src_idx -= src_stride;
 906                 }
 907
 908                 left = (dst_stride - height) * I915_GTT_PAGE_SIZE;
 909
 910                 if (!left)
 911                         continue;
 912
 913                 st->nents++;
 914
 915                 /*
 916                  * The DE ignores the PTEs for the padding tiles, the sg entry
 917                  * here is just a conenience to indicate how many padding PTEs
 918                  * to insert at this spot.
 919                  */
 920                 sg_set_page(sg, NULL, left, 0);
 921                 sg_dma_address(sg) = 0;
 922                 sg_dma_len(sg) = left;
 923                 sg = sg_next(sg);
 924         }
 925
 926         return sg;
 927 }
 928
 929 static noinline struct sg_table *
 930 intel_rotate_pages(struct intel_rotation_info *rot_info,
 931                    struct drm_i915_gem_object *obj)
 932 {
 933         unsigned int size = intel_rotation_info_size(rot_info);
 934         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 935         struct sg_table *st;
 936         struct scatterlist *sg;
 937         int ret = -ENOMEM;
 938         int i;
 939
 940         /* Allocate target SG list. */
 941         st = kmalloc(sizeof(*st), GFP_KERNEL);
 942         if (!st)
 943                 goto err_st_alloc;
 944
 945         ret = sg_alloc_table(st, size, GFP_KERNEL);
 946         if (ret)
 947                 goto err_sg_alloc;
 948
 949         st->nents = 0;
 950         sg = st->sgl;
 951
 952         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
 953                 sg = rotate_pages(obj, rot_info->plane[i].offset,
 954                                   rot_info->plane[i].width, rot_info->plane[i].height,
 955                                   rot_info->plane[i].src_stride,
 956                                   rot_info->plane[i].dst_stride,
 957                                   st, sg);
 958
 959         return st;
 960
 961 err_sg_alloc:
 962         kfree(st);
 963 err_st_alloc:
 964
 965         drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
 966                 obj->base.size, rot_info->plane[0].width,
 967                 rot_info->plane[0].height, size);
 968
 969         return ERR_PTR(ret);
 970 }
 971
 972 static struct scatterlist *
 973 remap_pages(struct drm_i915_gem_object *obj,
 974             unsigned int offset, unsigned int alignment_pad,
 975             unsigned int width, unsigned int height,
 976             unsigned int src_stride, unsigned int dst_stride,
 977             struct sg_table *st, struct scatterlist *sg)
 978 {
 979         unsigned int row;
 980
 981         if (!width || !height)
 982                 return sg;
 983
 984         if (alignment_pad) {
 985                 st->nents++;
 986
 987                 /*
 988                  * The DE ignores the PTEs for the padding tiles, the sg entry
 989                  * here is just a convenience to indicate how many padding PTEs
 990                  * to insert at this spot.
 991                  */
 992                 sg_set_page(sg, NULL, alignment_pad * 4096, 0);
 993                 sg_dma_address(sg) = 0;
 994                 sg_dma_len(sg) = alignment_pad * 4096;
 995                 sg = sg_next(sg);
 996         }
 997
 998         for (row = 0; row < height; row++) {
 999                 unsigned int left = width * I915_GTT_PAGE_SIZE;
1000
1001                 while (left) {
1002                         dma_addr_t addr;
1003                         unsigned int length;
1004
1005                         /*
1006                          * We don't need the pages, but need to initialize
1007                          * the entries so the sg list can be happily traversed.
1008                          * The only thing we need are DMA addresses.
1009                          */
1010
1011                         addr = i915_gem_object_get_dma_address_len(obj, offset, &length);
1012
1013                         length = min(left, length);
1014
1015                         st->nents++;
1016
1017                         sg_set_page(sg, NULL, length, 0);
1018                         sg_dma_address(sg) = addr;
1019                         sg_dma_len(sg) = length;
1020                         sg = sg_next(sg);
1021
1022                         offset += length / I915_GTT_PAGE_SIZE;
1023                         left -= length;
1024                 }
1025
1026                 offset += src_stride - width;
1027
1028                 left = (dst_stride - width) * I915_GTT_PAGE_SIZE;
1029
1030                 if (!left)
1031                         continue;
1032
1033                 st->nents++;
1034
1035                 /*
1036                  * The DE ignores the PTEs for the padding tiles, the sg entry
1037                  * here is just a conenience to indicate how many padding PTEs
1038                  * to insert at this spot.
1039                  */
1040                 sg_set_page(sg, NULL, left, 0);
1041                 sg_dma_address(sg) = 0;
1042                 sg_dma_len(sg) = left;
1043                 sg = sg_next(sg);
1044         }
1045
1046         return sg;
1047 }
1048
1049 static noinline struct sg_table *
1050 intel_remap_pages(struct intel_remapped_info *rem_info,
1051                   struct drm_i915_gem_object *obj)
1052 {
1053         unsigned int size = intel_remapped_info_size(rem_info);
1054         struct drm_i915_private *i915 = to_i915(obj->base.dev);
1055         struct sg_table *st;
1056         struct scatterlist *sg;
1057         unsigned int gtt_offset = 0;
1058         int ret = -ENOMEM;
1059         int i;
1060
1061         /* Allocate target SG list. */
1062         st = kmalloc(sizeof(*st), GFP_KERNEL);
1063         if (!st)
1064                 goto err_st_alloc;
1065
1066         ret = sg_alloc_table(st, size, GFP_KERNEL);
1067         if (ret)
1068                 goto err_sg_alloc;
1069
1070         st->nents = 0;
1071         sg = st->sgl;
1072
1073         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) {
1074                 unsigned int alignment_pad = 0;
1075
1076                 if (rem_info->plane_alignment)
1077                         alignment_pad = ALIGN(gtt_offset, rem_info->plane_alignment) - gtt_offset;
1078
1079                 sg = remap_pages(obj,
1080                                  rem_info->plane[i].offset, alignment_pad,
1081                                  rem_info->plane[i].width, rem_info->plane[i].height,
1082                                  rem_info->plane[i].src_stride, rem_info->plane[i].dst_stride,
1083                                  st, sg);
1084
1085                 gtt_offset += alignment_pad +
1086                               rem_info->plane[i].dst_stride * rem_info->plane[i].height;
1087         }
1088
1089         i915_sg_trim(st);
1090
1091         return st;
1092
1093 err_sg_alloc:
1094         kfree(st);
1095 err_st_alloc:
1096
1097         drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
1098                 obj->base.size, rem_info->plane[0].width,
1099                 rem_info->plane[0].height, size);
1100
1101         return ERR_PTR(ret);
1102 }
1103
1104 static noinline struct sg_table *
1105 intel_partial_pages(const struct i915_ggtt_view *view,
1106                     struct drm_i915_gem_object *obj)
1107 {
1108         struct sg_table *st;
1109         struct scatterlist *sg, *iter;
1110         unsigned int count = view->partial.size;
1111         unsigned int offset;
1112         int ret = -ENOMEM;
1113
1114         st = kmalloc(sizeof(*st), GFP_KERNEL);
1115         if (!st)
1116                 goto err_st_alloc;
1117
1118         ret = sg_alloc_table(st, count, GFP_KERNEL);
1119         if (ret)
1120                 goto err_sg_alloc;
1121
1122         iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset);
1123         GEM_BUG_ON(!iter);
1124
1125         sg = st->sgl;
1126         st->nents = 0;
1127         do {
1128                 unsigned int len;
1129
1130                 len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT),
1131                           count << PAGE_SHIFT);
1132                 sg_set_page(sg, NULL, len, 0);
1133                 sg_dma_address(sg) =
1134                         sg_dma_address(iter) + (offset << PAGE_SHIFT);
1135                 sg_dma_len(sg) = len;
1136
1137                 st->nents++;
1138                 count -= len >> PAGE_SHIFT;
1139                 if (count == 0) {
1140                         sg_mark_end(sg);
1141                         i915_sg_trim(st); /* Drop any unused tail entries. */
1142
1143                         return st;
1144                 }
1145
1146                 sg = __sg_next(sg);
1147                 iter = __sg_next(iter);
1148                 offset = 0;
1149         } while (1);
1150
1151 err_sg_alloc:
1152         kfree(st);
1153 err_st_alloc:
1154         return ERR_PTR(ret);
1155 }
1156
1157 static int
1158 __i915_vma_get_pages(struct i915_vma *vma)
1159 {
1160         struct sg_table *pages;
1161         int ret;
1162
1163         /*
1164          * The vma->pages are only valid within the lifespan of the borrowed
1165          * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
1166          * must be the vma->pages. A simple rule is that vma->pages must only
1167          * be accessed when the obj->mm.pages are pinned.
1168          */
1169         GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
1170
1171         switch (vma->ggtt_view.type) {
1172         default:
1173                 GEM_BUG_ON(vma->ggtt_view.type);
1174                 fallthrough;
1175         case I915_GGTT_VIEW_NORMAL:
1176                 pages = vma->obj->mm.pages;
1177                 break;
1178
1179         case I915_GGTT_VIEW_ROTATED:
1180                 pages =
1181                         intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
1182                 break;
1183
1184         case I915_GGTT_VIEW_REMAPPED:
1185                 pages =
1186                         intel_remap_pages(&vma->ggtt_view.remapped, vma->obj);
1187                 break;
1188
1189         case I915_GGTT_VIEW_PARTIAL:
1190                 pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
1191                 break;
1192         }
1193
1194         ret = 0;
1195         if (IS_ERR(pages)) {
1196                 ret = PTR_ERR(pages);
1197                 pages = NULL;
1198                 drm_err(&vma->vm->i915->drm,
1199                         "Failed to get pages for VMA view type %u (%d)!\n",
1200                         vma->ggtt_view.type, ret);
1201         }
1202
1203         vma->pages = pages;
1204
1205         return ret;
1206 }
1207
1208 I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
1209 {
1210         int err;
1211
1212         if (atomic_add_unless(&vma->pages_count, 1, 0))
1213                 return 0;
1214
1215         err = i915_gem_object_pin_pages(vma->obj);
1216         if (err)
1217                 return err;
1218
1219         err = __i915_vma_get_pages(vma);
1220         if (err)
1221                 goto err_unpin;
1222
1223         vma->page_sizes = vma->obj->mm.page_sizes;
1224         atomic_inc(&vma->pages_count);
1225
1226         return 0;
1227
1228 err_unpin:
1229         __i915_gem_object_unpin_pages(vma->obj);
1230
1231         return err;
1232 }
1233
1234 static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
1235 {
1236         /* We allocate under vma_get_pages, so beware the shrinker */
1237         struct sg_table *pages = READ_ONCE(vma->pages);
1238
1239         GEM_BUG_ON(atomic_read(&vma->pages_count) < count);
1240
1241         if (atomic_sub_return(count, &vma->pages_count) == 0) {
1242                 /*
1243                  * The atomic_sub_return is a read barrier for the READ_ONCE of
1244                  * vma->pages above.
1245                  *
1246                  * READ_ONCE is safe because this is either called from the same
1247                  * function (i915_vma_pin_ww), or guarded by vma->vm->mutex.
1248                  *
1249                  * TODO: We're leaving vma->pages dangling, until vma->obj->resv
1250                  * lock is required.
1251                  */
1252                 if (pages != vma->obj->mm.pages) {
1253                         sg_free_table(pages);
1254                         kfree(pages);
1255                 }
1256
1257                 i915_gem_object_unpin_pages(vma->obj);
1258         }
1259 }
1260
1261 I915_SELFTEST_EXPORT void i915_vma_put_pages(struct i915_vma *vma)
1262 {
1263         if (atomic_add_unless(&vma->pages_count, -1, 1))
1264                 return;
1265
1266         __vma_put_pages(vma, 1);
1267 }
1268
1269 static void vma_unbind_pages(struct i915_vma *vma)
1270 {
1271         unsigned int count;
1272
1273         lockdep_assert_held(&vma->vm->mutex);
1274
1275         /* The upper portion of pages_count is the number of bindings */
1276         count = atomic_read(&vma->pages_count);
1277         count >>= I915_VMA_PAGES_BIAS;
1278         GEM_BUG_ON(!count);
1279
1280         __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS);
1281 }
1282
1283 int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1284                     u64 size, u64 alignment, u64 flags)
1285 {
1286         struct i915_vma_work *work = NULL;
1287         struct dma_fence *moving = NULL;
1288         struct i915_vma_resource *vma_res = NULL;
1289         intel_wakeref_t wakeref = 0;
1290         unsigned int bound;
1291         int err;
1292
1293         assert_vma_held(vma);
1294         GEM_BUG_ON(!ww);
1295
1296         BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
1297         BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND);
1298
1299         GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL)));
1300
1301         /* First try and grab the pin without rebinding the vma */
1302         if (try_qad_pin(vma, flags & I915_VMA_BIND_MASK))
1303                 return 0;
1304
1305         err = i915_vma_get_pages(vma);
1306         if (err)
1307                 return err;
1308
1309         if (flags & PIN_GLOBAL)
1310                 wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm);
1311
1312         moving = vma->obj ? i915_gem_object_get_moving_fence(vma->obj) : NULL;
1313         if (flags & vma->vm->bind_async_flags || moving) {
1314                 /* lock VM */
1315                 err = i915_vm_lock_objects(vma->vm, ww);
1316                 if (err)
1317                         goto err_rpm;
1318
1319                 work = i915_vma_work();
1320                 if (!work) {
1321                         err = -ENOMEM;
1322                         goto err_rpm;
1323                 }
1324
1325                 work->vm = i915_vm_get(vma->vm);
1326
1327                 dma_fence_work_chain(&work->base, moving);
1328
1329                 /* Allocate enough page directories to used PTE */
1330                 if (vma->vm->allocate_va_range) {
1331                         err = i915_vm_alloc_pt_stash(vma->vm,
1332                                                      &work->stash,
1333                                                      vma->size);
1334                         if (err)
1335                                 goto err_fence;
1336
1337                         err = i915_vm_map_pt_stash(vma->vm, &work->stash);
1338                         if (err)
1339                                 goto err_fence;
1340                 }
1341         }
1342
1343         vma_res = i915_vma_resource_alloc();
1344         if (IS_ERR(vma_res)) {
1345                 err = PTR_ERR(vma_res);
1346                 goto err_fence;
1347         }
1348
1349         /*
1350          * Differentiate between user/kernel vma inside the aliasing-ppgtt.
1351          *
1352          * We conflate the Global GTT with the user's vma when using the
1353          * aliasing-ppgtt, but it is still vitally important to try and
1354          * keep the use cases distinct. For example, userptr objects are
1355          * not allowed inside the Global GTT as that will cause lock
1356          * inversions when we have to evict them the mmu_notifier callbacks -
1357          * but they are allowed to be part of the user ppGTT which can never
1358          * be mapped. As such we try to give the distinct users of the same
1359          * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt
1360          * and i915_ppgtt separate].
1361          *
1362          * NB this may cause us to mask real lock inversions -- while the
1363          * code is safe today, lockdep may not be able to spot future
1364          * transgressions.
1365          */
1366         err = mutex_lock_interruptible_nested(&vma->vm->mutex,
1367                                               !(flags & PIN_GLOBAL));
1368         if (err)
1369                 goto err_vma_res;
1370
1371         /* No more allocations allowed now we hold vm->mutex */
1372
1373         if (unlikely(i915_vma_is_closed(vma))) {
1374                 err = -ENOENT;
1375                 goto err_unlock;
1376         }
1377
1378         bound = atomic_read(&vma->flags);
1379         if (unlikely(bound & I915_VMA_ERROR)) {
1380                 err = -ENOMEM;
1381                 goto err_unlock;
1382         }
1383
1384         if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))) {
1385                 err = -EAGAIN; /* pins are meant to be fairly temporary */
1386                 goto err_unlock;
1387         }
1388
1389         if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) {
1390                 __i915_vma_pin(vma);
1391                 goto err_unlock;
1392         }
1393
1394         err = i915_active_acquire(&vma->active);
1395         if (err)
1396                 goto err_unlock;
1397
1398         if (!(bound & I915_VMA_BIND_MASK)) {
1399                 err = i915_vma_insert(vma, size, alignment, flags);
1400                 if (err)
1401                         goto err_active;
1402
1403                 if (i915_is_ggtt(vma->vm))
1404                         __i915_vma_set_map_and_fenceable(vma);
1405         }
1406
1407         GEM_BUG_ON(!vma->pages);
1408         err = i915_vma_bind(vma,
1409                             vma->obj->cache_level,
1410                             flags, work, vma_res);
1411         vma_res = NULL;
1412         if (err)
1413                 goto err_remove;
1414
1415         /* There should only be at most 2 active bindings (user, global) */
1416         GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound);
1417         atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count);
1418         list_move_tail(&vma->vm_link, &vma->vm->bound_list);
1419
1420         __i915_vma_pin(vma);
1421         GEM_BUG_ON(!i915_vma_is_pinned(vma));
1422         GEM_BUG_ON(!i915_vma_is_bound(vma, flags));
1423         GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
1424
1425 err_remove:
1426         if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) {
1427                 i915_vma_detach(vma);
1428                 drm_mm_remove_node(&vma->node);
1429         }
1430 err_active:
1431         i915_active_release(&vma->active);
1432 err_unlock:
1433         mutex_unlock(&vma->vm->mutex);
1434 err_vma_res:
1435         kfree(vma_res);
1436 err_fence:
1437         if (work)
1438                 dma_fence_work_commit_imm(&work->base);
1439 err_rpm:
1440         if (wakeref)
1441                 intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref);
1442
1443         if (moving)
1444                 dma_fence_put(moving);
1445
1446         i915_vma_put_pages(vma);
1447         return err;
1448 }
1449
1450 static void flush_idle_contexts(struct intel_gt *gt)
1451 {
1452         struct intel_engine_cs *engine;
1453         enum intel_engine_id id;
1454
1455         for_each_engine(engine, gt, id)
1456                 intel_engine_flush_barriers(engine);
1457
1458         intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT);
1459 }
1460
1461 static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1462                            u32 align, unsigned int flags)
1463 {
1464         struct i915_address_space *vm = vma->vm;
1465         int err;
1466
1467         do {
1468                 err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL);
1469
1470                 if (err != -ENOSPC) {
1471                         if (!err) {
1472                                 err = i915_vma_wait_for_bind(vma);
1473                                 if (err)
1474                                         i915_vma_unpin(vma);
1475                         }
1476                         return err;
1477                 }
1478
1479                 /* Unlike i915_vma_pin, we don't take no for an answer! */
1480                 flush_idle_contexts(vm->gt);
1481                 if (mutex_lock_interruptible(&vm->mutex) == 0) {
1482                         i915_gem_evict_vm(vm);
1483                         mutex_unlock(&vm->mutex);
1484                 }
1485         } while (1);
1486 }
1487
1488 int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
1489                   u32 align, unsigned int flags)
1490 {
1491         struct i915_gem_ww_ctx _ww;
1492         int err;
1493
1494         GEM_BUG_ON(!i915_vma_is_ggtt(vma));
1495
1496         if (ww)
1497                 return __i915_ggtt_pin(vma, ww, align, flags);
1498
1499 #ifdef CONFIG_LOCKDEP
1500         WARN_ON(dma_resv_held(vma->obj->base.resv));
1501 #endif
1502
1503         for_i915_gem_ww(&_ww, err, true) {
1504                 err = i915_gem_object_lock(vma->obj, &_ww);
1505                 if (!err)
1506                         err = __i915_ggtt_pin(vma, &_ww, align, flags);
1507         }
1508
1509         return err;
1510 }
1511
1512 static void __vma_close(struct i915_vma *vma, struct intel_gt *gt)
1513 {
1514         /*
1515          * We defer actually closing, unbinding and destroying the VMA until
1516          * the next idle point, or if the object is freed in the meantime. By
1517          * postponing the unbind, we allow for it to be resurrected by the
1518          * client, avoiding the work required to rebind the VMA. This is
1519          * advantageous for DRI, where the client/server pass objects
1520          * between themselves, temporarily opening a local VMA to the
1521          * object, and then closing it again. The same object is then reused
1522          * on the next frame (or two, depending on the depth of the swap queue)
1523          * causing us to rebind the VMA once more. This ends up being a lot
1524          * of wasted work for the steady state.
1525          */
1526         GEM_BUG_ON(i915_vma_is_closed(vma));
1527         list_add(&vma->closed_link, &gt->closed_vma);
1528 }
1529
1530 void i915_vma_close(struct i915_vma *vma)
1531 {
1532         struct intel_gt *gt = vma->vm->gt;
1533         unsigned long flags;
1534
1535         if (i915_vma_is_ggtt(vma))
1536                 return;
1537
1538         GEM_BUG_ON(!atomic_read(&vma->open_count));
1539         if (atomic_dec_and_lock_irqsave(&vma->open_count,
1540                                         &gt->closed_lock,
1541                                         flags)) {
1542                 __vma_close(vma, gt);
1543                 spin_unlock_irqrestore(&gt->closed_lock, flags);
1544         }
1545 }
1546
1547 static void __i915_vma_remove_closed(struct i915_vma *vma)
1548 {
1549         struct intel_gt *gt = vma->vm->gt;
1550
1551         spin_lock_irq(&gt->closed_lock);
1552         list_del_init(&vma->closed_link);
1553         spin_unlock_irq(&gt->closed_lock);
1554 }
1555
1556 void i915_vma_reopen(struct i915_vma *vma)
1557 {
1558         if (i915_vma_is_closed(vma))
1559                 __i915_vma_remove_closed(vma);
1560 }
1561
1562 void i915_vma_release(struct kref *ref)
1563 {
1564         struct i915_vma *vma = container_of(ref, typeof(*vma), ref);
1565         struct drm_i915_gem_object *obj = vma->obj;
1566
1567         if (drm_mm_node_allocated(&vma->node)) {
1568                 mutex_lock(&vma->vm->mutex);
1569                 atomic_and(~I915_VMA_PIN_MASK, &vma->flags);
1570                 WARN_ON(__i915_vma_unbind(vma));
1571                 mutex_unlock(&vma->vm->mutex);
1572                 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
1573         }
1574         GEM_BUG_ON(i915_vma_is_active(vma));
1575
1576         spin_lock(&obj->vma.lock);
1577         list_del(&vma->obj_link);
1578         if (!RB_EMPTY_NODE(&vma->obj_node))
1579                 rb_erase(&vma->obj_node, &obj->vma.tree);
1580         spin_unlock(&obj->vma.lock);
1581
1582         __i915_vma_remove_closed(vma);
1583         i915_vm_put(vma->vm);
1584
1585         i915_active_fini(&vma->active);
1586         GEM_WARN_ON(vma->resource);
1587         i915_vma_free(vma);
1588 }
1589
1590 void i915_vma_parked(struct intel_gt *gt)
1591 {
1592         struct i915_vma *vma, *next;
1593         LIST_HEAD(closed);
1594
1595         spin_lock_irq(&gt->closed_lock);
1596         list_for_each_entry_safe(vma, next, &gt->closed_vma, closed_link) {
1597                 struct drm_i915_gem_object *obj = vma->obj;
1598                 struct i915_address_space *vm = vma->vm;
1599
1600                 /* XXX All to avoid keeping a reference on i915_vma itself */
1601
1602                 if (!kref_get_unless_zero(&obj->base.refcount))
1603                         continue;
1604
1605                 if (!i915_vm_tryopen(vm)) {
1606                         i915_gem_object_put(obj);
1607                         continue;
1608                 }
1609
1610                 list_move(&vma->closed_link, &closed);
1611         }
1612         spin_unlock_irq(&gt->closed_lock);
1613
1614         /* As the GT is held idle, no vma can be reopened as we destroy them */
1615         list_for_each_entry_safe(vma, next, &closed, closed_link) {
1616                 struct drm_i915_gem_object *obj = vma->obj;
1617                 struct i915_address_space *vm = vma->vm;
1618
1619                 INIT_LIST_HEAD(&vma->closed_link);
1620                 __i915_vma_put(vma);
1621
1622                 i915_gem_object_put(obj);
1623                 i915_vm_close(vm);
1624         }
1625 }
1626
1627 static void __i915_vma_iounmap(struct i915_vma *vma)
1628 {
1629         GEM_BUG_ON(i915_vma_is_pinned(vma));
1630
1631         if (vma->iomap == NULL)
1632                 return;
1633
1634         io_mapping_unmap(vma->iomap);
1635         vma->iomap = NULL;
1636 }
1637
1638 void i915_vma_revoke_mmap(struct i915_vma *vma)
1639 {
1640         struct drm_vma_offset_node *node;
1641         u64 vma_offset;
1642
1643         if (!i915_vma_has_userfault(vma))
1644                 return;
1645
1646         GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
1647         GEM_BUG_ON(!vma->obj->userfault_count);
1648
1649         node = &vma->mmo->vma_node;
1650         vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
1651         unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
1652                             drm_vma_node_offset_addr(node) + vma_offset,
1653                             vma->size,
1654                             1);
1655
1656         i915_vma_unset_userfault(vma);
1657         if (!--vma->obj->userfault_count)
1658                 list_del(&vma->obj->userfault_link);
1659 }
1660
1661 static int
1662 __i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma)
1663 {
1664         return __i915_request_await_exclusive(rq, &vma->active);
1665 }
1666
1667 static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq)
1668 {
1669         int err;
1670
1671         GEM_BUG_ON(!i915_vma_is_pinned(vma));
1672
1673         /* Wait for the vma to be bound before we start! */
1674         err = __i915_request_await_bind(rq, vma);
1675         if (err)
1676                 return err;
1677
1678         return i915_active_add_request(&vma->active, rq);
1679 }
1680
1681 int _i915_vma_move_to_active(struct i915_vma *vma,
1682                              struct i915_request *rq,
1683                              struct dma_fence *fence,
1684                              unsigned int flags)
1685 {
1686         struct drm_i915_gem_object *obj = vma->obj;
1687         int err;
1688
1689         assert_object_held(obj);
1690
1691         err = __i915_vma_move_to_active(vma, rq);
1692         if (unlikely(err))
1693                 return err;
1694
1695         if (flags & EXEC_OBJECT_WRITE) {
1696                 struct intel_frontbuffer *front;
1697
1698                 front = __intel_frontbuffer_get(obj);
1699                 if (unlikely(front)) {
1700                         if (intel_frontbuffer_invalidate(front, ORIGIN_CS))
1701                                 i915_active_add_request(&front->write, rq);
1702                         intel_frontbuffer_put(front);
1703                 }
1704
1705                 if (fence) {
1706                         dma_resv_add_excl_fence(vma->obj->base.resv, fence);
1707                         obj->write_domain = I915_GEM_DOMAIN_RENDER;
1708                         obj->read_domains = 0;
1709                 }
1710         } else {
1711                 if (!(flags & __EXEC_OBJECT_NO_RESERVE)) {
1712                         err = dma_resv_reserve_shared(vma->obj->base.resv, 1);
1713                         if (unlikely(err))
1714                                 return err;
1715                 }
1716
1717                 if (fence) {
1718                         dma_resv_add_shared_fence(vma->obj->base.resv, fence);
1719                         obj->write_domain = 0;
1720                 }
1721         }
1722
1723         if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence)
1724                 i915_active_add_request(&vma->fence->active, rq);
1725
1726         obj->read_domains |= I915_GEM_GPU_DOMAINS;
1727         obj->mm.dirty = true;
1728
1729         GEM_BUG_ON(!i915_vma_is_active(vma));
1730         return 0;
1731 }
1732
1733 struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
1734 {
1735         struct i915_vma_resource *vma_res = vma->resource;
1736         struct dma_fence *unbind_fence;
1737
1738         GEM_BUG_ON(i915_vma_is_pinned(vma));
1739
1740         if (i915_vma_is_map_and_fenceable(vma)) {
1741                 /* Force a pagefault for domain tracking on next user access */
1742                 i915_vma_revoke_mmap(vma);
1743
1744                 /*
1745                  * Check that we have flushed all writes through the GGTT
1746                  * before the unbind, other due to non-strict nature of those
1747                  * indirect writes they may end up referencing the GGTT PTE
1748                  * after the unbind.
1749                  *
1750                  * Note that we may be concurrently poking at the GGTT_WRITE
1751                  * bit from set-domain, as we mark all GGTT vma associated
1752                  * with an object. We know this is for another vma, as we
1753                  * are currently unbinding this one -- so if this vma will be
1754                  * reused, it will be refaulted and have its dirty bit set
1755                  * before the next write.
1756                  */
1757                 i915_vma_flush_writes(vma);
1758
1759                 /* release the fence reg _after_ flushing */
1760                 i915_vma_revoke_fence(vma);
1761
1762                 __i915_vma_iounmap(vma);
1763                 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma));
1764         }
1765         GEM_BUG_ON(vma->fence);
1766         GEM_BUG_ON(i915_vma_has_userfault(vma));
1767
1768         /* Object backend must be async capable. */
1769         GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt);
1770
1771         /* If vm is not open, unbind is a nop. */
1772         vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND) &&
1773                 atomic_read(&vma->vm->open);
1774         trace_i915_vma_unbind(vma);
1775
1776         unbind_fence = i915_vma_resource_unbind(vma_res);
1777         vma->resource = NULL;
1778
1779         atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE),
1780                    &vma->flags);
1781
1782         i915_vma_detach(vma);
1783
1784         if (!async && unbind_fence) {
1785                 dma_fence_wait(unbind_fence, false);
1786                 dma_fence_put(unbind_fence);
1787                 unbind_fence = NULL;
1788         }
1789
1790         /*
1791          * Binding itself may not have completed until the unbind fence signals,
1792          * so don't drop the pages until that happens, unless the resource is
1793          * async_capable.
1794          */
1795
1796         vma_unbind_pages(vma);
1797         return unbind_fence;
1798 }
1799
1800 int __i915_vma_unbind(struct i915_vma *vma)
1801 {
1802         int ret;
1803
1804         lockdep_assert_held(&vma->vm->mutex);
1805
1806         if (!drm_mm_node_allocated(&vma->node))
1807                 return 0;
1808
1809         if (i915_vma_is_pinned(vma)) {
1810                 vma_print_allocator(vma, "is pinned");
1811                 return -EAGAIN;
1812         }
1813
1814         /*
1815          * After confirming that no one else is pinning this vma, wait for
1816          * any laggards who may have crept in during the wait (through
1817          * a residual pin skipping the vm->mutex) to complete.
1818          */
1819         ret = i915_vma_sync(vma);
1820         if (ret)
1821                 return ret;
1822
1823         GEM_BUG_ON(i915_vma_is_active(vma));
1824         __i915_vma_evict(vma, false);
1825
1826         drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
1827         return 0;
1828 }
1829
1830 static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma)
1831 {
1832         struct dma_fence *fence;
1833
1834         lockdep_assert_held(&vma->vm->mutex);
1835
1836         if (!drm_mm_node_allocated(&vma->node))
1837                 return NULL;
1838
1839         if (i915_vma_is_pinned(vma) ||
1840             &vma->obj->mm.rsgt->table != vma->resource->bi.pages)
1841                 return ERR_PTR(-EAGAIN);
1842
1843         /*
1844          * We probably need to replace this with awaiting the fences of the
1845          * object's dma_resv when the vma active goes away. When doing that
1846          * we need to be careful to not add the vma_resource unbind fence
1847          * immediately to the object's dma_resv, because then unbinding
1848          * the next vma from the object, in case there are many, will
1849          * actually await the unbinding of the previous vmas, which is
1850          * undesirable.
1851          */
1852         if (i915_sw_fence_await_active(&vma->resource->chain, &vma->active,
1853                                        I915_ACTIVE_AWAIT_EXCL |
1854                                        I915_ACTIVE_AWAIT_ACTIVE) < 0) {
1855                 return ERR_PTR(-EBUSY);
1856         }
1857
1858         fence = __i915_vma_evict(vma, true);
1859
1860         drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */
1861
1862         return fence;
1863 }
1864
1865 int i915_vma_unbind(struct i915_vma *vma)
1866 {
1867         struct i915_address_space *vm = vma->vm;
1868         intel_wakeref_t wakeref = 0;
1869         int err;
1870
1871         /* Optimistic wait before taking the mutex */
1872         err = i915_vma_sync(vma);
1873         if (err)
1874                 return err;
1875
1876         if (!drm_mm_node_allocated(&vma->node))
1877                 return 0;
1878
1879         if (i915_vma_is_pinned(vma)) {
1880                 vma_print_allocator(vma, "is pinned");
1881                 return -EAGAIN;
1882         }
1883
1884         if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
1885                 /* XXX not always required: nop_clear_range */
1886                 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
1887
1888         err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref);
1889         if (err)
1890                 goto out_rpm;
1891
1892         err = __i915_vma_unbind(vma);
1893         mutex_unlock(&vm->mutex);
1894
1895 out_rpm:
1896         if (wakeref)
1897                 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
1898         return err;
1899 }
1900
1901 int i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm)
1902 {
1903         struct drm_i915_gem_object *obj = vma->obj;
1904         struct i915_address_space *vm = vma->vm;
1905         intel_wakeref_t wakeref = 0;
1906         struct dma_fence *fence;
1907         int err;
1908
1909         /*
1910          * We need the dma-resv lock since we add the
1911          * unbind fence to the dma-resv object.
1912          */
1913         assert_object_held(obj);
1914
1915         if (!drm_mm_node_allocated(&vma->node))
1916                 return 0;
1917
1918         if (i915_vma_is_pinned(vma)) {
1919                 vma_print_allocator(vma, "is pinned");
1920                 return -EAGAIN;
1921         }
1922
1923         if (!obj->mm.rsgt)
1924                 return -EBUSY;
1925
1926         err = dma_resv_reserve_shared(obj->base.resv, 1);
1927         if (err)
1928                 return -EBUSY;
1929
1930         /*
1931          * It would be great if we could grab this wakeref from the
1932          * async unbind work if needed, but we can't because it uses
1933          * kmalloc and it's in the dma-fence signalling critical path.
1934          */
1935         if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
1936                 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
1937
1938         if (trylock_vm && !mutex_trylock(&vm->mutex)) {
1939                 err = -EBUSY;
1940                 goto out_rpm;
1941         } else if (!trylock_vm) {
1942                 err = mutex_lock_interruptible_nested(&vm->mutex, !wakeref);
1943                 if (err)
1944                         goto out_rpm;
1945         }
1946
1947         fence = __i915_vma_unbind_async(vma);
1948         mutex_unlock(&vm->mutex);
1949         if (IS_ERR_OR_NULL(fence)) {
1950                 err = PTR_ERR_OR_ZERO(fence);
1951                 goto out_rpm;
1952         }
1953
1954         dma_resv_add_shared_fence(obj->base.resv, fence);
1955         dma_fence_put(fence);
1956
1957 out_rpm:
1958         if (wakeref)
1959                 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
1960         return err;
1961 }
1962
1963 struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma)
1964 {
1965         i915_gem_object_make_unshrinkable(vma->obj);
1966         return vma;
1967 }
1968
1969 void i915_vma_make_shrinkable(struct i915_vma *vma)
1970 {
1971         i915_gem_object_make_shrinkable(vma->obj);
1972 }
1973
1974 void i915_vma_make_purgeable(struct i915_vma *vma)
1975 {
1976         i915_gem_object_make_purgeable(vma->obj);
1977 }
1978
1979 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1980 #include "selftests/i915_vma.c"
1981 #endif
1982
1983 void i915_vma_module_exit(void)
1984 {
1985         kmem_cache_destroy(slab_vmas);
1986 }
1987
1988 int __init i915_vma_module_init(void)
1989 {
1990         slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
1991         if (!slab_vmas)
1992                 return -ENOMEM;
1993
1994         return 0;
1995 }