drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_bo.h>
  35 #include <drm/ttm/ttm_placement.h>
  36 #include <drm/ttm/ttm_tt.h>
  37
  38 #include <linux/jiffies.h>
  39 #include <linux/slab.h>
  40 #include <linux/sched.h>
  41 #include <linux/mm.h>
  42 #include <linux/file.h>
  43 #include <linux/module.h>
  44 #include <linux/atomic.h>
  45 #include <linux/dma-resv.h>
  46
  47 #include "ttm_module.h"
  48
  49 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  50                                         struct ttm_placement *placement)
  51 {
  52         struct drm_printer p = drm_debug_printer(TTM_PFX);
  53         struct ttm_resource_manager *man;
  54         int i, mem_type;
  55
  56         for (i = 0; i < placement->num_placement; i++) {
  57                 mem_type = placement->placement[i].mem_type;
  58                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
  59                            i, placement->placement[i].flags, mem_type);
  60                 man = ttm_manager_type(bo->bdev, mem_type);
  61                 ttm_resource_manager_debug(man, &p);
  62         }
  63 }
  64
  65 /**
  66  * ttm_bo_move_to_lru_tail
  67  *
  68  * @bo: The buffer object.
  69  *
  70  * Move this BO to the tail of all lru lists used to lookup and reserve an
  71  * object. This function must be called with struct ttm_global::lru_lock
  72  * held, and is used to make a BO less likely to be considered for eviction.
  73  */
  74 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
  75 {
  76         dma_resv_assert_held(bo->base.resv);
  77
  78         if (bo->resource)
  79                 ttm_resource_move_to_lru_tail(bo->resource);
  80 }
  81 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
  82
  83 /**
  84  * ttm_bo_set_bulk_move - update BOs bulk move object
  85  *
  86  * @bo: The buffer object.
  87  * @bulk: bulk move structure
  88  *
  89  * Update the BOs bulk move object, making sure that resources are added/removed
  90  * as well. A bulk move allows to move many resource on the LRU at once,
  91  * resulting in much less overhead of maintaining the LRU.
  92  * The only requirement is that the resources stay together on the LRU and are
  93  * never separated. This is enforces by setting the bulk_move structure on a BO.
  94  * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of
  95  * their LRU list.
  96  */
  97 void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo,
  98                           struct ttm_lru_bulk_move *bulk)
  99 {
 100         dma_resv_assert_held(bo->base.resv);
 101
 102         if (bo->bulk_move == bulk)
 103                 return;
 104
 105         spin_lock(&bo->bdev->lru_lock);
 106         if (bo->resource)
 107                 ttm_resource_del_bulk_move(bo->resource, bo);
 108         bo->bulk_move = bulk;
 109         if (bo->resource)
 110                 ttm_resource_add_bulk_move(bo->resource, bo);
 111         spin_unlock(&bo->bdev->lru_lock);
 112 }
 113 EXPORT_SYMBOL(ttm_bo_set_bulk_move);
 114
 115 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 116                                   struct ttm_resource *mem, bool evict,
 117                                   struct ttm_operation_ctx *ctx,
 118                                   struct ttm_place *hop)
 119 {
 120         struct ttm_device *bdev = bo->bdev;
 121         bool old_use_tt, new_use_tt;
 122         int ret;
 123
 124         old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt;
 125         new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt;
 126
 127         ttm_bo_unmap_virtual(bo);
 128
 129         /*
 130          * Create and bind a ttm if required.
 131          */
 132
 133         if (new_use_tt) {
 134                 /* Zero init the new TTM structure if the old location should
 135                  * have used one as well.
 136                  */
 137                 ret = ttm_tt_create(bo, old_use_tt);
 138                 if (ret)
 139                         goto out_err;
 140
 141                 if (mem->mem_type != TTM_PL_SYSTEM) {
 142                         ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx);
 143                         if (ret)
 144                                 goto out_err;
 145                 }
 146         }
 147
 148         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 149         if (ret)
 150                 goto out_err;
 151
 152         ret = bdev->funcs->move(bo, evict, ctx, mem, hop);
 153         if (ret) {
 154                 if (ret == -EMULTIHOP)
 155                         return ret;
 156                 goto out_err;
 157         }
 158
 159         ctx->bytes_moved += bo->base.size;
 160         return 0;
 161
 162 out_err:
 163         if (!old_use_tt)
 164                 ttm_bo_tt_destroy(bo);
 165
 166         return ret;
 167 }
 168
 169 /*
 170  * Call bo::reserved.
 171  * Will release GPU memory type usage on destruction.
 172  * This is the place to put in driver specific hooks to release
 173  * driver private resources.
 174  * Will release the bo::reserved lock.
 175  */
 176
 177 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 178 {
 179         if (bo->bdev->funcs->delete_mem_notify)
 180                 bo->bdev->funcs->delete_mem_notify(bo);
 181
 182         ttm_bo_tt_destroy(bo);
 183         ttm_resource_free(bo, &bo->resource);
 184 }
 185
 186 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 187 {
 188         int r;
 189
 190         if (bo->base.resv == &bo->base._resv)
 191                 return 0;
 192
 193         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 194
 195         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 196         dma_resv_unlock(&bo->base._resv);
 197         if (r)
 198                 return r;
 199
 200         if (bo->type != ttm_bo_type_sg) {
 201                 /* This works because the BO is about to be destroyed and nobody
 202                  * reference it any more. The only tricky case is the trylock on
 203                  * the resv object while holding the lru_lock.
 204                  */
 205                 spin_lock(&bo->bdev->lru_lock);
 206                 bo->base.resv = &bo->base._resv;
 207                 spin_unlock(&bo->bdev->lru_lock);
 208         }
 209
 210         return r;
 211 }
 212
 213 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 214 {
 215         struct dma_resv *resv = &bo->base._resv;
 216         struct dma_resv_iter cursor;
 217         struct dma_fence *fence;
 218
 219         dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);
 220         dma_resv_for_each_fence_unlocked(&cursor, fence) {
 221                 if (!fence->ops->signaled)
 222                         dma_fence_enable_sw_signaling(fence);
 223         }
 224         dma_resv_iter_end(&cursor);
 225 }
 226
 227 /**
 228  * ttm_bo_cleanup_refs
 229  * If bo idle, remove from lru lists, and unref.
 230  * If not idle, block if possible.
 231  *
 232  * Must be called with lru_lock and reservation held, this function
 233  * will drop the lru lock and optionally the reservation lock before returning.
 234  *
 235  * @bo:                    The buffer object to clean-up
 236  * @interruptible:         Any sleeps should occur interruptibly.
 237  * @no_wait_gpu:           Never wait for gpu. Return -EBUSY instead.
 238  * @unlock_resv:           Unlock the reservation lock as well.
 239  */
 240
 241 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 242                                bool interruptible, bool no_wait_gpu,
 243                                bool unlock_resv)
 244 {
 245         struct dma_resv *resv = &bo->base._resv;
 246         int ret;
 247
 248         if (dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP))
 249                 ret = 0;
 250         else
 251                 ret = -EBUSY;
 252
 253         if (ret && !no_wait_gpu) {
 254                 long lret;
 255
 256                 if (unlock_resv)
 257                         dma_resv_unlock(bo->base.resv);
 258                 spin_unlock(&bo->bdev->lru_lock);
 259
 260                 lret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP,
 261                                              interruptible,
 262                                              30 * HZ);
 263
 264                 if (lret < 0)
 265                         return lret;
 266                 else if (lret == 0)
 267                         return -EBUSY;
 268
 269                 spin_lock(&bo->bdev->lru_lock);
 270                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 271                         /*
 272                          * We raced, and lost, someone else holds the reservation now,
 273                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 274                          *
 275                          * Even if it's not the case, because we finished waiting any
 276                          * delayed destruction would succeed, so just return success
 277                          * here.
 278                          */
 279                         spin_unlock(&bo->bdev->lru_lock);
 280                         return 0;
 281                 }
 282                 ret = 0;
 283         }
 284
 285         if (ret) {
 286                 if (unlock_resv)
 287                         dma_resv_unlock(bo->base.resv);
 288                 spin_unlock(&bo->bdev->lru_lock);
 289                 return ret;
 290         }
 291
 292         spin_unlock(&bo->bdev->lru_lock);
 293         ttm_bo_cleanup_memtype_use(bo);
 294
 295         if (unlock_resv)
 296                 dma_resv_unlock(bo->base.resv);
 297
 298         return 0;
 299 }
 300
 301 /*
 302  * Block for the dma_resv object to become idle, lock the buffer and clean up
 303  * the resource and tt object.
 304  */
 305 static void ttm_bo_delayed_delete(struct work_struct *work)
 306 {
 307         struct ttm_buffer_object *bo;
 308
 309         bo = container_of(work, typeof(*bo), delayed_delete);
 310
 311         dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false,
 312                               MAX_SCHEDULE_TIMEOUT);
 313         dma_resv_lock(bo->base.resv, NULL);
 314         ttm_bo_cleanup_memtype_use(bo);
 315         dma_resv_unlock(bo->base.resv);
 316         ttm_bo_put(bo);
 317 }
 318
 319 static void ttm_bo_release(struct kref *kref)
 320 {
 321         struct ttm_buffer_object *bo =
 322             container_of(kref, struct ttm_buffer_object, kref);
 323         struct ttm_device *bdev = bo->bdev;
 324         int ret;
 325
 326         WARN_ON_ONCE(bo->pin_count);
 327         WARN_ON_ONCE(bo->bulk_move);
 328
 329         if (!bo->deleted) {
 330                 ret = ttm_bo_individualize_resv(bo);
 331                 if (ret) {
 332                         /* Last resort, if we fail to allocate memory for the
 333                          * fences block for the BO to become idle
 334                          */
 335                         dma_resv_wait_timeout(bo->base.resv,
 336                                               DMA_RESV_USAGE_BOOKKEEP, false,
 337                                               30 * HZ);
 338                 }
 339
 340                 if (bo->bdev->funcs->release_notify)
 341                         bo->bdev->funcs->release_notify(bo);
 342
 343                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 344                 ttm_mem_io_free(bdev, bo->resource);
 345
 346                 if (!dma_resv_test_signaled(bo->base.resv,
 347                                             DMA_RESV_USAGE_BOOKKEEP) ||
 348                     !dma_resv_trylock(bo->base.resv)) {
 349                         /* The BO is not idle, resurrect it for delayed destroy */
 350                         ttm_bo_flush_all_fences(bo);
 351                         bo->deleted = true;
 352
 353                         spin_lock(&bo->bdev->lru_lock);
 354
 355                         /*
 356                          * Make pinned bos immediately available to
 357                          * shrinkers, now that they are queued for
 358                          * destruction.
 359                          *
 360                          * FIXME: QXL is triggering this. Can be removed when the
 361                          * driver is fixed.
 362                          */
 363                         if (bo->pin_count) {
 364                                 bo->pin_count = 0;
 365                                 ttm_resource_move_to_lru_tail(bo->resource);
 366                         }
 367
 368                         kref_init(&bo->kref);
 369                         spin_unlock(&bo->bdev->lru_lock);
 370
 371                         INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete);
 372                         queue_work(bdev->wq, &bo->delayed_delete);
 373                         return;
 374                 }
 375
 376                 ttm_bo_cleanup_memtype_use(bo);
 377                 dma_resv_unlock(bo->base.resv);
 378         }
 379
 380         atomic_dec(&ttm_glob.bo_count);
 381         bo->destroy(bo);
 382 }
 383
 384 /**
 385  * ttm_bo_put
 386  *
 387  * @bo: The buffer object.
 388  *
 389  * Unreference a buffer object.
 390  */
 391 void ttm_bo_put(struct ttm_buffer_object *bo)
 392 {
 393         kref_put(&bo->kref, ttm_bo_release);
 394 }
 395 EXPORT_SYMBOL(ttm_bo_put);
 396
 397 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo,
 398                                      struct ttm_resource **mem,
 399                                      struct ttm_operation_ctx *ctx,
 400                                      struct ttm_place *hop)
 401 {
 402         struct ttm_placement hop_placement;
 403         struct ttm_resource *hop_mem;
 404         int ret;
 405
 406         hop_placement.num_placement = hop_placement.num_busy_placement = 1;
 407         hop_placement.placement = hop_placement.busy_placement = hop;
 408
 409         /* find space in the bounce domain */
 410         ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx);
 411         if (ret)
 412                 return ret;
 413         /* move to the bounce domain */
 414         ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL);
 415         if (ret) {
 416                 ttm_resource_free(bo, &hop_mem);
 417                 return ret;
 418         }
 419         return 0;
 420 }
 421
 422 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 423                         struct ttm_operation_ctx *ctx)
 424 {
 425         struct ttm_device *bdev = bo->bdev;
 426         struct ttm_resource *evict_mem;
 427         struct ttm_placement placement;
 428         struct ttm_place hop;
 429         int ret = 0;
 430
 431         memset(&hop, 0, sizeof(hop));
 432
 433         dma_resv_assert_held(bo->base.resv);
 434
 435         placement.num_placement = 0;
 436         placement.num_busy_placement = 0;
 437         bdev->funcs->evict_flags(bo, &placement);
 438
 439         if (!placement.num_placement && !placement.num_busy_placement) {
 440                 ret = ttm_bo_wait_ctx(bo, ctx);
 441                 if (ret)
 442                         return ret;
 443
 444                 /*
 445                  * Since we've already synced, this frees backing store
 446                  * immediately.
 447                  */
 448                 return ttm_bo_pipeline_gutting(bo);
 449         }
 450
 451         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 452         if (ret) {
 453                 if (ret != -ERESTARTSYS) {
 454                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 455                                bo);
 456                         ttm_bo_mem_space_debug(bo, &placement);
 457                 }
 458                 goto out;
 459         }
 460
 461         do {
 462                 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop);
 463                 if (ret != -EMULTIHOP)
 464                         break;
 465
 466                 ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop);
 467         } while (!ret);
 468
 469         if (ret) {
 470                 ttm_resource_free(bo, &evict_mem);
 471                 if (ret != -ERESTARTSYS && ret != -EINTR)
 472                         pr_err("Buffer eviction failed\n");
 473         }
 474 out:
 475         return ret;
 476 }
 477
 478 /**
 479  * ttm_bo_eviction_valuable
 480  *
 481  * @bo: The buffer object to evict
 482  * @place: the placement we need to make room for
 483  *
 484  * Check if it is valuable to evict the BO to make room for the given placement.
 485  */
 486 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 487                               const struct ttm_place *place)
 488 {
 489         struct ttm_resource *res = bo->resource;
 490         struct ttm_device *bdev = bo->bdev;
 491
 492         dma_resv_assert_held(bo->base.resv);
 493         if (bo->resource->mem_type == TTM_PL_SYSTEM)
 494                 return true;
 495
 496         /* Don't evict this BO if it's outside of the
 497          * requested placement range
 498          */
 499         return ttm_resource_intersects(bdev, res, place, bo->base.size);
 500 }
 501 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 502
 503 /*
 504  * Check the target bo is allowable to be evicted or swapout, including cases:
 505  *
 506  * a. if share same reservation object with ctx->resv, have assumption
 507  * reservation objects should already be locked, so not lock again and
 508  * return true directly when either the opreation allow_reserved_eviction
 509  * or the target bo already is in delayed free list;
 510  *
 511  * b. Otherwise, trylock it.
 512  */
 513 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 514                                            struct ttm_operation_ctx *ctx,
 515                                            const struct ttm_place *place,
 516                                            bool *locked, bool *busy)
 517 {
 518         bool ret = false;
 519
 520         if (bo->pin_count) {
 521                 *locked = false;
 522                 if (busy)
 523                         *busy = false;
 524                 return false;
 525         }
 526
 527         if (bo->base.resv == ctx->resv) {
 528                 dma_resv_assert_held(bo->base.resv);
 529                 if (ctx->allow_res_evict)
 530                         ret = true;
 531                 *locked = false;
 532                 if (busy)
 533                         *busy = false;
 534         } else {
 535                 ret = dma_resv_trylock(bo->base.resv);
 536                 *locked = ret;
 537                 if (busy)
 538                         *busy = !ret;
 539         }
 540
 541         if (ret && place && (bo->resource->mem_type != place->mem_type ||
 542                 !bo->bdev->funcs->eviction_valuable(bo, place))) {
 543                 ret = false;
 544                 if (*locked) {
 545                         dma_resv_unlock(bo->base.resv);
 546                         *locked = false;
 547                 }
 548         }
 549
 550         return ret;
 551 }
 552
 553 /**
 554  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 555  *
 556  * @busy_bo: BO which couldn't be locked with trylock
 557  * @ctx: operation context
 558  * @ticket: acquire ticket
 559  *
 560  * Try to lock a busy buffer object to avoid failing eviction.
 561  */
 562 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 563                                    struct ttm_operation_ctx *ctx,
 564                                    struct ww_acquire_ctx *ticket)
 565 {
 566         int r;
 567
 568         if (!busy_bo || !ticket)
 569                 return -EBUSY;
 570
 571         if (ctx->interruptible)
 572                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 573                                                           ticket);
 574         else
 575                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 576
 577         /*
 578          * TODO: It would be better to keep the BO locked until allocation is at
 579          * least tried one more time, but that would mean a much larger rework
 580          * of TTM.
 581          */
 582         if (!r)
 583                 dma_resv_unlock(busy_bo->base.resv);
 584
 585         return r == -EDEADLK ? -EBUSY : r;
 586 }
 587
 588 int ttm_mem_evict_first(struct ttm_device *bdev,
 589                         struct ttm_resource_manager *man,
 590                         const struct ttm_place *place,
 591                         struct ttm_operation_ctx *ctx,
 592                         struct ww_acquire_ctx *ticket)
 593 {
 594         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 595         struct ttm_resource_cursor cursor;
 596         struct ttm_resource *res;
 597         bool locked = false;
 598         int ret;
 599
 600         spin_lock(&bdev->lru_lock);
 601         ttm_resource_manager_for_each_res(man, &cursor, res) {
 602                 bool busy;
 603
 604                 if (!ttm_bo_evict_swapout_allowable(res->bo, ctx, place,
 605                                                     &locked, &busy)) {
 606                         if (busy && !busy_bo && ticket !=
 607                             dma_resv_locking_ctx(res->bo->base.resv))
 608                                 busy_bo = res->bo;
 609                         continue;
 610                 }
 611
 612                 if (ttm_bo_get_unless_zero(res->bo)) {
 613                         bo = res->bo;
 614                         break;
 615                 }
 616                 if (locked)
 617                         dma_resv_unlock(res->bo->base.resv);
 618         }
 619
 620         if (!bo) {
 621                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 622                         busy_bo = NULL;
 623                 spin_unlock(&bdev->lru_lock);
 624                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 625                 if (busy_bo)
 626                         ttm_bo_put(busy_bo);
 627                 return ret;
 628         }
 629
 630         if (bo->deleted) {
 631                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 632                                           ctx->no_wait_gpu, locked);
 633                 ttm_bo_put(bo);
 634                 return ret;
 635         }
 636
 637         spin_unlock(&bdev->lru_lock);
 638
 639         ret = ttm_bo_evict(bo, ctx);
 640         if (locked)
 641                 ttm_bo_unreserve(bo);
 642         else
 643                 ttm_bo_move_to_lru_tail_unlocked(bo);
 644
 645         ttm_bo_put(bo);
 646         return ret;
 647 }
 648
 649 /**
 650  * ttm_bo_pin - Pin the buffer object.
 651  * @bo: The buffer object to pin
 652  *
 653  * Make sure the buffer is not evicted any more during memory pressure.
 654  * @bo must be unpinned again by calling ttm_bo_unpin().
 655  */
 656 void ttm_bo_pin(struct ttm_buffer_object *bo)
 657 {
 658         dma_resv_assert_held(bo->base.resv);
 659         WARN_ON_ONCE(!kref_read(&bo->kref));
 660         spin_lock(&bo->bdev->lru_lock);
 661         if (bo->resource)
 662                 ttm_resource_del_bulk_move(bo->resource, bo);
 663         ++bo->pin_count;
 664         spin_unlock(&bo->bdev->lru_lock);
 665 }
 666 EXPORT_SYMBOL(ttm_bo_pin);
 667
 668 /**
 669  * ttm_bo_unpin - Unpin the buffer object.
 670  * @bo: The buffer object to unpin
 671  *
 672  * Allows the buffer object to be evicted again during memory pressure.
 673  */
 674 void ttm_bo_unpin(struct ttm_buffer_object *bo)
 675 {
 676         dma_resv_assert_held(bo->base.resv);
 677         WARN_ON_ONCE(!kref_read(&bo->kref));
 678         if (WARN_ON_ONCE(!bo->pin_count))
 679                 return;
 680
 681         spin_lock(&bo->bdev->lru_lock);
 682         --bo->pin_count;
 683         if (bo->resource)
 684                 ttm_resource_add_bulk_move(bo->resource, bo);
 685         spin_unlock(&bo->bdev->lru_lock);
 686 }
 687 EXPORT_SYMBOL(ttm_bo_unpin);
 688
 689 /*
 690  * Add the last move fence to the BO as kernel dependency and reserve a new
 691  * fence slot.
 692  */
 693 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 694                                  struct ttm_resource_manager *man,
 695                                  struct ttm_resource *mem,
 696                                  bool no_wait_gpu)
 697 {
 698         struct dma_fence *fence;
 699         int ret;
 700
 701         spin_lock(&man->move_lock);
 702         fence = dma_fence_get(man->move);
 703         spin_unlock(&man->move_lock);
 704
 705         if (!fence)
 706                 return 0;
 707
 708         if (no_wait_gpu) {
 709                 ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY;
 710                 dma_fence_put(fence);
 711                 return ret;
 712         }
 713
 714         dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL);
 715
 716         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 717         dma_fence_put(fence);
 718         return ret;
 719 }
 720
 721 /*
 722  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 723  * space, or we've evicted everything and there isn't enough space.
 724  */
 725 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 726                                   const struct ttm_place *place,
 727                                   struct ttm_resource **mem,
 728                                   struct ttm_operation_ctx *ctx)
 729 {
 730         struct ttm_device *bdev = bo->bdev;
 731         struct ttm_resource_manager *man;
 732         struct ww_acquire_ctx *ticket;
 733         int ret;
 734
 735         man = ttm_manager_type(bdev, place->mem_type);
 736         ticket = dma_resv_locking_ctx(bo->base.resv);
 737         do {
 738                 ret = ttm_resource_alloc(bo, place, mem);
 739                 if (likely(!ret))
 740                         break;
 741                 if (unlikely(ret != -ENOSPC))
 742                         return ret;
 743                 ret = ttm_mem_evict_first(bdev, man, place, ctx,
 744                                           ticket);
 745                 if (unlikely(ret != 0))
 746                         return ret;
 747         } while (1);
 748
 749         return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 750 }
 751
 752 /**
 753  * ttm_bo_mem_space
 754  *
 755  * @bo: Pointer to a struct ttm_buffer_object. the data of which
 756  * we want to allocate space for.
 757  * @placement: Proposed new placement for the buffer object.
 758  * @mem: A struct ttm_resource.
 759  * @ctx: if and how to sleep, lock buffers and alloc memory
 760  *
 761  * Allocate memory space for the buffer object pointed to by @bo, using
 762  * the placement flags in @placement, potentially evicting other idle buffer objects.
 763  * This function may sleep while waiting for space to become available.
 764  * Returns:
 765  * -EBUSY: No space available (only if no_wait == 1).
 766  * -ENOMEM: Could not allocate memory for the buffer object, either due to
 767  * fragmentation or concurrent allocators.
 768  * -ERESTARTSYS: An interruptible sleep was interrupted by a signal.
 769  */
 770 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 771                         struct ttm_placement *placement,
 772                         struct ttm_resource **mem,
 773                         struct ttm_operation_ctx *ctx)
 774 {
 775         struct ttm_device *bdev = bo->bdev;
 776         bool type_found = false;
 777         int i, ret;
 778
 779         ret = dma_resv_reserve_fences(bo->base.resv, 1);
 780         if (unlikely(ret))
 781                 return ret;
 782
 783         for (i = 0; i < placement->num_placement; ++i) {
 784                 const struct ttm_place *place = &placement->placement[i];
 785                 struct ttm_resource_manager *man;
 786
 787                 man = ttm_manager_type(bdev, place->mem_type);
 788                 if (!man || !ttm_resource_manager_used(man))
 789                         continue;
 790
 791                 type_found = true;
 792                 ret = ttm_resource_alloc(bo, place, mem);
 793                 if (ret == -ENOSPC)
 794                         continue;
 795                 if (unlikely(ret))
 796                         goto error;
 797
 798                 ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu);
 799                 if (unlikely(ret)) {
 800                         ttm_resource_free(bo, mem);
 801                         if (ret == -EBUSY)
 802                                 continue;
 803
 804                         goto error;
 805                 }
 806                 return 0;
 807         }
 808
 809         for (i = 0; i < placement->num_busy_placement; ++i) {
 810                 const struct ttm_place *place = &placement->busy_placement[i];
 811                 struct ttm_resource_manager *man;
 812
 813                 man = ttm_manager_type(bdev, place->mem_type);
 814                 if (!man || !ttm_resource_manager_used(man))
 815                         continue;
 816
 817                 type_found = true;
 818                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
 819                 if (likely(!ret))
 820                         return 0;
 821
 822                 if (ret && ret != -EBUSY)
 823                         goto error;
 824         }
 825
 826         ret = -ENOMEM;
 827         if (!type_found) {
 828                 pr_err(TTM_PFX "No compatible memory type found\n");
 829                 ret = -EINVAL;
 830         }
 831
 832 error:
 833         return ret;
 834 }
 835 EXPORT_SYMBOL(ttm_bo_mem_space);
 836
 837 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 838                               struct ttm_placement *placement,
 839                               struct ttm_operation_ctx *ctx)
 840 {
 841         struct ttm_resource *mem;
 842         struct ttm_place hop;
 843         int ret;
 844
 845         dma_resv_assert_held(bo->base.resv);
 846
 847         /*
 848          * Determine where to move the buffer.
 849          *
 850          * If driver determines move is going to need
 851          * an extra step then it will return -EMULTIHOP
 852          * and the buffer will be moved to the temporary
 853          * stop and the driver will be called to make
 854          * the second hop.
 855          */
 856         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
 857         if (ret)
 858                 return ret;
 859 bounce:
 860         ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop);
 861         if (ret == -EMULTIHOP) {
 862                 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop);
 863                 if (ret)
 864                         goto out;
 865                 /* try and move to final place now. */
 866                 goto bounce;
 867         }
 868 out:
 869         if (ret)
 870                 ttm_resource_free(bo, &mem);
 871         return ret;
 872 }
 873
 874 /**
 875  * ttm_bo_validate
 876  *
 877  * @bo: The buffer object.
 878  * @placement: Proposed placement for the buffer object.
 879  * @ctx: validation parameters.
 880  *
 881  * Changes placement and caching policy of the buffer object
 882  * according proposed placement.
 883  * Returns
 884  * -EINVAL on invalid proposed placement.
 885  * -ENOMEM on out-of-memory condition.
 886  * -EBUSY if no_wait is true and buffer busy.
 887  * -ERESTARTSYS if interrupted by a signal.
 888  */
 889 int ttm_bo_validate(struct ttm_buffer_object *bo,
 890                     struct ttm_placement *placement,
 891                     struct ttm_operation_ctx *ctx)
 892 {
 893         int ret;
 894
 895         dma_resv_assert_held(bo->base.resv);
 896
 897         /*
 898          * Remove the backing store if no placement is given.
 899          */
 900         if (!placement->num_placement && !placement->num_busy_placement)
 901                 return ttm_bo_pipeline_gutting(bo);
 902
 903         /* Check whether we need to move buffer. */
 904         if (bo->resource && ttm_resource_compat(bo->resource, placement))
 905                 return 0;
 906
 907         /* Moving of pinned BOs is forbidden */
 908         if (bo->pin_count)
 909                 return -EINVAL;
 910
 911         ret = ttm_bo_move_buffer(bo, placement, ctx);
 912         if (ret)
 913                 return ret;
 914
 915         /*
 916          * We might need to add a TTM.
 917          */
 918         if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) {
 919                 ret = ttm_tt_create(bo, true);
 920                 if (ret)
 921                         return ret;
 922         }
 923         return 0;
 924 }
 925 EXPORT_SYMBOL(ttm_bo_validate);
 926
 927 /**
 928  * ttm_bo_init_reserved
 929  *
 930  * @bdev: Pointer to a ttm_device struct.
 931  * @bo: Pointer to a ttm_buffer_object to be initialized.
 932  * @type: Requested type of buffer object.
 933  * @placement: Initial placement for buffer object.
 934  * @alignment: Data alignment in pages.
 935  * @ctx: TTM operation context for memory allocation.
 936  * @sg: Scatter-gather table.
 937  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
 938  * @destroy: Destroy function. Use NULL for kfree().
 939  *
 940  * This function initializes a pre-allocated struct ttm_buffer_object.
 941  * As this object may be part of a larger structure, this function,
 942  * together with the @destroy function, enables driver-specific objects
 943  * derived from a ttm_buffer_object.
 944  *
 945  * On successful return, the caller owns an object kref to @bo. The kref and
 946  * list_kref are usually set to 1, but note that in some situations, other
 947  * tasks may already be holding references to @bo as well.
 948  * Furthermore, if resv == NULL, the buffer's reservation lock will be held,
 949  * and it is the caller's responsibility to call ttm_bo_unreserve.
 950  *
 951  * If a failure occurs, the function will call the @destroy function. Thus,
 952  * after a failure, dereferencing @bo is illegal and will likely cause memory
 953  * corruption.
 954  *
 955  * Returns
 956  * -ENOMEM: Out of memory.
 957  * -EINVAL: Invalid placement flags.
 958  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
 959  */
 960 int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
 961                          enum ttm_bo_type type, struct ttm_placement *placement,
 962                          uint32_t alignment, struct ttm_operation_ctx *ctx,
 963                          struct sg_table *sg, struct dma_resv *resv,
 964                          void (*destroy) (struct ttm_buffer_object *))
 965 {
 966         int ret;
 967
 968         kref_init(&bo->kref);
 969         bo->bdev = bdev;
 970         bo->type = type;
 971         bo->page_alignment = alignment;
 972         bo->destroy = destroy;
 973         bo->pin_count = 0;
 974         bo->sg = sg;
 975         bo->bulk_move = NULL;
 976         if (resv)
 977                 bo->base.resv = resv;
 978         else
 979                 bo->base.resv = &bo->base._resv;
 980         atomic_inc(&ttm_glob.bo_count);
 981
 982         /*
 983          * For ttm_bo_type_device buffers, allocate
 984          * address space from the device.
 985          */
 986         if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) {
 987                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
 988                                          PFN_UP(bo->base.size));
 989                 if (ret)
 990                         goto err_put;
 991         }
 992
 993         /* passed reservation objects should already be locked,
 994          * since otherwise lockdep will be angered in radeon.
 995          */
 996         if (!resv)
 997                 WARN_ON(!dma_resv_trylock(bo->base.resv));
 998         else
 999                 dma_resv_assert_held(resv);
1000
1001         ret = ttm_bo_validate(bo, placement, ctx);
1002         if (unlikely(ret))
1003                 goto err_unlock;
1004
1005         return 0;
1006
1007 err_unlock:
1008         if (!resv)
1009                 dma_resv_unlock(bo->base.resv);
1010
1011 err_put:
1012         ttm_bo_put(bo);
1013         return ret;
1014 }
1015 EXPORT_SYMBOL(ttm_bo_init_reserved);
1016
1017 /**
1018  * ttm_bo_init_validate
1019  *
1020  * @bdev: Pointer to a ttm_device struct.
1021  * @bo: Pointer to a ttm_buffer_object to be initialized.
1022  * @type: Requested type of buffer object.
1023  * @placement: Initial placement for buffer object.
1024  * @alignment: Data alignment in pages.
1025  * @interruptible: If needing to sleep to wait for GPU resources,
1026  * sleep interruptible.
1027  * pinned in physical memory. If this behaviour is not desired, this member
1028  * holds a pointer to a persistent shmem object. Typically, this would
1029  * point to the shmem object backing a GEM object if TTM is used to back a
1030  * GEM user interface.
1031  * @sg: Scatter-gather table.
1032  * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
1033  * @destroy: Destroy function. Use NULL for kfree().
1034  *
1035  * This function initializes a pre-allocated struct ttm_buffer_object.
1036  * As this object may be part of a larger structure, this function,
1037  * together with the @destroy function,
1038  * enables driver-specific objects derived from a ttm_buffer_object.
1039  *
1040  * On successful return, the caller owns an object kref to @bo. The kref and
1041  * list_kref are usually set to 1, but note that in some situations, other
1042  * tasks may already be holding references to @bo as well.
1043  *
1044  * If a failure occurs, the function will call the @destroy function, Thus,
1045  * after a failure, dereferencing @bo is illegal and will likely cause memory
1046  * corruption.
1047  *
1048  * Returns
1049  * -ENOMEM: Out of memory.
1050  * -EINVAL: Invalid placement flags.
1051  * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
1052  */
1053 int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo,
1054                          enum ttm_bo_type type, struct ttm_placement *placement,
1055                          uint32_t alignment, bool interruptible,
1056                          struct sg_table *sg, struct dma_resv *resv,
1057                          void (*destroy) (struct ttm_buffer_object *))
1058 {
1059         struct ttm_operation_ctx ctx = { interruptible, false };
1060         int ret;
1061
1062         ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx,
1063                                    sg, resv, destroy);
1064         if (ret)
1065                 return ret;
1066
1067         if (!resv)
1068                 ttm_bo_unreserve(bo);
1069
1070         return 0;
1071 }
1072 EXPORT_SYMBOL(ttm_bo_init_validate);
1073
1074 /*
1075  * buffer object vm functions.
1076  */
1077
1078 /**
1079  * ttm_bo_unmap_virtual
1080  *
1081  * @bo: tear down the virtual mappings for this BO
1082  */
1083 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1084 {
1085         struct ttm_device *bdev = bo->bdev;
1086
1087         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1088         ttm_mem_io_free(bdev, bo->resource);
1089 }
1090 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1091
1092 /**
1093  * ttm_bo_wait_ctx - wait for buffer idle.
1094  *
1095  * @bo:  The buffer object.
1096  * @ctx: defines how to wait
1097  *
1098  * Waits for the buffer to be idle. Used timeout depends on the context.
1099  * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or
1100  * zero on success.
1101  */
1102 int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx)
1103 {
1104         long ret;
1105
1106         if (ctx->no_wait_gpu) {
1107                 if (dma_resv_test_signaled(bo->base.resv,
1108                                            DMA_RESV_USAGE_BOOKKEEP))
1109                         return 0;
1110                 else
1111                         return -EBUSY;
1112         }
1113
1114         ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP,
1115                                     ctx->interruptible, 15 * HZ);
1116         if (unlikely(ret < 0))
1117                 return ret;
1118         if (unlikely(ret == 0))
1119                 return -EBUSY;
1120         return 0;
1121 }
1122 EXPORT_SYMBOL(ttm_bo_wait_ctx);
1123
1124 int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
1125                    gfp_t gfp_flags)
1126 {
1127         struct ttm_place place;
1128         bool locked;
1129         long ret;
1130
1131         /*
1132          * While the bo may already reside in SYSTEM placement, set
1133          * SYSTEM as new placement to cover also the move further below.
1134          * The driver may use the fact that we're moving from SYSTEM
1135          * as an indication that we're about to swap out.
1136          */
1137         memset(&place, 0, sizeof(place));
1138         place.mem_type = bo->resource->mem_type;
1139         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL))
1140                 return -EBUSY;
1141
1142         if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) ||
1143             bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL ||
1144             bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED ||
1145             !ttm_bo_get_unless_zero(bo)) {
1146                 if (locked)
1147                         dma_resv_unlock(bo->base.resv);
1148                 return -EBUSY;
1149         }
1150
1151         if (bo->deleted) {
1152                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1153                 ttm_bo_put(bo);
1154                 return ret == -EBUSY ? -ENOSPC : ret;
1155         }
1156
1157         /* TODO: Cleanup the locking */
1158         spin_unlock(&bo->bdev->lru_lock);
1159
1160         /*
1161          * Move to system cached
1162          */
1163         if (bo->resource->mem_type != TTM_PL_SYSTEM) {
1164                 struct ttm_operation_ctx ctx = { false, false };
1165                 struct ttm_resource *evict_mem;
1166                 struct ttm_place hop;
1167
1168                 memset(&hop, 0, sizeof(hop));
1169                 place.mem_type = TTM_PL_SYSTEM;
1170                 ret = ttm_resource_alloc(bo, &place, &evict_mem);
1171                 if (unlikely(ret))
1172                         goto out;
1173
1174                 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, &ctx, &hop);
1175                 if (unlikely(ret != 0)) {
1176                         WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n");
1177                         ttm_resource_free(bo, &evict_mem);
1178                         goto out;
1179                 }
1180         }
1181
1182         /*
1183          * Make sure BO is idle.
1184          */
1185         ret = ttm_bo_wait_ctx(bo, ctx);
1186         if (unlikely(ret != 0))
1187                 goto out;
1188
1189         ttm_bo_unmap_virtual(bo);
1190
1191         /*
1192          * Swap out. Buffer will be swapped in again as soon as
1193          * anyone tries to access a ttm page.
1194          */
1195         if (bo->bdev->funcs->swap_notify)
1196                 bo->bdev->funcs->swap_notify(bo);
1197
1198         if (ttm_tt_is_populated(bo->ttm))
1199                 ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags);
1200 out:
1201
1202         /*
1203          * Unreserve without putting on LRU to avoid swapping out an
1204          * already swapped buffer.
1205          */
1206         if (locked)
1207                 dma_resv_unlock(bo->base.resv);
1208         ttm_bo_put(bo);
1209         return ret == -EBUSY ? -ENOSPC : ret;
1210 }
1211
1212 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo)
1213 {
1214         if (bo->ttm == NULL)
1215                 return;
1216
1217         ttm_tt_unpopulate(bo->bdev, bo->ttm);
1218         ttm_tt_destroy(bo->bdev, bo->ttm);
1219         bo->ttm = NULL;
1220 }