drivers/gpu/drm/ttm/ttm_bo.c

   1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
   2 /**************************************************************************
   3  *
   4  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28 /*
  29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  30  */
  31
  32 #define pr_fmt(fmt) "[TTM] " fmt
  33
  34 #include <drm/ttm/ttm_module.h>
  35 #include <drm/ttm/ttm_bo_driver.h>
  36 #include <drm/ttm/ttm_placement.h>
  37 #include <linux/jiffies.h>
  38 #include <linux/slab.h>
  39 #include <linux/sched.h>
  40 #include <linux/mm.h>
  41 #include <linux/file.h>
  42 #include <linux/module.h>
  43 #include <linux/atomic.h>
  44 #include <linux/dma-resv.h>
  45
  46 static void ttm_bo_global_kobj_release(struct kobject *kobj);
  47
  48 /**
  49  * ttm_global_mutex - protecting the global BO state
  50  */
  51 DEFINE_MUTEX(ttm_global_mutex);
  52 unsigned ttm_bo_glob_use_count;
  53 struct ttm_bo_global ttm_bo_glob;
  54 EXPORT_SYMBOL(ttm_bo_glob);
  55
  56 static struct attribute ttm_bo_count = {
  57         .name = "bo_count",
  58         .mode = S_IRUGO
  59 };
  60
  61 /* default destructor */
  62 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo)
  63 {
  64         kfree(bo);
  65 }
  66
  67 static inline int ttm_mem_type_from_place(const struct ttm_place *place,
  68                                           uint32_t *mem_type)
  69 {
  70         int pos;
  71
  72         pos = ffs(place->flags & TTM_PL_MASK_MEM);
  73         if (unlikely(!pos))
  74                 return -EINVAL;
  75
  76         *mem_type = pos - 1;
  77         return 0;
  78 }
  79
  80 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, struct drm_printer *p,
  81                                int mem_type)
  82 {
  83         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  84
  85         drm_printf(p, "    has_type: %d\n", man->has_type);
  86         drm_printf(p, "    use_type: %d\n", man->use_type);
  87         drm_printf(p, "    flags: 0x%08X\n", man->flags);
  88         drm_printf(p, "    gpu_offset: 0x%08llX\n", man->gpu_offset);
  89         drm_printf(p, "    size: %llu\n", man->size);
  90         drm_printf(p, "    available_caching: 0x%08X\n", man->available_caching);
  91         drm_printf(p, "    default_caching: 0x%08X\n", man->default_caching);
  92         if (mem_type != TTM_PL_SYSTEM)
  93                 (*man->func->debug)(man, p);
  94 }
  95
  96 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  97                                         struct ttm_placement *placement)
  98 {
  99         struct drm_printer p = drm_debug_printer(TTM_PFX);
 100         int i, ret, mem_type;
 101
 102         drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n",
 103                    bo, bo->mem.num_pages, bo->mem.size >> 10,
 104                    bo->mem.size >> 20);
 105         for (i = 0; i < placement->num_placement; i++) {
 106                 ret = ttm_mem_type_from_place(&placement->placement[i],
 107                                                 &mem_type);
 108                 if (ret)
 109                         return;
 110                 drm_printf(&p, "  placement[%d]=0x%08X (%d)\n",
 111                            i, placement->placement[i].flags, mem_type);
 112                 ttm_mem_type_debug(bo->bdev, &p, mem_type);
 113         }
 114 }
 115
 116 static ssize_t ttm_bo_global_show(struct kobject *kobj,
 117                                   struct attribute *attr,
 118                                   char *buffer)
 119 {
 120         struct ttm_bo_global *glob =
 121                 container_of(kobj, struct ttm_bo_global, kobj);
 122
 123         return snprintf(buffer, PAGE_SIZE, "%d\n",
 124                                 atomic_read(&glob->bo_count));
 125 }
 126
 127 static struct attribute *ttm_bo_global_attrs[] = {
 128         &ttm_bo_count,
 129         NULL
 130 };
 131
 132 static const struct sysfs_ops ttm_bo_global_ops = {
 133         .show = &ttm_bo_global_show
 134 };
 135
 136 static struct kobj_type ttm_bo_glob_kobj_type  = {
 137         .release = &ttm_bo_global_kobj_release,
 138         .sysfs_ops = &ttm_bo_global_ops,
 139         .default_attrs = ttm_bo_global_attrs
 140 };
 141
 142
 143 static inline uint32_t ttm_bo_type_flags(unsigned type)
 144 {
 145         return 1 << (type);
 146 }
 147
 148 static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo,
 149                                   struct ttm_mem_reg *mem)
 150 {
 151         struct ttm_bo_device *bdev = bo->bdev;
 152         struct ttm_mem_type_manager *man;
 153
 154         if (!list_empty(&bo->lru))
 155                 return;
 156
 157         if (mem->placement & TTM_PL_FLAG_NO_EVICT)
 158                 return;
 159
 160         man = &bdev->man[mem->mem_type];
 161         list_add_tail(&bo->lru, &man->lru[bo->priority]);
 162
 163         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm &&
 164             !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG |
 165                                      TTM_PAGE_FLAG_SWAPPED))) {
 166                 list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]);
 167         }
 168 }
 169
 170 static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 171 {
 172         struct ttm_bo_device *bdev = bo->bdev;
 173         bool notify = false;
 174
 175         if (!list_empty(&bo->swap)) {
 176                 list_del_init(&bo->swap);
 177                 notify = true;
 178         }
 179         if (!list_empty(&bo->lru)) {
 180                 list_del_init(&bo->lru);
 181                 notify = true;
 182         }
 183
 184         if (notify && bdev->driver->del_from_lru_notify)
 185                 bdev->driver->del_from_lru_notify(bo);
 186 }
 187
 188 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
 189                                      struct ttm_buffer_object *bo)
 190 {
 191         if (!pos->first)
 192                 pos->first = bo;
 193         pos->last = bo;
 194 }
 195
 196 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
 197                              struct ttm_lru_bulk_move *bulk)
 198 {
 199         dma_resv_assert_held(bo->base.resv);
 200
 201         ttm_bo_del_from_lru(bo);
 202         ttm_bo_add_mem_to_lru(bo, &bo->mem);
 203
 204         if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 205                 switch (bo->mem.mem_type) {
 206                 case TTM_PL_TT:
 207                         ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
 208                         break;
 209
 210                 case TTM_PL_VRAM:
 211                         ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
 212                         break;
 213                 }
 214                 if (bo->ttm && !(bo->ttm->page_flags &
 215                                  (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED)))
 216                         ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo);
 217         }
 218 }
 219 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);
 220
 221 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
 222 {
 223         unsigned i;
 224
 225         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 226                 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
 227                 struct ttm_mem_type_manager *man;
 228
 229                 if (!pos->first)
 230                         continue;
 231
 232                 dma_resv_assert_held(pos->first->base.resv);
 233                 dma_resv_assert_held(pos->last->base.resv);
 234
 235                 man = &pos->first->bdev->man[TTM_PL_TT];
 236                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 237                                     &pos->last->lru);
 238         }
 239
 240         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 241                 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
 242                 struct ttm_mem_type_manager *man;
 243
 244                 if (!pos->first)
 245                         continue;
 246
 247                 dma_resv_assert_held(pos->first->base.resv);
 248                 dma_resv_assert_held(pos->last->base.resv);
 249
 250                 man = &pos->first->bdev->man[TTM_PL_VRAM];
 251                 list_bulk_move_tail(&man->lru[i], &pos->first->lru,
 252                                     &pos->last->lru);
 253         }
 254
 255         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 256                 struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i];
 257                 struct list_head *lru;
 258
 259                 if (!pos->first)
 260                         continue;
 261
 262                 dma_resv_assert_held(pos->first->base.resv);
 263                 dma_resv_assert_held(pos->last->base.resv);
 264
 265                 lru = &ttm_bo_glob.swap_lru[i];
 266                 list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap);
 267         }
 268 }
 269 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);
 270
 271 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 272                                   struct ttm_mem_reg *mem, bool evict,
 273                                   struct ttm_operation_ctx *ctx)
 274 {
 275         struct ttm_bo_device *bdev = bo->bdev;
 276         bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
 277         bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
 278         struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
 279         struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
 280         int ret = 0;
 281
 282         if (old_is_pci || new_is_pci ||
 283             ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
 284                 ret = ttm_mem_io_lock(old_man, true);
 285                 if (unlikely(ret != 0))
 286                         goto out_err;
 287                 ttm_bo_unmap_virtual_locked(bo);
 288                 ttm_mem_io_unlock(old_man);
 289         }
 290
 291         /*
 292          * Create and bind a ttm if required.
 293          */
 294
 295         if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
 296                 if (bo->ttm == NULL) {
 297                         bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
 298                         ret = ttm_tt_create(bo, zero);
 299                         if (ret)
 300                                 goto out_err;
 301                 }
 302
 303                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
 304                 if (ret)
 305                         goto out_err;
 306
 307                 if (mem->mem_type != TTM_PL_SYSTEM) {
 308                         ret = ttm_tt_bind(bo->ttm, mem, ctx);
 309                         if (ret)
 310                                 goto out_err;
 311                 }
 312
 313                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
 314                         if (bdev->driver->move_notify)
 315                                 bdev->driver->move_notify(bo, evict, mem);
 316                         bo->mem = *mem;
 317                         mem->mm_node = NULL;
 318                         goto moved;
 319                 }
 320         }
 321
 322         if (bdev->driver->move_notify)
 323                 bdev->driver->move_notify(bo, evict, mem);
 324
 325         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 326             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
 327                 ret = ttm_bo_move_ttm(bo, ctx, mem);
 328         else if (bdev->driver->move)
 329                 ret = bdev->driver->move(bo, evict, ctx, mem);
 330         else
 331                 ret = ttm_bo_move_memcpy(bo, ctx, mem);
 332
 333         if (ret) {
 334                 if (bdev->driver->move_notify) {
 335                         swap(*mem, bo->mem);
 336                         bdev->driver->move_notify(bo, false, mem);
 337                         swap(*mem, bo->mem);
 338                 }
 339
 340                 goto out_err;
 341         }
 342
 343 moved:
 344         bo->evicted = false;
 345
 346         if (bo->mem.mm_node)
 347                 bo->offset = (bo->mem.start << PAGE_SHIFT) +
 348                     bdev->man[bo->mem.mem_type].gpu_offset;
 349         else
 350                 bo->offset = 0;
 351
 352         ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
 353         return 0;
 354
 355 out_err:
 356         new_man = &bdev->man[bo->mem.mem_type];
 357         if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 358                 ttm_tt_destroy(bo->ttm);
 359                 bo->ttm = NULL;
 360         }
 361
 362         return ret;
 363 }
 364
 365 /**
 366  * Call bo::reserved.
 367  * Will release GPU memory type usage on destruction.
 368  * This is the place to put in driver specific hooks to release
 369  * driver private resources.
 370  * Will release the bo::reserved lock.
 371  */
 372
 373 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 374 {
 375         if (bo->bdev->driver->move_notify)
 376                 bo->bdev->driver->move_notify(bo, false, NULL);
 377
 378         ttm_tt_destroy(bo->ttm);
 379         bo->ttm = NULL;
 380         ttm_bo_mem_put(bo, &bo->mem);
 381 }
 382
 383 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
 384 {
 385         int r;
 386
 387         if (bo->base.resv == &bo->base._resv)
 388                 return 0;
 389
 390         BUG_ON(!dma_resv_trylock(&bo->base._resv));
 391
 392         r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv);
 393         dma_resv_unlock(&bo->base._resv);
 394         if (r)
 395                 return r;
 396
 397         if (bo->type != ttm_bo_type_sg) {
 398                 /* This works because the BO is about to be destroyed and nobody
 399                  * reference it any more. The only tricky case is the trylock on
 400                  * the resv object while holding the lru_lock.
 401                  */
 402                 spin_lock(&ttm_bo_glob.lru_lock);
 403                 bo->base.resv = &bo->base._resv;
 404                 spin_unlock(&ttm_bo_glob.lru_lock);
 405         }
 406
 407         return r;
 408 }
 409
 410 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
 411 {
 412         struct dma_resv *resv = &bo->base._resv;
 413         struct dma_resv_list *fobj;
 414         struct dma_fence *fence;
 415         int i;
 416
 417         rcu_read_lock();
 418         fobj = rcu_dereference(resv->fence);
 419         fence = rcu_dereference(resv->fence_excl);
 420         if (fence && !fence->ops->signaled)
 421                 dma_fence_enable_sw_signaling(fence);
 422
 423         for (i = 0; fobj && i < fobj->shared_count; ++i) {
 424                 fence = rcu_dereference(fobj->shared[i]);
 425
 426                 if (!fence->ops->signaled)
 427                         dma_fence_enable_sw_signaling(fence);
 428         }
 429         rcu_read_unlock();
 430 }
 431
 432 /**
 433  * function ttm_bo_cleanup_refs
 434  * If bo idle, remove from lru lists, and unref.
 435  * If not idle, block if possible.
 436  *
 437  * Must be called with lru_lock and reservation held, this function
 438  * will drop the lru lock and optionally the reservation lock before returning.
 439  *
 440  * @interruptible         Any sleeps should occur interruptibly.
 441  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
 442  * @unlock_resv           Unlock the reservation lock as well.
 443  */
 444
 445 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
 446                                bool interruptible, bool no_wait_gpu,
 447                                bool unlock_resv)
 448 {
 449         struct dma_resv *resv = &bo->base._resv;
 450         int ret;
 451
 452         if (dma_resv_test_signaled_rcu(resv, true))
 453                 ret = 0;
 454         else
 455                 ret = -EBUSY;
 456
 457         if (ret && !no_wait_gpu) {
 458                 long lret;
 459
 460                 if (unlock_resv)
 461                         dma_resv_unlock(bo->base.resv);
 462                 spin_unlock(&ttm_bo_glob.lru_lock);
 463
 464                 lret = dma_resv_wait_timeout_rcu(resv, true, interruptible,
 465                                                  30 * HZ);
 466
 467                 if (lret < 0)
 468                         return lret;
 469                 else if (lret == 0)
 470                         return -EBUSY;
 471
 472                 spin_lock(&ttm_bo_glob.lru_lock);
 473                 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) {
 474                         /*
 475                          * We raced, and lost, someone else holds the reservation now,
 476                          * and is probably busy in ttm_bo_cleanup_memtype_use.
 477                          *
 478                          * Even if it's not the case, because we finished waiting any
 479                          * delayed destruction would succeed, so just return success
 480                          * here.
 481                          */
 482                         spin_unlock(&ttm_bo_glob.lru_lock);
 483                         return 0;
 484                 }
 485                 ret = 0;
 486         }
 487
 488         if (ret || unlikely(list_empty(&bo->ddestroy))) {
 489                 if (unlock_resv)
 490                         dma_resv_unlock(bo->base.resv);
 491                 spin_unlock(&ttm_bo_glob.lru_lock);
 492                 return ret;
 493         }
 494
 495         ttm_bo_del_from_lru(bo);
 496         list_del_init(&bo->ddestroy);
 497         spin_unlock(&ttm_bo_glob.lru_lock);
 498         ttm_bo_cleanup_memtype_use(bo);
 499
 500         if (unlock_resv)
 501                 dma_resv_unlock(bo->base.resv);
 502
 503         ttm_bo_put(bo);
 504
 505         return 0;
 506 }
 507
 508 /**
 509  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 510  * encountered buffers.
 511  */
 512 static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 513 {
 514         struct ttm_bo_global *glob = &ttm_bo_glob;
 515         struct list_head removed;
 516         bool empty;
 517
 518         INIT_LIST_HEAD(&removed);
 519
 520         spin_lock(&glob->lru_lock);
 521         while (!list_empty(&bdev->ddestroy)) {
 522                 struct ttm_buffer_object *bo;
 523
 524                 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
 525                                       ddestroy);
 526                 list_move_tail(&bo->ddestroy, &removed);
 527                 if (!ttm_bo_get_unless_zero(bo))
 528                         continue;
 529
 530                 if (remove_all || bo->base.resv != &bo->base._resv) {
 531                         spin_unlock(&glob->lru_lock);
 532                         dma_resv_lock(bo->base.resv, NULL);
 533
 534                         spin_lock(&glob->lru_lock);
 535                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 536
 537                 } else if (dma_resv_trylock(bo->base.resv)) {
 538                         ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 539                 } else {
 540                         spin_unlock(&glob->lru_lock);
 541                 }
 542
 543                 ttm_bo_put(bo);
 544                 spin_lock(&glob->lru_lock);
 545         }
 546         list_splice_tail(&removed, &bdev->ddestroy);
 547         empty = list_empty(&bdev->ddestroy);
 548         spin_unlock(&glob->lru_lock);
 549
 550         return empty;
 551 }
 552
 553 static void ttm_bo_delayed_workqueue(struct work_struct *work)
 554 {
 555         struct ttm_bo_device *bdev =
 556             container_of(work, struct ttm_bo_device, wq.work);
 557
 558         if (!ttm_bo_delayed_delete(bdev, false))
 559                 schedule_delayed_work(&bdev->wq,
 560                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 561 }
 562
 563 static void ttm_bo_release(struct kref *kref)
 564 {
 565         struct ttm_buffer_object *bo =
 566             container_of(kref, struct ttm_buffer_object, kref);
 567         struct ttm_bo_device *bdev = bo->bdev;
 568         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 569         size_t acc_size = bo->acc_size;
 570         int ret;
 571
 572         if (!bo->deleted) {
 573                 ret = ttm_bo_individualize_resv(bo);
 574                 if (ret) {
 575                         /* Last resort, if we fail to allocate memory for the
 576                          * fences block for the BO to become idle
 577                          */
 578                         dma_resv_wait_timeout_rcu(bo->base.resv, true, false,
 579                                                   30 * HZ);
 580                 }
 581
 582                 if (bo->bdev->driver->release_notify)
 583                         bo->bdev->driver->release_notify(bo);
 584
 585                 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node);
 586                 ttm_mem_io_lock(man, false);
 587                 ttm_mem_io_free_vm(bo);
 588                 ttm_mem_io_unlock(man);
 589         }
 590
 591         if (!dma_resv_test_signaled_rcu(bo->base.resv, true) ||
 592             !dma_resv_trylock(bo->base.resv)) {
 593                 /* The BO is not idle, resurrect it for delayed destroy */
 594                 ttm_bo_flush_all_fences(bo);
 595                 bo->deleted = true;
 596
 597                 spin_lock(&ttm_bo_glob.lru_lock);
 598
 599                 /*
 600                  * Make NO_EVICT bos immediately available to
 601                  * shrinkers, now that they are queued for
 602                  * destruction.
 603                  */
 604                 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
 605                         bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
 606                         ttm_bo_del_from_lru(bo);
 607                         ttm_bo_add_mem_to_lru(bo, &bo->mem);
 608                 }
 609
 610                 kref_init(&bo->kref);
 611                 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 612                 spin_unlock(&ttm_bo_glob.lru_lock);
 613
 614                 schedule_delayed_work(&bdev->wq,
 615                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 616                 return;
 617         }
 618
 619         spin_lock(&ttm_bo_glob.lru_lock);
 620         ttm_bo_del_from_lru(bo);
 621         list_del(&bo->ddestroy);
 622         spin_unlock(&ttm_bo_glob.lru_lock);
 623
 624         ttm_bo_cleanup_memtype_use(bo);
 625         dma_resv_unlock(bo->base.resv);
 626
 627         BUG_ON(bo->mem.mm_node != NULL);
 628         atomic_dec(&ttm_bo_glob.bo_count);
 629         dma_fence_put(bo->moving);
 630         if (!ttm_bo_uses_embedded_gem_object(bo))
 631                 dma_resv_fini(&bo->base._resv);
 632         bo->destroy(bo);
 633         ttm_mem_global_free(&ttm_mem_glob, acc_size);
 634 }
 635
 636 void ttm_bo_put(struct ttm_buffer_object *bo)
 637 {
 638         kref_put(&bo->kref, ttm_bo_release);
 639 }
 640 EXPORT_SYMBOL(ttm_bo_put);
 641
 642 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 643 {
 644         return cancel_delayed_work_sync(&bdev->wq);
 645 }
 646 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 647
 648 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 649 {
 650         if (resched)
 651                 schedule_delayed_work(&bdev->wq,
 652                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 653 }
 654 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 655
 656 static int ttm_bo_evict(struct ttm_buffer_object *bo,
 657                         struct ttm_operation_ctx *ctx)
 658 {
 659         struct ttm_bo_device *bdev = bo->bdev;
 660         struct ttm_mem_reg evict_mem;
 661         struct ttm_placement placement;
 662         int ret = 0;
 663
 664         dma_resv_assert_held(bo->base.resv);
 665
 666         placement.num_placement = 0;
 667         placement.num_busy_placement = 0;
 668         bdev->driver->evict_flags(bo, &placement);
 669
 670         if (!placement.num_placement && !placement.num_busy_placement) {
 671                 ret = ttm_bo_pipeline_gutting(bo);
 672                 if (ret)
 673                         return ret;
 674
 675                 return ttm_tt_create(bo, false);
 676         }
 677
 678         evict_mem = bo->mem;
 679         evict_mem.mm_node = NULL;
 680         evict_mem.bus.io_reserved_vm = false;
 681         evict_mem.bus.io_reserved_count = 0;
 682
 683         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 684         if (ret) {
 685                 if (ret != -ERESTARTSYS) {
 686                         pr_err("Failed to find memory space for buffer 0x%p eviction\n",
 687                                bo);
 688                         ttm_bo_mem_space_debug(bo, &placement);
 689                 }
 690                 goto out;
 691         }
 692
 693         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
 694         if (unlikely(ret)) {
 695                 if (ret != -ERESTARTSYS)
 696                         pr_err("Buffer eviction failed\n");
 697                 ttm_bo_mem_put(bo, &evict_mem);
 698                 goto out;
 699         }
 700         bo->evicted = true;
 701 out:
 702         return ret;
 703 }
 704
 705 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 706                               const struct ttm_place *place)
 707 {
 708         /* Don't evict this BO if it's outside of the
 709          * requested placement range
 710          */
 711         if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
 712             (place->lpfn && place->lpfn <= bo->mem.start))
 713                 return false;
 714
 715         return true;
 716 }
 717 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 718
 719 /**
 720  * Check the target bo is allowable to be evicted or swapout, including cases:
 721  *
 722  * a. if share same reservation object with ctx->resv, have assumption
 723  * reservation objects should already be locked, so not lock again and
 724  * return true directly when either the opreation allow_reserved_eviction
 725  * or the target bo already is in delayed free list;
 726  *
 727  * b. Otherwise, trylock it.
 728  */
 729 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
 730                         struct ttm_operation_ctx *ctx, bool *locked, bool *busy)
 731 {
 732         bool ret = false;
 733
 734         if (bo->base.resv == ctx->resv) {
 735                 dma_resv_assert_held(bo->base.resv);
 736                 if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT)
 737                         ret = true;
 738                 *locked = false;
 739                 if (busy)
 740                         *busy = false;
 741         } else {
 742                 ret = dma_resv_trylock(bo->base.resv);
 743                 *locked = ret;
 744                 if (busy)
 745                         *busy = !ret;
 746         }
 747
 748         return ret;
 749 }
 750
 751 /**
 752  * ttm_mem_evict_wait_busy - wait for a busy BO to become available
 753  *
 754  * @busy_bo: BO which couldn't be locked with trylock
 755  * @ctx: operation context
 756  * @ticket: acquire ticket
 757  *
 758  * Try to lock a busy buffer object to avoid failing eviction.
 759  */
 760 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo,
 761                                    struct ttm_operation_ctx *ctx,
 762                                    struct ww_acquire_ctx *ticket)
 763 {
 764         int r;
 765
 766         if (!busy_bo || !ticket)
 767                 return -EBUSY;
 768
 769         if (ctx->interruptible)
 770                 r = dma_resv_lock_interruptible(busy_bo->base.resv,
 771                                                           ticket);
 772         else
 773                 r = dma_resv_lock(busy_bo->base.resv, ticket);
 774
 775         /*
 776          * TODO: It would be better to keep the BO locked until allocation is at
 777          * least tried one more time, but that would mean a much larger rework
 778          * of TTM.
 779          */
 780         if (!r)
 781                 dma_resv_unlock(busy_bo->base.resv);
 782
 783         return r == -EDEADLK ? -EBUSY : r;
 784 }
 785
 786 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 787                                uint32_t mem_type,
 788                                const struct ttm_place *place,
 789                                struct ttm_operation_ctx *ctx,
 790                                struct ww_acquire_ctx *ticket)
 791 {
 792         struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
 793         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 794         bool locked = false;
 795         unsigned i;
 796         int ret;
 797
 798         spin_lock(&ttm_bo_glob.lru_lock);
 799         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 800                 list_for_each_entry(bo, &man->lru[i], lru) {
 801                         bool busy;
 802
 803                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
 804                                                             &busy)) {
 805                                 if (busy && !busy_bo && ticket !=
 806                                     dma_resv_locking_ctx(bo->base.resv))
 807                                         busy_bo = bo;
 808                                 continue;
 809                         }
 810
 811                         if (place && !bdev->driver->eviction_valuable(bo,
 812                                                                       place)) {
 813                                 if (locked)
 814                                         dma_resv_unlock(bo->base.resv);
 815                                 continue;
 816                         }
 817                         if (!ttm_bo_get_unless_zero(bo)) {
 818                                 if (locked)
 819                                         dma_resv_unlock(bo->base.resv);
 820                                 continue;
 821                         }
 822                         break;
 823                 }
 824
 825                 /* If the inner loop terminated early, we have our candidate */
 826                 if (&bo->lru != &man->lru[i])
 827                         break;
 828
 829                 bo = NULL;
 830         }
 831
 832         if (!bo) {
 833                 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo))
 834                         busy_bo = NULL;
 835                 spin_unlock(&ttm_bo_glob.lru_lock);
 836                 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket);
 837                 if (busy_bo)
 838                         ttm_bo_put(busy_bo);
 839                 return ret;
 840         }
 841
 842         if (bo->deleted) {
 843                 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
 844                                           ctx->no_wait_gpu, locked);
 845                 ttm_bo_put(bo);
 846                 return ret;
 847         }
 848
 849         spin_unlock(&ttm_bo_glob.lru_lock);
 850
 851         ret = ttm_bo_evict(bo, ctx);
 852         if (locked)
 853                 ttm_bo_unreserve(bo);
 854
 855         ttm_bo_put(bo);
 856         return ret;
 857 }
 858
 859 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 860 {
 861         struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
 862
 863         if (mem->mm_node)
 864                 (*man->func->put_node)(man, mem);
 865 }
 866 EXPORT_SYMBOL(ttm_bo_mem_put);
 867
 868 /**
 869  * Add the last move fence to the BO and reserve a new shared slot.
 870  */
 871 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
 872                                  struct ttm_mem_type_manager *man,
 873                                  struct ttm_mem_reg *mem,
 874                                  bool no_wait_gpu)
 875 {
 876         struct dma_fence *fence;
 877         int ret;
 878
 879         spin_lock(&man->move_lock);
 880         fence = dma_fence_get(man->move);
 881         spin_unlock(&man->move_lock);
 882
 883         if (!fence)
 884                 return 0;
 885
 886         if (no_wait_gpu)
 887                 return -EBUSY;
 888
 889         dma_resv_add_shared_fence(bo->base.resv, fence);
 890
 891         ret = dma_resv_reserve_shared(bo->base.resv, 1);
 892         if (unlikely(ret)) {
 893                 dma_fence_put(fence);
 894                 return ret;
 895         }
 896
 897         dma_fence_put(bo->moving);
 898         bo->moving = fence;
 899         return 0;
 900 }
 901
 902 /**
 903  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 904  * space, or we've evicted everything and there isn't enough space.
 905  */
 906 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 907                                   const struct ttm_place *place,
 908                                   struct ttm_mem_reg *mem,
 909                                   struct ttm_operation_ctx *ctx)
 910 {
 911         struct ttm_bo_device *bdev = bo->bdev;
 912         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 913         struct ww_acquire_ctx *ticket;
 914         int ret;
 915
 916         ticket = dma_resv_locking_ctx(bo->base.resv);
 917         do {
 918                 ret = (*man->func->get_node)(man, bo, place, mem);
 919                 if (unlikely(ret != 0))
 920                         return ret;
 921                 if (mem->mm_node)
 922                         break;
 923                 ret = ttm_mem_evict_first(bdev, mem->mem_type, place, ctx,
 924                                           ticket);
 925                 if (unlikely(ret != 0))
 926                         return ret;
 927         } while (1);
 928
 929         return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
 930 }
 931
 932 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
 933                                       uint32_t cur_placement,
 934                                       uint32_t proposed_placement)
 935 {
 936         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
 937         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
 938
 939         /**
 940          * Keep current caching if possible.
 941          */
 942
 943         if ((cur_placement & caching) != 0)
 944                 result |= (cur_placement & caching);
 945         else if ((man->default_caching & caching) != 0)
 946                 result |= man->default_caching;
 947         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
 948                 result |= TTM_PL_FLAG_CACHED;
 949         else if ((TTM_PL_FLAG_WC & caching) != 0)
 950                 result |= TTM_PL_FLAG_WC;
 951         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
 952                 result |= TTM_PL_FLAG_UNCACHED;
 953
 954         return result;
 955 }
 956
 957 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 958                                  uint32_t mem_type,
 959                                  const struct ttm_place *place,
 960                                  uint32_t *masked_placement)
 961 {
 962         uint32_t cur_flags = ttm_bo_type_flags(mem_type);
 963
 964         if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
 965                 return false;
 966
 967         if ((place->flags & man->available_caching) == 0)
 968                 return false;
 969
 970         cur_flags |= (place->flags & man->available_caching);
 971
 972         *masked_placement = cur_flags;
 973         return true;
 974 }
 975
 976 /**
 977  * ttm_bo_mem_placement - check if placement is compatible
 978  * @bo: BO to find memory for
 979  * @place: where to search
 980  * @mem: the memory object to fill in
 981  * @ctx: operation context
 982  *
 983  * Check if placement is compatible and fill in mem structure.
 984  * Returns -EBUSY if placement won't work or negative error code.
 985  * 0 when placement can be used.
 986  */
 987 static int ttm_bo_mem_placement(struct ttm_buffer_object *bo,
 988                                 const struct ttm_place *place,
 989                                 struct ttm_mem_reg *mem,
 990                                 struct ttm_operation_ctx *ctx)
 991 {
 992         struct ttm_bo_device *bdev = bo->bdev;
 993         uint32_t mem_type = TTM_PL_SYSTEM;
 994         struct ttm_mem_type_manager *man;
 995         uint32_t cur_flags = 0;
 996         int ret;
 997
 998         ret = ttm_mem_type_from_place(place, &mem_type);
 999         if (ret)
1000                 return ret;
1001
1002         man = &bdev->man[mem_type];
1003         if (!man->has_type || !man->use_type)
1004                 return -EBUSY;
1005
1006         if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
1007                 return -EBUSY;
1008
1009         cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags);
1010         /*
1011          * Use the access and other non-mapping-related flag bits from
1012          * the memory placement flags to the current flags
1013          */
1014         ttm_flag_masked(&cur_flags, place->flags, ~TTM_PL_MASK_MEMTYPE);
1015
1016         mem->mem_type = mem_type;
1017         mem->placement = cur_flags;
1018
1019         spin_lock(&ttm_bo_glob.lru_lock);
1020         ttm_bo_del_from_lru(bo);
1021         ttm_bo_add_mem_to_lru(bo, mem);
1022         spin_unlock(&ttm_bo_glob.lru_lock);
1023
1024         return 0;
1025 }
1026
1027 /**
1028  * Creates space for memory region @mem according to its type.
1029  *
1030  * This function first searches for free space in compatible memory types in
1031  * the priority order defined by the driver.  If free space isn't found, then
1032  * ttm_bo_mem_force_space is attempted in priority order to evict and find
1033  * space.
1034  */
1035 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
1036                         struct ttm_placement *placement,
1037                         struct ttm_mem_reg *mem,
1038                         struct ttm_operation_ctx *ctx)
1039 {
1040         struct ttm_bo_device *bdev = bo->bdev;
1041         bool type_found = false;
1042         int i, ret;
1043
1044         ret = dma_resv_reserve_shared(bo->base.resv, 1);
1045         if (unlikely(ret))
1046                 return ret;
1047
1048         mem->mm_node = NULL;
1049         for (i = 0; i < placement->num_placement; ++i) {
1050                 const struct ttm_place *place = &placement->placement[i];
1051                 struct ttm_mem_type_manager *man;
1052
1053                 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
1054                 if (ret == -EBUSY)
1055                         continue;
1056                 if (ret)
1057                         goto error;
1058
1059                 type_found = true;
1060                 mem->mm_node = NULL;
1061                 if (mem->mem_type == TTM_PL_SYSTEM)
1062                         return 0;
1063
1064                 man = &bdev->man[mem->mem_type];
1065                 ret = (*man->func->get_node)(man, bo, place, mem);
1066                 if (unlikely(ret))
1067                         goto error;
1068
1069                 if (!mem->mm_node)
1070                         continue;
1071
1072                 ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu);
1073                 if (unlikely(ret)) {
1074                         (*man->func->put_node)(man, mem);
1075                         if (ret == -EBUSY)
1076                                 continue;
1077
1078                         goto error;
1079                 }
1080                 return 0;
1081         }
1082
1083         for (i = 0; i < placement->num_busy_placement; ++i) {
1084                 const struct ttm_place *place = &placement->busy_placement[i];
1085
1086                 ret = ttm_bo_mem_placement(bo, place, mem, ctx);
1087                 if (ret == -EBUSY)
1088                         continue;
1089                 if (ret)
1090                         goto error;
1091
1092                 type_found = true;
1093                 mem->mm_node = NULL;
1094                 if (mem->mem_type == TTM_PL_SYSTEM)
1095                         return 0;
1096
1097                 ret = ttm_bo_mem_force_space(bo, place, mem, ctx);
1098                 if (ret == 0 && mem->mm_node)
1099                         return 0;
1100
1101                 if (ret && ret != -EBUSY)
1102                         goto error;
1103         }
1104
1105         ret = -ENOMEM;
1106         if (!type_found) {
1107                 pr_err(TTM_PFX "No compatible memory type found\n");
1108                 ret = -EINVAL;
1109         }
1110
1111 error:
1112         if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) {
1113                 spin_lock(&ttm_bo_glob.lru_lock);
1114                 ttm_bo_move_to_lru_tail(bo, NULL);
1115                 spin_unlock(&ttm_bo_glob.lru_lock);
1116         }
1117
1118         return ret;
1119 }
1120 EXPORT_SYMBOL(ttm_bo_mem_space);
1121
1122 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1123                               struct ttm_placement *placement,
1124                               struct ttm_operation_ctx *ctx)
1125 {
1126         int ret = 0;
1127         struct ttm_mem_reg mem;
1128
1129         dma_resv_assert_held(bo->base.resv);
1130
1131         mem.num_pages = bo->num_pages;
1132         mem.size = mem.num_pages << PAGE_SHIFT;
1133         mem.page_alignment = bo->mem.page_alignment;
1134         mem.bus.io_reserved_vm = false;
1135         mem.bus.io_reserved_count = 0;
1136         /*
1137          * Determine where to move the buffer.
1138          */
1139         ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1140         if (ret)
1141                 goto out_unlock;
1142         ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1143 out_unlock:
1144         if (ret && mem.mm_node)
1145                 ttm_bo_mem_put(bo, &mem);
1146         return ret;
1147 }
1148
1149 static bool ttm_bo_places_compat(const struct ttm_place *places,
1150                                  unsigned num_placement,
1151                                  struct ttm_mem_reg *mem,
1152                                  uint32_t *new_flags)
1153 {
1154         unsigned i;
1155
1156         for (i = 0; i < num_placement; i++) {
1157                 const struct ttm_place *heap = &places[i];
1158
1159                 if (mem->mm_node && (mem->start < heap->fpfn ||
1160                      (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1161                         continue;
1162
1163                 *new_flags = heap->flags;
1164                 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1165                     (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
1166                     (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
1167                      (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1168                         return true;
1169         }
1170         return false;
1171 }
1172
1173 bool ttm_bo_mem_compat(struct ttm_placement *placement,
1174                        struct ttm_mem_reg *mem,
1175                        uint32_t *new_flags)
1176 {
1177         if (ttm_bo_places_compat(placement->placement, placement->num_placement,
1178                                  mem, new_flags))
1179                 return true;
1180
1181         if ((placement->busy_placement != placement->placement ||
1182              placement->num_busy_placement > placement->num_placement) &&
1183             ttm_bo_places_compat(placement->busy_placement,
1184                                  placement->num_busy_placement,
1185                                  mem, new_flags))
1186                 return true;
1187
1188         return false;
1189 }
1190 EXPORT_SYMBOL(ttm_bo_mem_compat);
1191
1192 int ttm_bo_validate(struct ttm_buffer_object *bo,
1193                     struct ttm_placement *placement,
1194                     struct ttm_operation_ctx *ctx)
1195 {
1196         int ret;
1197         uint32_t new_flags;
1198
1199         dma_resv_assert_held(bo->base.resv);
1200
1201         /*
1202          * Remove the backing store if no placement is given.
1203          */
1204         if (!placement->num_placement && !placement->num_busy_placement) {
1205                 ret = ttm_bo_pipeline_gutting(bo);
1206                 if (ret)
1207                         return ret;
1208
1209                 return ttm_tt_create(bo, false);
1210         }
1211
1212         /*
1213          * Check whether we need to move buffer.
1214          */
1215         if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1216                 ret = ttm_bo_move_buffer(bo, placement, ctx);
1217                 if (ret)
1218                         return ret;
1219         } else {
1220                 /*
1221                  * Use the access and other non-mapping-related flag bits from
1222                  * the compatible memory placement flags to the active flags
1223                  */
1224                 ttm_flag_masked(&bo->mem.placement, new_flags,
1225                                 ~TTM_PL_MASK_MEMTYPE);
1226         }
1227         /*
1228          * We might need to add a TTM.
1229          */
1230         if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1231                 ret = ttm_tt_create(bo, true);
1232                 if (ret)
1233                         return ret;
1234         }
1235         return 0;
1236 }
1237 EXPORT_SYMBOL(ttm_bo_validate);
1238
1239 int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
1240                          struct ttm_buffer_object *bo,
1241                          unsigned long size,
1242                          enum ttm_bo_type type,
1243                          struct ttm_placement *placement,
1244                          uint32_t page_alignment,
1245                          struct ttm_operation_ctx *ctx,
1246                          size_t acc_size,
1247                          struct sg_table *sg,
1248                          struct dma_resv *resv,
1249                          void (*destroy) (struct ttm_buffer_object *))
1250 {
1251         struct ttm_mem_global *mem_glob = &ttm_mem_glob;
1252         int ret = 0;
1253         unsigned long num_pages;
1254         bool locked;
1255
1256         ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
1257         if (ret) {
1258                 pr_err("Out of kernel memory\n");
1259                 if (destroy)
1260                         (*destroy)(bo);
1261                 else
1262                         kfree(bo);
1263                 return -ENOMEM;
1264         }
1265
1266         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1267         if (num_pages == 0) {
1268                 pr_err("Illegal buffer object size\n");
1269                 if (destroy)
1270                         (*destroy)(bo);
1271                 else
1272                         kfree(bo);
1273                 ttm_mem_global_free(mem_glob, acc_size);
1274                 return -EINVAL;
1275         }
1276         bo->destroy = destroy ? destroy : ttm_bo_default_destroy;
1277
1278         kref_init(&bo->kref);
1279         INIT_LIST_HEAD(&bo->lru);
1280         INIT_LIST_HEAD(&bo->ddestroy);
1281         INIT_LIST_HEAD(&bo->swap);
1282         INIT_LIST_HEAD(&bo->io_reserve_lru);
1283         bo->bdev = bdev;
1284         bo->type = type;
1285         bo->num_pages = num_pages;
1286         bo->mem.size = num_pages << PAGE_SHIFT;
1287         bo->mem.mem_type = TTM_PL_SYSTEM;
1288         bo->mem.num_pages = bo->num_pages;
1289         bo->mem.mm_node = NULL;
1290         bo->mem.page_alignment = page_alignment;
1291         bo->mem.bus.io_reserved_vm = false;
1292         bo->mem.bus.io_reserved_count = 0;
1293         bo->moving = NULL;
1294         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1295         bo->acc_size = acc_size;
1296         bo->sg = sg;
1297         if (resv) {
1298                 bo->base.resv = resv;
1299                 dma_resv_assert_held(bo->base.resv);
1300         } else {
1301                 bo->base.resv = &bo->base._resv;
1302         }
1303         if (!ttm_bo_uses_embedded_gem_object(bo)) {
1304                 /*
1305                  * bo.gem is not initialized, so we have to setup the
1306                  * struct elements we want use regardless.
1307                  */
1308                 dma_resv_init(&bo->base._resv);
1309                 drm_vma_node_reset(&bo->base.vma_node);
1310         }
1311         atomic_inc(&ttm_bo_glob.bo_count);
1312
1313         /*
1314          * For ttm_bo_type_device buffers, allocate
1315          * address space from the device.
1316          */
1317         if (bo->type == ttm_bo_type_device ||
1318             bo->type == ttm_bo_type_sg)
1319                 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
1320                                          bo->mem.num_pages);
1321
1322         /* passed reservation objects should already be locked,
1323          * since otherwise lockdep will be angered in radeon.
1324          */
1325         if (!resv) {
1326                 locked = dma_resv_trylock(bo->base.resv);
1327                 WARN_ON(!locked);
1328         }
1329
1330         if (likely(!ret))
1331                 ret = ttm_bo_validate(bo, placement, ctx);
1332
1333         if (unlikely(ret)) {
1334                 if (!resv)
1335                         ttm_bo_unreserve(bo);
1336
1337                 ttm_bo_put(bo);
1338                 return ret;
1339         }
1340
1341         spin_lock(&ttm_bo_glob.lru_lock);
1342         ttm_bo_move_to_lru_tail(bo, NULL);
1343         spin_unlock(&ttm_bo_glob.lru_lock);
1344
1345         return ret;
1346 }
1347 EXPORT_SYMBOL(ttm_bo_init_reserved);
1348
1349 int ttm_bo_init(struct ttm_bo_device *bdev,
1350                 struct ttm_buffer_object *bo,
1351                 unsigned long size,
1352                 enum ttm_bo_type type,
1353                 struct ttm_placement *placement,
1354                 uint32_t page_alignment,
1355                 bool interruptible,
1356                 size_t acc_size,
1357                 struct sg_table *sg,
1358                 struct dma_resv *resv,
1359                 void (*destroy) (struct ttm_buffer_object *))
1360 {
1361         struct ttm_operation_ctx ctx = { interruptible, false };
1362         int ret;
1363
1364         ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1365                                    page_alignment, &ctx, acc_size,
1366                                    sg, resv, destroy);
1367         if (ret)
1368                 return ret;
1369
1370         if (!resv)
1371                 ttm_bo_unreserve(bo);
1372
1373         return 0;
1374 }
1375 EXPORT_SYMBOL(ttm_bo_init);
1376
1377 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1378                        unsigned long bo_size,
1379                        unsigned struct_size)
1380 {
1381         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1382         size_t size = 0;
1383
1384         size += ttm_round_pot(struct_size);
1385         size += ttm_round_pot(npages * sizeof(void *));
1386         size += ttm_round_pot(sizeof(struct ttm_tt));
1387         return size;
1388 }
1389 EXPORT_SYMBOL(ttm_bo_acc_size);
1390
1391 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1392                            unsigned long bo_size,
1393                            unsigned struct_size)
1394 {
1395         unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1396         size_t size = 0;
1397
1398         size += ttm_round_pot(struct_size);
1399         size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1400         size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1401         return size;
1402 }
1403 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1404
1405 int ttm_bo_create(struct ttm_bo_device *bdev,
1406                         unsigned long size,
1407                         enum ttm_bo_type type,
1408                         struct ttm_placement *placement,
1409                         uint32_t page_alignment,
1410                         bool interruptible,
1411                         struct ttm_buffer_object **p_bo)
1412 {
1413         struct ttm_buffer_object *bo;
1414         size_t acc_size;
1415         int ret;
1416
1417         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1418         if (unlikely(bo == NULL))
1419                 return -ENOMEM;
1420
1421         acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1422         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1423                           interruptible, acc_size,
1424                           NULL, NULL, NULL);
1425         if (likely(ret == 0))
1426                 *p_bo = bo;
1427
1428         return ret;
1429 }
1430 EXPORT_SYMBOL(ttm_bo_create);
1431
1432 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1433                                    unsigned mem_type)
1434 {
1435         struct ttm_operation_ctx ctx = {
1436                 .interruptible = false,
1437                 .no_wait_gpu = false,
1438                 .flags = TTM_OPT_FLAG_FORCE_ALLOC
1439         };
1440         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1441         struct ttm_bo_global *glob = &ttm_bo_glob;
1442         struct dma_fence *fence;
1443         int ret;
1444         unsigned i;
1445
1446         /*
1447          * Can't use standard list traversal since we're unlocking.
1448          */
1449
1450         spin_lock(&glob->lru_lock);
1451         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1452                 while (!list_empty(&man->lru[i])) {
1453                         spin_unlock(&glob->lru_lock);
1454                         ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx,
1455                                                   NULL);
1456                         if (ret)
1457                                 return ret;
1458                         spin_lock(&glob->lru_lock);
1459                 }
1460         }
1461         spin_unlock(&glob->lru_lock);
1462
1463         spin_lock(&man->move_lock);
1464         fence = dma_fence_get(man->move);
1465         spin_unlock(&man->move_lock);
1466
1467         if (fence) {
1468                 ret = dma_fence_wait(fence, false);
1469                 dma_fence_put(fence);
1470                 if (ret)
1471                         return ret;
1472         }
1473
1474         return 0;
1475 }
1476
1477 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1478 {
1479         struct ttm_mem_type_manager *man;
1480         int ret = -EINVAL;
1481
1482         if (mem_type >= TTM_NUM_MEM_TYPES) {
1483                 pr_err("Illegal memory type %d\n", mem_type);
1484                 return ret;
1485         }
1486         man = &bdev->man[mem_type];
1487
1488         if (!man->has_type) {
1489                 pr_err("Trying to take down uninitialized memory manager type %u\n",
1490                        mem_type);
1491                 return ret;
1492         }
1493
1494         man->use_type = false;
1495         man->has_type = false;
1496
1497         ret = 0;
1498         if (mem_type > 0) {
1499                 ret = ttm_bo_force_list_clean(bdev, mem_type);
1500                 if (ret) {
1501                         pr_err("Cleanup eviction failed\n");
1502                         return ret;
1503                 }
1504
1505                 ret = (*man->func->takedown)(man);
1506         }
1507
1508         dma_fence_put(man->move);
1509         man->move = NULL;
1510
1511         return ret;
1512 }
1513 EXPORT_SYMBOL(ttm_bo_clean_mm);
1514
1515 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1516 {
1517         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1518
1519         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1520                 pr_err("Illegal memory manager memory type %u\n", mem_type);
1521                 return -EINVAL;
1522         }
1523
1524         if (!man->has_type) {
1525                 pr_err("Memory type %u has not been initialized\n", mem_type);
1526                 return 0;
1527         }
1528
1529         return ttm_bo_force_list_clean(bdev, mem_type);
1530 }
1531 EXPORT_SYMBOL(ttm_bo_evict_mm);
1532
1533 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1534                         unsigned long p_size)
1535 {
1536         int ret;
1537         struct ttm_mem_type_manager *man;
1538         unsigned i;
1539
1540         BUG_ON(type >= TTM_NUM_MEM_TYPES);
1541         man = &bdev->man[type];
1542         BUG_ON(man->has_type);
1543         man->io_reserve_fastpath = true;
1544         man->use_io_reserve_lru = false;
1545         mutex_init(&man->io_reserve_mutex);
1546         spin_lock_init(&man->move_lock);
1547         INIT_LIST_HEAD(&man->io_reserve_lru);
1548
1549         ret = bdev->driver->init_mem_type(bdev, type, man);
1550         if (ret)
1551                 return ret;
1552         man->bdev = bdev;
1553
1554         if (type != TTM_PL_SYSTEM) {
1555                 ret = (*man->func->init)(man, p_size);
1556                 if (ret)
1557                         return ret;
1558         }
1559         man->has_type = true;
1560         man->use_type = true;
1561         man->size = p_size;
1562
1563         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1564                 INIT_LIST_HEAD(&man->lru[i]);
1565         man->move = NULL;
1566
1567         return 0;
1568 }
1569 EXPORT_SYMBOL(ttm_bo_init_mm);
1570
1571 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1572 {
1573         struct ttm_bo_global *glob =
1574                 container_of(kobj, struct ttm_bo_global, kobj);
1575
1576         __free_page(glob->dummy_read_page);
1577 }
1578
1579 static void ttm_bo_global_release(void)
1580 {
1581         struct ttm_bo_global *glob = &ttm_bo_glob;
1582
1583         mutex_lock(&ttm_global_mutex);
1584         if (--ttm_bo_glob_use_count > 0)
1585                 goto out;
1586
1587         kobject_del(&glob->kobj);
1588         kobject_put(&glob->kobj);
1589         ttm_mem_global_release(&ttm_mem_glob);
1590         memset(glob, 0, sizeof(*glob));
1591 out:
1592         mutex_unlock(&ttm_global_mutex);
1593 }
1594
1595 static int ttm_bo_global_init(void)
1596 {
1597         struct ttm_bo_global *glob = &ttm_bo_glob;
1598         int ret = 0;
1599         unsigned i;
1600
1601         mutex_lock(&ttm_global_mutex);
1602         if (++ttm_bo_glob_use_count > 1)
1603                 goto out;
1604
1605         ret = ttm_mem_global_init(&ttm_mem_glob);
1606         if (ret)
1607                 goto out;
1608
1609         spin_lock_init(&glob->lru_lock);
1610         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1611
1612         if (unlikely(glob->dummy_read_page == NULL)) {
1613                 ret = -ENOMEM;
1614                 goto out;
1615         }
1616
1617         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1618                 INIT_LIST_HEAD(&glob->swap_lru[i]);
1619         INIT_LIST_HEAD(&glob->device_list);
1620         atomic_set(&glob->bo_count, 0);
1621
1622         ret = kobject_init_and_add(
1623                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1624         if (unlikely(ret != 0))
1625                 kobject_put(&glob->kobj);
1626 out:
1627         mutex_unlock(&ttm_global_mutex);
1628         return ret;
1629 }
1630
1631 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1632 {
1633         struct ttm_bo_global *glob = &ttm_bo_glob;
1634         int ret = 0;
1635         unsigned i = TTM_NUM_MEM_TYPES;
1636         struct ttm_mem_type_manager *man;
1637
1638         while (i--) {
1639                 man = &bdev->man[i];
1640                 if (man->has_type) {
1641                         man->use_type = false;
1642                         if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1643                                 ret = -EBUSY;
1644                                 pr_err("DRM memory manager type %d is not clean\n",
1645                                        i);
1646                         }
1647                         man->has_type = false;
1648                 }
1649         }
1650
1651         mutex_lock(&ttm_global_mutex);
1652         list_del(&bdev->device_list);
1653         mutex_unlock(&ttm_global_mutex);
1654
1655         cancel_delayed_work_sync(&bdev->wq);
1656
1657         if (ttm_bo_delayed_delete(bdev, true))
1658                 pr_debug("Delayed destroy list was clean\n");
1659
1660         spin_lock(&glob->lru_lock);
1661         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
1662                 if (list_empty(&bdev->man[0].lru[0]))
1663                         pr_debug("Swap list %d was clean\n", i);
1664         spin_unlock(&glob->lru_lock);
1665
1666         if (!ret)
1667                 ttm_bo_global_release();
1668
1669         return ret;
1670 }
1671 EXPORT_SYMBOL(ttm_bo_device_release);
1672
1673 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1674                        struct ttm_bo_driver *driver,
1675                        struct address_space *mapping,
1676                        struct drm_vma_offset_manager *vma_manager,
1677                        bool need_dma32)
1678 {
1679         struct ttm_bo_global *glob = &ttm_bo_glob;
1680         int ret;
1681
1682         if (WARN_ON(vma_manager == NULL))
1683                 return -EINVAL;
1684
1685         ret = ttm_bo_global_init();
1686         if (ret)
1687                 return ret;
1688
1689         bdev->driver = driver;
1690
1691         memset(bdev->man, 0, sizeof(bdev->man));
1692
1693         /*
1694          * Initialize the system memory buffer type.
1695          * Other types need to be driver / IOCTL initialized.
1696          */
1697         ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1698         if (unlikely(ret != 0))
1699                 goto out_no_sys;
1700
1701         bdev->vma_manager = vma_manager;
1702         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1703         INIT_LIST_HEAD(&bdev->ddestroy);
1704         bdev->dev_mapping = mapping;
1705         bdev->need_dma32 = need_dma32;
1706         mutex_lock(&ttm_global_mutex);
1707         list_add_tail(&bdev->device_list, &glob->device_list);
1708         mutex_unlock(&ttm_global_mutex);
1709
1710         return 0;
1711 out_no_sys:
1712         ttm_bo_global_release();
1713         return ret;
1714 }
1715 EXPORT_SYMBOL(ttm_bo_device_init);
1716
1717 /*
1718  * buffer object vm functions.
1719  */
1720
1721 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1722 {
1723         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1724
1725         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1726                 if (mem->mem_type == TTM_PL_SYSTEM)
1727                         return false;
1728
1729                 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1730                         return false;
1731
1732                 if (mem->placement & TTM_PL_FLAG_CACHED)
1733                         return false;
1734         }
1735         return true;
1736 }
1737
1738 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1739 {
1740         struct ttm_bo_device *bdev = bo->bdev;
1741
1742         drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
1743         ttm_mem_io_free_vm(bo);
1744 }
1745
1746 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1747 {
1748         struct ttm_bo_device *bdev = bo->bdev;
1749         struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1750
1751         ttm_mem_io_lock(man, false);
1752         ttm_bo_unmap_virtual_locked(bo);
1753         ttm_mem_io_unlock(man);
1754 }
1755
1756
1757 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1758
1759 int ttm_bo_wait(struct ttm_buffer_object *bo,
1760                 bool interruptible, bool no_wait)
1761 {
1762         long timeout = 15 * HZ;
1763
1764         if (no_wait) {
1765                 if (dma_resv_test_signaled_rcu(bo->base.resv, true))
1766                         return 0;
1767                 else
1768                         return -EBUSY;
1769         }
1770
1771         timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true,
1772                                                       interruptible, timeout);
1773         if (timeout < 0)
1774                 return timeout;
1775
1776         if (timeout == 0)
1777                 return -EBUSY;
1778
1779         dma_resv_add_excl_fence(bo->base.resv, NULL);
1780         return 0;
1781 }
1782 EXPORT_SYMBOL(ttm_bo_wait);
1783
1784 /**
1785  * A buffer object shrink method that tries to swap out the first
1786  * buffer object on the bo_global::swap_lru list.
1787  */
1788 int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
1789 {
1790         struct ttm_buffer_object *bo;
1791         int ret = -EBUSY;
1792         bool locked;
1793         unsigned i;
1794
1795         spin_lock(&glob->lru_lock);
1796         for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
1797                 list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1798                         if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked,
1799                                                             NULL))
1800                                 continue;
1801
1802                         if (!ttm_bo_get_unless_zero(bo)) {
1803                                 if (locked)
1804                                         dma_resv_unlock(bo->base.resv);
1805                                 continue;
1806                         }
1807
1808                         ret = 0;
1809                         break;
1810                 }
1811                 if (!ret)
1812                         break;
1813         }
1814
1815         if (ret) {
1816                 spin_unlock(&glob->lru_lock);
1817                 return ret;
1818         }
1819
1820         if (bo->deleted) {
1821                 ret = ttm_bo_cleanup_refs(bo, false, false, locked);
1822                 ttm_bo_put(bo);
1823                 return ret;
1824         }
1825
1826         ttm_bo_del_from_lru(bo);
1827         spin_unlock(&glob->lru_lock);
1828
1829         /**
1830          * Move to system cached
1831          */
1832
1833         if (bo->mem.mem_type != TTM_PL_SYSTEM ||
1834             bo->ttm->caching_state != tt_cached) {
1835                 struct ttm_operation_ctx ctx = { false, false };
1836                 struct ttm_mem_reg evict_mem;
1837
1838                 evict_mem = bo->mem;
1839                 evict_mem.mm_node = NULL;
1840                 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1841                 evict_mem.mem_type = TTM_PL_SYSTEM;
1842
1843                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1844                 if (unlikely(ret != 0))
1845                         goto out;
1846         }
1847
1848         /**
1849          * Make sure BO is idle.
1850          */
1851
1852         ret = ttm_bo_wait(bo, false, false);
1853         if (unlikely(ret != 0))
1854                 goto out;
1855
1856         ttm_bo_unmap_virtual(bo);
1857
1858         /**
1859          * Swap out. Buffer will be swapped in again as soon as
1860          * anyone tries to access a ttm page.
1861          */
1862
1863         if (bo->bdev->driver->swap_notify)
1864                 bo->bdev->driver->swap_notify(bo);
1865
1866         ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1867 out:
1868
1869         /**
1870          *
1871          * Unreserve without putting on LRU to avoid swapping out an
1872          * already swapped buffer.
1873          */
1874         if (locked)
1875                 dma_resv_unlock(bo->base.resv);
1876         ttm_bo_put(bo);
1877         return ret;
1878 }
1879 EXPORT_SYMBOL(ttm_bo_swapout);
1880
1881 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1882 {
1883         struct ttm_operation_ctx ctx = {
1884                 .interruptible = false,
1885                 .no_wait_gpu = false
1886         };
1887
1888         while (ttm_bo_swapout(&ttm_bo_glob, &ctx) == 0);
1889 }
1890 EXPORT_SYMBOL(ttm_bo_swapout_all);