drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #include <drm/ttm/ttm_bo_driver.h>
  32 #include <drm/ttm/ttm_placement.h>
  33 #include <drm/drm_vma_manager.h>
  34 #include <linux/io.h>
  35 #include <linux/highmem.h>
  36 #include <linux/wait.h>
  37 #include <linux/slab.h>
  38 #include <linux/vmalloc.h>
  39 #include <linux/module.h>
  40 #include <linux/reservation.h>
  41
  42 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  43 {
  44         ttm_bo_mem_put(bo, &bo->mem);
  45 }
  46
  47 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  48                     bool evict,
  49                     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
  50 {
  51         struct ttm_tt *ttm = bo->ttm;
  52         struct ttm_mem_reg *old_mem = &bo->mem;
  53         int ret;
  54
  55         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  56                 ttm_bo_free_old_node(bo);
  57                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  58                                 TTM_PL_MASK_MEM);
  59                 old_mem->mem_type = TTM_PL_SYSTEM;
  60         }
  61
  62         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  63         if (unlikely(ret != 0))
  64                 return ret;
  65
  66         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  67                 ret = ttm_tt_bind(ttm, new_mem);
  68                 if (unlikely(ret != 0))
  69                         return ret;
  70         }
  71
  72         *old_mem = *new_mem;
  73         new_mem->mm_node = NULL;
  74
  75         return 0;
  76 }
  77 EXPORT_SYMBOL(ttm_bo_move_ttm);
  78
  79 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
  80 {
  81         if (likely(man->io_reserve_fastpath))
  82                 return 0;
  83
  84         if (interruptible)
  85                 return mutex_lock_interruptible(&man->io_reserve_mutex);
  86
  87         mutex_lock(&man->io_reserve_mutex);
  88         return 0;
  89 }
  90 EXPORT_SYMBOL(ttm_mem_io_lock);
  91
  92 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
  93 {
  94         if (likely(man->io_reserve_fastpath))
  95                 return;
  96
  97         mutex_unlock(&man->io_reserve_mutex);
  98 }
  99 EXPORT_SYMBOL(ttm_mem_io_unlock);
 100
 101 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
 102 {
 103         struct ttm_buffer_object *bo;
 104
 105         if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
 106                 return -EAGAIN;
 107
 108         bo = list_first_entry(&man->io_reserve_lru,
 109                               struct ttm_buffer_object,
 110                               io_reserve_lru);
 111         list_del_init(&bo->io_reserve_lru);
 112         ttm_bo_unmap_virtual_locked(bo);
 113
 114         return 0;
 115 }
 116
 117
 118 int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
 119                        struct ttm_mem_reg *mem)
 120 {
 121         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 122         int ret = 0;
 123
 124         if (!bdev->driver->io_mem_reserve)
 125                 return 0;
 126         if (likely(man->io_reserve_fastpath))
 127                 return bdev->driver->io_mem_reserve(bdev, mem);
 128
 129         if (bdev->driver->io_mem_reserve &&
 130             mem->bus.io_reserved_count++ == 0) {
 131 retry:
 132                 ret = bdev->driver->io_mem_reserve(bdev, mem);
 133                 if (ret == -EAGAIN) {
 134                         ret = ttm_mem_io_evict(man);
 135                         if (ret == 0)
 136                                 goto retry;
 137                 }
 138         }
 139         return ret;
 140 }
 141 EXPORT_SYMBOL(ttm_mem_io_reserve);
 142
 143 void ttm_mem_io_free(struct ttm_bo_device *bdev,
 144                      struct ttm_mem_reg *mem)
 145 {
 146         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 147
 148         if (likely(man->io_reserve_fastpath))
 149                 return;
 150
 151         if (bdev->driver->io_mem_reserve &&
 152             --mem->bus.io_reserved_count == 0 &&
 153             bdev->driver->io_mem_free)
 154                 bdev->driver->io_mem_free(bdev, mem);
 155
 156 }
 157 EXPORT_SYMBOL(ttm_mem_io_free);
 158
 159 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
 160 {
 161         struct ttm_mem_reg *mem = &bo->mem;
 162         int ret;
 163
 164         if (!mem->bus.io_reserved_vm) {
 165                 struct ttm_mem_type_manager *man =
 166                         &bo->bdev->man[mem->mem_type];
 167
 168                 ret = ttm_mem_io_reserve(bo->bdev, mem);
 169                 if (unlikely(ret != 0))
 170                         return ret;
 171                 mem->bus.io_reserved_vm = true;
 172                 if (man->use_io_reserve_lru)
 173                         list_add_tail(&bo->io_reserve_lru,
 174                                       &man->io_reserve_lru);
 175         }
 176         return 0;
 177 }
 178
 179 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
 180 {
 181         struct ttm_mem_reg *mem = &bo->mem;
 182
 183         if (mem->bus.io_reserved_vm) {
 184                 mem->bus.io_reserved_vm = false;
 185                 list_del_init(&bo->io_reserve_lru);
 186                 ttm_mem_io_free(bo->bdev, mem);
 187         }
 188 }
 189
 190 static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 191                         void **virtual)
 192 {
 193         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
 194         int ret;
 195         void *addr;
 196
 197         *virtual = NULL;
 198         (void) ttm_mem_io_lock(man, false);
 199         ret = ttm_mem_io_reserve(bdev, mem);
 200         ttm_mem_io_unlock(man);
 201         if (ret || !mem->bus.is_iomem)
 202                 return ret;
 203
 204         if (mem->bus.addr) {
 205                 addr = mem->bus.addr;
 206         } else {
 207                 if (mem->placement & TTM_PL_FLAG_WC)
 208                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 209                 else
 210                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 211                 if (!addr) {
 212                         (void) ttm_mem_io_lock(man, false);
 213                         ttm_mem_io_free(bdev, mem);
 214                         ttm_mem_io_unlock(man);
 215                         return -ENOMEM;
 216                 }
 217         }
 218         *virtual = addr;
 219         return 0;
 220 }
 221
 222 static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 223                          void *virtual)
 224 {
 225         struct ttm_mem_type_manager *man;
 226
 227         man = &bdev->man[mem->mem_type];
 228
 229         if (virtual && mem->bus.addr == NULL)
 230                 iounmap(virtual);
 231         (void) ttm_mem_io_lock(man, false);
 232         ttm_mem_io_free(bdev, mem);
 233         ttm_mem_io_unlock(man);
 234 }
 235
 236 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 237 {
 238         uint32_t *dstP =
 239             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 240         uint32_t *srcP =
 241             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 242
 243         int i;
 244         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 245                 iowrite32(ioread32(srcP++), dstP++);
 246         return 0;
 247 }
 248
 249 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 250                                 unsigned long page,
 251                                 pgprot_t prot)
 252 {
 253         struct page *d = ttm->pages[page];
 254         void *dst;
 255
 256         if (!d)
 257                 return -ENOMEM;
 258
 259         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 260
 261 #ifdef CONFIG_X86
 262         dst = kmap_atomic_prot(d, prot);
 263 #else
 264         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 265                 dst = vmap(&d, 1, 0, prot);
 266         else
 267                 dst = kmap(d);
 268 #endif
 269         if (!dst)
 270                 return -ENOMEM;
 271
 272         memcpy_fromio(dst, src, PAGE_SIZE);
 273
 274 #ifdef CONFIG_X86
 275         kunmap_atomic(dst);
 276 #else
 277         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 278                 vunmap(dst);
 279         else
 280                 kunmap(d);
 281 #endif
 282
 283         return 0;
 284 }
 285
 286 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 287                                 unsigned long page,
 288                                 pgprot_t prot)
 289 {
 290         struct page *s = ttm->pages[page];
 291         void *src;
 292
 293         if (!s)
 294                 return -ENOMEM;
 295
 296         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 297 #ifdef CONFIG_X86
 298         src = kmap_atomic_prot(s, prot);
 299 #else
 300         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 301                 src = vmap(&s, 1, 0, prot);
 302         else
 303                 src = kmap(s);
 304 #endif
 305         if (!src)
 306                 return -ENOMEM;
 307
 308         memcpy_toio(dst, src, PAGE_SIZE);
 309
 310 #ifdef CONFIG_X86
 311         kunmap_atomic(src);
 312 #else
 313         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 314                 vunmap(src);
 315         else
 316                 kunmap(s);
 317 #endif
 318
 319         return 0;
 320 }
 321
 322 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 323                        bool evict, bool interruptible,
 324                        bool no_wait_gpu,
 325                        struct ttm_mem_reg *new_mem)
 326 {
 327         struct ttm_bo_device *bdev = bo->bdev;
 328         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 329         struct ttm_tt *ttm = bo->ttm;
 330         struct ttm_mem_reg *old_mem = &bo->mem;
 331         struct ttm_mem_reg old_copy = *old_mem;
 332         void *old_iomap;
 333         void *new_iomap;
 334         int ret;
 335         unsigned long i;
 336         unsigned long page;
 337         unsigned long add = 0;
 338         int dir;
 339
 340         ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
 341         if (ret)
 342                 return ret;
 343
 344         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 345         if (ret)
 346                 return ret;
 347         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 348         if (ret)
 349                 goto out;
 350
 351         /*
 352          * Single TTM move. NOP.
 353          */
 354         if (old_iomap == NULL && new_iomap == NULL)
 355                 goto out2;
 356
 357         /*
 358          * Don't move nonexistent data. Clear destination instead.
 359          */
 360         if (old_iomap == NULL &&
 361             (ttm == NULL || (ttm->state == tt_unpopulated &&
 362                              !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
 363                 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
 364                 goto out2;
 365         }
 366
 367         /*
 368          * TTM might be null for moves within the same region.
 369          */
 370         if (ttm && ttm->state == tt_unpopulated) {
 371                 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
 372                 if (ret)
 373                         goto out1;
 374         }
 375
 376         add = 0;
 377         dir = 1;
 378
 379         if ((old_mem->mem_type == new_mem->mem_type) &&
 380             (new_mem->start < old_mem->start + old_mem->size)) {
 381                 dir = -1;
 382                 add = new_mem->num_pages - 1;
 383         }
 384
 385         for (i = 0; i < new_mem->num_pages; ++i) {
 386                 page = i * dir + add;
 387                 if (old_iomap == NULL) {
 388                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 389                                                     PAGE_KERNEL);
 390                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 391                                                    prot);
 392                 } else if (new_iomap == NULL) {
 393                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 394                                                     PAGE_KERNEL);
 395                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 396                                                    prot);
 397                 } else
 398                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 399                 if (ret)
 400                         goto out1;
 401         }
 402         mb();
 403 out2:
 404         old_copy = *old_mem;
 405         *old_mem = *new_mem;
 406         new_mem->mm_node = NULL;
 407
 408         if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 409                 ttm_tt_destroy(ttm);
 410                 bo->ttm = NULL;
 411         }
 412
 413 out1:
 414         ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
 415 out:
 416         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 417
 418         /*
 419          * On error, keep the mm node!
 420          */
 421         if (!ret)
 422                 ttm_bo_mem_put(bo, &old_copy);
 423         return ret;
 424 }
 425 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 426
 427 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 428 {
 429         kfree(bo);
 430 }
 431
 432 /**
 433  * ttm_buffer_object_transfer
 434  *
 435  * @bo: A pointer to a struct ttm_buffer_object.
 436  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 437  * holding the data of @bo with the old placement.
 438  *
 439  * This is a utility function that may be called after an accelerated move
 440  * has been scheduled. A new buffer object is created as a placeholder for
 441  * the old data while it's being copied. When that buffer object is idle,
 442  * it can be destroyed, releasing the space of the old placement.
 443  * Returns:
 444  * !0: Failure.
 445  */
 446
 447 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 448                                       struct ttm_buffer_object **new_obj)
 449 {
 450         struct ttm_buffer_object *fbo;
 451         int ret;
 452
 453         fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
 454         if (!fbo)
 455                 return -ENOMEM;
 456
 457         *fbo = *bo;
 458
 459         /**
 460          * Fix up members that we shouldn't copy directly:
 461          * TODO: Explicit member copy would probably be better here.
 462          */
 463
 464         INIT_LIST_HEAD(&fbo->ddestroy);
 465         INIT_LIST_HEAD(&fbo->lru);
 466         INIT_LIST_HEAD(&fbo->swap);
 467         INIT_LIST_HEAD(&fbo->io_reserve_lru);
 468         fbo->moving = NULL;
 469         drm_vma_node_reset(&fbo->vma_node);
 470         atomic_set(&fbo->cpu_writers, 0);
 471
 472         kref_init(&fbo->list_kref);
 473         kref_init(&fbo->kref);
 474         fbo->destroy = &ttm_transfered_destroy;
 475         fbo->acc_size = 0;
 476         fbo->resv = &fbo->ttm_resv;
 477         reservation_object_init(fbo->resv);
 478         ret = ww_mutex_trylock(&fbo->resv->lock);
 479         WARN_ON(!ret);
 480
 481         *new_obj = fbo;
 482         return 0;
 483 }
 484
 485 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 486 {
 487         /* Cached mappings need no adjustment */
 488         if (caching_flags & TTM_PL_FLAG_CACHED)
 489                 return tmp;
 490
 491 #if defined(__i386__) || defined(__x86_64__)
 492         if (caching_flags & TTM_PL_FLAG_WC)
 493                 tmp = pgprot_writecombine(tmp);
 494         else if (boot_cpu_data.x86 > 3)
 495                 tmp = pgprot_noncached(tmp);
 496 #endif
 497 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
 498     defined(__powerpc__)
 499         if (caching_flags & TTM_PL_FLAG_WC)
 500                 tmp = pgprot_writecombine(tmp);
 501         else
 502                 tmp = pgprot_noncached(tmp);
 503 #endif
 504 #if defined(__sparc__) || defined(__mips__)
 505         tmp = pgprot_noncached(tmp);
 506 #endif
 507         return tmp;
 508 }
 509 EXPORT_SYMBOL(ttm_io_prot);
 510
 511 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 512                           unsigned long offset,
 513                           unsigned long size,
 514                           struct ttm_bo_kmap_obj *map)
 515 {
 516         struct ttm_mem_reg *mem = &bo->mem;
 517
 518         if (bo->mem.bus.addr) {
 519                 map->bo_kmap_type = ttm_bo_map_premapped;
 520                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 521         } else {
 522                 map->bo_kmap_type = ttm_bo_map_iomap;
 523                 if (mem->placement & TTM_PL_FLAG_WC)
 524                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 525                                                   size);
 526                 else
 527                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 528                                                        size);
 529         }
 530         return (!map->virtual) ? -ENOMEM : 0;
 531 }
 532
 533 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 534                            unsigned long start_page,
 535                            unsigned long num_pages,
 536                            struct ttm_bo_kmap_obj *map)
 537 {
 538         struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
 539         struct ttm_tt *ttm = bo->ttm;
 540         int ret;
 541
 542         BUG_ON(!ttm);
 543
 544         if (ttm->state == tt_unpopulated) {
 545                 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
 546                 if (ret)
 547                         return ret;
 548         }
 549
 550         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 551                 /*
 552                  * We're mapping a single page, and the desired
 553                  * page protection is consistent with the bo.
 554                  */
 555
 556                 map->bo_kmap_type = ttm_bo_map_kmap;
 557                 map->page = ttm->pages[start_page];
 558                 map->virtual = kmap(map->page);
 559         } else {
 560                 /*
 561                  * We need to use vmap to get the desired page protection
 562                  * or to make the buffer object look contiguous.
 563                  */
 564                 prot = ttm_io_prot(mem->placement, PAGE_KERNEL);
 565                 map->bo_kmap_type = ttm_bo_map_vmap;
 566                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 567                                     0, prot);
 568         }
 569         return (!map->virtual) ? -ENOMEM : 0;
 570 }
 571
 572 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 573                 unsigned long start_page, unsigned long num_pages,
 574                 struct ttm_bo_kmap_obj *map)
 575 {
 576         struct ttm_mem_type_manager *man =
 577                 &bo->bdev->man[bo->mem.mem_type];
 578         unsigned long offset, size;
 579         int ret;
 580
 581         BUG_ON(!list_empty(&bo->swap));
 582         map->virtual = NULL;
 583         map->bo = bo;
 584         if (num_pages > bo->num_pages)
 585                 return -EINVAL;
 586         if (start_page > bo->num_pages)
 587                 return -EINVAL;
 588 #if 0
 589         if (num_pages > 1 && !capable(CAP_SYS_ADMIN))
 590                 return -EPERM;
 591 #endif
 592         (void) ttm_mem_io_lock(man, false);
 593         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 594         ttm_mem_io_unlock(man);
 595         if (ret)
 596                 return ret;
 597         if (!bo->mem.bus.is_iomem) {
 598                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 599         } else {
 600                 offset = start_page << PAGE_SHIFT;
 601                 size = num_pages << PAGE_SHIFT;
 602                 return ttm_bo_ioremap(bo, offset, size, map);
 603         }
 604 }
 605 EXPORT_SYMBOL(ttm_bo_kmap);
 606
 607 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 608 {
 609         struct ttm_buffer_object *bo = map->bo;
 610         struct ttm_mem_type_manager *man =
 611                 &bo->bdev->man[bo->mem.mem_type];
 612
 613         if (!map->virtual)
 614                 return;
 615         switch (map->bo_kmap_type) {
 616         case ttm_bo_map_iomap:
 617                 iounmap(map->virtual);
 618                 break;
 619         case ttm_bo_map_vmap:
 620                 vunmap(map->virtual);
 621                 break;
 622         case ttm_bo_map_kmap:
 623                 kunmap(map->page);
 624                 break;
 625         case ttm_bo_map_premapped:
 626                 break;
 627         default:
 628                 BUG();
 629         }
 630         (void) ttm_mem_io_lock(man, false);
 631         ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 632         ttm_mem_io_unlock(man);
 633         map->virtual = NULL;
 634         map->page = NULL;
 635 }
 636 EXPORT_SYMBOL(ttm_bo_kunmap);
 637
 638 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 639                               struct fence *fence,
 640                               bool evict,
 641                               struct ttm_mem_reg *new_mem)
 642 {
 643         struct ttm_bo_device *bdev = bo->bdev;
 644         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 645         struct ttm_mem_reg *old_mem = &bo->mem;
 646         int ret;
 647         struct ttm_buffer_object *ghost_obj;
 648
 649         reservation_object_add_excl_fence(bo->resv, fence);
 650         if (evict) {
 651                 ret = ttm_bo_wait(bo, false, false);
 652                 if (ret)
 653                         return ret;
 654
 655                 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
 656                         ttm_tt_destroy(bo->ttm);
 657                         bo->ttm = NULL;
 658                 }
 659                 ttm_bo_free_old_node(bo);
 660         } else {
 661                 /**
 662                  * This should help pipeline ordinary buffer moves.
 663                  *
 664                  * Hang old buffer memory on a new buffer object,
 665                  * and leave it to be released when the GPU
 666                  * operation has completed.
 667                  */
 668
 669                 fence_put(bo->moving);
 670                 bo->moving = fence_get(fence);
 671
 672                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 673                 if (ret)
 674                         return ret;
 675
 676                 reservation_object_add_excl_fence(ghost_obj->resv, fence);
 677
 678                 /**
 679                  * If we're not moving to fixed memory, the TTM object
 680                  * needs to stay alive. Otherwhise hang it on the ghost
 681                  * bo to be unbound and destroyed.
 682                  */
 683
 684                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 685                         ghost_obj->ttm = NULL;
 686                 else
 687                         bo->ttm = NULL;
 688
 689                 ttm_bo_unreserve(ghost_obj);
 690                 ttm_bo_unref(&ghost_obj);
 691         }
 692
 693         *old_mem = *new_mem;
 694         new_mem->mm_node = NULL;
 695
 696         return 0;
 697 }
 698 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
 699
 700 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
 701                          struct fence *fence, bool evict,
 702                          struct ttm_mem_reg *new_mem)
 703 {
 704         struct ttm_bo_device *bdev = bo->bdev;
 705         struct ttm_mem_reg *old_mem = &bo->mem;
 706
 707         struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type];
 708         struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type];
 709
 710         int ret;
 711
 712         reservation_object_add_excl_fence(bo->resv, fence);
 713
 714         if (!evict) {
 715                 struct ttm_buffer_object *ghost_obj;
 716
 717                 /**
 718                  * This should help pipeline ordinary buffer moves.
 719                  *
 720                  * Hang old buffer memory on a new buffer object,
 721                  * and leave it to be released when the GPU
 722                  * operation has completed.
 723                  */
 724
 725                 fence_put(bo->moving);
 726                 bo->moving = fence_get(fence);
 727
 728                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 729                 if (ret)
 730                         return ret;
 731
 732                 reservation_object_add_excl_fence(ghost_obj->resv, fence);
 733
 734                 /**
 735                  * If we're not moving to fixed memory, the TTM object
 736                  * needs to stay alive. Otherwhise hang it on the ghost
 737                  * bo to be unbound and destroyed.
 738                  */
 739
 740                 if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED))
 741                         ghost_obj->ttm = NULL;
 742                 else
 743                         bo->ttm = NULL;
 744
 745                 ttm_bo_unreserve(ghost_obj);
 746                 ttm_bo_unref(&ghost_obj);
 747
 748         } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
 749
 750                 /**
 751                  * BO doesn't have a TTM we need to bind/unbind. Just remember
 752                  * this eviction and free up the allocation
 753                  */
 754
 755                 spin_lock(&from->move_lock);
 756                 if (!from->move || fence_is_later(fence, from->move)) {
 757                         fence_put(from->move);
 758                         from->move = fence_get(fence);
 759                 }
 760                 spin_unlock(&from->move_lock);
 761
 762                 ttm_bo_free_old_node(bo);
 763
 764                 fence_put(bo->moving);
 765                 bo->moving = fence_get(fence);
 766
 767         } else {
 768                 /**
 769                  * Last resort, wait for the move to be completed.
 770                  *
 771                  * Should never happen in pratice.
 772                  */
 773
 774                 ret = ttm_bo_wait(bo, false, false);
 775                 if (ret)
 776                         return ret;
 777
 778                 if (to->flags & TTM_MEMTYPE_FLAG_FIXED) {
 779                         ttm_tt_destroy(bo->ttm);
 780                         bo->ttm = NULL;
 781                 }
 782                 ttm_bo_free_old_node(bo);
 783         }
 784
 785         *old_mem = *new_mem;
 786         new_mem->mm_node = NULL;
 787
 788         return 0;
 789 }
 790 EXPORT_SYMBOL(ttm_bo_pipeline_move);