objtool: Re-arrange validate_functions()
[linux-block.git] / drivers / gpu / drm / ttm / ttm_bo_util.c
1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 /**************************************************************************
3  *
4  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
5  * All Rights Reserved.
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining a
8  * copy of this software and associated documentation files (the
9  * "Software"), to deal in the Software without restriction, including
10  * without limitation the rights to use, copy, modify, merge, publish,
11  * distribute, sub license, and/or sell copies of the Software, and to
12  * permit persons to whom the Software is furnished to do so, subject to
13  * the following conditions:
14  *
15  * The above copyright notice and this permission notice (including the
16  * next paragraph) shall be included in all copies or substantial portions
17  * of the Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25  * USE OR OTHER DEALINGS IN THE SOFTWARE.
26  *
27  **************************************************************************/
28 /*
29  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
30  */
31
32 #include <drm/ttm/ttm_bo_driver.h>
33 #include <drm/ttm/ttm_placement.h>
34 #include <drm/drm_vma_manager.h>
35 #include <linux/io.h>
36 #include <linux/highmem.h>
37 #include <linux/wait.h>
38 #include <linux/slab.h>
39 #include <linux/vmalloc.h>
40 #include <linux/module.h>
41 #include <linux/dma-resv.h>
42
43 struct ttm_transfer_obj {
44         struct ttm_buffer_object base;
45         struct ttm_buffer_object *bo;
46 };
47
48 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
49 {
50         ttm_bo_mem_put(bo, &bo->mem);
51 }
52
53 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
54                    struct ttm_operation_ctx *ctx,
55                     struct ttm_mem_reg *new_mem)
56 {
57         struct ttm_tt *ttm = bo->ttm;
58         struct ttm_mem_reg *old_mem = &bo->mem;
59         int ret;
60
61         if (old_mem->mem_type != TTM_PL_SYSTEM) {
62                 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
63
64                 if (unlikely(ret != 0)) {
65                         if (ret != -ERESTARTSYS)
66                                 pr_err("Failed to expire sync object before unbinding TTM\n");
67                         return ret;
68                 }
69
70                 ttm_tt_unbind(ttm);
71                 ttm_bo_free_old_node(bo);
72                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
73                                 TTM_PL_MASK_MEM);
74                 old_mem->mem_type = TTM_PL_SYSTEM;
75         }
76
77         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
78         if (unlikely(ret != 0))
79                 return ret;
80
81         if (new_mem->mem_type != TTM_PL_SYSTEM) {
82                 ret = ttm_tt_bind(ttm, new_mem, ctx);
83                 if (unlikely(ret != 0))
84                         return ret;
85         }
86
87         *old_mem = *new_mem;
88         new_mem->mm_node = NULL;
89
90         return 0;
91 }
92 EXPORT_SYMBOL(ttm_bo_move_ttm);
93
94 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible)
95 {
96         if (likely(man->io_reserve_fastpath))
97                 return 0;
98
99         if (interruptible)
100                 return mutex_lock_interruptible(&man->io_reserve_mutex);
101
102         mutex_lock(&man->io_reserve_mutex);
103         return 0;
104 }
105
106 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man)
107 {
108         if (likely(man->io_reserve_fastpath))
109                 return;
110
111         mutex_unlock(&man->io_reserve_mutex);
112 }
113
114 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man)
115 {
116         struct ttm_buffer_object *bo;
117
118         if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru))
119                 return -EAGAIN;
120
121         bo = list_first_entry(&man->io_reserve_lru,
122                               struct ttm_buffer_object,
123                               io_reserve_lru);
124         list_del_init(&bo->io_reserve_lru);
125         ttm_bo_unmap_virtual_locked(bo);
126
127         return 0;
128 }
129
130
131 int ttm_mem_io_reserve(struct ttm_bo_device *bdev,
132                        struct ttm_mem_reg *mem)
133 {
134         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
135         int ret = 0;
136
137         if (!bdev->driver->io_mem_reserve)
138                 return 0;
139         if (likely(man->io_reserve_fastpath))
140                 return bdev->driver->io_mem_reserve(bdev, mem);
141
142         if (bdev->driver->io_mem_reserve &&
143             mem->bus.io_reserved_count++ == 0) {
144 retry:
145                 ret = bdev->driver->io_mem_reserve(bdev, mem);
146                 if (ret == -EAGAIN) {
147                         ret = ttm_mem_io_evict(man);
148                         if (ret == 0)
149                                 goto retry;
150                 }
151         }
152         return ret;
153 }
154
155 void ttm_mem_io_free(struct ttm_bo_device *bdev,
156                      struct ttm_mem_reg *mem)
157 {
158         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
159
160         if (likely(man->io_reserve_fastpath))
161                 return;
162
163         if (bdev->driver->io_mem_reserve &&
164             --mem->bus.io_reserved_count == 0 &&
165             bdev->driver->io_mem_free)
166                 bdev->driver->io_mem_free(bdev, mem);
167
168 }
169
170 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo)
171 {
172         struct ttm_mem_reg *mem = &bo->mem;
173         int ret;
174
175         if (!mem->bus.io_reserved_vm) {
176                 struct ttm_mem_type_manager *man =
177                         &bo->bdev->man[mem->mem_type];
178
179                 ret = ttm_mem_io_reserve(bo->bdev, mem);
180                 if (unlikely(ret != 0))
181                         return ret;
182                 mem->bus.io_reserved_vm = true;
183                 if (man->use_io_reserve_lru)
184                         list_add_tail(&bo->io_reserve_lru,
185                                       &man->io_reserve_lru);
186         }
187         return 0;
188 }
189
190 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo)
191 {
192         struct ttm_mem_reg *mem = &bo->mem;
193
194         if (mem->bus.io_reserved_vm) {
195                 mem->bus.io_reserved_vm = false;
196                 list_del_init(&bo->io_reserve_lru);
197                 ttm_mem_io_free(bo->bdev, mem);
198         }
199 }
200
201 static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
202                         void **virtual)
203 {
204         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
205         int ret;
206         void *addr;
207
208         *virtual = NULL;
209         (void) ttm_mem_io_lock(man, false);
210         ret = ttm_mem_io_reserve(bdev, mem);
211         ttm_mem_io_unlock(man);
212         if (ret || !mem->bus.is_iomem)
213                 return ret;
214
215         if (mem->bus.addr) {
216                 addr = mem->bus.addr;
217         } else {
218                 if (mem->placement & TTM_PL_FLAG_WC)
219                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
220                 else
221                         addr = ioremap(mem->bus.base + mem->bus.offset, mem->bus.size);
222                 if (!addr) {
223                         (void) ttm_mem_io_lock(man, false);
224                         ttm_mem_io_free(bdev, mem);
225                         ttm_mem_io_unlock(man);
226                         return -ENOMEM;
227                 }
228         }
229         *virtual = addr;
230         return 0;
231 }
232
233 static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
234                          void *virtual)
235 {
236         struct ttm_mem_type_manager *man;
237
238         man = &bdev->man[mem->mem_type];
239
240         if (virtual && mem->bus.addr == NULL)
241                 iounmap(virtual);
242         (void) ttm_mem_io_lock(man, false);
243         ttm_mem_io_free(bdev, mem);
244         ttm_mem_io_unlock(man);
245 }
246
247 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
248 {
249         uint32_t *dstP =
250             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
251         uint32_t *srcP =
252             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
253
254         int i;
255         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
256                 iowrite32(ioread32(srcP++), dstP++);
257         return 0;
258 }
259
260 #ifdef CONFIG_X86
261 #define __ttm_kmap_atomic_prot(__page, __prot) kmap_atomic_prot(__page, __prot)
262 #define __ttm_kunmap_atomic(__addr) kunmap_atomic(__addr)
263 #else
264 #define __ttm_kmap_atomic_prot(__page, __prot) vmap(&__page, 1, 0,  __prot)
265 #define __ttm_kunmap_atomic(__addr) vunmap(__addr)
266 #endif
267
268
269 /**
270  * ttm_kmap_atomic_prot - Efficient kernel map of a single page with
271  * specified page protection.
272  *
273  * @page: The page to map.
274  * @prot: The page protection.
275  *
276  * This function maps a TTM page using the kmap_atomic api if available,
277  * otherwise falls back to vmap. The user must make sure that the
278  * specified page does not have an aliased mapping with a different caching
279  * policy unless the architecture explicitly allows it. Also mapping and
280  * unmapping using this api must be correctly nested. Unmapping should
281  * occur in the reverse order of mapping.
282  */
283 void *ttm_kmap_atomic_prot(struct page *page, pgprot_t prot)
284 {
285         if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
286                 return kmap_atomic(page);
287         else
288                 return __ttm_kmap_atomic_prot(page, prot);
289 }
290 EXPORT_SYMBOL(ttm_kmap_atomic_prot);
291
292 /**
293  * ttm_kunmap_atomic_prot - Unmap a page that was mapped using
294  * ttm_kmap_atomic_prot.
295  *
296  * @addr: The virtual address from the map.
297  * @prot: The page protection.
298  */
299 void ttm_kunmap_atomic_prot(void *addr, pgprot_t prot)
300 {
301         if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL))
302                 kunmap_atomic(addr);
303         else
304                 __ttm_kunmap_atomic(addr);
305 }
306 EXPORT_SYMBOL(ttm_kunmap_atomic_prot);
307
308 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
309                                 unsigned long page,
310                                 pgprot_t prot)
311 {
312         struct page *d = ttm->pages[page];
313         void *dst;
314
315         if (!d)
316                 return -ENOMEM;
317
318         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
319         dst = ttm_kmap_atomic_prot(d, prot);
320         if (!dst)
321                 return -ENOMEM;
322
323         memcpy_fromio(dst, src, PAGE_SIZE);
324
325         ttm_kunmap_atomic_prot(dst, prot);
326
327         return 0;
328 }
329
330 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
331                                 unsigned long page,
332                                 pgprot_t prot)
333 {
334         struct page *s = ttm->pages[page];
335         void *src;
336
337         if (!s)
338                 return -ENOMEM;
339
340         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
341         src = ttm_kmap_atomic_prot(s, prot);
342         if (!src)
343                 return -ENOMEM;
344
345         memcpy_toio(dst, src, PAGE_SIZE);
346
347         ttm_kunmap_atomic_prot(src, prot);
348
349         return 0;
350 }
351
352 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
353                        struct ttm_operation_ctx *ctx,
354                        struct ttm_mem_reg *new_mem)
355 {
356         struct ttm_bo_device *bdev = bo->bdev;
357         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
358         struct ttm_tt *ttm = bo->ttm;
359         struct ttm_mem_reg *old_mem = &bo->mem;
360         struct ttm_mem_reg old_copy = *old_mem;
361         void *old_iomap;
362         void *new_iomap;
363         int ret;
364         unsigned long i;
365         unsigned long page;
366         unsigned long add = 0;
367         int dir;
368
369         ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
370         if (ret)
371                 return ret;
372
373         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
374         if (ret)
375                 return ret;
376         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
377         if (ret)
378                 goto out;
379
380         /*
381          * Single TTM move. NOP.
382          */
383         if (old_iomap == NULL && new_iomap == NULL)
384                 goto out2;
385
386         /*
387          * Don't move nonexistent data. Clear destination instead.
388          */
389         if (old_iomap == NULL &&
390             (ttm == NULL || (ttm->state == tt_unpopulated &&
391                              !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) {
392                 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE);
393                 goto out2;
394         }
395
396         /*
397          * TTM might be null for moves within the same region.
398          */
399         if (ttm) {
400                 ret = ttm_tt_populate(ttm, ctx);
401                 if (ret)
402                         goto out1;
403         }
404
405         add = 0;
406         dir = 1;
407
408         if ((old_mem->mem_type == new_mem->mem_type) &&
409             (new_mem->start < old_mem->start + old_mem->size)) {
410                 dir = -1;
411                 add = new_mem->num_pages - 1;
412         }
413
414         for (i = 0; i < new_mem->num_pages; ++i) {
415                 page = i * dir + add;
416                 if (old_iomap == NULL) {
417                         pgprot_t prot = ttm_io_prot(old_mem->placement,
418                                                     PAGE_KERNEL);
419                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
420                                                    prot);
421                 } else if (new_iomap == NULL) {
422                         pgprot_t prot = ttm_io_prot(new_mem->placement,
423                                                     PAGE_KERNEL);
424                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
425                                                    prot);
426                 } else {
427                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
428                 }
429                 if (ret)
430                         goto out1;
431         }
432         mb();
433 out2:
434         old_copy = *old_mem;
435         *old_mem = *new_mem;
436         new_mem->mm_node = NULL;
437
438         if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
439                 ttm_tt_destroy(ttm);
440                 bo->ttm = NULL;
441         }
442
443 out1:
444         ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
445 out:
446         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
447
448         /*
449          * On error, keep the mm node!
450          */
451         if (!ret)
452                 ttm_bo_mem_put(bo, &old_copy);
453         return ret;
454 }
455 EXPORT_SYMBOL(ttm_bo_move_memcpy);
456
457 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
458 {
459         struct ttm_transfer_obj *fbo;
460
461         fbo = container_of(bo, struct ttm_transfer_obj, base);
462         ttm_bo_put(fbo->bo);
463         kfree(fbo);
464 }
465
466 /**
467  * ttm_buffer_object_transfer
468  *
469  * @bo: A pointer to a struct ttm_buffer_object.
470  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
471  * holding the data of @bo with the old placement.
472  *
473  * This is a utility function that may be called after an accelerated move
474  * has been scheduled. A new buffer object is created as a placeholder for
475  * the old data while it's being copied. When that buffer object is idle,
476  * it can be destroyed, releasing the space of the old placement.
477  * Returns:
478  * !0: Failure.
479  */
480
481 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
482                                       struct ttm_buffer_object **new_obj)
483 {
484         struct ttm_transfer_obj *fbo;
485         int ret;
486
487         fbo = kmalloc(sizeof(*fbo), GFP_KERNEL);
488         if (!fbo)
489                 return -ENOMEM;
490
491         fbo->base = *bo;
492         fbo->base.mem.placement |= TTM_PL_FLAG_NO_EVICT;
493
494         ttm_bo_get(bo);
495         fbo->bo = bo;
496
497         /**
498          * Fix up members that we shouldn't copy directly:
499          * TODO: Explicit member copy would probably be better here.
500          */
501
502         atomic_inc(&ttm_bo_glob.bo_count);
503         INIT_LIST_HEAD(&fbo->base.ddestroy);
504         INIT_LIST_HEAD(&fbo->base.lru);
505         INIT_LIST_HEAD(&fbo->base.swap);
506         INIT_LIST_HEAD(&fbo->base.io_reserve_lru);
507         fbo->base.moving = NULL;
508         drm_vma_node_reset(&fbo->base.base.vma_node);
509
510         kref_init(&fbo->base.list_kref);
511         kref_init(&fbo->base.kref);
512         fbo->base.destroy = &ttm_transfered_destroy;
513         fbo->base.acc_size = 0;
514         if (bo->base.resv == &bo->base._resv)
515                 fbo->base.base.resv = &fbo->base.base._resv;
516
517         dma_resv_init(&fbo->base.base._resv);
518         ret = dma_resv_trylock(&fbo->base.base._resv);
519         WARN_ON(!ret);
520
521         *new_obj = &fbo->base;
522         return 0;
523 }
524
525 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
526 {
527         /* Cached mappings need no adjustment */
528         if (caching_flags & TTM_PL_FLAG_CACHED)
529                 return tmp;
530
531 #if defined(__i386__) || defined(__x86_64__)
532         if (caching_flags & TTM_PL_FLAG_WC)
533                 tmp = pgprot_writecombine(tmp);
534         else if (boot_cpu_data.x86 > 3)
535                 tmp = pgprot_noncached(tmp);
536 #endif
537 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \
538     defined(__powerpc__) || defined(__mips__)
539         if (caching_flags & TTM_PL_FLAG_WC)
540                 tmp = pgprot_writecombine(tmp);
541         else
542                 tmp = pgprot_noncached(tmp);
543 #endif
544 #if defined(__sparc__)
545         tmp = pgprot_noncached(tmp);
546 #endif
547         return tmp;
548 }
549 EXPORT_SYMBOL(ttm_io_prot);
550
551 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
552                           unsigned long offset,
553                           unsigned long size,
554                           struct ttm_bo_kmap_obj *map)
555 {
556         struct ttm_mem_reg *mem = &bo->mem;
557
558         if (bo->mem.bus.addr) {
559                 map->bo_kmap_type = ttm_bo_map_premapped;
560                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
561         } else {
562                 map->bo_kmap_type = ttm_bo_map_iomap;
563                 if (mem->placement & TTM_PL_FLAG_WC)
564                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
565                                                   size);
566                 else
567                         map->virtual = ioremap(bo->mem.bus.base + bo->mem.bus.offset + offset,
568                                                        size);
569         }
570         return (!map->virtual) ? -ENOMEM : 0;
571 }
572
573 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
574                            unsigned long start_page,
575                            unsigned long num_pages,
576                            struct ttm_bo_kmap_obj *map)
577 {
578         struct ttm_mem_reg *mem = &bo->mem;
579         struct ttm_operation_ctx ctx = {
580                 .interruptible = false,
581                 .no_wait_gpu = false
582         };
583         struct ttm_tt *ttm = bo->ttm;
584         pgprot_t prot;
585         int ret;
586
587         BUG_ON(!ttm);
588
589         ret = ttm_tt_populate(ttm, &ctx);
590         if (ret)
591                 return ret;
592
593         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
594                 /*
595                  * We're mapping a single page, and the desired
596                  * page protection is consistent with the bo.
597                  */
598
599                 map->bo_kmap_type = ttm_bo_map_kmap;
600                 map->page = ttm->pages[start_page];
601                 map->virtual = kmap(map->page);
602         } else {
603                 /*
604                  * We need to use vmap to get the desired page protection
605                  * or to make the buffer object look contiguous.
606                  */
607                 prot = ttm_io_prot(mem->placement, PAGE_KERNEL);
608                 map->bo_kmap_type = ttm_bo_map_vmap;
609                 map->virtual = vmap(ttm->pages + start_page, num_pages,
610                                     0, prot);
611         }
612         return (!map->virtual) ? -ENOMEM : 0;
613 }
614
615 int ttm_bo_kmap(struct ttm_buffer_object *bo,
616                 unsigned long start_page, unsigned long num_pages,
617                 struct ttm_bo_kmap_obj *map)
618 {
619         struct ttm_mem_type_manager *man =
620                 &bo->bdev->man[bo->mem.mem_type];
621         unsigned long offset, size;
622         int ret;
623
624         map->virtual = NULL;
625         map->bo = bo;
626         if (num_pages > bo->num_pages)
627                 return -EINVAL;
628         if (start_page > bo->num_pages)
629                 return -EINVAL;
630
631         (void) ttm_mem_io_lock(man, false);
632         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
633         ttm_mem_io_unlock(man);
634         if (ret)
635                 return ret;
636         if (!bo->mem.bus.is_iomem) {
637                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
638         } else {
639                 offset = start_page << PAGE_SHIFT;
640                 size = num_pages << PAGE_SHIFT;
641                 return ttm_bo_ioremap(bo, offset, size, map);
642         }
643 }
644 EXPORT_SYMBOL(ttm_bo_kmap);
645
646 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
647 {
648         struct ttm_buffer_object *bo = map->bo;
649         struct ttm_mem_type_manager *man =
650                 &bo->bdev->man[bo->mem.mem_type];
651
652         if (!map->virtual)
653                 return;
654         switch (map->bo_kmap_type) {
655         case ttm_bo_map_iomap:
656                 iounmap(map->virtual);
657                 break;
658         case ttm_bo_map_vmap:
659                 vunmap(map->virtual);
660                 break;
661         case ttm_bo_map_kmap:
662                 kunmap(map->page);
663                 break;
664         case ttm_bo_map_premapped:
665                 break;
666         default:
667                 BUG();
668         }
669         (void) ttm_mem_io_lock(man, false);
670         ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
671         ttm_mem_io_unlock(man);
672         map->virtual = NULL;
673         map->page = NULL;
674 }
675 EXPORT_SYMBOL(ttm_bo_kunmap);
676
677 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
678                               struct dma_fence *fence,
679                               bool evict,
680                               struct ttm_mem_reg *new_mem)
681 {
682         struct ttm_bo_device *bdev = bo->bdev;
683         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
684         struct ttm_mem_reg *old_mem = &bo->mem;
685         int ret;
686         struct ttm_buffer_object *ghost_obj;
687
688         dma_resv_add_excl_fence(bo->base.resv, fence);
689         if (evict) {
690                 ret = ttm_bo_wait(bo, false, false);
691                 if (ret)
692                         return ret;
693
694                 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
695                         ttm_tt_destroy(bo->ttm);
696                         bo->ttm = NULL;
697                 }
698                 ttm_bo_free_old_node(bo);
699         } else {
700                 /**
701                  * This should help pipeline ordinary buffer moves.
702                  *
703                  * Hang old buffer memory on a new buffer object,
704                  * and leave it to be released when the GPU
705                  * operation has completed.
706                  */
707
708                 dma_fence_put(bo->moving);
709                 bo->moving = dma_fence_get(fence);
710
711                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
712                 if (ret)
713                         return ret;
714
715                 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence);
716
717                 /**
718                  * If we're not moving to fixed memory, the TTM object
719                  * needs to stay alive. Otherwhise hang it on the ghost
720                  * bo to be unbound and destroyed.
721                  */
722
723                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
724                         ghost_obj->ttm = NULL;
725                 else
726                         bo->ttm = NULL;
727
728                 dma_resv_unlock(&ghost_obj->base._resv);
729                 ttm_bo_put(ghost_obj);
730         }
731
732         *old_mem = *new_mem;
733         new_mem->mm_node = NULL;
734
735         return 0;
736 }
737 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
738
739 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
740                          struct dma_fence *fence, bool evict,
741                          struct ttm_mem_reg *new_mem)
742 {
743         struct ttm_bo_device *bdev = bo->bdev;
744         struct ttm_mem_reg *old_mem = &bo->mem;
745
746         struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type];
747         struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type];
748
749         int ret;
750
751         dma_resv_add_excl_fence(bo->base.resv, fence);
752
753         if (!evict) {
754                 struct ttm_buffer_object *ghost_obj;
755
756                 /**
757                  * This should help pipeline ordinary buffer moves.
758                  *
759                  * Hang old buffer memory on a new buffer object,
760                  * and leave it to be released when the GPU
761                  * operation has completed.
762                  */
763
764                 dma_fence_put(bo->moving);
765                 bo->moving = dma_fence_get(fence);
766
767                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
768                 if (ret)
769                         return ret;
770
771                 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence);
772
773                 /**
774                  * If we're not moving to fixed memory, the TTM object
775                  * needs to stay alive. Otherwhise hang it on the ghost
776                  * bo to be unbound and destroyed.
777                  */
778
779                 if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED))
780                         ghost_obj->ttm = NULL;
781                 else
782                         bo->ttm = NULL;
783
784                 dma_resv_unlock(&ghost_obj->base._resv);
785                 ttm_bo_put(ghost_obj);
786
787         } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
788
789                 /**
790                  * BO doesn't have a TTM we need to bind/unbind. Just remember
791                  * this eviction and free up the allocation
792                  */
793
794                 spin_lock(&from->move_lock);
795                 if (!from->move || dma_fence_is_later(fence, from->move)) {
796                         dma_fence_put(from->move);
797                         from->move = dma_fence_get(fence);
798                 }
799                 spin_unlock(&from->move_lock);
800
801                 ttm_bo_free_old_node(bo);
802
803                 dma_fence_put(bo->moving);
804                 bo->moving = dma_fence_get(fence);
805
806         } else {
807                 /**
808                  * Last resort, wait for the move to be completed.
809                  *
810                  * Should never happen in pratice.
811                  */
812
813                 ret = ttm_bo_wait(bo, false, false);
814                 if (ret)
815                         return ret;
816
817                 if (to->flags & TTM_MEMTYPE_FLAG_FIXED) {
818                         ttm_tt_destroy(bo->ttm);
819                         bo->ttm = NULL;
820                 }
821                 ttm_bo_free_old_node(bo);
822         }
823
824         *old_mem = *new_mem;
825         new_mem->mm_node = NULL;
826
827         return 0;
828 }
829 EXPORT_SYMBOL(ttm_bo_pipeline_move);
830
831 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo)
832 {
833         struct ttm_buffer_object *ghost;
834         int ret;
835
836         ret = ttm_buffer_object_transfer(bo, &ghost);
837         if (ret)
838                 return ret;
839
840         ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv);
841         /* Last resort, wait for the BO to be idle when we are OOM */
842         if (ret)
843                 ttm_bo_wait(bo, false, false);
844
845         memset(&bo->mem, 0, sizeof(bo->mem));
846         bo->mem.mem_type = TTM_PL_SYSTEM;
847         bo->ttm = NULL;
848
849         dma_resv_unlock(&ghost->base._resv);
850         ttm_bo_put(ghost);
851
852         return 0;
853 }