Commit | Line | Data |
---|---|---|
1297bf2e | 1 | /* SPDX-License-Identifier: GPL-2.0 OR MIT */ |
ba4e7d97 TH |
2 | /************************************************************************** |
3 | * | |
4 | * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA | |
5 | * All Rights Reserved. | |
6 | * | |
7 | * Permission is hereby granted, free of charge, to any person obtaining a | |
8 | * copy of this software and associated documentation files (the | |
9 | * "Software"), to deal in the Software without restriction, including | |
10 | * without limitation the rights to use, copy, modify, merge, publish, | |
11 | * distribute, sub license, and/or sell copies of the Software, and to | |
12 | * permit persons to whom the Software is furnished to do so, subject to | |
13 | * the following conditions: | |
14 | * | |
15 | * The above copyright notice and this permission notice (including the | |
16 | * next paragraph) shall be included in all copies or substantial portions | |
17 | * of the Software. | |
18 | * | |
19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
20 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
21 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
22 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
23 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
24 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
25 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
26 | * | |
27 | **************************************************************************/ | |
28 | /* | |
29 | * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> | |
30 | */ | |
31 | ||
760285e7 DH |
32 | #include <drm/ttm/ttm_bo_driver.h> |
33 | #include <drm/ttm/ttm_placement.h> | |
72525b3f | 34 | #include <drm/drm_vma_manager.h> |
ba4e7d97 TH |
35 | #include <linux/io.h> |
36 | #include <linux/highmem.h> | |
37 | #include <linux/wait.h> | |
5a0e3ad6 | 38 | #include <linux/slab.h> |
ba4e7d97 | 39 | #include <linux/vmalloc.h> |
ba4e7d97 | 40 | #include <linux/module.h> |
52791eee | 41 | #include <linux/dma-resv.h> |
ba4e7d97 | 42 | |
5452cf44 CK |
43 | struct ttm_transfer_obj { |
44 | struct ttm_buffer_object base; | |
45 | struct ttm_buffer_object *bo; | |
46 | }; | |
47 | ||
ba4e7d97 TH |
48 | void ttm_bo_free_old_node(struct ttm_buffer_object *bo) |
49 | { | |
42311ff9 | 50 | ttm_bo_mem_put(bo, &bo->mem); |
ba4e7d97 TH |
51 | } |
52 | ||
53 | int ttm_bo_move_ttm(struct ttm_buffer_object *bo, | |
3e98d829 | 54 | struct ttm_operation_ctx *ctx, |
4e2f0caa | 55 | struct ttm_mem_reg *new_mem) |
ba4e7d97 TH |
56 | { |
57 | struct ttm_tt *ttm = bo->ttm; | |
58 | struct ttm_mem_reg *old_mem = &bo->mem; | |
ba4e7d97 TH |
59 | int ret; |
60 | ||
61 | if (old_mem->mem_type != TTM_PL_SYSTEM) { | |
3e98d829 | 62 | ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); |
7b8082bc MD |
63 | |
64 | if (unlikely(ret != 0)) { | |
65 | if (ret != -ERESTARTSYS) | |
66 | pr_err("Failed to expire sync object before unbinding TTM\n"); | |
67 | return ret; | |
68 | } | |
69 | ||
2ff2bf1e | 70 | ttm_tt_unbind(ttm); |
ba4e7d97 TH |
71 | ttm_bo_free_old_node(bo); |
72 | ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, | |
73 | TTM_PL_MASK_MEM); | |
74 | old_mem->mem_type = TTM_PL_SYSTEM; | |
ba4e7d97 TH |
75 | } |
76 | ||
77 | ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); | |
78 | if (unlikely(ret != 0)) | |
79 | return ret; | |
80 | ||
81 | if (new_mem->mem_type != TTM_PL_SYSTEM) { | |
993baf15 | 82 | ret = ttm_tt_bind(ttm, new_mem, ctx); |
ba4e7d97 TH |
83 | if (unlikely(ret != 0)) |
84 | return ret; | |
85 | } | |
86 | ||
87 | *old_mem = *new_mem; | |
88 | new_mem->mm_node = NULL; | |
110b20c3 | 89 | |
ba4e7d97 TH |
90 | return 0; |
91 | } | |
92 | EXPORT_SYMBOL(ttm_bo_move_ttm); | |
93 | ||
eba67093 | 94 | int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) |
82c5da6b | 95 | { |
eba67093 TH |
96 | if (likely(man->io_reserve_fastpath)) |
97 | return 0; | |
98 | ||
99 | if (interruptible) | |
100 | return mutex_lock_interruptible(&man->io_reserve_mutex); | |
101 | ||
102 | mutex_lock(&man->io_reserve_mutex); | |
103 | return 0; | |
104 | } | |
82c5da6b | 105 | |
eba67093 TH |
106 | void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) |
107 | { | |
108 | if (likely(man->io_reserve_fastpath)) | |
109 | return; | |
110 | ||
111 | mutex_unlock(&man->io_reserve_mutex); | |
112 | } | |
113 | ||
114 | static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) | |
115 | { | |
116 | struct ttm_buffer_object *bo; | |
117 | ||
118 | if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) | |
119 | return -EAGAIN; | |
120 | ||
121 | bo = list_first_entry(&man->io_reserve_lru, | |
122 | struct ttm_buffer_object, | |
123 | io_reserve_lru); | |
124 | list_del_init(&bo->io_reserve_lru); | |
125 | ttm_bo_unmap_virtual_locked(bo); | |
126 | ||
127 | return 0; | |
128 | } | |
129 | ||
afe6804c DA |
130 | |
131 | int ttm_mem_io_reserve(struct ttm_bo_device *bdev, | |
132 | struct ttm_mem_reg *mem) | |
eba67093 TH |
133 | { |
134 | struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; | |
135 | int ret = 0; | |
136 | ||
137 | if (!bdev->driver->io_mem_reserve) | |
138 | return 0; | |
139 | if (likely(man->io_reserve_fastpath)) | |
140 | return bdev->driver->io_mem_reserve(bdev, mem); | |
141 | ||
142 | if (bdev->driver->io_mem_reserve && | |
143 | mem->bus.io_reserved_count++ == 0) { | |
144 | retry: | |
0c321c79 | 145 | ret = bdev->driver->io_mem_reserve(bdev, mem); |
eba67093 TH |
146 | if (ret == -EAGAIN) { |
147 | ret = ttm_mem_io_evict(man); | |
148 | if (ret == 0) | |
149 | goto retry; | |
150 | } | |
151 | } | |
152 | return ret; | |
153 | } | |
154 | ||
afe6804c DA |
155 | void ttm_mem_io_free(struct ttm_bo_device *bdev, |
156 | struct ttm_mem_reg *mem) | |
eba67093 TH |
157 | { |
158 | struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; | |
159 | ||
160 | if (likely(man->io_reserve_fastpath)) | |
161 | return; | |
162 | ||
163 | if (bdev->driver->io_mem_reserve && | |
164 | --mem->bus.io_reserved_count == 0 && | |
165 | bdev->driver->io_mem_free) | |
166 | bdev->driver->io_mem_free(bdev, mem); | |
167 | ||
168 | } | |
169 | ||
170 | int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) | |
171 | { | |
172 | struct ttm_mem_reg *mem = &bo->mem; | |
173 | int ret; | |
174 | ||
175 | if (!mem->bus.io_reserved_vm) { | |
176 | struct ttm_mem_type_manager *man = | |
177 | &bo->bdev->man[mem->mem_type]; | |
178 | ||
179 | ret = ttm_mem_io_reserve(bo->bdev, mem); | |
82c5da6b JG |
180 | if (unlikely(ret != 0)) |
181 | return ret; | |
eba67093 TH |
182 | mem->bus.io_reserved_vm = true; |
183 | if (man->use_io_reserve_lru) | |
184 | list_add_tail(&bo->io_reserve_lru, | |
185 | &man->io_reserve_lru); | |
82c5da6b JG |
186 | } |
187 | return 0; | |
188 | } | |
189 | ||
eba67093 | 190 | void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) |
82c5da6b | 191 | { |
eba67093 TH |
192 | struct ttm_mem_reg *mem = &bo->mem; |
193 | ||
194 | if (mem->bus.io_reserved_vm) { | |
195 | mem->bus.io_reserved_vm = false; | |
196 | list_del_init(&bo->io_reserve_lru); | |
197 | ttm_mem_io_free(bo->bdev, mem); | |
82c5da6b JG |
198 | } |
199 | } | |
200 | ||
dcbff15a | 201 | static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, |
ba4e7d97 TH |
202 | void **virtual) |
203 | { | |
eba67093 | 204 | struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; |
ba4e7d97 TH |
205 | int ret; |
206 | void *addr; | |
207 | ||
208 | *virtual = NULL; | |
eba67093 | 209 | (void) ttm_mem_io_lock(man, false); |
82c5da6b | 210 | ret = ttm_mem_io_reserve(bdev, mem); |
eba67093 | 211 | ttm_mem_io_unlock(man); |
9e51159c | 212 | if (ret || !mem->bus.is_iomem) |
ba4e7d97 TH |
213 | return ret; |
214 | ||
82c5da6b JG |
215 | if (mem->bus.addr) { |
216 | addr = mem->bus.addr; | |
217 | } else { | |
ba4e7d97 | 218 | if (mem->placement & TTM_PL_FLAG_WC) |
82c5da6b | 219 | addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size); |
ba4e7d97 | 220 | else |
4bdc0d67 | 221 | addr = ioremap(mem->bus.base + mem->bus.offset, mem->bus.size); |
82c5da6b | 222 | if (!addr) { |
eba67093 | 223 | (void) ttm_mem_io_lock(man, false); |
82c5da6b | 224 | ttm_mem_io_free(bdev, mem); |
eba67093 | 225 | ttm_mem_io_unlock(man); |
ba4e7d97 | 226 | return -ENOMEM; |
82c5da6b | 227 | } |
ba4e7d97 TH |
228 | } |
229 | *virtual = addr; | |
230 | return 0; | |
231 | } | |
232 | ||
dcbff15a | 233 | static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, |
ba4e7d97 TH |
234 | void *virtual) |
235 | { | |
236 | struct ttm_mem_type_manager *man; | |
237 | ||
238 | man = &bdev->man[mem->mem_type]; | |
239 | ||
0c321c79 | 240 | if (virtual && mem->bus.addr == NULL) |
ba4e7d97 | 241 | iounmap(virtual); |
eba67093 | 242 | (void) ttm_mem_io_lock(man, false); |
82c5da6b | 243 | ttm_mem_io_free(bdev, mem); |
eba67093 | 244 | ttm_mem_io_unlock(man); |
ba4e7d97 TH |
245 | } |
246 | ||
247 | static int ttm_copy_io_page(void *dst, void *src, unsigned long page) | |
248 | { | |
249 | uint32_t *dstP = | |
250 | (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); | |
251 | uint32_t *srcP = | |
252 | (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); | |
253 | ||
254 | int i; | |
255 | for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) | |
256 | iowrite32(ioread32(srcP++), dstP++); | |
257 | return 0; | |
258 | } | |
259 | ||
403c1826 TH |
260 | #ifdef CONFIG_X86 |
261 | #define __ttm_kmap_atomic_prot(__page, __prot) kmap_atomic_prot(__page, __prot) | |
262 | #define __ttm_kunmap_atomic(__addr) kunmap_atomic(__addr) | |
263 | #else | |
264 | #define __ttm_kmap_atomic_prot(__page, __prot) vmap(&__page, 1, 0, __prot) | |
265 | #define __ttm_kunmap_atomic(__addr) vunmap(__addr) | |
266 | #endif | |
267 | ||
9c11fcf1 TH |
268 | |
269 | /** | |
270 | * ttm_kmap_atomic_prot - Efficient kernel map of a single page with | |
271 | * specified page protection. | |
272 | * | |
273 | * @page: The page to map. | |
274 | * @prot: The page protection. | |
275 | * | |
276 | * This function maps a TTM page using the kmap_atomic api if available, | |
277 | * otherwise falls back to vmap. The user must make sure that the | |
278 | * specified page does not have an aliased mapping with a different caching | |
279 | * policy unless the architecture explicitly allows it. Also mapping and | |
280 | * unmapping using this api must be correctly nested. Unmapping should | |
281 | * occur in the reverse order of mapping. | |
282 | */ | |
283 | void *ttm_kmap_atomic_prot(struct page *page, pgprot_t prot) | |
403c1826 TH |
284 | { |
285 | if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) | |
286 | return kmap_atomic(page); | |
287 | else | |
288 | return __ttm_kmap_atomic_prot(page, prot); | |
289 | } | |
9c11fcf1 | 290 | EXPORT_SYMBOL(ttm_kmap_atomic_prot); |
403c1826 | 291 | |
9c11fcf1 TH |
292 | /** |
293 | * ttm_kunmap_atomic_prot - Unmap a page that was mapped using | |
294 | * ttm_kmap_atomic_prot. | |
295 | * | |
296 | * @addr: The virtual address from the map. | |
297 | * @prot: The page protection. | |
298 | */ | |
299 | void ttm_kunmap_atomic_prot(void *addr, pgprot_t prot) | |
403c1826 TH |
300 | { |
301 | if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) | |
302 | kunmap_atomic(addr); | |
303 | else | |
304 | __ttm_kunmap_atomic(addr); | |
305 | } | |
9c11fcf1 | 306 | EXPORT_SYMBOL(ttm_kunmap_atomic_prot); |
403c1826 | 307 | |
ba4e7d97 | 308 | static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, |
542c6f6d TH |
309 | unsigned long page, |
310 | pgprot_t prot) | |
ba4e7d97 | 311 | { |
b1e5f172 | 312 | struct page *d = ttm->pages[page]; |
ba4e7d97 TH |
313 | void *dst; |
314 | ||
315 | if (!d) | |
316 | return -ENOMEM; | |
317 | ||
318 | src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); | |
403c1826 | 319 | dst = ttm_kmap_atomic_prot(d, prot); |
ba4e7d97 TH |
320 | if (!dst) |
321 | return -ENOMEM; | |
322 | ||
323 | memcpy_fromio(dst, src, PAGE_SIZE); | |
542c6f6d | 324 | |
403c1826 | 325 | ttm_kunmap_atomic_prot(dst, prot); |
542c6f6d | 326 | |
ba4e7d97 TH |
327 | return 0; |
328 | } | |
329 | ||
330 | static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, | |
542c6f6d TH |
331 | unsigned long page, |
332 | pgprot_t prot) | |
ba4e7d97 | 333 | { |
b1e5f172 | 334 | struct page *s = ttm->pages[page]; |
ba4e7d97 TH |
335 | void *src; |
336 | ||
337 | if (!s) | |
338 | return -ENOMEM; | |
339 | ||
340 | dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); | |
403c1826 | 341 | src = ttm_kmap_atomic_prot(s, prot); |
ba4e7d97 TH |
342 | if (!src) |
343 | return -ENOMEM; | |
344 | ||
345 | memcpy_toio(dst, src, PAGE_SIZE); | |
542c6f6d | 346 | |
403c1826 | 347 | ttm_kunmap_atomic_prot(src, prot); |
542c6f6d | 348 | |
ba4e7d97 TH |
349 | return 0; |
350 | } | |
351 | ||
352 | int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, | |
3e98d829 | 353 | struct ttm_operation_ctx *ctx, |
9d87fa21 | 354 | struct ttm_mem_reg *new_mem) |
ba4e7d97 TH |
355 | { |
356 | struct ttm_bo_device *bdev = bo->bdev; | |
357 | struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; | |
358 | struct ttm_tt *ttm = bo->ttm; | |
359 | struct ttm_mem_reg *old_mem = &bo->mem; | |
e22469ca | 360 | struct ttm_mem_reg old_copy = *old_mem; |
ba4e7d97 TH |
361 | void *old_iomap; |
362 | void *new_iomap; | |
363 | int ret; | |
ba4e7d97 TH |
364 | unsigned long i; |
365 | unsigned long page; | |
366 | unsigned long add = 0; | |
367 | int dir; | |
368 | ||
3e98d829 | 369 | ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); |
77dfc28b CK |
370 | if (ret) |
371 | return ret; | |
372 | ||
ba4e7d97 TH |
373 | ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); |
374 | if (ret) | |
375 | return ret; | |
376 | ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); | |
377 | if (ret) | |
378 | goto out; | |
379 | ||
da95c788 TH |
380 | /* |
381 | * Single TTM move. NOP. | |
382 | */ | |
ba4e7d97 TH |
383 | if (old_iomap == NULL && new_iomap == NULL) |
384 | goto out2; | |
da95c788 TH |
385 | |
386 | /* | |
0bc25425 | 387 | * Don't move nonexistent data. Clear destination instead. |
da95c788 | 388 | */ |
0bc25425 | 389 | if (old_iomap == NULL && |
2e6d8b46 TH |
390 | (ttm == NULL || (ttm->state == tt_unpopulated && |
391 | !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { | |
0bc25425 | 392 | memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); |
ba4e7d97 | 393 | goto out2; |
0bc25425 | 394 | } |
ba4e7d97 | 395 | |
da95c788 TH |
396 | /* |
397 | * TTM might be null for moves within the same region. | |
9a0599dd | 398 | */ |
25893a14 CK |
399 | if (ttm) { |
400 | ret = ttm_tt_populate(ttm, ctx); | |
da95c788 | 401 | if (ret) |
b1e5f172 JG |
402 | goto out1; |
403 | } | |
404 | ||
ba4e7d97 TH |
405 | add = 0; |
406 | dir = 1; | |
407 | ||
408 | if ((old_mem->mem_type == new_mem->mem_type) && | |
d961db75 | 409 | (new_mem->start < old_mem->start + old_mem->size)) { |
ba4e7d97 TH |
410 | dir = -1; |
411 | add = new_mem->num_pages - 1; | |
412 | } | |
413 | ||
414 | for (i = 0; i < new_mem->num_pages; ++i) { | |
415 | page = i * dir + add; | |
542c6f6d TH |
416 | if (old_iomap == NULL) { |
417 | pgprot_t prot = ttm_io_prot(old_mem->placement, | |
418 | PAGE_KERNEL); | |
419 | ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, | |
420 | prot); | |
421 | } else if (new_iomap == NULL) { | |
422 | pgprot_t prot = ttm_io_prot(new_mem->placement, | |
423 | PAGE_KERNEL); | |
424 | ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, | |
425 | prot); | |
449f797a | 426 | } else { |
ba4e7d97 | 427 | ret = ttm_copy_io_page(new_iomap, old_iomap, page); |
449f797a | 428 | } |
da95c788 | 429 | if (ret) |
ba4e7d97 TH |
430 | goto out1; |
431 | } | |
432 | mb(); | |
433 | out2: | |
eba67093 | 434 | old_copy = *old_mem; |
ba4e7d97 TH |
435 | *old_mem = *new_mem; |
436 | new_mem->mm_node = NULL; | |
ba4e7d97 | 437 | |
4279cb14 | 438 | if (man->flags & TTM_MEMTYPE_FLAG_FIXED) { |
ba4e7d97 TH |
439 | ttm_tt_destroy(ttm); |
440 | bo->ttm = NULL; | |
441 | } | |
442 | ||
443 | out1: | |
eba67093 | 444 | ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); |
ba4e7d97 TH |
445 | out: |
446 | ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); | |
da95c788 TH |
447 | |
448 | /* | |
449 | * On error, keep the mm node! | |
450 | */ | |
451 | if (!ret) | |
452 | ttm_bo_mem_put(bo, &old_copy); | |
ba4e7d97 TH |
453 | return ret; |
454 | } | |
455 | EXPORT_SYMBOL(ttm_bo_move_memcpy); | |
456 | ||
457 | static void ttm_transfered_destroy(struct ttm_buffer_object *bo) | |
458 | { | |
5452cf44 CK |
459 | struct ttm_transfer_obj *fbo; |
460 | ||
461 | fbo = container_of(bo, struct ttm_transfer_obj, base); | |
f4490759 | 462 | ttm_bo_put(fbo->bo); |
5452cf44 | 463 | kfree(fbo); |
ba4e7d97 TH |
464 | } |
465 | ||
466 | /** | |
467 | * ttm_buffer_object_transfer | |
468 | * | |
469 | * @bo: A pointer to a struct ttm_buffer_object. | |
470 | * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, | |
471 | * holding the data of @bo with the old placement. | |
472 | * | |
473 | * This is a utility function that may be called after an accelerated move | |
474 | * has been scheduled. A new buffer object is created as a placeholder for | |
475 | * the old data while it's being copied. When that buffer object is idle, | |
476 | * it can be destroyed, releasing the space of the old placement. | |
477 | * Returns: | |
478 | * !0: Failure. | |
479 | */ | |
480 | ||
481 | static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, | |
482 | struct ttm_buffer_object **new_obj) | |
483 | { | |
5452cf44 | 484 | struct ttm_transfer_obj *fbo; |
5e338405 | 485 | int ret; |
ba4e7d97 | 486 | |
ff7c60c5 | 487 | fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); |
ba4e7d97 TH |
488 | if (!fbo) |
489 | return -ENOMEM; | |
490 | ||
5452cf44 | 491 | fbo->base = *bo; |
d6e820fc CK |
492 | fbo->base.mem.placement |= TTM_PL_FLAG_NO_EVICT; |
493 | ||
494 | ttm_bo_get(bo); | |
8129fdad | 495 | fbo->bo = bo; |
ba4e7d97 TH |
496 | |
497 | /** | |
498 | * Fix up members that we shouldn't copy directly: | |
499 | * TODO: Explicit member copy would probably be better here. | |
500 | */ | |
501 | ||
97588b5b | 502 | atomic_inc(&ttm_bo_glob.bo_count); |
5452cf44 CK |
503 | INIT_LIST_HEAD(&fbo->base.ddestroy); |
504 | INIT_LIST_HEAD(&fbo->base.lru); | |
505 | INIT_LIST_HEAD(&fbo->base.swap); | |
506 | INIT_LIST_HEAD(&fbo->base.io_reserve_lru); | |
5452cf44 | 507 | fbo->base.moving = NULL; |
b96f3e7c | 508 | drm_vma_node_reset(&fbo->base.base.vma_node); |
5452cf44 CK |
509 | |
510 | kref_init(&fbo->base.list_kref); | |
511 | kref_init(&fbo->base.kref); | |
512 | fbo->base.destroy = &ttm_transfered_destroy; | |
513 | fbo->base.acc_size = 0; | |
ef383218 CK |
514 | if (bo->base.resv == &bo->base._resv) |
515 | fbo->base.base.resv = &fbo->base.base._resv; | |
516 | ||
517 | dma_resv_init(&fbo->base.base._resv); | |
8c8c0620 | 518 | fbo->base.base.dev = NULL; |
ef383218 | 519 | ret = dma_resv_trylock(&fbo->base.base._resv); |
5e338405 | 520 | WARN_ON(!ret); |
ba4e7d97 | 521 | |
5452cf44 | 522 | *new_obj = &fbo->base; |
ba4e7d97 TH |
523 | return 0; |
524 | } | |
525 | ||
526 | pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) | |
527 | { | |
94318d50 BH |
528 | /* Cached mappings need no adjustment */ |
529 | if (caching_flags & TTM_PL_FLAG_CACHED) | |
530 | return tmp; | |
531 | ||
ba4e7d97 TH |
532 | #if defined(__i386__) || defined(__x86_64__) |
533 | if (caching_flags & TTM_PL_FLAG_WC) | |
534 | tmp = pgprot_writecombine(tmp); | |
535 | else if (boot_cpu_data.x86 > 3) | |
536 | tmp = pgprot_noncached(tmp); | |
ba4e7d97 | 537 | #endif |
f135b978 | 538 | #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ |
8a08e50c | 539 | defined(__powerpc__) || defined(__mips__) |
ba4e7d97 TH |
540 | if (caching_flags & TTM_PL_FLAG_WC) |
541 | tmp = pgprot_writecombine(tmp); | |
542 | else | |
543 | tmp = pgprot_noncached(tmp); | |
544 | #endif | |
8a08e50c | 545 | #if defined(__sparc__) |
94318d50 | 546 | tmp = pgprot_noncached(tmp); |
ba4e7d97 TH |
547 | #endif |
548 | return tmp; | |
549 | } | |
4bfd75cb | 550 | EXPORT_SYMBOL(ttm_io_prot); |
ba4e7d97 TH |
551 | |
552 | static int ttm_bo_ioremap(struct ttm_buffer_object *bo, | |
82c5da6b JG |
553 | unsigned long offset, |
554 | unsigned long size, | |
ba4e7d97 TH |
555 | struct ttm_bo_kmap_obj *map) |
556 | { | |
ba4e7d97 | 557 | struct ttm_mem_reg *mem = &bo->mem; |
ba4e7d97 | 558 | |
82c5da6b | 559 | if (bo->mem.bus.addr) { |
ba4e7d97 | 560 | map->bo_kmap_type = ttm_bo_map_premapped; |
82c5da6b | 561 | map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); |
ba4e7d97 TH |
562 | } else { |
563 | map->bo_kmap_type = ttm_bo_map_iomap; | |
564 | if (mem->placement & TTM_PL_FLAG_WC) | |
82c5da6b JG |
565 | map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset, |
566 | size); | |
ba4e7d97 | 567 | else |
4bdc0d67 | 568 | map->virtual = ioremap(bo->mem.bus.base + bo->mem.bus.offset + offset, |
82c5da6b | 569 | size); |
ba4e7d97 TH |
570 | } |
571 | return (!map->virtual) ? -ENOMEM : 0; | |
572 | } | |
573 | ||
574 | static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, | |
575 | unsigned long start_page, | |
576 | unsigned long num_pages, | |
577 | struct ttm_bo_kmap_obj *map) | |
578 | { | |
d0cef9fa RH |
579 | struct ttm_mem_reg *mem = &bo->mem; |
580 | struct ttm_operation_ctx ctx = { | |
581 | .interruptible = false, | |
582 | .no_wait_gpu = false | |
583 | }; | |
ba4e7d97 | 584 | struct ttm_tt *ttm = bo->ttm; |
d0cef9fa | 585 | pgprot_t prot; |
b1e5f172 | 586 | int ret; |
ba4e7d97 TH |
587 | |
588 | BUG_ON(!ttm); | |
b1e5f172 | 589 | |
25893a14 CK |
590 | ret = ttm_tt_populate(ttm, &ctx); |
591 | if (ret) | |
592 | return ret; | |
b1e5f172 | 593 | |
ba4e7d97 TH |
594 | if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { |
595 | /* | |
596 | * We're mapping a single page, and the desired | |
597 | * page protection is consistent with the bo. | |
598 | */ | |
599 | ||
600 | map->bo_kmap_type = ttm_bo_map_kmap; | |
b1e5f172 | 601 | map->page = ttm->pages[start_page]; |
ba4e7d97 TH |
602 | map->virtual = kmap(map->page); |
603 | } else { | |
ba4e7d97 TH |
604 | /* |
605 | * We need to use vmap to get the desired page protection | |
af901ca1 | 606 | * or to make the buffer object look contiguous. |
ba4e7d97 | 607 | */ |
94318d50 | 608 | prot = ttm_io_prot(mem->placement, PAGE_KERNEL); |
ba4e7d97 TH |
609 | map->bo_kmap_type = ttm_bo_map_vmap; |
610 | map->virtual = vmap(ttm->pages + start_page, num_pages, | |
611 | 0, prot); | |
612 | } | |
613 | return (!map->virtual) ? -ENOMEM : 0; | |
614 | } | |
615 | ||
616 | int ttm_bo_kmap(struct ttm_buffer_object *bo, | |
617 | unsigned long start_page, unsigned long num_pages, | |
618 | struct ttm_bo_kmap_obj *map) | |
619 | { | |
eba67093 TH |
620 | struct ttm_mem_type_manager *man = |
621 | &bo->bdev->man[bo->mem.mem_type]; | |
82c5da6b | 622 | unsigned long offset, size; |
ba4e7d97 | 623 | int ret; |
ba4e7d97 | 624 | |
ba4e7d97 | 625 | map->virtual = NULL; |
82c5da6b | 626 | map->bo = bo; |
ba4e7d97 TH |
627 | if (num_pages > bo->num_pages) |
628 | return -EINVAL; | |
629 | if (start_page > bo->num_pages) | |
630 | return -EINVAL; | |
02b29caf | 631 | |
eba67093 | 632 | (void) ttm_mem_io_lock(man, false); |
82c5da6b | 633 | ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); |
eba67093 | 634 | ttm_mem_io_unlock(man); |
ba4e7d97 TH |
635 | if (ret) |
636 | return ret; | |
82c5da6b | 637 | if (!bo->mem.bus.is_iomem) { |
ba4e7d97 TH |
638 | return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); |
639 | } else { | |
82c5da6b JG |
640 | offset = start_page << PAGE_SHIFT; |
641 | size = num_pages << PAGE_SHIFT; | |
642 | return ttm_bo_ioremap(bo, offset, size, map); | |
ba4e7d97 TH |
643 | } |
644 | } | |
645 | EXPORT_SYMBOL(ttm_bo_kmap); | |
646 | ||
647 | void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) | |
648 | { | |
eba67093 TH |
649 | struct ttm_buffer_object *bo = map->bo; |
650 | struct ttm_mem_type_manager *man = | |
651 | &bo->bdev->man[bo->mem.mem_type]; | |
652 | ||
ba4e7d97 TH |
653 | if (!map->virtual) |
654 | return; | |
655 | switch (map->bo_kmap_type) { | |
656 | case ttm_bo_map_iomap: | |
657 | iounmap(map->virtual); | |
658 | break; | |
659 | case ttm_bo_map_vmap: | |
660 | vunmap(map->virtual); | |
661 | break; | |
662 | case ttm_bo_map_kmap: | |
663 | kunmap(map->page); | |
664 | break; | |
665 | case ttm_bo_map_premapped: | |
666 | break; | |
667 | default: | |
668 | BUG(); | |
669 | } | |
eba67093 TH |
670 | (void) ttm_mem_io_lock(man, false); |
671 | ttm_mem_io_free(map->bo->bdev, &map->bo->mem); | |
672 | ttm_mem_io_unlock(man); | |
ba4e7d97 TH |
673 | map->virtual = NULL; |
674 | map->page = NULL; | |
675 | } | |
676 | EXPORT_SYMBOL(ttm_bo_kunmap); | |
677 | ||
ba4e7d97 | 678 | int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, |
f54d1867 | 679 | struct dma_fence *fence, |
97a875cb | 680 | bool evict, |
ba4e7d97 TH |
681 | struct ttm_mem_reg *new_mem) |
682 | { | |
683 | struct ttm_bo_device *bdev = bo->bdev; | |
ba4e7d97 TH |
684 | struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; |
685 | struct ttm_mem_reg *old_mem = &bo->mem; | |
686 | int ret; | |
ba4e7d97 | 687 | struct ttm_buffer_object *ghost_obj; |
ba4e7d97 | 688 | |
52791eee | 689 | dma_resv_add_excl_fence(bo->base.resv, fence); |
ba4e7d97 | 690 | if (evict) { |
8aa6d4fc | 691 | ret = ttm_bo_wait(bo, false, false); |
ba4e7d97 TH |
692 | if (ret) |
693 | return ret; | |
694 | ||
4279cb14 | 695 | if (man->flags & TTM_MEMTYPE_FLAG_FIXED) { |
ba4e7d97 TH |
696 | ttm_tt_destroy(bo->ttm); |
697 | bo->ttm = NULL; | |
698 | } | |
eac20953 | 699 | ttm_bo_free_old_node(bo); |
ba4e7d97 TH |
700 | } else { |
701 | /** | |
702 | * This should help pipeline ordinary buffer moves. | |
703 | * | |
704 | * Hang old buffer memory on a new buffer object, | |
705 | * and leave it to be released when the GPU | |
706 | * operation has completed. | |
707 | */ | |
708 | ||
f54d1867 CW |
709 | dma_fence_put(bo->moving); |
710 | bo->moving = dma_fence_get(fence); | |
ba4e7d97 | 711 | |
ff7c60c5 | 712 | ret = ttm_buffer_object_transfer(bo, &ghost_obj); |
ba4e7d97 TH |
713 | if (ret) |
714 | return ret; | |
715 | ||
ef383218 | 716 | dma_resv_add_excl_fence(&ghost_obj->base._resv, fence); |
f2c24b83 | 717 | |
ba4e7d97 TH |
718 | /** |
719 | * If we're not moving to fixed memory, the TTM object | |
720 | * needs to stay alive. Otherwhise hang it on the ghost | |
721 | * bo to be unbound and destroyed. | |
722 | */ | |
723 | ||
724 | if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) | |
725 | ghost_obj->ttm = NULL; | |
726 | else | |
727 | bo->ttm = NULL; | |
728 | ||
ef383218 | 729 | dma_resv_unlock(&ghost_obj->base._resv); |
f4490759 | 730 | ttm_bo_put(ghost_obj); |
ba4e7d97 TH |
731 | } |
732 | ||
733 | *old_mem = *new_mem; | |
734 | new_mem->mm_node = NULL; | |
110b20c3 | 735 | |
ba4e7d97 TH |
736 | return 0; |
737 | } | |
738 | EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); | |
3ddf4ad9 CK |
739 | |
740 | int ttm_bo_pipeline_move(struct ttm_buffer_object *bo, | |
f54d1867 | 741 | struct dma_fence *fence, bool evict, |
3ddf4ad9 CK |
742 | struct ttm_mem_reg *new_mem) |
743 | { | |
744 | struct ttm_bo_device *bdev = bo->bdev; | |
745 | struct ttm_mem_reg *old_mem = &bo->mem; | |
746 | ||
747 | struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type]; | |
748 | struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type]; | |
749 | ||
750 | int ret; | |
751 | ||
52791eee | 752 | dma_resv_add_excl_fence(bo->base.resv, fence); |
3ddf4ad9 CK |
753 | |
754 | if (!evict) { | |
755 | struct ttm_buffer_object *ghost_obj; | |
756 | ||
757 | /** | |
758 | * This should help pipeline ordinary buffer moves. | |
759 | * | |
760 | * Hang old buffer memory on a new buffer object, | |
761 | * and leave it to be released when the GPU | |
762 | * operation has completed. | |
763 | */ | |
764 | ||
f54d1867 CW |
765 | dma_fence_put(bo->moving); |
766 | bo->moving = dma_fence_get(fence); | |
3ddf4ad9 CK |
767 | |
768 | ret = ttm_buffer_object_transfer(bo, &ghost_obj); | |
769 | if (ret) | |
770 | return ret; | |
771 | ||
ef383218 | 772 | dma_resv_add_excl_fence(&ghost_obj->base._resv, fence); |
3ddf4ad9 CK |
773 | |
774 | /** | |
775 | * If we're not moving to fixed memory, the TTM object | |
776 | * needs to stay alive. Otherwhise hang it on the ghost | |
777 | * bo to be unbound and destroyed. | |
778 | */ | |
779 | ||
780 | if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED)) | |
781 | ghost_obj->ttm = NULL; | |
782 | else | |
783 | bo->ttm = NULL; | |
784 | ||
ef383218 | 785 | dma_resv_unlock(&ghost_obj->base._resv); |
f4490759 | 786 | ttm_bo_put(ghost_obj); |
3ddf4ad9 CK |
787 | |
788 | } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) { | |
789 | ||
790 | /** | |
791 | * BO doesn't have a TTM we need to bind/unbind. Just remember | |
792 | * this eviction and free up the allocation | |
793 | */ | |
794 | ||
795 | spin_lock(&from->move_lock); | |
f54d1867 CW |
796 | if (!from->move || dma_fence_is_later(fence, from->move)) { |
797 | dma_fence_put(from->move); | |
798 | from->move = dma_fence_get(fence); | |
3ddf4ad9 CK |
799 | } |
800 | spin_unlock(&from->move_lock); | |
801 | ||
802 | ttm_bo_free_old_node(bo); | |
803 | ||
f54d1867 CW |
804 | dma_fence_put(bo->moving); |
805 | bo->moving = dma_fence_get(fence); | |
3ddf4ad9 CK |
806 | |
807 | } else { | |
808 | /** | |
809 | * Last resort, wait for the move to be completed. | |
810 | * | |
811 | * Should never happen in pratice. | |
812 | */ | |
813 | ||
814 | ret = ttm_bo_wait(bo, false, false); | |
815 | if (ret) | |
816 | return ret; | |
817 | ||
818 | if (to->flags & TTM_MEMTYPE_FLAG_FIXED) { | |
819 | ttm_tt_destroy(bo->ttm); | |
820 | bo->ttm = NULL; | |
821 | } | |
822 | ttm_bo_free_old_node(bo); | |
823 | } | |
824 | ||
825 | *old_mem = *new_mem; | |
826 | new_mem->mm_node = NULL; | |
827 | ||
828 | return 0; | |
829 | } | |
830 | EXPORT_SYMBOL(ttm_bo_pipeline_move); | |
5d951098 CK |
831 | |
832 | int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) | |
833 | { | |
834 | struct ttm_buffer_object *ghost; | |
835 | int ret; | |
836 | ||
837 | ret = ttm_buffer_object_transfer(bo, &ghost); | |
838 | if (ret) | |
839 | return ret; | |
840 | ||
ef383218 | 841 | ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); |
5d951098 CK |
842 | /* Last resort, wait for the BO to be idle when we are OOM */ |
843 | if (ret) | |
844 | ttm_bo_wait(bo, false, false); | |
845 | ||
846 | memset(&bo->mem, 0, sizeof(bo->mem)); | |
847 | bo->mem.mem_type = TTM_PL_SYSTEM; | |
848 | bo->ttm = NULL; | |
849 | ||
ef383218 | 850 | dma_resv_unlock(&ghost->base._resv); |
f4490759 | 851 | ttm_bo_put(ghost); |
5d951098 CK |
852 | |
853 | return 0; | |
854 | } |