Commit | Line | Data |
---|---|---|
1297bf2e | 1 | /* SPDX-License-Identifier: GPL-2.0 OR MIT */ |
ba4e7d97 TH |
2 | /************************************************************************** |
3 | * | |
4 | * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA | |
5 | * All Rights Reserved. | |
6 | * | |
7 | * Permission is hereby granted, free of charge, to any person obtaining a | |
8 | * copy of this software and associated documentation files (the | |
9 | * "Software"), to deal in the Software without restriction, including | |
10 | * without limitation the rights to use, copy, modify, merge, publish, | |
11 | * distribute, sub license, and/or sell copies of the Software, and to | |
12 | * permit persons to whom the Software is furnished to do so, subject to | |
13 | * the following conditions: | |
14 | * | |
15 | * The above copyright notice and this permission notice (including the | |
16 | * next paragraph) shall be included in all copies or substantial portions | |
17 | * of the Software. | |
18 | * | |
19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
20 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
21 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
22 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
23 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
24 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
25 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
26 | * | |
27 | **************************************************************************/ | |
28 | /* | |
29 | * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> | |
30 | */ | |
31 | ||
59437c79 TZ |
32 | #include <linux/vmalloc.h> |
33 | ||
a3185f91 | 34 | #include <drm/ttm/ttm_bo.h> |
760285e7 | 35 | #include <drm/ttm/ttm_placement.h> |
a3185f91 CK |
36 | #include <drm/ttm/ttm_tt.h> |
37 | ||
053c5769 | 38 | #include <drm/drm_cache.h> |
ba4e7d97 | 39 | |
5452cf44 CK |
40 | struct ttm_transfer_obj { |
41 | struct ttm_buffer_object base; | |
42 | struct ttm_buffer_object *bo; | |
43 | }; | |
44 | ||
8af8a109 | 45 | int ttm_mem_io_reserve(struct ttm_device *bdev, |
2966141a | 46 | struct ttm_resource *mem) |
eba67093 | 47 | { |
54d04ea8 | 48 | if (mem->bus.offset || mem->bus.addr) |
c1c440d4 | 49 | return 0; |
eba67093 | 50 | |
fe662d84 | 51 | mem->bus.is_iomem = false; |
8af8a109 | 52 | if (!bdev->funcs->io_mem_reserve) |
eba67093 | 53 | return 0; |
eba67093 | 54 | |
8af8a109 | 55 | return bdev->funcs->io_mem_reserve(bdev, mem); |
eba67093 TH |
56 | } |
57 | ||
8af8a109 | 58 | void ttm_mem_io_free(struct ttm_device *bdev, |
2966141a | 59 | struct ttm_resource *mem) |
eba67093 | 60 | { |
2dbd9c27 PS |
61 | if (!mem) |
62 | return; | |
63 | ||
54d04ea8 | 64 | if (!mem->bus.offset && !mem->bus.addr) |
c1c440d4 | 65 | return; |
eba67093 | 66 | |
8af8a109 CK |
67 | if (bdev->funcs->io_mem_free) |
68 | bdev->funcs->io_mem_free(bdev, mem); | |
c1c440d4 | 69 | |
fe662d84 CK |
70 | mem->bus.offset = 0; |
71 | mem->bus.addr = NULL; | |
82c5da6b JG |
72 | } |
73 | ||
3bf3710e TH |
74 | /** |
75 | * ttm_move_memcpy - Helper to perform a memcpy ttm move operation. | |
66907633 TH |
76 | * @clear: Whether to clear rather than copy. |
77 | * @num_pages: Number of pages of the operation. | |
78 | * @dst_iter: A struct ttm_kmap_iter representing the destination resource. | |
3bf3710e TH |
79 | * @src_iter: A struct ttm_kmap_iter representing the source resource. |
80 | * | |
81 | * This function is intended to be able to move out async under a | |
82 | * dma-fence if desired. | |
83 | */ | |
66907633 | 84 | void ttm_move_memcpy(bool clear, |
3bf3710e TH |
85 | u32 num_pages, |
86 | struct ttm_kmap_iter *dst_iter, | |
87 | struct ttm_kmap_iter *src_iter) | |
ba4e7d97 | 88 | { |
3bf3710e TH |
89 | const struct ttm_kmap_iter_ops *dst_ops = dst_iter->ops; |
90 | const struct ttm_kmap_iter_ops *src_ops = src_iter->ops; | |
7938f421 | 91 | struct iosys_map src_map, dst_map; |
3bf3710e | 92 | pgoff_t i; |
ba4e7d97 | 93 | |
3bf3710e TH |
94 | /* Single TTM move. NOP */ |
95 | if (dst_ops->maps_tt && src_ops->maps_tt) | |
96 | return; | |
ebb21aa1 | 97 | |
3bf3710e | 98 | /* Don't move nonexistent data. Clear destination instead. */ |
66907633 | 99 | if (clear) { |
3bf3710e TH |
100 | for (i = 0; i < num_pages; ++i) { |
101 | dst_ops->map_local(dst_iter, &dst_map, i); | |
102 | if (dst_map.is_iomem) | |
103 | memset_io(dst_map.vaddr_iomem, 0, PAGE_SIZE); | |
104 | else | |
105 | memset(dst_map.vaddr, 0, PAGE_SIZE); | |
106 | if (dst_ops->unmap_local) | |
107 | dst_ops->unmap_local(dst_iter, &dst_map); | |
82c5da6b | 108 | } |
3bf3710e | 109 | return; |
ba4e7d97 | 110 | } |
ba4e7d97 | 111 | |
3bf3710e TH |
112 | for (i = 0; i < num_pages; ++i) { |
113 | dst_ops->map_local(dst_iter, &dst_map, i); | |
114 | src_ops->map_local(src_iter, &src_map, i); | |
115 | ||
053c5769 TH |
116 | drm_memcpy_from_wc(&dst_map, &src_map, PAGE_SIZE); |
117 | ||
3bf3710e TH |
118 | if (src_ops->unmap_local) |
119 | src_ops->unmap_local(src_iter, &src_map); | |
120 | if (dst_ops->unmap_local) | |
121 | dst_ops->unmap_local(dst_iter, &dst_map); | |
122 | } | |
ba4e7d97 | 123 | } |
3bf3710e | 124 | EXPORT_SYMBOL(ttm_move_memcpy); |
ba4e7d97 | 125 | |
a3185f91 CK |
126 | /** |
127 | * ttm_bo_move_memcpy | |
128 | * | |
129 | * @bo: A pointer to a struct ttm_buffer_object. | |
d3e83448 CK |
130 | * @ctx: operation context |
131 | * @dst_mem: struct ttm_resource indicating where to move. | |
a3185f91 CK |
132 | * |
133 | * Fallback move function for a mappable buffer object in mappable memory. | |
134 | * The function will, if successful, | |
135 | * free any old aperture space, and set (@new_mem)->mm_node to NULL, | |
136 | * and update the (@bo)->mem placement flags. If unsuccessful, the old | |
137 | * data remains untouched, and it's up to the caller to free the | |
138 | * memory space indicated by @new_mem. | |
139 | * Returns: | |
140 | * !0: Failure. | |
141 | */ | |
ba4e7d97 | 142 | int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, |
3e98d829 | 143 | struct ttm_operation_ctx *ctx, |
3bf3710e | 144 | struct ttm_resource *dst_mem) |
ba4e7d97 | 145 | { |
8af8a109 | 146 | struct ttm_device *bdev = bo->bdev; |
3bf3710e TH |
147 | struct ttm_resource_manager *dst_man = |
148 | ttm_manager_type(bo->bdev, dst_mem->mem_type); | |
ba4e7d97 | 149 | struct ttm_tt *ttm = bo->ttm; |
3bf3710e | 150 | struct ttm_resource *src_mem = bo->resource; |
4d8f6854 | 151 | struct ttm_resource_manager *src_man; |
3bf3710e TH |
152 | union { |
153 | struct ttm_kmap_iter_tt tt; | |
154 | struct ttm_kmap_iter_linear_io io; | |
155 | } _dst_iter, _src_iter; | |
156 | struct ttm_kmap_iter *dst_iter, *src_iter; | |
66907633 | 157 | bool clear; |
3bf3710e TH |
158 | int ret = 0; |
159 | ||
c604d319 MA |
160 | if (WARN_ON(!src_mem)) |
161 | return -EINVAL; | |
4d8f6854 CK |
162 | |
163 | src_man = ttm_manager_type(bdev, src_mem->mem_type); | |
43d46f0b | 164 | if (ttm && ((ttm->page_flags & TTM_TT_FLAG_SWAPPED) || |
3bf3710e | 165 | dst_man->use_tt)) { |
0a667b50 | 166 | ret = ttm_tt_populate(bdev, ttm, ctx); |
da95c788 | 167 | if (ret) |
3bf3710e | 168 | return ret; |
b1e5f172 JG |
169 | } |
170 | ||
3bf3710e TH |
171 | dst_iter = ttm_kmap_iter_linear_io_init(&_dst_iter.io, bdev, dst_mem); |
172 | if (PTR_ERR(dst_iter) == -EINVAL && dst_man->use_tt) | |
173 | dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm); | |
174 | if (IS_ERR(dst_iter)) | |
175 | return PTR_ERR(dst_iter); | |
176 | ||
177 | src_iter = ttm_kmap_iter_linear_io_init(&_src_iter.io, bdev, src_mem); | |
178 | if (PTR_ERR(src_iter) == -EINVAL && src_man->use_tt) | |
179 | src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm); | |
180 | if (IS_ERR(src_iter)) { | |
181 | ret = PTR_ERR(src_iter); | |
182 | goto out_src_iter; | |
bfa3357e | 183 | } |
2ee476f7 | 184 | |
66907633 | 185 | clear = src_iter->ops->maps_tt && (!ttm || !ttm_tt_is_populated(ttm)); |
1176d15f | 186 | if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) |
040b35c1 | 187 | ttm_move_memcpy(clear, PFN_UP(dst_mem->size), dst_iter, src_iter); |
ba4e7d97 | 188 | |
3bf3710e | 189 | if (!src_iter->ops->maps_tt) |
efcefc71 TH |
190 | ttm_kmap_iter_linear_io_fini(&_src_iter.io, bdev, src_mem); |
191 | ttm_bo_move_sync_cleanup(bo, dst_mem); | |
192 | ||
3bf3710e TH |
193 | out_src_iter: |
194 | if (!dst_iter->ops->maps_tt) | |
195 | ttm_kmap_iter_linear_io_fini(&_dst_iter.io, bdev, dst_mem); | |
ba4e7d97 | 196 | |
ba4e7d97 TH |
197 | return ret; |
198 | } | |
199 | EXPORT_SYMBOL(ttm_bo_move_memcpy); | |
200 | ||
201 | static void ttm_transfered_destroy(struct ttm_buffer_object *bo) | |
202 | { | |
5452cf44 CK |
203 | struct ttm_transfer_obj *fbo; |
204 | ||
205 | fbo = container_of(bo, struct ttm_transfer_obj, base); | |
0db55f9a | 206 | dma_resv_fini(&fbo->base.base._resv); |
f4490759 | 207 | ttm_bo_put(fbo->bo); |
5452cf44 | 208 | kfree(fbo); |
ba4e7d97 TH |
209 | } |
210 | ||
211 | /** | |
212 | * ttm_buffer_object_transfer | |
213 | * | |
214 | * @bo: A pointer to a struct ttm_buffer_object. | |
215 | * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, | |
216 | * holding the data of @bo with the old placement. | |
217 | * | |
218 | * This is a utility function that may be called after an accelerated move | |
219 | * has been scheduled. A new buffer object is created as a placeholder for | |
220 | * the old data while it's being copied. When that buffer object is idle, | |
221 | * it can be destroyed, releasing the space of the old placement. | |
222 | * Returns: | |
223 | * !0: Failure. | |
224 | */ | |
225 | ||
226 | static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, | |
227 | struct ttm_buffer_object **new_obj) | |
228 | { | |
5452cf44 | 229 | struct ttm_transfer_obj *fbo; |
5e338405 | 230 | int ret; |
ba4e7d97 | 231 | |
ff7c60c5 | 232 | fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); |
ba4e7d97 TH |
233 | if (!fbo) |
234 | return -ENOMEM; | |
235 | ||
5452cf44 | 236 | fbo->base = *bo; |
d6e820fc | 237 | |
ba4e7d97 TH |
238 | /** |
239 | * Fix up members that we shouldn't copy directly: | |
240 | * TODO: Explicit member copy would probably be better here. | |
241 | */ | |
242 | ||
8af8a109 | 243 | atomic_inc(&ttm_glob.bo_count); |
b96f3e7c | 244 | drm_vma_node_reset(&fbo->base.base.vma_node); |
5452cf44 | 245 | |
5452cf44 CK |
246 | kref_init(&fbo->base.kref); |
247 | fbo->base.destroy = &ttm_transfered_destroy; | |
b73cd1e2 | 248 | fbo->base.pin_count = 0; |
5b34406f | 249 | if (bo->type != ttm_bo_type_sg) |
ef383218 CK |
250 | fbo->base.base.resv = &fbo->base.base._resv; |
251 | ||
e3d3fd1c CK |
252 | dma_resv_init(&fbo->base.base._resv); |
253 | fbo->base.base.dev = NULL; | |
254 | ret = dma_resv_trylock(&fbo->base.base._resv); | |
255 | WARN_ON(!ret); | |
256 | ||
fda8d552 CK |
257 | if (fbo->base.resource) { |
258 | ttm_resource_set_bo(fbo->base.resource, &fbo->base); | |
259 | bo->resource = NULL; | |
d91c411c ZY |
260 | ttm_bo_set_bulk_move(&fbo->base, NULL); |
261 | } else { | |
262 | fbo->base.bulk_move = NULL; | |
fda8d552 CK |
263 | } |
264 | ||
c8d4c18b CK |
265 | ret = dma_resv_reserve_fences(&fbo->base.base._resv, 1); |
266 | if (ret) { | |
267 | kfree(fbo); | |
268 | return ret; | |
269 | } | |
270 | ||
271 | ttm_bo_get(bo); | |
272 | fbo->bo = bo; | |
273 | ||
b73cd1e2 CK |
274 | ttm_bo_move_to_lru_tail_unlocked(&fbo->base); |
275 | ||
5452cf44 | 276 | *new_obj = &fbo->base; |
ba4e7d97 TH |
277 | return 0; |
278 | } | |
279 | ||
a3185f91 CK |
280 | /** |
281 | * ttm_io_prot | |
282 | * | |
d3e83448 CK |
283 | * @bo: ttm buffer object |
284 | * @res: ttm resource object | |
a3185f91 CK |
285 | * @tmp: Page protection flag for a normal, cached mapping. |
286 | * | |
287 | * Utility function that returns the pgprot_t that should be used for | |
288 | * setting up a PTE with the caching model indicated by @c_state. | |
289 | */ | |
867bcecd CK |
290 | pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res, |
291 | pgprot_t tmp) | |
ba4e7d97 | 292 | { |
867bcecd CK |
293 | struct ttm_resource_manager *man; |
294 | enum ttm_caching caching; | |
295 | ||
296 | man = ttm_manager_type(bo->bdev, res->mem_type); | |
71ce0463 ZR |
297 | if (man->use_tt) { |
298 | caching = bo->ttm->caching; | |
299 | if (bo->ttm->page_flags & TTM_TT_FLAG_DECRYPTED) | |
300 | tmp = pgprot_decrypted(tmp); | |
301 | } else { | |
302 | caching = res->bus.caching; | |
303 | } | |
867bcecd | 304 | |
3bf3710e | 305 | return ttm_prot_from_caching(caching, tmp); |
ba4e7d97 | 306 | } |
4bfd75cb | 307 | EXPORT_SYMBOL(ttm_io_prot); |
ba4e7d97 TH |
308 | |
309 | static int ttm_bo_ioremap(struct ttm_buffer_object *bo, | |
82c5da6b JG |
310 | unsigned long offset, |
311 | unsigned long size, | |
ba4e7d97 TH |
312 | struct ttm_bo_kmap_obj *map) |
313 | { | |
d3116756 | 314 | struct ttm_resource *mem = bo->resource; |
ba4e7d97 | 315 | |
d3116756 | 316 | if (bo->resource->bus.addr) { |
ba4e7d97 | 317 | map->bo_kmap_type = ttm_bo_map_premapped; |
d3116756 | 318 | map->virtual = ((u8 *)bo->resource->bus.addr) + offset; |
ba4e7d97 | 319 | } else { |
d3116756 CK |
320 | resource_size_t res = bo->resource->bus.offset + offset; |
321 | ||
ba4e7d97 | 322 | map->bo_kmap_type = ttm_bo_map_iomap; |
ce65b874 | 323 | if (mem->bus.caching == ttm_write_combined) |
d3116756 | 324 | map->virtual = ioremap_wc(res, size); |
b849bec2 OZ |
325 | #ifdef CONFIG_X86 |
326 | else if (mem->bus.caching == ttm_cached) | |
d3116756 | 327 | map->virtual = ioremap_cache(res, size); |
b849bec2 | 328 | #endif |
ba4e7d97 | 329 | else |
d3116756 | 330 | map->virtual = ioremap(res, size); |
ba4e7d97 TH |
331 | } |
332 | return (!map->virtual) ? -ENOMEM : 0; | |
333 | } | |
334 | ||
335 | static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, | |
336 | unsigned long start_page, | |
337 | unsigned long num_pages, | |
338 | struct ttm_bo_kmap_obj *map) | |
339 | { | |
d3116756 | 340 | struct ttm_resource *mem = bo->resource; |
d0cef9fa RH |
341 | struct ttm_operation_ctx ctx = { |
342 | .interruptible = false, | |
343 | .no_wait_gpu = false | |
344 | }; | |
62975d27 | 345 | struct ttm_tt *ttm = bo->ttm; |
71ce0463 ZR |
346 | struct ttm_resource_manager *man = |
347 | ttm_manager_type(bo->bdev, bo->resource->mem_type); | |
d0cef9fa | 348 | pgprot_t prot; |
b1e5f172 | 349 | int ret; |
ba4e7d97 | 350 | |
62975d27 | 351 | BUG_ON(!ttm); |
b1e5f172 | 352 | |
0a667b50 | 353 | ret = ttm_tt_populate(bo->bdev, ttm, &ctx); |
25893a14 CK |
354 | if (ret) |
355 | return ret; | |
b1e5f172 | 356 | |
71ce0463 ZR |
357 | if (num_pages == 1 && ttm->caching == ttm_cached && |
358 | !(man->use_tt && (ttm->page_flags & TTM_TT_FLAG_DECRYPTED))) { | |
ba4e7d97 TH |
359 | /* |
360 | * We're mapping a single page, and the desired | |
361 | * page protection is consistent with the bo. | |
362 | */ | |
363 | ||
364 | map->bo_kmap_type = ttm_bo_map_kmap; | |
b1e5f172 | 365 | map->page = ttm->pages[start_page]; |
ba4e7d97 TH |
366 | map->virtual = kmap(map->page); |
367 | } else { | |
ba4e7d97 TH |
368 | /* |
369 | * We need to use vmap to get the desired page protection | |
af901ca1 | 370 | * or to make the buffer object look contiguous. |
ba4e7d97 | 371 | */ |
867bcecd | 372 | prot = ttm_io_prot(bo, mem, PAGE_KERNEL); |
ba4e7d97 TH |
373 | map->bo_kmap_type = ttm_bo_map_vmap; |
374 | map->virtual = vmap(ttm->pages + start_page, num_pages, | |
375 | 0, prot); | |
376 | } | |
377 | return (!map->virtual) ? -ENOMEM : 0; | |
378 | } | |
379 | ||
a3185f91 CK |
380 | /** |
381 | * ttm_bo_kmap | |
382 | * | |
383 | * @bo: The buffer object. | |
384 | * @start_page: The first page to map. | |
385 | * @num_pages: Number of pages to map. | |
386 | * @map: pointer to a struct ttm_bo_kmap_obj representing the map. | |
387 | * | |
388 | * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the | |
389 | * data in the buffer object. The ttm_kmap_obj_virtual function can then be | |
390 | * used to obtain a virtual address to the data. | |
391 | * | |
392 | * Returns | |
393 | * -ENOMEM: Out of memory. | |
394 | * -EINVAL: Invalid range. | |
395 | */ | |
ba4e7d97 TH |
396 | int ttm_bo_kmap(struct ttm_buffer_object *bo, |
397 | unsigned long start_page, unsigned long num_pages, | |
398 | struct ttm_bo_kmap_obj *map) | |
399 | { | |
82c5da6b | 400 | unsigned long offset, size; |
ba4e7d97 | 401 | int ret; |
ba4e7d97 | 402 | |
ba4e7d97 | 403 | map->virtual = NULL; |
82c5da6b | 404 | map->bo = bo; |
e3c92eb4 | 405 | if (num_pages > PFN_UP(bo->resource->size)) |
ba4e7d97 | 406 | return -EINVAL; |
e3c92eb4 | 407 | if ((start_page + num_pages) > PFN_UP(bo->resource->size)) |
ba4e7d97 | 408 | return -EINVAL; |
02b29caf | 409 | |
d3116756 | 410 | ret = ttm_mem_io_reserve(bo->bdev, bo->resource); |
ba4e7d97 TH |
411 | if (ret) |
412 | return ret; | |
d3116756 | 413 | if (!bo->resource->bus.is_iomem) { |
ba4e7d97 TH |
414 | return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); |
415 | } else { | |
82c5da6b JG |
416 | offset = start_page << PAGE_SHIFT; |
417 | size = num_pages << PAGE_SHIFT; | |
418 | return ttm_bo_ioremap(bo, offset, size, map); | |
ba4e7d97 TH |
419 | } |
420 | } | |
421 | EXPORT_SYMBOL(ttm_bo_kmap); | |
422 | ||
a3185f91 CK |
423 | /** |
424 | * ttm_bo_kunmap | |
425 | * | |
426 | * @map: Object describing the map to unmap. | |
427 | * | |
428 | * Unmaps a kernel map set up by ttm_bo_kmap. | |
429 | */ | |
ba4e7d97 TH |
430 | void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) |
431 | { | |
432 | if (!map->virtual) | |
433 | return; | |
434 | switch (map->bo_kmap_type) { | |
435 | case ttm_bo_map_iomap: | |
436 | iounmap(map->virtual); | |
437 | break; | |
438 | case ttm_bo_map_vmap: | |
439 | vunmap(map->virtual); | |
440 | break; | |
441 | case ttm_bo_map_kmap: | |
442 | kunmap(map->page); | |
443 | break; | |
444 | case ttm_bo_map_premapped: | |
445 | break; | |
446 | default: | |
447 | BUG(); | |
448 | } | |
d3116756 | 449 | ttm_mem_io_free(map->bo->bdev, map->bo->resource); |
ba4e7d97 TH |
450 | map->virtual = NULL; |
451 | map->page = NULL; | |
452 | } | |
453 | EXPORT_SYMBOL(ttm_bo_kunmap); | |
454 | ||
a3185f91 CK |
455 | /** |
456 | * ttm_bo_vmap | |
457 | * | |
458 | * @bo: The buffer object. | |
459 | * @map: pointer to a struct iosys_map representing the map. | |
460 | * | |
461 | * Sets up a kernel virtual mapping, using ioremap or vmap to the | |
462 | * data in the buffer object. The parameter @map returns the virtual | |
463 | * address as struct iosys_map. Unmap the buffer with ttm_bo_vunmap(). | |
464 | * | |
465 | * Returns | |
466 | * -ENOMEM: Out of memory. | |
467 | * -EINVAL: Invalid range. | |
468 | */ | |
7938f421 | 469 | int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map) |
43676605 | 470 | { |
d3116756 | 471 | struct ttm_resource *mem = bo->resource; |
43676605 TZ |
472 | int ret; |
473 | ||
d6d36cf2 CK |
474 | dma_resv_assert_held(bo->base.resv); |
475 | ||
43676605 TZ |
476 | ret = ttm_mem_io_reserve(bo->bdev, mem); |
477 | if (ret) | |
478 | return ret; | |
479 | ||
480 | if (mem->bus.is_iomem) { | |
481 | void __iomem *vaddr_iomem; | |
43676605 TZ |
482 | |
483 | if (mem->bus.addr) | |
484 | vaddr_iomem = (void __iomem *)mem->bus.addr; | |
485 | else if (mem->bus.caching == ttm_write_combined) | |
e11bfb99 CK |
486 | vaddr_iomem = ioremap_wc(mem->bus.offset, |
487 | bo->base.size); | |
b849bec2 OZ |
488 | #ifdef CONFIG_X86 |
489 | else if (mem->bus.caching == ttm_cached) | |
490 | vaddr_iomem = ioremap_cache(mem->bus.offset, | |
491 | bo->base.size); | |
492 | #endif | |
43676605 | 493 | else |
e11bfb99 | 494 | vaddr_iomem = ioremap(mem->bus.offset, bo->base.size); |
43676605 TZ |
495 | |
496 | if (!vaddr_iomem) | |
497 | return -ENOMEM; | |
498 | ||
7938f421 | 499 | iosys_map_set_vaddr_iomem(map, vaddr_iomem); |
43676605 TZ |
500 | |
501 | } else { | |
502 | struct ttm_operation_ctx ctx = { | |
503 | .interruptible = false, | |
504 | .no_wait_gpu = false | |
505 | }; | |
506 | struct ttm_tt *ttm = bo->ttm; | |
507 | pgprot_t prot; | |
508 | void *vaddr; | |
509 | ||
510 | ret = ttm_tt_populate(bo->bdev, ttm, &ctx); | |
511 | if (ret) | |
512 | return ret; | |
513 | ||
514 | /* | |
515 | * We need to use vmap to get the desired page protection | |
516 | * or to make the buffer object look contiguous. | |
517 | */ | |
518 | prot = ttm_io_prot(bo, mem, PAGE_KERNEL); | |
e11bfb99 | 519 | vaddr = vmap(ttm->pages, ttm->num_pages, 0, prot); |
43676605 TZ |
520 | if (!vaddr) |
521 | return -ENOMEM; | |
522 | ||
7938f421 | 523 | iosys_map_set_vaddr(map, vaddr); |
43676605 TZ |
524 | } |
525 | ||
526 | return 0; | |
527 | } | |
528 | EXPORT_SYMBOL(ttm_bo_vmap); | |
529 | ||
a3185f91 CK |
530 | /** |
531 | * ttm_bo_vunmap | |
532 | * | |
533 | * @bo: The buffer object. | |
534 | * @map: Object describing the map to unmap. | |
535 | * | |
536 | * Unmaps a kernel map set up by ttm_bo_vmap(). | |
537 | */ | |
7938f421 | 538 | void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map) |
43676605 | 539 | { |
d3116756 | 540 | struct ttm_resource *mem = bo->resource; |
43676605 | 541 | |
d6d36cf2 CK |
542 | dma_resv_assert_held(bo->base.resv); |
543 | ||
7938f421 | 544 | if (iosys_map_is_null(map)) |
43676605 TZ |
545 | return; |
546 | ||
547 | if (!map->is_iomem) | |
548 | vunmap(map->vaddr); | |
549 | else if (!mem->bus.addr) | |
550 | iounmap(map->vaddr_iomem); | |
7938f421 | 551 | iosys_map_clear(map); |
43676605 | 552 | |
d3116756 | 553 | ttm_mem_io_free(bo->bdev, bo->resource); |
43676605 TZ |
554 | } |
555 | EXPORT_SYMBOL(ttm_bo_vunmap); | |
556 | ||
92afce90 DA |
557 | static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo, |
558 | bool dst_use_tt) | |
559 | { | |
13acb368 CK |
560 | long ret; |
561 | ||
562 | ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, | |
563 | false, 15 * HZ); | |
564 | if (ret == 0) | |
565 | return -EBUSY; | |
566 | if (ret < 0) | |
92afce90 DA |
567 | return ret; |
568 | ||
569 | if (!dst_use_tt) | |
570 | ttm_bo_tt_destroy(bo); | |
bfa3357e | 571 | ttm_resource_free(bo, &bo->resource); |
92afce90 DA |
572 | return 0; |
573 | } | |
574 | ||
13a8f46d DA |
575 | static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, |
576 | struct dma_fence *fence, | |
577 | bool dst_use_tt) | |
578 | { | |
579 | struct ttm_buffer_object *ghost_obj; | |
580 | int ret; | |
581 | ||
582 | /** | |
583 | * This should help pipeline ordinary buffer moves. | |
584 | * | |
585 | * Hang old buffer memory on a new buffer object, | |
586 | * and leave it to be released when the GPU | |
587 | * operation has completed. | |
588 | */ | |
589 | ||
13a8f46d DA |
590 | ret = ttm_buffer_object_transfer(bo, &ghost_obj); |
591 | if (ret) | |
592 | return ret; | |
593 | ||
73511edf | 594 | dma_resv_add_fence(&ghost_obj->base._resv, fence, |
b29895e1 | 595 | DMA_RESV_USAGE_KERNEL); |
13a8f46d DA |
596 | |
597 | /** | |
598 | * If we're not moving to fixed memory, the TTM object | |
599 | * needs to stay alive. Otherwhise hang it on the ghost | |
600 | * bo to be unbound and destroyed. | |
601 | */ | |
602 | ||
603 | if (dst_use_tt) | |
604 | ghost_obj->ttm = NULL; | |
605 | else | |
606 | bo->ttm = NULL; | |
607 | ||
608 | dma_resv_unlock(&ghost_obj->base._resv); | |
609 | ttm_bo_put(ghost_obj); | |
610 | return 0; | |
611 | } | |
612 | ||
e46f468f DA |
613 | static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, |
614 | struct dma_fence *fence) | |
ba4e7d97 | 615 | { |
8af8a109 | 616 | struct ttm_device *bdev = bo->bdev; |
d3116756 CK |
617 | struct ttm_resource_manager *from; |
618 | ||
619 | from = ttm_manager_type(bdev, bo->resource->mem_type); | |
ba4e7d97 | 620 | |
e46f468f DA |
621 | /** |
622 | * BO doesn't have a TTM we need to bind/unbind. Just remember | |
623 | * this eviction and free up the allocation | |
624 | */ | |
625 | spin_lock(&from->move_lock); | |
626 | if (!from->move || dma_fence_is_later(fence, from->move)) { | |
627 | dma_fence_put(from->move); | |
628 | from->move = dma_fence_get(fence); | |
629 | } | |
630 | spin_unlock(&from->move_lock); | |
ba4e7d97 | 631 | |
bfa3357e | 632 | ttm_resource_free(bo, &bo->resource); |
ba4e7d97 | 633 | } |
3ddf4ad9 | 634 | |
a3185f91 | 635 | /** |
d3e83448 | 636 | * ttm_bo_move_accel_cleanup - cleanup helper for hw copies |
a3185f91 CK |
637 | * |
638 | * @bo: A pointer to a struct ttm_buffer_object. | |
639 | * @fence: A fence object that signals when moving is complete. | |
640 | * @evict: This is an evict move. Don't return until the buffer is idle. | |
641 | * @pipeline: evictions are to be pipelined. | |
642 | * @new_mem: struct ttm_resource indicating where to move. | |
643 | * | |
644 | * Accelerated move function to be called when an accelerated move | |
645 | * has been scheduled. The function will create a new temporary buffer object | |
646 | * representing the old placement, and put the sync object on both buffer | |
647 | * objects. After that the newly created buffer object is unref'd to be | |
648 | * destroyed when the move is complete. This will help pipeline | |
649 | * buffer moves. | |
650 | */ | |
e46f468f DA |
651 | int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, |
652 | struct dma_fence *fence, | |
653 | bool evict, | |
654 | bool pipeline, | |
655 | struct ttm_resource *new_mem) | |
3ddf4ad9 | 656 | { |
8af8a109 | 657 | struct ttm_device *bdev = bo->bdev; |
d3116756 | 658 | struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->resource->mem_type); |
e46f468f DA |
659 | struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); |
660 | int ret = 0; | |
3ddf4ad9 | 661 | |
b29895e1 | 662 | dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); |
e46f468f DA |
663 | if (!evict) |
664 | ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt); | |
665 | else if (!from->use_tt && pipeline) | |
666 | ttm_bo_move_pipeline_evict(bo, fence); | |
667 | else | |
668 | ret = ttm_bo_wait_free_node(bo, man->use_tt); | |
3ddf4ad9 | 669 | |
e46f468f DA |
670 | if (ret) |
671 | return ret; | |
3ddf4ad9 | 672 | |
2ee476f7 | 673 | ttm_bo_assign_mem(bo, new_mem); |
3ddf4ad9 CK |
674 | |
675 | return 0; | |
676 | } | |
e46f468f | 677 | EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); |
5d951098 | 678 | |
a3185f91 | 679 | /** |
d3e83448 | 680 | * ttm_bo_move_sync_cleanup - cleanup by waiting for the move to finish |
a3185f91 CK |
681 | * |
682 | * @bo: A pointer to a struct ttm_buffer_object. | |
683 | * @new_mem: struct ttm_resource indicating where to move. | |
684 | * | |
685 | * Special case of ttm_bo_move_accel_cleanup where the bo is guaranteed | |
686 | * by the caller to be idle. Typically used after memcpy buffer moves. | |
687 | */ | |
c6346218 MA |
688 | void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo, |
689 | struct ttm_resource *new_mem) | |
690 | { | |
691 | struct ttm_device *bdev = bo->bdev; | |
692 | struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); | |
693 | int ret; | |
694 | ||
695 | ret = ttm_bo_wait_free_node(bo, man->use_tt); | |
696 | if (WARN_ON(ret)) | |
697 | return; | |
698 | ||
699 | ttm_bo_assign_mem(bo, new_mem); | |
700 | } | |
701 | EXPORT_SYMBOL(ttm_bo_move_sync_cleanup); | |
702 | ||
a3be8cd7 TH |
703 | /** |
704 | * ttm_bo_pipeline_gutting - purge the contents of a bo | |
705 | * @bo: The buffer object | |
706 | * | |
707 | * Purge the contents of a bo, async if the bo is not idle. | |
708 | * After a successful call, the bo is left unpopulated in | |
709 | * system placement. The function may wait uninterruptible | |
710 | * for idle on OOM. | |
711 | * | |
712 | * Return: 0 if successful, negative error code on failure. | |
713 | */ | |
5d951098 CK |
714 | int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) |
715 | { | |
716 | struct ttm_buffer_object *ghost; | |
a3be8cd7 | 717 | struct ttm_tt *ttm; |
5d951098 CK |
718 | int ret; |
719 | ||
a3be8cd7 | 720 | /* If already idle, no need for ghost object dance. */ |
13acb368 | 721 | if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)) { |
a3be8cd7 TH |
722 | if (!bo->ttm) { |
723 | /* See comment below about clearing. */ | |
724 | ret = ttm_tt_create(bo, true); | |
725 | if (ret) | |
c7ea16f6 | 726 | return ret; |
a3be8cd7 TH |
727 | } else { |
728 | ttm_tt_unpopulate(bo->bdev, bo->ttm); | |
729 | if (bo->type == ttm_bo_type_device) | |
730 | ttm_tt_mark_for_clear(bo->ttm); | |
731 | } | |
732 | ttm_resource_free(bo, &bo->resource); | |
fc357bc8 | 733 | return 0; |
a3be8cd7 TH |
734 | } |
735 | ||
736 | /* | |
737 | * We need an unpopulated ttm_tt after giving our current one, | |
738 | * if any, to the ghost object. And we can't afford to fail | |
739 | * creating one *after* the operation. If the bo subsequently gets | |
740 | * resurrected, make sure it's cleared (if ttm_bo_type_device) | |
741 | * to avoid leaking sensitive information to user-space. | |
742 | */ | |
743 | ||
744 | ttm = bo->ttm; | |
745 | bo->ttm = NULL; | |
746 | ret = ttm_tt_create(bo, true); | |
747 | swap(bo->ttm, ttm); | |
5d951098 | 748 | if (ret) |
c7ea16f6 | 749 | return ret; |
5d951098 | 750 | |
a3be8cd7 | 751 | ret = ttm_buffer_object_transfer(bo, &ghost); |
fc357bc8 CK |
752 | if (ret) |
753 | goto error_destroy_tt; | |
a3be8cd7 | 754 | |
ef383218 | 755 | ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); |
5d951098 | 756 | /* Last resort, wait for the BO to be idle when we are OOM */ |
13acb368 CK |
757 | if (ret) { |
758 | dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, | |
759 | false, MAX_SCHEDULE_TIMEOUT); | |
760 | } | |
5d951098 | 761 | |
ef383218 | 762 | dma_resv_unlock(&ghost->base._resv); |
f4490759 | 763 | ttm_bo_put(ghost); |
a3be8cd7 | 764 | bo->ttm = ttm; |
fc357bc8 CK |
765 | return 0; |
766 | ||
767 | error_destroy_tt: | |
768 | ttm_tt_destroy(bo->bdev, ttm); | |
fc357bc8 | 769 | return ret; |
5d951098 | 770 | } |