Commit | Line | Data |
---|---|---|
f7749a54 | 1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
e6303f32 DK |
2 | /* |
3 | * Copyright (c) 2022 Red Hat. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the "Software"), | |
7 | * to deal in the Software without restriction, including without limitation | |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
9 | * and/or sell copies of the Software, and to permit persons to whom the | |
10 | * Software is furnished to do so, subject to the following conditions: | |
11 | * | |
12 | * The above copyright notice and this permission notice shall be included in | |
13 | * all copies or substantial portions of the Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
21 | * OTHER DEALINGS IN THE SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Danilo Krummrich <dakr@redhat.com> | |
25 | * | |
26 | */ | |
27 | ||
f72c2db4 | 28 | #include <drm/drm_gpuvm.h> |
e6303f32 DK |
29 | |
30 | #include <linux/interval_tree_generic.h> | |
31 | #include <linux/mm.h> | |
32 | ||
33 | /** | |
34 | * DOC: Overview | |
35 | * | |
f72c2db4 DK |
36 | * The DRM GPU VA Manager, represented by struct drm_gpuvm keeps track of a |
37 | * GPU's virtual address (VA) space and manages the corresponding virtual | |
e6303f32 DK |
38 | * mappings represented by &drm_gpuva objects. It also keeps track of the |
39 | * mapping's backing &drm_gem_object buffers. | |
40 | * | |
41 | * &drm_gem_object buffers maintain a list of &drm_gpuva objects representing | |
42 | * all existent GPU VA mappings using this &drm_gem_object as backing buffer. | |
43 | * | |
44 | * GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also | |
45 | * keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'. | |
46 | * | |
47 | * The GPU VA manager internally uses a rb-tree to manage the | |
48 | * &drm_gpuva mappings within a GPU's virtual address space. | |
49 | * | |
f72c2db4 | 50 | * The &drm_gpuvm structure contains a special &drm_gpuva representing the |
e6303f32 DK |
51 | * portion of VA space reserved by the kernel. This node is initialized together |
52 | * with the GPU VA manager instance and removed when the GPU VA manager is | |
53 | * destroyed. | |
54 | * | |
f72c2db4 | 55 | * In a typical application drivers would embed struct drm_gpuvm and |
e6303f32 DK |
56 | * struct drm_gpuva within their own driver specific structures, there won't be |
57 | * any memory allocations of its own nor memory allocations of &drm_gpuva | |
58 | * entries. | |
59 | * | |
f72c2db4 DK |
60 | * The data structures needed to store &drm_gpuvas within the &drm_gpuvm are |
61 | * contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva | |
62 | * entries from within dma-fence signalling critical sections it is enough to | |
63 | * pre-allocate the &drm_gpuva structures. | |
e6303f32 DK |
64 | */ |
65 | ||
66 | /** | |
67 | * DOC: Split and Merge | |
68 | * | |
69 | * Besides its capability to manage and represent a GPU VA space, the | |
f72c2db4 DK |
70 | * GPU VA manager also provides functions to let the &drm_gpuvm calculate a |
71 | * sequence of operations to satisfy a given map or unmap request. | |
e6303f32 DK |
72 | * |
73 | * Therefore the DRM GPU VA manager provides an algorithm implementing splitting | |
74 | * and merging of existent GPU VA mappings with the ones that are requested to | |
75 | * be mapped or unmapped. This feature is required by the Vulkan API to | |
76 | * implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this | |
77 | * as VM BIND. | |
78 | * | |
f72c2db4 | 79 | * Drivers can call drm_gpuvm_sm_map() to receive a sequence of callbacks |
e6303f32 DK |
80 | * containing map, unmap and remap operations for a given newly requested |
81 | * mapping. The sequence of callbacks represents the set of operations to | |
82 | * execute in order to integrate the new mapping cleanly into the current state | |
83 | * of the GPU VA space. | |
84 | * | |
85 | * Depending on how the new GPU VA mapping intersects with the existent mappings | |
f72c2db4 DK |
86 | * of the GPU VA space the &drm_gpuvm_ops callbacks contain an arbitrary amount |
87 | * of unmap operations, a maximum of two remap operations and a single map | |
88 | * operation. The caller might receive no callback at all if no operation is | |
e6303f32 DK |
89 | * required, e.g. if the requested mapping already exists in the exact same way. |
90 | * | |
91 | * The single map operation represents the original map operation requested by | |
92 | * the caller. | |
93 | * | |
94 | * &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the | |
95 | * &drm_gpuva to unmap is physically contiguous with the original mapping | |
96 | * request. Optionally, if 'keep' is set, drivers may keep the actual page table | |
97 | * entries for this &drm_gpuva, adding the missing page table entries only and | |
f72c2db4 | 98 | * update the &drm_gpuvm's view of things accordingly. |
e6303f32 DK |
99 | * |
100 | * Drivers may do the same optimization, namely delta page table updates, also | |
101 | * for remap operations. This is possible since &drm_gpuva_op_remap consists of | |
102 | * one unmap operation and one or two map operations, such that drivers can | |
103 | * derive the page table update delta accordingly. | |
104 | * | |
105 | * Note that there can't be more than two existent mappings to split up, one at | |
106 | * the beginning and one at the end of the new mapping, hence there is a | |
107 | * maximum of two remap operations. | |
108 | * | |
f72c2db4 DK |
109 | * Analogous to drm_gpuvm_sm_map() drm_gpuvm_sm_unmap() uses &drm_gpuvm_ops to |
110 | * call back into the driver in order to unmap a range of GPU VA space. The | |
e6303f32 DK |
111 | * logic behind this function is way simpler though: For all existent mappings |
112 | * enclosed by the given range unmap operations are created. For mappings which | |
113 | * are only partically located within the given range, remap operations are | |
114 | * created such that those mappings are split up and re-mapped partically. | |
115 | * | |
f72c2db4 DK |
116 | * As an alternative to drm_gpuvm_sm_map() and drm_gpuvm_sm_unmap(), |
117 | * drm_gpuvm_sm_map_ops_create() and drm_gpuvm_sm_unmap_ops_create() can be used | |
e6303f32 DK |
118 | * to directly obtain an instance of struct drm_gpuva_ops containing a list of |
119 | * &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list | |
120 | * contains the &drm_gpuva_ops analogous to the callbacks one would receive when | |
f72c2db4 | 121 | * calling drm_gpuvm_sm_map() or drm_gpuvm_sm_unmap(). While this way requires |
e6303f32 DK |
122 | * more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to |
123 | * iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory | |
124 | * allocations are possible (e.g. to allocate GPU page tables) and once in the | |
125 | * dma-fence signalling critical path. | |
126 | * | |
f72c2db4 DK |
127 | * To update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert() and |
128 | * drm_gpuva_remove() may be used. These functions can safely be used from | |
129 | * &drm_gpuvm_ops callbacks originating from drm_gpuvm_sm_map() or | |
130 | * drm_gpuvm_sm_unmap(). However, it might be more convenient to use the | |
131 | * provided helper functions drm_gpuva_map(), drm_gpuva_remap() and | |
132 | * drm_gpuva_unmap() instead. | |
e6303f32 DK |
133 | * |
134 | * The following diagram depicts the basic relationships of existent GPU VA | |
135 | * mappings, a newly requested mapping and the resulting mappings as implemented | |
f72c2db4 | 136 | * by drm_gpuvm_sm_map() - it doesn't cover any arbitrary combinations of these. |
e6303f32 DK |
137 | * |
138 | * 1) Requested mapping is identical. Replace it, but indicate the backing PTEs | |
139 | * could be kept. | |
140 | * | |
141 | * :: | |
142 | * | |
143 | * 0 a 1 | |
144 | * old: |-----------| (bo_offset=n) | |
145 | * | |
146 | * 0 a 1 | |
147 | * req: |-----------| (bo_offset=n) | |
148 | * | |
149 | * 0 a 1 | |
150 | * new: |-----------| (bo_offset=n) | |
151 | * | |
152 | * | |
153 | * 2) Requested mapping is identical, except for the BO offset, hence replace | |
154 | * the mapping. | |
155 | * | |
156 | * :: | |
157 | * | |
158 | * 0 a 1 | |
159 | * old: |-----------| (bo_offset=n) | |
160 | * | |
161 | * 0 a 1 | |
162 | * req: |-----------| (bo_offset=m) | |
163 | * | |
164 | * 0 a 1 | |
165 | * new: |-----------| (bo_offset=m) | |
166 | * | |
167 | * | |
168 | * 3) Requested mapping is identical, except for the backing BO, hence replace | |
169 | * the mapping. | |
170 | * | |
171 | * :: | |
172 | * | |
173 | * 0 a 1 | |
174 | * old: |-----------| (bo_offset=n) | |
175 | * | |
176 | * 0 b 1 | |
177 | * req: |-----------| (bo_offset=n) | |
178 | * | |
179 | * 0 b 1 | |
180 | * new: |-----------| (bo_offset=n) | |
181 | * | |
182 | * | |
183 | * 4) Existent mapping is a left aligned subset of the requested one, hence | |
184 | * replace the existent one. | |
185 | * | |
186 | * :: | |
187 | * | |
188 | * 0 a 1 | |
189 | * old: |-----| (bo_offset=n) | |
190 | * | |
191 | * 0 a 2 | |
192 | * req: |-----------| (bo_offset=n) | |
193 | * | |
194 | * 0 a 2 | |
195 | * new: |-----------| (bo_offset=n) | |
196 | * | |
197 | * .. note:: | |
198 | * We expect to see the same result for a request with a different BO | |
199 | * and/or non-contiguous BO offset. | |
200 | * | |
201 | * | |
202 | * 5) Requested mapping's range is a left aligned subset of the existent one, | |
203 | * but backed by a different BO. Hence, map the requested mapping and split | |
204 | * the existent one adjusting its BO offset. | |
205 | * | |
206 | * :: | |
207 | * | |
208 | * 0 a 2 | |
209 | * old: |-----------| (bo_offset=n) | |
210 | * | |
211 | * 0 b 1 | |
212 | * req: |-----| (bo_offset=n) | |
213 | * | |
214 | * 0 b 1 a' 2 | |
215 | * new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1) | |
216 | * | |
217 | * .. note:: | |
218 | * We expect to see the same result for a request with a different BO | |
219 | * and/or non-contiguous BO offset. | |
220 | * | |
221 | * | |
222 | * 6) Existent mapping is a superset of the requested mapping. Split it up, but | |
223 | * indicate that the backing PTEs could be kept. | |
224 | * | |
225 | * :: | |
226 | * | |
227 | * 0 a 2 | |
228 | * old: |-----------| (bo_offset=n) | |
229 | * | |
230 | * 0 a 1 | |
231 | * req: |-----| (bo_offset=n) | |
232 | * | |
233 | * 0 a 1 a' 2 | |
234 | * new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1) | |
235 | * | |
236 | * | |
237 | * 7) Requested mapping's range is a right aligned subset of the existent one, | |
238 | * but backed by a different BO. Hence, map the requested mapping and split | |
239 | * the existent one, without adjusting the BO offset. | |
240 | * | |
241 | * :: | |
242 | * | |
243 | * 0 a 2 | |
244 | * old: |-----------| (bo_offset=n) | |
245 | * | |
246 | * 1 b 2 | |
247 | * req: |-----| (bo_offset=m) | |
248 | * | |
249 | * 0 a 1 b 2 | |
250 | * new: |-----|-----| (a.bo_offset=n,b.bo_offset=m) | |
251 | * | |
252 | * | |
253 | * 8) Existent mapping is a superset of the requested mapping. Split it up, but | |
254 | * indicate that the backing PTEs could be kept. | |
255 | * | |
256 | * :: | |
257 | * | |
258 | * 0 a 2 | |
259 | * old: |-----------| (bo_offset=n) | |
260 | * | |
261 | * 1 a 2 | |
262 | * req: |-----| (bo_offset=n+1) | |
263 | * | |
264 | * 0 a' 1 a 2 | |
265 | * new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1) | |
266 | * | |
267 | * | |
268 | * 9) Existent mapping is overlapped at the end by the requested mapping backed | |
269 | * by a different BO. Hence, map the requested mapping and split up the | |
270 | * existent one, without adjusting the BO offset. | |
271 | * | |
272 | * :: | |
273 | * | |
274 | * 0 a 2 | |
275 | * old: |-----------| (bo_offset=n) | |
276 | * | |
277 | * 1 b 3 | |
278 | * req: |-----------| (bo_offset=m) | |
279 | * | |
280 | * 0 a 1 b 3 | |
281 | * new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m) | |
282 | * | |
283 | * | |
284 | * 10) Existent mapping is overlapped by the requested mapping, both having the | |
285 | * same backing BO with a contiguous offset. Indicate the backing PTEs of | |
286 | * the old mapping could be kept. | |
287 | * | |
288 | * :: | |
289 | * | |
290 | * 0 a 2 | |
291 | * old: |-----------| (bo_offset=n) | |
292 | * | |
293 | * 1 a 3 | |
294 | * req: |-----------| (bo_offset=n+1) | |
295 | * | |
296 | * 0 a' 1 a 3 | |
297 | * new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1) | |
298 | * | |
299 | * | |
300 | * 11) Requested mapping's range is a centered subset of the existent one | |
301 | * having a different backing BO. Hence, map the requested mapping and split | |
302 | * up the existent one in two mappings, adjusting the BO offset of the right | |
303 | * one accordingly. | |
304 | * | |
305 | * :: | |
306 | * | |
307 | * 0 a 3 | |
308 | * old: |-----------------| (bo_offset=n) | |
309 | * | |
310 | * 1 b 2 | |
311 | * req: |-----| (bo_offset=m) | |
312 | * | |
313 | * 0 a 1 b 2 a' 3 | |
314 | * new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2) | |
315 | * | |
316 | * | |
317 | * 12) Requested mapping is a contiguous subset of the existent one. Split it | |
318 | * up, but indicate that the backing PTEs could be kept. | |
319 | * | |
320 | * :: | |
321 | * | |
322 | * 0 a 3 | |
323 | * old: |-----------------| (bo_offset=n) | |
324 | * | |
325 | * 1 a 2 | |
326 | * req: |-----| (bo_offset=n+1) | |
327 | * | |
328 | * 0 a' 1 a 2 a'' 3 | |
329 | * old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2) | |
330 | * | |
331 | * | |
332 | * 13) Existent mapping is a right aligned subset of the requested one, hence | |
333 | * replace the existent one. | |
334 | * | |
335 | * :: | |
336 | * | |
337 | * 1 a 2 | |
338 | * old: |-----| (bo_offset=n+1) | |
339 | * | |
340 | * 0 a 2 | |
341 | * req: |-----------| (bo_offset=n) | |
342 | * | |
343 | * 0 a 2 | |
344 | * new: |-----------| (bo_offset=n) | |
345 | * | |
346 | * .. note:: | |
347 | * We expect to see the same result for a request with a different bo | |
348 | * and/or non-contiguous bo_offset. | |
349 | * | |
350 | * | |
351 | * 14) Existent mapping is a centered subset of the requested one, hence | |
352 | * replace the existent one. | |
353 | * | |
354 | * :: | |
355 | * | |
356 | * 1 a 2 | |
357 | * old: |-----| (bo_offset=n+1) | |
358 | * | |
359 | * 0 a 3 | |
360 | * req: |----------------| (bo_offset=n) | |
361 | * | |
362 | * 0 a 3 | |
363 | * new: |----------------| (bo_offset=n) | |
364 | * | |
365 | * .. note:: | |
366 | * We expect to see the same result for a request with a different bo | |
367 | * and/or non-contiguous bo_offset. | |
368 | * | |
369 | * | |
370 | * 15) Existent mappings is overlapped at the beginning by the requested mapping | |
371 | * backed by a different BO. Hence, map the requested mapping and split up | |
372 | * the existent one, adjusting its BO offset accordingly. | |
373 | * | |
374 | * :: | |
375 | * | |
376 | * 1 a 3 | |
377 | * old: |-----------| (bo_offset=n) | |
378 | * | |
379 | * 0 b 2 | |
380 | * req: |-----------| (bo_offset=m) | |
381 | * | |
382 | * 0 b 2 a' 3 | |
383 | * new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2) | |
384 | */ | |
385 | ||
386 | /** | |
387 | * DOC: Locking | |
388 | * | |
389 | * Generally, the GPU VA manager does not take care of locking itself, it is | |
390 | * the drivers responsibility to take care about locking. Drivers might want to | |
391 | * protect the following operations: inserting, removing and iterating | |
392 | * &drm_gpuva objects as well as generating all kinds of operations, such as | |
393 | * split / merge or prefetch. | |
394 | * | |
395 | * The GPU VA manager also does not take care of the locking of the backing | |
396 | * &drm_gem_object buffers GPU VA lists by itself; drivers are responsible to | |
397 | * enforce mutual exclusion using either the GEMs dma_resv lock or alternatively | |
398 | * a driver specific external lock. For the latter see also | |
399 | * drm_gem_gpuva_set_lock(). | |
400 | * | |
401 | * However, the GPU VA manager contains lockdep checks to ensure callers of its | |
402 | * API hold the corresponding lock whenever the &drm_gem_objects GPU VA list is | |
403 | * accessed by functions such as drm_gpuva_link() or drm_gpuva_unlink(). | |
404 | */ | |
405 | ||
406 | /** | |
407 | * DOC: Examples | |
408 | * | |
409 | * This section gives two examples on how to let the DRM GPUVA Manager generate | |
410 | * &drm_gpuva_op in order to satisfy a given map or unmap request and how to | |
411 | * make use of them. | |
412 | * | |
413 | * The below code is strictly limited to illustrate the generic usage pattern. | |
414 | * To maintain simplicitly, it doesn't make use of any abstractions for common | |
415 | * code, different (asyncronous) stages with fence signalling critical paths, | |
416 | * any other helpers or error handling in terms of freeing memory and dropping | |
417 | * previously taken locks. | |
418 | * | |
419 | * 1) Obtain a list of &drm_gpuva_op to create a new mapping:: | |
420 | * | |
421 | * // Allocates a new &drm_gpuva. | |
422 | * struct drm_gpuva * driver_gpuva_alloc(void); | |
423 | * | |
f72c2db4 | 424 | * // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva |
e6303f32 DK |
425 | * // structure in individual driver structures and lock the dma-resv with |
426 | * // drm_exec or similar helpers. | |
f72c2db4 | 427 | * int driver_mapping_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
428 | * u64 addr, u64 range, |
429 | * struct drm_gem_object *obj, u64 offset) | |
430 | * { | |
431 | * struct drm_gpuva_ops *ops; | |
432 | * struct drm_gpuva_op *op | |
433 | * | |
434 | * driver_lock_va_space(); | |
f72c2db4 | 435 | * ops = drm_gpuvm_sm_map_ops_create(gpuvm, addr, range, |
e6303f32 DK |
436 | * obj, offset); |
437 | * if (IS_ERR(ops)) | |
438 | * return PTR_ERR(ops); | |
439 | * | |
440 | * drm_gpuva_for_each_op(op, ops) { | |
441 | * struct drm_gpuva *va; | |
442 | * | |
443 | * switch (op->op) { | |
444 | * case DRM_GPUVA_OP_MAP: | |
445 | * va = driver_gpuva_alloc(); | |
446 | * if (!va) | |
447 | * ; // unwind previous VA space updates, | |
448 | * // free memory and unlock | |
449 | * | |
450 | * driver_vm_map(); | |
f72c2db4 | 451 | * drm_gpuva_map(gpuvm, va, &op->map); |
e6303f32 DK |
452 | * drm_gpuva_link(va); |
453 | * | |
454 | * break; | |
455 | * case DRM_GPUVA_OP_REMAP: { | |
456 | * struct drm_gpuva *prev = NULL, *next = NULL; | |
457 | * | |
458 | * va = op->remap.unmap->va; | |
459 | * | |
460 | * if (op->remap.prev) { | |
461 | * prev = driver_gpuva_alloc(); | |
462 | * if (!prev) | |
463 | * ; // unwind previous VA space | |
464 | * // updates, free memory and | |
465 | * // unlock | |
466 | * } | |
467 | * | |
468 | * if (op->remap.next) { | |
469 | * next = driver_gpuva_alloc(); | |
470 | * if (!next) | |
471 | * ; // unwind previous VA space | |
472 | * // updates, free memory and | |
473 | * // unlock | |
474 | * } | |
475 | * | |
476 | * driver_vm_remap(); | |
477 | * drm_gpuva_remap(prev, next, &op->remap); | |
478 | * | |
479 | * drm_gpuva_unlink(va); | |
480 | * if (prev) | |
481 | * drm_gpuva_link(prev); | |
482 | * if (next) | |
483 | * drm_gpuva_link(next); | |
484 | * | |
485 | * break; | |
486 | * } | |
487 | * case DRM_GPUVA_OP_UNMAP: | |
488 | * va = op->unmap->va; | |
489 | * | |
490 | * driver_vm_unmap(); | |
491 | * drm_gpuva_unlink(va); | |
492 | * drm_gpuva_unmap(&op->unmap); | |
493 | * | |
494 | * break; | |
495 | * default: | |
496 | * break; | |
497 | * } | |
498 | * } | |
499 | * driver_unlock_va_space(); | |
500 | * | |
501 | * return 0; | |
502 | * } | |
503 | * | |
504 | * 2) Receive a callback for each &drm_gpuva_op to create a new mapping:: | |
505 | * | |
506 | * struct driver_context { | |
f72c2db4 | 507 | * struct drm_gpuvm *gpuvm; |
e6303f32 DK |
508 | * struct drm_gpuva *new_va; |
509 | * struct drm_gpuva *prev_va; | |
510 | * struct drm_gpuva *next_va; | |
511 | * }; | |
512 | * | |
f72c2db4 DK |
513 | * // ops to pass to drm_gpuvm_init() |
514 | * static const struct drm_gpuvm_ops driver_gpuvm_ops = { | |
e6303f32 DK |
515 | * .sm_step_map = driver_gpuva_map, |
516 | * .sm_step_remap = driver_gpuva_remap, | |
517 | * .sm_step_unmap = driver_gpuva_unmap, | |
518 | * }; | |
519 | * | |
f72c2db4 | 520 | * // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva |
e6303f32 DK |
521 | * // structure in individual driver structures and lock the dma-resv with |
522 | * // drm_exec or similar helpers. | |
f72c2db4 | 523 | * int driver_mapping_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
524 | * u64 addr, u64 range, |
525 | * struct drm_gem_object *obj, u64 offset) | |
526 | * { | |
527 | * struct driver_context ctx; | |
528 | * struct drm_gpuva_ops *ops; | |
529 | * struct drm_gpuva_op *op; | |
530 | * int ret = 0; | |
531 | * | |
f72c2db4 | 532 | * ctx.gpuvm = gpuvm; |
e6303f32 DK |
533 | * |
534 | * ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL); | |
535 | * ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL); | |
536 | * ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL); | |
537 | * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va) { | |
538 | * ret = -ENOMEM; | |
539 | * goto out; | |
540 | * } | |
541 | * | |
542 | * driver_lock_va_space(); | |
f72c2db4 | 543 | * ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset); |
e6303f32 DK |
544 | * driver_unlock_va_space(); |
545 | * | |
546 | * out: | |
547 | * kfree(ctx.new_va); | |
548 | * kfree(ctx.prev_va); | |
549 | * kfree(ctx.next_va); | |
550 | * return ret; | |
551 | * } | |
552 | * | |
553 | * int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx) | |
554 | * { | |
555 | * struct driver_context *ctx = __ctx; | |
556 | * | |
f72c2db4 | 557 | * drm_gpuva_map(ctx->vm, ctx->new_va, &op->map); |
e6303f32 DK |
558 | * |
559 | * drm_gpuva_link(ctx->new_va); | |
560 | * | |
561 | * // prevent the new GPUVA from being freed in | |
562 | * // driver_mapping_create() | |
563 | * ctx->new_va = NULL; | |
564 | * | |
565 | * return 0; | |
566 | * } | |
567 | * | |
568 | * int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx) | |
569 | * { | |
570 | * struct driver_context *ctx = __ctx; | |
571 | * | |
572 | * drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap); | |
573 | * | |
574 | * drm_gpuva_unlink(op->remap.unmap->va); | |
575 | * kfree(op->remap.unmap->va); | |
576 | * | |
577 | * if (op->remap.prev) { | |
578 | * drm_gpuva_link(ctx->prev_va); | |
579 | * ctx->prev_va = NULL; | |
580 | * } | |
581 | * | |
582 | * if (op->remap.next) { | |
583 | * drm_gpuva_link(ctx->next_va); | |
584 | * ctx->next_va = NULL; | |
585 | * } | |
586 | * | |
587 | * return 0; | |
588 | * } | |
589 | * | |
590 | * int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx) | |
591 | * { | |
592 | * drm_gpuva_unlink(op->unmap.va); | |
593 | * drm_gpuva_unmap(&op->unmap); | |
594 | * kfree(op->unmap.va); | |
595 | * | |
596 | * return 0; | |
597 | * } | |
598 | */ | |
599 | ||
600 | #define to_drm_gpuva(__node) container_of((__node), struct drm_gpuva, rb.node) | |
601 | ||
602 | #define GPUVA_START(node) ((node)->va.addr) | |
603 | #define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1) | |
604 | ||
605 | /* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain | |
606 | * about this. | |
607 | */ | |
608 | INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last, | |
609 | GPUVA_START, GPUVA_LAST, static __maybe_unused, | |
610 | drm_gpuva_it) | |
611 | ||
f72c2db4 | 612 | static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
613 | struct drm_gpuva *va); |
614 | static void __drm_gpuva_remove(struct drm_gpuva *va); | |
615 | ||
616 | static bool | |
d1adea27 | 617 | drm_gpuvm_check_overflow(u64 addr, u64 range) |
e6303f32 DK |
618 | { |
619 | u64 end; | |
620 | ||
d1adea27 DK |
621 | return check_add_overflow(addr, range, &end); |
622 | } | |
623 | ||
624 | static bool | |
625 | drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range) | |
626 | { | |
627 | return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range), | |
628 | "GPUVA address limited to %zu bytes.\n", sizeof(addr)); | |
e6303f32 DK |
629 | } |
630 | ||
631 | static bool | |
f72c2db4 | 632 | drm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range) |
e6303f32 DK |
633 | { |
634 | u64 end = addr + range; | |
f72c2db4 DK |
635 | u64 mm_start = gpuvm->mm_start; |
636 | u64 mm_end = mm_start + gpuvm->mm_range; | |
e6303f32 DK |
637 | |
638 | return addr >= mm_start && end <= mm_end; | |
639 | } | |
640 | ||
641 | static bool | |
f72c2db4 | 642 | drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range) |
e6303f32 DK |
643 | { |
644 | u64 end = addr + range; | |
f72c2db4 DK |
645 | u64 kstart = gpuvm->kernel_alloc_node.va.addr; |
646 | u64 krange = gpuvm->kernel_alloc_node.va.range; | |
e6303f32 DK |
647 | u64 kend = kstart + krange; |
648 | ||
649 | return krange && addr < kend && kstart < end; | |
650 | } | |
651 | ||
652 | static bool | |
f72c2db4 | 653 | drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
654 | u64 addr, u64 range) |
655 | { | |
d1adea27 | 656 | return !drm_gpuvm_check_overflow(addr, range) && |
f72c2db4 DK |
657 | drm_gpuvm_in_mm_range(gpuvm, addr, range) && |
658 | !drm_gpuvm_in_kernel_node(gpuvm, addr, range); | |
e6303f32 DK |
659 | } |
660 | ||
661 | /** | |
f72c2db4 DK |
662 | * drm_gpuvm_init() - initialize a &drm_gpuvm |
663 | * @gpuvm: pointer to the &drm_gpuvm to initialize | |
e6303f32 | 664 | * @name: the name of the GPU VA space |
546ca4d3 | 665 | * @drm: the &drm_device this VM resides in |
e6303f32 DK |
666 | * @start_offset: the start offset of the GPU VA space |
667 | * @range: the size of the GPU VA space | |
668 | * @reserve_offset: the start of the kernel reserved GPU VA area | |
669 | * @reserve_range: the size of the kernel reserved GPU VA area | |
f72c2db4 | 670 | * @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap |
e6303f32 | 671 | * |
f72c2db4 | 672 | * The &drm_gpuvm must be initialized with this function before use. |
e6303f32 | 673 | * |
f72c2db4 | 674 | * Note that @gpuvm must be cleared to 0 before calling this function. The given |
e6303f32 DK |
675 | * &name is expected to be managed by the surrounding driver structures. |
676 | */ | |
677 | void | |
546ca4d3 DK |
678 | drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, |
679 | struct drm_device *drm, | |
f72c2db4 DK |
680 | u64 start_offset, u64 range, |
681 | u64 reserve_offset, u64 reserve_range, | |
682 | const struct drm_gpuvm_ops *ops) | |
e6303f32 | 683 | { |
f72c2db4 DK |
684 | gpuvm->rb.tree = RB_ROOT_CACHED; |
685 | INIT_LIST_HEAD(&gpuvm->rb.list); | |
e6303f32 | 686 | |
f72c2db4 DK |
687 | gpuvm->name = name ? name : "unknown"; |
688 | gpuvm->ops = ops; | |
546ca4d3 | 689 | gpuvm->drm = drm; |
e6303f32 | 690 | |
d1adea27 | 691 | drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range); |
546ca4d3 DK |
692 | gpuvm->mm_start = start_offset; |
693 | gpuvm->mm_range = range; | |
e6303f32 | 694 | |
546ca4d3 | 695 | memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva)); |
e6303f32 | 696 | if (reserve_range) { |
f72c2db4 DK |
697 | gpuvm->kernel_alloc_node.va.addr = reserve_offset; |
698 | gpuvm->kernel_alloc_node.va.range = reserve_range; | |
e6303f32 | 699 | |
d1adea27 DK |
700 | if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset, |
701 | reserve_range))) | |
f72c2db4 | 702 | __drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node); |
e6303f32 DK |
703 | } |
704 | } | |
f72c2db4 | 705 | EXPORT_SYMBOL_GPL(drm_gpuvm_init); |
e6303f32 DK |
706 | |
707 | /** | |
f72c2db4 DK |
708 | * drm_gpuvm_destroy() - cleanup a &drm_gpuvm |
709 | * @gpuvm: pointer to the &drm_gpuvm to clean up | |
e6303f32 DK |
710 | * |
711 | * Note that it is a bug to call this function on a manager that still | |
712 | * holds GPU VA mappings. | |
713 | */ | |
714 | void | |
f72c2db4 | 715 | drm_gpuvm_destroy(struct drm_gpuvm *gpuvm) |
e6303f32 | 716 | { |
f72c2db4 | 717 | gpuvm->name = NULL; |
e6303f32 | 718 | |
f72c2db4 DK |
719 | if (gpuvm->kernel_alloc_node.va.range) |
720 | __drm_gpuva_remove(&gpuvm->kernel_alloc_node); | |
e6303f32 | 721 | |
546ca4d3 DK |
722 | drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root), |
723 | "GPUVA tree is not empty, potentially leaking memory.\n"); | |
e6303f32 | 724 | } |
f72c2db4 | 725 | EXPORT_SYMBOL_GPL(drm_gpuvm_destroy); |
e6303f32 DK |
726 | |
727 | static int | |
f72c2db4 | 728 | __drm_gpuva_insert(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
729 | struct drm_gpuva *va) |
730 | { | |
731 | struct rb_node *node; | |
732 | struct list_head *head; | |
733 | ||
f72c2db4 | 734 | if (drm_gpuva_it_iter_first(&gpuvm->rb.tree, |
e6303f32 DK |
735 | GPUVA_START(va), |
736 | GPUVA_LAST(va))) | |
737 | return -EEXIST; | |
738 | ||
f72c2db4 | 739 | va->vm = gpuvm; |
e6303f32 | 740 | |
f72c2db4 | 741 | drm_gpuva_it_insert(va, &gpuvm->rb.tree); |
e6303f32 DK |
742 | |
743 | node = rb_prev(&va->rb.node); | |
744 | if (node) | |
745 | head = &(to_drm_gpuva(node))->rb.entry; | |
746 | else | |
f72c2db4 | 747 | head = &gpuvm->rb.list; |
e6303f32 DK |
748 | |
749 | list_add(&va->rb.entry, head); | |
750 | ||
751 | return 0; | |
752 | } | |
753 | ||
754 | /** | |
755 | * drm_gpuva_insert() - insert a &drm_gpuva | |
f72c2db4 | 756 | * @gpuvm: the &drm_gpuvm to insert the &drm_gpuva in |
e6303f32 DK |
757 | * @va: the &drm_gpuva to insert |
758 | * | |
759 | * Insert a &drm_gpuva with a given address and range into a | |
f72c2db4 | 760 | * &drm_gpuvm. |
e6303f32 DK |
761 | * |
762 | * It is safe to use this function using the safe versions of iterating the GPU | |
f72c2db4 DK |
763 | * VA space, such as drm_gpuvm_for_each_va_safe() and |
764 | * drm_gpuvm_for_each_va_range_safe(). | |
e6303f32 DK |
765 | * |
766 | * Returns: 0 on success, negative error code on failure. | |
767 | */ | |
768 | int | |
f72c2db4 | 769 | drm_gpuva_insert(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
770 | struct drm_gpuva *va) |
771 | { | |
772 | u64 addr = va->va.addr; | |
773 | u64 range = va->va.range; | |
774 | ||
f72c2db4 | 775 | if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range))) |
e6303f32 DK |
776 | return -EINVAL; |
777 | ||
f72c2db4 | 778 | return __drm_gpuva_insert(gpuvm, va); |
e6303f32 DK |
779 | } |
780 | EXPORT_SYMBOL_GPL(drm_gpuva_insert); | |
781 | ||
782 | static void | |
783 | __drm_gpuva_remove(struct drm_gpuva *va) | |
784 | { | |
f72c2db4 | 785 | drm_gpuva_it_remove(va, &va->vm->rb.tree); |
e6303f32 DK |
786 | list_del_init(&va->rb.entry); |
787 | } | |
788 | ||
789 | /** | |
790 | * drm_gpuva_remove() - remove a &drm_gpuva | |
791 | * @va: the &drm_gpuva to remove | |
792 | * | |
793 | * This removes the given &va from the underlaying tree. | |
794 | * | |
795 | * It is safe to use this function using the safe versions of iterating the GPU | |
f72c2db4 DK |
796 | * VA space, such as drm_gpuvm_for_each_va_safe() and |
797 | * drm_gpuvm_for_each_va_range_safe(). | |
e6303f32 DK |
798 | */ |
799 | void | |
800 | drm_gpuva_remove(struct drm_gpuva *va) | |
801 | { | |
f72c2db4 | 802 | struct drm_gpuvm *gpuvm = va->vm; |
e6303f32 | 803 | |
f72c2db4 | 804 | if (unlikely(va == &gpuvm->kernel_alloc_node)) { |
546ca4d3 DK |
805 | drm_WARN(gpuvm->drm, 1, |
806 | "Can't destroy kernel reserved node.\n"); | |
e6303f32 DK |
807 | return; |
808 | } | |
809 | ||
810 | __drm_gpuva_remove(va); | |
811 | } | |
812 | EXPORT_SYMBOL_GPL(drm_gpuva_remove); | |
813 | ||
814 | /** | |
815 | * drm_gpuva_link() - link a &drm_gpuva | |
816 | * @va: the &drm_gpuva to link | |
817 | * | |
818 | * This adds the given &va to the GPU VA list of the &drm_gem_object it is | |
819 | * associated with. | |
820 | * | |
821 | * This function expects the caller to protect the GEM's GPUVA list against | |
822 | * concurrent access using the GEMs dma_resv lock. | |
823 | */ | |
824 | void | |
825 | drm_gpuva_link(struct drm_gpuva *va) | |
826 | { | |
827 | struct drm_gem_object *obj = va->gem.obj; | |
828 | ||
829 | if (unlikely(!obj)) | |
830 | return; | |
831 | ||
832 | drm_gem_gpuva_assert_lock_held(obj); | |
833 | ||
834 | list_add_tail(&va->gem.entry, &obj->gpuva.list); | |
835 | } | |
836 | EXPORT_SYMBOL_GPL(drm_gpuva_link); | |
837 | ||
838 | /** | |
839 | * drm_gpuva_unlink() - unlink a &drm_gpuva | |
840 | * @va: the &drm_gpuva to unlink | |
841 | * | |
842 | * This removes the given &va from the GPU VA list of the &drm_gem_object it is | |
843 | * associated with. | |
844 | * | |
845 | * This function expects the caller to protect the GEM's GPUVA list against | |
846 | * concurrent access using the GEMs dma_resv lock. | |
847 | */ | |
848 | void | |
849 | drm_gpuva_unlink(struct drm_gpuva *va) | |
850 | { | |
851 | struct drm_gem_object *obj = va->gem.obj; | |
852 | ||
853 | if (unlikely(!obj)) | |
854 | return; | |
855 | ||
856 | drm_gem_gpuva_assert_lock_held(obj); | |
857 | ||
858 | list_del_init(&va->gem.entry); | |
859 | } | |
860 | EXPORT_SYMBOL_GPL(drm_gpuva_unlink); | |
861 | ||
862 | /** | |
863 | * drm_gpuva_find_first() - find the first &drm_gpuva in the given range | |
f72c2db4 | 864 | * @gpuvm: the &drm_gpuvm to search in |
e6303f32 DK |
865 | * @addr: the &drm_gpuvas address |
866 | * @range: the &drm_gpuvas range | |
867 | * | |
868 | * Returns: the first &drm_gpuva within the given range | |
869 | */ | |
870 | struct drm_gpuva * | |
f72c2db4 | 871 | drm_gpuva_find_first(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
872 | u64 addr, u64 range) |
873 | { | |
874 | u64 last = addr + range - 1; | |
875 | ||
f72c2db4 | 876 | return drm_gpuva_it_iter_first(&gpuvm->rb.tree, addr, last); |
e6303f32 DK |
877 | } |
878 | EXPORT_SYMBOL_GPL(drm_gpuva_find_first); | |
879 | ||
880 | /** | |
881 | * drm_gpuva_find() - find a &drm_gpuva | |
f72c2db4 | 882 | * @gpuvm: the &drm_gpuvm to search in |
e6303f32 DK |
883 | * @addr: the &drm_gpuvas address |
884 | * @range: the &drm_gpuvas range | |
885 | * | |
886 | * Returns: the &drm_gpuva at a given &addr and with a given &range | |
887 | */ | |
888 | struct drm_gpuva * | |
f72c2db4 | 889 | drm_gpuva_find(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
890 | u64 addr, u64 range) |
891 | { | |
892 | struct drm_gpuva *va; | |
893 | ||
f72c2db4 | 894 | va = drm_gpuva_find_first(gpuvm, addr, range); |
e6303f32 DK |
895 | if (!va) |
896 | goto out; | |
897 | ||
898 | if (va->va.addr != addr || | |
899 | va->va.range != range) | |
900 | goto out; | |
901 | ||
902 | return va; | |
903 | ||
904 | out: | |
905 | return NULL; | |
906 | } | |
907 | EXPORT_SYMBOL_GPL(drm_gpuva_find); | |
908 | ||
909 | /** | |
910 | * drm_gpuva_find_prev() - find the &drm_gpuva before the given address | |
f72c2db4 | 911 | * @gpuvm: the &drm_gpuvm to search in |
e6303f32 DK |
912 | * @start: the given GPU VA's start address |
913 | * | |
914 | * Find the adjacent &drm_gpuva before the GPU VA with given &start address. | |
915 | * | |
916 | * Note that if there is any free space between the GPU VA mappings no mapping | |
917 | * is returned. | |
918 | * | |
919 | * Returns: a pointer to the found &drm_gpuva or NULL if none was found | |
920 | */ | |
921 | struct drm_gpuva * | |
f72c2db4 | 922 | drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start) |
e6303f32 | 923 | { |
f72c2db4 | 924 | if (!drm_gpuvm_range_valid(gpuvm, start - 1, 1)) |
e6303f32 DK |
925 | return NULL; |
926 | ||
f72c2db4 | 927 | return drm_gpuva_it_iter_first(&gpuvm->rb.tree, start - 1, start); |
e6303f32 DK |
928 | } |
929 | EXPORT_SYMBOL_GPL(drm_gpuva_find_prev); | |
930 | ||
931 | /** | |
932 | * drm_gpuva_find_next() - find the &drm_gpuva after the given address | |
f72c2db4 | 933 | * @gpuvm: the &drm_gpuvm to search in |
e6303f32 DK |
934 | * @end: the given GPU VA's end address |
935 | * | |
936 | * Find the adjacent &drm_gpuva after the GPU VA with given &end address. | |
937 | * | |
938 | * Note that if there is any free space between the GPU VA mappings no mapping | |
939 | * is returned. | |
940 | * | |
941 | * Returns: a pointer to the found &drm_gpuva or NULL if none was found | |
942 | */ | |
943 | struct drm_gpuva * | |
f72c2db4 | 944 | drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end) |
e6303f32 | 945 | { |
f72c2db4 | 946 | if (!drm_gpuvm_range_valid(gpuvm, end, 1)) |
e6303f32 DK |
947 | return NULL; |
948 | ||
f72c2db4 | 949 | return drm_gpuva_it_iter_first(&gpuvm->rb.tree, end, end + 1); |
e6303f32 DK |
950 | } |
951 | EXPORT_SYMBOL_GPL(drm_gpuva_find_next); | |
952 | ||
953 | /** | |
f72c2db4 | 954 | * drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space |
e6303f32 | 955 | * is empty |
f72c2db4 | 956 | * @gpuvm: the &drm_gpuvm to check the range for |
e6303f32 DK |
957 | * @addr: the start address of the range |
958 | * @range: the range of the interval | |
959 | * | |
960 | * Returns: true if the interval is empty, false otherwise | |
961 | */ | |
962 | bool | |
f72c2db4 | 963 | drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range) |
e6303f32 | 964 | { |
f72c2db4 | 965 | return !drm_gpuva_find_first(gpuvm, addr, range); |
e6303f32 | 966 | } |
f72c2db4 | 967 | EXPORT_SYMBOL_GPL(drm_gpuvm_interval_empty); |
e6303f32 DK |
968 | |
969 | /** | |
970 | * drm_gpuva_map() - helper to insert a &drm_gpuva according to a | |
971 | * &drm_gpuva_op_map | |
f72c2db4 | 972 | * @gpuvm: the &drm_gpuvm |
e6303f32 DK |
973 | * @va: the &drm_gpuva to insert |
974 | * @op: the &drm_gpuva_op_map to initialize @va with | |
975 | * | |
f72c2db4 | 976 | * Initializes the @va from the @op and inserts it into the given @gpuvm. |
e6303f32 DK |
977 | */ |
978 | void | |
f72c2db4 | 979 | drm_gpuva_map(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
980 | struct drm_gpuva *va, |
981 | struct drm_gpuva_op_map *op) | |
982 | { | |
983 | drm_gpuva_init_from_op(va, op); | |
f72c2db4 | 984 | drm_gpuva_insert(gpuvm, va); |
e6303f32 DK |
985 | } |
986 | EXPORT_SYMBOL_GPL(drm_gpuva_map); | |
987 | ||
988 | /** | |
989 | * drm_gpuva_remap() - helper to remap a &drm_gpuva according to a | |
990 | * &drm_gpuva_op_remap | |
991 | * @prev: the &drm_gpuva to remap when keeping the start of a mapping | |
992 | * @next: the &drm_gpuva to remap when keeping the end of a mapping | |
993 | * @op: the &drm_gpuva_op_remap to initialize @prev and @next with | |
994 | * | |
995 | * Removes the currently mapped &drm_gpuva and remaps it using @prev and/or | |
996 | * @next. | |
997 | */ | |
998 | void | |
999 | drm_gpuva_remap(struct drm_gpuva *prev, | |
1000 | struct drm_gpuva *next, | |
1001 | struct drm_gpuva_op_remap *op) | |
1002 | { | |
1003 | struct drm_gpuva *curr = op->unmap->va; | |
f72c2db4 | 1004 | struct drm_gpuvm *gpuvm = curr->vm; |
e6303f32 DK |
1005 | |
1006 | drm_gpuva_remove(curr); | |
1007 | ||
1008 | if (op->prev) { | |
1009 | drm_gpuva_init_from_op(prev, op->prev); | |
f72c2db4 | 1010 | drm_gpuva_insert(gpuvm, prev); |
e6303f32 DK |
1011 | } |
1012 | ||
1013 | if (op->next) { | |
1014 | drm_gpuva_init_from_op(next, op->next); | |
f72c2db4 | 1015 | drm_gpuva_insert(gpuvm, next); |
e6303f32 DK |
1016 | } |
1017 | } | |
1018 | EXPORT_SYMBOL_GPL(drm_gpuva_remap); | |
1019 | ||
1020 | /** | |
1021 | * drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a | |
1022 | * &drm_gpuva_op_unmap | |
1023 | * @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove | |
1024 | * | |
1025 | * Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap. | |
1026 | */ | |
1027 | void | |
1028 | drm_gpuva_unmap(struct drm_gpuva_op_unmap *op) | |
1029 | { | |
1030 | drm_gpuva_remove(op->va); | |
1031 | } | |
1032 | EXPORT_SYMBOL_GPL(drm_gpuva_unmap); | |
1033 | ||
1034 | static int | |
f72c2db4 | 1035 | op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, |
e6303f32 DK |
1036 | u64 addr, u64 range, |
1037 | struct drm_gem_object *obj, u64 offset) | |
1038 | { | |
1039 | struct drm_gpuva_op op = {}; | |
1040 | ||
1041 | op.op = DRM_GPUVA_OP_MAP; | |
1042 | op.map.va.addr = addr; | |
1043 | op.map.va.range = range; | |
1044 | op.map.gem.obj = obj; | |
1045 | op.map.gem.offset = offset; | |
1046 | ||
1047 | return fn->sm_step_map(&op, priv); | |
1048 | } | |
1049 | ||
1050 | static int | |
f72c2db4 | 1051 | op_remap_cb(const struct drm_gpuvm_ops *fn, void *priv, |
e6303f32 DK |
1052 | struct drm_gpuva_op_map *prev, |
1053 | struct drm_gpuva_op_map *next, | |
1054 | struct drm_gpuva_op_unmap *unmap) | |
1055 | { | |
1056 | struct drm_gpuva_op op = {}; | |
1057 | struct drm_gpuva_op_remap *r; | |
1058 | ||
1059 | op.op = DRM_GPUVA_OP_REMAP; | |
1060 | r = &op.remap; | |
1061 | r->prev = prev; | |
1062 | r->next = next; | |
1063 | r->unmap = unmap; | |
1064 | ||
1065 | return fn->sm_step_remap(&op, priv); | |
1066 | } | |
1067 | ||
1068 | static int | |
f72c2db4 | 1069 | op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv, |
e6303f32 DK |
1070 | struct drm_gpuva *va, bool merge) |
1071 | { | |
1072 | struct drm_gpuva_op op = {}; | |
1073 | ||
1074 | op.op = DRM_GPUVA_OP_UNMAP; | |
1075 | op.unmap.va = va; | |
1076 | op.unmap.keep = merge; | |
1077 | ||
1078 | return fn->sm_step_unmap(&op, priv); | |
1079 | } | |
1080 | ||
1081 | static int | |
f72c2db4 DK |
1082 | __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, |
1083 | const struct drm_gpuvm_ops *ops, void *priv, | |
e6303f32 DK |
1084 | u64 req_addr, u64 req_range, |
1085 | struct drm_gem_object *req_obj, u64 req_offset) | |
1086 | { | |
cdf4100e | 1087 | struct drm_gpuva *va, *next; |
e6303f32 DK |
1088 | u64 req_end = req_addr + req_range; |
1089 | int ret; | |
1090 | ||
f72c2db4 | 1091 | if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) |
e6303f32 DK |
1092 | return -EINVAL; |
1093 | ||
f72c2db4 | 1094 | drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { |
e6303f32 DK |
1095 | struct drm_gem_object *obj = va->gem.obj; |
1096 | u64 offset = va->gem.offset; | |
1097 | u64 addr = va->va.addr; | |
1098 | u64 range = va->va.range; | |
1099 | u64 end = addr + range; | |
1100 | bool merge = !!va->gem.obj; | |
1101 | ||
1102 | if (addr == req_addr) { | |
1103 | merge &= obj == req_obj && | |
1104 | offset == req_offset; | |
1105 | ||
1106 | if (end == req_end) { | |
1107 | ret = op_unmap_cb(ops, priv, va, merge); | |
1108 | if (ret) | |
1109 | return ret; | |
1110 | break; | |
1111 | } | |
1112 | ||
1113 | if (end < req_end) { | |
1114 | ret = op_unmap_cb(ops, priv, va, merge); | |
1115 | if (ret) | |
1116 | return ret; | |
cdf4100e | 1117 | continue; |
e6303f32 DK |
1118 | } |
1119 | ||
1120 | if (end > req_end) { | |
1121 | struct drm_gpuva_op_map n = { | |
1122 | .va.addr = req_end, | |
1123 | .va.range = range - req_range, | |
1124 | .gem.obj = obj, | |
1125 | .gem.offset = offset + req_range, | |
1126 | }; | |
1127 | struct drm_gpuva_op_unmap u = { | |
1128 | .va = va, | |
1129 | .keep = merge, | |
1130 | }; | |
1131 | ||
1132 | ret = op_remap_cb(ops, priv, NULL, &n, &u); | |
1133 | if (ret) | |
1134 | return ret; | |
1135 | break; | |
1136 | } | |
1137 | } else if (addr < req_addr) { | |
1138 | u64 ls_range = req_addr - addr; | |
1139 | struct drm_gpuva_op_map p = { | |
1140 | .va.addr = addr, | |
1141 | .va.range = ls_range, | |
1142 | .gem.obj = obj, | |
1143 | .gem.offset = offset, | |
1144 | }; | |
1145 | struct drm_gpuva_op_unmap u = { .va = va }; | |
1146 | ||
1147 | merge &= obj == req_obj && | |
1148 | offset + ls_range == req_offset; | |
1149 | u.keep = merge; | |
1150 | ||
1151 | if (end == req_end) { | |
1152 | ret = op_remap_cb(ops, priv, &p, NULL, &u); | |
1153 | if (ret) | |
1154 | return ret; | |
1155 | break; | |
1156 | } | |
1157 | ||
1158 | if (end < req_end) { | |
1159 | ret = op_remap_cb(ops, priv, &p, NULL, &u); | |
1160 | if (ret) | |
1161 | return ret; | |
cdf4100e | 1162 | continue; |
e6303f32 DK |
1163 | } |
1164 | ||
1165 | if (end > req_end) { | |
1166 | struct drm_gpuva_op_map n = { | |
1167 | .va.addr = req_end, | |
1168 | .va.range = end - req_end, | |
1169 | .gem.obj = obj, | |
1170 | .gem.offset = offset + ls_range + | |
1171 | req_range, | |
1172 | }; | |
1173 | ||
1174 | ret = op_remap_cb(ops, priv, &p, &n, &u); | |
1175 | if (ret) | |
1176 | return ret; | |
1177 | break; | |
1178 | } | |
1179 | } else if (addr > req_addr) { | |
1180 | merge &= obj == req_obj && | |
1181 | offset == req_offset + | |
1182 | (addr - req_addr); | |
1183 | ||
1184 | if (end == req_end) { | |
1185 | ret = op_unmap_cb(ops, priv, va, merge); | |
1186 | if (ret) | |
1187 | return ret; | |
1188 | break; | |
1189 | } | |
1190 | ||
1191 | if (end < req_end) { | |
1192 | ret = op_unmap_cb(ops, priv, va, merge); | |
1193 | if (ret) | |
1194 | return ret; | |
cdf4100e | 1195 | continue; |
e6303f32 DK |
1196 | } |
1197 | ||
1198 | if (end > req_end) { | |
1199 | struct drm_gpuva_op_map n = { | |
1200 | .va.addr = req_end, | |
1201 | .va.range = end - req_end, | |
1202 | .gem.obj = obj, | |
1203 | .gem.offset = offset + req_end - addr, | |
1204 | }; | |
1205 | struct drm_gpuva_op_unmap u = { | |
1206 | .va = va, | |
1207 | .keep = merge, | |
1208 | }; | |
1209 | ||
1210 | ret = op_remap_cb(ops, priv, NULL, &n, &u); | |
1211 | if (ret) | |
1212 | return ret; | |
1213 | break; | |
1214 | } | |
1215 | } | |
e6303f32 DK |
1216 | } |
1217 | ||
1218 | return op_map_cb(ops, priv, | |
1219 | req_addr, req_range, | |
1220 | req_obj, req_offset); | |
1221 | } | |
1222 | ||
1223 | static int | |
f72c2db4 DK |
1224 | __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, |
1225 | const struct drm_gpuvm_ops *ops, void *priv, | |
e6303f32 DK |
1226 | u64 req_addr, u64 req_range) |
1227 | { | |
1228 | struct drm_gpuva *va, *next; | |
1229 | u64 req_end = req_addr + req_range; | |
1230 | int ret; | |
1231 | ||
f72c2db4 | 1232 | if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) |
e6303f32 DK |
1233 | return -EINVAL; |
1234 | ||
f72c2db4 | 1235 | drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { |
e6303f32 DK |
1236 | struct drm_gpuva_op_map prev = {}, next = {}; |
1237 | bool prev_split = false, next_split = false; | |
1238 | struct drm_gem_object *obj = va->gem.obj; | |
1239 | u64 offset = va->gem.offset; | |
1240 | u64 addr = va->va.addr; | |
1241 | u64 range = va->va.range; | |
1242 | u64 end = addr + range; | |
1243 | ||
1244 | if (addr < req_addr) { | |
1245 | prev.va.addr = addr; | |
1246 | prev.va.range = req_addr - addr; | |
1247 | prev.gem.obj = obj; | |
1248 | prev.gem.offset = offset; | |
1249 | ||
1250 | prev_split = true; | |
1251 | } | |
1252 | ||
1253 | if (end > req_end) { | |
1254 | next.va.addr = req_end; | |
1255 | next.va.range = end - req_end; | |
1256 | next.gem.obj = obj; | |
1257 | next.gem.offset = offset + (req_end - addr); | |
1258 | ||
1259 | next_split = true; | |
1260 | } | |
1261 | ||
1262 | if (prev_split || next_split) { | |
1263 | struct drm_gpuva_op_unmap unmap = { .va = va }; | |
1264 | ||
1265 | ret = op_remap_cb(ops, priv, | |
1266 | prev_split ? &prev : NULL, | |
1267 | next_split ? &next : NULL, | |
1268 | &unmap); | |
1269 | if (ret) | |
1270 | return ret; | |
1271 | } else { | |
1272 | ret = op_unmap_cb(ops, priv, va, false); | |
1273 | if (ret) | |
1274 | return ret; | |
1275 | } | |
1276 | } | |
1277 | ||
1278 | return 0; | |
1279 | } | |
1280 | ||
1281 | /** | |
f72c2db4 DK |
1282 | * drm_gpuvm_sm_map() - creates the &drm_gpuva_op split/merge steps |
1283 | * @gpuvm: the &drm_gpuvm representing the GPU VA space | |
e6303f32 DK |
1284 | * @req_addr: the start address of the new mapping |
1285 | * @req_range: the range of the new mapping | |
1286 | * @req_obj: the &drm_gem_object to map | |
1287 | * @req_offset: the offset within the &drm_gem_object | |
1288 | * @priv: pointer to a driver private data structure | |
1289 | * | |
1290 | * This function iterates the given range of the GPU VA space. It utilizes the | |
f72c2db4 | 1291 | * &drm_gpuvm_ops to call back into the driver providing the split and merge |
e6303f32 DK |
1292 | * steps. |
1293 | * | |
1294 | * Drivers may use these callbacks to update the GPU VA space right away within | |
1295 | * the callback. In case the driver decides to copy and store the operations for | |
f72c2db4 DK |
1296 | * later processing neither this function nor &drm_gpuvm_sm_unmap is allowed to |
1297 | * be called before the &drm_gpuvm's view of the GPU VA space was | |
e6303f32 | 1298 | * updated with the previous set of operations. To update the |
f72c2db4 | 1299 | * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), |
e6303f32 DK |
1300 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be |
1301 | * used. | |
1302 | * | |
1303 | * A sequence of callbacks can contain map, unmap and remap operations, but | |
1304 | * the sequence of callbacks might also be empty if no operation is required, | |
1305 | * e.g. if the requested mapping already exists in the exact same way. | |
1306 | * | |
1307 | * There can be an arbitrary amount of unmap operations, a maximum of two remap | |
1308 | * operations and a single map operation. The latter one represents the original | |
1309 | * map operation requested by the caller. | |
1310 | * | |
1311 | * Returns: 0 on success or a negative error code | |
1312 | */ | |
1313 | int | |
f72c2db4 | 1314 | drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, |
e6303f32 DK |
1315 | u64 req_addr, u64 req_range, |
1316 | struct drm_gem_object *req_obj, u64 req_offset) | |
1317 | { | |
f72c2db4 | 1318 | const struct drm_gpuvm_ops *ops = gpuvm->ops; |
e6303f32 DK |
1319 | |
1320 | if (unlikely(!(ops && ops->sm_step_map && | |
1321 | ops->sm_step_remap && | |
1322 | ops->sm_step_unmap))) | |
1323 | return -EINVAL; | |
1324 | ||
f72c2db4 | 1325 | return __drm_gpuvm_sm_map(gpuvm, ops, priv, |
e6303f32 DK |
1326 | req_addr, req_range, |
1327 | req_obj, req_offset); | |
1328 | } | |
f72c2db4 | 1329 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map); |
e6303f32 DK |
1330 | |
1331 | /** | |
f72c2db4 DK |
1332 | * drm_gpuvm_sm_unmap() - creates the &drm_gpuva_ops to split on unmap |
1333 | * @gpuvm: the &drm_gpuvm representing the GPU VA space | |
e6303f32 DK |
1334 | * @priv: pointer to a driver private data structure |
1335 | * @req_addr: the start address of the range to unmap | |
1336 | * @req_range: the range of the mappings to unmap | |
1337 | * | |
1338 | * This function iterates the given range of the GPU VA space. It utilizes the | |
f72c2db4 | 1339 | * &drm_gpuvm_ops to call back into the driver providing the operations to |
e6303f32 DK |
1340 | * unmap and, if required, split existent mappings. |
1341 | * | |
1342 | * Drivers may use these callbacks to update the GPU VA space right away within | |
1343 | * the callback. In case the driver decides to copy and store the operations for | |
f72c2db4 DK |
1344 | * later processing neither this function nor &drm_gpuvm_sm_map is allowed to be |
1345 | * called before the &drm_gpuvm's view of the GPU VA space was updated | |
1346 | * with the previous set of operations. To update the &drm_gpuvm's view | |
e6303f32 DK |
1347 | * of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or |
1348 | * drm_gpuva_destroy_unlocked() should be used. | |
1349 | * | |
1350 | * A sequence of callbacks can contain unmap and remap operations, depending on | |
1351 | * whether there are actual overlapping mappings to split. | |
1352 | * | |
1353 | * There can be an arbitrary amount of unmap operations and a maximum of two | |
1354 | * remap operations. | |
1355 | * | |
1356 | * Returns: 0 on success or a negative error code | |
1357 | */ | |
1358 | int | |
f72c2db4 | 1359 | drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, |
e6303f32 DK |
1360 | u64 req_addr, u64 req_range) |
1361 | { | |
f72c2db4 | 1362 | const struct drm_gpuvm_ops *ops = gpuvm->ops; |
e6303f32 DK |
1363 | |
1364 | if (unlikely(!(ops && ops->sm_step_remap && | |
1365 | ops->sm_step_unmap))) | |
1366 | return -EINVAL; | |
1367 | ||
f72c2db4 | 1368 | return __drm_gpuvm_sm_unmap(gpuvm, ops, priv, |
e6303f32 DK |
1369 | req_addr, req_range); |
1370 | } | |
f72c2db4 | 1371 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap); |
e6303f32 DK |
1372 | |
1373 | static struct drm_gpuva_op * | |
f72c2db4 | 1374 | gpuva_op_alloc(struct drm_gpuvm *gpuvm) |
e6303f32 | 1375 | { |
f72c2db4 | 1376 | const struct drm_gpuvm_ops *fn = gpuvm->ops; |
e6303f32 DK |
1377 | struct drm_gpuva_op *op; |
1378 | ||
1379 | if (fn && fn->op_alloc) | |
1380 | op = fn->op_alloc(); | |
1381 | else | |
1382 | op = kzalloc(sizeof(*op), GFP_KERNEL); | |
1383 | ||
1384 | if (unlikely(!op)) | |
1385 | return NULL; | |
1386 | ||
1387 | return op; | |
1388 | } | |
1389 | ||
1390 | static void | |
f72c2db4 | 1391 | gpuva_op_free(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1392 | struct drm_gpuva_op *op) |
1393 | { | |
f72c2db4 | 1394 | const struct drm_gpuvm_ops *fn = gpuvm->ops; |
e6303f32 DK |
1395 | |
1396 | if (fn && fn->op_free) | |
1397 | fn->op_free(op); | |
1398 | else | |
1399 | kfree(op); | |
1400 | } | |
1401 | ||
1402 | static int | |
1403 | drm_gpuva_sm_step(struct drm_gpuva_op *__op, | |
1404 | void *priv) | |
1405 | { | |
1406 | struct { | |
f72c2db4 | 1407 | struct drm_gpuvm *vm; |
e6303f32 DK |
1408 | struct drm_gpuva_ops *ops; |
1409 | } *args = priv; | |
f72c2db4 | 1410 | struct drm_gpuvm *gpuvm = args->vm; |
e6303f32 DK |
1411 | struct drm_gpuva_ops *ops = args->ops; |
1412 | struct drm_gpuva_op *op; | |
1413 | ||
f72c2db4 | 1414 | op = gpuva_op_alloc(gpuvm); |
e6303f32 DK |
1415 | if (unlikely(!op)) |
1416 | goto err; | |
1417 | ||
1418 | memcpy(op, __op, sizeof(*op)); | |
1419 | ||
1420 | if (op->op == DRM_GPUVA_OP_REMAP) { | |
1421 | struct drm_gpuva_op_remap *__r = &__op->remap; | |
1422 | struct drm_gpuva_op_remap *r = &op->remap; | |
1423 | ||
1424 | r->unmap = kmemdup(__r->unmap, sizeof(*r->unmap), | |
1425 | GFP_KERNEL); | |
1426 | if (unlikely(!r->unmap)) | |
1427 | goto err_free_op; | |
1428 | ||
1429 | if (__r->prev) { | |
1430 | r->prev = kmemdup(__r->prev, sizeof(*r->prev), | |
1431 | GFP_KERNEL); | |
1432 | if (unlikely(!r->prev)) | |
1433 | goto err_free_unmap; | |
1434 | } | |
1435 | ||
1436 | if (__r->next) { | |
1437 | r->next = kmemdup(__r->next, sizeof(*r->next), | |
1438 | GFP_KERNEL); | |
1439 | if (unlikely(!r->next)) | |
1440 | goto err_free_prev; | |
1441 | } | |
1442 | } | |
1443 | ||
1444 | list_add_tail(&op->entry, &ops->list); | |
1445 | ||
1446 | return 0; | |
1447 | ||
1448 | err_free_unmap: | |
1449 | kfree(op->remap.unmap); | |
1450 | err_free_prev: | |
1451 | kfree(op->remap.prev); | |
1452 | err_free_op: | |
f72c2db4 | 1453 | gpuva_op_free(gpuvm, op); |
e6303f32 DK |
1454 | err: |
1455 | return -ENOMEM; | |
1456 | } | |
1457 | ||
f72c2db4 | 1458 | static const struct drm_gpuvm_ops gpuvm_list_ops = { |
e6303f32 DK |
1459 | .sm_step_map = drm_gpuva_sm_step, |
1460 | .sm_step_remap = drm_gpuva_sm_step, | |
1461 | .sm_step_unmap = drm_gpuva_sm_step, | |
1462 | }; | |
1463 | ||
1464 | /** | |
f72c2db4 DK |
1465 | * drm_gpuvm_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge |
1466 | * @gpuvm: the &drm_gpuvm representing the GPU VA space | |
e6303f32 DK |
1467 | * @req_addr: the start address of the new mapping |
1468 | * @req_range: the range of the new mapping | |
1469 | * @req_obj: the &drm_gem_object to map | |
1470 | * @req_offset: the offset within the &drm_gem_object | |
1471 | * | |
1472 | * This function creates a list of operations to perform splitting and merging | |
1473 | * of existent mapping(s) with the newly requested one. | |
1474 | * | |
1475 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed | |
1476 | * in the given order. It can contain map, unmap and remap operations, but it | |
1477 | * also can be empty if no operation is required, e.g. if the requested mapping | |
1478 | * already exists is the exact same way. | |
1479 | * | |
1480 | * There can be an arbitrary amount of unmap operations, a maximum of two remap | |
1481 | * operations and a single map operation. The latter one represents the original | |
1482 | * map operation requested by the caller. | |
1483 | * | |
1484 | * Note that before calling this function again with another mapping request it | |
f72c2db4 | 1485 | * is necessary to update the &drm_gpuvm's view of the GPU VA space. The |
e6303f32 | 1486 | * previously obtained operations must be either processed or abandoned. To |
f72c2db4 | 1487 | * update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), |
e6303f32 DK |
1488 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be |
1489 | * used. | |
1490 | * | |
1491 | * After the caller finished processing the returned &drm_gpuva_ops, they must | |
1492 | * be freed with &drm_gpuva_ops_free. | |
1493 | * | |
1494 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure | |
1495 | */ | |
1496 | struct drm_gpuva_ops * | |
f72c2db4 | 1497 | drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1498 | u64 req_addr, u64 req_range, |
1499 | struct drm_gem_object *req_obj, u64 req_offset) | |
1500 | { | |
1501 | struct drm_gpuva_ops *ops; | |
1502 | struct { | |
f72c2db4 | 1503 | struct drm_gpuvm *vm; |
e6303f32 DK |
1504 | struct drm_gpuva_ops *ops; |
1505 | } args; | |
1506 | int ret; | |
1507 | ||
1508 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); | |
1509 | if (unlikely(!ops)) | |
1510 | return ERR_PTR(-ENOMEM); | |
1511 | ||
1512 | INIT_LIST_HEAD(&ops->list); | |
1513 | ||
f72c2db4 | 1514 | args.vm = gpuvm; |
e6303f32 DK |
1515 | args.ops = ops; |
1516 | ||
f72c2db4 | 1517 | ret = __drm_gpuvm_sm_map(gpuvm, &gpuvm_list_ops, &args, |
e6303f32 DK |
1518 | req_addr, req_range, |
1519 | req_obj, req_offset); | |
1520 | if (ret) | |
1521 | goto err_free_ops; | |
1522 | ||
1523 | return ops; | |
1524 | ||
1525 | err_free_ops: | |
f72c2db4 | 1526 | drm_gpuva_ops_free(gpuvm, ops); |
e6303f32 DK |
1527 | return ERR_PTR(ret); |
1528 | } | |
f72c2db4 | 1529 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_ops_create); |
e6303f32 DK |
1530 | |
1531 | /** | |
f72c2db4 | 1532 | * drm_gpuvm_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on |
e6303f32 | 1533 | * unmap |
f72c2db4 | 1534 | * @gpuvm: the &drm_gpuvm representing the GPU VA space |
e6303f32 DK |
1535 | * @req_addr: the start address of the range to unmap |
1536 | * @req_range: the range of the mappings to unmap | |
1537 | * | |
1538 | * This function creates a list of operations to perform unmapping and, if | |
1539 | * required, splitting of the mappings overlapping the unmap range. | |
1540 | * | |
1541 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed | |
1542 | * in the given order. It can contain unmap and remap operations, depending on | |
1543 | * whether there are actual overlapping mappings to split. | |
1544 | * | |
1545 | * There can be an arbitrary amount of unmap operations and a maximum of two | |
1546 | * remap operations. | |
1547 | * | |
1548 | * Note that before calling this function again with another range to unmap it | |
f72c2db4 | 1549 | * is necessary to update the &drm_gpuvm's view of the GPU VA space. The |
e6303f32 | 1550 | * previously obtained operations must be processed or abandoned. To update the |
f72c2db4 | 1551 | * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), |
e6303f32 DK |
1552 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be |
1553 | * used. | |
1554 | * | |
1555 | * After the caller finished processing the returned &drm_gpuva_ops, they must | |
1556 | * be freed with &drm_gpuva_ops_free. | |
1557 | * | |
1558 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure | |
1559 | */ | |
1560 | struct drm_gpuva_ops * | |
f72c2db4 | 1561 | drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1562 | u64 req_addr, u64 req_range) |
1563 | { | |
1564 | struct drm_gpuva_ops *ops; | |
1565 | struct { | |
f72c2db4 | 1566 | struct drm_gpuvm *vm; |
e6303f32 DK |
1567 | struct drm_gpuva_ops *ops; |
1568 | } args; | |
1569 | int ret; | |
1570 | ||
1571 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); | |
1572 | if (unlikely(!ops)) | |
1573 | return ERR_PTR(-ENOMEM); | |
1574 | ||
1575 | INIT_LIST_HEAD(&ops->list); | |
1576 | ||
f72c2db4 | 1577 | args.vm = gpuvm; |
e6303f32 DK |
1578 | args.ops = ops; |
1579 | ||
f72c2db4 | 1580 | ret = __drm_gpuvm_sm_unmap(gpuvm, &gpuvm_list_ops, &args, |
e6303f32 DK |
1581 | req_addr, req_range); |
1582 | if (ret) | |
1583 | goto err_free_ops; | |
1584 | ||
1585 | return ops; | |
1586 | ||
1587 | err_free_ops: | |
f72c2db4 | 1588 | drm_gpuva_ops_free(gpuvm, ops); |
e6303f32 DK |
1589 | return ERR_PTR(ret); |
1590 | } | |
f72c2db4 | 1591 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_ops_create); |
e6303f32 DK |
1592 | |
1593 | /** | |
f72c2db4 DK |
1594 | * drm_gpuvm_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch |
1595 | * @gpuvm: the &drm_gpuvm representing the GPU VA space | |
e6303f32 DK |
1596 | * @addr: the start address of the range to prefetch |
1597 | * @range: the range of the mappings to prefetch | |
1598 | * | |
1599 | * This function creates a list of operations to perform prefetching. | |
1600 | * | |
1601 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed | |
1602 | * in the given order. It can contain prefetch operations. | |
1603 | * | |
1604 | * There can be an arbitrary amount of prefetch operations. | |
1605 | * | |
1606 | * After the caller finished processing the returned &drm_gpuva_ops, they must | |
1607 | * be freed with &drm_gpuva_ops_free. | |
1608 | * | |
1609 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure | |
1610 | */ | |
1611 | struct drm_gpuva_ops * | |
f72c2db4 | 1612 | drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1613 | u64 addr, u64 range) |
1614 | { | |
1615 | struct drm_gpuva_ops *ops; | |
1616 | struct drm_gpuva_op *op; | |
1617 | struct drm_gpuva *va; | |
1618 | u64 end = addr + range; | |
1619 | int ret; | |
1620 | ||
1621 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); | |
1622 | if (!ops) | |
1623 | return ERR_PTR(-ENOMEM); | |
1624 | ||
1625 | INIT_LIST_HEAD(&ops->list); | |
1626 | ||
f72c2db4 DK |
1627 | drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) { |
1628 | op = gpuva_op_alloc(gpuvm); | |
e6303f32 DK |
1629 | if (!op) { |
1630 | ret = -ENOMEM; | |
1631 | goto err_free_ops; | |
1632 | } | |
1633 | ||
1634 | op->op = DRM_GPUVA_OP_PREFETCH; | |
1635 | op->prefetch.va = va; | |
1636 | list_add_tail(&op->entry, &ops->list); | |
1637 | } | |
1638 | ||
1639 | return ops; | |
1640 | ||
1641 | err_free_ops: | |
f72c2db4 | 1642 | drm_gpuva_ops_free(gpuvm, ops); |
e6303f32 DK |
1643 | return ERR_PTR(ret); |
1644 | } | |
f72c2db4 | 1645 | EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create); |
e6303f32 DK |
1646 | |
1647 | /** | |
f72c2db4 DK |
1648 | * drm_gpuvm_gem_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM |
1649 | * @gpuvm: the &drm_gpuvm representing the GPU VA space | |
e6303f32 DK |
1650 | * @obj: the &drm_gem_object to unmap |
1651 | * | |
1652 | * This function creates a list of operations to perform unmapping for every | |
1653 | * GPUVA attached to a GEM. | |
1654 | * | |
1655 | * The list can be iterated with &drm_gpuva_for_each_op and consists out of an | |
1656 | * arbitrary amount of unmap operations. | |
1657 | * | |
1658 | * After the caller finished processing the returned &drm_gpuva_ops, they must | |
1659 | * be freed with &drm_gpuva_ops_free. | |
1660 | * | |
1661 | * It is the callers responsibility to protect the GEMs GPUVA list against | |
1662 | * concurrent access using the GEMs dma_resv lock. | |
1663 | * | |
1664 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure | |
1665 | */ | |
1666 | struct drm_gpuva_ops * | |
f72c2db4 | 1667 | drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1668 | struct drm_gem_object *obj) |
1669 | { | |
1670 | struct drm_gpuva_ops *ops; | |
1671 | struct drm_gpuva_op *op; | |
1672 | struct drm_gpuva *va; | |
1673 | int ret; | |
1674 | ||
1675 | drm_gem_gpuva_assert_lock_held(obj); | |
1676 | ||
1677 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); | |
1678 | if (!ops) | |
1679 | return ERR_PTR(-ENOMEM); | |
1680 | ||
1681 | INIT_LIST_HEAD(&ops->list); | |
1682 | ||
1683 | drm_gem_for_each_gpuva(va, obj) { | |
f72c2db4 | 1684 | op = gpuva_op_alloc(gpuvm); |
e6303f32 DK |
1685 | if (!op) { |
1686 | ret = -ENOMEM; | |
1687 | goto err_free_ops; | |
1688 | } | |
1689 | ||
1690 | op->op = DRM_GPUVA_OP_UNMAP; | |
1691 | op->unmap.va = va; | |
1692 | list_add_tail(&op->entry, &ops->list); | |
1693 | } | |
1694 | ||
1695 | return ops; | |
1696 | ||
1697 | err_free_ops: | |
f72c2db4 | 1698 | drm_gpuva_ops_free(gpuvm, ops); |
e6303f32 DK |
1699 | return ERR_PTR(ret); |
1700 | } | |
f72c2db4 | 1701 | EXPORT_SYMBOL_GPL(drm_gpuvm_gem_unmap_ops_create); |
e6303f32 DK |
1702 | |
1703 | /** | |
1704 | * drm_gpuva_ops_free() - free the given &drm_gpuva_ops | |
f72c2db4 | 1705 | * @gpuvm: the &drm_gpuvm the ops were created for |
e6303f32 DK |
1706 | * @ops: the &drm_gpuva_ops to free |
1707 | * | |
1708 | * Frees the given &drm_gpuva_ops structure including all the ops associated | |
1709 | * with it. | |
1710 | */ | |
1711 | void | |
f72c2db4 | 1712 | drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1713 | struct drm_gpuva_ops *ops) |
1714 | { | |
1715 | struct drm_gpuva_op *op, *next; | |
1716 | ||
1717 | drm_gpuva_for_each_op_safe(op, next, ops) { | |
1718 | list_del(&op->entry); | |
1719 | ||
1720 | if (op->op == DRM_GPUVA_OP_REMAP) { | |
1721 | kfree(op->remap.prev); | |
1722 | kfree(op->remap.next); | |
1723 | kfree(op->remap.unmap); | |
1724 | } | |
1725 | ||
f72c2db4 | 1726 | gpuva_op_free(gpuvm, op); |
e6303f32 DK |
1727 | } |
1728 | ||
1729 | kfree(ops); | |
1730 | } | |
1731 | EXPORT_SYMBOL_GPL(drm_gpuva_ops_free); | |
fe7acaa7 DK |
1732 | |
1733 | MODULE_DESCRIPTION("DRM GPUVM"); | |
1734 | MODULE_LICENSE("GPL"); |