Commit | Line | Data |
---|---|---|
b9c02e10 | 1 | // SPDX-License-Identifier: GPL-2.0-only OR MIT |
e6303f32 DK |
2 | /* |
3 | * Copyright (c) 2022 Red Hat. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the "Software"), | |
7 | * to deal in the Software without restriction, including without limitation | |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
9 | * and/or sell copies of the Software, and to permit persons to whom the | |
10 | * Software is furnished to do so, subject to the following conditions: | |
11 | * | |
12 | * The above copyright notice and this permission notice shall be included in | |
13 | * all copies or substantial portions of the Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
21 | * OTHER DEALINGS IN THE SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Danilo Krummrich <dakr@redhat.com> | |
25 | * | |
26 | */ | |
27 | ||
f72c2db4 | 28 | #include <drm/drm_gpuvm.h> |
e6303f32 DK |
29 | |
30 | #include <linux/interval_tree_generic.h> | |
31 | #include <linux/mm.h> | |
32 | ||
33 | /** | |
34 | * DOC: Overview | |
35 | * | |
f72c2db4 DK |
36 | * The DRM GPU VA Manager, represented by struct drm_gpuvm keeps track of a |
37 | * GPU's virtual address (VA) space and manages the corresponding virtual | |
e6303f32 DK |
38 | * mappings represented by &drm_gpuva objects. It also keeps track of the |
39 | * mapping's backing &drm_gem_object buffers. | |
40 | * | |
41 | * &drm_gem_object buffers maintain a list of &drm_gpuva objects representing | |
42 | * all existent GPU VA mappings using this &drm_gem_object as backing buffer. | |
43 | * | |
44 | * GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also | |
45 | * keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'. | |
46 | * | |
47 | * The GPU VA manager internally uses a rb-tree to manage the | |
48 | * &drm_gpuva mappings within a GPU's virtual address space. | |
49 | * | |
f72c2db4 | 50 | * The &drm_gpuvm structure contains a special &drm_gpuva representing the |
e6303f32 DK |
51 | * portion of VA space reserved by the kernel. This node is initialized together |
52 | * with the GPU VA manager instance and removed when the GPU VA manager is | |
53 | * destroyed. | |
54 | * | |
f72c2db4 | 55 | * In a typical application drivers would embed struct drm_gpuvm and |
e6303f32 DK |
56 | * struct drm_gpuva within their own driver specific structures, there won't be |
57 | * any memory allocations of its own nor memory allocations of &drm_gpuva | |
58 | * entries. | |
59 | * | |
f72c2db4 DK |
60 | * The data structures needed to store &drm_gpuvas within the &drm_gpuvm are |
61 | * contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva | |
62 | * entries from within dma-fence signalling critical sections it is enough to | |
63 | * pre-allocate the &drm_gpuva structures. | |
bbe84580 DK |
64 | * |
65 | * &drm_gem_objects which are private to a single VM can share a common | |
66 | * &dma_resv in order to improve locking efficiency (e.g. with &drm_exec). | |
67 | * For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in | |
68 | * the following called 'resv object', which serves as the container of the | |
69 | * GPUVM's shared &dma_resv. This resv object can be a driver specific | |
70 | * &drm_gem_object, such as the &drm_gem_object containing the root page table, | |
71 | * but it can also be a 'dummy' object, which can be allocated with | |
72 | * drm_gpuvm_resv_object_alloc(). | |
94bc2249 DK |
73 | * |
74 | * In order to connect a struct drm_gpuva its backing &drm_gem_object each | |
75 | * &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each | |
76 | * &drm_gpuvm_bo contains a list of &drm_gpuva structures. | |
77 | * | |
78 | * A &drm_gpuvm_bo is an abstraction that represents a combination of a | |
79 | * &drm_gpuvm and a &drm_gem_object. Every such combination should be unique. | |
80 | * This is ensured by the API through drm_gpuvm_bo_obtain() and | |
81 | * drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding | |
82 | * &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this | |
83 | * particular combination. If not existent a new instance is created and linked | |
84 | * to the &drm_gem_object. | |
50c1a36f DK |
85 | * |
86 | * &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used | |
87 | * as entry for the &drm_gpuvm's lists of external and evicted objects. Those | |
88 | * lists are maintained in order to accelerate locking of dma-resv locks and | |
89 | * validation of evicted objects bound in a &drm_gpuvm. For instance, all | |
90 | * &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling | |
91 | * drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in | |
92 | * order to validate all evicted &drm_gem_objects. It is also possible to lock | |
93 | * additional &drm_gem_objects by providing the corresponding parameters to | |
94 | * drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making | |
95 | * use of helper functions such as drm_gpuvm_prepare_range() or | |
96 | * drm_gpuvm_prepare_objects(). | |
97 | * | |
98 | * Every bound &drm_gem_object is treated as external object when its &dma_resv | |
99 | * structure is different than the &drm_gpuvm's common &dma_resv structure. | |
e6303f32 DK |
100 | */ |
101 | ||
102 | /** | |
103 | * DOC: Split and Merge | |
104 | * | |
105 | * Besides its capability to manage and represent a GPU VA space, the | |
f72c2db4 DK |
106 | * GPU VA manager also provides functions to let the &drm_gpuvm calculate a |
107 | * sequence of operations to satisfy a given map or unmap request. | |
e6303f32 DK |
108 | * |
109 | * Therefore the DRM GPU VA manager provides an algorithm implementing splitting | |
110 | * and merging of existent GPU VA mappings with the ones that are requested to | |
111 | * be mapped or unmapped. This feature is required by the Vulkan API to | |
112 | * implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this | |
113 | * as VM BIND. | |
114 | * | |
f72c2db4 | 115 | * Drivers can call drm_gpuvm_sm_map() to receive a sequence of callbacks |
e6303f32 DK |
116 | * containing map, unmap and remap operations for a given newly requested |
117 | * mapping. The sequence of callbacks represents the set of operations to | |
118 | * execute in order to integrate the new mapping cleanly into the current state | |
119 | * of the GPU VA space. | |
120 | * | |
121 | * Depending on how the new GPU VA mapping intersects with the existent mappings | |
f72c2db4 DK |
122 | * of the GPU VA space the &drm_gpuvm_ops callbacks contain an arbitrary amount |
123 | * of unmap operations, a maximum of two remap operations and a single map | |
124 | * operation. The caller might receive no callback at all if no operation is | |
e6303f32 DK |
125 | * required, e.g. if the requested mapping already exists in the exact same way. |
126 | * | |
127 | * The single map operation represents the original map operation requested by | |
128 | * the caller. | |
129 | * | |
130 | * &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the | |
131 | * &drm_gpuva to unmap is physically contiguous with the original mapping | |
132 | * request. Optionally, if 'keep' is set, drivers may keep the actual page table | |
133 | * entries for this &drm_gpuva, adding the missing page table entries only and | |
f72c2db4 | 134 | * update the &drm_gpuvm's view of things accordingly. |
e6303f32 DK |
135 | * |
136 | * Drivers may do the same optimization, namely delta page table updates, also | |
137 | * for remap operations. This is possible since &drm_gpuva_op_remap consists of | |
138 | * one unmap operation and one or two map operations, such that drivers can | |
139 | * derive the page table update delta accordingly. | |
140 | * | |
141 | * Note that there can't be more than two existent mappings to split up, one at | |
142 | * the beginning and one at the end of the new mapping, hence there is a | |
143 | * maximum of two remap operations. | |
144 | * | |
f72c2db4 DK |
145 | * Analogous to drm_gpuvm_sm_map() drm_gpuvm_sm_unmap() uses &drm_gpuvm_ops to |
146 | * call back into the driver in order to unmap a range of GPU VA space. The | |
e6303f32 DK |
147 | * logic behind this function is way simpler though: For all existent mappings |
148 | * enclosed by the given range unmap operations are created. For mappings which | |
149 | * are only partically located within the given range, remap operations are | |
150 | * created such that those mappings are split up and re-mapped partically. | |
151 | * | |
f72c2db4 DK |
152 | * As an alternative to drm_gpuvm_sm_map() and drm_gpuvm_sm_unmap(), |
153 | * drm_gpuvm_sm_map_ops_create() and drm_gpuvm_sm_unmap_ops_create() can be used | |
e6303f32 DK |
154 | * to directly obtain an instance of struct drm_gpuva_ops containing a list of |
155 | * &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list | |
156 | * contains the &drm_gpuva_ops analogous to the callbacks one would receive when | |
f72c2db4 | 157 | * calling drm_gpuvm_sm_map() or drm_gpuvm_sm_unmap(). While this way requires |
e6303f32 DK |
158 | * more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to |
159 | * iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory | |
160 | * allocations are possible (e.g. to allocate GPU page tables) and once in the | |
161 | * dma-fence signalling critical path. | |
162 | * | |
f72c2db4 DK |
163 | * To update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert() and |
164 | * drm_gpuva_remove() may be used. These functions can safely be used from | |
165 | * &drm_gpuvm_ops callbacks originating from drm_gpuvm_sm_map() or | |
166 | * drm_gpuvm_sm_unmap(). However, it might be more convenient to use the | |
167 | * provided helper functions drm_gpuva_map(), drm_gpuva_remap() and | |
168 | * drm_gpuva_unmap() instead. | |
e6303f32 DK |
169 | * |
170 | * The following diagram depicts the basic relationships of existent GPU VA | |
171 | * mappings, a newly requested mapping and the resulting mappings as implemented | |
f72c2db4 | 172 | * by drm_gpuvm_sm_map() - it doesn't cover any arbitrary combinations of these. |
e6303f32 DK |
173 | * |
174 | * 1) Requested mapping is identical. Replace it, but indicate the backing PTEs | |
175 | * could be kept. | |
176 | * | |
177 | * :: | |
178 | * | |
179 | * 0 a 1 | |
180 | * old: |-----------| (bo_offset=n) | |
181 | * | |
182 | * 0 a 1 | |
183 | * req: |-----------| (bo_offset=n) | |
184 | * | |
185 | * 0 a 1 | |
186 | * new: |-----------| (bo_offset=n) | |
187 | * | |
188 | * | |
189 | * 2) Requested mapping is identical, except for the BO offset, hence replace | |
190 | * the mapping. | |
191 | * | |
192 | * :: | |
193 | * | |
194 | * 0 a 1 | |
195 | * old: |-----------| (bo_offset=n) | |
196 | * | |
197 | * 0 a 1 | |
198 | * req: |-----------| (bo_offset=m) | |
199 | * | |
200 | * 0 a 1 | |
201 | * new: |-----------| (bo_offset=m) | |
202 | * | |
203 | * | |
204 | * 3) Requested mapping is identical, except for the backing BO, hence replace | |
205 | * the mapping. | |
206 | * | |
207 | * :: | |
208 | * | |
209 | * 0 a 1 | |
210 | * old: |-----------| (bo_offset=n) | |
211 | * | |
212 | * 0 b 1 | |
213 | * req: |-----------| (bo_offset=n) | |
214 | * | |
215 | * 0 b 1 | |
216 | * new: |-----------| (bo_offset=n) | |
217 | * | |
218 | * | |
219 | * 4) Existent mapping is a left aligned subset of the requested one, hence | |
220 | * replace the existent one. | |
221 | * | |
222 | * :: | |
223 | * | |
224 | * 0 a 1 | |
225 | * old: |-----| (bo_offset=n) | |
226 | * | |
227 | * 0 a 2 | |
228 | * req: |-----------| (bo_offset=n) | |
229 | * | |
230 | * 0 a 2 | |
231 | * new: |-----------| (bo_offset=n) | |
232 | * | |
233 | * .. note:: | |
234 | * We expect to see the same result for a request with a different BO | |
235 | * and/or non-contiguous BO offset. | |
236 | * | |
237 | * | |
238 | * 5) Requested mapping's range is a left aligned subset of the existent one, | |
239 | * but backed by a different BO. Hence, map the requested mapping and split | |
240 | * the existent one adjusting its BO offset. | |
241 | * | |
242 | * :: | |
243 | * | |
244 | * 0 a 2 | |
245 | * old: |-----------| (bo_offset=n) | |
246 | * | |
247 | * 0 b 1 | |
248 | * req: |-----| (bo_offset=n) | |
249 | * | |
250 | * 0 b 1 a' 2 | |
251 | * new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1) | |
252 | * | |
253 | * .. note:: | |
254 | * We expect to see the same result for a request with a different BO | |
255 | * and/or non-contiguous BO offset. | |
256 | * | |
257 | * | |
258 | * 6) Existent mapping is a superset of the requested mapping. Split it up, but | |
259 | * indicate that the backing PTEs could be kept. | |
260 | * | |
261 | * :: | |
262 | * | |
263 | * 0 a 2 | |
264 | * old: |-----------| (bo_offset=n) | |
265 | * | |
266 | * 0 a 1 | |
267 | * req: |-----| (bo_offset=n) | |
268 | * | |
269 | * 0 a 1 a' 2 | |
270 | * new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1) | |
271 | * | |
272 | * | |
273 | * 7) Requested mapping's range is a right aligned subset of the existent one, | |
274 | * but backed by a different BO. Hence, map the requested mapping and split | |
275 | * the existent one, without adjusting the BO offset. | |
276 | * | |
277 | * :: | |
278 | * | |
279 | * 0 a 2 | |
280 | * old: |-----------| (bo_offset=n) | |
281 | * | |
282 | * 1 b 2 | |
283 | * req: |-----| (bo_offset=m) | |
284 | * | |
285 | * 0 a 1 b 2 | |
286 | * new: |-----|-----| (a.bo_offset=n,b.bo_offset=m) | |
287 | * | |
288 | * | |
289 | * 8) Existent mapping is a superset of the requested mapping. Split it up, but | |
290 | * indicate that the backing PTEs could be kept. | |
291 | * | |
292 | * :: | |
293 | * | |
294 | * 0 a 2 | |
295 | * old: |-----------| (bo_offset=n) | |
296 | * | |
297 | * 1 a 2 | |
298 | * req: |-----| (bo_offset=n+1) | |
299 | * | |
300 | * 0 a' 1 a 2 | |
301 | * new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1) | |
302 | * | |
303 | * | |
304 | * 9) Existent mapping is overlapped at the end by the requested mapping backed | |
305 | * by a different BO. Hence, map the requested mapping and split up the | |
306 | * existent one, without adjusting the BO offset. | |
307 | * | |
308 | * :: | |
309 | * | |
310 | * 0 a 2 | |
311 | * old: |-----------| (bo_offset=n) | |
312 | * | |
313 | * 1 b 3 | |
314 | * req: |-----------| (bo_offset=m) | |
315 | * | |
316 | * 0 a 1 b 3 | |
317 | * new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m) | |
318 | * | |
319 | * | |
320 | * 10) Existent mapping is overlapped by the requested mapping, both having the | |
321 | * same backing BO with a contiguous offset. Indicate the backing PTEs of | |
322 | * the old mapping could be kept. | |
323 | * | |
324 | * :: | |
325 | * | |
326 | * 0 a 2 | |
327 | * old: |-----------| (bo_offset=n) | |
328 | * | |
329 | * 1 a 3 | |
330 | * req: |-----------| (bo_offset=n+1) | |
331 | * | |
332 | * 0 a' 1 a 3 | |
333 | * new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1) | |
334 | * | |
335 | * | |
336 | * 11) Requested mapping's range is a centered subset of the existent one | |
337 | * having a different backing BO. Hence, map the requested mapping and split | |
338 | * up the existent one in two mappings, adjusting the BO offset of the right | |
339 | * one accordingly. | |
340 | * | |
341 | * :: | |
342 | * | |
343 | * 0 a 3 | |
344 | * old: |-----------------| (bo_offset=n) | |
345 | * | |
346 | * 1 b 2 | |
347 | * req: |-----| (bo_offset=m) | |
348 | * | |
349 | * 0 a 1 b 2 a' 3 | |
350 | * new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2) | |
351 | * | |
352 | * | |
353 | * 12) Requested mapping is a contiguous subset of the existent one. Split it | |
354 | * up, but indicate that the backing PTEs could be kept. | |
355 | * | |
356 | * :: | |
357 | * | |
358 | * 0 a 3 | |
359 | * old: |-----------------| (bo_offset=n) | |
360 | * | |
361 | * 1 a 2 | |
362 | * req: |-----| (bo_offset=n+1) | |
363 | * | |
364 | * 0 a' 1 a 2 a'' 3 | |
365 | * old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2) | |
366 | * | |
367 | * | |
368 | * 13) Existent mapping is a right aligned subset of the requested one, hence | |
369 | * replace the existent one. | |
370 | * | |
371 | * :: | |
372 | * | |
373 | * 1 a 2 | |
374 | * old: |-----| (bo_offset=n+1) | |
375 | * | |
376 | * 0 a 2 | |
377 | * req: |-----------| (bo_offset=n) | |
378 | * | |
379 | * 0 a 2 | |
380 | * new: |-----------| (bo_offset=n) | |
381 | * | |
382 | * .. note:: | |
383 | * We expect to see the same result for a request with a different bo | |
384 | * and/or non-contiguous bo_offset. | |
385 | * | |
386 | * | |
387 | * 14) Existent mapping is a centered subset of the requested one, hence | |
388 | * replace the existent one. | |
389 | * | |
390 | * :: | |
391 | * | |
392 | * 1 a 2 | |
393 | * old: |-----| (bo_offset=n+1) | |
394 | * | |
395 | * 0 a 3 | |
396 | * req: |----------------| (bo_offset=n) | |
397 | * | |
398 | * 0 a 3 | |
399 | * new: |----------------| (bo_offset=n) | |
400 | * | |
401 | * .. note:: | |
402 | * We expect to see the same result for a request with a different bo | |
403 | * and/or non-contiguous bo_offset. | |
404 | * | |
405 | * | |
406 | * 15) Existent mappings is overlapped at the beginning by the requested mapping | |
407 | * backed by a different BO. Hence, map the requested mapping and split up | |
408 | * the existent one, adjusting its BO offset accordingly. | |
409 | * | |
410 | * :: | |
411 | * | |
412 | * 1 a 3 | |
413 | * old: |-----------| (bo_offset=n) | |
414 | * | |
415 | * 0 b 2 | |
416 | * req: |-----------| (bo_offset=m) | |
417 | * | |
418 | * 0 b 2 a' 3 | |
419 | * new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2) | |
420 | */ | |
421 | ||
422 | /** | |
423 | * DOC: Locking | |
424 | * | |
94bc2249 DK |
425 | * In terms of managing &drm_gpuva entries DRM GPUVM does not take care of |
426 | * locking itself, it is the drivers responsibility to take care about locking. | |
427 | * Drivers might want to protect the following operations: inserting, removing | |
428 | * and iterating &drm_gpuva objects as well as generating all kinds of | |
429 | * operations, such as split / merge or prefetch. | |
430 | * | |
431 | * DRM GPUVM also does not take care of the locking of the backing | |
432 | * &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by | |
433 | * itself; drivers are responsible to enforce mutual exclusion using either the | |
434 | * GEMs dma_resv lock or alternatively a driver specific external lock. For the | |
435 | * latter see also drm_gem_gpuva_set_lock(). | |
436 | * | |
437 | * However, DRM GPUVM contains lockdep checks to ensure callers of its API hold | |
438 | * the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed | |
439 | * by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also | |
440 | * drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put(). | |
441 | * | |
442 | * The latter is required since on creation and destruction of a &drm_gpuvm_bo | |
443 | * the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list. | |
444 | * Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and | |
445 | * &drm_gem_object must be able to observe previous creations and destructions | |
446 | * of &drm_gpuvm_bos in order to keep instances unique. | |
50c1a36f DK |
447 | * |
448 | * The &drm_gpuvm's lists for keeping track of external and evicted objects are | |
449 | * protected against concurrent insertion / removal and iteration internally. | |
450 | * | |
451 | * However, drivers still need ensure to protect concurrent calls to functions | |
452 | * iterating those lists, namely drm_gpuvm_prepare_objects() and | |
453 | * drm_gpuvm_validate(). | |
454 | * | |
455 | * Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate | |
456 | * that the corresponding &dma_resv locks are held in order to protect the | |
457 | * lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and | |
458 | * the corresponding lockdep checks are enabled. This is an optimization for | |
459 | * drivers which are capable of taking the corresponding &dma_resv locks and | |
460 | * hence do not require internal locking. | |
e6303f32 DK |
461 | */ |
462 | ||
463 | /** | |
464 | * DOC: Examples | |
465 | * | |
466 | * This section gives two examples on how to let the DRM GPUVA Manager generate | |
467 | * &drm_gpuva_op in order to satisfy a given map or unmap request and how to | |
468 | * make use of them. | |
469 | * | |
470 | * The below code is strictly limited to illustrate the generic usage pattern. | |
471 | * To maintain simplicitly, it doesn't make use of any abstractions for common | |
472 | * code, different (asyncronous) stages with fence signalling critical paths, | |
473 | * any other helpers or error handling in terms of freeing memory and dropping | |
474 | * previously taken locks. | |
475 | * | |
476 | * 1) Obtain a list of &drm_gpuva_op to create a new mapping:: | |
477 | * | |
478 | * // Allocates a new &drm_gpuva. | |
479 | * struct drm_gpuva * driver_gpuva_alloc(void); | |
480 | * | |
f72c2db4 | 481 | * // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva |
e6303f32 DK |
482 | * // structure in individual driver structures and lock the dma-resv with |
483 | * // drm_exec or similar helpers. | |
f72c2db4 | 484 | * int driver_mapping_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
485 | * u64 addr, u64 range, |
486 | * struct drm_gem_object *obj, u64 offset) | |
487 | * { | |
488 | * struct drm_gpuva_ops *ops; | |
489 | * struct drm_gpuva_op *op | |
94bc2249 | 490 | * struct drm_gpuvm_bo *vm_bo; |
e6303f32 DK |
491 | * |
492 | * driver_lock_va_space(); | |
f72c2db4 | 493 | * ops = drm_gpuvm_sm_map_ops_create(gpuvm, addr, range, |
e6303f32 DK |
494 | * obj, offset); |
495 | * if (IS_ERR(ops)) | |
496 | * return PTR_ERR(ops); | |
497 | * | |
94bc2249 DK |
498 | * vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj); |
499 | * if (IS_ERR(vm_bo)) | |
500 | * return PTR_ERR(vm_bo); | |
501 | * | |
e6303f32 DK |
502 | * drm_gpuva_for_each_op(op, ops) { |
503 | * struct drm_gpuva *va; | |
504 | * | |
505 | * switch (op->op) { | |
506 | * case DRM_GPUVA_OP_MAP: | |
507 | * va = driver_gpuva_alloc(); | |
508 | * if (!va) | |
509 | * ; // unwind previous VA space updates, | |
510 | * // free memory and unlock | |
511 | * | |
512 | * driver_vm_map(); | |
f72c2db4 | 513 | * drm_gpuva_map(gpuvm, va, &op->map); |
94bc2249 | 514 | * drm_gpuva_link(va, vm_bo); |
e6303f32 DK |
515 | * |
516 | * break; | |
517 | * case DRM_GPUVA_OP_REMAP: { | |
518 | * struct drm_gpuva *prev = NULL, *next = NULL; | |
519 | * | |
520 | * va = op->remap.unmap->va; | |
521 | * | |
522 | * if (op->remap.prev) { | |
523 | * prev = driver_gpuva_alloc(); | |
524 | * if (!prev) | |
525 | * ; // unwind previous VA space | |
526 | * // updates, free memory and | |
527 | * // unlock | |
528 | * } | |
529 | * | |
530 | * if (op->remap.next) { | |
531 | * next = driver_gpuva_alloc(); | |
532 | * if (!next) | |
533 | * ; // unwind previous VA space | |
534 | * // updates, free memory and | |
535 | * // unlock | |
536 | * } | |
537 | * | |
538 | * driver_vm_remap(); | |
539 | * drm_gpuva_remap(prev, next, &op->remap); | |
540 | * | |
e6303f32 | 541 | * if (prev) |
94bc2249 | 542 | * drm_gpuva_link(prev, va->vm_bo); |
e6303f32 | 543 | * if (next) |
94bc2249 DK |
544 | * drm_gpuva_link(next, va->vm_bo); |
545 | * drm_gpuva_unlink(va); | |
e6303f32 DK |
546 | * |
547 | * break; | |
548 | * } | |
549 | * case DRM_GPUVA_OP_UNMAP: | |
550 | * va = op->unmap->va; | |
551 | * | |
552 | * driver_vm_unmap(); | |
553 | * drm_gpuva_unlink(va); | |
554 | * drm_gpuva_unmap(&op->unmap); | |
555 | * | |
556 | * break; | |
557 | * default: | |
558 | * break; | |
559 | * } | |
560 | * } | |
94bc2249 | 561 | * drm_gpuvm_bo_put(vm_bo); |
e6303f32 DK |
562 | * driver_unlock_va_space(); |
563 | * | |
564 | * return 0; | |
565 | * } | |
566 | * | |
567 | * 2) Receive a callback for each &drm_gpuva_op to create a new mapping:: | |
568 | * | |
569 | * struct driver_context { | |
f72c2db4 | 570 | * struct drm_gpuvm *gpuvm; |
94bc2249 | 571 | * struct drm_gpuvm_bo *vm_bo; |
e6303f32 DK |
572 | * struct drm_gpuva *new_va; |
573 | * struct drm_gpuva *prev_va; | |
574 | * struct drm_gpuva *next_va; | |
575 | * }; | |
576 | * | |
f72c2db4 DK |
577 | * // ops to pass to drm_gpuvm_init() |
578 | * static const struct drm_gpuvm_ops driver_gpuvm_ops = { | |
e6303f32 DK |
579 | * .sm_step_map = driver_gpuva_map, |
580 | * .sm_step_remap = driver_gpuva_remap, | |
581 | * .sm_step_unmap = driver_gpuva_unmap, | |
582 | * }; | |
583 | * | |
f72c2db4 | 584 | * // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva |
e6303f32 DK |
585 | * // structure in individual driver structures and lock the dma-resv with |
586 | * // drm_exec or similar helpers. | |
f72c2db4 | 587 | * int driver_mapping_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
588 | * u64 addr, u64 range, |
589 | * struct drm_gem_object *obj, u64 offset) | |
590 | * { | |
591 | * struct driver_context ctx; | |
94bc2249 | 592 | * struct drm_gpuvm_bo *vm_bo; |
e6303f32 DK |
593 | * struct drm_gpuva_ops *ops; |
594 | * struct drm_gpuva_op *op; | |
595 | * int ret = 0; | |
596 | * | |
f72c2db4 | 597 | * ctx.gpuvm = gpuvm; |
e6303f32 DK |
598 | * |
599 | * ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL); | |
600 | * ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL); | |
601 | * ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL); | |
94bc2249 DK |
602 | * ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj); |
603 | * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) { | |
e6303f32 DK |
604 | * ret = -ENOMEM; |
605 | * goto out; | |
606 | * } | |
607 | * | |
94bc2249 DK |
608 | * // Typically protected with a driver specific GEM gpuva lock |
609 | * // used in the fence signaling path for drm_gpuva_link() and | |
610 | * // drm_gpuva_unlink(), hence pre-allocate. | |
611 | * ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo); | |
612 | * | |
e6303f32 | 613 | * driver_lock_va_space(); |
f72c2db4 | 614 | * ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset); |
e6303f32 DK |
615 | * driver_unlock_va_space(); |
616 | * | |
617 | * out: | |
94bc2249 | 618 | * drm_gpuvm_bo_put(ctx.vm_bo); |
e6303f32 DK |
619 | * kfree(ctx.new_va); |
620 | * kfree(ctx.prev_va); | |
621 | * kfree(ctx.next_va); | |
622 | * return ret; | |
623 | * } | |
624 | * | |
625 | * int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx) | |
626 | * { | |
627 | * struct driver_context *ctx = __ctx; | |
628 | * | |
f72c2db4 | 629 | * drm_gpuva_map(ctx->vm, ctx->new_va, &op->map); |
e6303f32 | 630 | * |
94bc2249 | 631 | * drm_gpuva_link(ctx->new_va, ctx->vm_bo); |
e6303f32 DK |
632 | * |
633 | * // prevent the new GPUVA from being freed in | |
634 | * // driver_mapping_create() | |
635 | * ctx->new_va = NULL; | |
636 | * | |
637 | * return 0; | |
638 | * } | |
639 | * | |
640 | * int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx) | |
641 | * { | |
642 | * struct driver_context *ctx = __ctx; | |
94bc2249 | 643 | * struct drm_gpuva *va = op->remap.unmap->va; |
e6303f32 DK |
644 | * |
645 | * drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap); | |
646 | * | |
e6303f32 | 647 | * if (op->remap.prev) { |
94bc2249 | 648 | * drm_gpuva_link(ctx->prev_va, va->vm_bo); |
e6303f32 DK |
649 | * ctx->prev_va = NULL; |
650 | * } | |
651 | * | |
652 | * if (op->remap.next) { | |
94bc2249 | 653 | * drm_gpuva_link(ctx->next_va, va->vm_bo); |
e6303f32 DK |
654 | * ctx->next_va = NULL; |
655 | * } | |
656 | * | |
94bc2249 DK |
657 | * drm_gpuva_unlink(va); |
658 | * kfree(va); | |
659 | * | |
e6303f32 DK |
660 | * return 0; |
661 | * } | |
662 | * | |
663 | * int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx) | |
664 | * { | |
665 | * drm_gpuva_unlink(op->unmap.va); | |
666 | * drm_gpuva_unmap(&op->unmap); | |
667 | * kfree(op->unmap.va); | |
668 | * | |
669 | * return 0; | |
670 | * } | |
671 | */ | |
672 | ||
50c1a36f DK |
673 | /** |
674 | * get_next_vm_bo_from_list() - get the next vm_bo element | |
675 | * @__gpuvm: the &drm_gpuvm | |
676 | * @__list_name: the name of the list we're iterating on | |
677 | * @__local_list: a pointer to the local list used to store already iterated items | |
678 | * @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list() | |
679 | * | |
680 | * This helper is here to provide lockless list iteration. Lockless as in, the | |
681 | * iterator releases the lock immediately after picking the first element from | |
682 | * the list, so list insertion deletion can happen concurrently. | |
683 | * | |
684 | * Elements popped from the original list are kept in a local list, so removal | |
685 | * and is_empty checks can still happen while we're iterating the list. | |
686 | */ | |
687 | #define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo) \ | |
688 | ({ \ | |
689 | struct drm_gpuvm_bo *__vm_bo = NULL; \ | |
690 | \ | |
691 | drm_gpuvm_bo_put(__prev_vm_bo); \ | |
692 | \ | |
693 | spin_lock(&(__gpuvm)->__list_name.lock); \ | |
694 | if (!(__gpuvm)->__list_name.local_list) \ | |
695 | (__gpuvm)->__list_name.local_list = __local_list; \ | |
696 | else \ | |
697 | drm_WARN_ON((__gpuvm)->drm, \ | |
698 | (__gpuvm)->__list_name.local_list != __local_list); \ | |
699 | \ | |
700 | while (!list_empty(&(__gpuvm)->__list_name.list)) { \ | |
701 | __vm_bo = list_first_entry(&(__gpuvm)->__list_name.list, \ | |
702 | struct drm_gpuvm_bo, \ | |
703 | list.entry.__list_name); \ | |
704 | if (kref_get_unless_zero(&__vm_bo->kref)) { \ | |
705 | list_move_tail(&(__vm_bo)->list.entry.__list_name, \ | |
706 | __local_list); \ | |
707 | break; \ | |
708 | } else { \ | |
709 | list_del_init(&(__vm_bo)->list.entry.__list_name); \ | |
710 | __vm_bo = NULL; \ | |
711 | } \ | |
712 | } \ | |
713 | spin_unlock(&(__gpuvm)->__list_name.lock); \ | |
714 | \ | |
715 | __vm_bo; \ | |
716 | }) | |
717 | ||
718 | /** | |
719 | * for_each_vm_bo_in_list() - internal vm_bo list iterator | |
720 | * @__gpuvm: the &drm_gpuvm | |
721 | * @__list_name: the name of the list we're iterating on | |
722 | * @__local_list: a pointer to the local list used to store already iterated items | |
723 | * @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step | |
724 | * | |
725 | * This helper is here to provide lockless list iteration. Lockless as in, the | |
726 | * iterator releases the lock immediately after picking the first element from the | |
727 | * list, hence list insertion and deletion can happen concurrently. | |
728 | * | |
729 | * It is not allowed to re-assign the vm_bo pointer from inside this loop. | |
730 | * | |
731 | * Typical use: | |
732 | * | |
733 | * struct drm_gpuvm_bo *vm_bo; | |
734 | * LIST_HEAD(my_local_list); | |
735 | * | |
736 | * ret = 0; | |
737 | * for_each_vm_bo_in_list(gpuvm, <list_name>, &my_local_list, vm_bo) { | |
738 | * ret = do_something_with_vm_bo(..., vm_bo); | |
739 | * if (ret) | |
740 | * break; | |
741 | * } | |
742 | * // Drop ref in case we break out of the loop. | |
743 | * drm_gpuvm_bo_put(vm_bo); | |
744 | * restore_vm_bo_list(gpuvm, <list_name>, &my_local_list); | |
745 | * | |
746 | * | |
747 | * Only used for internal list iterations, not meant to be exposed to the outside | |
748 | * world. | |
749 | */ | |
750 | #define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo) \ | |
751 | for (__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \ | |
752 | __local_list, NULL); \ | |
753 | __vm_bo; \ | |
754 | __vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \ | |
755 | __local_list, __vm_bo)) | |
756 | ||
757 | static void | |
758 | __restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock, | |
759 | struct list_head *list, struct list_head **local_list) | |
760 | { | |
761 | /* Merge back the two lists, moving local list elements to the | |
762 | * head to preserve previous ordering, in case it matters. | |
763 | */ | |
764 | spin_lock(lock); | |
765 | if (*local_list) { | |
766 | list_splice(*local_list, list); | |
767 | *local_list = NULL; | |
768 | } | |
769 | spin_unlock(lock); | |
770 | } | |
771 | ||
772 | /** | |
773 | * restore_vm_bo_list() - move vm_bo elements back to their original list | |
774 | * @__gpuvm: the &drm_gpuvm | |
775 | * @__list_name: the name of the list we're iterating on | |
776 | * | |
777 | * When we're done iterating a vm_bo list, we should call restore_vm_bo_list() | |
778 | * to restore the original state and let new iterations take place. | |
779 | */ | |
780 | #define restore_vm_bo_list(__gpuvm, __list_name) \ | |
781 | __restore_vm_bo_list((__gpuvm), &(__gpuvm)->__list_name.lock, \ | |
782 | &(__gpuvm)->__list_name.list, \ | |
783 | &(__gpuvm)->__list_name.local_list) | |
784 | ||
785 | static void | |
786 | cond_spin_lock(spinlock_t *lock, bool cond) | |
787 | { | |
788 | if (cond) | |
789 | spin_lock(lock); | |
790 | } | |
791 | ||
792 | static void | |
793 | cond_spin_unlock(spinlock_t *lock, bool cond) | |
794 | { | |
795 | if (cond) | |
796 | spin_unlock(lock); | |
797 | } | |
798 | ||
799 | static void | |
800 | __drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock, | |
801 | struct list_head *entry, struct list_head *list) | |
802 | { | |
803 | cond_spin_lock(lock, !!lock); | |
804 | if (list_empty(entry)) | |
805 | list_add_tail(entry, list); | |
806 | cond_spin_unlock(lock, !!lock); | |
807 | } | |
808 | ||
809 | /** | |
810 | * drm_gpuvm_bo_list_add() - insert a vm_bo into the given list | |
811 | * @__vm_bo: the &drm_gpuvm_bo | |
812 | * @__list_name: the name of the list to insert into | |
813 | * @__lock: whether to lock with the internal spinlock | |
814 | * | |
815 | * Inserts the given @__vm_bo into the list specified by @__list_name. | |
816 | */ | |
817 | #define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock) \ | |
818 | __drm_gpuvm_bo_list_add((__vm_bo)->vm, \ | |
819 | __lock ? &(__vm_bo)->vm->__list_name.lock : \ | |
820 | NULL, \ | |
821 | &(__vm_bo)->list.entry.__list_name, \ | |
822 | &(__vm_bo)->vm->__list_name.list) | |
823 | ||
824 | static void | |
825 | __drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock, | |
826 | struct list_head *entry, bool init) | |
827 | { | |
828 | cond_spin_lock(lock, !!lock); | |
829 | if (init) { | |
830 | if (!list_empty(entry)) | |
831 | list_del_init(entry); | |
832 | } else { | |
833 | list_del(entry); | |
834 | } | |
835 | cond_spin_unlock(lock, !!lock); | |
836 | } | |
837 | ||
838 | /** | |
839 | * drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list | |
840 | * @__vm_bo: the &drm_gpuvm_bo | |
841 | * @__list_name: the name of the list to insert into | |
842 | * @__lock: whether to lock with the internal spinlock | |
843 | * | |
844 | * Removes the given @__vm_bo from the list specified by @__list_name. | |
845 | */ | |
846 | #define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock) \ | |
847 | __drm_gpuvm_bo_list_del((__vm_bo)->vm, \ | |
848 | __lock ? &(__vm_bo)->vm->__list_name.lock : \ | |
849 | NULL, \ | |
850 | &(__vm_bo)->list.entry.__list_name, \ | |
851 | true) | |
852 | ||
853 | /** | |
854 | * drm_gpuvm_bo_list_del() - remove a vm_bo from the given list | |
855 | * @__vm_bo: the &drm_gpuvm_bo | |
856 | * @__list_name: the name of the list to insert into | |
857 | * @__lock: whether to lock with the internal spinlock | |
858 | * | |
859 | * Removes the given @__vm_bo from the list specified by @__list_name. | |
860 | */ | |
861 | #define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock) \ | |
862 | __drm_gpuvm_bo_list_del((__vm_bo)->vm, \ | |
863 | __lock ? &(__vm_bo)->vm->__list_name.lock : \ | |
864 | NULL, \ | |
865 | &(__vm_bo)->list.entry.__list_name, \ | |
866 | false) | |
867 | ||
e6303f32 DK |
868 | #define to_drm_gpuva(__node) container_of((__node), struct drm_gpuva, rb.node) |
869 | ||
870 | #define GPUVA_START(node) ((node)->va.addr) | |
871 | #define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1) | |
872 | ||
873 | /* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain | |
874 | * about this. | |
875 | */ | |
876 | INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last, | |
877 | GPUVA_START, GPUVA_LAST, static __maybe_unused, | |
878 | drm_gpuva_it) | |
879 | ||
f72c2db4 | 880 | static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
881 | struct drm_gpuva *va); |
882 | static void __drm_gpuva_remove(struct drm_gpuva *va); | |
883 | ||
884 | static bool | |
d1adea27 | 885 | drm_gpuvm_check_overflow(u64 addr, u64 range) |
e6303f32 DK |
886 | { |
887 | u64 end; | |
888 | ||
d1adea27 DK |
889 | return check_add_overflow(addr, range, &end); |
890 | } | |
891 | ||
892 | static bool | |
893 | drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range) | |
894 | { | |
895 | return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range), | |
896 | "GPUVA address limited to %zu bytes.\n", sizeof(addr)); | |
e6303f32 DK |
897 | } |
898 | ||
899 | static bool | |
f72c2db4 | 900 | drm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range) |
e6303f32 DK |
901 | { |
902 | u64 end = addr + range; | |
f72c2db4 DK |
903 | u64 mm_start = gpuvm->mm_start; |
904 | u64 mm_end = mm_start + gpuvm->mm_range; | |
e6303f32 DK |
905 | |
906 | return addr >= mm_start && end <= mm_end; | |
907 | } | |
908 | ||
909 | static bool | |
f72c2db4 | 910 | drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range) |
e6303f32 DK |
911 | { |
912 | u64 end = addr + range; | |
f72c2db4 DK |
913 | u64 kstart = gpuvm->kernel_alloc_node.va.addr; |
914 | u64 krange = gpuvm->kernel_alloc_node.va.range; | |
e6303f32 DK |
915 | u64 kend = kstart + krange; |
916 | ||
917 | return krange && addr < kend && kstart < end; | |
918 | } | |
919 | ||
9297cfc9 DK |
920 | /** |
921 | * drm_gpuvm_range_valid() - checks whether the given range is valid for the | |
922 | * given &drm_gpuvm | |
923 | * @gpuvm: the GPUVM to check the range for | |
924 | * @addr: the base address | |
925 | * @range: the range starting from the base address | |
926 | * | |
927 | * Checks whether the range is within the GPUVM's managed boundaries. | |
928 | * | |
929 | * Returns: true for a valid range, false otherwise | |
930 | */ | |
931 | bool | |
f72c2db4 | 932 | drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
933 | u64 addr, u64 range) |
934 | { | |
d1adea27 | 935 | return !drm_gpuvm_check_overflow(addr, range) && |
f72c2db4 DK |
936 | drm_gpuvm_in_mm_range(gpuvm, addr, range) && |
937 | !drm_gpuvm_in_kernel_node(gpuvm, addr, range); | |
e6303f32 | 938 | } |
9297cfc9 | 939 | EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid); |
e6303f32 | 940 | |
bbe84580 DK |
941 | static void |
942 | drm_gpuvm_gem_object_free(struct drm_gem_object *obj) | |
943 | { | |
944 | drm_gem_object_release(obj); | |
945 | kfree(obj); | |
946 | } | |
947 | ||
948 | static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = { | |
949 | .free = drm_gpuvm_gem_object_free, | |
950 | }; | |
951 | ||
952 | /** | |
953 | * drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object | |
954 | * @drm: the drivers &drm_device | |
955 | * | |
956 | * Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in | |
957 | * order to serve as root GEM object providing the &drm_resv shared across | |
958 | * &drm_gem_objects local to a single GPUVM. | |
959 | * | |
960 | * Returns: the &drm_gem_object on success, NULL on failure | |
961 | */ | |
962 | struct drm_gem_object * | |
963 | drm_gpuvm_resv_object_alloc(struct drm_device *drm) | |
964 | { | |
965 | struct drm_gem_object *obj; | |
966 | ||
967 | obj = kzalloc(sizeof(*obj), GFP_KERNEL); | |
968 | if (!obj) | |
969 | return NULL; | |
970 | ||
971 | obj->funcs = &drm_gpuvm_object_funcs; | |
972 | drm_gem_private_object_init(drm, obj, 0); | |
973 | ||
974 | return obj; | |
975 | } | |
976 | EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc); | |
977 | ||
e6303f32 | 978 | /** |
f72c2db4 DK |
979 | * drm_gpuvm_init() - initialize a &drm_gpuvm |
980 | * @gpuvm: pointer to the &drm_gpuvm to initialize | |
e6303f32 | 981 | * @name: the name of the GPU VA space |
809ef191 | 982 | * @flags: the &drm_gpuvm_flags for this GPUVM |
546ca4d3 | 983 | * @drm: the &drm_device this VM resides in |
bbe84580 | 984 | * @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv |
e6303f32 DK |
985 | * @start_offset: the start offset of the GPU VA space |
986 | * @range: the size of the GPU VA space | |
987 | * @reserve_offset: the start of the kernel reserved GPU VA area | |
988 | * @reserve_range: the size of the kernel reserved GPU VA area | |
f72c2db4 | 989 | * @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap |
e6303f32 | 990 | * |
f72c2db4 | 991 | * The &drm_gpuvm must be initialized with this function before use. |
e6303f32 | 992 | * |
f72c2db4 | 993 | * Note that @gpuvm must be cleared to 0 before calling this function. The given |
e6303f32 DK |
994 | * &name is expected to be managed by the surrounding driver structures. |
995 | */ | |
996 | void | |
546ca4d3 | 997 | drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, |
809ef191 | 998 | enum drm_gpuvm_flags flags, |
546ca4d3 | 999 | struct drm_device *drm, |
bbe84580 | 1000 | struct drm_gem_object *r_obj, |
f72c2db4 DK |
1001 | u64 start_offset, u64 range, |
1002 | u64 reserve_offset, u64 reserve_range, | |
1003 | const struct drm_gpuvm_ops *ops) | |
e6303f32 | 1004 | { |
f72c2db4 DK |
1005 | gpuvm->rb.tree = RB_ROOT_CACHED; |
1006 | INIT_LIST_HEAD(&gpuvm->rb.list); | |
e6303f32 | 1007 | |
50c1a36f DK |
1008 | INIT_LIST_HEAD(&gpuvm->extobj.list); |
1009 | spin_lock_init(&gpuvm->extobj.lock); | |
1010 | ||
1011 | INIT_LIST_HEAD(&gpuvm->evict.list); | |
1012 | spin_lock_init(&gpuvm->evict.lock); | |
1013 | ||
8af72338 DK |
1014 | kref_init(&gpuvm->kref); |
1015 | ||
f72c2db4 | 1016 | gpuvm->name = name ? name : "unknown"; |
809ef191 | 1017 | gpuvm->flags = flags; |
f72c2db4 | 1018 | gpuvm->ops = ops; |
546ca4d3 | 1019 | gpuvm->drm = drm; |
bbe84580 DK |
1020 | gpuvm->r_obj = r_obj; |
1021 | ||
1022 | drm_gem_object_get(r_obj); | |
e6303f32 | 1023 | |
d1adea27 | 1024 | drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range); |
546ca4d3 DK |
1025 | gpuvm->mm_start = start_offset; |
1026 | gpuvm->mm_range = range; | |
e6303f32 | 1027 | |
546ca4d3 | 1028 | memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva)); |
e6303f32 | 1029 | if (reserve_range) { |
f72c2db4 DK |
1030 | gpuvm->kernel_alloc_node.va.addr = reserve_offset; |
1031 | gpuvm->kernel_alloc_node.va.range = reserve_range; | |
e6303f32 | 1032 | |
d1adea27 DK |
1033 | if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset, |
1034 | reserve_range))) | |
f72c2db4 | 1035 | __drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node); |
e6303f32 DK |
1036 | } |
1037 | } | |
f72c2db4 | 1038 | EXPORT_SYMBOL_GPL(drm_gpuvm_init); |
e6303f32 | 1039 | |
8af72338 DK |
1040 | static void |
1041 | drm_gpuvm_fini(struct drm_gpuvm *gpuvm) | |
e6303f32 | 1042 | { |
f72c2db4 | 1043 | gpuvm->name = NULL; |
e6303f32 | 1044 | |
f72c2db4 DK |
1045 | if (gpuvm->kernel_alloc_node.va.range) |
1046 | __drm_gpuva_remove(&gpuvm->kernel_alloc_node); | |
e6303f32 | 1047 | |
546ca4d3 DK |
1048 | drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root), |
1049 | "GPUVA tree is not empty, potentially leaking memory.\n"); | |
bbe84580 | 1050 | |
50c1a36f DK |
1051 | drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list), |
1052 | "Extobj list should be empty.\n"); | |
1053 | drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list), | |
1054 | "Evict list should be empty.\n"); | |
1055 | ||
bbe84580 | 1056 | drm_gem_object_put(gpuvm->r_obj); |
e6303f32 | 1057 | } |
8af72338 DK |
1058 | |
1059 | static void | |
1060 | drm_gpuvm_free(struct kref *kref) | |
1061 | { | |
1062 | struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref); | |
1063 | ||
1064 | drm_gpuvm_fini(gpuvm); | |
1065 | ||
1066 | if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free)) | |
1067 | return; | |
1068 | ||
1069 | gpuvm->ops->vm_free(gpuvm); | |
1070 | } | |
1071 | ||
1072 | /** | |
1073 | * drm_gpuvm_put() - drop a struct drm_gpuvm reference | |
1074 | * @gpuvm: the &drm_gpuvm to release the reference of | |
1075 | * | |
1076 | * This releases a reference to @gpuvm. | |
1077 | * | |
1078 | * This function may be called from atomic context. | |
1079 | */ | |
1080 | void | |
1081 | drm_gpuvm_put(struct drm_gpuvm *gpuvm) | |
1082 | { | |
1083 | if (gpuvm) | |
1084 | kref_put(&gpuvm->kref, drm_gpuvm_free); | |
1085 | } | |
1086 | EXPORT_SYMBOL_GPL(drm_gpuvm_put); | |
e6303f32 | 1087 | |
e759f2ca DK |
1088 | static int |
1089 | exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj, | |
1090 | unsigned int num_fences) | |
1091 | { | |
1092 | return num_fences ? drm_exec_prepare_obj(exec, obj, num_fences) : | |
1093 | drm_exec_lock_obj(exec, obj); | |
1094 | } | |
1095 | ||
1096 | /** | |
1097 | * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv | |
1098 | * @gpuvm: the &drm_gpuvm | |
1099 | * @exec: the &drm_exec context | |
1100 | * @num_fences: the amount of &dma_fences to reserve | |
1101 | * | |
1102 | * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object; if | |
1103 | * @num_fences is zero drm_exec_lock_obj() is called instead. | |
1104 | * | |
1105 | * Using this function directly, it is the drivers responsibility to call | |
1106 | * drm_exec_init() and drm_exec_fini() accordingly. | |
1107 | * | |
1108 | * Returns: 0 on success, negative error code on failure. | |
1109 | */ | |
1110 | int | |
1111 | drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, | |
1112 | struct drm_exec *exec, | |
1113 | unsigned int num_fences) | |
1114 | { | |
1115 | return exec_prepare_obj(exec, gpuvm->r_obj, num_fences); | |
1116 | } | |
1117 | EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_vm); | |
1118 | ||
50c1a36f DK |
1119 | static int |
1120 | __drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, | |
1121 | struct drm_exec *exec, | |
1122 | unsigned int num_fences) | |
1123 | { | |
1124 | struct drm_gpuvm_bo *vm_bo; | |
1125 | LIST_HEAD(extobjs); | |
1126 | int ret = 0; | |
1127 | ||
1128 | for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) { | |
e759f2ca | 1129 | ret = exec_prepare_obj(exec, vm_bo->obj, num_fences); |
50c1a36f DK |
1130 | if (ret) |
1131 | break; | |
1132 | } | |
1133 | /* Drop ref in case we break out of the loop. */ | |
1134 | drm_gpuvm_bo_put(vm_bo); | |
1135 | restore_vm_bo_list(gpuvm, extobj); | |
1136 | ||
1137 | return ret; | |
1138 | } | |
1139 | ||
1140 | static int | |
1141 | drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm, | |
1142 | struct drm_exec *exec, | |
1143 | unsigned int num_fences) | |
1144 | { | |
1145 | struct drm_gpuvm_bo *vm_bo; | |
1146 | int ret = 0; | |
1147 | ||
1148 | drm_gpuvm_resv_assert_held(gpuvm); | |
1149 | list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) { | |
e759f2ca | 1150 | ret = exec_prepare_obj(exec, vm_bo->obj, num_fences); |
50c1a36f DK |
1151 | if (ret) |
1152 | break; | |
1153 | ||
1154 | if (vm_bo->evicted) | |
1155 | drm_gpuvm_bo_list_add(vm_bo, evict, false); | |
1156 | } | |
1157 | ||
1158 | return ret; | |
1159 | } | |
1160 | ||
1161 | /** | |
1162 | * drm_gpuvm_prepare_objects() - prepare all assoiciated BOs | |
1163 | * @gpuvm: the &drm_gpuvm | |
1164 | * @exec: the &drm_exec locking context | |
1165 | * @num_fences: the amount of &dma_fences to reserve | |
1166 | * | |
1167 | * Calls drm_exec_prepare_obj() for all &drm_gem_objects the given | |
e759f2ca DK |
1168 | * &drm_gpuvm contains mappings of; if @num_fences is zero drm_exec_lock_obj() |
1169 | * is called instead. | |
50c1a36f DK |
1170 | * |
1171 | * Using this function directly, it is the drivers responsibility to call | |
1172 | * drm_exec_init() and drm_exec_fini() accordingly. | |
1173 | * | |
1174 | * Note: This function is safe against concurrent insertion and removal of | |
1175 | * external objects, however it is not safe against concurrent usage itself. | |
1176 | * | |
1177 | * Drivers need to make sure to protect this case with either an outer VM lock | |
1178 | * or by calling drm_gpuvm_prepare_vm() before this function within the | |
1179 | * drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures | |
1180 | * mutual exclusion. | |
1181 | * | |
1182 | * Returns: 0 on success, negative error code on failure. | |
1183 | */ | |
1184 | int | |
1185 | drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, | |
1186 | struct drm_exec *exec, | |
1187 | unsigned int num_fences) | |
1188 | { | |
1189 | if (drm_gpuvm_resv_protected(gpuvm)) | |
1190 | return drm_gpuvm_prepare_objects_locked(gpuvm, exec, | |
1191 | num_fences); | |
1192 | else | |
1193 | return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences); | |
1194 | } | |
1195 | EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects); | |
1196 | ||
1197 | /** | |
1198 | * drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range | |
1199 | * @gpuvm: the &drm_gpuvm | |
1200 | * @exec: the &drm_exec locking context | |
1201 | * @addr: the start address within the VA space | |
1202 | * @range: the range to iterate within the VA space | |
1203 | * @num_fences: the amount of &dma_fences to reserve | |
1204 | * | |
1205 | * Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr | |
e759f2ca DK |
1206 | * and @addr + @range; if @num_fences is zero drm_exec_lock_obj() is called |
1207 | * instead. | |
50c1a36f DK |
1208 | * |
1209 | * Returns: 0 on success, negative error code on failure. | |
1210 | */ | |
1211 | int | |
1212 | drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec, | |
1213 | u64 addr, u64 range, unsigned int num_fences) | |
1214 | { | |
1215 | struct drm_gpuva *va; | |
1216 | u64 end = addr + range; | |
1217 | int ret; | |
1218 | ||
1219 | drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) { | |
1220 | struct drm_gem_object *obj = va->gem.obj; | |
1221 | ||
e759f2ca | 1222 | ret = exec_prepare_obj(exec, obj, num_fences); |
50c1a36f DK |
1223 | if (ret) |
1224 | return ret; | |
1225 | } | |
1226 | ||
1227 | return 0; | |
1228 | } | |
1229 | EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_range); | |
1230 | ||
1231 | /** | |
1232 | * drm_gpuvm_exec_lock() - lock all dma-resv of all assoiciated BOs | |
1233 | * @vm_exec: the &drm_gpuvm_exec wrapper | |
1234 | * | |
1235 | * Acquires all dma-resv locks of all &drm_gem_objects the given | |
1236 | * &drm_gpuvm contains mappings of. | |
1237 | * | |
1238 | * Addionally, when calling this function with struct drm_gpuvm_exec::extra | |
1239 | * being set the driver receives the given @fn callback to lock additional | |
1240 | * dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers | |
1241 | * would call drm_exec_prepare_obj() from within this callback. | |
1242 | * | |
1243 | * Returns: 0 on success, negative error code on failure. | |
1244 | */ | |
1245 | int | |
1246 | drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec) | |
1247 | { | |
1248 | struct drm_gpuvm *gpuvm = vm_exec->vm; | |
1249 | struct drm_exec *exec = &vm_exec->exec; | |
1250 | unsigned int num_fences = vm_exec->num_fences; | |
1251 | int ret; | |
1252 | ||
05d24935 | 1253 | drm_exec_init(exec, vm_exec->flags, 0); |
50c1a36f DK |
1254 | |
1255 | drm_exec_until_all_locked(exec) { | |
1256 | ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences); | |
1257 | drm_exec_retry_on_contention(exec); | |
1258 | if (ret) | |
1259 | goto err; | |
1260 | ||
1261 | ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences); | |
1262 | drm_exec_retry_on_contention(exec); | |
1263 | if (ret) | |
1264 | goto err; | |
1265 | ||
1266 | if (vm_exec->extra.fn) { | |
1267 | ret = vm_exec->extra.fn(vm_exec); | |
1268 | drm_exec_retry_on_contention(exec); | |
1269 | if (ret) | |
1270 | goto err; | |
1271 | } | |
1272 | } | |
1273 | ||
1274 | return 0; | |
1275 | ||
1276 | err: | |
1277 | drm_exec_fini(exec); | |
1278 | return ret; | |
1279 | } | |
1280 | EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock); | |
1281 | ||
1282 | static int | |
1283 | fn_lock_array(struct drm_gpuvm_exec *vm_exec) | |
1284 | { | |
1285 | struct { | |
1286 | struct drm_gem_object **objs; | |
1287 | unsigned int num_objs; | |
1288 | } *args = vm_exec->extra.priv; | |
1289 | ||
1290 | return drm_exec_prepare_array(&vm_exec->exec, args->objs, | |
1291 | args->num_objs, vm_exec->num_fences); | |
1292 | } | |
1293 | ||
1294 | /** | |
1295 | * drm_gpuvm_exec_lock_array() - lock all dma-resv of all assoiciated BOs | |
1296 | * @vm_exec: the &drm_gpuvm_exec wrapper | |
1297 | * @objs: additional &drm_gem_objects to lock | |
1298 | * @num_objs: the number of additional &drm_gem_objects to lock | |
1299 | * | |
1300 | * Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm | |
1301 | * contains mappings of, plus the ones given through @objs. | |
1302 | * | |
1303 | * Returns: 0 on success, negative error code on failure. | |
1304 | */ | |
1305 | int | |
1306 | drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec, | |
1307 | struct drm_gem_object **objs, | |
1308 | unsigned int num_objs) | |
1309 | { | |
1310 | struct { | |
1311 | struct drm_gem_object **objs; | |
1312 | unsigned int num_objs; | |
1313 | } args; | |
1314 | ||
1315 | args.objs = objs; | |
1316 | args.num_objs = num_objs; | |
1317 | ||
1318 | vm_exec->extra.fn = fn_lock_array; | |
1319 | vm_exec->extra.priv = &args; | |
1320 | ||
1321 | return drm_gpuvm_exec_lock(vm_exec); | |
1322 | } | |
1323 | EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_array); | |
1324 | ||
1325 | /** | |
1326 | * drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range | |
1327 | * @vm_exec: the &drm_gpuvm_exec wrapper | |
1328 | * @addr: the start address within the VA space | |
1329 | * @range: the range to iterate within the VA space | |
1330 | * | |
1331 | * Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and | |
1332 | * @addr + @range. | |
1333 | * | |
1334 | * Returns: 0 on success, negative error code on failure. | |
1335 | */ | |
1336 | int | |
1337 | drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec, | |
1338 | u64 addr, u64 range) | |
1339 | { | |
1340 | struct drm_gpuvm *gpuvm = vm_exec->vm; | |
1341 | struct drm_exec *exec = &vm_exec->exec; | |
1342 | int ret; | |
1343 | ||
05d24935 | 1344 | drm_exec_init(exec, vm_exec->flags, 0); |
50c1a36f DK |
1345 | |
1346 | drm_exec_until_all_locked(exec) { | |
1347 | ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range, | |
1348 | vm_exec->num_fences); | |
1349 | drm_exec_retry_on_contention(exec); | |
1350 | if (ret) | |
1351 | goto err; | |
1352 | } | |
1353 | ||
1354 | return ret; | |
1355 | ||
1356 | err: | |
1357 | drm_exec_fini(exec); | |
1358 | return ret; | |
1359 | } | |
1360 | EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_range); | |
1361 | ||
1362 | static int | |
1363 | __drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec) | |
1364 | { | |
1365 | const struct drm_gpuvm_ops *ops = gpuvm->ops; | |
1366 | struct drm_gpuvm_bo *vm_bo; | |
1367 | LIST_HEAD(evict); | |
1368 | int ret = 0; | |
1369 | ||
1370 | for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) { | |
1371 | ret = ops->vm_bo_validate(vm_bo, exec); | |
1372 | if (ret) | |
1373 | break; | |
1374 | } | |
1375 | /* Drop ref in case we break out of the loop. */ | |
1376 | drm_gpuvm_bo_put(vm_bo); | |
1377 | restore_vm_bo_list(gpuvm, evict); | |
1378 | ||
1379 | return ret; | |
1380 | } | |
1381 | ||
1382 | static int | |
1383 | drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec) | |
1384 | { | |
1385 | const struct drm_gpuvm_ops *ops = gpuvm->ops; | |
1386 | struct drm_gpuvm_bo *vm_bo, *next; | |
1387 | int ret = 0; | |
1388 | ||
1389 | drm_gpuvm_resv_assert_held(gpuvm); | |
1390 | ||
1391 | list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list, | |
1392 | list.entry.evict) { | |
1393 | ret = ops->vm_bo_validate(vm_bo, exec); | |
1394 | if (ret) | |
1395 | break; | |
1396 | ||
1397 | dma_resv_assert_held(vm_bo->obj->resv); | |
1398 | if (!vm_bo->evicted) | |
1399 | drm_gpuvm_bo_list_del_init(vm_bo, evict, false); | |
1400 | } | |
1401 | ||
1402 | return ret; | |
1403 | } | |
1404 | ||
1405 | /** | |
1406 | * drm_gpuvm_validate() - validate all BOs marked as evicted | |
1407 | * @gpuvm: the &drm_gpuvm to validate evicted BOs | |
1408 | * @exec: the &drm_exec instance used for locking the GPUVM | |
1409 | * | |
1410 | * Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer | |
1411 | * objects being mapped in the given &drm_gpuvm. | |
1412 | * | |
1413 | * Returns: 0 on success, negative error code on failure. | |
1414 | */ | |
1415 | int | |
1416 | drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec) | |
1417 | { | |
1418 | const struct drm_gpuvm_ops *ops = gpuvm->ops; | |
1419 | ||
1420 | if (unlikely(!ops || !ops->vm_bo_validate)) | |
1421 | return -EOPNOTSUPP; | |
1422 | ||
1423 | if (drm_gpuvm_resv_protected(gpuvm)) | |
1424 | return drm_gpuvm_validate_locked(gpuvm, exec); | |
1425 | else | |
1426 | return __drm_gpuvm_validate(gpuvm, exec); | |
1427 | } | |
1428 | EXPORT_SYMBOL_GPL(drm_gpuvm_validate); | |
1429 | ||
1430 | /** | |
1431 | * drm_gpuvm_resv_add_fence - add fence to private and all extobj | |
1432 | * dma-resv | |
1433 | * @gpuvm: the &drm_gpuvm to add a fence to | |
1434 | * @exec: the &drm_exec locking context | |
1435 | * @fence: fence to add | |
1436 | * @private_usage: private dma-resv usage | |
1437 | * @extobj_usage: extobj dma-resv usage | |
1438 | */ | |
1439 | void | |
1440 | drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm, | |
1441 | struct drm_exec *exec, | |
1442 | struct dma_fence *fence, | |
1443 | enum dma_resv_usage private_usage, | |
1444 | enum dma_resv_usage extobj_usage) | |
1445 | { | |
1446 | struct drm_gem_object *obj; | |
1447 | unsigned long index; | |
1448 | ||
1449 | drm_exec_for_each_locked_object(exec, index, obj) { | |
1450 | dma_resv_assert_held(obj->resv); | |
1451 | dma_resv_add_fence(obj->resv, fence, | |
1452 | drm_gpuvm_is_extobj(gpuvm, obj) ? | |
1453 | extobj_usage : private_usage); | |
1454 | } | |
1455 | } | |
1456 | EXPORT_SYMBOL_GPL(drm_gpuvm_resv_add_fence); | |
1457 | ||
94bc2249 DK |
1458 | /** |
1459 | * drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo | |
1460 | * @gpuvm: The &drm_gpuvm the @obj is mapped in. | |
1461 | * @obj: The &drm_gem_object being mapped in the @gpuvm. | |
1462 | * | |
1463 | * If provided by the driver, this function uses the &drm_gpuvm_ops | |
1464 | * vm_bo_alloc() callback to allocate. | |
1465 | * | |
1466 | * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure | |
1467 | */ | |
1468 | struct drm_gpuvm_bo * | |
1469 | drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm, | |
1470 | struct drm_gem_object *obj) | |
1471 | { | |
1472 | const struct drm_gpuvm_ops *ops = gpuvm->ops; | |
1473 | struct drm_gpuvm_bo *vm_bo; | |
1474 | ||
1475 | if (ops && ops->vm_bo_alloc) | |
1476 | vm_bo = ops->vm_bo_alloc(); | |
1477 | else | |
1478 | vm_bo = kzalloc(sizeof(*vm_bo), GFP_KERNEL); | |
1479 | ||
1480 | if (unlikely(!vm_bo)) | |
1481 | return NULL; | |
1482 | ||
1483 | vm_bo->vm = drm_gpuvm_get(gpuvm); | |
1484 | vm_bo->obj = obj; | |
1485 | drm_gem_object_get(obj); | |
1486 | ||
1487 | kref_init(&vm_bo->kref); | |
1488 | INIT_LIST_HEAD(&vm_bo->list.gpuva); | |
1489 | INIT_LIST_HEAD(&vm_bo->list.entry.gem); | |
1490 | ||
50c1a36f DK |
1491 | INIT_LIST_HEAD(&vm_bo->list.entry.extobj); |
1492 | INIT_LIST_HEAD(&vm_bo->list.entry.evict); | |
1493 | ||
94bc2249 DK |
1494 | return vm_bo; |
1495 | } | |
1496 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create); | |
1497 | ||
1498 | static void | |
1499 | drm_gpuvm_bo_destroy(struct kref *kref) | |
1500 | { | |
1501 | struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo, | |
1502 | kref); | |
1503 | struct drm_gpuvm *gpuvm = vm_bo->vm; | |
1504 | const struct drm_gpuvm_ops *ops = gpuvm->ops; | |
1505 | struct drm_gem_object *obj = vm_bo->obj; | |
1506 | bool lock = !drm_gpuvm_resv_protected(gpuvm); | |
1507 | ||
1508 | if (!lock) | |
1509 | drm_gpuvm_resv_assert_held(gpuvm); | |
1510 | ||
50c1a36f DK |
1511 | drm_gpuvm_bo_list_del(vm_bo, extobj, lock); |
1512 | drm_gpuvm_bo_list_del(vm_bo, evict, lock); | |
1513 | ||
94bc2249 DK |
1514 | drm_gem_gpuva_assert_lock_held(obj); |
1515 | list_del(&vm_bo->list.entry.gem); | |
1516 | ||
1517 | if (ops && ops->vm_bo_free) | |
1518 | ops->vm_bo_free(vm_bo); | |
1519 | else | |
1520 | kfree(vm_bo); | |
1521 | ||
1522 | drm_gpuvm_put(gpuvm); | |
1523 | drm_gem_object_put(obj); | |
1524 | } | |
1525 | ||
1526 | /** | |
1527 | * drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference | |
1528 | * @vm_bo: the &drm_gpuvm_bo to release the reference of | |
1529 | * | |
1530 | * This releases a reference to @vm_bo. | |
1531 | * | |
1532 | * If the reference count drops to zero, the &gpuvm_bo is destroyed, which | |
1533 | * includes removing it from the GEMs gpuva list. Hence, if a call to this | |
1534 | * function can potentially let the reference count drop to zero the caller must | |
1535 | * hold the dma-resv or driver specific GEM gpuva lock. | |
1536 | * | |
1537 | * This function may only be called from non-atomic context. | |
c50a291d BB |
1538 | * |
1539 | * Returns: true if vm_bo was destroyed, false otherwise. | |
94bc2249 | 1540 | */ |
c50a291d | 1541 | bool |
94bc2249 DK |
1542 | drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo) |
1543 | { | |
1544 | might_sleep(); | |
1545 | ||
1546 | if (vm_bo) | |
c50a291d BB |
1547 | return !!kref_put(&vm_bo->kref, drm_gpuvm_bo_destroy); |
1548 | ||
1549 | return false; | |
94bc2249 DK |
1550 | } |
1551 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put); | |
1552 | ||
1553 | static struct drm_gpuvm_bo * | |
1554 | __drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, | |
1555 | struct drm_gem_object *obj) | |
1556 | { | |
1557 | struct drm_gpuvm_bo *vm_bo; | |
1558 | ||
1559 | drm_gem_gpuva_assert_lock_held(obj); | |
1560 | drm_gem_for_each_gpuvm_bo(vm_bo, obj) | |
1561 | if (vm_bo->vm == gpuvm) | |
1562 | return vm_bo; | |
1563 | ||
1564 | return NULL; | |
1565 | } | |
1566 | ||
1567 | /** | |
1568 | * drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given | |
1569 | * &drm_gpuvm and &drm_gem_object | |
1570 | * @gpuvm: The &drm_gpuvm the @obj is mapped in. | |
1571 | * @obj: The &drm_gem_object being mapped in the @gpuvm. | |
1572 | * | |
1573 | * Find the &drm_gpuvm_bo representing the combination of the given | |
1574 | * &drm_gpuvm and &drm_gem_object. If found, increases the reference | |
1575 | * count of the &drm_gpuvm_bo accordingly. | |
1576 | * | |
1577 | * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure | |
1578 | */ | |
1579 | struct drm_gpuvm_bo * | |
1580 | drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, | |
1581 | struct drm_gem_object *obj) | |
1582 | { | |
1583 | struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj); | |
1584 | ||
1585 | return vm_bo ? drm_gpuvm_bo_get(vm_bo) : NULL; | |
1586 | } | |
1587 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_find); | |
1588 | ||
1589 | /** | |
1590 | * drm_gpuvm_bo_obtain() - obtains and instance of the &drm_gpuvm_bo for the | |
1591 | * given &drm_gpuvm and &drm_gem_object | |
1592 | * @gpuvm: The &drm_gpuvm the @obj is mapped in. | |
1593 | * @obj: The &drm_gem_object being mapped in the @gpuvm. | |
1594 | * | |
1595 | * Find the &drm_gpuvm_bo representing the combination of the given | |
1596 | * &drm_gpuvm and &drm_gem_object. If found, increases the reference | |
1597 | * count of the &drm_gpuvm_bo accordingly. If not found, allocates a new | |
1598 | * &drm_gpuvm_bo. | |
1599 | * | |
1600 | * A new &drm_gpuvm_bo is added to the GEMs gpuva list. | |
1601 | * | |
1602 | * Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure | |
1603 | */ | |
1604 | struct drm_gpuvm_bo * | |
1605 | drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm, | |
1606 | struct drm_gem_object *obj) | |
1607 | { | |
1608 | struct drm_gpuvm_bo *vm_bo; | |
1609 | ||
1610 | vm_bo = drm_gpuvm_bo_find(gpuvm, obj); | |
1611 | if (vm_bo) | |
1612 | return vm_bo; | |
1613 | ||
1614 | vm_bo = drm_gpuvm_bo_create(gpuvm, obj); | |
1615 | if (!vm_bo) | |
1616 | return ERR_PTR(-ENOMEM); | |
1617 | ||
1618 | drm_gem_gpuva_assert_lock_held(obj); | |
1619 | list_add_tail(&vm_bo->list.entry.gem, &obj->gpuva.list); | |
1620 | ||
1621 | return vm_bo; | |
1622 | } | |
1623 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain); | |
1624 | ||
1625 | /** | |
1626 | * drm_gpuvm_bo_obtain_prealloc() - obtains and instance of the &drm_gpuvm_bo | |
1627 | * for the given &drm_gpuvm and &drm_gem_object | |
1628 | * @__vm_bo: A pre-allocated struct drm_gpuvm_bo. | |
1629 | * | |
1630 | * Find the &drm_gpuvm_bo representing the combination of the given | |
1631 | * &drm_gpuvm and &drm_gem_object. If found, increases the reference | |
1632 | * count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference | |
1633 | * count is decreased. If not found @__vm_bo is returned without further | |
1634 | * increase of the reference count. | |
1635 | * | |
1636 | * A new &drm_gpuvm_bo is added to the GEMs gpuva list. | |
1637 | * | |
1638 | * Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing | |
1639 | * &drm_gpuvm_bo was found | |
1640 | */ | |
1641 | struct drm_gpuvm_bo * | |
1642 | drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo) | |
1643 | { | |
1644 | struct drm_gpuvm *gpuvm = __vm_bo->vm; | |
1645 | struct drm_gem_object *obj = __vm_bo->obj; | |
1646 | struct drm_gpuvm_bo *vm_bo; | |
1647 | ||
1648 | vm_bo = drm_gpuvm_bo_find(gpuvm, obj); | |
1649 | if (vm_bo) { | |
1650 | drm_gpuvm_bo_put(__vm_bo); | |
1651 | return vm_bo; | |
1652 | } | |
1653 | ||
1654 | drm_gem_gpuva_assert_lock_held(obj); | |
1655 | list_add_tail(&__vm_bo->list.entry.gem, &obj->gpuva.list); | |
1656 | ||
1657 | return __vm_bo; | |
1658 | } | |
1659 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc); | |
1660 | ||
50c1a36f DK |
1661 | /** |
1662 | * drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's | |
1663 | * extobj list | |
1664 | * @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list. | |
1665 | * | |
1666 | * Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list | |
1667 | * already and if the corresponding &drm_gem_object is an external object, | |
1668 | * actually. | |
1669 | */ | |
1670 | void | |
1671 | drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo) | |
1672 | { | |
1673 | struct drm_gpuvm *gpuvm = vm_bo->vm; | |
1674 | bool lock = !drm_gpuvm_resv_protected(gpuvm); | |
1675 | ||
1676 | if (!lock) | |
1677 | drm_gpuvm_resv_assert_held(gpuvm); | |
1678 | ||
1679 | if (drm_gpuvm_is_extobj(gpuvm, vm_bo->obj)) | |
1680 | drm_gpuvm_bo_list_add(vm_bo, extobj, lock); | |
1681 | } | |
1682 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_extobj_add); | |
1683 | ||
1684 | /** | |
1685 | * drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms | |
1686 | * evicted list | |
1687 | * @vm_bo: the &drm_gpuvm_bo to add or remove | |
1688 | * @evict: indicates whether the object is evicted | |
1689 | * | |
1690 | * Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvms evicted list. | |
1691 | */ | |
1692 | void | |
1693 | drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict) | |
1694 | { | |
1695 | struct drm_gpuvm *gpuvm = vm_bo->vm; | |
1696 | struct drm_gem_object *obj = vm_bo->obj; | |
1697 | bool lock = !drm_gpuvm_resv_protected(gpuvm); | |
1698 | ||
1699 | dma_resv_assert_held(obj->resv); | |
1700 | vm_bo->evicted = evict; | |
1701 | ||
1702 | /* Can't add external objects to the evicted list directly if not using | |
1703 | * internal spinlocks, since in this case the evicted list is protected | |
1704 | * with the VM's common dma-resv lock. | |
1705 | */ | |
1706 | if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock) | |
1707 | return; | |
1708 | ||
1709 | if (evict) | |
1710 | drm_gpuvm_bo_list_add(vm_bo, evict, lock); | |
1711 | else | |
1712 | drm_gpuvm_bo_list_del_init(vm_bo, evict, lock); | |
1713 | } | |
1714 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_evict); | |
1715 | ||
e6303f32 | 1716 | static int |
f72c2db4 | 1717 | __drm_gpuva_insert(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1718 | struct drm_gpuva *va) |
1719 | { | |
1720 | struct rb_node *node; | |
1721 | struct list_head *head; | |
1722 | ||
f72c2db4 | 1723 | if (drm_gpuva_it_iter_first(&gpuvm->rb.tree, |
e6303f32 DK |
1724 | GPUVA_START(va), |
1725 | GPUVA_LAST(va))) | |
1726 | return -EEXIST; | |
1727 | ||
f72c2db4 | 1728 | va->vm = gpuvm; |
e6303f32 | 1729 | |
f72c2db4 | 1730 | drm_gpuva_it_insert(va, &gpuvm->rb.tree); |
e6303f32 DK |
1731 | |
1732 | node = rb_prev(&va->rb.node); | |
1733 | if (node) | |
1734 | head = &(to_drm_gpuva(node))->rb.entry; | |
1735 | else | |
f72c2db4 | 1736 | head = &gpuvm->rb.list; |
e6303f32 DK |
1737 | |
1738 | list_add(&va->rb.entry, head); | |
1739 | ||
1740 | return 0; | |
1741 | } | |
1742 | ||
1743 | /** | |
1744 | * drm_gpuva_insert() - insert a &drm_gpuva | |
f72c2db4 | 1745 | * @gpuvm: the &drm_gpuvm to insert the &drm_gpuva in |
e6303f32 DK |
1746 | * @va: the &drm_gpuva to insert |
1747 | * | |
1748 | * Insert a &drm_gpuva with a given address and range into a | |
f72c2db4 | 1749 | * &drm_gpuvm. |
e6303f32 DK |
1750 | * |
1751 | * It is safe to use this function using the safe versions of iterating the GPU | |
f72c2db4 DK |
1752 | * VA space, such as drm_gpuvm_for_each_va_safe() and |
1753 | * drm_gpuvm_for_each_va_range_safe(). | |
e6303f32 DK |
1754 | * |
1755 | * Returns: 0 on success, negative error code on failure. | |
1756 | */ | |
1757 | int | |
f72c2db4 | 1758 | drm_gpuva_insert(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1759 | struct drm_gpuva *va) |
1760 | { | |
1761 | u64 addr = va->va.addr; | |
1762 | u64 range = va->va.range; | |
8af72338 | 1763 | int ret; |
e6303f32 | 1764 | |
f72c2db4 | 1765 | if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range))) |
e6303f32 DK |
1766 | return -EINVAL; |
1767 | ||
8af72338 DK |
1768 | ret = __drm_gpuva_insert(gpuvm, va); |
1769 | if (likely(!ret)) | |
1770 | /* Take a reference of the GPUVM for the successfully inserted | |
1771 | * drm_gpuva. We can't take the reference in | |
1772 | * __drm_gpuva_insert() itself, since we don't want to increse | |
1773 | * the reference count for the GPUVM's kernel_alloc_node. | |
1774 | */ | |
1775 | drm_gpuvm_get(gpuvm); | |
1776 | ||
1777 | return ret; | |
e6303f32 DK |
1778 | } |
1779 | EXPORT_SYMBOL_GPL(drm_gpuva_insert); | |
1780 | ||
1781 | static void | |
1782 | __drm_gpuva_remove(struct drm_gpuva *va) | |
1783 | { | |
f72c2db4 | 1784 | drm_gpuva_it_remove(va, &va->vm->rb.tree); |
e6303f32 DK |
1785 | list_del_init(&va->rb.entry); |
1786 | } | |
1787 | ||
1788 | /** | |
1789 | * drm_gpuva_remove() - remove a &drm_gpuva | |
1790 | * @va: the &drm_gpuva to remove | |
1791 | * | |
1792 | * This removes the given &va from the underlaying tree. | |
1793 | * | |
1794 | * It is safe to use this function using the safe versions of iterating the GPU | |
f72c2db4 DK |
1795 | * VA space, such as drm_gpuvm_for_each_va_safe() and |
1796 | * drm_gpuvm_for_each_va_range_safe(). | |
e6303f32 DK |
1797 | */ |
1798 | void | |
1799 | drm_gpuva_remove(struct drm_gpuva *va) | |
1800 | { | |
f72c2db4 | 1801 | struct drm_gpuvm *gpuvm = va->vm; |
e6303f32 | 1802 | |
f72c2db4 | 1803 | if (unlikely(va == &gpuvm->kernel_alloc_node)) { |
546ca4d3 DK |
1804 | drm_WARN(gpuvm->drm, 1, |
1805 | "Can't destroy kernel reserved node.\n"); | |
e6303f32 DK |
1806 | return; |
1807 | } | |
1808 | ||
1809 | __drm_gpuva_remove(va); | |
8af72338 | 1810 | drm_gpuvm_put(va->vm); |
e6303f32 DK |
1811 | } |
1812 | EXPORT_SYMBOL_GPL(drm_gpuva_remove); | |
1813 | ||
1814 | /** | |
1815 | * drm_gpuva_link() - link a &drm_gpuva | |
1816 | * @va: the &drm_gpuva to link | |
94bc2249 | 1817 | * @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to |
e6303f32 | 1818 | * |
94bc2249 DK |
1819 | * This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the |
1820 | * &drm_gpuvm_bo to the &drm_gem_object it is associated with. | |
1821 | * | |
1822 | * For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional | |
1823 | * reference of the latter is taken. | |
e6303f32 DK |
1824 | * |
1825 | * This function expects the caller to protect the GEM's GPUVA list against | |
94bc2249 DK |
1826 | * concurrent access using either the GEMs dma_resv lock or a driver specific |
1827 | * lock set through drm_gem_gpuva_set_lock(). | |
e6303f32 DK |
1828 | */ |
1829 | void | |
94bc2249 | 1830 | drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo) |
e6303f32 DK |
1831 | { |
1832 | struct drm_gem_object *obj = va->gem.obj; | |
94bc2249 | 1833 | struct drm_gpuvm *gpuvm = va->vm; |
e6303f32 DK |
1834 | |
1835 | if (unlikely(!obj)) | |
1836 | return; | |
1837 | ||
94bc2249 | 1838 | drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj); |
e6303f32 | 1839 | |
94bc2249 DK |
1840 | va->vm_bo = drm_gpuvm_bo_get(vm_bo); |
1841 | ||
1842 | drm_gem_gpuva_assert_lock_held(obj); | |
1843 | list_add_tail(&va->gem.entry, &vm_bo->list.gpuva); | |
e6303f32 DK |
1844 | } |
1845 | EXPORT_SYMBOL_GPL(drm_gpuva_link); | |
1846 | ||
1847 | /** | |
1848 | * drm_gpuva_unlink() - unlink a &drm_gpuva | |
1849 | * @va: the &drm_gpuva to unlink | |
1850 | * | |
1851 | * This removes the given &va from the GPU VA list of the &drm_gem_object it is | |
1852 | * associated with. | |
1853 | * | |
94bc2249 DK |
1854 | * This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and |
1855 | * the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case | |
1856 | * this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo. | |
1857 | * | |
1858 | * For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of | |
1859 | * the latter is dropped. | |
1860 | * | |
e6303f32 | 1861 | * This function expects the caller to protect the GEM's GPUVA list against |
94bc2249 DK |
1862 | * concurrent access using either the GEMs dma_resv lock or a driver specific |
1863 | * lock set through drm_gem_gpuva_set_lock(). | |
e6303f32 DK |
1864 | */ |
1865 | void | |
1866 | drm_gpuva_unlink(struct drm_gpuva *va) | |
1867 | { | |
1868 | struct drm_gem_object *obj = va->gem.obj; | |
94bc2249 | 1869 | struct drm_gpuvm_bo *vm_bo = va->vm_bo; |
e6303f32 DK |
1870 | |
1871 | if (unlikely(!obj)) | |
1872 | return; | |
1873 | ||
1874 | drm_gem_gpuva_assert_lock_held(obj); | |
e6303f32 | 1875 | list_del_init(&va->gem.entry); |
94bc2249 DK |
1876 | |
1877 | va->vm_bo = NULL; | |
1878 | drm_gpuvm_bo_put(vm_bo); | |
e6303f32 DK |
1879 | } |
1880 | EXPORT_SYMBOL_GPL(drm_gpuva_unlink); | |
1881 | ||
1882 | /** | |
1883 | * drm_gpuva_find_first() - find the first &drm_gpuva in the given range | |
f72c2db4 | 1884 | * @gpuvm: the &drm_gpuvm to search in |
e6303f32 DK |
1885 | * @addr: the &drm_gpuvas address |
1886 | * @range: the &drm_gpuvas range | |
1887 | * | |
1888 | * Returns: the first &drm_gpuva within the given range | |
1889 | */ | |
1890 | struct drm_gpuva * | |
f72c2db4 | 1891 | drm_gpuva_find_first(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1892 | u64 addr, u64 range) |
1893 | { | |
1894 | u64 last = addr + range - 1; | |
1895 | ||
f72c2db4 | 1896 | return drm_gpuva_it_iter_first(&gpuvm->rb.tree, addr, last); |
e6303f32 DK |
1897 | } |
1898 | EXPORT_SYMBOL_GPL(drm_gpuva_find_first); | |
1899 | ||
1900 | /** | |
1901 | * drm_gpuva_find() - find a &drm_gpuva | |
f72c2db4 | 1902 | * @gpuvm: the &drm_gpuvm to search in |
e6303f32 DK |
1903 | * @addr: the &drm_gpuvas address |
1904 | * @range: the &drm_gpuvas range | |
1905 | * | |
1906 | * Returns: the &drm_gpuva at a given &addr and with a given &range | |
1907 | */ | |
1908 | struct drm_gpuva * | |
f72c2db4 | 1909 | drm_gpuva_find(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
1910 | u64 addr, u64 range) |
1911 | { | |
1912 | struct drm_gpuva *va; | |
1913 | ||
f72c2db4 | 1914 | va = drm_gpuva_find_first(gpuvm, addr, range); |
e6303f32 DK |
1915 | if (!va) |
1916 | goto out; | |
1917 | ||
1918 | if (va->va.addr != addr || | |
1919 | va->va.range != range) | |
1920 | goto out; | |
1921 | ||
1922 | return va; | |
1923 | ||
1924 | out: | |
1925 | return NULL; | |
1926 | } | |
1927 | EXPORT_SYMBOL_GPL(drm_gpuva_find); | |
1928 | ||
1929 | /** | |
1930 | * drm_gpuva_find_prev() - find the &drm_gpuva before the given address | |
f72c2db4 | 1931 | * @gpuvm: the &drm_gpuvm to search in |
e6303f32 DK |
1932 | * @start: the given GPU VA's start address |
1933 | * | |
1934 | * Find the adjacent &drm_gpuva before the GPU VA with given &start address. | |
1935 | * | |
1936 | * Note that if there is any free space between the GPU VA mappings no mapping | |
1937 | * is returned. | |
1938 | * | |
1939 | * Returns: a pointer to the found &drm_gpuva or NULL if none was found | |
1940 | */ | |
1941 | struct drm_gpuva * | |
f72c2db4 | 1942 | drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start) |
e6303f32 | 1943 | { |
f72c2db4 | 1944 | if (!drm_gpuvm_range_valid(gpuvm, start - 1, 1)) |
e6303f32 DK |
1945 | return NULL; |
1946 | ||
f72c2db4 | 1947 | return drm_gpuva_it_iter_first(&gpuvm->rb.tree, start - 1, start); |
e6303f32 DK |
1948 | } |
1949 | EXPORT_SYMBOL_GPL(drm_gpuva_find_prev); | |
1950 | ||
1951 | /** | |
1952 | * drm_gpuva_find_next() - find the &drm_gpuva after the given address | |
f72c2db4 | 1953 | * @gpuvm: the &drm_gpuvm to search in |
e6303f32 DK |
1954 | * @end: the given GPU VA's end address |
1955 | * | |
1956 | * Find the adjacent &drm_gpuva after the GPU VA with given &end address. | |
1957 | * | |
1958 | * Note that if there is any free space between the GPU VA mappings no mapping | |
1959 | * is returned. | |
1960 | * | |
1961 | * Returns: a pointer to the found &drm_gpuva or NULL if none was found | |
1962 | */ | |
1963 | struct drm_gpuva * | |
f72c2db4 | 1964 | drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end) |
e6303f32 | 1965 | { |
f72c2db4 | 1966 | if (!drm_gpuvm_range_valid(gpuvm, end, 1)) |
e6303f32 DK |
1967 | return NULL; |
1968 | ||
f72c2db4 | 1969 | return drm_gpuva_it_iter_first(&gpuvm->rb.tree, end, end + 1); |
e6303f32 DK |
1970 | } |
1971 | EXPORT_SYMBOL_GPL(drm_gpuva_find_next); | |
1972 | ||
1973 | /** | |
f72c2db4 | 1974 | * drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space |
e6303f32 | 1975 | * is empty |
f72c2db4 | 1976 | * @gpuvm: the &drm_gpuvm to check the range for |
e6303f32 DK |
1977 | * @addr: the start address of the range |
1978 | * @range: the range of the interval | |
1979 | * | |
1980 | * Returns: true if the interval is empty, false otherwise | |
1981 | */ | |
1982 | bool | |
f72c2db4 | 1983 | drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range) |
e6303f32 | 1984 | { |
f72c2db4 | 1985 | return !drm_gpuva_find_first(gpuvm, addr, range); |
e6303f32 | 1986 | } |
f72c2db4 | 1987 | EXPORT_SYMBOL_GPL(drm_gpuvm_interval_empty); |
e6303f32 DK |
1988 | |
1989 | /** | |
1990 | * drm_gpuva_map() - helper to insert a &drm_gpuva according to a | |
1991 | * &drm_gpuva_op_map | |
f72c2db4 | 1992 | * @gpuvm: the &drm_gpuvm |
e6303f32 DK |
1993 | * @va: the &drm_gpuva to insert |
1994 | * @op: the &drm_gpuva_op_map to initialize @va with | |
1995 | * | |
f72c2db4 | 1996 | * Initializes the @va from the @op and inserts it into the given @gpuvm. |
e6303f32 DK |
1997 | */ |
1998 | void | |
f72c2db4 | 1999 | drm_gpuva_map(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
2000 | struct drm_gpuva *va, |
2001 | struct drm_gpuva_op_map *op) | |
2002 | { | |
2003 | drm_gpuva_init_from_op(va, op); | |
f72c2db4 | 2004 | drm_gpuva_insert(gpuvm, va); |
e6303f32 DK |
2005 | } |
2006 | EXPORT_SYMBOL_GPL(drm_gpuva_map); | |
2007 | ||
2008 | /** | |
2009 | * drm_gpuva_remap() - helper to remap a &drm_gpuva according to a | |
2010 | * &drm_gpuva_op_remap | |
2011 | * @prev: the &drm_gpuva to remap when keeping the start of a mapping | |
2012 | * @next: the &drm_gpuva to remap when keeping the end of a mapping | |
2013 | * @op: the &drm_gpuva_op_remap to initialize @prev and @next with | |
2014 | * | |
2015 | * Removes the currently mapped &drm_gpuva and remaps it using @prev and/or | |
2016 | * @next. | |
2017 | */ | |
2018 | void | |
2019 | drm_gpuva_remap(struct drm_gpuva *prev, | |
2020 | struct drm_gpuva *next, | |
2021 | struct drm_gpuva_op_remap *op) | |
2022 | { | |
94bc2249 DK |
2023 | struct drm_gpuva *va = op->unmap->va; |
2024 | struct drm_gpuvm *gpuvm = va->vm; | |
e6303f32 | 2025 | |
94bc2249 | 2026 | drm_gpuva_remove(va); |
e6303f32 DK |
2027 | |
2028 | if (op->prev) { | |
2029 | drm_gpuva_init_from_op(prev, op->prev); | |
f72c2db4 | 2030 | drm_gpuva_insert(gpuvm, prev); |
e6303f32 DK |
2031 | } |
2032 | ||
2033 | if (op->next) { | |
2034 | drm_gpuva_init_from_op(next, op->next); | |
f72c2db4 | 2035 | drm_gpuva_insert(gpuvm, next); |
e6303f32 DK |
2036 | } |
2037 | } | |
2038 | EXPORT_SYMBOL_GPL(drm_gpuva_remap); | |
2039 | ||
2040 | /** | |
2041 | * drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a | |
2042 | * &drm_gpuva_op_unmap | |
2043 | * @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove | |
2044 | * | |
2045 | * Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap. | |
2046 | */ | |
2047 | void | |
2048 | drm_gpuva_unmap(struct drm_gpuva_op_unmap *op) | |
2049 | { | |
2050 | drm_gpuva_remove(op->va); | |
2051 | } | |
2052 | EXPORT_SYMBOL_GPL(drm_gpuva_unmap); | |
2053 | ||
2054 | static int | |
f72c2db4 | 2055 | op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, |
e6303f32 DK |
2056 | u64 addr, u64 range, |
2057 | struct drm_gem_object *obj, u64 offset) | |
2058 | { | |
2059 | struct drm_gpuva_op op = {}; | |
2060 | ||
2061 | op.op = DRM_GPUVA_OP_MAP; | |
2062 | op.map.va.addr = addr; | |
2063 | op.map.va.range = range; | |
2064 | op.map.gem.obj = obj; | |
2065 | op.map.gem.offset = offset; | |
2066 | ||
2067 | return fn->sm_step_map(&op, priv); | |
2068 | } | |
2069 | ||
2070 | static int | |
f72c2db4 | 2071 | op_remap_cb(const struct drm_gpuvm_ops *fn, void *priv, |
e6303f32 DK |
2072 | struct drm_gpuva_op_map *prev, |
2073 | struct drm_gpuva_op_map *next, | |
2074 | struct drm_gpuva_op_unmap *unmap) | |
2075 | { | |
2076 | struct drm_gpuva_op op = {}; | |
2077 | struct drm_gpuva_op_remap *r; | |
2078 | ||
2079 | op.op = DRM_GPUVA_OP_REMAP; | |
2080 | r = &op.remap; | |
2081 | r->prev = prev; | |
2082 | r->next = next; | |
2083 | r->unmap = unmap; | |
2084 | ||
2085 | return fn->sm_step_remap(&op, priv); | |
2086 | } | |
2087 | ||
2088 | static int | |
f72c2db4 | 2089 | op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv, |
e6303f32 DK |
2090 | struct drm_gpuva *va, bool merge) |
2091 | { | |
2092 | struct drm_gpuva_op op = {}; | |
2093 | ||
2094 | op.op = DRM_GPUVA_OP_UNMAP; | |
2095 | op.unmap.va = va; | |
2096 | op.unmap.keep = merge; | |
2097 | ||
2098 | return fn->sm_step_unmap(&op, priv); | |
2099 | } | |
2100 | ||
2101 | static int | |
f72c2db4 DK |
2102 | __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, |
2103 | const struct drm_gpuvm_ops *ops, void *priv, | |
e6303f32 DK |
2104 | u64 req_addr, u64 req_range, |
2105 | struct drm_gem_object *req_obj, u64 req_offset) | |
2106 | { | |
cdf4100e | 2107 | struct drm_gpuva *va, *next; |
e6303f32 DK |
2108 | u64 req_end = req_addr + req_range; |
2109 | int ret; | |
2110 | ||
f72c2db4 | 2111 | if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) |
e6303f32 DK |
2112 | return -EINVAL; |
2113 | ||
f72c2db4 | 2114 | drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { |
e6303f32 DK |
2115 | struct drm_gem_object *obj = va->gem.obj; |
2116 | u64 offset = va->gem.offset; | |
2117 | u64 addr = va->va.addr; | |
2118 | u64 range = va->va.range; | |
2119 | u64 end = addr + range; | |
2120 | bool merge = !!va->gem.obj; | |
2121 | ||
2122 | if (addr == req_addr) { | |
2123 | merge &= obj == req_obj && | |
2124 | offset == req_offset; | |
2125 | ||
2126 | if (end == req_end) { | |
2127 | ret = op_unmap_cb(ops, priv, va, merge); | |
2128 | if (ret) | |
2129 | return ret; | |
2130 | break; | |
2131 | } | |
2132 | ||
2133 | if (end < req_end) { | |
2134 | ret = op_unmap_cb(ops, priv, va, merge); | |
2135 | if (ret) | |
2136 | return ret; | |
cdf4100e | 2137 | continue; |
e6303f32 DK |
2138 | } |
2139 | ||
2140 | if (end > req_end) { | |
2141 | struct drm_gpuva_op_map n = { | |
2142 | .va.addr = req_end, | |
2143 | .va.range = range - req_range, | |
2144 | .gem.obj = obj, | |
2145 | .gem.offset = offset + req_range, | |
2146 | }; | |
2147 | struct drm_gpuva_op_unmap u = { | |
2148 | .va = va, | |
2149 | .keep = merge, | |
2150 | }; | |
2151 | ||
2152 | ret = op_remap_cb(ops, priv, NULL, &n, &u); | |
2153 | if (ret) | |
2154 | return ret; | |
2155 | break; | |
2156 | } | |
2157 | } else if (addr < req_addr) { | |
2158 | u64 ls_range = req_addr - addr; | |
2159 | struct drm_gpuva_op_map p = { | |
2160 | .va.addr = addr, | |
2161 | .va.range = ls_range, | |
2162 | .gem.obj = obj, | |
2163 | .gem.offset = offset, | |
2164 | }; | |
2165 | struct drm_gpuva_op_unmap u = { .va = va }; | |
2166 | ||
2167 | merge &= obj == req_obj && | |
2168 | offset + ls_range == req_offset; | |
2169 | u.keep = merge; | |
2170 | ||
2171 | if (end == req_end) { | |
2172 | ret = op_remap_cb(ops, priv, &p, NULL, &u); | |
2173 | if (ret) | |
2174 | return ret; | |
2175 | break; | |
2176 | } | |
2177 | ||
2178 | if (end < req_end) { | |
2179 | ret = op_remap_cb(ops, priv, &p, NULL, &u); | |
2180 | if (ret) | |
2181 | return ret; | |
cdf4100e | 2182 | continue; |
e6303f32 DK |
2183 | } |
2184 | ||
2185 | if (end > req_end) { | |
2186 | struct drm_gpuva_op_map n = { | |
2187 | .va.addr = req_end, | |
2188 | .va.range = end - req_end, | |
2189 | .gem.obj = obj, | |
2190 | .gem.offset = offset + ls_range + | |
2191 | req_range, | |
2192 | }; | |
2193 | ||
2194 | ret = op_remap_cb(ops, priv, &p, &n, &u); | |
2195 | if (ret) | |
2196 | return ret; | |
2197 | break; | |
2198 | } | |
2199 | } else if (addr > req_addr) { | |
2200 | merge &= obj == req_obj && | |
2201 | offset == req_offset + | |
2202 | (addr - req_addr); | |
2203 | ||
2204 | if (end == req_end) { | |
2205 | ret = op_unmap_cb(ops, priv, va, merge); | |
2206 | if (ret) | |
2207 | return ret; | |
2208 | break; | |
2209 | } | |
2210 | ||
2211 | if (end < req_end) { | |
2212 | ret = op_unmap_cb(ops, priv, va, merge); | |
2213 | if (ret) | |
2214 | return ret; | |
cdf4100e | 2215 | continue; |
e6303f32 DK |
2216 | } |
2217 | ||
2218 | if (end > req_end) { | |
2219 | struct drm_gpuva_op_map n = { | |
2220 | .va.addr = req_end, | |
2221 | .va.range = end - req_end, | |
2222 | .gem.obj = obj, | |
2223 | .gem.offset = offset + req_end - addr, | |
2224 | }; | |
2225 | struct drm_gpuva_op_unmap u = { | |
2226 | .va = va, | |
2227 | .keep = merge, | |
2228 | }; | |
2229 | ||
2230 | ret = op_remap_cb(ops, priv, NULL, &n, &u); | |
2231 | if (ret) | |
2232 | return ret; | |
2233 | break; | |
2234 | } | |
2235 | } | |
e6303f32 DK |
2236 | } |
2237 | ||
2238 | return op_map_cb(ops, priv, | |
2239 | req_addr, req_range, | |
2240 | req_obj, req_offset); | |
2241 | } | |
2242 | ||
2243 | static int | |
f72c2db4 DK |
2244 | __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, |
2245 | const struct drm_gpuvm_ops *ops, void *priv, | |
e6303f32 DK |
2246 | u64 req_addr, u64 req_range) |
2247 | { | |
2248 | struct drm_gpuva *va, *next; | |
2249 | u64 req_end = req_addr + req_range; | |
2250 | int ret; | |
2251 | ||
f72c2db4 | 2252 | if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) |
e6303f32 DK |
2253 | return -EINVAL; |
2254 | ||
f72c2db4 | 2255 | drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { |
e6303f32 DK |
2256 | struct drm_gpuva_op_map prev = {}, next = {}; |
2257 | bool prev_split = false, next_split = false; | |
2258 | struct drm_gem_object *obj = va->gem.obj; | |
2259 | u64 offset = va->gem.offset; | |
2260 | u64 addr = va->va.addr; | |
2261 | u64 range = va->va.range; | |
2262 | u64 end = addr + range; | |
2263 | ||
2264 | if (addr < req_addr) { | |
2265 | prev.va.addr = addr; | |
2266 | prev.va.range = req_addr - addr; | |
2267 | prev.gem.obj = obj; | |
2268 | prev.gem.offset = offset; | |
2269 | ||
2270 | prev_split = true; | |
2271 | } | |
2272 | ||
2273 | if (end > req_end) { | |
2274 | next.va.addr = req_end; | |
2275 | next.va.range = end - req_end; | |
2276 | next.gem.obj = obj; | |
2277 | next.gem.offset = offset + (req_end - addr); | |
2278 | ||
2279 | next_split = true; | |
2280 | } | |
2281 | ||
2282 | if (prev_split || next_split) { | |
2283 | struct drm_gpuva_op_unmap unmap = { .va = va }; | |
2284 | ||
2285 | ret = op_remap_cb(ops, priv, | |
2286 | prev_split ? &prev : NULL, | |
2287 | next_split ? &next : NULL, | |
2288 | &unmap); | |
2289 | if (ret) | |
2290 | return ret; | |
2291 | } else { | |
2292 | ret = op_unmap_cb(ops, priv, va, false); | |
2293 | if (ret) | |
2294 | return ret; | |
2295 | } | |
2296 | } | |
2297 | ||
2298 | return 0; | |
2299 | } | |
2300 | ||
2301 | /** | |
f72c2db4 DK |
2302 | * drm_gpuvm_sm_map() - creates the &drm_gpuva_op split/merge steps |
2303 | * @gpuvm: the &drm_gpuvm representing the GPU VA space | |
e6303f32 DK |
2304 | * @req_addr: the start address of the new mapping |
2305 | * @req_range: the range of the new mapping | |
2306 | * @req_obj: the &drm_gem_object to map | |
2307 | * @req_offset: the offset within the &drm_gem_object | |
2308 | * @priv: pointer to a driver private data structure | |
2309 | * | |
2310 | * This function iterates the given range of the GPU VA space. It utilizes the | |
f72c2db4 | 2311 | * &drm_gpuvm_ops to call back into the driver providing the split and merge |
e6303f32 DK |
2312 | * steps. |
2313 | * | |
2314 | * Drivers may use these callbacks to update the GPU VA space right away within | |
2315 | * the callback. In case the driver decides to copy and store the operations for | |
f72c2db4 DK |
2316 | * later processing neither this function nor &drm_gpuvm_sm_unmap is allowed to |
2317 | * be called before the &drm_gpuvm's view of the GPU VA space was | |
e6303f32 | 2318 | * updated with the previous set of operations. To update the |
f72c2db4 | 2319 | * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), |
e6303f32 DK |
2320 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be |
2321 | * used. | |
2322 | * | |
2323 | * A sequence of callbacks can contain map, unmap and remap operations, but | |
2324 | * the sequence of callbacks might also be empty if no operation is required, | |
2325 | * e.g. if the requested mapping already exists in the exact same way. | |
2326 | * | |
2327 | * There can be an arbitrary amount of unmap operations, a maximum of two remap | |
2328 | * operations and a single map operation. The latter one represents the original | |
2329 | * map operation requested by the caller. | |
2330 | * | |
2331 | * Returns: 0 on success or a negative error code | |
2332 | */ | |
2333 | int | |
f72c2db4 | 2334 | drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, |
e6303f32 DK |
2335 | u64 req_addr, u64 req_range, |
2336 | struct drm_gem_object *req_obj, u64 req_offset) | |
2337 | { | |
f72c2db4 | 2338 | const struct drm_gpuvm_ops *ops = gpuvm->ops; |
e6303f32 DK |
2339 | |
2340 | if (unlikely(!(ops && ops->sm_step_map && | |
2341 | ops->sm_step_remap && | |
2342 | ops->sm_step_unmap))) | |
2343 | return -EINVAL; | |
2344 | ||
f72c2db4 | 2345 | return __drm_gpuvm_sm_map(gpuvm, ops, priv, |
e6303f32 DK |
2346 | req_addr, req_range, |
2347 | req_obj, req_offset); | |
2348 | } | |
f72c2db4 | 2349 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map); |
e6303f32 DK |
2350 | |
2351 | /** | |
f72c2db4 DK |
2352 | * drm_gpuvm_sm_unmap() - creates the &drm_gpuva_ops to split on unmap |
2353 | * @gpuvm: the &drm_gpuvm representing the GPU VA space | |
e6303f32 DK |
2354 | * @priv: pointer to a driver private data structure |
2355 | * @req_addr: the start address of the range to unmap | |
2356 | * @req_range: the range of the mappings to unmap | |
2357 | * | |
2358 | * This function iterates the given range of the GPU VA space. It utilizes the | |
f72c2db4 | 2359 | * &drm_gpuvm_ops to call back into the driver providing the operations to |
e6303f32 DK |
2360 | * unmap and, if required, split existent mappings. |
2361 | * | |
2362 | * Drivers may use these callbacks to update the GPU VA space right away within | |
2363 | * the callback. In case the driver decides to copy and store the operations for | |
f72c2db4 DK |
2364 | * later processing neither this function nor &drm_gpuvm_sm_map is allowed to be |
2365 | * called before the &drm_gpuvm's view of the GPU VA space was updated | |
2366 | * with the previous set of operations. To update the &drm_gpuvm's view | |
e6303f32 DK |
2367 | * of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or |
2368 | * drm_gpuva_destroy_unlocked() should be used. | |
2369 | * | |
2370 | * A sequence of callbacks can contain unmap and remap operations, depending on | |
2371 | * whether there are actual overlapping mappings to split. | |
2372 | * | |
2373 | * There can be an arbitrary amount of unmap operations and a maximum of two | |
2374 | * remap operations. | |
2375 | * | |
2376 | * Returns: 0 on success or a negative error code | |
2377 | */ | |
2378 | int | |
f72c2db4 | 2379 | drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, |
e6303f32 DK |
2380 | u64 req_addr, u64 req_range) |
2381 | { | |
f72c2db4 | 2382 | const struct drm_gpuvm_ops *ops = gpuvm->ops; |
e6303f32 DK |
2383 | |
2384 | if (unlikely(!(ops && ops->sm_step_remap && | |
2385 | ops->sm_step_unmap))) | |
2386 | return -EINVAL; | |
2387 | ||
f72c2db4 | 2388 | return __drm_gpuvm_sm_unmap(gpuvm, ops, priv, |
e6303f32 DK |
2389 | req_addr, req_range); |
2390 | } | |
f72c2db4 | 2391 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap); |
e6303f32 DK |
2392 | |
2393 | static struct drm_gpuva_op * | |
f72c2db4 | 2394 | gpuva_op_alloc(struct drm_gpuvm *gpuvm) |
e6303f32 | 2395 | { |
f72c2db4 | 2396 | const struct drm_gpuvm_ops *fn = gpuvm->ops; |
e6303f32 DK |
2397 | struct drm_gpuva_op *op; |
2398 | ||
2399 | if (fn && fn->op_alloc) | |
2400 | op = fn->op_alloc(); | |
2401 | else | |
2402 | op = kzalloc(sizeof(*op), GFP_KERNEL); | |
2403 | ||
2404 | if (unlikely(!op)) | |
2405 | return NULL; | |
2406 | ||
2407 | return op; | |
2408 | } | |
2409 | ||
2410 | static void | |
f72c2db4 | 2411 | gpuva_op_free(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
2412 | struct drm_gpuva_op *op) |
2413 | { | |
f72c2db4 | 2414 | const struct drm_gpuvm_ops *fn = gpuvm->ops; |
e6303f32 DK |
2415 | |
2416 | if (fn && fn->op_free) | |
2417 | fn->op_free(op); | |
2418 | else | |
2419 | kfree(op); | |
2420 | } | |
2421 | ||
2422 | static int | |
2423 | drm_gpuva_sm_step(struct drm_gpuva_op *__op, | |
2424 | void *priv) | |
2425 | { | |
2426 | struct { | |
f72c2db4 | 2427 | struct drm_gpuvm *vm; |
e6303f32 DK |
2428 | struct drm_gpuva_ops *ops; |
2429 | } *args = priv; | |
f72c2db4 | 2430 | struct drm_gpuvm *gpuvm = args->vm; |
e6303f32 DK |
2431 | struct drm_gpuva_ops *ops = args->ops; |
2432 | struct drm_gpuva_op *op; | |
2433 | ||
f72c2db4 | 2434 | op = gpuva_op_alloc(gpuvm); |
e6303f32 DK |
2435 | if (unlikely(!op)) |
2436 | goto err; | |
2437 | ||
2438 | memcpy(op, __op, sizeof(*op)); | |
2439 | ||
2440 | if (op->op == DRM_GPUVA_OP_REMAP) { | |
2441 | struct drm_gpuva_op_remap *__r = &__op->remap; | |
2442 | struct drm_gpuva_op_remap *r = &op->remap; | |
2443 | ||
2444 | r->unmap = kmemdup(__r->unmap, sizeof(*r->unmap), | |
2445 | GFP_KERNEL); | |
2446 | if (unlikely(!r->unmap)) | |
2447 | goto err_free_op; | |
2448 | ||
2449 | if (__r->prev) { | |
2450 | r->prev = kmemdup(__r->prev, sizeof(*r->prev), | |
2451 | GFP_KERNEL); | |
2452 | if (unlikely(!r->prev)) | |
2453 | goto err_free_unmap; | |
2454 | } | |
2455 | ||
2456 | if (__r->next) { | |
2457 | r->next = kmemdup(__r->next, sizeof(*r->next), | |
2458 | GFP_KERNEL); | |
2459 | if (unlikely(!r->next)) | |
2460 | goto err_free_prev; | |
2461 | } | |
2462 | } | |
2463 | ||
2464 | list_add_tail(&op->entry, &ops->list); | |
2465 | ||
2466 | return 0; | |
2467 | ||
2468 | err_free_unmap: | |
2469 | kfree(op->remap.unmap); | |
2470 | err_free_prev: | |
2471 | kfree(op->remap.prev); | |
2472 | err_free_op: | |
f72c2db4 | 2473 | gpuva_op_free(gpuvm, op); |
e6303f32 DK |
2474 | err: |
2475 | return -ENOMEM; | |
2476 | } | |
2477 | ||
f72c2db4 | 2478 | static const struct drm_gpuvm_ops gpuvm_list_ops = { |
e6303f32 DK |
2479 | .sm_step_map = drm_gpuva_sm_step, |
2480 | .sm_step_remap = drm_gpuva_sm_step, | |
2481 | .sm_step_unmap = drm_gpuva_sm_step, | |
2482 | }; | |
2483 | ||
2484 | /** | |
f72c2db4 DK |
2485 | * drm_gpuvm_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge |
2486 | * @gpuvm: the &drm_gpuvm representing the GPU VA space | |
e6303f32 DK |
2487 | * @req_addr: the start address of the new mapping |
2488 | * @req_range: the range of the new mapping | |
2489 | * @req_obj: the &drm_gem_object to map | |
2490 | * @req_offset: the offset within the &drm_gem_object | |
2491 | * | |
2492 | * This function creates a list of operations to perform splitting and merging | |
2493 | * of existent mapping(s) with the newly requested one. | |
2494 | * | |
2495 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed | |
2496 | * in the given order. It can contain map, unmap and remap operations, but it | |
2497 | * also can be empty if no operation is required, e.g. if the requested mapping | |
2498 | * already exists is the exact same way. | |
2499 | * | |
2500 | * There can be an arbitrary amount of unmap operations, a maximum of two remap | |
2501 | * operations and a single map operation. The latter one represents the original | |
2502 | * map operation requested by the caller. | |
2503 | * | |
2504 | * Note that before calling this function again with another mapping request it | |
f72c2db4 | 2505 | * is necessary to update the &drm_gpuvm's view of the GPU VA space. The |
e6303f32 | 2506 | * previously obtained operations must be either processed or abandoned. To |
f72c2db4 | 2507 | * update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), |
e6303f32 DK |
2508 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be |
2509 | * used. | |
2510 | * | |
2511 | * After the caller finished processing the returned &drm_gpuva_ops, they must | |
2512 | * be freed with &drm_gpuva_ops_free. | |
2513 | * | |
2514 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure | |
2515 | */ | |
2516 | struct drm_gpuva_ops * | |
f72c2db4 | 2517 | drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
2518 | u64 req_addr, u64 req_range, |
2519 | struct drm_gem_object *req_obj, u64 req_offset) | |
2520 | { | |
2521 | struct drm_gpuva_ops *ops; | |
2522 | struct { | |
f72c2db4 | 2523 | struct drm_gpuvm *vm; |
e6303f32 DK |
2524 | struct drm_gpuva_ops *ops; |
2525 | } args; | |
2526 | int ret; | |
2527 | ||
2528 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); | |
2529 | if (unlikely(!ops)) | |
2530 | return ERR_PTR(-ENOMEM); | |
2531 | ||
2532 | INIT_LIST_HEAD(&ops->list); | |
2533 | ||
f72c2db4 | 2534 | args.vm = gpuvm; |
e6303f32 DK |
2535 | args.ops = ops; |
2536 | ||
f72c2db4 | 2537 | ret = __drm_gpuvm_sm_map(gpuvm, &gpuvm_list_ops, &args, |
e6303f32 DK |
2538 | req_addr, req_range, |
2539 | req_obj, req_offset); | |
2540 | if (ret) | |
2541 | goto err_free_ops; | |
2542 | ||
2543 | return ops; | |
2544 | ||
2545 | err_free_ops: | |
f72c2db4 | 2546 | drm_gpuva_ops_free(gpuvm, ops); |
e6303f32 DK |
2547 | return ERR_PTR(ret); |
2548 | } | |
f72c2db4 | 2549 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_ops_create); |
e6303f32 DK |
2550 | |
2551 | /** | |
f72c2db4 | 2552 | * drm_gpuvm_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on |
e6303f32 | 2553 | * unmap |
f72c2db4 | 2554 | * @gpuvm: the &drm_gpuvm representing the GPU VA space |
e6303f32 DK |
2555 | * @req_addr: the start address of the range to unmap |
2556 | * @req_range: the range of the mappings to unmap | |
2557 | * | |
2558 | * This function creates a list of operations to perform unmapping and, if | |
2559 | * required, splitting of the mappings overlapping the unmap range. | |
2560 | * | |
2561 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed | |
2562 | * in the given order. It can contain unmap and remap operations, depending on | |
2563 | * whether there are actual overlapping mappings to split. | |
2564 | * | |
2565 | * There can be an arbitrary amount of unmap operations and a maximum of two | |
2566 | * remap operations. | |
2567 | * | |
2568 | * Note that before calling this function again with another range to unmap it | |
f72c2db4 | 2569 | * is necessary to update the &drm_gpuvm's view of the GPU VA space. The |
e6303f32 | 2570 | * previously obtained operations must be processed or abandoned. To update the |
f72c2db4 | 2571 | * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), |
e6303f32 DK |
2572 | * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be |
2573 | * used. | |
2574 | * | |
2575 | * After the caller finished processing the returned &drm_gpuva_ops, they must | |
2576 | * be freed with &drm_gpuva_ops_free. | |
2577 | * | |
2578 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure | |
2579 | */ | |
2580 | struct drm_gpuva_ops * | |
f72c2db4 | 2581 | drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
2582 | u64 req_addr, u64 req_range) |
2583 | { | |
2584 | struct drm_gpuva_ops *ops; | |
2585 | struct { | |
f72c2db4 | 2586 | struct drm_gpuvm *vm; |
e6303f32 DK |
2587 | struct drm_gpuva_ops *ops; |
2588 | } args; | |
2589 | int ret; | |
2590 | ||
2591 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); | |
2592 | if (unlikely(!ops)) | |
2593 | return ERR_PTR(-ENOMEM); | |
2594 | ||
2595 | INIT_LIST_HEAD(&ops->list); | |
2596 | ||
f72c2db4 | 2597 | args.vm = gpuvm; |
e6303f32 DK |
2598 | args.ops = ops; |
2599 | ||
f72c2db4 | 2600 | ret = __drm_gpuvm_sm_unmap(gpuvm, &gpuvm_list_ops, &args, |
e6303f32 DK |
2601 | req_addr, req_range); |
2602 | if (ret) | |
2603 | goto err_free_ops; | |
2604 | ||
2605 | return ops; | |
2606 | ||
2607 | err_free_ops: | |
f72c2db4 | 2608 | drm_gpuva_ops_free(gpuvm, ops); |
e6303f32 DK |
2609 | return ERR_PTR(ret); |
2610 | } | |
f72c2db4 | 2611 | EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_ops_create); |
e6303f32 DK |
2612 | |
2613 | /** | |
f72c2db4 DK |
2614 | * drm_gpuvm_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch |
2615 | * @gpuvm: the &drm_gpuvm representing the GPU VA space | |
e6303f32 DK |
2616 | * @addr: the start address of the range to prefetch |
2617 | * @range: the range of the mappings to prefetch | |
2618 | * | |
2619 | * This function creates a list of operations to perform prefetching. | |
2620 | * | |
2621 | * The list can be iterated with &drm_gpuva_for_each_op and must be processed | |
2622 | * in the given order. It can contain prefetch operations. | |
2623 | * | |
2624 | * There can be an arbitrary amount of prefetch operations. | |
2625 | * | |
2626 | * After the caller finished processing the returned &drm_gpuva_ops, they must | |
2627 | * be freed with &drm_gpuva_ops_free. | |
2628 | * | |
2629 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure | |
2630 | */ | |
2631 | struct drm_gpuva_ops * | |
f72c2db4 | 2632 | drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
2633 | u64 addr, u64 range) |
2634 | { | |
2635 | struct drm_gpuva_ops *ops; | |
2636 | struct drm_gpuva_op *op; | |
2637 | struct drm_gpuva *va; | |
2638 | u64 end = addr + range; | |
2639 | int ret; | |
2640 | ||
2641 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); | |
2642 | if (!ops) | |
2643 | return ERR_PTR(-ENOMEM); | |
2644 | ||
2645 | INIT_LIST_HEAD(&ops->list); | |
2646 | ||
f72c2db4 DK |
2647 | drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) { |
2648 | op = gpuva_op_alloc(gpuvm); | |
e6303f32 DK |
2649 | if (!op) { |
2650 | ret = -ENOMEM; | |
2651 | goto err_free_ops; | |
2652 | } | |
2653 | ||
2654 | op->op = DRM_GPUVA_OP_PREFETCH; | |
2655 | op->prefetch.va = va; | |
2656 | list_add_tail(&op->entry, &ops->list); | |
2657 | } | |
2658 | ||
2659 | return ops; | |
2660 | ||
2661 | err_free_ops: | |
f72c2db4 | 2662 | drm_gpuva_ops_free(gpuvm, ops); |
e6303f32 DK |
2663 | return ERR_PTR(ret); |
2664 | } | |
f72c2db4 | 2665 | EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create); |
e6303f32 DK |
2666 | |
2667 | /** | |
94bc2249 DK |
2668 | * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM |
2669 | * @vm_bo: the &drm_gpuvm_bo abstraction | |
e6303f32 DK |
2670 | * |
2671 | * This function creates a list of operations to perform unmapping for every | |
2672 | * GPUVA attached to a GEM. | |
2673 | * | |
2674 | * The list can be iterated with &drm_gpuva_for_each_op and consists out of an | |
2675 | * arbitrary amount of unmap operations. | |
2676 | * | |
2677 | * After the caller finished processing the returned &drm_gpuva_ops, they must | |
2678 | * be freed with &drm_gpuva_ops_free. | |
2679 | * | |
2680 | * It is the callers responsibility to protect the GEMs GPUVA list against | |
2681 | * concurrent access using the GEMs dma_resv lock. | |
2682 | * | |
2683 | * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure | |
2684 | */ | |
2685 | struct drm_gpuva_ops * | |
94bc2249 | 2686 | drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo) |
e6303f32 DK |
2687 | { |
2688 | struct drm_gpuva_ops *ops; | |
2689 | struct drm_gpuva_op *op; | |
2690 | struct drm_gpuva *va; | |
2691 | int ret; | |
2692 | ||
94bc2249 | 2693 | drm_gem_gpuva_assert_lock_held(vm_bo->obj); |
e6303f32 DK |
2694 | |
2695 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); | |
2696 | if (!ops) | |
2697 | return ERR_PTR(-ENOMEM); | |
2698 | ||
2699 | INIT_LIST_HEAD(&ops->list); | |
2700 | ||
94bc2249 DK |
2701 | drm_gpuvm_bo_for_each_va(va, vm_bo) { |
2702 | op = gpuva_op_alloc(vm_bo->vm); | |
e6303f32 DK |
2703 | if (!op) { |
2704 | ret = -ENOMEM; | |
2705 | goto err_free_ops; | |
2706 | } | |
2707 | ||
2708 | op->op = DRM_GPUVA_OP_UNMAP; | |
2709 | op->unmap.va = va; | |
2710 | list_add_tail(&op->entry, &ops->list); | |
2711 | } | |
2712 | ||
2713 | return ops; | |
2714 | ||
2715 | err_free_ops: | |
94bc2249 | 2716 | drm_gpuva_ops_free(vm_bo->vm, ops); |
e6303f32 DK |
2717 | return ERR_PTR(ret); |
2718 | } | |
94bc2249 | 2719 | EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap_ops_create); |
e6303f32 DK |
2720 | |
2721 | /** | |
2722 | * drm_gpuva_ops_free() - free the given &drm_gpuva_ops | |
f72c2db4 | 2723 | * @gpuvm: the &drm_gpuvm the ops were created for |
e6303f32 DK |
2724 | * @ops: the &drm_gpuva_ops to free |
2725 | * | |
2726 | * Frees the given &drm_gpuva_ops structure including all the ops associated | |
2727 | * with it. | |
2728 | */ | |
2729 | void | |
f72c2db4 | 2730 | drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, |
e6303f32 DK |
2731 | struct drm_gpuva_ops *ops) |
2732 | { | |
2733 | struct drm_gpuva_op *op, *next; | |
2734 | ||
2735 | drm_gpuva_for_each_op_safe(op, next, ops) { | |
2736 | list_del(&op->entry); | |
2737 | ||
2738 | if (op->op == DRM_GPUVA_OP_REMAP) { | |
2739 | kfree(op->remap.prev); | |
2740 | kfree(op->remap.next); | |
2741 | kfree(op->remap.unmap); | |
2742 | } | |
2743 | ||
f72c2db4 | 2744 | gpuva_op_free(gpuvm, op); |
e6303f32 DK |
2745 | } |
2746 | ||
2747 | kfree(ops); | |
2748 | } | |
2749 | EXPORT_SYMBOL_GPL(drm_gpuva_ops_free); | |
fe7acaa7 DK |
2750 | |
2751 | MODULE_DESCRIPTION("DRM GPUVM"); | |
2752 | MODULE_LICENSE("GPL"); |