Commit | Line | Data |
---|---|---|
068d9d75 | 1 | // SPDX-License-Identifier: MIT |
786d7257 | 2 | /* |
04a5faa8 | 3 | * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst) |
786d7257 ML |
4 | * |
5 | * Based on bo.c which bears the following copyright notice, | |
6 | * but is dual licensed: | |
7 | * | |
8 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA | |
9 | * All Rights Reserved. | |
10 | * | |
11 | * Permission is hereby granted, free of charge, to any person obtaining a | |
12 | * copy of this software and associated documentation files (the | |
13 | * "Software"), to deal in the Software without restriction, including | |
14 | * without limitation the rights to use, copy, modify, merge, publish, | |
15 | * distribute, sub license, and/or sell copies of the Software, and to | |
16 | * permit persons to whom the Software is furnished to do so, subject to | |
17 | * the following conditions: | |
18 | * | |
19 | * The above copyright notice and this permission notice (including the | |
20 | * next paragraph) shall be included in all copies or substantial portions | |
21 | * of the Software. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
24 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
25 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
26 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
27 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
28 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
29 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
30 | * | |
31 | **************************************************************************/ | |
32 | /* | |
33 | * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> | |
34 | */ | |
35 | ||
52791eee | 36 | #include <linux/dma-resv.h> |
92cedee6 | 37 | #include <linux/dma-fence-array.h> |
786d7257 | 38 | #include <linux/export.h> |
0adf65f5 | 39 | #include <linux/mm.h> |
b2a8116e | 40 | #include <linux/sched/mm.h> |
d0b9a9ae | 41 | #include <linux/mmu_notifier.h> |
a25efb38 | 42 | #include <linux/seq_file.h> |
786d7257 | 43 | |
dad6c394 RC |
44 | /** |
45 | * DOC: Reservation Object Overview | |
46 | * | |
047a1b87 CK |
47 | * The reservation object provides a mechanism to manage a container of |
48 | * dma_fence object associated with a resource. A reservation object | |
49 | * can have any number of fences attaches to it. Each fence carries an usage | |
50 | * parameter determining how the operation represented by the fence is using the | |
51 | * resource. The RCU mechanism is used to protect read access to fences from | |
52 | * locked write-side updates. | |
d9edf92d DV |
53 | * |
54 | * See struct dma_resv for more details. | |
dad6c394 RC |
55 | */ |
56 | ||
08295b3b | 57 | DEFINE_WD_CLASS(reservation_ww_class); |
786d7257 | 58 | EXPORT_SYMBOL(reservation_ww_class); |
04a5faa8 | 59 | |
047a1b87 CK |
60 | /* Mask for the lower fence pointer bits */ |
61 | #define DMA_RESV_LIST_MASK 0x3 | |
62 | ||
8938d484 CK |
63 | struct dma_resv_list { |
64 | struct rcu_head rcu; | |
047a1b87 CK |
65 | u32 num_fences, max_fences; |
66 | struct dma_fence __rcu *table[]; | |
8938d484 CK |
67 | }; |
68 | ||
047a1b87 CK |
69 | /* Extract the fence and usage flags from an RCU protected entry in the list. */ |
70 | static void dma_resv_list_entry(struct dma_resv_list *list, unsigned int index, | |
71 | struct dma_resv *resv, struct dma_fence **fence, | |
72 | enum dma_resv_usage *usage) | |
73 | { | |
74 | long tmp; | |
75 | ||
76 | tmp = (long)rcu_dereference_check(list->table[index], | |
77 | resv ? dma_resv_held(resv) : true); | |
78 | *fence = (struct dma_fence *)(tmp & ~DMA_RESV_LIST_MASK); | |
79 | if (usage) | |
80 | *usage = tmp & DMA_RESV_LIST_MASK; | |
81 | } | |
82 | ||
83 | /* Set the fence and usage flags at the specific index in the list. */ | |
84 | static void dma_resv_list_set(struct dma_resv_list *list, | |
85 | unsigned int index, | |
86 | struct dma_fence *fence, | |
87 | enum dma_resv_usage usage) | |
88 | { | |
89 | long tmp = ((long)fence) | usage; | |
90 | ||
91 | RCU_INIT_POINTER(list->table[index], (struct dma_fence *)tmp); | |
92 | } | |
93 | ||
94 | /* | |
52791eee | 95 | * Allocate a new dma_resv_list and make sure to correctly initialize |
047a1b87 | 96 | * max_fences. |
96e95496 | 97 | */ |
047a1b87 | 98 | static struct dma_resv_list *dma_resv_list_alloc(unsigned int max_fences) |
96e95496 | 99 | { |
52791eee | 100 | struct dma_resv_list *list; |
cd536db0 | 101 | size_t size; |
96e95496 | 102 | |
cd536db0 KC |
103 | /* Round up to the next kmalloc bucket size. */ |
104 | size = kmalloc_size_roundup(struct_size(list, table, max_fences)); | |
105 | ||
106 | list = kmalloc(size, GFP_KERNEL); | |
96e95496 CK |
107 | if (!list) |
108 | return NULL; | |
109 | ||
cd536db0 KC |
110 | /* Given the resulting bucket size, recalculated max_fences. */ |
111 | list->max_fences = (size - offsetof(typeof(*list), table)) / | |
047a1b87 | 112 | sizeof(*list->table); |
96e95496 CK |
113 | |
114 | return list; | |
115 | } | |
116 | ||
047a1b87 | 117 | /* Free a dma_resv_list and make sure to drop all references. */ |
52791eee | 118 | static void dma_resv_list_free(struct dma_resv_list *list) |
96e95496 CK |
119 | { |
120 | unsigned int i; | |
121 | ||
122 | if (!list) | |
123 | return; | |
124 | ||
047a1b87 CK |
125 | for (i = 0; i < list->num_fences; ++i) { |
126 | struct dma_fence *fence; | |
96e95496 | 127 | |
047a1b87 CK |
128 | dma_resv_list_entry(list, i, NULL, &fence, NULL); |
129 | dma_fence_put(fence); | |
130 | } | |
96e95496 CK |
131 | kfree_rcu(list, rcu); |
132 | } | |
133 | ||
8735f168 | 134 | /** |
52791eee | 135 | * dma_resv_init - initialize a reservation object |
8735f168 CK |
136 | * @obj: the reservation object |
137 | */ | |
52791eee | 138 | void dma_resv_init(struct dma_resv *obj) |
8735f168 CK |
139 | { |
140 | ww_mutex_init(&obj->lock, &reservation_ww_class); | |
b016cd6e | 141 | |
047a1b87 | 142 | RCU_INIT_POINTER(obj->fences, NULL); |
8735f168 | 143 | } |
52791eee | 144 | EXPORT_SYMBOL(dma_resv_init); |
8735f168 CK |
145 | |
146 | /** | |
52791eee | 147 | * dma_resv_fini - destroys a reservation object |
8735f168 CK |
148 | * @obj: the reservation object |
149 | */ | |
52791eee | 150 | void dma_resv_fini(struct dma_resv *obj) |
8735f168 | 151 | { |
8735f168 CK |
152 | /* |
153 | * This object should be dead and all references must have | |
154 | * been released to it, so no need to be protected with rcu. | |
155 | */ | |
047a1b87 | 156 | dma_resv_list_free(rcu_dereference_protected(obj->fences, true)); |
8735f168 CK |
157 | ww_mutex_destroy(&obj->lock); |
158 | } | |
52791eee | 159 | EXPORT_SYMBOL(dma_resv_fini); |
8735f168 | 160 | |
047a1b87 CK |
161 | /* Dereference the fences while ensuring RCU rules */ |
162 | static inline struct dma_resv_list *dma_resv_fences_list(struct dma_resv *obj) | |
8938d484 | 163 | { |
047a1b87 | 164 | return rcu_dereference_check(obj->fences, dma_resv_held(obj)); |
8938d484 CK |
165 | } |
166 | ||
dad6c394 | 167 | /** |
047a1b87 | 168 | * dma_resv_reserve_fences - Reserve space to add fences to a dma_resv object. |
dad6c394 | 169 | * @obj: reservation object |
ca05359f | 170 | * @num_fences: number of fences we want to add |
dad6c394 | 171 | * |
047a1b87 CK |
172 | * Should be called before dma_resv_add_fence(). Must be called with @obj |
173 | * locked through dma_resv_lock(). | |
d9edf92d DV |
174 | * |
175 | * Note that the preallocated slots need to be re-reserved if @obj is unlocked | |
047a1b87 CK |
176 | * at any time before calling dma_resv_add_fence(). This is validated when |
177 | * CONFIG_DEBUG_MUTEXES is enabled. | |
dad6c394 RC |
178 | * |
179 | * RETURNS | |
180 | * Zero for success, or -errno | |
04a5faa8 | 181 | */ |
c8d4c18b | 182 | int dma_resv_reserve_fences(struct dma_resv *obj, unsigned int num_fences) |
04a5faa8 | 183 | { |
52791eee | 184 | struct dma_resv_list *old, *new; |
27836b64 | 185 | unsigned int i, j, k, max; |
04a5faa8 | 186 | |
52791eee | 187 | dma_resv_assert_held(obj); |
547c7138 | 188 | |
047a1b87 CK |
189 | old = dma_resv_fences_list(obj); |
190 | if (old && old->max_fences) { | |
191 | if ((old->num_fences + num_fences) <= old->max_fences) | |
04a5faa8 | 192 | return 0; |
047a1b87 | 193 | max = max(old->num_fences + num_fences, old->max_fences * 2); |
ca25fe5e | 194 | } else { |
bf897583 | 195 | max = max(4ul, roundup_pow_of_two(num_fences)); |
ca25fe5e | 196 | } |
3c3b177a | 197 | |
52791eee | 198 | new = dma_resv_list_alloc(max); |
27836b64 CK |
199 | if (!new) |
200 | return -ENOMEM; | |
04a5faa8 ML |
201 | |
202 | /* | |
203 | * no need to bump fence refcounts, rcu_read access | |
204 | * requires the use of kref_get_unless_zero, and the | |
205 | * references from the old struct are carried over to | |
206 | * the new. | |
207 | */ | |
047a1b87 CK |
208 | for (i = 0, j = 0, k = max; i < (old ? old->num_fences : 0); ++i) { |
209 | enum dma_resv_usage usage; | |
27836b64 | 210 | struct dma_fence *fence; |
3c3b177a | 211 | |
047a1b87 | 212 | dma_resv_list_entry(old, i, obj, &fence, &usage); |
27836b64 | 213 | if (dma_fence_is_signaled(fence)) |
047a1b87 | 214 | RCU_INIT_POINTER(new->table[--k], fence); |
4d9c62e8 | 215 | else |
047a1b87 | 216 | dma_resv_list_set(new, j++, fence, usage); |
04a5faa8 | 217 | } |
047a1b87 | 218 | new->num_fences = j; |
04a5faa8 | 219 | |
3c3b177a | 220 | /* |
30fe7b07 CW |
221 | * We are not changing the effective set of fences here so can |
222 | * merely update the pointer to the new array; both existing | |
223 | * readers and new readers will see exactly the same set of | |
047a1b87 | 224 | * active (unsignaled) fences. Individual fences and the |
30fe7b07 CW |
225 | * old array are protected by RCU and so will not vanish under |
226 | * the gaze of the rcu_read_lock() readers. | |
3c3b177a | 227 | */ |
047a1b87 | 228 | rcu_assign_pointer(obj->fences, new); |
3c3b177a | 229 | |
4d9c62e8 | 230 | if (!old) |
27836b64 | 231 | return 0; |
3c3b177a | 232 | |
4d9c62e8 | 233 | /* Drop the references to the signaled fences */ |
94eb1e10 | 234 | for (i = k; i < max; ++i) { |
27836b64 | 235 | struct dma_fence *fence; |
4d9c62e8 | 236 | |
047a1b87 | 237 | fence = rcu_dereference_protected(new->table[i], |
52791eee | 238 | dma_resv_held(obj)); |
27836b64 | 239 | dma_fence_put(fence); |
4d9c62e8 CK |
240 | } |
241 | kfree_rcu(old, rcu); | |
27836b64 CK |
242 | |
243 | return 0; | |
04a5faa8 | 244 | } |
c8d4c18b | 245 | EXPORT_SYMBOL(dma_resv_reserve_fences); |
04a5faa8 | 246 | |
0c6b522a CK |
247 | #ifdef CONFIG_DEBUG_MUTEXES |
248 | /** | |
047a1b87 | 249 | * dma_resv_reset_max_fences - reset fences for debugging |
0c6b522a CK |
250 | * @obj: the dma_resv object to reset |
251 | * | |
047a1b87 | 252 | * Reset the number of pre-reserved fence slots to test that drivers do |
c8d4c18b | 253 | * correct slot allocation using dma_resv_reserve_fences(). See also |
047a1b87 | 254 | * &dma_resv_list.max_fences. |
0c6b522a | 255 | */ |
73511edf | 256 | void dma_resv_reset_max_fences(struct dma_resv *obj) |
0c6b522a | 257 | { |
047a1b87 | 258 | struct dma_resv_list *fences = dma_resv_fences_list(obj); |
0c6b522a | 259 | |
fb5ce730 CK |
260 | dma_resv_assert_held(obj); |
261 | ||
047a1b87 | 262 | /* Test fence slot reservation */ |
fb5ce730 | 263 | if (fences) |
047a1b87 | 264 | fences->max_fences = fences->num_fences; |
0c6b522a | 265 | } |
73511edf | 266 | EXPORT_SYMBOL(dma_resv_reset_max_fences); |
0c6b522a CK |
267 | #endif |
268 | ||
dad6c394 | 269 | /** |
047a1b87 | 270 | * dma_resv_add_fence - Add a fence to the dma_resv obj |
dad6c394 | 271 | * @obj: the reservation object |
047a1b87 CK |
272 | * @fence: the fence to add |
273 | * @usage: how the fence is used, see enum dma_resv_usage | |
dad6c394 | 274 | * |
047a1b87 | 275 | * Add a fence to a slot, @obj must be locked with dma_resv_lock(), and |
c8d4c18b | 276 | * dma_resv_reserve_fences() has been called. |
d9edf92d DV |
277 | * |
278 | * See also &dma_resv.fence for a discussion of the semantics. | |
04a5faa8 | 279 | */ |
047a1b87 CK |
280 | void dma_resv_add_fence(struct dma_resv *obj, struct dma_fence *fence, |
281 | enum dma_resv_usage usage) | |
04a5faa8 | 282 | { |
52791eee | 283 | struct dma_resv_list *fobj; |
93505ee7 | 284 | struct dma_fence *old; |
a590d0fd | 285 | unsigned int i, count; |
04a5faa8 | 286 | |
27836b64 CK |
287 | dma_fence_get(fence); |
288 | ||
52791eee | 289 | dma_resv_assert_held(obj); |
547c7138 | 290 | |
68129f43 CK |
291 | /* Drivers should not add containers here, instead add each fence |
292 | * individually. | |
293 | */ | |
294 | WARN_ON(dma_fence_is_container(fence)); | |
295 | ||
047a1b87 CK |
296 | fobj = dma_resv_fences_list(obj); |
297 | count = fobj->num_fences; | |
04a5faa8 | 298 | |
a590d0fd | 299 | for (i = 0; i < count; ++i) { |
047a1b87 | 300 | enum dma_resv_usage old_usage; |
27836b64 | 301 | |
047a1b87 | 302 | dma_resv_list_entry(fobj, i, obj, &old, &old_usage); |
a3f7c10a CK |
303 | if ((old->context == fence->context && old_usage >= usage && |
304 | dma_fence_is_later(fence, old)) || | |
8f94eda3 CK |
305 | dma_fence_is_signaled(old)) { |
306 | dma_resv_list_set(fobj, i, fence, usage); | |
307 | dma_fence_put(old); | |
308 | return; | |
309 | } | |
27836b64 CK |
310 | } |
311 | ||
047a1b87 | 312 | BUG_ON(fobj->num_fences >= fobj->max_fences); |
a590d0fd | 313 | count++; |
27836b64 | 314 | |
047a1b87 CK |
315 | dma_resv_list_set(fobj, i, fence, usage); |
316 | /* pointer update must be visible before we extend the num_fences */ | |
317 | smp_store_mb(fobj->num_fences, count); | |
04a5faa8 | 318 | } |
047a1b87 | 319 | EXPORT_SYMBOL(dma_resv_add_fence); |
04a5faa8 | 320 | |
548e7432 CK |
321 | /** |
322 | * dma_resv_replace_fences - replace fences in the dma_resv obj | |
323 | * @obj: the reservation object | |
324 | * @context: the context of the fences to replace | |
325 | * @replacement: the new fence to use instead | |
73511edf | 326 | * @usage: how the new fence is used, see enum dma_resv_usage |
548e7432 CK |
327 | * |
328 | * Replace fences with a specified context with a new fence. Only valid if the | |
329 | * operation represented by the original fence has no longer access to the | |
330 | * resources represented by the dma_resv object when the new fence completes. | |
331 | * | |
332 | * And example for using this is replacing a preemption fence with a page table | |
333 | * update fence which makes the resource inaccessible. | |
334 | */ | |
335 | void dma_resv_replace_fences(struct dma_resv *obj, uint64_t context, | |
73511edf CK |
336 | struct dma_fence *replacement, |
337 | enum dma_resv_usage usage) | |
548e7432 CK |
338 | { |
339 | struct dma_resv_list *list; | |
548e7432 CK |
340 | unsigned int i; |
341 | ||
342 | dma_resv_assert_held(obj); | |
343 | ||
047a1b87 | 344 | list = dma_resv_fences_list(obj); |
047a1b87 CK |
345 | for (i = 0; list && i < list->num_fences; ++i) { |
346 | struct dma_fence *old; | |
548e7432 | 347 | |
047a1b87 | 348 | dma_resv_list_entry(list, i, obj, &old, NULL); |
548e7432 CK |
349 | if (old->context != context) |
350 | continue; | |
351 | ||
7c1aeba7 | 352 | dma_resv_list_set(list, i, dma_fence_get(replacement), usage); |
548e7432 CK |
353 | dma_fence_put(old); |
354 | } | |
548e7432 CK |
355 | } |
356 | EXPORT_SYMBOL(dma_resv_replace_fences); | |
357 | ||
047a1b87 | 358 | /* Restart the unlocked iteration by initializing the cursor object. */ |
c921ff37 CK |
359 | static void dma_resv_iter_restart_unlocked(struct dma_resv_iter *cursor) |
360 | { | |
047a1b87 CK |
361 | cursor->index = 0; |
362 | cursor->num_fences = 0; | |
363 | cursor->fences = dma_resv_fences_list(cursor->obj); | |
364 | if (cursor->fences) | |
365 | cursor->num_fences = cursor->fences->num_fences; | |
c921ff37 CK |
366 | cursor->is_restarted = true; |
367 | } | |
368 | ||
d80976d9 | 369 | /* Walk to the next not signaled fence and grab a reference to it */ |
c921ff37 CK |
370 | static void dma_resv_iter_walk_unlocked(struct dma_resv_iter *cursor) |
371 | { | |
047a1b87 CK |
372 | if (!cursor->fences) |
373 | return; | |
c921ff37 CK |
374 | |
375 | do { | |
376 | /* Drop the reference from the previous round */ | |
377 | dma_fence_put(cursor->fence); | |
378 | ||
047a1b87 | 379 | if (cursor->index >= cursor->num_fences) { |
c921ff37 CK |
380 | cursor->fence = NULL; |
381 | break; | |
382 | ||
c921ff37 | 383 | } |
047a1b87 CK |
384 | |
385 | dma_resv_list_entry(cursor->fences, cursor->index++, | |
386 | cursor->obj, &cursor->fence, | |
387 | &cursor->fence_usage); | |
c921ff37 | 388 | cursor->fence = dma_fence_get_rcu(cursor->fence); |
8f94eda3 CK |
389 | if (!cursor->fence) { |
390 | dma_resv_iter_restart_unlocked(cursor); | |
391 | continue; | |
392 | } | |
047a1b87 CK |
393 | |
394 | if (!dma_fence_is_signaled(cursor->fence) && | |
395 | cursor->usage >= cursor->fence_usage) | |
c921ff37 CK |
396 | break; |
397 | } while (true); | |
398 | } | |
399 | ||
400 | /** | |
401 | * dma_resv_iter_first_unlocked - first fence in an unlocked dma_resv obj. | |
402 | * @cursor: the cursor with the current position | |
403 | * | |
d80976d9 DV |
404 | * Subsequent fences are iterated with dma_resv_iter_next_unlocked(). |
405 | * | |
406 | * Beware that the iterator can be restarted. Code which accumulates statistics | |
407 | * or similar needs to check for this with dma_resv_iter_is_restarted(). For | |
408 | * this reason prefer the locked dma_resv_iter_first() whenver possible. | |
409 | * | |
c921ff37 CK |
410 | * Returns the first fence from an unlocked dma_resv obj. |
411 | */ | |
412 | struct dma_fence *dma_resv_iter_first_unlocked(struct dma_resv_iter *cursor) | |
413 | { | |
414 | rcu_read_lock(); | |
415 | do { | |
416 | dma_resv_iter_restart_unlocked(cursor); | |
417 | dma_resv_iter_walk_unlocked(cursor); | |
8f94eda3 | 418 | } while (dma_resv_fences_list(cursor->obj) != cursor->fences); |
c921ff37 CK |
419 | rcu_read_unlock(); |
420 | ||
421 | return cursor->fence; | |
422 | } | |
423 | EXPORT_SYMBOL(dma_resv_iter_first_unlocked); | |
424 | ||
425 | /** | |
426 | * dma_resv_iter_next_unlocked - next fence in an unlocked dma_resv obj. | |
427 | * @cursor: the cursor with the current position | |
428 | * | |
d80976d9 DV |
429 | * Beware that the iterator can be restarted. Code which accumulates statistics |
430 | * or similar needs to check for this with dma_resv_iter_is_restarted(). For | |
431 | * this reason prefer the locked dma_resv_iter_next() whenver possible. | |
432 | * | |
c921ff37 CK |
433 | * Returns the next fence from an unlocked dma_resv obj. |
434 | */ | |
435 | struct dma_fence *dma_resv_iter_next_unlocked(struct dma_resv_iter *cursor) | |
436 | { | |
437 | bool restart; | |
438 | ||
439 | rcu_read_lock(); | |
440 | cursor->is_restarted = false; | |
8f94eda3 | 441 | restart = dma_resv_fences_list(cursor->obj) != cursor->fences; |
c921ff37 CK |
442 | do { |
443 | if (restart) | |
444 | dma_resv_iter_restart_unlocked(cursor); | |
445 | dma_resv_iter_walk_unlocked(cursor); | |
446 | restart = true; | |
8f94eda3 | 447 | } while (dma_resv_fences_list(cursor->obj) != cursor->fences); |
c921ff37 CK |
448 | rcu_read_unlock(); |
449 | ||
450 | return cursor->fence; | |
451 | } | |
452 | EXPORT_SYMBOL(dma_resv_iter_next_unlocked); | |
453 | ||
5baaac31 CK |
454 | /** |
455 | * dma_resv_iter_first - first fence from a locked dma_resv object | |
456 | * @cursor: cursor to record the current position | |
457 | * | |
d80976d9 DV |
458 | * Subsequent fences are iterated with dma_resv_iter_next_unlocked(). |
459 | * | |
5baaac31 CK |
460 | * Return the first fence in the dma_resv object while holding the |
461 | * &dma_resv.lock. | |
462 | */ | |
463 | struct dma_fence *dma_resv_iter_first(struct dma_resv_iter *cursor) | |
464 | { | |
465 | struct dma_fence *fence; | |
466 | ||
467 | dma_resv_assert_held(cursor->obj); | |
468 | ||
469 | cursor->index = 0; | |
047a1b87 | 470 | cursor->fences = dma_resv_fences_list(cursor->obj); |
5baaac31 | 471 | |
047a1b87 | 472 | fence = dma_resv_iter_next(cursor); |
5baaac31 CK |
473 | cursor->is_restarted = true; |
474 | return fence; | |
475 | } | |
476 | EXPORT_SYMBOL_GPL(dma_resv_iter_first); | |
477 | ||
478 | /** | |
479 | * dma_resv_iter_next - next fence from a locked dma_resv object | |
480 | * @cursor: cursor to record the current position | |
481 | * | |
482 | * Return the next fences from the dma_resv object while holding the | |
483 | * &dma_resv.lock. | |
484 | */ | |
485 | struct dma_fence *dma_resv_iter_next(struct dma_resv_iter *cursor) | |
486 | { | |
047a1b87 | 487 | struct dma_fence *fence; |
5baaac31 CK |
488 | |
489 | dma_resv_assert_held(cursor->obj); | |
490 | ||
491 | cursor->is_restarted = false; | |
5baaac31 | 492 | |
047a1b87 CK |
493 | do { |
494 | if (!cursor->fences || | |
495 | cursor->index >= cursor->fences->num_fences) | |
496 | return NULL; | |
497 | ||
498 | dma_resv_list_entry(cursor->fences, cursor->index++, | |
499 | cursor->obj, &fence, &cursor->fence_usage); | |
500 | } while (cursor->fence_usage > cursor->usage); | |
501 | ||
502 | return fence; | |
5baaac31 CK |
503 | } |
504 | EXPORT_SYMBOL_GPL(dma_resv_iter_next); | |
505 | ||
7faf952a | 506 | /** |
068d9d75 CK |
507 | * dma_resv_copy_fences - Copy all fences from src to dst. |
508 | * @dst: the destination reservation object | |
509 | * @src: the source reservation object | |
510 | * | |
511 | * Copy all fences from src to dst. dst-lock must be held. | |
512 | */ | |
52791eee | 513 | int dma_resv_copy_fences(struct dma_resv *dst, struct dma_resv *src) |
7faf952a | 514 | { |
96601e8a CK |
515 | struct dma_resv_iter cursor; |
516 | struct dma_resv_list *list; | |
047a1b87 | 517 | struct dma_fence *f; |
7faf952a | 518 | |
52791eee | 519 | dma_resv_assert_held(dst); |
547c7138 | 520 | |
96601e8a | 521 | list = NULL; |
7faf952a | 522 | |
0cc848a7 | 523 | dma_resv_iter_begin(&cursor, src, DMA_RESV_USAGE_BOOKKEEP); |
96601e8a | 524 | dma_resv_for_each_fence_unlocked(&cursor, f) { |
b016cd6e | 525 | |
96601e8a CK |
526 | if (dma_resv_iter_is_restarted(&cursor)) { |
527 | dma_resv_list_free(list); | |
39e16ba1 | 528 | |
047a1b87 CK |
529 | list = dma_resv_list_alloc(cursor.num_fences); |
530 | if (!list) { | |
531 | dma_resv_iter_end(&cursor); | |
532 | return -ENOMEM; | |
39e16ba1 | 533 | } |
047a1b87 | 534 | list->num_fences = 0; |
39e16ba1 | 535 | } |
39e16ba1 | 536 | |
96601e8a | 537 | dma_fence_get(f); |
047a1b87 CK |
538 | dma_resv_list_set(list, list->num_fences++, f, |
539 | dma_resv_iter_usage(&cursor)); | |
96601e8a CK |
540 | } |
541 | dma_resv_iter_end(&cursor); | |
7faf952a | 542 | |
047a1b87 | 543 | list = rcu_replace_pointer(dst->fences, list, dma_resv_held(dst)); |
96601e8a | 544 | dma_resv_list_free(list); |
7faf952a CK |
545 | return 0; |
546 | } | |
52791eee | 547 | EXPORT_SYMBOL(dma_resv_copy_fences); |
7faf952a | 548 | |
dad6c394 | 549 | /** |
047a1b87 | 550 | * dma_resv_get_fences - Get an object's fences |
dad6c394 RC |
551 | * fences without update side lock held |
552 | * @obj: the reservation object | |
7bc80a54 | 553 | * @usage: controls which fences to include, see enum dma_resv_usage. |
75ab2b36 CK |
554 | * @num_fences: the number of fences returned |
555 | * @fences: the array of fence ptrs returned (array is krealloc'd to the | |
556 | * required size, and must be freed by caller) | |
dad6c394 | 557 | * |
75ab2b36 CK |
558 | * Retrieve all fences from the reservation object. |
559 | * Returns either zero or -ENOMEM. | |
dad6c394 | 560 | */ |
7bc80a54 | 561 | int dma_resv_get_fences(struct dma_resv *obj, enum dma_resv_usage usage, |
75ab2b36 | 562 | unsigned int *num_fences, struct dma_fence ***fences) |
3c3b177a | 563 | { |
d3c80698 CK |
564 | struct dma_resv_iter cursor; |
565 | struct dma_fence *fence; | |
fedf5413 | 566 | |
75ab2b36 CK |
567 | *num_fences = 0; |
568 | *fences = NULL; | |
3c3b177a | 569 | |
7bc80a54 | 570 | dma_resv_iter_begin(&cursor, obj, usage); |
d3c80698 | 571 | dma_resv_for_each_fence_unlocked(&cursor, fence) { |
a35f2f34 | 572 | |
d3c80698 CK |
573 | if (dma_resv_iter_is_restarted(&cursor)) { |
574 | unsigned int count; | |
3c3b177a | 575 | |
75ab2b36 CK |
576 | while (*num_fences) |
577 | dma_fence_put((*fences)[--(*num_fences)]); | |
f5b07b04 | 578 | |
047a1b87 | 579 | count = cursor.num_fences + 1; |
3c3b177a | 580 | |
d3c80698 | 581 | /* Eventually re-allocate the array */ |
75ab2b36 | 582 | *fences = krealloc_array(*fences, count, |
d3c80698 CK |
583 | sizeof(void *), |
584 | GFP_KERNEL); | |
75ab2b36 | 585 | if (count && !*fences) { |
d3c80698 CK |
586 | dma_resv_iter_end(&cursor); |
587 | return -ENOMEM; | |
3c3b177a | 588 | } |
fedf5413 | 589 | } |
3c3b177a | 590 | |
75ab2b36 | 591 | (*fences)[(*num_fences)++] = dma_fence_get(fence); |
3c3b177a | 592 | } |
d3c80698 | 593 | dma_resv_iter_end(&cursor); |
fedf5413 | 594 | |
d3c80698 | 595 | return 0; |
3c3b177a | 596 | } |
d3fae3b3 | 597 | EXPORT_SYMBOL_GPL(dma_resv_get_fences); |
3c3b177a | 598 | |
92cedee6 CK |
599 | /** |
600 | * dma_resv_get_singleton - Get a single fence for all the fences | |
601 | * @obj: the reservation object | |
7bc80a54 | 602 | * @usage: controls which fences to include, see enum dma_resv_usage. |
92cedee6 CK |
603 | * @fence: the resulting fence |
604 | * | |
605 | * Get a single fence representing all the fences inside the resv object. | |
606 | * Returns either 0 for success or -ENOMEM. | |
607 | * | |
608 | * Warning: This can't be used like this when adding the fence back to the resv | |
609 | * object since that can lead to stack corruption when finalizing the | |
610 | * dma_fence_array. | |
611 | * | |
612 | * Returns 0 on success and negative error values on failure. | |
613 | */ | |
7bc80a54 | 614 | int dma_resv_get_singleton(struct dma_resv *obj, enum dma_resv_usage usage, |
92cedee6 CK |
615 | struct dma_fence **fence) |
616 | { | |
617 | struct dma_fence_array *array; | |
618 | struct dma_fence **fences; | |
619 | unsigned count; | |
620 | int r; | |
621 | ||
7bc80a54 | 622 | r = dma_resv_get_fences(obj, usage, &count, &fences); |
92cedee6 CK |
623 | if (r) |
624 | return r; | |
625 | ||
626 | if (count == 0) { | |
627 | *fence = NULL; | |
628 | return 0; | |
629 | } | |
630 | ||
631 | if (count == 1) { | |
632 | *fence = fences[0]; | |
633 | kfree(fences); | |
634 | return 0; | |
635 | } | |
636 | ||
637 | array = dma_fence_array_create(count, fences, | |
638 | dma_fence_context_alloc(1), | |
639 | 1, false); | |
640 | if (!array) { | |
641 | while (count--) | |
642 | dma_fence_put(fences[count]); | |
643 | kfree(fences); | |
644 | return -ENOMEM; | |
645 | } | |
646 | ||
647 | *fence = &array->base; | |
648 | return 0; | |
649 | } | |
650 | EXPORT_SYMBOL_GPL(dma_resv_get_singleton); | |
651 | ||
dad6c394 | 652 | /** |
047a1b87 | 653 | * dma_resv_wait_timeout - Wait on reservation's objects fences |
dad6c394 | 654 | * @obj: the reservation object |
7bc80a54 | 655 | * @usage: controls which fences to include, see enum dma_resv_usage. |
dad6c394 RC |
656 | * @intr: if true, do interruptible wait |
657 | * @timeout: timeout value in jiffies or zero to return immediately | |
658 | * | |
d3fae3b3 CK |
659 | * Callers are not required to hold specific locks, but maybe hold |
660 | * dma_resv_lock() already | |
dad6c394 RC |
661 | * RETURNS |
662 | * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or | |
e28f6966 | 663 | * greater than zero on success. |
dad6c394 | 664 | */ |
7bc80a54 CK |
665 | long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage, |
666 | bool intr, unsigned long timeout) | |
3c3b177a | 667 | { |
06a66b5c | 668 | long ret = timeout ? timeout : 1; |
ada5c48b | 669 | struct dma_resv_iter cursor; |
068d9d75 | 670 | struct dma_fence *fence; |
3c3b177a | 671 | |
7bc80a54 | 672 | dma_resv_iter_begin(&cursor, obj, usage); |
ada5c48b | 673 | dma_resv_for_each_fence_unlocked(&cursor, fence) { |
3c3b177a | 674 | |
ada5c48b CK |
675 | ret = dma_fence_wait_timeout(fence, intr, ret); |
676 | if (ret <= 0) { | |
677 | dma_resv_iter_end(&cursor); | |
678 | return ret; | |
3c3b177a ML |
679 | } |
680 | } | |
ada5c48b | 681 | dma_resv_iter_end(&cursor); |
3c3b177a | 682 | |
3c3b177a | 683 | return ret; |
3c3b177a | 684 | } |
d3fae3b3 | 685 | EXPORT_SYMBOL_GPL(dma_resv_wait_timeout); |
3c3b177a | 686 | |
d7d5a21d RC |
687 | /** |
688 | * dma_resv_set_deadline - Set a deadline on reservation's objects fences | |
689 | * @obj: the reservation object | |
690 | * @usage: controls which fences to include, see enum dma_resv_usage. | |
691 | * @deadline: the requested deadline (MONOTONIC) | |
692 | * | |
693 | * May be called without holding the dma_resv lock. Sets @deadline on | |
694 | * all fences filtered by @usage. | |
695 | */ | |
696 | void dma_resv_set_deadline(struct dma_resv *obj, enum dma_resv_usage usage, | |
697 | ktime_t deadline) | |
698 | { | |
699 | struct dma_resv_iter cursor; | |
700 | struct dma_fence *fence; | |
701 | ||
702 | dma_resv_iter_begin(&cursor, obj, usage); | |
703 | dma_resv_for_each_fence_unlocked(&cursor, fence) { | |
704 | dma_fence_set_deadline(fence, deadline); | |
705 | } | |
706 | dma_resv_iter_end(&cursor); | |
707 | } | |
708 | EXPORT_SYMBOL_GPL(dma_resv_set_deadline); | |
3c3b177a | 709 | |
dad6c394 | 710 | /** |
d3fae3b3 CK |
711 | * dma_resv_test_signaled - Test if a reservation object's fences have been |
712 | * signaled. | |
dad6c394 | 713 | * @obj: the reservation object |
7bc80a54 | 714 | * @usage: controls which fences to include, see enum dma_resv_usage. |
dad6c394 | 715 | * |
d3fae3b3 | 716 | * Callers are not required to hold specific locks, but maybe hold |
d9edf92d DV |
717 | * dma_resv_lock() already. |
718 | * | |
dad6c394 | 719 | * RETURNS |
d9edf92d DV |
720 | * |
721 | * True if all fences signaled, else false. | |
dad6c394 | 722 | */ |
7bc80a54 | 723 | bool dma_resv_test_signaled(struct dma_resv *obj, enum dma_resv_usage usage) |
3c3b177a | 724 | { |
7fa828cb | 725 | struct dma_resv_iter cursor; |
9d38814d | 726 | struct dma_fence *fence; |
b016cd6e | 727 | |
7bc80a54 | 728 | dma_resv_iter_begin(&cursor, obj, usage); |
7fa828cb CK |
729 | dma_resv_for_each_fence_unlocked(&cursor, fence) { |
730 | dma_resv_iter_end(&cursor); | |
731 | return false; | |
b016cd6e | 732 | } |
7fa828cb CK |
733 | dma_resv_iter_end(&cursor); |
734 | return true; | |
3c3b177a | 735 | } |
d3fae3b3 | 736 | EXPORT_SYMBOL_GPL(dma_resv_test_signaled); |
068d9d75 | 737 | |
a25efb38 CK |
738 | /** |
739 | * dma_resv_describe - Dump description of the resv object into seq_file | |
740 | * @obj: the reservation object | |
741 | * @seq: the seq_file to dump the description into | |
742 | * | |
743 | * Dump a textual description of the fences inside an dma_resv object into the | |
744 | * seq_file. | |
745 | */ | |
746 | void dma_resv_describe(struct dma_resv *obj, struct seq_file *seq) | |
747 | { | |
0cc848a7 | 748 | static const char *usage[] = { "kernel", "write", "read", "bookkeep" }; |
a25efb38 CK |
749 | struct dma_resv_iter cursor; |
750 | struct dma_fence *fence; | |
751 | ||
7bc80a54 | 752 | dma_resv_for_each_fence(&cursor, obj, DMA_RESV_USAGE_READ, fence) { |
a25efb38 | 753 | seq_printf(seq, "\t%s fence:", |
73511edf | 754 | usage[dma_resv_iter_usage(&cursor)]); |
a25efb38 CK |
755 | dma_fence_describe(fence, seq); |
756 | } | |
757 | } | |
758 | EXPORT_SYMBOL_GPL(dma_resv_describe); | |
759 | ||
068d9d75 CK |
760 | #if IS_ENABLED(CONFIG_LOCKDEP) |
761 | static int __init dma_resv_lockdep(void) | |
762 | { | |
763 | struct mm_struct *mm = mm_alloc(); | |
764 | struct ww_acquire_ctx ctx; | |
765 | struct dma_resv obj; | |
766 | struct address_space mapping; | |
767 | int ret; | |
768 | ||
769 | if (!mm) | |
770 | return -ENOMEM; | |
771 | ||
772 | dma_resv_init(&obj); | |
773 | address_space_init_once(&mapping); | |
774 | ||
775 | mmap_read_lock(mm); | |
776 | ww_acquire_init(&ctx, &reservation_ww_class); | |
777 | ret = dma_resv_lock(&obj, &ctx); | |
778 | if (ret == -EDEADLK) | |
779 | dma_resv_lock_slow(&obj, &ctx); | |
780 | fs_reclaim_acquire(GFP_KERNEL); | |
781 | /* for unmap_mapping_range on trylocked buffer objects in shrinkers */ | |
782 | i_mmap_lock_write(&mapping); | |
783 | i_mmap_unlock_write(&mapping); | |
784 | #ifdef CONFIG_MMU_NOTIFIER | |
785 | lock_map_acquire(&__mmu_notifier_invalidate_range_start_map); | |
786 | __dma_fence_might_wait(); | |
787 | lock_map_release(&__mmu_notifier_invalidate_range_start_map); | |
788 | #else | |
789 | __dma_fence_might_wait(); | |
790 | #endif | |
791 | fs_reclaim_release(GFP_KERNEL); | |
792 | ww_mutex_unlock(&obj.lock); | |
793 | ww_acquire_fini(&ctx); | |
794 | mmap_read_unlock(mm); | |
795 | ||
796 | mmput(mm); | |
797 | ||
798 | return 0; | |
799 | } | |
800 | subsys_initcall(dma_resv_lockdep); | |
801 | #endif |