Commit | Line | Data |
---|---|---|
068d9d75 | 1 | // SPDX-License-Identifier: MIT |
786d7257 | 2 | /* |
04a5faa8 | 3 | * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst) |
786d7257 ML |
4 | * |
5 | * Based on bo.c which bears the following copyright notice, | |
6 | * but is dual licensed: | |
7 | * | |
8 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA | |
9 | * All Rights Reserved. | |
10 | * | |
11 | * Permission is hereby granted, free of charge, to any person obtaining a | |
12 | * copy of this software and associated documentation files (the | |
13 | * "Software"), to deal in the Software without restriction, including | |
14 | * without limitation the rights to use, copy, modify, merge, publish, | |
15 | * distribute, sub license, and/or sell copies of the Software, and to | |
16 | * permit persons to whom the Software is furnished to do so, subject to | |
17 | * the following conditions: | |
18 | * | |
19 | * The above copyright notice and this permission notice (including the | |
20 | * next paragraph) shall be included in all copies or substantial portions | |
21 | * of the Software. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
24 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
25 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
26 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
27 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
28 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
29 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
30 | * | |
31 | **************************************************************************/ | |
32 | /* | |
33 | * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> | |
34 | */ | |
35 | ||
52791eee | 36 | #include <linux/dma-resv.h> |
92cedee6 | 37 | #include <linux/dma-fence-array.h> |
786d7257 | 38 | #include <linux/export.h> |
0adf65f5 | 39 | #include <linux/mm.h> |
b2a8116e | 40 | #include <linux/sched/mm.h> |
d0b9a9ae | 41 | #include <linux/mmu_notifier.h> |
a25efb38 | 42 | #include <linux/seq_file.h> |
786d7257 | 43 | |
dad6c394 RC |
44 | /** |
45 | * DOC: Reservation Object Overview | |
46 | * | |
047a1b87 CK |
47 | * The reservation object provides a mechanism to manage a container of |
48 | * dma_fence object associated with a resource. A reservation object | |
49 | * can have any number of fences attaches to it. Each fence carries an usage | |
50 | * parameter determining how the operation represented by the fence is using the | |
51 | * resource. The RCU mechanism is used to protect read access to fences from | |
52 | * locked write-side updates. | |
d9edf92d DV |
53 | * |
54 | * See struct dma_resv for more details. | |
dad6c394 RC |
55 | */ |
56 | ||
08295b3b | 57 | DEFINE_WD_CLASS(reservation_ww_class); |
786d7257 | 58 | EXPORT_SYMBOL(reservation_ww_class); |
04a5faa8 | 59 | |
047a1b87 CK |
60 | /* Mask for the lower fence pointer bits */ |
61 | #define DMA_RESV_LIST_MASK 0x3 | |
62 | ||
8938d484 CK |
63 | struct dma_resv_list { |
64 | struct rcu_head rcu; | |
047a1b87 CK |
65 | u32 num_fences, max_fences; |
66 | struct dma_fence __rcu *table[]; | |
8938d484 CK |
67 | }; |
68 | ||
047a1b87 CK |
69 | /* Extract the fence and usage flags from an RCU protected entry in the list. */ |
70 | static void dma_resv_list_entry(struct dma_resv_list *list, unsigned int index, | |
71 | struct dma_resv *resv, struct dma_fence **fence, | |
72 | enum dma_resv_usage *usage) | |
73 | { | |
74 | long tmp; | |
75 | ||
76 | tmp = (long)rcu_dereference_check(list->table[index], | |
77 | resv ? dma_resv_held(resv) : true); | |
78 | *fence = (struct dma_fence *)(tmp & ~DMA_RESV_LIST_MASK); | |
79 | if (usage) | |
80 | *usage = tmp & DMA_RESV_LIST_MASK; | |
81 | } | |
82 | ||
83 | /* Set the fence and usage flags at the specific index in the list. */ | |
84 | static void dma_resv_list_set(struct dma_resv_list *list, | |
85 | unsigned int index, | |
86 | struct dma_fence *fence, | |
87 | enum dma_resv_usage usage) | |
88 | { | |
89 | long tmp = ((long)fence) | usage; | |
90 | ||
91 | RCU_INIT_POINTER(list->table[index], (struct dma_fence *)tmp); | |
92 | } | |
93 | ||
94 | /* | |
52791eee | 95 | * Allocate a new dma_resv_list and make sure to correctly initialize |
047a1b87 | 96 | * max_fences. |
96e95496 | 97 | */ |
047a1b87 | 98 | static struct dma_resv_list *dma_resv_list_alloc(unsigned int max_fences) |
96e95496 | 99 | { |
52791eee | 100 | struct dma_resv_list *list; |
96e95496 | 101 | |
047a1b87 | 102 | list = kmalloc(struct_size(list, table, max_fences), GFP_KERNEL); |
96e95496 CK |
103 | if (!list) |
104 | return NULL; | |
105 | ||
047a1b87 CK |
106 | list->max_fences = (ksize(list) - offsetof(typeof(*list), table)) / |
107 | sizeof(*list->table); | |
96e95496 CK |
108 | |
109 | return list; | |
110 | } | |
111 | ||
047a1b87 | 112 | /* Free a dma_resv_list and make sure to drop all references. */ |
52791eee | 113 | static void dma_resv_list_free(struct dma_resv_list *list) |
96e95496 CK |
114 | { |
115 | unsigned int i; | |
116 | ||
117 | if (!list) | |
118 | return; | |
119 | ||
047a1b87 CK |
120 | for (i = 0; i < list->num_fences; ++i) { |
121 | struct dma_fence *fence; | |
96e95496 | 122 | |
047a1b87 CK |
123 | dma_resv_list_entry(list, i, NULL, &fence, NULL); |
124 | dma_fence_put(fence); | |
125 | } | |
96e95496 CK |
126 | kfree_rcu(list, rcu); |
127 | } | |
128 | ||
8735f168 | 129 | /** |
52791eee | 130 | * dma_resv_init - initialize a reservation object |
8735f168 CK |
131 | * @obj: the reservation object |
132 | */ | |
52791eee | 133 | void dma_resv_init(struct dma_resv *obj) |
8735f168 CK |
134 | { |
135 | ww_mutex_init(&obj->lock, &reservation_ww_class); | |
b016cd6e | 136 | |
047a1b87 | 137 | RCU_INIT_POINTER(obj->fences, NULL); |
8735f168 | 138 | } |
52791eee | 139 | EXPORT_SYMBOL(dma_resv_init); |
8735f168 CK |
140 | |
141 | /** | |
52791eee | 142 | * dma_resv_fini - destroys a reservation object |
8735f168 CK |
143 | * @obj: the reservation object |
144 | */ | |
52791eee | 145 | void dma_resv_fini(struct dma_resv *obj) |
8735f168 | 146 | { |
8735f168 CK |
147 | /* |
148 | * This object should be dead and all references must have | |
149 | * been released to it, so no need to be protected with rcu. | |
150 | */ | |
047a1b87 | 151 | dma_resv_list_free(rcu_dereference_protected(obj->fences, true)); |
8735f168 CK |
152 | ww_mutex_destroy(&obj->lock); |
153 | } | |
52791eee | 154 | EXPORT_SYMBOL(dma_resv_fini); |
8735f168 | 155 | |
047a1b87 CK |
156 | /* Dereference the fences while ensuring RCU rules */ |
157 | static inline struct dma_resv_list *dma_resv_fences_list(struct dma_resv *obj) | |
8938d484 | 158 | { |
047a1b87 | 159 | return rcu_dereference_check(obj->fences, dma_resv_held(obj)); |
8938d484 CK |
160 | } |
161 | ||
dad6c394 | 162 | /** |
047a1b87 | 163 | * dma_resv_reserve_fences - Reserve space to add fences to a dma_resv object. |
dad6c394 | 164 | * @obj: reservation object |
ca05359f | 165 | * @num_fences: number of fences we want to add |
dad6c394 | 166 | * |
047a1b87 CK |
167 | * Should be called before dma_resv_add_fence(). Must be called with @obj |
168 | * locked through dma_resv_lock(). | |
d9edf92d DV |
169 | * |
170 | * Note that the preallocated slots need to be re-reserved if @obj is unlocked | |
047a1b87 CK |
171 | * at any time before calling dma_resv_add_fence(). This is validated when |
172 | * CONFIG_DEBUG_MUTEXES is enabled. | |
dad6c394 RC |
173 | * |
174 | * RETURNS | |
175 | * Zero for success, or -errno | |
04a5faa8 | 176 | */ |
c8d4c18b | 177 | int dma_resv_reserve_fences(struct dma_resv *obj, unsigned int num_fences) |
04a5faa8 | 178 | { |
52791eee | 179 | struct dma_resv_list *old, *new; |
27836b64 | 180 | unsigned int i, j, k, max; |
04a5faa8 | 181 | |
52791eee | 182 | dma_resv_assert_held(obj); |
547c7138 | 183 | |
047a1b87 CK |
184 | old = dma_resv_fences_list(obj); |
185 | if (old && old->max_fences) { | |
186 | if ((old->num_fences + num_fences) <= old->max_fences) | |
04a5faa8 | 187 | return 0; |
047a1b87 | 188 | max = max(old->num_fences + num_fences, old->max_fences * 2); |
ca25fe5e | 189 | } else { |
bf897583 | 190 | max = max(4ul, roundup_pow_of_two(num_fences)); |
ca25fe5e | 191 | } |
3c3b177a | 192 | |
52791eee | 193 | new = dma_resv_list_alloc(max); |
27836b64 CK |
194 | if (!new) |
195 | return -ENOMEM; | |
04a5faa8 ML |
196 | |
197 | /* | |
198 | * no need to bump fence refcounts, rcu_read access | |
199 | * requires the use of kref_get_unless_zero, and the | |
200 | * references from the old struct are carried over to | |
201 | * the new. | |
202 | */ | |
047a1b87 CK |
203 | for (i = 0, j = 0, k = max; i < (old ? old->num_fences : 0); ++i) { |
204 | enum dma_resv_usage usage; | |
27836b64 | 205 | struct dma_fence *fence; |
3c3b177a | 206 | |
047a1b87 | 207 | dma_resv_list_entry(old, i, obj, &fence, &usage); |
27836b64 | 208 | if (dma_fence_is_signaled(fence)) |
047a1b87 | 209 | RCU_INIT_POINTER(new->table[--k], fence); |
4d9c62e8 | 210 | else |
047a1b87 | 211 | dma_resv_list_set(new, j++, fence, usage); |
04a5faa8 | 212 | } |
047a1b87 | 213 | new->num_fences = j; |
04a5faa8 | 214 | |
3c3b177a | 215 | /* |
30fe7b07 CW |
216 | * We are not changing the effective set of fences here so can |
217 | * merely update the pointer to the new array; both existing | |
218 | * readers and new readers will see exactly the same set of | |
047a1b87 | 219 | * active (unsignaled) fences. Individual fences and the |
30fe7b07 CW |
220 | * old array are protected by RCU and so will not vanish under |
221 | * the gaze of the rcu_read_lock() readers. | |
3c3b177a | 222 | */ |
047a1b87 | 223 | rcu_assign_pointer(obj->fences, new); |
3c3b177a | 224 | |
4d9c62e8 | 225 | if (!old) |
27836b64 | 226 | return 0; |
3c3b177a | 227 | |
4d9c62e8 | 228 | /* Drop the references to the signaled fences */ |
94eb1e10 | 229 | for (i = k; i < max; ++i) { |
27836b64 | 230 | struct dma_fence *fence; |
4d9c62e8 | 231 | |
047a1b87 | 232 | fence = rcu_dereference_protected(new->table[i], |
52791eee | 233 | dma_resv_held(obj)); |
27836b64 | 234 | dma_fence_put(fence); |
4d9c62e8 CK |
235 | } |
236 | kfree_rcu(old, rcu); | |
27836b64 CK |
237 | |
238 | return 0; | |
04a5faa8 | 239 | } |
c8d4c18b | 240 | EXPORT_SYMBOL(dma_resv_reserve_fences); |
04a5faa8 | 241 | |
0c6b522a CK |
242 | #ifdef CONFIG_DEBUG_MUTEXES |
243 | /** | |
047a1b87 | 244 | * dma_resv_reset_max_fences - reset fences for debugging |
0c6b522a CK |
245 | * @obj: the dma_resv object to reset |
246 | * | |
047a1b87 | 247 | * Reset the number of pre-reserved fence slots to test that drivers do |
c8d4c18b | 248 | * correct slot allocation using dma_resv_reserve_fences(). See also |
047a1b87 | 249 | * &dma_resv_list.max_fences. |
0c6b522a | 250 | */ |
73511edf | 251 | void dma_resv_reset_max_fences(struct dma_resv *obj) |
0c6b522a | 252 | { |
047a1b87 | 253 | struct dma_resv_list *fences = dma_resv_fences_list(obj); |
0c6b522a | 254 | |
fb5ce730 CK |
255 | dma_resv_assert_held(obj); |
256 | ||
047a1b87 | 257 | /* Test fence slot reservation */ |
fb5ce730 | 258 | if (fences) |
047a1b87 | 259 | fences->max_fences = fences->num_fences; |
0c6b522a | 260 | } |
73511edf | 261 | EXPORT_SYMBOL(dma_resv_reset_max_fences); |
0c6b522a CK |
262 | #endif |
263 | ||
dad6c394 | 264 | /** |
047a1b87 | 265 | * dma_resv_add_fence - Add a fence to the dma_resv obj |
dad6c394 | 266 | * @obj: the reservation object |
047a1b87 CK |
267 | * @fence: the fence to add |
268 | * @usage: how the fence is used, see enum dma_resv_usage | |
dad6c394 | 269 | * |
047a1b87 | 270 | * Add a fence to a slot, @obj must be locked with dma_resv_lock(), and |
c8d4c18b | 271 | * dma_resv_reserve_fences() has been called. |
d9edf92d DV |
272 | * |
273 | * See also &dma_resv.fence for a discussion of the semantics. | |
04a5faa8 | 274 | */ |
047a1b87 CK |
275 | void dma_resv_add_fence(struct dma_resv *obj, struct dma_fence *fence, |
276 | enum dma_resv_usage usage) | |
04a5faa8 | 277 | { |
52791eee | 278 | struct dma_resv_list *fobj; |
93505ee7 | 279 | struct dma_fence *old; |
a590d0fd | 280 | unsigned int i, count; |
04a5faa8 | 281 | |
27836b64 CK |
282 | dma_fence_get(fence); |
283 | ||
52791eee | 284 | dma_resv_assert_held(obj); |
547c7138 | 285 | |
68129f43 CK |
286 | /* Drivers should not add containers here, instead add each fence |
287 | * individually. | |
288 | */ | |
289 | WARN_ON(dma_fence_is_container(fence)); | |
290 | ||
047a1b87 CK |
291 | fobj = dma_resv_fences_list(obj); |
292 | count = fobj->num_fences; | |
04a5faa8 | 293 | |
a590d0fd | 294 | for (i = 0; i < count; ++i) { |
047a1b87 | 295 | enum dma_resv_usage old_usage; |
27836b64 | 296 | |
047a1b87 CK |
297 | dma_resv_list_entry(fobj, i, obj, &old, &old_usage); |
298 | if ((old->context == fence->context && old_usage >= usage) || | |
8f94eda3 CK |
299 | dma_fence_is_signaled(old)) { |
300 | dma_resv_list_set(fobj, i, fence, usage); | |
301 | dma_fence_put(old); | |
302 | return; | |
303 | } | |
27836b64 CK |
304 | } |
305 | ||
047a1b87 | 306 | BUG_ON(fobj->num_fences >= fobj->max_fences); |
a590d0fd | 307 | count++; |
27836b64 | 308 | |
047a1b87 CK |
309 | dma_resv_list_set(fobj, i, fence, usage); |
310 | /* pointer update must be visible before we extend the num_fences */ | |
311 | smp_store_mb(fobj->num_fences, count); | |
04a5faa8 | 312 | } |
047a1b87 | 313 | EXPORT_SYMBOL(dma_resv_add_fence); |
04a5faa8 | 314 | |
548e7432 CK |
315 | /** |
316 | * dma_resv_replace_fences - replace fences in the dma_resv obj | |
317 | * @obj: the reservation object | |
318 | * @context: the context of the fences to replace | |
319 | * @replacement: the new fence to use instead | |
73511edf | 320 | * @usage: how the new fence is used, see enum dma_resv_usage |
548e7432 CK |
321 | * |
322 | * Replace fences with a specified context with a new fence. Only valid if the | |
323 | * operation represented by the original fence has no longer access to the | |
324 | * resources represented by the dma_resv object when the new fence completes. | |
325 | * | |
326 | * And example for using this is replacing a preemption fence with a page table | |
327 | * update fence which makes the resource inaccessible. | |
328 | */ | |
329 | void dma_resv_replace_fences(struct dma_resv *obj, uint64_t context, | |
73511edf CK |
330 | struct dma_fence *replacement, |
331 | enum dma_resv_usage usage) | |
548e7432 CK |
332 | { |
333 | struct dma_resv_list *list; | |
548e7432 CK |
334 | unsigned int i; |
335 | ||
336 | dma_resv_assert_held(obj); | |
337 | ||
047a1b87 | 338 | list = dma_resv_fences_list(obj); |
047a1b87 CK |
339 | for (i = 0; list && i < list->num_fences; ++i) { |
340 | struct dma_fence *old; | |
548e7432 | 341 | |
047a1b87 | 342 | dma_resv_list_entry(list, i, obj, &old, NULL); |
548e7432 CK |
343 | if (old->context != context) |
344 | continue; | |
345 | ||
7c1aeba7 | 346 | dma_resv_list_set(list, i, dma_fence_get(replacement), usage); |
548e7432 CK |
347 | dma_fence_put(old); |
348 | } | |
548e7432 CK |
349 | } |
350 | EXPORT_SYMBOL(dma_resv_replace_fences); | |
351 | ||
047a1b87 | 352 | /* Restart the unlocked iteration by initializing the cursor object. */ |
c921ff37 CK |
353 | static void dma_resv_iter_restart_unlocked(struct dma_resv_iter *cursor) |
354 | { | |
047a1b87 CK |
355 | cursor->index = 0; |
356 | cursor->num_fences = 0; | |
357 | cursor->fences = dma_resv_fences_list(cursor->obj); | |
358 | if (cursor->fences) | |
359 | cursor->num_fences = cursor->fences->num_fences; | |
c921ff37 CK |
360 | cursor->is_restarted = true; |
361 | } | |
362 | ||
d80976d9 | 363 | /* Walk to the next not signaled fence and grab a reference to it */ |
c921ff37 CK |
364 | static void dma_resv_iter_walk_unlocked(struct dma_resv_iter *cursor) |
365 | { | |
047a1b87 CK |
366 | if (!cursor->fences) |
367 | return; | |
c921ff37 CK |
368 | |
369 | do { | |
370 | /* Drop the reference from the previous round */ | |
371 | dma_fence_put(cursor->fence); | |
372 | ||
047a1b87 | 373 | if (cursor->index >= cursor->num_fences) { |
c921ff37 CK |
374 | cursor->fence = NULL; |
375 | break; | |
376 | ||
c921ff37 | 377 | } |
047a1b87 CK |
378 | |
379 | dma_resv_list_entry(cursor->fences, cursor->index++, | |
380 | cursor->obj, &cursor->fence, | |
381 | &cursor->fence_usage); | |
c921ff37 | 382 | cursor->fence = dma_fence_get_rcu(cursor->fence); |
8f94eda3 CK |
383 | if (!cursor->fence) { |
384 | dma_resv_iter_restart_unlocked(cursor); | |
385 | continue; | |
386 | } | |
047a1b87 CK |
387 | |
388 | if (!dma_fence_is_signaled(cursor->fence) && | |
389 | cursor->usage >= cursor->fence_usage) | |
c921ff37 CK |
390 | break; |
391 | } while (true); | |
392 | } | |
393 | ||
394 | /** | |
395 | * dma_resv_iter_first_unlocked - first fence in an unlocked dma_resv obj. | |
396 | * @cursor: the cursor with the current position | |
397 | * | |
d80976d9 DV |
398 | * Subsequent fences are iterated with dma_resv_iter_next_unlocked(). |
399 | * | |
400 | * Beware that the iterator can be restarted. Code which accumulates statistics | |
401 | * or similar needs to check for this with dma_resv_iter_is_restarted(). For | |
402 | * this reason prefer the locked dma_resv_iter_first() whenver possible. | |
403 | * | |
c921ff37 CK |
404 | * Returns the first fence from an unlocked dma_resv obj. |
405 | */ | |
406 | struct dma_fence *dma_resv_iter_first_unlocked(struct dma_resv_iter *cursor) | |
407 | { | |
408 | rcu_read_lock(); | |
409 | do { | |
410 | dma_resv_iter_restart_unlocked(cursor); | |
411 | dma_resv_iter_walk_unlocked(cursor); | |
8f94eda3 | 412 | } while (dma_resv_fences_list(cursor->obj) != cursor->fences); |
c921ff37 CK |
413 | rcu_read_unlock(); |
414 | ||
415 | return cursor->fence; | |
416 | } | |
417 | EXPORT_SYMBOL(dma_resv_iter_first_unlocked); | |
418 | ||
419 | /** | |
420 | * dma_resv_iter_next_unlocked - next fence in an unlocked dma_resv obj. | |
421 | * @cursor: the cursor with the current position | |
422 | * | |
d80976d9 DV |
423 | * Beware that the iterator can be restarted. Code which accumulates statistics |
424 | * or similar needs to check for this with dma_resv_iter_is_restarted(). For | |
425 | * this reason prefer the locked dma_resv_iter_next() whenver possible. | |
426 | * | |
c921ff37 CK |
427 | * Returns the next fence from an unlocked dma_resv obj. |
428 | */ | |
429 | struct dma_fence *dma_resv_iter_next_unlocked(struct dma_resv_iter *cursor) | |
430 | { | |
431 | bool restart; | |
432 | ||
433 | rcu_read_lock(); | |
434 | cursor->is_restarted = false; | |
8f94eda3 | 435 | restart = dma_resv_fences_list(cursor->obj) != cursor->fences; |
c921ff37 CK |
436 | do { |
437 | if (restart) | |
438 | dma_resv_iter_restart_unlocked(cursor); | |
439 | dma_resv_iter_walk_unlocked(cursor); | |
440 | restart = true; | |
8f94eda3 | 441 | } while (dma_resv_fences_list(cursor->obj) != cursor->fences); |
c921ff37 CK |
442 | rcu_read_unlock(); |
443 | ||
444 | return cursor->fence; | |
445 | } | |
446 | EXPORT_SYMBOL(dma_resv_iter_next_unlocked); | |
447 | ||
5baaac31 CK |
448 | /** |
449 | * dma_resv_iter_first - first fence from a locked dma_resv object | |
450 | * @cursor: cursor to record the current position | |
451 | * | |
d80976d9 DV |
452 | * Subsequent fences are iterated with dma_resv_iter_next_unlocked(). |
453 | * | |
5baaac31 CK |
454 | * Return the first fence in the dma_resv object while holding the |
455 | * &dma_resv.lock. | |
456 | */ | |
457 | struct dma_fence *dma_resv_iter_first(struct dma_resv_iter *cursor) | |
458 | { | |
459 | struct dma_fence *fence; | |
460 | ||
461 | dma_resv_assert_held(cursor->obj); | |
462 | ||
463 | cursor->index = 0; | |
047a1b87 | 464 | cursor->fences = dma_resv_fences_list(cursor->obj); |
5baaac31 | 465 | |
047a1b87 | 466 | fence = dma_resv_iter_next(cursor); |
5baaac31 CK |
467 | cursor->is_restarted = true; |
468 | return fence; | |
469 | } | |
470 | EXPORT_SYMBOL_GPL(dma_resv_iter_first); | |
471 | ||
472 | /** | |
473 | * dma_resv_iter_next - next fence from a locked dma_resv object | |
474 | * @cursor: cursor to record the current position | |
475 | * | |
476 | * Return the next fences from the dma_resv object while holding the | |
477 | * &dma_resv.lock. | |
478 | */ | |
479 | struct dma_fence *dma_resv_iter_next(struct dma_resv_iter *cursor) | |
480 | { | |
047a1b87 | 481 | struct dma_fence *fence; |
5baaac31 CK |
482 | |
483 | dma_resv_assert_held(cursor->obj); | |
484 | ||
485 | cursor->is_restarted = false; | |
5baaac31 | 486 | |
047a1b87 CK |
487 | do { |
488 | if (!cursor->fences || | |
489 | cursor->index >= cursor->fences->num_fences) | |
490 | return NULL; | |
491 | ||
492 | dma_resv_list_entry(cursor->fences, cursor->index++, | |
493 | cursor->obj, &fence, &cursor->fence_usage); | |
494 | } while (cursor->fence_usage > cursor->usage); | |
495 | ||
496 | return fence; | |
5baaac31 CK |
497 | } |
498 | EXPORT_SYMBOL_GPL(dma_resv_iter_next); | |
499 | ||
7faf952a | 500 | /** |
068d9d75 CK |
501 | * dma_resv_copy_fences - Copy all fences from src to dst. |
502 | * @dst: the destination reservation object | |
503 | * @src: the source reservation object | |
504 | * | |
505 | * Copy all fences from src to dst. dst-lock must be held. | |
506 | */ | |
52791eee | 507 | int dma_resv_copy_fences(struct dma_resv *dst, struct dma_resv *src) |
7faf952a | 508 | { |
96601e8a CK |
509 | struct dma_resv_iter cursor; |
510 | struct dma_resv_list *list; | |
047a1b87 | 511 | struct dma_fence *f; |
7faf952a | 512 | |
52791eee | 513 | dma_resv_assert_held(dst); |
547c7138 | 514 | |
96601e8a | 515 | list = NULL; |
7faf952a | 516 | |
0cc848a7 | 517 | dma_resv_iter_begin(&cursor, src, DMA_RESV_USAGE_BOOKKEEP); |
96601e8a | 518 | dma_resv_for_each_fence_unlocked(&cursor, f) { |
b016cd6e | 519 | |
96601e8a CK |
520 | if (dma_resv_iter_is_restarted(&cursor)) { |
521 | dma_resv_list_free(list); | |
39e16ba1 | 522 | |
047a1b87 CK |
523 | list = dma_resv_list_alloc(cursor.num_fences); |
524 | if (!list) { | |
525 | dma_resv_iter_end(&cursor); | |
526 | return -ENOMEM; | |
39e16ba1 | 527 | } |
047a1b87 | 528 | list->num_fences = 0; |
39e16ba1 | 529 | } |
39e16ba1 | 530 | |
96601e8a | 531 | dma_fence_get(f); |
047a1b87 CK |
532 | dma_resv_list_set(list, list->num_fences++, f, |
533 | dma_resv_iter_usage(&cursor)); | |
96601e8a CK |
534 | } |
535 | dma_resv_iter_end(&cursor); | |
7faf952a | 536 | |
047a1b87 | 537 | list = rcu_replace_pointer(dst->fences, list, dma_resv_held(dst)); |
96601e8a | 538 | dma_resv_list_free(list); |
7faf952a CK |
539 | return 0; |
540 | } | |
52791eee | 541 | EXPORT_SYMBOL(dma_resv_copy_fences); |
7faf952a | 542 | |
dad6c394 | 543 | /** |
047a1b87 | 544 | * dma_resv_get_fences - Get an object's fences |
dad6c394 RC |
545 | * fences without update side lock held |
546 | * @obj: the reservation object | |
7bc80a54 | 547 | * @usage: controls which fences to include, see enum dma_resv_usage. |
75ab2b36 CK |
548 | * @num_fences: the number of fences returned |
549 | * @fences: the array of fence ptrs returned (array is krealloc'd to the | |
550 | * required size, and must be freed by caller) | |
dad6c394 | 551 | * |
75ab2b36 CK |
552 | * Retrieve all fences from the reservation object. |
553 | * Returns either zero or -ENOMEM. | |
dad6c394 | 554 | */ |
7bc80a54 | 555 | int dma_resv_get_fences(struct dma_resv *obj, enum dma_resv_usage usage, |
75ab2b36 | 556 | unsigned int *num_fences, struct dma_fence ***fences) |
3c3b177a | 557 | { |
d3c80698 CK |
558 | struct dma_resv_iter cursor; |
559 | struct dma_fence *fence; | |
fedf5413 | 560 | |
75ab2b36 CK |
561 | *num_fences = 0; |
562 | *fences = NULL; | |
3c3b177a | 563 | |
7bc80a54 | 564 | dma_resv_iter_begin(&cursor, obj, usage); |
d3c80698 | 565 | dma_resv_for_each_fence_unlocked(&cursor, fence) { |
a35f2f34 | 566 | |
d3c80698 CK |
567 | if (dma_resv_iter_is_restarted(&cursor)) { |
568 | unsigned int count; | |
3c3b177a | 569 | |
75ab2b36 CK |
570 | while (*num_fences) |
571 | dma_fence_put((*fences)[--(*num_fences)]); | |
f5b07b04 | 572 | |
047a1b87 | 573 | count = cursor.num_fences + 1; |
3c3b177a | 574 | |
d3c80698 | 575 | /* Eventually re-allocate the array */ |
75ab2b36 | 576 | *fences = krealloc_array(*fences, count, |
d3c80698 CK |
577 | sizeof(void *), |
578 | GFP_KERNEL); | |
75ab2b36 | 579 | if (count && !*fences) { |
d3c80698 CK |
580 | dma_resv_iter_end(&cursor); |
581 | return -ENOMEM; | |
3c3b177a | 582 | } |
fedf5413 | 583 | } |
3c3b177a | 584 | |
75ab2b36 | 585 | (*fences)[(*num_fences)++] = dma_fence_get(fence); |
3c3b177a | 586 | } |
d3c80698 | 587 | dma_resv_iter_end(&cursor); |
fedf5413 | 588 | |
d3c80698 | 589 | return 0; |
3c3b177a | 590 | } |
d3fae3b3 | 591 | EXPORT_SYMBOL_GPL(dma_resv_get_fences); |
3c3b177a | 592 | |
92cedee6 CK |
593 | /** |
594 | * dma_resv_get_singleton - Get a single fence for all the fences | |
595 | * @obj: the reservation object | |
7bc80a54 | 596 | * @usage: controls which fences to include, see enum dma_resv_usage. |
92cedee6 CK |
597 | * @fence: the resulting fence |
598 | * | |
599 | * Get a single fence representing all the fences inside the resv object. | |
600 | * Returns either 0 for success or -ENOMEM. | |
601 | * | |
602 | * Warning: This can't be used like this when adding the fence back to the resv | |
603 | * object since that can lead to stack corruption when finalizing the | |
604 | * dma_fence_array. | |
605 | * | |
606 | * Returns 0 on success and negative error values on failure. | |
607 | */ | |
7bc80a54 | 608 | int dma_resv_get_singleton(struct dma_resv *obj, enum dma_resv_usage usage, |
92cedee6 CK |
609 | struct dma_fence **fence) |
610 | { | |
611 | struct dma_fence_array *array; | |
612 | struct dma_fence **fences; | |
613 | unsigned count; | |
614 | int r; | |
615 | ||
7bc80a54 | 616 | r = dma_resv_get_fences(obj, usage, &count, &fences); |
92cedee6 CK |
617 | if (r) |
618 | return r; | |
619 | ||
620 | if (count == 0) { | |
621 | *fence = NULL; | |
622 | return 0; | |
623 | } | |
624 | ||
625 | if (count == 1) { | |
626 | *fence = fences[0]; | |
627 | kfree(fences); | |
628 | return 0; | |
629 | } | |
630 | ||
631 | array = dma_fence_array_create(count, fences, | |
632 | dma_fence_context_alloc(1), | |
633 | 1, false); | |
634 | if (!array) { | |
635 | while (count--) | |
636 | dma_fence_put(fences[count]); | |
637 | kfree(fences); | |
638 | return -ENOMEM; | |
639 | } | |
640 | ||
641 | *fence = &array->base; | |
642 | return 0; | |
643 | } | |
644 | EXPORT_SYMBOL_GPL(dma_resv_get_singleton); | |
645 | ||
dad6c394 | 646 | /** |
047a1b87 | 647 | * dma_resv_wait_timeout - Wait on reservation's objects fences |
dad6c394 | 648 | * @obj: the reservation object |
7bc80a54 | 649 | * @usage: controls which fences to include, see enum dma_resv_usage. |
dad6c394 RC |
650 | * @intr: if true, do interruptible wait |
651 | * @timeout: timeout value in jiffies or zero to return immediately | |
652 | * | |
d3fae3b3 CK |
653 | * Callers are not required to hold specific locks, but maybe hold |
654 | * dma_resv_lock() already | |
dad6c394 RC |
655 | * RETURNS |
656 | * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or | |
657 | * greater than zer on success. | |
658 | */ | |
7bc80a54 CK |
659 | long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage, |
660 | bool intr, unsigned long timeout) | |
3c3b177a | 661 | { |
06a66b5c | 662 | long ret = timeout ? timeout : 1; |
ada5c48b | 663 | struct dma_resv_iter cursor; |
068d9d75 | 664 | struct dma_fence *fence; |
3c3b177a | 665 | |
7bc80a54 | 666 | dma_resv_iter_begin(&cursor, obj, usage); |
ada5c48b | 667 | dma_resv_for_each_fence_unlocked(&cursor, fence) { |
3c3b177a | 668 | |
ada5c48b CK |
669 | ret = dma_fence_wait_timeout(fence, intr, ret); |
670 | if (ret <= 0) { | |
671 | dma_resv_iter_end(&cursor); | |
672 | return ret; | |
3c3b177a ML |
673 | } |
674 | } | |
ada5c48b | 675 | dma_resv_iter_end(&cursor); |
3c3b177a | 676 | |
3c3b177a | 677 | return ret; |
3c3b177a | 678 | } |
d3fae3b3 | 679 | EXPORT_SYMBOL_GPL(dma_resv_wait_timeout); |
3c3b177a ML |
680 | |
681 | ||
dad6c394 | 682 | /** |
d3fae3b3 CK |
683 | * dma_resv_test_signaled - Test if a reservation object's fences have been |
684 | * signaled. | |
dad6c394 | 685 | * @obj: the reservation object |
7bc80a54 | 686 | * @usage: controls which fences to include, see enum dma_resv_usage. |
dad6c394 | 687 | * |
d3fae3b3 | 688 | * Callers are not required to hold specific locks, but maybe hold |
d9edf92d DV |
689 | * dma_resv_lock() already. |
690 | * | |
dad6c394 | 691 | * RETURNS |
d9edf92d DV |
692 | * |
693 | * True if all fences signaled, else false. | |
dad6c394 | 694 | */ |
7bc80a54 | 695 | bool dma_resv_test_signaled(struct dma_resv *obj, enum dma_resv_usage usage) |
3c3b177a | 696 | { |
7fa828cb | 697 | struct dma_resv_iter cursor; |
9d38814d | 698 | struct dma_fence *fence; |
b016cd6e | 699 | |
7bc80a54 | 700 | dma_resv_iter_begin(&cursor, obj, usage); |
7fa828cb CK |
701 | dma_resv_for_each_fence_unlocked(&cursor, fence) { |
702 | dma_resv_iter_end(&cursor); | |
703 | return false; | |
b016cd6e | 704 | } |
7fa828cb CK |
705 | dma_resv_iter_end(&cursor); |
706 | return true; | |
3c3b177a | 707 | } |
d3fae3b3 | 708 | EXPORT_SYMBOL_GPL(dma_resv_test_signaled); |
068d9d75 | 709 | |
a25efb38 CK |
710 | /** |
711 | * dma_resv_describe - Dump description of the resv object into seq_file | |
712 | * @obj: the reservation object | |
713 | * @seq: the seq_file to dump the description into | |
714 | * | |
715 | * Dump a textual description of the fences inside an dma_resv object into the | |
716 | * seq_file. | |
717 | */ | |
718 | void dma_resv_describe(struct dma_resv *obj, struct seq_file *seq) | |
719 | { | |
0cc848a7 | 720 | static const char *usage[] = { "kernel", "write", "read", "bookkeep" }; |
a25efb38 CK |
721 | struct dma_resv_iter cursor; |
722 | struct dma_fence *fence; | |
723 | ||
7bc80a54 | 724 | dma_resv_for_each_fence(&cursor, obj, DMA_RESV_USAGE_READ, fence) { |
a25efb38 | 725 | seq_printf(seq, "\t%s fence:", |
73511edf | 726 | usage[dma_resv_iter_usage(&cursor)]); |
a25efb38 CK |
727 | dma_fence_describe(fence, seq); |
728 | } | |
729 | } | |
730 | EXPORT_SYMBOL_GPL(dma_resv_describe); | |
731 | ||
068d9d75 CK |
732 | #if IS_ENABLED(CONFIG_LOCKDEP) |
733 | static int __init dma_resv_lockdep(void) | |
734 | { | |
735 | struct mm_struct *mm = mm_alloc(); | |
736 | struct ww_acquire_ctx ctx; | |
737 | struct dma_resv obj; | |
738 | struct address_space mapping; | |
739 | int ret; | |
740 | ||
741 | if (!mm) | |
742 | return -ENOMEM; | |
743 | ||
744 | dma_resv_init(&obj); | |
745 | address_space_init_once(&mapping); | |
746 | ||
747 | mmap_read_lock(mm); | |
748 | ww_acquire_init(&ctx, &reservation_ww_class); | |
749 | ret = dma_resv_lock(&obj, &ctx); | |
750 | if (ret == -EDEADLK) | |
751 | dma_resv_lock_slow(&obj, &ctx); | |
752 | fs_reclaim_acquire(GFP_KERNEL); | |
753 | /* for unmap_mapping_range on trylocked buffer objects in shrinkers */ | |
754 | i_mmap_lock_write(&mapping); | |
755 | i_mmap_unlock_write(&mapping); | |
756 | #ifdef CONFIG_MMU_NOTIFIER | |
757 | lock_map_acquire(&__mmu_notifier_invalidate_range_start_map); | |
758 | __dma_fence_might_wait(); | |
759 | lock_map_release(&__mmu_notifier_invalidate_range_start_map); | |
760 | #else | |
761 | __dma_fence_might_wait(); | |
762 | #endif | |
763 | fs_reclaim_release(GFP_KERNEL); | |
764 | ww_mutex_unlock(&obj.lock); | |
765 | ww_acquire_fini(&ctx); | |
766 | mmap_read_unlock(mm); | |
767 | ||
768 | mmput(mm); | |
769 | ||
770 | return 0; | |
771 | } | |
772 | subsys_initcall(dma_resv_lockdep); | |
773 | #endif |