Commit | Line | Data |
---|---|---|
786d7257 | 1 | /* |
04a5faa8 | 2 | * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst) |
786d7257 ML |
3 | * |
4 | * Based on bo.c which bears the following copyright notice, | |
5 | * but is dual licensed: | |
6 | * | |
7 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA | |
8 | * All Rights Reserved. | |
9 | * | |
10 | * Permission is hereby granted, free of charge, to any person obtaining a | |
11 | * copy of this software and associated documentation files (the | |
12 | * "Software"), to deal in the Software without restriction, including | |
13 | * without limitation the rights to use, copy, modify, merge, publish, | |
14 | * distribute, sub license, and/or sell copies of the Software, and to | |
15 | * permit persons to whom the Software is furnished to do so, subject to | |
16 | * the following conditions: | |
17 | * | |
18 | * The above copyright notice and this permission notice (including the | |
19 | * next paragraph) shall be included in all copies or substantial portions | |
20 | * of the Software. | |
21 | * | |
22 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
23 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
24 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
25 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
26 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
27 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
28 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
29 | * | |
30 | **************************************************************************/ | |
31 | /* | |
32 | * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> | |
33 | */ | |
34 | ||
35 | #include <linux/reservation.h> | |
36 | #include <linux/export.h> | |
37 | ||
dad6c394 RC |
38 | /** |
39 | * DOC: Reservation Object Overview | |
40 | * | |
41 | * The reservation object provides a mechanism to manage shared and | |
42 | * exclusive fences associated with a buffer. A reservation object | |
43 | * can have attached one exclusive fence (normally associated with | |
44 | * write operations) or N shared fences (read operations). The RCU | |
45 | * mechanism is used to protect read access to fences from locked | |
46 | * write-side updates. | |
47 | */ | |
48 | ||
08295b3b | 49 | DEFINE_WD_CLASS(reservation_ww_class); |
786d7257 | 50 | EXPORT_SYMBOL(reservation_ww_class); |
04a5faa8 | 51 | |
3c3b177a ML |
52 | struct lock_class_key reservation_seqcount_class; |
53 | EXPORT_SYMBOL(reservation_seqcount_class); | |
54 | ||
55 | const char reservation_seqcount_string[] = "reservation_seqcount"; | |
56 | EXPORT_SYMBOL(reservation_seqcount_string); | |
dad6c394 RC |
57 | |
58 | /** | |
ca05359f CK |
59 | * reservation_object_reserve_shared - Reserve space to add shared fences to |
60 | * a reservation_object. | |
dad6c394 | 61 | * @obj: reservation object |
ca05359f | 62 | * @num_fences: number of fences we want to add |
dad6c394 RC |
63 | * |
64 | * Should be called before reservation_object_add_shared_fence(). Must | |
65 | * be called with obj->lock held. | |
66 | * | |
67 | * RETURNS | |
68 | * Zero for success, or -errno | |
04a5faa8 | 69 | */ |
ca05359f CK |
70 | int reservation_object_reserve_shared(struct reservation_object *obj, |
71 | unsigned int num_fences) | |
04a5faa8 | 72 | { |
27836b64 CK |
73 | struct reservation_object_list *old, *new; |
74 | unsigned int i, j, k, max; | |
04a5faa8 | 75 | |
547c7138 LS |
76 | reservation_object_assert_held(obj); |
77 | ||
04a5faa8 ML |
78 | old = reservation_object_get_list(obj); |
79 | ||
80 | if (old && old->shared_max) { | |
ca05359f | 81 | if ((old->shared_count + num_fences) <= old->shared_max) |
04a5faa8 | 82 | return 0; |
27836b64 | 83 | else |
ca05359f CK |
84 | max = max(old->shared_count + num_fences, |
85 | old->shared_max * 2); | |
ca25fe5e | 86 | } else { |
27836b64 | 87 | max = 4; |
ca25fe5e | 88 | } |
3c3b177a | 89 | |
27836b64 CK |
90 | new = kmalloc(offsetof(typeof(*new), shared[max]), GFP_KERNEL); |
91 | if (!new) | |
92 | return -ENOMEM; | |
04a5faa8 ML |
93 | |
94 | /* | |
95 | * no need to bump fence refcounts, rcu_read access | |
96 | * requires the use of kref_get_unless_zero, and the | |
97 | * references from the old struct are carried over to | |
98 | * the new. | |
99 | */ | |
27836b64 CK |
100 | for (i = 0, j = 0, k = max; i < (old ? old->shared_count : 0); ++i) { |
101 | struct dma_fence *fence; | |
3c3b177a | 102 | |
27836b64 CK |
103 | fence = rcu_dereference_protected(old->shared[i], |
104 | reservation_object_held(obj)); | |
105 | if (dma_fence_is_signaled(fence)) | |
106 | RCU_INIT_POINTER(new->shared[--k], fence); | |
4d9c62e8 | 107 | else |
27836b64 | 108 | RCU_INIT_POINTER(new->shared[j++], fence); |
04a5faa8 | 109 | } |
27836b64 CK |
110 | new->shared_count = j; |
111 | new->shared_max = max; | |
04a5faa8 | 112 | |
3c3b177a ML |
113 | preempt_disable(); |
114 | write_seqcount_begin(&obj->seq); | |
115 | /* | |
116 | * RCU_INIT_POINTER can be used here, | |
117 | * seqcount provides the necessary barriers | |
118 | */ | |
27836b64 | 119 | RCU_INIT_POINTER(obj->fence, new); |
3c3b177a ML |
120 | write_seqcount_end(&obj->seq); |
121 | preempt_enable(); | |
122 | ||
4d9c62e8 | 123 | if (!old) |
27836b64 | 124 | return 0; |
3c3b177a | 125 | |
4d9c62e8 | 126 | /* Drop the references to the signaled fences */ |
27836b64 CK |
127 | for (i = k; i < new->shared_max; ++i) { |
128 | struct dma_fence *fence; | |
4d9c62e8 | 129 | |
27836b64 CK |
130 | fence = rcu_dereference_protected(new->shared[i], |
131 | reservation_object_held(obj)); | |
132 | dma_fence_put(fence); | |
4d9c62e8 CK |
133 | } |
134 | kfree_rcu(old, rcu); | |
27836b64 CK |
135 | |
136 | return 0; | |
04a5faa8 | 137 | } |
27836b64 | 138 | EXPORT_SYMBOL(reservation_object_reserve_shared); |
04a5faa8 | 139 | |
dad6c394 RC |
140 | /** |
141 | * reservation_object_add_shared_fence - Add a fence to a shared slot | |
142 | * @obj: the reservation object | |
143 | * @fence: the shared fence to add | |
144 | * | |
04a5faa8 | 145 | * Add a fence to a shared slot, obj->lock must be held, and |
f5bef0b8 | 146 | * reservation_object_reserve_shared() has been called. |
04a5faa8 ML |
147 | */ |
148 | void reservation_object_add_shared_fence(struct reservation_object *obj, | |
f54d1867 | 149 | struct dma_fence *fence) |
04a5faa8 | 150 | { |
27836b64 | 151 | struct reservation_object_list *fobj; |
a590d0fd | 152 | unsigned int i, count; |
04a5faa8 | 153 | |
27836b64 CK |
154 | dma_fence_get(fence); |
155 | ||
547c7138 LS |
156 | reservation_object_assert_held(obj); |
157 | ||
27836b64 | 158 | fobj = reservation_object_get_list(obj); |
a590d0fd | 159 | count = fobj->shared_count; |
04a5faa8 | 160 | |
27836b64 CK |
161 | preempt_disable(); |
162 | write_seqcount_begin(&obj->seq); | |
163 | ||
a590d0fd | 164 | for (i = 0; i < count; ++i) { |
27836b64 CK |
165 | struct dma_fence *old_fence; |
166 | ||
167 | old_fence = rcu_dereference_protected(fobj->shared[i], | |
168 | reservation_object_held(obj)); | |
169 | if (old_fence->context == fence->context || | |
170 | dma_fence_is_signaled(old_fence)) { | |
171 | dma_fence_put(old_fence); | |
172 | goto replace; | |
173 | } | |
174 | } | |
175 | ||
176 | BUG_ON(fobj->shared_count >= fobj->shared_max); | |
a590d0fd | 177 | count++; |
27836b64 CK |
178 | |
179 | replace: | |
27836b64 | 180 | RCU_INIT_POINTER(fobj->shared[i], fence); |
a590d0fd CW |
181 | /* pointer update must be visible before we extend the shared_count */ |
182 | smp_store_mb(fobj->shared_count, count); | |
183 | ||
27836b64 CK |
184 | write_seqcount_end(&obj->seq); |
185 | preempt_enable(); | |
04a5faa8 ML |
186 | } |
187 | EXPORT_SYMBOL(reservation_object_add_shared_fence); | |
188 | ||
dad6c394 RC |
189 | /** |
190 | * reservation_object_add_excl_fence - Add an exclusive fence. | |
191 | * @obj: the reservation object | |
192 | * @fence: the shared fence to add | |
193 | * | |
194 | * Add a fence to the exclusive slot. The obj->lock must be held. | |
195 | */ | |
04a5faa8 | 196 | void reservation_object_add_excl_fence(struct reservation_object *obj, |
f54d1867 | 197 | struct dma_fence *fence) |
04a5faa8 | 198 | { |
f54d1867 | 199 | struct dma_fence *old_fence = reservation_object_get_excl(obj); |
04a5faa8 ML |
200 | struct reservation_object_list *old; |
201 | u32 i = 0; | |
202 | ||
547c7138 LS |
203 | reservation_object_assert_held(obj); |
204 | ||
04a5faa8 | 205 | old = reservation_object_get_list(obj); |
3c3b177a | 206 | if (old) |
04a5faa8 | 207 | i = old->shared_count; |
04a5faa8 ML |
208 | |
209 | if (fence) | |
f54d1867 | 210 | dma_fence_get(fence); |
04a5faa8 | 211 | |
3c3b177a ML |
212 | preempt_disable(); |
213 | write_seqcount_begin(&obj->seq); | |
214 | /* write_seqcount_begin provides the necessary memory barrier */ | |
215 | RCU_INIT_POINTER(obj->fence_excl, fence); | |
216 | if (old) | |
217 | old->shared_count = 0; | |
218 | write_seqcount_end(&obj->seq); | |
219 | preempt_enable(); | |
04a5faa8 ML |
220 | |
221 | /* inplace update, no shared fences */ | |
222 | while (i--) | |
f54d1867 | 223 | dma_fence_put(rcu_dereference_protected(old->shared[i], |
3c3b177a | 224 | reservation_object_held(obj))); |
04a5faa8 | 225 | |
f3e31b73 | 226 | dma_fence_put(old_fence); |
04a5faa8 ML |
227 | } |
228 | EXPORT_SYMBOL(reservation_object_add_excl_fence); | |
3c3b177a | 229 | |
7faf952a CK |
230 | /** |
231 | * reservation_object_copy_fences - Copy all fences from src to dst. | |
232 | * @dst: the destination reservation object | |
233 | * @src: the source reservation object | |
234 | * | |
39e16ba1 | 235 | * Copy all fences from src to dst. dst-lock must be held. |
7faf952a CK |
236 | */ |
237 | int reservation_object_copy_fences(struct reservation_object *dst, | |
238 | struct reservation_object *src) | |
239 | { | |
240 | struct reservation_object_list *src_list, *dst_list; | |
241 | struct dma_fence *old, *new; | |
242 | size_t size; | |
243 | unsigned i; | |
244 | ||
547c7138 LS |
245 | reservation_object_assert_held(dst); |
246 | ||
39e16ba1 CK |
247 | rcu_read_lock(); |
248 | src_list = rcu_dereference(src->fence); | |
7faf952a | 249 | |
39e16ba1 | 250 | retry: |
7faf952a | 251 | if (src_list) { |
39e16ba1 CK |
252 | unsigned shared_count = src_list->shared_count; |
253 | ||
254 | size = offsetof(typeof(*src_list), shared[shared_count]); | |
255 | rcu_read_unlock(); | |
256 | ||
7faf952a CK |
257 | dst_list = kmalloc(size, GFP_KERNEL); |
258 | if (!dst_list) | |
259 | return -ENOMEM; | |
260 | ||
39e16ba1 CK |
261 | rcu_read_lock(); |
262 | src_list = rcu_dereference(src->fence); | |
263 | if (!src_list || src_list->shared_count > shared_count) { | |
264 | kfree(dst_list); | |
265 | goto retry; | |
266 | } | |
267 | ||
268 | dst_list->shared_count = 0; | |
269 | dst_list->shared_max = shared_count; | |
270 | for (i = 0; i < src_list->shared_count; ++i) { | |
271 | struct dma_fence *fence; | |
272 | ||
273 | fence = rcu_dereference(src_list->shared[i]); | |
274 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, | |
275 | &fence->flags)) | |
276 | continue; | |
277 | ||
278 | if (!dma_fence_get_rcu(fence)) { | |
279 | kfree(dst_list); | |
280 | src_list = rcu_dereference(src->fence); | |
281 | goto retry; | |
282 | } | |
283 | ||
284 | if (dma_fence_is_signaled(fence)) { | |
285 | dma_fence_put(fence); | |
286 | continue; | |
287 | } | |
288 | ||
ad46d7b8 | 289 | rcu_assign_pointer(dst_list->shared[dst_list->shared_count++], fence); |
39e16ba1 | 290 | } |
7faf952a CK |
291 | } else { |
292 | dst_list = NULL; | |
293 | } | |
294 | ||
39e16ba1 CK |
295 | new = dma_fence_get_rcu_safe(&src->fence_excl); |
296 | rcu_read_unlock(); | |
297 | ||
7faf952a | 298 | src_list = reservation_object_get_list(dst); |
7faf952a | 299 | old = reservation_object_get_excl(dst); |
7faf952a CK |
300 | |
301 | preempt_disable(); | |
302 | write_seqcount_begin(&dst->seq); | |
303 | /* write_seqcount_begin provides the necessary memory barrier */ | |
304 | RCU_INIT_POINTER(dst->fence_excl, new); | |
305 | RCU_INIT_POINTER(dst->fence, dst_list); | |
306 | write_seqcount_end(&dst->seq); | |
307 | preempt_enable(); | |
308 | ||
309 | if (src_list) | |
310 | kfree_rcu(src_list, rcu); | |
311 | dma_fence_put(old); | |
312 | ||
313 | return 0; | |
314 | } | |
315 | EXPORT_SYMBOL(reservation_object_copy_fences); | |
316 | ||
dad6c394 RC |
317 | /** |
318 | * reservation_object_get_fences_rcu - Get an object's shared and exclusive | |
319 | * fences without update side lock held | |
320 | * @obj: the reservation object | |
321 | * @pfence_excl: the returned exclusive fence (or NULL) | |
322 | * @pshared_count: the number of shared fences returned | |
323 | * @pshared: the array of shared fence ptrs returned (array is krealloc'd to | |
324 | * the required size, and must be freed by caller) | |
325 | * | |
a35f2f34 CK |
326 | * Retrieve all fences from the reservation object. If the pointer for the |
327 | * exclusive fence is not specified the fence is put into the array of the | |
328 | * shared fences as well. Returns either zero or -ENOMEM. | |
dad6c394 | 329 | */ |
3c3b177a | 330 | int reservation_object_get_fences_rcu(struct reservation_object *obj, |
f54d1867 | 331 | struct dma_fence **pfence_excl, |
3c3b177a | 332 | unsigned *pshared_count, |
f54d1867 | 333 | struct dma_fence ***pshared) |
3c3b177a | 334 | { |
f54d1867 CW |
335 | struct dma_fence **shared = NULL; |
336 | struct dma_fence *fence_excl; | |
fedf5413 CW |
337 | unsigned int shared_count; |
338 | int ret = 1; | |
3c3b177a | 339 | |
fedf5413 | 340 | do { |
3c3b177a | 341 | struct reservation_object_list *fobj; |
a35f2f34 CK |
342 | unsigned int i, seq; |
343 | size_t sz = 0; | |
3c3b177a | 344 | |
fedf5413 | 345 | shared_count = i = 0; |
3c3b177a ML |
346 | |
347 | rcu_read_lock(); | |
fedf5413 CW |
348 | seq = read_seqcount_begin(&obj->seq); |
349 | ||
350 | fence_excl = rcu_dereference(obj->fence_excl); | |
f54d1867 | 351 | if (fence_excl && !dma_fence_get_rcu(fence_excl)) |
fedf5413 | 352 | goto unlock; |
3c3b177a ML |
353 | |
354 | fobj = rcu_dereference(obj->fence); | |
a35f2f34 CK |
355 | if (fobj) |
356 | sz += sizeof(*shared) * fobj->shared_max; | |
357 | ||
358 | if (!pfence_excl && fence_excl) | |
359 | sz += sizeof(*shared); | |
360 | ||
361 | if (sz) { | |
f54d1867 | 362 | struct dma_fence **nshared; |
3c3b177a ML |
363 | |
364 | nshared = krealloc(shared, sz, | |
365 | GFP_NOWAIT | __GFP_NOWARN); | |
366 | if (!nshared) { | |
367 | rcu_read_unlock(); | |
368 | nshared = krealloc(shared, sz, GFP_KERNEL); | |
369 | if (nshared) { | |
370 | shared = nshared; | |
371 | continue; | |
372 | } | |
373 | ||
374 | ret = -ENOMEM; | |
3c3b177a ML |
375 | break; |
376 | } | |
377 | shared = nshared; | |
a35f2f34 | 378 | shared_count = fobj ? fobj->shared_count : 0; |
3c3b177a | 379 | for (i = 0; i < shared_count; ++i) { |
fedf5413 | 380 | shared[i] = rcu_dereference(fobj->shared[i]); |
f54d1867 | 381 | if (!dma_fence_get_rcu(shared[i])) |
fedf5413 | 382 | break; |
3c3b177a | 383 | } |
a35f2f34 CK |
384 | |
385 | if (!pfence_excl && fence_excl) { | |
386 | shared[i] = fence_excl; | |
387 | fence_excl = NULL; | |
388 | ++i; | |
389 | ++shared_count; | |
390 | } | |
fedf5413 | 391 | } |
3c3b177a | 392 | |
fedf5413 CW |
393 | if (i != shared_count || read_seqcount_retry(&obj->seq, seq)) { |
394 | while (i--) | |
f54d1867 CW |
395 | dma_fence_put(shared[i]); |
396 | dma_fence_put(fence_excl); | |
fedf5413 CW |
397 | goto unlock; |
398 | } | |
399 | ||
400 | ret = 0; | |
3c3b177a ML |
401 | unlock: |
402 | rcu_read_unlock(); | |
fedf5413 CW |
403 | } while (ret); |
404 | ||
405 | if (!shared_count) { | |
3c3b177a | 406 | kfree(shared); |
fedf5413 | 407 | shared = NULL; |
3c3b177a | 408 | } |
fedf5413 CW |
409 | |
410 | *pshared_count = shared_count; | |
411 | *pshared = shared; | |
a35f2f34 CK |
412 | if (pfence_excl) |
413 | *pfence_excl = fence_excl; | |
3c3b177a ML |
414 | |
415 | return ret; | |
416 | } | |
417 | EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu); | |
418 | ||
dad6c394 RC |
419 | /** |
420 | * reservation_object_wait_timeout_rcu - Wait on reservation's objects | |
421 | * shared and/or exclusive fences. | |
422 | * @obj: the reservation object | |
423 | * @wait_all: if true, wait on all fences, else wait on just exclusive fence | |
424 | * @intr: if true, do interruptible wait | |
425 | * @timeout: timeout value in jiffies or zero to return immediately | |
426 | * | |
427 | * RETURNS | |
428 | * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or | |
429 | * greater than zer on success. | |
430 | */ | |
3c3b177a ML |
431 | long reservation_object_wait_timeout_rcu(struct reservation_object *obj, |
432 | bool wait_all, bool intr, | |
433 | unsigned long timeout) | |
434 | { | |
f54d1867 | 435 | struct dma_fence *fence; |
5bffee86 | 436 | unsigned seq, shared_count; |
06a66b5c | 437 | long ret = timeout ? timeout : 1; |
5bffee86 | 438 | int i; |
fb8b7d2b | 439 | |
3c3b177a | 440 | retry: |
3c3b177a ML |
441 | shared_count = 0; |
442 | seq = read_seqcount_begin(&obj->seq); | |
443 | rcu_read_lock(); | |
5bffee86 | 444 | i = -1; |
3c3b177a | 445 | |
b88fa004 CK |
446 | fence = rcu_dereference(obj->fence_excl); |
447 | if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { | |
448 | if (!dma_fence_get_rcu(fence)) | |
449 | goto unlock_retry; | |
450 | ||
451 | if (dma_fence_is_signaled(fence)) { | |
452 | dma_fence_put(fence); | |
453 | fence = NULL; | |
454 | } | |
455 | ||
456 | } else { | |
457 | fence = NULL; | |
458 | } | |
459 | ||
5bffee86 | 460 | if (wait_all) { |
5136629d JT |
461 | struct reservation_object_list *fobj = |
462 | rcu_dereference(obj->fence); | |
3c3b177a ML |
463 | |
464 | if (fobj) | |
465 | shared_count = fobj->shared_count; | |
466 | ||
5bffee86 | 467 | for (i = 0; !fence && i < shared_count; ++i) { |
f54d1867 | 468 | struct dma_fence *lfence = rcu_dereference(fobj->shared[i]); |
3c3b177a | 469 | |
f54d1867 CW |
470 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, |
471 | &lfence->flags)) | |
3c3b177a ML |
472 | continue; |
473 | ||
f54d1867 | 474 | if (!dma_fence_get_rcu(lfence)) |
3c3b177a ML |
475 | goto unlock_retry; |
476 | ||
f54d1867 CW |
477 | if (dma_fence_is_signaled(lfence)) { |
478 | dma_fence_put(lfence); | |
3c3b177a ML |
479 | continue; |
480 | } | |
481 | ||
482 | fence = lfence; | |
483 | break; | |
484 | } | |
485 | } | |
486 | ||
3c3b177a ML |
487 | rcu_read_unlock(); |
488 | if (fence) { | |
1cec20f0 | 489 | if (read_seqcount_retry(&obj->seq, seq)) { |
f54d1867 | 490 | dma_fence_put(fence); |
1cec20f0 CW |
491 | goto retry; |
492 | } | |
493 | ||
f54d1867 CW |
494 | ret = dma_fence_wait_timeout(fence, intr, ret); |
495 | dma_fence_put(fence); | |
3c3b177a ML |
496 | if (ret > 0 && wait_all && (i + 1 < shared_count)) |
497 | goto retry; | |
498 | } | |
499 | return ret; | |
500 | ||
501 | unlock_retry: | |
502 | rcu_read_unlock(); | |
503 | goto retry; | |
504 | } | |
505 | EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu); | |
506 | ||
507 | ||
508 | static inline int | |
f54d1867 | 509 | reservation_object_test_signaled_single(struct dma_fence *passed_fence) |
3c3b177a | 510 | { |
f54d1867 | 511 | struct dma_fence *fence, *lfence = passed_fence; |
3c3b177a ML |
512 | int ret = 1; |
513 | ||
f54d1867 CW |
514 | if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) { |
515 | fence = dma_fence_get_rcu(lfence); | |
3c3b177a ML |
516 | if (!fence) |
517 | return -1; | |
518 | ||
f54d1867 CW |
519 | ret = !!dma_fence_is_signaled(fence); |
520 | dma_fence_put(fence); | |
3c3b177a ML |
521 | } |
522 | return ret; | |
523 | } | |
524 | ||
dad6c394 RC |
525 | /** |
526 | * reservation_object_test_signaled_rcu - Test if a reservation object's | |
527 | * fences have been signaled. | |
528 | * @obj: the reservation object | |
529 | * @test_all: if true, test all fences, otherwise only test the exclusive | |
530 | * fence | |
531 | * | |
532 | * RETURNS | |
533 | * true if all fences signaled, else false | |
534 | */ | |
3c3b177a ML |
535 | bool reservation_object_test_signaled_rcu(struct reservation_object *obj, |
536 | bool test_all) | |
537 | { | |
538 | unsigned seq, shared_count; | |
b68d8379 | 539 | int ret; |
3c3b177a | 540 | |
b68d8379 | 541 | rcu_read_lock(); |
3c3b177a | 542 | retry: |
b68d8379 | 543 | ret = true; |
3c3b177a ML |
544 | shared_count = 0; |
545 | seq = read_seqcount_begin(&obj->seq); | |
3c3b177a ML |
546 | |
547 | if (test_all) { | |
548 | unsigned i; | |
549 | ||
5136629d JT |
550 | struct reservation_object_list *fobj = |
551 | rcu_dereference(obj->fence); | |
3c3b177a ML |
552 | |
553 | if (fobj) | |
554 | shared_count = fobj->shared_count; | |
555 | ||
3c3b177a | 556 | for (i = 0; i < shared_count; ++i) { |
f54d1867 | 557 | struct dma_fence *fence = rcu_dereference(fobj->shared[i]); |
3c3b177a ML |
558 | |
559 | ret = reservation_object_test_signaled_single(fence); | |
560 | if (ret < 0) | |
b68d8379 | 561 | goto retry; |
3c3b177a ML |
562 | else if (!ret) |
563 | break; | |
564 | } | |
565 | ||
b68d8379 CW |
566 | if (read_seqcount_retry(&obj->seq, seq)) |
567 | goto retry; | |
3c3b177a ML |
568 | } |
569 | ||
570 | if (!shared_count) { | |
f54d1867 | 571 | struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl); |
3c3b177a | 572 | |
3c3b177a | 573 | if (fence_excl) { |
5136629d JT |
574 | ret = reservation_object_test_signaled_single( |
575 | fence_excl); | |
3c3b177a | 576 | if (ret < 0) |
b68d8379 CW |
577 | goto retry; |
578 | ||
579 | if (read_seqcount_retry(&obj->seq, seq)) | |
580 | goto retry; | |
3c3b177a ML |
581 | } |
582 | } | |
583 | ||
584 | rcu_read_unlock(); | |
585 | return ret; | |
3c3b177a ML |
586 | } |
587 | EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu); |