Commit | Line | Data |
---|---|---|
e68a139f | 1 | /* |
635b3bc6 | 2 | * SPDX-License-Identifier: MIT |
e68a139f | 3 | * |
635b3bc6 | 4 | * (C) Copyright 2016 Intel Corporation |
e68a139f CW |
5 | */ |
6 | ||
7 | #include <linux/slab.h> | |
f54d1867 | 8 | #include <linux/dma-fence.h> |
81c0ed21 | 9 | #include <linux/irq_work.h> |
52791eee | 10 | #include <linux/dma-resv.h> |
e68a139f CW |
11 | |
12 | #include "i915_sw_fence.h" | |
47624cc3 | 13 | #include "i915_selftest.h" |
e68a139f | 14 | |
67a3acaa CW |
15 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG) |
16 | #define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr) | |
17 | #else | |
18 | #define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) | |
19 | #endif | |
20 | ||
7e941861 CW |
21 | #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ |
22 | ||
e68a139f CW |
23 | static DEFINE_SPINLOCK(i915_sw_fence_lock); |
24 | ||
fc158405 CW |
25 | enum { |
26 | DEBUG_FENCE_IDLE = 0, | |
27 | DEBUG_FENCE_NOTIFY, | |
28 | }; | |
29 | ||
fc158405 CW |
30 | static void *i915_sw_fence_debug_hint(void *addr) |
31 | { | |
32 | return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); | |
33 | } | |
34 | ||
5791bad4 CW |
35 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
36 | ||
fc158405 CW |
37 | static struct debug_obj_descr i915_sw_fence_debug_descr = { |
38 | .name = "i915_sw_fence", | |
39 | .debug_hint = i915_sw_fence_debug_hint, | |
40 | }; | |
41 | ||
42 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
43 | { | |
44 | debug_object_init(fence, &i915_sw_fence_debug_descr); | |
45 | } | |
46 | ||
214707fc CW |
47 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
48 | { | |
49 | debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); | |
50 | } | |
51 | ||
fc158405 CW |
52 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
53 | { | |
54 | debug_object_activate(fence, &i915_sw_fence_debug_descr); | |
55 | } | |
56 | ||
57 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
58 | int old, int new) | |
59 | { | |
60 | debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); | |
61 | } | |
62 | ||
63 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
64 | { | |
65 | debug_object_deactivate(fence, &i915_sw_fence_debug_descr); | |
66 | } | |
67 | ||
68 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
69 | { | |
70 | debug_object_destroy(fence, &i915_sw_fence_debug_descr); | |
71 | } | |
72 | ||
73 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
74 | { | |
75 | debug_object_free(fence, &i915_sw_fence_debug_descr); | |
6f13f29f | 76 | smp_wmb(); /* flush the change in state before reallocation */ |
fc158405 CW |
77 | } |
78 | ||
79 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
80 | { | |
81 | debug_object_assert_init(fence, &i915_sw_fence_debug_descr); | |
82 | } | |
83 | ||
84 | #else | |
85 | ||
86 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
87 | { | |
88 | } | |
89 | ||
214707fc CW |
90 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
91 | { | |
92 | } | |
93 | ||
fc158405 CW |
94 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
95 | { | |
96 | } | |
97 | ||
98 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
99 | int old, int new) | |
100 | { | |
101 | } | |
102 | ||
103 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
104 | { | |
105 | } | |
106 | ||
107 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
108 | { | |
109 | } | |
110 | ||
111 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
112 | { | |
113 | } | |
114 | ||
115 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
116 | { | |
117 | } | |
118 | ||
119 | #endif | |
120 | ||
e68a139f CW |
121 | static int __i915_sw_fence_notify(struct i915_sw_fence *fence, |
122 | enum i915_sw_fence_notify state) | |
123 | { | |
124 | i915_sw_fence_notify_t fn; | |
125 | ||
126 | fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); | |
127 | return fn(fence, state); | |
128 | } | |
129 | ||
fc158405 CW |
130 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
131 | void i915_sw_fence_fini(struct i915_sw_fence *fence) | |
132 | { | |
133 | debug_fence_free(fence); | |
134 | } | |
135 | #endif | |
136 | ||
e68a139f CW |
137 | static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, |
138 | struct list_head *continuation) | |
139 | { | |
140 | wait_queue_head_t *x = &fence->wait; | |
ac6424b9 | 141 | wait_queue_entry_t *pos, *next; |
e68a139f CW |
142 | unsigned long flags; |
143 | ||
fc158405 | 144 | debug_fence_deactivate(fence); |
e68a139f CW |
145 | atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ |
146 | ||
147 | /* | |
148 | * To prevent unbounded recursion as we traverse the graph of | |
2055da97 IM |
149 | * i915_sw_fences, we move the entry list from this, the next ready |
150 | * fence, to the tail of the original fence's entry list | |
e68a139f CW |
151 | * (and so added to the list to be woken). |
152 | */ | |
153 | ||
154 | spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); | |
155 | if (continuation) { | |
2055da97 | 156 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
e68a139f CW |
157 | if (pos->func == autoremove_wake_function) |
158 | pos->func(pos, TASK_NORMAL, 0, continuation); | |
159 | else | |
2055da97 | 160 | list_move_tail(&pos->entry, continuation); |
e68a139f CW |
161 | } |
162 | } else { | |
163 | LIST_HEAD(extra); | |
164 | ||
165 | do { | |
ef468849 | 166 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
20612303 CW |
167 | int wake_flags; |
168 | ||
169 | wake_flags = fence->error; | |
170 | if (pos->func == autoremove_wake_function) | |
171 | wake_flags = 0; | |
172 | ||
173 | pos->func(pos, TASK_NORMAL, wake_flags, &extra); | |
ef468849 | 174 | } |
e68a139f CW |
175 | |
176 | if (list_empty(&extra)) | |
177 | break; | |
178 | ||
2055da97 | 179 | list_splice_tail_init(&extra, &x->head); |
e68a139f CW |
180 | } while (1); |
181 | } | |
182 | spin_unlock_irqrestore(&x->lock, flags); | |
fc158405 CW |
183 | |
184 | debug_fence_assert(fence); | |
e68a139f CW |
185 | } |
186 | ||
187 | static void __i915_sw_fence_complete(struct i915_sw_fence *fence, | |
188 | struct list_head *continuation) | |
189 | { | |
fc158405 CW |
190 | debug_fence_assert(fence); |
191 | ||
e68a139f CW |
192 | if (!atomic_dec_and_test(&fence->pending)) |
193 | return; | |
194 | ||
fc158405 CW |
195 | debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); |
196 | ||
9310cb7f | 197 | if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) |
e68a139f CW |
198 | return; |
199 | ||
fc158405 CW |
200 | debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); |
201 | ||
e68a139f | 202 | __i915_sw_fence_wake_up_all(fence, continuation); |
9310cb7f CW |
203 | |
204 | debug_fence_destroy(fence); | |
205 | __i915_sw_fence_notify(fence, FENCE_FREE); | |
e68a139f CW |
206 | } |
207 | ||
e8861964 | 208 | void i915_sw_fence_complete(struct i915_sw_fence *fence) |
e68a139f | 209 | { |
fc158405 CW |
210 | debug_fence_assert(fence); |
211 | ||
e68a139f CW |
212 | if (WARN_ON(i915_sw_fence_done(fence))) |
213 | return; | |
214 | ||
215 | __i915_sw_fence_complete(fence, NULL); | |
216 | } | |
217 | ||
42fb60de | 218 | bool i915_sw_fence_await(struct i915_sw_fence *fence) |
e68a139f | 219 | { |
42fb60de CW |
220 | int pending; |
221 | ||
222 | /* | |
223 | * It is only safe to add a new await to the fence while it has | |
224 | * not yet been signaled (i.e. there are still existing signalers). | |
225 | */ | |
226 | pending = atomic_read(&fence->pending); | |
227 | do { | |
228 | if (pending < 1) | |
229 | return false; | |
230 | } while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1)); | |
231 | ||
232 | return true; | |
e68a139f CW |
233 | } |
234 | ||
556b7487 CW |
235 | void __i915_sw_fence_init(struct i915_sw_fence *fence, |
236 | i915_sw_fence_notify_t fn, | |
237 | const char *name, | |
238 | struct lock_class_key *key) | |
e68a139f | 239 | { |
9310cb7f | 240 | BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK); |
e68a139f | 241 | |
67a3acaa CW |
242 | __init_waitqueue_head(&fence->wait, name, key); |
243 | fence->flags = (unsigned long)fn; | |
244 | ||
245 | i915_sw_fence_reinit(fence); | |
246 | } | |
247 | ||
248 | void i915_sw_fence_reinit(struct i915_sw_fence *fence) | |
249 | { | |
fc158405 CW |
250 | debug_fence_init(fence); |
251 | ||
e68a139f | 252 | atomic_set(&fence->pending, 1); |
ef468849 CW |
253 | fence->error = 0; |
254 | ||
67a3acaa CW |
255 | I915_SW_FENCE_BUG_ON(!fence->flags); |
256 | I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head)); | |
e68a139f CW |
257 | } |
258 | ||
fc158405 CW |
259 | void i915_sw_fence_commit(struct i915_sw_fence *fence) |
260 | { | |
261 | debug_fence_activate(fence); | |
9310cb7f | 262 | i915_sw_fence_complete(fence); |
fc158405 CW |
263 | } |
264 | ||
ac6424b9 | 265 | static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) |
e68a139f | 266 | { |
ef468849 CW |
267 | i915_sw_fence_set_error_once(wq->private, flags); |
268 | ||
2055da97 | 269 | list_del(&wq->entry); |
e68a139f | 270 | __i915_sw_fence_complete(wq->private, key); |
9310cb7f | 271 | |
7e941861 CW |
272 | if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) |
273 | kfree(wq); | |
e68a139f CW |
274 | return 0; |
275 | } | |
276 | ||
277 | static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
278 | const struct i915_sw_fence * const signaler) | |
279 | { | |
ac6424b9 | 280 | wait_queue_entry_t *wq; |
e68a139f CW |
281 | |
282 | if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
283 | return false; | |
284 | ||
285 | if (fence == signaler) | |
286 | return true; | |
287 | ||
2055da97 | 288 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
289 | if (wq->func != i915_sw_fence_wake) |
290 | continue; | |
291 | ||
292 | if (__i915_sw_fence_check_if_after(wq->private, signaler)) | |
293 | return true; | |
294 | } | |
295 | ||
296 | return false; | |
297 | } | |
298 | ||
299 | static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) | |
300 | { | |
ac6424b9 | 301 | wait_queue_entry_t *wq; |
e68a139f CW |
302 | |
303 | if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
304 | return; | |
305 | ||
2055da97 | 306 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
307 | if (wq->func != i915_sw_fence_wake) |
308 | continue; | |
309 | ||
310 | __i915_sw_fence_clear_checked_bit(wq->private); | |
311 | } | |
312 | } | |
313 | ||
314 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
315 | const struct i915_sw_fence * const signaler) | |
316 | { | |
317 | unsigned long flags; | |
318 | bool err; | |
319 | ||
47624cc3 | 320 | if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG)) |
e68a139f CW |
321 | return false; |
322 | ||
323 | spin_lock_irqsave(&i915_sw_fence_lock, flags); | |
324 | err = __i915_sw_fence_check_if_after(fence, signaler); | |
325 | __i915_sw_fence_clear_checked_bit(fence); | |
326 | spin_unlock_irqrestore(&i915_sw_fence_lock, flags); | |
327 | ||
328 | return err; | |
329 | } | |
330 | ||
7e941861 CW |
331 | static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
332 | struct i915_sw_fence *signaler, | |
ac6424b9 | 333 | wait_queue_entry_t *wq, gfp_t gfp) |
e68a139f CW |
334 | { |
335 | unsigned long flags; | |
336 | int pending; | |
337 | ||
fc158405 | 338 | debug_fence_assert(fence); |
e30a7581 | 339 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 340 | |
ef468849 CW |
341 | if (i915_sw_fence_done(signaler)) { |
342 | i915_sw_fence_set_error_once(fence, signaler->error); | |
e68a139f | 343 | return 0; |
ef468849 | 344 | } |
e68a139f | 345 | |
fc158405 CW |
346 | debug_fence_assert(signaler); |
347 | ||
e68a139f CW |
348 | /* The dependency graph must be acyclic. */ |
349 | if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) | |
350 | return -EINVAL; | |
351 | ||
7e941861 CW |
352 | pending = 0; |
353 | if (!wq) { | |
354 | wq = kmalloc(sizeof(*wq), gfp); | |
355 | if (!wq) { | |
356 | if (!gfpflags_allow_blocking(gfp)) | |
357 | return -ENOMEM; | |
358 | ||
359 | i915_sw_fence_wait(signaler); | |
ef468849 | 360 | i915_sw_fence_set_error_once(fence, signaler->error); |
7e941861 CW |
361 | return 0; |
362 | } | |
363 | ||
364 | pending |= I915_SW_FENCE_FLAG_ALLOC; | |
365 | } | |
366 | ||
2055da97 | 367 | INIT_LIST_HEAD(&wq->entry); |
7e941861 | 368 | wq->flags = pending; |
e68a139f | 369 | wq->func = i915_sw_fence_wake; |
9310cb7f | 370 | wq->private = fence; |
e68a139f CW |
371 | |
372 | i915_sw_fence_await(fence); | |
373 | ||
374 | spin_lock_irqsave(&signaler->wait.lock, flags); | |
375 | if (likely(!i915_sw_fence_done(signaler))) { | |
ac6424b9 | 376 | __add_wait_queue_entry_tail(&signaler->wait, wq); |
e68a139f CW |
377 | pending = 1; |
378 | } else { | |
ef468849 | 379 | i915_sw_fence_wake(wq, 0, signaler->error, NULL); |
e68a139f CW |
380 | pending = 0; |
381 | } | |
382 | spin_unlock_irqrestore(&signaler->wait.lock, flags); | |
383 | ||
384 | return pending; | |
385 | } | |
386 | ||
7e941861 CW |
387 | int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
388 | struct i915_sw_fence *signaler, | |
ac6424b9 | 389 | wait_queue_entry_t *wq) |
7e941861 CW |
390 | { |
391 | return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); | |
392 | } | |
393 | ||
394 | int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, | |
395 | struct i915_sw_fence *signaler, | |
396 | gfp_t gfp) | |
397 | { | |
398 | return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); | |
399 | } | |
400 | ||
f255c1e9 CW |
401 | struct i915_sw_dma_fence_cb_timer { |
402 | struct i915_sw_dma_fence_cb base; | |
f54d1867 | 403 | struct dma_fence *dma; |
e68a139f | 404 | struct timer_list timer; |
81c0ed21 | 405 | struct irq_work work; |
7d622351 | 406 | struct rcu_head rcu; |
e68a139f CW |
407 | }; |
408 | ||
f255c1e9 CW |
409 | static void dma_i915_sw_fence_wake(struct dma_fence *dma, |
410 | struct dma_fence_cb *data) | |
411 | { | |
412 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
413 | ||
ef468849 | 414 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
f255c1e9 CW |
415 | i915_sw_fence_complete(cb->fence); |
416 | kfree(cb); | |
417 | } | |
418 | ||
39cbf2aa | 419 | static void timer_i915_sw_fence_wake(struct timer_list *t) |
e68a139f | 420 | { |
f255c1e9 | 421 | struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); |
81c0ed21 CW |
422 | struct i915_sw_fence *fence; |
423 | ||
f255c1e9 | 424 | fence = xchg(&cb->base.fence, NULL); |
81c0ed21 CW |
425 | if (!fence) |
426 | return; | |
e68a139f | 427 | |
2386b492 | 428 | pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n", |
5791bad4 CW |
429 | cb->dma->ops->get_driver_name(cb->dma), |
430 | cb->dma->ops->get_timeline_name(cb->dma), | |
431 | cb->dma->seqno, | |
432 | i915_sw_fence_debug_hint(fence)); | |
e68a139f | 433 | |
ef468849 | 434 | i915_sw_fence_set_error_once(fence, -ETIMEDOUT); |
81c0ed21 | 435 | i915_sw_fence_complete(fence); |
e68a139f CW |
436 | } |
437 | ||
f255c1e9 CW |
438 | static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, |
439 | struct dma_fence_cb *data) | |
e68a139f | 440 | { |
f255c1e9 CW |
441 | struct i915_sw_dma_fence_cb_timer *cb = |
442 | container_of(data, typeof(*cb), base.base); | |
81c0ed21 CW |
443 | struct i915_sw_fence *fence; |
444 | ||
f255c1e9 | 445 | fence = xchg(&cb->base.fence, NULL); |
cbab8d87 CW |
446 | if (fence) { |
447 | i915_sw_fence_set_error_once(fence, dma->error); | |
81c0ed21 | 448 | i915_sw_fence_complete(fence); |
cbab8d87 | 449 | } |
81c0ed21 | 450 | |
f255c1e9 | 451 | irq_work_queue(&cb->work); |
81c0ed21 CW |
452 | } |
453 | ||
454 | static void irq_i915_sw_fence_work(struct irq_work *wrk) | |
455 | { | |
f255c1e9 CW |
456 | struct i915_sw_dma_fence_cb_timer *cb = |
457 | container_of(wrk, typeof(*cb), work); | |
e68a139f CW |
458 | |
459 | del_timer_sync(&cb->timer); | |
f54d1867 | 460 | dma_fence_put(cb->dma); |
e68a139f | 461 | |
7d622351 | 462 | kfree_rcu(cb, rcu); |
e68a139f CW |
463 | } |
464 | ||
465 | int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
f54d1867 | 466 | struct dma_fence *dma, |
e68a139f CW |
467 | unsigned long timeout, |
468 | gfp_t gfp) | |
469 | { | |
f54d1867 | 470 | struct i915_sw_dma_fence_cb *cb; |
f255c1e9 | 471 | dma_fence_func_t func; |
e68a139f CW |
472 | int ret; |
473 | ||
fc158405 | 474 | debug_fence_assert(fence); |
e30a7581 | 475 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 476 | |
cbab8d87 CW |
477 | if (dma_fence_is_signaled(dma)) { |
478 | i915_sw_fence_set_error_once(fence, dma->error); | |
e68a139f | 479 | return 0; |
cbab8d87 | 480 | } |
e68a139f | 481 | |
f255c1e9 CW |
482 | cb = kmalloc(timeout ? |
483 | sizeof(struct i915_sw_dma_fence_cb_timer) : | |
484 | sizeof(struct i915_sw_dma_fence_cb), | |
485 | gfp); | |
e68a139f CW |
486 | if (!cb) { |
487 | if (!gfpflags_allow_blocking(gfp)) | |
488 | return -ENOMEM; | |
489 | ||
cbab8d87 CW |
490 | ret = dma_fence_wait(dma, false); |
491 | if (ret) | |
492 | return ret; | |
493 | ||
494 | i915_sw_fence_set_error_once(fence, dma->error); | |
495 | return 0; | |
e68a139f CW |
496 | } |
497 | ||
9310cb7f | 498 | cb->fence = fence; |
e68a139f CW |
499 | i915_sw_fence_await(fence); |
500 | ||
f255c1e9 | 501 | func = dma_i915_sw_fence_wake; |
e68a139f | 502 | if (timeout) { |
f255c1e9 CW |
503 | struct i915_sw_dma_fence_cb_timer *timer = |
504 | container_of(cb, typeof(*timer), base); | |
c32164b1 | 505 | |
f255c1e9 CW |
506 | timer->dma = dma_fence_get(dma); |
507 | init_irq_work(&timer->work, irq_i915_sw_fence_work); | |
508 | ||
509 | timer_setup(&timer->timer, | |
c32164b1 | 510 | timer_i915_sw_fence_wake, TIMER_IRQSAFE); |
f255c1e9 CW |
511 | mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout)); |
512 | ||
513 | func = dma_i915_sw_fence_wake_timer; | |
e68a139f CW |
514 | } |
515 | ||
f255c1e9 | 516 | ret = dma_fence_add_callback(dma, &cb->base, func); |
e68a139f CW |
517 | if (ret == 0) { |
518 | ret = 1; | |
519 | } else { | |
f255c1e9 | 520 | func(dma, &cb->base); |
e68a139f CW |
521 | if (ret == -ENOENT) /* fence already signaled */ |
522 | ret = 0; | |
523 | } | |
524 | ||
525 | return ret; | |
526 | } | |
527 | ||
ea593dbb CW |
528 | static void __dma_i915_sw_fence_wake(struct dma_fence *dma, |
529 | struct dma_fence_cb *data) | |
530 | { | |
531 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
532 | ||
ef468849 | 533 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
ea593dbb CW |
534 | i915_sw_fence_complete(cb->fence); |
535 | } | |
536 | ||
537 | int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
538 | struct dma_fence *dma, | |
539 | struct i915_sw_dma_fence_cb *cb) | |
540 | { | |
541 | int ret; | |
542 | ||
543 | debug_fence_assert(fence); | |
544 | ||
cbab8d87 CW |
545 | if (dma_fence_is_signaled(dma)) { |
546 | i915_sw_fence_set_error_once(fence, dma->error); | |
ea593dbb | 547 | return 0; |
cbab8d87 | 548 | } |
ea593dbb CW |
549 | |
550 | cb->fence = fence; | |
551 | i915_sw_fence_await(fence); | |
552 | ||
a80d7367 CW |
553 | ret = 1; |
554 | if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) { | |
555 | /* fence already signaled */ | |
ef468849 | 556 | __dma_i915_sw_fence_wake(dma, &cb->base); |
a80d7367 | 557 | ret = 0; |
ea593dbb CW |
558 | } |
559 | ||
560 | return ret; | |
561 | } | |
562 | ||
e68a139f | 563 | int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, |
52791eee | 564 | struct dma_resv *resv, |
f54d1867 | 565 | const struct dma_fence_ops *exclude, |
e68a139f CW |
566 | bool write, |
567 | unsigned long timeout, | |
568 | gfp_t gfp) | |
569 | { | |
f54d1867 | 570 | struct dma_fence *excl; |
e68a139f CW |
571 | int ret = 0, pending; |
572 | ||
fc158405 | 573 | debug_fence_assert(fence); |
e30a7581 | 574 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 575 | |
e68a139f | 576 | if (write) { |
f54d1867 | 577 | struct dma_fence **shared; |
e68a139f CW |
578 | unsigned int count, i; |
579 | ||
cbab8d87 | 580 | ret = dma_resv_get_fences_rcu(resv, &excl, &count, &shared); |
e68a139f CW |
581 | if (ret) |
582 | return ret; | |
583 | ||
584 | for (i = 0; i < count; i++) { | |
585 | if (shared[i]->ops == exclude) | |
586 | continue; | |
587 | ||
588 | pending = i915_sw_fence_await_dma_fence(fence, | |
589 | shared[i], | |
590 | timeout, | |
591 | gfp); | |
592 | if (pending < 0) { | |
593 | ret = pending; | |
594 | break; | |
595 | } | |
596 | ||
597 | ret |= pending; | |
598 | } | |
599 | ||
600 | for (i = 0; i < count; i++) | |
f54d1867 | 601 | dma_fence_put(shared[i]); |
e68a139f CW |
602 | kfree(shared); |
603 | } else { | |
52791eee | 604 | excl = dma_resv_get_excl_rcu(resv); |
e68a139f CW |
605 | } |
606 | ||
607 | if (ret >= 0 && excl && excl->ops != exclude) { | |
608 | pending = i915_sw_fence_await_dma_fence(fence, | |
609 | excl, | |
610 | timeout, | |
611 | gfp); | |
612 | if (pending < 0) | |
613 | ret = pending; | |
614 | else | |
615 | ret |= pending; | |
616 | } | |
617 | ||
f54d1867 | 618 | dma_fence_put(excl); |
e68a139f CW |
619 | |
620 | return ret; | |
621 | } | |
47624cc3 CW |
622 | |
623 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
214707fc | 624 | #include "selftests/lib_sw_fence.c" |
47624cc3 CW |
625 | #include "selftests/i915_sw_fence.c" |
626 | #endif |