Commit | Line | Data |
---|---|---|
e68a139f | 1 | /* |
635b3bc6 | 2 | * SPDX-License-Identifier: MIT |
e68a139f | 3 | * |
635b3bc6 | 4 | * (C) Copyright 2016 Intel Corporation |
e68a139f CW |
5 | */ |
6 | ||
7 | #include <linux/slab.h> | |
f54d1867 | 8 | #include <linux/dma-fence.h> |
81c0ed21 | 9 | #include <linux/irq_work.h> |
52791eee | 10 | #include <linux/dma-resv.h> |
e68a139f CW |
11 | |
12 | #include "i915_sw_fence.h" | |
47624cc3 | 13 | #include "i915_selftest.h" |
e68a139f | 14 | |
67a3acaa CW |
15 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG) |
16 | #define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr) | |
17 | #else | |
18 | #define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) | |
19 | #endif | |
20 | ||
7e941861 CW |
21 | #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ |
22 | ||
e68a139f CW |
23 | static DEFINE_SPINLOCK(i915_sw_fence_lock); |
24 | ||
fc158405 CW |
25 | enum { |
26 | DEBUG_FENCE_IDLE = 0, | |
27 | DEBUG_FENCE_NOTIFY, | |
28 | }; | |
29 | ||
fc158405 CW |
30 | static void *i915_sw_fence_debug_hint(void *addr) |
31 | { | |
32 | return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); | |
33 | } | |
34 | ||
5791bad4 CW |
35 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
36 | ||
fc158405 CW |
37 | static struct debug_obj_descr i915_sw_fence_debug_descr = { |
38 | .name = "i915_sw_fence", | |
39 | .debug_hint = i915_sw_fence_debug_hint, | |
40 | }; | |
41 | ||
42 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
43 | { | |
44 | debug_object_init(fence, &i915_sw_fence_debug_descr); | |
45 | } | |
46 | ||
214707fc CW |
47 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
48 | { | |
49 | debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); | |
50 | } | |
51 | ||
fc158405 CW |
52 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
53 | { | |
54 | debug_object_activate(fence, &i915_sw_fence_debug_descr); | |
55 | } | |
56 | ||
57 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
58 | int old, int new) | |
59 | { | |
60 | debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); | |
61 | } | |
62 | ||
63 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
64 | { | |
65 | debug_object_deactivate(fence, &i915_sw_fence_debug_descr); | |
66 | } | |
67 | ||
68 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
69 | { | |
70 | debug_object_destroy(fence, &i915_sw_fence_debug_descr); | |
71 | } | |
72 | ||
73 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
74 | { | |
75 | debug_object_free(fence, &i915_sw_fence_debug_descr); | |
6f13f29f | 76 | smp_wmb(); /* flush the change in state before reallocation */ |
fc158405 CW |
77 | } |
78 | ||
79 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
80 | { | |
81 | debug_object_assert_init(fence, &i915_sw_fence_debug_descr); | |
82 | } | |
83 | ||
84 | #else | |
85 | ||
86 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
87 | { | |
88 | } | |
89 | ||
214707fc CW |
90 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
91 | { | |
92 | } | |
93 | ||
fc158405 CW |
94 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
95 | { | |
96 | } | |
97 | ||
98 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
99 | int old, int new) | |
100 | { | |
101 | } | |
102 | ||
103 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
104 | { | |
105 | } | |
106 | ||
107 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
108 | { | |
109 | } | |
110 | ||
111 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
112 | { | |
113 | } | |
114 | ||
115 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
116 | { | |
117 | } | |
118 | ||
119 | #endif | |
120 | ||
e68a139f CW |
121 | static int __i915_sw_fence_notify(struct i915_sw_fence *fence, |
122 | enum i915_sw_fence_notify state) | |
123 | { | |
124 | i915_sw_fence_notify_t fn; | |
125 | ||
126 | fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); | |
127 | return fn(fence, state); | |
128 | } | |
129 | ||
fc158405 CW |
130 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
131 | void i915_sw_fence_fini(struct i915_sw_fence *fence) | |
132 | { | |
133 | debug_fence_free(fence); | |
134 | } | |
135 | #endif | |
136 | ||
e68a139f CW |
137 | static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, |
138 | struct list_head *continuation) | |
139 | { | |
140 | wait_queue_head_t *x = &fence->wait; | |
ac6424b9 | 141 | wait_queue_entry_t *pos, *next; |
e68a139f CW |
142 | unsigned long flags; |
143 | ||
fc158405 | 144 | debug_fence_deactivate(fence); |
e68a139f CW |
145 | atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ |
146 | ||
147 | /* | |
148 | * To prevent unbounded recursion as we traverse the graph of | |
2055da97 IM |
149 | * i915_sw_fences, we move the entry list from this, the next ready |
150 | * fence, to the tail of the original fence's entry list | |
e68a139f CW |
151 | * (and so added to the list to be woken). |
152 | */ | |
153 | ||
154 | spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); | |
155 | if (continuation) { | |
2055da97 | 156 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
e68a139f CW |
157 | if (pos->func == autoremove_wake_function) |
158 | pos->func(pos, TASK_NORMAL, 0, continuation); | |
159 | else | |
2055da97 | 160 | list_move_tail(&pos->entry, continuation); |
e68a139f CW |
161 | } |
162 | } else { | |
163 | LIST_HEAD(extra); | |
164 | ||
165 | do { | |
ef468849 CW |
166 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
167 | pos->func(pos, | |
168 | TASK_NORMAL, fence->error, | |
169 | &extra); | |
170 | } | |
e68a139f CW |
171 | |
172 | if (list_empty(&extra)) | |
173 | break; | |
174 | ||
2055da97 | 175 | list_splice_tail_init(&extra, &x->head); |
e68a139f CW |
176 | } while (1); |
177 | } | |
178 | spin_unlock_irqrestore(&x->lock, flags); | |
fc158405 CW |
179 | |
180 | debug_fence_assert(fence); | |
e68a139f CW |
181 | } |
182 | ||
183 | static void __i915_sw_fence_complete(struct i915_sw_fence *fence, | |
184 | struct list_head *continuation) | |
185 | { | |
fc158405 CW |
186 | debug_fence_assert(fence); |
187 | ||
e68a139f CW |
188 | if (!atomic_dec_and_test(&fence->pending)) |
189 | return; | |
190 | ||
fc158405 CW |
191 | debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); |
192 | ||
9310cb7f | 193 | if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) |
e68a139f CW |
194 | return; |
195 | ||
fc158405 CW |
196 | debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); |
197 | ||
e68a139f | 198 | __i915_sw_fence_wake_up_all(fence, continuation); |
9310cb7f CW |
199 | |
200 | debug_fence_destroy(fence); | |
201 | __i915_sw_fence_notify(fence, FENCE_FREE); | |
e68a139f CW |
202 | } |
203 | ||
e8861964 | 204 | void i915_sw_fence_complete(struct i915_sw_fence *fence) |
e68a139f | 205 | { |
fc158405 CW |
206 | debug_fence_assert(fence); |
207 | ||
e68a139f CW |
208 | if (WARN_ON(i915_sw_fence_done(fence))) |
209 | return; | |
210 | ||
211 | __i915_sw_fence_complete(fence, NULL); | |
212 | } | |
213 | ||
42fb60de | 214 | bool i915_sw_fence_await(struct i915_sw_fence *fence) |
e68a139f | 215 | { |
42fb60de CW |
216 | int pending; |
217 | ||
218 | /* | |
219 | * It is only safe to add a new await to the fence while it has | |
220 | * not yet been signaled (i.e. there are still existing signalers). | |
221 | */ | |
222 | pending = atomic_read(&fence->pending); | |
223 | do { | |
224 | if (pending < 1) | |
225 | return false; | |
226 | } while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1)); | |
227 | ||
228 | return true; | |
e68a139f CW |
229 | } |
230 | ||
556b7487 CW |
231 | void __i915_sw_fence_init(struct i915_sw_fence *fence, |
232 | i915_sw_fence_notify_t fn, | |
233 | const char *name, | |
234 | struct lock_class_key *key) | |
e68a139f | 235 | { |
9310cb7f | 236 | BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK); |
e68a139f | 237 | |
67a3acaa CW |
238 | __init_waitqueue_head(&fence->wait, name, key); |
239 | fence->flags = (unsigned long)fn; | |
240 | ||
241 | i915_sw_fence_reinit(fence); | |
242 | } | |
243 | ||
244 | void i915_sw_fence_reinit(struct i915_sw_fence *fence) | |
245 | { | |
fc158405 CW |
246 | debug_fence_init(fence); |
247 | ||
e68a139f | 248 | atomic_set(&fence->pending, 1); |
ef468849 CW |
249 | fence->error = 0; |
250 | ||
67a3acaa CW |
251 | I915_SW_FENCE_BUG_ON(!fence->flags); |
252 | I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head)); | |
e68a139f CW |
253 | } |
254 | ||
fc158405 CW |
255 | void i915_sw_fence_commit(struct i915_sw_fence *fence) |
256 | { | |
257 | debug_fence_activate(fence); | |
9310cb7f | 258 | i915_sw_fence_complete(fence); |
fc158405 CW |
259 | } |
260 | ||
ac6424b9 | 261 | static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) |
e68a139f | 262 | { |
ef468849 CW |
263 | i915_sw_fence_set_error_once(wq->private, flags); |
264 | ||
2055da97 | 265 | list_del(&wq->entry); |
e68a139f | 266 | __i915_sw_fence_complete(wq->private, key); |
9310cb7f | 267 | |
7e941861 CW |
268 | if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) |
269 | kfree(wq); | |
e68a139f CW |
270 | return 0; |
271 | } | |
272 | ||
273 | static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
274 | const struct i915_sw_fence * const signaler) | |
275 | { | |
ac6424b9 | 276 | wait_queue_entry_t *wq; |
e68a139f CW |
277 | |
278 | if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
279 | return false; | |
280 | ||
281 | if (fence == signaler) | |
282 | return true; | |
283 | ||
2055da97 | 284 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
285 | if (wq->func != i915_sw_fence_wake) |
286 | continue; | |
287 | ||
288 | if (__i915_sw_fence_check_if_after(wq->private, signaler)) | |
289 | return true; | |
290 | } | |
291 | ||
292 | return false; | |
293 | } | |
294 | ||
295 | static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) | |
296 | { | |
ac6424b9 | 297 | wait_queue_entry_t *wq; |
e68a139f CW |
298 | |
299 | if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
300 | return; | |
301 | ||
2055da97 | 302 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
303 | if (wq->func != i915_sw_fence_wake) |
304 | continue; | |
305 | ||
306 | __i915_sw_fence_clear_checked_bit(wq->private); | |
307 | } | |
308 | } | |
309 | ||
310 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
311 | const struct i915_sw_fence * const signaler) | |
312 | { | |
313 | unsigned long flags; | |
314 | bool err; | |
315 | ||
47624cc3 | 316 | if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG)) |
e68a139f CW |
317 | return false; |
318 | ||
319 | spin_lock_irqsave(&i915_sw_fence_lock, flags); | |
320 | err = __i915_sw_fence_check_if_after(fence, signaler); | |
321 | __i915_sw_fence_clear_checked_bit(fence); | |
322 | spin_unlock_irqrestore(&i915_sw_fence_lock, flags); | |
323 | ||
324 | return err; | |
325 | } | |
326 | ||
7e941861 CW |
327 | static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
328 | struct i915_sw_fence *signaler, | |
ac6424b9 | 329 | wait_queue_entry_t *wq, gfp_t gfp) |
e68a139f CW |
330 | { |
331 | unsigned long flags; | |
332 | int pending; | |
333 | ||
fc158405 | 334 | debug_fence_assert(fence); |
e30a7581 | 335 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 336 | |
ef468849 CW |
337 | if (i915_sw_fence_done(signaler)) { |
338 | i915_sw_fence_set_error_once(fence, signaler->error); | |
e68a139f | 339 | return 0; |
ef468849 | 340 | } |
e68a139f | 341 | |
fc158405 CW |
342 | debug_fence_assert(signaler); |
343 | ||
e68a139f CW |
344 | /* The dependency graph must be acyclic. */ |
345 | if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) | |
346 | return -EINVAL; | |
347 | ||
7e941861 CW |
348 | pending = 0; |
349 | if (!wq) { | |
350 | wq = kmalloc(sizeof(*wq), gfp); | |
351 | if (!wq) { | |
352 | if (!gfpflags_allow_blocking(gfp)) | |
353 | return -ENOMEM; | |
354 | ||
355 | i915_sw_fence_wait(signaler); | |
ef468849 | 356 | i915_sw_fence_set_error_once(fence, signaler->error); |
7e941861 CW |
357 | return 0; |
358 | } | |
359 | ||
360 | pending |= I915_SW_FENCE_FLAG_ALLOC; | |
361 | } | |
362 | ||
2055da97 | 363 | INIT_LIST_HEAD(&wq->entry); |
7e941861 | 364 | wq->flags = pending; |
e68a139f | 365 | wq->func = i915_sw_fence_wake; |
9310cb7f | 366 | wq->private = fence; |
e68a139f CW |
367 | |
368 | i915_sw_fence_await(fence); | |
369 | ||
370 | spin_lock_irqsave(&signaler->wait.lock, flags); | |
371 | if (likely(!i915_sw_fence_done(signaler))) { | |
ac6424b9 | 372 | __add_wait_queue_entry_tail(&signaler->wait, wq); |
e68a139f CW |
373 | pending = 1; |
374 | } else { | |
ef468849 | 375 | i915_sw_fence_wake(wq, 0, signaler->error, NULL); |
e68a139f CW |
376 | pending = 0; |
377 | } | |
378 | spin_unlock_irqrestore(&signaler->wait.lock, flags); | |
379 | ||
380 | return pending; | |
381 | } | |
382 | ||
7e941861 CW |
383 | int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
384 | struct i915_sw_fence *signaler, | |
ac6424b9 | 385 | wait_queue_entry_t *wq) |
7e941861 CW |
386 | { |
387 | return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); | |
388 | } | |
389 | ||
390 | int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, | |
391 | struct i915_sw_fence *signaler, | |
392 | gfp_t gfp) | |
393 | { | |
394 | return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); | |
395 | } | |
396 | ||
f255c1e9 CW |
397 | struct i915_sw_dma_fence_cb_timer { |
398 | struct i915_sw_dma_fence_cb base; | |
f54d1867 | 399 | struct dma_fence *dma; |
e68a139f | 400 | struct timer_list timer; |
81c0ed21 | 401 | struct irq_work work; |
7d622351 | 402 | struct rcu_head rcu; |
e68a139f CW |
403 | }; |
404 | ||
f255c1e9 CW |
405 | static void dma_i915_sw_fence_wake(struct dma_fence *dma, |
406 | struct dma_fence_cb *data) | |
407 | { | |
408 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
409 | ||
ef468849 | 410 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
f255c1e9 CW |
411 | i915_sw_fence_complete(cb->fence); |
412 | kfree(cb); | |
413 | } | |
414 | ||
39cbf2aa | 415 | static void timer_i915_sw_fence_wake(struct timer_list *t) |
e68a139f | 416 | { |
f255c1e9 | 417 | struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); |
81c0ed21 CW |
418 | struct i915_sw_fence *fence; |
419 | ||
f255c1e9 | 420 | fence = xchg(&cb->base.fence, NULL); |
81c0ed21 CW |
421 | if (!fence) |
422 | return; | |
e68a139f | 423 | |
2386b492 | 424 | pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n", |
5791bad4 CW |
425 | cb->dma->ops->get_driver_name(cb->dma), |
426 | cb->dma->ops->get_timeline_name(cb->dma), | |
427 | cb->dma->seqno, | |
428 | i915_sw_fence_debug_hint(fence)); | |
e68a139f | 429 | |
ef468849 | 430 | i915_sw_fence_set_error_once(fence, -ETIMEDOUT); |
81c0ed21 | 431 | i915_sw_fence_complete(fence); |
e68a139f CW |
432 | } |
433 | ||
f255c1e9 CW |
434 | static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, |
435 | struct dma_fence_cb *data) | |
e68a139f | 436 | { |
f255c1e9 CW |
437 | struct i915_sw_dma_fence_cb_timer *cb = |
438 | container_of(data, typeof(*cb), base.base); | |
81c0ed21 CW |
439 | struct i915_sw_fence *fence; |
440 | ||
f255c1e9 | 441 | fence = xchg(&cb->base.fence, NULL); |
cbab8d87 CW |
442 | if (fence) { |
443 | i915_sw_fence_set_error_once(fence, dma->error); | |
81c0ed21 | 444 | i915_sw_fence_complete(fence); |
cbab8d87 | 445 | } |
81c0ed21 | 446 | |
f255c1e9 | 447 | irq_work_queue(&cb->work); |
81c0ed21 CW |
448 | } |
449 | ||
450 | static void irq_i915_sw_fence_work(struct irq_work *wrk) | |
451 | { | |
f255c1e9 CW |
452 | struct i915_sw_dma_fence_cb_timer *cb = |
453 | container_of(wrk, typeof(*cb), work); | |
e68a139f CW |
454 | |
455 | del_timer_sync(&cb->timer); | |
f54d1867 | 456 | dma_fence_put(cb->dma); |
e68a139f | 457 | |
7d622351 | 458 | kfree_rcu(cb, rcu); |
e68a139f CW |
459 | } |
460 | ||
461 | int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
f54d1867 | 462 | struct dma_fence *dma, |
e68a139f CW |
463 | unsigned long timeout, |
464 | gfp_t gfp) | |
465 | { | |
f54d1867 | 466 | struct i915_sw_dma_fence_cb *cb; |
f255c1e9 | 467 | dma_fence_func_t func; |
e68a139f CW |
468 | int ret; |
469 | ||
fc158405 | 470 | debug_fence_assert(fence); |
e30a7581 | 471 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 472 | |
cbab8d87 CW |
473 | if (dma_fence_is_signaled(dma)) { |
474 | i915_sw_fence_set_error_once(fence, dma->error); | |
e68a139f | 475 | return 0; |
cbab8d87 | 476 | } |
e68a139f | 477 | |
f255c1e9 CW |
478 | cb = kmalloc(timeout ? |
479 | sizeof(struct i915_sw_dma_fence_cb_timer) : | |
480 | sizeof(struct i915_sw_dma_fence_cb), | |
481 | gfp); | |
e68a139f CW |
482 | if (!cb) { |
483 | if (!gfpflags_allow_blocking(gfp)) | |
484 | return -ENOMEM; | |
485 | ||
cbab8d87 CW |
486 | ret = dma_fence_wait(dma, false); |
487 | if (ret) | |
488 | return ret; | |
489 | ||
490 | i915_sw_fence_set_error_once(fence, dma->error); | |
491 | return 0; | |
e68a139f CW |
492 | } |
493 | ||
9310cb7f | 494 | cb->fence = fence; |
e68a139f CW |
495 | i915_sw_fence_await(fence); |
496 | ||
f255c1e9 | 497 | func = dma_i915_sw_fence_wake; |
e68a139f | 498 | if (timeout) { |
f255c1e9 CW |
499 | struct i915_sw_dma_fence_cb_timer *timer = |
500 | container_of(cb, typeof(*timer), base); | |
c32164b1 | 501 | |
f255c1e9 CW |
502 | timer->dma = dma_fence_get(dma); |
503 | init_irq_work(&timer->work, irq_i915_sw_fence_work); | |
504 | ||
505 | timer_setup(&timer->timer, | |
c32164b1 | 506 | timer_i915_sw_fence_wake, TIMER_IRQSAFE); |
f255c1e9 CW |
507 | mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout)); |
508 | ||
509 | func = dma_i915_sw_fence_wake_timer; | |
e68a139f CW |
510 | } |
511 | ||
f255c1e9 | 512 | ret = dma_fence_add_callback(dma, &cb->base, func); |
e68a139f CW |
513 | if (ret == 0) { |
514 | ret = 1; | |
515 | } else { | |
f255c1e9 | 516 | func(dma, &cb->base); |
e68a139f CW |
517 | if (ret == -ENOENT) /* fence already signaled */ |
518 | ret = 0; | |
519 | } | |
520 | ||
521 | return ret; | |
522 | } | |
523 | ||
ea593dbb CW |
524 | static void __dma_i915_sw_fence_wake(struct dma_fence *dma, |
525 | struct dma_fence_cb *data) | |
526 | { | |
527 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
528 | ||
ef468849 | 529 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
ea593dbb CW |
530 | i915_sw_fence_complete(cb->fence); |
531 | } | |
532 | ||
533 | int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
534 | struct dma_fence *dma, | |
535 | struct i915_sw_dma_fence_cb *cb) | |
536 | { | |
537 | int ret; | |
538 | ||
539 | debug_fence_assert(fence); | |
540 | ||
cbab8d87 CW |
541 | if (dma_fence_is_signaled(dma)) { |
542 | i915_sw_fence_set_error_once(fence, dma->error); | |
ea593dbb | 543 | return 0; |
cbab8d87 | 544 | } |
ea593dbb CW |
545 | |
546 | cb->fence = fence; | |
547 | i915_sw_fence_await(fence); | |
548 | ||
549 | ret = dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake); | |
550 | if (ret == 0) { | |
551 | ret = 1; | |
552 | } else { | |
ef468849 | 553 | __dma_i915_sw_fence_wake(dma, &cb->base); |
ea593dbb CW |
554 | if (ret == -ENOENT) /* fence already signaled */ |
555 | ret = 0; | |
556 | } | |
557 | ||
558 | return ret; | |
559 | } | |
560 | ||
e68a139f | 561 | int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, |
52791eee | 562 | struct dma_resv *resv, |
f54d1867 | 563 | const struct dma_fence_ops *exclude, |
e68a139f CW |
564 | bool write, |
565 | unsigned long timeout, | |
566 | gfp_t gfp) | |
567 | { | |
f54d1867 | 568 | struct dma_fence *excl; |
e68a139f CW |
569 | int ret = 0, pending; |
570 | ||
fc158405 | 571 | debug_fence_assert(fence); |
e30a7581 | 572 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 573 | |
e68a139f | 574 | if (write) { |
f54d1867 | 575 | struct dma_fence **shared; |
e68a139f CW |
576 | unsigned int count, i; |
577 | ||
cbab8d87 | 578 | ret = dma_resv_get_fences_rcu(resv, &excl, &count, &shared); |
e68a139f CW |
579 | if (ret) |
580 | return ret; | |
581 | ||
582 | for (i = 0; i < count; i++) { | |
583 | if (shared[i]->ops == exclude) | |
584 | continue; | |
585 | ||
586 | pending = i915_sw_fence_await_dma_fence(fence, | |
587 | shared[i], | |
588 | timeout, | |
589 | gfp); | |
590 | if (pending < 0) { | |
591 | ret = pending; | |
592 | break; | |
593 | } | |
594 | ||
595 | ret |= pending; | |
596 | } | |
597 | ||
598 | for (i = 0; i < count; i++) | |
f54d1867 | 599 | dma_fence_put(shared[i]); |
e68a139f CW |
600 | kfree(shared); |
601 | } else { | |
52791eee | 602 | excl = dma_resv_get_excl_rcu(resv); |
e68a139f CW |
603 | } |
604 | ||
605 | if (ret >= 0 && excl && excl->ops != exclude) { | |
606 | pending = i915_sw_fence_await_dma_fence(fence, | |
607 | excl, | |
608 | timeout, | |
609 | gfp); | |
610 | if (pending < 0) | |
611 | ret = pending; | |
612 | else | |
613 | ret |= pending; | |
614 | } | |
615 | ||
f54d1867 | 616 | dma_fence_put(excl); |
e68a139f CW |
617 | |
618 | return ret; | |
619 | } | |
47624cc3 CW |
620 | |
621 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
214707fc | 622 | #include "selftests/lib_sw_fence.c" |
47624cc3 CW |
623 | #include "selftests/i915_sw_fence.c" |
624 | #endif |