Commit | Line | Data |
---|---|---|
e68a139f | 1 | /* |
635b3bc6 | 2 | * SPDX-License-Identifier: MIT |
e68a139f | 3 | * |
635b3bc6 | 4 | * (C) Copyright 2016 Intel Corporation |
e68a139f CW |
5 | */ |
6 | ||
7 | #include <linux/slab.h> | |
f54d1867 | 8 | #include <linux/dma-fence.h> |
81c0ed21 | 9 | #include <linux/irq_work.h> |
52791eee | 10 | #include <linux/dma-resv.h> |
e68a139f CW |
11 | |
12 | #include "i915_sw_fence.h" | |
47624cc3 | 13 | #include "i915_selftest.h" |
e68a139f | 14 | |
67a3acaa CW |
15 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG) |
16 | #define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr) | |
17 | #else | |
18 | #define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) | |
19 | #endif | |
20 | ||
e68a139f CW |
21 | static DEFINE_SPINLOCK(i915_sw_fence_lock); |
22 | ||
460d02ba CW |
23 | #define WQ_FLAG_BITS \ |
24 | BITS_PER_TYPE(typeof_member(struct wait_queue_entry, flags)) | |
25 | ||
26 | /* after WQ_FLAG_* for safety */ | |
27 | #define I915_SW_FENCE_FLAG_FENCE BIT(WQ_FLAG_BITS - 1) | |
28 | #define I915_SW_FENCE_FLAG_ALLOC BIT(WQ_FLAG_BITS - 2) | |
29 | ||
fc158405 CW |
30 | enum { |
31 | DEBUG_FENCE_IDLE = 0, | |
32 | DEBUG_FENCE_NOTIFY, | |
33 | }; | |
34 | ||
fc158405 CW |
35 | static void *i915_sw_fence_debug_hint(void *addr) |
36 | { | |
37 | return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); | |
38 | } | |
39 | ||
5791bad4 CW |
40 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
41 | ||
f9e62f31 | 42 | static const struct debug_obj_descr i915_sw_fence_debug_descr = { |
fc158405 CW |
43 | .name = "i915_sw_fence", |
44 | .debug_hint = i915_sw_fence_debug_hint, | |
45 | }; | |
46 | ||
47 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
48 | { | |
49 | debug_object_init(fence, &i915_sw_fence_debug_descr); | |
50 | } | |
51 | ||
214707fc CW |
52 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
53 | { | |
54 | debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); | |
55 | } | |
56 | ||
fc158405 CW |
57 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
58 | { | |
59 | debug_object_activate(fence, &i915_sw_fence_debug_descr); | |
60 | } | |
61 | ||
62 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
63 | int old, int new) | |
64 | { | |
65 | debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); | |
66 | } | |
67 | ||
68 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
69 | { | |
70 | debug_object_deactivate(fence, &i915_sw_fence_debug_descr); | |
71 | } | |
72 | ||
73 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
74 | { | |
75 | debug_object_destroy(fence, &i915_sw_fence_debug_descr); | |
76 | } | |
77 | ||
78 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
79 | { | |
80 | debug_object_free(fence, &i915_sw_fence_debug_descr); | |
6f13f29f | 81 | smp_wmb(); /* flush the change in state before reallocation */ |
fc158405 CW |
82 | } |
83 | ||
84 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
85 | { | |
86 | debug_object_assert_init(fence, &i915_sw_fence_debug_descr); | |
87 | } | |
88 | ||
89 | #else | |
90 | ||
91 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
92 | { | |
93 | } | |
94 | ||
214707fc CW |
95 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
96 | { | |
97 | } | |
98 | ||
fc158405 CW |
99 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
100 | { | |
101 | } | |
102 | ||
103 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
104 | int old, int new) | |
105 | { | |
106 | } | |
107 | ||
108 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
109 | { | |
110 | } | |
111 | ||
112 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
113 | { | |
114 | } | |
115 | ||
116 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
117 | { | |
118 | } | |
119 | ||
120 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
121 | { | |
122 | } | |
123 | ||
124 | #endif | |
125 | ||
e68a139f CW |
126 | static int __i915_sw_fence_notify(struct i915_sw_fence *fence, |
127 | enum i915_sw_fence_notify state) | |
128 | { | |
129 | i915_sw_fence_notify_t fn; | |
130 | ||
131 | fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); | |
132 | return fn(fence, state); | |
133 | } | |
134 | ||
fc158405 CW |
135 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
136 | void i915_sw_fence_fini(struct i915_sw_fence *fence) | |
137 | { | |
138 | debug_fence_free(fence); | |
139 | } | |
140 | #endif | |
141 | ||
e68a139f CW |
142 | static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, |
143 | struct list_head *continuation) | |
144 | { | |
145 | wait_queue_head_t *x = &fence->wait; | |
ac6424b9 | 146 | wait_queue_entry_t *pos, *next; |
e68a139f CW |
147 | unsigned long flags; |
148 | ||
fc158405 | 149 | debug_fence_deactivate(fence); |
e68a139f CW |
150 | atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ |
151 | ||
152 | /* | |
153 | * To prevent unbounded recursion as we traverse the graph of | |
2055da97 IM |
154 | * i915_sw_fences, we move the entry list from this, the next ready |
155 | * fence, to the tail of the original fence's entry list | |
e68a139f CW |
156 | * (and so added to the list to be woken). |
157 | */ | |
158 | ||
159 | spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); | |
160 | if (continuation) { | |
2055da97 | 161 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
460d02ba | 162 | if (pos->flags & I915_SW_FENCE_FLAG_FENCE) |
2055da97 | 163 | list_move_tail(&pos->entry, continuation); |
460d02ba CW |
164 | else |
165 | pos->func(pos, TASK_NORMAL, 0, continuation); | |
e68a139f CW |
166 | } |
167 | } else { | |
168 | LIST_HEAD(extra); | |
169 | ||
170 | do { | |
ef468849 | 171 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
20612303 CW |
172 | int wake_flags; |
173 | ||
460d02ba CW |
174 | wake_flags = 0; |
175 | if (pos->flags & I915_SW_FENCE_FLAG_FENCE) | |
176 | wake_flags = fence->error; | |
20612303 CW |
177 | |
178 | pos->func(pos, TASK_NORMAL, wake_flags, &extra); | |
ef468849 | 179 | } |
e68a139f CW |
180 | |
181 | if (list_empty(&extra)) | |
182 | break; | |
183 | ||
2055da97 | 184 | list_splice_tail_init(&extra, &x->head); |
e68a139f CW |
185 | } while (1); |
186 | } | |
187 | spin_unlock_irqrestore(&x->lock, flags); | |
fc158405 CW |
188 | |
189 | debug_fence_assert(fence); | |
e68a139f CW |
190 | } |
191 | ||
192 | static void __i915_sw_fence_complete(struct i915_sw_fence *fence, | |
193 | struct list_head *continuation) | |
194 | { | |
fc158405 CW |
195 | debug_fence_assert(fence); |
196 | ||
e68a139f CW |
197 | if (!atomic_dec_and_test(&fence->pending)) |
198 | return; | |
199 | ||
fc158405 CW |
200 | debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); |
201 | ||
9310cb7f | 202 | if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) |
e68a139f CW |
203 | return; |
204 | ||
fc158405 CW |
205 | debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); |
206 | ||
e68a139f | 207 | __i915_sw_fence_wake_up_all(fence, continuation); |
9310cb7f CW |
208 | |
209 | debug_fence_destroy(fence); | |
210 | __i915_sw_fence_notify(fence, FENCE_FREE); | |
e68a139f CW |
211 | } |
212 | ||
e8861964 | 213 | void i915_sw_fence_complete(struct i915_sw_fence *fence) |
e68a139f | 214 | { |
fc158405 CW |
215 | debug_fence_assert(fence); |
216 | ||
e68a139f CW |
217 | if (WARN_ON(i915_sw_fence_done(fence))) |
218 | return; | |
219 | ||
220 | __i915_sw_fence_complete(fence, NULL); | |
221 | } | |
222 | ||
42fb60de | 223 | bool i915_sw_fence_await(struct i915_sw_fence *fence) |
e68a139f | 224 | { |
42fb60de CW |
225 | int pending; |
226 | ||
227 | /* | |
228 | * It is only safe to add a new await to the fence while it has | |
229 | * not yet been signaled (i.e. there are still existing signalers). | |
230 | */ | |
231 | pending = atomic_read(&fence->pending); | |
232 | do { | |
233 | if (pending < 1) | |
234 | return false; | |
235 | } while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1)); | |
236 | ||
237 | return true; | |
e68a139f CW |
238 | } |
239 | ||
556b7487 CW |
240 | void __i915_sw_fence_init(struct i915_sw_fence *fence, |
241 | i915_sw_fence_notify_t fn, | |
242 | const char *name, | |
243 | struct lock_class_key *key) | |
e68a139f | 244 | { |
9310cb7f | 245 | BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK); |
e68a139f | 246 | |
67a3acaa CW |
247 | __init_waitqueue_head(&fence->wait, name, key); |
248 | fence->flags = (unsigned long)fn; | |
249 | ||
250 | i915_sw_fence_reinit(fence); | |
251 | } | |
252 | ||
253 | void i915_sw_fence_reinit(struct i915_sw_fence *fence) | |
254 | { | |
fc158405 CW |
255 | debug_fence_init(fence); |
256 | ||
e68a139f | 257 | atomic_set(&fence->pending, 1); |
ef468849 CW |
258 | fence->error = 0; |
259 | ||
67a3acaa CW |
260 | I915_SW_FENCE_BUG_ON(!fence->flags); |
261 | I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head)); | |
e68a139f CW |
262 | } |
263 | ||
fc158405 CW |
264 | void i915_sw_fence_commit(struct i915_sw_fence *fence) |
265 | { | |
266 | debug_fence_activate(fence); | |
9310cb7f | 267 | i915_sw_fence_complete(fence); |
fc158405 CW |
268 | } |
269 | ||
ac6424b9 | 270 | static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) |
e68a139f | 271 | { |
ef468849 CW |
272 | i915_sw_fence_set_error_once(wq->private, flags); |
273 | ||
2055da97 | 274 | list_del(&wq->entry); |
e68a139f | 275 | __i915_sw_fence_complete(wq->private, key); |
9310cb7f | 276 | |
7e941861 CW |
277 | if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) |
278 | kfree(wq); | |
e68a139f CW |
279 | return 0; |
280 | } | |
281 | ||
282 | static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
283 | const struct i915_sw_fence * const signaler) | |
284 | { | |
ac6424b9 | 285 | wait_queue_entry_t *wq; |
e68a139f CW |
286 | |
287 | if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
288 | return false; | |
289 | ||
290 | if (fence == signaler) | |
291 | return true; | |
292 | ||
2055da97 | 293 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
294 | if (wq->func != i915_sw_fence_wake) |
295 | continue; | |
296 | ||
297 | if (__i915_sw_fence_check_if_after(wq->private, signaler)) | |
298 | return true; | |
299 | } | |
300 | ||
301 | return false; | |
302 | } | |
303 | ||
304 | static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) | |
305 | { | |
ac6424b9 | 306 | wait_queue_entry_t *wq; |
e68a139f CW |
307 | |
308 | if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
309 | return; | |
310 | ||
2055da97 | 311 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
312 | if (wq->func != i915_sw_fence_wake) |
313 | continue; | |
314 | ||
315 | __i915_sw_fence_clear_checked_bit(wq->private); | |
316 | } | |
317 | } | |
318 | ||
319 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
320 | const struct i915_sw_fence * const signaler) | |
321 | { | |
322 | unsigned long flags; | |
323 | bool err; | |
324 | ||
47624cc3 | 325 | if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG)) |
e68a139f CW |
326 | return false; |
327 | ||
328 | spin_lock_irqsave(&i915_sw_fence_lock, flags); | |
329 | err = __i915_sw_fence_check_if_after(fence, signaler); | |
330 | __i915_sw_fence_clear_checked_bit(fence); | |
331 | spin_unlock_irqrestore(&i915_sw_fence_lock, flags); | |
332 | ||
333 | return err; | |
334 | } | |
335 | ||
7e941861 CW |
336 | static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
337 | struct i915_sw_fence *signaler, | |
ac6424b9 | 338 | wait_queue_entry_t *wq, gfp_t gfp) |
e68a139f | 339 | { |
460d02ba | 340 | unsigned int pending; |
e68a139f | 341 | unsigned long flags; |
e68a139f | 342 | |
fc158405 | 343 | debug_fence_assert(fence); |
e30a7581 | 344 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 345 | |
ef468849 CW |
346 | if (i915_sw_fence_done(signaler)) { |
347 | i915_sw_fence_set_error_once(fence, signaler->error); | |
e68a139f | 348 | return 0; |
ef468849 | 349 | } |
e68a139f | 350 | |
fc158405 CW |
351 | debug_fence_assert(signaler); |
352 | ||
e68a139f CW |
353 | /* The dependency graph must be acyclic. */ |
354 | if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) | |
355 | return -EINVAL; | |
356 | ||
460d02ba | 357 | pending = I915_SW_FENCE_FLAG_FENCE; |
7e941861 CW |
358 | if (!wq) { |
359 | wq = kmalloc(sizeof(*wq), gfp); | |
360 | if (!wq) { | |
361 | if (!gfpflags_allow_blocking(gfp)) | |
362 | return -ENOMEM; | |
363 | ||
364 | i915_sw_fence_wait(signaler); | |
ef468849 | 365 | i915_sw_fence_set_error_once(fence, signaler->error); |
7e941861 CW |
366 | return 0; |
367 | } | |
368 | ||
369 | pending |= I915_SW_FENCE_FLAG_ALLOC; | |
370 | } | |
371 | ||
2055da97 | 372 | INIT_LIST_HEAD(&wq->entry); |
7e941861 | 373 | wq->flags = pending; |
e68a139f | 374 | wq->func = i915_sw_fence_wake; |
9310cb7f | 375 | wq->private = fence; |
e68a139f CW |
376 | |
377 | i915_sw_fence_await(fence); | |
378 | ||
379 | spin_lock_irqsave(&signaler->wait.lock, flags); | |
380 | if (likely(!i915_sw_fence_done(signaler))) { | |
ac6424b9 | 381 | __add_wait_queue_entry_tail(&signaler->wait, wq); |
e68a139f CW |
382 | pending = 1; |
383 | } else { | |
ef468849 | 384 | i915_sw_fence_wake(wq, 0, signaler->error, NULL); |
e68a139f CW |
385 | pending = 0; |
386 | } | |
387 | spin_unlock_irqrestore(&signaler->wait.lock, flags); | |
388 | ||
389 | return pending; | |
390 | } | |
391 | ||
7e941861 CW |
392 | int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
393 | struct i915_sw_fence *signaler, | |
ac6424b9 | 394 | wait_queue_entry_t *wq) |
7e941861 CW |
395 | { |
396 | return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); | |
397 | } | |
398 | ||
399 | int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, | |
400 | struct i915_sw_fence *signaler, | |
401 | gfp_t gfp) | |
402 | { | |
403 | return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); | |
404 | } | |
405 | ||
f255c1e9 CW |
406 | struct i915_sw_dma_fence_cb_timer { |
407 | struct i915_sw_dma_fence_cb base; | |
f54d1867 | 408 | struct dma_fence *dma; |
e68a139f | 409 | struct timer_list timer; |
81c0ed21 | 410 | struct irq_work work; |
7d622351 | 411 | struct rcu_head rcu; |
e68a139f CW |
412 | }; |
413 | ||
f255c1e9 CW |
414 | static void dma_i915_sw_fence_wake(struct dma_fence *dma, |
415 | struct dma_fence_cb *data) | |
416 | { | |
417 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
418 | ||
ef468849 | 419 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
f255c1e9 CW |
420 | i915_sw_fence_complete(cb->fence); |
421 | kfree(cb); | |
422 | } | |
423 | ||
39cbf2aa | 424 | static void timer_i915_sw_fence_wake(struct timer_list *t) |
e68a139f | 425 | { |
f255c1e9 | 426 | struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); |
81c0ed21 CW |
427 | struct i915_sw_fence *fence; |
428 | ||
f255c1e9 | 429 | fence = xchg(&cb->base.fence, NULL); |
81c0ed21 CW |
430 | if (!fence) |
431 | return; | |
e68a139f | 432 | |
2386b492 | 433 | pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n", |
5791bad4 CW |
434 | cb->dma->ops->get_driver_name(cb->dma), |
435 | cb->dma->ops->get_timeline_name(cb->dma), | |
436 | cb->dma->seqno, | |
437 | i915_sw_fence_debug_hint(fence)); | |
e68a139f | 438 | |
ef468849 | 439 | i915_sw_fence_set_error_once(fence, -ETIMEDOUT); |
81c0ed21 | 440 | i915_sw_fence_complete(fence); |
e68a139f CW |
441 | } |
442 | ||
f255c1e9 CW |
443 | static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, |
444 | struct dma_fence_cb *data) | |
e68a139f | 445 | { |
f255c1e9 CW |
446 | struct i915_sw_dma_fence_cb_timer *cb = |
447 | container_of(data, typeof(*cb), base.base); | |
81c0ed21 CW |
448 | struct i915_sw_fence *fence; |
449 | ||
f255c1e9 | 450 | fence = xchg(&cb->base.fence, NULL); |
cbab8d87 CW |
451 | if (fence) { |
452 | i915_sw_fence_set_error_once(fence, dma->error); | |
81c0ed21 | 453 | i915_sw_fence_complete(fence); |
cbab8d87 | 454 | } |
81c0ed21 | 455 | |
f255c1e9 | 456 | irq_work_queue(&cb->work); |
81c0ed21 CW |
457 | } |
458 | ||
459 | static void irq_i915_sw_fence_work(struct irq_work *wrk) | |
460 | { | |
f255c1e9 CW |
461 | struct i915_sw_dma_fence_cb_timer *cb = |
462 | container_of(wrk, typeof(*cb), work); | |
e68a139f CW |
463 | |
464 | del_timer_sync(&cb->timer); | |
f54d1867 | 465 | dma_fence_put(cb->dma); |
e68a139f | 466 | |
7d622351 | 467 | kfree_rcu(cb, rcu); |
e68a139f CW |
468 | } |
469 | ||
470 | int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
f54d1867 | 471 | struct dma_fence *dma, |
e68a139f CW |
472 | unsigned long timeout, |
473 | gfp_t gfp) | |
474 | { | |
f54d1867 | 475 | struct i915_sw_dma_fence_cb *cb; |
f255c1e9 | 476 | dma_fence_func_t func; |
e68a139f CW |
477 | int ret; |
478 | ||
fc158405 | 479 | debug_fence_assert(fence); |
e30a7581 | 480 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 481 | |
cbab8d87 CW |
482 | if (dma_fence_is_signaled(dma)) { |
483 | i915_sw_fence_set_error_once(fence, dma->error); | |
e68a139f | 484 | return 0; |
cbab8d87 | 485 | } |
e68a139f | 486 | |
f255c1e9 CW |
487 | cb = kmalloc(timeout ? |
488 | sizeof(struct i915_sw_dma_fence_cb_timer) : | |
489 | sizeof(struct i915_sw_dma_fence_cb), | |
490 | gfp); | |
e68a139f CW |
491 | if (!cb) { |
492 | if (!gfpflags_allow_blocking(gfp)) | |
493 | return -ENOMEM; | |
494 | ||
cbab8d87 CW |
495 | ret = dma_fence_wait(dma, false); |
496 | if (ret) | |
497 | return ret; | |
498 | ||
499 | i915_sw_fence_set_error_once(fence, dma->error); | |
500 | return 0; | |
e68a139f CW |
501 | } |
502 | ||
9310cb7f | 503 | cb->fence = fence; |
e68a139f CW |
504 | i915_sw_fence_await(fence); |
505 | ||
f255c1e9 | 506 | func = dma_i915_sw_fence_wake; |
e68a139f | 507 | if (timeout) { |
f255c1e9 CW |
508 | struct i915_sw_dma_fence_cb_timer *timer = |
509 | container_of(cb, typeof(*timer), base); | |
c32164b1 | 510 | |
f255c1e9 CW |
511 | timer->dma = dma_fence_get(dma); |
512 | init_irq_work(&timer->work, irq_i915_sw_fence_work); | |
513 | ||
514 | timer_setup(&timer->timer, | |
c32164b1 | 515 | timer_i915_sw_fence_wake, TIMER_IRQSAFE); |
f255c1e9 CW |
516 | mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout)); |
517 | ||
518 | func = dma_i915_sw_fence_wake_timer; | |
e68a139f CW |
519 | } |
520 | ||
f255c1e9 | 521 | ret = dma_fence_add_callback(dma, &cb->base, func); |
e68a139f CW |
522 | if (ret == 0) { |
523 | ret = 1; | |
524 | } else { | |
f255c1e9 | 525 | func(dma, &cb->base); |
e68a139f CW |
526 | if (ret == -ENOENT) /* fence already signaled */ |
527 | ret = 0; | |
528 | } | |
529 | ||
530 | return ret; | |
531 | } | |
532 | ||
ea593dbb CW |
533 | static void __dma_i915_sw_fence_wake(struct dma_fence *dma, |
534 | struct dma_fence_cb *data) | |
535 | { | |
536 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
537 | ||
ef468849 | 538 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
ea593dbb CW |
539 | i915_sw_fence_complete(cb->fence); |
540 | } | |
541 | ||
542 | int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
543 | struct dma_fence *dma, | |
544 | struct i915_sw_dma_fence_cb *cb) | |
545 | { | |
546 | int ret; | |
547 | ||
548 | debug_fence_assert(fence); | |
549 | ||
cbab8d87 CW |
550 | if (dma_fence_is_signaled(dma)) { |
551 | i915_sw_fence_set_error_once(fence, dma->error); | |
ea593dbb | 552 | return 0; |
cbab8d87 | 553 | } |
ea593dbb CW |
554 | |
555 | cb->fence = fence; | |
556 | i915_sw_fence_await(fence); | |
557 | ||
a80d7367 CW |
558 | ret = 1; |
559 | if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) { | |
560 | /* fence already signaled */ | |
ef468849 | 561 | __dma_i915_sw_fence_wake(dma, &cb->base); |
a80d7367 | 562 | ret = 0; |
ea593dbb CW |
563 | } |
564 | ||
565 | return ret; | |
566 | } | |
567 | ||
e68a139f | 568 | int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, |
52791eee | 569 | struct dma_resv *resv, |
f54d1867 | 570 | const struct dma_fence_ops *exclude, |
e68a139f CW |
571 | bool write, |
572 | unsigned long timeout, | |
573 | gfp_t gfp) | |
574 | { | |
f54d1867 | 575 | struct dma_fence *excl; |
e68a139f CW |
576 | int ret = 0, pending; |
577 | ||
fc158405 | 578 | debug_fence_assert(fence); |
e30a7581 | 579 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 580 | |
e68a139f | 581 | if (write) { |
f54d1867 | 582 | struct dma_fence **shared; |
e68a139f CW |
583 | unsigned int count, i; |
584 | ||
cbab8d87 | 585 | ret = dma_resv_get_fences_rcu(resv, &excl, &count, &shared); |
e68a139f CW |
586 | if (ret) |
587 | return ret; | |
588 | ||
589 | for (i = 0; i < count; i++) { | |
590 | if (shared[i]->ops == exclude) | |
591 | continue; | |
592 | ||
593 | pending = i915_sw_fence_await_dma_fence(fence, | |
594 | shared[i], | |
595 | timeout, | |
596 | gfp); | |
597 | if (pending < 0) { | |
598 | ret = pending; | |
599 | break; | |
600 | } | |
601 | ||
602 | ret |= pending; | |
603 | } | |
604 | ||
605 | for (i = 0; i < count; i++) | |
f54d1867 | 606 | dma_fence_put(shared[i]); |
e68a139f CW |
607 | kfree(shared); |
608 | } else { | |
6b41323a | 609 | excl = dma_resv_get_excl_unlocked(resv); |
e68a139f CW |
610 | } |
611 | ||
612 | if (ret >= 0 && excl && excl->ops != exclude) { | |
613 | pending = i915_sw_fence_await_dma_fence(fence, | |
614 | excl, | |
615 | timeout, | |
616 | gfp); | |
617 | if (pending < 0) | |
618 | ret = pending; | |
619 | else | |
620 | ret |= pending; | |
621 | } | |
622 | ||
f54d1867 | 623 | dma_fence_put(excl); |
e68a139f CW |
624 | |
625 | return ret; | |
626 | } | |
47624cc3 CW |
627 | |
628 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
214707fc | 629 | #include "selftests/lib_sw_fence.c" |
47624cc3 CW |
630 | #include "selftests/i915_sw_fence.c" |
631 | #endif |