Commit | Line | Data |
---|---|---|
e68a139f | 1 | /* |
635b3bc6 | 2 | * SPDX-License-Identifier: MIT |
e68a139f | 3 | * |
635b3bc6 | 4 | * (C) Copyright 2016 Intel Corporation |
e68a139f CW |
5 | */ |
6 | ||
7 | #include <linux/slab.h> | |
f54d1867 | 8 | #include <linux/dma-fence.h> |
81c0ed21 | 9 | #include <linux/irq_work.h> |
52791eee | 10 | #include <linux/dma-resv.h> |
e68a139f CW |
11 | |
12 | #include "i915_sw_fence.h" | |
47624cc3 | 13 | #include "i915_selftest.h" |
e68a139f | 14 | |
67a3acaa CW |
15 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG) |
16 | #define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr) | |
17 | #else | |
18 | #define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) | |
19 | #endif | |
20 | ||
44505168 | 21 | #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG |
e68a139f | 22 | static DEFINE_SPINLOCK(i915_sw_fence_lock); |
44505168 | 23 | #endif |
e68a139f | 24 | |
460d02ba CW |
25 | #define WQ_FLAG_BITS \ |
26 | BITS_PER_TYPE(typeof_member(struct wait_queue_entry, flags)) | |
27 | ||
28 | /* after WQ_FLAG_* for safety */ | |
29 | #define I915_SW_FENCE_FLAG_FENCE BIT(WQ_FLAG_BITS - 1) | |
30 | #define I915_SW_FENCE_FLAG_ALLOC BIT(WQ_FLAG_BITS - 2) | |
31 | ||
fc158405 CW |
32 | enum { |
33 | DEBUG_FENCE_IDLE = 0, | |
34 | DEBUG_FENCE_NOTIFY, | |
35 | }; | |
36 | ||
fc158405 CW |
37 | static void *i915_sw_fence_debug_hint(void *addr) |
38 | { | |
44505168 | 39 | return (void *)(((struct i915_sw_fence *)addr)->fn); |
fc158405 CW |
40 | } |
41 | ||
5791bad4 CW |
42 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
43 | ||
f9e62f31 | 44 | static const struct debug_obj_descr i915_sw_fence_debug_descr = { |
fc158405 CW |
45 | .name = "i915_sw_fence", |
46 | .debug_hint = i915_sw_fence_debug_hint, | |
47 | }; | |
48 | ||
49 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
50 | { | |
51 | debug_object_init(fence, &i915_sw_fence_debug_descr); | |
52 | } | |
53 | ||
214707fc CW |
54 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
55 | { | |
56 | debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); | |
57 | } | |
58 | ||
fc158405 CW |
59 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
60 | { | |
61 | debug_object_activate(fence, &i915_sw_fence_debug_descr); | |
62 | } | |
63 | ||
64 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
65 | int old, int new) | |
66 | { | |
67 | debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); | |
68 | } | |
69 | ||
70 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
71 | { | |
72 | debug_object_deactivate(fence, &i915_sw_fence_debug_descr); | |
73 | } | |
74 | ||
75 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
76 | { | |
77 | debug_object_destroy(fence, &i915_sw_fence_debug_descr); | |
78 | } | |
79 | ||
80 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
81 | { | |
82 | debug_object_free(fence, &i915_sw_fence_debug_descr); | |
6f13f29f | 83 | smp_wmb(); /* flush the change in state before reallocation */ |
fc158405 CW |
84 | } |
85 | ||
86 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
87 | { | |
88 | debug_object_assert_init(fence, &i915_sw_fence_debug_descr); | |
89 | } | |
90 | ||
91 | #else | |
92 | ||
93 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
94 | { | |
95 | } | |
96 | ||
214707fc CW |
97 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
98 | { | |
99 | } | |
100 | ||
fc158405 CW |
101 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
102 | { | |
103 | } | |
104 | ||
105 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
106 | int old, int new) | |
107 | { | |
108 | } | |
109 | ||
110 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
111 | { | |
112 | } | |
113 | ||
114 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
115 | { | |
116 | } | |
117 | ||
118 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
119 | { | |
120 | } | |
121 | ||
122 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
123 | { | |
124 | } | |
125 | ||
126 | #endif | |
127 | ||
e68a139f CW |
128 | static int __i915_sw_fence_notify(struct i915_sw_fence *fence, |
129 | enum i915_sw_fence_notify state) | |
130 | { | |
44505168 | 131 | return fence->fn(fence, state); |
e68a139f CW |
132 | } |
133 | ||
fc158405 CW |
134 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
135 | void i915_sw_fence_fini(struct i915_sw_fence *fence) | |
136 | { | |
137 | debug_fence_free(fence); | |
138 | } | |
139 | #endif | |
140 | ||
e68a139f CW |
141 | static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, |
142 | struct list_head *continuation) | |
143 | { | |
144 | wait_queue_head_t *x = &fence->wait; | |
ac6424b9 | 145 | wait_queue_entry_t *pos, *next; |
e68a139f CW |
146 | unsigned long flags; |
147 | ||
fc158405 | 148 | debug_fence_deactivate(fence); |
e68a139f CW |
149 | atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ |
150 | ||
151 | /* | |
152 | * To prevent unbounded recursion as we traverse the graph of | |
2055da97 IM |
153 | * i915_sw_fences, we move the entry list from this, the next ready |
154 | * fence, to the tail of the original fence's entry list | |
e68a139f CW |
155 | * (and so added to the list to be woken). |
156 | */ | |
157 | ||
158 | spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); | |
159 | if (continuation) { | |
2055da97 | 160 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
460d02ba | 161 | if (pos->flags & I915_SW_FENCE_FLAG_FENCE) |
2055da97 | 162 | list_move_tail(&pos->entry, continuation); |
460d02ba CW |
163 | else |
164 | pos->func(pos, TASK_NORMAL, 0, continuation); | |
e68a139f CW |
165 | } |
166 | } else { | |
167 | LIST_HEAD(extra); | |
168 | ||
169 | do { | |
ef468849 | 170 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
20612303 CW |
171 | int wake_flags; |
172 | ||
460d02ba CW |
173 | wake_flags = 0; |
174 | if (pos->flags & I915_SW_FENCE_FLAG_FENCE) | |
175 | wake_flags = fence->error; | |
20612303 CW |
176 | |
177 | pos->func(pos, TASK_NORMAL, wake_flags, &extra); | |
ef468849 | 178 | } |
e68a139f CW |
179 | |
180 | if (list_empty(&extra)) | |
181 | break; | |
182 | ||
2055da97 | 183 | list_splice_tail_init(&extra, &x->head); |
e68a139f CW |
184 | } while (1); |
185 | } | |
186 | spin_unlock_irqrestore(&x->lock, flags); | |
fc158405 CW |
187 | |
188 | debug_fence_assert(fence); | |
e68a139f CW |
189 | } |
190 | ||
191 | static void __i915_sw_fence_complete(struct i915_sw_fence *fence, | |
192 | struct list_head *continuation) | |
193 | { | |
fc158405 CW |
194 | debug_fence_assert(fence); |
195 | ||
e68a139f CW |
196 | if (!atomic_dec_and_test(&fence->pending)) |
197 | return; | |
198 | ||
fc158405 CW |
199 | debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); |
200 | ||
9310cb7f | 201 | if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) |
e68a139f CW |
202 | return; |
203 | ||
fc158405 CW |
204 | debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); |
205 | ||
e68a139f | 206 | __i915_sw_fence_wake_up_all(fence, continuation); |
9310cb7f CW |
207 | |
208 | debug_fence_destroy(fence); | |
209 | __i915_sw_fence_notify(fence, FENCE_FREE); | |
e68a139f CW |
210 | } |
211 | ||
e8861964 | 212 | void i915_sw_fence_complete(struct i915_sw_fence *fence) |
e68a139f | 213 | { |
fc158405 CW |
214 | debug_fence_assert(fence); |
215 | ||
e68a139f CW |
216 | if (WARN_ON(i915_sw_fence_done(fence))) |
217 | return; | |
218 | ||
219 | __i915_sw_fence_complete(fence, NULL); | |
220 | } | |
221 | ||
42fb60de | 222 | bool i915_sw_fence_await(struct i915_sw_fence *fence) |
e68a139f | 223 | { |
42fb60de CW |
224 | int pending; |
225 | ||
226 | /* | |
227 | * It is only safe to add a new await to the fence while it has | |
228 | * not yet been signaled (i.e. there are still existing signalers). | |
229 | */ | |
230 | pending = atomic_read(&fence->pending); | |
231 | do { | |
232 | if (pending < 1) | |
233 | return false; | |
234 | } while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1)); | |
235 | ||
236 | return true; | |
e68a139f CW |
237 | } |
238 | ||
556b7487 CW |
239 | void __i915_sw_fence_init(struct i915_sw_fence *fence, |
240 | i915_sw_fence_notify_t fn, | |
241 | const char *name, | |
242 | struct lock_class_key *key) | |
e68a139f | 243 | { |
44505168 | 244 | BUG_ON(!fn); |
e68a139f | 245 | |
67a3acaa | 246 | __init_waitqueue_head(&fence->wait, name, key); |
44505168 MB |
247 | fence->fn = fn; |
248 | #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG | |
249 | fence->flags = 0; | |
250 | #endif | |
67a3acaa CW |
251 | |
252 | i915_sw_fence_reinit(fence); | |
253 | } | |
254 | ||
255 | void i915_sw_fence_reinit(struct i915_sw_fence *fence) | |
256 | { | |
fc158405 CW |
257 | debug_fence_init(fence); |
258 | ||
e68a139f | 259 | atomic_set(&fence->pending, 1); |
ef468849 CW |
260 | fence->error = 0; |
261 | ||
67a3acaa | 262 | I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head)); |
e68a139f CW |
263 | } |
264 | ||
fc158405 CW |
265 | void i915_sw_fence_commit(struct i915_sw_fence *fence) |
266 | { | |
267 | debug_fence_activate(fence); | |
9310cb7f | 268 | i915_sw_fence_complete(fence); |
fc158405 CW |
269 | } |
270 | ||
ac6424b9 | 271 | static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) |
e68a139f | 272 | { |
ef468849 CW |
273 | i915_sw_fence_set_error_once(wq->private, flags); |
274 | ||
2055da97 | 275 | list_del(&wq->entry); |
e68a139f | 276 | __i915_sw_fence_complete(wq->private, key); |
9310cb7f | 277 | |
7e941861 CW |
278 | if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) |
279 | kfree(wq); | |
e68a139f CW |
280 | return 0; |
281 | } | |
282 | ||
44505168 | 283 | #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG |
e68a139f CW |
284 | static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, |
285 | const struct i915_sw_fence * const signaler) | |
286 | { | |
ac6424b9 | 287 | wait_queue_entry_t *wq; |
e68a139f CW |
288 | |
289 | if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
290 | return false; | |
291 | ||
292 | if (fence == signaler) | |
293 | return true; | |
294 | ||
2055da97 | 295 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
296 | if (wq->func != i915_sw_fence_wake) |
297 | continue; | |
298 | ||
299 | if (__i915_sw_fence_check_if_after(wq->private, signaler)) | |
300 | return true; | |
301 | } | |
302 | ||
303 | return false; | |
304 | } | |
305 | ||
306 | static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) | |
307 | { | |
ac6424b9 | 308 | wait_queue_entry_t *wq; |
e68a139f CW |
309 | |
310 | if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
311 | return; | |
312 | ||
2055da97 | 313 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
314 | if (wq->func != i915_sw_fence_wake) |
315 | continue; | |
316 | ||
317 | __i915_sw_fence_clear_checked_bit(wq->private); | |
318 | } | |
319 | } | |
320 | ||
321 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
322 | const struct i915_sw_fence * const signaler) | |
323 | { | |
324 | unsigned long flags; | |
325 | bool err; | |
326 | ||
e68a139f CW |
327 | spin_lock_irqsave(&i915_sw_fence_lock, flags); |
328 | err = __i915_sw_fence_check_if_after(fence, signaler); | |
329 | __i915_sw_fence_clear_checked_bit(fence); | |
330 | spin_unlock_irqrestore(&i915_sw_fence_lock, flags); | |
331 | ||
332 | return err; | |
333 | } | |
44505168 MB |
334 | #else |
335 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
336 | const struct i915_sw_fence * const signaler) | |
337 | { | |
338 | return false; | |
339 | } | |
340 | #endif | |
e68a139f | 341 | |
7e941861 CW |
342 | static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
343 | struct i915_sw_fence *signaler, | |
ac6424b9 | 344 | wait_queue_entry_t *wq, gfp_t gfp) |
e68a139f | 345 | { |
460d02ba | 346 | unsigned int pending; |
e68a139f | 347 | unsigned long flags; |
e68a139f | 348 | |
fc158405 | 349 | debug_fence_assert(fence); |
e30a7581 | 350 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 351 | |
ef468849 CW |
352 | if (i915_sw_fence_done(signaler)) { |
353 | i915_sw_fence_set_error_once(fence, signaler->error); | |
e68a139f | 354 | return 0; |
ef468849 | 355 | } |
e68a139f | 356 | |
fc158405 CW |
357 | debug_fence_assert(signaler); |
358 | ||
e68a139f CW |
359 | /* The dependency graph must be acyclic. */ |
360 | if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) | |
361 | return -EINVAL; | |
362 | ||
460d02ba | 363 | pending = I915_SW_FENCE_FLAG_FENCE; |
7e941861 CW |
364 | if (!wq) { |
365 | wq = kmalloc(sizeof(*wq), gfp); | |
366 | if (!wq) { | |
367 | if (!gfpflags_allow_blocking(gfp)) | |
368 | return -ENOMEM; | |
369 | ||
370 | i915_sw_fence_wait(signaler); | |
ef468849 | 371 | i915_sw_fence_set_error_once(fence, signaler->error); |
7e941861 CW |
372 | return 0; |
373 | } | |
374 | ||
375 | pending |= I915_SW_FENCE_FLAG_ALLOC; | |
376 | } | |
377 | ||
2055da97 | 378 | INIT_LIST_HEAD(&wq->entry); |
7e941861 | 379 | wq->flags = pending; |
e68a139f | 380 | wq->func = i915_sw_fence_wake; |
9310cb7f | 381 | wq->private = fence; |
e68a139f CW |
382 | |
383 | i915_sw_fence_await(fence); | |
384 | ||
385 | spin_lock_irqsave(&signaler->wait.lock, flags); | |
386 | if (likely(!i915_sw_fence_done(signaler))) { | |
ac6424b9 | 387 | __add_wait_queue_entry_tail(&signaler->wait, wq); |
e68a139f CW |
388 | pending = 1; |
389 | } else { | |
ef468849 | 390 | i915_sw_fence_wake(wq, 0, signaler->error, NULL); |
e68a139f CW |
391 | pending = 0; |
392 | } | |
393 | spin_unlock_irqrestore(&signaler->wait.lock, flags); | |
394 | ||
395 | return pending; | |
396 | } | |
397 | ||
7e941861 CW |
398 | int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
399 | struct i915_sw_fence *signaler, | |
ac6424b9 | 400 | wait_queue_entry_t *wq) |
7e941861 CW |
401 | { |
402 | return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); | |
403 | } | |
404 | ||
405 | int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, | |
406 | struct i915_sw_fence *signaler, | |
407 | gfp_t gfp) | |
408 | { | |
409 | return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); | |
410 | } | |
411 | ||
f255c1e9 CW |
412 | struct i915_sw_dma_fence_cb_timer { |
413 | struct i915_sw_dma_fence_cb base; | |
f54d1867 | 414 | struct dma_fence *dma; |
e68a139f | 415 | struct timer_list timer; |
81c0ed21 | 416 | struct irq_work work; |
7d622351 | 417 | struct rcu_head rcu; |
e68a139f CW |
418 | }; |
419 | ||
f255c1e9 CW |
420 | static void dma_i915_sw_fence_wake(struct dma_fence *dma, |
421 | struct dma_fence_cb *data) | |
422 | { | |
423 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
424 | ||
ef468849 | 425 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
f255c1e9 CW |
426 | i915_sw_fence_complete(cb->fence); |
427 | kfree(cb); | |
428 | } | |
429 | ||
39cbf2aa | 430 | static void timer_i915_sw_fence_wake(struct timer_list *t) |
e68a139f | 431 | { |
f255c1e9 | 432 | struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); |
81c0ed21 CW |
433 | struct i915_sw_fence *fence; |
434 | ||
f255c1e9 | 435 | fence = xchg(&cb->base.fence, NULL); |
81c0ed21 CW |
436 | if (!fence) |
437 | return; | |
e68a139f | 438 | |
2386b492 | 439 | pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n", |
5791bad4 CW |
440 | cb->dma->ops->get_driver_name(cb->dma), |
441 | cb->dma->ops->get_timeline_name(cb->dma), | |
442 | cb->dma->seqno, | |
443 | i915_sw_fence_debug_hint(fence)); | |
e68a139f | 444 | |
ef468849 | 445 | i915_sw_fence_set_error_once(fence, -ETIMEDOUT); |
81c0ed21 | 446 | i915_sw_fence_complete(fence); |
e68a139f CW |
447 | } |
448 | ||
f255c1e9 CW |
449 | static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, |
450 | struct dma_fence_cb *data) | |
e68a139f | 451 | { |
f255c1e9 CW |
452 | struct i915_sw_dma_fence_cb_timer *cb = |
453 | container_of(data, typeof(*cb), base.base); | |
81c0ed21 CW |
454 | struct i915_sw_fence *fence; |
455 | ||
f255c1e9 | 456 | fence = xchg(&cb->base.fence, NULL); |
cbab8d87 CW |
457 | if (fence) { |
458 | i915_sw_fence_set_error_once(fence, dma->error); | |
81c0ed21 | 459 | i915_sw_fence_complete(fence); |
cbab8d87 | 460 | } |
81c0ed21 | 461 | |
f255c1e9 | 462 | irq_work_queue(&cb->work); |
81c0ed21 CW |
463 | } |
464 | ||
465 | static void irq_i915_sw_fence_work(struct irq_work *wrk) | |
466 | { | |
f255c1e9 CW |
467 | struct i915_sw_dma_fence_cb_timer *cb = |
468 | container_of(wrk, typeof(*cb), work); | |
e68a139f CW |
469 | |
470 | del_timer_sync(&cb->timer); | |
f54d1867 | 471 | dma_fence_put(cb->dma); |
e68a139f | 472 | |
7d622351 | 473 | kfree_rcu(cb, rcu); |
e68a139f CW |
474 | } |
475 | ||
476 | int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
f54d1867 | 477 | struct dma_fence *dma, |
e68a139f CW |
478 | unsigned long timeout, |
479 | gfp_t gfp) | |
480 | { | |
f54d1867 | 481 | struct i915_sw_dma_fence_cb *cb; |
f255c1e9 | 482 | dma_fence_func_t func; |
e68a139f CW |
483 | int ret; |
484 | ||
fc158405 | 485 | debug_fence_assert(fence); |
e30a7581 | 486 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 487 | |
cbab8d87 CW |
488 | if (dma_fence_is_signaled(dma)) { |
489 | i915_sw_fence_set_error_once(fence, dma->error); | |
e68a139f | 490 | return 0; |
cbab8d87 | 491 | } |
e68a139f | 492 | |
f255c1e9 CW |
493 | cb = kmalloc(timeout ? |
494 | sizeof(struct i915_sw_dma_fence_cb_timer) : | |
495 | sizeof(struct i915_sw_dma_fence_cb), | |
496 | gfp); | |
e68a139f CW |
497 | if (!cb) { |
498 | if (!gfpflags_allow_blocking(gfp)) | |
499 | return -ENOMEM; | |
500 | ||
cbab8d87 CW |
501 | ret = dma_fence_wait(dma, false); |
502 | if (ret) | |
503 | return ret; | |
504 | ||
505 | i915_sw_fence_set_error_once(fence, dma->error); | |
506 | return 0; | |
e68a139f CW |
507 | } |
508 | ||
9310cb7f | 509 | cb->fence = fence; |
e68a139f CW |
510 | i915_sw_fence_await(fence); |
511 | ||
f255c1e9 | 512 | func = dma_i915_sw_fence_wake; |
e68a139f | 513 | if (timeout) { |
f255c1e9 CW |
514 | struct i915_sw_dma_fence_cb_timer *timer = |
515 | container_of(cb, typeof(*timer), base); | |
c32164b1 | 516 | |
f255c1e9 CW |
517 | timer->dma = dma_fence_get(dma); |
518 | init_irq_work(&timer->work, irq_i915_sw_fence_work); | |
519 | ||
520 | timer_setup(&timer->timer, | |
c32164b1 | 521 | timer_i915_sw_fence_wake, TIMER_IRQSAFE); |
f255c1e9 CW |
522 | mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout)); |
523 | ||
524 | func = dma_i915_sw_fence_wake_timer; | |
e68a139f CW |
525 | } |
526 | ||
f255c1e9 | 527 | ret = dma_fence_add_callback(dma, &cb->base, func); |
e68a139f CW |
528 | if (ret == 0) { |
529 | ret = 1; | |
530 | } else { | |
f255c1e9 | 531 | func(dma, &cb->base); |
e68a139f CW |
532 | if (ret == -ENOENT) /* fence already signaled */ |
533 | ret = 0; | |
534 | } | |
535 | ||
536 | return ret; | |
537 | } | |
538 | ||
ea593dbb CW |
539 | static void __dma_i915_sw_fence_wake(struct dma_fence *dma, |
540 | struct dma_fence_cb *data) | |
541 | { | |
542 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
543 | ||
ef468849 | 544 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
ea593dbb CW |
545 | i915_sw_fence_complete(cb->fence); |
546 | } | |
547 | ||
548 | int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
549 | struct dma_fence *dma, | |
550 | struct i915_sw_dma_fence_cb *cb) | |
551 | { | |
552 | int ret; | |
553 | ||
554 | debug_fence_assert(fence); | |
555 | ||
cbab8d87 CW |
556 | if (dma_fence_is_signaled(dma)) { |
557 | i915_sw_fence_set_error_once(fence, dma->error); | |
ea593dbb | 558 | return 0; |
cbab8d87 | 559 | } |
ea593dbb CW |
560 | |
561 | cb->fence = fence; | |
562 | i915_sw_fence_await(fence); | |
563 | ||
a80d7367 CW |
564 | ret = 1; |
565 | if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) { | |
566 | /* fence already signaled */ | |
ef468849 | 567 | __dma_i915_sw_fence_wake(dma, &cb->base); |
a80d7367 | 568 | ret = 0; |
ea593dbb CW |
569 | } |
570 | ||
571 | return ret; | |
572 | } | |
573 | ||
e68a139f | 574 | int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, |
52791eee | 575 | struct dma_resv *resv, |
f54d1867 | 576 | const struct dma_fence_ops *exclude, |
e68a139f CW |
577 | bool write, |
578 | unsigned long timeout, | |
579 | gfp_t gfp) | |
580 | { | |
1b5bdf07 CK |
581 | struct dma_resv_iter cursor; |
582 | struct dma_fence *f; | |
e68a139f CW |
583 | int ret = 0, pending; |
584 | ||
fc158405 | 585 | debug_fence_assert(fence); |
e30a7581 | 586 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 587 | |
1b5bdf07 CK |
588 | dma_resv_iter_begin(&cursor, resv, write); |
589 | dma_resv_for_each_fence_unlocked(&cursor, f) { | |
590 | pending = i915_sw_fence_await_dma_fence(fence, f, timeout, | |
e68a139f | 591 | gfp); |
1b5bdf07 | 592 | if (pending < 0) { |
e68a139f | 593 | ret = pending; |
1b5bdf07 CK |
594 | break; |
595 | } | |
e68a139f | 596 | |
1b5bdf07 CK |
597 | ret |= pending; |
598 | } | |
599 | dma_resv_iter_end(&cursor); | |
e68a139f CW |
600 | return ret; |
601 | } | |
47624cc3 CW |
602 | |
603 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
214707fc | 604 | #include "selftests/lib_sw_fence.c" |
47624cc3 CW |
605 | #include "selftests/i915_sw_fence.c" |
606 | #endif |