Commit | Line | Data |
---|---|---|
e68a139f | 1 | /* |
635b3bc6 | 2 | * SPDX-License-Identifier: MIT |
e68a139f | 3 | * |
635b3bc6 | 4 | * (C) Copyright 2016 Intel Corporation |
e68a139f CW |
5 | */ |
6 | ||
7 | #include <linux/slab.h> | |
f54d1867 | 8 | #include <linux/dma-fence.h> |
81c0ed21 | 9 | #include <linux/irq_work.h> |
52791eee | 10 | #include <linux/dma-resv.h> |
e68a139f CW |
11 | |
12 | #include "i915_sw_fence.h" | |
47624cc3 | 13 | #include "i915_selftest.h" |
e68a139f | 14 | |
67a3acaa CW |
15 | #if IS_ENABLED(CONFIG_DRM_I915_DEBUG) |
16 | #define I915_SW_FENCE_BUG_ON(expr) BUG_ON(expr) | |
17 | #else | |
18 | #define I915_SW_FENCE_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) | |
19 | #endif | |
20 | ||
44505168 | 21 | #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG |
e68a139f | 22 | static DEFINE_SPINLOCK(i915_sw_fence_lock); |
44505168 | 23 | #endif |
e68a139f | 24 | |
460d02ba CW |
25 | #define WQ_FLAG_BITS \ |
26 | BITS_PER_TYPE(typeof_member(struct wait_queue_entry, flags)) | |
27 | ||
28 | /* after WQ_FLAG_* for safety */ | |
29 | #define I915_SW_FENCE_FLAG_FENCE BIT(WQ_FLAG_BITS - 1) | |
30 | #define I915_SW_FENCE_FLAG_ALLOC BIT(WQ_FLAG_BITS - 2) | |
31 | ||
fc158405 CW |
32 | enum { |
33 | DEBUG_FENCE_IDLE = 0, | |
34 | DEBUG_FENCE_NOTIFY, | |
35 | }; | |
36 | ||
fc158405 CW |
37 | static void *i915_sw_fence_debug_hint(void *addr) |
38 | { | |
44505168 | 39 | return (void *)(((struct i915_sw_fence *)addr)->fn); |
fc158405 CW |
40 | } |
41 | ||
5791bad4 CW |
42 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
43 | ||
f9e62f31 | 44 | static const struct debug_obj_descr i915_sw_fence_debug_descr = { |
fc158405 CW |
45 | .name = "i915_sw_fence", |
46 | .debug_hint = i915_sw_fence_debug_hint, | |
47 | }; | |
48 | ||
49 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
50 | { | |
51 | debug_object_init(fence, &i915_sw_fence_debug_descr); | |
52 | } | |
53 | ||
214707fc CW |
54 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
55 | { | |
56 | debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); | |
57 | } | |
58 | ||
fc158405 CW |
59 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
60 | { | |
61 | debug_object_activate(fence, &i915_sw_fence_debug_descr); | |
62 | } | |
63 | ||
64 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
65 | int old, int new) | |
66 | { | |
67 | debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); | |
68 | } | |
69 | ||
70 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
71 | { | |
72 | debug_object_deactivate(fence, &i915_sw_fence_debug_descr); | |
73 | } | |
74 | ||
75 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
76 | { | |
77 | debug_object_destroy(fence, &i915_sw_fence_debug_descr); | |
78 | } | |
79 | ||
80 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
81 | { | |
82 | debug_object_free(fence, &i915_sw_fence_debug_descr); | |
6f13f29f | 83 | smp_wmb(); /* flush the change in state before reallocation */ |
fc158405 CW |
84 | } |
85 | ||
86 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
87 | { | |
88 | debug_object_assert_init(fence, &i915_sw_fence_debug_descr); | |
89 | } | |
90 | ||
91 | #else | |
92 | ||
93 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
94 | { | |
95 | } | |
96 | ||
214707fc CW |
97 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
98 | { | |
99 | } | |
100 | ||
fc158405 CW |
101 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
102 | { | |
103 | } | |
104 | ||
105 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
106 | int old, int new) | |
107 | { | |
108 | } | |
109 | ||
110 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
111 | { | |
112 | } | |
113 | ||
114 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
115 | { | |
116 | } | |
117 | ||
118 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
119 | { | |
120 | } | |
121 | ||
122 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
123 | { | |
124 | } | |
125 | ||
126 | #endif | |
127 | ||
e68a139f CW |
128 | static int __i915_sw_fence_notify(struct i915_sw_fence *fence, |
129 | enum i915_sw_fence_notify state) | |
130 | { | |
44505168 | 131 | return fence->fn(fence, state); |
e68a139f CW |
132 | } |
133 | ||
fc158405 CW |
134 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
135 | void i915_sw_fence_fini(struct i915_sw_fence *fence) | |
136 | { | |
137 | debug_fence_free(fence); | |
138 | } | |
139 | #endif | |
140 | ||
e68a139f CW |
141 | static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, |
142 | struct list_head *continuation) | |
143 | { | |
144 | wait_queue_head_t *x = &fence->wait; | |
ac6424b9 | 145 | wait_queue_entry_t *pos, *next; |
e68a139f CW |
146 | unsigned long flags; |
147 | ||
fc158405 | 148 | debug_fence_deactivate(fence); |
e68a139f CW |
149 | atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ |
150 | ||
151 | /* | |
152 | * To prevent unbounded recursion as we traverse the graph of | |
2055da97 IM |
153 | * i915_sw_fences, we move the entry list from this, the next ready |
154 | * fence, to the tail of the original fence's entry list | |
e68a139f CW |
155 | * (and so added to the list to be woken). |
156 | */ | |
157 | ||
158 | spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); | |
159 | if (continuation) { | |
2055da97 | 160 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
460d02ba | 161 | if (pos->flags & I915_SW_FENCE_FLAG_FENCE) |
2055da97 | 162 | list_move_tail(&pos->entry, continuation); |
460d02ba CW |
163 | else |
164 | pos->func(pos, TASK_NORMAL, 0, continuation); | |
e68a139f CW |
165 | } |
166 | } else { | |
167 | LIST_HEAD(extra); | |
168 | ||
169 | do { | |
ef468849 | 170 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
20612303 CW |
171 | int wake_flags; |
172 | ||
460d02ba CW |
173 | wake_flags = 0; |
174 | if (pos->flags & I915_SW_FENCE_FLAG_FENCE) | |
175 | wake_flags = fence->error; | |
20612303 CW |
176 | |
177 | pos->func(pos, TASK_NORMAL, wake_flags, &extra); | |
ef468849 | 178 | } |
e68a139f CW |
179 | |
180 | if (list_empty(&extra)) | |
181 | break; | |
182 | ||
2055da97 | 183 | list_splice_tail_init(&extra, &x->head); |
e68a139f CW |
184 | } while (1); |
185 | } | |
186 | spin_unlock_irqrestore(&x->lock, flags); | |
fc158405 CW |
187 | |
188 | debug_fence_assert(fence); | |
e68a139f CW |
189 | } |
190 | ||
191 | static void __i915_sw_fence_complete(struct i915_sw_fence *fence, | |
192 | struct list_head *continuation) | |
193 | { | |
fc158405 CW |
194 | debug_fence_assert(fence); |
195 | ||
e68a139f CW |
196 | if (!atomic_dec_and_test(&fence->pending)) |
197 | return; | |
198 | ||
fc158405 CW |
199 | debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); |
200 | ||
9310cb7f | 201 | if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) |
e68a139f CW |
202 | return; |
203 | ||
fc158405 CW |
204 | debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); |
205 | ||
e68a139f | 206 | __i915_sw_fence_wake_up_all(fence, continuation); |
9310cb7f CW |
207 | |
208 | debug_fence_destroy(fence); | |
209 | __i915_sw_fence_notify(fence, FENCE_FREE); | |
e68a139f CW |
210 | } |
211 | ||
e8861964 | 212 | void i915_sw_fence_complete(struct i915_sw_fence *fence) |
e68a139f | 213 | { |
fc158405 CW |
214 | debug_fence_assert(fence); |
215 | ||
e68a139f CW |
216 | if (WARN_ON(i915_sw_fence_done(fence))) |
217 | return; | |
218 | ||
219 | __i915_sw_fence_complete(fence, NULL); | |
220 | } | |
221 | ||
42fb60de | 222 | bool i915_sw_fence_await(struct i915_sw_fence *fence) |
e68a139f | 223 | { |
42fb60de CW |
224 | int pending; |
225 | ||
226 | /* | |
227 | * It is only safe to add a new await to the fence while it has | |
228 | * not yet been signaled (i.e. there are still existing signalers). | |
229 | */ | |
230 | pending = atomic_read(&fence->pending); | |
231 | do { | |
232 | if (pending < 1) | |
233 | return false; | |
234 | } while (!atomic_try_cmpxchg(&fence->pending, &pending, pending + 1)); | |
235 | ||
236 | return true; | |
e68a139f CW |
237 | } |
238 | ||
556b7487 CW |
239 | void __i915_sw_fence_init(struct i915_sw_fence *fence, |
240 | i915_sw_fence_notify_t fn, | |
241 | const char *name, | |
242 | struct lock_class_key *key) | |
e68a139f | 243 | { |
67a3acaa | 244 | __init_waitqueue_head(&fence->wait, name, key); |
44505168 MB |
245 | fence->fn = fn; |
246 | #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG | |
247 | fence->flags = 0; | |
248 | #endif | |
67a3acaa CW |
249 | |
250 | i915_sw_fence_reinit(fence); | |
251 | } | |
252 | ||
253 | void i915_sw_fence_reinit(struct i915_sw_fence *fence) | |
254 | { | |
fc158405 CW |
255 | debug_fence_init(fence); |
256 | ||
e68a139f | 257 | atomic_set(&fence->pending, 1); |
ef468849 CW |
258 | fence->error = 0; |
259 | ||
67a3acaa | 260 | I915_SW_FENCE_BUG_ON(!list_empty(&fence->wait.head)); |
e68a139f CW |
261 | } |
262 | ||
fc158405 CW |
263 | void i915_sw_fence_commit(struct i915_sw_fence *fence) |
264 | { | |
265 | debug_fence_activate(fence); | |
9310cb7f | 266 | i915_sw_fence_complete(fence); |
fc158405 CW |
267 | } |
268 | ||
ac6424b9 | 269 | static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) |
e68a139f | 270 | { |
ef468849 CW |
271 | i915_sw_fence_set_error_once(wq->private, flags); |
272 | ||
2055da97 | 273 | list_del(&wq->entry); |
e68a139f | 274 | __i915_sw_fence_complete(wq->private, key); |
9310cb7f | 275 | |
7e941861 CW |
276 | if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) |
277 | kfree(wq); | |
e68a139f CW |
278 | return 0; |
279 | } | |
280 | ||
44505168 | 281 | #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG |
e68a139f CW |
282 | static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, |
283 | const struct i915_sw_fence * const signaler) | |
284 | { | |
ac6424b9 | 285 | wait_queue_entry_t *wq; |
e68a139f CW |
286 | |
287 | if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
288 | return false; | |
289 | ||
290 | if (fence == signaler) | |
291 | return true; | |
292 | ||
2055da97 | 293 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
294 | if (wq->func != i915_sw_fence_wake) |
295 | continue; | |
296 | ||
297 | if (__i915_sw_fence_check_if_after(wq->private, signaler)) | |
298 | return true; | |
299 | } | |
300 | ||
301 | return false; | |
302 | } | |
303 | ||
304 | static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) | |
305 | { | |
ac6424b9 | 306 | wait_queue_entry_t *wq; |
e68a139f CW |
307 | |
308 | if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
309 | return; | |
310 | ||
2055da97 | 311 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
312 | if (wq->func != i915_sw_fence_wake) |
313 | continue; | |
314 | ||
315 | __i915_sw_fence_clear_checked_bit(wq->private); | |
316 | } | |
317 | } | |
318 | ||
319 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
320 | const struct i915_sw_fence * const signaler) | |
321 | { | |
322 | unsigned long flags; | |
323 | bool err; | |
324 | ||
e68a139f CW |
325 | spin_lock_irqsave(&i915_sw_fence_lock, flags); |
326 | err = __i915_sw_fence_check_if_after(fence, signaler); | |
327 | __i915_sw_fence_clear_checked_bit(fence); | |
328 | spin_unlock_irqrestore(&i915_sw_fence_lock, flags); | |
329 | ||
330 | return err; | |
331 | } | |
44505168 MB |
332 | #else |
333 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
334 | const struct i915_sw_fence * const signaler) | |
335 | { | |
336 | return false; | |
337 | } | |
338 | #endif | |
e68a139f | 339 | |
7e941861 CW |
340 | static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
341 | struct i915_sw_fence *signaler, | |
ac6424b9 | 342 | wait_queue_entry_t *wq, gfp_t gfp) |
e68a139f | 343 | { |
460d02ba | 344 | unsigned int pending; |
e68a139f | 345 | unsigned long flags; |
e68a139f | 346 | |
fc158405 | 347 | debug_fence_assert(fence); |
e30a7581 | 348 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 349 | |
ef468849 CW |
350 | if (i915_sw_fence_done(signaler)) { |
351 | i915_sw_fence_set_error_once(fence, signaler->error); | |
e68a139f | 352 | return 0; |
ef468849 | 353 | } |
e68a139f | 354 | |
fc158405 CW |
355 | debug_fence_assert(signaler); |
356 | ||
e68a139f CW |
357 | /* The dependency graph must be acyclic. */ |
358 | if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) | |
359 | return -EINVAL; | |
360 | ||
460d02ba | 361 | pending = I915_SW_FENCE_FLAG_FENCE; |
7e941861 CW |
362 | if (!wq) { |
363 | wq = kmalloc(sizeof(*wq), gfp); | |
364 | if (!wq) { | |
365 | if (!gfpflags_allow_blocking(gfp)) | |
366 | return -ENOMEM; | |
367 | ||
368 | i915_sw_fence_wait(signaler); | |
ef468849 | 369 | i915_sw_fence_set_error_once(fence, signaler->error); |
7e941861 CW |
370 | return 0; |
371 | } | |
372 | ||
373 | pending |= I915_SW_FENCE_FLAG_ALLOC; | |
374 | } | |
375 | ||
2055da97 | 376 | INIT_LIST_HEAD(&wq->entry); |
7e941861 | 377 | wq->flags = pending; |
e68a139f | 378 | wq->func = i915_sw_fence_wake; |
9310cb7f | 379 | wq->private = fence; |
e68a139f CW |
380 | |
381 | i915_sw_fence_await(fence); | |
382 | ||
383 | spin_lock_irqsave(&signaler->wait.lock, flags); | |
384 | if (likely(!i915_sw_fence_done(signaler))) { | |
ac6424b9 | 385 | __add_wait_queue_entry_tail(&signaler->wait, wq); |
e68a139f CW |
386 | pending = 1; |
387 | } else { | |
ef468849 | 388 | i915_sw_fence_wake(wq, 0, signaler->error, NULL); |
e68a139f CW |
389 | pending = 0; |
390 | } | |
391 | spin_unlock_irqrestore(&signaler->wait.lock, flags); | |
392 | ||
393 | return pending; | |
394 | } | |
395 | ||
7e941861 CW |
396 | int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
397 | struct i915_sw_fence *signaler, | |
ac6424b9 | 398 | wait_queue_entry_t *wq) |
7e941861 CW |
399 | { |
400 | return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); | |
401 | } | |
402 | ||
403 | int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, | |
404 | struct i915_sw_fence *signaler, | |
405 | gfp_t gfp) | |
406 | { | |
407 | return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); | |
408 | } | |
409 | ||
f255c1e9 CW |
410 | struct i915_sw_dma_fence_cb_timer { |
411 | struct i915_sw_dma_fence_cb base; | |
f54d1867 | 412 | struct dma_fence *dma; |
e68a139f | 413 | struct timer_list timer; |
81c0ed21 | 414 | struct irq_work work; |
7d622351 | 415 | struct rcu_head rcu; |
e68a139f CW |
416 | }; |
417 | ||
f255c1e9 CW |
418 | static void dma_i915_sw_fence_wake(struct dma_fence *dma, |
419 | struct dma_fence_cb *data) | |
420 | { | |
421 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
422 | ||
ef468849 | 423 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
f255c1e9 CW |
424 | i915_sw_fence_complete(cb->fence); |
425 | kfree(cb); | |
426 | } | |
427 | ||
39cbf2aa | 428 | static void timer_i915_sw_fence_wake(struct timer_list *t) |
e68a139f | 429 | { |
f255c1e9 | 430 | struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); |
81c0ed21 CW |
431 | struct i915_sw_fence *fence; |
432 | ||
f255c1e9 | 433 | fence = xchg(&cb->base.fence, NULL); |
81c0ed21 CW |
434 | if (!fence) |
435 | return; | |
e68a139f | 436 | |
2386b492 | 437 | pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n", |
5791bad4 CW |
438 | cb->dma->ops->get_driver_name(cb->dma), |
439 | cb->dma->ops->get_timeline_name(cb->dma), | |
440 | cb->dma->seqno, | |
441 | i915_sw_fence_debug_hint(fence)); | |
e68a139f | 442 | |
ef468849 | 443 | i915_sw_fence_set_error_once(fence, -ETIMEDOUT); |
81c0ed21 | 444 | i915_sw_fence_complete(fence); |
e68a139f CW |
445 | } |
446 | ||
f255c1e9 CW |
447 | static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, |
448 | struct dma_fence_cb *data) | |
e68a139f | 449 | { |
f255c1e9 CW |
450 | struct i915_sw_dma_fence_cb_timer *cb = |
451 | container_of(data, typeof(*cb), base.base); | |
81c0ed21 CW |
452 | struct i915_sw_fence *fence; |
453 | ||
f255c1e9 | 454 | fence = xchg(&cb->base.fence, NULL); |
cbab8d87 CW |
455 | if (fence) { |
456 | i915_sw_fence_set_error_once(fence, dma->error); | |
81c0ed21 | 457 | i915_sw_fence_complete(fence); |
cbab8d87 | 458 | } |
81c0ed21 | 459 | |
f255c1e9 | 460 | irq_work_queue(&cb->work); |
81c0ed21 CW |
461 | } |
462 | ||
463 | static void irq_i915_sw_fence_work(struct irq_work *wrk) | |
464 | { | |
f255c1e9 CW |
465 | struct i915_sw_dma_fence_cb_timer *cb = |
466 | container_of(wrk, typeof(*cb), work); | |
e68a139f | 467 | |
292a089d | 468 | timer_shutdown_sync(&cb->timer); |
f54d1867 | 469 | dma_fence_put(cb->dma); |
e68a139f | 470 | |
7d622351 | 471 | kfree_rcu(cb, rcu); |
e68a139f CW |
472 | } |
473 | ||
474 | int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
f54d1867 | 475 | struct dma_fence *dma, |
e68a139f CW |
476 | unsigned long timeout, |
477 | gfp_t gfp) | |
478 | { | |
f54d1867 | 479 | struct i915_sw_dma_fence_cb *cb; |
f255c1e9 | 480 | dma_fence_func_t func; |
e68a139f CW |
481 | int ret; |
482 | ||
fc158405 | 483 | debug_fence_assert(fence); |
e30a7581 | 484 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 485 | |
cbab8d87 CW |
486 | if (dma_fence_is_signaled(dma)) { |
487 | i915_sw_fence_set_error_once(fence, dma->error); | |
e68a139f | 488 | return 0; |
cbab8d87 | 489 | } |
e68a139f | 490 | |
f255c1e9 CW |
491 | cb = kmalloc(timeout ? |
492 | sizeof(struct i915_sw_dma_fence_cb_timer) : | |
493 | sizeof(struct i915_sw_dma_fence_cb), | |
494 | gfp); | |
e68a139f CW |
495 | if (!cb) { |
496 | if (!gfpflags_allow_blocking(gfp)) | |
497 | return -ENOMEM; | |
498 | ||
cbab8d87 CW |
499 | ret = dma_fence_wait(dma, false); |
500 | if (ret) | |
501 | return ret; | |
502 | ||
503 | i915_sw_fence_set_error_once(fence, dma->error); | |
504 | return 0; | |
e68a139f CW |
505 | } |
506 | ||
9310cb7f | 507 | cb->fence = fence; |
e68a139f CW |
508 | i915_sw_fence_await(fence); |
509 | ||
f255c1e9 | 510 | func = dma_i915_sw_fence_wake; |
e68a139f | 511 | if (timeout) { |
f255c1e9 CW |
512 | struct i915_sw_dma_fence_cb_timer *timer = |
513 | container_of(cb, typeof(*timer), base); | |
c32164b1 | 514 | |
f255c1e9 CW |
515 | timer->dma = dma_fence_get(dma); |
516 | init_irq_work(&timer->work, irq_i915_sw_fence_work); | |
517 | ||
518 | timer_setup(&timer->timer, | |
c32164b1 | 519 | timer_i915_sw_fence_wake, TIMER_IRQSAFE); |
f255c1e9 CW |
520 | mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout)); |
521 | ||
522 | func = dma_i915_sw_fence_wake_timer; | |
e68a139f CW |
523 | } |
524 | ||
f255c1e9 | 525 | ret = dma_fence_add_callback(dma, &cb->base, func); |
e68a139f CW |
526 | if (ret == 0) { |
527 | ret = 1; | |
528 | } else { | |
f255c1e9 | 529 | func(dma, &cb->base); |
e68a139f CW |
530 | if (ret == -ENOENT) /* fence already signaled */ |
531 | ret = 0; | |
532 | } | |
533 | ||
534 | return ret; | |
535 | } | |
536 | ||
ea593dbb CW |
537 | static void __dma_i915_sw_fence_wake(struct dma_fence *dma, |
538 | struct dma_fence_cb *data) | |
539 | { | |
540 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
541 | ||
ef468849 | 542 | i915_sw_fence_set_error_once(cb->fence, dma->error); |
ea593dbb CW |
543 | i915_sw_fence_complete(cb->fence); |
544 | } | |
545 | ||
546 | int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
547 | struct dma_fence *dma, | |
548 | struct i915_sw_dma_fence_cb *cb) | |
549 | { | |
550 | int ret; | |
551 | ||
552 | debug_fence_assert(fence); | |
553 | ||
cbab8d87 CW |
554 | if (dma_fence_is_signaled(dma)) { |
555 | i915_sw_fence_set_error_once(fence, dma->error); | |
ea593dbb | 556 | return 0; |
cbab8d87 | 557 | } |
ea593dbb CW |
558 | |
559 | cb->fence = fence; | |
560 | i915_sw_fence_await(fence); | |
561 | ||
a80d7367 CW |
562 | ret = 1; |
563 | if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) { | |
564 | /* fence already signaled */ | |
ef468849 | 565 | __dma_i915_sw_fence_wake(dma, &cb->base); |
a80d7367 | 566 | ret = 0; |
ea593dbb CW |
567 | } |
568 | ||
569 | return ret; | |
570 | } | |
571 | ||
e68a139f | 572 | int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, |
52791eee | 573 | struct dma_resv *resv, |
e68a139f CW |
574 | bool write, |
575 | unsigned long timeout, | |
576 | gfp_t gfp) | |
577 | { | |
1b5bdf07 CK |
578 | struct dma_resv_iter cursor; |
579 | struct dma_fence *f; | |
e68a139f CW |
580 | int ret = 0, pending; |
581 | ||
fc158405 | 582 | debug_fence_assert(fence); |
e30a7581 | 583 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 584 | |
7bc80a54 | 585 | dma_resv_iter_begin(&cursor, resv, dma_resv_usage_rw(write)); |
1b5bdf07 CK |
586 | dma_resv_for_each_fence_unlocked(&cursor, f) { |
587 | pending = i915_sw_fence_await_dma_fence(fence, f, timeout, | |
e68a139f | 588 | gfp); |
1b5bdf07 | 589 | if (pending < 0) { |
e68a139f | 590 | ret = pending; |
1b5bdf07 CK |
591 | break; |
592 | } | |
e68a139f | 593 | |
1b5bdf07 CK |
594 | ret |= pending; |
595 | } | |
596 | dma_resv_iter_end(&cursor); | |
e68a139f CW |
597 | return ret; |
598 | } | |
47624cc3 CW |
599 | |
600 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
214707fc | 601 | #include "selftests/lib_sw_fence.c" |
47624cc3 CW |
602 | #include "selftests/i915_sw_fence.c" |
603 | #endif |