Commit | Line | Data |
---|---|---|
e68a139f CW |
1 | /* |
2 | * (C) Copyright 2016 Intel Corporation | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; version 2 | |
7 | * of the License. | |
8 | */ | |
9 | ||
10 | #include <linux/slab.h> | |
f54d1867 | 11 | #include <linux/dma-fence.h> |
81c0ed21 | 12 | #include <linux/irq_work.h> |
e68a139f CW |
13 | #include <linux/reservation.h> |
14 | ||
15 | #include "i915_sw_fence.h" | |
47624cc3 | 16 | #include "i915_selftest.h" |
e68a139f | 17 | |
7e941861 CW |
18 | #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ |
19 | ||
e68a139f CW |
20 | static DEFINE_SPINLOCK(i915_sw_fence_lock); |
21 | ||
fc158405 CW |
22 | enum { |
23 | DEBUG_FENCE_IDLE = 0, | |
24 | DEBUG_FENCE_NOTIFY, | |
25 | }; | |
26 | ||
27 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS | |
28 | ||
29 | static void *i915_sw_fence_debug_hint(void *addr) | |
30 | { | |
31 | return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); | |
32 | } | |
33 | ||
34 | static struct debug_obj_descr i915_sw_fence_debug_descr = { | |
35 | .name = "i915_sw_fence", | |
36 | .debug_hint = i915_sw_fence_debug_hint, | |
37 | }; | |
38 | ||
39 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
40 | { | |
41 | debug_object_init(fence, &i915_sw_fence_debug_descr); | |
42 | } | |
43 | ||
214707fc CW |
44 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
45 | { | |
46 | debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); | |
47 | } | |
48 | ||
fc158405 CW |
49 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
50 | { | |
51 | debug_object_activate(fence, &i915_sw_fence_debug_descr); | |
52 | } | |
53 | ||
54 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
55 | int old, int new) | |
56 | { | |
57 | debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); | |
58 | } | |
59 | ||
60 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
61 | { | |
62 | debug_object_deactivate(fence, &i915_sw_fence_debug_descr); | |
63 | } | |
64 | ||
65 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
66 | { | |
67 | debug_object_destroy(fence, &i915_sw_fence_debug_descr); | |
68 | } | |
69 | ||
70 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
71 | { | |
72 | debug_object_free(fence, &i915_sw_fence_debug_descr); | |
6f13f29f | 73 | smp_wmb(); /* flush the change in state before reallocation */ |
fc158405 CW |
74 | } |
75 | ||
76 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
77 | { | |
78 | debug_object_assert_init(fence, &i915_sw_fence_debug_descr); | |
79 | } | |
80 | ||
81 | #else | |
82 | ||
83 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
84 | { | |
85 | } | |
86 | ||
214707fc CW |
87 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
88 | { | |
89 | } | |
90 | ||
fc158405 CW |
91 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
92 | { | |
93 | } | |
94 | ||
95 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
96 | int old, int new) | |
97 | { | |
98 | } | |
99 | ||
100 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
101 | { | |
102 | } | |
103 | ||
104 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
105 | { | |
106 | } | |
107 | ||
108 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
109 | { | |
110 | } | |
111 | ||
112 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
113 | { | |
114 | } | |
115 | ||
116 | #endif | |
117 | ||
e68a139f CW |
118 | static int __i915_sw_fence_notify(struct i915_sw_fence *fence, |
119 | enum i915_sw_fence_notify state) | |
120 | { | |
121 | i915_sw_fence_notify_t fn; | |
122 | ||
123 | fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); | |
124 | return fn(fence, state); | |
125 | } | |
126 | ||
fc158405 CW |
127 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
128 | void i915_sw_fence_fini(struct i915_sw_fence *fence) | |
129 | { | |
130 | debug_fence_free(fence); | |
131 | } | |
132 | #endif | |
133 | ||
e68a139f CW |
134 | static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, |
135 | struct list_head *continuation) | |
136 | { | |
137 | wait_queue_head_t *x = &fence->wait; | |
ac6424b9 | 138 | wait_queue_entry_t *pos, *next; |
e68a139f CW |
139 | unsigned long flags; |
140 | ||
fc158405 | 141 | debug_fence_deactivate(fence); |
e68a139f CW |
142 | atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ |
143 | ||
144 | /* | |
145 | * To prevent unbounded recursion as we traverse the graph of | |
2055da97 IM |
146 | * i915_sw_fences, we move the entry list from this, the next ready |
147 | * fence, to the tail of the original fence's entry list | |
e68a139f CW |
148 | * (and so added to the list to be woken). |
149 | */ | |
150 | ||
151 | spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); | |
152 | if (continuation) { | |
2055da97 | 153 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
e68a139f CW |
154 | if (pos->func == autoremove_wake_function) |
155 | pos->func(pos, TASK_NORMAL, 0, continuation); | |
156 | else | |
2055da97 | 157 | list_move_tail(&pos->entry, continuation); |
e68a139f CW |
158 | } |
159 | } else { | |
160 | LIST_HEAD(extra); | |
161 | ||
162 | do { | |
2055da97 | 163 | list_for_each_entry_safe(pos, next, &x->head, entry) |
e68a139f CW |
164 | pos->func(pos, TASK_NORMAL, 0, &extra); |
165 | ||
166 | if (list_empty(&extra)) | |
167 | break; | |
168 | ||
2055da97 | 169 | list_splice_tail_init(&extra, &x->head); |
e68a139f CW |
170 | } while (1); |
171 | } | |
172 | spin_unlock_irqrestore(&x->lock, flags); | |
fc158405 CW |
173 | |
174 | debug_fence_assert(fence); | |
e68a139f CW |
175 | } |
176 | ||
177 | static void __i915_sw_fence_complete(struct i915_sw_fence *fence, | |
178 | struct list_head *continuation) | |
179 | { | |
fc158405 CW |
180 | debug_fence_assert(fence); |
181 | ||
e68a139f CW |
182 | if (!atomic_dec_and_test(&fence->pending)) |
183 | return; | |
184 | ||
fc158405 CW |
185 | debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); |
186 | ||
9310cb7f | 187 | if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) |
e68a139f CW |
188 | return; |
189 | ||
fc158405 CW |
190 | debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); |
191 | ||
e68a139f | 192 | __i915_sw_fence_wake_up_all(fence, continuation); |
9310cb7f CW |
193 | |
194 | debug_fence_destroy(fence); | |
195 | __i915_sw_fence_notify(fence, FENCE_FREE); | |
e68a139f CW |
196 | } |
197 | ||
198 | static void i915_sw_fence_complete(struct i915_sw_fence *fence) | |
199 | { | |
fc158405 CW |
200 | debug_fence_assert(fence); |
201 | ||
e68a139f CW |
202 | if (WARN_ON(i915_sw_fence_done(fence))) |
203 | return; | |
204 | ||
205 | __i915_sw_fence_complete(fence, NULL); | |
206 | } | |
207 | ||
208 | static void i915_sw_fence_await(struct i915_sw_fence *fence) | |
209 | { | |
fc158405 | 210 | debug_fence_assert(fence); |
e68a139f CW |
211 | WARN_ON(atomic_inc_return(&fence->pending) <= 1); |
212 | } | |
213 | ||
556b7487 CW |
214 | void __i915_sw_fence_init(struct i915_sw_fence *fence, |
215 | i915_sw_fence_notify_t fn, | |
216 | const char *name, | |
217 | struct lock_class_key *key) | |
e68a139f | 218 | { |
9310cb7f | 219 | BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK); |
e68a139f | 220 | |
fc158405 CW |
221 | debug_fence_init(fence); |
222 | ||
556b7487 | 223 | __init_waitqueue_head(&fence->wait, name, key); |
e68a139f CW |
224 | atomic_set(&fence->pending, 1); |
225 | fence->flags = (unsigned long)fn; | |
226 | } | |
227 | ||
fc158405 CW |
228 | void i915_sw_fence_commit(struct i915_sw_fence *fence) |
229 | { | |
230 | debug_fence_activate(fence); | |
9310cb7f | 231 | i915_sw_fence_complete(fence); |
fc158405 CW |
232 | } |
233 | ||
ac6424b9 | 234 | static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) |
e68a139f | 235 | { |
2055da97 | 236 | list_del(&wq->entry); |
e68a139f | 237 | __i915_sw_fence_complete(wq->private, key); |
9310cb7f | 238 | |
7e941861 CW |
239 | if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) |
240 | kfree(wq); | |
e68a139f CW |
241 | return 0; |
242 | } | |
243 | ||
244 | static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
245 | const struct i915_sw_fence * const signaler) | |
246 | { | |
ac6424b9 | 247 | wait_queue_entry_t *wq; |
e68a139f CW |
248 | |
249 | if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
250 | return false; | |
251 | ||
252 | if (fence == signaler) | |
253 | return true; | |
254 | ||
2055da97 | 255 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
256 | if (wq->func != i915_sw_fence_wake) |
257 | continue; | |
258 | ||
259 | if (__i915_sw_fence_check_if_after(wq->private, signaler)) | |
260 | return true; | |
261 | } | |
262 | ||
263 | return false; | |
264 | } | |
265 | ||
266 | static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) | |
267 | { | |
ac6424b9 | 268 | wait_queue_entry_t *wq; |
e68a139f CW |
269 | |
270 | if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
271 | return; | |
272 | ||
2055da97 | 273 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
274 | if (wq->func != i915_sw_fence_wake) |
275 | continue; | |
276 | ||
277 | __i915_sw_fence_clear_checked_bit(wq->private); | |
278 | } | |
279 | } | |
280 | ||
281 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
282 | const struct i915_sw_fence * const signaler) | |
283 | { | |
284 | unsigned long flags; | |
285 | bool err; | |
286 | ||
47624cc3 | 287 | if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG)) |
e68a139f CW |
288 | return false; |
289 | ||
290 | spin_lock_irqsave(&i915_sw_fence_lock, flags); | |
291 | err = __i915_sw_fence_check_if_after(fence, signaler); | |
292 | __i915_sw_fence_clear_checked_bit(fence); | |
293 | spin_unlock_irqrestore(&i915_sw_fence_lock, flags); | |
294 | ||
295 | return err; | |
296 | } | |
297 | ||
7e941861 CW |
298 | static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
299 | struct i915_sw_fence *signaler, | |
ac6424b9 | 300 | wait_queue_entry_t *wq, gfp_t gfp) |
e68a139f CW |
301 | { |
302 | unsigned long flags; | |
303 | int pending; | |
304 | ||
fc158405 CW |
305 | debug_fence_assert(fence); |
306 | ||
e68a139f CW |
307 | if (i915_sw_fence_done(signaler)) |
308 | return 0; | |
309 | ||
fc158405 CW |
310 | debug_fence_assert(signaler); |
311 | ||
e68a139f CW |
312 | /* The dependency graph must be acyclic. */ |
313 | if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) | |
314 | return -EINVAL; | |
315 | ||
7e941861 CW |
316 | pending = 0; |
317 | if (!wq) { | |
318 | wq = kmalloc(sizeof(*wq), gfp); | |
319 | if (!wq) { | |
320 | if (!gfpflags_allow_blocking(gfp)) | |
321 | return -ENOMEM; | |
322 | ||
323 | i915_sw_fence_wait(signaler); | |
324 | return 0; | |
325 | } | |
326 | ||
327 | pending |= I915_SW_FENCE_FLAG_ALLOC; | |
328 | } | |
329 | ||
2055da97 | 330 | INIT_LIST_HEAD(&wq->entry); |
7e941861 | 331 | wq->flags = pending; |
e68a139f | 332 | wq->func = i915_sw_fence_wake; |
9310cb7f | 333 | wq->private = fence; |
e68a139f CW |
334 | |
335 | i915_sw_fence_await(fence); | |
336 | ||
337 | spin_lock_irqsave(&signaler->wait.lock, flags); | |
338 | if (likely(!i915_sw_fence_done(signaler))) { | |
ac6424b9 | 339 | __add_wait_queue_entry_tail(&signaler->wait, wq); |
e68a139f CW |
340 | pending = 1; |
341 | } else { | |
342 | i915_sw_fence_wake(wq, 0, 0, NULL); | |
343 | pending = 0; | |
344 | } | |
345 | spin_unlock_irqrestore(&signaler->wait.lock, flags); | |
346 | ||
347 | return pending; | |
348 | } | |
349 | ||
7e941861 CW |
350 | int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
351 | struct i915_sw_fence *signaler, | |
ac6424b9 | 352 | wait_queue_entry_t *wq) |
7e941861 CW |
353 | { |
354 | return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); | |
355 | } | |
356 | ||
357 | int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, | |
358 | struct i915_sw_fence *signaler, | |
359 | gfp_t gfp) | |
360 | { | |
361 | return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); | |
362 | } | |
363 | ||
f54d1867 CW |
364 | struct i915_sw_dma_fence_cb { |
365 | struct dma_fence_cb base; | |
e68a139f | 366 | struct i915_sw_fence *fence; |
f54d1867 | 367 | struct dma_fence *dma; |
e68a139f | 368 | struct timer_list timer; |
81c0ed21 | 369 | struct irq_work work; |
2cf654db | 370 | struct rcu_head rcu; |
e68a139f CW |
371 | }; |
372 | ||
39cbf2aa | 373 | static void timer_i915_sw_fence_wake(struct timer_list *t) |
e68a139f | 374 | { |
39cbf2aa | 375 | struct i915_sw_dma_fence_cb *cb = from_timer(cb, t, timer); |
81c0ed21 CW |
376 | struct i915_sw_fence *fence; |
377 | ||
378 | fence = xchg(&cb->fence, NULL); | |
379 | if (!fence) | |
380 | return; | |
e68a139f | 381 | |
8dfe162a JP |
382 | pr_warn("asynchronous wait on fence %s:%s:%x timed out\n", |
383 | cb->dma->ops->get_driver_name(cb->dma), | |
384 | cb->dma->ops->get_timeline_name(cb->dma), | |
385 | cb->dma->seqno); | |
e68a139f | 386 | |
81c0ed21 | 387 | i915_sw_fence_complete(fence); |
e68a139f CW |
388 | } |
389 | ||
f54d1867 CW |
390 | static void dma_i915_sw_fence_wake(struct dma_fence *dma, |
391 | struct dma_fence_cb *data) | |
e68a139f | 392 | { |
f54d1867 | 393 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); |
81c0ed21 CW |
394 | struct i915_sw_fence *fence; |
395 | ||
396 | fence = xchg(&cb->fence, NULL); | |
397 | if (fence) | |
398 | i915_sw_fence_complete(fence); | |
399 | ||
400 | irq_work_queue(&cb->work); | |
401 | } | |
402 | ||
403 | static void irq_i915_sw_fence_work(struct irq_work *wrk) | |
404 | { | |
405 | struct i915_sw_dma_fence_cb *cb = container_of(wrk, typeof(*cb), work); | |
e68a139f CW |
406 | |
407 | del_timer_sync(&cb->timer); | |
f54d1867 | 408 | dma_fence_put(cb->dma); |
e68a139f | 409 | |
2cf654db | 410 | kfree_rcu(cb, rcu); |
e68a139f CW |
411 | } |
412 | ||
413 | int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
f54d1867 | 414 | struct dma_fence *dma, |
e68a139f CW |
415 | unsigned long timeout, |
416 | gfp_t gfp) | |
417 | { | |
f54d1867 | 418 | struct i915_sw_dma_fence_cb *cb; |
e68a139f CW |
419 | int ret; |
420 | ||
fc158405 CW |
421 | debug_fence_assert(fence); |
422 | ||
f54d1867 | 423 | if (dma_fence_is_signaled(dma)) |
e68a139f CW |
424 | return 0; |
425 | ||
426 | cb = kmalloc(sizeof(*cb), gfp); | |
427 | if (!cb) { | |
428 | if (!gfpflags_allow_blocking(gfp)) | |
429 | return -ENOMEM; | |
430 | ||
f54d1867 | 431 | return dma_fence_wait(dma, false); |
e68a139f CW |
432 | } |
433 | ||
9310cb7f | 434 | cb->fence = fence; |
e68a139f CW |
435 | i915_sw_fence_await(fence); |
436 | ||
437 | cb->dma = NULL; | |
39cbf2aa | 438 | timer_setup(&cb->timer, timer_i915_sw_fence_wake, TIMER_IRQSAFE); |
81c0ed21 | 439 | init_irq_work(&cb->work, irq_i915_sw_fence_work); |
e68a139f | 440 | if (timeout) { |
f54d1867 | 441 | cb->dma = dma_fence_get(dma); |
e68a139f CW |
442 | mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout)); |
443 | } | |
444 | ||
f54d1867 | 445 | ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake); |
e68a139f CW |
446 | if (ret == 0) { |
447 | ret = 1; | |
448 | } else { | |
449 | dma_i915_sw_fence_wake(dma, &cb->base); | |
450 | if (ret == -ENOENT) /* fence already signaled */ | |
451 | ret = 0; | |
452 | } | |
453 | ||
454 | return ret; | |
455 | } | |
456 | ||
457 | int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, | |
458 | struct reservation_object *resv, | |
f54d1867 | 459 | const struct dma_fence_ops *exclude, |
e68a139f CW |
460 | bool write, |
461 | unsigned long timeout, | |
462 | gfp_t gfp) | |
463 | { | |
f54d1867 | 464 | struct dma_fence *excl; |
e68a139f CW |
465 | int ret = 0, pending; |
466 | ||
fc158405 CW |
467 | debug_fence_assert(fence); |
468 | ||
e68a139f | 469 | if (write) { |
f54d1867 | 470 | struct dma_fence **shared; |
e68a139f CW |
471 | unsigned int count, i; |
472 | ||
473 | ret = reservation_object_get_fences_rcu(resv, | |
474 | &excl, &count, &shared); | |
475 | if (ret) | |
476 | return ret; | |
477 | ||
478 | for (i = 0; i < count; i++) { | |
479 | if (shared[i]->ops == exclude) | |
480 | continue; | |
481 | ||
482 | pending = i915_sw_fence_await_dma_fence(fence, | |
483 | shared[i], | |
484 | timeout, | |
485 | gfp); | |
486 | if (pending < 0) { | |
487 | ret = pending; | |
488 | break; | |
489 | } | |
490 | ||
491 | ret |= pending; | |
492 | } | |
493 | ||
494 | for (i = 0; i < count; i++) | |
f54d1867 | 495 | dma_fence_put(shared[i]); |
e68a139f CW |
496 | kfree(shared); |
497 | } else { | |
498 | excl = reservation_object_get_excl_rcu(resv); | |
499 | } | |
500 | ||
501 | if (ret >= 0 && excl && excl->ops != exclude) { | |
502 | pending = i915_sw_fence_await_dma_fence(fence, | |
503 | excl, | |
504 | timeout, | |
505 | gfp); | |
506 | if (pending < 0) | |
507 | ret = pending; | |
508 | else | |
509 | ret |= pending; | |
510 | } | |
511 | ||
f54d1867 | 512 | dma_fence_put(excl); |
e68a139f CW |
513 | |
514 | return ret; | |
515 | } | |
47624cc3 CW |
516 | |
517 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
214707fc | 518 | #include "selftests/lib_sw_fence.c" |
47624cc3 CW |
519 | #include "selftests/i915_sw_fence.c" |
520 | #endif |