Commit | Line | Data |
---|---|---|
e68a139f CW |
1 | /* |
2 | * (C) Copyright 2016 Intel Corporation | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation; version 2 | |
7 | * of the License. | |
8 | */ | |
9 | ||
10 | #include <linux/slab.h> | |
f54d1867 | 11 | #include <linux/dma-fence.h> |
81c0ed21 | 12 | #include <linux/irq_work.h> |
e68a139f CW |
13 | #include <linux/reservation.h> |
14 | ||
15 | #include "i915_sw_fence.h" | |
47624cc3 | 16 | #include "i915_selftest.h" |
e68a139f | 17 | |
7e941861 CW |
18 | #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ |
19 | ||
e68a139f CW |
20 | static DEFINE_SPINLOCK(i915_sw_fence_lock); |
21 | ||
fc158405 CW |
22 | enum { |
23 | DEBUG_FENCE_IDLE = 0, | |
24 | DEBUG_FENCE_NOTIFY, | |
25 | }; | |
26 | ||
27 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS | |
28 | ||
29 | static void *i915_sw_fence_debug_hint(void *addr) | |
30 | { | |
31 | return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); | |
32 | } | |
33 | ||
34 | static struct debug_obj_descr i915_sw_fence_debug_descr = { | |
35 | .name = "i915_sw_fence", | |
36 | .debug_hint = i915_sw_fence_debug_hint, | |
37 | }; | |
38 | ||
39 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
40 | { | |
41 | debug_object_init(fence, &i915_sw_fence_debug_descr); | |
42 | } | |
43 | ||
44 | static inline void debug_fence_activate(struct i915_sw_fence *fence) | |
45 | { | |
46 | debug_object_activate(fence, &i915_sw_fence_debug_descr); | |
47 | } | |
48 | ||
49 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
50 | int old, int new) | |
51 | { | |
52 | debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); | |
53 | } | |
54 | ||
55 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
56 | { | |
57 | debug_object_deactivate(fence, &i915_sw_fence_debug_descr); | |
58 | } | |
59 | ||
60 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
61 | { | |
62 | debug_object_destroy(fence, &i915_sw_fence_debug_descr); | |
63 | } | |
64 | ||
65 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
66 | { | |
67 | debug_object_free(fence, &i915_sw_fence_debug_descr); | |
6f13f29f | 68 | smp_wmb(); /* flush the change in state before reallocation */ |
fc158405 CW |
69 | } |
70 | ||
71 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
72 | { | |
73 | debug_object_assert_init(fence, &i915_sw_fence_debug_descr); | |
74 | } | |
75 | ||
76 | #else | |
77 | ||
78 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
79 | { | |
80 | } | |
81 | ||
82 | static inline void debug_fence_activate(struct i915_sw_fence *fence) | |
83 | { | |
84 | } | |
85 | ||
86 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
87 | int old, int new) | |
88 | { | |
89 | } | |
90 | ||
91 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
92 | { | |
93 | } | |
94 | ||
95 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
96 | { | |
97 | } | |
98 | ||
99 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
100 | { | |
101 | } | |
102 | ||
103 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
104 | { | |
105 | } | |
106 | ||
107 | #endif | |
108 | ||
e68a139f CW |
109 | static int __i915_sw_fence_notify(struct i915_sw_fence *fence, |
110 | enum i915_sw_fence_notify state) | |
111 | { | |
112 | i915_sw_fence_notify_t fn; | |
113 | ||
114 | fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); | |
115 | return fn(fence, state); | |
116 | } | |
117 | ||
fc158405 CW |
118 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
119 | void i915_sw_fence_fini(struct i915_sw_fence *fence) | |
120 | { | |
121 | debug_fence_free(fence); | |
122 | } | |
123 | #endif | |
124 | ||
e68a139f CW |
125 | static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, |
126 | struct list_head *continuation) | |
127 | { | |
128 | wait_queue_head_t *x = &fence->wait; | |
ac6424b9 | 129 | wait_queue_entry_t *pos, *next; |
e68a139f CW |
130 | unsigned long flags; |
131 | ||
fc158405 | 132 | debug_fence_deactivate(fence); |
e68a139f CW |
133 | atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ |
134 | ||
135 | /* | |
136 | * To prevent unbounded recursion as we traverse the graph of | |
2055da97 IM |
137 | * i915_sw_fences, we move the entry list from this, the next ready |
138 | * fence, to the tail of the original fence's entry list | |
e68a139f CW |
139 | * (and so added to the list to be woken). |
140 | */ | |
141 | ||
142 | spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); | |
143 | if (continuation) { | |
2055da97 | 144 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
e68a139f CW |
145 | if (pos->func == autoremove_wake_function) |
146 | pos->func(pos, TASK_NORMAL, 0, continuation); | |
147 | else | |
2055da97 | 148 | list_move_tail(&pos->entry, continuation); |
e68a139f CW |
149 | } |
150 | } else { | |
151 | LIST_HEAD(extra); | |
152 | ||
153 | do { | |
2055da97 | 154 | list_for_each_entry_safe(pos, next, &x->head, entry) |
e68a139f CW |
155 | pos->func(pos, TASK_NORMAL, 0, &extra); |
156 | ||
157 | if (list_empty(&extra)) | |
158 | break; | |
159 | ||
2055da97 | 160 | list_splice_tail_init(&extra, &x->head); |
e68a139f CW |
161 | } while (1); |
162 | } | |
163 | spin_unlock_irqrestore(&x->lock, flags); | |
fc158405 CW |
164 | |
165 | debug_fence_assert(fence); | |
e68a139f CW |
166 | } |
167 | ||
168 | static void __i915_sw_fence_complete(struct i915_sw_fence *fence, | |
169 | struct list_head *continuation) | |
170 | { | |
fc158405 CW |
171 | debug_fence_assert(fence); |
172 | ||
e68a139f CW |
173 | if (!atomic_dec_and_test(&fence->pending)) |
174 | return; | |
175 | ||
fc158405 CW |
176 | debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); |
177 | ||
9310cb7f | 178 | if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) |
e68a139f CW |
179 | return; |
180 | ||
fc158405 CW |
181 | debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); |
182 | ||
e68a139f | 183 | __i915_sw_fence_wake_up_all(fence, continuation); |
9310cb7f CW |
184 | |
185 | debug_fence_destroy(fence); | |
186 | __i915_sw_fence_notify(fence, FENCE_FREE); | |
e68a139f CW |
187 | } |
188 | ||
189 | static void i915_sw_fence_complete(struct i915_sw_fence *fence) | |
190 | { | |
fc158405 CW |
191 | debug_fence_assert(fence); |
192 | ||
e68a139f CW |
193 | if (WARN_ON(i915_sw_fence_done(fence))) |
194 | return; | |
195 | ||
196 | __i915_sw_fence_complete(fence, NULL); | |
197 | } | |
198 | ||
199 | static void i915_sw_fence_await(struct i915_sw_fence *fence) | |
200 | { | |
fc158405 | 201 | debug_fence_assert(fence); |
e68a139f CW |
202 | WARN_ON(atomic_inc_return(&fence->pending) <= 1); |
203 | } | |
204 | ||
556b7487 CW |
205 | void __i915_sw_fence_init(struct i915_sw_fence *fence, |
206 | i915_sw_fence_notify_t fn, | |
207 | const char *name, | |
208 | struct lock_class_key *key) | |
e68a139f | 209 | { |
9310cb7f | 210 | BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK); |
e68a139f | 211 | |
fc158405 CW |
212 | debug_fence_init(fence); |
213 | ||
556b7487 | 214 | __init_waitqueue_head(&fence->wait, name, key); |
e68a139f CW |
215 | atomic_set(&fence->pending, 1); |
216 | fence->flags = (unsigned long)fn; | |
217 | } | |
218 | ||
fc158405 CW |
219 | void i915_sw_fence_commit(struct i915_sw_fence *fence) |
220 | { | |
221 | debug_fence_activate(fence); | |
9310cb7f | 222 | i915_sw_fence_complete(fence); |
fc158405 CW |
223 | } |
224 | ||
ac6424b9 | 225 | static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) |
e68a139f | 226 | { |
2055da97 | 227 | list_del(&wq->entry); |
e68a139f | 228 | __i915_sw_fence_complete(wq->private, key); |
9310cb7f | 229 | |
7e941861 CW |
230 | if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) |
231 | kfree(wq); | |
e68a139f CW |
232 | return 0; |
233 | } | |
234 | ||
235 | static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
236 | const struct i915_sw_fence * const signaler) | |
237 | { | |
ac6424b9 | 238 | wait_queue_entry_t *wq; |
e68a139f CW |
239 | |
240 | if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
241 | return false; | |
242 | ||
243 | if (fence == signaler) | |
244 | return true; | |
245 | ||
2055da97 | 246 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
247 | if (wq->func != i915_sw_fence_wake) |
248 | continue; | |
249 | ||
250 | if (__i915_sw_fence_check_if_after(wq->private, signaler)) | |
251 | return true; | |
252 | } | |
253 | ||
254 | return false; | |
255 | } | |
256 | ||
257 | static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) | |
258 | { | |
ac6424b9 | 259 | wait_queue_entry_t *wq; |
e68a139f CW |
260 | |
261 | if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
262 | return; | |
263 | ||
2055da97 | 264 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
265 | if (wq->func != i915_sw_fence_wake) |
266 | continue; | |
267 | ||
268 | __i915_sw_fence_clear_checked_bit(wq->private); | |
269 | } | |
270 | } | |
271 | ||
272 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
273 | const struct i915_sw_fence * const signaler) | |
274 | { | |
275 | unsigned long flags; | |
276 | bool err; | |
277 | ||
47624cc3 | 278 | if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG)) |
e68a139f CW |
279 | return false; |
280 | ||
281 | spin_lock_irqsave(&i915_sw_fence_lock, flags); | |
282 | err = __i915_sw_fence_check_if_after(fence, signaler); | |
283 | __i915_sw_fence_clear_checked_bit(fence); | |
284 | spin_unlock_irqrestore(&i915_sw_fence_lock, flags); | |
285 | ||
286 | return err; | |
287 | } | |
288 | ||
7e941861 CW |
289 | static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
290 | struct i915_sw_fence *signaler, | |
ac6424b9 | 291 | wait_queue_entry_t *wq, gfp_t gfp) |
e68a139f CW |
292 | { |
293 | unsigned long flags; | |
294 | int pending; | |
295 | ||
fc158405 CW |
296 | debug_fence_assert(fence); |
297 | ||
e68a139f CW |
298 | if (i915_sw_fence_done(signaler)) |
299 | return 0; | |
300 | ||
fc158405 CW |
301 | debug_fence_assert(signaler); |
302 | ||
e68a139f CW |
303 | /* The dependency graph must be acyclic. */ |
304 | if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) | |
305 | return -EINVAL; | |
306 | ||
7e941861 CW |
307 | pending = 0; |
308 | if (!wq) { | |
309 | wq = kmalloc(sizeof(*wq), gfp); | |
310 | if (!wq) { | |
311 | if (!gfpflags_allow_blocking(gfp)) | |
312 | return -ENOMEM; | |
313 | ||
314 | i915_sw_fence_wait(signaler); | |
315 | return 0; | |
316 | } | |
317 | ||
318 | pending |= I915_SW_FENCE_FLAG_ALLOC; | |
319 | } | |
320 | ||
2055da97 | 321 | INIT_LIST_HEAD(&wq->entry); |
7e941861 | 322 | wq->flags = pending; |
e68a139f | 323 | wq->func = i915_sw_fence_wake; |
9310cb7f | 324 | wq->private = fence; |
e68a139f CW |
325 | |
326 | i915_sw_fence_await(fence); | |
327 | ||
328 | spin_lock_irqsave(&signaler->wait.lock, flags); | |
329 | if (likely(!i915_sw_fence_done(signaler))) { | |
ac6424b9 | 330 | __add_wait_queue_entry_tail(&signaler->wait, wq); |
e68a139f CW |
331 | pending = 1; |
332 | } else { | |
333 | i915_sw_fence_wake(wq, 0, 0, NULL); | |
334 | pending = 0; | |
335 | } | |
336 | spin_unlock_irqrestore(&signaler->wait.lock, flags); | |
337 | ||
338 | return pending; | |
339 | } | |
340 | ||
7e941861 CW |
341 | int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
342 | struct i915_sw_fence *signaler, | |
ac6424b9 | 343 | wait_queue_entry_t *wq) |
7e941861 CW |
344 | { |
345 | return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); | |
346 | } | |
347 | ||
348 | int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, | |
349 | struct i915_sw_fence *signaler, | |
350 | gfp_t gfp) | |
351 | { | |
352 | return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); | |
353 | } | |
354 | ||
f54d1867 CW |
355 | struct i915_sw_dma_fence_cb { |
356 | struct dma_fence_cb base; | |
e68a139f | 357 | struct i915_sw_fence *fence; |
f54d1867 | 358 | struct dma_fence *dma; |
e68a139f | 359 | struct timer_list timer; |
81c0ed21 | 360 | struct irq_work work; |
e68a139f CW |
361 | }; |
362 | ||
363 | static void timer_i915_sw_fence_wake(unsigned long data) | |
364 | { | |
f54d1867 | 365 | struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data; |
81c0ed21 CW |
366 | struct i915_sw_fence *fence; |
367 | ||
368 | fence = xchg(&cb->fence, NULL); | |
369 | if (!fence) | |
370 | return; | |
e68a139f | 371 | |
8dfe162a JP |
372 | pr_warn("asynchronous wait on fence %s:%s:%x timed out\n", |
373 | cb->dma->ops->get_driver_name(cb->dma), | |
374 | cb->dma->ops->get_timeline_name(cb->dma), | |
375 | cb->dma->seqno); | |
e68a139f | 376 | |
81c0ed21 | 377 | i915_sw_fence_complete(fence); |
e68a139f CW |
378 | } |
379 | ||
f54d1867 CW |
380 | static void dma_i915_sw_fence_wake(struct dma_fence *dma, |
381 | struct dma_fence_cb *data) | |
e68a139f | 382 | { |
f54d1867 | 383 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); |
81c0ed21 CW |
384 | struct i915_sw_fence *fence; |
385 | ||
386 | fence = xchg(&cb->fence, NULL); | |
387 | if (fence) | |
388 | i915_sw_fence_complete(fence); | |
389 | ||
390 | irq_work_queue(&cb->work); | |
391 | } | |
392 | ||
393 | static void irq_i915_sw_fence_work(struct irq_work *wrk) | |
394 | { | |
395 | struct i915_sw_dma_fence_cb *cb = container_of(wrk, typeof(*cb), work); | |
e68a139f CW |
396 | |
397 | del_timer_sync(&cb->timer); | |
f54d1867 | 398 | dma_fence_put(cb->dma); |
e68a139f CW |
399 | |
400 | kfree(cb); | |
401 | } | |
402 | ||
403 | int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
f54d1867 | 404 | struct dma_fence *dma, |
e68a139f CW |
405 | unsigned long timeout, |
406 | gfp_t gfp) | |
407 | { | |
f54d1867 | 408 | struct i915_sw_dma_fence_cb *cb; |
e68a139f CW |
409 | int ret; |
410 | ||
fc158405 CW |
411 | debug_fence_assert(fence); |
412 | ||
f54d1867 | 413 | if (dma_fence_is_signaled(dma)) |
e68a139f CW |
414 | return 0; |
415 | ||
416 | cb = kmalloc(sizeof(*cb), gfp); | |
417 | if (!cb) { | |
418 | if (!gfpflags_allow_blocking(gfp)) | |
419 | return -ENOMEM; | |
420 | ||
f54d1867 | 421 | return dma_fence_wait(dma, false); |
e68a139f CW |
422 | } |
423 | ||
9310cb7f | 424 | cb->fence = fence; |
e68a139f CW |
425 | i915_sw_fence_await(fence); |
426 | ||
427 | cb->dma = NULL; | |
428 | __setup_timer(&cb->timer, | |
429 | timer_i915_sw_fence_wake, (unsigned long)cb, | |
430 | TIMER_IRQSAFE); | |
81c0ed21 | 431 | init_irq_work(&cb->work, irq_i915_sw_fence_work); |
e68a139f | 432 | if (timeout) { |
f54d1867 | 433 | cb->dma = dma_fence_get(dma); |
e68a139f CW |
434 | mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout)); |
435 | } | |
436 | ||
f54d1867 | 437 | ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake); |
e68a139f CW |
438 | if (ret == 0) { |
439 | ret = 1; | |
440 | } else { | |
441 | dma_i915_sw_fence_wake(dma, &cb->base); | |
442 | if (ret == -ENOENT) /* fence already signaled */ | |
443 | ret = 0; | |
444 | } | |
445 | ||
446 | return ret; | |
447 | } | |
448 | ||
449 | int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, | |
450 | struct reservation_object *resv, | |
f54d1867 | 451 | const struct dma_fence_ops *exclude, |
e68a139f CW |
452 | bool write, |
453 | unsigned long timeout, | |
454 | gfp_t gfp) | |
455 | { | |
f54d1867 | 456 | struct dma_fence *excl; |
e68a139f CW |
457 | int ret = 0, pending; |
458 | ||
fc158405 CW |
459 | debug_fence_assert(fence); |
460 | ||
e68a139f | 461 | if (write) { |
f54d1867 | 462 | struct dma_fence **shared; |
e68a139f CW |
463 | unsigned int count, i; |
464 | ||
465 | ret = reservation_object_get_fences_rcu(resv, | |
466 | &excl, &count, &shared); | |
467 | if (ret) | |
468 | return ret; | |
469 | ||
470 | for (i = 0; i < count; i++) { | |
471 | if (shared[i]->ops == exclude) | |
472 | continue; | |
473 | ||
474 | pending = i915_sw_fence_await_dma_fence(fence, | |
475 | shared[i], | |
476 | timeout, | |
477 | gfp); | |
478 | if (pending < 0) { | |
479 | ret = pending; | |
480 | break; | |
481 | } | |
482 | ||
483 | ret |= pending; | |
484 | } | |
485 | ||
486 | for (i = 0; i < count; i++) | |
f54d1867 | 487 | dma_fence_put(shared[i]); |
e68a139f CW |
488 | kfree(shared); |
489 | } else { | |
490 | excl = reservation_object_get_excl_rcu(resv); | |
491 | } | |
492 | ||
493 | if (ret >= 0 && excl && excl->ops != exclude) { | |
494 | pending = i915_sw_fence_await_dma_fence(fence, | |
495 | excl, | |
496 | timeout, | |
497 | gfp); | |
498 | if (pending < 0) | |
499 | ret = pending; | |
500 | else | |
501 | ret |= pending; | |
502 | } | |
503 | ||
f54d1867 | 504 | dma_fence_put(excl); |
e68a139f CW |
505 | |
506 | return ret; | |
507 | } | |
47624cc3 CW |
508 | |
509 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
510 | #include "selftests/i915_sw_fence.c" | |
511 | #endif |