Commit | Line | Data |
---|---|---|
e68a139f | 1 | /* |
635b3bc6 | 2 | * SPDX-License-Identifier: MIT |
e68a139f | 3 | * |
635b3bc6 | 4 | * (C) Copyright 2016 Intel Corporation |
e68a139f CW |
5 | */ |
6 | ||
7 | #include <linux/slab.h> | |
f54d1867 | 8 | #include <linux/dma-fence.h> |
81c0ed21 | 9 | #include <linux/irq_work.h> |
e68a139f CW |
10 | #include <linux/reservation.h> |
11 | ||
12 | #include "i915_sw_fence.h" | |
47624cc3 | 13 | #include "i915_selftest.h" |
e68a139f | 14 | |
7e941861 CW |
15 | #define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ |
16 | ||
e68a139f CW |
17 | static DEFINE_SPINLOCK(i915_sw_fence_lock); |
18 | ||
fc158405 CW |
19 | enum { |
20 | DEBUG_FENCE_IDLE = 0, | |
21 | DEBUG_FENCE_NOTIFY, | |
22 | }; | |
23 | ||
fc158405 CW |
24 | static void *i915_sw_fence_debug_hint(void *addr) |
25 | { | |
26 | return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK); | |
27 | } | |
28 | ||
5791bad4 CW |
29 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
30 | ||
fc158405 CW |
31 | static struct debug_obj_descr i915_sw_fence_debug_descr = { |
32 | .name = "i915_sw_fence", | |
33 | .debug_hint = i915_sw_fence_debug_hint, | |
34 | }; | |
35 | ||
36 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
37 | { | |
38 | debug_object_init(fence, &i915_sw_fence_debug_descr); | |
39 | } | |
40 | ||
214707fc CW |
41 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
42 | { | |
43 | debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr); | |
44 | } | |
45 | ||
fc158405 CW |
46 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
47 | { | |
48 | debug_object_activate(fence, &i915_sw_fence_debug_descr); | |
49 | } | |
50 | ||
51 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
52 | int old, int new) | |
53 | { | |
54 | debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new); | |
55 | } | |
56 | ||
57 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
58 | { | |
59 | debug_object_deactivate(fence, &i915_sw_fence_debug_descr); | |
60 | } | |
61 | ||
62 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
63 | { | |
64 | debug_object_destroy(fence, &i915_sw_fence_debug_descr); | |
65 | } | |
66 | ||
67 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
68 | { | |
69 | debug_object_free(fence, &i915_sw_fence_debug_descr); | |
6f13f29f | 70 | smp_wmb(); /* flush the change in state before reallocation */ |
fc158405 CW |
71 | } |
72 | ||
73 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
74 | { | |
75 | debug_object_assert_init(fence, &i915_sw_fence_debug_descr); | |
76 | } | |
77 | ||
78 | #else | |
79 | ||
80 | static inline void debug_fence_init(struct i915_sw_fence *fence) | |
81 | { | |
82 | } | |
83 | ||
214707fc CW |
84 | static inline void debug_fence_init_onstack(struct i915_sw_fence *fence) |
85 | { | |
86 | } | |
87 | ||
fc158405 CW |
88 | static inline void debug_fence_activate(struct i915_sw_fence *fence) |
89 | { | |
90 | } | |
91 | ||
92 | static inline void debug_fence_set_state(struct i915_sw_fence *fence, | |
93 | int old, int new) | |
94 | { | |
95 | } | |
96 | ||
97 | static inline void debug_fence_deactivate(struct i915_sw_fence *fence) | |
98 | { | |
99 | } | |
100 | ||
101 | static inline void debug_fence_destroy(struct i915_sw_fence *fence) | |
102 | { | |
103 | } | |
104 | ||
105 | static inline void debug_fence_free(struct i915_sw_fence *fence) | |
106 | { | |
107 | } | |
108 | ||
109 | static inline void debug_fence_assert(struct i915_sw_fence *fence) | |
110 | { | |
111 | } | |
112 | ||
113 | #endif | |
114 | ||
e68a139f CW |
115 | static int __i915_sw_fence_notify(struct i915_sw_fence *fence, |
116 | enum i915_sw_fence_notify state) | |
117 | { | |
118 | i915_sw_fence_notify_t fn; | |
119 | ||
120 | fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK); | |
121 | return fn(fence, state); | |
122 | } | |
123 | ||
fc158405 CW |
124 | #ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS |
125 | void i915_sw_fence_fini(struct i915_sw_fence *fence) | |
126 | { | |
127 | debug_fence_free(fence); | |
128 | } | |
129 | #endif | |
130 | ||
e68a139f CW |
131 | static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence, |
132 | struct list_head *continuation) | |
133 | { | |
134 | wait_queue_head_t *x = &fence->wait; | |
ac6424b9 | 135 | wait_queue_entry_t *pos, *next; |
e68a139f CW |
136 | unsigned long flags; |
137 | ||
fc158405 | 138 | debug_fence_deactivate(fence); |
e68a139f CW |
139 | atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */ |
140 | ||
141 | /* | |
142 | * To prevent unbounded recursion as we traverse the graph of | |
2055da97 IM |
143 | * i915_sw_fences, we move the entry list from this, the next ready |
144 | * fence, to the tail of the original fence's entry list | |
e68a139f CW |
145 | * (and so added to the list to be woken). |
146 | */ | |
147 | ||
148 | spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation); | |
149 | if (continuation) { | |
2055da97 | 150 | list_for_each_entry_safe(pos, next, &x->head, entry) { |
e68a139f CW |
151 | if (pos->func == autoremove_wake_function) |
152 | pos->func(pos, TASK_NORMAL, 0, continuation); | |
153 | else | |
2055da97 | 154 | list_move_tail(&pos->entry, continuation); |
e68a139f CW |
155 | } |
156 | } else { | |
157 | LIST_HEAD(extra); | |
158 | ||
159 | do { | |
2055da97 | 160 | list_for_each_entry_safe(pos, next, &x->head, entry) |
e68a139f CW |
161 | pos->func(pos, TASK_NORMAL, 0, &extra); |
162 | ||
163 | if (list_empty(&extra)) | |
164 | break; | |
165 | ||
2055da97 | 166 | list_splice_tail_init(&extra, &x->head); |
e68a139f CW |
167 | } while (1); |
168 | } | |
169 | spin_unlock_irqrestore(&x->lock, flags); | |
fc158405 CW |
170 | |
171 | debug_fence_assert(fence); | |
e68a139f CW |
172 | } |
173 | ||
174 | static void __i915_sw_fence_complete(struct i915_sw_fence *fence, | |
175 | struct list_head *continuation) | |
176 | { | |
fc158405 CW |
177 | debug_fence_assert(fence); |
178 | ||
e68a139f CW |
179 | if (!atomic_dec_and_test(&fence->pending)) |
180 | return; | |
181 | ||
fc158405 CW |
182 | debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY); |
183 | ||
9310cb7f | 184 | if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE) |
e68a139f CW |
185 | return; |
186 | ||
fc158405 CW |
187 | debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE); |
188 | ||
e68a139f | 189 | __i915_sw_fence_wake_up_all(fence, continuation); |
9310cb7f CW |
190 | |
191 | debug_fence_destroy(fence); | |
192 | __i915_sw_fence_notify(fence, FENCE_FREE); | |
e68a139f CW |
193 | } |
194 | ||
e8861964 | 195 | void i915_sw_fence_complete(struct i915_sw_fence *fence) |
e68a139f | 196 | { |
fc158405 CW |
197 | debug_fence_assert(fence); |
198 | ||
e68a139f CW |
199 | if (WARN_ON(i915_sw_fence_done(fence))) |
200 | return; | |
201 | ||
202 | __i915_sw_fence_complete(fence, NULL); | |
203 | } | |
204 | ||
e8861964 | 205 | void i915_sw_fence_await(struct i915_sw_fence *fence) |
e68a139f | 206 | { |
fc158405 | 207 | debug_fence_assert(fence); |
e68a139f CW |
208 | WARN_ON(atomic_inc_return(&fence->pending) <= 1); |
209 | } | |
210 | ||
556b7487 CW |
211 | void __i915_sw_fence_init(struct i915_sw_fence *fence, |
212 | i915_sw_fence_notify_t fn, | |
213 | const char *name, | |
214 | struct lock_class_key *key) | |
e68a139f | 215 | { |
9310cb7f | 216 | BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK); |
e68a139f | 217 | |
fc158405 CW |
218 | debug_fence_init(fence); |
219 | ||
556b7487 | 220 | __init_waitqueue_head(&fence->wait, name, key); |
e68a139f CW |
221 | atomic_set(&fence->pending, 1); |
222 | fence->flags = (unsigned long)fn; | |
223 | } | |
224 | ||
fc158405 CW |
225 | void i915_sw_fence_commit(struct i915_sw_fence *fence) |
226 | { | |
227 | debug_fence_activate(fence); | |
9310cb7f | 228 | i915_sw_fence_complete(fence); |
fc158405 CW |
229 | } |
230 | ||
ac6424b9 | 231 | static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key) |
e68a139f | 232 | { |
2055da97 | 233 | list_del(&wq->entry); |
e68a139f | 234 | __i915_sw_fence_complete(wq->private, key); |
9310cb7f | 235 | |
7e941861 CW |
236 | if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) |
237 | kfree(wq); | |
e68a139f CW |
238 | return 0; |
239 | } | |
240 | ||
241 | static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
242 | const struct i915_sw_fence * const signaler) | |
243 | { | |
ac6424b9 | 244 | wait_queue_entry_t *wq; |
e68a139f CW |
245 | |
246 | if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
247 | return false; | |
248 | ||
249 | if (fence == signaler) | |
250 | return true; | |
251 | ||
2055da97 | 252 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
253 | if (wq->func != i915_sw_fence_wake) |
254 | continue; | |
255 | ||
256 | if (__i915_sw_fence_check_if_after(wq->private, signaler)) | |
257 | return true; | |
258 | } | |
259 | ||
260 | return false; | |
261 | } | |
262 | ||
263 | static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence) | |
264 | { | |
ac6424b9 | 265 | wait_queue_entry_t *wq; |
e68a139f CW |
266 | |
267 | if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags)) | |
268 | return; | |
269 | ||
2055da97 | 270 | list_for_each_entry(wq, &fence->wait.head, entry) { |
e68a139f CW |
271 | if (wq->func != i915_sw_fence_wake) |
272 | continue; | |
273 | ||
274 | __i915_sw_fence_clear_checked_bit(wq->private); | |
275 | } | |
276 | } | |
277 | ||
278 | static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, | |
279 | const struct i915_sw_fence * const signaler) | |
280 | { | |
281 | unsigned long flags; | |
282 | bool err; | |
283 | ||
47624cc3 | 284 | if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG)) |
e68a139f CW |
285 | return false; |
286 | ||
287 | spin_lock_irqsave(&i915_sw_fence_lock, flags); | |
288 | err = __i915_sw_fence_check_if_after(fence, signaler); | |
289 | __i915_sw_fence_clear_checked_bit(fence); | |
290 | spin_unlock_irqrestore(&i915_sw_fence_lock, flags); | |
291 | ||
292 | return err; | |
293 | } | |
294 | ||
7e941861 CW |
295 | static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
296 | struct i915_sw_fence *signaler, | |
ac6424b9 | 297 | wait_queue_entry_t *wq, gfp_t gfp) |
e68a139f CW |
298 | { |
299 | unsigned long flags; | |
300 | int pending; | |
301 | ||
fc158405 | 302 | debug_fence_assert(fence); |
e30a7581 | 303 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 304 | |
e68a139f CW |
305 | if (i915_sw_fence_done(signaler)) |
306 | return 0; | |
307 | ||
fc158405 CW |
308 | debug_fence_assert(signaler); |
309 | ||
e68a139f CW |
310 | /* The dependency graph must be acyclic. */ |
311 | if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) | |
312 | return -EINVAL; | |
313 | ||
7e941861 CW |
314 | pending = 0; |
315 | if (!wq) { | |
316 | wq = kmalloc(sizeof(*wq), gfp); | |
317 | if (!wq) { | |
318 | if (!gfpflags_allow_blocking(gfp)) | |
319 | return -ENOMEM; | |
320 | ||
321 | i915_sw_fence_wait(signaler); | |
322 | return 0; | |
323 | } | |
324 | ||
325 | pending |= I915_SW_FENCE_FLAG_ALLOC; | |
326 | } | |
327 | ||
2055da97 | 328 | INIT_LIST_HEAD(&wq->entry); |
7e941861 | 329 | wq->flags = pending; |
e68a139f | 330 | wq->func = i915_sw_fence_wake; |
9310cb7f | 331 | wq->private = fence; |
e68a139f CW |
332 | |
333 | i915_sw_fence_await(fence); | |
334 | ||
335 | spin_lock_irqsave(&signaler->wait.lock, flags); | |
336 | if (likely(!i915_sw_fence_done(signaler))) { | |
ac6424b9 | 337 | __add_wait_queue_entry_tail(&signaler->wait, wq); |
e68a139f CW |
338 | pending = 1; |
339 | } else { | |
340 | i915_sw_fence_wake(wq, 0, 0, NULL); | |
341 | pending = 0; | |
342 | } | |
343 | spin_unlock_irqrestore(&signaler->wait.lock, flags); | |
344 | ||
345 | return pending; | |
346 | } | |
347 | ||
7e941861 CW |
348 | int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, |
349 | struct i915_sw_fence *signaler, | |
ac6424b9 | 350 | wait_queue_entry_t *wq) |
7e941861 CW |
351 | { |
352 | return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); | |
353 | } | |
354 | ||
355 | int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, | |
356 | struct i915_sw_fence *signaler, | |
357 | gfp_t gfp) | |
358 | { | |
359 | return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); | |
360 | } | |
361 | ||
f54d1867 CW |
362 | struct i915_sw_dma_fence_cb { |
363 | struct dma_fence_cb base; | |
e68a139f | 364 | struct i915_sw_fence *fence; |
f255c1e9 CW |
365 | }; |
366 | ||
367 | struct i915_sw_dma_fence_cb_timer { | |
368 | struct i915_sw_dma_fence_cb base; | |
f54d1867 | 369 | struct dma_fence *dma; |
e68a139f | 370 | struct timer_list timer; |
81c0ed21 | 371 | struct irq_work work; |
7d622351 | 372 | struct rcu_head rcu; |
e68a139f CW |
373 | }; |
374 | ||
f255c1e9 CW |
375 | static void dma_i915_sw_fence_wake(struct dma_fence *dma, |
376 | struct dma_fence_cb *data) | |
377 | { | |
378 | struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base); | |
379 | ||
380 | i915_sw_fence_complete(cb->fence); | |
381 | kfree(cb); | |
382 | } | |
383 | ||
39cbf2aa | 384 | static void timer_i915_sw_fence_wake(struct timer_list *t) |
e68a139f | 385 | { |
f255c1e9 | 386 | struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer); |
81c0ed21 CW |
387 | struct i915_sw_fence *fence; |
388 | ||
f255c1e9 | 389 | fence = xchg(&cb->base.fence, NULL); |
81c0ed21 CW |
390 | if (!fence) |
391 | return; | |
e68a139f | 392 | |
b312d8ca | 393 | pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%pS)\n", |
5791bad4 CW |
394 | cb->dma->ops->get_driver_name(cb->dma), |
395 | cb->dma->ops->get_timeline_name(cb->dma), | |
396 | cb->dma->seqno, | |
397 | i915_sw_fence_debug_hint(fence)); | |
e68a139f | 398 | |
81c0ed21 | 399 | i915_sw_fence_complete(fence); |
e68a139f CW |
400 | } |
401 | ||
f255c1e9 CW |
402 | static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma, |
403 | struct dma_fence_cb *data) | |
e68a139f | 404 | { |
f255c1e9 CW |
405 | struct i915_sw_dma_fence_cb_timer *cb = |
406 | container_of(data, typeof(*cb), base.base); | |
81c0ed21 CW |
407 | struct i915_sw_fence *fence; |
408 | ||
f255c1e9 | 409 | fence = xchg(&cb->base.fence, NULL); |
81c0ed21 CW |
410 | if (fence) |
411 | i915_sw_fence_complete(fence); | |
412 | ||
f255c1e9 | 413 | irq_work_queue(&cb->work); |
81c0ed21 CW |
414 | } |
415 | ||
416 | static void irq_i915_sw_fence_work(struct irq_work *wrk) | |
417 | { | |
f255c1e9 CW |
418 | struct i915_sw_dma_fence_cb_timer *cb = |
419 | container_of(wrk, typeof(*cb), work); | |
e68a139f CW |
420 | |
421 | del_timer_sync(&cb->timer); | |
f54d1867 | 422 | dma_fence_put(cb->dma); |
e68a139f | 423 | |
7d622351 | 424 | kfree_rcu(cb, rcu); |
e68a139f CW |
425 | } |
426 | ||
427 | int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, | |
f54d1867 | 428 | struct dma_fence *dma, |
e68a139f CW |
429 | unsigned long timeout, |
430 | gfp_t gfp) | |
431 | { | |
f54d1867 | 432 | struct i915_sw_dma_fence_cb *cb; |
f255c1e9 | 433 | dma_fence_func_t func; |
e68a139f CW |
434 | int ret; |
435 | ||
fc158405 | 436 | debug_fence_assert(fence); |
e30a7581 | 437 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 438 | |
f54d1867 | 439 | if (dma_fence_is_signaled(dma)) |
e68a139f CW |
440 | return 0; |
441 | ||
f255c1e9 CW |
442 | cb = kmalloc(timeout ? |
443 | sizeof(struct i915_sw_dma_fence_cb_timer) : | |
444 | sizeof(struct i915_sw_dma_fence_cb), | |
445 | gfp); | |
e68a139f CW |
446 | if (!cb) { |
447 | if (!gfpflags_allow_blocking(gfp)) | |
448 | return -ENOMEM; | |
449 | ||
f54d1867 | 450 | return dma_fence_wait(dma, false); |
e68a139f CW |
451 | } |
452 | ||
9310cb7f | 453 | cb->fence = fence; |
e68a139f CW |
454 | i915_sw_fence_await(fence); |
455 | ||
f255c1e9 | 456 | func = dma_i915_sw_fence_wake; |
e68a139f | 457 | if (timeout) { |
f255c1e9 CW |
458 | struct i915_sw_dma_fence_cb_timer *timer = |
459 | container_of(cb, typeof(*timer), base); | |
c32164b1 | 460 | |
f255c1e9 CW |
461 | timer->dma = dma_fence_get(dma); |
462 | init_irq_work(&timer->work, irq_i915_sw_fence_work); | |
463 | ||
464 | timer_setup(&timer->timer, | |
c32164b1 | 465 | timer_i915_sw_fence_wake, TIMER_IRQSAFE); |
f255c1e9 CW |
466 | mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout)); |
467 | ||
468 | func = dma_i915_sw_fence_wake_timer; | |
e68a139f CW |
469 | } |
470 | ||
f255c1e9 | 471 | ret = dma_fence_add_callback(dma, &cb->base, func); |
e68a139f CW |
472 | if (ret == 0) { |
473 | ret = 1; | |
474 | } else { | |
f255c1e9 | 475 | func(dma, &cb->base); |
e68a139f CW |
476 | if (ret == -ENOENT) /* fence already signaled */ |
477 | ret = 0; | |
478 | } | |
479 | ||
480 | return ret; | |
481 | } | |
482 | ||
483 | int i915_sw_fence_await_reservation(struct i915_sw_fence *fence, | |
484 | struct reservation_object *resv, | |
f54d1867 | 485 | const struct dma_fence_ops *exclude, |
e68a139f CW |
486 | bool write, |
487 | unsigned long timeout, | |
488 | gfp_t gfp) | |
489 | { | |
f54d1867 | 490 | struct dma_fence *excl; |
e68a139f CW |
491 | int ret = 0, pending; |
492 | ||
fc158405 | 493 | debug_fence_assert(fence); |
e30a7581 | 494 | might_sleep_if(gfpflags_allow_blocking(gfp)); |
fc158405 | 495 | |
e68a139f | 496 | if (write) { |
f54d1867 | 497 | struct dma_fence **shared; |
e68a139f CW |
498 | unsigned int count, i; |
499 | ||
500 | ret = reservation_object_get_fences_rcu(resv, | |
501 | &excl, &count, &shared); | |
502 | if (ret) | |
503 | return ret; | |
504 | ||
505 | for (i = 0; i < count; i++) { | |
506 | if (shared[i]->ops == exclude) | |
507 | continue; | |
508 | ||
509 | pending = i915_sw_fence_await_dma_fence(fence, | |
510 | shared[i], | |
511 | timeout, | |
512 | gfp); | |
513 | if (pending < 0) { | |
514 | ret = pending; | |
515 | break; | |
516 | } | |
517 | ||
518 | ret |= pending; | |
519 | } | |
520 | ||
521 | for (i = 0; i < count; i++) | |
f54d1867 | 522 | dma_fence_put(shared[i]); |
e68a139f CW |
523 | kfree(shared); |
524 | } else { | |
525 | excl = reservation_object_get_excl_rcu(resv); | |
526 | } | |
527 | ||
528 | if (ret >= 0 && excl && excl->ops != exclude) { | |
529 | pending = i915_sw_fence_await_dma_fence(fence, | |
530 | excl, | |
531 | timeout, | |
532 | gfp); | |
533 | if (pending < 0) | |
534 | ret = pending; | |
535 | else | |
536 | ret |= pending; | |
537 | } | |
538 | ||
f54d1867 | 539 | dma_fence_put(excl); |
e68a139f CW |
540 | |
541 | return ret; | |
542 | } | |
47624cc3 CW |
543 | |
544 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
214707fc | 545 | #include "selftests/lib_sw_fence.c" |
47624cc3 CW |
546 | #include "selftests/i915_sw_fence.c" |
547 | #endif |