Commit | Line | Data |
---|---|---|
1802d0be | 1 | // SPDX-License-Identifier: GPL-2.0-only |
e941759c ML |
2 | /* |
3 | * Fence mechanism for dma-buf and to allow for asynchronous dma access | |
4 | * | |
5 | * Copyright (C) 2012 Canonical Ltd | |
6 | * Copyright (C) 2012 Texas Instruments | |
7 | * | |
8 | * Authors: | |
9 | * Rob Clark <robdclark@gmail.com> | |
10 | * Maarten Lankhorst <maarten.lankhorst@canonical.com> | |
e941759c ML |
11 | */ |
12 | ||
13 | #include <linux/slab.h> | |
14 | #include <linux/export.h> | |
15 | #include <linux/atomic.h> | |
f54d1867 | 16 | #include <linux/dma-fence.h> |
174cd4b1 | 17 | #include <linux/sched/signal.h> |
e941759c ML |
18 | |
19 | #define CREATE_TRACE_POINTS | |
f54d1867 | 20 | #include <trace/events/dma_fence.h> |
e941759c | 21 | |
f54d1867 | 22 | EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit); |
8c96c678 | 23 | EXPORT_TRACEPOINT_SYMBOL(dma_fence_enable_signal); |
c36beba6 | 24 | EXPORT_TRACEPOINT_SYMBOL(dma_fence_signaled); |
e941759c | 25 | |
078dec33 CK |
26 | static DEFINE_SPINLOCK(dma_fence_stub_lock); |
27 | static struct dma_fence dma_fence_stub; | |
28 | ||
e9f3b796 | 29 | /* |
e941759c ML |
30 | * fence context counter: each execution context should have its own |
31 | * fence context, this allows checking if fences belong to the same | |
32 | * context or not. One device can have multiple separate contexts, | |
33 | * and they're used if some engine can run independently of another. | |
34 | */ | |
078dec33 | 35 | static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(1); |
e941759c | 36 | |
4dd3cdb2 DV |
37 | /** |
38 | * DOC: DMA fences overview | |
39 | * | |
40 | * DMA fences, represented by &struct dma_fence, are the kernel internal | |
41 | * synchronization primitive for DMA operations like GPU rendering, video | |
42 | * encoding/decoding, or displaying buffers on a screen. | |
43 | * | |
44 | * A fence is initialized using dma_fence_init() and completed using | |
45 | * dma_fence_signal(). Fences are associated with a context, allocated through | |
46 | * dma_fence_context_alloc(), and all fences on the same context are | |
47 | * fully ordered. | |
48 | * | |
49 | * Since the purposes of fences is to facilitate cross-device and | |
50 | * cross-application synchronization, there's multiple ways to use one: | |
51 | * | |
52 | * - Individual fences can be exposed as a &sync_file, accessed as a file | |
53 | * descriptor from userspace, created by calling sync_file_create(). This is | |
54 | * called explicit fencing, since userspace passes around explicit | |
55 | * synchronization points. | |
56 | * | |
57 | * - Some subsystems also have their own explicit fencing primitives, like | |
58 | * &drm_syncobj. Compared to &sync_file, a &drm_syncobj allows the underlying | |
59 | * fence to be updated. | |
60 | * | |
61 | * - Then there's also implicit fencing, where the synchronization points are | |
62 | * implicitly passed around as part of shared &dma_buf instances. Such | |
52791eee | 63 | * implicit fences are stored in &struct dma_resv through the |
4dd3cdb2 DV |
64 | * &dma_buf.resv pointer. |
65 | */ | |
66 | ||
d0b9a9ae DV |
67 | /** |
68 | * DOC: fence cross-driver contract | |
69 | * | |
70 | * Since &dma_fence provide a cross driver contract, all drivers must follow the | |
71 | * same rules: | |
72 | * | |
73 | * * Fences must complete in a reasonable time. Fences which represent kernels | |
74 | * and shaders submitted by userspace, which could run forever, must be backed | |
75 | * up by timeout and gpu hang recovery code. Minimally that code must prevent | |
76 | * further command submission and force complete all in-flight fences, e.g. | |
77 | * when the driver or hardware do not support gpu reset, or if the gpu reset | |
78 | * failed for some reason. Ideally the driver supports gpu recovery which only | |
79 | * affects the offending userspace context, and no other userspace | |
80 | * submissions. | |
81 | * | |
82 | * * Drivers may have different ideas of what completion within a reasonable | |
83 | * time means. Some hang recovery code uses a fixed timeout, others a mix | |
84 | * between observing forward progress and increasingly strict timeouts. | |
85 | * Drivers should not try to second guess timeout handling of fences from | |
86 | * other drivers. | |
87 | * | |
88 | * * To ensure there's no deadlocks of dma_fence_wait() against other locks | |
89 | * drivers should annotate all code required to reach dma_fence_signal(), | |
90 | * which completes the fences, with dma_fence_begin_signalling() and | |
91 | * dma_fence_end_signalling(). | |
92 | * | |
93 | * * Drivers are allowed to call dma_fence_wait() while holding dma_resv_lock(). | |
94 | * This means any code required for fence completion cannot acquire a | |
95 | * &dma_resv lock. Note that this also pulls in the entire established | |
96 | * locking hierarchy around dma_resv_lock() and dma_resv_unlock(). | |
97 | * | |
98 | * * Drivers are allowed to call dma_fence_wait() from their &shrinker | |
99 | * callbacks. This means any code required for fence completion cannot | |
100 | * allocate memory with GFP_KERNEL. | |
101 | * | |
102 | * * Drivers are allowed to call dma_fence_wait() from their &mmu_notifier | |
103 | * respectively &mmu_interval_notifier callbacks. This means any code required | |
104 | * for fence completeion cannot allocate memory with GFP_NOFS or GFP_NOIO. | |
105 | * Only GFP_ATOMIC is permissible, which might fail. | |
106 | * | |
107 | * Note that only GPU drivers have a reasonable excuse for both requiring | |
108 | * &mmu_interval_notifier and &shrinker callbacks at the same time as having to | |
109 | * track asynchronous compute work using &dma_fence. No driver outside of | |
110 | * drivers/gpu should ever call dma_fence_wait() in such contexts. | |
111 | */ | |
112 | ||
078dec33 CK |
113 | static const char *dma_fence_stub_get_name(struct dma_fence *fence) |
114 | { | |
115 | return "stub"; | |
116 | } | |
117 | ||
118 | static const struct dma_fence_ops dma_fence_stub_ops = { | |
119 | .get_driver_name = dma_fence_stub_get_name, | |
120 | .get_timeline_name = dma_fence_stub_get_name, | |
121 | }; | |
122 | ||
123 | /** | |
124 | * dma_fence_get_stub - return a signaled fence | |
125 | * | |
fd921693 DS |
126 | * Return a stub fence which is already signaled. The fence's |
127 | * timestamp corresponds to the first time after boot this | |
128 | * function is called. | |
078dec33 CK |
129 | */ |
130 | struct dma_fence *dma_fence_get_stub(void) | |
131 | { | |
132 | spin_lock(&dma_fence_stub_lock); | |
133 | if (!dma_fence_stub.ops) { | |
134 | dma_fence_init(&dma_fence_stub, | |
135 | &dma_fence_stub_ops, | |
136 | &dma_fence_stub_lock, | |
137 | 0, 0); | |
138 | dma_fence_signal_locked(&dma_fence_stub); | |
139 | } | |
140 | spin_unlock(&dma_fence_stub_lock); | |
141 | ||
142 | return dma_fence_get(&dma_fence_stub); | |
143 | } | |
144 | EXPORT_SYMBOL(dma_fence_get_stub); | |
145 | ||
fd921693 DS |
146 | /** |
147 | * dma_fence_allocate_private_stub - return a private, signaled fence | |
148 | * | |
149 | * Return a newly allocated and signaled stub fence. | |
150 | */ | |
151 | struct dma_fence *dma_fence_allocate_private_stub(void) | |
152 | { | |
153 | struct dma_fence *fence; | |
154 | ||
155 | fence = kzalloc(sizeof(*fence), GFP_KERNEL); | |
156 | if (fence == NULL) | |
157 | return ERR_PTR(-ENOMEM); | |
158 | ||
159 | dma_fence_init(fence, | |
160 | &dma_fence_stub_ops, | |
161 | &dma_fence_stub_lock, | |
162 | 0, 0); | |
163 | dma_fence_signal(fence); | |
164 | ||
165 | return fence; | |
166 | } | |
167 | EXPORT_SYMBOL(dma_fence_allocate_private_stub); | |
168 | ||
e941759c | 169 | /** |
f54d1867 | 170 | * dma_fence_context_alloc - allocate an array of fence contexts |
4dd3cdb2 | 171 | * @num: amount of contexts to allocate |
e941759c | 172 | * |
4dd3cdb2 DV |
173 | * This function will return the first index of the number of fence contexts |
174 | * allocated. The fence context is used for setting &dma_fence.context to a | |
175 | * unique number by passing the context to dma_fence_init(). | |
e941759c | 176 | */ |
f54d1867 | 177 | u64 dma_fence_context_alloc(unsigned num) |
e941759c | 178 | { |
6ce31263 | 179 | WARN_ON(!num); |
1c530d43 | 180 | return atomic64_fetch_add(num, &dma_fence_context_counter); |
e941759c | 181 | } |
f54d1867 | 182 | EXPORT_SYMBOL(dma_fence_context_alloc); |
e941759c | 183 | |
5fbff813 DV |
184 | /** |
185 | * DOC: fence signalling annotation | |
186 | * | |
187 | * Proving correctness of all the kernel code around &dma_fence through code | |
188 | * review and testing is tricky for a few reasons: | |
189 | * | |
190 | * * It is a cross-driver contract, and therefore all drivers must follow the | |
191 | * same rules for lock nesting order, calling contexts for various functions | |
192 | * and anything else significant for in-kernel interfaces. But it is also | |
193 | * impossible to test all drivers in a single machine, hence brute-force N vs. | |
194 | * N testing of all combinations is impossible. Even just limiting to the | |
195 | * possible combinations is infeasible. | |
196 | * | |
197 | * * There is an enormous amount of driver code involved. For render drivers | |
198 | * there's the tail of command submission, after fences are published, | |
199 | * scheduler code, interrupt and workers to process job completion, | |
200 | * and timeout, gpu reset and gpu hang recovery code. Plus for integration | |
201 | * with core mm with have &mmu_notifier, respectively &mmu_interval_notifier, | |
202 | * and &shrinker. For modesetting drivers there's the commit tail functions | |
203 | * between when fences for an atomic modeset are published, and when the | |
204 | * corresponding vblank completes, including any interrupt processing and | |
205 | * related workers. Auditing all that code, across all drivers, is not | |
206 | * feasible. | |
207 | * | |
208 | * * Due to how many other subsystems are involved and the locking hierarchies | |
209 | * this pulls in there is extremely thin wiggle-room for driver-specific | |
210 | * differences. &dma_fence interacts with almost all of the core memory | |
211 | * handling through page fault handlers via &dma_resv, dma_resv_lock() and | |
212 | * dma_resv_unlock(). On the other side it also interacts through all | |
213 | * allocation sites through &mmu_notifier and &shrinker. | |
214 | * | |
215 | * Furthermore lockdep does not handle cross-release dependencies, which means | |
216 | * any deadlocks between dma_fence_wait() and dma_fence_signal() can't be caught | |
217 | * at runtime with some quick testing. The simplest example is one thread | |
218 | * waiting on a &dma_fence while holding a lock:: | |
219 | * | |
220 | * lock(A); | |
221 | * dma_fence_wait(B); | |
222 | * unlock(A); | |
223 | * | |
224 | * while the other thread is stuck trying to acquire the same lock, which | |
225 | * prevents it from signalling the fence the previous thread is stuck waiting | |
226 | * on:: | |
227 | * | |
228 | * lock(A); | |
229 | * unlock(A); | |
230 | * dma_fence_signal(B); | |
231 | * | |
232 | * By manually annotating all code relevant to signalling a &dma_fence we can | |
233 | * teach lockdep about these dependencies, which also helps with the validation | |
234 | * headache since now lockdep can check all the rules for us:: | |
235 | * | |
236 | * cookie = dma_fence_begin_signalling(); | |
237 | * lock(A); | |
238 | * unlock(A); | |
239 | * dma_fence_signal(B); | |
240 | * dma_fence_end_signalling(cookie); | |
241 | * | |
242 | * For using dma_fence_begin_signalling() and dma_fence_end_signalling() to | |
243 | * annotate critical sections the following rules need to be observed: | |
244 | * | |
245 | * * All code necessary to complete a &dma_fence must be annotated, from the | |
246 | * point where a fence is accessible to other threads, to the point where | |
247 | * dma_fence_signal() is called. Un-annotated code can contain deadlock issues, | |
248 | * and due to the very strict rules and many corner cases it is infeasible to | |
249 | * catch these just with review or normal stress testing. | |
250 | * | |
251 | * * &struct dma_resv deserves a special note, since the readers are only | |
252 | * protected by rcu. This means the signalling critical section starts as soon | |
253 | * as the new fences are installed, even before dma_resv_unlock() is called. | |
254 | * | |
255 | * * The only exception are fast paths and opportunistic signalling code, which | |
256 | * calls dma_fence_signal() purely as an optimization, but is not required to | |
257 | * guarantee completion of a &dma_fence. The usual example is a wait IOCTL | |
258 | * which calls dma_fence_signal(), while the mandatory completion path goes | |
259 | * through a hardware interrupt and possible job completion worker. | |
260 | * | |
261 | * * To aid composability of code, the annotations can be freely nested, as long | |
262 | * as the overall locking hierarchy is consistent. The annotations also work | |
263 | * both in interrupt and process context. Due to implementation details this | |
264 | * requires that callers pass an opaque cookie from | |
265 | * dma_fence_begin_signalling() to dma_fence_end_signalling(). | |
266 | * | |
267 | * * Validation against the cross driver contract is implemented by priming | |
268 | * lockdep with the relevant hierarchy at boot-up. This means even just | |
269 | * testing with a single device is enough to validate a driver, at least as | |
270 | * far as deadlocks with dma_fence_wait() against dma_fence_signal() are | |
271 | * concerned. | |
272 | */ | |
273 | #ifdef CONFIG_LOCKDEP | |
fa07634d | 274 | static struct lockdep_map dma_fence_lockdep_map = { |
5fbff813 DV |
275 | .name = "dma_fence_map" |
276 | }; | |
277 | ||
278 | /** | |
279 | * dma_fence_begin_signalling - begin a critical DMA fence signalling section | |
280 | * | |
281 | * Drivers should use this to annotate the beginning of any code section | |
282 | * required to eventually complete &dma_fence by calling dma_fence_signal(). | |
283 | * | |
284 | * The end of these critical sections are annotated with | |
285 | * dma_fence_end_signalling(). | |
286 | * | |
287 | * Returns: | |
288 | * | |
289 | * Opaque cookie needed by the implementation, which needs to be passed to | |
290 | * dma_fence_end_signalling(). | |
291 | */ | |
292 | bool dma_fence_begin_signalling(void) | |
293 | { | |
294 | /* explicitly nesting ... */ | |
295 | if (lock_is_held_type(&dma_fence_lockdep_map, 1)) | |
296 | return true; | |
297 | ||
298 | /* rely on might_sleep check for soft/hardirq locks */ | |
299 | if (in_atomic()) | |
300 | return true; | |
301 | ||
302 | /* ... and non-recursive readlock */ | |
303 | lock_acquire(&dma_fence_lockdep_map, 0, 0, 1, 1, NULL, _RET_IP_); | |
304 | ||
305 | return false; | |
306 | } | |
307 | EXPORT_SYMBOL(dma_fence_begin_signalling); | |
308 | ||
309 | /** | |
310 | * dma_fence_end_signalling - end a critical DMA fence signalling section | |
e44cd6bc | 311 | * @cookie: opaque cookie from dma_fence_begin_signalling() |
5fbff813 DV |
312 | * |
313 | * Closes a critical section annotation opened by dma_fence_begin_signalling(). | |
314 | */ | |
315 | void dma_fence_end_signalling(bool cookie) | |
316 | { | |
317 | if (cookie) | |
318 | return; | |
319 | ||
320 | lock_release(&dma_fence_lockdep_map, _RET_IP_); | |
321 | } | |
322 | EXPORT_SYMBOL(dma_fence_end_signalling); | |
323 | ||
324 | void __dma_fence_might_wait(void) | |
325 | { | |
326 | bool tmp; | |
327 | ||
328 | tmp = lock_is_held_type(&dma_fence_lockdep_map, 1); | |
329 | if (tmp) | |
330 | lock_release(&dma_fence_lockdep_map, _THIS_IP_); | |
331 | lock_map_acquire(&dma_fence_lockdep_map); | |
332 | lock_map_release(&dma_fence_lockdep_map); | |
333 | if (tmp) | |
334 | lock_acquire(&dma_fence_lockdep_map, 0, 0, 1, 1, NULL, _THIS_IP_); | |
335 | } | |
336 | #endif | |
337 | ||
338 | ||
e941759c | 339 | /** |
5a164ac4 | 340 | * dma_fence_signal_timestamp_locked - signal completion of a fence |
e941759c | 341 | * @fence: the fence to signal |
5a164ac4 | 342 | * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain |
e941759c ML |
343 | * |
344 | * Signal completion for software callbacks on a fence, this will unblock | |
f54d1867 CW |
345 | * dma_fence_wait() calls and run all the callbacks added with |
346 | * dma_fence_add_callback(). Can be called multiple times, but since a fence | |
4dd3cdb2 | 347 | * can only go from the unsignaled to the signaled state and not back, it will |
5a164ac4 VSS |
348 | * only be effective the first time. Set the timestamp provided as the fence |
349 | * signal timestamp. | |
4dd3cdb2 | 350 | * |
5a164ac4 VSS |
351 | * Unlike dma_fence_signal_timestamp(), this function must be called with |
352 | * &dma_fence.lock held. | |
e941759c | 353 | * |
4dd3cdb2 DV |
354 | * Returns 0 on success and a negative error value when @fence has been |
355 | * signalled already. | |
e941759c | 356 | */ |
5a164ac4 VSS |
357 | int dma_fence_signal_timestamp_locked(struct dma_fence *fence, |
358 | ktime_t timestamp) | |
e941759c | 359 | { |
f54d1867 | 360 | struct dma_fence_cb *cur, *tmp; |
f2cb60e9 | 361 | struct list_head cb_list; |
e941759c | 362 | |
78010cd9 RC |
363 | lockdep_assert_held(fence->lock); |
364 | ||
0fc89b68 CW |
365 | if (unlikely(test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, |
366 | &fence->flags))) | |
e941759c ML |
367 | return -EINVAL; |
368 | ||
f2cb60e9 CW |
369 | /* Stash the cb_list before replacing it with the timestamp */ |
370 | list_replace(&fence->cb_list, &cb_list); | |
371 | ||
5a164ac4 | 372 | fence->timestamp = timestamp; |
0fc89b68 CW |
373 | set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); |
374 | trace_dma_fence_signaled(fence); | |
e941759c | 375 | |
f2cb60e9 CW |
376 | list_for_each_entry_safe(cur, tmp, &cb_list, node) { |
377 | INIT_LIST_HEAD(&cur->node); | |
378 | cur->func(fence, cur); | |
e941759c | 379 | } |
0fc89b68 CW |
380 | |
381 | return 0; | |
e941759c | 382 | } |
5a164ac4 VSS |
383 | EXPORT_SYMBOL(dma_fence_signal_timestamp_locked); |
384 | ||
385 | /** | |
386 | * dma_fence_signal_timestamp - signal completion of a fence | |
387 | * @fence: the fence to signal | |
388 | * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain | |
389 | * | |
390 | * Signal completion for software callbacks on a fence, this will unblock | |
391 | * dma_fence_wait() calls and run all the callbacks added with | |
392 | * dma_fence_add_callback(). Can be called multiple times, but since a fence | |
393 | * can only go from the unsignaled to the signaled state and not back, it will | |
394 | * only be effective the first time. Set the timestamp provided as the fence | |
395 | * signal timestamp. | |
396 | * | |
397 | * Returns 0 on success and a negative error value when @fence has been | |
398 | * signalled already. | |
399 | */ | |
400 | int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp) | |
401 | { | |
402 | unsigned long flags; | |
403 | int ret; | |
404 | ||
405 | if (!fence) | |
406 | return -EINVAL; | |
407 | ||
408 | spin_lock_irqsave(fence->lock, flags); | |
409 | ret = dma_fence_signal_timestamp_locked(fence, timestamp); | |
410 | spin_unlock_irqrestore(fence->lock, flags); | |
411 | ||
412 | return ret; | |
413 | } | |
414 | EXPORT_SYMBOL(dma_fence_signal_timestamp); | |
415 | ||
416 | /** | |
417 | * dma_fence_signal_locked - signal completion of a fence | |
418 | * @fence: the fence to signal | |
419 | * | |
420 | * Signal completion for software callbacks on a fence, this will unblock | |
421 | * dma_fence_wait() calls and run all the callbacks added with | |
422 | * dma_fence_add_callback(). Can be called multiple times, but since a fence | |
423 | * can only go from the unsignaled to the signaled state and not back, it will | |
424 | * only be effective the first time. | |
425 | * | |
426 | * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock | |
427 | * held. | |
428 | * | |
429 | * Returns 0 on success and a negative error value when @fence has been | |
430 | * signalled already. | |
431 | */ | |
432 | int dma_fence_signal_locked(struct dma_fence *fence) | |
433 | { | |
434 | return dma_fence_signal_timestamp_locked(fence, ktime_get()); | |
435 | } | |
f54d1867 | 436 | EXPORT_SYMBOL(dma_fence_signal_locked); |
e941759c ML |
437 | |
438 | /** | |
f54d1867 | 439 | * dma_fence_signal - signal completion of a fence |
e941759c ML |
440 | * @fence: the fence to signal |
441 | * | |
442 | * Signal completion for software callbacks on a fence, this will unblock | |
f54d1867 CW |
443 | * dma_fence_wait() calls and run all the callbacks added with |
444 | * dma_fence_add_callback(). Can be called multiple times, but since a fence | |
4dd3cdb2 DV |
445 | * can only go from the unsignaled to the signaled state and not back, it will |
446 | * only be effective the first time. | |
447 | * | |
448 | * Returns 0 on success and a negative error value when @fence has been | |
449 | * signalled already. | |
e941759c | 450 | */ |
f54d1867 | 451 | int dma_fence_signal(struct dma_fence *fence) |
e941759c ML |
452 | { |
453 | unsigned long flags; | |
0fc89b68 | 454 | int ret; |
5fbff813 | 455 | bool tmp; |
e941759c ML |
456 | |
457 | if (!fence) | |
458 | return -EINVAL; | |
459 | ||
5fbff813 DV |
460 | tmp = dma_fence_begin_signalling(); |
461 | ||
0fc89b68 | 462 | spin_lock_irqsave(fence->lock, flags); |
5a164ac4 | 463 | ret = dma_fence_signal_timestamp_locked(fence, ktime_get()); |
0fc89b68 | 464 | spin_unlock_irqrestore(fence->lock, flags); |
e941759c | 465 | |
5fbff813 DV |
466 | dma_fence_end_signalling(tmp); |
467 | ||
0fc89b68 | 468 | return ret; |
e941759c | 469 | } |
f54d1867 | 470 | EXPORT_SYMBOL(dma_fence_signal); |
e941759c ML |
471 | |
472 | /** | |
f54d1867 | 473 | * dma_fence_wait_timeout - sleep until the fence gets signaled |
e941759c | 474 | * or until timeout elapses |
4dd3cdb2 DV |
475 | * @fence: the fence to wait on |
476 | * @intr: if true, do an interruptible wait | |
477 | * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT | |
e941759c ML |
478 | * |
479 | * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the | |
480 | * remaining timeout in jiffies on success. Other error values may be | |
481 | * returned on custom implementations. | |
482 | * | |
483 | * Performs a synchronous wait on this fence. It is assumed the caller | |
484 | * directly or indirectly (buf-mgr between reservation and committing) | |
485 | * holds a reference to the fence, otherwise the fence might be | |
486 | * freed before return, resulting in undefined behavior. | |
4dd3cdb2 DV |
487 | * |
488 | * See also dma_fence_wait() and dma_fence_wait_any_timeout(). | |
e941759c ML |
489 | */ |
490 | signed long | |
f54d1867 | 491 | dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout) |
e941759c ML |
492 | { |
493 | signed long ret; | |
494 | ||
495 | if (WARN_ON(timeout < 0)) | |
496 | return -EINVAL; | |
497 | ||
ef825550 DV |
498 | might_sleep(); |
499 | ||
5fbff813 DV |
500 | __dma_fence_might_wait(); |
501 | ||
f54d1867 | 502 | trace_dma_fence_wait_start(fence); |
418cc6ca DV |
503 | if (fence->ops->wait) |
504 | ret = fence->ops->wait(fence, intr, timeout); | |
505 | else | |
506 | ret = dma_fence_default_wait(fence, intr, timeout); | |
f54d1867 | 507 | trace_dma_fence_wait_end(fence); |
e941759c ML |
508 | return ret; |
509 | } | |
f54d1867 | 510 | EXPORT_SYMBOL(dma_fence_wait_timeout); |
e941759c | 511 | |
4dd3cdb2 DV |
512 | /** |
513 | * dma_fence_release - default relese function for fences | |
514 | * @kref: &dma_fence.recfount | |
515 | * | |
516 | * This is the default release functions for &dma_fence. Drivers shouldn't call | |
517 | * this directly, but instead call dma_fence_put(). | |
518 | */ | |
f54d1867 | 519 | void dma_fence_release(struct kref *kref) |
e941759c | 520 | { |
f54d1867 CW |
521 | struct dma_fence *fence = |
522 | container_of(kref, struct dma_fence, refcount); | |
e941759c | 523 | |
f54d1867 | 524 | trace_dma_fence_destroy(fence); |
e941759c | 525 | |
f2cb60e9 CW |
526 | if (WARN(!list_empty(&fence->cb_list) && |
527 | !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags), | |
427231bc CW |
528 | "Fence %s:%s:%llx:%llx released with pending signals!\n", |
529 | fence->ops->get_driver_name(fence), | |
530 | fence->ops->get_timeline_name(fence), | |
531 | fence->context, fence->seqno)) { | |
532 | unsigned long flags; | |
533 | ||
534 | /* | |
535 | * Failed to signal before release, likely a refcounting issue. | |
536 | * | |
537 | * This should never happen, but if it does make sure that we | |
538 | * don't leave chains dangling. We set the error flag first | |
539 | * so that the callbacks know this signal is due to an error. | |
540 | */ | |
541 | spin_lock_irqsave(fence->lock, flags); | |
542 | fence->error = -EDEADLK; | |
543 | dma_fence_signal_locked(fence); | |
544 | spin_unlock_irqrestore(fence->lock, flags); | |
545 | } | |
e941759c ML |
546 | |
547 | if (fence->ops->release) | |
548 | fence->ops->release(fence); | |
549 | else | |
f54d1867 | 550 | dma_fence_free(fence); |
e941759c | 551 | } |
f54d1867 | 552 | EXPORT_SYMBOL(dma_fence_release); |
e941759c | 553 | |
4dd3cdb2 DV |
554 | /** |
555 | * dma_fence_free - default release function for &dma_fence. | |
556 | * @fence: fence to release | |
557 | * | |
558 | * This is the default implementation for &dma_fence_ops.release. It calls | |
559 | * kfree_rcu() on @fence. | |
560 | */ | |
f54d1867 | 561 | void dma_fence_free(struct dma_fence *fence) |
e941759c | 562 | { |
3c3b177a | 563 | kfree_rcu(fence, rcu); |
e941759c | 564 | } |
f54d1867 | 565 | EXPORT_SYMBOL(dma_fence_free); |
e941759c | 566 | |
9c98f021 CW |
567 | static bool __dma_fence_enable_signaling(struct dma_fence *fence) |
568 | { | |
569 | bool was_set; | |
570 | ||
571 | lockdep_assert_held(fence->lock); | |
572 | ||
573 | was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, | |
574 | &fence->flags); | |
575 | ||
576 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) | |
577 | return false; | |
578 | ||
579 | if (!was_set && fence->ops->enable_signaling) { | |
580 | trace_dma_fence_enable_signal(fence); | |
581 | ||
582 | if (!fence->ops->enable_signaling(fence)) { | |
583 | dma_fence_signal_locked(fence); | |
584 | return false; | |
585 | } | |
586 | } | |
587 | ||
588 | return true; | |
589 | } | |
590 | ||
e941759c | 591 | /** |
f54d1867 | 592 | * dma_fence_enable_sw_signaling - enable signaling on fence |
4dd3cdb2 | 593 | * @fence: the fence to enable |
e941759c | 594 | * |
4dd3cdb2 DV |
595 | * This will request for sw signaling to be enabled, to make the fence |
596 | * complete as soon as possible. This calls &dma_fence_ops.enable_signaling | |
597 | * internally. | |
e941759c | 598 | */ |
f54d1867 | 599 | void dma_fence_enable_sw_signaling(struct dma_fence *fence) |
e941759c ML |
600 | { |
601 | unsigned long flags; | |
602 | ||
9c98f021 CW |
603 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) |
604 | return; | |
e941759c | 605 | |
9c98f021 CW |
606 | spin_lock_irqsave(fence->lock, flags); |
607 | __dma_fence_enable_signaling(fence); | |
608 | spin_unlock_irqrestore(fence->lock, flags); | |
e941759c | 609 | } |
f54d1867 | 610 | EXPORT_SYMBOL(dma_fence_enable_sw_signaling); |
e941759c ML |
611 | |
612 | /** | |
f54d1867 | 613 | * dma_fence_add_callback - add a callback to be called when the fence |
e941759c | 614 | * is signaled |
4dd3cdb2 DV |
615 | * @fence: the fence to wait on |
616 | * @cb: the callback to register | |
617 | * @func: the function to call | |
e941759c | 618 | * |
4dd3cdb2 | 619 | * @cb will be initialized by dma_fence_add_callback(), no initialization |
e941759c ML |
620 | * by the caller is required. Any number of callbacks can be registered |
621 | * to a fence, but a callback can only be registered to one fence at a time. | |
622 | * | |
623 | * Note that the callback can be called from an atomic context. If | |
624 | * fence is already signaled, this function will return -ENOENT (and | |
4dd3cdb2 | 625 | * *not* call the callback). |
e941759c ML |
626 | * |
627 | * Add a software callback to the fence. Same restrictions apply to | |
4dd3cdb2 DV |
628 | * refcount as it does to dma_fence_wait(), however the caller doesn't need to |
629 | * keep a refcount to fence afterward dma_fence_add_callback() has returned: | |
630 | * when software access is enabled, the creator of the fence is required to keep | |
631 | * the fence alive until after it signals with dma_fence_signal(). The callback | |
632 | * itself can be called from irq context. | |
e941759c | 633 | * |
f642de16 GP |
634 | * Returns 0 in case of success, -ENOENT if the fence is already signaled |
635 | * and -EINVAL in case of error. | |
e941759c | 636 | */ |
f54d1867 CW |
637 | int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb, |
638 | dma_fence_func_t func) | |
e941759c ML |
639 | { |
640 | unsigned long flags; | |
641 | int ret = 0; | |
e941759c ML |
642 | |
643 | if (WARN_ON(!fence || !func)) | |
644 | return -EINVAL; | |
645 | ||
f54d1867 | 646 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { |
e941759c ML |
647 | INIT_LIST_HEAD(&cb->node); |
648 | return -ENOENT; | |
649 | } | |
650 | ||
651 | spin_lock_irqsave(fence->lock, flags); | |
652 | ||
9c98f021 | 653 | if (__dma_fence_enable_signaling(fence)) { |
e941759c ML |
654 | cb->func = func; |
655 | list_add_tail(&cb->node, &fence->cb_list); | |
9c98f021 | 656 | } else { |
e941759c | 657 | INIT_LIST_HEAD(&cb->node); |
9c98f021 CW |
658 | ret = -ENOENT; |
659 | } | |
660 | ||
e941759c ML |
661 | spin_unlock_irqrestore(fence->lock, flags); |
662 | ||
663 | return ret; | |
664 | } | |
f54d1867 | 665 | EXPORT_SYMBOL(dma_fence_add_callback); |
e941759c | 666 | |
d6c99f4b CW |
667 | /** |
668 | * dma_fence_get_status - returns the status upon completion | |
4dd3cdb2 | 669 | * @fence: the dma_fence to query |
d6c99f4b CW |
670 | * |
671 | * This wraps dma_fence_get_status_locked() to return the error status | |
672 | * condition on a signaled fence. See dma_fence_get_status_locked() for more | |
673 | * details. | |
674 | * | |
675 | * Returns 0 if the fence has not yet been signaled, 1 if the fence has | |
676 | * been signaled without an error condition, or a negative error code | |
677 | * if the fence has been completed in err. | |
678 | */ | |
679 | int dma_fence_get_status(struct dma_fence *fence) | |
680 | { | |
681 | unsigned long flags; | |
682 | int status; | |
683 | ||
684 | spin_lock_irqsave(fence->lock, flags); | |
685 | status = dma_fence_get_status_locked(fence); | |
686 | spin_unlock_irqrestore(fence->lock, flags); | |
687 | ||
688 | return status; | |
689 | } | |
690 | EXPORT_SYMBOL(dma_fence_get_status); | |
691 | ||
e941759c | 692 | /** |
f54d1867 | 693 | * dma_fence_remove_callback - remove a callback from the signaling list |
4dd3cdb2 DV |
694 | * @fence: the fence to wait on |
695 | * @cb: the callback to remove | |
e941759c ML |
696 | * |
697 | * Remove a previously queued callback from the fence. This function returns | |
f353d71f | 698 | * true if the callback is successfully removed, or false if the fence has |
e941759c ML |
699 | * already been signaled. |
700 | * | |
701 | * *WARNING*: | |
702 | * Cancelling a callback should only be done if you really know what you're | |
703 | * doing, since deadlocks and race conditions could occur all too easily. For | |
704 | * this reason, it should only ever be done on hardware lockup recovery, | |
705 | * with a reference held to the fence. | |
4dd3cdb2 DV |
706 | * |
707 | * Behaviour is undefined if @cb has not been added to @fence using | |
708 | * dma_fence_add_callback() beforehand. | |
e941759c ML |
709 | */ |
710 | bool | |
f54d1867 | 711 | dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb) |
e941759c ML |
712 | { |
713 | unsigned long flags; | |
714 | bool ret; | |
715 | ||
716 | spin_lock_irqsave(fence->lock, flags); | |
717 | ||
718 | ret = !list_empty(&cb->node); | |
719 | if (ret) | |
720 | list_del_init(&cb->node); | |
721 | ||
722 | spin_unlock_irqrestore(fence->lock, flags); | |
723 | ||
724 | return ret; | |
725 | } | |
f54d1867 | 726 | EXPORT_SYMBOL(dma_fence_remove_callback); |
e941759c ML |
727 | |
728 | struct default_wait_cb { | |
f54d1867 | 729 | struct dma_fence_cb base; |
e941759c ML |
730 | struct task_struct *task; |
731 | }; | |
732 | ||
733 | static void | |
f54d1867 | 734 | dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb) |
e941759c ML |
735 | { |
736 | struct default_wait_cb *wait = | |
737 | container_of(cb, struct default_wait_cb, base); | |
738 | ||
739 | wake_up_state(wait->task, TASK_NORMAL); | |
740 | } | |
741 | ||
742 | /** | |
f54d1867 | 743 | * dma_fence_default_wait - default sleep until the fence gets signaled |
e941759c | 744 | * or until timeout elapses |
4dd3cdb2 DV |
745 | * @fence: the fence to wait on |
746 | * @intr: if true, do an interruptible wait | |
747 | * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT | |
e941759c ML |
748 | * |
749 | * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the | |
bcc004b6 AD |
750 | * remaining timeout in jiffies on success. If timeout is zero the value one is |
751 | * returned if the fence is already signaled for consistency with other | |
752 | * functions taking a jiffies timeout. | |
e941759c ML |
753 | */ |
754 | signed long | |
f54d1867 | 755 | dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout) |
e941759c ML |
756 | { |
757 | struct default_wait_cb cb; | |
758 | unsigned long flags; | |
bcc004b6 | 759 | signed long ret = timeout ? timeout : 1; |
e941759c | 760 | |
f54d1867 | 761 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) |
bcc004b6 | 762 | return ret; |
e941759c ML |
763 | |
764 | spin_lock_irqsave(fence->lock, flags); | |
765 | ||
766 | if (intr && signal_pending(current)) { | |
767 | ret = -ERESTARTSYS; | |
768 | goto out; | |
769 | } | |
770 | ||
9c98f021 | 771 | if (!__dma_fence_enable_signaling(fence)) |
e941759c ML |
772 | goto out; |
773 | ||
03c0c5f6 AR |
774 | if (!timeout) { |
775 | ret = 0; | |
776 | goto out; | |
777 | } | |
778 | ||
f54d1867 | 779 | cb.base.func = dma_fence_default_wait_cb; |
e941759c ML |
780 | cb.task = current; |
781 | list_add(&cb.base.node, &fence->cb_list); | |
782 | ||
f54d1867 | 783 | while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) { |
e941759c ML |
784 | if (intr) |
785 | __set_current_state(TASK_INTERRUPTIBLE); | |
786 | else | |
787 | __set_current_state(TASK_UNINTERRUPTIBLE); | |
788 | spin_unlock_irqrestore(fence->lock, flags); | |
789 | ||
790 | ret = schedule_timeout(ret); | |
791 | ||
792 | spin_lock_irqsave(fence->lock, flags); | |
793 | if (ret > 0 && intr && signal_pending(current)) | |
794 | ret = -ERESTARTSYS; | |
795 | } | |
796 | ||
797 | if (!list_empty(&cb.base.node)) | |
798 | list_del(&cb.base.node); | |
799 | __set_current_state(TASK_RUNNING); | |
800 | ||
801 | out: | |
802 | spin_unlock_irqrestore(fence->lock, flags); | |
803 | return ret; | |
804 | } | |
f54d1867 | 805 | EXPORT_SYMBOL(dma_fence_default_wait); |
e941759c | 806 | |
a519435a | 807 | static bool |
7392b4bb | 808 | dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count, |
809 | uint32_t *idx) | |
a519435a CK |
810 | { |
811 | int i; | |
812 | ||
813 | for (i = 0; i < count; ++i) { | |
f54d1867 | 814 | struct dma_fence *fence = fences[i]; |
7392b4bb | 815 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { |
816 | if (idx) | |
817 | *idx = i; | |
a519435a | 818 | return true; |
7392b4bb | 819 | } |
a519435a CK |
820 | } |
821 | return false; | |
822 | } | |
823 | ||
824 | /** | |
f54d1867 | 825 | * dma_fence_wait_any_timeout - sleep until any fence gets signaled |
a519435a | 826 | * or until timeout elapses |
4dd3cdb2 DV |
827 | * @fences: array of fences to wait on |
828 | * @count: number of fences to wait on | |
829 | * @intr: if true, do an interruptible wait | |
830 | * @timeout: timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT | |
831 | * @idx: used to store the first signaled fence index, meaningful only on | |
832 | * positive return | |
a519435a CK |
833 | * |
834 | * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if | |
835 | * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies | |
836 | * on success. | |
837 | * | |
838 | * Synchronous waits for the first fence in the array to be signaled. The | |
839 | * caller needs to hold a reference to all fences in the array, otherwise a | |
840 | * fence might be freed before return, resulting in undefined behavior. | |
4dd3cdb2 DV |
841 | * |
842 | * See also dma_fence_wait() and dma_fence_wait_timeout(). | |
a519435a CK |
843 | */ |
844 | signed long | |
f54d1867 | 845 | dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count, |
7392b4bb | 846 | bool intr, signed long timeout, uint32_t *idx) |
a519435a CK |
847 | { |
848 | struct default_wait_cb *cb; | |
849 | signed long ret = timeout; | |
850 | unsigned i; | |
851 | ||
852 | if (WARN_ON(!fences || !count || timeout < 0)) | |
853 | return -EINVAL; | |
854 | ||
855 | if (timeout == 0) { | |
856 | for (i = 0; i < count; ++i) | |
7392b4bb | 857 | if (dma_fence_is_signaled(fences[i])) { |
858 | if (idx) | |
859 | *idx = i; | |
a519435a | 860 | return 1; |
7392b4bb | 861 | } |
a519435a CK |
862 | |
863 | return 0; | |
864 | } | |
865 | ||
866 | cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL); | |
867 | if (cb == NULL) { | |
868 | ret = -ENOMEM; | |
869 | goto err_free_cb; | |
870 | } | |
871 | ||
872 | for (i = 0; i < count; ++i) { | |
f54d1867 | 873 | struct dma_fence *fence = fences[i]; |
a519435a | 874 | |
a519435a | 875 | cb[i].task = current; |
f54d1867 CW |
876 | if (dma_fence_add_callback(fence, &cb[i].base, |
877 | dma_fence_default_wait_cb)) { | |
a519435a | 878 | /* This fence is already signaled */ |
7392b4bb | 879 | if (idx) |
880 | *idx = i; | |
a519435a CK |
881 | goto fence_rm_cb; |
882 | } | |
883 | } | |
884 | ||
885 | while (ret > 0) { | |
886 | if (intr) | |
887 | set_current_state(TASK_INTERRUPTIBLE); | |
888 | else | |
889 | set_current_state(TASK_UNINTERRUPTIBLE); | |
890 | ||
7392b4bb | 891 | if (dma_fence_test_signaled_any(fences, count, idx)) |
a519435a CK |
892 | break; |
893 | ||
894 | ret = schedule_timeout(ret); | |
895 | ||
896 | if (ret > 0 && intr && signal_pending(current)) | |
897 | ret = -ERESTARTSYS; | |
898 | } | |
899 | ||
900 | __set_current_state(TASK_RUNNING); | |
901 | ||
902 | fence_rm_cb: | |
903 | while (i-- > 0) | |
f54d1867 | 904 | dma_fence_remove_callback(fences[i], &cb[i].base); |
a519435a CK |
905 | |
906 | err_free_cb: | |
907 | kfree(cb); | |
908 | ||
909 | return ret; | |
910 | } | |
f54d1867 | 911 | EXPORT_SYMBOL(dma_fence_wait_any_timeout); |
a519435a | 912 | |
e941759c | 913 | /** |
f54d1867 | 914 | * dma_fence_init - Initialize a custom fence. |
4dd3cdb2 DV |
915 | * @fence: the fence to initialize |
916 | * @ops: the dma_fence_ops for operations on this fence | |
917 | * @lock: the irqsafe spinlock to use for locking this fence | |
918 | * @context: the execution context this fence is run on | |
919 | * @seqno: a linear increasing sequence number for this context | |
e941759c ML |
920 | * |
921 | * Initializes an allocated fence, the caller doesn't have to keep its | |
922 | * refcount after committing with this fence, but it will need to hold a | |
4dd3cdb2 | 923 | * refcount again if &dma_fence_ops.enable_signaling gets called. |
e941759c ML |
924 | * |
925 | * context and seqno are used for easy comparison between fences, allowing | |
4dd3cdb2 | 926 | * to check which fence is later by simply using dma_fence_later(). |
e941759c ML |
927 | */ |
928 | void | |
f54d1867 | 929 | dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, |
b312d8ca | 930 | spinlock_t *lock, u64 context, u64 seqno) |
e941759c ML |
931 | { |
932 | BUG_ON(!lock); | |
418cc6ca | 933 | BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name); |
e941759c ML |
934 | |
935 | kref_init(&fence->refcount); | |
936 | fence->ops = ops; | |
937 | INIT_LIST_HEAD(&fence->cb_list); | |
938 | fence->lock = lock; | |
939 | fence->context = context; | |
940 | fence->seqno = seqno; | |
941 | fence->flags = 0UL; | |
a009e975 | 942 | fence->error = 0; |
e941759c | 943 | |
f54d1867 | 944 | trace_dma_fence_init(fence); |
e941759c | 945 | } |
f54d1867 | 946 | EXPORT_SYMBOL(dma_fence_init); |