Commit | Line | Data |
---|---|---|
1802d0be | 1 | // SPDX-License-Identifier: GPL-2.0-only |
7bf60c52 CK |
2 | /* |
3 | * fence-chain: chain fences together in a timeline | |
4 | * | |
5 | * Copyright (C) 2018 Advanced Micro Devices, Inc. | |
6 | * Authors: | |
7 | * Christian König <christian.koenig@amd.com> | |
7bf60c52 CK |
8 | */ |
9 | ||
10 | #include <linux/dma-fence-chain.h> | |
11 | ||
12 | static bool dma_fence_chain_enable_signaling(struct dma_fence *fence); | |
13 | ||
14 | /** | |
15 | * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence | |
16 | * @chain: chain node to get the previous node from | |
17 | * | |
18 | * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the | |
19 | * chain node. | |
20 | */ | |
21 | static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain) | |
22 | { | |
23 | struct dma_fence *prev; | |
24 | ||
25 | rcu_read_lock(); | |
26 | prev = dma_fence_get_rcu_safe(&chain->prev); | |
27 | rcu_read_unlock(); | |
28 | return prev; | |
29 | } | |
30 | ||
31 | /** | |
32 | * dma_fence_chain_walk - chain walking function | |
33 | * @fence: current chain node | |
34 | * | |
35 | * Walk the chain to the next node. Returns the next fence or NULL if we are at | |
36 | * the end of the chain. Garbage collects chain nodes which are already | |
37 | * signaled. | |
38 | */ | |
39 | struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence) | |
40 | { | |
41 | struct dma_fence_chain *chain, *prev_chain; | |
42 | struct dma_fence *prev, *replacement, *tmp; | |
43 | ||
44 | chain = to_dma_fence_chain(fence); | |
45 | if (!chain) { | |
46 | dma_fence_put(fence); | |
47 | return NULL; | |
48 | } | |
49 | ||
50 | while ((prev = dma_fence_chain_get_prev(chain))) { | |
51 | ||
52 | prev_chain = to_dma_fence_chain(prev); | |
53 | if (prev_chain) { | |
54 | if (!dma_fence_is_signaled(prev_chain->fence)) | |
55 | break; | |
56 | ||
57 | replacement = dma_fence_chain_get_prev(prev_chain); | |
58 | } else { | |
59 | if (!dma_fence_is_signaled(prev)) | |
60 | break; | |
61 | ||
62 | replacement = NULL; | |
63 | } | |
64 | ||
29da47cf CW |
65 | tmp = cmpxchg((struct dma_fence __force **)&chain->prev, |
66 | prev, replacement); | |
7bf60c52 CK |
67 | if (tmp == prev) |
68 | dma_fence_put(tmp); | |
69 | else | |
70 | dma_fence_put(replacement); | |
71 | dma_fence_put(prev); | |
72 | } | |
73 | ||
74 | dma_fence_put(fence); | |
75 | return prev; | |
76 | } | |
77 | EXPORT_SYMBOL(dma_fence_chain_walk); | |
78 | ||
79 | /** | |
80 | * dma_fence_chain_find_seqno - find fence chain node by seqno | |
81 | * @pfence: pointer to the chain node where to start | |
82 | * @seqno: the sequence number to search for | |
83 | * | |
84 | * Advance the fence pointer to the chain node which will signal this sequence | |
85 | * number. If no sequence number is provided then this is a no-op. | |
86 | * | |
87 | * Returns EINVAL if the fence is not a chain node or the sequence number has | |
88 | * not yet advanced far enough. | |
89 | */ | |
90 | int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno) | |
91 | { | |
92 | struct dma_fence_chain *chain; | |
93 | ||
94 | if (!seqno) | |
95 | return 0; | |
96 | ||
97 | chain = to_dma_fence_chain(*pfence); | |
98 | if (!chain || chain->base.seqno < seqno) | |
99 | return -EINVAL; | |
100 | ||
101 | dma_fence_chain_for_each(*pfence, &chain->base) { | |
102 | if ((*pfence)->context != chain->base.context || | |
103 | to_dma_fence_chain(*pfence)->prev_seqno < seqno) | |
104 | break; | |
105 | } | |
106 | dma_fence_put(&chain->base); | |
107 | ||
108 | return 0; | |
109 | } | |
110 | EXPORT_SYMBOL(dma_fence_chain_find_seqno); | |
111 | ||
112 | static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence) | |
113 | { | |
114 | return "dma_fence_chain"; | |
115 | } | |
116 | ||
117 | static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence) | |
118 | { | |
119 | return "unbound"; | |
120 | } | |
121 | ||
122 | static void dma_fence_chain_irq_work(struct irq_work *work) | |
123 | { | |
124 | struct dma_fence_chain *chain; | |
125 | ||
126 | chain = container_of(work, typeof(*chain), work); | |
127 | ||
128 | /* Try to rearm the callback */ | |
129 | if (!dma_fence_chain_enable_signaling(&chain->base)) | |
130 | /* Ok, we are done. No more unsignaled fences left */ | |
131 | dma_fence_signal(&chain->base); | |
132 | dma_fence_put(&chain->base); | |
133 | } | |
134 | ||
135 | static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb) | |
136 | { | |
137 | struct dma_fence_chain *chain; | |
138 | ||
139 | chain = container_of(cb, typeof(*chain), cb); | |
140 | irq_work_queue(&chain->work); | |
141 | dma_fence_put(f); | |
142 | } | |
143 | ||
144 | static bool dma_fence_chain_enable_signaling(struct dma_fence *fence) | |
145 | { | |
146 | struct dma_fence_chain *head = to_dma_fence_chain(fence); | |
147 | ||
148 | dma_fence_get(&head->base); | |
149 | dma_fence_chain_for_each(fence, &head->base) { | |
150 | struct dma_fence_chain *chain = to_dma_fence_chain(fence); | |
151 | struct dma_fence *f = chain ? chain->fence : fence; | |
152 | ||
153 | dma_fence_get(f); | |
154 | if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) { | |
155 | dma_fence_put(fence); | |
156 | return true; | |
157 | } | |
158 | dma_fence_put(f); | |
159 | } | |
160 | dma_fence_put(&head->base); | |
161 | return false; | |
162 | } | |
163 | ||
164 | static bool dma_fence_chain_signaled(struct dma_fence *fence) | |
165 | { | |
166 | dma_fence_chain_for_each(fence, fence) { | |
167 | struct dma_fence_chain *chain = to_dma_fence_chain(fence); | |
168 | struct dma_fence *f = chain ? chain->fence : fence; | |
169 | ||
170 | if (!dma_fence_is_signaled(f)) { | |
171 | dma_fence_put(fence); | |
172 | return false; | |
173 | } | |
174 | } | |
175 | ||
176 | return true; | |
177 | } | |
178 | ||
179 | static void dma_fence_chain_release(struct dma_fence *fence) | |
180 | { | |
181 | struct dma_fence_chain *chain = to_dma_fence_chain(fence); | |
92cb3e59 CK |
182 | struct dma_fence *prev; |
183 | ||
184 | /* Manually unlink the chain as much as possible to avoid recursion | |
185 | * and potential stack overflow. | |
186 | */ | |
187 | while ((prev = rcu_dereference_protected(chain->prev, true))) { | |
188 | struct dma_fence_chain *prev_chain; | |
189 | ||
190 | if (kref_read(&prev->refcount) > 1) | |
191 | break; | |
192 | ||
193 | prev_chain = to_dma_fence_chain(prev); | |
194 | if (!prev_chain) | |
195 | break; | |
196 | ||
197 | /* No need for atomic operations since we hold the last | |
198 | * reference to prev_chain. | |
199 | */ | |
200 | chain->prev = prev_chain->prev; | |
201 | RCU_INIT_POINTER(prev_chain->prev, NULL); | |
202 | dma_fence_put(prev); | |
203 | } | |
204 | dma_fence_put(prev); | |
7bf60c52 | 205 | |
7bf60c52 CK |
206 | dma_fence_put(chain->fence); |
207 | dma_fence_free(fence); | |
208 | } | |
209 | ||
210 | const struct dma_fence_ops dma_fence_chain_ops = { | |
5e498abf | 211 | .use_64bit_seqno = true, |
7bf60c52 CK |
212 | .get_driver_name = dma_fence_chain_get_driver_name, |
213 | .get_timeline_name = dma_fence_chain_get_timeline_name, | |
214 | .enable_signaling = dma_fence_chain_enable_signaling, | |
215 | .signaled = dma_fence_chain_signaled, | |
216 | .release = dma_fence_chain_release, | |
217 | }; | |
218 | EXPORT_SYMBOL(dma_fence_chain_ops); | |
219 | ||
220 | /** | |
221 | * dma_fence_chain_init - initialize a fence chain | |
222 | * @chain: the chain node to initialize | |
223 | * @prev: the previous fence | |
224 | * @fence: the current fence | |
8d0441cf | 225 | * @seqno: the sequence number to use for the fence chain |
7bf60c52 CK |
226 | * |
227 | * Initialize a new chain node and either start a new chain or add the node to | |
228 | * the existing chain of the previous fence. | |
229 | */ | |
230 | void dma_fence_chain_init(struct dma_fence_chain *chain, | |
231 | struct dma_fence *prev, | |
232 | struct dma_fence *fence, | |
233 | uint64_t seqno) | |
234 | { | |
235 | struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev); | |
236 | uint64_t context; | |
237 | ||
238 | spin_lock_init(&chain->lock); | |
239 | rcu_assign_pointer(chain->prev, prev); | |
240 | chain->fence = fence; | |
241 | chain->prev_seqno = 0; | |
242 | init_irq_work(&chain->work, dma_fence_chain_irq_work); | |
243 | ||
244 | /* Try to reuse the context of the previous chain node. */ | |
5e498abf | 245 | if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) { |
7bf60c52 CK |
246 | context = prev->context; |
247 | chain->prev_seqno = prev->seqno; | |
248 | } else { | |
249 | context = dma_fence_context_alloc(1); | |
250 | /* Make sure that we always have a valid sequence number. */ | |
251 | if (prev_chain) | |
252 | seqno = max(prev->seqno, seqno); | |
253 | } | |
254 | ||
255 | dma_fence_init(&chain->base, &dma_fence_chain_ops, | |
256 | &chain->lock, context, seqno); | |
257 | } | |
258 | EXPORT_SYMBOL(dma_fence_chain_init); |