Commit | Line | Data |
---|---|---|
cafe5635 KO |
1 | #ifndef _LINUX_CLOSURE_H |
2 | #define _LINUX_CLOSURE_H | |
3 | ||
4 | #include <linux/llist.h> | |
5 | #include <linux/sched.h> | |
68db0cf1 | 6 | #include <linux/sched/task_stack.h> |
cafe5635 KO |
7 | #include <linux/workqueue.h> |
8 | ||
9 | /* | |
10 | * Closure is perhaps the most overused and abused term in computer science, but | |
11 | * since I've been unable to come up with anything better you're stuck with it | |
12 | * again. | |
13 | * | |
14 | * What are closures? | |
15 | * | |
16 | * They embed a refcount. The basic idea is they count "things that are in | |
17 | * progress" - in flight bios, some other thread that's doing something else - | |
18 | * anything you might want to wait on. | |
19 | * | |
20 | * The refcount may be manipulated with closure_get() and closure_put(). | |
21 | * closure_put() is where many of the interesting things happen, when it causes | |
22 | * the refcount to go to 0. | |
23 | * | |
24 | * Closures can be used to wait on things both synchronously and asynchronously, | |
25 | * and synchronous and asynchronous use can be mixed without restriction. To | |
26 | * wait synchronously, use closure_sync() - you will sleep until your closure's | |
27 | * refcount hits 1. | |
28 | * | |
29 | * To wait asynchronously, use | |
30 | * continue_at(cl, next_function, workqueue); | |
31 | * | |
32 | * passing it, as you might expect, the function to run when nothing is pending | |
33 | * and the workqueue to run that function out of. | |
34 | * | |
7abc70d7 YW |
35 | * continue_at() also, critically, requires a 'return' immediately following the |
36 | * location where this macro is referenced, to return to the calling function. | |
cafe5635 KO |
37 | * There's good reason for this. |
38 | * | |
39 | * To use safely closures asynchronously, they must always have a refcount while | |
40 | * they are running owned by the thread that is running them. Otherwise, suppose | |
41 | * you submit some bios and wish to have a function run when they all complete: | |
42 | * | |
4246a0b6 | 43 | * foo_endio(struct bio *bio) |
cafe5635 KO |
44 | * { |
45 | * closure_put(cl); | |
46 | * } | |
47 | * | |
48 | * closure_init(cl); | |
49 | * | |
50 | * do_stuff(); | |
51 | * closure_get(cl); | |
52 | * bio1->bi_endio = foo_endio; | |
53 | * bio_submit(bio1); | |
54 | * | |
55 | * do_more_stuff(); | |
56 | * closure_get(cl); | |
57 | * bio2->bi_endio = foo_endio; | |
58 | * bio_submit(bio2); | |
59 | * | |
60 | * continue_at(cl, complete_some_read, system_wq); | |
61 | * | |
62 | * If closure's refcount started at 0, complete_some_read() could run before the | |
63 | * second bio was submitted - which is almost always not what you want! More | |
64 | * importantly, it wouldn't be possible to say whether the original thread or | |
65 | * complete_some_read()'s thread owned the closure - and whatever state it was | |
66 | * associated with! | |
67 | * | |
68 | * So, closure_init() initializes a closure's refcount to 1 - and when a | |
69 | * closure_fn is run, the refcount will be reset to 1 first. | |
70 | * | |
71 | * Then, the rule is - if you got the refcount with closure_get(), release it | |
72 | * with closure_put() (i.e, in a bio->bi_endio function). If you have a refcount | |
73 | * on a closure because you called closure_init() or you were run out of a | |
74 | * closure - _always_ use continue_at(). Doing so consistently will help | |
75 | * eliminate an entire class of particularly pernicious races. | |
76 | * | |
cafe5635 KO |
77 | * Lastly, you might have a wait list dedicated to a specific event, and have no |
78 | * need for specifying the condition - you just want to wait until someone runs | |
79 | * closure_wake_up() on the appropriate wait list. In that case, just use | |
80 | * closure_wait(). It will return either true or false, depending on whether the | |
81 | * closure was already on a wait list or not - a closure can only be on one wait | |
82 | * list at a time. | |
83 | * | |
84 | * Parents: | |
85 | * | |
86 | * closure_init() takes two arguments - it takes the closure to initialize, and | |
87 | * a (possibly null) parent. | |
88 | * | |
89 | * If parent is non null, the new closure will have a refcount for its lifetime; | |
90 | * a closure is considered to be "finished" when its refcount hits 0 and the | |
91 | * function to run is null. Hence | |
92 | * | |
93 | * continue_at(cl, NULL, NULL); | |
94 | * | |
95 | * returns up the (spaghetti) stack of closures, precisely like normal return | |
96 | * returns up the C stack. continue_at() with non null fn is better thought of | |
97 | * as doing a tail call. | |
98 | * | |
99 | * All this implies that a closure should typically be embedded in a particular | |
100 | * struct (which its refcount will normally control the lifetime of), and that | |
101 | * struct can very much be thought of as a stack frame. | |
cafe5635 KO |
102 | */ |
103 | ||
104 | struct closure; | |
105 | typedef void (closure_fn) (struct closure *); | |
106 | ||
107 | struct closure_waitlist { | |
108 | struct llist_head list; | |
109 | }; | |
110 | ||
cafe5635 KO |
111 | enum closure_state { |
112 | /* | |
cafe5635 KO |
113 | * CLOSURE_WAITING: Set iff the closure is on a waitlist. Must be set by |
114 | * the thread that owns the closure, and cleared by the thread that's | |
115 | * waking up the closure. | |
116 | * | |
117 | * CLOSURE_SLEEPING: Must be set before a thread uses a closure to sleep | |
118 | * - indicates that cl->task is valid and closure_put() may wake it up. | |
119 | * Only set or cleared by the thread that owns the closure. | |
120 | * | |
cafe5635 KO |
121 | * The rest are for debugging and don't affect behaviour: |
122 | * | |
123 | * CLOSURE_RUNNING: Set when a closure is running (i.e. by | |
124 | * closure_init() and when closure_put() runs then next function), and | |
125 | * must be cleared before remaining hits 0. Primarily to help guard | |
126 | * against incorrect usage and accidentally transferring references. | |
127 | * continue_at() and closure_return() clear it for you, if you're doing | |
128 | * something unusual you can use closure_set_dead() which also helps | |
129 | * annotate where references are being transferred. | |
130 | * | |
131 | * CLOSURE_STACK: Sanity check - remaining should never hit 0 on a | |
132 | * closure with this flag set | |
133 | */ | |
134 | ||
faadf0c9 KO |
135 | CLOSURE_BITS_START = (1 << 23), |
136 | CLOSURE_DESTRUCTOR = (1 << 23), | |
137 | CLOSURE_WAITING = (1 << 25), | |
138 | CLOSURE_SLEEPING = (1 << 27), | |
cafe5635 KO |
139 | CLOSURE_RUNNING = (1 << 29), |
140 | CLOSURE_STACK = (1 << 31), | |
141 | }; | |
142 | ||
143 | #define CLOSURE_GUARD_MASK \ | |
faadf0c9 KO |
144 | ((CLOSURE_DESTRUCTOR|CLOSURE_WAITING|CLOSURE_SLEEPING| \ |
145 | CLOSURE_RUNNING|CLOSURE_STACK) << 1) | |
cafe5635 KO |
146 | |
147 | #define CLOSURE_REMAINING_MASK (CLOSURE_BITS_START - 1) | |
148 | #define CLOSURE_REMAINING_INITIALIZER (1|CLOSURE_RUNNING) | |
149 | ||
150 | struct closure { | |
151 | union { | |
152 | struct { | |
153 | struct workqueue_struct *wq; | |
154 | struct task_struct *task; | |
155 | struct llist_node list; | |
156 | closure_fn *fn; | |
157 | }; | |
158 | struct work_struct work; | |
159 | }; | |
160 | ||
161 | struct closure *parent; | |
162 | ||
163 | atomic_t remaining; | |
164 | ||
cafe5635 KO |
165 | #ifdef CONFIG_BCACHE_CLOSURES_DEBUG |
166 | #define CLOSURE_MAGIC_DEAD 0xc054dead | |
167 | #define CLOSURE_MAGIC_ALIVE 0xc054a11e | |
168 | ||
169 | unsigned magic; | |
170 | struct list_head all; | |
171 | unsigned long ip; | |
172 | unsigned long waiting_on; | |
173 | #endif | |
174 | }; | |
175 | ||
cafe5635 KO |
176 | void closure_sub(struct closure *cl, int v); |
177 | void closure_put(struct closure *cl); | |
cafe5635 KO |
178 | void __closure_wake_up(struct closure_waitlist *list); |
179 | bool closure_wait(struct closure_waitlist *list, struct closure *cl); | |
180 | void closure_sync(struct closure *cl); | |
181 | ||
cafe5635 KO |
182 | #ifdef CONFIG_BCACHE_CLOSURES_DEBUG |
183 | ||
07e86ccb | 184 | void closure_debug_init(void); |
cafe5635 KO |
185 | void closure_debug_create(struct closure *cl); |
186 | void closure_debug_destroy(struct closure *cl); | |
187 | ||
188 | #else | |
189 | ||
07e86ccb | 190 | static inline void closure_debug_init(void) {} |
cafe5635 KO |
191 | static inline void closure_debug_create(struct closure *cl) {} |
192 | static inline void closure_debug_destroy(struct closure *cl) {} | |
193 | ||
194 | #endif | |
195 | ||
196 | static inline void closure_set_ip(struct closure *cl) | |
197 | { | |
198 | #ifdef CONFIG_BCACHE_CLOSURES_DEBUG | |
199 | cl->ip = _THIS_IP_; | |
200 | #endif | |
201 | } | |
202 | ||
203 | static inline void closure_set_ret_ip(struct closure *cl) | |
204 | { | |
205 | #ifdef CONFIG_BCACHE_CLOSURES_DEBUG | |
206 | cl->ip = _RET_IP_; | |
207 | #endif | |
208 | } | |
209 | ||
1dd13c8d | 210 | static inline void closure_set_waiting(struct closure *cl, unsigned long f) |
cafe5635 KO |
211 | { |
212 | #ifdef CONFIG_BCACHE_CLOSURES_DEBUG | |
1dd13c8d | 213 | cl->waiting_on = f; |
cafe5635 KO |
214 | #endif |
215 | } | |
216 | ||
1dd13c8d KO |
217 | static inline void __closure_end_sleep(struct closure *cl) |
218 | { | |
219 | __set_current_state(TASK_RUNNING); | |
220 | ||
221 | if (atomic_read(&cl->remaining) & CLOSURE_SLEEPING) | |
222 | atomic_sub(CLOSURE_SLEEPING, &cl->remaining); | |
223 | } | |
224 | ||
225 | static inline void __closure_start_sleep(struct closure *cl) | |
226 | { | |
227 | closure_set_ip(cl); | |
228 | cl->task = current; | |
229 | set_current_state(TASK_UNINTERRUPTIBLE); | |
230 | ||
231 | if (!(atomic_read(&cl->remaining) & CLOSURE_SLEEPING)) | |
232 | atomic_add(CLOSURE_SLEEPING, &cl->remaining); | |
233 | } | |
234 | ||
cafe5635 KO |
235 | static inline void closure_set_stopped(struct closure *cl) |
236 | { | |
237 | atomic_sub(CLOSURE_RUNNING, &cl->remaining); | |
238 | } | |
239 | ||
1dd13c8d KO |
240 | static inline void set_closure_fn(struct closure *cl, closure_fn *fn, |
241 | struct workqueue_struct *wq) | |
cafe5635 | 242 | { |
1dd13c8d KO |
243 | BUG_ON(object_is_on_stack(cl)); |
244 | closure_set_ip(cl); | |
245 | cl->fn = fn; | |
246 | cl->wq = wq; | |
247 | /* between atomic_dec() in closure_put() */ | |
4e857c58 | 248 | smp_mb__before_atomic(); |
cafe5635 KO |
249 | } |
250 | ||
1dd13c8d | 251 | static inline void closure_queue(struct closure *cl) |
cafe5635 | 252 | { |
1dd13c8d KO |
253 | struct workqueue_struct *wq = cl->wq; |
254 | if (wq) { | |
255 | INIT_WORK(&cl->work, cl->work.func); | |
256 | BUG_ON(!queue_work(wq, &cl->work)); | |
cafe5635 | 257 | } else |
1dd13c8d | 258 | cl->fn(cl); |
cafe5635 KO |
259 | } |
260 | ||
1dd13c8d KO |
261 | /** |
262 | * closure_get - increment a closure's refcount | |
cafe5635 | 263 | */ |
1dd13c8d KO |
264 | static inline void closure_get(struct closure *cl) |
265 | { | |
266 | #ifdef CONFIG_BCACHE_CLOSURES_DEBUG | |
267 | BUG_ON((atomic_inc_return(&cl->remaining) & | |
268 | CLOSURE_REMAINING_MASK) <= 1); | |
269 | #else | |
270 | atomic_inc(&cl->remaining); | |
271 | #endif | |
272 | } | |
cafe5635 | 273 | |
cafe5635 | 274 | /** |
1dd13c8d | 275 | * closure_init - Initialize a closure, setting the refcount to 1 |
cafe5635 KO |
276 | * @cl: closure to initialize |
277 | * @parent: parent of the new closure. cl will take a refcount on it for its | |
278 | * lifetime; may be NULL. | |
279 | */ | |
1dd13c8d | 280 | static inline void closure_init(struct closure *cl, struct closure *parent) |
cafe5635 KO |
281 | { |
282 | memset(cl, 0, sizeof(struct closure)); | |
1dd13c8d KO |
283 | cl->parent = parent; |
284 | if (parent) | |
285 | closure_get(parent); | |
cafe5635 | 286 | |
1dd13c8d | 287 | atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER); |
cafe5635 | 288 | |
1dd13c8d KO |
289 | closure_debug_create(cl); |
290 | closure_set_ip(cl); | |
cafe5635 KO |
291 | } |
292 | ||
1dd13c8d | 293 | static inline void closure_init_stack(struct closure *cl) |
cafe5635 | 294 | { |
1dd13c8d KO |
295 | memset(cl, 0, sizeof(struct closure)); |
296 | atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER|CLOSURE_STACK); | |
cafe5635 KO |
297 | } |
298 | ||
cafe5635 | 299 | /** |
1dd13c8d | 300 | * closure_wake_up - wake up all closures on a wait list. |
cafe5635 KO |
301 | */ |
302 | static inline void closure_wake_up(struct closure_waitlist *list) | |
303 | { | |
304 | smp_mb(); | |
305 | __closure_wake_up(list); | |
306 | } | |
307 | ||
1dd13c8d KO |
308 | /** |
309 | * continue_at - jump to another function with barrier | |
310 | * | |
311 | * After @cl is no longer waiting on anything (i.e. all outstanding refs have | |
312 | * been dropped with closure_put()), it will resume execution at @fn running out | |
313 | * of @wq (or, if @wq is NULL, @fn will be called by closure_put() directly). | |
314 | * | |
315 | * NOTE: This macro expands to a return in the calling function! | |
316 | * | |
317 | * This is because after calling continue_at() you no longer have a ref on @cl, | |
318 | * and whatever @cl owns may be freed out from under you - a running closure fn | |
319 | * has a ref on its own closure which continue_at() drops. | |
cafe5635 | 320 | */ |
cafe5635 KO |
321 | #define continue_at(_cl, _fn, _wq) \ |
322 | do { \ | |
323 | set_closure_fn(_cl, _fn, _wq); \ | |
324 | closure_sub(_cl, CLOSURE_RUNNING + 1); \ | |
cafe5635 KO |
325 | } while (0) |
326 | ||
1dd13c8d KO |
327 | /** |
328 | * closure_return - finish execution of a closure | |
329 | * | |
330 | * This is used to indicate that @cl is finished: when all outstanding refs on | |
331 | * @cl have been dropped @cl's ref on its parent closure (as passed to | |
332 | * closure_init()) will be dropped, if one was specified - thus this can be | |
333 | * thought of as returning to the parent closure. | |
334 | */ | |
cafe5635 KO |
335 | #define closure_return(_cl) continue_at((_cl), NULL, NULL) |
336 | ||
1dd13c8d KO |
337 | /** |
338 | * continue_at_nobarrier - jump to another function without barrier | |
339 | * | |
340 | * Causes @fn to be executed out of @cl, in @wq context (or called directly if | |
341 | * @wq is NULL). | |
342 | * | |
343 | * NOTE: like continue_at(), this macro expands to a return in the caller! | |
344 | * | |
345 | * The ref the caller of continue_at_nobarrier() had on @cl is now owned by @fn, | |
346 | * thus it's not safe to touch anything protected by @cl after a | |
347 | * continue_at_nobarrier(). | |
348 | */ | |
cafe5635 KO |
349 | #define continue_at_nobarrier(_cl, _fn, _wq) \ |
350 | do { \ | |
351 | set_closure_fn(_cl, _fn, _wq); \ | |
a34a8bfd | 352 | closure_queue(_cl); \ |
cafe5635 KO |
353 | } while (0) |
354 | ||
1dd13c8d KO |
355 | /** |
356 | * closure_return - finish execution of a closure, with destructor | |
357 | * | |
358 | * Works like closure_return(), except @destructor will be called when all | |
359 | * outstanding refs on @cl have been dropped; @destructor may be used to safely | |
360 | * free the memory occupied by @cl, and it is called with the ref on the parent | |
361 | * closure still held - so @destructor could safely return an item to a | |
362 | * freelist protected by @cl's parent. | |
363 | */ | |
cafe5635 KO |
364 | #define closure_return_with_destructor(_cl, _destructor) \ |
365 | do { \ | |
366 | set_closure_fn(_cl, _destructor, NULL); \ | |
367 | closure_sub(_cl, CLOSURE_RUNNING - CLOSURE_DESTRUCTOR + 1); \ | |
cafe5635 KO |
368 | } while (0) |
369 | ||
1dd13c8d KO |
370 | /** |
371 | * closure_call - execute @fn out of a new, uninitialized closure | |
372 | * | |
373 | * Typically used when running out of one closure, and we want to run @fn | |
374 | * asynchronously out of a new closure - @parent will then wait for @cl to | |
375 | * finish. | |
376 | */ | |
cafe5635 KO |
377 | static inline void closure_call(struct closure *cl, closure_fn fn, |
378 | struct workqueue_struct *wq, | |
379 | struct closure *parent) | |
380 | { | |
381 | closure_init(cl, parent); | |
382 | continue_at_nobarrier(cl, fn, wq); | |
383 | } | |
384 | ||
cafe5635 | 385 | #endif /* _LINUX_CLOSURE_H */ |