Commit | Line | Data |
---|---|---|
87418ef9 | 1 | // SPDX-License-Identifier: GPL-2.0 |
cafe5635 KO |
2 | /* |
3 | * Asynchronous refcounty things | |
4 | * | |
5 | * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> | |
6 | * Copyright 2012 Google, Inc. | |
7 | */ | |
8 | ||
8c8d2d96 | 9 | #include <linux/closure.h> |
cafe5635 | 10 | #include <linux/debugfs.h> |
8c8d2d96 | 11 | #include <linux/export.h> |
b414e8ec | 12 | #include <linux/rcupdate.h> |
cafe5635 | 13 | #include <linux/seq_file.h> |
ce439bf7 | 14 | #include <linux/sched/debug.h> |
cafe5635 | 15 | |
cafe5635 KO |
16 | static inline void closure_put_after_sub(struct closure *cl, int flags) |
17 | { | |
18 | int r = flags & CLOSURE_REMAINING_MASK; | |
19 | ||
20 | BUG_ON(flags & CLOSURE_GUARD_MASK); | |
faadf0c9 | 21 | BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR)); |
cafe5635 | 22 | |
cafe5635 | 23 | if (!r) { |
2bce6368 KO |
24 | smp_acquire__after_ctrl_dep(); |
25 | ||
ee526b88 KO |
26 | cl->closure_get_happened = false; |
27 | ||
cafe5635 | 28 | if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) { |
cafe5635 KO |
29 | atomic_set(&cl->remaining, |
30 | CLOSURE_REMAINING_INITIALIZER); | |
31 | closure_queue(cl); | |
32 | } else { | |
33 | struct closure *parent = cl->parent; | |
6aa8f1a6 | 34 | closure_fn *destructor = cl->fn; |
cafe5635 KO |
35 | |
36 | closure_debug_destroy(cl); | |
37 | ||
6aa8f1a6 | 38 | if (destructor) |
d4e3b928 | 39 | destructor(&cl->work); |
cafe5635 KO |
40 | |
41 | if (parent) | |
42 | closure_put(parent); | |
43 | } | |
44 | } | |
45 | } | |
46 | ||
47 | /* For clearing flags with the same atomic op as a put */ | |
48 | void closure_sub(struct closure *cl, int v) | |
49 | { | |
2bce6368 | 50 | closure_put_after_sub(cl, atomic_sub_return_release(v, &cl->remaining)); |
cafe5635 | 51 | } |
8c8d2d96 | 52 | EXPORT_SYMBOL(closure_sub); |
cafe5635 | 53 | |
47344e33 | 54 | /* |
1dd13c8d KO |
55 | * closure_put - decrement a closure's refcount |
56 | */ | |
cafe5635 KO |
57 | void closure_put(struct closure *cl) |
58 | { | |
2bce6368 | 59 | closure_put_after_sub(cl, atomic_dec_return_release(&cl->remaining)); |
cafe5635 | 60 | } |
8c8d2d96 | 61 | EXPORT_SYMBOL(closure_put); |
cafe5635 | 62 | |
47344e33 | 63 | /* |
1dd13c8d KO |
64 | * closure_wake_up - wake up all closures on a wait list, without memory barrier |
65 | */ | |
cafe5635 KO |
66 | void __closure_wake_up(struct closure_waitlist *wait_list) |
67 | { | |
68 | struct llist_node *list; | |
a5f3d8a5 | 69 | struct closure *cl, *t; |
cafe5635 KO |
70 | struct llist_node *reverse = NULL; |
71 | ||
72 | list = llist_del_all(&wait_list->list); | |
73 | ||
74 | /* We first reverse the list to preserve FIFO ordering and fairness */ | |
09b3efec | 75 | reverse = llist_reverse_order(list); |
cafe5635 KO |
76 | |
77 | /* Then do the wakeups */ | |
a5f3d8a5 | 78 | llist_for_each_entry_safe(cl, t, reverse, list) { |
1dd13c8d | 79 | closure_set_waiting(cl, 0); |
cafe5635 KO |
80 | closure_sub(cl, CLOSURE_WAITING + 1); |
81 | } | |
82 | } | |
8c8d2d96 | 83 | EXPORT_SYMBOL(__closure_wake_up); |
cafe5635 | 84 | |
1dd13c8d KO |
85 | /** |
86 | * closure_wait - add a closure to a waitlist | |
47344e33 | 87 | * @waitlist: will own a ref on @cl, which will be released when |
1dd13c8d | 88 | * closure_wake_up() is called on @waitlist. |
47344e33 | 89 | * @cl: closure pointer. |
1dd13c8d KO |
90 | * |
91 | */ | |
92 | bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl) | |
cafe5635 KO |
93 | { |
94 | if (atomic_read(&cl->remaining) & CLOSURE_WAITING) | |
95 | return false; | |
96 | ||
ee526b88 | 97 | cl->closure_get_happened = true; |
1dd13c8d | 98 | closure_set_waiting(cl, _RET_IP_); |
cafe5635 | 99 | atomic_add(CLOSURE_WAITING + 1, &cl->remaining); |
1dd13c8d | 100 | llist_add(&cl->list, &waitlist->list); |
cafe5635 KO |
101 | |
102 | return true; | |
103 | } | |
8c8d2d96 | 104 | EXPORT_SYMBOL(closure_wait); |
cafe5635 | 105 | |
e4bf7919 KO |
106 | struct closure_syncer { |
107 | struct task_struct *task; | |
108 | int done; | |
109 | }; | |
110 | ||
d4e3b928 | 111 | static CLOSURE_CALLBACK(closure_sync_fn) |
cafe5635 | 112 | { |
d4e3b928 | 113 | struct closure *cl = container_of(ws, struct closure, work); |
a22a9602 KO |
114 | struct closure_syncer *s = cl->s; |
115 | struct task_struct *p; | |
116 | ||
117 | rcu_read_lock(); | |
118 | p = READ_ONCE(s->task); | |
119 | s->done = 1; | |
120 | wake_up_process(p); | |
121 | rcu_read_unlock(); | |
e4bf7919 | 122 | } |
cafe5635 | 123 | |
ce439bf7 | 124 | void __sched __closure_sync(struct closure *cl) |
e4bf7919 KO |
125 | { |
126 | struct closure_syncer s = { .task = current }; | |
cafe5635 | 127 | |
e4bf7919 KO |
128 | cl->s = &s; |
129 | continue_at(cl, closure_sync_fn, NULL); | |
130 | ||
131 | while (1) { | |
132 | set_current_state(TASK_UNINTERRUPTIBLE); | |
133 | if (s.done) | |
134 | break; | |
cafe5635 KO |
135 | schedule(); |
136 | } | |
137 | ||
e4bf7919 | 138 | __set_current_state(TASK_RUNNING); |
cafe5635 | 139 | } |
8c8d2d96 | 140 | EXPORT_SYMBOL(__closure_sync); |
cafe5635 | 141 | |
8c8d2d96 | 142 | #ifdef CONFIG_DEBUG_CLOSURES |
cafe5635 KO |
143 | |
144 | static LIST_HEAD(closure_list); | |
145 | static DEFINE_SPINLOCK(closure_list_lock); | |
146 | ||
147 | void closure_debug_create(struct closure *cl) | |
148 | { | |
149 | unsigned long flags; | |
150 | ||
151 | BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE); | |
152 | cl->magic = CLOSURE_MAGIC_ALIVE; | |
153 | ||
154 | spin_lock_irqsave(&closure_list_lock, flags); | |
155 | list_add(&cl->all, &closure_list); | |
156 | spin_unlock_irqrestore(&closure_list_lock, flags); | |
157 | } | |
8c8d2d96 | 158 | EXPORT_SYMBOL(closure_debug_create); |
cafe5635 KO |
159 | |
160 | void closure_debug_destroy(struct closure *cl) | |
161 | { | |
162 | unsigned long flags; | |
163 | ||
164 | BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE); | |
165 | cl->magic = CLOSURE_MAGIC_DEAD; | |
166 | ||
167 | spin_lock_irqsave(&closure_list_lock, flags); | |
168 | list_del(&cl->all); | |
169 | spin_unlock_irqrestore(&closure_list_lock, flags); | |
170 | } | |
8c8d2d96 | 171 | EXPORT_SYMBOL(closure_debug_destroy); |
cafe5635 | 172 | |
84e5d136 | 173 | static int debug_show(struct seq_file *f, void *data) |
cafe5635 KO |
174 | { |
175 | struct closure *cl; | |
1fae7cf0 | 176 | |
cafe5635 KO |
177 | spin_lock_irq(&closure_list_lock); |
178 | ||
179 | list_for_each_entry(cl, &closure_list, all) { | |
180 | int r = atomic_read(&cl->remaining); | |
181 | ||
d9c61d30 | 182 | seq_printf(f, "%p: %pS -> %pS p %p r %i ", |
cafe5635 KO |
183 | cl, (void *) cl->ip, cl->fn, cl->parent, |
184 | r & CLOSURE_REMAINING_MASK); | |
185 | ||
e4bf7919 | 186 | seq_printf(f, "%s%s\n", |
8d090f47 | 187 | test_bit(WORK_STRUCT_PENDING_BIT, |
cafe5635 | 188 | work_data_bits(&cl->work)) ? "Q" : "", |
e4bf7919 | 189 | r & CLOSURE_RUNNING ? "R" : ""); |
cafe5635 KO |
190 | |
191 | if (r & CLOSURE_WAITING) | |
d9c61d30 | 192 | seq_printf(f, " W %pS\n", |
cafe5635 KO |
193 | (void *) cl->waiting_on); |
194 | ||
8c8d2d96 | 195 | seq_puts(f, "\n"); |
cafe5635 KO |
196 | } |
197 | ||
198 | spin_unlock_irq(&closure_list_lock); | |
199 | return 0; | |
200 | } | |
201 | ||
84e5d136 | 202 | DEFINE_SHOW_ATTRIBUTE(debug); |
cafe5635 | 203 | |
8c8d2d96 | 204 | static int __init closure_debug_init(void) |
cafe5635 | 205 | { |
8c8d2d96 KO |
206 | debugfs_create_file("closures", 0400, NULL, NULL, &debug_fops); |
207 | return 0; | |
cafe5635 | 208 | } |
8c8d2d96 | 209 | late_initcall(closure_debug_init) |
cafe5635 | 210 | |
8c8d2d96 | 211 | #endif |