Commit | Line | Data |
---|---|---|
87418ef9 | 1 | // SPDX-License-Identifier: GPL-2.0 |
cafe5635 KO |
2 | /* |
3 | * Asynchronous refcounty things | |
4 | * | |
5 | * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> | |
6 | * Copyright 2012 Google, Inc. | |
7 | */ | |
8 | ||
8c8d2d96 | 9 | #include <linux/closure.h> |
cafe5635 | 10 | #include <linux/debugfs.h> |
8c8d2d96 | 11 | #include <linux/export.h> |
b414e8ec | 12 | #include <linux/rcupdate.h> |
cafe5635 | 13 | #include <linux/seq_file.h> |
ce439bf7 | 14 | #include <linux/sched/debug.h> |
cafe5635 | 15 | |
cafe5635 KO |
16 | static inline void closure_put_after_sub(struct closure *cl, int flags) |
17 | { | |
18 | int r = flags & CLOSURE_REMAINING_MASK; | |
19 | ||
20 | BUG_ON(flags & CLOSURE_GUARD_MASK); | |
faadf0c9 | 21 | BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR)); |
cafe5635 | 22 | |
cafe5635 KO |
23 | if (!r) { |
24 | if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) { | |
cafe5635 KO |
25 | atomic_set(&cl->remaining, |
26 | CLOSURE_REMAINING_INITIALIZER); | |
27 | closure_queue(cl); | |
28 | } else { | |
29 | struct closure *parent = cl->parent; | |
6aa8f1a6 | 30 | closure_fn *destructor = cl->fn; |
cafe5635 KO |
31 | |
32 | closure_debug_destroy(cl); | |
33 | ||
6aa8f1a6 KO |
34 | if (destructor) |
35 | destructor(cl); | |
cafe5635 KO |
36 | |
37 | if (parent) | |
38 | closure_put(parent); | |
39 | } | |
40 | } | |
41 | } | |
42 | ||
43 | /* For clearing flags with the same atomic op as a put */ | |
44 | void closure_sub(struct closure *cl, int v) | |
45 | { | |
46 | closure_put_after_sub(cl, atomic_sub_return(v, &cl->remaining)); | |
47 | } | |
8c8d2d96 | 48 | EXPORT_SYMBOL(closure_sub); |
cafe5635 | 49 | |
47344e33 | 50 | /* |
1dd13c8d KO |
51 | * closure_put - decrement a closure's refcount |
52 | */ | |
cafe5635 KO |
53 | void closure_put(struct closure *cl) |
54 | { | |
55 | closure_put_after_sub(cl, atomic_dec_return(&cl->remaining)); | |
56 | } | |
8c8d2d96 | 57 | EXPORT_SYMBOL(closure_put); |
cafe5635 | 58 | |
47344e33 | 59 | /* |
1dd13c8d KO |
60 | * closure_wake_up - wake up all closures on a wait list, without memory barrier |
61 | */ | |
cafe5635 KO |
62 | void __closure_wake_up(struct closure_waitlist *wait_list) |
63 | { | |
64 | struct llist_node *list; | |
a5f3d8a5 | 65 | struct closure *cl, *t; |
cafe5635 KO |
66 | struct llist_node *reverse = NULL; |
67 | ||
68 | list = llist_del_all(&wait_list->list); | |
69 | ||
70 | /* We first reverse the list to preserve FIFO ordering and fairness */ | |
09b3efec | 71 | reverse = llist_reverse_order(list); |
cafe5635 KO |
72 | |
73 | /* Then do the wakeups */ | |
a5f3d8a5 | 74 | llist_for_each_entry_safe(cl, t, reverse, list) { |
1dd13c8d | 75 | closure_set_waiting(cl, 0); |
cafe5635 KO |
76 | closure_sub(cl, CLOSURE_WAITING + 1); |
77 | } | |
78 | } | |
8c8d2d96 | 79 | EXPORT_SYMBOL(__closure_wake_up); |
cafe5635 | 80 | |
1dd13c8d KO |
81 | /** |
82 | * closure_wait - add a closure to a waitlist | |
47344e33 | 83 | * @waitlist: will own a ref on @cl, which will be released when |
1dd13c8d | 84 | * closure_wake_up() is called on @waitlist. |
47344e33 | 85 | * @cl: closure pointer. |
1dd13c8d KO |
86 | * |
87 | */ | |
88 | bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl) | |
cafe5635 KO |
89 | { |
90 | if (atomic_read(&cl->remaining) & CLOSURE_WAITING) | |
91 | return false; | |
92 | ||
1dd13c8d | 93 | closure_set_waiting(cl, _RET_IP_); |
cafe5635 | 94 | atomic_add(CLOSURE_WAITING + 1, &cl->remaining); |
1dd13c8d | 95 | llist_add(&cl->list, &waitlist->list); |
cafe5635 KO |
96 | |
97 | return true; | |
98 | } | |
8c8d2d96 | 99 | EXPORT_SYMBOL(closure_wait); |
cafe5635 | 100 | |
e4bf7919 KO |
101 | struct closure_syncer { |
102 | struct task_struct *task; | |
103 | int done; | |
104 | }; | |
105 | ||
106 | static void closure_sync_fn(struct closure *cl) | |
cafe5635 | 107 | { |
a22a9602 KO |
108 | struct closure_syncer *s = cl->s; |
109 | struct task_struct *p; | |
110 | ||
111 | rcu_read_lock(); | |
112 | p = READ_ONCE(s->task); | |
113 | s->done = 1; | |
114 | wake_up_process(p); | |
115 | rcu_read_unlock(); | |
e4bf7919 | 116 | } |
cafe5635 | 117 | |
ce439bf7 | 118 | void __sched __closure_sync(struct closure *cl) |
e4bf7919 KO |
119 | { |
120 | struct closure_syncer s = { .task = current }; | |
cafe5635 | 121 | |
e4bf7919 KO |
122 | cl->s = &s; |
123 | continue_at(cl, closure_sync_fn, NULL); | |
124 | ||
125 | while (1) { | |
126 | set_current_state(TASK_UNINTERRUPTIBLE); | |
127 | if (s.done) | |
128 | break; | |
cafe5635 KO |
129 | schedule(); |
130 | } | |
131 | ||
e4bf7919 | 132 | __set_current_state(TASK_RUNNING); |
cafe5635 | 133 | } |
8c8d2d96 | 134 | EXPORT_SYMBOL(__closure_sync); |
cafe5635 | 135 | |
8c8d2d96 | 136 | #ifdef CONFIG_DEBUG_CLOSURES |
cafe5635 KO |
137 | |
138 | static LIST_HEAD(closure_list); | |
139 | static DEFINE_SPINLOCK(closure_list_lock); | |
140 | ||
141 | void closure_debug_create(struct closure *cl) | |
142 | { | |
143 | unsigned long flags; | |
144 | ||
145 | BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE); | |
146 | cl->magic = CLOSURE_MAGIC_ALIVE; | |
147 | ||
148 | spin_lock_irqsave(&closure_list_lock, flags); | |
149 | list_add(&cl->all, &closure_list); | |
150 | spin_unlock_irqrestore(&closure_list_lock, flags); | |
151 | } | |
8c8d2d96 | 152 | EXPORT_SYMBOL(closure_debug_create); |
cafe5635 KO |
153 | |
154 | void closure_debug_destroy(struct closure *cl) | |
155 | { | |
156 | unsigned long flags; | |
157 | ||
158 | BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE); | |
159 | cl->magic = CLOSURE_MAGIC_DEAD; | |
160 | ||
161 | spin_lock_irqsave(&closure_list_lock, flags); | |
162 | list_del(&cl->all); | |
163 | spin_unlock_irqrestore(&closure_list_lock, flags); | |
164 | } | |
8c8d2d96 | 165 | EXPORT_SYMBOL(closure_debug_destroy); |
cafe5635 | 166 | |
84e5d136 | 167 | static int debug_show(struct seq_file *f, void *data) |
cafe5635 KO |
168 | { |
169 | struct closure *cl; | |
1fae7cf0 | 170 | |
cafe5635 KO |
171 | spin_lock_irq(&closure_list_lock); |
172 | ||
173 | list_for_each_entry(cl, &closure_list, all) { | |
174 | int r = atomic_read(&cl->remaining); | |
175 | ||
d9c61d30 | 176 | seq_printf(f, "%p: %pS -> %pS p %p r %i ", |
cafe5635 KO |
177 | cl, (void *) cl->ip, cl->fn, cl->parent, |
178 | r & CLOSURE_REMAINING_MASK); | |
179 | ||
e4bf7919 | 180 | seq_printf(f, "%s%s\n", |
8d090f47 | 181 | test_bit(WORK_STRUCT_PENDING_BIT, |
cafe5635 | 182 | work_data_bits(&cl->work)) ? "Q" : "", |
e4bf7919 | 183 | r & CLOSURE_RUNNING ? "R" : ""); |
cafe5635 KO |
184 | |
185 | if (r & CLOSURE_WAITING) | |
d9c61d30 | 186 | seq_printf(f, " W %pS\n", |
cafe5635 KO |
187 | (void *) cl->waiting_on); |
188 | ||
8c8d2d96 | 189 | seq_puts(f, "\n"); |
cafe5635 KO |
190 | } |
191 | ||
192 | spin_unlock_irq(&closure_list_lock); | |
193 | return 0; | |
194 | } | |
195 | ||
84e5d136 | 196 | DEFINE_SHOW_ATTRIBUTE(debug); |
cafe5635 | 197 | |
8c8d2d96 | 198 | static int __init closure_debug_init(void) |
cafe5635 | 199 | { |
8c8d2d96 KO |
200 | debugfs_create_file("closures", 0400, NULL, NULL, &debug_fops); |
201 | return 0; | |
cafe5635 | 202 | } |
8c8d2d96 | 203 | late_initcall(closure_debug_init) |
cafe5635 | 204 | |
8c8d2d96 | 205 | #endif |