Commit | Line | Data |
---|---|---|
cafe5635 KO |
1 | /* |
2 | * Asynchronous refcounty things | |
3 | * | |
4 | * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com> | |
5 | * Copyright 2012 Google, Inc. | |
6 | */ | |
7 | ||
8 | #include <linux/debugfs.h> | |
9 | #include <linux/module.h> | |
10 | #include <linux/seq_file.h> | |
ce439bf7 | 11 | #include <linux/sched/debug.h> |
cafe5635 KO |
12 | |
13 | #include "closure.h" | |
14 | ||
cafe5635 KO |
15 | static inline void closure_put_after_sub(struct closure *cl, int flags) |
16 | { | |
17 | int r = flags & CLOSURE_REMAINING_MASK; | |
18 | ||
19 | BUG_ON(flags & CLOSURE_GUARD_MASK); | |
faadf0c9 | 20 | BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR)); |
cafe5635 | 21 | |
cafe5635 KO |
22 | if (!r) { |
23 | if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) { | |
cafe5635 KO |
24 | atomic_set(&cl->remaining, |
25 | CLOSURE_REMAINING_INITIALIZER); | |
26 | closure_queue(cl); | |
27 | } else { | |
28 | struct closure *parent = cl->parent; | |
6aa8f1a6 | 29 | closure_fn *destructor = cl->fn; |
cafe5635 KO |
30 | |
31 | closure_debug_destroy(cl); | |
32 | ||
6aa8f1a6 KO |
33 | if (destructor) |
34 | destructor(cl); | |
cafe5635 KO |
35 | |
36 | if (parent) | |
37 | closure_put(parent); | |
38 | } | |
39 | } | |
40 | } | |
41 | ||
42 | /* For clearing flags with the same atomic op as a put */ | |
43 | void closure_sub(struct closure *cl, int v) | |
44 | { | |
45 | closure_put_after_sub(cl, atomic_sub_return(v, &cl->remaining)); | |
46 | } | |
faadf0c9 | 47 | EXPORT_SYMBOL(closure_sub); |
cafe5635 | 48 | |
47344e33 | 49 | /* |
1dd13c8d KO |
50 | * closure_put - decrement a closure's refcount |
51 | */ | |
cafe5635 KO |
52 | void closure_put(struct closure *cl) |
53 | { | |
54 | closure_put_after_sub(cl, atomic_dec_return(&cl->remaining)); | |
55 | } | |
faadf0c9 | 56 | EXPORT_SYMBOL(closure_put); |
cafe5635 | 57 | |
47344e33 | 58 | /* |
1dd13c8d KO |
59 | * closure_wake_up - wake up all closures on a wait list, without memory barrier |
60 | */ | |
cafe5635 KO |
61 | void __closure_wake_up(struct closure_waitlist *wait_list) |
62 | { | |
63 | struct llist_node *list; | |
a5f3d8a5 | 64 | struct closure *cl, *t; |
cafe5635 KO |
65 | struct llist_node *reverse = NULL; |
66 | ||
67 | list = llist_del_all(&wait_list->list); | |
68 | ||
69 | /* We first reverse the list to preserve FIFO ordering and fairness */ | |
09b3efec | 70 | reverse = llist_reverse_order(list); |
cafe5635 KO |
71 | |
72 | /* Then do the wakeups */ | |
a5f3d8a5 | 73 | llist_for_each_entry_safe(cl, t, reverse, list) { |
1dd13c8d | 74 | closure_set_waiting(cl, 0); |
cafe5635 KO |
75 | closure_sub(cl, CLOSURE_WAITING + 1); |
76 | } | |
77 | } | |
faadf0c9 | 78 | EXPORT_SYMBOL(__closure_wake_up); |
cafe5635 | 79 | |
1dd13c8d KO |
80 | /** |
81 | * closure_wait - add a closure to a waitlist | |
47344e33 | 82 | * @waitlist: will own a ref on @cl, which will be released when |
1dd13c8d | 83 | * closure_wake_up() is called on @waitlist. |
47344e33 | 84 | * @cl: closure pointer. |
1dd13c8d KO |
85 | * |
86 | */ | |
87 | bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl) | |
cafe5635 KO |
88 | { |
89 | if (atomic_read(&cl->remaining) & CLOSURE_WAITING) | |
90 | return false; | |
91 | ||
1dd13c8d | 92 | closure_set_waiting(cl, _RET_IP_); |
cafe5635 | 93 | atomic_add(CLOSURE_WAITING + 1, &cl->remaining); |
1dd13c8d | 94 | llist_add(&cl->list, &waitlist->list); |
cafe5635 KO |
95 | |
96 | return true; | |
97 | } | |
faadf0c9 | 98 | EXPORT_SYMBOL(closure_wait); |
cafe5635 | 99 | |
e4bf7919 KO |
100 | struct closure_syncer { |
101 | struct task_struct *task; | |
102 | int done; | |
103 | }; | |
104 | ||
105 | static void closure_sync_fn(struct closure *cl) | |
cafe5635 | 106 | { |
e4bf7919 KO |
107 | cl->s->done = 1; |
108 | wake_up_process(cl->s->task); | |
109 | } | |
cafe5635 | 110 | |
ce439bf7 | 111 | void __sched __closure_sync(struct closure *cl) |
e4bf7919 KO |
112 | { |
113 | struct closure_syncer s = { .task = current }; | |
cafe5635 | 114 | |
e4bf7919 KO |
115 | cl->s = &s; |
116 | continue_at(cl, closure_sync_fn, NULL); | |
117 | ||
118 | while (1) { | |
119 | set_current_state(TASK_UNINTERRUPTIBLE); | |
120 | if (s.done) | |
121 | break; | |
cafe5635 KO |
122 | schedule(); |
123 | } | |
124 | ||
e4bf7919 | 125 | __set_current_state(TASK_RUNNING); |
cafe5635 | 126 | } |
e4bf7919 | 127 | EXPORT_SYMBOL(__closure_sync); |
cafe5635 | 128 | |
cafe5635 KO |
129 | #ifdef CONFIG_BCACHE_CLOSURES_DEBUG |
130 | ||
131 | static LIST_HEAD(closure_list); | |
132 | static DEFINE_SPINLOCK(closure_list_lock); | |
133 | ||
134 | void closure_debug_create(struct closure *cl) | |
135 | { | |
136 | unsigned long flags; | |
137 | ||
138 | BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE); | |
139 | cl->magic = CLOSURE_MAGIC_ALIVE; | |
140 | ||
141 | spin_lock_irqsave(&closure_list_lock, flags); | |
142 | list_add(&cl->all, &closure_list); | |
143 | spin_unlock_irqrestore(&closure_list_lock, flags); | |
144 | } | |
faadf0c9 | 145 | EXPORT_SYMBOL(closure_debug_create); |
cafe5635 KO |
146 | |
147 | void closure_debug_destroy(struct closure *cl) | |
148 | { | |
149 | unsigned long flags; | |
150 | ||
151 | BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE); | |
152 | cl->magic = CLOSURE_MAGIC_DEAD; | |
153 | ||
154 | spin_lock_irqsave(&closure_list_lock, flags); | |
155 | list_del(&cl->all); | |
156 | spin_unlock_irqrestore(&closure_list_lock, flags); | |
157 | } | |
faadf0c9 | 158 | EXPORT_SYMBOL(closure_debug_destroy); |
cafe5635 | 159 | |
df2b9431 | 160 | static struct dentry *closure_debug; |
cafe5635 | 161 | |
cafe5635 KO |
162 | static int debug_seq_show(struct seq_file *f, void *data) |
163 | { | |
164 | struct closure *cl; | |
1fae7cf0 | 165 | |
cafe5635 KO |
166 | spin_lock_irq(&closure_list_lock); |
167 | ||
168 | list_for_each_entry(cl, &closure_list, all) { | |
169 | int r = atomic_read(&cl->remaining); | |
170 | ||
d9c61d30 | 171 | seq_printf(f, "%p: %pS -> %pS p %p r %i ", |
cafe5635 KO |
172 | cl, (void *) cl->ip, cl->fn, cl->parent, |
173 | r & CLOSURE_REMAINING_MASK); | |
174 | ||
e4bf7919 | 175 | seq_printf(f, "%s%s\n", |
8d090f47 | 176 | test_bit(WORK_STRUCT_PENDING_BIT, |
cafe5635 | 177 | work_data_bits(&cl->work)) ? "Q" : "", |
e4bf7919 | 178 | r & CLOSURE_RUNNING ? "R" : ""); |
cafe5635 KO |
179 | |
180 | if (r & CLOSURE_WAITING) | |
d9c61d30 | 181 | seq_printf(f, " W %pS\n", |
cafe5635 KO |
182 | (void *) cl->waiting_on); |
183 | ||
184 | seq_printf(f, "\n"); | |
185 | } | |
186 | ||
187 | spin_unlock_irq(&closure_list_lock); | |
188 | return 0; | |
189 | } | |
190 | ||
191 | static int debug_seq_open(struct inode *inode, struct file *file) | |
192 | { | |
193 | return single_open(file, debug_seq_show, NULL); | |
194 | } | |
195 | ||
196 | static const struct file_operations debug_ops = { | |
197 | .owner = THIS_MODULE, | |
198 | .open = debug_seq_open, | |
199 | .read = seq_read, | |
200 | .release = single_release | |
201 | }; | |
202 | ||
78ac2107 | 203 | void __init closure_debug_init(void) |
cafe5635 | 204 | { |
78ac2107 CL |
205 | if (!IS_ERR_OR_NULL(bcache_debug)) |
206 | /* | |
207 | * it is unnecessary to check return value of | |
208 | * debugfs_create_file(), we should not care | |
209 | * about this. | |
210 | */ | |
211 | closure_debug = debugfs_create_file( | |
212 | "closures", 0400, bcache_debug, NULL, &debug_ops); | |
cafe5635 | 213 | } |
cafe5635 KO |
214 | #endif |
215 | ||
216 | MODULE_AUTHOR("Kent Overstreet <koverstreet@google.com>"); | |
217 | MODULE_LICENSE("GPL"); |