Commit | Line | Data |
---|---|---|
5dd43ce2 IM |
1 | /* |
2 | * The implementation of the wait_bit*() and related waiting APIs: | |
3 | */ | |
325ea10c | 4 | #include "sched.h" |
5822a454 IM |
5 | |
6 | #define WAIT_TABLE_BITS 8 | |
7 | #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS) | |
8 | ||
9 | static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned; | |
10 | ||
11 | wait_queue_head_t *bit_waitqueue(void *word, int bit) | |
12 | { | |
13 | const int shift = BITS_PER_LONG == 32 ? 5 : 6; | |
14 | unsigned long val = (unsigned long)word << shift | bit; | |
15 | ||
16 | return bit_wait_table + hash_long(val, WAIT_TABLE_BITS); | |
17 | } | |
18 | EXPORT_SYMBOL(bit_waitqueue); | |
5dd43ce2 IM |
19 | |
20 | int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg) | |
21 | { | |
22 | struct wait_bit_key *key = arg; | |
23 | struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); | |
24 | ||
25 | if (wait_bit->key.flags != key->flags || | |
26 | wait_bit->key.bit_nr != key->bit_nr || | |
27 | test_bit(key->bit_nr, key->flags)) | |
28 | return 0; | |
97fb7a0a IM |
29 | |
30 | return autoremove_wake_function(wq_entry, mode, sync, key); | |
5dd43ce2 IM |
31 | } |
32 | EXPORT_SYMBOL(wake_bit_function); | |
33 | ||
34 | /* | |
35 | * To allow interruptible waiting and asynchronous (i.e. nonblocking) | |
36 | * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are | |
37 | * permitted return codes. Nonzero return codes halt waiting and return. | |
38 | */ | |
39 | int __sched | |
40 | __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, | |
41 | wait_bit_action_f *action, unsigned mode) | |
42 | { | |
43 | int ret = 0; | |
44 | ||
45 | do { | |
46 | prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode); | |
47 | if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) | |
48 | ret = (*action)(&wbq_entry->key, mode); | |
49 | } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); | |
97fb7a0a | 50 | |
5dd43ce2 | 51 | finish_wait(wq_head, &wbq_entry->wq_entry); |
97fb7a0a | 52 | |
5dd43ce2 IM |
53 | return ret; |
54 | } | |
55 | EXPORT_SYMBOL(__wait_on_bit); | |
56 | ||
57 | int __sched out_of_line_wait_on_bit(void *word, int bit, | |
58 | wait_bit_action_f *action, unsigned mode) | |
59 | { | |
60 | struct wait_queue_head *wq_head = bit_waitqueue(word, bit); | |
61 | DEFINE_WAIT_BIT(wq_entry, word, bit); | |
62 | ||
63 | return __wait_on_bit(wq_head, &wq_entry, action, mode); | |
64 | } | |
65 | EXPORT_SYMBOL(out_of_line_wait_on_bit); | |
66 | ||
67 | int __sched out_of_line_wait_on_bit_timeout( | |
68 | void *word, int bit, wait_bit_action_f *action, | |
69 | unsigned mode, unsigned long timeout) | |
70 | { | |
71 | struct wait_queue_head *wq_head = bit_waitqueue(word, bit); | |
72 | DEFINE_WAIT_BIT(wq_entry, word, bit); | |
73 | ||
74 | wq_entry.key.timeout = jiffies + timeout; | |
97fb7a0a | 75 | |
5dd43ce2 IM |
76 | return __wait_on_bit(wq_head, &wq_entry, action, mode); |
77 | } | |
78 | EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout); | |
79 | ||
80 | int __sched | |
81 | __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry, | |
82 | wait_bit_action_f *action, unsigned mode) | |
83 | { | |
84 | int ret = 0; | |
85 | ||
86 | for (;;) { | |
87 | prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode); | |
88 | if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) { | |
89 | ret = action(&wbq_entry->key, mode); | |
90 | /* | |
91 | * See the comment in prepare_to_wait_event(). | |
92 | * finish_wait() does not necessarily takes wwq_head->lock, | |
93 | * but test_and_set_bit() implies mb() which pairs with | |
94 | * smp_mb__after_atomic() before wake_up_page(). | |
95 | */ | |
96 | if (ret) | |
97 | finish_wait(wq_head, &wbq_entry->wq_entry); | |
98 | } | |
99 | if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) { | |
100 | if (!ret) | |
101 | finish_wait(wq_head, &wbq_entry->wq_entry); | |
102 | return 0; | |
103 | } else if (ret) { | |
104 | return ret; | |
105 | } | |
106 | } | |
107 | } | |
108 | EXPORT_SYMBOL(__wait_on_bit_lock); | |
109 | ||
110 | int __sched out_of_line_wait_on_bit_lock(void *word, int bit, | |
111 | wait_bit_action_f *action, unsigned mode) | |
112 | { | |
113 | struct wait_queue_head *wq_head = bit_waitqueue(word, bit); | |
114 | DEFINE_WAIT_BIT(wq_entry, word, bit); | |
115 | ||
116 | return __wait_on_bit_lock(wq_head, &wq_entry, action, mode); | |
117 | } | |
118 | EXPORT_SYMBOL(out_of_line_wait_on_bit_lock); | |
119 | ||
120 | void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit) | |
121 | { | |
122 | struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit); | |
97fb7a0a | 123 | |
5dd43ce2 IM |
124 | if (waitqueue_active(wq_head)) |
125 | __wake_up(wq_head, TASK_NORMAL, 1, &key); | |
126 | } | |
127 | EXPORT_SYMBOL(__wake_up_bit); | |
128 | ||
129 | /** | |
130 | * wake_up_bit - wake up a waiter on a bit | |
131 | * @word: the word being waited on, a kernel virtual address | |
132 | * @bit: the bit of the word being waited on | |
133 | * | |
134 | * There is a standard hashed waitqueue table for generic use. This | |
135 | * is the part of the hashtable's accessor API that wakes up waiters | |
136 | * on a bit. For instance, if one were to have waiters on a bitflag, | |
137 | * one would call wake_up_bit() after clearing the bit. | |
138 | * | |
139 | * In order for this to function properly, as it uses waitqueue_active() | |
140 | * internally, some kind of memory barrier must be done prior to calling | |
141 | * this. Typically, this will be smp_mb__after_atomic(), but in some | |
142 | * cases where bitflags are manipulated non-atomically under a lock, one | |
143 | * may need to use a less regular barrier, such fs/inode.c's smp_mb(), | |
144 | * because spin_unlock() does not guarantee a memory barrier. | |
145 | */ | |
146 | void wake_up_bit(void *word, int bit) | |
147 | { | |
148 | __wake_up_bit(bit_waitqueue(word, bit), word, bit); | |
149 | } | |
150 | EXPORT_SYMBOL(wake_up_bit); | |
151 | ||
6b2bb726 PZ |
152 | wait_queue_head_t *__var_waitqueue(void *p) |
153 | { | |
b3fc5c9b | 154 | return bit_wait_table + hash_ptr(p, WAIT_TABLE_BITS); |
6b2bb726 PZ |
155 | } |
156 | EXPORT_SYMBOL(__var_waitqueue); | |
157 | ||
158 | static int | |
159 | var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, | |
160 | int sync, void *arg) | |
161 | { | |
162 | struct wait_bit_key *key = arg; | |
163 | struct wait_bit_queue_entry *wbq_entry = | |
164 | container_of(wq_entry, struct wait_bit_queue_entry, wq_entry); | |
165 | ||
166 | if (wbq_entry->key.flags != key->flags || | |
167 | wbq_entry->key.bit_nr != key->bit_nr) | |
168 | return 0; | |
169 | ||
170 | return autoremove_wake_function(wq_entry, mode, sync, key); | |
171 | } | |
172 | ||
173 | void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, int flags) | |
174 | { | |
175 | *wbq_entry = (struct wait_bit_queue_entry){ | |
176 | .key = { | |
177 | .flags = (var), | |
178 | .bit_nr = -1, | |
179 | }, | |
180 | .wq_entry = { | |
181 | .private = current, | |
182 | .func = var_wake_function, | |
183 | .entry = LIST_HEAD_INIT(wbq_entry->wq_entry.entry), | |
184 | }, | |
185 | }; | |
186 | } | |
187 | EXPORT_SYMBOL(init_wait_var_entry); | |
188 | ||
189 | void wake_up_var(void *var) | |
190 | { | |
191 | __wake_up_bit(__var_waitqueue(var), var, -1); | |
192 | } | |
193 | EXPORT_SYMBOL(wake_up_var); | |
194 | ||
5dd43ce2 IM |
195 | __sched int bit_wait(struct wait_bit_key *word, int mode) |
196 | { | |
197 | schedule(); | |
198 | if (signal_pending_state(mode, current)) | |
199 | return -EINTR; | |
97fb7a0a | 200 | |
5dd43ce2 IM |
201 | return 0; |
202 | } | |
203 | EXPORT_SYMBOL(bit_wait); | |
204 | ||
205 | __sched int bit_wait_io(struct wait_bit_key *word, int mode) | |
206 | { | |
207 | io_schedule(); | |
208 | if (signal_pending_state(mode, current)) | |
209 | return -EINTR; | |
97fb7a0a | 210 | |
5dd43ce2 IM |
211 | return 0; |
212 | } | |
213 | EXPORT_SYMBOL(bit_wait_io); | |
214 | ||
215 | __sched int bit_wait_timeout(struct wait_bit_key *word, int mode) | |
216 | { | |
217 | unsigned long now = READ_ONCE(jiffies); | |
97fb7a0a | 218 | |
5dd43ce2 IM |
219 | if (time_after_eq(now, word->timeout)) |
220 | return -EAGAIN; | |
221 | schedule_timeout(word->timeout - now); | |
222 | if (signal_pending_state(mode, current)) | |
223 | return -EINTR; | |
97fb7a0a | 224 | |
5dd43ce2 IM |
225 | return 0; |
226 | } | |
227 | EXPORT_SYMBOL_GPL(bit_wait_timeout); | |
228 | ||
229 | __sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode) | |
230 | { | |
231 | unsigned long now = READ_ONCE(jiffies); | |
97fb7a0a | 232 | |
5dd43ce2 IM |
233 | if (time_after_eq(now, word->timeout)) |
234 | return -EAGAIN; | |
235 | io_schedule_timeout(word->timeout - now); | |
236 | if (signal_pending_state(mode, current)) | |
237 | return -EINTR; | |
97fb7a0a | 238 | |
5dd43ce2 IM |
239 | return 0; |
240 | } | |
241 | EXPORT_SYMBOL_GPL(bit_wait_io_timeout); | |
5822a454 IM |
242 | |
243 | void __init wait_bit_init(void) | |
244 | { | |
245 | int i; | |
246 | ||
247 | for (i = 0; i < WAIT_TABLE_SIZE; i++) | |
248 | init_waitqueue_head(bit_wait_table + i); | |
249 | } |