Commit | Line | Data |
---|---|---|
8b712842 CM |
1 | /* |
2 | * Copyright (C) 2007 Oracle. All rights reserved. | |
08a9ff32 | 3 | * Copyright (C) 2014 Fujitsu. All rights reserved. |
8b712842 CM |
4 | * |
5 | * This program is free software; you can redistribute it and/or | |
6 | * modify it under the terms of the GNU General Public | |
7 | * License v2 as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
12 | * General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public | |
15 | * License along with this program; if not, write to the | |
16 | * Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
17 | * Boston, MA 021110-1307, USA. | |
18 | */ | |
19 | ||
20 | #include <linux/kthread.h> | |
5a0e3ad6 | 21 | #include <linux/slab.h> |
8b712842 CM |
22 | #include <linux/list.h> |
23 | #include <linux/spinlock.h> | |
b51912c9 | 24 | #include <linux/freezer.h> |
8b712842 | 25 | #include "async-thread.h" |
52483bc2 | 26 | #include "ctree.h" |
8b712842 | 27 | |
a046e9c8 QW |
28 | #define WORK_DONE_BIT 0 |
29 | #define WORK_ORDER_DONE_BIT 1 | |
30 | #define WORK_HIGH_PRIO_BIT 2 | |
4a69a410 | 31 | |
0bd9289c QW |
32 | #define NO_THRESHOLD (-1) |
33 | #define DFT_THRESHOLD (32) | |
34 | ||
d458b054 | 35 | struct __btrfs_workqueue { |
08a9ff32 | 36 | struct workqueue_struct *normal_wq; |
cb001095 JM |
37 | |
38 | /* File system this workqueue services */ | |
39 | struct btrfs_fs_info *fs_info; | |
40 | ||
08a9ff32 QW |
41 | /* List head pointing to ordered work list */ |
42 | struct list_head ordered_list; | |
43 | ||
44 | /* Spinlock for ordered_list */ | |
45 | spinlock_t list_lock; | |
0bd9289c QW |
46 | |
47 | /* Thresholding related variants */ | |
48 | atomic_t pending; | |
c6dd6ea5 QW |
49 | |
50 | /* Up limit of concurrency workers */ | |
51 | int limit_active; | |
52 | ||
53 | /* Current number of concurrency workers */ | |
54 | int current_active; | |
55 | ||
56 | /* Threshold to change current_active */ | |
0bd9289c QW |
57 | int thresh; |
58 | unsigned int count; | |
59 | spinlock_t thres_lock; | |
08a9ff32 QW |
60 | }; |
61 | ||
d458b054 QW |
62 | struct btrfs_workqueue { |
63 | struct __btrfs_workqueue *normal; | |
64 | struct __btrfs_workqueue *high; | |
1ca08976 QW |
65 | }; |
66 | ||
9e0af237 LB |
67 | static void normal_work_helper(struct btrfs_work *work); |
68 | ||
69 | #define BTRFS_WORK_HELPER(name) \ | |
6939f667 | 70 | noinline_for_stack void btrfs_##name(struct work_struct *arg) \ |
9e0af237 LB |
71 | { \ |
72 | struct btrfs_work *work = container_of(arg, struct btrfs_work, \ | |
73 | normal_work); \ | |
74 | normal_work_helper(work); \ | |
75 | } | |
76 | ||
cb001095 | 77 | struct btrfs_fs_info * |
9a35b637 | 78 | btrfs_workqueue_owner(const struct __btrfs_workqueue *wq) |
cb001095 JM |
79 | { |
80 | return wq->fs_info; | |
81 | } | |
82 | ||
83 | struct btrfs_fs_info * | |
9a35b637 | 84 | btrfs_work_owner(const struct btrfs_work *work) |
cb001095 JM |
85 | { |
86 | return work->wq->fs_info; | |
87 | } | |
88 | ||
9a35b637 | 89 | bool btrfs_workqueue_normal_congested(const struct btrfs_workqueue *wq) |
2939e1a8 MP |
90 | { |
91 | /* | |
92 | * We could compare wq->normal->pending with num_online_cpus() | |
93 | * to support "thresh == NO_THRESHOLD" case, but it requires | |
94 | * moving up atomic_inc/dec in thresh_queue/exec_hook. Let's | |
95 | * postpone it until someone needs the support of that case. | |
96 | */ | |
97 | if (wq->normal->thresh == NO_THRESHOLD) | |
98 | return false; | |
99 | ||
100 | return atomic_read(&wq->normal->pending) > wq->normal->thresh * 2; | |
101 | } | |
102 | ||
9e0af237 LB |
103 | BTRFS_WORK_HELPER(worker_helper); |
104 | BTRFS_WORK_HELPER(delalloc_helper); | |
105 | BTRFS_WORK_HELPER(flush_delalloc_helper); | |
106 | BTRFS_WORK_HELPER(cache_helper); | |
107 | BTRFS_WORK_HELPER(submit_helper); | |
108 | BTRFS_WORK_HELPER(fixup_helper); | |
109 | BTRFS_WORK_HELPER(endio_helper); | |
110 | BTRFS_WORK_HELPER(endio_meta_helper); | |
111 | BTRFS_WORK_HELPER(endio_meta_write_helper); | |
112 | BTRFS_WORK_HELPER(endio_raid56_helper); | |
8b110e39 | 113 | BTRFS_WORK_HELPER(endio_repair_helper); |
9e0af237 LB |
114 | BTRFS_WORK_HELPER(rmw_helper); |
115 | BTRFS_WORK_HELPER(endio_write_helper); | |
116 | BTRFS_WORK_HELPER(freespace_write_helper); | |
117 | BTRFS_WORK_HELPER(delayed_meta_helper); | |
118 | BTRFS_WORK_HELPER(readahead_helper); | |
119 | BTRFS_WORK_HELPER(qgroup_rescan_helper); | |
120 | BTRFS_WORK_HELPER(extent_refs_helper); | |
121 | BTRFS_WORK_HELPER(scrub_helper); | |
122 | BTRFS_WORK_HELPER(scrubwrc_helper); | |
123 | BTRFS_WORK_HELPER(scrubnc_helper); | |
20b2e302 | 124 | BTRFS_WORK_HELPER(scrubparity_helper); |
9e0af237 LB |
125 | |
126 | static struct __btrfs_workqueue * | |
cb001095 JM |
127 | __btrfs_alloc_workqueue(struct btrfs_fs_info *fs_info, const char *name, |
128 | unsigned int flags, int limit_active, int thresh) | |
1ca08976 | 129 | { |
61dd5ae6 | 130 | struct __btrfs_workqueue *ret = kzalloc(sizeof(*ret), GFP_KERNEL); |
1ca08976 | 131 | |
5d99a998 | 132 | if (!ret) |
1ca08976 QW |
133 | return NULL; |
134 | ||
cb001095 | 135 | ret->fs_info = fs_info; |
c6dd6ea5 | 136 | ret->limit_active = limit_active; |
0bd9289c QW |
137 | atomic_set(&ret->pending, 0); |
138 | if (thresh == 0) | |
139 | thresh = DFT_THRESHOLD; | |
140 | /* For low threshold, disabling threshold is a better choice */ | |
141 | if (thresh < DFT_THRESHOLD) { | |
c6dd6ea5 | 142 | ret->current_active = limit_active; |
0bd9289c QW |
143 | ret->thresh = NO_THRESHOLD; |
144 | } else { | |
c6dd6ea5 QW |
145 | /* |
146 | * For threshold-able wq, let its concurrency grow on demand. | |
147 | * Use minimal max_active at alloc time to reduce resource | |
148 | * usage. | |
149 | */ | |
150 | ret->current_active = 1; | |
0bd9289c QW |
151 | ret->thresh = thresh; |
152 | } | |
153 | ||
1ca08976 QW |
154 | if (flags & WQ_HIGHPRI) |
155 | ret->normal_wq = alloc_workqueue("%s-%s-high", flags, | |
c6dd6ea5 QW |
156 | ret->current_active, "btrfs", |
157 | name); | |
1ca08976 QW |
158 | else |
159 | ret->normal_wq = alloc_workqueue("%s-%s", flags, | |
c6dd6ea5 | 160 | ret->current_active, "btrfs", |
0bd9289c | 161 | name); |
5d99a998 | 162 | if (!ret->normal_wq) { |
1ca08976 QW |
163 | kfree(ret); |
164 | return NULL; | |
165 | } | |
166 | ||
167 | INIT_LIST_HEAD(&ret->ordered_list); | |
168 | spin_lock_init(&ret->list_lock); | |
0bd9289c | 169 | spin_lock_init(&ret->thres_lock); |
c3a46891 | 170 | trace_btrfs_workqueue_alloc(ret, name, flags & WQ_HIGHPRI); |
1ca08976 QW |
171 | return ret; |
172 | } | |
173 | ||
174 | static inline void | |
d458b054 | 175 | __btrfs_destroy_workqueue(struct __btrfs_workqueue *wq); |
1ca08976 | 176 | |
cb001095 JM |
177 | struct btrfs_workqueue *btrfs_alloc_workqueue(struct btrfs_fs_info *fs_info, |
178 | const char *name, | |
6f011058 | 179 | unsigned int flags, |
c6dd6ea5 | 180 | int limit_active, |
d458b054 | 181 | int thresh) |
08a9ff32 | 182 | { |
61dd5ae6 | 183 | struct btrfs_workqueue *ret = kzalloc(sizeof(*ret), GFP_KERNEL); |
08a9ff32 | 184 | |
5d99a998 | 185 | if (!ret) |
08a9ff32 QW |
186 | return NULL; |
187 | ||
cb001095 JM |
188 | ret->normal = __btrfs_alloc_workqueue(fs_info, name, |
189 | flags & ~WQ_HIGHPRI, | |
c6dd6ea5 | 190 | limit_active, thresh); |
5d99a998 | 191 | if (!ret->normal) { |
08a9ff32 QW |
192 | kfree(ret); |
193 | return NULL; | |
194 | } | |
195 | ||
1ca08976 | 196 | if (flags & WQ_HIGHPRI) { |
cb001095 JM |
197 | ret->high = __btrfs_alloc_workqueue(fs_info, name, flags, |
198 | limit_active, thresh); | |
5d99a998 | 199 | if (!ret->high) { |
1ca08976 QW |
200 | __btrfs_destroy_workqueue(ret->normal); |
201 | kfree(ret); | |
202 | return NULL; | |
203 | } | |
204 | } | |
08a9ff32 QW |
205 | return ret; |
206 | } | |
207 | ||
0bd9289c QW |
208 | /* |
209 | * Hook for threshold which will be called in btrfs_queue_work. | |
210 | * This hook WILL be called in IRQ handler context, | |
211 | * so workqueue_set_max_active MUST NOT be called in this hook | |
212 | */ | |
d458b054 | 213 | static inline void thresh_queue_hook(struct __btrfs_workqueue *wq) |
0bd9289c QW |
214 | { |
215 | if (wq->thresh == NO_THRESHOLD) | |
216 | return; | |
217 | atomic_inc(&wq->pending); | |
218 | } | |
219 | ||
220 | /* | |
221 | * Hook for threshold which will be called before executing the work, | |
222 | * This hook is called in kthread content. | |
223 | * So workqueue_set_max_active is called here. | |
224 | */ | |
d458b054 | 225 | static inline void thresh_exec_hook(struct __btrfs_workqueue *wq) |
0bd9289c | 226 | { |
c6dd6ea5 | 227 | int new_current_active; |
0bd9289c QW |
228 | long pending; |
229 | int need_change = 0; | |
230 | ||
231 | if (wq->thresh == NO_THRESHOLD) | |
232 | return; | |
233 | ||
234 | atomic_dec(&wq->pending); | |
235 | spin_lock(&wq->thres_lock); | |
236 | /* | |
237 | * Use wq->count to limit the calling frequency of | |
238 | * workqueue_set_max_active. | |
239 | */ | |
240 | wq->count++; | |
241 | wq->count %= (wq->thresh / 4); | |
242 | if (!wq->count) | |
243 | goto out; | |
c6dd6ea5 | 244 | new_current_active = wq->current_active; |
0bd9289c QW |
245 | |
246 | /* | |
247 | * pending may be changed later, but it's OK since we really | |
248 | * don't need it so accurate to calculate new_max_active. | |
249 | */ | |
250 | pending = atomic_read(&wq->pending); | |
251 | if (pending > wq->thresh) | |
c6dd6ea5 | 252 | new_current_active++; |
0bd9289c | 253 | if (pending < wq->thresh / 2) |
c6dd6ea5 QW |
254 | new_current_active--; |
255 | new_current_active = clamp_val(new_current_active, 1, wq->limit_active); | |
256 | if (new_current_active != wq->current_active) { | |
0bd9289c | 257 | need_change = 1; |
c6dd6ea5 | 258 | wq->current_active = new_current_active; |
0bd9289c QW |
259 | } |
260 | out: | |
261 | spin_unlock(&wq->thres_lock); | |
262 | ||
263 | if (need_change) { | |
c6dd6ea5 | 264 | workqueue_set_max_active(wq->normal_wq, wq->current_active); |
0bd9289c QW |
265 | } |
266 | } | |
267 | ||
d458b054 | 268 | static void run_ordered_work(struct __btrfs_workqueue *wq) |
08a9ff32 QW |
269 | { |
270 | struct list_head *list = &wq->ordered_list; | |
d458b054 | 271 | struct btrfs_work *work; |
08a9ff32 QW |
272 | spinlock_t *lock = &wq->list_lock; |
273 | unsigned long flags; | |
274 | ||
275 | while (1) { | |
ac0c7cf8 DS |
276 | void *wtag; |
277 | ||
08a9ff32 QW |
278 | spin_lock_irqsave(lock, flags); |
279 | if (list_empty(list)) | |
280 | break; | |
d458b054 | 281 | work = list_entry(list->next, struct btrfs_work, |
08a9ff32 QW |
282 | ordered_list); |
283 | if (!test_bit(WORK_DONE_BIT, &work->flags)) | |
284 | break; | |
285 | ||
286 | /* | |
287 | * we are going to call the ordered done function, but | |
288 | * we leave the work item on the list as a barrier so | |
289 | * that later work items that are done don't have their | |
290 | * functions called before this one returns | |
291 | */ | |
292 | if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags)) | |
293 | break; | |
52483bc2 | 294 | trace_btrfs_ordered_sched(work); |
08a9ff32 QW |
295 | spin_unlock_irqrestore(lock, flags); |
296 | work->ordered_func(work); | |
297 | ||
298 | /* now take the lock again and drop our item from the list */ | |
299 | spin_lock_irqsave(lock, flags); | |
300 | list_del(&work->ordered_list); | |
301 | spin_unlock_irqrestore(lock, flags); | |
302 | ||
303 | /* | |
ac0c7cf8 DS |
304 | * We don't want to call the ordered free functions with the |
305 | * lock held though. Save the work as tag for the trace event, | |
306 | * because the callback could free the structure. | |
08a9ff32 | 307 | */ |
ac0c7cf8 | 308 | wtag = work; |
08a9ff32 | 309 | work->ordered_free(work); |
ac0c7cf8 | 310 | trace_btrfs_all_work_done(wq->fs_info, wtag); |
08a9ff32 QW |
311 | } |
312 | spin_unlock_irqrestore(lock, flags); | |
313 | } | |
314 | ||
9e0af237 | 315 | static void normal_work_helper(struct btrfs_work *work) |
08a9ff32 | 316 | { |
d458b054 | 317 | struct __btrfs_workqueue *wq; |
ac0c7cf8 | 318 | void *wtag; |
08a9ff32 QW |
319 | int need_order = 0; |
320 | ||
08a9ff32 QW |
321 | /* |
322 | * We should not touch things inside work in the following cases: | |
323 | * 1) after work->func() if it has no ordered_free | |
324 | * Since the struct is freed in work->func(). | |
325 | * 2) after setting WORK_DONE_BIT | |
326 | * The work may be freed in other threads almost instantly. | |
327 | * So we save the needed things here. | |
328 | */ | |
329 | if (work->ordered_func) | |
330 | need_order = 1; | |
331 | wq = work->wq; | |
ac0c7cf8 DS |
332 | /* Safe for tracepoints in case work gets freed by the callback */ |
333 | wtag = work; | |
08a9ff32 | 334 | |
52483bc2 | 335 | trace_btrfs_work_sched(work); |
0bd9289c | 336 | thresh_exec_hook(wq); |
08a9ff32 QW |
337 | work->func(work); |
338 | if (need_order) { | |
339 | set_bit(WORK_DONE_BIT, &work->flags); | |
340 | run_ordered_work(wq); | |
341 | } | |
52483bc2 | 342 | if (!need_order) |
ac0c7cf8 | 343 | trace_btrfs_all_work_done(wq->fs_info, wtag); |
08a9ff32 QW |
344 | } |
345 | ||
9e0af237 | 346 | void btrfs_init_work(struct btrfs_work *work, btrfs_work_func_t uniq_func, |
6db8914f QW |
347 | btrfs_func_t func, |
348 | btrfs_func_t ordered_func, | |
349 | btrfs_func_t ordered_free) | |
08a9ff32 QW |
350 | { |
351 | work->func = func; | |
352 | work->ordered_func = ordered_func; | |
353 | work->ordered_free = ordered_free; | |
9e0af237 | 354 | INIT_WORK(&work->normal_work, uniq_func); |
08a9ff32 QW |
355 | INIT_LIST_HEAD(&work->ordered_list); |
356 | work->flags = 0; | |
357 | } | |
358 | ||
d458b054 QW |
359 | static inline void __btrfs_queue_work(struct __btrfs_workqueue *wq, |
360 | struct btrfs_work *work) | |
08a9ff32 QW |
361 | { |
362 | unsigned long flags; | |
363 | ||
364 | work->wq = wq; | |
0bd9289c | 365 | thresh_queue_hook(wq); |
08a9ff32 QW |
366 | if (work->ordered_func) { |
367 | spin_lock_irqsave(&wq->list_lock, flags); | |
368 | list_add_tail(&work->ordered_list, &wq->ordered_list); | |
369 | spin_unlock_irqrestore(&wq->list_lock, flags); | |
370 | } | |
52483bc2 | 371 | trace_btrfs_work_queued(work); |
0a95b851 | 372 | queue_work(wq->normal_wq, &work->normal_work); |
08a9ff32 QW |
373 | } |
374 | ||
d458b054 QW |
375 | void btrfs_queue_work(struct btrfs_workqueue *wq, |
376 | struct btrfs_work *work) | |
1ca08976 | 377 | { |
d458b054 | 378 | struct __btrfs_workqueue *dest_wq; |
1ca08976 QW |
379 | |
380 | if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags) && wq->high) | |
381 | dest_wq = wq->high; | |
382 | else | |
383 | dest_wq = wq->normal; | |
384 | __btrfs_queue_work(dest_wq, work); | |
385 | } | |
386 | ||
387 | static inline void | |
d458b054 | 388 | __btrfs_destroy_workqueue(struct __btrfs_workqueue *wq) |
08a9ff32 QW |
389 | { |
390 | destroy_workqueue(wq->normal_wq); | |
c3a46891 | 391 | trace_btrfs_workqueue_destroy(wq); |
08a9ff32 QW |
392 | kfree(wq); |
393 | } | |
394 | ||
d458b054 | 395 | void btrfs_destroy_workqueue(struct btrfs_workqueue *wq) |
1ca08976 QW |
396 | { |
397 | if (!wq) | |
398 | return; | |
399 | if (wq->high) | |
400 | __btrfs_destroy_workqueue(wq->high); | |
401 | __btrfs_destroy_workqueue(wq->normal); | |
ef66af10 | 402 | kfree(wq); |
1ca08976 QW |
403 | } |
404 | ||
c6dd6ea5 | 405 | void btrfs_workqueue_set_max(struct btrfs_workqueue *wq, int limit_active) |
08a9ff32 | 406 | { |
800ee224 ST |
407 | if (!wq) |
408 | return; | |
c6dd6ea5 | 409 | wq->normal->limit_active = limit_active; |
1ca08976 | 410 | if (wq->high) |
c6dd6ea5 | 411 | wq->high->limit_active = limit_active; |
1ca08976 QW |
412 | } |
413 | ||
d458b054 | 414 | void btrfs_set_work_high_priority(struct btrfs_work *work) |
1ca08976 QW |
415 | { |
416 | set_bit(WORK_HIGH_PRIO_BIT, &work->flags); | |
08a9ff32 | 417 | } |