Commit | Line | Data |
---|---|---|
8b712842 CM |
1 | /* |
2 | * Copyright (C) 2007 Oracle. All rights reserved. | |
08a9ff32 | 3 | * Copyright (C) 2014 Fujitsu. All rights reserved. |
8b712842 CM |
4 | * |
5 | * This program is free software; you can redistribute it and/or | |
6 | * modify it under the terms of the GNU General Public | |
7 | * License v2 as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
12 | * General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public | |
15 | * License along with this program; if not, write to the | |
16 | * Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
17 | * Boston, MA 021110-1307, USA. | |
18 | */ | |
19 | ||
20 | #include <linux/kthread.h> | |
5a0e3ad6 | 21 | #include <linux/slab.h> |
8b712842 CM |
22 | #include <linux/list.h> |
23 | #include <linux/spinlock.h> | |
b51912c9 | 24 | #include <linux/freezer.h> |
8b712842 | 25 | #include "async-thread.h" |
52483bc2 | 26 | #include "ctree.h" |
8b712842 | 27 | |
a046e9c8 QW |
28 | #define WORK_DONE_BIT 0 |
29 | #define WORK_ORDER_DONE_BIT 1 | |
30 | #define WORK_HIGH_PRIO_BIT 2 | |
4a69a410 | 31 | |
0bd9289c QW |
32 | #define NO_THRESHOLD (-1) |
33 | #define DFT_THRESHOLD (32) | |
34 | ||
d458b054 | 35 | struct __btrfs_workqueue { |
08a9ff32 QW |
36 | struct workqueue_struct *normal_wq; |
37 | /* List head pointing to ordered work list */ | |
38 | struct list_head ordered_list; | |
39 | ||
40 | /* Spinlock for ordered_list */ | |
41 | spinlock_t list_lock; | |
0bd9289c QW |
42 | |
43 | /* Thresholding related variants */ | |
44 | atomic_t pending; | |
45 | int max_active; | |
46 | int current_max; | |
47 | int thresh; | |
48 | unsigned int count; | |
49 | spinlock_t thres_lock; | |
08a9ff32 QW |
50 | }; |
51 | ||
d458b054 QW |
52 | struct btrfs_workqueue { |
53 | struct __btrfs_workqueue *normal; | |
54 | struct __btrfs_workqueue *high; | |
1ca08976 QW |
55 | }; |
56 | ||
9e0af237 LB |
57 | static void normal_work_helper(struct btrfs_work *work); |
58 | ||
59 | #define BTRFS_WORK_HELPER(name) \ | |
60 | void btrfs_##name(struct work_struct *arg) \ | |
61 | { \ | |
62 | struct btrfs_work *work = container_of(arg, struct btrfs_work, \ | |
63 | normal_work); \ | |
64 | normal_work_helper(work); \ | |
65 | } | |
66 | ||
67 | BTRFS_WORK_HELPER(worker_helper); | |
68 | BTRFS_WORK_HELPER(delalloc_helper); | |
69 | BTRFS_WORK_HELPER(flush_delalloc_helper); | |
70 | BTRFS_WORK_HELPER(cache_helper); | |
71 | BTRFS_WORK_HELPER(submit_helper); | |
72 | BTRFS_WORK_HELPER(fixup_helper); | |
73 | BTRFS_WORK_HELPER(endio_helper); | |
74 | BTRFS_WORK_HELPER(endio_meta_helper); | |
75 | BTRFS_WORK_HELPER(endio_meta_write_helper); | |
76 | BTRFS_WORK_HELPER(endio_raid56_helper); | |
8b110e39 | 77 | BTRFS_WORK_HELPER(endio_repair_helper); |
9e0af237 LB |
78 | BTRFS_WORK_HELPER(rmw_helper); |
79 | BTRFS_WORK_HELPER(endio_write_helper); | |
80 | BTRFS_WORK_HELPER(freespace_write_helper); | |
81 | BTRFS_WORK_HELPER(delayed_meta_helper); | |
82 | BTRFS_WORK_HELPER(readahead_helper); | |
83 | BTRFS_WORK_HELPER(qgroup_rescan_helper); | |
84 | BTRFS_WORK_HELPER(extent_refs_helper); | |
85 | BTRFS_WORK_HELPER(scrub_helper); | |
86 | BTRFS_WORK_HELPER(scrubwrc_helper); | |
87 | BTRFS_WORK_HELPER(scrubnc_helper); | |
20b2e302 | 88 | BTRFS_WORK_HELPER(scrubparity_helper); |
9e0af237 LB |
89 | |
90 | static struct __btrfs_workqueue * | |
6f011058 | 91 | __btrfs_alloc_workqueue(const char *name, unsigned int flags, int max_active, |
c3a46891 | 92 | int thresh) |
1ca08976 | 93 | { |
d458b054 | 94 | struct __btrfs_workqueue *ret = kzalloc(sizeof(*ret), GFP_NOFS); |
1ca08976 | 95 | |
5d99a998 | 96 | if (!ret) |
1ca08976 QW |
97 | return NULL; |
98 | ||
0bd9289c QW |
99 | ret->max_active = max_active; |
100 | atomic_set(&ret->pending, 0); | |
101 | if (thresh == 0) | |
102 | thresh = DFT_THRESHOLD; | |
103 | /* For low threshold, disabling threshold is a better choice */ | |
104 | if (thresh < DFT_THRESHOLD) { | |
105 | ret->current_max = max_active; | |
106 | ret->thresh = NO_THRESHOLD; | |
107 | } else { | |
108 | ret->current_max = 1; | |
109 | ret->thresh = thresh; | |
110 | } | |
111 | ||
1ca08976 QW |
112 | if (flags & WQ_HIGHPRI) |
113 | ret->normal_wq = alloc_workqueue("%s-%s-high", flags, | |
0bd9289c QW |
114 | ret->max_active, |
115 | "btrfs", name); | |
1ca08976 QW |
116 | else |
117 | ret->normal_wq = alloc_workqueue("%s-%s", flags, | |
0bd9289c QW |
118 | ret->max_active, "btrfs", |
119 | name); | |
5d99a998 | 120 | if (!ret->normal_wq) { |
1ca08976 QW |
121 | kfree(ret); |
122 | return NULL; | |
123 | } | |
124 | ||
125 | INIT_LIST_HEAD(&ret->ordered_list); | |
126 | spin_lock_init(&ret->list_lock); | |
0bd9289c | 127 | spin_lock_init(&ret->thres_lock); |
c3a46891 | 128 | trace_btrfs_workqueue_alloc(ret, name, flags & WQ_HIGHPRI); |
1ca08976 QW |
129 | return ret; |
130 | } | |
131 | ||
132 | static inline void | |
d458b054 | 133 | __btrfs_destroy_workqueue(struct __btrfs_workqueue *wq); |
1ca08976 | 134 | |
c3a46891 | 135 | struct btrfs_workqueue *btrfs_alloc_workqueue(const char *name, |
6f011058 | 136 | unsigned int flags, |
d458b054 QW |
137 | int max_active, |
138 | int thresh) | |
08a9ff32 | 139 | { |
d458b054 | 140 | struct btrfs_workqueue *ret = kzalloc(sizeof(*ret), GFP_NOFS); |
08a9ff32 | 141 | |
5d99a998 | 142 | if (!ret) |
08a9ff32 QW |
143 | return NULL; |
144 | ||
1ca08976 | 145 | ret->normal = __btrfs_alloc_workqueue(name, flags & ~WQ_HIGHPRI, |
0bd9289c | 146 | max_active, thresh); |
5d99a998 | 147 | if (!ret->normal) { |
08a9ff32 QW |
148 | kfree(ret); |
149 | return NULL; | |
150 | } | |
151 | ||
1ca08976 | 152 | if (flags & WQ_HIGHPRI) { |
0bd9289c QW |
153 | ret->high = __btrfs_alloc_workqueue(name, flags, max_active, |
154 | thresh); | |
5d99a998 | 155 | if (!ret->high) { |
1ca08976 QW |
156 | __btrfs_destroy_workqueue(ret->normal); |
157 | kfree(ret); | |
158 | return NULL; | |
159 | } | |
160 | } | |
08a9ff32 QW |
161 | return ret; |
162 | } | |
163 | ||
0bd9289c QW |
164 | /* |
165 | * Hook for threshold which will be called in btrfs_queue_work. | |
166 | * This hook WILL be called in IRQ handler context, | |
167 | * so workqueue_set_max_active MUST NOT be called in this hook | |
168 | */ | |
d458b054 | 169 | static inline void thresh_queue_hook(struct __btrfs_workqueue *wq) |
0bd9289c QW |
170 | { |
171 | if (wq->thresh == NO_THRESHOLD) | |
172 | return; | |
173 | atomic_inc(&wq->pending); | |
174 | } | |
175 | ||
176 | /* | |
177 | * Hook for threshold which will be called before executing the work, | |
178 | * This hook is called in kthread content. | |
179 | * So workqueue_set_max_active is called here. | |
180 | */ | |
d458b054 | 181 | static inline void thresh_exec_hook(struct __btrfs_workqueue *wq) |
0bd9289c QW |
182 | { |
183 | int new_max_active; | |
184 | long pending; | |
185 | int need_change = 0; | |
186 | ||
187 | if (wq->thresh == NO_THRESHOLD) | |
188 | return; | |
189 | ||
190 | atomic_dec(&wq->pending); | |
191 | spin_lock(&wq->thres_lock); | |
192 | /* | |
193 | * Use wq->count to limit the calling frequency of | |
194 | * workqueue_set_max_active. | |
195 | */ | |
196 | wq->count++; | |
197 | wq->count %= (wq->thresh / 4); | |
198 | if (!wq->count) | |
199 | goto out; | |
200 | new_max_active = wq->current_max; | |
201 | ||
202 | /* | |
203 | * pending may be changed later, but it's OK since we really | |
204 | * don't need it so accurate to calculate new_max_active. | |
205 | */ | |
206 | pending = atomic_read(&wq->pending); | |
207 | if (pending > wq->thresh) | |
208 | new_max_active++; | |
209 | if (pending < wq->thresh / 2) | |
210 | new_max_active--; | |
211 | new_max_active = clamp_val(new_max_active, 1, wq->max_active); | |
212 | if (new_max_active != wq->current_max) { | |
213 | need_change = 1; | |
214 | wq->current_max = new_max_active; | |
215 | } | |
216 | out: | |
217 | spin_unlock(&wq->thres_lock); | |
218 | ||
219 | if (need_change) { | |
220 | workqueue_set_max_active(wq->normal_wq, wq->current_max); | |
221 | } | |
222 | } | |
223 | ||
d458b054 | 224 | static void run_ordered_work(struct __btrfs_workqueue *wq) |
08a9ff32 QW |
225 | { |
226 | struct list_head *list = &wq->ordered_list; | |
d458b054 | 227 | struct btrfs_work *work; |
08a9ff32 QW |
228 | spinlock_t *lock = &wq->list_lock; |
229 | unsigned long flags; | |
230 | ||
231 | while (1) { | |
232 | spin_lock_irqsave(lock, flags); | |
233 | if (list_empty(list)) | |
234 | break; | |
d458b054 | 235 | work = list_entry(list->next, struct btrfs_work, |
08a9ff32 QW |
236 | ordered_list); |
237 | if (!test_bit(WORK_DONE_BIT, &work->flags)) | |
238 | break; | |
239 | ||
240 | /* | |
241 | * we are going to call the ordered done function, but | |
242 | * we leave the work item on the list as a barrier so | |
243 | * that later work items that are done don't have their | |
244 | * functions called before this one returns | |
245 | */ | |
246 | if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags)) | |
247 | break; | |
52483bc2 | 248 | trace_btrfs_ordered_sched(work); |
08a9ff32 QW |
249 | spin_unlock_irqrestore(lock, flags); |
250 | work->ordered_func(work); | |
251 | ||
252 | /* now take the lock again and drop our item from the list */ | |
253 | spin_lock_irqsave(lock, flags); | |
254 | list_del(&work->ordered_list); | |
255 | spin_unlock_irqrestore(lock, flags); | |
256 | ||
257 | /* | |
258 | * we don't want to call the ordered free functions | |
259 | * with the lock held though | |
260 | */ | |
261 | work->ordered_free(work); | |
52483bc2 | 262 | trace_btrfs_all_work_done(work); |
08a9ff32 QW |
263 | } |
264 | spin_unlock_irqrestore(lock, flags); | |
265 | } | |
266 | ||
9e0af237 | 267 | static void normal_work_helper(struct btrfs_work *work) |
08a9ff32 | 268 | { |
d458b054 | 269 | struct __btrfs_workqueue *wq; |
08a9ff32 QW |
270 | int need_order = 0; |
271 | ||
08a9ff32 QW |
272 | /* |
273 | * We should not touch things inside work in the following cases: | |
274 | * 1) after work->func() if it has no ordered_free | |
275 | * Since the struct is freed in work->func(). | |
276 | * 2) after setting WORK_DONE_BIT | |
277 | * The work may be freed in other threads almost instantly. | |
278 | * So we save the needed things here. | |
279 | */ | |
280 | if (work->ordered_func) | |
281 | need_order = 1; | |
282 | wq = work->wq; | |
283 | ||
52483bc2 | 284 | trace_btrfs_work_sched(work); |
0bd9289c | 285 | thresh_exec_hook(wq); |
08a9ff32 QW |
286 | work->func(work); |
287 | if (need_order) { | |
288 | set_bit(WORK_DONE_BIT, &work->flags); | |
289 | run_ordered_work(wq); | |
290 | } | |
52483bc2 QW |
291 | if (!need_order) |
292 | trace_btrfs_all_work_done(work); | |
08a9ff32 QW |
293 | } |
294 | ||
9e0af237 | 295 | void btrfs_init_work(struct btrfs_work *work, btrfs_work_func_t uniq_func, |
6db8914f QW |
296 | btrfs_func_t func, |
297 | btrfs_func_t ordered_func, | |
298 | btrfs_func_t ordered_free) | |
08a9ff32 QW |
299 | { |
300 | work->func = func; | |
301 | work->ordered_func = ordered_func; | |
302 | work->ordered_free = ordered_free; | |
9e0af237 | 303 | INIT_WORK(&work->normal_work, uniq_func); |
08a9ff32 QW |
304 | INIT_LIST_HEAD(&work->ordered_list); |
305 | work->flags = 0; | |
306 | } | |
307 | ||
d458b054 QW |
308 | static inline void __btrfs_queue_work(struct __btrfs_workqueue *wq, |
309 | struct btrfs_work *work) | |
08a9ff32 QW |
310 | { |
311 | unsigned long flags; | |
312 | ||
313 | work->wq = wq; | |
0bd9289c | 314 | thresh_queue_hook(wq); |
08a9ff32 QW |
315 | if (work->ordered_func) { |
316 | spin_lock_irqsave(&wq->list_lock, flags); | |
317 | list_add_tail(&work->ordered_list, &wq->ordered_list); | |
318 | spin_unlock_irqrestore(&wq->list_lock, flags); | |
319 | } | |
320 | queue_work(wq->normal_wq, &work->normal_work); | |
52483bc2 | 321 | trace_btrfs_work_queued(work); |
08a9ff32 QW |
322 | } |
323 | ||
d458b054 QW |
324 | void btrfs_queue_work(struct btrfs_workqueue *wq, |
325 | struct btrfs_work *work) | |
1ca08976 | 326 | { |
d458b054 | 327 | struct __btrfs_workqueue *dest_wq; |
1ca08976 QW |
328 | |
329 | if (test_bit(WORK_HIGH_PRIO_BIT, &work->flags) && wq->high) | |
330 | dest_wq = wq->high; | |
331 | else | |
332 | dest_wq = wq->normal; | |
333 | __btrfs_queue_work(dest_wq, work); | |
334 | } | |
335 | ||
336 | static inline void | |
d458b054 | 337 | __btrfs_destroy_workqueue(struct __btrfs_workqueue *wq) |
08a9ff32 QW |
338 | { |
339 | destroy_workqueue(wq->normal_wq); | |
c3a46891 | 340 | trace_btrfs_workqueue_destroy(wq); |
08a9ff32 QW |
341 | kfree(wq); |
342 | } | |
343 | ||
d458b054 | 344 | void btrfs_destroy_workqueue(struct btrfs_workqueue *wq) |
1ca08976 QW |
345 | { |
346 | if (!wq) | |
347 | return; | |
348 | if (wq->high) | |
349 | __btrfs_destroy_workqueue(wq->high); | |
350 | __btrfs_destroy_workqueue(wq->normal); | |
ef66af10 | 351 | kfree(wq); |
1ca08976 QW |
352 | } |
353 | ||
d458b054 | 354 | void btrfs_workqueue_set_max(struct btrfs_workqueue *wq, int max) |
08a9ff32 | 355 | { |
800ee224 ST |
356 | if (!wq) |
357 | return; | |
0bd9289c | 358 | wq->normal->max_active = max; |
1ca08976 | 359 | if (wq->high) |
0bd9289c | 360 | wq->high->max_active = max; |
1ca08976 QW |
361 | } |
362 | ||
d458b054 | 363 | void btrfs_set_work_high_priority(struct btrfs_work *work) |
1ca08976 QW |
364 | { |
365 | set_bit(WORK_HIGH_PRIO_BIT, &work->flags); | |
08a9ff32 | 366 | } |