Commit | Line | Data |
---|---|---|
b0643e59 DZ |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | #include <linux/jiffies.h> | |
4 | #include <linux/kernel.h> | |
5 | #include <linux/ktime.h> | |
6 | #include <linux/list.h> | |
e93591bb | 7 | #include <linux/math64.h> |
b0643e59 DZ |
8 | #include <linux/sizes.h> |
9 | #include <linux/workqueue.h> | |
10 | #include "ctree.h" | |
11 | #include "block-group.h" | |
12 | #include "discard.h" | |
13 | #include "free-space-cache.h" | |
fc97a410 | 14 | #include "fs.h" |
b0643e59 | 15 | |
dbc2a8c9 DZ |
16 | /* |
17 | * This contains the logic to handle async discard. | |
18 | * | |
19 | * Async discard manages trimming of free space outside of transaction commit. | |
20 | * Discarding is done by managing the block_groups on a LRU list based on free | |
21 | * space recency. Two passes are used to first prioritize discarding extents | |
22 | * and then allow for trimming in the bitmap the best opportunity to coalesce. | |
23 | * The block_groups are maintained on multiple lists to allow for multiple | |
24 | * passes with different discard filter requirements. A delayed work item is | |
25 | * used to manage discarding with timeout determined by a max of the delay | |
26 | * incurred by the iops rate limit, the byte rate limit, and the max delay of | |
27 | * BTRFS_DISCARD_MAX_DELAY. | |
28 | * | |
29 | * Note, this only keeps track of block_groups that are explicitly for data. | |
30 | * Mixed block_groups are not supported. | |
31 | * | |
32 | * The first list is special to manage discarding of fully free block groups. | |
33 | * This is necessary because we issue a final trim for a full free block group | |
34 | * after forgetting it. When a block group becomes unused, instead of directly | |
35 | * being added to the unused_bgs list, we add it to this first list. Then | |
36 | * from there, if it becomes fully discarded, we place it onto the unused_bgs | |
37 | * list. | |
38 | * | |
39 | * The in-memory free space cache serves as the backing state for discard. | |
40 | * Consequently this means there is no persistence. We opt to load all the | |
41 | * block groups in as not discarded, so the mount case degenerates to the | |
42 | * crashing case. | |
43 | * | |
44 | * As the free space cache uses bitmaps, there exists a tradeoff between | |
45 | * ease/efficiency for find_free_extent() and the accuracy of discard state. | |
46 | * Here we opt to let untrimmed regions merge with everything while only letting | |
47 | * trimmed regions merge with other trimmed regions. This can cause | |
48 | * overtrimming, but the coalescing benefit seems to be worth it. Additionally, | |
49 | * bitmap state is tracked as a whole. If we're able to fully trim a bitmap, | |
50 | * the trimmed flag is set on the bitmap. Otherwise, if an allocation comes in, | |
51 | * this resets the state and we will retry trimming the whole bitmap. This is a | |
52 | * tradeoff between discard state accuracy and the cost of accounting. | |
53 | */ | |
54 | ||
b0643e59 DZ |
55 | /* This is an initial delay to give some chance for block reuse */ |
56 | #define BTRFS_DISCARD_DELAY (120ULL * NSEC_PER_SEC) | |
6e80d4f8 | 57 | #define BTRFS_DISCARD_UNUSED_DELAY (10ULL * NSEC_PER_SEC) |
b0643e59 | 58 | |
a2309300 DZ |
59 | #define BTRFS_DISCARD_MIN_DELAY_MSEC (1UL) |
60 | #define BTRFS_DISCARD_MAX_DELAY_MSEC (1000UL) | |
e9f59429 | 61 | #define BTRFS_DISCARD_MAX_IOPS (1000U) |
a2309300 | 62 | |
43dd529a | 63 | /* Monotonically decreasing minimum length filters after index 0 */ |
7fe6d45e DZ |
64 | static int discard_minlen[BTRFS_NR_DISCARD_LISTS] = { |
65 | 0, | |
66 | BTRFS_ASYNC_DISCARD_MAX_FILTER, | |
67 | BTRFS_ASYNC_DISCARD_MIN_FILTER | |
68 | }; | |
69 | ||
b0643e59 DZ |
70 | static struct list_head *get_discard_list(struct btrfs_discard_ctl *discard_ctl, |
71 | struct btrfs_block_group *block_group) | |
72 | { | |
73 | return &discard_ctl->discard_list[block_group->discard_index]; | |
74 | } | |
75 | ||
bb5167e6 JT |
76 | /* |
77 | * Determine if async discard should be running. | |
78 | * | |
79 | * @discard_ctl: discard control | |
80 | * | |
81 | * Check if the file system is writeable and BTRFS_FS_DISCARD_RUNNING is set. | |
82 | */ | |
83 | static bool btrfs_run_discard_work(struct btrfs_discard_ctl *discard_ctl) | |
84 | { | |
85 | struct btrfs_fs_info *fs_info = container_of(discard_ctl, | |
86 | struct btrfs_fs_info, | |
87 | discard_ctl); | |
88 | ||
89 | return (!(fs_info->sb->s_flags & SB_RDONLY) && | |
90 | test_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags)); | |
91 | } | |
92 | ||
2bee7eb8 DZ |
93 | static void __add_to_discard_list(struct btrfs_discard_ctl *discard_ctl, |
94 | struct btrfs_block_group *block_group) | |
b0643e59 | 95 | { |
2b5463fc | 96 | lockdep_assert_held(&discard_ctl->lock); |
2bee7eb8 | 97 | if (!btrfs_run_discard_work(discard_ctl)) |
b0643e59 | 98 | return; |
b0643e59 | 99 | |
6e80d4f8 DZ |
100 | if (list_empty(&block_group->discard_list) || |
101 | block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED) { | |
102 | if (block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED) | |
103 | block_group->discard_index = BTRFS_DISCARD_INDEX_START; | |
b0643e59 DZ |
104 | block_group->discard_eligible_time = (ktime_get_ns() + |
105 | BTRFS_DISCARD_DELAY); | |
2bee7eb8 | 106 | block_group->discard_state = BTRFS_DISCARD_RESET_CURSOR; |
6e80d4f8 | 107 | } |
2b5463fc BB |
108 | if (list_empty(&block_group->discard_list)) |
109 | btrfs_get_block_group(block_group); | |
b0643e59 DZ |
110 | |
111 | list_move_tail(&block_group->discard_list, | |
112 | get_discard_list(discard_ctl, block_group)); | |
2bee7eb8 | 113 | } |
b0643e59 | 114 | |
2bee7eb8 DZ |
115 | static void add_to_discard_list(struct btrfs_discard_ctl *discard_ctl, |
116 | struct btrfs_block_group *block_group) | |
117 | { | |
5cb0724e DZ |
118 | if (!btrfs_is_block_group_data_only(block_group)) |
119 | return; | |
120 | ||
2bee7eb8 DZ |
121 | spin_lock(&discard_ctl->lock); |
122 | __add_to_discard_list(discard_ctl, block_group); | |
b0643e59 DZ |
123 | spin_unlock(&discard_ctl->lock); |
124 | } | |
125 | ||
6e80d4f8 DZ |
126 | static void add_to_discard_unused_list(struct btrfs_discard_ctl *discard_ctl, |
127 | struct btrfs_block_group *block_group) | |
128 | { | |
2b5463fc BB |
129 | bool queued; |
130 | ||
6e80d4f8 DZ |
131 | spin_lock(&discard_ctl->lock); |
132 | ||
2b5463fc BB |
133 | queued = !list_empty(&block_group->discard_list); |
134 | ||
6e80d4f8 DZ |
135 | if (!btrfs_run_discard_work(discard_ctl)) { |
136 | spin_unlock(&discard_ctl->lock); | |
137 | return; | |
138 | } | |
139 | ||
140 | list_del_init(&block_group->discard_list); | |
141 | ||
142 | block_group->discard_index = BTRFS_DISCARD_INDEX_UNUSED; | |
143 | block_group->discard_eligible_time = (ktime_get_ns() + | |
144 | BTRFS_DISCARD_UNUSED_DELAY); | |
2bee7eb8 | 145 | block_group->discard_state = BTRFS_DISCARD_RESET_CURSOR; |
2b5463fc BB |
146 | if (!queued) |
147 | btrfs_get_block_group(block_group); | |
6e80d4f8 DZ |
148 | list_add_tail(&block_group->discard_list, |
149 | &discard_ctl->discard_list[BTRFS_DISCARD_INDEX_UNUSED]); | |
150 | ||
151 | spin_unlock(&discard_ctl->lock); | |
152 | } | |
153 | ||
b0643e59 DZ |
154 | static bool remove_from_discard_list(struct btrfs_discard_ctl *discard_ctl, |
155 | struct btrfs_block_group *block_group) | |
156 | { | |
157 | bool running = false; | |
2b5463fc | 158 | bool queued = false; |
b0643e59 DZ |
159 | |
160 | spin_lock(&discard_ctl->lock); | |
161 | ||
162 | if (block_group == discard_ctl->block_group) { | |
163 | running = true; | |
164 | discard_ctl->block_group = NULL; | |
165 | } | |
166 | ||
167 | block_group->discard_eligible_time = 0; | |
2b5463fc | 168 | queued = !list_empty(&block_group->discard_list); |
b0643e59 | 169 | list_del_init(&block_group->discard_list); |
2b5463fc BB |
170 | /* |
171 | * If the block group is currently running in the discard workfn, we | |
172 | * don't want to deref it, since it's still being used by the workfn. | |
173 | * The workfn will notice this case and deref the block group when it is | |
174 | * finished. | |
175 | */ | |
176 | if (queued && !running) | |
177 | btrfs_put_block_group(block_group); | |
b0643e59 DZ |
178 | |
179 | spin_unlock(&discard_ctl->lock); | |
180 | ||
181 | return running; | |
182 | } | |
183 | ||
43dd529a DS |
184 | /* |
185 | * Find block_group that's up next for discarding. | |
186 | * | |
187 | * @discard_ctl: discard control | |
188 | * @now: current time | |
b0643e59 DZ |
189 | * |
190 | * Iterate over the discard lists to find the next block_group up for | |
191 | * discarding checking the discard_eligible_time of block_group. | |
192 | */ | |
193 | static struct btrfs_block_group *find_next_block_group( | |
194 | struct btrfs_discard_ctl *discard_ctl, | |
195 | u64 now) | |
196 | { | |
197 | struct btrfs_block_group *ret_block_group = NULL, *block_group; | |
198 | int i; | |
199 | ||
200 | for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++) { | |
201 | struct list_head *discard_list = &discard_ctl->discard_list[i]; | |
202 | ||
203 | if (!list_empty(discard_list)) { | |
204 | block_group = list_first_entry(discard_list, | |
205 | struct btrfs_block_group, | |
206 | discard_list); | |
207 | ||
208 | if (!ret_block_group) | |
209 | ret_block_group = block_group; | |
210 | ||
211 | if (ret_block_group->discard_eligible_time < now) | |
212 | break; | |
213 | ||
214 | if (ret_block_group->discard_eligible_time > | |
215 | block_group->discard_eligible_time) | |
216 | ret_block_group = block_group; | |
217 | } | |
218 | } | |
219 | ||
220 | return ret_block_group; | |
221 | } | |
222 | ||
43dd529a DS |
223 | /* |
224 | * Look up next block group and set it for use. | |
92419695 NB |
225 | * |
226 | * @discard_ctl: discard control | |
2bee7eb8 | 227 | * @discard_state: the discard_state of the block_group after state management |
7fe6d45e | 228 | * @discard_index: the discard_index of the block_group after state management |
92419695 | 229 | * @now: time when discard was invoked, in ns |
b0643e59 | 230 | * |
43dd529a DS |
231 | * Wrap find_next_block_group() and set the block_group to be in use. |
232 | * @discard_state's control flow is managed here. Variables related to | |
233 | * @discard_state are reset here as needed (eg. @discard_cursor). @discard_state | |
7fe6d45e DZ |
234 | * and @discard_index are remembered as it may change while we're discarding, |
235 | * but we want the discard to execute in the context determined here. | |
b0643e59 DZ |
236 | */ |
237 | static struct btrfs_block_group *peek_discard_list( | |
2bee7eb8 | 238 | struct btrfs_discard_ctl *discard_ctl, |
7fe6d45e | 239 | enum btrfs_discard_state *discard_state, |
ea9ed87c | 240 | int *discard_index, u64 now) |
b0643e59 DZ |
241 | { |
242 | struct btrfs_block_group *block_group; | |
b0643e59 DZ |
243 | |
244 | spin_lock(&discard_ctl->lock); | |
2bee7eb8 | 245 | again: |
b0643e59 DZ |
246 | block_group = find_next_block_group(discard_ctl, now); |
247 | ||
ea9ed87c | 248 | if (block_group && now >= block_group->discard_eligible_time) { |
2bee7eb8 DZ |
249 | if (block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED && |
250 | block_group->used != 0) { | |
2b5463fc | 251 | if (btrfs_is_block_group_data_only(block_group)) { |
5cb0724e | 252 | __add_to_discard_list(discard_ctl, block_group); |
2b5463fc | 253 | } else { |
5cb0724e | 254 | list_del_init(&block_group->discard_list); |
2b5463fc BB |
255 | btrfs_put_block_group(block_group); |
256 | } | |
2bee7eb8 DZ |
257 | goto again; |
258 | } | |
259 | if (block_group->discard_state == BTRFS_DISCARD_RESET_CURSOR) { | |
260 | block_group->discard_cursor = block_group->start; | |
261 | block_group->discard_state = BTRFS_DISCARD_EXTENTS; | |
262 | } | |
263 | discard_ctl->block_group = block_group; | |
ea9ed87c PB |
264 | } |
265 | if (block_group) { | |
2bee7eb8 | 266 | *discard_state = block_group->discard_state; |
7fe6d45e | 267 | *discard_index = block_group->discard_index; |
2bee7eb8 | 268 | } |
b0643e59 DZ |
269 | spin_unlock(&discard_ctl->lock); |
270 | ||
271 | return block_group; | |
272 | } | |
273 | ||
43dd529a DS |
274 | /* |
275 | * Update a block group's filters. | |
276 | * | |
277 | * @block_group: block group of interest | |
278 | * @bytes: recently freed region size after coalescing | |
7fe6d45e DZ |
279 | * |
280 | * Async discard maintains multiple lists with progressively smaller filters | |
281 | * to prioritize discarding based on size. Should a free space that matches | |
282 | * a larger filter be returned to the free_space_cache, prioritize that discard | |
283 | * by moving @block_group to the proper filter. | |
284 | */ | |
285 | void btrfs_discard_check_filter(struct btrfs_block_group *block_group, | |
286 | u64 bytes) | |
287 | { | |
288 | struct btrfs_discard_ctl *discard_ctl; | |
289 | ||
290 | if (!block_group || | |
291 | !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC)) | |
292 | return; | |
293 | ||
294 | discard_ctl = &block_group->fs_info->discard_ctl; | |
295 | ||
296 | if (block_group->discard_index > BTRFS_DISCARD_INDEX_START && | |
297 | bytes >= discard_minlen[block_group->discard_index - 1]) { | |
298 | int i; | |
299 | ||
300 | remove_from_discard_list(discard_ctl, block_group); | |
301 | ||
302 | for (i = BTRFS_DISCARD_INDEX_START; i < BTRFS_NR_DISCARD_LISTS; | |
303 | i++) { | |
304 | if (bytes >= discard_minlen[i]) { | |
305 | block_group->discard_index = i; | |
306 | add_to_discard_list(discard_ctl, block_group); | |
307 | break; | |
308 | } | |
309 | } | |
310 | } | |
311 | } | |
312 | ||
43dd529a DS |
313 | /* |
314 | * Move a block group along the discard lists. | |
315 | * | |
7fe6d45e DZ |
316 | * @discard_ctl: discard control |
317 | * @block_group: block_group of interest | |
318 | * | |
319 | * Increment @block_group's discard_index. If it falls of the list, let it be. | |
320 | * Otherwise add it back to the appropriate list. | |
321 | */ | |
322 | static void btrfs_update_discard_index(struct btrfs_discard_ctl *discard_ctl, | |
323 | struct btrfs_block_group *block_group) | |
324 | { | |
325 | block_group->discard_index++; | |
326 | if (block_group->discard_index == BTRFS_NR_DISCARD_LISTS) { | |
327 | block_group->discard_index = 1; | |
328 | return; | |
329 | } | |
330 | ||
331 | add_to_discard_list(discard_ctl, block_group); | |
332 | } | |
333 | ||
43dd529a DS |
334 | /* |
335 | * Remove a block_group from the discard lists. | |
336 | * | |
b0643e59 DZ |
337 | * @discard_ctl: discard control |
338 | * @block_group: block_group of interest | |
339 | * | |
43dd529a DS |
340 | * Remove @block_group from the discard lists. If necessary, wait on the |
341 | * current work and then reschedule the delayed work. | |
b0643e59 DZ |
342 | */ |
343 | void btrfs_discard_cancel_work(struct btrfs_discard_ctl *discard_ctl, | |
344 | struct btrfs_block_group *block_group) | |
345 | { | |
346 | if (remove_from_discard_list(discard_ctl, block_group)) { | |
347 | cancel_delayed_work_sync(&discard_ctl->work); | |
348 | btrfs_discard_schedule_work(discard_ctl, true); | |
349 | } | |
350 | } | |
351 | ||
43dd529a DS |
352 | /* |
353 | * Handles queuing the block_groups. | |
354 | * | |
b0643e59 DZ |
355 | * @discard_ctl: discard control |
356 | * @block_group: block_group of interest | |
357 | * | |
43dd529a | 358 | * Maintain the LRU order of the discard lists. |
b0643e59 DZ |
359 | */ |
360 | void btrfs_discard_queue_work(struct btrfs_discard_ctl *discard_ctl, | |
361 | struct btrfs_block_group *block_group) | |
362 | { | |
363 | if (!block_group || !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC)) | |
364 | return; | |
365 | ||
6e80d4f8 DZ |
366 | if (block_group->used == 0) |
367 | add_to_discard_unused_list(discard_ctl, block_group); | |
368 | else | |
369 | add_to_discard_list(discard_ctl, block_group); | |
b0643e59 DZ |
370 | |
371 | if (!delayed_work_pending(&discard_ctl->work)) | |
372 | btrfs_discard_schedule_work(discard_ctl, false); | |
373 | } | |
374 | ||
8fc05859 PB |
375 | static void __btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, |
376 | u64 now, bool override) | |
b0643e59 DZ |
377 | { |
378 | struct btrfs_block_group *block_group; | |
b0643e59 DZ |
379 | |
380 | if (!btrfs_run_discard_work(discard_ctl)) | |
8fc05859 | 381 | return; |
b0643e59 | 382 | if (!override && delayed_work_pending(&discard_ctl->work)) |
8fc05859 | 383 | return; |
b0643e59 DZ |
384 | |
385 | block_group = find_next_block_group(discard_ctl, now); | |
386 | if (block_group) { | |
6e88f116 | 387 | u64 delay = discard_ctl->delay_ms * NSEC_PER_MSEC; |
e93591bb DZ |
388 | u32 kbps_limit = READ_ONCE(discard_ctl->kbps_limit); |
389 | ||
390 | /* | |
391 | * A single delayed workqueue item is responsible for | |
392 | * discarding, so we can manage the bytes rate limit by keeping | |
393 | * track of the previous discard. | |
394 | */ | |
395 | if (kbps_limit && discard_ctl->prev_discard) { | |
396 | u64 bps_limit = ((u64)kbps_limit) * SZ_1K; | |
397 | u64 bps_delay = div64_u64(discard_ctl->prev_discard * | |
6e88f116 | 398 | NSEC_PER_SEC, bps_limit); |
e93591bb | 399 | |
6e88f116 | 400 | delay = max(delay, bps_delay); |
e93591bb | 401 | } |
a2309300 DZ |
402 | |
403 | /* | |
404 | * This timeout is to hopefully prevent immediate discarding | |
405 | * in a recently allocated block group. | |
406 | */ | |
407 | if (now < block_group->discard_eligible_time) { | |
408 | u64 bg_timeout = block_group->discard_eligible_time - now; | |
b0643e59 | 409 | |
6e88f116 | 410 | delay = max(delay, bg_timeout); |
a2309300 | 411 | } |
b0643e59 | 412 | |
df903e5d PB |
413 | if (override && discard_ctl->prev_discard) { |
414 | u64 elapsed = now - discard_ctl->prev_discard_time; | |
415 | ||
416 | if (delay > elapsed) | |
417 | delay -= elapsed; | |
418 | else | |
419 | delay = 0; | |
420 | } | |
421 | ||
b0643e59 | 422 | mod_delayed_work(discard_ctl->discard_workers, |
6e88f116 | 423 | &discard_ctl->work, nsecs_to_jiffies(delay)); |
b0643e59 | 424 | } |
8fc05859 PB |
425 | } |
426 | ||
427 | /* | |
43dd529a DS |
428 | * Responsible for scheduling the discard work. |
429 | * | |
8fc05859 PB |
430 | * @discard_ctl: discard control |
431 | * @override: override the current timer | |
432 | * | |
433 | * Discards are issued by a delayed workqueue item. @override is used to | |
434 | * update the current delay as the baseline delay interval is reevaluated on | |
435 | * transaction commit. This is also maxed with any other rate limit. | |
436 | */ | |
437 | void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, | |
438 | bool override) | |
439 | { | |
440 | const u64 now = ktime_get_ns(); | |
441 | ||
442 | spin_lock(&discard_ctl->lock); | |
443 | __btrfs_discard_schedule_work(discard_ctl, now, override); | |
b0643e59 DZ |
444 | spin_unlock(&discard_ctl->lock); |
445 | } | |
446 | ||
43dd529a DS |
447 | /* |
448 | * Determine next step of a block_group. | |
449 | * | |
6e80d4f8 DZ |
450 | * @discard_ctl: discard control |
451 | * @block_group: block_group of interest | |
452 | * | |
43dd529a DS |
453 | * Determine the next step for a block group after it's finished going through |
454 | * a pass on a discard list. If it is unused and fully trimmed, we can mark it | |
455 | * unused and send it to the unused_bgs path. Otherwise, pass it onto the | |
456 | * appropriate filter list or let it fall off. | |
6e80d4f8 DZ |
457 | */ |
458 | static void btrfs_finish_discard_pass(struct btrfs_discard_ctl *discard_ctl, | |
459 | struct btrfs_block_group *block_group) | |
460 | { | |
461 | remove_from_discard_list(discard_ctl, block_group); | |
462 | ||
463 | if (block_group->used == 0) { | |
464 | if (btrfs_is_free_space_trimmed(block_group)) | |
465 | btrfs_mark_bg_unused(block_group); | |
466 | else | |
467 | add_to_discard_unused_list(discard_ctl, block_group); | |
7fe6d45e DZ |
468 | } else { |
469 | btrfs_update_discard_index(discard_ctl, block_group); | |
6e80d4f8 DZ |
470 | } |
471 | } | |
472 | ||
43dd529a DS |
473 | /* |
474 | * Discard work queue callback | |
475 | * | |
b0643e59 DZ |
476 | * @work: work |
477 | * | |
43dd529a DS |
478 | * Find the next block_group to start discarding and then discard a single |
479 | * region. It does this in a two-pass fashion: first extents and second | |
2bee7eb8 | 480 | * bitmaps. Completely discarded block groups are sent to the unused_bgs path. |
b0643e59 DZ |
481 | */ |
482 | static void btrfs_discard_workfn(struct work_struct *work) | |
483 | { | |
484 | struct btrfs_discard_ctl *discard_ctl; | |
485 | struct btrfs_block_group *block_group; | |
2bee7eb8 | 486 | enum btrfs_discard_state discard_state; |
7fe6d45e | 487 | int discard_index = 0; |
b0643e59 | 488 | u64 trimmed = 0; |
7fe6d45e | 489 | u64 minlen = 0; |
ea9ed87c | 490 | u64 now = ktime_get_ns(); |
b0643e59 DZ |
491 | |
492 | discard_ctl = container_of(work, struct btrfs_discard_ctl, work.work); | |
493 | ||
7fe6d45e | 494 | block_group = peek_discard_list(discard_ctl, &discard_state, |
ea9ed87c | 495 | &discard_index, now); |
b0643e59 DZ |
496 | if (!block_group || !btrfs_run_discard_work(discard_ctl)) |
497 | return; | |
ea9ed87c PB |
498 | if (now < block_group->discard_eligible_time) { |
499 | btrfs_discard_schedule_work(discard_ctl, false); | |
500 | return; | |
501 | } | |
b0643e59 | 502 | |
2bee7eb8 | 503 | /* Perform discarding */ |
7fe6d45e DZ |
504 | minlen = discard_minlen[discard_index]; |
505 | ||
506 | if (discard_state == BTRFS_DISCARD_BITMAPS) { | |
507 | u64 maxlen = 0; | |
508 | ||
509 | /* | |
510 | * Use the previous levels minimum discard length as the max | |
511 | * length filter. In the case something is added to make a | |
512 | * region go beyond the max filter, the entire bitmap is set | |
513 | * back to BTRFS_TRIM_STATE_UNTRIMMED. | |
514 | */ | |
515 | if (discard_index != BTRFS_DISCARD_INDEX_UNUSED) | |
516 | maxlen = discard_minlen[discard_index - 1]; | |
517 | ||
2bee7eb8 DZ |
518 | btrfs_trim_block_group_bitmaps(block_group, &trimmed, |
519 | block_group->discard_cursor, | |
520 | btrfs_block_group_end(block_group), | |
7fe6d45e | 521 | minlen, maxlen, true); |
9ddf648f | 522 | discard_ctl->discard_bitmap_bytes += trimmed; |
7fe6d45e | 523 | } else { |
2bee7eb8 DZ |
524 | btrfs_trim_block_group_extents(block_group, &trimmed, |
525 | block_group->discard_cursor, | |
526 | btrfs_block_group_end(block_group), | |
7fe6d45e | 527 | minlen, true); |
9ddf648f | 528 | discard_ctl->discard_extent_bytes += trimmed; |
7fe6d45e | 529 | } |
2bee7eb8 DZ |
530 | |
531 | /* Determine next steps for a block_group */ | |
532 | if (block_group->discard_cursor >= btrfs_block_group_end(block_group)) { | |
533 | if (discard_state == BTRFS_DISCARD_BITMAPS) { | |
534 | btrfs_finish_discard_pass(discard_ctl, block_group); | |
535 | } else { | |
536 | block_group->discard_cursor = block_group->start; | |
537 | spin_lock(&discard_ctl->lock); | |
538 | if (block_group->discard_state != | |
539 | BTRFS_DISCARD_RESET_CURSOR) | |
540 | block_group->discard_state = | |
541 | BTRFS_DISCARD_BITMAPS; | |
542 | spin_unlock(&discard_ctl->lock); | |
543 | } | |
544 | } | |
545 | ||
1ea2872f | 546 | now = ktime_get_ns(); |
2bee7eb8 | 547 | spin_lock(&discard_ctl->lock); |
1ea2872f PB |
548 | discard_ctl->prev_discard = trimmed; |
549 | discard_ctl->prev_discard_time = now; | |
2b5463fc BB |
550 | /* |
551 | * If the block group was removed from the discard list while it was | |
552 | * running in this workfn, then we didn't deref it, since this function | |
553 | * still owned that reference. But we set the discard_ctl->block_group | |
554 | * back to NULL, so we can use that condition to know that now we need | |
555 | * to deref the block_group. | |
556 | */ | |
557 | if (discard_ctl->block_group == NULL) | |
558 | btrfs_put_block_group(block_group); | |
2bee7eb8 | 559 | discard_ctl->block_group = NULL; |
8fc05859 | 560 | __btrfs_discard_schedule_work(discard_ctl, now, false); |
2bee7eb8 | 561 | spin_unlock(&discard_ctl->lock); |
b0643e59 DZ |
562 | } |
563 | ||
43dd529a DS |
564 | /* |
565 | * Recalculate the base delay. | |
566 | * | |
a2309300 DZ |
567 | * @discard_ctl: discard control |
568 | * | |
569 | * Recalculate the base delay which is based off the total number of | |
570 | * discardable_extents. Clamp this between the lower_limit (iops_limit or 1ms) | |
571 | * and the upper_limit (BTRFS_DISCARD_MAX_DELAY_MSEC). | |
572 | */ | |
573 | void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) | |
574 | { | |
575 | s32 discardable_extents; | |
81b29a3b | 576 | s64 discardable_bytes; |
a2309300 | 577 | u32 iops_limit; |
ef9cddfe | 578 | unsigned long min_delay = BTRFS_DISCARD_MIN_DELAY_MSEC; |
a2309300 | 579 | unsigned long delay; |
a2309300 DZ |
580 | |
581 | discardable_extents = atomic_read(&discard_ctl->discardable_extents); | |
582 | if (!discardable_extents) | |
583 | return; | |
584 | ||
585 | spin_lock(&discard_ctl->lock); | |
586 | ||
81b29a3b | 587 | /* |
43dd529a | 588 | * The following is to fix a potential -1 discrepancy that we're not |
81b29a3b DZ |
589 | * sure how to reproduce. But given that this is the only place that |
590 | * utilizes these numbers and this is only called by from | |
591 | * btrfs_finish_extent_commit() which is synchronized, we can correct | |
592 | * here. | |
593 | */ | |
594 | if (discardable_extents < 0) | |
595 | atomic_add(-discardable_extents, | |
596 | &discard_ctl->discardable_extents); | |
597 | ||
598 | discardable_bytes = atomic64_read(&discard_ctl->discardable_bytes); | |
599 | if (discardable_bytes < 0) | |
600 | atomic64_add(-discardable_bytes, | |
601 | &discard_ctl->discardable_bytes); | |
602 | ||
603 | if (discardable_extents <= 0) { | |
604 | spin_unlock(&discard_ctl->lock); | |
605 | return; | |
606 | } | |
607 | ||
a2309300 | 608 | iops_limit = READ_ONCE(discard_ctl->iops_limit); |
ef9cddfe BB |
609 | |
610 | if (iops_limit) { | |
e50404a8 | 611 | delay = MSEC_PER_SEC / iops_limit; |
ef9cddfe BB |
612 | } else { |
613 | /* | |
614 | * Unset iops_limit means go as fast as possible, so allow a | |
615 | * delay of 0. | |
616 | */ | |
617 | delay = 0; | |
618 | min_delay = 0; | |
619 | } | |
a2309300 | 620 | |
ef9cddfe | 621 | delay = clamp(delay, min_delay, BTRFS_DISCARD_MAX_DELAY_MSEC); |
6e88f116 | 622 | discard_ctl->delay_ms = delay; |
a2309300 DZ |
623 | |
624 | spin_unlock(&discard_ctl->lock); | |
625 | } | |
626 | ||
43dd529a DS |
627 | /* |
628 | * Propagate discard counters. | |
629 | * | |
dfb79ddb | 630 | * @block_group: block_group of interest |
dfb79ddb | 631 | * |
43dd529a DS |
632 | * Propagate deltas of counters up to the discard_ctl. It maintains a current |
633 | * counter and a previous counter passing the delta up to the global stat. | |
634 | * Then the current counter value becomes the previous counter value. | |
dfb79ddb | 635 | */ |
66b53bae | 636 | void btrfs_discard_update_discardable(struct btrfs_block_group *block_group) |
dfb79ddb | 637 | { |
66b53bae | 638 | struct btrfs_free_space_ctl *ctl; |
dfb79ddb DZ |
639 | struct btrfs_discard_ctl *discard_ctl; |
640 | s32 extents_delta; | |
5dc7c10b | 641 | s64 bytes_delta; |
dfb79ddb | 642 | |
5cb0724e DZ |
643 | if (!block_group || |
644 | !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC) || | |
645 | !btrfs_is_block_group_data_only(block_group)) | |
dfb79ddb DZ |
646 | return; |
647 | ||
66b53bae | 648 | ctl = block_group->free_space_ctl; |
dfb79ddb DZ |
649 | discard_ctl = &block_group->fs_info->discard_ctl; |
650 | ||
66b53bae | 651 | lockdep_assert_held(&ctl->tree_lock); |
dfb79ddb DZ |
652 | extents_delta = ctl->discardable_extents[BTRFS_STAT_CURR] - |
653 | ctl->discardable_extents[BTRFS_STAT_PREV]; | |
654 | if (extents_delta) { | |
655 | atomic_add(extents_delta, &discard_ctl->discardable_extents); | |
656 | ctl->discardable_extents[BTRFS_STAT_PREV] = | |
657 | ctl->discardable_extents[BTRFS_STAT_CURR]; | |
658 | } | |
5dc7c10b DZ |
659 | |
660 | bytes_delta = ctl->discardable_bytes[BTRFS_STAT_CURR] - | |
661 | ctl->discardable_bytes[BTRFS_STAT_PREV]; | |
662 | if (bytes_delta) { | |
663 | atomic64_add(bytes_delta, &discard_ctl->discardable_bytes); | |
664 | ctl->discardable_bytes[BTRFS_STAT_PREV] = | |
665 | ctl->discardable_bytes[BTRFS_STAT_CURR]; | |
666 | } | |
dfb79ddb DZ |
667 | } |
668 | ||
43dd529a DS |
669 | /* |
670 | * Punt unused_bgs list to discard lists. | |
671 | * | |
6e80d4f8 DZ |
672 | * @fs_info: fs_info of interest |
673 | * | |
674 | * The unused_bgs list needs to be punted to the discard lists because the | |
1a9fd417 | 675 | * order of operations is changed. In the normal synchronous discard path, the |
6e80d4f8 DZ |
676 | * block groups are trimmed via a single large trim in transaction commit. This |
677 | * is ultimately what we are trying to avoid with asynchronous discard. Thus, | |
678 | * it must be done before going down the unused_bgs path. | |
679 | */ | |
680 | void btrfs_discard_punt_unused_bgs_list(struct btrfs_fs_info *fs_info) | |
681 | { | |
682 | struct btrfs_block_group *block_group, *next; | |
683 | ||
684 | spin_lock(&fs_info->unused_bgs_lock); | |
685 | /* We enabled async discard, so punt all to the queue */ | |
686 | list_for_each_entry_safe(block_group, next, &fs_info->unused_bgs, | |
687 | bg_list) { | |
688 | list_del_init(&block_group->bg_list); | |
689 | btrfs_discard_queue_work(&fs_info->discard_ctl, block_group); | |
2b5463fc BB |
690 | /* |
691 | * This put is for the get done by btrfs_mark_bg_unused. | |
692 | * Queueing discard incremented it for discard's reference. | |
693 | */ | |
694 | btrfs_put_block_group(block_group); | |
6e80d4f8 DZ |
695 | } |
696 | spin_unlock(&fs_info->unused_bgs_lock); | |
697 | } | |
698 | ||
43dd529a DS |
699 | /* |
700 | * Purge discard lists. | |
701 | * | |
6e80d4f8 DZ |
702 | * @discard_ctl: discard control |
703 | * | |
704 | * If we are disabling async discard, we may have intercepted block groups that | |
705 | * are completely free and ready for the unused_bgs path. As discarding will | |
706 | * now happen in transaction commit or not at all, we can safely mark the | |
707 | * corresponding block groups as unused and they will be sent on their merry | |
708 | * way to the unused_bgs list. | |
709 | */ | |
710 | static void btrfs_discard_purge_list(struct btrfs_discard_ctl *discard_ctl) | |
711 | { | |
712 | struct btrfs_block_group *block_group, *next; | |
713 | int i; | |
714 | ||
715 | spin_lock(&discard_ctl->lock); | |
716 | for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++) { | |
717 | list_for_each_entry_safe(block_group, next, | |
718 | &discard_ctl->discard_list[i], | |
719 | discard_list) { | |
720 | list_del_init(&block_group->discard_list); | |
721 | spin_unlock(&discard_ctl->lock); | |
722 | if (block_group->used == 0) | |
723 | btrfs_mark_bg_unused(block_group); | |
724 | spin_lock(&discard_ctl->lock); | |
2b5463fc | 725 | btrfs_put_block_group(block_group); |
6e80d4f8 DZ |
726 | } |
727 | } | |
728 | spin_unlock(&discard_ctl->lock); | |
729 | } | |
730 | ||
b0643e59 DZ |
731 | void btrfs_discard_resume(struct btrfs_fs_info *fs_info) |
732 | { | |
733 | if (!btrfs_test_opt(fs_info, DISCARD_ASYNC)) { | |
734 | btrfs_discard_cleanup(fs_info); | |
735 | return; | |
736 | } | |
737 | ||
6e80d4f8 DZ |
738 | btrfs_discard_punt_unused_bgs_list(fs_info); |
739 | ||
b0643e59 DZ |
740 | set_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags); |
741 | } | |
742 | ||
743 | void btrfs_discard_stop(struct btrfs_fs_info *fs_info) | |
744 | { | |
745 | clear_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags); | |
746 | } | |
747 | ||
748 | void btrfs_discard_init(struct btrfs_fs_info *fs_info) | |
749 | { | |
750 | struct btrfs_discard_ctl *discard_ctl = &fs_info->discard_ctl; | |
751 | int i; | |
752 | ||
753 | spin_lock_init(&discard_ctl->lock); | |
754 | INIT_DELAYED_WORK(&discard_ctl->work, btrfs_discard_workfn); | |
755 | ||
756 | for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++) | |
757 | INIT_LIST_HEAD(&discard_ctl->discard_list[i]); | |
dfb79ddb | 758 | |
e93591bb | 759 | discard_ctl->prev_discard = 0; |
df903e5d | 760 | discard_ctl->prev_discard_time = 0; |
dfb79ddb | 761 | atomic_set(&discard_ctl->discardable_extents, 0); |
5dc7c10b | 762 | atomic64_set(&discard_ctl->discardable_bytes, 0); |
19b2a2c7 | 763 | discard_ctl->max_discard_size = BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE; |
6e88f116 | 764 | discard_ctl->delay_ms = BTRFS_DISCARD_MAX_DELAY_MSEC; |
a2309300 | 765 | discard_ctl->iops_limit = BTRFS_DISCARD_MAX_IOPS; |
e93591bb | 766 | discard_ctl->kbps_limit = 0; |
9ddf648f DZ |
767 | discard_ctl->discard_extent_bytes = 0; |
768 | discard_ctl->discard_bitmap_bytes = 0; | |
769 | atomic64_set(&discard_ctl->discard_bytes_saved, 0); | |
b0643e59 DZ |
770 | } |
771 | ||
772 | void btrfs_discard_cleanup(struct btrfs_fs_info *fs_info) | |
773 | { | |
774 | btrfs_discard_stop(fs_info); | |
775 | cancel_delayed_work_sync(&fs_info->discard_ctl.work); | |
6e80d4f8 | 776 | btrfs_discard_purge_list(&fs_info->discard_ctl); |
b0643e59 | 777 | } |