1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2013 Fusion IO. All rights reserved.
6 #include <linux/slab.h>
7 #include "btrfs-tests.h"
9 #include "../disk-io.h"
10 #include "../free-space-cache.h"
11 #include "../block-group.h"
13 #define BITS_PER_BITMAP (PAGE_SIZE * 8UL)
16 * This test just does basic sanity checking, making sure we can add an extent
17 * entry and remove space from either end and the middle, and make sure we can
18 * remove space that covers adjacent extent entries.
20 static int test_extents(struct btrfs_block_group_cache *cache)
24 test_msg("running extent only tests");
26 /* First just make sure we can remove an entire entry */
27 ret = btrfs_add_free_space(cache, 0, SZ_4M);
29 test_err("error adding initial extents %d", ret);
33 ret = btrfs_remove_free_space(cache, 0, SZ_4M);
35 test_err("error removing extent %d", ret);
39 if (test_check_exists(cache, 0, SZ_4M)) {
40 test_err("full remove left some lingering space");
44 /* Ok edge and middle cases now */
45 ret = btrfs_add_free_space(cache, 0, SZ_4M);
47 test_err("error adding half extent %d", ret);
51 ret = btrfs_remove_free_space(cache, 3 * SZ_1M, SZ_1M);
53 test_err("error removing tail end %d", ret);
57 ret = btrfs_remove_free_space(cache, 0, SZ_1M);
59 test_err("error removing front end %d", ret);
63 ret = btrfs_remove_free_space(cache, SZ_2M, 4096);
65 test_err("error removing middle piece %d", ret);
69 if (test_check_exists(cache, 0, SZ_1M)) {
70 test_err("still have space at the front");
74 if (test_check_exists(cache, SZ_2M, 4096)) {
75 test_err("still have space in the middle");
79 if (test_check_exists(cache, 3 * SZ_1M, SZ_1M)) {
80 test_err("still have space at the end");
85 __btrfs_remove_free_space_cache(cache->free_space_ctl);
90 static int test_bitmaps(struct btrfs_block_group_cache *cache,
93 u64 next_bitmap_offset;
96 test_msg("running bitmap only tests");
98 ret = test_add_free_space_entry(cache, 0, SZ_4M, 1);
100 test_err("couldn't create a bitmap entry %d", ret);
104 ret = btrfs_remove_free_space(cache, 0, SZ_4M);
106 test_err("error removing bitmap full range %d", ret);
110 if (test_check_exists(cache, 0, SZ_4M)) {
111 test_err("left some space in bitmap");
115 ret = test_add_free_space_entry(cache, 0, SZ_4M, 1);
117 test_err("couldn't add to our bitmap entry %d", ret);
121 ret = btrfs_remove_free_space(cache, SZ_1M, SZ_2M);
123 test_err("couldn't remove middle chunk %d", ret);
128 * The first bitmap we have starts at offset 0 so the next one is just
129 * at the end of the first bitmap.
131 next_bitmap_offset = (u64)(BITS_PER_BITMAP * sectorsize);
133 /* Test a bit straddling two bitmaps */
134 ret = test_add_free_space_entry(cache, next_bitmap_offset - SZ_2M,
137 test_err("couldn't add space that straddles two bitmaps %d",
142 ret = btrfs_remove_free_space(cache, next_bitmap_offset - SZ_1M, SZ_2M);
144 test_err("couldn't remove overlapping space %d", ret);
148 if (test_check_exists(cache, next_bitmap_offset - SZ_1M, SZ_2M)) {
149 test_err("left some space when removing overlapping");
153 __btrfs_remove_free_space_cache(cache->free_space_ctl);
158 /* This is the high grade jackassery */
159 static int test_bitmaps_and_extents(struct btrfs_block_group_cache *cache,
162 u64 bitmap_offset = (u64)(BITS_PER_BITMAP * sectorsize);
165 test_msg("running bitmap and extent tests");
168 * First let's do something simple, an extent at the same offset as the
169 * bitmap, but the free space completely in the extent and then
170 * completely in the bitmap.
172 ret = test_add_free_space_entry(cache, SZ_4M, SZ_1M, 1);
174 test_err("couldn't create bitmap entry %d", ret);
178 ret = test_add_free_space_entry(cache, 0, SZ_1M, 0);
180 test_err("couldn't add extent entry %d", ret);
184 ret = btrfs_remove_free_space(cache, 0, SZ_1M);
186 test_err("couldn't remove extent entry %d", ret);
190 if (test_check_exists(cache, 0, SZ_1M)) {
191 test_err("left remnants after our remove");
195 /* Now to add back the extent entry and remove from the bitmap */
196 ret = test_add_free_space_entry(cache, 0, SZ_1M, 0);
198 test_err("couldn't re-add extent entry %d", ret);
202 ret = btrfs_remove_free_space(cache, SZ_4M, SZ_1M);
204 test_err("couldn't remove from bitmap %d", ret);
208 if (test_check_exists(cache, SZ_4M, SZ_1M)) {
209 test_err("left remnants in the bitmap");
214 * Ok so a little more evil, extent entry and bitmap at the same offset,
215 * removing an overlapping chunk.
217 ret = test_add_free_space_entry(cache, SZ_1M, SZ_4M, 1);
219 test_err("couldn't add to a bitmap %d", ret);
223 ret = btrfs_remove_free_space(cache, SZ_512K, 3 * SZ_1M);
225 test_err("couldn't remove overlapping space %d", ret);
229 if (test_check_exists(cache, SZ_512K, 3 * SZ_1M)) {
230 test_err("left over pieces after removing overlapping");
234 __btrfs_remove_free_space_cache(cache->free_space_ctl);
236 /* Now with the extent entry offset into the bitmap */
237 ret = test_add_free_space_entry(cache, SZ_4M, SZ_4M, 1);
239 test_err("couldn't add space to the bitmap %d", ret);
243 ret = test_add_free_space_entry(cache, SZ_2M, SZ_2M, 0);
245 test_err("couldn't add extent to the cache %d", ret);
249 ret = btrfs_remove_free_space(cache, 3 * SZ_1M, SZ_4M);
251 test_err("problem removing overlapping space %d", ret);
255 if (test_check_exists(cache, 3 * SZ_1M, SZ_4M)) {
256 test_err("left something behind when removing space");
261 * This has blown up in the past, the extent entry starts before the
262 * bitmap entry, but we're trying to remove an offset that falls
263 * completely within the bitmap range and is in both the extent entry
264 * and the bitmap entry, looks like this
270 __btrfs_remove_free_space_cache(cache->free_space_ctl);
271 ret = test_add_free_space_entry(cache, bitmap_offset + SZ_4M, SZ_4M, 1);
273 test_err("couldn't add bitmap %d", ret);
277 ret = test_add_free_space_entry(cache, bitmap_offset - SZ_1M,
280 test_err("couldn't add extent entry %d", ret);
284 ret = btrfs_remove_free_space(cache, bitmap_offset + SZ_1M, 5 * SZ_1M);
286 test_err("failed to free our space %d", ret);
290 if (test_check_exists(cache, bitmap_offset + SZ_1M, 5 * SZ_1M)) {
291 test_err("left stuff over");
295 __btrfs_remove_free_space_cache(cache->free_space_ctl);
298 * This blew up before, we have part of the free space in a bitmap and
299 * then the entirety of the rest of the space in an extent. This used
300 * to return -EAGAIN back from btrfs_remove_extent, make sure this
303 ret = test_add_free_space_entry(cache, SZ_1M, SZ_2M, 1);
305 test_err("couldn't add bitmap entry %d", ret);
309 ret = test_add_free_space_entry(cache, 3 * SZ_1M, SZ_1M, 0);
311 test_err("couldn't add extent entry %d", ret);
315 ret = btrfs_remove_free_space(cache, SZ_1M, 3 * SZ_1M);
317 test_err("error removing bitmap and extent overlapping %d", ret);
321 __btrfs_remove_free_space_cache(cache->free_space_ctl);
325 /* Used by test_steal_space_from_bitmap_to_extent(). */
326 static bool test_use_bitmap(struct btrfs_free_space_ctl *ctl,
327 struct btrfs_free_space *info)
329 return ctl->free_extents > 0;
332 /* Used by test_steal_space_from_bitmap_to_extent(). */
334 check_num_extents_and_bitmaps(const struct btrfs_block_group_cache *cache,
335 const int num_extents,
336 const int num_bitmaps)
338 if (cache->free_space_ctl->free_extents != num_extents) {
340 "incorrect # of extent entries in the cache: %d, expected %d",
341 cache->free_space_ctl->free_extents, num_extents);
344 if (cache->free_space_ctl->total_bitmaps != num_bitmaps) {
346 "incorrect # of extent entries in the cache: %d, expected %d",
347 cache->free_space_ctl->total_bitmaps, num_bitmaps);
353 /* Used by test_steal_space_from_bitmap_to_extent(). */
354 static int check_cache_empty(struct btrfs_block_group_cache *cache)
360 * Now lets confirm that there's absolutely no free space left to
363 if (cache->free_space_ctl->free_space != 0) {
364 test_err("cache free space is not 0");
368 /* And any allocation request, no matter how small, should fail now. */
369 offset = btrfs_find_space_for_alloc(cache, 0, 4096, 0,
372 test_err("space allocation did not fail, returned offset: %llu",
377 /* And no extent nor bitmap entries in the cache anymore. */
378 return check_num_extents_and_bitmaps(cache, 0, 0);
382 * Before we were able to steal free space from a bitmap entry to an extent
383 * entry, we could end up with 2 entries representing a contiguous free space.
384 * One would be an extent entry and the other a bitmap entry. Since in order
385 * to allocate space to a caller we use only 1 entry, we couldn't return that
386 * whole range to the caller if it was requested. This forced the caller to
387 * either assume ENOSPC or perform several smaller space allocations, which
388 * wasn't optimal as they could be spread all over the block group while under
389 * concurrency (extra overhead and fragmentation).
391 * This stealing approach is beneficial, since we always prefer to allocate
392 * from extent entries, both for clustered and non-clustered allocation
396 test_steal_space_from_bitmap_to_extent(struct btrfs_block_group_cache *cache,
402 const struct btrfs_free_space_op test_free_space_ops = {
403 .recalc_thresholds = cache->free_space_ctl->op->recalc_thresholds,
404 .use_bitmap = test_use_bitmap,
406 const struct btrfs_free_space_op *orig_free_space_ops;
408 test_msg("running space stealing from bitmap to extent tests");
411 * For this test, we want to ensure we end up with an extent entry
412 * immediately adjacent to a bitmap entry, where the bitmap starts
413 * at an offset where the extent entry ends. We keep adding and
414 * removing free space to reach into this state, but to get there
415 * we need to reach a point where marking new free space doesn't
416 * result in adding new extent entries or merging the new space
417 * with existing extent entries - the space ends up being marked
418 * in an existing bitmap that covers the new free space range.
420 * To get there, we need to reach the threshold defined set at
421 * cache->free_space_ctl->extents_thresh, which currently is
422 * 256 extents on a x86_64 system at least, and a few other
423 * conditions (check free_space_cache.c). Instead of making the
424 * test much longer and complicated, use a "use_bitmap" operation
425 * that forces use of bitmaps as soon as we have at least 1
428 orig_free_space_ops = cache->free_space_ctl->op;
429 cache->free_space_ctl->op = &test_free_space_ops;
432 * Extent entry covering free space range [128Mb - 256Kb, 128Mb - 128Kb[
434 ret = test_add_free_space_entry(cache, SZ_128M - SZ_256K, SZ_128K, 0);
436 test_err("couldn't add extent entry %d", ret);
440 /* Bitmap entry covering free space range [128Mb + 512Kb, 256Mb[ */
441 ret = test_add_free_space_entry(cache, SZ_128M + SZ_512K,
442 SZ_128M - SZ_512K, 1);
444 test_err("couldn't add bitmap entry %d", ret);
448 ret = check_num_extents_and_bitmaps(cache, 2, 1);
453 * Now make only the first 256Kb of the bitmap marked as free, so that
454 * we end up with only the following ranges marked as free space:
456 * [128Mb - 256Kb, 128Mb - 128Kb[
457 * [128Mb + 512Kb, 128Mb + 768Kb[
459 ret = btrfs_remove_free_space(cache,
460 SZ_128M + 768 * SZ_1K,
461 SZ_128M - 768 * SZ_1K);
463 test_err("failed to free part of bitmap space %d", ret);
467 /* Confirm that only those 2 ranges are marked as free. */
468 if (!test_check_exists(cache, SZ_128M - SZ_256K, SZ_128K)) {
469 test_err("free space range missing");
472 if (!test_check_exists(cache, SZ_128M + SZ_512K, SZ_256K)) {
473 test_err("free space range missing");
478 * Confirm that the bitmap range [128Mb + 768Kb, 256Mb[ isn't marked
481 if (test_check_exists(cache, SZ_128M + 768 * SZ_1K,
482 SZ_128M - 768 * SZ_1K)) {
483 test_err("bitmap region not removed from space cache");
488 * Confirm that the region [128Mb + 256Kb, 128Mb + 512Kb[, which is
489 * covered by the bitmap, isn't marked as free.
491 if (test_check_exists(cache, SZ_128M + SZ_256K, SZ_256K)) {
492 test_err("invalid bitmap region marked as free");
497 * Confirm that the region [128Mb, 128Mb + 256Kb[, which is covered
498 * by the bitmap too, isn't marked as free either.
500 if (test_check_exists(cache, SZ_128M, SZ_256K)) {
501 test_err("invalid bitmap region marked as free");
506 * Now lets mark the region [128Mb, 128Mb + 512Kb[ as free too. But,
507 * lets make sure the free space cache marks it as free in the bitmap,
508 * and doesn't insert a new extent entry to represent this region.
510 ret = btrfs_add_free_space(cache, SZ_128M, SZ_512K);
512 test_err("error adding free space: %d", ret);
515 /* Confirm the region is marked as free. */
516 if (!test_check_exists(cache, SZ_128M, SZ_512K)) {
517 test_err("bitmap region not marked as free");
522 * Confirm that no new extent entries or bitmap entries were added to
523 * the cache after adding that free space region.
525 ret = check_num_extents_and_bitmaps(cache, 2, 1);
530 * Now lets add a small free space region to the right of the previous
531 * one, which is not contiguous with it and is part of the bitmap too.
532 * The goal is to test that the bitmap entry space stealing doesn't
533 * steal this space region.
535 ret = btrfs_add_free_space(cache, SZ_128M + SZ_16M, sectorsize);
537 test_err("error adding free space: %d", ret);
542 * Confirm that no new extent entries or bitmap entries were added to
543 * the cache after adding that free space region.
545 ret = check_num_extents_and_bitmaps(cache, 2, 1);
550 * Now mark the region [128Mb - 128Kb, 128Mb[ as free too. This will
551 * expand the range covered by the existing extent entry that represents
552 * the free space [128Mb - 256Kb, 128Mb - 128Kb[.
554 ret = btrfs_add_free_space(cache, SZ_128M - SZ_128K, SZ_128K);
556 test_err("error adding free space: %d", ret);
559 /* Confirm the region is marked as free. */
560 if (!test_check_exists(cache, SZ_128M - SZ_128K, SZ_128K)) {
561 test_err("extent region not marked as free");
566 * Confirm that our extent entry didn't stole all free space from the
567 * bitmap, because of the small 4Kb free space region.
569 ret = check_num_extents_and_bitmaps(cache, 2, 1);
574 * So now we have the range [128Mb - 256Kb, 128Mb + 768Kb[ as free
575 * space. Without stealing bitmap free space into extent entry space,
576 * we would have all this free space represented by 2 entries in the
579 * extent entry covering range: [128Mb - 256Kb, 128Mb[
580 * bitmap entry covering range: [128Mb, 128Mb + 768Kb[
582 * Attempting to allocate the whole free space (1Mb) would fail, because
583 * we can't allocate from multiple entries.
584 * With the bitmap free space stealing, we get a single extent entry
585 * that represents the 1Mb free space, and therefore we're able to
586 * allocate the whole free space at once.
588 if (!test_check_exists(cache, SZ_128M - SZ_256K, SZ_1M)) {
589 test_err("expected region not marked as free");
593 if (cache->free_space_ctl->free_space != (SZ_1M + sectorsize)) {
594 test_err("cache free space is not 1Mb + %u", sectorsize);
598 offset = btrfs_find_space_for_alloc(cache,
601 if (offset != (SZ_128M - SZ_256K)) {
603 "failed to allocate 1Mb from space cache, returned offset is: %llu",
609 * All that remains is a sectorsize free space region in a bitmap.
612 ret = check_num_extents_and_bitmaps(cache, 1, 1);
616 if (cache->free_space_ctl->free_space != sectorsize) {
617 test_err("cache free space is not %u", sectorsize);
621 offset = btrfs_find_space_for_alloc(cache,
624 if (offset != (SZ_128M + SZ_16M)) {
625 test_err("failed to allocate %u, returned offset : %llu",
630 ret = check_cache_empty(cache);
634 __btrfs_remove_free_space_cache(cache->free_space_ctl);
637 * Now test a similar scenario, but where our extent entry is located
638 * to the right of the bitmap entry, so that we can check that stealing
639 * space from a bitmap to the front of an extent entry works.
643 * Extent entry covering free space range [128Mb + 128Kb, 128Mb + 256Kb[
645 ret = test_add_free_space_entry(cache, SZ_128M + SZ_128K, SZ_128K, 0);
647 test_err("couldn't add extent entry %d", ret);
651 /* Bitmap entry covering free space range [0, 128Mb - 512Kb[ */
652 ret = test_add_free_space_entry(cache, 0, SZ_128M - SZ_512K, 1);
654 test_err("couldn't add bitmap entry %d", ret);
658 ret = check_num_extents_and_bitmaps(cache, 2, 1);
663 * Now make only the last 256Kb of the bitmap marked as free, so that
664 * we end up with only the following ranges marked as free space:
666 * [128Mb + 128b, 128Mb + 256Kb[
667 * [128Mb - 768Kb, 128Mb - 512Kb[
669 ret = btrfs_remove_free_space(cache, 0, SZ_128M - 768 * SZ_1K);
671 test_err("failed to free part of bitmap space %d", ret);
675 /* Confirm that only those 2 ranges are marked as free. */
676 if (!test_check_exists(cache, SZ_128M + SZ_128K, SZ_128K)) {
677 test_err("free space range missing");
680 if (!test_check_exists(cache, SZ_128M - 768 * SZ_1K, SZ_256K)) {
681 test_err("free space range missing");
686 * Confirm that the bitmap range [0, 128Mb - 768Kb[ isn't marked
689 if (test_check_exists(cache, 0, SZ_128M - 768 * SZ_1K)) {
690 test_err("bitmap region not removed from space cache");
695 * Confirm that the region [128Mb - 512Kb, 128Mb[, which is
696 * covered by the bitmap, isn't marked as free.
698 if (test_check_exists(cache, SZ_128M - SZ_512K, SZ_512K)) {
699 test_err("invalid bitmap region marked as free");
704 * Now lets mark the region [128Mb - 512Kb, 128Mb[ as free too. But,
705 * lets make sure the free space cache marks it as free in the bitmap,
706 * and doesn't insert a new extent entry to represent this region.
708 ret = btrfs_add_free_space(cache, SZ_128M - SZ_512K, SZ_512K);
710 test_err("error adding free space: %d", ret);
713 /* Confirm the region is marked as free. */
714 if (!test_check_exists(cache, SZ_128M - SZ_512K, SZ_512K)) {
715 test_err("bitmap region not marked as free");
720 * Confirm that no new extent entries or bitmap entries were added to
721 * the cache after adding that free space region.
723 ret = check_num_extents_and_bitmaps(cache, 2, 1);
728 * Now lets add a small free space region to the left of the previous
729 * one, which is not contiguous with it and is part of the bitmap too.
730 * The goal is to test that the bitmap entry space stealing doesn't
731 * steal this space region.
733 ret = btrfs_add_free_space(cache, SZ_32M, 2 * sectorsize);
735 test_err("error adding free space: %d", ret);
740 * Now mark the region [128Mb, 128Mb + 128Kb[ as free too. This will
741 * expand the range covered by the existing extent entry that represents
742 * the free space [128Mb + 128Kb, 128Mb + 256Kb[.
744 ret = btrfs_add_free_space(cache, SZ_128M, SZ_128K);
746 test_err("error adding free space: %d", ret);
749 /* Confirm the region is marked as free. */
750 if (!test_check_exists(cache, SZ_128M, SZ_128K)) {
751 test_err("extent region not marked as free");
756 * Confirm that our extent entry didn't stole all free space from the
757 * bitmap, because of the small 2 * sectorsize free space region.
759 ret = check_num_extents_and_bitmaps(cache, 2, 1);
764 * So now we have the range [128Mb - 768Kb, 128Mb + 256Kb[ as free
765 * space. Without stealing bitmap free space into extent entry space,
766 * we would have all this free space represented by 2 entries in the
769 * extent entry covering range: [128Mb, 128Mb + 256Kb[
770 * bitmap entry covering range: [128Mb - 768Kb, 128Mb[
772 * Attempting to allocate the whole free space (1Mb) would fail, because
773 * we can't allocate from multiple entries.
774 * With the bitmap free space stealing, we get a single extent entry
775 * that represents the 1Mb free space, and therefore we're able to
776 * allocate the whole free space at once.
778 if (!test_check_exists(cache, SZ_128M - 768 * SZ_1K, SZ_1M)) {
779 test_err("expected region not marked as free");
783 if (cache->free_space_ctl->free_space != (SZ_1M + 2 * sectorsize)) {
784 test_err("cache free space is not 1Mb + %u", 2 * sectorsize);
788 offset = btrfs_find_space_for_alloc(cache, 0, SZ_1M, 0,
790 if (offset != (SZ_128M - 768 * SZ_1K)) {
792 "failed to allocate 1Mb from space cache, returned offset is: %llu",
798 * All that remains is 2 * sectorsize free space region
799 * in a bitmap. Confirm.
801 ret = check_num_extents_and_bitmaps(cache, 1, 1);
805 if (cache->free_space_ctl->free_space != 2 * sectorsize) {
806 test_err("cache free space is not %u", 2 * sectorsize);
810 offset = btrfs_find_space_for_alloc(cache,
811 0, 2 * sectorsize, 0,
813 if (offset != SZ_32M) {
814 test_err("failed to allocate %u, offset: %llu",
815 2 * sectorsize, offset);
819 ret = check_cache_empty(cache);
823 cache->free_space_ctl->op = orig_free_space_ops;
824 __btrfs_remove_free_space_cache(cache->free_space_ctl);
829 int btrfs_test_free_space_cache(u32 sectorsize, u32 nodesize)
831 struct btrfs_fs_info *fs_info;
832 struct btrfs_block_group_cache *cache;
833 struct btrfs_root *root = NULL;
836 test_msg("running btrfs free space cache tests");
837 fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize);
839 test_std_err(TEST_ALLOC_FS_INFO);
844 * For ppc64 (with 64k page size), bytes per bitmap might be
845 * larger than 1G. To make bitmap test available in ppc64,
846 * alloc dummy block group whose size cross bitmaps.
848 cache = btrfs_alloc_dummy_block_group(fs_info,
849 BITS_PER_BITMAP * sectorsize + PAGE_SIZE);
851 test_std_err(TEST_ALLOC_BLOCK_GROUP);
852 btrfs_free_dummy_fs_info(fs_info);
856 root = btrfs_alloc_dummy_root(fs_info);
858 test_std_err(TEST_ALLOC_ROOT);
863 root->fs_info->extent_root = root;
865 ret = test_extents(cache);
868 ret = test_bitmaps(cache, sectorsize);
871 ret = test_bitmaps_and_extents(cache, sectorsize);
875 ret = test_steal_space_from_bitmap_to_extent(cache, sectorsize);
877 btrfs_free_dummy_block_group(cache);
878 btrfs_free_dummy_root(root);
879 btrfs_free_dummy_fs_info(fs_info);