From 15f969326ee296f7b7faf7704105a99fa02c288d Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Sat, 17 May 2025 15:05:26 -0400 Subject: [PATCH] bcachefs: Improve bucket_bitmap code Add some more helpers, and mismatches is now a superset of the empty bitmap - simplifies most checks. Signed-off-by: Kent Overstreet --- fs/bcachefs/backpointers.c | 123 ++++++++++++++++++++++--------------- fs/bcachefs/backpointers.h | 7 +++ fs/bcachefs/bcachefs.h | 3 +- fs/bcachefs/buckets.c | 25 ++------ fs/bcachefs/movinggc.c | 6 +- fs/bcachefs/super.c | 8 +-- 6 files changed, 92 insertions(+), 80 deletions(-) diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c index d9ddfc4b5dcc..6b98ce1ed6c9 100644 --- a/fs/bcachefs/backpointers.c +++ b/fs/bcachefs/backpointers.c @@ -15,6 +15,8 @@ #include +static int bch2_bucket_bitmap_set(struct bch_dev *, struct bucket_bitmap *, u64); + static inline struct bbpos bp_to_bbpos(struct bch_backpointer bp) { return (struct bbpos) { @@ -685,31 +687,28 @@ static int check_extent_to_backpointers(struct btree_trans *trans, continue; } - u64 b = PTR_BUCKET_NR(ca, &p.ptr); - bool set[2]; - - for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) { - unsigned long *bitmap = - READ_ONCE(ca->bucket_backpointer_mismatches[i].buckets); - set[i] = bitmap && test_bit(b, bitmap); + if (p.ptr.cached && dev_ptr_stale_rcu(ca, &p.ptr)) { + rcu_read_unlock(); + continue; } - bool check = set[0]; - bool empty = set[1]; + u64 b = PTR_BUCKET_NR(ca, &p.ptr); + if (!bch2_bucket_bitmap_test(&ca->bucket_backpointer_mismatch, b)) { + rcu_read_unlock(); + continue; + } - bool stale = p.ptr.cached && (!ca || dev_ptr_stale_rcu(ca, &p.ptr)); + bool empty = bch2_bucket_bitmap_test(&ca->bucket_backpointer_empty, b); rcu_read_unlock(); - if ((check || empty) && !stale) { - struct bkey_i_backpointer bp; - bch2_extent_ptr_to_bp(c, btree, level, k, p, entry, &bp); + struct bkey_i_backpointer bp; + bch2_extent_ptr_to_bp(c, btree, level, k, p, entry, &bp); - int ret = check - ? check_bp_exists(trans, s, &bp, k) - : bch2_bucket_backpointer_mod(trans, k, &bp, true); - if (ret) - return ret; - } + int ret = !empty + ? check_bp_exists(trans, s, &bp, k) + : bch2_bucket_backpointer_mod(trans, k, &bp, true); + if (ret) + return ret; } return 0; @@ -952,21 +951,12 @@ static int check_bucket_backpointer_mismatch(struct btree_trans *trans, struct b sectors[ALLOC_stripe] + sectors[ALLOC_cached]) == 0; - struct bucket_bitmap *bitmap = &ca->bucket_backpointer_mismatches[empty]; - - mutex_lock(&bitmap->lock); - if (!bitmap->buckets) { - bitmap->buckets = kvcalloc(BITS_TO_LONGS(ca->mi.nbuckets), - sizeof(unsigned long), GFP_KERNEL); - if (!bitmap->buckets) { - mutex_unlock(&bitmap->lock); - ret = -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap; - goto err; - } - } - - bitmap->nr += !__test_and_set_bit(alloc_k.k->p.offset, bitmap->buckets); - mutex_unlock(&bitmap->lock); + ret = bch2_bucket_bitmap_set(ca, &ca->bucket_backpointer_mismatch, + alloc_k.k->p.offset) ?: + (empty + ? bch2_bucket_bitmap_set(ca, &ca->bucket_backpointer_empty, + alloc_k.k->p.offset) + : 0); } err: bch2_dev_put(ca); @@ -992,15 +982,10 @@ static bool backpointer_node_has_missing(struct bch_fs *c, struct bkey_s_c k) struct bpos bucket = bp_pos_to_bucket(ca, pos); u64 next = ca->mi.nbuckets; - for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) { - unsigned long *bitmap = - READ_ONCE(ca->bucket_backpointer_mismatches[i].buckets); - if (bitmap) - next = min_t(u64, next, - find_next_bit(bitmap, - ca->mi.nbuckets, - bucket.offset)); - } + unsigned long *bitmap = READ_ONCE(ca->bucket_backpointer_mismatch.buckets); + if (bitmap) + next = min_t(u64, next, + find_next_bit(bitmap, ca->mi.nbuckets, bucket.offset)); bucket.offset = next; if (bucket.offset == ca->mi.nbuckets) @@ -1124,18 +1109,17 @@ int bch2_check_extents_to_backpointers(struct bch_fs *c) if (ret) goto err; - u64 nr_buckets = 0, nr_mismatches = 0, nr_empty = 0; + u64 nr_buckets = 0, nr_mismatches = 0; for_each_member_device(c, ca) { nr_buckets += ca->mi.nbuckets; - nr_mismatches += ca->bucket_backpointer_mismatches[0].nr; - nr_empty += ca->bucket_backpointer_mismatches[1].nr; + nr_mismatches += ca->bucket_backpointer_mismatch.nr; } - if (!nr_mismatches && !nr_empty) + if (!nr_mismatches) goto err; bch_info(c, "scanning for missing backpointers in %llu/%llu buckets", - nr_mismatches + nr_empty, nr_buckets); + nr_mismatches, nr_buckets); while (1) { ret = bch2_pin_backpointer_nodes_with_missing(trans, s.bp_start, &s.bp_end); @@ -1171,9 +1155,10 @@ err: bch2_bkey_buf_exit(&s.last_flushed, c); bch2_btree_cache_unpin(c); - for_each_member_device(c, ca) - for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) - bch2_bucket_bitmap_free(&ca->bucket_backpointer_mismatches[i]); + for_each_member_device(c, ca) { + bch2_bucket_bitmap_free(&ca->bucket_backpointer_mismatch); + bch2_bucket_bitmap_free(&ca->bucket_backpointer_empty); + } bch_err_fn(c, ret); return ret; @@ -1297,6 +1282,42 @@ int bch2_check_backpointers_to_extents(struct bch_fs *c) return ret; } +static int bch2_bucket_bitmap_set(struct bch_dev *ca, struct bucket_bitmap *b, u64 bit) +{ + scoped_guard(mutex, &b->lock) { + if (!b->buckets) { + b->buckets = kvcalloc(BITS_TO_LONGS(ca->mi.nbuckets), + sizeof(unsigned long), GFP_KERNEL); + if (!b->buckets) + return -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap; + } + + b->nr += !__test_and_set_bit(bit, b->buckets); + } + + return 0; +} + +int bch2_bucket_bitmap_resize(struct bucket_bitmap *b, u64 old_size, u64 new_size) +{ + scoped_guard(mutex, &b->lock) { + if (!b->buckets) + return 0; + + unsigned long *n = kvcalloc(BITS_TO_LONGS(new_size), + sizeof(unsigned long), GFP_KERNEL); + if (!n) + return -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap; + + memcpy(n, b->buckets, + BITS_TO_LONGS(min(old_size, new_size)) * sizeof(unsigned long)); + kvfree(b->buckets); + b->buckets = n; + } + + return 0; +} + void bch2_bucket_bitmap_free(struct bucket_bitmap *b) { mutex_lock(&b->lock); diff --git a/fs/bcachefs/backpointers.h b/fs/bcachefs/backpointers.h index f57098c32143..fe7149a2fbf5 100644 --- a/fs/bcachefs/backpointers.h +++ b/fs/bcachefs/backpointers.h @@ -188,6 +188,13 @@ int bch2_check_btree_backpointers(struct bch_fs *); int bch2_check_extents_to_backpointers(struct bch_fs *); int bch2_check_backpointers_to_extents(struct bch_fs *); +static inline bool bch2_bucket_bitmap_test(struct bucket_bitmap *b, u64 i) +{ + unsigned long *bitmap = READ_ONCE(b->buckets); + return bitmap && test_bit(i, bitmap); +} + +int bch2_bucket_bitmap_resize(struct bucket_bitmap *, u64, u64); void bch2_bucket_bitmap_free(struct bucket_bitmap *); #endif /* _BCACHEFS_BACKPOINTERS_BACKGROUND_H */ diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h index e1680b635fe1..b58fad743fc4 100644 --- a/fs/bcachefs/bcachefs.h +++ b/fs/bcachefs/bcachefs.h @@ -626,7 +626,8 @@ struct bch_dev { u8 *oldest_gen; unsigned long *buckets_nouse; - struct bucket_bitmap bucket_backpointer_mismatches[2]; + struct bucket_bitmap bucket_backpointer_mismatch; + struct bucket_bitmap bucket_backpointer_empty; struct bch_dev_usage_full __percpu *usage; diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c index ca6e58d6fbc8..8bb6384190c5 100644 --- a/fs/bcachefs/buckets.c +++ b/fs/bcachefs/buckets.c @@ -1324,27 +1324,10 @@ int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) sizeof(bucket_gens->b[0]) * copy); } - for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) { - struct bucket_bitmap *bitmap = &ca->bucket_backpointer_mismatches[i]; - - mutex_lock(&bitmap->lock); - if (bitmap->buckets) { - unsigned long *n = kvcalloc(BITS_TO_LONGS(nbuckets), - sizeof(unsigned long), GFP_KERNEL); - if (!n) { - mutex_unlock(&bitmap->lock); - ret = -BCH_ERR_ENOMEM_backpointer_mismatches_bitmap; - goto err; - } - - memcpy(n, bitmap->buckets, - BITS_TO_LONGS(ca->mi.nbuckets) * sizeof(unsigned long)); - kvfree(bitmap->buckets); - bitmap->buckets = n; - - } - mutex_unlock(&bitmap->lock); - } + ret = bch2_bucket_bitmap_resize(&ca->bucket_backpointer_mismatch, + ca->mi.nbuckets, nbuckets) ?: + bch2_bucket_bitmap_resize(&ca->bucket_backpointer_empty, + ca->mi.nbuckets, nbuckets); rcu_assign_pointer(ca->bucket_gens, bucket_gens); bucket_gens = old_bucket_gens; diff --git a/fs/bcachefs/movinggc.c b/fs/bcachefs/movinggc.c index 4bfdb1befb9a..0a751a65386f 100644 --- a/fs/bcachefs/movinggc.c +++ b/fs/bcachefs/movinggc.c @@ -8,6 +8,7 @@ #include "bcachefs.h" #include "alloc_background.h" #include "alloc_foreground.h" +#include "backpointers.h" #include "btree_iter.h" #include "btree_update.h" #include "btree_write_buffer.h" @@ -76,7 +77,7 @@ static int bch2_bucket_is_movable(struct btree_trans *trans, if (ca->mi.state != BCH_MEMBER_STATE_rw || !bch2_dev_is_online(ca)) - goto out_put; + goto out; struct bch_alloc_v4 _a; const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &_a); @@ -85,9 +86,8 @@ static int bch2_bucket_is_movable(struct btree_trans *trans, u64 lru_idx = alloc_lru_idx_fragmentation(*a, ca); ret = lru_idx && lru_idx <= time; -out_put: - bch2_dev_put(ca); out: + bch2_dev_put(ca); bch2_trans_iter_exit(trans, &iter); return ret; } diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c index 170b0f26c018..24658bf450ab 100644 --- a/fs/bcachefs/super.c +++ b/fs/bcachefs/super.c @@ -1366,8 +1366,8 @@ static void bch2_dev_free(struct bch_dev *ca) if (ca->kobj.state_in_sysfs) kobject_del(&ca->kobj); - for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) - bch2_bucket_bitmap_free(&ca->bucket_backpointer_mismatches[i]); + bch2_bucket_bitmap_free(&ca->bucket_backpointer_mismatch); + bch2_bucket_bitmap_free(&ca->bucket_backpointer_empty); bch2_free_super(&ca->disk_sb); bch2_dev_allocator_background_exit(ca); @@ -1499,8 +1499,8 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c, atomic_long_set(&ca->ref, 1); #endif - for (unsigned i = 0; i < ARRAY_SIZE(ca->bucket_backpointer_mismatches); i++) - mutex_init(&ca->bucket_backpointer_mismatches[i].lock); + mutex_init(&ca->bucket_backpointer_mismatch.lock); + mutex_init(&ca->bucket_backpointer_empty.lock); bch2_dev_allocator_background_init(ca); -- 2.25.1