From 237e80483a6466f3c1968c2a8bb115b3e24d951b Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Tue, 18 Feb 2020 17:15:32 -0500 Subject: [PATCH] bcachefs: introduce b->hash_val This is partly prep work for introducing bch_btree_ptr_v2, but it'll also be a bit of a performance boost by moving the full key out of the hot part of struct btree. Signed-off-by: Kent Overstreet Signed-off-by: Kent Overstreet --- fs/bcachefs/btree_cache.c | 24 +++++++++++--------- fs/bcachefs/btree_cache.h | 13 ++++++++--- fs/bcachefs/btree_io.c | 9 ++------ fs/bcachefs/btree_types.h | 7 +++--- fs/bcachefs/btree_update.h | 2 +- fs/bcachefs/btree_update_interior.c | 35 ++++++++++++++++------------- fs/bcachefs/migrate.c | 6 ++--- 7 files changed, 52 insertions(+), 44 deletions(-) diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c index 8eed82ac41f1..ee3c1f40b500 100644 --- a/fs/bcachefs/btree_cache.c +++ b/fs/bcachefs/btree_cache.c @@ -62,13 +62,13 @@ static int bch2_btree_cache_cmp_fn(struct rhashtable_compare_arg *arg, const struct btree *b = obj; const u64 *v = arg->key; - return PTR_HASH(&b->key) == *v ? 0 : 1; + return b->hash_val == *v ? 0 : 1; } static const struct rhashtable_params bch_btree_cache_params = { .head_offset = offsetof(struct btree, hash), - .key_offset = offsetof(struct btree, key.v), - .key_len = sizeof(struct bch_extent_ptr), + .key_offset = offsetof(struct btree, hash_val), + .key_len = sizeof(u64), .obj_cmpfn = bch2_btree_cache_cmp_fn, }; @@ -115,11 +115,14 @@ void bch2_btree_node_hash_remove(struct btree_cache *bc, struct btree *b) rhashtable_remove_fast(&bc->table, &b->hash, bch_btree_cache_params); /* Cause future lookups for this node to fail: */ - PTR_HASH(&b->key) = 0; + b->hash_val = 0; } int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b) { + BUG_ON(b->hash_val); + b->hash_val = btree_ptr_hash_val(&b->key); + return rhashtable_lookup_insert_fast(&bc->table, &b->hash, bch_btree_cache_params); } @@ -145,8 +148,9 @@ __flatten static inline struct btree *btree_cache_find(struct btree_cache *bc, const struct bkey_i *k) { - return rhashtable_lookup_fast(&bc->table, &PTR_HASH(k), - bch_btree_cache_params); + u64 v = btree_ptr_hash_val(k); + + return rhashtable_lookup_fast(&bc->table, &v, bch_btree_cache_params); } /* @@ -200,7 +204,7 @@ static int __btree_node_reclaim(struct bch_fs *c, struct btree *b, bool flush) btree_node_wait_on_io(b); } out: - if (PTR_HASH(&b->key) && !ret) + if (b->hash_val && !ret) trace_btree_node_reap(c, b); return ret; out_unlock: @@ -608,7 +612,7 @@ static noinline struct btree *bch2_btree_node_fill(struct bch_fs *c, /* raced with another fill: */ /* mark as unhashed... */ - PTR_HASH(&b->key) = 0; + b->hash_val = 0; mutex_lock(&bc->lock); list_add(&b->list, &bc->freeable); @@ -711,7 +715,7 @@ retry: * free it: * * To guard against this, btree nodes are evicted from the cache - * when they're freed - and PTR_HASH() is zeroed out, which we + * when they're freed - and b->hash_val is zeroed out, which we * check for after we lock the node. * * Then, bch2_btree_node_relock() on the parent will fail - because @@ -724,7 +728,7 @@ retry: if (!btree_node_lock(b, k->k.p, level, iter, lock_type)) return ERR_PTR(-EINTR); - if (unlikely(PTR_HASH(&b->key) != PTR_HASH(k) || + if (unlikely(b->hash_val != btree_ptr_hash_val(k) || b->c.level != level || race_fault())) { six_unlock_type(&b->c.lock, lock_type); diff --git a/fs/bcachefs/btree_cache.h b/fs/bcachefs/btree_cache.h index adacb0a06703..270f7f8fb140 100644 --- a/fs/bcachefs/btree_cache.h +++ b/fs/bcachefs/btree_cache.h @@ -35,13 +35,20 @@ void bch2_fs_btree_cache_exit(struct bch_fs *); int bch2_fs_btree_cache_init(struct bch_fs *); void bch2_fs_btree_cache_init_early(struct btree_cache *); -#define PTR_HASH(_k) *((u64 *) &bkey_i_to_btree_ptr_c(_k)->v) +static inline u64 btree_ptr_hash_val(const struct bkey_i *k) +{ + switch (k->k.type) { + case KEY_TYPE_btree_ptr: + return *((u64 *) bkey_i_to_btree_ptr_c(k)->v.start); + default: + return 0; + } +} /* is btree node in hash table? */ static inline bool btree_node_hashed(struct btree *b) { - return b->key.k.type == KEY_TYPE_btree_ptr && - PTR_HASH(&b->key); + return b->hash_val != 0; } #define for_each_cached_btree(_b, _c, _tbl, _iter, _pos) \ diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c index 83f61443c8bb..9df8d4f785bf 100644 --- a/fs/bcachefs/btree_io.c +++ b/fs/bcachefs/btree_io.c @@ -1254,8 +1254,6 @@ static void bch2_btree_node_write_error(struct bch_fs *c, { struct btree *b = wbio->wbio.bio.bi_private; __BKEY_PADDED(k, BKEY_BTREE_PTR_VAL_U64s_MAX) tmp; - struct bkey_i_btree_ptr *new_key; - struct bkey_s_btree_ptr bp; struct bch_extent_ptr *ptr; struct btree_trans trans; struct btree_iter *iter; @@ -1281,16 +1279,13 @@ retry: bkey_copy(&tmp.k, &b->key); - new_key = bkey_i_to_btree_ptr(&tmp.k); - bp = btree_ptr_i_to_s(new_key); - bch2_bkey_drop_ptrs(bkey_i_to_s(&tmp.k), ptr, bch2_dev_list_has_dev(wbio->wbio.failed, ptr->dev)); - if (!bch2_bkey_nr_ptrs(bp.s_c)) + if (!bch2_bkey_nr_ptrs(bkey_i_to_s_c(&tmp.k))) goto err; - ret = bch2_btree_node_update_key(c, iter, b, new_key); + ret = bch2_btree_node_update_key(c, iter, b, &tmp.k); if (ret == -EINTR) goto retry; if (ret) diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index 85d4a6d2f7e9..4636b4fd1222 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -71,9 +71,7 @@ struct btree { struct btree_bkey_cached_common c; struct rhash_head hash; - - /* Key/pointer for this btree node */ - __BKEY_PADDED(key, BKEY_BTREE_PTR_VAL_U64s_MAX); + u64 hash_val; unsigned long flags; u16 written; @@ -136,6 +134,9 @@ struct btree { #ifdef CONFIG_BCACHEFS_DEBUG bool *expensive_debug_checks; #endif + + /* Key/pointer for this btree node */ + __BKEY_PADDED(key, BKEY_BTREE_PTR_VAL_U64s_MAX); }; struct btree_cache { diff --git a/fs/bcachefs/btree_update.h b/fs/bcachefs/btree_update.h index 2c34bae64281..be4fe818eac8 100644 --- a/fs/bcachefs/btree_update.h +++ b/fs/bcachefs/btree_update.h @@ -70,7 +70,7 @@ int bch2_btree_delete_range(struct bch_fs *, enum btree_id, int bch2_btree_node_rewrite(struct bch_fs *c, struct btree_iter *, __le64, unsigned); int bch2_btree_node_update_key(struct bch_fs *, struct btree_iter *, - struct btree *, struct bkey_i_btree_ptr *); + struct btree *, struct bkey_i *); int bch2_trans_update(struct btree_trans *, struct btree_iter *, struct bkey_i *, enum btree_trigger_flags); diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c index 713f2d41e6c9..ff8cb37ed9a1 100644 --- a/fs/bcachefs/btree_update_interior.c +++ b/fs/bcachefs/btree_update_interior.c @@ -1944,7 +1944,7 @@ static void __bch2_btree_node_update_key(struct bch_fs *c, struct btree_update *as, struct btree_iter *iter, struct btree *b, struct btree *new_hash, - struct bkey_i_btree_ptr *new_key) + struct bkey_i *new_key) { struct btree *parent; int ret; @@ -1989,20 +1989,20 @@ static void __bch2_btree_node_update_key(struct bch_fs *c, */ ret = bch2_disk_reservation_add(c, &as->reserve->disk_res, c->opts.btree_node_size * - bch2_bkey_nr_ptrs(bkey_i_to_s_c(&new_key->k_i)), + bch2_bkey_nr_ptrs(bkey_i_to_s_c(new_key)), BCH_DISK_RESERVATION_NOFAIL); BUG_ON(ret); parent = btree_node_parent(iter, b); if (parent) { if (new_hash) { - bkey_copy(&new_hash->key, &new_key->k_i); + bkey_copy(&new_hash->key, new_key); ret = bch2_btree_node_hash_insert(&c->btree_cache, new_hash, b->c.level, b->c.btree_id); BUG_ON(ret); } - bch2_keylist_add(&as->parent_keys, &new_key->k_i); + bch2_keylist_add(&as->parent_keys, new_key); bch2_btree_insert_node(as, parent, iter, &as->parent_keys, 0); if (new_hash) { @@ -2011,12 +2011,12 @@ static void __bch2_btree_node_update_key(struct bch_fs *c, bch2_btree_node_hash_remove(&c->btree_cache, b); - bkey_copy(&b->key, &new_key->k_i); + bkey_copy(&b->key, new_key); ret = __bch2_btree_node_hash_insert(&c->btree_cache, b); BUG_ON(ret); mutex_unlock(&c->btree_cache.lock); } else { - bkey_copy(&b->key, &new_key->k_i); + bkey_copy(&b->key, new_key); } } else { struct bch_fs_usage_online *fs_usage; @@ -2029,11 +2029,11 @@ static void __bch2_btree_node_update_key(struct bch_fs *c, percpu_down_read(&c->mark_lock); fs_usage = bch2_fs_usage_scratch_get(c); - bch2_mark_key_locked(c, bkey_i_to_s_c(&new_key->k_i), + bch2_mark_key_locked(c, bkey_i_to_s_c(new_key), 0, 0, &fs_usage->u, 0, BTREE_TRIGGER_INSERT); if (gc_visited(c, gc_pos_btree_root(b->c.btree_id))) - bch2_mark_key_locked(c, bkey_i_to_s_c(&new_key->k_i), + bch2_mark_key_locked(c, bkey_i_to_s_c(new_key), 0, 0, NULL, 0, BTREE_TRIGGER_INSERT|| BTREE_TRIGGER_GC); @@ -2047,16 +2047,16 @@ static void __bch2_btree_node_update_key(struct bch_fs *c, percpu_up_read(&c->mark_lock); mutex_unlock(&c->btree_interior_update_lock); - if (PTR_HASH(&new_key->k_i) != PTR_HASH(&b->key)) { + if (btree_ptr_hash_val(new_key) != b->hash_val) { mutex_lock(&c->btree_cache.lock); bch2_btree_node_hash_remove(&c->btree_cache, b); - bkey_copy(&b->key, &new_key->k_i); + bkey_copy(&b->key, new_key); ret = __bch2_btree_node_hash_insert(&c->btree_cache, b); BUG_ON(ret); mutex_unlock(&c->btree_cache.lock); } else { - bkey_copy(&b->key, &new_key->k_i); + bkey_copy(&b->key, new_key); } btree_update_updated_root(as); @@ -2068,7 +2068,7 @@ static void __bch2_btree_node_update_key(struct bch_fs *c, int bch2_btree_node_update_key(struct bch_fs *c, struct btree_iter *iter, struct btree *b, - struct bkey_i_btree_ptr *new_key) + struct bkey_i *new_key) { struct btree *parent = btree_node_parent(iter, b); struct btree_update *as = NULL; @@ -2091,8 +2091,11 @@ int bch2_btree_node_update_key(struct bch_fs *c, struct btree_iter *iter, } } - /* check PTR_HASH() after @b is locked by btree_iter_traverse(): */ - if (PTR_HASH(&new_key->k_i) != PTR_HASH(&b->key)) { + /* + * check btree_ptr_hash_val() after @b is locked by + * btree_iter_traverse(): + */ + if (btree_ptr_hash_val(new_key) != b->hash_val) { /* bch2_btree_reserve_get will unlock */ ret = bch2_btree_cache_cannibalize_lock(c, &cl); if (ret) { @@ -2134,7 +2137,7 @@ int bch2_btree_node_update_key(struct bch_fs *c, struct btree_iter *iter, goto err; } - ret = bch2_mark_bkey_replicas(c, bkey_i_to_s_c(&new_key->k_i)); + ret = bch2_mark_bkey_replicas(c, bkey_i_to_s_c(new_key)); if (ret) goto err_free_update; @@ -2193,7 +2196,7 @@ void bch2_btree_root_alloc(struct bch_fs *c, enum btree_id id) bkey_btree_ptr_init(&b->key); b->key.k.p = POS_MAX; - PTR_HASH(&b->key) = U64_MAX - id; + *((u64 *) bkey_i_to_btree_ptr(&b->key)->v.start) = U64_MAX - id; bch2_bset_init_first(b, &b->data->keys); bch2_btree_build_aux_trees(b); diff --git a/fs/bcachefs/migrate.c b/fs/bcachefs/migrate.c index 1ef62a189e33..e26fa1608f39 100644 --- a/fs/bcachefs/migrate.c +++ b/fs/bcachefs/migrate.c @@ -123,23 +123,21 @@ static int bch2_dev_metadata_drop(struct bch_fs *c, unsigned dev_idx, int flags) for_each_btree_node(&trans, iter, id, POS_MIN, BTREE_ITER_PREFETCH, b) { __BKEY_PADDED(k, BKEY_BTREE_PTR_VAL_U64s_MAX) tmp; - struct bkey_i_btree_ptr *new_key; retry: if (!bch2_bkey_has_device(bkey_i_to_s_c(&b->key), dev_idx)) continue; bkey_copy(&tmp.k, &b->key); - new_key = bkey_i_to_btree_ptr(&tmp.k); - ret = drop_dev_ptrs(c, bkey_i_to_s(&new_key->k_i), + ret = drop_dev_ptrs(c, bkey_i_to_s(&tmp.k), dev_idx, flags, true); if (ret) { bch_err(c, "Cannot drop device without losing data"); goto err; } - ret = bch2_btree_node_update_key(c, iter, b, new_key); + ret = bch2_btree_node_update_key(c, iter, b, &tmp.k); if (ret == -EINTR) { b = bch2_btree_iter_peek_node(iter); goto retry; -- 2.25.1