bcachefs: Switch to local_clock() for fastpath time source

local_clock() isn't always completely accurate - e.g. on machines with
TSC drift - but ktime_get_ns() overhead is too high, unfortunately.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c
index c87129a1164011d5a312231051be1135ca104d03..f928de6692aec73e59f5c05a10ff55c8cde31df5 100644 (file)
--- a/fs/bcachefs/btree_iter.c
+++ b/fs/bcachefs/btree_iter.c
@@ -2787,7 +2787,7 @@ u32 bch2_trans_begin(struct btree_trans *trans)
 
        if (!trans->restarted &&
            (need_resched() ||
-            ktime_get_ns() - trans->last_begin_time > BTREE_TRANS_MAX_LOCK_HOLD_TIME_NS)) {
+            local_clock() - trans->last_begin_time > BTREE_TRANS_MAX_LOCK_HOLD_TIME_NS)) {
                bch2_trans_unlock(trans);
                cond_resched();
                bch2_trans_relock(trans);
@@ -2797,7 +2797,7 @@ u32 bch2_trans_begin(struct btree_trans *trans)
        if (trans->restarted)
                bch2_btree_path_traverse_all(trans);
 
-       trans->last_begin_time = ktime_get_ns();
+       trans->last_begin_time = local_clock();
        return trans->restart_count;
 }
 
@@ -2851,7 +2851,7 @@ void __bch2_trans_init(struct btree_trans *trans, struct bch_fs *c, const char *
        memset(trans, 0, sizeof(*trans));
        trans->c                = c;
        trans->fn               = fn;
-       trans->last_begin_time  = ktime_get_ns();
+       trans->last_begin_time  = local_clock();
        trans->fn_idx           = bch2_trans_get_fn_idx(trans, c, fn);
        trans->locking_wait.task = current;
        trans->journal_replay_not_finished =

diff --git a/fs/bcachefs/btree_locking.h b/fs/bcachefs/btree_locking.h
index c74a5fd4d908ea977ff74eff575e78c1fff72afc..3356f089e268d452a80b583929a022c598f935ed 100644 (file)
--- a/fs/bcachefs/btree_locking.h
+++ b/fs/bcachefs/btree_locking.h
@@ -87,7 +87,7 @@ static inline void mark_btree_node_locked(struct btree_trans *trans,
 {
        mark_btree_node_locked_noreset(path, level, type);
 #ifdef CONFIG_BCACHEFS_LOCK_TIME_STATS
-       path->l[level].lock_taken_time = ktime_get_ns();
+       path->l[level].lock_taken_time = local_clock();
 #endif
 }
 
@@ -119,7 +119,7 @@ static void btree_trans_lock_hold_time_update(struct btree_trans *trans,
        if (s)
                __bch2_time_stats_update(&s->lock_hold_times,
                                         path->l[level].lock_taken_time,
-                                        ktime_get_ns());
+                                        local_clock());
 #endif
 }
 
@@ -259,7 +259,7 @@ static inline int btree_node_lock(struct btree_trans *trans,
            btree_node_lock_increment(trans, b, level, type) ||
            !(ret = btree_node_lock_nopath(trans, b, type))) {
 #ifdef CONFIG_BCACHEFS_LOCK_TIME_STATS
-               path->l[b->level].lock_taken_time = ktime_get_ns();
+               path->l[b->level].lock_taken_time = local_clock();
 #endif
        }