Commit | Line | Data |
---|---|---|
1c6fdbd8 KO |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _BCACHEFS_BTREE_LOCKING_H | |
3 | #define _BCACHEFS_BTREE_LOCKING_H | |
4 | ||
5 | /* | |
6 | * Only for internal btree use: | |
7 | * | |
8 | * The btree iterator tracks what locks it wants to take, and what locks it | |
9 | * currently has - here we have wrappers for locking/unlocking btree nodes and | |
10 | * updating the iterator state | |
11 | */ | |
12 | ||
13 | #include "btree_iter.h" | |
1c6fdbd8 KO |
14 | #include "six.h" |
15 | ||
16 | /* matches six lock types */ | |
17 | enum btree_node_locked_type { | |
18 | BTREE_NODE_UNLOCKED = -1, | |
19 | BTREE_NODE_READ_LOCKED = SIX_LOCK_read, | |
20 | BTREE_NODE_INTENT_LOCKED = SIX_LOCK_intent, | |
21 | }; | |
22 | ||
23 | static inline int btree_node_locked_type(struct btree_iter *iter, | |
24 | unsigned level) | |
25 | { | |
26 | /* | |
27 | * We're relying on the fact that if nodes_intent_locked is set | |
28 | * nodes_locked must be set as well, so that we can compute without | |
29 | * branches: | |
30 | */ | |
31 | return BTREE_NODE_UNLOCKED + | |
32 | ((iter->nodes_locked >> level) & 1) + | |
33 | ((iter->nodes_intent_locked >> level) & 1); | |
34 | } | |
35 | ||
36 | static inline bool btree_node_intent_locked(struct btree_iter *iter, | |
37 | unsigned level) | |
38 | { | |
39 | return btree_node_locked_type(iter, level) == BTREE_NODE_INTENT_LOCKED; | |
40 | } | |
41 | ||
42 | static inline bool btree_node_read_locked(struct btree_iter *iter, | |
43 | unsigned level) | |
44 | { | |
45 | return btree_node_locked_type(iter, level) == BTREE_NODE_READ_LOCKED; | |
46 | } | |
47 | ||
48 | static inline bool btree_node_locked(struct btree_iter *iter, unsigned level) | |
49 | { | |
50 | return iter->nodes_locked & (1 << level); | |
51 | } | |
52 | ||
53 | static inline void mark_btree_node_unlocked(struct btree_iter *iter, | |
54 | unsigned level) | |
55 | { | |
56 | iter->nodes_locked &= ~(1 << level); | |
57 | iter->nodes_intent_locked &= ~(1 << level); | |
58 | } | |
59 | ||
60 | static inline void mark_btree_node_locked(struct btree_iter *iter, | |
61 | unsigned level, | |
62 | enum six_lock_type type) | |
63 | { | |
64 | /* relying on this to avoid a branch */ | |
65 | BUILD_BUG_ON(SIX_LOCK_read != 0); | |
66 | BUILD_BUG_ON(SIX_LOCK_intent != 1); | |
67 | ||
68 | iter->nodes_locked |= 1 << level; | |
69 | iter->nodes_intent_locked |= type << level; | |
70 | } | |
71 | ||
72 | static inline void mark_btree_node_intent_locked(struct btree_iter *iter, | |
73 | unsigned level) | |
74 | { | |
75 | mark_btree_node_locked(iter, level, SIX_LOCK_intent); | |
76 | } | |
77 | ||
78 | static inline enum six_lock_type __btree_lock_want(struct btree_iter *iter, int level) | |
79 | { | |
80 | return level < iter->locks_want | |
81 | ? SIX_LOCK_intent | |
82 | : SIX_LOCK_read; | |
83 | } | |
84 | ||
85 | static inline enum btree_node_locked_type | |
86 | btree_lock_want(struct btree_iter *iter, int level) | |
87 | { | |
88 | if (level < iter->level) | |
89 | return BTREE_NODE_UNLOCKED; | |
90 | if (level < iter->locks_want) | |
91 | return BTREE_NODE_INTENT_LOCKED; | |
92 | if (level == iter->level) | |
93 | return BTREE_NODE_READ_LOCKED; | |
94 | return BTREE_NODE_UNLOCKED; | |
95 | } | |
96 | ||
ad7ae8d6 | 97 | static inline void __btree_node_unlock(struct btree_iter *iter, unsigned level) |
1c6fdbd8 KO |
98 | { |
99 | int lock_type = btree_node_locked_type(iter, level); | |
100 | ||
101 | EBUG_ON(level >= BTREE_MAX_DEPTH); | |
102 | ||
103 | if (lock_type != BTREE_NODE_UNLOCKED) | |
c43a6ef9 | 104 | six_unlock_type(&iter->l[level].b->c.lock, lock_type); |
1c6fdbd8 KO |
105 | mark_btree_node_unlocked(iter, level); |
106 | } | |
107 | ||
ad7ae8d6 KO |
108 | static inline void btree_node_unlock(struct btree_iter *iter, unsigned level) |
109 | { | |
60755344 | 110 | EBUG_ON(!level && iter->trans->nounlock); |
ad7ae8d6 KO |
111 | |
112 | __btree_node_unlock(iter, level); | |
113 | } | |
114 | ||
1c6fdbd8 KO |
115 | static inline void __bch2_btree_iter_unlock(struct btree_iter *iter) |
116 | { | |
117 | btree_iter_set_dirty(iter, BTREE_ITER_NEED_RELOCK); | |
118 | ||
119 | while (iter->nodes_locked) | |
120 | btree_node_unlock(iter, __ffs(iter->nodes_locked)); | |
121 | } | |
122 | ||
123 | static inline enum bch_time_stats lock_to_time_stat(enum six_lock_type type) | |
124 | { | |
125 | switch (type) { | |
126 | case SIX_LOCK_read: | |
127 | return BCH_TIME_btree_lock_contended_read; | |
128 | case SIX_LOCK_intent: | |
129 | return BCH_TIME_btree_lock_contended_intent; | |
130 | case SIX_LOCK_write: | |
131 | return BCH_TIME_btree_lock_contended_write; | |
132 | default: | |
133 | BUG(); | |
134 | } | |
135 | } | |
136 | ||
137 | /* | |
138 | * wrapper around six locks that just traces lock contended time | |
139 | */ | |
140 | static inline void __btree_node_lock_type(struct bch_fs *c, struct btree *b, | |
141 | enum six_lock_type type) | |
142 | { | |
143 | u64 start_time = local_clock(); | |
144 | ||
c43a6ef9 | 145 | six_lock_type(&b->c.lock, type, NULL, NULL); |
1c6fdbd8 KO |
146 | bch2_time_stats_update(&c->times[lock_to_time_stat(type)], start_time); |
147 | } | |
148 | ||
149 | static inline void btree_node_lock_type(struct bch_fs *c, struct btree *b, | |
150 | enum six_lock_type type) | |
151 | { | |
c43a6ef9 | 152 | if (!six_trylock_type(&b->c.lock, type)) |
1c6fdbd8 KO |
153 | __btree_node_lock_type(c, b, type); |
154 | } | |
155 | ||
647d7b60 KO |
156 | /* |
157 | * Lock a btree node if we already have it locked on one of our linked | |
158 | * iterators: | |
159 | */ | |
515282ac | 160 | static inline bool btree_node_lock_increment(struct btree_trans *trans, |
647d7b60 KO |
161 | struct btree *b, unsigned level, |
162 | enum btree_node_locked_type want) | |
163 | { | |
515282ac | 164 | struct btree_iter *iter; |
647d7b60 | 165 | |
515282ac KO |
166 | trans_for_each_iter(trans, iter) |
167 | if (iter->l[level].b == b && | |
168 | btree_node_locked_type(iter, level) >= want) { | |
c43a6ef9 | 169 | six_lock_increment(&b->c.lock, want); |
647d7b60 KO |
170 | return true; |
171 | } | |
172 | ||
173 | return false; | |
174 | } | |
175 | ||
1c6fdbd8 | 176 | bool __bch2_btree_node_lock(struct btree *, struct bpos, unsigned, |
bd2bb273 | 177 | struct btree_iter *, enum six_lock_type, |
a301dc38 KO |
178 | six_lock_should_sleep_fn, void *, |
179 | unsigned long); | |
1c6fdbd8 | 180 | |
bd2bb273 KO |
181 | static inline bool btree_node_lock(struct btree *b, |
182 | struct bpos pos, unsigned level, | |
183 | struct btree_iter *iter, | |
184 | enum six_lock_type type, | |
a301dc38 KO |
185 | six_lock_should_sleep_fn should_sleep_fn, void *p, |
186 | unsigned long ip) | |
1c6fdbd8 | 187 | { |
515282ac | 188 | struct btree_trans *trans = iter->trans; |
495aabed KO |
189 | bool ret; |
190 | ||
1c6fdbd8 | 191 | EBUG_ON(level >= BTREE_MAX_DEPTH); |
bd2bb273 KO |
192 | EBUG_ON(!(trans->iters_linked & (1ULL << iter->idx))); |
193 | ||
495aabed | 194 | #ifdef CONFIG_BCACHEFS_DEBUG |
515282ac KO |
195 | trans->locking = b; |
196 | trans->locking_iter_idx = iter->idx; | |
197 | trans->locking_pos = pos; | |
198 | trans->locking_btree_id = iter->btree_id; | |
199 | trans->locking_level = level; | |
495aabed | 200 | #endif |
515282ac KO |
201 | ret = likely(six_trylock_type(&b->c.lock, type)) || |
202 | btree_node_lock_increment(trans, b, level, type) || | |
bd2bb273 | 203 | __bch2_btree_node_lock(b, pos, level, iter, type, |
a301dc38 | 204 | should_sleep_fn, p, ip); |
495aabed KO |
205 | |
206 | #ifdef CONFIG_BCACHEFS_DEBUG | |
515282ac | 207 | trans->locking = NULL; |
495aabed KO |
208 | #endif |
209 | return ret; | |
1c6fdbd8 KO |
210 | } |
211 | ||
212 | bool __bch2_btree_node_relock(struct btree_iter *, unsigned); | |
213 | ||
214 | static inline bool bch2_btree_node_relock(struct btree_iter *iter, | |
215 | unsigned level) | |
216 | { | |
217 | EBUG_ON(btree_node_locked(iter, level) && | |
218 | btree_node_locked_type(iter, level) != | |
219 | __btree_lock_want(iter, level)); | |
220 | ||
221 | return likely(btree_node_locked(iter, level)) || | |
222 | __bch2_btree_node_relock(iter, level); | |
223 | } | |
224 | ||
b7ba66c8 KO |
225 | /* |
226 | * Updates the saved lock sequence number, so that bch2_btree_node_relock() will | |
227 | * succeed: | |
228 | */ | |
229 | static inline void | |
230 | bch2_btree_node_unlock_write_inlined(struct btree *b, struct btree_iter *iter) | |
231 | { | |
232 | struct btree_iter *linked; | |
233 | ||
234 | EBUG_ON(iter->l[b->c.level].b != b); | |
235 | EBUG_ON(iter->l[b->c.level].lock_seq + 1 != b->c.lock.state.seq); | |
236 | ||
237 | trans_for_each_iter_with_node(iter->trans, b, linked) | |
238 | linked->l[b->c.level].lock_seq += 2; | |
239 | ||
240 | six_unlock_write(&b->c.lock); | |
241 | } | |
242 | ||
1c6fdbd8 KO |
243 | void bch2_btree_node_unlock_write(struct btree *, struct btree_iter *); |
244 | ||
245 | void __bch2_btree_node_lock_write(struct btree *, struct btree_iter *); | |
246 | ||
247 | static inline void bch2_btree_node_lock_write(struct btree *b, struct btree_iter *iter) | |
248 | { | |
c43a6ef9 KO |
249 | EBUG_ON(iter->l[b->c.level].b != b); |
250 | EBUG_ON(iter->l[b->c.level].lock_seq != b->c.lock.state.seq); | |
1c6fdbd8 | 251 | |
fdfab313 | 252 | if (unlikely(!six_trylock_write(&b->c.lock))) |
1c6fdbd8 KO |
253 | __bch2_btree_node_lock_write(b, iter); |
254 | } | |
255 | ||
256 | #endif /* _BCACHEFS_BTREE_LOCKING_H */ | |
257 | ||
258 |