Commit | Line | Data |
---|---|---|
1c6fdbd8 KO |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | #include "bcachefs.h" | |
36e916e1 | 4 | #include "btree_gc.h" |
1c6fdbd8 KO |
5 | #include "btree_io.h" |
6 | #include "btree_iter.h" | |
401585fe | 7 | #include "btree_journal_iter.h" |
2ca88e5a | 8 | #include "btree_key_cache.h" |
8079aab0 | 9 | #include "btree_update_interior.h" |
920e69bc | 10 | #include "btree_write_buffer.h" |
b35b1925 | 11 | #include "buckets.h" |
549d173c | 12 | #include "errcode.h" |
3636ed48 | 13 | #include "error.h" |
1c6fdbd8 KO |
14 | #include "journal.h" |
15 | #include "journal_reclaim.h" | |
1d25849c | 16 | #include "replicas.h" |
8e877caa | 17 | #include "snapshot.h" |
1c6fdbd8 | 18 | |
b7ba66c8 | 19 | #include <linux/prefetch.h> |
a52a4da4 | 20 | |
08f78031 KO |
21 | static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i) |
22 | { | |
23 | #ifdef CONFIG_BCACHEFS_DEBUG | |
24 | struct bch_fs *c = trans->c; | |
25 | struct bkey u; | |
26 | struct bkey_s_c k = bch2_btree_path_peek_slot_exact(i->path, &u); | |
27 | ||
28 | if (unlikely(trans->journal_replay_not_finished)) { | |
29 | struct bkey_i *j_k = | |
30 | bch2_journal_keys_peek_slot(c, i->btree_id, i->level, i->k->k.p); | |
31 | ||
32 | if (j_k) | |
33 | k = bkey_i_to_s_c(j_k); | |
34 | } | |
35 | ||
70f0b0fd KO |
36 | u = *k.k; |
37 | u.needs_whiteout = i->old_k.needs_whiteout; | |
08f78031 | 38 | |
70f0b0fd | 39 | BUG_ON(memcmp(&i->old_k, &u, sizeof(struct bkey))); |
08f78031 KO |
40 | BUG_ON(i->old_v != k.v); |
41 | #endif | |
42 | } | |
43 | ||
67e0dd8f | 44 | static inline struct btree_path_level *insert_l(struct btree_insert_entry *i) |
6fba6b83 | 45 | { |
67e0dd8f | 46 | return i->path->l + i->level; |
6fba6b83 KO |
47 | } |
48 | ||
36e9d698 | 49 | static inline bool same_leaf_as_prev(struct btree_trans *trans, |
24326cd1 | 50 | struct btree_insert_entry *i) |
36e9d698 | 51 | { |
cd8319fd | 52 | return i != trans->updates && |
6fba6b83 | 53 | insert_l(&i[0])->b == insert_l(&i[-1])->b; |
36e9d698 KO |
54 | } |
55 | ||
05046a96 KO |
56 | static inline bool same_leaf_as_next(struct btree_trans *trans, |
57 | struct btree_insert_entry *i) | |
58 | { | |
59 | return i + 1 < trans->updates + trans->nr_updates && | |
6fba6b83 | 60 | insert_l(&i[0])->b == insert_l(&i[1])->b; |
05046a96 KO |
61 | } |
62 | ||
1ff7849f KO |
63 | inline void bch2_btree_node_prep_for_write(struct btree_trans *trans, |
64 | struct btree_path *path, | |
65 | struct btree *b) | |
9623ab27 | 66 | { |
e3a67bdb KO |
67 | struct bch_fs *c = trans->c; |
68 | ||
fdfab313 | 69 | if (unlikely(btree_node_just_written(b)) && |
9623ab27 | 70 | bch2_btree_post_write_cleanup(c, b)) |
f7a966a3 | 71 | bch2_trans_node_reinit_iter(trans, b); |
9623ab27 KO |
72 | |
73 | /* | |
74 | * If the last bset has been written, or if it's gotten too big - start | |
75 | * a new bset to insert into: | |
76 | */ | |
77 | if (want_new_bset(c, b)) | |
f7a966a3 | 78 | bch2_btree_init_next(trans, b); |
9623ab27 KO |
79 | } |
80 | ||
1c6fdbd8 KO |
81 | /* Inserting into a given leaf node (last stage of insert): */ |
82 | ||
83 | /* Handle overwrites and do insert, for non extents: */ | |
9f6bd307 | 84 | bool bch2_btree_bset_insert_key(struct btree_trans *trans, |
67e0dd8f | 85 | struct btree_path *path, |
1c6fdbd8 KO |
86 | struct btree *b, |
87 | struct btree_node_iter *node_iter, | |
88 | struct bkey_i *insert) | |
89 | { | |
1c6fdbd8 | 90 | struct bkey_packed *k; |
fdf22400 | 91 | unsigned clobber_u64s = 0, new_u64s = 0; |
1c6fdbd8 KO |
92 | |
93 | EBUG_ON(btree_node_just_written(b)); | |
94 | EBUG_ON(bset_written(b, btree_bset_last(b))); | |
95 | EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k)); | |
e88a75eb KO |
96 | EBUG_ON(bpos_lt(insert->k.p, b->data->min_key)); |
97 | EBUG_ON(bpos_gt(insert->k.p, b->data->max_key)); | |
e3e464ac | 98 | EBUG_ON(insert->k.u64s > |
9f6bd307 | 99 | bch_btree_keys_u64s_remaining(trans->c, b)); |
da525760 | 100 | EBUG_ON(!b->c.level && !bpos_eq(insert->k.p, path->pos)); |
1c6fdbd8 KO |
101 | |
102 | k = bch2_btree_node_iter_peek_all(node_iter, b); | |
811d2bcd | 103 | if (k && bkey_cmp_left_packed(b, k, &insert->k.p)) |
ae54c453 | 104 | k = NULL; |
1c6fdbd8 | 105 | |
ae54c453 | 106 | /* @k is the key being overwritten/deleted, if any: */ |
c052cf82 | 107 | EBUG_ON(k && bkey_deleted(k)); |
1c6fdbd8 | 108 | |
fdf22400 | 109 | /* Deleting, but not found? nothing to do: */ |
c052cf82 | 110 | if (bkey_deleted(&insert->k) && !k) |
fdf22400 KO |
111 | return false; |
112 | ||
c052cf82 | 113 | if (bkey_deleted(&insert->k)) { |
ae54c453 | 114 | /* Deleting: */ |
ae54c453 KO |
115 | btree_account_key_drop(b, k); |
116 | k->type = KEY_TYPE_deleted; | |
c9bebae6 | 117 | |
fdf22400 | 118 | if (k->needs_whiteout) |
9f6bd307 | 119 | push_whiteout(trans->c, b, insert->k.p); |
fdf22400 | 120 | k->needs_whiteout = false; |
1c6fdbd8 | 121 | |
ae54c453 KO |
122 | if (k >= btree_bset_last(b)->start) { |
123 | clobber_u64s = k->u64s; | |
ae54c453 | 124 | bch2_bset_delete(b, k, clobber_u64s); |
fdf22400 | 125 | goto fix_iter; |
ae54c453 | 126 | } else { |
67e0dd8f | 127 | bch2_btree_path_fix_key_modified(trans, b, k); |
ae54c453 KO |
128 | } |
129 | ||
130 | return true; | |
131 | } | |
c9bebae6 | 132 | |
ae54c453 KO |
133 | if (k) { |
134 | /* Overwriting: */ | |
ae54c453 KO |
135 | btree_account_key_drop(b, k); |
136 | k->type = KEY_TYPE_deleted; | |
137 | ||
f2e8c69f KO |
138 | insert->k.needs_whiteout = k->needs_whiteout; |
139 | k->needs_whiteout = false; | |
140 | ||
c9bebae6 KO |
141 | if (k >= btree_bset_last(b)->start) { |
142 | clobber_u64s = k->u64s; | |
1c6fdbd8 | 143 | goto overwrite; |
ae54c453 | 144 | } else { |
67e0dd8f | 145 | bch2_btree_path_fix_key_modified(trans, b, k); |
1c6fdbd8 | 146 | } |
1c6fdbd8 KO |
147 | } |
148 | ||
216c9fac | 149 | k = bch2_btree_node_iter_bset_pos(node_iter, b, bset_tree_last(b)); |
1c6fdbd8 KO |
150 | overwrite: |
151 | bch2_bset_insert(b, node_iter, k, insert, clobber_u64s); | |
fdf22400 KO |
152 | new_u64s = k->u64s; |
153 | fix_iter: | |
154 | if (clobber_u64s != new_u64s) | |
67e0dd8f | 155 | bch2_btree_node_iter_fix(trans, path, b, node_iter, k, |
fdf22400 | 156 | clobber_u64s, new_u64s); |
1c6fdbd8 KO |
157 | return true; |
158 | } | |
159 | ||
2940295c | 160 | static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, |
1c6fdbd8 KO |
161 | unsigned i, u64 seq) |
162 | { | |
163 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
164 | struct btree_write *w = container_of(pin, struct btree_write, journal); | |
165 | struct btree *b = container_of(w, struct btree, writes[i]); | |
6bd68ec2 | 166 | struct btree_trans *trans = bch2_trans_get(c); |
6f5f747c KO |
167 | unsigned long old, new, v; |
168 | unsigned idx = w - b->writes; | |
1c6fdbd8 | 169 | |
6bd68ec2 | 170 | btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read); |
6f5f747c KO |
171 | v = READ_ONCE(b->flags); |
172 | ||
173 | do { | |
174 | old = new = v; | |
175 | ||
176 | if (!(old & (1 << BTREE_NODE_dirty)) || | |
177 | !!(old & (1 << BTREE_NODE_write_idx)) != idx || | |
178 | w->journal.seq != seq) | |
179 | break; | |
180 | ||
42af0ad5 KO |
181 | new &= ~BTREE_WRITE_TYPE_MASK; |
182 | new |= BTREE_WRITE_journal_reclaim; | |
6f5f747c KO |
183 | new |= 1 << BTREE_NODE_need_write; |
184 | } while ((v = cmpxchg(&b->flags, old, new)) != old); | |
185 | ||
186 | btree_node_write_if_need(c, b, SIX_LOCK_read); | |
c43a6ef9 | 187 | six_unlock_read(&b->c.lock); |
ca7d8fca | 188 | |
6bd68ec2 | 189 | bch2_trans_put(trans); |
2940295c | 190 | return 0; |
1c6fdbd8 KO |
191 | } |
192 | ||
83ec519a | 193 | int bch2_btree_node_flush0(struct journal *j, struct journal_entry_pin *pin, u64 seq) |
1c6fdbd8 KO |
194 | { |
195 | return __btree_node_flush(j, pin, 0, seq); | |
196 | } | |
197 | ||
83ec519a | 198 | int bch2_btree_node_flush1(struct journal *j, struct journal_entry_pin *pin, u64 seq) |
1c6fdbd8 KO |
199 | { |
200 | return __btree_node_flush(j, pin, 1, seq); | |
201 | } | |
202 | ||
6357d607 KO |
203 | inline void bch2_btree_add_journal_pin(struct bch_fs *c, |
204 | struct btree *b, u64 seq) | |
205 | { | |
206 | struct btree_write *w = btree_current_write(b); | |
207 | ||
208 | bch2_journal_pin_add(&c->journal, seq, &w->journal, | |
209 | btree_node_write_idx(b) == 0 | |
83ec519a KO |
210 | ? bch2_btree_node_flush0 |
211 | : bch2_btree_node_flush1); | |
6357d607 KO |
212 | } |
213 | ||
1c6fdbd8 | 214 | /** |
96dea3d5 KO |
215 | * bch2_btree_insert_key_leaf() - insert a key one key into a leaf node |
216 | * @trans: btree transaction object | |
217 | * @path: path pointing to @insert's pos | |
218 | * @insert: key to insert | |
219 | * @journal_seq: sequence number of journal reservation | |
1c6fdbd8 | 220 | */ |
920e69bc KO |
221 | inline void bch2_btree_insert_key_leaf(struct btree_trans *trans, |
222 | struct btree_path *path, | |
223 | struct bkey_i *insert, | |
224 | u64 journal_seq) | |
1c6fdbd8 KO |
225 | { |
226 | struct bch_fs *c = trans->c; | |
920e69bc | 227 | struct btree *b = path_l(path)->b; |
2a9101a9 | 228 | struct bset_tree *t = bset_tree_last(b); |
4e8224ed | 229 | struct bset *i = bset(b, t); |
2a9101a9 | 230 | int old_u64s = bset_u64s(t); |
1c6fdbd8 KO |
231 | int old_live_u64s = b->nr.live_u64s; |
232 | int live_u64s_added, u64s_added; | |
233 | ||
920e69bc KO |
234 | if (unlikely(!bch2_btree_bset_insert_key(trans, path, b, |
235 | &path_l(path)->iter, insert))) | |
0576ba9a | 236 | return; |
4e8224ed | 237 | |
920e69bc | 238 | i->journal_seq = cpu_to_le64(max(journal_seq, le64_to_cpu(i->journal_seq))); |
bcd6f3e0 | 239 | |
920e69bc | 240 | bch2_btree_add_journal_pin(c, b, journal_seq); |
4e8224ed | 241 | |
30a8278a KO |
242 | if (unlikely(!btree_node_dirty(b))) { |
243 | EBUG_ON(test_bit(BCH_FS_CLEAN_SHUTDOWN, &c->flags)); | |
de517c95 | 244 | set_btree_node_dirty_acct(c, b); |
30a8278a | 245 | } |
1c6fdbd8 KO |
246 | |
247 | live_u64s_added = (int) b->nr.live_u64s - old_live_u64s; | |
2a9101a9 | 248 | u64s_added = (int) bset_u64s(t) - old_u64s; |
1c6fdbd8 KO |
249 | |
250 | if (b->sib_u64s[0] != U16_MAX && live_u64s_added < 0) | |
251 | b->sib_u64s[0] = max(0, (int) b->sib_u64s[0] + live_u64s_added); | |
252 | if (b->sib_u64s[1] != U16_MAX && live_u64s_added < 0) | |
253 | b->sib_u64s[1] = max(0, (int) b->sib_u64s[1] + live_u64s_added); | |
254 | ||
255 | if (u64s_added > live_u64s_added && | |
256 | bch2_maybe_compact_whiteouts(c, b)) | |
f7a966a3 | 257 | bch2_trans_node_reinit_iter(trans, b); |
1c6fdbd8 KO |
258 | } |
259 | ||
2ca88e5a KO |
260 | /* Cached btree updates: */ |
261 | ||
9623ab27 | 262 | /* Normal update interface: */ |
1c6fdbd8 | 263 | |
1a470560 | 264 | static inline void btree_insert_entry_checks(struct btree_trans *trans, |
6333bd2f | 265 | struct btree_insert_entry *i) |
1c6fdbd8 | 266 | { |
e88a75eb | 267 | BUG_ON(!bpos_eq(i->k->k.p, i->path->pos)); |
67e0dd8f KO |
268 | BUG_ON(i->cached != i->path->cached); |
269 | BUG_ON(i->level != i->path->level); | |
270 | BUG_ON(i->btree_id != i->path->btree_id); | |
14b393ee KO |
271 | EBUG_ON(!i->level && |
272 | !(i->flags & BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) && | |
273 | test_bit(JOURNAL_REPLAY_DONE, &trans->c->journal.flags) && | |
274 | i->k->k.p.snapshot && | |
8479938d | 275 | bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot)); |
9623ab27 | 276 | } |
1c6fdbd8 | 277 | |
2a9101a9 | 278 | static noinline int |
30ca6ece | 279 | bch2_trans_journal_preres_get_cold(struct btree_trans *trans, unsigned flags, |
531a0095 | 280 | unsigned long trace_ip) |
1c6fdbd8 | 281 | { |
b5fd7566 KO |
282 | return drop_locks_do(trans, |
283 | bch2_journal_preres_get(&trans->c->journal, | |
30ca6ece KO |
284 | &trans->journal_preres, |
285 | trans->journal_preres_u64s, | |
ec14fc60 | 286 | (flags & BCH_WATERMARK_MASK))); |
1c6fdbd8 KO |
287 | } |
288 | ||
0cc455b3 | 289 | static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans, |
30ca6ece | 290 | unsigned flags) |
c8cc5b3e | 291 | { |
87ced107 | 292 | return bch2_journal_res_get(&trans->c->journal, &trans->journal_res, |
30ca6ece | 293 | trans->journal_u64s, flags); |
9623ab27 | 294 | } |
1c6fdbd8 | 295 | |
fb64f3fd KO |
296 | #define JSET_ENTRY_LOG_U64s 4 |
297 | ||
298 | static noinline void journal_transaction_name(struct btree_trans *trans) | |
299 | { | |
300 | struct bch_fs *c = trans->c; | |
43ddf448 KO |
301 | struct journal *j = &c->journal; |
302 | struct jset_entry *entry = | |
303 | bch2_journal_add_entry(j, &trans->journal_res, | |
304 | BCH_JSET_ENTRY_log, 0, 0, | |
305 | JSET_ENTRY_LOG_U64s); | |
306 | struct jset_entry_log *l = | |
307 | container_of(entry, struct jset_entry_log, entry); | |
308 | ||
309 | strncpy(l->d, trans->fn, JSET_ENTRY_LOG_U64s * sizeof(u64)); | |
fb64f3fd KO |
310 | } |
311 | ||
ac9fa4bd KO |
312 | static inline int btree_key_can_insert(struct btree_trans *trans, |
313 | struct btree *b, unsigned u64s) | |
b0004d8d KO |
314 | { |
315 | struct bch_fs *c = trans->c; | |
b0004d8d | 316 | |
f8058242 | 317 | if (!bch2_btree_node_insert_fits(c, b, u64s)) |
ac9fa4bd | 318 | return -BCH_ERR_btree_insert_btree_node_full; |
b0004d8d | 319 | |
ac9fa4bd | 320 | return 0; |
b0004d8d KO |
321 | } |
322 | ||
30ca6ece KO |
323 | static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags, |
324 | struct btree_path *path, unsigned u64s) | |
2ca88e5a | 325 | { |
f0f41a6d | 326 | struct bch_fs *c = trans->c; |
67e0dd8f | 327 | struct bkey_cached *ck = (void *) path->l[0].b; |
08f78031 | 328 | struct btree_insert_entry *i; |
f83009cd | 329 | unsigned new_u64s; |
2ca88e5a KO |
330 | struct bkey_i *new_k; |
331 | ||
67e0dd8f | 332 | EBUG_ON(path->level); |
2ca88e5a | 333 | |
d5425a3b | 334 | if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags) && |
f0f41a6d | 335 | bch2_btree_key_cache_must_wait(c) && |
30ca6ece | 336 | !(flags & BTREE_INSERT_JOURNAL_RECLAIM)) |
ac9fa4bd | 337 | return -BCH_ERR_btree_insert_need_journal_reclaim; |
d5425a3b | 338 | |
bc2e5d5c KO |
339 | /* |
340 | * bch2_varint_decode can read past the end of the buffer by at most 7 | |
341 | * bytes (it won't be used): | |
342 | */ | |
343 | u64s += 1; | |
344 | ||
64f2a880 | 345 | if (u64s <= ck->u64s) |
ac9fa4bd | 346 | return 0; |
2ca88e5a | 347 | |
64f2a880 | 348 | new_u64s = roundup_pow_of_two(u64s); |
2ca88e5a | 349 | new_k = krealloc(ck->k, new_u64s * sizeof(u64), GFP_NOFS); |
f0f41a6d KO |
350 | if (!new_k) { |
351 | bch_err(c, "error allocating memory for key cache key, btree %s u64s %u", | |
352 | bch2_btree_ids[path->btree_id], new_u64s); | |
65d48e35 | 353 | return -BCH_ERR_ENOMEM_btree_key_cache_insert; |
f0f41a6d | 354 | } |
2ca88e5a | 355 | |
08f78031 KO |
356 | trans_for_each_update(trans, i) |
357 | if (i->old_v == &ck->k->v) | |
358 | i->old_v = &new_k->v; | |
359 | ||
2ca88e5a KO |
360 | ck->u64s = new_u64s; |
361 | ck->k = new_k; | |
f83009cd | 362 | return 0; |
2ca88e5a KO |
363 | } |
364 | ||
3598c56e KO |
365 | /* Triggers: */ |
366 | ||
367 | static int run_one_mem_trigger(struct btree_trans *trans, | |
368 | struct btree_insert_entry *i, | |
369 | unsigned flags) | |
370 | { | |
96d3a0af | 371 | struct bkey_s_c old = { &i->old_k, i->old_v }; |
3598c56e | 372 | struct bkey_i *new = i->k; |
183e9c43 KO |
373 | const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type); |
374 | const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); | |
3598c56e KO |
375 | int ret; |
376 | ||
08f78031 KO |
377 | verify_update_old_key(trans, i); |
378 | ||
3598c56e KO |
379 | if (unlikely(flags & BTREE_TRIGGER_NORUN)) |
380 | return 0; | |
381 | ||
bf5a261c | 382 | if (!btree_node_type_needs_gc((enum btree_node_type) i->btree_id)) |
3598c56e KO |
383 | return 0; |
384 | ||
183e9c43 | 385 | if (old_ops->atomic_trigger == new_ops->atomic_trigger && |
3598c56e | 386 | ((1U << old.k->type) & BTREE_TRIGGER_WANTS_OLD_AND_NEW)) { |
2611a041 KO |
387 | ret = bch2_mark_key(trans, i->btree_id, i->level, |
388 | old, bkey_i_to_s_c(new), | |
3598c56e KO |
389 | BTREE_TRIGGER_INSERT|BTREE_TRIGGER_OVERWRITE|flags); |
390 | } else { | |
96d3a0af KO |
391 | struct bkey _deleted = KEY(0, 0, 0); |
392 | struct bkey_s_c deleted = (struct bkey_s_c) { &_deleted, NULL }; | |
393 | ||
394 | _deleted.p = i->path->pos; | |
395 | ||
2611a041 KO |
396 | ret = bch2_mark_key(trans, i->btree_id, i->level, |
397 | deleted, bkey_i_to_s_c(new), | |
3598c56e | 398 | BTREE_TRIGGER_INSERT|flags) ?: |
2611a041 KO |
399 | bch2_mark_key(trans, i->btree_id, i->level, |
400 | old, deleted, | |
3598c56e KO |
401 | BTREE_TRIGGER_OVERWRITE|flags); |
402 | } | |
403 | ||
404 | return ret; | |
405 | } | |
406 | ||
96d3a0af | 407 | static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i, |
f13fd87a | 408 | bool overwrite) |
3598c56e | 409 | { |
96d3a0af KO |
410 | /* |
411 | * Transactional triggers create new btree_insert_entries, so we can't | |
412 | * pass them a pointer to a btree_insert_entry, that memory is going to | |
413 | * move: | |
414 | */ | |
415 | struct bkey old_k = i->old_k; | |
416 | struct bkey_s_c old = { &old_k, i->old_v }; | |
183e9c43 KO |
417 | const struct bkey_ops *old_ops = bch2_bkey_type_ops(old.k->type); |
418 | const struct bkey_ops *new_ops = bch2_bkey_type_ops(i->k->k.type); | |
3598c56e | 419 | |
08f78031 KO |
420 | verify_update_old_key(trans, i); |
421 | ||
3598c56e KO |
422 | if ((i->flags & BTREE_TRIGGER_NORUN) || |
423 | !(BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type))) | |
424 | return 0; | |
425 | ||
f13fd87a KO |
426 | if (!i->insert_trigger_run && |
427 | !i->overwrite_trigger_run && | |
183e9c43 | 428 | old_ops->trans_trigger == new_ops->trans_trigger && |
3598c56e KO |
429 | ((1U << old.k->type) & BTREE_TRIGGER_WANTS_OLD_AND_NEW)) { |
430 | i->overwrite_trigger_run = true; | |
f13fd87a | 431 | i->insert_trigger_run = true; |
e1b8f5f5 | 432 | return bch2_trans_mark_key(trans, i->btree_id, i->level, old, i->k, |
f13fd87a KO |
433 | BTREE_TRIGGER_INSERT| |
434 | BTREE_TRIGGER_OVERWRITE| | |
435 | i->flags) ?: 1; | |
436 | } else if (overwrite && !i->overwrite_trigger_run) { | |
437 | i->overwrite_trigger_run = true; | |
e1b8f5f5 | 438 | return bch2_trans_mark_old(trans, i->btree_id, i->level, old, i->flags) ?: 1; |
f13fd87a KO |
439 | } else if (!overwrite && !i->insert_trigger_run) { |
440 | i->insert_trigger_run = true; | |
e1b8f5f5 | 441 | return bch2_trans_mark_new(trans, i->btree_id, i->level, i->k, i->flags) ?: 1; |
3598c56e | 442 | } else { |
f13fd87a | 443 | return 0; |
3598c56e | 444 | } |
3598c56e KO |
445 | } |
446 | ||
447 | static int run_btree_triggers(struct btree_trans *trans, enum btree_id btree_id, | |
448 | struct btree_insert_entry *btree_id_start) | |
449 | { | |
450 | struct btree_insert_entry *i; | |
451 | bool trans_trigger_run; | |
452 | int ret, overwrite; | |
453 | ||
f13fd87a | 454 | for (overwrite = 1; overwrite >= 0; --overwrite) { |
3598c56e KO |
455 | |
456 | /* | |
457 | * Running triggers will append more updates to the list of updates as | |
458 | * we're walking it: | |
459 | */ | |
460 | do { | |
461 | trans_trigger_run = false; | |
462 | ||
463 | for (i = btree_id_start; | |
464 | i < trans->updates + trans->nr_updates && i->btree_id <= btree_id; | |
465 | i++) { | |
0c10cf85 KO |
466 | if (i->btree_id != btree_id) |
467 | continue; | |
468 | ||
3598c56e KO |
469 | ret = run_one_trans_trigger(trans, i, overwrite); |
470 | if (ret < 0) | |
471 | return ret; | |
472 | if (ret) | |
473 | trans_trigger_run = true; | |
474 | } | |
475 | } while (trans_trigger_run); | |
476 | } | |
477 | ||
478 | return 0; | |
479 | } | |
480 | ||
481 | static int bch2_trans_commit_run_triggers(struct btree_trans *trans) | |
482 | { | |
483 | struct btree_insert_entry *i = NULL, *btree_id_start = trans->updates; | |
484 | unsigned btree_id = 0; | |
485 | int ret = 0; | |
486 | ||
487 | /* | |
488 | * | |
489 | * For a given btree, this algorithm runs insert triggers before | |
490 | * overwrite triggers: this is so that when extents are being moved | |
491 | * (e.g. by FALLOCATE_FL_INSERT_RANGE), we don't drop references before | |
492 | * they are re-added. | |
493 | */ | |
494 | for (btree_id = 0; btree_id < BTREE_ID_NR; btree_id++) { | |
0c10cf85 KO |
495 | if (btree_id == BTREE_ID_alloc) |
496 | continue; | |
497 | ||
3598c56e KO |
498 | while (btree_id_start < trans->updates + trans->nr_updates && |
499 | btree_id_start->btree_id < btree_id) | |
500 | btree_id_start++; | |
501 | ||
502 | ret = run_btree_triggers(trans, btree_id, btree_id_start); | |
503 | if (ret) | |
504 | return ret; | |
505 | } | |
506 | ||
0c10cf85 KO |
507 | trans_for_each_update(trans, i) { |
508 | if (i->btree_id > BTREE_ID_alloc) | |
509 | break; | |
510 | if (i->btree_id == BTREE_ID_alloc) { | |
511 | ret = run_btree_triggers(trans, BTREE_ID_alloc, i); | |
512 | if (ret) | |
513 | return ret; | |
514 | break; | |
515 | } | |
516 | } | |
517 | ||
c9ee99ad | 518 | #ifdef CONFIG_BCACHEFS_DEBUG |
3598c56e KO |
519 | trans_for_each_update(trans, i) |
520 | BUG_ON(!(i->flags & BTREE_TRIGGER_NORUN) && | |
521 | (BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type)) && | |
522 | (!i->insert_trigger_run || !i->overwrite_trigger_run)); | |
c9ee99ad | 523 | #endif |
3598c56e KO |
524 | return 0; |
525 | } | |
526 | ||
527 | static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans) | |
1c6fdbd8 KO |
528 | { |
529 | struct bch_fs *c = trans->c; | |
530 | struct btree_insert_entry *i; | |
21aec962 | 531 | int ret = 0; |
1c6fdbd8 | 532 | |
2ca88e5a KO |
533 | trans_for_each_update(trans, i) { |
534 | /* | |
535 | * XXX: synchronization of cached update triggers with gc | |
67e0dd8f | 536 | * XXX: synchronization of interior node updates with gc |
2ca88e5a | 537 | */ |
6fba6b83 | 538 | BUG_ON(i->cached || i->level); |
2ca88e5a | 539 | |
21aec962 | 540 | if (gc_visited(c, gc_pos_btree_node(insert_l(i)->b))) { |
3598c56e | 541 | ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_GC); |
21aec962 KO |
542 | if (ret) |
543 | break; | |
544 | } | |
2ca88e5a | 545 | } |
21aec962 KO |
546 | |
547 | return ret; | |
2a9101a9 | 548 | } |
36e9d698 | 549 | |
2a9101a9 | 550 | static inline int |
30ca6ece | 551 | bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, |
531a0095 KO |
552 | struct btree_insert_entry **stopped_at, |
553 | unsigned long trace_ip) | |
2a9101a9 KO |
554 | { |
555 | struct bch_fs *c = trans->c; | |
2a9101a9 | 556 | struct btree_insert_entry *i; |
920e69bc | 557 | struct btree_write_buffered_key *wb; |
43d00243 | 558 | struct btree_trans_commit_hook *h; |
24326cd1 | 559 | unsigned u64s = 0; |
2a9101a9 | 560 | int ret; |
4d8100da | 561 | |
1c6fdbd8 | 562 | if (race_fault()) { |
674cfc26 | 563 | trace_and_count(c, trans_restart_fault_inject, trans, trace_ip); |
549d173c | 564 | return btree_trans_restart_nounlock(trans, BCH_ERR_transaction_restart_fault_inject); |
1c6fdbd8 KO |
565 | } |
566 | ||
b0004d8d KO |
567 | /* |
568 | * Check if the insert will fit in the leaf node with the write lock | |
569 | * held, otherwise another thread could write the node changing the | |
570 | * amount of space available: | |
571 | */ | |
c8cc5b3e | 572 | |
b7ba66c8 | 573 | prefetch(&trans->c->journal.flags); |
932aa837 | 574 | |
cd8319fd | 575 | trans_for_each_update(trans, i) { |
b7ba66c8 | 576 | /* Multiple inserts might go to same leaf: */ |
24326cd1 | 577 | if (!same_leaf_as_prev(trans, i)) |
b7ba66c8 | 578 | u64s = 0; |
932aa837 | 579 | |
b7ba66c8 | 580 | u64s += i->k->k.u64s; |
6fba6b83 KO |
581 | ret = !i->cached |
582 | ? btree_key_can_insert(trans, insert_l(i)->b, u64s) | |
30ca6ece | 583 | : btree_key_can_insert_cached(trans, flags, i->path, u64s); |
b7ba66c8 KO |
584 | if (ret) { |
585 | *stopped_at = i; | |
586 | return ret; | |
932aa837 | 587 | } |
b7ba66c8 KO |
588 | } |
589 | ||
920e69bc KO |
590 | if (trans->nr_wb_updates && |
591 | trans->nr_wb_updates + c->btree_write_buffer.state.nr > c->btree_write_buffer.size) | |
592 | return -BCH_ERR_btree_insert_need_flush_buffer; | |
593 | ||
9623ab27 KO |
594 | /* |
595 | * Don't get journal reservation until after we know insert will | |
596 | * succeed: | |
597 | */ | |
30ca6ece | 598 | if (likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) { |
2a9101a9 | 599 | ret = bch2_trans_journal_res_get(trans, |
ec14fc60 | 600 | (flags & BCH_WATERMARK_MASK)| |
2a9101a9 | 601 | JOURNAL_RES_GET_NONBLOCK); |
87c3beb4 | 602 | if (ret) |
502cfb35 | 603 | return ret; |
fb64f3fd KO |
604 | |
605 | if (unlikely(trans->journal_transaction_names)) | |
606 | journal_transaction_name(trans); | |
4e8224ed KO |
607 | } else { |
608 | trans->journal_res.seq = c->journal.replay_journal_seq; | |
87c3beb4 | 609 | } |
c8cc5b3e | 610 | |
2a9101a9 KO |
611 | /* |
612 | * Not allowed to fail after we've gotten our journal reservation - we | |
613 | * have to use it: | |
614 | */ | |
615 | ||
1ae40fd8 | 616 | if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) && |
30ca6ece | 617 | !(flags & BTREE_INSERT_JOURNAL_REPLAY)) { |
29364f34 | 618 | if (bch2_journal_seq_verify) |
cd8319fd | 619 | trans_for_each_update(trans, i) |
1c6fdbd8 | 620 | i->k->k.version.lo = trans->journal_res.seq; |
29364f34 | 621 | else if (bch2_inject_invalid_keys) |
cd8319fd | 622 | trans_for_each_update(trans, i) |
1c6fdbd8 KO |
623 | i->k->k.version = MAX_VERSION; |
624 | } | |
625 | ||
58e1ea4b KO |
626 | if (trans->fs_usage_deltas && |
627 | bch2_trans_fs_usage_apply(trans, trans->fs_usage_deltas)) | |
ac9fa4bd | 628 | return -BCH_ERR_btree_insert_need_mark_replicas; |
502cfb35 | 629 | |
920e69bc KO |
630 | if (trans->nr_wb_updates) { |
631 | EBUG_ON(flags & BTREE_INSERT_JOURNAL_REPLAY); | |
632 | ||
633 | ret = bch2_btree_insert_keys_write_buffer(trans); | |
634 | if (ret) | |
635 | goto revert_fs_usage; | |
636 | } | |
637 | ||
56cc033d KO |
638 | h = trans->hooks; |
639 | while (h) { | |
640 | ret = h->fn(trans, h); | |
641 | if (ret) | |
642 | goto revert_fs_usage; | |
643 | h = h->next; | |
644 | } | |
645 | ||
a7199432 | 646 | trans_for_each_update(trans, i) |
21aec962 | 647 | if (BTREE_NODE_TYPE_HAS_MEM_TRIGGERS & (1U << i->bkey_type)) { |
3598c56e | 648 | ret = run_one_mem_trigger(trans, i, i->flags); |
21aec962 | 649 | if (ret) |
920e69bc | 650 | goto fatal_err; |
21aec962 | 651 | } |
932aa837 | 652 | |
21aec962 | 653 | if (unlikely(c->gc_pos.phase)) { |
3598c56e | 654 | ret = bch2_trans_commit_run_gc_triggers(trans); |
21aec962 | 655 | if (ret) |
920e69bc | 656 | goto fatal_err; |
21aec962 | 657 | } |
932aa837 | 658 | |
1ae40fd8 KO |
659 | if (unlikely(trans->extra_journal_entries.nr)) { |
660 | memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res), | |
661 | trans->extra_journal_entries.data, | |
662 | trans->extra_journal_entries.nr); | |
663 | ||
664 | trans->journal_res.offset += trans->extra_journal_entries.nr; | |
665 | trans->journal_res.u64s -= trans->extra_journal_entries.nr; | |
666 | } | |
667 | ||
30ca6ece | 668 | if (likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) { |
920e69bc KO |
669 | struct journal *j = &c->journal; |
670 | struct jset_entry *entry; | |
cb685ce7 | 671 | |
920e69bc | 672 | trans_for_each_update(trans, i) { |
cb685ce7 KO |
673 | if (i->key_cache_already_flushed) |
674 | continue; | |
675 | ||
676 | if (i->flags & BTREE_UPDATE_NOJOURNAL) | |
677 | continue; | |
678 | ||
08f78031 KO |
679 | verify_update_old_key(trans, i); |
680 | ||
cb685ce7 KO |
681 | if (trans->journal_transaction_names) { |
682 | entry = bch2_journal_add_entry(j, &trans->journal_res, | |
683 | BCH_JSET_ENTRY_overwrite, | |
684 | i->btree_id, i->level, | |
685 | i->old_k.u64s); | |
686 | bkey_reassemble(&entry->start[0], | |
687 | (struct bkey_s_c) { &i->old_k, i->old_v }); | |
688 | } | |
689 | ||
690 | entry = bch2_journal_add_entry(j, &trans->journal_res, | |
691 | BCH_JSET_ENTRY_btree_keys, | |
692 | i->btree_id, i->level, | |
693 | i->k->k.u64s); | |
694 | bkey_copy(&entry->start[0], i->k); | |
695 | } | |
696 | ||
920e69bc KO |
697 | trans_for_each_wb_update(trans, wb) { |
698 | entry = bch2_journal_add_entry(j, &trans->journal_res, | |
699 | BCH_JSET_ENTRY_btree_keys, | |
700 | wb->btree, 0, | |
701 | wb->k.k.u64s); | |
702 | bkey_copy(&entry->start[0], &wb->k); | |
703 | } | |
704 | ||
cb685ce7 KO |
705 | if (trans->journal_seq) |
706 | *trans->journal_seq = trans->journal_res.seq; | |
707 | } | |
708 | ||
709 | trans_for_each_update(trans, i) { | |
710 | i->k->k.needs_whiteout = false; | |
711 | ||
eabb10dc BF |
712 | if (!i->cached) { |
713 | u64 seq = trans->journal_res.seq; | |
714 | ||
715 | if (i->flags & BTREE_UPDATE_PREJOURNAL) | |
716 | seq = i->seq; | |
717 | ||
718 | bch2_btree_insert_key_leaf(trans, i->path, i->k, seq); | |
719 | } else if (!i->key_cache_already_flushed) | |
e53d03fe | 720 | bch2_btree_insert_key_cached(trans, flags, i); |
7c812ab7 | 721 | else { |
cb685ce7 | 722 | bch2_btree_key_cache_drop(trans, i->path); |
7c812ab7 KO |
723 | btree_path_set_dirty(i->path, BTREE_ITER_NEED_TRAVERSE); |
724 | } | |
cb685ce7 | 725 | } |
502cfb35 | 726 | |
920e69bc KO |
727 | return 0; |
728 | fatal_err: | |
729 | bch2_fatal_error(c); | |
730 | revert_fs_usage: | |
731 | if (trans->fs_usage_deltas) | |
732 | bch2_trans_fs_usage_revert(trans, trans->fs_usage_deltas); | |
2a9101a9 KO |
733 | return ret; |
734 | } | |
735 | ||
0cc455b3 KO |
736 | static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i) |
737 | { | |
738 | while (--i >= trans->updates) { | |
739 | if (same_leaf_as_prev(trans, i)) | |
740 | continue; | |
741 | ||
742 | bch2_btree_node_unlock_write(trans, i->path, insert_l(i)->b); | |
743 | } | |
744 | ||
745 | trace_and_count(trans->c, trans_restart_would_deadlock_write, trans); | |
746 | return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write); | |
747 | } | |
748 | ||
caaa66aa KO |
749 | static inline int trans_lock_write(struct btree_trans *trans) |
750 | { | |
751 | struct btree_insert_entry *i; | |
752 | ||
753 | trans_for_each_update(trans, i) { | |
754 | if (same_leaf_as_prev(trans, i)) | |
755 | continue; | |
756 | ||
0cc455b3 KO |
757 | if (bch2_btree_node_lock_write(trans, i->path, &insert_l(i)->b->c)) |
758 | return trans_lock_write_fail(trans, i); | |
caaa66aa | 759 | |
920e69bc KO |
760 | if (!i->cached) |
761 | bch2_btree_node_prep_for_write(trans, i->path, insert_l(i)->b); | |
caaa66aa KO |
762 | } |
763 | ||
764 | return 0; | |
caaa66aa KO |
765 | } |
766 | ||
dfd41fb9 KO |
767 | static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans) |
768 | { | |
769 | struct btree_insert_entry *i; | |
920e69bc | 770 | struct btree_write_buffered_key *wb; |
dfd41fb9 KO |
771 | |
772 | trans_for_each_update(trans, i) | |
773 | bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p); | |
920e69bc KO |
774 | |
775 | trans_for_each_wb_update(trans, wb) | |
776 | bch2_journal_key_overwritten(trans->c, wb->btree, 0, wb->k.k.p); | |
dfd41fb9 KO |
777 | } |
778 | ||
30ca6ece | 779 | static noinline int bch2_trans_commit_bkey_invalid(struct btree_trans *trans, unsigned flags, |
03e83f63 KO |
780 | struct btree_insert_entry *i, |
781 | struct printbuf *err) | |
782 | { | |
783 | struct bch_fs *c = trans->c; | |
30ca6ece | 784 | int rw = (flags & BTREE_INSERT_JOURNAL_REPLAY) ? READ : WRITE; |
03e83f63 KO |
785 | |
786 | printbuf_reset(err); | |
787 | prt_printf(err, "invalid bkey on insert from %s -> %ps", | |
788 | trans->fn, (void *) i->ip_allocated); | |
789 | prt_newline(err); | |
790 | printbuf_indent_add(err, 2); | |
791 | ||
792 | bch2_bkey_val_to_text(err, c, bkey_i_to_s_c(i->k)); | |
793 | prt_newline(err); | |
794 | ||
795 | bch2_bkey_invalid(c, bkey_i_to_s_c(i->k), | |
796 | i->bkey_type, rw, err); | |
797 | bch2_print_string_as_lines(KERN_ERR, err->buf); | |
798 | ||
799 | bch2_inconsistent_error(c); | |
800 | bch2_dump_trans_updates(trans); | |
03e83f63 KO |
801 | |
802 | return -EINVAL; | |
803 | } | |
804 | ||
2a9101a9 KO |
805 | /* |
806 | * Get journal reservation, take write locks, and attempt to do btree update(s): | |
807 | */ | |
30ca6ece | 808 | static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags, |
531a0095 KO |
809 | struct btree_insert_entry **stopped_at, |
810 | unsigned long trace_ip) | |
2a9101a9 | 811 | { |
b182ff60 | 812 | struct bch_fs *c = trans->c; |
2a9101a9 | 813 | struct btree_insert_entry *i; |
3f4ab4c1 | 814 | int ret = 0, u64s_delta = 0; |
2a9101a9 | 815 | |
cd8319fd | 816 | trans_for_each_update(trans, i) { |
2e63e180 | 817 | if (i->cached) |
05046a96 | 818 | continue; |
b182ff60 | 819 | |
05046a96 | 820 | u64s_delta += !bkey_deleted(&i->k->k) ? i->k->k.u64s : 0; |
2e63e180 | 821 | u64s_delta -= i->old_btree_u64s; |
05046a96 KO |
822 | |
823 | if (!same_leaf_as_next(trans, i)) { | |
824 | if (u64s_delta <= 0) { | |
67e0dd8f | 825 | ret = bch2_foreground_maybe_merge(trans, i->path, |
30ca6ece | 826 | i->level, flags); |
05046a96 KO |
827 | if (unlikely(ret)) |
828 | return ret; | |
829 | } | |
b182ff60 | 830 | |
05046a96 | 831 | u64s_delta = 0; |
b182ff60 KO |
832 | } |
833 | } | |
834 | ||
b182ff60 | 835 | ret = bch2_journal_preres_get(&c->journal, |
8b3bbe2c | 836 | &trans->journal_preres, trans->journal_preres_u64s, |
ec14fc60 | 837 | (flags & BCH_WATERMARK_MASK)|JOURNAL_RES_GET_NONBLOCK); |
87ced107 | 838 | if (unlikely(ret == -BCH_ERR_journal_preres_get_blocked)) |
30ca6ece | 839 | ret = bch2_trans_journal_preres_get_cold(trans, flags, trace_ip); |
2a9101a9 KO |
840 | if (unlikely(ret)) |
841 | return ret; | |
842 | ||
caaa66aa KO |
843 | ret = trans_lock_write(trans); |
844 | if (unlikely(ret)) | |
845 | return ret; | |
b7ba66c8 | 846 | |
30ca6ece | 847 | ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip); |
b7ba66c8 | 848 | |
5222a460 | 849 | if (!ret && unlikely(trans->journal_replay_not_finished)) |
dfd41fb9 KO |
850 | bch2_drop_overwrites_from_journal(trans); |
851 | ||
cd8319fd | 852 | trans_for_each_update(trans, i) |
24326cd1 | 853 | if (!same_leaf_as_prev(trans, i)) |
67e0dd8f | 854 | bch2_btree_node_unlock_write_inlined(trans, i->path, |
6fba6b83 | 855 | insert_l(i)->b); |
2a9101a9 | 856 | |
00b8ccf7 | 857 | if (!ret && trans->journal_pin) |
b182ff60 | 858 | bch2_journal_pin_add(&c->journal, trans->journal_res.seq, |
00b8ccf7 KO |
859 | trans->journal_pin, NULL); |
860 | ||
2a9101a9 KO |
861 | /* |
862 | * Drop journal reservation after dropping write locks, since dropping | |
863 | * the journal reservation may kick off a journal write: | |
864 | */ | |
b182ff60 | 865 | bch2_journal_res_put(&c->journal, &trans->journal_res); |
2a9101a9 KO |
866 | |
867 | if (unlikely(ret)) | |
868 | return ret; | |
869 | ||
72545b5e | 870 | bch2_trans_downgrade(trans); |
2a9101a9 KO |
871 | |
872 | return 0; | |
1c6fdbd8 KO |
873 | } |
874 | ||
24db24c7 KO |
875 | static int journal_reclaim_wait_done(struct bch_fs *c) |
876 | { | |
0ef10785 KO |
877 | int ret = bch2_journal_error(&c->journal) ?: |
878 | !bch2_btree_key_cache_must_wait(c); | |
24db24c7 KO |
879 | |
880 | if (!ret) | |
0ef10785 | 881 | journal_reclaim_kick(&c->journal); |
24db24c7 KO |
882 | return ret; |
883 | } | |
884 | ||
11e6f19a | 885 | static noinline |
30ca6ece | 886 | int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags, |
11e6f19a | 887 | struct btree_insert_entry *i, |
531a0095 | 888 | int ret, unsigned long trace_ip) |
1c6fdbd8 KO |
889 | { |
890 | struct bch_fs *c = trans->c; | |
1c6fdbd8 | 891 | |
1d25849c | 892 | switch (ret) { |
ac9fa4bd | 893 | case -BCH_ERR_btree_insert_btree_node_full: |
30ca6ece | 894 | ret = bch2_btree_split_leaf(trans, i->path, flags); |
549d173c | 895 | if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) |
674cfc26 | 896 | trace_and_count(c, trans_restart_btree_node_split, trans, trace_ip, i->path); |
1d25849c | 897 | break; |
ac9fa4bd | 898 | case -BCH_ERR_btree_insert_need_mark_replicas: |
49c7cd9d KO |
899 | ret = drop_locks_do(trans, |
900 | bch2_replicas_delta_list_mark(c, trans->fs_usage_deltas)); | |
1d25849c | 901 | break; |
87ced107 | 902 | case -BCH_ERR_journal_res_get_blocked: |
01e691e8 KO |
903 | /* |
904 | * XXX: this should probably be a separate BTREE_INSERT_NONBLOCK | |
905 | * flag | |
906 | */ | |
30ca6ece | 907 | if ((flags & BTREE_INSERT_JOURNAL_RECLAIM) && |
ec14fc60 | 908 | (flags & BCH_WATERMARK_MASK) != BCH_WATERMARK_reclaim) { |
549d173c | 909 | ret = -BCH_ERR_journal_reclaim_would_deadlock; |
e5af273f KO |
910 | break; |
911 | } | |
2940295c | 912 | |
49c7cd9d KO |
913 | ret = drop_locks_do(trans, |
914 | bch2_trans_journal_res_get(trans, | |
ec14fc60 | 915 | (flags & BCH_WATERMARK_MASK)| |
49c7cd9d | 916 | JOURNAL_RES_GET_CHECK)); |
d5425a3b | 917 | break; |
ac9fa4bd | 918 | case -BCH_ERR_btree_insert_need_journal_reclaim: |
d5425a3b KO |
919 | bch2_trans_unlock(trans); |
920 | ||
674cfc26 | 921 | trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip); |
4f6dad46 | 922 | |
0ef10785 KO |
923 | wait_event_freezable(c->journal.reclaim_wait, |
924 | (ret = journal_reclaim_wait_done(c))); | |
24db24c7 | 925 | if (ret < 0) |
e5af273f | 926 | break; |
d5425a3b | 927 | |
549d173c | 928 | ret = bch2_trans_relock(trans); |
9623ab27 | 929 | break; |
920e69bc KO |
930 | case -BCH_ERR_btree_insert_need_flush_buffer: { |
931 | struct btree_write_buffer *wb = &c->btree_write_buffer; | |
932 | ||
933 | ret = 0; | |
934 | ||
935 | if (wb->state.nr > wb->size * 3 / 4) { | |
920e69bc | 936 | bch2_trans_unlock(trans); |
920e69bc KO |
937 | mutex_lock(&wb->flush_lock); |
938 | ||
49c7cd9d KO |
939 | if (wb->state.nr > wb->size * 3 / 4) { |
940 | bch2_trans_begin(trans); | |
920e69bc KO |
941 | ret = __bch2_btree_write_buffer_flush(trans, |
942 | flags|BTREE_INSERT_NOCHECK_RW, true); | |
49c7cd9d KO |
943 | if (!ret) { |
944 | trace_and_count(c, trans_restart_write_buffer_flush, trans, _THIS_IP_); | |
945 | ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_write_buffer_flush); | |
946 | } | |
947 | } else { | |
920e69bc | 948 | mutex_unlock(&wb->flush_lock); |
49c7cd9d | 949 | ret = bch2_trans_relock(trans); |
920e69bc KO |
950 | } |
951 | } | |
952 | break; | |
953 | } | |
1d25849c KO |
954 | default: |
955 | BUG_ON(ret >= 0); | |
956 | break; | |
1c6fdbd8 KO |
957 | } |
958 | ||
549d173c | 959 | BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted); |
098ef98d KO |
960 | |
961 | bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOSPC) && | |
30ca6ece KO |
962 | !(flags & BTREE_INSERT_NOWAIT) && |
963 | (flags & BTREE_INSERT_NOFAIL), c, | |
098ef98d | 964 | "%s: incorrectly got %s\n", __func__, bch2_err_str(ret)); |
a6336910 | 965 | |
11e6f19a KO |
966 | return ret; |
967 | } | |
968 | ||
2a9101a9 | 969 | static noinline int |
30ca6ece | 970 | bch2_trans_commit_get_rw_cold(struct btree_trans *trans, unsigned flags) |
11e6f19a KO |
971 | { |
972 | struct bch_fs *c = trans->c; | |
11e6f19a KO |
973 | int ret; |
974 | ||
30ca6ece | 975 | if (likely(!(flags & BTREE_INSERT_LAZY_RW)) || |
6214485b | 976 | test_bit(BCH_FS_STARTED, &c->flags)) |
858536c7 | 977 | return -BCH_ERR_erofs_trans_commit; |
11e6f19a | 978 | |
b5fd7566 | 979 | ret = drop_locks_do(trans, bch2_fs_read_write_early(c)); |
2a9101a9 KO |
980 | if (ret) |
981 | return ret; | |
11e6f19a | 982 | |
d94189ad | 983 | bch2_write_ref_get(c, BCH_WRITE_REF_trans); |
2a9101a9 | 984 | return 0; |
1c6fdbd8 KO |
985 | } |
986 | ||
78c8fe20 KO |
987 | /* |
988 | * This is for updates done in the early part of fsck - btree_gc - before we've | |
989 | * gone RW. we only add the new key to the list of keys for journal replay to | |
990 | * do. | |
991 | */ | |
992 | static noinline int | |
993 | do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans) | |
994 | { | |
995 | struct bch_fs *c = trans->c; | |
996 | struct btree_insert_entry *i; | |
997 | int ret = 0; | |
998 | ||
999 | trans_for_each_update(trans, i) { | |
1000 | ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->k); | |
1001 | if (ret) | |
1002 | break; | |
1003 | } | |
1004 | ||
1005 | return ret; | |
1006 | } | |
1007 | ||
30ca6ece | 1008 | int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) |
1c6fdbd8 | 1009 | { |
fb64f3fd | 1010 | struct bch_fs *c = trans->c; |
d9b022fe | 1011 | struct btree_insert_entry *i = NULL; |
920e69bc | 1012 | struct btree_write_buffered_key *wb; |
955af634 | 1013 | unsigned u64s; |
dc3b63dc | 1014 | int ret = 0; |
1c6fdbd8 | 1015 | |
9f1833ca | 1016 | if (!trans->nr_updates && |
920e69bc | 1017 | !trans->nr_wb_updates && |
2a6870ad | 1018 | !trans->extra_journal_entries.nr) |
b7cf4bd7 | 1019 | goto out_reset; |
1c6fdbd8 | 1020 | |
30ca6ece | 1021 | if (flags & BTREE_INSERT_GC_LOCK_HELD) |
fb64f3fd | 1022 | lockdep_assert_held(&c->gc_lock); |
11e6f19a | 1023 | |
06a98c96 KO |
1024 | ret = bch2_trans_commit_run_triggers(trans); |
1025 | if (ret) | |
1026 | goto out_reset; | |
1027 | ||
cc07773f KO |
1028 | trans_for_each_update(trans, i) { |
1029 | struct printbuf buf = PRINTBUF; | |
1030 | enum bkey_invalid_flags invalid_flags = 0; | |
1031 | ||
1032 | if (!(flags & BTREE_INSERT_JOURNAL_REPLAY)) | |
1033 | invalid_flags |= BKEY_INVALID_WRITE|BKEY_INVALID_COMMIT; | |
1034 | ||
1035 | if (unlikely(bch2_bkey_invalid(c, bkey_i_to_s_c(i->k), | |
1036 | i->bkey_type, invalid_flags, &buf))) | |
1037 | ret = bch2_trans_commit_bkey_invalid(trans, flags, i, &buf); | |
1038 | btree_insert_entry_checks(trans, i); | |
1039 | printbuf_exit(&buf); | |
1040 | ||
1041 | if (ret) | |
1042 | return ret; | |
1043 | } | |
cc07773f | 1044 | |
78c8fe20 KO |
1045 | if (unlikely(!test_bit(BCH_FS_MAY_GO_RW, &c->flags))) { |
1046 | ret = do_bch2_trans_commit_to_journal_replay(trans); | |
1047 | goto out_reset; | |
1048 | } | |
1049 | ||
30ca6ece | 1050 | if (!(flags & BTREE_INSERT_NOCHECK_RW) && |
d94189ad | 1051 | unlikely(!bch2_write_ref_tryget(c, BCH_WRITE_REF_trans))) { |
30ca6ece | 1052 | ret = bch2_trans_commit_get_rw_cold(trans, flags); |
06a98c96 KO |
1053 | if (ret) |
1054 | goto out_reset; | |
1055 | } | |
1056 | ||
920e69bc KO |
1057 | if (c->btree_write_buffer.state.nr > c->btree_write_buffer.size / 2 && |
1058 | mutex_trylock(&c->btree_write_buffer.flush_lock)) { | |
1059 | bch2_trans_begin(trans); | |
1060 | bch2_trans_unlock(trans); | |
1061 | ||
1062 | ret = __bch2_btree_write_buffer_flush(trans, | |
1063 | flags|BTREE_INSERT_NOCHECK_RW, true); | |
1064 | if (!ret) { | |
1065 | trace_and_count(c, trans_restart_write_buffer_flush, trans, _THIS_IP_); | |
1066 | ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_write_buffer_flush); | |
1067 | } | |
1068 | goto out; | |
1069 | } | |
1070 | ||
c0960603 KO |
1071 | EBUG_ON(test_bit(BCH_FS_CLEAN_SHUTDOWN, &c->flags)); |
1072 | ||
9623ab27 | 1073 | memset(&trans->journal_preres, 0, sizeof(trans->journal_preres)); |
0dc17247 | 1074 | |
2a6870ad | 1075 | trans->journal_u64s = trans->extra_journal_entries.nr; |
8b3bbe2c KO |
1076 | trans->journal_preres_u64s = 0; |
1077 | ||
fb64f3fd KO |
1078 | trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names); |
1079 | ||
1080 | if (trans->journal_transaction_names) | |
43ddf448 | 1081 | trans->journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s); |
fb64f3fd | 1082 | |
8b3bbe2c | 1083 | trans_for_each_update(trans, i) { |
c9ee99ad | 1084 | EBUG_ON(!i->path->should_be_locked); |
9f631dc1 | 1085 | |
367d72dd KO |
1086 | ret = bch2_btree_path_upgrade(trans, i->path, i->level + 1); |
1087 | if (unlikely(ret)) | |
6333bd2f | 1088 | goto out; |
6333bd2f | 1089 | |
c9ee99ad | 1090 | EBUG_ON(!btree_node_intent_locked(i->path, i->level)); |
5c1d808a | 1091 | |
cb685ce7 KO |
1092 | if (i->key_cache_already_flushed) |
1093 | continue; | |
1094 | ||
1095 | /* we're going to journal the key being updated: */ | |
8b3bbe2c | 1096 | u64s = jset_u64s(i->k->k.u64s); |
6fba6b83 | 1097 | if (i->cached && |
30ca6ece | 1098 | likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) |
8b3bbe2c | 1099 | trans->journal_preres_u64s += u64s; |
34d74830 | 1100 | |
cb685ce7 KO |
1101 | if (i->flags & BTREE_UPDATE_NOJOURNAL) |
1102 | continue; | |
1103 | ||
1104 | trans->journal_u64s += u64s; | |
1105 | ||
1106 | /* and we're also going to log the overwrite: */ | |
1107 | if (trans->journal_transaction_names) | |
1108 | trans->journal_u64s += jset_u64s(i->old_k.u64s); | |
8b3bbe2c | 1109 | } |
a49e9a05 | 1110 | |
920e69bc KO |
1111 | trans_for_each_wb_update(trans, wb) |
1112 | trans->journal_u64s += jset_u64s(wb->k.k.u64s); | |
1113 | ||
a49e9a05 | 1114 | if (trans->extra_journal_res) { |
fb64f3fd | 1115 | ret = bch2_disk_reservation_add(c, trans->disk_res, |
a49e9a05 | 1116 | trans->extra_journal_res, |
30ca6ece | 1117 | (flags & BTREE_INSERT_NOFAIL) |
a49e9a05 KO |
1118 | ? BCH_DISK_RESERVATION_NOFAIL : 0); |
1119 | if (ret) | |
1120 | goto err; | |
1121 | } | |
2a9101a9 | 1122 | retry: |
12344c7c | 1123 | bch2_trans_verify_not_in_restart(trans); |
2a9101a9 | 1124 | memset(&trans->journal_res, 0, sizeof(trans->journal_res)); |
134915f3 | 1125 | |
30ca6ece | 1126 | ret = do_bch2_trans_commit(trans, flags, &i, _RET_IP_); |
134915f3 | 1127 | |
2a9101a9 | 1128 | /* make sure we didn't drop or screw up locks: */ |
a0a56879 | 1129 | bch2_trans_verify_locks(trans); |
2a9101a9 | 1130 | |
11e6f19a KO |
1131 | if (ret) |
1132 | goto err; | |
1f93726e | 1133 | |
674cfc26 | 1134 | trace_and_count(c, transaction_commit, trans, _RET_IP_); |
11e6f19a | 1135 | out: |
fb64f3fd | 1136 | bch2_journal_preres_put(&c->journal, &trans->journal_preres); |
9623ab27 | 1137 | |
30ca6ece | 1138 | if (likely(!(flags & BTREE_INSERT_NOCHECK_RW))) |
d94189ad | 1139 | bch2_write_ref_put(c, BCH_WRITE_REF_trans); |
b7cf4bd7 | 1140 | out_reset: |
0fbf71f8 | 1141 | bch2_trans_reset_updates(trans); |
955af634 | 1142 | |
0dc17247 | 1143 | return ret; |
11e6f19a | 1144 | err: |
30ca6ece | 1145 | ret = bch2_trans_commit_error(trans, flags, i, ret, _RET_IP_); |
2a9101a9 KO |
1146 | if (ret) |
1147 | goto out; | |
932aa837 | 1148 | |
2a9101a9 | 1149 | goto retry; |
1c6fdbd8 | 1150 | } |