Commit | Line | Data |
---|---|---|
1c6fdbd8 KO |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include "bcachefs.h" | |
7b3f84ea | 3 | #include "alloc_foreground.h" |
39fb2983 | 4 | #include "btree_io.h" |
00b8ccf7 | 5 | #include "btree_update_interior.h" |
1c6fdbd8 KO |
6 | #include "buckets.h" |
7 | #include "checksum.h" | |
d042b040 | 8 | #include "disk_groups.h" |
1c6fdbd8 | 9 | #include "error.h" |
63b214e7 | 10 | #include "io.h" |
1c6fdbd8 KO |
11 | #include "journal.h" |
12 | #include "journal_io.h" | |
13 | #include "journal_reclaim.h" | |
adbcada4 | 14 | #include "journal_seq_blacklist.h" |
1c6fdbd8 KO |
15 | #include "replicas.h" |
16 | #include "trace.h" | |
17 | ||
adbcada4 KO |
18 | static void __journal_replay_free(struct journal_replay *i) |
19 | { | |
20 | list_del(&i->list); | |
21 | kvpfree(i, offsetof(struct journal_replay, j) + | |
22 | vstruct_bytes(&i->j)); | |
23 | ||
24 | } | |
25 | ||
26 | static void journal_replay_free(struct bch_fs *c, struct journal_replay *i) | |
27 | { | |
28 | i->ignore = true; | |
29 | ||
30 | if (!c->opts.read_entire_journal) | |
31 | __journal_replay_free(i); | |
32 | } | |
33 | ||
1c6fdbd8 KO |
34 | struct journal_list { |
35 | struct closure cl; | |
36 | struct mutex lock; | |
37 | struct list_head *head; | |
38 | int ret; | |
39 | }; | |
40 | ||
41 | #define JOURNAL_ENTRY_ADD_OK 0 | |
42 | #define JOURNAL_ENTRY_ADD_OUT_OF_RANGE 5 | |
43 | ||
44 | /* | |
45 | * Given a journal entry we just read, add it to the list of journal entries to | |
46 | * be replayed: | |
47 | */ | |
48 | static int journal_entry_add(struct bch_fs *c, struct bch_dev *ca, | |
e4c3f386 | 49 | struct bch_extent_ptr entry_ptr, |
ca73852a KO |
50 | struct journal_list *jlist, struct jset *j, |
51 | bool bad) | |
1c6fdbd8 | 52 | { |
e4c3f386 KO |
53 | struct journal_replay *i, *pos, *dup = NULL; |
54 | struct bch_extent_ptr *ptr; | |
1c6fdbd8 KO |
55 | struct list_head *where; |
56 | size_t bytes = vstruct_bytes(j); | |
adbcada4 | 57 | u64 last_seq = 0; |
e4c3f386 | 58 | int ret = JOURNAL_ENTRY_ADD_OK; |
1c6fdbd8 | 59 | |
adbcada4 KO |
60 | list_for_each_entry_reverse(i, jlist->head, list) { |
61 | if (!JSET_NO_FLUSH(&i->j)) { | |
62 | last_seq = le64_to_cpu(i->j.last_seq); | |
63 | break; | |
7fffc85b | 64 | } |
adbcada4 | 65 | } |
1c6fdbd8 | 66 | |
adbcada4 KO |
67 | /* Is this entry older than the range we need? */ |
68 | if (!c->opts.read_entire_journal && | |
69 | le64_to_cpu(j->seq) < last_seq) { | |
70 | ret = JOURNAL_ENTRY_ADD_OUT_OF_RANGE; | |
71 | goto out; | |
72 | } | |
73 | ||
74 | /* Drop entries we don't need anymore */ | |
75 | if (!JSET_NO_FLUSH(j)) { | |
7fffc85b KO |
76 | list_for_each_entry_safe(i, pos, jlist->head, list) { |
77 | if (le64_to_cpu(i->j.seq) >= le64_to_cpu(j->last_seq)) | |
78 | break; | |
adbcada4 | 79 | journal_replay_free(c, i); |
7fffc85b | 80 | } |
1c6fdbd8 KO |
81 | } |
82 | ||
83 | list_for_each_entry_reverse(i, jlist->head, list) { | |
ca73852a KO |
84 | if (le64_to_cpu(j->seq) > le64_to_cpu(i->j.seq)) { |
85 | where = &i->list; | |
86 | goto add; | |
87 | } | |
88 | } | |
89 | ||
90 | where = jlist->head; | |
91 | add: | |
e4c3f386 | 92 | dup = where->next != jlist->head |
ca73852a KO |
93 | ? container_of(where->next, struct journal_replay, list) |
94 | : NULL; | |
95 | ||
e4c3f386 KO |
96 | if (dup && le64_to_cpu(j->seq) != le64_to_cpu(dup->j.seq)) |
97 | dup = NULL; | |
98 | ||
ca73852a KO |
99 | /* |
100 | * Duplicate journal entries? If so we want the one that didn't have a | |
101 | * checksum error: | |
102 | */ | |
e4c3f386 KO |
103 | if (dup) { |
104 | if (dup->bad) { | |
105 | /* we'll replace @dup: */ | |
ca73852a | 106 | } else if (bad) { |
e4c3f386 | 107 | i = dup; |
ca73852a KO |
108 | goto found; |
109 | } else { | |
e4c3f386 KO |
110 | fsck_err_on(bytes != vstruct_bytes(&dup->j) || |
111 | memcmp(j, &dup->j, bytes), c, | |
1c6fdbd8 KO |
112 | "found duplicate but non identical journal entries (seq %llu)", |
113 | le64_to_cpu(j->seq)); | |
e4c3f386 | 114 | i = dup; |
1c6fdbd8 KO |
115 | goto found; |
116 | } | |
1c6fdbd8 KO |
117 | } |
118 | ||
1c6fdbd8 KO |
119 | i = kvpmalloc(offsetof(struct journal_replay, j) + bytes, GFP_KERNEL); |
120 | if (!i) { | |
121 | ret = -ENOMEM; | |
122 | goto out; | |
123 | } | |
124 | ||
e4c3f386 KO |
125 | i->nr_ptrs = 0; |
126 | i->bad = bad; | |
127 | i->ignore = false; | |
1c6fdbd8 | 128 | unsafe_memcpy(&i->j, j, bytes, "embedded variable length struct"); |
e4c3f386 KO |
129 | |
130 | if (dup) { | |
131 | i->nr_ptrs = dup->nr_ptrs; | |
132 | memcpy(i->ptrs, dup->ptrs, sizeof(dup->ptrs)); | |
133 | __journal_replay_free(dup); | |
134 | } | |
135 | ||
136 | list_add(&i->list, where); | |
1c6fdbd8 | 137 | found: |
e4c3f386 KO |
138 | for (ptr = i->ptrs; ptr < i->ptrs + i->nr_ptrs; ptr++) { |
139 | if (ptr->dev == ca->dev_idx) { | |
140 | bch_err(c, "duplicate journal entry %llu on same device", | |
141 | le64_to_cpu(i->j.seq)); | |
142 | goto out; | |
143 | } | |
144 | } | |
145 | ||
146 | if (i->nr_ptrs >= ARRAY_SIZE(i->ptrs)) { | |
147 | bch_err(c, "found too many copies of journal entry %llu", | |
148 | le64_to_cpu(i->j.seq)); | |
149 | goto out; | |
150 | } | |
151 | ||
152 | i->ptrs[i->nr_ptrs++] = entry_ptr; | |
1c6fdbd8 KO |
153 | out: |
154 | fsck_err: | |
155 | return ret; | |
156 | } | |
157 | ||
158 | static struct nonce journal_nonce(const struct jset *jset) | |
159 | { | |
160 | return (struct nonce) {{ | |
161 | [0] = 0, | |
162 | [1] = ((__le32 *) &jset->seq)[0], | |
163 | [2] = ((__le32 *) &jset->seq)[1], | |
164 | [3] = BCH_NONCE_JOURNAL, | |
165 | }}; | |
166 | } | |
167 | ||
168 | /* this fills in a range with empty jset_entries: */ | |
169 | static void journal_entry_null_range(void *start, void *end) | |
170 | { | |
171 | struct jset_entry *entry; | |
172 | ||
173 | for (entry = start; entry != end; entry = vstruct_next(entry)) | |
174 | memset(entry, 0, sizeof(*entry)); | |
175 | } | |
176 | ||
177 | #define JOURNAL_ENTRY_REREAD 5 | |
178 | #define JOURNAL_ENTRY_NONE 6 | |
179 | #define JOURNAL_ENTRY_BAD 7 | |
180 | ||
181 | #define journal_entry_err(c, msg, ...) \ | |
182 | ({ \ | |
183 | switch (write) { \ | |
184 | case READ: \ | |
185 | mustfix_fsck_err(c, msg, ##__VA_ARGS__); \ | |
186 | break; \ | |
187 | case WRITE: \ | |
188 | bch_err(c, "corrupt metadata before write:\n" \ | |
189 | msg, ##__VA_ARGS__); \ | |
190 | if (bch2_fs_inconsistent(c)) { \ | |
191 | ret = BCH_FSCK_ERRORS_NOT_FIXED; \ | |
192 | goto fsck_err; \ | |
193 | } \ | |
194 | break; \ | |
195 | } \ | |
196 | true; \ | |
197 | }) | |
198 | ||
199 | #define journal_entry_err_on(cond, c, msg, ...) \ | |
200 | ((cond) ? journal_entry_err(c, msg, ##__VA_ARGS__) : false) | |
201 | ||
4d54337c KO |
202 | #define FSCK_DELETED_KEY 5 |
203 | ||
7d6f07ed | 204 | static int journal_validate_key(struct bch_fs *c, const char *where, |
1c6fdbd8 | 205 | struct jset_entry *entry, |
39fb2983 | 206 | unsigned level, enum btree_id btree_id, |
7d6f07ed KO |
207 | struct bkey_i *k, const char *type, |
208 | unsigned version, int big_endian, int write) | |
1c6fdbd8 KO |
209 | { |
210 | void *next = vstruct_next(entry); | |
211 | const char *invalid; | |
1c6fdbd8 KO |
212 | int ret = 0; |
213 | ||
214 | if (journal_entry_err_on(!k->k.u64s, c, | |
7d6f07ed KO |
215 | "invalid %s in %s entry offset %zi/%u: k->u64s 0", |
216 | type, where, | |
4d54337c KO |
217 | (u64 *) k - entry->_data, |
218 | le16_to_cpu(entry->u64s))) { | |
1c6fdbd8 KO |
219 | entry->u64s = cpu_to_le16((u64 *) k - entry->_data); |
220 | journal_entry_null_range(vstruct_next(entry), next); | |
4d54337c | 221 | return FSCK_DELETED_KEY; |
1c6fdbd8 KO |
222 | } |
223 | ||
224 | if (journal_entry_err_on((void *) bkey_next(k) > | |
225 | (void *) vstruct_next(entry), c, | |
7d6f07ed KO |
226 | "invalid %s in %s entry offset %zi/%u: extends past end of journal entry", |
227 | type, where, | |
4d54337c KO |
228 | (u64 *) k - entry->_data, |
229 | le16_to_cpu(entry->u64s))) { | |
1c6fdbd8 KO |
230 | entry->u64s = cpu_to_le16((u64 *) k - entry->_data); |
231 | journal_entry_null_range(vstruct_next(entry), next); | |
4d54337c | 232 | return FSCK_DELETED_KEY; |
1c6fdbd8 KO |
233 | } |
234 | ||
235 | if (journal_entry_err_on(k->k.format != KEY_FORMAT_CURRENT, c, | |
7d6f07ed KO |
236 | "invalid %s in %s entry offset %zi/%u: bad format %u", |
237 | type, where, | |
4d54337c KO |
238 | (u64 *) k - entry->_data, |
239 | le16_to_cpu(entry->u64s), | |
ed0d631f | 240 | k->k.format)) { |
4d54337c | 241 | le16_add_cpu(&entry->u64s, -((u16) k->k.u64s)); |
1c6fdbd8 KO |
242 | memmove(k, bkey_next(k), next - (void *) bkey_next(k)); |
243 | journal_entry_null_range(vstruct_next(entry), next); | |
4d54337c | 244 | return FSCK_DELETED_KEY; |
1c6fdbd8 KO |
245 | } |
246 | ||
39fb2983 | 247 | if (!write) |
7d6f07ed KO |
248 | bch2_bkey_compat(level, btree_id, version, big_endian, |
249 | write, NULL, bkey_to_packed(k)); | |
26609b61 | 250 | |
39fb2983 KO |
251 | invalid = bch2_bkey_invalid(c, bkey_i_to_s_c(k), |
252 | __btree_node_type(level, btree_id)); | |
1c6fdbd8 | 253 | if (invalid) { |
319f9ac3 KO |
254 | char buf[160]; |
255 | ||
26609b61 | 256 | bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(k)); |
7d6f07ed KO |
257 | mustfix_fsck_err(c, "invalid %s in %s entry offset %zi/%u: %s\n%s", |
258 | type, where, | |
4d54337c KO |
259 | (u64 *) k - entry->_data, |
260 | le16_to_cpu(entry->u64s), | |
ed0d631f | 261 | invalid, buf); |
1c6fdbd8 | 262 | |
4d54337c | 263 | le16_add_cpu(&entry->u64s, -((u16) k->k.u64s)); |
1c6fdbd8 KO |
264 | memmove(k, bkey_next(k), next - (void *) bkey_next(k)); |
265 | journal_entry_null_range(vstruct_next(entry), next); | |
4d54337c | 266 | return FSCK_DELETED_KEY; |
1c6fdbd8 | 267 | } |
26609b61 | 268 | |
39fb2983 | 269 | if (write) |
7d6f07ed KO |
270 | bch2_bkey_compat(level, btree_id, version, big_endian, |
271 | write, NULL, bkey_to_packed(k)); | |
1c6fdbd8 KO |
272 | fsck_err: |
273 | return ret; | |
274 | } | |
275 | ||
276 | static int journal_entry_validate_btree_keys(struct bch_fs *c, | |
7d6f07ed | 277 | const char *where, |
1c6fdbd8 | 278 | struct jset_entry *entry, |
7d6f07ed | 279 | unsigned version, int big_endian, int write) |
1c6fdbd8 | 280 | { |
4d54337c | 281 | struct bkey_i *k = entry->start; |
1c6fdbd8 | 282 | |
4d54337c | 283 | while (k != vstruct_last(entry)) { |
7d6f07ed | 284 | int ret = journal_validate_key(c, where, entry, |
39fb2983 KO |
285 | entry->level, |
286 | entry->btree_id, | |
7d6f07ed | 287 | k, "key", version, big_endian, write); |
4d54337c KO |
288 | if (ret == FSCK_DELETED_KEY) |
289 | continue; | |
290 | ||
291 | k = bkey_next(k); | |
1c6fdbd8 KO |
292 | } |
293 | ||
294 | return 0; | |
295 | } | |
296 | ||
297 | static int journal_entry_validate_btree_root(struct bch_fs *c, | |
7d6f07ed | 298 | const char *where, |
1c6fdbd8 | 299 | struct jset_entry *entry, |
7d6f07ed | 300 | unsigned version, int big_endian, int write) |
1c6fdbd8 KO |
301 | { |
302 | struct bkey_i *k = entry->start; | |
303 | int ret = 0; | |
304 | ||
305 | if (journal_entry_err_on(!entry->u64s || | |
306 | le16_to_cpu(entry->u64s) != k->k.u64s, c, | |
307 | "invalid btree root journal entry: wrong number of keys")) { | |
308 | void *next = vstruct_next(entry); | |
309 | /* | |
310 | * we don't want to null out this jset_entry, | |
311 | * just the contents, so that later we can tell | |
312 | * we were _supposed_ to have a btree root | |
313 | */ | |
314 | entry->u64s = 0; | |
315 | journal_entry_null_range(vstruct_next(entry), next); | |
316 | return 0; | |
317 | } | |
318 | ||
7d6f07ed KO |
319 | return journal_validate_key(c, where, entry, 1, entry->btree_id, k, |
320 | "btree root", version, big_endian, write); | |
1c6fdbd8 KO |
321 | fsck_err: |
322 | return ret; | |
323 | } | |
324 | ||
325 | static int journal_entry_validate_prio_ptrs(struct bch_fs *c, | |
7d6f07ed | 326 | const char *where, |
1c6fdbd8 | 327 | struct jset_entry *entry, |
7d6f07ed | 328 | unsigned version, int big_endian, int write) |
1c6fdbd8 KO |
329 | { |
330 | /* obsolete, don't care: */ | |
331 | return 0; | |
332 | } | |
333 | ||
334 | static int journal_entry_validate_blacklist(struct bch_fs *c, | |
7d6f07ed | 335 | const char *where, |
1c6fdbd8 | 336 | struct jset_entry *entry, |
7d6f07ed | 337 | unsigned version, int big_endian, int write) |
1c6fdbd8 KO |
338 | { |
339 | int ret = 0; | |
340 | ||
341 | if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 1, c, | |
342 | "invalid journal seq blacklist entry: bad size")) { | |
343 | journal_entry_null_range(entry, vstruct_next(entry)); | |
344 | } | |
345 | fsck_err: | |
346 | return ret; | |
347 | } | |
348 | ||
349 | static int journal_entry_validate_blacklist_v2(struct bch_fs *c, | |
7d6f07ed | 350 | const char *where, |
1c6fdbd8 | 351 | struct jset_entry *entry, |
7d6f07ed | 352 | unsigned version, int big_endian, int write) |
1c6fdbd8 KO |
353 | { |
354 | struct jset_entry_blacklist_v2 *bl_entry; | |
355 | int ret = 0; | |
356 | ||
357 | if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 2, c, | |
358 | "invalid journal seq blacklist entry: bad size")) { | |
359 | journal_entry_null_range(entry, vstruct_next(entry)); | |
2c5af169 | 360 | goto out; |
1c6fdbd8 KO |
361 | } |
362 | ||
363 | bl_entry = container_of(entry, struct jset_entry_blacklist_v2, entry); | |
364 | ||
365 | if (journal_entry_err_on(le64_to_cpu(bl_entry->start) > | |
366 | le64_to_cpu(bl_entry->end), c, | |
367 | "invalid journal seq blacklist entry: start > end")) { | |
368 | journal_entry_null_range(entry, vstruct_next(entry)); | |
369 | } | |
2c5af169 KO |
370 | out: |
371 | fsck_err: | |
372 | return ret; | |
373 | } | |
374 | ||
375 | static int journal_entry_validate_usage(struct bch_fs *c, | |
7d6f07ed | 376 | const char *where, |
2c5af169 | 377 | struct jset_entry *entry, |
7d6f07ed | 378 | unsigned version, int big_endian, int write) |
2c5af169 KO |
379 | { |
380 | struct jset_entry_usage *u = | |
381 | container_of(entry, struct jset_entry_usage, entry); | |
382 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); | |
383 | int ret = 0; | |
384 | ||
3577df5f KO |
385 | if (journal_entry_err_on(bytes < sizeof(*u), |
386 | c, | |
387 | "invalid journal entry usage: bad size")) { | |
388 | journal_entry_null_range(entry, vstruct_next(entry)); | |
389 | return ret; | |
390 | } | |
391 | ||
392 | fsck_err: | |
393 | return ret; | |
394 | } | |
395 | ||
396 | static int journal_entry_validate_data_usage(struct bch_fs *c, | |
7d6f07ed | 397 | const char *where, |
3577df5f | 398 | struct jset_entry *entry, |
7d6f07ed | 399 | unsigned version, int big_endian, int write) |
3577df5f KO |
400 | { |
401 | struct jset_entry_data_usage *u = | |
402 | container_of(entry, struct jset_entry_data_usage, entry); | |
403 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); | |
404 | int ret = 0; | |
405 | ||
2c5af169 KO |
406 | if (journal_entry_err_on(bytes < sizeof(*u) || |
407 | bytes < sizeof(*u) + u->r.nr_devs, | |
408 | c, | |
409 | "invalid journal entry usage: bad size")) { | |
410 | journal_entry_null_range(entry, vstruct_next(entry)); | |
411 | return ret; | |
412 | } | |
1c6fdbd8 KO |
413 | |
414 | fsck_err: | |
415 | return ret; | |
416 | } | |
417 | ||
2abe5420 | 418 | static int journal_entry_validate_clock(struct bch_fs *c, |
7d6f07ed | 419 | const char *where, |
2abe5420 | 420 | struct jset_entry *entry, |
7d6f07ed | 421 | unsigned version, int big_endian, int write) |
2abe5420 KO |
422 | { |
423 | struct jset_entry_clock *clock = | |
424 | container_of(entry, struct jset_entry_clock, entry); | |
425 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); | |
426 | int ret = 0; | |
427 | ||
428 | if (journal_entry_err_on(bytes != sizeof(*clock), | |
429 | c, "invalid journal entry clock: bad size")) { | |
430 | journal_entry_null_range(entry, vstruct_next(entry)); | |
431 | return ret; | |
432 | } | |
433 | ||
434 | if (journal_entry_err_on(clock->rw > 1, | |
435 | c, "invalid journal entry clock: bad rw")) { | |
436 | journal_entry_null_range(entry, vstruct_next(entry)); | |
437 | return ret; | |
438 | } | |
439 | ||
440 | fsck_err: | |
441 | return ret; | |
442 | } | |
443 | ||
180fb49d | 444 | static int journal_entry_validate_dev_usage(struct bch_fs *c, |
7d6f07ed | 445 | const char *where, |
180fb49d | 446 | struct jset_entry *entry, |
7d6f07ed | 447 | unsigned version, int big_endian, int write) |
180fb49d KO |
448 | { |
449 | struct jset_entry_dev_usage *u = | |
450 | container_of(entry, struct jset_entry_dev_usage, entry); | |
451 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); | |
452 | unsigned expected = sizeof(*u) + sizeof(u->d[0]) * 7; /* Current value of BCH_DATA_NR */ | |
453 | unsigned dev; | |
454 | int ret = 0; | |
455 | ||
456 | if (journal_entry_err_on(bytes < expected, | |
457 | c, "invalid journal entry dev usage: bad size (%u < %u)", | |
458 | bytes, expected)) { | |
459 | journal_entry_null_range(entry, vstruct_next(entry)); | |
460 | return ret; | |
461 | } | |
462 | ||
463 | dev = le32_to_cpu(u->dev); | |
464 | ||
465 | if (journal_entry_err_on(!bch2_dev_exists2(c, dev), | |
466 | c, "invalid journal entry dev usage: bad dev")) { | |
467 | journal_entry_null_range(entry, vstruct_next(entry)); | |
468 | return ret; | |
469 | } | |
470 | ||
471 | if (journal_entry_err_on(u->pad, | |
472 | c, "invalid journal entry dev usage: bad pad")) { | |
473 | journal_entry_null_range(entry, vstruct_next(entry)); | |
474 | return ret; | |
475 | } | |
476 | ||
477 | fsck_err: | |
478 | return ret; | |
479 | } | |
480 | ||
1c6fdbd8 | 481 | struct jset_entry_ops { |
7d6f07ed KO |
482 | int (*validate)(struct bch_fs *, const char *, |
483 | struct jset_entry *, unsigned, int, int); | |
1c6fdbd8 KO |
484 | }; |
485 | ||
486 | static const struct jset_entry_ops bch2_jset_entry_ops[] = { | |
487 | #define x(f, nr) \ | |
488 | [BCH_JSET_ENTRY_##f] = (struct jset_entry_ops) { \ | |
489 | .validate = journal_entry_validate_##f, \ | |
490 | }, | |
491 | BCH_JSET_ENTRY_TYPES() | |
492 | #undef x | |
493 | }; | |
494 | ||
7d6f07ed KO |
495 | int bch2_journal_entry_validate(struct bch_fs *c, const char *where, |
496 | struct jset_entry *entry, | |
497 | unsigned version, int big_endian, int write) | |
1c6fdbd8 | 498 | { |
2c5af169 | 499 | return entry->type < BCH_JSET_ENTRY_NR |
7d6f07ed KO |
500 | ? bch2_jset_entry_ops[entry->type].validate(c, where, entry, |
501 | version, big_endian, write) | |
2c5af169 | 502 | : 0; |
1c6fdbd8 KO |
503 | } |
504 | ||
505 | static int jset_validate_entries(struct bch_fs *c, struct jset *jset, | |
506 | int write) | |
507 | { | |
7d6f07ed | 508 | char buf[100]; |
1c6fdbd8 KO |
509 | struct jset_entry *entry; |
510 | int ret = 0; | |
511 | ||
512 | vstruct_for_each(jset, entry) { | |
7d6f07ed KO |
513 | scnprintf(buf, sizeof(buf), "jset %llu entry offset %zi/%u", |
514 | le64_to_cpu(jset->seq), | |
515 | (u64 *) entry - jset->_data, | |
516 | le32_to_cpu(jset->u64s)); | |
517 | ||
1c6fdbd8 KO |
518 | if (journal_entry_err_on(vstruct_next(entry) > |
519 | vstruct_last(jset), c, | |
520 | "journal entry extends past end of jset")) { | |
521 | jset->u64s = cpu_to_le32((u64 *) entry - jset->_data); | |
522 | break; | |
523 | } | |
524 | ||
7d6f07ed KO |
525 | ret = bch2_journal_entry_validate(c, buf, entry, |
526 | le32_to_cpu(jset->version), | |
527 | JSET_BIG_ENDIAN(jset), write); | |
1c6fdbd8 KO |
528 | if (ret) |
529 | break; | |
530 | } | |
531 | fsck_err: | |
532 | return ret; | |
533 | } | |
534 | ||
535 | static int jset_validate(struct bch_fs *c, | |
ca73852a | 536 | struct bch_dev *ca, |
1c6fdbd8 KO |
537 | struct jset *jset, u64 sector, |
538 | unsigned bucket_sectors_left, | |
539 | unsigned sectors_read, | |
540 | int write) | |
541 | { | |
542 | size_t bytes = vstruct_bytes(jset); | |
543 | struct bch_csum csum; | |
26609b61 | 544 | unsigned version; |
1c6fdbd8 KO |
545 | int ret = 0; |
546 | ||
547 | if (le64_to_cpu(jset->magic) != jset_magic(c)) | |
548 | return JOURNAL_ENTRY_NONE; | |
549 | ||
26609b61 | 550 | version = le32_to_cpu(jset->version); |
ca73852a KO |
551 | if (journal_entry_err_on((version != BCH_JSET_VERSION_OLD && |
552 | version < bcachefs_metadata_version_min) || | |
553 | version >= bcachefs_metadata_version_max, c, | |
554 | "%s sector %llu seq %llu: unknown journal entry version %u", | |
ed9d58a2 KO |
555 | ca ? ca->name : c->name, |
556 | sector, le64_to_cpu(jset->seq), | |
ca73852a | 557 | version)) { |
35ef6df5 KO |
558 | /* don't try to continue: */ |
559 | return EINVAL; | |
1c6fdbd8 KO |
560 | } |
561 | ||
35ef6df5 KO |
562 | if (bytes > (sectors_read << 9) && |
563 | sectors_read < bucket_sectors_left) | |
564 | return JOURNAL_ENTRY_REREAD; | |
565 | ||
1c6fdbd8 | 566 | if (journal_entry_err_on(bytes > bucket_sectors_left << 9, c, |
ca73852a | 567 | "%s sector %llu seq %llu: journal entry too big (%zu bytes)", |
ed9d58a2 KO |
568 | ca ? ca->name : c->name, |
569 | sector, le64_to_cpu(jset->seq), bytes)) { | |
35ef6df5 KO |
570 | ret = JOURNAL_ENTRY_BAD; |
571 | le32_add_cpu(&jset->u64s, | |
572 | -((bytes - (bucket_sectors_left << 9)) / 8)); | |
1c6fdbd8 KO |
573 | } |
574 | ||
ed9d58a2 | 575 | if (journal_entry_err_on(!bch2_checksum_type_valid(c, JSET_CSUM_TYPE(jset)), c, |
ca73852a | 576 | "%s sector %llu seq %llu: journal entry with unknown csum type %llu", |
ed9d58a2 KO |
577 | ca ? ca->name : c->name, |
578 | sector, le64_to_cpu(jset->seq), | |
35ef6df5 KO |
579 | JSET_CSUM_TYPE(jset))) { |
580 | ret = JOURNAL_ENTRY_BAD; | |
ed9d58a2 | 581 | goto csum_done; |
35ef6df5 | 582 | } |
1c6fdbd8 | 583 | |
ed9d58a2 KO |
584 | if (write) |
585 | goto csum_done; | |
586 | ||
1c6fdbd8 KO |
587 | csum = csum_vstruct(c, JSET_CSUM_TYPE(jset), journal_nonce(jset), jset); |
588 | if (journal_entry_err_on(bch2_crc_cmp(csum, jset->csum), c, | |
ca73852a | 589 | "%s sector %llu seq %llu: journal checksum bad", |
ed9d58a2 KO |
590 | ca ? ca->name : c->name, |
591 | sector, le64_to_cpu(jset->seq))) | |
35ef6df5 | 592 | ret = JOURNAL_ENTRY_BAD; |
1c6fdbd8 KO |
593 | |
594 | bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset), | |
595 | jset->encrypted_start, | |
596 | vstruct_end(jset) - (void *) jset->encrypted_start); | |
ed9d58a2 KO |
597 | csum_done: |
598 | /* last_seq is ignored when JSET_NO_FLUSH is true */ | |
599 | if (journal_entry_err_on(!JSET_NO_FLUSH(jset) && | |
600 | le64_to_cpu(jset->last_seq) > le64_to_cpu(jset->seq), c, | |
601 | "invalid journal entry: last_seq > seq (%llu > %llu)", | |
602 | le64_to_cpu(jset->last_seq), | |
603 | le64_to_cpu(jset->seq))) { | |
1c6fdbd8 | 604 | jset->last_seq = jset->seq; |
ca73852a KO |
605 | return JOURNAL_ENTRY_BAD; |
606 | } | |
1c6fdbd8 KO |
607 | fsck_err: |
608 | return ret; | |
609 | } | |
610 | ||
ed9d58a2 KO |
611 | static int jset_validate_for_write(struct bch_fs *c, struct jset *jset) |
612 | { | |
613 | unsigned sectors = vstruct_sectors(jset, c->block_bits); | |
614 | ||
615 | return jset_validate(c, NULL, jset, 0, sectors, sectors, WRITE) ?: | |
616 | jset_validate_entries(c, jset, WRITE); | |
617 | } | |
618 | ||
1c6fdbd8 KO |
619 | struct journal_read_buf { |
620 | void *data; | |
621 | size_t size; | |
622 | }; | |
623 | ||
624 | static int journal_read_buf_realloc(struct journal_read_buf *b, | |
625 | size_t new_size) | |
626 | { | |
627 | void *n; | |
628 | ||
629 | /* the bios are sized for this many pages, max: */ | |
630 | if (new_size > JOURNAL_ENTRY_SIZE_MAX) | |
631 | return -ENOMEM; | |
632 | ||
633 | new_size = roundup_pow_of_two(new_size); | |
634 | n = kvpmalloc(new_size, GFP_KERNEL); | |
635 | if (!n) | |
636 | return -ENOMEM; | |
637 | ||
638 | kvpfree(b->data, b->size); | |
639 | b->data = n; | |
640 | b->size = new_size; | |
641 | return 0; | |
642 | } | |
643 | ||
644 | static int journal_read_bucket(struct bch_dev *ca, | |
645 | struct journal_read_buf *buf, | |
646 | struct journal_list *jlist, | |
a9ec3454 | 647 | unsigned bucket) |
1c6fdbd8 KO |
648 | { |
649 | struct bch_fs *c = ca->fs; | |
650 | struct journal_device *ja = &ca->journal; | |
1c6fdbd8 KO |
651 | struct jset *j = NULL; |
652 | unsigned sectors, sectors_read = 0; | |
653 | u64 offset = bucket_to_sector(ca, ja->buckets[bucket]), | |
654 | end = offset + ca->mi.bucket_size; | |
655 | bool saw_bad = false; | |
656 | int ret = 0; | |
657 | ||
658 | pr_debug("reading %u", bucket); | |
659 | ||
660 | while (offset < end) { | |
661 | if (!sectors_read) { | |
ac10a961 KO |
662 | struct bio *bio; |
663 | unsigned nr_bvecs; | |
664 | reread: | |
665 | sectors_read = min_t(unsigned, | |
1c6fdbd8 | 666 | end - offset, buf->size >> 9); |
ac10a961 KO |
667 | nr_bvecs = buf_pages(buf->data, sectors_read << 9); |
668 | ||
669 | bio = bio_kmalloc(nr_bvecs, GFP_KERNEL); | |
670 | bio_init(bio, ca->disk_sb.bdev, bio->bi_inline_vecs, nr_bvecs, REQ_OP_READ); | |
1c6fdbd8 | 671 | |
885678f6 KO |
672 | bio->bi_iter.bi_sector = offset; |
673 | bch2_bio_map(bio, buf->data, sectors_read << 9); | |
1c6fdbd8 KO |
674 | |
675 | ret = submit_bio_wait(bio); | |
ac10a961 | 676 | kfree(bio); |
1c6fdbd8 KO |
677 | |
678 | if (bch2_dev_io_err_on(ret, ca, | |
0fefe8d8 | 679 | "journal read error: sector %llu", |
1c6fdbd8 | 680 | offset) || |
29d90f61 KO |
681 | bch2_meta_read_fault("journal")) { |
682 | /* | |
683 | * We don't error out of the recovery process | |
684 | * here, since the relevant journal entry may be | |
685 | * found on a different device, and missing or | |
686 | * no journal entries will be handled later | |
687 | */ | |
688 | return 0; | |
689 | } | |
1c6fdbd8 KO |
690 | |
691 | j = buf->data; | |
692 | } | |
693 | ||
ca73852a | 694 | ret = jset_validate(c, ca, j, offset, |
1c6fdbd8 KO |
695 | end - offset, sectors_read, |
696 | READ); | |
697 | switch (ret) { | |
698 | case BCH_FSCK_OK: | |
ca73852a | 699 | sectors = vstruct_sectors(j, c->block_bits); |
1c6fdbd8 KO |
700 | break; |
701 | case JOURNAL_ENTRY_REREAD: | |
702 | if (vstruct_bytes(j) > buf->size) { | |
703 | ret = journal_read_buf_realloc(buf, | |
704 | vstruct_bytes(j)); | |
705 | if (ret) | |
706 | return ret; | |
707 | } | |
708 | goto reread; | |
709 | case JOURNAL_ENTRY_NONE: | |
710 | if (!saw_bad) | |
711 | return 0; | |
712 | sectors = c->opts.block_size; | |
713 | goto next_block; | |
714 | case JOURNAL_ENTRY_BAD: | |
715 | saw_bad = true; | |
ca73852a KO |
716 | /* |
717 | * On checksum error we don't really trust the size | |
718 | * field of the journal entry we read, so try reading | |
719 | * again at next block boundary: | |
720 | */ | |
1c6fdbd8 | 721 | sectors = c->opts.block_size; |
ca73852a | 722 | break; |
1c6fdbd8 KO |
723 | default: |
724 | return ret; | |
725 | } | |
726 | ||
727 | /* | |
728 | * This happens sometimes if we don't have discards on - | |
729 | * when we've partially overwritten a bucket with new | |
730 | * journal entries. We don't need the rest of the | |
731 | * bucket: | |
732 | */ | |
733 | if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket]) | |
734 | return 0; | |
735 | ||
736 | ja->bucket_seq[bucket] = le64_to_cpu(j->seq); | |
737 | ||
738 | mutex_lock(&jlist->lock); | |
e4c3f386 KO |
739 | ret = journal_entry_add(c, ca, (struct bch_extent_ptr) { |
740 | .dev = ca->dev_idx, | |
741 | .offset = offset, | |
742 | }, jlist, j, ret != 0); | |
1c6fdbd8 KO |
743 | mutex_unlock(&jlist->lock); |
744 | ||
745 | switch (ret) { | |
746 | case JOURNAL_ENTRY_ADD_OK: | |
1c6fdbd8 KO |
747 | break; |
748 | case JOURNAL_ENTRY_ADD_OUT_OF_RANGE: | |
749 | break; | |
750 | default: | |
751 | return ret; | |
752 | } | |
1c6fdbd8 KO |
753 | next_block: |
754 | pr_debug("next"); | |
755 | offset += sectors; | |
756 | sectors_read -= sectors; | |
757 | j = ((void *) j) + (sectors << 9); | |
758 | } | |
759 | ||
760 | return 0; | |
761 | } | |
762 | ||
763 | static void bch2_journal_read_device(struct closure *cl) | |
764 | { | |
1c6fdbd8 KO |
765 | struct journal_device *ja = |
766 | container_of(cl, struct journal_device, read); | |
767 | struct bch_dev *ca = container_of(ja, struct bch_dev, journal); | |
768 | struct journal_list *jlist = | |
769 | container_of(cl->parent, struct journal_list, cl); | |
1c6fdbd8 | 770 | struct journal_read_buf buf = { NULL, 0 }; |
a9ec3454 KO |
771 | u64 min_seq = U64_MAX; |
772 | unsigned i; | |
1c6fdbd8 KO |
773 | int ret; |
774 | ||
775 | if (!ja->nr) | |
776 | goto out; | |
777 | ||
1c6fdbd8 KO |
778 | ret = journal_read_buf_realloc(&buf, PAGE_SIZE); |
779 | if (ret) | |
780 | goto err; | |
781 | ||
782 | pr_debug("%u journal buckets", ja->nr); | |
783 | ||
1c6fdbd8 | 784 | for (i = 0; i < ja->nr; i++) { |
a9ec3454 KO |
785 | ret = journal_read_bucket(ca, &buf, jlist, i); |
786 | if (ret) | |
787 | goto err; | |
1c6fdbd8 KO |
788 | } |
789 | ||
a9ec3454 KO |
790 | /* Find the journal bucket with the highest sequence number: */ |
791 | for (i = 0; i < ja->nr; i++) { | |
792 | if (ja->bucket_seq[i] > ja->bucket_seq[ja->cur_idx]) | |
793 | ja->cur_idx = i; | |
1c6fdbd8 | 794 | |
a9ec3454 | 795 | min_seq = min(ja->bucket_seq[i], min_seq); |
1c6fdbd8 KO |
796 | } |
797 | ||
1c6fdbd8 | 798 | /* |
1c6fdbd8 KO |
799 | * If there's duplicate journal entries in multiple buckets (which |
800 | * definitely isn't supposed to happen, but...) - make sure to start | |
801 | * cur_idx at the last of those buckets, so we don't deadlock trying to | |
802 | * allocate | |
803 | */ | |
a9ec3454 KO |
804 | while (ja->bucket_seq[ja->cur_idx] > min_seq && |
805 | ja->bucket_seq[ja->cur_idx] > | |
806 | ja->bucket_seq[(ja->cur_idx + 1) % ja->nr]) | |
a36d3685 | 807 | ja->cur_idx = (ja->cur_idx + 1) % ja->nr; |
a9ec3454 KO |
808 | |
809 | ja->sectors_free = 0; | |
1c6fdbd8 KO |
810 | |
811 | /* | |
0ce2dbbe | 812 | * Set dirty_idx to indicate the entire journal is full and needs to be |
1c6fdbd8 KO |
813 | * reclaimed - journal reclaim will immediately reclaim whatever isn't |
814 | * pinned when it first runs: | |
815 | */ | |
0ce2dbbe KO |
816 | ja->discard_idx = ja->dirty_idx_ondisk = |
817 | ja->dirty_idx = (ja->cur_idx + 1) % ja->nr; | |
1c6fdbd8 KO |
818 | out: |
819 | kvpfree(buf.data, buf.size); | |
1c6fdbd8 KO |
820 | percpu_ref_put(&ca->io_ref); |
821 | closure_return(cl); | |
822 | return; | |
823 | err: | |
824 | mutex_lock(&jlist->lock); | |
825 | jlist->ret = ret; | |
826 | mutex_unlock(&jlist->lock); | |
827 | goto out; | |
1c6fdbd8 KO |
828 | } |
829 | ||
e4c3f386 KO |
830 | static void bch2_journal_ptrs_to_text(struct printbuf *out, struct bch_fs *c, |
831 | struct journal_replay *j) | |
832 | { | |
833 | unsigned i; | |
834 | ||
835 | for (i = 0; i < j->nr_ptrs; i++) { | |
836 | struct bch_dev *ca = c->devs[j->ptrs[i].dev]; | |
514852c2 KO |
837 | u64 offset; |
838 | ||
839 | div64_u64_rem(j->ptrs[i].offset, ca->mi.bucket_size, &offset); | |
e4c3f386 KO |
840 | |
841 | if (i) | |
842 | pr_buf(out, " "); | |
843 | pr_buf(out, "%u:%llu (offset %llu)", | |
844 | j->ptrs[i].dev, | |
514852c2 | 845 | (u64) j->ptrs[i].offset, offset); |
e4c3f386 KO |
846 | } |
847 | } | |
848 | ||
adbcada4 KO |
849 | int bch2_journal_read(struct bch_fs *c, struct list_head *list, |
850 | u64 *blacklist_seq, u64 *start_seq) | |
1c6fdbd8 | 851 | { |
1c6fdbd8 | 852 | struct journal_list jlist; |
adbcada4 | 853 | struct journal_replay *i, *t; |
1c6fdbd8 | 854 | struct bch_dev *ca; |
1c6fdbd8 KO |
855 | unsigned iter; |
856 | size_t keys = 0, entries = 0; | |
857 | bool degraded = false; | |
adbcada4 | 858 | u64 seq, last_seq = 0; |
1c6fdbd8 KO |
859 | int ret = 0; |
860 | ||
861 | closure_init_stack(&jlist.cl); | |
862 | mutex_init(&jlist.lock); | |
863 | jlist.head = list; | |
864 | jlist.ret = 0; | |
865 | ||
866 | for_each_member_device(ca, c, iter) { | |
6bdbfa87 | 867 | if (!test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) && |
89fd25be | 868 | !(bch2_dev_has_data(c, ca) & (1 << BCH_DATA_journal))) |
1c6fdbd8 KO |
869 | continue; |
870 | ||
2436cb9f KO |
871 | if ((ca->mi.state == BCH_MEMBER_STATE_rw || |
872 | ca->mi.state == BCH_MEMBER_STATE_ro) && | |
1c6fdbd8 KO |
873 | percpu_ref_tryget(&ca->io_ref)) |
874 | closure_call(&ca->journal.read, | |
875 | bch2_journal_read_device, | |
876 | system_unbound_wq, | |
877 | &jlist.cl); | |
878 | else | |
879 | degraded = true; | |
880 | } | |
881 | ||
882 | closure_sync(&jlist.cl); | |
883 | ||
884 | if (jlist.ret) | |
885 | return jlist.ret; | |
886 | ||
adbcada4 KO |
887 | if (list_empty(list)) { |
888 | bch_info(c, "journal read done, but no entries found"); | |
889 | return 0; | |
890 | } | |
891 | ||
892 | i = list_last_entry(list, struct journal_replay, list); | |
893 | *start_seq = le64_to_cpu(i->j.seq) + 1; | |
894 | ||
895 | /* | |
896 | * Find most recent flush entry, and ignore newer non flush entries - | |
897 | * those entries will be blacklisted: | |
898 | */ | |
899 | list_for_each_entry_safe_reverse(i, t, list, list) { | |
900 | if (i->ignore) | |
901 | continue; | |
902 | ||
903 | if (!JSET_NO_FLUSH(&i->j)) { | |
904 | last_seq = le64_to_cpu(i->j.last_seq); | |
905 | *blacklist_seq = le64_to_cpu(i->j.seq) + 1; | |
906 | break; | |
907 | } | |
908 | ||
909 | journal_replay_free(c, i); | |
910 | } | |
911 | ||
912 | if (!last_seq) { | |
913 | fsck_err(c, "journal read done, but no entries found after dropping non-flushes"); | |
914 | return -1; | |
915 | } | |
916 | ||
917 | /* Drop blacklisted entries and entries older than last_seq: */ | |
918 | list_for_each_entry_safe(i, t, list, list) { | |
919 | if (i->ignore) | |
920 | continue; | |
921 | ||
922 | seq = le64_to_cpu(i->j.seq); | |
923 | if (seq < last_seq) { | |
924 | journal_replay_free(c, i); | |
925 | continue; | |
926 | } | |
927 | ||
928 | if (bch2_journal_seq_is_blacklisted(c, seq, true)) { | |
929 | fsck_err_on(!JSET_NO_FLUSH(&i->j), c, | |
930 | "found blacklisted journal entry %llu", seq); | |
931 | ||
932 | journal_replay_free(c, i); | |
933 | } | |
934 | } | |
935 | ||
936 | /* Check for missing entries: */ | |
937 | seq = last_seq; | |
938 | list_for_each_entry(i, list, list) { | |
939 | if (i->ignore) | |
940 | continue; | |
941 | ||
942 | BUG_ON(seq > le64_to_cpu(i->j.seq)); | |
943 | ||
944 | while (seq < le64_to_cpu(i->j.seq)) { | |
945 | u64 missing_start, missing_end; | |
e4c3f386 | 946 | char buf1[200], buf2[200]; |
adbcada4 KO |
947 | |
948 | while (seq < le64_to_cpu(i->j.seq) && | |
949 | bch2_journal_seq_is_blacklisted(c, seq, false)) | |
950 | seq++; | |
951 | ||
952 | if (seq == le64_to_cpu(i->j.seq)) | |
953 | break; | |
954 | ||
955 | missing_start = seq; | |
956 | ||
957 | while (seq < le64_to_cpu(i->j.seq) && | |
958 | !bch2_journal_seq_is_blacklisted(c, seq, false)) | |
959 | seq++; | |
960 | ||
e4c3f386 KO |
961 | if (i->list.prev != list) { |
962 | struct printbuf out = PBUF(buf1); | |
963 | struct journal_replay *p = list_prev_entry(i, list); | |
964 | ||
965 | bch2_journal_ptrs_to_text(&out, c, p); | |
966 | pr_buf(&out, " size %llu", vstruct_sectors(&p->j, c->block_bits)); | |
967 | } else | |
968 | sprintf(buf1, "(none)"); | |
969 | bch2_journal_ptrs_to_text(&PBUF(buf2), c, i); | |
970 | ||
adbcada4 | 971 | missing_end = seq - 1; |
e4c3f386 KO |
972 | fsck_err(c, "journal entries %llu-%llu missing! (replaying %llu-%llu)\n" |
973 | " prev at %s\n" | |
974 | " next at %s", | |
adbcada4 | 975 | missing_start, missing_end, |
e4c3f386 KO |
976 | last_seq, *blacklist_seq - 1, |
977 | buf1, buf2); | |
adbcada4 KO |
978 | } |
979 | ||
980 | seq++; | |
981 | } | |
982 | ||
1c6fdbd8 | 983 | list_for_each_entry(i, list, list) { |
1dd7f9d9 KO |
984 | struct jset_entry *entry; |
985 | struct bkey_i *k, *_n; | |
e4c3f386 KO |
986 | struct bch_replicas_padded replicas = { |
987 | .e.data_type = BCH_DATA_journal, | |
988 | .e.nr_required = 1, | |
989 | }; | |
990 | unsigned ptr; | |
7ef2a73a KO |
991 | char buf[80]; |
992 | ||
adbcada4 KO |
993 | if (i->ignore) |
994 | continue; | |
995 | ||
1c6fdbd8 KO |
996 | ret = jset_validate_entries(c, &i->j, READ); |
997 | if (ret) | |
998 | goto fsck_err; | |
999 | ||
e4c3f386 KO |
1000 | for (ptr = 0; ptr < i->nr_ptrs; ptr++) |
1001 | replicas.e.devs[replicas.e.nr_devs++] = i->ptrs[ptr].dev; | |
1002 | ||
26452d1d KO |
1003 | bch2_replicas_entry_sort(&replicas.e); |
1004 | ||
1c6fdbd8 KO |
1005 | /* |
1006 | * If we're mounting in degraded mode - if we didn't read all | |
1007 | * the devices - this is wrong: | |
1008 | */ | |
1009 | ||
1010 | if (!degraded && | |
1011 | (test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) || | |
988e98cf | 1012 | fsck_err_on(!bch2_replicas_marked(c, &replicas.e), c, |
7ef2a73a KO |
1013 | "superblock not marked as containing replicas %s", |
1014 | (bch2_replicas_entry_to_text(&PBUF(buf), | |
1015 | &replicas.e), buf)))) { | |
1016 | ret = bch2_mark_replicas(c, &replicas.e); | |
1c6fdbd8 KO |
1017 | if (ret) |
1018 | return ret; | |
1019 | } | |
1c6fdbd8 KO |
1020 | |
1021 | for_each_jset_key(k, _n, entry, &i->j) | |
1022 | keys++; | |
1023 | entries++; | |
1024 | } | |
1025 | ||
adbcada4 KO |
1026 | bch_info(c, "journal read done, %zu keys in %zu entries, seq %llu", |
1027 | keys, entries, *start_seq); | |
1dd7f9d9 | 1028 | |
adbcada4 KO |
1029 | if (*start_seq != *blacklist_seq) |
1030 | bch_info(c, "dropped unflushed entries %llu-%llu", | |
1031 | *blacklist_seq, *start_seq - 1); | |
1c6fdbd8 KO |
1032 | fsck_err: |
1033 | return ret; | |
1034 | } | |
1035 | ||
1c6fdbd8 KO |
1036 | /* journal write: */ |
1037 | ||
a9ec3454 KO |
1038 | static void __journal_write_alloc(struct journal *j, |
1039 | struct journal_buf *w, | |
1040 | struct dev_alloc_list *devs_sorted, | |
1041 | unsigned sectors, | |
1042 | unsigned *replicas, | |
1043 | unsigned replicas_want) | |
1c6fdbd8 KO |
1044 | { |
1045 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
1c6fdbd8 KO |
1046 | struct journal_device *ja; |
1047 | struct bch_dev *ca; | |
a9ec3454 | 1048 | unsigned i; |
a2753581 | 1049 | |
a9ec3454 KO |
1050 | if (*replicas >= replicas_want) |
1051 | return; | |
1c6fdbd8 | 1052 | |
a9ec3454 KO |
1053 | for (i = 0; i < devs_sorted->nr; i++) { |
1054 | ca = rcu_dereference(c->devs[devs_sorted->devs[i]]); | |
1c6fdbd8 KO |
1055 | if (!ca) |
1056 | continue; | |
1057 | ||
1c6fdbd8 | 1058 | ja = &ca->journal; |
1c6fdbd8 KO |
1059 | |
1060 | /* | |
1061 | * Check that we can use this device, and aren't already using | |
1062 | * it: | |
1063 | */ | |
a9ec3454 | 1064 | if (!ca->mi.durability || |
2436cb9f | 1065 | ca->mi.state != BCH_MEMBER_STATE_rw || |
a9ec3454 | 1066 | !ja->nr || |
26609b61 KO |
1067 | bch2_bkey_has_device(bkey_i_to_s_c(&w->key), |
1068 | ca->dev_idx) || | |
a9ec3454 | 1069 | sectors > ja->sectors_free) |
1c6fdbd8 KO |
1070 | continue; |
1071 | ||
3d080aa5 | 1072 | bch2_dev_stripe_increment(ca, &j->wp.stripe); |
1c6fdbd8 | 1073 | |
26609b61 | 1074 | bch2_bkey_append_ptr(&w->key, |
1c6fdbd8 KO |
1075 | (struct bch_extent_ptr) { |
1076 | .offset = bucket_to_sector(ca, | |
a9ec3454 KO |
1077 | ja->buckets[ja->cur_idx]) + |
1078 | ca->mi.bucket_size - | |
1079 | ja->sectors_free, | |
1c6fdbd8 KO |
1080 | .dev = ca->dev_idx, |
1081 | }); | |
1082 | ||
a9ec3454 KO |
1083 | ja->sectors_free -= sectors; |
1084 | ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq); | |
1085 | ||
1086 | *replicas += ca->mi.durability; | |
1087 | ||
1088 | if (*replicas >= replicas_want) | |
1089 | break; | |
1c6fdbd8 | 1090 | } |
a9ec3454 | 1091 | } |
1c6fdbd8 | 1092 | |
a9ec3454 KO |
1093 | /** |
1094 | * journal_next_bucket - move on to the next journal bucket if possible | |
1095 | */ | |
1096 | static int journal_write_alloc(struct journal *j, struct journal_buf *w, | |
1097 | unsigned sectors) | |
1098 | { | |
1099 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
d042b040 | 1100 | struct bch_devs_mask devs; |
a9ec3454 KO |
1101 | struct journal_device *ja; |
1102 | struct bch_dev *ca; | |
1103 | struct dev_alloc_list devs_sorted; | |
d042b040 KO |
1104 | unsigned target = c->opts.metadata_target ?: |
1105 | c->opts.foreground_target; | |
a9ec3454 KO |
1106 | unsigned i, replicas = 0, replicas_want = |
1107 | READ_ONCE(c->opts.metadata_replicas); | |
1c6fdbd8 | 1108 | |
a9ec3454 | 1109 | rcu_read_lock(); |
d042b040 KO |
1110 | retry: |
1111 | devs = target_rw_devs(c, BCH_DATA_journal, target); | |
1c6fdbd8 | 1112 | |
d042b040 | 1113 | devs_sorted = bch2_dev_alloc_list(c, &j->wp.stripe, &devs); |
1c6fdbd8 | 1114 | |
a9ec3454 KO |
1115 | __journal_write_alloc(j, w, &devs_sorted, |
1116 | sectors, &replicas, replicas_want); | |
1c6fdbd8 | 1117 | |
a9ec3454 KO |
1118 | if (replicas >= replicas_want) |
1119 | goto done; | |
1120 | ||
1121 | for (i = 0; i < devs_sorted.nr; i++) { | |
1122 | ca = rcu_dereference(c->devs[devs_sorted.devs[i]]); | |
1123 | if (!ca) | |
1124 | continue; | |
1125 | ||
1126 | ja = &ca->journal; | |
1127 | ||
1128 | if (sectors > ja->sectors_free && | |
1129 | sectors <= ca->mi.bucket_size && | |
03d5eaed KO |
1130 | bch2_journal_dev_buckets_available(j, ja, |
1131 | journal_space_discarded)) { | |
a9ec3454 KO |
1132 | ja->cur_idx = (ja->cur_idx + 1) % ja->nr; |
1133 | ja->sectors_free = ca->mi.bucket_size; | |
68ef94a6 KO |
1134 | |
1135 | /* | |
1136 | * ja->bucket_seq[ja->cur_idx] must always have | |
1137 | * something sensible: | |
1138 | */ | |
1139 | ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq); | |
a9ec3454 KO |
1140 | } |
1141 | } | |
1142 | ||
1143 | __journal_write_alloc(j, w, &devs_sorted, | |
1144 | sectors, &replicas, replicas_want); | |
d042b040 KO |
1145 | |
1146 | if (replicas < replicas_want && target) { | |
1147 | /* Retry from all devices: */ | |
1148 | target = 0; | |
1149 | goto retry; | |
1150 | } | |
a9ec3454 | 1151 | done: |
a9ec3454 KO |
1152 | rcu_read_unlock(); |
1153 | ||
07a1006a KO |
1154 | BUG_ON(bkey_val_u64s(&w->key.k) > BCH_REPLICAS_MAX); |
1155 | ||
57cb2142 | 1156 | return replicas >= c->opts.metadata_replicas_required ? 0 : -EROFS; |
1c6fdbd8 KO |
1157 | } |
1158 | ||
1159 | static void journal_write_compact(struct jset *jset) | |
1160 | { | |
1161 | struct jset_entry *i, *next, *prev = NULL; | |
1162 | ||
1163 | /* | |
1164 | * Simple compaction, dropping empty jset_entries (from journal | |
1165 | * reservations that weren't fully used) and merging jset_entries that | |
1166 | * can be. | |
1167 | * | |
1168 | * If we wanted to be really fancy here, we could sort all the keys in | |
1169 | * the jset and drop keys that were overwritten - probably not worth it: | |
1170 | */ | |
1171 | vstruct_for_each_safe(jset, i, next) { | |
1172 | unsigned u64s = le16_to_cpu(i->u64s); | |
1173 | ||
1174 | /* Empty entry: */ | |
1175 | if (!u64s) | |
1176 | continue; | |
1177 | ||
1178 | /* Can we merge with previous entry? */ | |
1179 | if (prev && | |
1180 | i->btree_id == prev->btree_id && | |
1181 | i->level == prev->level && | |
1182 | i->type == prev->type && | |
1183 | i->type == BCH_JSET_ENTRY_btree_keys && | |
1184 | le16_to_cpu(prev->u64s) + u64s <= U16_MAX) { | |
1185 | memmove_u64s_down(vstruct_next(prev), | |
1186 | i->_data, | |
1187 | u64s); | |
1188 | le16_add_cpu(&prev->u64s, u64s); | |
1189 | continue; | |
1190 | } | |
1191 | ||
1192 | /* Couldn't merge, move i into new position (after prev): */ | |
1193 | prev = prev ? vstruct_next(prev) : jset->start; | |
1194 | if (i != prev) | |
1195 | memmove_u64s_down(prev, i, jset_u64s(u64s)); | |
1196 | } | |
1197 | ||
1198 | prev = prev ? vstruct_next(prev) : jset->start; | |
1199 | jset->u64s = cpu_to_le32((u64 *) prev - jset->_data); | |
1200 | } | |
1201 | ||
1202 | static void journal_buf_realloc(struct journal *j, struct journal_buf *buf) | |
1203 | { | |
1204 | /* we aren't holding j->lock: */ | |
1205 | unsigned new_size = READ_ONCE(j->buf_size_want); | |
1206 | void *new_buf; | |
1207 | ||
d16b4a77 | 1208 | if (buf->buf_size >= new_size) |
1c6fdbd8 KO |
1209 | return; |
1210 | ||
1211 | new_buf = kvpmalloc(new_size, GFP_NOIO|__GFP_NOWARN); | |
1212 | if (!new_buf) | |
1213 | return; | |
1214 | ||
d16b4a77 | 1215 | memcpy(new_buf, buf->data, buf->buf_size); |
c859430b KO |
1216 | |
1217 | spin_lock(&j->lock); | |
1218 | swap(buf->data, new_buf); | |
1219 | swap(buf->buf_size, new_size); | |
1220 | spin_unlock(&j->lock); | |
1221 | ||
1222 | kvpfree(new_buf, new_size); | |
1c6fdbd8 KO |
1223 | } |
1224 | ||
ebb84d09 KO |
1225 | static inline struct journal_buf *journal_last_unwritten_buf(struct journal *j) |
1226 | { | |
1227 | return j->buf + j->reservations.unwritten_idx; | |
1228 | } | |
1229 | ||
1c6fdbd8 KO |
1230 | static void journal_write_done(struct closure *cl) |
1231 | { | |
1232 | struct journal *j = container_of(cl, struct journal, io); | |
1233 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
ebb84d09 | 1234 | struct journal_buf *w = journal_last_unwritten_buf(j); |
1c6fdbd8 | 1235 | struct bch_devs_list devs = |
26609b61 | 1236 | bch2_bkey_devs(bkey_i_to_s_c(&w->key)); |
7ef2a73a | 1237 | struct bch_replicas_padded replicas; |
ebb84d09 | 1238 | union journal_res_state old, new; |
1784d43a | 1239 | u64 v, seq; |
158eecb8 | 1240 | int err = 0; |
1c6fdbd8 | 1241 | |
9c859dc9 KO |
1242 | bch2_time_stats_update(j->write_time, j->write_start_time); |
1243 | ||
1c6fdbd8 KO |
1244 | if (!devs.nr) { |
1245 | bch_err(c, "unable to write journal to sufficient devices"); | |
158eecb8 KO |
1246 | err = -EIO; |
1247 | } else { | |
1248 | bch2_devlist_to_replicas(&replicas.e, BCH_DATA_journal, devs); | |
1249 | if (bch2_mark_replicas(c, &replicas.e)) | |
1250 | err = -EIO; | |
1c6fdbd8 KO |
1251 | } |
1252 | ||
158eecb8 KO |
1253 | if (err) |
1254 | bch2_fatal_error(c); | |
1c6fdbd8 KO |
1255 | |
1256 | spin_lock(&j->lock); | |
ed9d58a2 | 1257 | seq = le64_to_cpu(w->data->seq); |
ed9d58a2 | 1258 | |
1c6fdbd8 KO |
1259 | if (seq >= j->pin.front) |
1260 | journal_seq_pin(j, seq)->devs = devs; | |
1261 | ||
0ce2dbbe | 1262 | j->seq_ondisk = seq; |
158eecb8 KO |
1263 | if (err && (!j->err_seq || seq < j->err_seq)) |
1264 | j->err_seq = seq; | |
adbcada4 | 1265 | |
ed9d58a2 | 1266 | if (!JSET_NO_FLUSH(w->data)) { |
adbcada4 | 1267 | j->flushed_seq_ondisk = seq; |
1784d43a | 1268 | j->last_seq_ondisk = w->last_seq; |
adbcada4 | 1269 | } |
0ce2dbbe | 1270 | |
1c6fdbd8 KO |
1271 | /* |
1272 | * Updating last_seq_ondisk may let bch2_journal_reclaim_work() discard | |
1273 | * more buckets: | |
1274 | * | |
1275 | * Must come before signaling write completion, for | |
1276 | * bch2_fs_journal_stop(): | |
1277 | */ | |
b7a9bbfc | 1278 | journal_reclaim_kick(&c->journal); |
158eecb8 | 1279 | |
1c6fdbd8 KO |
1280 | /* also must come before signalling write completion: */ |
1281 | closure_debug_destroy(cl); | |
1282 | ||
ebb84d09 KO |
1283 | v = atomic64_read(&j->reservations.counter); |
1284 | do { | |
1285 | old.v = new.v = v; | |
1286 | BUG_ON(new.idx == new.unwritten_idx); | |
1287 | ||
1288 | new.unwritten_idx++; | |
1289 | } while ((v = atomic64_cmpxchg(&j->reservations.counter, | |
1290 | old.v, new.v)) != old.v); | |
1c6fdbd8 | 1291 | |
5d32c5bb KO |
1292 | bch2_journal_space_available(j); |
1293 | ||
1c6fdbd8 KO |
1294 | closure_wake_up(&w->wait); |
1295 | journal_wake(j); | |
1296 | ||
1297 | if (test_bit(JOURNAL_NEED_WRITE, &j->flags)) | |
731bdd2e | 1298 | mod_delayed_work(c->io_complete_wq, &j->write_work, 0); |
1c6fdbd8 | 1299 | spin_unlock(&j->lock); |
ebb84d09 KO |
1300 | |
1301 | if (new.unwritten_idx != new.idx && | |
1302 | !journal_state_count(new, new.unwritten_idx)) | |
731bdd2e | 1303 | closure_call(&j->io, bch2_journal_write, c->io_complete_wq, NULL); |
1c6fdbd8 KO |
1304 | } |
1305 | ||
1306 | static void journal_write_endio(struct bio *bio) | |
1307 | { | |
1308 | struct bch_dev *ca = bio->bi_private; | |
1309 | struct journal *j = &ca->fs->journal; | |
1310 | ||
0fefe8d8 | 1311 | if (bch2_dev_io_err_on(bio->bi_status, ca, "journal write error: %s", |
63b214e7 | 1312 | bch2_blk_status_to_str(bio->bi_status)) || |
1c6fdbd8 | 1313 | bch2_meta_write_fault("journal")) { |
ebb84d09 | 1314 | struct journal_buf *w = journal_last_unwritten_buf(j); |
1c6fdbd8 KO |
1315 | unsigned long flags; |
1316 | ||
1317 | spin_lock_irqsave(&j->err_lock, flags); | |
26609b61 | 1318 | bch2_bkey_drop_device(bkey_i_to_s(&w->key), ca->dev_idx); |
1c6fdbd8 KO |
1319 | spin_unlock_irqrestore(&j->err_lock, flags); |
1320 | } | |
1321 | ||
1322 | closure_put(&j->io); | |
1323 | percpu_ref_put(&ca->io_ref); | |
1324 | } | |
1325 | ||
280249b9 KO |
1326 | static void do_journal_write(struct closure *cl) |
1327 | { | |
1328 | struct journal *j = container_of(cl, struct journal, io); | |
1329 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
1330 | struct bch_dev *ca; | |
1331 | struct journal_buf *w = journal_last_unwritten_buf(j); | |
1332 | struct bch_extent_ptr *ptr; | |
1333 | struct bio *bio; | |
1334 | unsigned sectors = vstruct_sectors(w->data, c->block_bits); | |
1335 | ||
1336 | extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) { | |
1337 | ca = bch_dev_bkey_exists(c, ptr->dev); | |
1338 | if (!percpu_ref_tryget(&ca->io_ref)) { | |
1339 | /* XXX: fix this */ | |
1340 | bch_err(c, "missing device for journal write\n"); | |
1341 | continue; | |
1342 | } | |
1343 | ||
1344 | this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_journal], | |
1345 | sectors); | |
1346 | ||
1347 | bio = ca->journal.bio; | |
1348 | bio_reset(bio, ca->disk_sb.bdev, REQ_OP_WRITE|REQ_SYNC|REQ_META); | |
1349 | bio->bi_iter.bi_sector = ptr->offset; | |
1350 | bio->bi_end_io = journal_write_endio; | |
1351 | bio->bi_private = ca; | |
1352 | ||
a28bd48a KO |
1353 | BUG_ON(bio->bi_iter.bi_sector == ca->prev_journal_sector); |
1354 | ca->prev_journal_sector = bio->bi_iter.bi_sector; | |
1355 | ||
280249b9 KO |
1356 | if (!JSET_NO_FLUSH(w->data)) |
1357 | bio->bi_opf |= REQ_FUA; | |
1358 | if (!JSET_NO_FLUSH(w->data) && !w->separate_flush) | |
1359 | bio->bi_opf |= REQ_PREFLUSH; | |
1360 | ||
1361 | bch2_bio_map(bio, w->data, sectors << 9); | |
1362 | ||
1363 | trace_journal_write(bio); | |
1364 | closure_bio_submit(bio, cl); | |
1365 | ||
1366 | ca->journal.bucket_seq[ca->journal.cur_idx] = | |
1367 | le64_to_cpu(w->data->seq); | |
1368 | } | |
1369 | ||
731bdd2e | 1370 | continue_at(cl, journal_write_done, c->io_complete_wq); |
280249b9 KO |
1371 | return; |
1372 | } | |
1373 | ||
1c6fdbd8 KO |
1374 | void bch2_journal_write(struct closure *cl) |
1375 | { | |
1376 | struct journal *j = container_of(cl, struct journal, io); | |
1377 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
1378 | struct bch_dev *ca; | |
ebb84d09 | 1379 | struct journal_buf *w = journal_last_unwritten_buf(j); |
3ccc5c50 | 1380 | struct jset_entry *start, *end; |
1c6fdbd8 KO |
1381 | struct jset *jset; |
1382 | struct bio *bio; | |
85674154 | 1383 | char *journal_debug_buf = NULL; |
26609b61 | 1384 | bool validate_before_checksum = false; |
280249b9 | 1385 | unsigned i, sectors, bytes, u64s, nr_rw_members = 0; |
e5a66496 KO |
1386 | int ret; |
1387 | ||
b7a9bbfc KO |
1388 | BUG_ON(BCH_SB_CLEAN(c->disk_sb.sb)); |
1389 | ||
1c6fdbd8 KO |
1390 | journal_buf_realloc(j, w); |
1391 | jset = w->data; | |
1392 | ||
1393 | j->write_start_time = local_clock(); | |
1c6fdbd8 | 1394 | |
adbcada4 KO |
1395 | spin_lock(&j->lock); |
1396 | if (c->sb.features & (1ULL << BCH_FEATURE_journal_no_flush) && | |
1397 | !w->must_flush && | |
1398 | (jiffies - j->last_flush_write) < msecs_to_jiffies(j->write_delay_ms) && | |
1399 | test_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags)) { | |
1400 | w->noflush = true; | |
1401 | SET_JSET_NO_FLUSH(jset, true); | |
1784d43a | 1402 | jset->last_seq = w->last_seq = 0; |
adbcada4 KO |
1403 | |
1404 | j->nr_noflush_writes++; | |
1405 | } else { | |
1406 | j->last_flush_write = jiffies; | |
1407 | j->nr_flush_writes++; | |
1408 | } | |
1409 | spin_unlock(&j->lock); | |
1410 | ||
00b8ccf7 KO |
1411 | /* |
1412 | * New btree roots are set by journalling them; when the journal entry | |
1413 | * gets written we have to propagate them to c->btree_roots | |
1414 | * | |
1415 | * But, every journal entry we write has to contain all the btree roots | |
1416 | * (at least for now); so after we copy btree roots to c->btree_roots we | |
1417 | * have to get any missing btree roots and add them to this journal | |
1418 | * entry: | |
1419 | */ | |
1420 | ||
1421 | bch2_journal_entries_to_btree_roots(c, jset); | |
1422 | ||
1423 | start = end = vstruct_last(jset); | |
1424 | ||
1425 | end = bch2_btree_roots_to_journal_entries(c, jset->start, end); | |
1426 | ||
2abe5420 KO |
1427 | bch2_journal_super_entries_add_common(c, &end, |
1428 | le64_to_cpu(jset->seq)); | |
3ccc5c50 KO |
1429 | u64s = (u64 *) end - (u64 *) start; |
1430 | BUG_ON(u64s > j->entry_u64s_reserved); | |
1431 | ||
d16b4a77 KO |
1432 | le32_add_cpu(&jset->u64s, u64s); |
1433 | BUG_ON(vstruct_sectors(jset, c->block_bits) > w->sectors); | |
1c6fdbd8 KO |
1434 | |
1435 | journal_write_compact(jset); | |
1436 | ||
1c6fdbd8 | 1437 | jset->magic = cpu_to_le64(jset_magic(c)); |
26609b61 KO |
1438 | jset->version = c->sb.version < bcachefs_metadata_version_new_versioning |
1439 | ? cpu_to_le32(BCH_JSET_VERSION_OLD) | |
1440 | : cpu_to_le32(c->sb.version); | |
1c6fdbd8 KO |
1441 | |
1442 | SET_JSET_BIG_ENDIAN(jset, CPU_BIG_ENDIAN); | |
1443 | SET_JSET_CSUM_TYPE(jset, bch2_meta_checksum_type(c)); | |
1444 | ||
158eecb8 KO |
1445 | if (journal_entry_empty(jset)) |
1446 | j->last_empty_seq = le64_to_cpu(jset->seq); | |
1447 | ||
26609b61 KO |
1448 | if (bch2_csum_type_is_encryption(JSET_CSUM_TYPE(jset))) |
1449 | validate_before_checksum = true; | |
1450 | ||
e751c01a | 1451 | if (le32_to_cpu(jset->version) < bcachefs_metadata_version_current) |
26609b61 KO |
1452 | validate_before_checksum = true; |
1453 | ||
1454 | if (validate_before_checksum && | |
ed9d58a2 | 1455 | jset_validate_for_write(c, jset)) |
1c6fdbd8 KO |
1456 | goto err; |
1457 | ||
1458 | bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset), | |
1459 | jset->encrypted_start, | |
1460 | vstruct_end(jset) - (void *) jset->encrypted_start); | |
1461 | ||
1462 | jset->csum = csum_vstruct(c, JSET_CSUM_TYPE(jset), | |
1463 | journal_nonce(jset), jset); | |
1464 | ||
26609b61 | 1465 | if (!validate_before_checksum && |
ed9d58a2 | 1466 | jset_validate_for_write(c, jset)) |
1c6fdbd8 KO |
1467 | goto err; |
1468 | ||
1469 | sectors = vstruct_sectors(jset, c->block_bits); | |
d16b4a77 | 1470 | BUG_ON(sectors > w->sectors); |
1c6fdbd8 | 1471 | |
d16b4a77 KO |
1472 | bytes = vstruct_bytes(jset); |
1473 | memset((void *) jset + bytes, 0, (sectors << 9) - bytes); | |
1c6fdbd8 | 1474 | |
c18dade6 | 1475 | retry_alloc: |
e5a66496 KO |
1476 | spin_lock(&j->lock); |
1477 | ret = journal_write_alloc(j, w, sectors); | |
1478 | ||
c18dade6 KO |
1479 | if (ret && j->can_discard) { |
1480 | spin_unlock(&j->lock); | |
1481 | bch2_journal_do_discards(j); | |
1482 | goto retry_alloc; | |
1483 | } | |
1484 | ||
85674154 KO |
1485 | if (ret) { |
1486 | journal_debug_buf = kmalloc(4096, GFP_ATOMIC); | |
1487 | if (journal_debug_buf) | |
1488 | __bch2_journal_debug_to_text(&_PBUF(journal_debug_buf, 4096), j); | |
1489 | } | |
1490 | ||
e5a66496 KO |
1491 | /* |
1492 | * write is allocated, no longer need to account for it in | |
1493 | * bch2_journal_space_available(): | |
1494 | */ | |
1495 | w->sectors = 0; | |
1496 | ||
1497 | /* | |
1498 | * journal entry has been compacted and allocated, recalculate space | |
1499 | * available: | |
1500 | */ | |
1501 | bch2_journal_space_available(j); | |
1502 | spin_unlock(&j->lock); | |
1503 | ||
1504 | if (ret) { | |
85674154 KO |
1505 | bch_err(c, "Unable to allocate journal write:\n%s", |
1506 | journal_debug_buf); | |
1507 | kfree(journal_debug_buf); | |
1c6fdbd8 | 1508 | bch2_fatal_error(c); |
731bdd2e | 1509 | continue_at(cl, journal_write_done, c->io_complete_wq); |
1c6fdbd8 KO |
1510 | return; |
1511 | } | |
1512 | ||
1513 | /* | |
1514 | * XXX: we really should just disable the entire journal in nochanges | |
1515 | * mode | |
1516 | */ | |
1517 | if (c->opts.nochanges) | |
1518 | goto no_io; | |
1519 | ||
280249b9 KO |
1520 | for_each_rw_member(ca, c, i) |
1521 | nr_rw_members++; | |
1c6fdbd8 | 1522 | |
280249b9 KO |
1523 | if (nr_rw_members > 1) |
1524 | w->separate_flush = true; | |
1c6fdbd8 | 1525 | |
280249b9 KO |
1526 | if (!JSET_NO_FLUSH(jset) && w->separate_flush) { |
1527 | for_each_rw_member(ca, c, i) { | |
1528 | percpu_ref_get(&ca->io_ref); | |
1c6fdbd8 | 1529 | |
280249b9 KO |
1530 | bio = ca->journal.bio; |
1531 | bio_reset(bio, ca->disk_sb.bdev, REQ_OP_FLUSH); | |
1532 | bio->bi_end_io = journal_write_endio; | |
1533 | bio->bi_private = ca; | |
1534 | closure_bio_submit(bio, cl); | |
1535 | } | |
1c6fdbd8 KO |
1536 | } |
1537 | ||
280249b9 KO |
1538 | bch2_bucket_seq_cleanup(c); |
1539 | ||
731bdd2e | 1540 | continue_at(cl, do_journal_write, c->io_complete_wq); |
280249b9 | 1541 | return; |
1c6fdbd8 | 1542 | no_io: |
c6923995 KO |
1543 | bch2_bucket_seq_cleanup(c); |
1544 | ||
731bdd2e | 1545 | continue_at(cl, journal_write_done, c->io_complete_wq); |
1c6fdbd8 KO |
1546 | return; |
1547 | err: | |
1548 | bch2_inconsistent_error(c); | |
731bdd2e | 1549 | continue_at(cl, journal_write_done, c->io_complete_wq); |
1c6fdbd8 | 1550 | } |