Commit | Line | Data |
---|---|---|
1c6fdbd8 KO |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include "bcachefs.h" | |
59cc38b8 | 3 | #include "alloc_background.h" |
7b3f84ea | 4 | #include "alloc_foreground.h" |
39fb2983 | 5 | #include "btree_io.h" |
00b8ccf7 | 6 | #include "btree_update_interior.h" |
1c6fdbd8 KO |
7 | #include "buckets.h" |
8 | #include "checksum.h" | |
d042b040 | 9 | #include "disk_groups.h" |
1c6fdbd8 KO |
10 | #include "error.h" |
11 | #include "journal.h" | |
12 | #include "journal_io.h" | |
13 | #include "journal_reclaim.h" | |
adbcada4 | 14 | #include "journal_seq_blacklist.h" |
1c6fdbd8 | 15 | #include "replicas.h" |
a37ad1a3 | 16 | #include "sb-clean.h" |
1c6fdbd8 KO |
17 | #include "trace.h" |
18 | ||
17fe3b64 KO |
19 | static struct nonce journal_nonce(const struct jset *jset) |
20 | { | |
21 | return (struct nonce) {{ | |
22 | [0] = 0, | |
23 | [1] = ((__le32 *) &jset->seq)[0], | |
24 | [2] = ((__le32 *) &jset->seq)[1], | |
25 | [3] = BCH_NONCE_JOURNAL, | |
26 | }}; | |
27 | } | |
28 | ||
29 | static bool jset_csum_good(struct bch_fs *c, struct jset *j) | |
30 | { | |
31 | return bch2_checksum_type_valid(c, JSET_CSUM_TYPE(j)) && | |
32 | !bch2_crc_cmp(j->csum, | |
33 | csum_vstruct(c, JSET_CSUM_TYPE(j), journal_nonce(j), j)); | |
34 | } | |
35 | ||
ec7ccbde | 36 | static inline u32 journal_entry_radix_idx(struct bch_fs *c, u64 seq) |
adbcada4 | 37 | { |
ec7ccbde | 38 | return (seq - c->journal_entries_base_seq) & (~0U >> 1); |
ce6201c4 KO |
39 | } |
40 | ||
41 | static void __journal_replay_free(struct bch_fs *c, | |
42 | struct journal_replay *i) | |
43 | { | |
44 | struct journal_replay **p = | |
ec7ccbde KO |
45 | genradix_ptr(&c->journal_entries, |
46 | journal_entry_radix_idx(c, le64_to_cpu(i->j.seq))); | |
ce6201c4 KO |
47 | |
48 | BUG_ON(*p != i); | |
49 | *p = NULL; | |
adbcada4 KO |
50 | kvpfree(i, offsetof(struct journal_replay, j) + |
51 | vstruct_bytes(&i->j)); | |
adbcada4 KO |
52 | } |
53 | ||
54 | static void journal_replay_free(struct bch_fs *c, struct journal_replay *i) | |
55 | { | |
56 | i->ignore = true; | |
57 | ||
58 | if (!c->opts.read_entire_journal) | |
ce6201c4 | 59 | __journal_replay_free(c, i); |
adbcada4 KO |
60 | } |
61 | ||
1c6fdbd8 KO |
62 | struct journal_list { |
63 | struct closure cl; | |
ec7ccbde | 64 | u64 last_seq; |
1c6fdbd8 | 65 | struct mutex lock; |
1c6fdbd8 KO |
66 | int ret; |
67 | }; | |
68 | ||
69 | #define JOURNAL_ENTRY_ADD_OK 0 | |
70 | #define JOURNAL_ENTRY_ADD_OUT_OF_RANGE 5 | |
71 | ||
72 | /* | |
73 | * Given a journal entry we just read, add it to the list of journal entries to | |
74 | * be replayed: | |
75 | */ | |
76 | static int journal_entry_add(struct bch_fs *c, struct bch_dev *ca, | |
72b7d633 | 77 | struct journal_ptr entry_ptr, |
17fe3b64 | 78 | struct journal_list *jlist, struct jset *j) |
1c6fdbd8 | 79 | { |
ce6201c4 KO |
80 | struct genradix_iter iter; |
81 | struct journal_replay **_i, *i, *dup; | |
72b7d633 | 82 | struct journal_ptr *ptr; |
1c6fdbd8 | 83 | size_t bytes = vstruct_bytes(j); |
ec7ccbde | 84 | u64 last_seq = !JSET_NO_FLUSH(j) ? le64_to_cpu(j->last_seq) : 0; |
e4c3f386 | 85 | int ret = JOURNAL_ENTRY_ADD_OK; |
1c6fdbd8 | 86 | |
ec7ccbde KO |
87 | /* Is this entry older than the range we need? */ |
88 | if (!c->opts.read_entire_journal && | |
89 | le64_to_cpu(j->seq) < jlist->last_seq) | |
90 | return JOURNAL_ENTRY_ADD_OUT_OF_RANGE; | |
91 | ||
ce6201c4 | 92 | /* |
ec7ccbde KO |
93 | * genradixes are indexed by a ulong, not a u64, so we can't index them |
94 | * by sequence number directly: Assume instead that they will all fall | |
95 | * within the range of +-2billion of the filrst one we find. | |
ce6201c4 KO |
96 | */ |
97 | if (!c->journal_entries_base_seq) | |
98 | c->journal_entries_base_seq = max_t(s64, 1, le64_to_cpu(j->seq) - S32_MAX); | |
99 | ||
adbcada4 | 100 | /* Drop entries we don't need anymore */ |
ec7ccbde KO |
101 | if (last_seq > jlist->last_seq && !c->opts.read_entire_journal) { |
102 | genradix_for_each_from(&c->journal_entries, iter, _i, | |
103 | journal_entry_radix_idx(c, jlist->last_seq)) { | |
ce6201c4 KO |
104 | i = *_i; |
105 | ||
ec7ccbde | 106 | if (!i || i->ignore) |
ce6201c4 KO |
107 | continue; |
108 | ||
ec7ccbde | 109 | if (le64_to_cpu(i->j.seq) >= last_seq) |
7fffc85b | 110 | break; |
adbcada4 | 111 | journal_replay_free(c, i); |
7fffc85b | 112 | } |
1c6fdbd8 KO |
113 | } |
114 | ||
ec7ccbde KO |
115 | jlist->last_seq = max(jlist->last_seq, last_seq); |
116 | ||
117 | _i = genradix_ptr_alloc(&c->journal_entries, | |
118 | journal_entry_radix_idx(c, le64_to_cpu(j->seq)), | |
119 | GFP_KERNEL); | |
120 | if (!_i) | |
65d48e35 | 121 | return -BCH_ERR_ENOMEM_journal_entry_add; |
e4c3f386 | 122 | |
ca73852a KO |
123 | /* |
124 | * Duplicate journal entries? If so we want the one that didn't have a | |
125 | * checksum error: | |
126 | */ | |
ec7ccbde | 127 | dup = *_i; |
e4c3f386 | 128 | if (dup) { |
17fe3b64 KO |
129 | if (bytes == vstruct_bytes(&dup->j) && |
130 | !memcmp(j, &dup->j, bytes)) { | |
e4c3f386 | 131 | i = dup; |
ca73852a | 132 | goto found; |
17fe3b64 KO |
133 | } |
134 | ||
135 | if (!entry_ptr.csum_good) { | |
e4c3f386 | 136 | i = dup; |
1c6fdbd8 KO |
137 | goto found; |
138 | } | |
1c6fdbd8 | 139 | |
17fe3b64 KO |
140 | if (!dup->csum_good) |
141 | goto replace; | |
142 | ||
b65db750 KO |
143 | fsck_err(c, journal_entry_replicas_data_mismatch, |
144 | "found duplicate but non identical journal entries (seq %llu)", | |
17fe3b64 KO |
145 | le64_to_cpu(j->seq)); |
146 | i = dup; | |
147 | goto found; | |
148 | } | |
149 | replace: | |
1c6fdbd8 | 150 | i = kvpmalloc(offsetof(struct journal_replay, j) + bytes, GFP_KERNEL); |
ec7ccbde | 151 | if (!i) |
65d48e35 | 152 | return -BCH_ERR_ENOMEM_journal_entry_add; |
1c6fdbd8 | 153 | |
17fe3b64 KO |
154 | i->nr_ptrs = 0; |
155 | i->csum_good = entry_ptr.csum_good; | |
e4c3f386 | 156 | i->ignore = false; |
1c6fdbd8 | 157 | unsafe_memcpy(&i->j, j, bytes, "embedded variable length struct"); |
17fe3b64 | 158 | i->ptrs[i->nr_ptrs++] = entry_ptr; |
e4c3f386 KO |
159 | |
160 | if (dup) { | |
17fe3b64 KO |
161 | if (dup->nr_ptrs >= ARRAY_SIZE(dup->ptrs)) { |
162 | bch_err(c, "found too many copies of journal entry %llu", | |
163 | le64_to_cpu(i->j.seq)); | |
164 | dup->nr_ptrs = ARRAY_SIZE(dup->ptrs) - 1; | |
165 | } | |
166 | ||
167 | /* The first ptr should represent the jset we kept: */ | |
168 | memcpy(i->ptrs + i->nr_ptrs, | |
169 | dup->ptrs, | |
170 | sizeof(dup->ptrs[0]) * dup->nr_ptrs); | |
171 | i->nr_ptrs += dup->nr_ptrs; | |
ce6201c4 | 172 | __journal_replay_free(c, dup); |
e4c3f386 KO |
173 | } |
174 | ||
ce6201c4 | 175 | *_i = i; |
17fe3b64 | 176 | return 0; |
1c6fdbd8 | 177 | found: |
e4c3f386 KO |
178 | for (ptr = i->ptrs; ptr < i->ptrs + i->nr_ptrs; ptr++) { |
179 | if (ptr->dev == ca->dev_idx) { | |
180 | bch_err(c, "duplicate journal entry %llu on same device", | |
181 | le64_to_cpu(i->j.seq)); | |
182 | goto out; | |
183 | } | |
184 | } | |
185 | ||
186 | if (i->nr_ptrs >= ARRAY_SIZE(i->ptrs)) { | |
187 | bch_err(c, "found too many copies of journal entry %llu", | |
188 | le64_to_cpu(i->j.seq)); | |
189 | goto out; | |
190 | } | |
191 | ||
192 | i->ptrs[i->nr_ptrs++] = entry_ptr; | |
1c6fdbd8 KO |
193 | out: |
194 | fsck_err: | |
195 | return ret; | |
196 | } | |
197 | ||
1c6fdbd8 KO |
198 | /* this fills in a range with empty jset_entries: */ |
199 | static void journal_entry_null_range(void *start, void *end) | |
200 | { | |
201 | struct jset_entry *entry; | |
202 | ||
203 | for (entry = start; entry != end; entry = vstruct_next(entry)) | |
204 | memset(entry, 0, sizeof(*entry)); | |
205 | } | |
206 | ||
207 | #define JOURNAL_ENTRY_REREAD 5 | |
208 | #define JOURNAL_ENTRY_NONE 6 | |
209 | #define JOURNAL_ENTRY_BAD 7 | |
210 | ||
c23a9e08 | 211 | static void journal_entry_err_msg(struct printbuf *out, |
a8712967 | 212 | u32 version, |
c23a9e08 KO |
213 | struct jset *jset, |
214 | struct jset_entry *entry) | |
215 | { | |
a8712967 KO |
216 | prt_str(out, "invalid journal entry, version="); |
217 | bch2_version_to_text(out, version); | |
218 | ||
219 | if (entry) { | |
220 | prt_str(out, " type="); | |
221 | prt_str(out, bch2_jset_entry_types[entry->type]); | |
222 | } | |
223 | ||
224 | if (!jset) { | |
225 | prt_printf(out, " in superblock"); | |
226 | } else { | |
227 | ||
228 | prt_printf(out, " seq=%llu", le64_to_cpu(jset->seq)); | |
229 | ||
230 | if (entry) | |
231 | prt_printf(out, " offset=%zi/%u", | |
232 | (u64 *) entry - jset->_data, | |
233 | le32_to_cpu(jset->u64s)); | |
234 | } | |
235 | ||
c23a9e08 KO |
236 | prt_str(out, ": "); |
237 | } | |
238 | ||
b65db750 | 239 | #define journal_entry_err(c, version, jset, entry, _err, msg, ...) \ |
1c6fdbd8 | 240 | ({ \ |
96dea3d5 | 241 | struct printbuf _buf = PRINTBUF; \ |
c23a9e08 | 242 | \ |
96dea3d5 KO |
243 | journal_entry_err_msg(&_buf, version, jset, entry); \ |
244 | prt_printf(&_buf, msg, ##__VA_ARGS__); \ | |
c23a9e08 | 245 | \ |
c4e382e2 | 246 | switch (flags & BKEY_INVALID_WRITE) { \ |
1c6fdbd8 | 247 | case READ: \ |
b65db750 | 248 | mustfix_fsck_err(c, _err, "%s", _buf.buf); \ |
1c6fdbd8 KO |
249 | break; \ |
250 | case WRITE: \ | |
b65db750 | 251 | bch2_sb_error_count(c, BCH_FSCK_ERR_##_err); \ |
96dea3d5 | 252 | bch_err(c, "corrupt metadata before write: %s\n", _buf.buf);\ |
1c6fdbd8 | 253 | if (bch2_fs_inconsistent(c)) { \ |
1ed0a5d2 | 254 | ret = -BCH_ERR_fsck_errors_not_fixed; \ |
1c6fdbd8 KO |
255 | goto fsck_err; \ |
256 | } \ | |
257 | break; \ | |
258 | } \ | |
c23a9e08 | 259 | \ |
96dea3d5 | 260 | printbuf_exit(&_buf); \ |
1c6fdbd8 KO |
261 | true; \ |
262 | }) | |
263 | ||
b65db750 KO |
264 | #define journal_entry_err_on(cond, ...) \ |
265 | ((cond) ? journal_entry_err(__VA_ARGS__) : false) | |
1c6fdbd8 | 266 | |
4d54337c KO |
267 | #define FSCK_DELETED_KEY 5 |
268 | ||
c23a9e08 KO |
269 | static int journal_validate_key(struct bch_fs *c, |
270 | struct jset *jset, | |
1c6fdbd8 | 271 | struct jset_entry *entry, |
39fb2983 | 272 | unsigned level, enum btree_id btree_id, |
cb685ce7 | 273 | struct bkey_i *k, |
c4e382e2 KO |
274 | unsigned version, int big_endian, |
275 | enum bkey_invalid_flags flags) | |
1c6fdbd8 | 276 | { |
c4e382e2 | 277 | int write = flags & BKEY_INVALID_WRITE; |
1c6fdbd8 | 278 | void *next = vstruct_next(entry); |
f0ac7df2 | 279 | struct printbuf buf = PRINTBUF; |
1c6fdbd8 KO |
280 | int ret = 0; |
281 | ||
b65db750 KO |
282 | if (journal_entry_err_on(!k->k.u64s, |
283 | c, version, jset, entry, | |
284 | journal_entry_bkey_u64s_0, | |
285 | "k->u64s 0")) { | |
1c6fdbd8 KO |
286 | entry->u64s = cpu_to_le16((u64 *) k - entry->_data); |
287 | journal_entry_null_range(vstruct_next(entry), next); | |
4d54337c | 288 | return FSCK_DELETED_KEY; |
1c6fdbd8 KO |
289 | } |
290 | ||
291 | if (journal_entry_err_on((void *) bkey_next(k) > | |
c23a9e08 | 292 | (void *) vstruct_next(entry), |
a8712967 | 293 | c, version, jset, entry, |
b65db750 | 294 | journal_entry_bkey_past_end, |
c23a9e08 | 295 | "extends past end of journal entry")) { |
1c6fdbd8 KO |
296 | entry->u64s = cpu_to_le16((u64 *) k - entry->_data); |
297 | journal_entry_null_range(vstruct_next(entry), next); | |
4d54337c | 298 | return FSCK_DELETED_KEY; |
1c6fdbd8 KO |
299 | } |
300 | ||
c23a9e08 | 301 | if (journal_entry_err_on(k->k.format != KEY_FORMAT_CURRENT, |
a8712967 | 302 | c, version, jset, entry, |
b65db750 | 303 | journal_entry_bkey_bad_format, |
c23a9e08 | 304 | "bad format %u", k->k.format)) { |
4d54337c | 305 | le16_add_cpu(&entry->u64s, -((u16) k->k.u64s)); |
1c6fdbd8 KO |
306 | memmove(k, bkey_next(k), next - (void *) bkey_next(k)); |
307 | journal_entry_null_range(vstruct_next(entry), next); | |
4d54337c | 308 | return FSCK_DELETED_KEY; |
1c6fdbd8 KO |
309 | } |
310 | ||
39fb2983 | 311 | if (!write) |
7d6f07ed KO |
312 | bch2_bkey_compat(level, btree_id, version, big_endian, |
313 | write, NULL, bkey_to_packed(k)); | |
26609b61 | 314 | |
f0ac7df2 | 315 | if (bch2_bkey_invalid(c, bkey_i_to_s_c(k), |
275c8426 | 316 | __btree_node_type(level, btree_id), write, &buf)) { |
f0ac7df2 | 317 | printbuf_reset(&buf); |
a8712967 | 318 | journal_entry_err_msg(&buf, version, jset, entry); |
401ec4db KO |
319 | prt_newline(&buf); |
320 | printbuf_indent_add(&buf, 2); | |
319f9ac3 | 321 | |
fa8e94fa | 322 | bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(k)); |
401ec4db | 323 | prt_newline(&buf); |
f0ac7df2 | 324 | bch2_bkey_invalid(c, bkey_i_to_s_c(k), |
275c8426 | 325 | __btree_node_type(level, btree_id), write, &buf); |
f0ac7df2 | 326 | |
b65db750 KO |
327 | mustfix_fsck_err(c, journal_entry_bkey_invalid, |
328 | "%s", buf.buf); | |
1c6fdbd8 | 329 | |
4d54337c | 330 | le16_add_cpu(&entry->u64s, -((u16) k->k.u64s)); |
1c6fdbd8 KO |
331 | memmove(k, bkey_next(k), next - (void *) bkey_next(k)); |
332 | journal_entry_null_range(vstruct_next(entry), next); | |
f0ac7df2 KO |
333 | |
334 | printbuf_exit(&buf); | |
4d54337c | 335 | return FSCK_DELETED_KEY; |
1c6fdbd8 | 336 | } |
26609b61 | 337 | |
39fb2983 | 338 | if (write) |
7d6f07ed KO |
339 | bch2_bkey_compat(level, btree_id, version, big_endian, |
340 | write, NULL, bkey_to_packed(k)); | |
1c6fdbd8 | 341 | fsck_err: |
f0ac7df2 | 342 | printbuf_exit(&buf); |
1c6fdbd8 KO |
343 | return ret; |
344 | } | |
345 | ||
528b18e6 | 346 | static int journal_entry_btree_keys_validate(struct bch_fs *c, |
c4e382e2 KO |
347 | struct jset *jset, |
348 | struct jset_entry *entry, | |
349 | unsigned version, int big_endian, | |
350 | enum bkey_invalid_flags flags) | |
1c6fdbd8 | 351 | { |
4d54337c | 352 | struct bkey_i *k = entry->start; |
1c6fdbd8 | 353 | |
4d54337c | 354 | while (k != vstruct_last(entry)) { |
c23a9e08 | 355 | int ret = journal_validate_key(c, jset, entry, |
39fb2983 KO |
356 | entry->level, |
357 | entry->btree_id, | |
8726dc93 | 358 | k, version, big_endian, |
c4e382e2 | 359 | flags|BKEY_INVALID_JOURNAL); |
4d54337c KO |
360 | if (ret == FSCK_DELETED_KEY) |
361 | continue; | |
362 | ||
363 | k = bkey_next(k); | |
1c6fdbd8 KO |
364 | } |
365 | ||
366 | return 0; | |
367 | } | |
368 | ||
528b18e6 KO |
369 | static void journal_entry_btree_keys_to_text(struct printbuf *out, struct bch_fs *c, |
370 | struct jset_entry *entry) | |
371 | { | |
372 | struct bkey_i *k; | |
e7bc7cdf | 373 | bool first = true; |
528b18e6 | 374 | |
ac2ccddc | 375 | jset_entry_for_each_key(entry, k) { |
e7bc7cdf | 376 | if (!first) { |
401ec4db KO |
377 | prt_newline(out); |
378 | prt_printf(out, "%s: ", bch2_jset_entry_types[entry->type]); | |
e7bc7cdf | 379 | } |
88dfe193 | 380 | prt_printf(out, "btree=%s l=%u ", bch2_btree_id_str(entry->btree_id), entry->level); |
528b18e6 | 381 | bch2_bkey_val_to_text(out, c, bkey_i_to_s_c(k)); |
e7bc7cdf KO |
382 | first = false; |
383 | } | |
528b18e6 KO |
384 | } |
385 | ||
386 | static int journal_entry_btree_root_validate(struct bch_fs *c, | |
c4e382e2 KO |
387 | struct jset *jset, |
388 | struct jset_entry *entry, | |
389 | unsigned version, int big_endian, | |
390 | enum bkey_invalid_flags flags) | |
1c6fdbd8 KO |
391 | { |
392 | struct bkey_i *k = entry->start; | |
393 | int ret = 0; | |
394 | ||
395 | if (journal_entry_err_on(!entry->u64s || | |
c23a9e08 | 396 | le16_to_cpu(entry->u64s) != k->k.u64s, |
a8712967 | 397 | c, version, jset, entry, |
b65db750 | 398 | journal_entry_btree_root_bad_size, |
1c6fdbd8 KO |
399 | "invalid btree root journal entry: wrong number of keys")) { |
400 | void *next = vstruct_next(entry); | |
401 | /* | |
402 | * we don't want to null out this jset_entry, | |
403 | * just the contents, so that later we can tell | |
404 | * we were _supposed_ to have a btree root | |
405 | */ | |
406 | entry->u64s = 0; | |
407 | journal_entry_null_range(vstruct_next(entry), next); | |
408 | return 0; | |
409 | } | |
410 | ||
c23a9e08 | 411 | return journal_validate_key(c, jset, entry, 1, entry->btree_id, k, |
c4e382e2 | 412 | version, big_endian, flags); |
1c6fdbd8 KO |
413 | fsck_err: |
414 | return ret; | |
415 | } | |
416 | ||
528b18e6 KO |
417 | static void journal_entry_btree_root_to_text(struct printbuf *out, struct bch_fs *c, |
418 | struct jset_entry *entry) | |
419 | { | |
420 | journal_entry_btree_keys_to_text(out, c, entry); | |
421 | } | |
422 | ||
423 | static int journal_entry_prio_ptrs_validate(struct bch_fs *c, | |
c4e382e2 KO |
424 | struct jset *jset, |
425 | struct jset_entry *entry, | |
426 | unsigned version, int big_endian, | |
427 | enum bkey_invalid_flags flags) | |
1c6fdbd8 KO |
428 | { |
429 | /* obsolete, don't care: */ | |
430 | return 0; | |
431 | } | |
432 | ||
528b18e6 KO |
433 | static void journal_entry_prio_ptrs_to_text(struct printbuf *out, struct bch_fs *c, |
434 | struct jset_entry *entry) | |
435 | { | |
436 | } | |
437 | ||
438 | static int journal_entry_blacklist_validate(struct bch_fs *c, | |
c4e382e2 KO |
439 | struct jset *jset, |
440 | struct jset_entry *entry, | |
441 | unsigned version, int big_endian, | |
442 | enum bkey_invalid_flags flags) | |
1c6fdbd8 KO |
443 | { |
444 | int ret = 0; | |
445 | ||
c23a9e08 | 446 | if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 1, |
a8712967 | 447 | c, version, jset, entry, |
b65db750 | 448 | journal_entry_blacklist_bad_size, |
1c6fdbd8 KO |
449 | "invalid journal seq blacklist entry: bad size")) { |
450 | journal_entry_null_range(entry, vstruct_next(entry)); | |
451 | } | |
452 | fsck_err: | |
453 | return ret; | |
454 | } | |
455 | ||
528b18e6 KO |
456 | static void journal_entry_blacklist_to_text(struct printbuf *out, struct bch_fs *c, |
457 | struct jset_entry *entry) | |
458 | { | |
459 | struct jset_entry_blacklist *bl = | |
460 | container_of(entry, struct jset_entry_blacklist, entry); | |
461 | ||
401ec4db | 462 | prt_printf(out, "seq=%llu", le64_to_cpu(bl->seq)); |
528b18e6 KO |
463 | } |
464 | ||
465 | static int journal_entry_blacklist_v2_validate(struct bch_fs *c, | |
c4e382e2 KO |
466 | struct jset *jset, |
467 | struct jset_entry *entry, | |
468 | unsigned version, int big_endian, | |
469 | enum bkey_invalid_flags flags) | |
1c6fdbd8 KO |
470 | { |
471 | struct jset_entry_blacklist_v2 *bl_entry; | |
472 | int ret = 0; | |
473 | ||
c23a9e08 | 474 | if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 2, |
a8712967 | 475 | c, version, jset, entry, |
b65db750 | 476 | journal_entry_blacklist_v2_bad_size, |
1c6fdbd8 KO |
477 | "invalid journal seq blacklist entry: bad size")) { |
478 | journal_entry_null_range(entry, vstruct_next(entry)); | |
2c5af169 | 479 | goto out; |
1c6fdbd8 KO |
480 | } |
481 | ||
482 | bl_entry = container_of(entry, struct jset_entry_blacklist_v2, entry); | |
483 | ||
484 | if (journal_entry_err_on(le64_to_cpu(bl_entry->start) > | |
c23a9e08 | 485 | le64_to_cpu(bl_entry->end), |
a8712967 | 486 | c, version, jset, entry, |
b65db750 | 487 | journal_entry_blacklist_v2_start_past_end, |
1c6fdbd8 KO |
488 | "invalid journal seq blacklist entry: start > end")) { |
489 | journal_entry_null_range(entry, vstruct_next(entry)); | |
490 | } | |
2c5af169 KO |
491 | out: |
492 | fsck_err: | |
493 | return ret; | |
494 | } | |
495 | ||
528b18e6 KO |
496 | static void journal_entry_blacklist_v2_to_text(struct printbuf *out, struct bch_fs *c, |
497 | struct jset_entry *entry) | |
498 | { | |
499 | struct jset_entry_blacklist_v2 *bl = | |
500 | container_of(entry, struct jset_entry_blacklist_v2, entry); | |
501 | ||
401ec4db | 502 | prt_printf(out, "start=%llu end=%llu", |
528b18e6 KO |
503 | le64_to_cpu(bl->start), |
504 | le64_to_cpu(bl->end)); | |
505 | } | |
506 | ||
507 | static int journal_entry_usage_validate(struct bch_fs *c, | |
c4e382e2 KO |
508 | struct jset *jset, |
509 | struct jset_entry *entry, | |
510 | unsigned version, int big_endian, | |
511 | enum bkey_invalid_flags flags) | |
2c5af169 KO |
512 | { |
513 | struct jset_entry_usage *u = | |
514 | container_of(entry, struct jset_entry_usage, entry); | |
515 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); | |
516 | int ret = 0; | |
517 | ||
3577df5f | 518 | if (journal_entry_err_on(bytes < sizeof(*u), |
a8712967 | 519 | c, version, jset, entry, |
b65db750 | 520 | journal_entry_usage_bad_size, |
3577df5f KO |
521 | "invalid journal entry usage: bad size")) { |
522 | journal_entry_null_range(entry, vstruct_next(entry)); | |
523 | return ret; | |
524 | } | |
525 | ||
526 | fsck_err: | |
527 | return ret; | |
528 | } | |
529 | ||
528b18e6 KO |
530 | static void journal_entry_usage_to_text(struct printbuf *out, struct bch_fs *c, |
531 | struct jset_entry *entry) | |
532 | { | |
533 | struct jset_entry_usage *u = | |
534 | container_of(entry, struct jset_entry_usage, entry); | |
535 | ||
401ec4db | 536 | prt_printf(out, "type=%s v=%llu", |
528b18e6 KO |
537 | bch2_fs_usage_types[u->entry.btree_id], |
538 | le64_to_cpu(u->v)); | |
539 | } | |
540 | ||
541 | static int journal_entry_data_usage_validate(struct bch_fs *c, | |
c4e382e2 KO |
542 | struct jset *jset, |
543 | struct jset_entry *entry, | |
544 | unsigned version, int big_endian, | |
545 | enum bkey_invalid_flags flags) | |
3577df5f KO |
546 | { |
547 | struct jset_entry_data_usage *u = | |
548 | container_of(entry, struct jset_entry_data_usage, entry); | |
549 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); | |
550 | int ret = 0; | |
551 | ||
2c5af169 KO |
552 | if (journal_entry_err_on(bytes < sizeof(*u) || |
553 | bytes < sizeof(*u) + u->r.nr_devs, | |
a8712967 | 554 | c, version, jset, entry, |
b65db750 | 555 | journal_entry_data_usage_bad_size, |
2c5af169 KO |
556 | "invalid journal entry usage: bad size")) { |
557 | journal_entry_null_range(entry, vstruct_next(entry)); | |
558 | return ret; | |
559 | } | |
1c6fdbd8 KO |
560 | |
561 | fsck_err: | |
562 | return ret; | |
563 | } | |
564 | ||
528b18e6 KO |
565 | static void journal_entry_data_usage_to_text(struct printbuf *out, struct bch_fs *c, |
566 | struct jset_entry *entry) | |
567 | { | |
568 | struct jset_entry_data_usage *u = | |
569 | container_of(entry, struct jset_entry_data_usage, entry); | |
570 | ||
571 | bch2_replicas_entry_to_text(out, &u->r); | |
401ec4db | 572 | prt_printf(out, "=%llu", le64_to_cpu(u->v)); |
528b18e6 KO |
573 | } |
574 | ||
575 | static int journal_entry_clock_validate(struct bch_fs *c, | |
c4e382e2 KO |
576 | struct jset *jset, |
577 | struct jset_entry *entry, | |
578 | unsigned version, int big_endian, | |
579 | enum bkey_invalid_flags flags) | |
2abe5420 KO |
580 | { |
581 | struct jset_entry_clock *clock = | |
582 | container_of(entry, struct jset_entry_clock, entry); | |
583 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); | |
584 | int ret = 0; | |
585 | ||
586 | if (journal_entry_err_on(bytes != sizeof(*clock), | |
b65db750 KO |
587 | c, version, jset, entry, |
588 | journal_entry_clock_bad_size, | |
589 | "bad size")) { | |
2abe5420 KO |
590 | journal_entry_null_range(entry, vstruct_next(entry)); |
591 | return ret; | |
592 | } | |
593 | ||
594 | if (journal_entry_err_on(clock->rw > 1, | |
b65db750 KO |
595 | c, version, jset, entry, |
596 | journal_entry_clock_bad_rw, | |
597 | "bad rw")) { | |
2abe5420 KO |
598 | journal_entry_null_range(entry, vstruct_next(entry)); |
599 | return ret; | |
600 | } | |
601 | ||
602 | fsck_err: | |
603 | return ret; | |
604 | } | |
605 | ||
528b18e6 KO |
606 | static void journal_entry_clock_to_text(struct printbuf *out, struct bch_fs *c, |
607 | struct jset_entry *entry) | |
608 | { | |
609 | struct jset_entry_clock *clock = | |
610 | container_of(entry, struct jset_entry_clock, entry); | |
611 | ||
401ec4db | 612 | prt_printf(out, "%s=%llu", clock->rw ? "write" : "read", le64_to_cpu(clock->time)); |
528b18e6 KO |
613 | } |
614 | ||
615 | static int journal_entry_dev_usage_validate(struct bch_fs *c, | |
c4e382e2 KO |
616 | struct jset *jset, |
617 | struct jset_entry *entry, | |
618 | unsigned version, int big_endian, | |
619 | enum bkey_invalid_flags flags) | |
180fb49d KO |
620 | { |
621 | struct jset_entry_dev_usage *u = | |
622 | container_of(entry, struct jset_entry_dev_usage, entry); | |
623 | unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64); | |
45c2e33f | 624 | unsigned expected = sizeof(*u); |
180fb49d KO |
625 | unsigned dev; |
626 | int ret = 0; | |
627 | ||
628 | if (journal_entry_err_on(bytes < expected, | |
b65db750 KO |
629 | c, version, jset, entry, |
630 | journal_entry_dev_usage_bad_size, | |
631 | "bad size (%u < %u)", | |
180fb49d KO |
632 | bytes, expected)) { |
633 | journal_entry_null_range(entry, vstruct_next(entry)); | |
634 | return ret; | |
635 | } | |
636 | ||
637 | dev = le32_to_cpu(u->dev); | |
638 | ||
639 | if (journal_entry_err_on(!bch2_dev_exists2(c, dev), | |
b65db750 KO |
640 | c, version, jset, entry, |
641 | journal_entry_dev_usage_bad_dev, | |
642 | "bad dev")) { | |
180fb49d KO |
643 | journal_entry_null_range(entry, vstruct_next(entry)); |
644 | return ret; | |
645 | } | |
646 | ||
647 | if (journal_entry_err_on(u->pad, | |
b65db750 KO |
648 | c, version, jset, entry, |
649 | journal_entry_dev_usage_bad_pad, | |
650 | "bad pad")) { | |
180fb49d KO |
651 | journal_entry_null_range(entry, vstruct_next(entry)); |
652 | return ret; | |
653 | } | |
654 | ||
655 | fsck_err: | |
656 | return ret; | |
657 | } | |
658 | ||
528b18e6 KO |
659 | static void journal_entry_dev_usage_to_text(struct printbuf *out, struct bch_fs *c, |
660 | struct jset_entry *entry) | |
661 | { | |
662 | struct jset_entry_dev_usage *u = | |
663 | container_of(entry, struct jset_entry_dev_usage, entry); | |
664 | unsigned i, nr_types = jset_entry_dev_usage_nr_types(u); | |
665 | ||
401ec4db | 666 | prt_printf(out, "dev=%u", le32_to_cpu(u->dev)); |
528b18e6 KO |
667 | |
668 | for (i = 0; i < nr_types; i++) { | |
669 | if (i < BCH_DATA_NR) | |
401ec4db | 670 | prt_printf(out, " %s", bch2_data_types[i]); |
528b18e6 | 671 | else |
401ec4db KO |
672 | prt_printf(out, " (unknown data type %u)", i); |
673 | prt_printf(out, ": buckets=%llu sectors=%llu fragmented=%llu", | |
528b18e6 KO |
674 | le64_to_cpu(u->d[i].buckets), |
675 | le64_to_cpu(u->d[i].sectors), | |
676 | le64_to_cpu(u->d[i].fragmented)); | |
677 | } | |
678 | ||
401ec4db | 679 | prt_printf(out, " buckets_ec: %llu", le64_to_cpu(u->buckets_ec)); |
528b18e6 KO |
680 | } |
681 | ||
682 | static int journal_entry_log_validate(struct bch_fs *c, | |
c4e382e2 KO |
683 | struct jset *jset, |
684 | struct jset_entry *entry, | |
685 | unsigned version, int big_endian, | |
686 | enum bkey_invalid_flags flags) | |
fb64f3fd KO |
687 | { |
688 | return 0; | |
689 | } | |
690 | ||
528b18e6 KO |
691 | static void journal_entry_log_to_text(struct printbuf *out, struct bch_fs *c, |
692 | struct jset_entry *entry) | |
693 | { | |
694 | struct jset_entry_log *l = container_of(entry, struct jset_entry_log, entry); | |
695 | unsigned bytes = vstruct_bytes(entry) - offsetof(struct jset_entry_log, d); | |
696 | ||
401ec4db | 697 | prt_printf(out, "%.*s", bytes, l->d); |
528b18e6 KO |
698 | } |
699 | ||
c23a9e08 | 700 | static int journal_entry_overwrite_validate(struct bch_fs *c, |
c4e382e2 KO |
701 | struct jset *jset, |
702 | struct jset_entry *entry, | |
703 | unsigned version, int big_endian, | |
704 | enum bkey_invalid_flags flags) | |
cb685ce7 | 705 | { |
dbe17f18 KO |
706 | return journal_entry_btree_keys_validate(c, jset, entry, |
707 | version, big_endian, READ); | |
cb685ce7 KO |
708 | } |
709 | ||
710 | static void journal_entry_overwrite_to_text(struct printbuf *out, struct bch_fs *c, | |
711 | struct jset_entry *entry) | |
712 | { | |
713 | journal_entry_btree_keys_to_text(out, c, entry); | |
714 | } | |
715 | ||
1c6fdbd8 | 716 | struct jset_entry_ops { |
c23a9e08 | 717 | int (*validate)(struct bch_fs *, struct jset *, |
c4e382e2 KO |
718 | struct jset_entry *, unsigned, int, |
719 | enum bkey_invalid_flags); | |
528b18e6 | 720 | void (*to_text)(struct printbuf *, struct bch_fs *, struct jset_entry *); |
1c6fdbd8 KO |
721 | }; |
722 | ||
723 | static const struct jset_entry_ops bch2_jset_entry_ops[] = { | |
724 | #define x(f, nr) \ | |
725 | [BCH_JSET_ENTRY_##f] = (struct jset_entry_ops) { \ | |
528b18e6 KO |
726 | .validate = journal_entry_##f##_validate, \ |
727 | .to_text = journal_entry_##f##_to_text, \ | |
1c6fdbd8 KO |
728 | }, |
729 | BCH_JSET_ENTRY_TYPES() | |
730 | #undef x | |
731 | }; | |
732 | ||
c23a9e08 KO |
733 | int bch2_journal_entry_validate(struct bch_fs *c, |
734 | struct jset *jset, | |
7d6f07ed | 735 | struct jset_entry *entry, |
c4e382e2 KO |
736 | unsigned version, int big_endian, |
737 | enum bkey_invalid_flags flags) | |
1c6fdbd8 | 738 | { |
2c5af169 | 739 | return entry->type < BCH_JSET_ENTRY_NR |
c23a9e08 | 740 | ? bch2_jset_entry_ops[entry->type].validate(c, jset, entry, |
c4e382e2 | 741 | version, big_endian, flags) |
2c5af169 | 742 | : 0; |
1c6fdbd8 KO |
743 | } |
744 | ||
528b18e6 KO |
745 | void bch2_journal_entry_to_text(struct printbuf *out, struct bch_fs *c, |
746 | struct jset_entry *entry) | |
747 | { | |
748 | if (entry->type < BCH_JSET_ENTRY_NR) { | |
401ec4db | 749 | prt_printf(out, "%s: ", bch2_jset_entry_types[entry->type]); |
528b18e6 KO |
750 | bch2_jset_entry_ops[entry->type].to_text(out, c, entry); |
751 | } else { | |
401ec4db | 752 | prt_printf(out, "(unknown type %u)", entry->type); |
528b18e6 KO |
753 | } |
754 | } | |
755 | ||
1c6fdbd8 | 756 | static int jset_validate_entries(struct bch_fs *c, struct jset *jset, |
c4e382e2 | 757 | enum bkey_invalid_flags flags) |
1c6fdbd8 KO |
758 | { |
759 | struct jset_entry *entry; | |
a8712967 | 760 | unsigned version = le32_to_cpu(jset->version); |
1c6fdbd8 KO |
761 | int ret = 0; |
762 | ||
763 | vstruct_for_each(jset, entry) { | |
a8712967 | 764 | if (journal_entry_err_on(vstruct_next(entry) > vstruct_last(jset), |
b65db750 KO |
765 | c, version, jset, entry, |
766 | journal_entry_past_jset_end, | |
1c6fdbd8 KO |
767 | "journal entry extends past end of jset")) { |
768 | jset->u64s = cpu_to_le32((u64 *) entry - jset->_data); | |
769 | break; | |
770 | } | |
771 | ||
c23a9e08 | 772 | ret = bch2_journal_entry_validate(c, jset, entry, |
c4e382e2 | 773 | version, JSET_BIG_ENDIAN(jset), flags); |
1c6fdbd8 KO |
774 | if (ret) |
775 | break; | |
776 | } | |
777 | fsck_err: | |
778 | return ret; | |
779 | } | |
780 | ||
781 | static int jset_validate(struct bch_fs *c, | |
ca73852a | 782 | struct bch_dev *ca, |
1c6fdbd8 | 783 | struct jset *jset, u64 sector, |
c4e382e2 | 784 | enum bkey_invalid_flags flags) |
1c6fdbd8 | 785 | { |
26609b61 | 786 | unsigned version; |
1c6fdbd8 KO |
787 | int ret = 0; |
788 | ||
789 | if (le64_to_cpu(jset->magic) != jset_magic(c)) | |
790 | return JOURNAL_ENTRY_NONE; | |
791 | ||
26609b61 | 792 | version = le32_to_cpu(jset->version); |
a8712967 KO |
793 | if (journal_entry_err_on(!bch2_version_compatible(version), |
794 | c, version, jset, NULL, | |
b65db750 | 795 | jset_unsupported_version, |
ba8eeae8 | 796 | "%s sector %llu seq %llu: incompatible journal entry version %u.%u", |
ed9d58a2 | 797 | ca ? ca->name : c->name, |
ba8eeae8 KO |
798 | sector, le64_to_cpu(jset->seq), |
799 | BCH_VERSION_MAJOR(version), | |
800 | BCH_VERSION_MINOR(version))) { | |
35ef6df5 | 801 | /* don't try to continue: */ |
d1b2c864 | 802 | return -EINVAL; |
1c6fdbd8 KO |
803 | } |
804 | ||
c23a9e08 | 805 | if (journal_entry_err_on(!bch2_checksum_type_valid(c, JSET_CSUM_TYPE(jset)), |
b65db750 KO |
806 | c, version, jset, NULL, |
807 | jset_unknown_csum, | |
ca73852a | 808 | "%s sector %llu seq %llu: journal entry with unknown csum type %llu", |
ed9d58a2 KO |
809 | ca ? ca->name : c->name, |
810 | sector, le64_to_cpu(jset->seq), | |
d1b2c864 | 811 | JSET_CSUM_TYPE(jset))) |
35ef6df5 | 812 | ret = JOURNAL_ENTRY_BAD; |
1c6fdbd8 | 813 | |
ed9d58a2 KO |
814 | /* last_seq is ignored when JSET_NO_FLUSH is true */ |
815 | if (journal_entry_err_on(!JSET_NO_FLUSH(jset) && | |
c23a9e08 | 816 | le64_to_cpu(jset->last_seq) > le64_to_cpu(jset->seq), |
a8712967 | 817 | c, version, jset, NULL, |
b65db750 | 818 | jset_last_seq_newer_than_seq, |
ed9d58a2 KO |
819 | "invalid journal entry: last_seq > seq (%llu > %llu)", |
820 | le64_to_cpu(jset->last_seq), | |
821 | le64_to_cpu(jset->seq))) { | |
1c6fdbd8 | 822 | jset->last_seq = jset->seq; |
ca73852a KO |
823 | return JOURNAL_ENTRY_BAD; |
824 | } | |
d1b2c864 | 825 | |
c4e382e2 | 826 | ret = jset_validate_entries(c, jset, flags); |
1c6fdbd8 KO |
827 | fsck_err: |
828 | return ret; | |
829 | } | |
830 | ||
d1b2c864 KO |
831 | static int jset_validate_early(struct bch_fs *c, |
832 | struct bch_dev *ca, | |
833 | struct jset *jset, u64 sector, | |
834 | unsigned bucket_sectors_left, | |
835 | unsigned sectors_read) | |
ed9d58a2 | 836 | { |
d1b2c864 KO |
837 | size_t bytes = vstruct_bytes(jset); |
838 | unsigned version; | |
c4e382e2 | 839 | enum bkey_invalid_flags flags = BKEY_INVALID_JOURNAL; |
d1b2c864 KO |
840 | int ret = 0; |
841 | ||
842 | if (le64_to_cpu(jset->magic) != jset_magic(c)) | |
843 | return JOURNAL_ENTRY_NONE; | |
844 | ||
845 | version = le32_to_cpu(jset->version); | |
a8712967 | 846 | if (journal_entry_err_on(!bch2_version_compatible(version), |
b65db750 KO |
847 | c, version, jset, NULL, |
848 | jset_unsupported_version, | |
ba8eeae8 | 849 | "%s sector %llu seq %llu: unknown journal entry version %u.%u", |
d1b2c864 | 850 | ca ? ca->name : c->name, |
ba8eeae8 KO |
851 | sector, le64_to_cpu(jset->seq), |
852 | BCH_VERSION_MAJOR(version), | |
853 | BCH_VERSION_MINOR(version))) { | |
d1b2c864 KO |
854 | /* don't try to continue: */ |
855 | return -EINVAL; | |
856 | } | |
857 | ||
858 | if (bytes > (sectors_read << 9) && | |
859 | sectors_read < bucket_sectors_left) | |
860 | return JOURNAL_ENTRY_REREAD; | |
ed9d58a2 | 861 | |
d1b2c864 | 862 | if (journal_entry_err_on(bytes > bucket_sectors_left << 9, |
b65db750 KO |
863 | c, version, jset, NULL, |
864 | jset_past_bucket_end, | |
d1b2c864 KO |
865 | "%s sector %llu seq %llu: journal entry too big (%zu bytes)", |
866 | ca ? ca->name : c->name, | |
867 | sector, le64_to_cpu(jset->seq), bytes)) | |
868 | le32_add_cpu(&jset->u64s, | |
869 | -((bytes - (bucket_sectors_left << 9)) / 8)); | |
870 | fsck_err: | |
871 | return ret; | |
ed9d58a2 KO |
872 | } |
873 | ||
1c6fdbd8 KO |
874 | struct journal_read_buf { |
875 | void *data; | |
876 | size_t size; | |
877 | }; | |
878 | ||
879 | static int journal_read_buf_realloc(struct journal_read_buf *b, | |
880 | size_t new_size) | |
881 | { | |
882 | void *n; | |
883 | ||
884 | /* the bios are sized for this many pages, max: */ | |
885 | if (new_size > JOURNAL_ENTRY_SIZE_MAX) | |
65d48e35 | 886 | return -BCH_ERR_ENOMEM_journal_read_buf_realloc; |
1c6fdbd8 KO |
887 | |
888 | new_size = roundup_pow_of_two(new_size); | |
889 | n = kvpmalloc(new_size, GFP_KERNEL); | |
890 | if (!n) | |
65d48e35 | 891 | return -BCH_ERR_ENOMEM_journal_read_buf_realloc; |
1c6fdbd8 KO |
892 | |
893 | kvpfree(b->data, b->size); | |
894 | b->data = n; | |
895 | b->size = new_size; | |
896 | return 0; | |
897 | } | |
898 | ||
899 | static int journal_read_bucket(struct bch_dev *ca, | |
900 | struct journal_read_buf *buf, | |
901 | struct journal_list *jlist, | |
a9ec3454 | 902 | unsigned bucket) |
1c6fdbd8 KO |
903 | { |
904 | struct bch_fs *c = ca->fs; | |
905 | struct journal_device *ja = &ca->journal; | |
1c6fdbd8 KO |
906 | struct jset *j = NULL; |
907 | unsigned sectors, sectors_read = 0; | |
908 | u64 offset = bucket_to_sector(ca, ja->buckets[bucket]), | |
909 | end = offset + ca->mi.bucket_size; | |
17fe3b64 | 910 | bool saw_bad = false, csum_good; |
1c6fdbd8 KO |
911 | int ret = 0; |
912 | ||
913 | pr_debug("reading %u", bucket); | |
914 | ||
915 | while (offset < end) { | |
916 | if (!sectors_read) { | |
ac10a961 KO |
917 | struct bio *bio; |
918 | unsigned nr_bvecs; | |
919 | reread: | |
920 | sectors_read = min_t(unsigned, | |
1c6fdbd8 | 921 | end - offset, buf->size >> 9); |
ac10a961 KO |
922 | nr_bvecs = buf_pages(buf->data, sectors_read << 9); |
923 | ||
924 | bio = bio_kmalloc(nr_bvecs, GFP_KERNEL); | |
925 | bio_init(bio, ca->disk_sb.bdev, bio->bi_inline_vecs, nr_bvecs, REQ_OP_READ); | |
1c6fdbd8 | 926 | |
885678f6 KO |
927 | bio->bi_iter.bi_sector = offset; |
928 | bch2_bio_map(bio, buf->data, sectors_read << 9); | |
1c6fdbd8 KO |
929 | |
930 | ret = submit_bio_wait(bio); | |
ac10a961 | 931 | kfree(bio); |
1c6fdbd8 | 932 | |
94119eeb | 933 | if (bch2_dev_io_err_on(ret, ca, BCH_MEMBER_ERROR_read, |
0fefe8d8 | 934 | "journal read error: sector %llu", |
1c6fdbd8 | 935 | offset) || |
29d90f61 KO |
936 | bch2_meta_read_fault("journal")) { |
937 | /* | |
938 | * We don't error out of the recovery process | |
939 | * here, since the relevant journal entry may be | |
940 | * found on a different device, and missing or | |
941 | * no journal entries will be handled later | |
942 | */ | |
943 | return 0; | |
944 | } | |
1c6fdbd8 KO |
945 | |
946 | j = buf->data; | |
947 | } | |
948 | ||
d1b2c864 KO |
949 | ret = jset_validate_early(c, ca, j, offset, |
950 | end - offset, sectors_read); | |
1c6fdbd8 | 951 | switch (ret) { |
1ed0a5d2 | 952 | case 0: |
ca73852a | 953 | sectors = vstruct_sectors(j, c->block_bits); |
1c6fdbd8 KO |
954 | break; |
955 | case JOURNAL_ENTRY_REREAD: | |
956 | if (vstruct_bytes(j) > buf->size) { | |
957 | ret = journal_read_buf_realloc(buf, | |
958 | vstruct_bytes(j)); | |
959 | if (ret) | |
960 | return ret; | |
961 | } | |
962 | goto reread; | |
963 | case JOURNAL_ENTRY_NONE: | |
964 | if (!saw_bad) | |
965 | return 0; | |
ca73852a KO |
966 | /* |
967 | * On checksum error we don't really trust the size | |
968 | * field of the journal entry we read, so try reading | |
969 | * again at next block boundary: | |
970 | */ | |
8244f320 | 971 | sectors = block_sectors(c); |
d1b2c864 | 972 | goto next_block; |
1c6fdbd8 KO |
973 | default: |
974 | return ret; | |
975 | } | |
976 | ||
977 | /* | |
978 | * This happens sometimes if we don't have discards on - | |
979 | * when we've partially overwritten a bucket with new | |
980 | * journal entries. We don't need the rest of the | |
981 | * bucket: | |
982 | */ | |
983 | if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket]) | |
984 | return 0; | |
985 | ||
986 | ja->bucket_seq[bucket] = le64_to_cpu(j->seq); | |
987 | ||
17fe3b64 | 988 | csum_good = jset_csum_good(c, j); |
94119eeb KO |
989 | if (bch2_dev_io_err_on(!csum_good, ca, BCH_MEMBER_ERROR_checksum, |
990 | "journal checksum error")) | |
17fe3b64 KO |
991 | saw_bad = true; |
992 | ||
d1b2c864 KO |
993 | ret = bch2_encrypt(c, JSET_CSUM_TYPE(j), journal_nonce(j), |
994 | j->encrypted_start, | |
995 | vstruct_end(j) - (void *) j->encrypted_start); | |
996 | bch2_fs_fatal_err_on(ret, c, | |
997 | "error decrypting journal entry: %i", ret); | |
998 | ||
1c6fdbd8 | 999 | mutex_lock(&jlist->lock); |
72b7d633 | 1000 | ret = journal_entry_add(c, ca, (struct journal_ptr) { |
17fe3b64 | 1001 | .csum_good = csum_good, |
72b7d633 KO |
1002 | .dev = ca->dev_idx, |
1003 | .bucket = bucket, | |
1004 | .bucket_offset = offset - | |
1005 | bucket_to_sector(ca, ja->buckets[bucket]), | |
1006 | .sector = offset, | |
17fe3b64 | 1007 | }, jlist, j); |
1c6fdbd8 KO |
1008 | mutex_unlock(&jlist->lock); |
1009 | ||
1010 | switch (ret) { | |
1011 | case JOURNAL_ENTRY_ADD_OK: | |
1c6fdbd8 KO |
1012 | break; |
1013 | case JOURNAL_ENTRY_ADD_OUT_OF_RANGE: | |
1014 | break; | |
1015 | default: | |
1016 | return ret; | |
1017 | } | |
1c6fdbd8 KO |
1018 | next_block: |
1019 | pr_debug("next"); | |
1020 | offset += sectors; | |
1021 | sectors_read -= sectors; | |
1022 | j = ((void *) j) + (sectors << 9); | |
1023 | } | |
1024 | ||
1025 | return 0; | |
1026 | } | |
1027 | ||
1028 | static void bch2_journal_read_device(struct closure *cl) | |
1029 | { | |
1c6fdbd8 KO |
1030 | struct journal_device *ja = |
1031 | container_of(cl, struct journal_device, read); | |
1032 | struct bch_dev *ca = container_of(ja, struct bch_dev, journal); | |
365f64f3 | 1033 | struct bch_fs *c = ca->fs; |
1c6fdbd8 KO |
1034 | struct journal_list *jlist = |
1035 | container_of(cl->parent, struct journal_list, cl); | |
ce6201c4 KO |
1036 | struct journal_replay *r, **_r; |
1037 | struct genradix_iter iter; | |
1c6fdbd8 | 1038 | struct journal_read_buf buf = { NULL, 0 }; |
a9ec3454 | 1039 | unsigned i; |
9714baaa | 1040 | int ret = 0; |
1c6fdbd8 KO |
1041 | |
1042 | if (!ja->nr) | |
1043 | goto out; | |
1044 | ||
1c6fdbd8 KO |
1045 | ret = journal_read_buf_realloc(&buf, PAGE_SIZE); |
1046 | if (ret) | |
1047 | goto err; | |
1048 | ||
1049 | pr_debug("%u journal buckets", ja->nr); | |
1050 | ||
1c6fdbd8 | 1051 | for (i = 0; i < ja->nr; i++) { |
a9ec3454 KO |
1052 | ret = journal_read_bucket(ca, &buf, jlist, i); |
1053 | if (ret) | |
1054 | goto err; | |
1c6fdbd8 KO |
1055 | } |
1056 | ||
062afcba KO |
1057 | ja->sectors_free = ca->mi.bucket_size; |
1058 | ||
1059 | mutex_lock(&jlist->lock); | |
230fa1c7 | 1060 | genradix_for_each_reverse(&c->journal_entries, iter, _r) { |
ce6201c4 KO |
1061 | r = *_r; |
1062 | ||
1063 | if (!r) | |
1064 | continue; | |
1065 | ||
062afcba | 1066 | for (i = 0; i < r->nr_ptrs; i++) { |
230fa1c7 | 1067 | if (r->ptrs[i].dev == ca->dev_idx) { |
502f973d | 1068 | unsigned wrote = bucket_remainder(ca, r->ptrs[i].sector) + |
062afcba KO |
1069 | vstruct_sectors(&r->j, c->block_bits); |
1070 | ||
230fa1c7 KO |
1071 | ja->cur_idx = r->ptrs[i].bucket; |
1072 | ja->sectors_free = ca->mi.bucket_size - wrote; | |
1073 | goto found; | |
062afcba KO |
1074 | } |
1075 | } | |
1076 | } | |
230fa1c7 | 1077 | found: |
062afcba KO |
1078 | mutex_unlock(&jlist->lock); |
1079 | ||
b0be2fcf KO |
1080 | if (ja->bucket_seq[ja->cur_idx] && |
1081 | ja->sectors_free == ca->mi.bucket_size) { | |
1082 | bch_err(c, "ja->sectors_free == ca->mi.bucket_size"); | |
1083 | bch_err(c, "cur_idx %u/%u", ja->cur_idx, ja->nr); | |
1084 | for (i = 0; i < 3; i++) { | |
1085 | unsigned idx = (ja->cur_idx + ja->nr - 1 + i) % ja->nr; | |
1e81f89b | 1086 | |
b0be2fcf KO |
1087 | bch_err(c, "bucket_seq[%u] = %llu", idx, ja->bucket_seq[idx]); |
1088 | } | |
1089 | ja->sectors_free = 0; | |
1090 | } | |
1c6fdbd8 KO |
1091 | |
1092 | /* | |
0ce2dbbe | 1093 | * Set dirty_idx to indicate the entire journal is full and needs to be |
1c6fdbd8 KO |
1094 | * reclaimed - journal reclaim will immediately reclaim whatever isn't |
1095 | * pinned when it first runs: | |
1096 | */ | |
0ce2dbbe KO |
1097 | ja->discard_idx = ja->dirty_idx_ondisk = |
1098 | ja->dirty_idx = (ja->cur_idx + 1) % ja->nr; | |
1c6fdbd8 | 1099 | out: |
365f64f3 | 1100 | bch_verbose(c, "journal read done on device %s, ret %i", ca->name, ret); |
1c6fdbd8 | 1101 | kvpfree(buf.data, buf.size); |
1c6fdbd8 KO |
1102 | percpu_ref_put(&ca->io_ref); |
1103 | closure_return(cl); | |
1104 | return; | |
1105 | err: | |
1106 | mutex_lock(&jlist->lock); | |
1107 | jlist->ret = ret; | |
1108 | mutex_unlock(&jlist->lock); | |
1109 | goto out; | |
1c6fdbd8 KO |
1110 | } |
1111 | ||
72b7d633 KO |
1112 | void bch2_journal_ptrs_to_text(struct printbuf *out, struct bch_fs *c, |
1113 | struct journal_replay *j) | |
e4c3f386 KO |
1114 | { |
1115 | unsigned i; | |
1116 | ||
1117 | for (i = 0; i < j->nr_ptrs; i++) { | |
c0ebe3e4 | 1118 | struct bch_dev *ca = bch_dev_bkey_exists(c, j->ptrs[i].dev); |
514852c2 KO |
1119 | u64 offset; |
1120 | ||
72b7d633 | 1121 | div64_u64_rem(j->ptrs[i].sector, ca->mi.bucket_size, &offset); |
e4c3f386 KO |
1122 | |
1123 | if (i) | |
401ec4db KO |
1124 | prt_printf(out, " "); |
1125 | prt_printf(out, "%u:%u:%u (sector %llu)", | |
e4c3f386 | 1126 | j->ptrs[i].dev, |
72b7d633 KO |
1127 | j->ptrs[i].bucket, |
1128 | j->ptrs[i].bucket_offset, | |
1129 | j->ptrs[i].sector); | |
e4c3f386 KO |
1130 | } |
1131 | } | |
1132 | ||
5bbe3f2d KO |
1133 | int bch2_journal_read(struct bch_fs *c, |
1134 | u64 *last_seq, | |
1135 | u64 *blacklist_seq, | |
1136 | u64 *start_seq) | |
1c6fdbd8 | 1137 | { |
1c6fdbd8 | 1138 | struct journal_list jlist; |
ce6201c4 KO |
1139 | struct journal_replay *i, **_i, *prev = NULL; |
1140 | struct genradix_iter radix_iter; | |
1c6fdbd8 | 1141 | struct bch_dev *ca; |
1c6fdbd8 | 1142 | unsigned iter; |
fa8e94fa | 1143 | struct printbuf buf = PRINTBUF; |
dab1e248 | 1144 | bool degraded = false, last_write_torn = false; |
5bbe3f2d | 1145 | u64 seq; |
1c6fdbd8 KO |
1146 | int ret = 0; |
1147 | ||
1148 | closure_init_stack(&jlist.cl); | |
1149 | mutex_init(&jlist.lock); | |
ec7ccbde | 1150 | jlist.last_seq = 0; |
1c6fdbd8 KO |
1151 | jlist.ret = 0; |
1152 | ||
1153 | for_each_member_device(ca, c, iter) { | |
75c8d030 | 1154 | if (!c->opts.fsck && |
89fd25be | 1155 | !(bch2_dev_has_data(c, ca) & (1 << BCH_DATA_journal))) |
1c6fdbd8 KO |
1156 | continue; |
1157 | ||
2436cb9f KO |
1158 | if ((ca->mi.state == BCH_MEMBER_STATE_rw || |
1159 | ca->mi.state == BCH_MEMBER_STATE_ro) && | |
1c6fdbd8 KO |
1160 | percpu_ref_tryget(&ca->io_ref)) |
1161 | closure_call(&ca->journal.read, | |
1162 | bch2_journal_read_device, | |
1163 | system_unbound_wq, | |
1164 | &jlist.cl); | |
1165 | else | |
1166 | degraded = true; | |
1167 | } | |
1168 | ||
1169 | closure_sync(&jlist.cl); | |
1170 | ||
1171 | if (jlist.ret) | |
1172 | return jlist.ret; | |
1173 | ||
5bbe3f2d | 1174 | *last_seq = 0; |
ff56d68c KO |
1175 | *start_seq = 0; |
1176 | *blacklist_seq = 0; | |
adbcada4 KO |
1177 | |
1178 | /* | |
1179 | * Find most recent flush entry, and ignore newer non flush entries - | |
1180 | * those entries will be blacklisted: | |
1181 | */ | |
ce6201c4 | 1182 | genradix_for_each_reverse(&c->journal_entries, radix_iter, _i) { |
c4e382e2 | 1183 | enum bkey_invalid_flags flags = BKEY_INVALID_JOURNAL; |
dab1e248 | 1184 | |
ce6201c4 KO |
1185 | i = *_i; |
1186 | ||
1187 | if (!i || i->ignore) | |
adbcada4 KO |
1188 | continue; |
1189 | ||
ce6201c4 | 1190 | if (!*start_seq) |
ff56d68c | 1191 | *blacklist_seq = *start_seq = le64_to_cpu(i->j.seq) + 1; |
ce6201c4 | 1192 | |
dab1e248 | 1193 | if (JSET_NO_FLUSH(&i->j)) { |
1ba8a796 | 1194 | i->ignore = true; |
dab1e248 | 1195 | continue; |
adbcada4 KO |
1196 | } |
1197 | ||
dab1e248 KO |
1198 | if (!last_write_torn && !i->csum_good) { |
1199 | last_write_torn = true; | |
1ba8a796 | 1200 | i->ignore = true; |
dab1e248 KO |
1201 | continue; |
1202 | } | |
1203 | ||
1204 | if (journal_entry_err_on(le64_to_cpu(i->j.last_seq) > le64_to_cpu(i->j.seq), | |
a8712967 | 1205 | c, le32_to_cpu(i->j.version), &i->j, NULL, |
b65db750 | 1206 | jset_last_seq_newer_than_seq, |
dab1e248 KO |
1207 | "invalid journal entry: last_seq > seq (%llu > %llu)", |
1208 | le64_to_cpu(i->j.last_seq), | |
1209 | le64_to_cpu(i->j.seq))) | |
1210 | i->j.last_seq = i->j.seq; | |
1211 | ||
5bbe3f2d | 1212 | *last_seq = le64_to_cpu(i->j.last_seq); |
dab1e248 KO |
1213 | *blacklist_seq = le64_to_cpu(i->j.seq) + 1; |
1214 | break; | |
adbcada4 KO |
1215 | } |
1216 | ||
ce6201c4 KO |
1217 | if (!*start_seq) { |
1218 | bch_info(c, "journal read done, but no entries found"); | |
1219 | return 0; | |
1220 | } | |
1221 | ||
5bbe3f2d | 1222 | if (!*last_seq) { |
b65db750 KO |
1223 | fsck_err(c, dirty_but_no_journal_entries_post_drop_nonflushes, |
1224 | "journal read done, but no entries found after dropping non-flushes"); | |
e0de429a | 1225 | return 0; |
adbcada4 KO |
1226 | } |
1227 | ||
ff56d68c | 1228 | bch_info(c, "journal read done, replaying entries %llu-%llu", |
5bbe3f2d | 1229 | *last_seq, *blacklist_seq - 1); |
ff56d68c KO |
1230 | |
1231 | if (*start_seq != *blacklist_seq) | |
1232 | bch_info(c, "dropped unflushed entries %llu-%llu", | |
1233 | *blacklist_seq, *start_seq - 1); | |
1234 | ||
adbcada4 | 1235 | /* Drop blacklisted entries and entries older than last_seq: */ |
ce6201c4 KO |
1236 | genradix_for_each(&c->journal_entries, radix_iter, _i) { |
1237 | i = *_i; | |
1238 | ||
1239 | if (!i || i->ignore) | |
adbcada4 KO |
1240 | continue; |
1241 | ||
1242 | seq = le64_to_cpu(i->j.seq); | |
5bbe3f2d | 1243 | if (seq < *last_seq) { |
adbcada4 KO |
1244 | journal_replay_free(c, i); |
1245 | continue; | |
1246 | } | |
1247 | ||
1248 | if (bch2_journal_seq_is_blacklisted(c, seq, true)) { | |
1249 | fsck_err_on(!JSET_NO_FLUSH(&i->j), c, | |
b65db750 | 1250 | jset_seq_blacklisted, |
adbcada4 | 1251 | "found blacklisted journal entry %llu", seq); |
1ba8a796 | 1252 | i->ignore = true; |
adbcada4 KO |
1253 | } |
1254 | } | |
1255 | ||
1256 | /* Check for missing entries: */ | |
5bbe3f2d | 1257 | seq = *last_seq; |
ce6201c4 KO |
1258 | genradix_for_each(&c->journal_entries, radix_iter, _i) { |
1259 | i = *_i; | |
1260 | ||
1261 | if (!i || i->ignore) | |
adbcada4 KO |
1262 | continue; |
1263 | ||
1264 | BUG_ON(seq > le64_to_cpu(i->j.seq)); | |
1265 | ||
1266 | while (seq < le64_to_cpu(i->j.seq)) { | |
1267 | u64 missing_start, missing_end; | |
fa8e94fa | 1268 | struct printbuf buf1 = PRINTBUF, buf2 = PRINTBUF; |
adbcada4 KO |
1269 | |
1270 | while (seq < le64_to_cpu(i->j.seq) && | |
1271 | bch2_journal_seq_is_blacklisted(c, seq, false)) | |
1272 | seq++; | |
1273 | ||
1274 | if (seq == le64_to_cpu(i->j.seq)) | |
1275 | break; | |
1276 | ||
1277 | missing_start = seq; | |
1278 | ||
1279 | while (seq < le64_to_cpu(i->j.seq) && | |
1280 | !bch2_journal_seq_is_blacklisted(c, seq, false)) | |
1281 | seq++; | |
1282 | ||
ce6201c4 KO |
1283 | if (prev) { |
1284 | bch2_journal_ptrs_to_text(&buf1, c, prev); | |
401ec4db | 1285 | prt_printf(&buf1, " size %zu", vstruct_sectors(&prev->j, c->block_bits)); |
e4c3f386 | 1286 | } else |
401ec4db | 1287 | prt_printf(&buf1, "(none)"); |
fa8e94fa | 1288 | bch2_journal_ptrs_to_text(&buf2, c, i); |
e4c3f386 | 1289 | |
adbcada4 | 1290 | missing_end = seq - 1; |
b65db750 KO |
1291 | fsck_err(c, journal_entries_missing, |
1292 | "journal entries %llu-%llu missing! (replaying %llu-%llu)\n" | |
e4c3f386 KO |
1293 | " prev at %s\n" |
1294 | " next at %s", | |
adbcada4 | 1295 | missing_start, missing_end, |
5bbe3f2d | 1296 | *last_seq, *blacklist_seq - 1, |
fa8e94fa KO |
1297 | buf1.buf, buf2.buf); |
1298 | ||
1299 | printbuf_exit(&buf1); | |
1300 | printbuf_exit(&buf2); | |
adbcada4 KO |
1301 | } |
1302 | ||
ce6201c4 | 1303 | prev = i; |
adbcada4 KO |
1304 | seq++; |
1305 | } | |
1306 | ||
ce6201c4 | 1307 | genradix_for_each(&c->journal_entries, radix_iter, _i) { |
e4c3f386 KO |
1308 | struct bch_replicas_padded replicas = { |
1309 | .e.data_type = BCH_DATA_journal, | |
1310 | .e.nr_required = 1, | |
1311 | }; | |
1312 | unsigned ptr; | |
7ef2a73a | 1313 | |
ce6201c4 KO |
1314 | i = *_i; |
1315 | if (!i || i->ignore) | |
adbcada4 KO |
1316 | continue; |
1317 | ||
d1b2c864 | 1318 | for (ptr = 0; ptr < i->nr_ptrs; ptr++) { |
96dea3d5 | 1319 | ca = bch_dev_bkey_exists(c, i->ptrs[ptr].dev); |
d1b2c864 KO |
1320 | |
1321 | if (!i->ptrs[ptr].csum_good) | |
7fec8266 KO |
1322 | bch_err_dev_offset(ca, i->ptrs[ptr].sector, |
1323 | "invalid journal checksum, seq %llu%s", | |
1324 | le64_to_cpu(i->j.seq), | |
1325 | i->csum_good ? " (had good copy on another device)" : ""); | |
d1b2c864 KO |
1326 | } |
1327 | ||
1328 | ret = jset_validate(c, | |
1329 | bch_dev_bkey_exists(c, i->ptrs[0].dev), | |
1330 | &i->j, | |
1331 | i->ptrs[0].sector, | |
1332 | READ); | |
1c6fdbd8 | 1333 | if (ret) |
fa8e94fa | 1334 | goto err; |
1c6fdbd8 | 1335 | |
e4c3f386 KO |
1336 | for (ptr = 0; ptr < i->nr_ptrs; ptr++) |
1337 | replicas.e.devs[replicas.e.nr_devs++] = i->ptrs[ptr].dev; | |
1338 | ||
26452d1d KO |
1339 | bch2_replicas_entry_sort(&replicas.e); |
1340 | ||
fa8e94fa KO |
1341 | printbuf_reset(&buf); |
1342 | bch2_replicas_entry_to_text(&buf, &replicas.e); | |
1343 | ||
1c6fdbd8 | 1344 | if (!degraded && |
83b3d959 KO |
1345 | !bch2_replicas_marked(c, &replicas.e) && |
1346 | (le64_to_cpu(i->j.seq) == *last_seq || | |
b65db750 KO |
1347 | fsck_err(c, journal_entry_replicas_not_marked, |
1348 | "superblock not marked as containing replicas for journal entry %llu\n %s", | |
83b3d959 | 1349 | le64_to_cpu(i->j.seq), buf.buf))) { |
7ef2a73a | 1350 | ret = bch2_mark_replicas(c, &replicas.e); |
1c6fdbd8 | 1351 | if (ret) |
fa8e94fa | 1352 | goto err; |
1c6fdbd8 | 1353 | } |
1c6fdbd8 | 1354 | } |
fa8e94fa | 1355 | err: |
1c6fdbd8 | 1356 | fsck_err: |
fa8e94fa | 1357 | printbuf_exit(&buf); |
1c6fdbd8 KO |
1358 | return ret; |
1359 | } | |
1360 | ||
1c6fdbd8 KO |
1361 | /* journal write: */ |
1362 | ||
a9ec3454 KO |
1363 | static void __journal_write_alloc(struct journal *j, |
1364 | struct journal_buf *w, | |
1365 | struct dev_alloc_list *devs_sorted, | |
1366 | unsigned sectors, | |
1367 | unsigned *replicas, | |
1368 | unsigned replicas_want) | |
1c6fdbd8 KO |
1369 | { |
1370 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
1c6fdbd8 KO |
1371 | struct journal_device *ja; |
1372 | struct bch_dev *ca; | |
a9ec3454 | 1373 | unsigned i; |
a2753581 | 1374 | |
a9ec3454 KO |
1375 | if (*replicas >= replicas_want) |
1376 | return; | |
1c6fdbd8 | 1377 | |
a9ec3454 KO |
1378 | for (i = 0; i < devs_sorted->nr; i++) { |
1379 | ca = rcu_dereference(c->devs[devs_sorted->devs[i]]); | |
1c6fdbd8 KO |
1380 | if (!ca) |
1381 | continue; | |
1382 | ||
1c6fdbd8 | 1383 | ja = &ca->journal; |
1c6fdbd8 KO |
1384 | |
1385 | /* | |
1386 | * Check that we can use this device, and aren't already using | |
1387 | * it: | |
1388 | */ | |
a9ec3454 | 1389 | if (!ca->mi.durability || |
2436cb9f | 1390 | ca->mi.state != BCH_MEMBER_STATE_rw || |
a9ec3454 | 1391 | !ja->nr || |
702ffea2 | 1392 | bch2_bkey_has_device_c(bkey_i_to_s_c(&w->key), ca->dev_idx) || |
a9ec3454 | 1393 | sectors > ja->sectors_free) |
1c6fdbd8 KO |
1394 | continue; |
1395 | ||
3d080aa5 | 1396 | bch2_dev_stripe_increment(ca, &j->wp.stripe); |
1c6fdbd8 | 1397 | |
26609b61 | 1398 | bch2_bkey_append_ptr(&w->key, |
1c6fdbd8 KO |
1399 | (struct bch_extent_ptr) { |
1400 | .offset = bucket_to_sector(ca, | |
a9ec3454 KO |
1401 | ja->buckets[ja->cur_idx]) + |
1402 | ca->mi.bucket_size - | |
1403 | ja->sectors_free, | |
1c6fdbd8 KO |
1404 | .dev = ca->dev_idx, |
1405 | }); | |
1406 | ||
a9ec3454 KO |
1407 | ja->sectors_free -= sectors; |
1408 | ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq); | |
1409 | ||
1410 | *replicas += ca->mi.durability; | |
1411 | ||
1412 | if (*replicas >= replicas_want) | |
1413 | break; | |
1c6fdbd8 | 1414 | } |
a9ec3454 | 1415 | } |
1c6fdbd8 | 1416 | |
a9ec3454 | 1417 | /** |
96dea3d5 KO |
1418 | * journal_write_alloc - decide where to write next journal entry |
1419 | * | |
1420 | * @j: journal object | |
1421 | * @w: journal buf (entry to be written) | |
1422 | * | |
1423 | * Returns: 0 on success, or -EROFS on failure | |
a9ec3454 | 1424 | */ |
96dea3d5 | 1425 | static int journal_write_alloc(struct journal *j, struct journal_buf *w) |
a9ec3454 KO |
1426 | { |
1427 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
d042b040 | 1428 | struct bch_devs_mask devs; |
a9ec3454 KO |
1429 | struct journal_device *ja; |
1430 | struct bch_dev *ca; | |
1431 | struct dev_alloc_list devs_sorted; | |
96dea3d5 | 1432 | unsigned sectors = vstruct_sectors(w->data, c->block_bits); |
d042b040 KO |
1433 | unsigned target = c->opts.metadata_target ?: |
1434 | c->opts.foreground_target; | |
a9ec3454 KO |
1435 | unsigned i, replicas = 0, replicas_want = |
1436 | READ_ONCE(c->opts.metadata_replicas); | |
1c6fdbd8 | 1437 | |
a9ec3454 | 1438 | rcu_read_lock(); |
d042b040 KO |
1439 | retry: |
1440 | devs = target_rw_devs(c, BCH_DATA_journal, target); | |
1c6fdbd8 | 1441 | |
d042b040 | 1442 | devs_sorted = bch2_dev_alloc_list(c, &j->wp.stripe, &devs); |
1c6fdbd8 | 1443 | |
a9ec3454 KO |
1444 | __journal_write_alloc(j, w, &devs_sorted, |
1445 | sectors, &replicas, replicas_want); | |
1c6fdbd8 | 1446 | |
a9ec3454 KO |
1447 | if (replicas >= replicas_want) |
1448 | goto done; | |
1449 | ||
1450 | for (i = 0; i < devs_sorted.nr; i++) { | |
1451 | ca = rcu_dereference(c->devs[devs_sorted.devs[i]]); | |
1452 | if (!ca) | |
1453 | continue; | |
1454 | ||
1455 | ja = &ca->journal; | |
1456 | ||
1457 | if (sectors > ja->sectors_free && | |
1458 | sectors <= ca->mi.bucket_size && | |
03d5eaed KO |
1459 | bch2_journal_dev_buckets_available(j, ja, |
1460 | journal_space_discarded)) { | |
a9ec3454 KO |
1461 | ja->cur_idx = (ja->cur_idx + 1) % ja->nr; |
1462 | ja->sectors_free = ca->mi.bucket_size; | |
68ef94a6 KO |
1463 | |
1464 | /* | |
1465 | * ja->bucket_seq[ja->cur_idx] must always have | |
1466 | * something sensible: | |
1467 | */ | |
1468 | ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq); | |
a9ec3454 KO |
1469 | } |
1470 | } | |
1471 | ||
1472 | __journal_write_alloc(j, w, &devs_sorted, | |
1473 | sectors, &replicas, replicas_want); | |
d042b040 KO |
1474 | |
1475 | if (replicas < replicas_want && target) { | |
1476 | /* Retry from all devices: */ | |
1477 | target = 0; | |
1478 | goto retry; | |
1479 | } | |
a9ec3454 | 1480 | done: |
a9ec3454 KO |
1481 | rcu_read_unlock(); |
1482 | ||
07a1006a KO |
1483 | BUG_ON(bkey_val_u64s(&w->key.k) > BCH_REPLICAS_MAX); |
1484 | ||
57cb2142 | 1485 | return replicas >= c->opts.metadata_replicas_required ? 0 : -EROFS; |
1c6fdbd8 KO |
1486 | } |
1487 | ||
1c6fdbd8 KO |
1488 | static void journal_buf_realloc(struct journal *j, struct journal_buf *buf) |
1489 | { | |
1490 | /* we aren't holding j->lock: */ | |
1491 | unsigned new_size = READ_ONCE(j->buf_size_want); | |
1492 | void *new_buf; | |
1493 | ||
d16b4a77 | 1494 | if (buf->buf_size >= new_size) |
1c6fdbd8 KO |
1495 | return; |
1496 | ||
19c304be | 1497 | new_buf = kvpmalloc(new_size, GFP_NOFS|__GFP_NOWARN); |
1c6fdbd8 KO |
1498 | if (!new_buf) |
1499 | return; | |
1500 | ||
d16b4a77 | 1501 | memcpy(new_buf, buf->data, buf->buf_size); |
c859430b KO |
1502 | |
1503 | spin_lock(&j->lock); | |
1504 | swap(buf->data, new_buf); | |
1505 | swap(buf->buf_size, new_size); | |
1506 | spin_unlock(&j->lock); | |
1507 | ||
1508 | kvpfree(new_buf, new_size); | |
1c6fdbd8 KO |
1509 | } |
1510 | ||
ebb84d09 KO |
1511 | static inline struct journal_buf *journal_last_unwritten_buf(struct journal *j) |
1512 | { | |
30ef633a | 1513 | return j->buf + (journal_last_unwritten_seq(j) & JOURNAL_BUF_MASK); |
ebb84d09 KO |
1514 | } |
1515 | ||
1c6fdbd8 KO |
1516 | static void journal_write_done(struct closure *cl) |
1517 | { | |
1518 | struct journal *j = container_of(cl, struct journal, io); | |
1519 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
ebb84d09 | 1520 | struct journal_buf *w = journal_last_unwritten_buf(j); |
83b3d959 | 1521 | struct bch_replicas_padded replicas; |
ebb84d09 | 1522 | union journal_res_state old, new; |
1784d43a | 1523 | u64 v, seq; |
158eecb8 | 1524 | int err = 0; |
1c6fdbd8 | 1525 | |
991ba021 KO |
1526 | bch2_time_stats_update(!JSET_NO_FLUSH(w->data) |
1527 | ? j->flush_write_time | |
1528 | : j->noflush_write_time, j->write_start_time); | |
9c859dc9 | 1529 | |
d797ca3d | 1530 | if (!w->devs_written.nr) { |
1c6fdbd8 | 1531 | bch_err(c, "unable to write journal to sufficient devices"); |
158eecb8 | 1532 | err = -EIO; |
83b3d959 KO |
1533 | } else { |
1534 | bch2_devlist_to_replicas(&replicas.e, BCH_DATA_journal, | |
1535 | w->devs_written); | |
1536 | if (bch2_mark_replicas(c, &replicas.e)) | |
1537 | err = -EIO; | |
1c6fdbd8 | 1538 | } |
83b3d959 | 1539 | |
158eecb8 KO |
1540 | if (err) |
1541 | bch2_fatal_error(c); | |
1c6fdbd8 KO |
1542 | |
1543 | spin_lock(&j->lock); | |
ed9d58a2 | 1544 | seq = le64_to_cpu(w->data->seq); |
ed9d58a2 | 1545 | |
1c6fdbd8 | 1546 | if (seq >= j->pin.front) |
d797ca3d | 1547 | journal_seq_pin(j, seq)->devs = w->devs_written; |
1c6fdbd8 | 1548 | |
9be1efe9 | 1549 | if (!err) { |
9be1efe9 KO |
1550 | if (!JSET_NO_FLUSH(w->data)) { |
1551 | j->flushed_seq_ondisk = seq; | |
1552 | j->last_seq_ondisk = w->last_seq; | |
f25d8215 | 1553 | |
59cc38b8 | 1554 | bch2_do_discards(c); |
f25d8215 KO |
1555 | closure_wake_up(&c->freelist_wait); |
1556 | ||
1557 | bch2_reset_alloc_cursors(c); | |
9be1efe9 KO |
1558 | } |
1559 | } else if (!j->err_seq || seq < j->err_seq) | |
1560 | j->err_seq = seq; | |
0ce2dbbe | 1561 | |
f0a3a2cc KO |
1562 | j->seq_ondisk = seq; |
1563 | ||
1c6fdbd8 KO |
1564 | /* |
1565 | * Updating last_seq_ondisk may let bch2_journal_reclaim_work() discard | |
1566 | * more buckets: | |
1567 | * | |
1568 | * Must come before signaling write completion, for | |
1569 | * bch2_fs_journal_stop(): | |
1570 | */ | |
ec14fc60 | 1571 | if (j->watermark != BCH_WATERMARK_stripe) |
8cc052db | 1572 | journal_reclaim_kick(&c->journal); |
158eecb8 | 1573 | |
1c6fdbd8 KO |
1574 | /* also must come before signalling write completion: */ |
1575 | closure_debug_destroy(cl); | |
1576 | ||
ebb84d09 KO |
1577 | v = atomic64_read(&j->reservations.counter); |
1578 | do { | |
1579 | old.v = new.v = v; | |
24a3d53b | 1580 | BUG_ON(journal_state_count(new, new.unwritten_idx)); |
ebb84d09 KO |
1581 | |
1582 | new.unwritten_idx++; | |
1583 | } while ((v = atomic64_cmpxchg(&j->reservations.counter, | |
1584 | old.v, new.v)) != old.v); | |
1c6fdbd8 | 1585 | |
5d32c5bb KO |
1586 | bch2_journal_space_available(j); |
1587 | ||
1c6fdbd8 KO |
1588 | closure_wake_up(&w->wait); |
1589 | journal_wake(j); | |
1590 | ||
24a3d53b KO |
1591 | if (!journal_state_count(new, new.unwritten_idx) && |
1592 | journal_last_unwritten_seq(j) <= journal_cur_seq(j)) { | |
cfda31c0 | 1593 | spin_unlock(&j->lock); |
24a3d53b KO |
1594 | closure_call(&j->io, bch2_journal_write, c->io_complete_wq, NULL); |
1595 | } else if (journal_last_unwritten_seq(j) == journal_cur_seq(j) && | |
1596 | new.cur_entry_offset < JOURNAL_ENTRY_CLOSED_VAL) { | |
fbec3b88 KO |
1597 | struct journal_buf *buf = journal_cur_buf(j); |
1598 | long delta = buf->expires - jiffies; | |
ebb84d09 | 1599 | |
24a3d53b KO |
1600 | /* |
1601 | * We don't close a journal entry to write it while there's | |
1602 | * previous entries still in flight - the current journal entry | |
1603 | * might want to be written now: | |
1604 | */ | |
1605 | ||
cfda31c0 | 1606 | spin_unlock(&j->lock); |
fbec3b88 | 1607 | mod_delayed_work(c->io_complete_wq, &j->write_work, max(0L, delta)); |
cfda31c0 KO |
1608 | } else { |
1609 | spin_unlock(&j->lock); | |
24a3d53b | 1610 | } |
1c6fdbd8 KO |
1611 | } |
1612 | ||
1613 | static void journal_write_endio(struct bio *bio) | |
1614 | { | |
1615 | struct bch_dev *ca = bio->bi_private; | |
1616 | struct journal *j = &ca->fs->journal; | |
d797ca3d KO |
1617 | struct journal_buf *w = journal_last_unwritten_buf(j); |
1618 | unsigned long flags; | |
1c6fdbd8 | 1619 | |
94119eeb KO |
1620 | if (bch2_dev_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_write, |
1621 | "error writing journal entry %llu: %s", | |
d797ca3d | 1622 | le64_to_cpu(w->data->seq), |
63b214e7 | 1623 | bch2_blk_status_to_str(bio->bi_status)) || |
1c6fdbd8 | 1624 | bch2_meta_write_fault("journal")) { |
1c6fdbd8 | 1625 | spin_lock_irqsave(&j->err_lock, flags); |
d797ca3d | 1626 | bch2_dev_list_drop_dev(&w->devs_written, ca->dev_idx); |
1c6fdbd8 KO |
1627 | spin_unlock_irqrestore(&j->err_lock, flags); |
1628 | } | |
1629 | ||
1630 | closure_put(&j->io); | |
1631 | percpu_ref_put(&ca->io_ref); | |
1632 | } | |
1633 | ||
280249b9 KO |
1634 | static void do_journal_write(struct closure *cl) |
1635 | { | |
1636 | struct journal *j = container_of(cl, struct journal, io); | |
1637 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
1638 | struct bch_dev *ca; | |
1639 | struct journal_buf *w = journal_last_unwritten_buf(j); | |
1640 | struct bch_extent_ptr *ptr; | |
1641 | struct bio *bio; | |
1642 | unsigned sectors = vstruct_sectors(w->data, c->block_bits); | |
1643 | ||
1644 | extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) { | |
1645 | ca = bch_dev_bkey_exists(c, ptr->dev); | |
1646 | if (!percpu_ref_tryget(&ca->io_ref)) { | |
1647 | /* XXX: fix this */ | |
1648 | bch_err(c, "missing device for journal write\n"); | |
1649 | continue; | |
1650 | } | |
1651 | ||
1652 | this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_journal], | |
1653 | sectors); | |
1654 | ||
1655 | bio = ca->journal.bio; | |
1656 | bio_reset(bio, ca->disk_sb.bdev, REQ_OP_WRITE|REQ_SYNC|REQ_META); | |
1657 | bio->bi_iter.bi_sector = ptr->offset; | |
1658 | bio->bi_end_io = journal_write_endio; | |
1659 | bio->bi_private = ca; | |
1660 | ||
a28bd48a KO |
1661 | BUG_ON(bio->bi_iter.bi_sector == ca->prev_journal_sector); |
1662 | ca->prev_journal_sector = bio->bi_iter.bi_sector; | |
1663 | ||
280249b9 KO |
1664 | if (!JSET_NO_FLUSH(w->data)) |
1665 | bio->bi_opf |= REQ_FUA; | |
1666 | if (!JSET_NO_FLUSH(w->data) && !w->separate_flush) | |
1667 | bio->bi_opf |= REQ_PREFLUSH; | |
1668 | ||
1669 | bch2_bio_map(bio, w->data, sectors << 9); | |
1670 | ||
674cfc26 | 1671 | trace_and_count(c, journal_write, bio); |
280249b9 KO |
1672 | closure_bio_submit(bio, cl); |
1673 | ||
1674 | ca->journal.bucket_seq[ca->journal.cur_idx] = | |
1675 | le64_to_cpu(w->data->seq); | |
1676 | } | |
1677 | ||
731bdd2e | 1678 | continue_at(cl, journal_write_done, c->io_complete_wq); |
280249b9 KO |
1679 | } |
1680 | ||
9f6db127 KO |
1681 | static void bch2_journal_entries_postprocess(struct bch_fs *c, struct jset *jset) |
1682 | { | |
1683 | struct jset_entry *i, *next, *prev = NULL; | |
1684 | ||
1685 | /* | |
1686 | * Simple compaction, dropping empty jset_entries (from journal | |
1687 | * reservations that weren't fully used) and merging jset_entries that | |
1688 | * can be. | |
1689 | * | |
1690 | * If we wanted to be really fancy here, we could sort all the keys in | |
1691 | * the jset and drop keys that were overwritten - probably not worth it: | |
1692 | */ | |
1693 | vstruct_for_each_safe(jset, i, next) { | |
1694 | unsigned u64s = le16_to_cpu(i->u64s); | |
1695 | ||
1696 | /* Empty entry: */ | |
1697 | if (!u64s) | |
1698 | continue; | |
1699 | ||
1700 | if (i->type == BCH_JSET_ENTRY_btree_root) | |
1701 | bch2_journal_entry_to_btree_root(c, i); | |
1702 | ||
1703 | /* Can we merge with previous entry? */ | |
1704 | if (prev && | |
1705 | i->btree_id == prev->btree_id && | |
1706 | i->level == prev->level && | |
1707 | i->type == prev->type && | |
1708 | i->type == BCH_JSET_ENTRY_btree_keys && | |
1709 | le16_to_cpu(prev->u64s) + u64s <= U16_MAX) { | |
1710 | memmove_u64s_down(vstruct_next(prev), | |
1711 | i->_data, | |
1712 | u64s); | |
1713 | le16_add_cpu(&prev->u64s, u64s); | |
1714 | continue; | |
1715 | } | |
1716 | ||
1717 | /* Couldn't merge, move i into new position (after prev): */ | |
1718 | prev = prev ? vstruct_next(prev) : jset->start; | |
1719 | if (i != prev) | |
1720 | memmove_u64s_down(prev, i, jset_u64s(u64s)); | |
1721 | } | |
1722 | ||
1723 | prev = prev ? vstruct_next(prev) : jset->start; | |
1724 | jset->u64s = cpu_to_le32((u64 *) prev - jset->_data); | |
1725 | } | |
1726 | ||
80396a47 | 1727 | static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w) |
1c6fdbd8 | 1728 | { |
1c6fdbd8 | 1729 | struct bch_fs *c = container_of(j, struct bch_fs, journal); |
3ccc5c50 | 1730 | struct jset_entry *start, *end; |
1c6fdbd8 | 1731 | struct jset *jset; |
80396a47 | 1732 | unsigned sectors, bytes, u64s; |
26609b61 | 1733 | bool validate_before_checksum = false; |
e5a66496 KO |
1734 | int ret; |
1735 | ||
1c6fdbd8 KO |
1736 | journal_buf_realloc(j, w); |
1737 | jset = w->data; | |
1738 | ||
00b8ccf7 KO |
1739 | /* |
1740 | * New btree roots are set by journalling them; when the journal entry | |
1741 | * gets written we have to propagate them to c->btree_roots | |
1742 | * | |
1743 | * But, every journal entry we write has to contain all the btree roots | |
1744 | * (at least for now); so after we copy btree roots to c->btree_roots we | |
1745 | * have to get any missing btree roots and add them to this journal | |
1746 | * entry: | |
1747 | */ | |
1748 | ||
9f6db127 | 1749 | bch2_journal_entries_postprocess(c, jset); |
00b8ccf7 KO |
1750 | |
1751 | start = end = vstruct_last(jset); | |
1752 | ||
1753 | end = bch2_btree_roots_to_journal_entries(c, jset->start, end); | |
1754 | ||
2abe5420 KO |
1755 | bch2_journal_super_entries_add_common(c, &end, |
1756 | le64_to_cpu(jset->seq)); | |
3ccc5c50 KO |
1757 | u64s = (u64 *) end - (u64 *) start; |
1758 | BUG_ON(u64s > j->entry_u64s_reserved); | |
1759 | ||
d16b4a77 | 1760 | le32_add_cpu(&jset->u64s, u64s); |
4a2e5d7b KO |
1761 | |
1762 | sectors = vstruct_sectors(jset, c->block_bits); | |
1763 | bytes = vstruct_bytes(jset); | |
1764 | ||
1765 | if (sectors > w->sectors) { | |
1766 | bch2_fs_fatal_error(c, "aieeee! journal write overran available space, %zu > %u (extra %u reserved %u/%u)", | |
1767 | vstruct_bytes(jset), w->sectors << 9, | |
1768 | u64s, w->u64s_reserved, j->entry_u64s_reserved); | |
80396a47 | 1769 | return -EINVAL; |
4a2e5d7b | 1770 | } |
1c6fdbd8 | 1771 | |
1c6fdbd8 | 1772 | jset->magic = cpu_to_le64(jset_magic(c)); |
a02a0121 | 1773 | jset->version = cpu_to_le32(c->sb.version); |
1c6fdbd8 KO |
1774 | |
1775 | SET_JSET_BIG_ENDIAN(jset, CPU_BIG_ENDIAN); | |
1776 | SET_JSET_CSUM_TYPE(jset, bch2_meta_checksum_type(c)); | |
1777 | ||
4141fde0 | 1778 | if (!JSET_NO_FLUSH(jset) && journal_entry_empty(jset)) |
158eecb8 KO |
1779 | j->last_empty_seq = le64_to_cpu(jset->seq); |
1780 | ||
26609b61 KO |
1781 | if (bch2_csum_type_is_encryption(JSET_CSUM_TYPE(jset))) |
1782 | validate_before_checksum = true; | |
1783 | ||
e751c01a | 1784 | if (le32_to_cpu(jset->version) < bcachefs_metadata_version_current) |
26609b61 KO |
1785 | validate_before_checksum = true; |
1786 | ||
1787 | if (validate_before_checksum && | |
80396a47 KO |
1788 | (ret = jset_validate(c, NULL, jset, 0, WRITE))) |
1789 | return ret; | |
1c6fdbd8 | 1790 | |
a9de137b | 1791 | ret = bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset), |
1c6fdbd8 KO |
1792 | jset->encrypted_start, |
1793 | vstruct_end(jset) - (void *) jset->encrypted_start); | |
a9de137b KO |
1794 | if (bch2_fs_fatal_err_on(ret, c, |
1795 | "error decrypting journal entry: %i", ret)) | |
80396a47 | 1796 | return ret; |
1c6fdbd8 KO |
1797 | |
1798 | jset->csum = csum_vstruct(c, JSET_CSUM_TYPE(jset), | |
1799 | journal_nonce(jset), jset); | |
1800 | ||
26609b61 | 1801 | if (!validate_before_checksum && |
80396a47 KO |
1802 | (ret = jset_validate(c, NULL, jset, 0, WRITE))) |
1803 | return ret; | |
1c6fdbd8 | 1804 | |
d16b4a77 | 1805 | memset((void *) jset + bytes, 0, (sectors << 9) - bytes); |
80396a47 KO |
1806 | return 0; |
1807 | } | |
1808 | ||
1809 | static int bch2_journal_write_pick_flush(struct journal *j, struct journal_buf *w) | |
1810 | { | |
1811 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
1812 | int error = bch2_journal_error(j); | |
1813 | ||
1814 | /* | |
1815 | * If the journal is in an error state - we did an emergency shutdown - | |
1816 | * we prefer to continue doing journal writes. We just mark them as | |
1817 | * noflush so they'll never be used, but they'll still be visible by the | |
1818 | * list_journal tool - this helps in debugging. | |
1819 | * | |
1820 | * There's a caveat: the first journal write after marking the | |
1821 | * superblock dirty must always be a flush write, because on startup | |
1822 | * from a clean shutdown we didn't necessarily read the journal and the | |
1823 | * new journal write might overwrite whatever was in the journal | |
1824 | * previously - we can't leave the journal without any flush writes in | |
1825 | * it. | |
1826 | * | |
1827 | * So if we're in an error state, and we're still starting up, we don't | |
1828 | * write anything at all. | |
1829 | */ | |
1830 | if (error && test_bit(JOURNAL_NEED_FLUSH_WRITE, &j->flags)) | |
1831 | return -EIO; | |
1832 | ||
1833 | if (error || | |
1834 | w->noflush || | |
1835 | (!w->must_flush && | |
1836 | (jiffies - j->last_flush_write) < msecs_to_jiffies(c->opts.journal_flush_delay) && | |
1837 | test_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags))) { | |
1838 | w->noflush = true; | |
1839 | SET_JSET_NO_FLUSH(w->data, true); | |
1840 | w->data->last_seq = 0; | |
1841 | w->last_seq = 0; | |
1842 | ||
1843 | j->nr_noflush_writes++; | |
1844 | } else { | |
1845 | j->last_flush_write = jiffies; | |
1846 | j->nr_flush_writes++; | |
1847 | clear_bit(JOURNAL_NEED_FLUSH_WRITE, &j->flags); | |
1848 | } | |
1849 | ||
1850 | return 0; | |
1851 | } | |
1852 | ||
1853 | void bch2_journal_write(struct closure *cl) | |
1854 | { | |
1855 | struct journal *j = container_of(cl, struct journal, io); | |
1856 | struct bch_fs *c = container_of(j, struct bch_fs, journal); | |
1857 | struct bch_dev *ca; | |
1858 | struct journal_buf *w = journal_last_unwritten_buf(j); | |
1859 | struct bch_replicas_padded replicas; | |
1860 | struct bio *bio; | |
1861 | struct printbuf journal_debug_buf = PRINTBUF; | |
1862 | unsigned i, nr_rw_members = 0; | |
1863 | int ret; | |
1864 | ||
1865 | BUG_ON(BCH_SB_CLEAN(c->disk_sb.sb)); | |
1866 | ||
1867 | j->write_start_time = local_clock(); | |
1c6fdbd8 | 1868 | |
e5a66496 | 1869 | spin_lock(&j->lock); |
80396a47 KO |
1870 | ret = bch2_journal_write_pick_flush(j, w); |
1871 | spin_unlock(&j->lock); | |
1872 | if (ret) | |
1873 | goto err; | |
1874 | ||
1875 | ret = bch2_journal_write_prep(j, w); | |
1876 | if (ret) | |
1877 | goto err; | |
1878 | ||
1879 | while (1) { | |
1880 | spin_lock(&j->lock); | |
1881 | ret = journal_write_alloc(j, w); | |
1882 | if (!ret || !j->can_discard) | |
1883 | break; | |
e5a66496 | 1884 | |
c18dade6 KO |
1885 | spin_unlock(&j->lock); |
1886 | bch2_journal_do_discards(j); | |
c18dade6 KO |
1887 | } |
1888 | ||
80396a47 | 1889 | if (ret) { |
fa8e94fa | 1890 | __bch2_journal_debug_to_text(&journal_debug_buf, j); |
80396a47 KO |
1891 | spin_unlock(&j->lock); |
1892 | bch_err(c, "Unable to allocate journal write:\n%s", | |
1893 | journal_debug_buf.buf); | |
1894 | printbuf_exit(&journal_debug_buf); | |
1895 | goto err; | |
1896 | } | |
85674154 | 1897 | |
e5a66496 KO |
1898 | /* |
1899 | * write is allocated, no longer need to account for it in | |
1900 | * bch2_journal_space_available(): | |
1901 | */ | |
1902 | w->sectors = 0; | |
1903 | ||
1904 | /* | |
1905 | * journal entry has been compacted and allocated, recalculate space | |
1906 | * available: | |
1907 | */ | |
1908 | bch2_journal_space_available(j); | |
1909 | spin_unlock(&j->lock); | |
1910 | ||
d797ca3d KO |
1911 | w->devs_written = bch2_bkey_devs(bkey_i_to_s_c(&w->key)); |
1912 | ||
b66b2bc0 | 1913 | if (c->opts.nochanges) |
1c6fdbd8 KO |
1914 | goto no_io; |
1915 | ||
280249b9 KO |
1916 | for_each_rw_member(ca, c, i) |
1917 | nr_rw_members++; | |
1c6fdbd8 | 1918 | |
280249b9 KO |
1919 | if (nr_rw_members > 1) |
1920 | w->separate_flush = true; | |
1c6fdbd8 | 1921 | |
a7b29b8d BF |
1922 | /* |
1923 | * Mark journal replicas before we submit the write to guarantee | |
1924 | * recovery will find the journal entries after a crash. | |
1925 | */ | |
1926 | bch2_devlist_to_replicas(&replicas.e, BCH_DATA_journal, | |
1927 | w->devs_written); | |
1928 | ret = bch2_mark_replicas(c, &replicas.e); | |
1929 | if (ret) | |
1930 | goto err; | |
1931 | ||
80396a47 | 1932 | if (!JSET_NO_FLUSH(w->data) && w->separate_flush) { |
280249b9 KO |
1933 | for_each_rw_member(ca, c, i) { |
1934 | percpu_ref_get(&ca->io_ref); | |
1c6fdbd8 | 1935 | |
280249b9 KO |
1936 | bio = ca->journal.bio; |
1937 | bio_reset(bio, ca->disk_sb.bdev, REQ_OP_FLUSH); | |
1938 | bio->bi_end_io = journal_write_endio; | |
1939 | bio->bi_private = ca; | |
1940 | closure_bio_submit(bio, cl); | |
1941 | } | |
1c6fdbd8 KO |
1942 | } |
1943 | ||
731bdd2e | 1944 | continue_at(cl, do_journal_write, c->io_complete_wq); |
280249b9 | 1945 | return; |
1c6fdbd8 | 1946 | no_io: |
731bdd2e | 1947 | continue_at(cl, journal_write_done, c->io_complete_wq); |
1c6fdbd8 KO |
1948 | return; |
1949 | err: | |
b74b147d | 1950 | bch2_fatal_error(c); |
731bdd2e | 1951 | continue_at(cl, journal_write_done, c->io_complete_wq); |
1c6fdbd8 | 1952 | } |