fs/bcachefs/journal_io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include "bcachefs.h"
   3 #include "alloc_background.h"
   4 #include "alloc_foreground.h"
   5 #include "btree_gc.h"
   6 #include "btree_update.h"
   7 #include "buckets.h"
   8 #include "checksum.h"
   9 #include "error.h"
  10 #include "journal.h"
  11 #include "journal_io.h"
  12 #include "journal_reclaim.h"
  13 #include "replicas.h"
  14 #include "trace.h"
  15
  16 struct journal_list {
  17         struct closure          cl;
  18         struct mutex            lock;
  19         struct list_head        *head;
  20         int                     ret;
  21 };
  22
  23 #define JOURNAL_ENTRY_ADD_OK            0
  24 #define JOURNAL_ENTRY_ADD_OUT_OF_RANGE  5
  25
  26 /*
  27  * Given a journal entry we just read, add it to the list of journal entries to
  28  * be replayed:
  29  */
  30 static int journal_entry_add(struct bch_fs *c, struct bch_dev *ca,
  31                              struct journal_list *jlist, struct jset *j)
  32 {
  33         struct journal_replay *i, *pos;
  34         struct list_head *where;
  35         size_t bytes = vstruct_bytes(j);
  36         __le64 last_seq;
  37         int ret;
  38
  39         last_seq = !list_empty(jlist->head)
  40                 ? list_last_entry(jlist->head, struct journal_replay,
  41                                   list)->j.last_seq
  42                 : 0;
  43
  44         /* Is this entry older than the range we need? */
  45         if (le64_to_cpu(j->seq) < le64_to_cpu(last_seq)) {
  46                 ret = JOURNAL_ENTRY_ADD_OUT_OF_RANGE;
  47                 goto out;
  48         }
  49
  50         /* Drop entries we don't need anymore */
  51         list_for_each_entry_safe(i, pos, jlist->head, list) {
  52                 if (le64_to_cpu(i->j.seq) >= le64_to_cpu(j->last_seq))
  53                         break;
  54                 list_del(&i->list);
  55                 kvpfree(i, offsetof(struct journal_replay, j) +
  56                         vstruct_bytes(&i->j));
  57         }
  58
  59         list_for_each_entry_reverse(i, jlist->head, list) {
  60                 /* Duplicate? */
  61                 if (le64_to_cpu(j->seq) == le64_to_cpu(i->j.seq)) {
  62                         fsck_err_on(bytes != vstruct_bytes(&i->j) ||
  63                                     memcmp(j, &i->j, bytes), c,
  64                                     "found duplicate but non identical journal entries (seq %llu)",
  65                                     le64_to_cpu(j->seq));
  66                         goto found;
  67                 }
  68
  69                 if (le64_to_cpu(j->seq) > le64_to_cpu(i->j.seq)) {
  70                         where = &i->list;
  71                         goto add;
  72                 }
  73         }
  74
  75         where = jlist->head;
  76 add:
  77         i = kvpmalloc(offsetof(struct journal_replay, j) + bytes, GFP_KERNEL);
  78         if (!i) {
  79                 ret = -ENOMEM;
  80                 goto out;
  81         }
  82
  83         list_add(&i->list, where);
  84         i->devs.nr = 0;
  85         unsafe_memcpy(&i->j, j, bytes, "embedded variable length struct");
  86 found:
  87         if (!bch2_dev_list_has_dev(i->devs, ca->dev_idx))
  88                 bch2_dev_list_add_dev(&i->devs, ca->dev_idx);
  89         else
  90                 fsck_err_on(1, c, "duplicate journal entries on same device");
  91         ret = JOURNAL_ENTRY_ADD_OK;
  92 out:
  93 fsck_err:
  94         return ret;
  95 }
  96
  97 static struct nonce journal_nonce(const struct jset *jset)
  98 {
  99         return (struct nonce) {{
 100                 [0] = 0,
 101                 [1] = ((__le32 *) &jset->seq)[0],
 102                 [2] = ((__le32 *) &jset->seq)[1],
 103                 [3] = BCH_NONCE_JOURNAL,
 104         }};
 105 }
 106
 107 /* this fills in a range with empty jset_entries: */
 108 static void journal_entry_null_range(void *start, void *end)
 109 {
 110         struct jset_entry *entry;
 111
 112         for (entry = start; entry != end; entry = vstruct_next(entry))
 113                 memset(entry, 0, sizeof(*entry));
 114 }
 115
 116 #define JOURNAL_ENTRY_REREAD    5
 117 #define JOURNAL_ENTRY_NONE      6
 118 #define JOURNAL_ENTRY_BAD       7
 119
 120 #define journal_entry_err(c, msg, ...)                                  \
 121 ({                                                                      \
 122         switch (write) {                                                \
 123         case READ:                                                      \
 124                 mustfix_fsck_err(c, msg, ##__VA_ARGS__);                \
 125                 break;                                                  \
 126         case WRITE:                                                     \
 127                 bch_err(c, "corrupt metadata before write:\n"           \
 128                         msg, ##__VA_ARGS__);                            \
 129                 if (bch2_fs_inconsistent(c)) {                          \
 130                         ret = BCH_FSCK_ERRORS_NOT_FIXED;                \
 131                         goto fsck_err;                                  \
 132                 }                                                       \
 133                 break;                                                  \
 134         }                                                               \
 135         true;                                                           \
 136 })
 137
 138 #define journal_entry_err_on(cond, c, msg, ...)                         \
 139         ((cond) ? journal_entry_err(c, msg, ##__VA_ARGS__) : false)
 140
 141 static int journal_validate_key(struct bch_fs *c, struct jset *jset,
 142                                 struct jset_entry *entry,
 143                                 struct bkey_i *k, enum btree_node_type key_type,
 144                                 const char *type, int write)
 145 {
 146         void *next = vstruct_next(entry);
 147         const char *invalid;
 148         unsigned version = le32_to_cpu(jset->version);
 149         int ret = 0;
 150
 151         if (journal_entry_err_on(!k->k.u64s, c,
 152                         "invalid %s in journal: k->u64s 0", type)) {
 153                 entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
 154                 journal_entry_null_range(vstruct_next(entry), next);
 155                 return 0;
 156         }
 157
 158         if (journal_entry_err_on((void *) bkey_next(k) >
 159                                 (void *) vstruct_next(entry), c,
 160                         "invalid %s in journal: extends past end of journal entry",
 161                         type)) {
 162                 entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
 163                 journal_entry_null_range(vstruct_next(entry), next);
 164                 return 0;
 165         }
 166
 167         if (journal_entry_err_on(k->k.format != KEY_FORMAT_CURRENT, c,
 168                         "invalid %s in journal: bad format %u",
 169                         type, k->k.format)) {
 170                 le16_add_cpu(&entry->u64s, -k->k.u64s);
 171                 memmove(k, bkey_next(k), next - (void *) bkey_next(k));
 172                 journal_entry_null_range(vstruct_next(entry), next);
 173                 return 0;
 174         }
 175
 176         if (JSET_BIG_ENDIAN(jset) != CPU_BIG_ENDIAN)
 177                 bch2_bkey_swab(NULL, bkey_to_packed(k));
 178
 179         if (!write &&
 180             version < bcachefs_metadata_version_bkey_renumber)
 181                 bch2_bkey_renumber(key_type, bkey_to_packed(k), write);
 182
 183         invalid = bch2_bkey_invalid(c, bkey_i_to_s_c(k), key_type);
 184         if (invalid) {
 185                 char buf[160];
 186
 187                 bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(k));
 188                 mustfix_fsck_err(c, "invalid %s in journal: %s\n%s",
 189                                  type, invalid, buf);
 190
 191                 le16_add_cpu(&entry->u64s, -k->k.u64s);
 192                 memmove(k, bkey_next(k), next - (void *) bkey_next(k));
 193                 journal_entry_null_range(vstruct_next(entry), next);
 194                 return 0;
 195         }
 196
 197         if (write &&
 198             version < bcachefs_metadata_version_bkey_renumber)
 199                 bch2_bkey_renumber(key_type, bkey_to_packed(k), write);
 200 fsck_err:
 201         return ret;
 202 }
 203
 204 static int journal_entry_validate_btree_keys(struct bch_fs *c,
 205                                              struct jset *jset,
 206                                              struct jset_entry *entry,
 207                                              int write)
 208 {
 209         struct bkey_i *k;
 210
 211         vstruct_for_each(entry, k) {
 212                 int ret = journal_validate_key(c, jset, entry, k,
 213                                 __btree_node_type(entry->level,
 214                                                   entry->btree_id),
 215                                 "key", write);
 216                 if (ret)
 217                         return ret;
 218         }
 219
 220         return 0;
 221 }
 222
 223 static int journal_entry_validate_btree_root(struct bch_fs *c,
 224                                              struct jset *jset,
 225                                              struct jset_entry *entry,
 226                                              int write)
 227 {
 228         struct bkey_i *k = entry->start;
 229         int ret = 0;
 230
 231         if (journal_entry_err_on(!entry->u64s ||
 232                                  le16_to_cpu(entry->u64s) != k->k.u64s, c,
 233                                  "invalid btree root journal entry: wrong number of keys")) {
 234                 void *next = vstruct_next(entry);
 235                 /*
 236                  * we don't want to null out this jset_entry,
 237                  * just the contents, so that later we can tell
 238                  * we were _supposed_ to have a btree root
 239                  */
 240                 entry->u64s = 0;
 241                 journal_entry_null_range(vstruct_next(entry), next);
 242                 return 0;
 243         }
 244
 245         return journal_validate_key(c, jset, entry, k, BKEY_TYPE_BTREE,
 246                                     "btree root", write);
 247 fsck_err:
 248         return ret;
 249 }
 250
 251 static int journal_entry_validate_prio_ptrs(struct bch_fs *c,
 252                                             struct jset *jset,
 253                                             struct jset_entry *entry,
 254                                             int write)
 255 {
 256         /* obsolete, don't care: */
 257         return 0;
 258 }
 259
 260 static int journal_entry_validate_blacklist(struct bch_fs *c,
 261                                             struct jset *jset,
 262                                             struct jset_entry *entry,
 263                                             int write)
 264 {
 265         int ret = 0;
 266
 267         if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 1, c,
 268                 "invalid journal seq blacklist entry: bad size")) {
 269                 journal_entry_null_range(entry, vstruct_next(entry));
 270         }
 271 fsck_err:
 272         return ret;
 273 }
 274
 275 static int journal_entry_validate_blacklist_v2(struct bch_fs *c,
 276                                                struct jset *jset,
 277                                                struct jset_entry *entry,
 278                                                int write)
 279 {
 280         struct jset_entry_blacklist_v2 *bl_entry;
 281         int ret = 0;
 282
 283         if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 2, c,
 284                 "invalid journal seq blacklist entry: bad size")) {
 285                 journal_entry_null_range(entry, vstruct_next(entry));
 286                 goto out;
 287         }
 288
 289         bl_entry = container_of(entry, struct jset_entry_blacklist_v2, entry);
 290
 291         if (journal_entry_err_on(le64_to_cpu(bl_entry->start) >
 292                                  le64_to_cpu(bl_entry->end), c,
 293                 "invalid journal seq blacklist entry: start > end")) {
 294                 journal_entry_null_range(entry, vstruct_next(entry));
 295         }
 296 out:
 297 fsck_err:
 298         return ret;
 299 }
 300
 301 static int journal_entry_validate_usage(struct bch_fs *c,
 302                                         struct jset *jset,
 303                                         struct jset_entry *entry,
 304                                         int write)
 305 {
 306         struct jset_entry_usage *u =
 307                 container_of(entry, struct jset_entry_usage, entry);
 308         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 309         int ret = 0;
 310
 311         if (journal_entry_err_on(bytes < sizeof(*u),
 312                                  c,
 313                                  "invalid journal entry usage: bad size")) {
 314                 journal_entry_null_range(entry, vstruct_next(entry));
 315                 return ret;
 316         }
 317
 318 fsck_err:
 319         return ret;
 320 }
 321
 322 static int journal_entry_validate_data_usage(struct bch_fs *c,
 323                                         struct jset *jset,
 324                                         struct jset_entry *entry,
 325                                         int write)
 326 {
 327         struct jset_entry_data_usage *u =
 328                 container_of(entry, struct jset_entry_data_usage, entry);
 329         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 330         int ret = 0;
 331
 332         if (journal_entry_err_on(bytes < sizeof(*u) ||
 333                                  bytes < sizeof(*u) + u->r.nr_devs,
 334                                  c,
 335                                  "invalid journal entry usage: bad size")) {
 336                 journal_entry_null_range(entry, vstruct_next(entry));
 337                 return ret;
 338         }
 339
 340 fsck_err:
 341         return ret;
 342 }
 343
 344 struct jset_entry_ops {
 345         int (*validate)(struct bch_fs *, struct jset *,
 346                         struct jset_entry *, int);
 347 };
 348
 349 static const struct jset_entry_ops bch2_jset_entry_ops[] = {
 350 #define x(f, nr)                                                \
 351         [BCH_JSET_ENTRY_##f]    = (struct jset_entry_ops) {     \
 352                 .validate       = journal_entry_validate_##f,   \
 353         },
 354         BCH_JSET_ENTRY_TYPES()
 355 #undef x
 356 };
 357
 358 static int journal_entry_validate(struct bch_fs *c, struct jset *jset,
 359                                   struct jset_entry *entry, int write)
 360 {
 361         return entry->type < BCH_JSET_ENTRY_NR
 362                 ? bch2_jset_entry_ops[entry->type].validate(c, jset,
 363                                                             entry, write)
 364                 : 0;
 365 }
 366
 367 static int jset_validate_entries(struct bch_fs *c, struct jset *jset,
 368                                  int write)
 369 {
 370         struct jset_entry *entry;
 371         int ret = 0;
 372
 373         vstruct_for_each(jset, entry) {
 374                 if (journal_entry_err_on(vstruct_next(entry) >
 375                                          vstruct_last(jset), c,
 376                                 "journal entry extends past end of jset")) {
 377                         jset->u64s = cpu_to_le32((u64 *) entry - jset->_data);
 378                         break;
 379                 }
 380
 381                 ret = journal_entry_validate(c, jset, entry, write);
 382                 if (ret)
 383                         break;
 384         }
 385 fsck_err:
 386         return ret;
 387 }
 388
 389 static int jset_validate(struct bch_fs *c,
 390                          struct jset *jset, u64 sector,
 391                          unsigned bucket_sectors_left,
 392                          unsigned sectors_read,
 393                          int write)
 394 {
 395         size_t bytes = vstruct_bytes(jset);
 396         struct bch_csum csum;
 397         unsigned version;
 398         int ret = 0;
 399
 400         if (le64_to_cpu(jset->magic) != jset_magic(c))
 401                 return JOURNAL_ENTRY_NONE;
 402
 403         version = le32_to_cpu(jset->version);
 404         if ((version != BCH_JSET_VERSION_OLD &&
 405              version < bcachefs_metadata_version_min) ||
 406             version >= bcachefs_metadata_version_max) {
 407                 bch_err(c, "unknown journal entry version %u", jset->version);
 408                 return BCH_FSCK_UNKNOWN_VERSION;
 409         }
 410
 411         if (journal_entry_err_on(bytes > bucket_sectors_left << 9, c,
 412                                  "journal entry too big (%zu bytes), sector %lluu",
 413                                  bytes, sector)) {
 414                 /* XXX: note we might have missing journal entries */
 415                 return JOURNAL_ENTRY_BAD;
 416         }
 417
 418         if (bytes > sectors_read << 9)
 419                 return JOURNAL_ENTRY_REREAD;
 420
 421         if (fsck_err_on(!bch2_checksum_type_valid(c, JSET_CSUM_TYPE(jset)), c,
 422                         "journal entry with unknown csum type %llu sector %lluu",
 423                         JSET_CSUM_TYPE(jset), sector))
 424                 return JOURNAL_ENTRY_BAD;
 425
 426         csum = csum_vstruct(c, JSET_CSUM_TYPE(jset), journal_nonce(jset), jset);
 427         if (journal_entry_err_on(bch2_crc_cmp(csum, jset->csum), c,
 428                                  "journal checksum bad, sector %llu", sector)) {
 429                 /* XXX: retry IO, when we start retrying checksum errors */
 430                 /* XXX: note we might have missing journal entries */
 431                 return JOURNAL_ENTRY_BAD;
 432         }
 433
 434         bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset),
 435                      jset->encrypted_start,
 436                      vstruct_end(jset) - (void *) jset->encrypted_start);
 437
 438         if (journal_entry_err_on(le64_to_cpu(jset->last_seq) > le64_to_cpu(jset->seq), c,
 439                                  "invalid journal entry: last_seq > seq"))
 440                 jset->last_seq = jset->seq;
 441
 442         return 0;
 443 fsck_err:
 444         return ret;
 445 }
 446
 447 struct journal_read_buf {
 448         void            *data;
 449         size_t          size;
 450 };
 451
 452 static int journal_read_buf_realloc(struct journal_read_buf *b,
 453                                     size_t new_size)
 454 {
 455         void *n;
 456
 457         /* the bios are sized for this many pages, max: */
 458         if (new_size > JOURNAL_ENTRY_SIZE_MAX)
 459                 return -ENOMEM;
 460
 461         new_size = roundup_pow_of_two(new_size);
 462         n = kvpmalloc(new_size, GFP_KERNEL);
 463         if (!n)
 464                 return -ENOMEM;
 465
 466         kvpfree(b->data, b->size);
 467         b->data = n;
 468         b->size = new_size;
 469         return 0;
 470 }
 471
 472 static int journal_read_bucket(struct bch_dev *ca,
 473                                struct journal_read_buf *buf,
 474                                struct journal_list *jlist,
 475                                unsigned bucket)
 476 {
 477         struct bch_fs *c = ca->fs;
 478         struct journal_device *ja = &ca->journal;
 479         struct jset *j = NULL;
 480         unsigned sectors, sectors_read = 0;
 481         u64 offset = bucket_to_sector(ca, ja->buckets[bucket]),
 482             end = offset + ca->mi.bucket_size;
 483         bool saw_bad = false;
 484         int ret = 0;
 485
 486         pr_debug("reading %u", bucket);
 487
 488         while (offset < end) {
 489                 if (!sectors_read) {
 490                         struct bio *bio;
 491                         unsigned nr_bvecs;
 492 reread:
 493                         sectors_read = min_t(unsigned,
 494                                 end - offset, buf->size >> 9);
 495                         nr_bvecs = buf_pages(buf->data, sectors_read << 9);
 496
 497                         bio = bio_kmalloc(nr_bvecs, GFP_KERNEL);
 498                         bio_init(bio, ca->disk_sb.bdev, bio->bi_inline_vecs, nr_bvecs, REQ_OP_READ);
 499
 500                         bio->bi_iter.bi_sector  = offset;
 501                         bio->bi_iter.bi_size    = sectors_read << 9;
 502                         bch2_bio_map(bio, buf->data);
 503
 504                         ret = submit_bio_wait(bio);
 505                         kfree(bio);
 506
 507                         if (bch2_dev_io_err_on(ret, ca,
 508                                                "journal read from sector %llu",
 509                                                offset) ||
 510                             bch2_meta_read_fault("journal"))
 511                                 return -EIO;
 512
 513                         j = buf->data;
 514                 }
 515
 516                 ret = jset_validate(c, j, offset,
 517                                     end - offset, sectors_read,
 518                                     READ);
 519                 switch (ret) {
 520                 case BCH_FSCK_OK:
 521                         break;
 522                 case JOURNAL_ENTRY_REREAD:
 523                         if (vstruct_bytes(j) > buf->size) {
 524                                 ret = journal_read_buf_realloc(buf,
 525                                                         vstruct_bytes(j));
 526                                 if (ret)
 527                                         return ret;
 528                         }
 529                         goto reread;
 530                 case JOURNAL_ENTRY_NONE:
 531                         if (!saw_bad)
 532                                 return 0;
 533                         sectors = c->opts.block_size;
 534                         goto next_block;
 535                 case JOURNAL_ENTRY_BAD:
 536                         saw_bad = true;
 537                         sectors = c->opts.block_size;
 538                         goto next_block;
 539                 default:
 540                         return ret;
 541                 }
 542
 543                 /*
 544                  * This happens sometimes if we don't have discards on -
 545                  * when we've partially overwritten a bucket with new
 546                  * journal entries. We don't need the rest of the
 547                  * bucket:
 548                  */
 549                 if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket])
 550                         return 0;
 551
 552                 ja->bucket_seq[bucket] = le64_to_cpu(j->seq);
 553
 554                 mutex_lock(&jlist->lock);
 555                 ret = journal_entry_add(c, ca, jlist, j);
 556                 mutex_unlock(&jlist->lock);
 557
 558                 switch (ret) {
 559                 case JOURNAL_ENTRY_ADD_OK:
 560                         break;
 561                 case JOURNAL_ENTRY_ADD_OUT_OF_RANGE:
 562                         break;
 563                 default:
 564                         return ret;
 565                 }
 566
 567                 sectors = vstruct_sectors(j, c->block_bits);
 568 next_block:
 569                 pr_debug("next");
 570                 offset          += sectors;
 571                 sectors_read    -= sectors;
 572                 j = ((void *) j) + (sectors << 9);
 573         }
 574
 575         return 0;
 576 }
 577
 578 static void bch2_journal_read_device(struct closure *cl)
 579 {
 580         struct journal_device *ja =
 581                 container_of(cl, struct journal_device, read);
 582         struct bch_dev *ca = container_of(ja, struct bch_dev, journal);
 583         struct journal_list *jlist =
 584                 container_of(cl->parent, struct journal_list, cl);
 585         struct journal_read_buf buf = { NULL, 0 };
 586         u64 min_seq = U64_MAX;
 587         unsigned i;
 588         int ret;
 589
 590         if (!ja->nr)
 591                 goto out;
 592
 593         ret = journal_read_buf_realloc(&buf, PAGE_SIZE);
 594         if (ret)
 595                 goto err;
 596
 597         pr_debug("%u journal buckets", ja->nr);
 598
 599         for (i = 0; i < ja->nr; i++) {
 600                 ret = journal_read_bucket(ca, &buf, jlist, i);
 601                 if (ret)
 602                         goto err;
 603         }
 604
 605         /* Find the journal bucket with the highest sequence number: */
 606         for (i = 0; i < ja->nr; i++) {
 607                 if (ja->bucket_seq[i] > ja->bucket_seq[ja->cur_idx])
 608                         ja->cur_idx = i;
 609
 610                 min_seq = min(ja->bucket_seq[i], min_seq);
 611         }
 612
 613         /*
 614          * If there's duplicate journal entries in multiple buckets (which
 615          * definitely isn't supposed to happen, but...) - make sure to start
 616          * cur_idx at the last of those buckets, so we don't deadlock trying to
 617          * allocate
 618          */
 619         while (ja->bucket_seq[ja->cur_idx] > min_seq &&
 620                ja->bucket_seq[ja->cur_idx] >
 621                ja->bucket_seq[(ja->cur_idx + 1) % ja->nr])
 622                 ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
 623
 624         ja->sectors_free = 0;
 625
 626         /*
 627          * Set dirty_idx to indicate the entire journal is full and needs to be
 628          * reclaimed - journal reclaim will immediately reclaim whatever isn't
 629          * pinned when it first runs:
 630          */
 631         ja->discard_idx = ja->dirty_idx_ondisk =
 632                 ja->dirty_idx = (ja->cur_idx + 1) % ja->nr;
 633 out:
 634         kvpfree(buf.data, buf.size);
 635         percpu_ref_put(&ca->io_ref);
 636         closure_return(cl);
 637         return;
 638 err:
 639         mutex_lock(&jlist->lock);
 640         jlist->ret = ret;
 641         mutex_unlock(&jlist->lock);
 642         goto out;
 643 }
 644
 645 void bch2_journal_entries_free(struct list_head *list)
 646 {
 647
 648         while (!list_empty(list)) {
 649                 struct journal_replay *i =
 650                         list_first_entry(list, struct journal_replay, list);
 651                 list_del(&i->list);
 652                 kvpfree(i, offsetof(struct journal_replay, j) +
 653                         vstruct_bytes(&i->j));
 654         }
 655 }
 656
 657 int bch2_journal_read(struct bch_fs *c, struct list_head *list)
 658 {
 659         struct journal_list jlist;
 660         struct journal_replay *i;
 661         struct bch_dev *ca;
 662         unsigned iter;
 663         size_t keys = 0, entries = 0;
 664         bool degraded = false;
 665         int ret = 0;
 666
 667         closure_init_stack(&jlist.cl);
 668         mutex_init(&jlist.lock);
 669         jlist.head = list;
 670         jlist.ret = 0;
 671
 672         for_each_member_device(ca, c, iter) {
 673                 if (!test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) &&
 674                     !(bch2_dev_has_data(c, ca) & (1 << BCH_DATA_JOURNAL)))
 675                         continue;
 676
 677                 if ((ca->mi.state == BCH_MEMBER_STATE_RW ||
 678                      ca->mi.state == BCH_MEMBER_STATE_RO) &&
 679                     percpu_ref_tryget(&ca->io_ref))
 680                         closure_call(&ca->journal.read,
 681                                      bch2_journal_read_device,
 682                                      system_unbound_wq,
 683                                      &jlist.cl);
 684                 else
 685                         degraded = true;
 686         }
 687
 688         closure_sync(&jlist.cl);
 689
 690         if (jlist.ret)
 691                 return jlist.ret;
 692
 693         list_for_each_entry(i, list, list) {
 694                 struct jset_entry *entry;
 695                 struct bkey_i *k, *_n;
 696                 struct bch_replicas_padded replicas;
 697                 char buf[80];
 698
 699                 ret = jset_validate_entries(c, &i->j, READ);
 700                 if (ret)
 701                         goto fsck_err;
 702
 703                 /*
 704                  * If we're mounting in degraded mode - if we didn't read all
 705                  * the devices - this is wrong:
 706                  */
 707
 708                 bch2_devlist_to_replicas(&replicas.e, BCH_DATA_JOURNAL, i->devs);
 709
 710                 if (!degraded &&
 711                     (test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) ||
 712                      fsck_err_on(!bch2_replicas_marked(c, &replicas.e, false), c,
 713                                  "superblock not marked as containing replicas %s",
 714                                  (bch2_replicas_entry_to_text(&PBUF(buf),
 715                                                               &replicas.e), buf)))) {
 716                         ret = bch2_mark_replicas(c, &replicas.e);
 717                         if (ret)
 718                                 return ret;
 719                 }
 720
 721                 for_each_jset_key(k, _n, entry, &i->j)
 722                         keys++;
 723                 entries++;
 724         }
 725
 726         if (!list_empty(list)) {
 727                 i = list_last_entry(list, struct journal_replay, list);
 728
 729                 bch_info(c, "journal read done, %zu keys in %zu entries, seq %llu",
 730                          keys, entries, le64_to_cpu(i->j.seq));
 731         }
 732 fsck_err:
 733         return ret;
 734 }
 735
 736 /* journal replay: */
 737
 738 static int bch2_extent_replay_key(struct bch_fs *c, struct bkey_i *k)
 739 {
 740         struct btree_trans trans;
 741         struct btree_iter *iter;
 742         /*
 743          * We might cause compressed extents to be
 744          * split, so we need to pass in a
 745          * disk_reservation:
 746          */
 747         struct disk_reservation disk_res =
 748                 bch2_disk_reservation_init(c, 0);
 749         BKEY_PADDED(k) split;
 750         int ret;
 751
 752         bch2_trans_init(&trans, c);
 753
 754         iter = bch2_trans_get_iter(&trans, BTREE_ID_EXTENTS,
 755                                    bkey_start_pos(&k->k),
 756                                    BTREE_ITER_INTENT);
 757         do {
 758                 ret = bch2_btree_iter_traverse(iter);
 759                 if (ret)
 760                         break;
 761
 762                 bkey_copy(&split.k, k);
 763                 bch2_cut_front(iter->pos, &split.k);
 764                 bch2_extent_trim_atomic(&split.k, iter);
 765
 766                 ret = bch2_disk_reservation_add(c, &disk_res,
 767                                 split.k.k.size *
 768                                 bch2_bkey_nr_dirty_ptrs(bkey_i_to_s_c(&split.k)),
 769                                 BCH_DISK_RESERVATION_NOFAIL);
 770                 BUG_ON(ret);
 771
 772                 bch2_trans_update(&trans, BTREE_INSERT_ENTRY(iter, &split.k));
 773                 ret = bch2_trans_commit(&trans, &disk_res, NULL,
 774                                         BTREE_INSERT_ATOMIC|
 775                                         BTREE_INSERT_NOFAIL|
 776                                         BTREE_INSERT_LAZY_RW|
 777                                         BTREE_INSERT_JOURNAL_REPLAY);
 778         } while ((!ret || ret == -EINTR) &&
 779                  bkey_cmp(k->k.p, iter->pos));
 780
 781         bch2_disk_reservation_put(c, &disk_res);
 782
 783         /*
 784          * This isn't strictly correct - we should only be relying on the btree
 785          * node lock for synchronization with gc when we've got a write lock
 786          * held.
 787          *
 788          * but - there are other correctness issues if btree gc were to run
 789          * before journal replay finishes
 790          */
 791         BUG_ON(c->gc_pos.phase);
 792
 793         bch2_mark_key(c, bkey_i_to_s_c(k), false, -((s64) k->k.size),
 794                       NULL, 0, 0);
 795         bch2_trans_exit(&trans);
 796
 797         return ret;
 798 }
 799
 800 int bch2_journal_replay(struct bch_fs *c, struct list_head *list)
 801 {
 802         struct journal *j = &c->journal;
 803         struct bkey_i *k, *_n;
 804         struct jset_entry *entry;
 805         struct journal_replay *i, *n;
 806         int ret = 0;
 807
 808         list_for_each_entry_safe(i, n, list, list) {
 809                 j->replay_journal_seq = le64_to_cpu(i->j.seq);
 810
 811                 for_each_jset_key(k, _n, entry, &i->j) {
 812                         switch (entry->btree_id) {
 813                         case BTREE_ID_ALLOC:
 814                                 ret = bch2_alloc_replay_key(c, k);
 815                                 break;
 816                         case BTREE_ID_EXTENTS:
 817                                 ret = bch2_extent_replay_key(c, k);
 818                                 break;
 819                         default:
 820                                 ret = bch2_btree_insert(c, entry->btree_id, k,
 821                                                 NULL, NULL,
 822                                                 BTREE_INSERT_NOFAIL|
 823                                                 BTREE_INSERT_LAZY_RW|
 824                                                 BTREE_INSERT_JOURNAL_REPLAY|
 825                                                 BTREE_INSERT_NOMARK);
 826                                 break;
 827                         }
 828
 829                         if (ret) {
 830                                 bch_err(c, "journal replay: error %d while replaying key",
 831                                         ret);
 832                                 goto err;
 833                         }
 834
 835                         cond_resched();
 836                 }
 837
 838                 bch2_journal_pin_put(j, j->replay_journal_seq);
 839         }
 840
 841         j->replay_journal_seq = 0;
 842
 843         bch2_journal_set_replay_done(j);
 844         bch2_journal_flush_all_pins(j);
 845         ret = bch2_journal_error(j);
 846 err:
 847         bch2_journal_entries_free(list);
 848         return ret;
 849 }
 850
 851 /* journal write: */
 852
 853 static void __journal_write_alloc(struct journal *j,
 854                                   struct journal_buf *w,
 855                                   struct dev_alloc_list *devs_sorted,
 856                                   unsigned sectors,
 857                                   unsigned *replicas,
 858                                   unsigned replicas_want)
 859 {
 860         struct bch_fs *c = container_of(j, struct bch_fs, journal);
 861         struct journal_device *ja;
 862         struct bch_dev *ca;
 863         unsigned i;
 864
 865         if (*replicas >= replicas_want)
 866                 return;
 867
 868         for (i = 0; i < devs_sorted->nr; i++) {
 869                 ca = rcu_dereference(c->devs[devs_sorted->devs[i]]);
 870                 if (!ca)
 871                         continue;
 872
 873                 ja = &ca->journal;
 874
 875                 /*
 876                  * Check that we can use this device, and aren't already using
 877                  * it:
 878                  */
 879                 if (!ca->mi.durability ||
 880                     ca->mi.state != BCH_MEMBER_STATE_RW ||
 881                     !ja->nr ||
 882                     bch2_bkey_has_device(bkey_i_to_s_c(&w->key),
 883                                          ca->dev_idx) ||
 884                     sectors > ja->sectors_free)
 885                         continue;
 886
 887                 bch2_dev_stripe_increment(c, ca, &j->wp.stripe);
 888
 889                 bch2_bkey_append_ptr(&w->key,
 890                         (struct bch_extent_ptr) {
 891                                   .offset = bucket_to_sector(ca,
 892                                         ja->buckets[ja->cur_idx]) +
 893                                         ca->mi.bucket_size -
 894                                         ja->sectors_free,
 895                                   .dev = ca->dev_idx,
 896                 });
 897
 898                 ja->sectors_free -= sectors;
 899                 ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq);
 900
 901                 *replicas += ca->mi.durability;
 902
 903                 if (*replicas >= replicas_want)
 904                         break;
 905         }
 906 }
 907
 908 /**
 909  * journal_next_bucket - move on to the next journal bucket if possible
 910  */
 911 static int journal_write_alloc(struct journal *j, struct journal_buf *w,
 912                                unsigned sectors)
 913 {
 914         struct bch_fs *c = container_of(j, struct bch_fs, journal);
 915         struct journal_device *ja;
 916         struct bch_dev *ca;
 917         struct dev_alloc_list devs_sorted;
 918         unsigned i, replicas = 0, replicas_want =
 919                 READ_ONCE(c->opts.metadata_replicas);
 920
 921         rcu_read_lock();
 922
 923         devs_sorted = bch2_dev_alloc_list(c, &j->wp.stripe,
 924                                           &c->rw_devs[BCH_DATA_JOURNAL]);
 925
 926         __journal_write_alloc(j, w, &devs_sorted,
 927                               sectors, &replicas, replicas_want);
 928
 929         if (replicas >= replicas_want)
 930                 goto done;
 931
 932         for (i = 0; i < devs_sorted.nr; i++) {
 933                 ca = rcu_dereference(c->devs[devs_sorted.devs[i]]);
 934                 if (!ca)
 935                         continue;
 936
 937                 ja = &ca->journal;
 938
 939                 if (sectors > ja->sectors_free &&
 940                     sectors <= ca->mi.bucket_size &&
 941                     bch2_journal_dev_buckets_available(j, ja,
 942                                         journal_space_discarded)) {
 943                         ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
 944                         ja->sectors_free = ca->mi.bucket_size;
 945
 946                         /*
 947                          * ja->bucket_seq[ja->cur_idx] must always have
 948                          * something sensible:
 949                          */
 950                         ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq);
 951                 }
 952         }
 953
 954         __journal_write_alloc(j, w, &devs_sorted,
 955                               sectors, &replicas, replicas_want);
 956 done:
 957         rcu_read_unlock();
 958
 959         return replicas >= c->opts.metadata_replicas_required ? 0 : -EROFS;
 960 }
 961
 962 static void journal_write_compact(struct jset *jset)
 963 {
 964         struct jset_entry *i, *next, *prev = NULL;
 965
 966         /*
 967          * Simple compaction, dropping empty jset_entries (from journal
 968          * reservations that weren't fully used) and merging jset_entries that
 969          * can be.
 970          *
 971          * If we wanted to be really fancy here, we could sort all the keys in
 972          * the jset and drop keys that were overwritten - probably not worth it:
 973          */
 974         vstruct_for_each_safe(jset, i, next) {
 975                 unsigned u64s = le16_to_cpu(i->u64s);
 976
 977                 /* Empty entry: */
 978                 if (!u64s)
 979                         continue;
 980
 981                 /* Can we merge with previous entry? */
 982                 if (prev &&
 983                     i->btree_id == prev->btree_id &&
 984                     i->level    == prev->level &&
 985                     i->type     == prev->type &&
 986                     i->type     == BCH_JSET_ENTRY_btree_keys &&
 987                     le16_to_cpu(prev->u64s) + u64s <= U16_MAX) {
 988                         memmove_u64s_down(vstruct_next(prev),
 989                                           i->_data,
 990                                           u64s);
 991                         le16_add_cpu(&prev->u64s, u64s);
 992                         continue;
 993                 }
 994
 995                 /* Couldn't merge, move i into new position (after prev): */
 996                 prev = prev ? vstruct_next(prev) : jset->start;
 997                 if (i != prev)
 998                         memmove_u64s_down(prev, i, jset_u64s(u64s));
 999         }
1000
1001         prev = prev ? vstruct_next(prev) : jset->start;
1002         jset->u64s = cpu_to_le32((u64 *) prev - jset->_data);
1003 }
1004
1005 static void journal_buf_realloc(struct journal *j, struct journal_buf *buf)
1006 {
1007         /* we aren't holding j->lock: */
1008         unsigned new_size = READ_ONCE(j->buf_size_want);
1009         void *new_buf;
1010
1011         if (buf->buf_size >= new_size)
1012                 return;
1013
1014         new_buf = kvpmalloc(new_size, GFP_NOIO|__GFP_NOWARN);
1015         if (!new_buf)
1016                 return;
1017
1018         memcpy(new_buf, buf->data, buf->buf_size);
1019         kvpfree(buf->data, buf->buf_size);
1020         buf->data       = new_buf;
1021         buf->buf_size   = new_size;
1022 }
1023
1024 static void journal_write_done(struct closure *cl)
1025 {
1026         struct journal *j = container_of(cl, struct journal, io);
1027         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1028         struct journal_buf *w = journal_prev_buf(j);
1029         struct bch_devs_list devs =
1030                 bch2_bkey_devs(bkey_i_to_s_c(&w->key));
1031         struct bch_replicas_padded replicas;
1032         u64 seq = le64_to_cpu(w->data->seq);
1033         u64 last_seq = le64_to_cpu(w->data->last_seq);
1034
1035         bch2_time_stats_update(j->write_time, j->write_start_time);
1036
1037         if (!devs.nr) {
1038                 bch_err(c, "unable to write journal to sufficient devices");
1039                 goto err;
1040         }
1041
1042         bch2_devlist_to_replicas(&replicas.e, BCH_DATA_JOURNAL, devs);
1043
1044         if (bch2_mark_replicas(c, &replicas.e))
1045                 goto err;
1046
1047         spin_lock(&j->lock);
1048         if (seq >= j->pin.front)
1049                 journal_seq_pin(j, seq)->devs = devs;
1050
1051         j->seq_ondisk           = seq;
1052         j->last_seq_ondisk      = last_seq;
1053         bch2_journal_space_available(j);
1054
1055         /*
1056          * Updating last_seq_ondisk may let bch2_journal_reclaim_work() discard
1057          * more buckets:
1058          *
1059          * Must come before signaling write completion, for
1060          * bch2_fs_journal_stop():
1061          */
1062         mod_delayed_work(c->journal_reclaim_wq, &j->reclaim_work, 0);
1063 out:
1064         /* also must come before signalling write completion: */
1065         closure_debug_destroy(cl);
1066
1067         BUG_ON(!j->reservations.prev_buf_unwritten);
1068         atomic64_sub(((union journal_res_state) { .prev_buf_unwritten = 1 }).v,
1069                      &j->reservations.counter);
1070
1071         closure_wake_up(&w->wait);
1072         journal_wake(j);
1073
1074         if (test_bit(JOURNAL_NEED_WRITE, &j->flags))
1075                 mod_delayed_work(system_freezable_wq, &j->write_work, 0);
1076         spin_unlock(&j->lock);
1077         return;
1078 err:
1079         bch2_fatal_error(c);
1080         bch2_journal_halt(j);
1081         spin_lock(&j->lock);
1082         goto out;
1083 }
1084
1085 static void journal_write_endio(struct bio *bio)
1086 {
1087         struct bch_dev *ca = bio->bi_private;
1088         struct journal *j = &ca->fs->journal;
1089
1090         if (bch2_dev_io_err_on(bio->bi_status, ca, "journal write") ||
1091             bch2_meta_write_fault("journal")) {
1092                 struct journal_buf *w = journal_prev_buf(j);
1093                 unsigned long flags;
1094
1095                 spin_lock_irqsave(&j->err_lock, flags);
1096                 bch2_bkey_drop_device(bkey_i_to_s(&w->key), ca->dev_idx);
1097                 spin_unlock_irqrestore(&j->err_lock, flags);
1098         }
1099
1100         closure_put(&j->io);
1101         percpu_ref_put(&ca->io_ref);
1102 }
1103
1104 void bch2_journal_write(struct closure *cl)
1105 {
1106         struct journal *j = container_of(cl, struct journal, io);
1107         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1108         struct bch_dev *ca;
1109         struct journal_buf *w = journal_prev_buf(j);
1110         struct jset_entry *start, *end;
1111         struct jset *jset;
1112         struct bio *bio;
1113         struct bch_extent_ptr *ptr;
1114         bool validate_before_checksum = false;
1115         unsigned i, sectors, bytes, u64s;
1116         int ret;
1117
1118         bch2_journal_pin_put(j, le64_to_cpu(w->data->seq));
1119
1120         journal_buf_realloc(j, w);
1121         jset = w->data;
1122
1123         j->write_start_time = local_clock();
1124
1125         start   = vstruct_last(jset);
1126         end     = bch2_journal_super_entries_add_common(c, start,
1127                                                 le64_to_cpu(jset->seq));
1128         u64s    = (u64 *) end - (u64 *) start;
1129         BUG_ON(u64s > j->entry_u64s_reserved);
1130
1131         le32_add_cpu(&jset->u64s, u64s);
1132         BUG_ON(vstruct_sectors(jset, c->block_bits) > w->sectors);
1133
1134         journal_write_compact(jset);
1135
1136         jset->read_clock        = cpu_to_le16(c->bucket_clock[READ].hand);
1137         jset->write_clock       = cpu_to_le16(c->bucket_clock[WRITE].hand);
1138         jset->magic             = cpu_to_le64(jset_magic(c));
1139
1140         jset->version           = c->sb.version < bcachefs_metadata_version_new_versioning
1141                 ? cpu_to_le32(BCH_JSET_VERSION_OLD)
1142                 : cpu_to_le32(c->sb.version);
1143
1144         SET_JSET_BIG_ENDIAN(jset, CPU_BIG_ENDIAN);
1145         SET_JSET_CSUM_TYPE(jset, bch2_meta_checksum_type(c));
1146
1147         if (bch2_csum_type_is_encryption(JSET_CSUM_TYPE(jset)))
1148                 validate_before_checksum = true;
1149
1150         if (le32_to_cpu(jset->version) <
1151             bcachefs_metadata_version_bkey_renumber)
1152                 validate_before_checksum = true;
1153
1154         if (validate_before_checksum &&
1155             jset_validate_entries(c, jset, WRITE))
1156                 goto err;
1157
1158         bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset),
1159                     jset->encrypted_start,
1160                     vstruct_end(jset) - (void *) jset->encrypted_start);
1161
1162         jset->csum = csum_vstruct(c, JSET_CSUM_TYPE(jset),
1163                                   journal_nonce(jset), jset);
1164
1165         if (!validate_before_checksum &&
1166             jset_validate_entries(c, jset, WRITE))
1167                 goto err;
1168
1169         sectors = vstruct_sectors(jset, c->block_bits);
1170         BUG_ON(sectors > w->sectors);
1171
1172         bytes = vstruct_bytes(jset);
1173         memset((void *) jset + bytes, 0, (sectors << 9) - bytes);
1174
1175         spin_lock(&j->lock);
1176         ret = journal_write_alloc(j, w, sectors);
1177
1178         /*
1179          * write is allocated, no longer need to account for it in
1180          * bch2_journal_space_available():
1181          */
1182         w->sectors = 0;
1183
1184         /*
1185          * journal entry has been compacted and allocated, recalculate space
1186          * available:
1187          */
1188         bch2_journal_space_available(j);
1189         spin_unlock(&j->lock);
1190
1191         if (ret) {
1192                 bch2_journal_halt(j);
1193                 bch_err(c, "Unable to allocate journal write");
1194                 bch2_fatal_error(c);
1195                 continue_at(cl, journal_write_done, system_highpri_wq);
1196                 return;
1197         }
1198
1199         /*
1200          * XXX: we really should just disable the entire journal in nochanges
1201          * mode
1202          */
1203         if (c->opts.nochanges)
1204                 goto no_io;
1205
1206         extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) {
1207                 ca = bch_dev_bkey_exists(c, ptr->dev);
1208                 if (!percpu_ref_tryget(&ca->io_ref)) {
1209                         /* XXX: fix this */
1210                         bch_err(c, "missing device for journal write\n");
1211                         continue;
1212                 }
1213
1214                 this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_JOURNAL],
1215                              sectors);
1216
1217                 bio = ca->journal.bio;
1218                 bio_reset(bio, ca->disk_sb.bdev,
1219                           REQ_OP_WRITE|REQ_SYNC|REQ_META|REQ_PREFLUSH|REQ_FUA);
1220                 bio->bi_iter.bi_sector  = ptr->offset;
1221                 bio->bi_iter.bi_size    = sectors << 9;
1222                 bio->bi_end_io          = journal_write_endio;
1223                 bio->bi_private         = ca;
1224                 bch2_bio_map(bio, jset);
1225
1226                 trace_journal_write(bio);
1227                 closure_bio_submit(bio, cl);
1228
1229                 ca->journal.bucket_seq[ca->journal.cur_idx] = le64_to_cpu(jset->seq);
1230         }
1231
1232         for_each_rw_member(ca, c, i)
1233                 if (journal_flushes_device(ca) &&
1234                     !bch2_extent_has_device(bkey_i_to_s_c_extent(&w->key), i)) {
1235                         percpu_ref_get(&ca->io_ref);
1236
1237                         bio = ca->journal.bio;
1238                         bio_reset(bio, ca->disk_sb.bdev, REQ_OP_FLUSH);
1239                         bio->bi_end_io          = journal_write_endio;
1240                         bio->bi_private         = ca;
1241                         closure_bio_submit(bio, cl);
1242                 }
1243
1244 no_io:
1245         bch2_bucket_seq_cleanup(c);
1246
1247         continue_at(cl, journal_write_done, system_highpri_wq);
1248         return;
1249 err:
1250         bch2_inconsistent_error(c);
1251         continue_at(cl, journal_write_done, system_highpri_wq);
1252 }