fs/btrfs/disk-io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/fs.h>
   7 #include <linux/blkdev.h>
   8 #include <linux/radix-tree.h>
   9 #include <linux/writeback.h>
  10 #include <linux/workqueue.h>
  11 #include <linux/kthread.h>
  12 #include <linux/slab.h>
  13 #include <linux/migrate.h>
  14 #include <linux/ratelimit.h>
  15 #include <linux/uuid.h>
  16 #include <linux/semaphore.h>
  17 #include <linux/error-injection.h>
  18 #include <linux/crc32c.h>
  19 #include <linux/sched/mm.h>
  20 #include <asm/unaligned.h>
  21 #include <crypto/hash.h>
  22 #include "ctree.h"
  23 #include "disk-io.h"
  24 #include "transaction.h"
  25 #include "btrfs_inode.h"
  26 #include "bio.h"
  27 #include "print-tree.h"
  28 #include "locking.h"
  29 #include "tree-log.h"
  30 #include "free-space-cache.h"
  31 #include "free-space-tree.h"
  32 #include "check-integrity.h"
  33 #include "rcu-string.h"
  34 #include "dev-replace.h"
  35 #include "raid56.h"
  36 #include "sysfs.h"
  37 #include "qgroup.h"
  38 #include "compression.h"
  39 #include "tree-checker.h"
  40 #include "ref-verify.h"
  41 #include "block-group.h"
  42 #include "discard.h"
  43 #include "space-info.h"
  44 #include "zoned.h"
  45 #include "subpage.h"
  46 #include "fs.h"
  47 #include "accessors.h"
  48 #include "extent-tree.h"
  49 #include "root-tree.h"
  50 #include "defrag.h"
  51 #include "uuid-tree.h"
  52 #include "relocation.h"
  53 #include "scrub.h"
  54 #include "super.h"
  55
  56 #define BTRFS_SUPER_FLAG_SUPP   (BTRFS_HEADER_FLAG_WRITTEN |\
  57                                  BTRFS_HEADER_FLAG_RELOC |\
  58                                  BTRFS_SUPER_FLAG_ERROR |\
  59                                  BTRFS_SUPER_FLAG_SEEDING |\
  60                                  BTRFS_SUPER_FLAG_METADUMP |\
  61                                  BTRFS_SUPER_FLAG_METADUMP_V2)
  62
  63 static void btrfs_destroy_ordered_extents(struct btrfs_root *root);
  64 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
  65                                       struct btrfs_fs_info *fs_info);
  66 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root);
  67 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
  68                                         struct extent_io_tree *dirty_pages,
  69                                         int mark);
  70 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
  71                                        struct extent_io_tree *pinned_extents);
  72 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info);
  73 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info);
  74
  75 static void btrfs_free_csum_hash(struct btrfs_fs_info *fs_info)
  76 {
  77         if (fs_info->csum_shash)
  78                 crypto_free_shash(fs_info->csum_shash);
  79 }
  80
  81 /*
  82  * Compute the csum of a btree block and store the result to provided buffer.
  83  */
  84 static void csum_tree_block(struct extent_buffer *buf, u8 *result)
  85 {
  86         struct btrfs_fs_info *fs_info = buf->fs_info;
  87         const int num_pages = num_extent_pages(buf);
  88         const int first_page_part = min_t(u32, PAGE_SIZE, fs_info->nodesize);
  89         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
  90         char *kaddr;
  91         int i;
  92
  93         shash->tfm = fs_info->csum_shash;
  94         crypto_shash_init(shash);
  95         kaddr = page_address(buf->pages[0]) + offset_in_page(buf->start);
  96         crypto_shash_update(shash, kaddr + BTRFS_CSUM_SIZE,
  97                             first_page_part - BTRFS_CSUM_SIZE);
  98
  99         for (i = 1; i < num_pages; i++) {
 100                 kaddr = page_address(buf->pages[i]);
 101                 crypto_shash_update(shash, kaddr, PAGE_SIZE);
 102         }
 103         memset(result, 0, BTRFS_CSUM_SIZE);
 104         crypto_shash_final(shash, result);
 105 }
 106
 107 /*
 108  * we can't consider a given block up to date unless the transid of the
 109  * block matches the transid in the parent node's pointer.  This is how we
 110  * detect blocks that either didn't get written at all or got written
 111  * in the wrong place.
 112  */
 113 static int verify_parent_transid(struct extent_io_tree *io_tree,
 114                                  struct extent_buffer *eb, u64 parent_transid,
 115                                  int atomic)
 116 {
 117         struct extent_state *cached_state = NULL;
 118         int ret;
 119
 120         if (!parent_transid || btrfs_header_generation(eb) == parent_transid)
 121                 return 0;
 122
 123         if (atomic)
 124                 return -EAGAIN;
 125
 126         lock_extent(io_tree, eb->start, eb->start + eb->len - 1, &cached_state);
 127         if (extent_buffer_uptodate(eb) &&
 128             btrfs_header_generation(eb) == parent_transid) {
 129                 ret = 0;
 130                 goto out;
 131         }
 132         btrfs_err_rl(eb->fs_info,
 133 "parent transid verify failed on logical %llu mirror %u wanted %llu found %llu",
 134                         eb->start, eb->read_mirror,
 135                         parent_transid, btrfs_header_generation(eb));
 136         ret = 1;
 137         clear_extent_buffer_uptodate(eb);
 138 out:
 139         unlock_extent(io_tree, eb->start, eb->start + eb->len - 1,
 140                       &cached_state);
 141         return ret;
 142 }
 143
 144 static bool btrfs_supported_super_csum(u16 csum_type)
 145 {
 146         switch (csum_type) {
 147         case BTRFS_CSUM_TYPE_CRC32:
 148         case BTRFS_CSUM_TYPE_XXHASH:
 149         case BTRFS_CSUM_TYPE_SHA256:
 150         case BTRFS_CSUM_TYPE_BLAKE2:
 151                 return true;
 152         default:
 153                 return false;
 154         }
 155 }
 156
 157 /*
 158  * Return 0 if the superblock checksum type matches the checksum value of that
 159  * algorithm. Pass the raw disk superblock data.
 160  */
 161 int btrfs_check_super_csum(struct btrfs_fs_info *fs_info,
 162                            const struct btrfs_super_block *disk_sb)
 163 {
 164         char result[BTRFS_CSUM_SIZE];
 165         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
 166
 167         shash->tfm = fs_info->csum_shash;
 168
 169         /*
 170          * The super_block structure does not span the whole
 171          * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is
 172          * filled with zeros and is included in the checksum.
 173          */
 174         crypto_shash_digest(shash, (const u8 *)disk_sb + BTRFS_CSUM_SIZE,
 175                             BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result);
 176
 177         if (memcmp(disk_sb->csum, result, fs_info->csum_size))
 178                 return 1;
 179
 180         return 0;
 181 }
 182
 183 int btrfs_verify_level_key(struct extent_buffer *eb, int level,
 184                            struct btrfs_key *first_key, u64 parent_transid)
 185 {
 186         struct btrfs_fs_info *fs_info = eb->fs_info;
 187         int found_level;
 188         struct btrfs_key found_key;
 189         int ret;
 190
 191         found_level = btrfs_header_level(eb);
 192         if (found_level != level) {
 193                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 194                      KERN_ERR "BTRFS: tree level check failed\n");
 195                 btrfs_err(fs_info,
 196 "tree level mismatch detected, bytenr=%llu level expected=%u has=%u",
 197                           eb->start, level, found_level);
 198                 return -EIO;
 199         }
 200
 201         if (!first_key)
 202                 return 0;
 203
 204         /*
 205          * For live tree block (new tree blocks in current transaction),
 206          * we need proper lock context to avoid race, which is impossible here.
 207          * So we only checks tree blocks which is read from disk, whose
 208          * generation <= fs_info->last_trans_committed.
 209          */
 210         if (btrfs_header_generation(eb) > fs_info->last_trans_committed)
 211                 return 0;
 212
 213         /* We have @first_key, so this @eb must have at least one item */
 214         if (btrfs_header_nritems(eb) == 0) {
 215                 btrfs_err(fs_info,
 216                 "invalid tree nritems, bytenr=%llu nritems=0 expect >0",
 217                           eb->start);
 218                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
 219                 return -EUCLEAN;
 220         }
 221
 222         if (found_level)
 223                 btrfs_node_key_to_cpu(eb, &found_key, 0);
 224         else
 225                 btrfs_item_key_to_cpu(eb, &found_key, 0);
 226         ret = btrfs_comp_cpu_keys(first_key, &found_key);
 227
 228         if (ret) {
 229                 WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
 230                      KERN_ERR "BTRFS: tree first key check failed\n");
 231                 btrfs_err(fs_info,
 232 "tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)",
 233                           eb->start, parent_transid, first_key->objectid,
 234                           first_key->type, first_key->offset,
 235                           found_key.objectid, found_key.type,
 236                           found_key.offset);
 237         }
 238         return ret;
 239 }
 240
 241 static int btrfs_repair_eb_io_failure(const struct extent_buffer *eb,
 242                                       int mirror_num)
 243 {
 244         struct btrfs_fs_info *fs_info = eb->fs_info;
 245         u64 start = eb->start;
 246         int i, num_pages = num_extent_pages(eb);
 247         int ret = 0;
 248
 249         if (sb_rdonly(fs_info->sb))
 250                 return -EROFS;
 251
 252         for (i = 0; i < num_pages; i++) {
 253                 struct page *p = eb->pages[i];
 254
 255                 ret = btrfs_repair_io_failure(fs_info, 0, start, PAGE_SIZE,
 256                                 start, p, start - page_offset(p), mirror_num);
 257                 if (ret)
 258                         break;
 259                 start += PAGE_SIZE;
 260         }
 261
 262         return ret;
 263 }
 264
 265 /*
 266  * helper to read a given tree block, doing retries as required when
 267  * the checksums don't match and we have alternate mirrors to try.
 268  *
 269  * @check:              expected tree parentness check, see the comments of the
 270  *                      structure for details.
 271  */
 272 int btrfs_read_extent_buffer(struct extent_buffer *eb,
 273                              struct btrfs_tree_parent_check *check)
 274 {
 275         struct btrfs_fs_info *fs_info = eb->fs_info;
 276         int failed = 0;
 277         int ret;
 278         int num_copies = 0;
 279         int mirror_num = 0;
 280         int failed_mirror = 0;
 281
 282         ASSERT(check);
 283
 284         while (1) {
 285                 clear_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 286                 ret = read_extent_buffer_pages(eb, WAIT_COMPLETE, mirror_num, check);
 287                 if (!ret)
 288                         break;
 289
 290                 num_copies = btrfs_num_copies(fs_info,
 291                                               eb->start, eb->len);
 292                 if (num_copies == 1)
 293                         break;
 294
 295                 if (!failed_mirror) {
 296                         failed = 1;
 297                         failed_mirror = eb->read_mirror;
 298                 }
 299
 300                 mirror_num++;
 301                 if (mirror_num == failed_mirror)
 302                         mirror_num++;
 303
 304                 if (mirror_num > num_copies)
 305                         break;
 306         }
 307
 308         if (failed && !ret && failed_mirror)
 309                 btrfs_repair_eb_io_failure(eb, failed_mirror);
 310
 311         return ret;
 312 }
 313
 314 static int csum_one_extent_buffer(struct extent_buffer *eb)
 315 {
 316         struct btrfs_fs_info *fs_info = eb->fs_info;
 317         u8 result[BTRFS_CSUM_SIZE];
 318         int ret;
 319
 320         ASSERT(memcmp_extent_buffer(eb, fs_info->fs_devices->metadata_uuid,
 321                                     offsetof(struct btrfs_header, fsid),
 322                                     BTRFS_FSID_SIZE) == 0);
 323         csum_tree_block(eb, result);
 324
 325         if (btrfs_header_level(eb))
 326                 ret = btrfs_check_node(eb);
 327         else
 328                 ret = btrfs_check_leaf_full(eb);
 329
 330         if (ret < 0)
 331                 goto error;
 332
 333         /*
 334          * Also check the generation, the eb reached here must be newer than
 335          * last committed. Or something seriously wrong happened.
 336          */
 337         if (unlikely(btrfs_header_generation(eb) <= fs_info->last_trans_committed)) {
 338                 ret = -EUCLEAN;
 339                 btrfs_err(fs_info,
 340                         "block=%llu bad generation, have %llu expect > %llu",
 341                           eb->start, btrfs_header_generation(eb),
 342                           fs_info->last_trans_committed);
 343                 goto error;
 344         }
 345         write_extent_buffer(eb, result, 0, fs_info->csum_size);
 346
 347         return 0;
 348
 349 error:
 350         btrfs_print_tree(eb, 0);
 351         btrfs_err(fs_info, "block=%llu write time tree block corruption detected",
 352                   eb->start);
 353         /*
 354          * Be noisy if this is an extent buffer from a log tree. We don't abort
 355          * a transaction in case there's a bad log tree extent buffer, we just
 356          * fallback to a transaction commit. Still we want to know when there is
 357          * a bad log tree extent buffer, as that may signal a bug somewhere.
 358          */
 359         WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG) ||
 360                 btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID);
 361         return ret;
 362 }
 363
 364 /* Checksum all dirty extent buffers in one bio_vec */
 365 static int csum_dirty_subpage_buffers(struct btrfs_fs_info *fs_info,
 366                                       struct bio_vec *bvec)
 367 {
 368         struct page *page = bvec->bv_page;
 369         u64 bvec_start = page_offset(page) + bvec->bv_offset;
 370         u64 cur;
 371         int ret = 0;
 372
 373         for (cur = bvec_start; cur < bvec_start + bvec->bv_len;
 374              cur += fs_info->nodesize) {
 375                 struct extent_buffer *eb;
 376                 bool uptodate;
 377
 378                 eb = find_extent_buffer(fs_info, cur);
 379                 uptodate = btrfs_subpage_test_uptodate(fs_info, page, cur,
 380                                                        fs_info->nodesize);
 381
 382                 /* A dirty eb shouldn't disappear from buffer_radix */
 383                 if (WARN_ON(!eb))
 384                         return -EUCLEAN;
 385
 386                 if (WARN_ON(cur != btrfs_header_bytenr(eb))) {
 387                         free_extent_buffer(eb);
 388                         return -EUCLEAN;
 389                 }
 390                 if (WARN_ON(!uptodate)) {
 391                         free_extent_buffer(eb);
 392                         return -EUCLEAN;
 393                 }
 394
 395                 ret = csum_one_extent_buffer(eb);
 396                 free_extent_buffer(eb);
 397                 if (ret < 0)
 398                         return ret;
 399         }
 400         return ret;
 401 }
 402
 403 /*
 404  * Checksum a dirty tree block before IO.  This has extra checks to make sure
 405  * we only fill in the checksum field in the first page of a multi-page block.
 406  * For subpage extent buffers we need bvec to also read the offset in the page.
 407  */
 408 static int csum_dirty_buffer(struct btrfs_fs_info *fs_info, struct bio_vec *bvec)
 409 {
 410         struct page *page = bvec->bv_page;
 411         u64 start = page_offset(page);
 412         u64 found_start;
 413         struct extent_buffer *eb;
 414
 415         if (fs_info->nodesize < PAGE_SIZE)
 416                 return csum_dirty_subpage_buffers(fs_info, bvec);
 417
 418         eb = (struct extent_buffer *)page->private;
 419         if (page != eb->pages[0])
 420                 return 0;
 421
 422         found_start = btrfs_header_bytenr(eb);
 423
 424         if (test_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags)) {
 425                 WARN_ON(found_start != 0);
 426                 return 0;
 427         }
 428
 429         /*
 430          * Please do not consolidate these warnings into a single if.
 431          * It is useful to know what went wrong.
 432          */
 433         if (WARN_ON(found_start != start))
 434                 return -EUCLEAN;
 435         if (WARN_ON(!PageUptodate(page)))
 436                 return -EUCLEAN;
 437
 438         return csum_one_extent_buffer(eb);
 439 }
 440
 441 blk_status_t btree_csum_one_bio(struct btrfs_bio *bbio)
 442 {
 443         struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info;
 444         struct bvec_iter iter;
 445         struct bio_vec bv;
 446         int ret = 0;
 447
 448         bio_for_each_segment(bv, &bbio->bio, iter) {
 449                 ret = csum_dirty_buffer(fs_info, &bv);
 450                 if (ret)
 451                         break;
 452         }
 453
 454         return errno_to_blk_status(ret);
 455 }
 456
 457 static int check_tree_block_fsid(struct extent_buffer *eb)
 458 {
 459         struct btrfs_fs_info *fs_info = eb->fs_info;
 460         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
 461         u8 fsid[BTRFS_FSID_SIZE];
 462         u8 *metadata_uuid;
 463
 464         read_extent_buffer(eb, fsid, offsetof(struct btrfs_header, fsid),
 465                            BTRFS_FSID_SIZE);
 466         /*
 467          * Checking the incompat flag is only valid for the current fs. For
 468          * seed devices it's forbidden to have their uuid changed so reading
 469          * ->fsid in this case is fine
 470          */
 471         if (btrfs_fs_incompat(fs_info, METADATA_UUID))
 472                 metadata_uuid = fs_devices->metadata_uuid;
 473         else
 474                 metadata_uuid = fs_devices->fsid;
 475
 476         if (!memcmp(fsid, metadata_uuid, BTRFS_FSID_SIZE))
 477                 return 0;
 478
 479         list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list)
 480                 if (!memcmp(fsid, seed_devs->fsid, BTRFS_FSID_SIZE))
 481                         return 0;
 482
 483         return 1;
 484 }
 485
 486 /* Do basic extent buffer checks at read time */
 487 static int validate_extent_buffer(struct extent_buffer *eb,
 488                                   struct btrfs_tree_parent_check *check)
 489 {
 490         struct btrfs_fs_info *fs_info = eb->fs_info;
 491         u64 found_start;
 492         const u32 csum_size = fs_info->csum_size;
 493         u8 found_level;
 494         u8 result[BTRFS_CSUM_SIZE];
 495         const u8 *header_csum;
 496         int ret = 0;
 497
 498         ASSERT(check);
 499
 500         found_start = btrfs_header_bytenr(eb);
 501         if (found_start != eb->start) {
 502                 btrfs_err_rl(fs_info,
 503                         "bad tree block start, mirror %u want %llu have %llu",
 504                              eb->read_mirror, eb->start, found_start);
 505                 ret = -EIO;
 506                 goto out;
 507         }
 508         if (check_tree_block_fsid(eb)) {
 509                 btrfs_err_rl(fs_info, "bad fsid on logical %llu mirror %u",
 510                              eb->start, eb->read_mirror);
 511                 ret = -EIO;
 512                 goto out;
 513         }
 514         found_level = btrfs_header_level(eb);
 515         if (found_level >= BTRFS_MAX_LEVEL) {
 516                 btrfs_err(fs_info,
 517                         "bad tree block level, mirror %u level %d on logical %llu",
 518                         eb->read_mirror, btrfs_header_level(eb), eb->start);
 519                 ret = -EIO;
 520                 goto out;
 521         }
 522
 523         csum_tree_block(eb, result);
 524         header_csum = page_address(eb->pages[0]) +
 525                 get_eb_offset_in_page(eb, offsetof(struct btrfs_header, csum));
 526
 527         if (memcmp(result, header_csum, csum_size) != 0) {
 528                 btrfs_warn_rl(fs_info,
 529 "checksum verify failed on logical %llu mirror %u wanted " CSUM_FMT " found " CSUM_FMT " level %d",
 530                               eb->start, eb->read_mirror,
 531                               CSUM_FMT_VALUE(csum_size, header_csum),
 532                               CSUM_FMT_VALUE(csum_size, result),
 533                               btrfs_header_level(eb));
 534                 ret = -EUCLEAN;
 535                 goto out;
 536         }
 537
 538         if (found_level != check->level) {
 539                 btrfs_err(fs_info,
 540                 "level verify failed on logical %llu mirror %u wanted %u found %u",
 541                           eb->start, eb->read_mirror, check->level, found_level);
 542                 ret = -EIO;
 543                 goto out;
 544         }
 545         if (unlikely(check->transid &&
 546                      btrfs_header_generation(eb) != check->transid)) {
 547                 btrfs_err_rl(eb->fs_info,
 548 "parent transid verify failed on logical %llu mirror %u wanted %llu found %llu",
 549                                 eb->start, eb->read_mirror, check->transid,
 550                                 btrfs_header_generation(eb));
 551                 ret = -EIO;
 552                 goto out;
 553         }
 554         if (check->has_first_key) {
 555                 struct btrfs_key *expect_key = &check->first_key;
 556                 struct btrfs_key found_key;
 557
 558                 if (found_level)
 559                         btrfs_node_key_to_cpu(eb, &found_key, 0);
 560                 else
 561                         btrfs_item_key_to_cpu(eb, &found_key, 0);
 562                 if (unlikely(btrfs_comp_cpu_keys(expect_key, &found_key))) {
 563                         btrfs_err(fs_info,
 564 "tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)",
 565                                   eb->start, check->transid,
 566                                   expect_key->objectid,
 567                                   expect_key->type, expect_key->offset,
 568                                   found_key.objectid, found_key.type,
 569                                   found_key.offset);
 570                         ret = -EUCLEAN;
 571                         goto out;
 572                 }
 573         }
 574         if (check->owner_root) {
 575                 ret = btrfs_check_eb_owner(eb, check->owner_root);
 576                 if (ret < 0)
 577                         goto out;
 578         }
 579
 580         /*
 581          * If this is a leaf block and it is corrupt, set the corrupt bit so
 582          * that we don't try and read the other copies of this block, just
 583          * return -EIO.
 584          */
 585         if (found_level == 0 && btrfs_check_leaf_full(eb)) {
 586                 set_bit(EXTENT_BUFFER_CORRUPT, &eb->bflags);
 587                 ret = -EIO;
 588         }
 589
 590         if (found_level > 0 && btrfs_check_node(eb))
 591                 ret = -EIO;
 592
 593         if (!ret)
 594                 set_extent_buffer_uptodate(eb);
 595         else
 596                 btrfs_err(fs_info,
 597                 "read time tree block corruption detected on logical %llu mirror %u",
 598                           eb->start, eb->read_mirror);
 599 out:
 600         return ret;
 601 }
 602
 603 static int validate_subpage_buffer(struct page *page, u64 start, u64 end,
 604                                    int mirror, struct btrfs_tree_parent_check *check)
 605 {
 606         struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb);
 607         struct extent_buffer *eb;
 608         bool reads_done;
 609         int ret = 0;
 610
 611         ASSERT(check);
 612
 613         /*
 614          * We don't allow bio merge for subpage metadata read, so we should
 615          * only get one eb for each endio hook.
 616          */
 617         ASSERT(end == start + fs_info->nodesize - 1);
 618         ASSERT(PagePrivate(page));
 619
 620         eb = find_extent_buffer(fs_info, start);
 621         /*
 622          * When we are reading one tree block, eb must have been inserted into
 623          * the radix tree. If not, something is wrong.
 624          */
 625         ASSERT(eb);
 626
 627         reads_done = atomic_dec_and_test(&eb->io_pages);
 628         /* Subpage read must finish in page read */
 629         ASSERT(reads_done);
 630
 631         eb->read_mirror = mirror;
 632         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 633                 ret = -EIO;
 634                 goto err;
 635         }
 636         ret = validate_extent_buffer(eb, check);
 637         if (ret < 0)
 638                 goto err;
 639
 640         set_extent_buffer_uptodate(eb);
 641
 642         free_extent_buffer(eb);
 643         return ret;
 644 err:
 645         /*
 646          * end_bio_extent_readpage decrements io_pages in case of error,
 647          * make sure it has something to decrement.
 648          */
 649         atomic_inc(&eb->io_pages);
 650         clear_extent_buffer_uptodate(eb);
 651         free_extent_buffer(eb);
 652         return ret;
 653 }
 654
 655 int btrfs_validate_metadata_buffer(struct btrfs_bio *bbio,
 656                                    struct page *page, u64 start, u64 end,
 657                                    int mirror)
 658 {
 659         struct extent_buffer *eb;
 660         int ret = 0;
 661         int reads_done;
 662
 663         ASSERT(page->private);
 664
 665         if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
 666                 return validate_subpage_buffer(page, start, end, mirror,
 667                                                &bbio->parent_check);
 668
 669         eb = (struct extent_buffer *)page->private;
 670
 671         /*
 672          * The pending IO might have been the only thing that kept this buffer
 673          * in memory.  Make sure we have a ref for all this other checks
 674          */
 675         atomic_inc(&eb->refs);
 676
 677         reads_done = atomic_dec_and_test(&eb->io_pages);
 678         if (!reads_done)
 679                 goto err;
 680
 681         eb->read_mirror = mirror;
 682         if (test_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags)) {
 683                 ret = -EIO;
 684                 goto err;
 685         }
 686         ret = validate_extent_buffer(eb, &bbio->parent_check);
 687 err:
 688         if (ret) {
 689                 /*
 690                  * our io error hook is going to dec the io pages
 691                  * again, we have to make sure it has something
 692                  * to decrement
 693                  */
 694                 atomic_inc(&eb->io_pages);
 695                 clear_extent_buffer_uptodate(eb);
 696         }
 697         free_extent_buffer(eb);
 698
 699         return ret;
 700 }
 701
 702 #ifdef CONFIG_MIGRATION
 703 static int btree_migrate_folio(struct address_space *mapping,
 704                 struct folio *dst, struct folio *src, enum migrate_mode mode)
 705 {
 706         /*
 707          * we can't safely write a btree page from here,
 708          * we haven't done the locking hook
 709          */
 710         if (folio_test_dirty(src))
 711                 return -EAGAIN;
 712         /*
 713          * Buffers may be managed in a filesystem specific way.
 714          * We must have no buffers or drop them.
 715          */
 716         if (folio_get_private(src) &&
 717             !filemap_release_folio(src, GFP_KERNEL))
 718                 return -EAGAIN;
 719         return migrate_folio(mapping, dst, src, mode);
 720 }
 721 #else
 722 #define btree_migrate_folio NULL
 723 #endif
 724
 725 static int btree_writepages(struct address_space *mapping,
 726                             struct writeback_control *wbc)
 727 {
 728         struct btrfs_fs_info *fs_info;
 729         int ret;
 730
 731         if (wbc->sync_mode == WB_SYNC_NONE) {
 732
 733                 if (wbc->for_kupdate)
 734                         return 0;
 735
 736                 fs_info = BTRFS_I(mapping->host)->root->fs_info;
 737                 /* this is a bit racy, but that's ok */
 738                 ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
 739                                              BTRFS_DIRTY_METADATA_THRESH,
 740                                              fs_info->dirty_metadata_batch);
 741                 if (ret < 0)
 742                         return 0;
 743         }
 744         return btree_write_cache_pages(mapping, wbc);
 745 }
 746
 747 static bool btree_release_folio(struct folio *folio, gfp_t gfp_flags)
 748 {
 749         if (folio_test_writeback(folio) || folio_test_dirty(folio))
 750                 return false;
 751
 752         return try_release_extent_buffer(&folio->page);
 753 }
 754
 755 static void btree_invalidate_folio(struct folio *folio, size_t offset,
 756                                  size_t length)
 757 {
 758         struct extent_io_tree *tree;
 759         tree = &BTRFS_I(folio->mapping->host)->io_tree;
 760         extent_invalidate_folio(tree, folio, offset);
 761         btree_release_folio(folio, GFP_NOFS);
 762         if (folio_get_private(folio)) {
 763                 btrfs_warn(BTRFS_I(folio->mapping->host)->root->fs_info,
 764                            "folio private not zero on folio %llu",
 765                            (unsigned long long)folio_pos(folio));
 766                 folio_detach_private(folio);
 767         }
 768 }
 769
 770 #ifdef DEBUG
 771 static bool btree_dirty_folio(struct address_space *mapping,
 772                 struct folio *folio)
 773 {
 774         struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb);
 775         struct btrfs_subpage *subpage;
 776         struct extent_buffer *eb;
 777         int cur_bit = 0;
 778         u64 page_start = folio_pos(folio);
 779
 780         if (fs_info->sectorsize == PAGE_SIZE) {
 781                 eb = folio_get_private(folio);
 782                 BUG_ON(!eb);
 783                 BUG_ON(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
 784                 BUG_ON(!atomic_read(&eb->refs));
 785                 btrfs_assert_tree_write_locked(eb);
 786                 return filemap_dirty_folio(mapping, folio);
 787         }
 788         subpage = folio_get_private(folio);
 789
 790         ASSERT(subpage->dirty_bitmap);
 791         while (cur_bit < BTRFS_SUBPAGE_BITMAP_SIZE) {
 792                 unsigned long flags;
 793                 u64 cur;
 794                 u16 tmp = (1 << cur_bit);
 795
 796                 spin_lock_irqsave(&subpage->lock, flags);
 797                 if (!(tmp & subpage->dirty_bitmap)) {
 798                         spin_unlock_irqrestore(&subpage->lock, flags);
 799                         cur_bit++;
 800                         continue;
 801                 }
 802                 spin_unlock_irqrestore(&subpage->lock, flags);
 803                 cur = page_start + cur_bit * fs_info->sectorsize;
 804
 805                 eb = find_extent_buffer(fs_info, cur);
 806                 ASSERT(eb);
 807                 ASSERT(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
 808                 ASSERT(atomic_read(&eb->refs));
 809                 btrfs_assert_tree_write_locked(eb);
 810                 free_extent_buffer(eb);
 811
 812                 cur_bit += (fs_info->nodesize >> fs_info->sectorsize_bits);
 813         }
 814         return filemap_dirty_folio(mapping, folio);
 815 }
 816 #else
 817 #define btree_dirty_folio filemap_dirty_folio
 818 #endif
 819
 820 static const struct address_space_operations btree_aops = {
 821         .writepages     = btree_writepages,
 822         .release_folio  = btree_release_folio,
 823         .invalidate_folio = btree_invalidate_folio,
 824         .migrate_folio  = btree_migrate_folio,
 825         .dirty_folio    = btree_dirty_folio,
 826 };
 827
 828 struct extent_buffer *btrfs_find_create_tree_block(
 829                                                 struct btrfs_fs_info *fs_info,
 830                                                 u64 bytenr, u64 owner_root,
 831                                                 int level)
 832 {
 833         if (btrfs_is_testing(fs_info))
 834                 return alloc_test_extent_buffer(fs_info, bytenr);
 835         return alloc_extent_buffer(fs_info, bytenr, owner_root, level);
 836 }
 837
 838 /*
 839  * Read tree block at logical address @bytenr and do variant basic but critical
 840  * verification.
 841  *
 842  * @check:              expected tree parentness check, see comments of the
 843  *                      structure for details.
 844  */
 845 struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr,
 846                                       struct btrfs_tree_parent_check *check)
 847 {
 848         struct extent_buffer *buf = NULL;
 849         int ret;
 850
 851         ASSERT(check);
 852
 853         buf = btrfs_find_create_tree_block(fs_info, bytenr, check->owner_root,
 854                                            check->level);
 855         if (IS_ERR(buf))
 856                 return buf;
 857
 858         ret = btrfs_read_extent_buffer(buf, check);
 859         if (ret) {
 860                 free_extent_buffer_stale(buf);
 861                 return ERR_PTR(ret);
 862         }
 863         if (btrfs_check_eb_owner(buf, check->owner_root)) {
 864                 free_extent_buffer_stale(buf);
 865                 return ERR_PTR(-EUCLEAN);
 866         }
 867         return buf;
 868
 869 }
 870
 871 static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info,
 872                          u64 objectid)
 873 {
 874         bool dummy = test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state);
 875
 876         memset(&root->root_key, 0, sizeof(root->root_key));
 877         memset(&root->root_item, 0, sizeof(root->root_item));
 878         memset(&root->defrag_progress, 0, sizeof(root->defrag_progress));
 879         root->fs_info = fs_info;
 880         root->root_key.objectid = objectid;
 881         root->node = NULL;
 882         root->commit_root = NULL;
 883         root->state = 0;
 884         RB_CLEAR_NODE(&root->rb_node);
 885
 886         root->last_trans = 0;
 887         root->free_objectid = 0;
 888         root->nr_delalloc_inodes = 0;
 889         root->nr_ordered_extents = 0;
 890         root->inode_tree = RB_ROOT;
 891         INIT_RADIX_TREE(&root->delayed_nodes_tree, GFP_ATOMIC);
 892
 893         btrfs_init_root_block_rsv(root);
 894
 895         INIT_LIST_HEAD(&root->dirty_list);
 896         INIT_LIST_HEAD(&root->root_list);
 897         INIT_LIST_HEAD(&root->delalloc_inodes);
 898         INIT_LIST_HEAD(&root->delalloc_root);
 899         INIT_LIST_HEAD(&root->ordered_extents);
 900         INIT_LIST_HEAD(&root->ordered_root);
 901         INIT_LIST_HEAD(&root->reloc_dirty_list);
 902         INIT_LIST_HEAD(&root->logged_list[0]);
 903         INIT_LIST_HEAD(&root->logged_list[1]);
 904         spin_lock_init(&root->inode_lock);
 905         spin_lock_init(&root->delalloc_lock);
 906         spin_lock_init(&root->ordered_extent_lock);
 907         spin_lock_init(&root->accounting_lock);
 908         spin_lock_init(&root->log_extents_lock[0]);
 909         spin_lock_init(&root->log_extents_lock[1]);
 910         spin_lock_init(&root->qgroup_meta_rsv_lock);
 911         mutex_init(&root->objectid_mutex);
 912         mutex_init(&root->log_mutex);
 913         mutex_init(&root->ordered_extent_mutex);
 914         mutex_init(&root->delalloc_mutex);
 915         init_waitqueue_head(&root->qgroup_flush_wait);
 916         init_waitqueue_head(&root->log_writer_wait);
 917         init_waitqueue_head(&root->log_commit_wait[0]);
 918         init_waitqueue_head(&root->log_commit_wait[1]);
 919         INIT_LIST_HEAD(&root->log_ctxs[0]);
 920         INIT_LIST_HEAD(&root->log_ctxs[1]);
 921         atomic_set(&root->log_commit[0], 0);
 922         atomic_set(&root->log_commit[1], 0);
 923         atomic_set(&root->log_writers, 0);
 924         atomic_set(&root->log_batch, 0);
 925         refcount_set(&root->refs, 1);
 926         atomic_set(&root->snapshot_force_cow, 0);
 927         atomic_set(&root->nr_swapfiles, 0);
 928         root->log_transid = 0;
 929         root->log_transid_committed = -1;
 930         root->last_log_commit = 0;
 931         root->anon_dev = 0;
 932         if (!dummy) {
 933                 extent_io_tree_init(fs_info, &root->dirty_log_pages,
 934                                     IO_TREE_ROOT_DIRTY_LOG_PAGES);
 935                 extent_io_tree_init(fs_info, &root->log_csum_range,
 936                                     IO_TREE_LOG_CSUM_RANGE);
 937         }
 938
 939         spin_lock_init(&root->root_item_lock);
 940         btrfs_qgroup_init_swapped_blocks(&root->swapped_blocks);
 941 #ifdef CONFIG_BTRFS_DEBUG
 942         INIT_LIST_HEAD(&root->leak_list);
 943         spin_lock(&fs_info->fs_roots_radix_lock);
 944         list_add_tail(&root->leak_list, &fs_info->allocated_roots);
 945         spin_unlock(&fs_info->fs_roots_radix_lock);
 946 #endif
 947 }
 948
 949 static struct btrfs_root *btrfs_alloc_root(struct btrfs_fs_info *fs_info,
 950                                            u64 objectid, gfp_t flags)
 951 {
 952         struct btrfs_root *root = kzalloc(sizeof(*root), flags);
 953         if (root)
 954                 __setup_root(root, fs_info, objectid);
 955         return root;
 956 }
 957
 958 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
 959 /* Should only be used by the testing infrastructure */
 960 struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info)
 961 {
 962         struct btrfs_root *root;
 963
 964         if (!fs_info)
 965                 return ERR_PTR(-EINVAL);
 966
 967         root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID, GFP_KERNEL);
 968         if (!root)
 969                 return ERR_PTR(-ENOMEM);
 970
 971         /* We don't use the stripesize in selftest, set it as sectorsize */
 972         root->alloc_bytenr = 0;
 973
 974         return root;
 975 }
 976 #endif
 977
 978 static int global_root_cmp(struct rb_node *a_node, const struct rb_node *b_node)
 979 {
 980         const struct btrfs_root *a = rb_entry(a_node, struct btrfs_root, rb_node);
 981         const struct btrfs_root *b = rb_entry(b_node, struct btrfs_root, rb_node);
 982
 983         return btrfs_comp_cpu_keys(&a->root_key, &b->root_key);
 984 }
 985
 986 static int global_root_key_cmp(const void *k, const struct rb_node *node)
 987 {
 988         const struct btrfs_key *key = k;
 989         const struct btrfs_root *root = rb_entry(node, struct btrfs_root, rb_node);
 990
 991         return btrfs_comp_cpu_keys(key, &root->root_key);
 992 }
 993
 994 int btrfs_global_root_insert(struct btrfs_root *root)
 995 {
 996         struct btrfs_fs_info *fs_info = root->fs_info;
 997         struct rb_node *tmp;
 998
 999         write_lock(&fs_info->global_root_lock);
1000         tmp = rb_find_add(&root->rb_node, &fs_info->global_root_tree, global_root_cmp);
1001         write_unlock(&fs_info->global_root_lock);
1002         ASSERT(!tmp);
1003
1004         return tmp ? -EEXIST : 0;
1005 }
1006
1007 void btrfs_global_root_delete(struct btrfs_root *root)
1008 {
1009         struct btrfs_fs_info *fs_info = root->fs_info;
1010
1011         write_lock(&fs_info->global_root_lock);
1012         rb_erase(&root->rb_node, &fs_info->global_root_tree);
1013         write_unlock(&fs_info->global_root_lock);
1014 }
1015
1016 struct btrfs_root *btrfs_global_root(struct btrfs_fs_info *fs_info,
1017                                      struct btrfs_key *key)
1018 {
1019         struct rb_node *node;
1020         struct btrfs_root *root = NULL;
1021
1022         read_lock(&fs_info->global_root_lock);
1023         node = rb_find(key, &fs_info->global_root_tree, global_root_key_cmp);
1024         if (node)
1025                 root = container_of(node, struct btrfs_root, rb_node);
1026         read_unlock(&fs_info->global_root_lock);
1027
1028         return root;
1029 }
1030
1031 static u64 btrfs_global_root_id(struct btrfs_fs_info *fs_info, u64 bytenr)
1032 {
1033         struct btrfs_block_group *block_group;
1034         u64 ret;
1035
1036         if (!btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))
1037                 return 0;
1038
1039         if (bytenr)
1040                 block_group = btrfs_lookup_block_group(fs_info, bytenr);
1041         else
1042                 block_group = btrfs_lookup_first_block_group(fs_info, bytenr);
1043         ASSERT(block_group);
1044         if (!block_group)
1045                 return 0;
1046         ret = block_group->global_root_id;
1047         btrfs_put_block_group(block_group);
1048
1049         return ret;
1050 }
1051
1052 struct btrfs_root *btrfs_csum_root(struct btrfs_fs_info *fs_info, u64 bytenr)
1053 {
1054         struct btrfs_key key = {
1055                 .objectid = BTRFS_CSUM_TREE_OBJECTID,
1056                 .type = BTRFS_ROOT_ITEM_KEY,
1057                 .offset = btrfs_global_root_id(fs_info, bytenr),
1058         };
1059
1060         return btrfs_global_root(fs_info, &key);
1061 }
1062
1063 struct btrfs_root *btrfs_extent_root(struct btrfs_fs_info *fs_info, u64 bytenr)
1064 {
1065         struct btrfs_key key = {
1066                 .objectid = BTRFS_EXTENT_TREE_OBJECTID,
1067                 .type = BTRFS_ROOT_ITEM_KEY,
1068                 .offset = btrfs_global_root_id(fs_info, bytenr),
1069         };
1070
1071         return btrfs_global_root(fs_info, &key);
1072 }
1073
1074 struct btrfs_root *btrfs_block_group_root(struct btrfs_fs_info *fs_info)
1075 {
1076         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE))
1077                 return fs_info->block_group_root;
1078         return btrfs_extent_root(fs_info, 0);
1079 }
1080
1081 struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans,
1082                                      u64 objectid)
1083 {
1084         struct btrfs_fs_info *fs_info = trans->fs_info;
1085         struct extent_buffer *leaf;
1086         struct btrfs_root *tree_root = fs_info->tree_root;
1087         struct btrfs_root *root;
1088         struct btrfs_key key;
1089         unsigned int nofs_flag;
1090         int ret = 0;
1091
1092         /*
1093          * We're holding a transaction handle, so use a NOFS memory allocation
1094          * context to avoid deadlock if reclaim happens.
1095          */
1096         nofs_flag = memalloc_nofs_save();
1097         root = btrfs_alloc_root(fs_info, objectid, GFP_KERNEL);
1098         memalloc_nofs_restore(nofs_flag);
1099         if (!root)
1100                 return ERR_PTR(-ENOMEM);
1101
1102         root->root_key.objectid = objectid;
1103         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1104         root->root_key.offset = 0;
1105
1106         leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0,
1107                                       BTRFS_NESTING_NORMAL);
1108         if (IS_ERR(leaf)) {
1109                 ret = PTR_ERR(leaf);
1110                 leaf = NULL;
1111                 goto fail;
1112         }
1113
1114         root->node = leaf;
1115         btrfs_mark_buffer_dirty(leaf);
1116
1117         root->commit_root = btrfs_root_node(root);
1118         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
1119
1120         btrfs_set_root_flags(&root->root_item, 0);
1121         btrfs_set_root_limit(&root->root_item, 0);
1122         btrfs_set_root_bytenr(&root->root_item, leaf->start);
1123         btrfs_set_root_generation(&root->root_item, trans->transid);
1124         btrfs_set_root_level(&root->root_item, 0);
1125         btrfs_set_root_refs(&root->root_item, 1);
1126         btrfs_set_root_used(&root->root_item, leaf->len);
1127         btrfs_set_root_last_snapshot(&root->root_item, 0);
1128         btrfs_set_root_dirid(&root->root_item, 0);
1129         if (is_fstree(objectid))
1130                 generate_random_guid(root->root_item.uuid);
1131         else
1132                 export_guid(root->root_item.uuid, &guid_null);
1133         btrfs_set_root_drop_level(&root->root_item, 0);
1134
1135         btrfs_tree_unlock(leaf);
1136
1137         key.objectid = objectid;
1138         key.type = BTRFS_ROOT_ITEM_KEY;
1139         key.offset = 0;
1140         ret = btrfs_insert_root(trans, tree_root, &key, &root->root_item);
1141         if (ret)
1142                 goto fail;
1143
1144         return root;
1145
1146 fail:
1147         btrfs_put_root(root);
1148
1149         return ERR_PTR(ret);
1150 }
1151
1152 static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans,
1153                                          struct btrfs_fs_info *fs_info)
1154 {
1155         struct btrfs_root *root;
1156
1157         root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID, GFP_NOFS);
1158         if (!root)
1159                 return ERR_PTR(-ENOMEM);
1160
1161         root->root_key.objectid = BTRFS_TREE_LOG_OBJECTID;
1162         root->root_key.type = BTRFS_ROOT_ITEM_KEY;
1163         root->root_key.offset = BTRFS_TREE_LOG_OBJECTID;
1164
1165         return root;
1166 }
1167
1168 int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans,
1169                               struct btrfs_root *root)
1170 {
1171         struct extent_buffer *leaf;
1172
1173         /*
1174          * DON'T set SHAREABLE bit for log trees.
1175          *
1176          * Log trees are not exposed to user space thus can't be snapshotted,
1177          * and they go away before a real commit is actually done.
1178          *
1179          * They do store pointers to file data extents, and those reference
1180          * counts still get updated (along with back refs to the log tree).
1181          */
1182
1183         leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID,
1184                         NULL, 0, 0, 0, BTRFS_NESTING_NORMAL);
1185         if (IS_ERR(leaf))
1186                 return PTR_ERR(leaf);
1187
1188         root->node = leaf;
1189
1190         btrfs_mark_buffer_dirty(root->node);
1191         btrfs_tree_unlock(root->node);
1192
1193         return 0;
1194 }
1195
1196 int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans,
1197                              struct btrfs_fs_info *fs_info)
1198 {
1199         struct btrfs_root *log_root;
1200
1201         log_root = alloc_log_tree(trans, fs_info);
1202         if (IS_ERR(log_root))
1203                 return PTR_ERR(log_root);
1204
1205         if (!btrfs_is_zoned(fs_info)) {
1206                 int ret = btrfs_alloc_log_tree_node(trans, log_root);
1207
1208                 if (ret) {
1209                         btrfs_put_root(log_root);
1210                         return ret;
1211                 }
1212         }
1213
1214         WARN_ON(fs_info->log_root_tree);
1215         fs_info->log_root_tree = log_root;
1216         return 0;
1217 }
1218
1219 int btrfs_add_log_tree(struct btrfs_trans_handle *trans,
1220                        struct btrfs_root *root)
1221 {
1222         struct btrfs_fs_info *fs_info = root->fs_info;
1223         struct btrfs_root *log_root;
1224         struct btrfs_inode_item *inode_item;
1225         int ret;
1226
1227         log_root = alloc_log_tree(trans, fs_info);
1228         if (IS_ERR(log_root))
1229                 return PTR_ERR(log_root);
1230
1231         ret = btrfs_alloc_log_tree_node(trans, log_root);
1232         if (ret) {
1233                 btrfs_put_root(log_root);
1234                 return ret;
1235         }
1236
1237         log_root->last_trans = trans->transid;
1238         log_root->root_key.offset = root->root_key.objectid;
1239
1240         inode_item = &log_root->root_item.inode;
1241         btrfs_set_stack_inode_generation(inode_item, 1);
1242         btrfs_set_stack_inode_size(inode_item, 3);
1243         btrfs_set_stack_inode_nlink(inode_item, 1);
1244         btrfs_set_stack_inode_nbytes(inode_item,
1245                                      fs_info->nodesize);
1246         btrfs_set_stack_inode_mode(inode_item, S_IFDIR | 0755);
1247
1248         btrfs_set_root_node(&log_root->root_item, log_root->node);
1249
1250         WARN_ON(root->log_root);
1251         root->log_root = log_root;
1252         root->log_transid = 0;
1253         root->log_transid_committed = -1;
1254         root->last_log_commit = 0;
1255         return 0;
1256 }
1257
1258 static struct btrfs_root *read_tree_root_path(struct btrfs_root *tree_root,
1259                                               struct btrfs_path *path,
1260                                               struct btrfs_key *key)
1261 {
1262         struct btrfs_root *root;
1263         struct btrfs_tree_parent_check check = { 0 };
1264         struct btrfs_fs_info *fs_info = tree_root->fs_info;
1265         u64 generation;
1266         int ret;
1267         int level;
1268
1269         root = btrfs_alloc_root(fs_info, key->objectid, GFP_NOFS);
1270         if (!root)
1271                 return ERR_PTR(-ENOMEM);
1272
1273         ret = btrfs_find_root(tree_root, key, path,
1274                               &root->root_item, &root->root_key);
1275         if (ret) {
1276                 if (ret > 0)
1277                         ret = -ENOENT;
1278                 goto fail;
1279         }
1280
1281         generation = btrfs_root_generation(&root->root_item);
1282         level = btrfs_root_level(&root->root_item);
1283         check.level = level;
1284         check.transid = generation;
1285         check.owner_root = key->objectid;
1286         root->node = read_tree_block(fs_info, btrfs_root_bytenr(&root->root_item),
1287                                      &check);
1288         if (IS_ERR(root->node)) {
1289                 ret = PTR_ERR(root->node);
1290                 root->node = NULL;
1291                 goto fail;
1292         }
1293         if (!btrfs_buffer_uptodate(root->node, generation, 0)) {
1294                 ret = -EIO;
1295                 goto fail;
1296         }
1297
1298         /*
1299          * For real fs, and not log/reloc trees, root owner must
1300          * match its root node owner
1301          */
1302         if (!test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state) &&
1303             root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
1304             root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID &&
1305             root->root_key.objectid != btrfs_header_owner(root->node)) {
1306                 btrfs_crit(fs_info,
1307 "root=%llu block=%llu, tree root owner mismatch, have %llu expect %llu",
1308                            root->root_key.objectid, root->node->start,
1309                            btrfs_header_owner(root->node),
1310                            root->root_key.objectid);
1311                 ret = -EUCLEAN;
1312                 goto fail;
1313         }
1314         root->commit_root = btrfs_root_node(root);
1315         return root;
1316 fail:
1317         btrfs_put_root(root);
1318         return ERR_PTR(ret);
1319 }
1320
1321 struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root,
1322                                         struct btrfs_key *key)
1323 {
1324         struct btrfs_root *root;
1325         struct btrfs_path *path;
1326
1327         path = btrfs_alloc_path();
1328         if (!path)
1329                 return ERR_PTR(-ENOMEM);
1330         root = read_tree_root_path(tree_root, path, key);
1331         btrfs_free_path(path);
1332
1333         return root;
1334 }
1335
1336 /*
1337  * Initialize subvolume root in-memory structure
1338  *
1339  * @anon_dev:   anonymous device to attach to the root, if zero, allocate new
1340  */
1341 static int btrfs_init_fs_root(struct btrfs_root *root, dev_t anon_dev)
1342 {
1343         int ret;
1344         unsigned int nofs_flag;
1345
1346         /*
1347          * We might be called under a transaction (e.g. indirect backref
1348          * resolution) which could deadlock if it triggers memory reclaim
1349          */
1350         nofs_flag = memalloc_nofs_save();
1351         ret = btrfs_drew_lock_init(&root->snapshot_lock);
1352         memalloc_nofs_restore(nofs_flag);
1353         if (ret)
1354                 goto fail;
1355
1356         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID &&
1357             !btrfs_is_data_reloc_root(root)) {
1358                 set_bit(BTRFS_ROOT_SHAREABLE, &root->state);
1359                 btrfs_check_and_init_root_item(&root->root_item);
1360         }
1361
1362         /*
1363          * Don't assign anonymous block device to roots that are not exposed to
1364          * userspace, the id pool is limited to 1M
1365          */
1366         if (is_fstree(root->root_key.objectid) &&
1367             btrfs_root_refs(&root->root_item) > 0) {
1368                 if (!anon_dev) {
1369                         ret = get_anon_bdev(&root->anon_dev);
1370                         if (ret)
1371                                 goto fail;
1372                 } else {
1373                         root->anon_dev = anon_dev;
1374                 }
1375         }
1376
1377         mutex_lock(&root->objectid_mutex);
1378         ret = btrfs_init_root_free_objectid(root);
1379         if (ret) {
1380                 mutex_unlock(&root->objectid_mutex);
1381                 goto fail;
1382         }
1383
1384         ASSERT(root->free_objectid <= BTRFS_LAST_FREE_OBJECTID);
1385
1386         mutex_unlock(&root->objectid_mutex);
1387
1388         return 0;
1389 fail:
1390         /* The caller is responsible to call btrfs_free_fs_root */
1391         return ret;
1392 }
1393
1394 static struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info,
1395                                                u64 root_id)
1396 {
1397         struct btrfs_root *root;
1398
1399         spin_lock(&fs_info->fs_roots_radix_lock);
1400         root = radix_tree_lookup(&fs_info->fs_roots_radix,
1401                                  (unsigned long)root_id);
1402         if (root)
1403                 root = btrfs_grab_root(root);
1404         spin_unlock(&fs_info->fs_roots_radix_lock);
1405         return root;
1406 }
1407
1408 static struct btrfs_root *btrfs_get_global_root(struct btrfs_fs_info *fs_info,
1409                                                 u64 objectid)
1410 {
1411         struct btrfs_key key = {
1412                 .objectid = objectid,
1413                 .type = BTRFS_ROOT_ITEM_KEY,
1414                 .offset = 0,
1415         };
1416
1417         if (objectid == BTRFS_ROOT_TREE_OBJECTID)
1418                 return btrfs_grab_root(fs_info->tree_root);
1419         if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
1420                 return btrfs_grab_root(btrfs_global_root(fs_info, &key));
1421         if (objectid == BTRFS_CHUNK_TREE_OBJECTID)
1422                 return btrfs_grab_root(fs_info->chunk_root);
1423         if (objectid == BTRFS_DEV_TREE_OBJECTID)
1424                 return btrfs_grab_root(fs_info->dev_root);
1425         if (objectid == BTRFS_CSUM_TREE_OBJECTID)
1426                 return btrfs_grab_root(btrfs_global_root(fs_info, &key));
1427         if (objectid == BTRFS_QUOTA_TREE_OBJECTID)
1428                 return btrfs_grab_root(fs_info->quota_root) ?
1429                         fs_info->quota_root : ERR_PTR(-ENOENT);
1430         if (objectid == BTRFS_UUID_TREE_OBJECTID)
1431                 return btrfs_grab_root(fs_info->uuid_root) ?
1432                         fs_info->uuid_root : ERR_PTR(-ENOENT);
1433         if (objectid == BTRFS_BLOCK_GROUP_TREE_OBJECTID)
1434                 return btrfs_grab_root(fs_info->block_group_root) ?
1435                         fs_info->block_group_root : ERR_PTR(-ENOENT);
1436         if (objectid == BTRFS_FREE_SPACE_TREE_OBJECTID) {
1437                 struct btrfs_root *root = btrfs_global_root(fs_info, &key);
1438
1439                 return btrfs_grab_root(root) ? root : ERR_PTR(-ENOENT);
1440         }
1441         return NULL;
1442 }
1443
1444 int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info,
1445                          struct btrfs_root *root)
1446 {
1447         int ret;
1448
1449         ret = radix_tree_preload(GFP_NOFS);
1450         if (ret)
1451                 return ret;
1452
1453         spin_lock(&fs_info->fs_roots_radix_lock);
1454         ret = radix_tree_insert(&fs_info->fs_roots_radix,
1455                                 (unsigned long)root->root_key.objectid,
1456                                 root);
1457         if (ret == 0) {
1458                 btrfs_grab_root(root);
1459                 set_bit(BTRFS_ROOT_IN_RADIX, &root->state);
1460         }
1461         spin_unlock(&fs_info->fs_roots_radix_lock);
1462         radix_tree_preload_end();
1463
1464         return ret;
1465 }
1466
1467 void btrfs_check_leaked_roots(struct btrfs_fs_info *fs_info)
1468 {
1469 #ifdef CONFIG_BTRFS_DEBUG
1470         struct btrfs_root *root;
1471
1472         while (!list_empty(&fs_info->allocated_roots)) {
1473                 char buf[BTRFS_ROOT_NAME_BUF_LEN];
1474
1475                 root = list_first_entry(&fs_info->allocated_roots,
1476                                         struct btrfs_root, leak_list);
1477                 btrfs_err(fs_info, "leaked root %s refcount %d",
1478                           btrfs_root_name(&root->root_key, buf),
1479                           refcount_read(&root->refs));
1480                 while (refcount_read(&root->refs) > 1)
1481                         btrfs_put_root(root);
1482                 btrfs_put_root(root);
1483         }
1484 #endif
1485 }
1486
1487 static void free_global_roots(struct btrfs_fs_info *fs_info)
1488 {
1489         struct btrfs_root *root;
1490         struct rb_node *node;
1491
1492         while ((node = rb_first_postorder(&fs_info->global_root_tree)) != NULL) {
1493                 root = rb_entry(node, struct btrfs_root, rb_node);
1494                 rb_erase(&root->rb_node, &fs_info->global_root_tree);
1495                 btrfs_put_root(root);
1496         }
1497 }
1498
1499 void btrfs_free_fs_info(struct btrfs_fs_info *fs_info)
1500 {
1501         percpu_counter_destroy(&fs_info->dirty_metadata_bytes);
1502         percpu_counter_destroy(&fs_info->delalloc_bytes);
1503         percpu_counter_destroy(&fs_info->ordered_bytes);
1504         percpu_counter_destroy(&fs_info->dev_replace.bio_counter);
1505         btrfs_free_csum_hash(fs_info);
1506         btrfs_free_stripe_hash_table(fs_info);
1507         btrfs_free_ref_cache(fs_info);
1508         kfree(fs_info->balance_ctl);
1509         kfree(fs_info->delayed_root);
1510         free_global_roots(fs_info);
1511         btrfs_put_root(fs_info->tree_root);
1512         btrfs_put_root(fs_info->chunk_root);
1513         btrfs_put_root(fs_info->dev_root);
1514         btrfs_put_root(fs_info->quota_root);
1515         btrfs_put_root(fs_info->uuid_root);
1516         btrfs_put_root(fs_info->fs_root);
1517         btrfs_put_root(fs_info->data_reloc_root);
1518         btrfs_put_root(fs_info->block_group_root);
1519         btrfs_check_leaked_roots(fs_info);
1520         btrfs_extent_buffer_leak_debug_check(fs_info);
1521         kfree(fs_info->super_copy);
1522         kfree(fs_info->super_for_commit);
1523         kfree(fs_info->subpage_info);
1524         kvfree(fs_info);
1525 }
1526
1527
1528 /*
1529  * Get an in-memory reference of a root structure.
1530  *
1531  * For essential trees like root/extent tree, we grab it from fs_info directly.
1532  * For subvolume trees, we check the cached filesystem roots first. If not
1533  * found, then read it from disk and add it to cached fs roots.
1534  *
1535  * Caller should release the root by calling btrfs_put_root() after the usage.
1536  *
1537  * NOTE: Reloc and log trees can't be read by this function as they share the
1538  *       same root objectid.
1539  *
1540  * @objectid:   root id
1541  * @anon_dev:   preallocated anonymous block device number for new roots,
1542  *              pass 0 for new allocation.
1543  * @check_ref:  whether to check root item references, If true, return -ENOENT
1544  *              for orphan roots
1545  */
1546 static struct btrfs_root *btrfs_get_root_ref(struct btrfs_fs_info *fs_info,
1547                                              u64 objectid, dev_t anon_dev,
1548                                              bool check_ref)
1549 {
1550         struct btrfs_root *root;
1551         struct btrfs_path *path;
1552         struct btrfs_key key;
1553         int ret;
1554
1555         root = btrfs_get_global_root(fs_info, objectid);
1556         if (root)
1557                 return root;
1558 again:
1559         root = btrfs_lookup_fs_root(fs_info, objectid);
1560         if (root) {
1561                 /* Shouldn't get preallocated anon_dev for cached roots */
1562                 ASSERT(!anon_dev);
1563                 if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1564                         btrfs_put_root(root);
1565                         return ERR_PTR(-ENOENT);
1566                 }
1567                 return root;
1568         }
1569
1570         key.objectid = objectid;
1571         key.type = BTRFS_ROOT_ITEM_KEY;
1572         key.offset = (u64)-1;
1573         root = btrfs_read_tree_root(fs_info->tree_root, &key);
1574         if (IS_ERR(root))
1575                 return root;
1576
1577         if (check_ref && btrfs_root_refs(&root->root_item) == 0) {
1578                 ret = -ENOENT;
1579                 goto fail;
1580         }
1581
1582         ret = btrfs_init_fs_root(root, anon_dev);
1583         if (ret)
1584                 goto fail;
1585
1586         path = btrfs_alloc_path();
1587         if (!path) {
1588                 ret = -ENOMEM;
1589                 goto fail;
1590         }
1591         key.objectid = BTRFS_ORPHAN_OBJECTID;
1592         key.type = BTRFS_ORPHAN_ITEM_KEY;
1593         key.offset = objectid;
1594
1595         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
1596         btrfs_free_path(path);
1597         if (ret < 0)
1598                 goto fail;
1599         if (ret == 0)
1600                 set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state);
1601
1602         ret = btrfs_insert_fs_root(fs_info, root);
1603         if (ret) {
1604                 if (ret == -EEXIST) {
1605                         btrfs_put_root(root);
1606                         goto again;
1607                 }
1608                 goto fail;
1609         }
1610         return root;
1611 fail:
1612         /*
1613          * If our caller provided us an anonymous device, then it's his
1614          * responsibility to free it in case we fail. So we have to set our
1615          * root's anon_dev to 0 to avoid a double free, once by btrfs_put_root()
1616          * and once again by our caller.
1617          */
1618         if (anon_dev)
1619                 root->anon_dev = 0;
1620         btrfs_put_root(root);
1621         return ERR_PTR(ret);
1622 }
1623
1624 /*
1625  * Get in-memory reference of a root structure
1626  *
1627  * @objectid:   tree objectid
1628  * @check_ref:  if set, verify that the tree exists and the item has at least
1629  *              one reference
1630  */
1631 struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info,
1632                                      u64 objectid, bool check_ref)
1633 {
1634         return btrfs_get_root_ref(fs_info, objectid, 0, check_ref);
1635 }
1636
1637 /*
1638  * Get in-memory reference of a root structure, created as new, optionally pass
1639  * the anonymous block device id
1640  *
1641  * @objectid:   tree objectid
1642  * @anon_dev:   if zero, allocate a new anonymous block device or use the
1643  *              parameter value
1644  */
1645 struct btrfs_root *btrfs_get_new_fs_root(struct btrfs_fs_info *fs_info,
1646                                          u64 objectid, dev_t anon_dev)
1647 {
1648         return btrfs_get_root_ref(fs_info, objectid, anon_dev, true);
1649 }
1650
1651 /*
1652  * btrfs_get_fs_root_commit_root - return a root for the given objectid
1653  * @fs_info:    the fs_info
1654  * @objectid:   the objectid we need to lookup
1655  *
1656  * This is exclusively used for backref walking, and exists specifically because
1657  * of how qgroups does lookups.  Qgroups will do a backref lookup at delayed ref
1658  * creation time, which means we may have to read the tree_root in order to look
1659  * up a fs root that is not in memory.  If the root is not in memory we will
1660  * read the tree root commit root and look up the fs root from there.  This is a
1661  * temporary root, it will not be inserted into the radix tree as it doesn't
1662  * have the most uptodate information, it'll simply be discarded once the
1663  * backref code is finished using the root.
1664  */
1665 struct btrfs_root *btrfs_get_fs_root_commit_root(struct btrfs_fs_info *fs_info,
1666                                                  struct btrfs_path *path,
1667                                                  u64 objectid)
1668 {
1669         struct btrfs_root *root;
1670         struct btrfs_key key;
1671
1672         ASSERT(path->search_commit_root && path->skip_locking);
1673
1674         /*
1675          * This can return -ENOENT if we ask for a root that doesn't exist, but
1676          * since this is called via the backref walking code we won't be looking
1677          * up a root that doesn't exist, unless there's corruption.  So if root
1678          * != NULL just return it.
1679          */
1680         root = btrfs_get_global_root(fs_info, objectid);
1681         if (root)
1682                 return root;
1683
1684         root = btrfs_lookup_fs_root(fs_info, objectid);
1685         if (root)
1686                 return root;
1687
1688         key.objectid = objectid;
1689         key.type = BTRFS_ROOT_ITEM_KEY;
1690         key.offset = (u64)-1;
1691         root = read_tree_root_path(fs_info->tree_root, path, &key);
1692         btrfs_release_path(path);
1693
1694         return root;
1695 }
1696
1697 static int cleaner_kthread(void *arg)
1698 {
1699         struct btrfs_fs_info *fs_info = arg;
1700         int again;
1701
1702         while (1) {
1703                 again = 0;
1704
1705                 set_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1706
1707                 /* Make the cleaner go to sleep early. */
1708                 if (btrfs_need_cleaner_sleep(fs_info))
1709                         goto sleep;
1710
1711                 /*
1712                  * Do not do anything if we might cause open_ctree() to block
1713                  * before we have finished mounting the filesystem.
1714                  */
1715                 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1716                         goto sleep;
1717
1718                 if (!mutex_trylock(&fs_info->cleaner_mutex))
1719                         goto sleep;
1720
1721                 /*
1722                  * Avoid the problem that we change the status of the fs
1723                  * during the above check and trylock.
1724                  */
1725                 if (btrfs_need_cleaner_sleep(fs_info)) {
1726                         mutex_unlock(&fs_info->cleaner_mutex);
1727                         goto sleep;
1728                 }
1729
1730                 if (test_and_clear_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags))
1731                         btrfs_sysfs_feature_update(fs_info);
1732
1733                 btrfs_run_delayed_iputs(fs_info);
1734
1735                 again = btrfs_clean_one_deleted_snapshot(fs_info);
1736                 mutex_unlock(&fs_info->cleaner_mutex);
1737
1738                 /*
1739                  * The defragger has dealt with the R/O remount and umount,
1740                  * needn't do anything special here.
1741                  */
1742                 btrfs_run_defrag_inodes(fs_info);
1743
1744                 /*
1745                  * Acquires fs_info->reclaim_bgs_lock to avoid racing
1746                  * with relocation (btrfs_relocate_chunk) and relocation
1747                  * acquires fs_info->cleaner_mutex (btrfs_relocate_block_group)
1748                  * after acquiring fs_info->reclaim_bgs_lock. So we
1749                  * can't hold, nor need to, fs_info->cleaner_mutex when deleting
1750                  * unused block groups.
1751                  */
1752                 btrfs_delete_unused_bgs(fs_info);
1753
1754                 /*
1755                  * Reclaim block groups in the reclaim_bgs list after we deleted
1756                  * all unused block_groups. This possibly gives us some more free
1757                  * space.
1758                  */
1759                 btrfs_reclaim_bgs(fs_info);
1760 sleep:
1761                 clear_and_wake_up_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags);
1762                 if (kthread_should_park())
1763                         kthread_parkme();
1764                 if (kthread_should_stop())
1765                         return 0;
1766                 if (!again) {
1767                         set_current_state(TASK_INTERRUPTIBLE);
1768                         schedule();
1769                         __set_current_state(TASK_RUNNING);
1770                 }
1771         }
1772 }
1773
1774 static int transaction_kthread(void *arg)
1775 {
1776         struct btrfs_root *root = arg;
1777         struct btrfs_fs_info *fs_info = root->fs_info;
1778         struct btrfs_trans_handle *trans;
1779         struct btrfs_transaction *cur;
1780         u64 transid;
1781         time64_t delta;
1782         unsigned long delay;
1783         bool cannot_commit;
1784
1785         do {
1786                 cannot_commit = false;
1787                 delay = msecs_to_jiffies(fs_info->commit_interval * 1000);
1788                 mutex_lock(&fs_info->transaction_kthread_mutex);
1789
1790                 spin_lock(&fs_info->trans_lock);
1791                 cur = fs_info->running_transaction;
1792                 if (!cur) {
1793                         spin_unlock(&fs_info->trans_lock);
1794                         goto sleep;
1795                 }
1796
1797                 delta = ktime_get_seconds() - cur->start_time;
1798                 if (!test_and_clear_bit(BTRFS_FS_COMMIT_TRANS, &fs_info->flags) &&
1799                     cur->state < TRANS_STATE_COMMIT_START &&
1800                     delta < fs_info->commit_interval) {
1801                         spin_unlock(&fs_info->trans_lock);
1802                         delay -= msecs_to_jiffies((delta - 1) * 1000);
1803                         delay = min(delay,
1804                                     msecs_to_jiffies(fs_info->commit_interval * 1000));
1805                         goto sleep;
1806                 }
1807                 transid = cur->transid;
1808                 spin_unlock(&fs_info->trans_lock);
1809
1810                 /* If the file system is aborted, this will always fail. */
1811                 trans = btrfs_attach_transaction(root);
1812                 if (IS_ERR(trans)) {
1813                         if (PTR_ERR(trans) != -ENOENT)
1814                                 cannot_commit = true;
1815                         goto sleep;
1816                 }
1817                 if (transid == trans->transid) {
1818                         btrfs_commit_transaction(trans);
1819                 } else {
1820                         btrfs_end_transaction(trans);
1821                 }
1822 sleep:
1823                 wake_up_process(fs_info->cleaner_kthread);
1824                 mutex_unlock(&fs_info->transaction_kthread_mutex);
1825
1826                 if (BTRFS_FS_ERROR(fs_info))
1827                         btrfs_cleanup_transaction(fs_info);
1828                 if (!kthread_should_stop() &&
1829                                 (!btrfs_transaction_blocked(fs_info) ||
1830                                  cannot_commit))
1831                         schedule_timeout_interruptible(delay);
1832         } while (!kthread_should_stop());
1833         return 0;
1834 }
1835
1836 /*
1837  * This will find the highest generation in the array of root backups.  The
1838  * index of the highest array is returned, or -EINVAL if we can't find
1839  * anything.
1840  *
1841  * We check to make sure the array is valid by comparing the
1842  * generation of the latest  root in the array with the generation
1843  * in the super block.  If they don't match we pitch it.
1844  */
1845 static int find_newest_super_backup(struct btrfs_fs_info *info)
1846 {
1847         const u64 newest_gen = btrfs_super_generation(info->super_copy);
1848         u64 cur;
1849         struct btrfs_root_backup *root_backup;
1850         int i;
1851
1852         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
1853                 root_backup = info->super_copy->super_roots + i;
1854                 cur = btrfs_backup_tree_root_gen(root_backup);
1855                 if (cur == newest_gen)
1856                         return i;
1857         }
1858
1859         return -EINVAL;
1860 }
1861
1862 /*
1863  * copy all the root pointers into the super backup array.
1864  * this will bump the backup pointer by one when it is
1865  * done
1866  */
1867 static void backup_super_roots(struct btrfs_fs_info *info)
1868 {
1869         const int next_backup = info->backup_root_index;
1870         struct btrfs_root_backup *root_backup;
1871
1872         root_backup = info->super_for_commit->super_roots + next_backup;
1873
1874         /*
1875          * make sure all of our padding and empty slots get zero filled
1876          * regardless of which ones we use today
1877          */
1878         memset(root_backup, 0, sizeof(*root_backup));
1879
1880         info->backup_root_index = (next_backup + 1) % BTRFS_NUM_BACKUP_ROOTS;
1881
1882         btrfs_set_backup_tree_root(root_backup, info->tree_root->node->start);
1883         btrfs_set_backup_tree_root_gen(root_backup,
1884                                btrfs_header_generation(info->tree_root->node));
1885
1886         btrfs_set_backup_tree_root_level(root_backup,
1887                                btrfs_header_level(info->tree_root->node));
1888
1889         btrfs_set_backup_chunk_root(root_backup, info->chunk_root->node->start);
1890         btrfs_set_backup_chunk_root_gen(root_backup,
1891                                btrfs_header_generation(info->chunk_root->node));
1892         btrfs_set_backup_chunk_root_level(root_backup,
1893                                btrfs_header_level(info->chunk_root->node));
1894
1895         if (!btrfs_fs_compat_ro(info, BLOCK_GROUP_TREE)) {
1896                 struct btrfs_root *extent_root = btrfs_extent_root(info, 0);
1897                 struct btrfs_root *csum_root = btrfs_csum_root(info, 0);
1898
1899                 btrfs_set_backup_extent_root(root_backup,
1900                                              extent_root->node->start);
1901                 btrfs_set_backup_extent_root_gen(root_backup,
1902                                 btrfs_header_generation(extent_root->node));
1903                 btrfs_set_backup_extent_root_level(root_backup,
1904                                         btrfs_header_level(extent_root->node));
1905
1906                 btrfs_set_backup_csum_root(root_backup, csum_root->node->start);
1907                 btrfs_set_backup_csum_root_gen(root_backup,
1908                                                btrfs_header_generation(csum_root->node));
1909                 btrfs_set_backup_csum_root_level(root_backup,
1910                                                  btrfs_header_level(csum_root->node));
1911         }
1912
1913         /*
1914          * we might commit during log recovery, which happens before we set
1915          * the fs_root.  Make sure it is valid before we fill it in.
1916          */
1917         if (info->fs_root && info->fs_root->node) {
1918                 btrfs_set_backup_fs_root(root_backup,
1919                                          info->fs_root->node->start);
1920                 btrfs_set_backup_fs_root_gen(root_backup,
1921                                btrfs_header_generation(info->fs_root->node));
1922                 btrfs_set_backup_fs_root_level(root_backup,
1923                                btrfs_header_level(info->fs_root->node));
1924         }
1925
1926         btrfs_set_backup_dev_root(root_backup, info->dev_root->node->start);
1927         btrfs_set_backup_dev_root_gen(root_backup,
1928                                btrfs_header_generation(info->dev_root->node));
1929         btrfs_set_backup_dev_root_level(root_backup,
1930                                        btrfs_header_level(info->dev_root->node));
1931
1932         btrfs_set_backup_total_bytes(root_backup,
1933                              btrfs_super_total_bytes(info->super_copy));
1934         btrfs_set_backup_bytes_used(root_backup,
1935                              btrfs_super_bytes_used(info->super_copy));
1936         btrfs_set_backup_num_devices(root_backup,
1937                              btrfs_super_num_devices(info->super_copy));
1938
1939         /*
1940          * if we don't copy this out to the super_copy, it won't get remembered
1941          * for the next commit
1942          */
1943         memcpy(&info->super_copy->super_roots,
1944                &info->super_for_commit->super_roots,
1945                sizeof(*root_backup) * BTRFS_NUM_BACKUP_ROOTS);
1946 }
1947
1948 /*
1949  * read_backup_root - Reads a backup root based on the passed priority. Prio 0
1950  * is the newest, prio 1/2/3 are 2nd newest/3rd newest/4th (oldest) backup roots
1951  *
1952  * fs_info - filesystem whose backup roots need to be read
1953  * priority - priority of backup root required
1954  *
1955  * Returns backup root index on success and -EINVAL otherwise.
1956  */
1957 static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority)
1958 {
1959         int backup_index = find_newest_super_backup(fs_info);
1960         struct btrfs_super_block *super = fs_info->super_copy;
1961         struct btrfs_root_backup *root_backup;
1962
1963         if (priority < BTRFS_NUM_BACKUP_ROOTS && backup_index >= 0) {
1964                 if (priority == 0)
1965                         return backup_index;
1966
1967                 backup_index = backup_index + BTRFS_NUM_BACKUP_ROOTS - priority;
1968                 backup_index %= BTRFS_NUM_BACKUP_ROOTS;
1969         } else {
1970                 return -EINVAL;
1971         }
1972
1973         root_backup = super->super_roots + backup_index;
1974
1975         btrfs_set_super_generation(super,
1976                                    btrfs_backup_tree_root_gen(root_backup));
1977         btrfs_set_super_root(super, btrfs_backup_tree_root(root_backup));
1978         btrfs_set_super_root_level(super,
1979                                    btrfs_backup_tree_root_level(root_backup));
1980         btrfs_set_super_bytes_used(super, btrfs_backup_bytes_used(root_backup));
1981
1982         /*
1983          * Fixme: the total bytes and num_devices need to match or we should
1984          * need a fsck
1985          */
1986         btrfs_set_super_total_bytes(super, btrfs_backup_total_bytes(root_backup));
1987         btrfs_set_super_num_devices(super, btrfs_backup_num_devices(root_backup));
1988
1989         return backup_index;
1990 }
1991
1992 /* helper to cleanup workers */
1993 static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info)
1994 {
1995         btrfs_destroy_workqueue(fs_info->fixup_workers);
1996         btrfs_destroy_workqueue(fs_info->delalloc_workers);
1997         btrfs_destroy_workqueue(fs_info->hipri_workers);
1998         btrfs_destroy_workqueue(fs_info->workers);
1999         if (fs_info->endio_workers)
2000                 destroy_workqueue(fs_info->endio_workers);
2001         if (fs_info->rmw_workers)
2002                 destroy_workqueue(fs_info->rmw_workers);
2003         if (fs_info->compressed_write_workers)
2004                 destroy_workqueue(fs_info->compressed_write_workers);
2005         btrfs_destroy_workqueue(fs_info->endio_write_workers);
2006         btrfs_destroy_workqueue(fs_info->endio_freespace_worker);
2007         btrfs_destroy_workqueue(fs_info->delayed_workers);
2008         btrfs_destroy_workqueue(fs_info->caching_workers);
2009         btrfs_destroy_workqueue(fs_info->flush_workers);
2010         btrfs_destroy_workqueue(fs_info->qgroup_rescan_workers);
2011         if (fs_info->discard_ctl.discard_workers)
2012                 destroy_workqueue(fs_info->discard_ctl.discard_workers);
2013         /*
2014          * Now that all other work queues are destroyed, we can safely destroy
2015          * the queues used for metadata I/O, since tasks from those other work
2016          * queues can do metadata I/O operations.
2017          */
2018         if (fs_info->endio_meta_workers)
2019                 destroy_workqueue(fs_info->endio_meta_workers);
2020 }
2021
2022 static void free_root_extent_buffers(struct btrfs_root *root)
2023 {
2024         if (root) {
2025                 free_extent_buffer(root->node);
2026                 free_extent_buffer(root->commit_root);
2027                 root->node = NULL;
2028                 root->commit_root = NULL;
2029         }
2030 }
2031
2032 static void free_global_root_pointers(struct btrfs_fs_info *fs_info)
2033 {
2034         struct btrfs_root *root, *tmp;
2035
2036         rbtree_postorder_for_each_entry_safe(root, tmp,
2037                                              &fs_info->global_root_tree,
2038                                              rb_node)
2039                 free_root_extent_buffers(root);
2040 }
2041
2042 /* helper to cleanup tree roots */
2043 static void free_root_pointers(struct btrfs_fs_info *info, bool free_chunk_root)
2044 {
2045         free_root_extent_buffers(info->tree_root);
2046
2047         free_global_root_pointers(info);
2048         free_root_extent_buffers(info->dev_root);
2049         free_root_extent_buffers(info->quota_root);
2050         free_root_extent_buffers(info->uuid_root);
2051         free_root_extent_buffers(info->fs_root);
2052         free_root_extent_buffers(info->data_reloc_root);
2053         free_root_extent_buffers(info->block_group_root);
2054         if (free_chunk_root)
2055                 free_root_extent_buffers(info->chunk_root);
2056 }
2057
2058 void btrfs_put_root(struct btrfs_root *root)
2059 {
2060         if (!root)
2061                 return;
2062
2063         if (refcount_dec_and_test(&root->refs)) {
2064                 WARN_ON(!RB_EMPTY_ROOT(&root->inode_tree));
2065                 WARN_ON(test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state));
2066                 if (root->anon_dev)
2067                         free_anon_bdev(root->anon_dev);
2068                 btrfs_drew_lock_destroy(&root->snapshot_lock);
2069                 free_root_extent_buffers(root);
2070 #ifdef CONFIG_BTRFS_DEBUG
2071                 spin_lock(&root->fs_info->fs_roots_radix_lock);
2072                 list_del_init(&root->leak_list);
2073                 spin_unlock(&root->fs_info->fs_roots_radix_lock);
2074 #endif
2075                 kfree(root);
2076         }
2077 }
2078
2079 void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info)
2080 {
2081         int ret;
2082         struct btrfs_root *gang[8];
2083         int i;
2084
2085         while (!list_empty(&fs_info->dead_roots)) {
2086                 gang[0] = list_entry(fs_info->dead_roots.next,
2087                                      struct btrfs_root, root_list);
2088                 list_del(&gang[0]->root_list);
2089
2090                 if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state))
2091                         btrfs_drop_and_free_fs_root(fs_info, gang[0]);
2092                 btrfs_put_root(gang[0]);
2093         }
2094
2095         while (1) {
2096                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
2097                                              (void **)gang, 0,
2098                                              ARRAY_SIZE(gang));
2099                 if (!ret)
2100                         break;
2101                 for (i = 0; i < ret; i++)
2102                         btrfs_drop_and_free_fs_root(fs_info, gang[i]);
2103         }
2104 }
2105
2106 static void btrfs_init_scrub(struct btrfs_fs_info *fs_info)
2107 {
2108         mutex_init(&fs_info->scrub_lock);
2109         atomic_set(&fs_info->scrubs_running, 0);
2110         atomic_set(&fs_info->scrub_pause_req, 0);
2111         atomic_set(&fs_info->scrubs_paused, 0);
2112         atomic_set(&fs_info->scrub_cancel_req, 0);
2113         init_waitqueue_head(&fs_info->scrub_pause_wait);
2114         refcount_set(&fs_info->scrub_workers_refcnt, 0);
2115 }
2116
2117 static void btrfs_init_balance(struct btrfs_fs_info *fs_info)
2118 {
2119         spin_lock_init(&fs_info->balance_lock);
2120         mutex_init(&fs_info->balance_mutex);
2121         atomic_set(&fs_info->balance_pause_req, 0);
2122         atomic_set(&fs_info->balance_cancel_req, 0);
2123         fs_info->balance_ctl = NULL;
2124         init_waitqueue_head(&fs_info->balance_wait_q);
2125         atomic_set(&fs_info->reloc_cancel_req, 0);
2126 }
2127
2128 static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info)
2129 {
2130         struct inode *inode = fs_info->btree_inode;
2131         unsigned long hash = btrfs_inode_hash(BTRFS_BTREE_INODE_OBJECTID,
2132                                               fs_info->tree_root);
2133
2134         inode->i_ino = BTRFS_BTREE_INODE_OBJECTID;
2135         set_nlink(inode, 1);
2136         /*
2137          * we set the i_size on the btree inode to the max possible int.
2138          * the real end of the address space is determined by all of
2139          * the devices in the system
2140          */
2141         inode->i_size = OFFSET_MAX;
2142         inode->i_mapping->a_ops = &btree_aops;
2143
2144         RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
2145         extent_io_tree_init(fs_info, &BTRFS_I(inode)->io_tree,
2146                             IO_TREE_BTREE_INODE_IO);
2147         extent_map_tree_init(&BTRFS_I(inode)->extent_tree);
2148
2149         BTRFS_I(inode)->root = btrfs_grab_root(fs_info->tree_root);
2150         BTRFS_I(inode)->location.objectid = BTRFS_BTREE_INODE_OBJECTID;
2151         BTRFS_I(inode)->location.type = 0;
2152         BTRFS_I(inode)->location.offset = 0;
2153         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
2154         __insert_inode_hash(inode, hash);
2155 }
2156
2157 static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info)
2158 {
2159         mutex_init(&fs_info->dev_replace.lock_finishing_cancel_unmount);
2160         init_rwsem(&fs_info->dev_replace.rwsem);
2161         init_waitqueue_head(&fs_info->dev_replace.replace_wait);
2162 }
2163
2164 static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info)
2165 {
2166         spin_lock_init(&fs_info->qgroup_lock);
2167         mutex_init(&fs_info->qgroup_ioctl_lock);
2168         fs_info->qgroup_tree = RB_ROOT;
2169         INIT_LIST_HEAD(&fs_info->dirty_qgroups);
2170         fs_info->qgroup_seq = 1;
2171         fs_info->qgroup_ulist = NULL;
2172         fs_info->qgroup_rescan_running = false;
2173         fs_info->qgroup_drop_subtree_thres = BTRFS_MAX_LEVEL;
2174         mutex_init(&fs_info->qgroup_rescan_lock);
2175 }
2176
2177 static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info)
2178 {
2179         u32 max_active = fs_info->thread_pool_size;
2180         unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND;
2181
2182         fs_info->workers =
2183                 btrfs_alloc_workqueue(fs_info, "worker", flags, max_active, 16);
2184         fs_info->hipri_workers =
2185                 btrfs_alloc_workqueue(fs_info, "worker-high",
2186                                       flags | WQ_HIGHPRI, max_active, 16);
2187
2188         fs_info->delalloc_workers =
2189                 btrfs_alloc_workqueue(fs_info, "delalloc",
2190                                       flags, max_active, 2);
2191
2192         fs_info->flush_workers =
2193                 btrfs_alloc_workqueue(fs_info, "flush_delalloc",
2194                                       flags, max_active, 0);
2195
2196         fs_info->caching_workers =
2197                 btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0);
2198
2199         fs_info->fixup_workers =
2200                 btrfs_alloc_workqueue(fs_info, "fixup", flags, 1, 0);
2201
2202         fs_info->endio_workers =
2203                 alloc_workqueue("btrfs-endio", flags, max_active);
2204         fs_info->endio_meta_workers =
2205                 alloc_workqueue("btrfs-endio-meta", flags, max_active);
2206         fs_info->rmw_workers = alloc_workqueue("btrfs-rmw", flags, max_active);
2207         fs_info->endio_write_workers =
2208                 btrfs_alloc_workqueue(fs_info, "endio-write", flags,
2209                                       max_active, 2);
2210         fs_info->compressed_write_workers =
2211                 alloc_workqueue("btrfs-compressed-write", flags, max_active);
2212         fs_info->endio_freespace_worker =
2213                 btrfs_alloc_workqueue(fs_info, "freespace-write", flags,
2214                                       max_active, 0);
2215         fs_info->delayed_workers =
2216                 btrfs_alloc_workqueue(fs_info, "delayed-meta", flags,
2217                                       max_active, 0);
2218         fs_info->qgroup_rescan_workers =
2219                 btrfs_alloc_workqueue(fs_info, "qgroup-rescan", flags, 1, 0);
2220         fs_info->discard_ctl.discard_workers =
2221                 alloc_workqueue("btrfs_discard", WQ_UNBOUND | WQ_FREEZABLE, 1);
2222
2223         if (!(fs_info->workers && fs_info->hipri_workers &&
2224               fs_info->delalloc_workers && fs_info->flush_workers &&
2225               fs_info->endio_workers && fs_info->endio_meta_workers &&
2226               fs_info->compressed_write_workers &&
2227               fs_info->endio_write_workers &&
2228               fs_info->endio_freespace_worker && fs_info->rmw_workers &&
2229               fs_info->caching_workers && fs_info->fixup_workers &&
2230               fs_info->delayed_workers && fs_info->qgroup_rescan_workers &&
2231               fs_info->discard_ctl.discard_workers)) {
2232                 return -ENOMEM;
2233         }
2234
2235         return 0;
2236 }
2237
2238 static int btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type)
2239 {
2240         struct crypto_shash *csum_shash;
2241         const char *csum_driver = btrfs_super_csum_driver(csum_type);
2242
2243         csum_shash = crypto_alloc_shash(csum_driver, 0, 0);
2244
2245         if (IS_ERR(csum_shash)) {
2246                 btrfs_err(fs_info, "error allocating %s hash for checksum",
2247                           csum_driver);
2248                 return PTR_ERR(csum_shash);
2249         }
2250
2251         fs_info->csum_shash = csum_shash;
2252
2253         btrfs_info(fs_info, "using %s (%s) checksum algorithm",
2254                         btrfs_super_csum_name(csum_type),
2255                         crypto_shash_driver_name(csum_shash));
2256         return 0;
2257 }
2258
2259 static int btrfs_replay_log(struct btrfs_fs_info *fs_info,
2260                             struct btrfs_fs_devices *fs_devices)
2261 {
2262         int ret;
2263         struct btrfs_tree_parent_check check = { 0 };
2264         struct btrfs_root *log_tree_root;
2265         struct btrfs_super_block *disk_super = fs_info->super_copy;
2266         u64 bytenr = btrfs_super_log_root(disk_super);
2267         int level = btrfs_super_log_root_level(disk_super);
2268
2269         if (fs_devices->rw_devices == 0) {
2270                 btrfs_warn(fs_info, "log replay required on RO media");
2271                 return -EIO;
2272         }
2273
2274         log_tree_root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID,
2275                                          GFP_KERNEL);
2276         if (!log_tree_root)
2277                 return -ENOMEM;
2278
2279         check.level = level;
2280         check.transid = fs_info->generation + 1;
2281         check.owner_root = BTRFS_TREE_LOG_OBJECTID;
2282         log_tree_root->node = read_tree_block(fs_info, bytenr, &check);
2283         if (IS_ERR(log_tree_root->node)) {
2284                 btrfs_warn(fs_info, "failed to read log tree");
2285                 ret = PTR_ERR(log_tree_root->node);
2286                 log_tree_root->node = NULL;
2287                 btrfs_put_root(log_tree_root);
2288                 return ret;
2289         }
2290         if (!extent_buffer_uptodate(log_tree_root->node)) {
2291                 btrfs_err(fs_info, "failed to read log tree");
2292                 btrfs_put_root(log_tree_root);
2293                 return -EIO;
2294         }
2295
2296         /* returns with log_tree_root freed on success */
2297         ret = btrfs_recover_log_trees(log_tree_root);
2298         if (ret) {
2299                 btrfs_handle_fs_error(fs_info, ret,
2300                                       "Failed to recover log tree");
2301                 btrfs_put_root(log_tree_root);
2302                 return ret;
2303         }
2304
2305         if (sb_rdonly(fs_info->sb)) {
2306                 ret = btrfs_commit_super(fs_info);
2307                 if (ret)
2308                         return ret;
2309         }
2310
2311         return 0;
2312 }
2313
2314 static int load_global_roots_objectid(struct btrfs_root *tree_root,
2315                                       struct btrfs_path *path, u64 objectid,
2316                                       const char *name)
2317 {
2318         struct btrfs_fs_info *fs_info = tree_root->fs_info;
2319         struct btrfs_root *root;
2320         u64 max_global_id = 0;
2321         int ret;
2322         struct btrfs_key key = {
2323                 .objectid = objectid,
2324                 .type = BTRFS_ROOT_ITEM_KEY,
2325                 .offset = 0,
2326         };
2327         bool found = false;
2328
2329         /* If we have IGNOREDATACSUMS skip loading these roots. */
2330         if (objectid == BTRFS_CSUM_TREE_OBJECTID &&
2331             btrfs_test_opt(fs_info, IGNOREDATACSUMS)) {
2332                 set_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
2333                 return 0;
2334         }
2335
2336         while (1) {
2337                 ret = btrfs_search_slot(NULL, tree_root, &key, path, 0, 0);
2338                 if (ret < 0)
2339                         break;
2340
2341                 if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
2342                         ret = btrfs_next_leaf(tree_root, path);
2343                         if (ret) {
2344                                 if (ret > 0)
2345                                         ret = 0;
2346                                 break;
2347                         }
2348                 }
2349                 ret = 0;
2350
2351                 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
2352                 if (key.objectid != objectid)
2353                         break;
2354                 btrfs_release_path(path);
2355
2356                 /*
2357                  * Just worry about this for extent tree, it'll be the same for
2358                  * everybody.
2359                  */
2360                 if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
2361                         max_global_id = max(max_global_id, key.offset);
2362
2363                 found = true;
2364                 root = read_tree_root_path(tree_root, path, &key);
2365                 if (IS_ERR(root)) {
2366                         if (!btrfs_test_opt(fs_info, IGNOREBADROOTS))
2367                                 ret = PTR_ERR(root);
2368                         break;
2369                 }
2370                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2371                 ret = btrfs_global_root_insert(root);
2372                 if (ret) {
2373                         btrfs_put_root(root);
2374                         break;
2375                 }
2376                 key.offset++;
2377         }
2378         btrfs_release_path(path);
2379
2380         if (objectid == BTRFS_EXTENT_TREE_OBJECTID)
2381                 fs_info->nr_global_roots = max_global_id + 1;
2382
2383         if (!found || ret) {
2384                 if (objectid == BTRFS_CSUM_TREE_OBJECTID)
2385                         set_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state);
2386
2387                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS))
2388                         ret = ret ? ret : -ENOENT;
2389                 else
2390                         ret = 0;
2391                 btrfs_err(fs_info, "failed to load root %s", name);
2392         }
2393         return ret;
2394 }
2395
2396 static int load_global_roots(struct btrfs_root *tree_root)
2397 {
2398         struct btrfs_path *path;
2399         int ret = 0;
2400
2401         path = btrfs_alloc_path();
2402         if (!path)
2403                 return -ENOMEM;
2404
2405         ret = load_global_roots_objectid(tree_root, path,
2406                                          BTRFS_EXTENT_TREE_OBJECTID, "extent");
2407         if (ret)
2408                 goto out;
2409         ret = load_global_roots_objectid(tree_root, path,
2410                                          BTRFS_CSUM_TREE_OBJECTID, "csum");
2411         if (ret)
2412                 goto out;
2413         if (!btrfs_fs_compat_ro(tree_root->fs_info, FREE_SPACE_TREE))
2414                 goto out;
2415         ret = load_global_roots_objectid(tree_root, path,
2416                                          BTRFS_FREE_SPACE_TREE_OBJECTID,
2417                                          "free space");
2418 out:
2419         btrfs_free_path(path);
2420         return ret;
2421 }
2422
2423 static int btrfs_read_roots(struct btrfs_fs_info *fs_info)
2424 {
2425         struct btrfs_root *tree_root = fs_info->tree_root;
2426         struct btrfs_root *root;
2427         struct btrfs_key location;
2428         int ret;
2429
2430         BUG_ON(!fs_info->tree_root);
2431
2432         ret = load_global_roots(tree_root);
2433         if (ret)
2434                 return ret;
2435
2436         location.type = BTRFS_ROOT_ITEM_KEY;
2437         location.offset = 0;
2438
2439         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) {
2440                 location.objectid = BTRFS_BLOCK_GROUP_TREE_OBJECTID;
2441                 root = btrfs_read_tree_root(tree_root, &location);
2442                 if (IS_ERR(root)) {
2443                         if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2444                                 ret = PTR_ERR(root);
2445                                 goto out;
2446                         }
2447                 } else {
2448                         set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2449                         fs_info->block_group_root = root;
2450                 }
2451         }
2452
2453         location.objectid = BTRFS_DEV_TREE_OBJECTID;
2454         root = btrfs_read_tree_root(tree_root, &location);
2455         if (IS_ERR(root)) {
2456                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2457                         ret = PTR_ERR(root);
2458                         goto out;
2459                 }
2460         } else {
2461                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2462                 fs_info->dev_root = root;
2463         }
2464         /* Initialize fs_info for all devices in any case */
2465         ret = btrfs_init_devices_late(fs_info);
2466         if (ret)
2467                 goto out;
2468
2469         /*
2470          * This tree can share blocks with some other fs tree during relocation
2471          * and we need a proper setup by btrfs_get_fs_root
2472          */
2473         root = btrfs_get_fs_root(tree_root->fs_info,
2474                                  BTRFS_DATA_RELOC_TREE_OBJECTID, true);
2475         if (IS_ERR(root)) {
2476                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2477                         ret = PTR_ERR(root);
2478                         goto out;
2479                 }
2480         } else {
2481                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2482                 fs_info->data_reloc_root = root;
2483         }
2484
2485         location.objectid = BTRFS_QUOTA_TREE_OBJECTID;
2486         root = btrfs_read_tree_root(tree_root, &location);
2487         if (!IS_ERR(root)) {
2488                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2489                 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
2490                 fs_info->quota_root = root;
2491         }
2492
2493         location.objectid = BTRFS_UUID_TREE_OBJECTID;
2494         root = btrfs_read_tree_root(tree_root, &location);
2495         if (IS_ERR(root)) {
2496                 if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) {
2497                         ret = PTR_ERR(root);
2498                         if (ret != -ENOENT)
2499                                 goto out;
2500                 }
2501         } else {
2502                 set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state);
2503                 fs_info->uuid_root = root;
2504         }
2505
2506         return 0;
2507 out:
2508         btrfs_warn(fs_info, "failed to read root (objectid=%llu): %d",
2509                    location.objectid, ret);
2510         return ret;
2511 }
2512
2513 /*
2514  * Real super block validation
2515  * NOTE: super csum type and incompat features will not be checked here.
2516  *
2517  * @sb:         super block to check
2518  * @mirror_num: the super block number to check its bytenr:
2519  *              0       the primary (1st) sb
2520  *              1, 2    2nd and 3rd backup copy
2521  *             -1       skip bytenr check
2522  */
2523 int btrfs_validate_super(struct btrfs_fs_info *fs_info,
2524                          struct btrfs_super_block *sb, int mirror_num)
2525 {
2526         u64 nodesize = btrfs_super_nodesize(sb);
2527         u64 sectorsize = btrfs_super_sectorsize(sb);
2528         int ret = 0;
2529
2530         if (btrfs_super_magic(sb) != BTRFS_MAGIC) {
2531                 btrfs_err(fs_info, "no valid FS found");
2532                 ret = -EINVAL;
2533         }
2534         if (btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP) {
2535                 btrfs_err(fs_info, "unrecognized or unsupported super flag: %llu",
2536                                 btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP);
2537                 ret = -EINVAL;
2538         }
2539         if (btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL) {
2540                 btrfs_err(fs_info, "tree_root level too big: %d >= %d",
2541                                 btrfs_super_root_level(sb), BTRFS_MAX_LEVEL);
2542                 ret = -EINVAL;
2543         }
2544         if (btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL) {
2545                 btrfs_err(fs_info, "chunk_root level too big: %d >= %d",
2546                                 btrfs_super_chunk_root_level(sb), BTRFS_MAX_LEVEL);
2547                 ret = -EINVAL;
2548         }
2549         if (btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL) {
2550                 btrfs_err(fs_info, "log_root level too big: %d >= %d",
2551                                 btrfs_super_log_root_level(sb), BTRFS_MAX_LEVEL);
2552                 ret = -EINVAL;
2553         }
2554
2555         /*
2556          * Check sectorsize and nodesize first, other check will need it.
2557          * Check all possible sectorsize(4K, 8K, 16K, 32K, 64K) here.
2558          */
2559         if (!is_power_of_2(sectorsize) || sectorsize < 4096 ||
2560             sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2561                 btrfs_err(fs_info, "invalid sectorsize %llu", sectorsize);
2562                 ret = -EINVAL;
2563         }
2564
2565         /*
2566          * We only support at most two sectorsizes: 4K and PAGE_SIZE.
2567          *
2568          * We can support 16K sectorsize with 64K page size without problem,
2569          * but such sectorsize/pagesize combination doesn't make much sense.
2570          * 4K will be our future standard, PAGE_SIZE is supported from the very
2571          * beginning.
2572          */
2573         if (sectorsize > PAGE_SIZE || (sectorsize != SZ_4K && sectorsize != PAGE_SIZE)) {
2574                 btrfs_err(fs_info,
2575                         "sectorsize %llu not yet supported for page size %lu",
2576                         sectorsize, PAGE_SIZE);
2577                 ret = -EINVAL;
2578         }
2579
2580         if (!is_power_of_2(nodesize) || nodesize < sectorsize ||
2581             nodesize > BTRFS_MAX_METADATA_BLOCKSIZE) {
2582                 btrfs_err(fs_info, "invalid nodesize %llu", nodesize);
2583                 ret = -EINVAL;
2584         }
2585         if (nodesize != le32_to_cpu(sb->__unused_leafsize)) {
2586                 btrfs_err(fs_info, "invalid leafsize %u, should be %llu",
2587                           le32_to_cpu(sb->__unused_leafsize), nodesize);
2588                 ret = -EINVAL;
2589         }
2590
2591         /* Root alignment check */
2592         if (!IS_ALIGNED(btrfs_super_root(sb), sectorsize)) {
2593                 btrfs_warn(fs_info, "tree_root block unaligned: %llu",
2594                            btrfs_super_root(sb));
2595                 ret = -EINVAL;
2596         }
2597         if (!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize)) {
2598                 btrfs_warn(fs_info, "chunk_root block unaligned: %llu",
2599                            btrfs_super_chunk_root(sb));
2600                 ret = -EINVAL;
2601         }
2602         if (!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize)) {
2603                 btrfs_warn(fs_info, "log_root block unaligned: %llu",
2604                            btrfs_super_log_root(sb));
2605                 ret = -EINVAL;
2606         }
2607
2608         if (memcmp(fs_info->fs_devices->fsid, fs_info->super_copy->fsid,
2609                    BTRFS_FSID_SIZE)) {
2610                 btrfs_err(fs_info,
2611                 "superblock fsid doesn't match fsid of fs_devices: %pU != %pU",
2612                         fs_info->super_copy->fsid, fs_info->fs_devices->fsid);
2613                 ret = -EINVAL;
2614         }
2615
2616         if (btrfs_fs_incompat(fs_info, METADATA_UUID) &&
2617             memcmp(fs_info->fs_devices->metadata_uuid,
2618                    fs_info->super_copy->metadata_uuid, BTRFS_FSID_SIZE)) {
2619                 btrfs_err(fs_info,
2620 "superblock metadata_uuid doesn't match metadata uuid of fs_devices: %pU != %pU",
2621                         fs_info->super_copy->metadata_uuid,
2622                         fs_info->fs_devices->metadata_uuid);
2623                 ret = -EINVAL;
2624         }
2625
2626         /*
2627          * Artificial requirement for block-group-tree to force newer features
2628          * (free-space-tree, no-holes) so the test matrix is smaller.
2629          */
2630         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) &&
2631             (!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) ||
2632              !btrfs_fs_incompat(fs_info, NO_HOLES))) {
2633                 btrfs_err(fs_info,
2634                 "block-group-tree feature requires fres-space-tree and no-holes");
2635                 ret = -EINVAL;
2636         }
2637
2638         if (memcmp(fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid,
2639                    BTRFS_FSID_SIZE) != 0) {
2640                 btrfs_err(fs_info,
2641                         "dev_item UUID does not match metadata fsid: %pU != %pU",
2642                         fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid);
2643                 ret = -EINVAL;
2644         }
2645
2646         /*
2647          * Hint to catch really bogus numbers, bitflips or so, more exact checks are
2648          * done later
2649          */
2650         if (btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb)) {
2651                 btrfs_err(fs_info, "bytes_used is too small %llu",
2652                           btrfs_super_bytes_used(sb));
2653                 ret = -EINVAL;
2654         }
2655         if (!is_power_of_2(btrfs_super_stripesize(sb))) {
2656                 btrfs_err(fs_info, "invalid stripesize %u",
2657                           btrfs_super_stripesize(sb));
2658                 ret = -EINVAL;
2659         }
2660         if (btrfs_super_num_devices(sb) > (1UL << 31))
2661                 btrfs_warn(fs_info, "suspicious number of devices: %llu",
2662                            btrfs_super_num_devices(sb));
2663         if (btrfs_super_num_devices(sb) == 0) {
2664                 btrfs_err(fs_info, "number of devices is 0");
2665                 ret = -EINVAL;
2666         }
2667
2668         if (mirror_num >= 0 &&
2669             btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num)) {
2670                 btrfs_err(fs_info, "super offset mismatch %llu != %u",
2671                           btrfs_super_bytenr(sb), BTRFS_SUPER_INFO_OFFSET);
2672                 ret = -EINVAL;
2673         }
2674
2675         /*
2676          * Obvious sys_chunk_array corruptions, it must hold at least one key
2677          * and one chunk
2678          */
2679         if (btrfs_super_sys_array_size(sb) > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
2680                 btrfs_err(fs_info, "system chunk array too big %u > %u",
2681                           btrfs_super_sys_array_size(sb),
2682                           BTRFS_SYSTEM_CHUNK_ARRAY_SIZE);
2683                 ret = -EINVAL;
2684         }
2685         if (btrfs_super_sys_array_size(sb) < sizeof(struct btrfs_disk_key)
2686                         + sizeof(struct btrfs_chunk)) {
2687                 btrfs_err(fs_info, "system chunk array too small %u < %zu",
2688                           btrfs_super_sys_array_size(sb),
2689                           sizeof(struct btrfs_disk_key)
2690                           + sizeof(struct btrfs_chunk));
2691                 ret = -EINVAL;
2692         }
2693
2694         /*
2695          * The generation is a global counter, we'll trust it more than the others
2696          * but it's still possible that it's the one that's wrong.
2697          */
2698         if (btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb))
2699                 btrfs_warn(fs_info,
2700                         "suspicious: generation < chunk_root_generation: %llu < %llu",
2701                         btrfs_super_generation(sb),
2702                         btrfs_super_chunk_root_generation(sb));
2703         if (btrfs_super_generation(sb) < btrfs_super_cache_generation(sb)
2704             && btrfs_super_cache_generation(sb) != (u64)-1)
2705                 btrfs_warn(fs_info,
2706                         "suspicious: generation < cache_generation: %llu < %llu",
2707                         btrfs_super_generation(sb),
2708                         btrfs_super_cache_generation(sb));
2709
2710         return ret;
2711 }
2712
2713 /*
2714  * Validation of super block at mount time.
2715  * Some checks already done early at mount time, like csum type and incompat
2716  * flags will be skipped.
2717  */
2718 static int btrfs_validate_mount_super(struct btrfs_fs_info *fs_info)
2719 {
2720         return btrfs_validate_super(fs_info, fs_info->super_copy, 0);
2721 }
2722
2723 /*
2724  * Validation of super block at write time.
2725  * Some checks like bytenr check will be skipped as their values will be
2726  * overwritten soon.
2727  * Extra checks like csum type and incompat flags will be done here.
2728  */
2729 static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info,
2730                                       struct btrfs_super_block *sb)
2731 {
2732         int ret;
2733
2734         ret = btrfs_validate_super(fs_info, sb, -1);
2735         if (ret < 0)
2736                 goto out;
2737         if (!btrfs_supported_super_csum(btrfs_super_csum_type(sb))) {
2738                 ret = -EUCLEAN;
2739                 btrfs_err(fs_info, "invalid csum type, has %u want %u",
2740                           btrfs_super_csum_type(sb), BTRFS_CSUM_TYPE_CRC32);
2741                 goto out;
2742         }
2743         if (btrfs_super_incompat_flags(sb) & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
2744                 ret = -EUCLEAN;
2745                 btrfs_err(fs_info,
2746                 "invalid incompat flags, has 0x%llx valid mask 0x%llx",
2747                           btrfs_super_incompat_flags(sb),
2748                           (unsigned long long)BTRFS_FEATURE_INCOMPAT_SUPP);
2749                 goto out;
2750         }
2751 out:
2752         if (ret < 0)
2753                 btrfs_err(fs_info,
2754                 "super block corruption detected before writing it to disk");
2755         return ret;
2756 }
2757
2758 static int load_super_root(struct btrfs_root *root, u64 bytenr, u64 gen, int level)
2759 {
2760         struct btrfs_tree_parent_check check = {
2761                 .level = level,
2762                 .transid = gen,
2763                 .owner_root = root->root_key.objectid
2764         };
2765         int ret = 0;
2766
2767         root->node = read_tree_block(root->fs_info, bytenr, &check);
2768         if (IS_ERR(root->node)) {
2769                 ret = PTR_ERR(root->node);
2770                 root->node = NULL;
2771                 return ret;
2772         }
2773         if (!extent_buffer_uptodate(root->node)) {
2774                 free_extent_buffer(root->node);
2775                 root->node = NULL;
2776                 return -EIO;
2777         }
2778
2779         btrfs_set_root_node(&root->root_item, root->node);
2780         root->commit_root = btrfs_root_node(root);
2781         btrfs_set_root_refs(&root->root_item, 1);
2782         return ret;
2783 }
2784
2785 static int load_important_roots(struct btrfs_fs_info *fs_info)
2786 {
2787         struct btrfs_super_block *sb = fs_info->super_copy;
2788         u64 gen, bytenr;
2789         int level, ret;
2790
2791         bytenr = btrfs_super_root(sb);
2792         gen = btrfs_super_generation(sb);
2793         level = btrfs_super_root_level(sb);
2794         ret = load_super_root(fs_info->tree_root, bytenr, gen, level);
2795         if (ret) {
2796                 btrfs_warn(fs_info, "couldn't read tree root");
2797                 return ret;
2798         }
2799         return 0;
2800 }
2801
2802 static int __cold init_tree_roots(struct btrfs_fs_info *fs_info)
2803 {
2804         int backup_index = find_newest_super_backup(fs_info);
2805         struct btrfs_super_block *sb = fs_info->super_copy;
2806         struct btrfs_root *tree_root = fs_info->tree_root;
2807         bool handle_error = false;
2808         int ret = 0;
2809         int i;
2810
2811         for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) {
2812                 if (handle_error) {
2813                         if (!IS_ERR(tree_root->node))
2814                                 free_extent_buffer(tree_root->node);
2815                         tree_root->node = NULL;
2816
2817                         if (!btrfs_test_opt(fs_info, USEBACKUPROOT))
2818                                 break;
2819
2820                         free_root_pointers(fs_info, 0);
2821
2822                         /*
2823                          * Don't use the log in recovery mode, it won't be
2824                          * valid
2825                          */
2826                         btrfs_set_super_log_root(sb, 0);
2827
2828                         /* We can't trust the free space cache either */
2829                         btrfs_set_opt(fs_info->mount_opt, CLEAR_CACHE);
2830
2831                         ret = read_backup_root(fs_info, i);
2832                         backup_index = ret;
2833                         if (ret < 0)
2834                                 return ret;
2835                 }
2836
2837                 ret = load_important_roots(fs_info);
2838                 if (ret) {
2839                         handle_error = true;
2840                         continue;
2841                 }
2842
2843                 /*
2844                  * No need to hold btrfs_root::objectid_mutex since the fs
2845                  * hasn't been fully initialised and we are the only user
2846                  */
2847                 ret = btrfs_init_root_free_objectid(tree_root);
2848                 if (ret < 0) {
2849                         handle_error = true;
2850                         continue;
2851                 }
2852
2853                 ASSERT(tree_root->free_objectid <= BTRFS_LAST_FREE_OBJECTID);
2854
2855                 ret = btrfs_read_roots(fs_info);
2856                 if (ret < 0) {
2857                         handle_error = true;
2858                         continue;
2859                 }
2860
2861                 /* All successful */
2862                 fs_info->generation = btrfs_header_generation(tree_root->node);
2863                 fs_info->last_trans_committed = fs_info->generation;
2864                 fs_info->last_reloc_trans = 0;
2865
2866                 /* Always begin writing backup roots after the one being used */
2867                 if (backup_index < 0) {
2868                         fs_info->backup_root_index = 0;
2869                 } else {
2870                         fs_info->backup_root_index = backup_index + 1;
2871                         fs_info->backup_root_index %= BTRFS_NUM_BACKUP_ROOTS;
2872                 }
2873                 break;
2874         }
2875
2876         return ret;
2877 }
2878
2879 void btrfs_init_fs_info(struct btrfs_fs_info *fs_info)
2880 {
2881         INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC);
2882         INIT_RADIX_TREE(&fs_info->buffer_radix, GFP_ATOMIC);
2883         INIT_LIST_HEAD(&fs_info->trans_list);
2884         INIT_LIST_HEAD(&fs_info->dead_roots);
2885         INIT_LIST_HEAD(&fs_info->delayed_iputs);
2886         INIT_LIST_HEAD(&fs_info->delalloc_roots);
2887         INIT_LIST_HEAD(&fs_info->caching_block_groups);
2888         spin_lock_init(&fs_info->delalloc_root_lock);
2889         spin_lock_init(&fs_info->trans_lock);
2890         spin_lock_init(&fs_info->fs_roots_radix_lock);
2891         spin_lock_init(&fs_info->delayed_iput_lock);
2892         spin_lock_init(&fs_info->defrag_inodes_lock);
2893         spin_lock_init(&fs_info->super_lock);
2894         spin_lock_init(&fs_info->buffer_lock);
2895         spin_lock_init(&fs_info->unused_bgs_lock);
2896         spin_lock_init(&fs_info->treelog_bg_lock);
2897         spin_lock_init(&fs_info->zone_active_bgs_lock);
2898         spin_lock_init(&fs_info->relocation_bg_lock);
2899         rwlock_init(&fs_info->tree_mod_log_lock);
2900         rwlock_init(&fs_info->global_root_lock);
2901         mutex_init(&fs_info->unused_bg_unpin_mutex);
2902         mutex_init(&fs_info->reclaim_bgs_lock);
2903         mutex_init(&fs_info->reloc_mutex);
2904         mutex_init(&fs_info->delalloc_root_mutex);
2905         mutex_init(&fs_info->zoned_meta_io_lock);
2906         mutex_init(&fs_info->zoned_data_reloc_io_lock);
2907         seqlock_init(&fs_info->profiles_lock);
2908
2909         btrfs_lockdep_init_map(fs_info, btrfs_trans_num_writers);
2910         btrfs_lockdep_init_map(fs_info, btrfs_trans_num_extwriters);
2911         btrfs_lockdep_init_map(fs_info, btrfs_trans_pending_ordered);
2912         btrfs_lockdep_init_map(fs_info, btrfs_ordered_extent);
2913         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_commit_start,
2914                                      BTRFS_LOCKDEP_TRANS_COMMIT_START);
2915         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_unblocked,
2916                                      BTRFS_LOCKDEP_TRANS_UNBLOCKED);
2917         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_super_committed,
2918                                      BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED);
2919         btrfs_state_lockdep_init_map(fs_info, btrfs_trans_completed,
2920                                      BTRFS_LOCKDEP_TRANS_COMPLETED);
2921
2922         INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
2923         INIT_LIST_HEAD(&fs_info->space_info);
2924         INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
2925         INIT_LIST_HEAD(&fs_info->unused_bgs);
2926         INIT_LIST_HEAD(&fs_info->reclaim_bgs);
2927         INIT_LIST_HEAD(&fs_info->zone_active_bgs);
2928 #ifdef CONFIG_BTRFS_DEBUG
2929         INIT_LIST_HEAD(&fs_info->allocated_roots);
2930         INIT_LIST_HEAD(&fs_info->allocated_ebs);
2931         spin_lock_init(&fs_info->eb_leak_lock);
2932 #endif
2933         extent_map_tree_init(&fs_info->mapping_tree);
2934         btrfs_init_block_rsv(&fs_info->global_block_rsv,
2935                              BTRFS_BLOCK_RSV_GLOBAL);
2936         btrfs_init_block_rsv(&fs_info->trans_block_rsv, BTRFS_BLOCK_RSV_TRANS);
2937         btrfs_init_block_rsv(&fs_info->chunk_block_rsv, BTRFS_BLOCK_RSV_CHUNK);
2938         btrfs_init_block_rsv(&fs_info->empty_block_rsv, BTRFS_BLOCK_RSV_EMPTY);
2939         btrfs_init_block_rsv(&fs_info->delayed_block_rsv,
2940                              BTRFS_BLOCK_RSV_DELOPS);
2941         btrfs_init_block_rsv(&fs_info->delayed_refs_rsv,
2942                              BTRFS_BLOCK_RSV_DELREFS);
2943
2944         atomic_set(&fs_info->async_delalloc_pages, 0);
2945         atomic_set(&fs_info->defrag_running, 0);
2946         atomic_set(&fs_info->nr_delayed_iputs, 0);
2947         atomic64_set(&fs_info->tree_mod_seq, 0);
2948         fs_info->global_root_tree = RB_ROOT;
2949         fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE;
2950         fs_info->metadata_ratio = 0;
2951         fs_info->defrag_inodes = RB_ROOT;
2952         atomic64_set(&fs_info->free_chunk_space, 0);
2953         fs_info->tree_mod_log = RB_ROOT;
2954         fs_info->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL;
2955         fs_info->avg_delayed_ref_runtime = NSEC_PER_SEC >> 6; /* div by 64 */
2956         btrfs_init_ref_verify(fs_info);
2957
2958         fs_info->thread_pool_size = min_t(unsigned long,
2959                                           num_online_cpus() + 2, 8);
2960
2961         INIT_LIST_HEAD(&fs_info->ordered_roots);
2962         spin_lock_init(&fs_info->ordered_root_lock);
2963
2964         btrfs_init_scrub(fs_info);
2965 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
2966         fs_info->check_integrity_print_mask = 0;
2967 #endif
2968         btrfs_init_balance(fs_info);
2969         btrfs_init_async_reclaim_work(fs_info);
2970
2971         rwlock_init(&fs_info->block_group_cache_lock);
2972         fs_info->block_group_cache_tree = RB_ROOT_CACHED;
2973
2974         extent_io_tree_init(fs_info, &fs_info->excluded_extents,
2975                             IO_TREE_FS_EXCLUDED_EXTENTS);
2976
2977         mutex_init(&fs_info->ordered_operations_mutex);
2978         mutex_init(&fs_info->tree_log_mutex);
2979         mutex_init(&fs_info->chunk_mutex);
2980         mutex_init(&fs_info->transaction_kthread_mutex);
2981         mutex_init(&fs_info->cleaner_mutex);
2982         mutex_init(&fs_info->ro_block_group_mutex);
2983         init_rwsem(&fs_info->commit_root_sem);
2984         init_rwsem(&fs_info->cleanup_work_sem);
2985         init_rwsem(&fs_info->subvol_sem);
2986         sema_init(&fs_info->uuid_tree_rescan_sem, 1);
2987
2988         btrfs_init_dev_replace_locks(fs_info);
2989         btrfs_init_qgroup(fs_info);
2990         btrfs_discard_init(fs_info);
2991
2992         btrfs_init_free_cluster(&fs_info->meta_alloc_cluster);
2993         btrfs_init_free_cluster(&fs_info->data_alloc_cluster);
2994
2995         init_waitqueue_head(&fs_info->transaction_throttle);
2996         init_waitqueue_head(&fs_info->transaction_wait);
2997         init_waitqueue_head(&fs_info->transaction_blocked_wait);
2998         init_waitqueue_head(&fs_info->async_submit_wait);
2999         init_waitqueue_head(&fs_info->delayed_iputs_wait);
3000
3001         /* Usable values until the real ones are cached from the superblock */
3002         fs_info->nodesize = 4096;
3003         fs_info->sectorsize = 4096;
3004         fs_info->sectorsize_bits = ilog2(4096);
3005         fs_info->stripesize = 4096;
3006
3007         fs_info->max_extent_size = BTRFS_MAX_EXTENT_SIZE;
3008
3009         spin_lock_init(&fs_info->swapfile_pins_lock);
3010         fs_info->swapfile_pins = RB_ROOT;
3011
3012         fs_info->bg_reclaim_threshold = BTRFS_DEFAULT_RECLAIM_THRESH;
3013         INIT_WORK(&fs_info->reclaim_bgs_work, btrfs_reclaim_bgs_work);
3014 }
3015
3016 static int init_mount_fs_info(struct btrfs_fs_info *fs_info, struct super_block *sb)
3017 {
3018         int ret;
3019
3020         fs_info->sb = sb;
3021         sb->s_blocksize = BTRFS_BDEV_BLOCKSIZE;
3022         sb->s_blocksize_bits = blksize_bits(BTRFS_BDEV_BLOCKSIZE);
3023
3024         ret = percpu_counter_init(&fs_info->ordered_bytes, 0, GFP_KERNEL);
3025         if (ret)
3026                 return ret;
3027
3028         ret = percpu_counter_init(&fs_info->dirty_metadata_bytes, 0, GFP_KERNEL);
3029         if (ret)
3030                 return ret;
3031
3032         fs_info->dirty_metadata_batch = PAGE_SIZE *
3033                                         (1 + ilog2(nr_cpu_ids));
3034
3035         ret = percpu_counter_init(&fs_info->delalloc_bytes, 0, GFP_KERNEL);
3036         if (ret)
3037                 return ret;
3038
3039         ret = percpu_counter_init(&fs_info->dev_replace.bio_counter, 0,
3040                         GFP_KERNEL);
3041         if (ret)
3042                 return ret;
3043
3044         fs_info->delayed_root = kmalloc(sizeof(struct btrfs_delayed_root),
3045                                         GFP_KERNEL);
3046         if (!fs_info->delayed_root)
3047                 return -ENOMEM;
3048         btrfs_init_delayed_root(fs_info->delayed_root);
3049
3050         if (sb_rdonly(sb))
3051                 set_bit(BTRFS_FS_STATE_RO, &fs_info->fs_state);
3052
3053         return btrfs_alloc_stripe_hash_table(fs_info);
3054 }
3055
3056 static int btrfs_uuid_rescan_kthread(void *data)
3057 {
3058         struct btrfs_fs_info *fs_info = data;
3059         int ret;
3060
3061         /*
3062          * 1st step is to iterate through the existing UUID tree and
3063          * to delete all entries that contain outdated data.
3064          * 2nd step is to add all missing entries to the UUID tree.
3065          */
3066         ret = btrfs_uuid_tree_iterate(fs_info);
3067         if (ret < 0) {
3068                 if (ret != -EINTR)
3069                         btrfs_warn(fs_info, "iterating uuid_tree failed %d",
3070                                    ret);
3071                 up(&fs_info->uuid_tree_rescan_sem);
3072                 return ret;
3073         }
3074         return btrfs_uuid_scan_kthread(data);
3075 }
3076
3077 static int btrfs_check_uuid_tree(struct btrfs_fs_info *fs_info)
3078 {
3079         struct task_struct *task;
3080
3081         down(&fs_info->uuid_tree_rescan_sem);
3082         task = kthread_run(btrfs_uuid_rescan_kthread, fs_info, "btrfs-uuid");
3083         if (IS_ERR(task)) {
3084                 /* fs_info->update_uuid_tree_gen remains 0 in all error case */
3085                 btrfs_warn(fs_info, "failed to start uuid_rescan task");
3086                 up(&fs_info->uuid_tree_rescan_sem);
3087                 return PTR_ERR(task);
3088         }
3089
3090         return 0;
3091 }
3092
3093 /*
3094  * Some options only have meaning at mount time and shouldn't persist across
3095  * remounts, or be displayed. Clear these at the end of mount and remount
3096  * code paths.
3097  */
3098 void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info)
3099 {
3100         btrfs_clear_opt(fs_info->mount_opt, USEBACKUPROOT);
3101         btrfs_clear_opt(fs_info->mount_opt, CLEAR_CACHE);
3102 }
3103
3104 /*
3105  * Mounting logic specific to read-write file systems. Shared by open_ctree
3106  * and btrfs_remount when remounting from read-only to read-write.
3107  */
3108 int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info)
3109 {
3110         int ret;
3111         const bool cache_opt = btrfs_test_opt(fs_info, SPACE_CACHE);
3112         bool clear_free_space_tree = false;
3113
3114         if (btrfs_test_opt(fs_info, CLEAR_CACHE) &&
3115             btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3116                 clear_free_space_tree = true;
3117         } else if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) &&
3118                    !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID)) {
3119                 btrfs_warn(fs_info, "free space tree is invalid");
3120                 clear_free_space_tree = true;
3121         }
3122
3123         if (clear_free_space_tree) {
3124                 btrfs_info(fs_info, "clearing free space tree");
3125                 ret = btrfs_clear_free_space_tree(fs_info);
3126                 if (ret) {
3127                         btrfs_warn(fs_info,
3128                                    "failed to clear free space tree: %d", ret);
3129                         goto out;
3130                 }
3131         }
3132
3133         /*
3134          * btrfs_find_orphan_roots() is responsible for finding all the dead
3135          * roots (with 0 refs), flag them with BTRFS_ROOT_DEAD_TREE and load
3136          * them into the fs_info->fs_roots_radix tree. This must be done before
3137          * calling btrfs_orphan_cleanup() on the tree root. If we don't do it
3138          * first, then btrfs_orphan_cleanup() will delete a dead root's orphan
3139          * item before the root's tree is deleted - this means that if we unmount
3140          * or crash before the deletion completes, on the next mount we will not
3141          * delete what remains of the tree because the orphan item does not
3142          * exists anymore, which is what tells us we have a pending deletion.
3143          */
3144         ret = btrfs_find_orphan_roots(fs_info);
3145         if (ret)
3146                 goto out;
3147
3148         ret = btrfs_cleanup_fs_roots(fs_info);
3149         if (ret)
3150                 goto out;
3151
3152         down_read(&fs_info->cleanup_work_sem);
3153         if ((ret = btrfs_orphan_cleanup(fs_info->fs_root)) ||
3154             (ret = btrfs_orphan_cleanup(fs_info->tree_root))) {
3155                 up_read(&fs_info->cleanup_work_sem);
3156                 goto out;
3157         }
3158         up_read(&fs_info->cleanup_work_sem);
3159
3160         mutex_lock(&fs_info->cleaner_mutex);
3161         ret = btrfs_recover_relocation(fs_info);
3162         mutex_unlock(&fs_info->cleaner_mutex);
3163         if (ret < 0) {
3164                 btrfs_warn(fs_info, "failed to recover relocation: %d", ret);
3165                 goto out;
3166         }
3167
3168         if (btrfs_test_opt(fs_info, FREE_SPACE_TREE) &&
3169             !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) {
3170                 btrfs_info(fs_info, "creating free space tree");
3171                 ret = btrfs_create_free_space_tree(fs_info);
3172                 if (ret) {
3173                         btrfs_warn(fs_info,
3174                                 "failed to create free space tree: %d", ret);
3175                         goto out;
3176                 }
3177         }
3178
3179         if (cache_opt != btrfs_free_space_cache_v1_active(fs_info)) {
3180                 ret = btrfs_set_free_space_cache_v1_active(fs_info, cache_opt);
3181                 if (ret)
3182                         goto out;
3183         }
3184
3185         ret = btrfs_resume_balance_async(fs_info);
3186         if (ret)
3187                 goto out;
3188
3189         ret = btrfs_resume_dev_replace_async(fs_info);
3190         if (ret) {
3191                 btrfs_warn(fs_info, "failed to resume dev_replace");
3192                 goto out;
3193         }
3194
3195         btrfs_qgroup_rescan_resume(fs_info);
3196
3197         if (!fs_info->uuid_root) {
3198                 btrfs_info(fs_info, "creating UUID tree");
3199                 ret = btrfs_create_uuid_tree(fs_info);
3200                 if (ret) {
3201                         btrfs_warn(fs_info,
3202                                    "failed to create the UUID tree %d", ret);
3203                         goto out;
3204                 }
3205         }
3206
3207 out:
3208         return ret;
3209 }
3210
3211 /*
3212  * Do various sanity and dependency checks of different features.
3213  *
3214  * @is_rw_mount:        If the mount is read-write.
3215  *
3216  * This is the place for less strict checks (like for subpage or artificial
3217  * feature dependencies).
3218  *
3219  * For strict checks or possible corruption detection, see
3220  * btrfs_validate_super().
3221  *
3222  * This should be called after btrfs_parse_options(), as some mount options
3223  * (space cache related) can modify on-disk format like free space tree and
3224  * screw up certain feature dependencies.
3225  */
3226 int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount)
3227 {
3228         struct btrfs_super_block *disk_super = fs_info->super_copy;
3229         u64 incompat = btrfs_super_incompat_flags(disk_super);
3230         const u64 compat_ro = btrfs_super_compat_ro_flags(disk_super);
3231         const u64 compat_ro_unsupp = (compat_ro & ~BTRFS_FEATURE_COMPAT_RO_SUPP);
3232
3233         if (incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP) {
3234                 btrfs_err(fs_info,
3235                 "cannot mount because of unknown incompat features (0x%llx)",
3236                     incompat);
3237                 return -EINVAL;
3238         }
3239
3240         /* Runtime limitation for mixed block groups. */
3241         if ((incompat & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) &&
3242             (fs_info->sectorsize != fs_info->nodesize)) {
3243                 btrfs_err(fs_info,
3244 "unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups",
3245                         fs_info->nodesize, fs_info->sectorsize);
3246                 return -EINVAL;
3247         }
3248
3249         /* Mixed backref is an always-enabled feature. */
3250         incompat |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
3251
3252         /* Set compression related flags just in case. */
3253         if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
3254                 incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
3255         else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
3256                 incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
3257
3258         /*
3259          * An ancient flag, which should really be marked deprecated.
3260          * Such runtime limitation doesn't really need a incompat flag.
3261          */
3262         if (btrfs_super_nodesize(disk_super) > PAGE_SIZE)
3263                 incompat |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA;
3264
3265         if (compat_ro_unsupp && is_rw_mount) {
3266                 btrfs_err(fs_info,
3267         "cannot mount read-write because of unknown compat_ro features (0x%llx)",
3268                        compat_ro);
3269                 return -EINVAL;
3270         }
3271
3272         /*
3273          * We have unsupported RO compat features, although RO mounted, we
3274          * should not cause any metadata writes, including log replay.
3275          * Or we could screw up whatever the new feature requires.
3276          */
3277         if (compat_ro_unsupp && btrfs_super_log_root(disk_super) &&
3278             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3279                 btrfs_err(fs_info,
3280 "cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay",
3281                           compat_ro);
3282                 return -EINVAL;
3283         }
3284
3285         /*
3286          * Artificial limitations for block group tree, to force
3287          * block-group-tree to rely on no-holes and free-space-tree.
3288          */
3289         if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) &&
3290             (!btrfs_fs_incompat(fs_info, NO_HOLES) ||
3291              !btrfs_test_opt(fs_info, FREE_SPACE_TREE))) {
3292                 btrfs_err(fs_info,
3293 "block-group-tree feature requires no-holes and free-space-tree features");
3294                 return -EINVAL;
3295         }
3296
3297         /*
3298          * Subpage runtime limitation on v1 cache.
3299          *
3300          * V1 space cache still has some hard codeed PAGE_SIZE usage, while
3301          * we're already defaulting to v2 cache, no need to bother v1 as it's
3302          * going to be deprecated anyway.
3303          */
3304         if (fs_info->sectorsize < PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) {
3305                 btrfs_warn(fs_info,
3306         "v1 space cache is not supported for page size %lu with sectorsize %u",
3307                            PAGE_SIZE, fs_info->sectorsize);
3308                 return -EINVAL;
3309         }
3310
3311         /* This can be called by remount, we need to protect the super block. */
3312         spin_lock(&fs_info->super_lock);
3313         btrfs_set_super_incompat_flags(disk_super, incompat);
3314         spin_unlock(&fs_info->super_lock);
3315
3316         return 0;
3317 }
3318
3319 int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices,
3320                       char *options)
3321 {
3322         u32 sectorsize;
3323         u32 nodesize;
3324         u32 stripesize;
3325         u64 generation;
3326         u64 features;
3327         u16 csum_type;
3328         struct btrfs_super_block *disk_super;
3329         struct btrfs_fs_info *fs_info = btrfs_sb(sb);
3330         struct btrfs_root *tree_root;
3331         struct btrfs_root *chunk_root;
3332         int ret;
3333         int err = -EINVAL;
3334         int level;
3335
3336         ret = init_mount_fs_info(fs_info, sb);
3337         if (ret) {
3338                 err = ret;
3339                 goto fail;
3340         }
3341
3342         /* These need to be init'ed before we start creating inodes and such. */
3343         tree_root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID,
3344                                      GFP_KERNEL);
3345         fs_info->tree_root = tree_root;
3346         chunk_root = btrfs_alloc_root(fs_info, BTRFS_CHUNK_TREE_OBJECTID,
3347                                       GFP_KERNEL);
3348         fs_info->chunk_root = chunk_root;
3349         if (!tree_root || !chunk_root) {
3350                 err = -ENOMEM;
3351                 goto fail;
3352         }
3353
3354         fs_info->btree_inode = new_inode(sb);
3355         if (!fs_info->btree_inode) {
3356                 err = -ENOMEM;
3357                 goto fail;
3358         }
3359         mapping_set_gfp_mask(fs_info->btree_inode->i_mapping, GFP_NOFS);
3360         btrfs_init_btree_inode(fs_info);
3361
3362         invalidate_bdev(fs_devices->latest_dev->bdev);
3363
3364         /*
3365          * Read super block and check the signature bytes only
3366          */
3367         disk_super = btrfs_read_dev_super(fs_devices->latest_dev->bdev);
3368         if (IS_ERR(disk_super)) {
3369                 err = PTR_ERR(disk_super);
3370                 goto fail_alloc;
3371         }
3372
3373         /*
3374          * Verify the type first, if that or the checksum value are
3375          * corrupted, we'll find out
3376          */
3377         csum_type = btrfs_super_csum_type(disk_super);
3378         if (!btrfs_supported_super_csum(csum_type)) {
3379                 btrfs_err(fs_info, "unsupported checksum algorithm: %u",
3380                           csum_type);
3381                 err = -EINVAL;
3382                 btrfs_release_disk_super(disk_super);
3383                 goto fail_alloc;
3384         }
3385
3386         fs_info->csum_size = btrfs_super_csum_size(disk_super);
3387
3388         ret = btrfs_init_csum_hash(fs_info, csum_type);
3389         if (ret) {
3390                 err = ret;
3391                 btrfs_release_disk_super(disk_super);
3392                 goto fail_alloc;
3393         }
3394
3395         /*
3396          * We want to check superblock checksum, the type is stored inside.
3397          * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k).
3398          */
3399         if (btrfs_check_super_csum(fs_info, disk_super)) {
3400                 btrfs_err(fs_info, "superblock checksum mismatch");
3401                 err = -EINVAL;
3402                 btrfs_release_disk_super(disk_super);
3403                 goto fail_alloc;
3404         }
3405
3406         /*
3407          * super_copy is zeroed at allocation time and we never touch the
3408          * following bytes up to INFO_SIZE, the checksum is calculated from
3409          * the whole block of INFO_SIZE
3410          */
3411         memcpy(fs_info->super_copy, disk_super, sizeof(*fs_info->super_copy));
3412         btrfs_release_disk_super(disk_super);
3413
3414         disk_super = fs_info->super_copy;
3415
3416
3417         features = btrfs_super_flags(disk_super);
3418         if (features & BTRFS_SUPER_FLAG_CHANGING_FSID_V2) {
3419                 features &= ~BTRFS_SUPER_FLAG_CHANGING_FSID_V2;
3420                 btrfs_set_super_flags(disk_super, features);
3421                 btrfs_info(fs_info,
3422                         "found metadata UUID change in progress flag, clearing");
3423         }
3424
3425         memcpy(fs_info->super_for_commit, fs_info->super_copy,
3426                sizeof(*fs_info->super_for_commit));
3427
3428         ret = btrfs_validate_mount_super(fs_info);
3429         if (ret) {
3430                 btrfs_err(fs_info, "superblock contains fatal errors");
3431                 err = -EINVAL;
3432                 goto fail_alloc;
3433         }
3434
3435         if (!btrfs_super_root(disk_super))
3436                 goto fail_alloc;
3437
3438         /* check FS state, whether FS is broken. */
3439         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR)
3440                 set_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state);
3441
3442         /*
3443          * In the long term, we'll store the compression type in the super
3444          * block, and it'll be used for per file compression control.
3445          */
3446         fs_info->compress_type = BTRFS_COMPRESS_ZLIB;
3447
3448
3449         /* Set up fs_info before parsing mount options */
3450         nodesize = btrfs_super_nodesize(disk_super);
3451         sectorsize = btrfs_super_sectorsize(disk_super);
3452         stripesize = sectorsize;
3453         fs_info->dirty_metadata_batch = nodesize * (1 + ilog2(nr_cpu_ids));
3454         fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids));
3455
3456         fs_info->nodesize = nodesize;
3457         fs_info->sectorsize = sectorsize;
3458         fs_info->sectorsize_bits = ilog2(sectorsize);
3459         fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size;
3460         fs_info->stripesize = stripesize;
3461
3462         ret = btrfs_parse_options(fs_info, options, sb->s_flags);
3463         if (ret) {
3464                 err = ret;
3465                 goto fail_alloc;
3466         }
3467
3468         ret = btrfs_check_features(fs_info, !sb_rdonly(sb));
3469         if (ret < 0) {
3470                 err = ret;
3471                 goto fail_alloc;
3472         }
3473
3474         if (sectorsize < PAGE_SIZE) {
3475                 struct btrfs_subpage_info *subpage_info;
3476
3477                 /*
3478                  * V1 space cache has some hardcoded PAGE_SIZE usage, and is
3479                  * going to be deprecated.
3480                  *
3481                  * Force to use v2 cache for subpage case.
3482                  */
3483                 btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE);
3484                 btrfs_set_and_info(fs_info, FREE_SPACE_TREE,
3485                         "forcing free space tree for sector size %u with page size %lu",
3486                         sectorsize, PAGE_SIZE);
3487
3488                 btrfs_warn(fs_info,
3489                 "read-write for sector size %u with page size %lu is experimental",
3490                            sectorsize, PAGE_SIZE);
3491                 subpage_info = kzalloc(sizeof(*subpage_info), GFP_KERNEL);
3492                 if (!subpage_info)
3493                         goto fail_alloc;
3494                 btrfs_init_subpage_info(subpage_info, sectorsize);
3495                 fs_info->subpage_info = subpage_info;
3496         }
3497
3498         ret = btrfs_init_workqueues(fs_info);
3499         if (ret) {
3500                 err = ret;
3501                 goto fail_sb_buffer;
3502         }
3503
3504         sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super);
3505         sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE);
3506
3507         sb->s_blocksize = sectorsize;
3508         sb->s_blocksize_bits = blksize_bits(sectorsize);
3509         memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE);
3510
3511         mutex_lock(&fs_info->chunk_mutex);
3512         ret = btrfs_read_sys_array(fs_info);
3513         mutex_unlock(&fs_info->chunk_mutex);
3514         if (ret) {
3515                 btrfs_err(fs_info, "failed to read the system array: %d", ret);
3516                 goto fail_sb_buffer;
3517         }
3518
3519         generation = btrfs_super_chunk_root_generation(disk_super);
3520         level = btrfs_super_chunk_root_level(disk_super);
3521         ret = load_super_root(chunk_root, btrfs_super_chunk_root(disk_super),
3522                               generation, level);
3523         if (ret) {
3524                 btrfs_err(fs_info, "failed to read chunk root");
3525                 goto fail_tree_roots;
3526         }
3527
3528         read_extent_buffer(chunk_root->node, fs_info->chunk_tree_uuid,
3529                            offsetof(struct btrfs_header, chunk_tree_uuid),
3530                            BTRFS_UUID_SIZE);
3531
3532         ret = btrfs_read_chunk_tree(fs_info);
3533         if (ret) {
3534                 btrfs_err(fs_info, "failed to read chunk tree: %d", ret);
3535                 goto fail_tree_roots;
3536         }
3537
3538         /*
3539          * At this point we know all the devices that make this filesystem,
3540          * including the seed devices but we don't know yet if the replace
3541          * target is required. So free devices that are not part of this
3542          * filesystem but skip the replace target device which is checked
3543          * below in btrfs_init_dev_replace().
3544          */
3545         btrfs_free_extra_devids(fs_devices);
3546         if (!fs_devices->latest_dev->bdev) {
3547                 btrfs_err(fs_info, "failed to read devices");
3548                 goto fail_tree_roots;
3549         }
3550
3551         ret = init_tree_roots(fs_info);
3552         if (ret)
3553                 goto fail_tree_roots;
3554
3555         /*
3556          * Get zone type information of zoned block devices. This will also
3557          * handle emulation of a zoned filesystem if a regular device has the
3558          * zoned incompat feature flag set.
3559          */
3560         ret = btrfs_get_dev_zone_info_all_devices(fs_info);
3561         if (ret) {
3562                 btrfs_err(fs_info,
3563                           "zoned: failed to read device zone info: %d",
3564                           ret);
3565                 goto fail_block_groups;
3566         }
3567
3568         /*
3569          * If we have a uuid root and we're not being told to rescan we need to
3570          * check the generation here so we can set the
3571          * BTRFS_FS_UPDATE_UUID_TREE_GEN bit.  Otherwise we could commit the
3572          * transaction during a balance or the log replay without updating the
3573          * uuid generation, and then if we crash we would rescan the uuid tree,
3574          * even though it was perfectly fine.
3575          */
3576         if (fs_info->uuid_root && !btrfs_test_opt(fs_info, RESCAN_UUID_TREE) &&
3577             fs_info->generation == btrfs_super_uuid_tree_generation(disk_super))
3578                 set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
3579
3580         ret = btrfs_verify_dev_extents(fs_info);
3581         if (ret) {
3582                 btrfs_err(fs_info,
3583                           "failed to verify dev extents against chunks: %d",
3584                           ret);
3585                 goto fail_block_groups;
3586         }
3587         ret = btrfs_recover_balance(fs_info);
3588         if (ret) {
3589                 btrfs_err(fs_info, "failed to recover balance: %d", ret);
3590                 goto fail_block_groups;
3591         }
3592
3593         ret = btrfs_init_dev_stats(fs_info);
3594         if (ret) {
3595                 btrfs_err(fs_info, "failed to init dev_stats: %d", ret);
3596                 goto fail_block_groups;
3597         }
3598
3599         ret = btrfs_init_dev_replace(fs_info);
3600         if (ret) {
3601                 btrfs_err(fs_info, "failed to init dev_replace: %d", ret);
3602                 goto fail_block_groups;
3603         }
3604
3605         ret = btrfs_check_zoned_mode(fs_info);
3606         if (ret) {
3607                 btrfs_err(fs_info, "failed to initialize zoned mode: %d",
3608                           ret);
3609                 goto fail_block_groups;
3610         }
3611
3612         ret = btrfs_sysfs_add_fsid(fs_devices);
3613         if (ret) {
3614                 btrfs_err(fs_info, "failed to init sysfs fsid interface: %d",
3615                                 ret);
3616                 goto fail_block_groups;
3617         }
3618
3619         ret = btrfs_sysfs_add_mounted(fs_info);
3620         if (ret) {
3621                 btrfs_err(fs_info, "failed to init sysfs interface: %d", ret);
3622                 goto fail_fsdev_sysfs;
3623         }
3624
3625         ret = btrfs_init_space_info(fs_info);
3626         if (ret) {
3627                 btrfs_err(fs_info, "failed to initialize space info: %d", ret);
3628                 goto fail_sysfs;
3629         }
3630
3631         ret = btrfs_read_block_groups(fs_info);
3632         if (ret) {
3633                 btrfs_err(fs_info, "failed to read block groups: %d", ret);
3634                 goto fail_sysfs;
3635         }
3636
3637         btrfs_free_zone_cache(fs_info);
3638
3639         if (!sb_rdonly(sb) && fs_info->fs_devices->missing_devices &&
3640             !btrfs_check_rw_degradable(fs_info, NULL)) {
3641                 btrfs_warn(fs_info,
3642                 "writable mount is not allowed due to too many missing devices");
3643                 goto fail_sysfs;
3644         }
3645
3646         fs_info->cleaner_kthread = kthread_run(cleaner_kthread, fs_info,
3647                                                "btrfs-cleaner");
3648         if (IS_ERR(fs_info->cleaner_kthread))
3649                 goto fail_sysfs;
3650
3651         fs_info->transaction_kthread = kthread_run(transaction_kthread,
3652                                                    tree_root,
3653                                                    "btrfs-transaction");
3654         if (IS_ERR(fs_info->transaction_kthread))
3655                 goto fail_cleaner;
3656
3657         if (!btrfs_test_opt(fs_info, NOSSD) &&
3658             !fs_info->fs_devices->rotating) {
3659                 btrfs_set_and_info(fs_info, SSD, "enabling ssd optimizations");
3660         }
3661
3662         /*
3663          * For devices supporting discard turn on discard=async automatically,
3664          * unless it's already set or disabled. This could be turned off by
3665          * nodiscard for the same mount.
3666          */
3667         if (!(btrfs_test_opt(fs_info, DISCARD_SYNC) ||
3668               btrfs_test_opt(fs_info, DISCARD_ASYNC) ||
3669               btrfs_test_opt(fs_info, NODISCARD)) &&
3670             fs_info->fs_devices->discardable) {
3671                 btrfs_set_and_info(fs_info, DISCARD_ASYNC,
3672                                    "auto enabling async discard");
3673                 btrfs_clear_opt(fs_info->mount_opt, NODISCARD);
3674         }
3675
3676 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
3677         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY)) {
3678                 ret = btrfsic_mount(fs_info, fs_devices,
3679                                     btrfs_test_opt(fs_info,
3680                                         CHECK_INTEGRITY_DATA) ? 1 : 0,
3681                                     fs_info->check_integrity_print_mask);
3682                 if (ret)
3683                         btrfs_warn(fs_info,
3684                                 "failed to initialize integrity check module: %d",
3685                                 ret);
3686         }
3687 #endif
3688         ret = btrfs_read_qgroup_config(fs_info);
3689         if (ret)
3690                 goto fail_trans_kthread;
3691
3692         if (btrfs_build_ref_tree(fs_info))
3693                 btrfs_err(fs_info, "couldn't build ref tree");
3694
3695         /* do not make disk changes in broken FS or nologreplay is given */
3696         if (btrfs_super_log_root(disk_super) != 0 &&
3697             !btrfs_test_opt(fs_info, NOLOGREPLAY)) {
3698                 btrfs_info(fs_info, "start tree-log replay");
3699                 ret = btrfs_replay_log(fs_info, fs_devices);
3700                 if (ret) {
3701                         err = ret;
3702                         goto fail_qgroup;
3703                 }
3704         }
3705
3706         fs_info->fs_root = btrfs_get_fs_root(fs_info, BTRFS_FS_TREE_OBJECTID, true);
3707         if (IS_ERR(fs_info->fs_root)) {
3708                 err = PTR_ERR(fs_info->fs_root);
3709                 btrfs_warn(fs_info, "failed to read fs tree: %d", err);
3710                 fs_info->fs_root = NULL;
3711                 goto fail_qgroup;
3712         }
3713
3714         if (sb_rdonly(sb))
3715                 goto clear_oneshot;
3716
3717         ret = btrfs_start_pre_rw_mount(fs_info);
3718         if (ret) {
3719                 close_ctree(fs_info);
3720                 return ret;
3721         }
3722         btrfs_discard_resume(fs_info);
3723
3724         if (fs_info->uuid_root &&
3725             (btrfs_test_opt(fs_info, RESCAN_UUID_TREE) ||
3726              fs_info->generation != btrfs_super_uuid_tree_generation(disk_super))) {
3727                 btrfs_info(fs_info, "checking UUID tree");
3728                 ret = btrfs_check_uuid_tree(fs_info);
3729                 if (ret) {
3730                         btrfs_warn(fs_info,
3731                                 "failed to check the UUID tree: %d", ret);
3732                         close_ctree(fs_info);
3733                         return ret;
3734                 }
3735         }
3736
3737         set_bit(BTRFS_FS_OPEN, &fs_info->flags);
3738
3739         /* Kick the cleaner thread so it'll start deleting snapshots. */
3740         if (test_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags))
3741                 wake_up_process(fs_info->cleaner_kthread);
3742
3743 clear_oneshot:
3744         btrfs_clear_oneshot_options(fs_info);
3745         return 0;
3746
3747 fail_qgroup:
3748         btrfs_free_qgroup_config(fs_info);
3749 fail_trans_kthread:
3750         kthread_stop(fs_info->transaction_kthread);
3751         btrfs_cleanup_transaction(fs_info);
3752         btrfs_free_fs_roots(fs_info);
3753 fail_cleaner:
3754         kthread_stop(fs_info->cleaner_kthread);
3755
3756         /*
3757          * make sure we're done with the btree inode before we stop our
3758          * kthreads
3759          */
3760         filemap_write_and_wait(fs_info->btree_inode->i_mapping);
3761
3762 fail_sysfs:
3763         btrfs_sysfs_remove_mounted(fs_info);
3764
3765 fail_fsdev_sysfs:
3766         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
3767
3768 fail_block_groups:
3769         btrfs_put_block_group_cache(fs_info);
3770
3771 fail_tree_roots:
3772         if (fs_info->data_reloc_root)
3773                 btrfs_drop_and_free_fs_root(fs_info, fs_info->data_reloc_root);
3774         free_root_pointers(fs_info, true);
3775         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
3776
3777 fail_sb_buffer:
3778         btrfs_stop_all_workers(fs_info);
3779         btrfs_free_block_groups(fs_info);
3780 fail_alloc:
3781         btrfs_mapping_tree_free(&fs_info->mapping_tree);
3782
3783         iput(fs_info->btree_inode);
3784 fail:
3785         btrfs_close_devices(fs_info->fs_devices);
3786         return err;
3787 }
3788 ALLOW_ERROR_INJECTION(open_ctree, ERRNO);
3789
3790 static void btrfs_end_super_write(struct bio *bio)
3791 {
3792         struct btrfs_device *device = bio->bi_private;
3793         struct bio_vec *bvec;
3794         struct bvec_iter_all iter_all;
3795         struct page *page;
3796
3797         bio_for_each_segment_all(bvec, bio, iter_all) {
3798                 page = bvec->bv_page;
3799
3800                 if (bio->bi_status) {
3801                         btrfs_warn_rl_in_rcu(device->fs_info,
3802                                 "lost page write due to IO error on %s (%d)",
3803                                 btrfs_dev_name(device),
3804                                 blk_status_to_errno(bio->bi_status));
3805                         ClearPageUptodate(page);
3806                         SetPageError(page);
3807                         btrfs_dev_stat_inc_and_print(device,
3808                                                      BTRFS_DEV_STAT_WRITE_ERRS);
3809                 } else {
3810                         SetPageUptodate(page);
3811                 }
3812
3813                 put_page(page);
3814                 unlock_page(page);
3815         }
3816
3817         bio_put(bio);
3818 }
3819
3820 struct btrfs_super_block *btrfs_read_dev_one_super(struct block_device *bdev,
3821                                                    int copy_num, bool drop_cache)
3822 {
3823         struct btrfs_super_block *super;
3824         struct page *page;
3825         u64 bytenr, bytenr_orig;
3826         struct address_space *mapping = bdev->bd_inode->i_mapping;
3827         int ret;
3828
3829         bytenr_orig = btrfs_sb_offset(copy_num);
3830         ret = btrfs_sb_log_location_bdev(bdev, copy_num, READ, &bytenr);
3831         if (ret == -ENOENT)
3832                 return ERR_PTR(-EINVAL);
3833         else if (ret)
3834                 return ERR_PTR(ret);
3835
3836         if (bytenr + BTRFS_SUPER_INFO_SIZE >= bdev_nr_bytes(bdev))
3837                 return ERR_PTR(-EINVAL);
3838
3839         if (drop_cache) {
3840                 /* This should only be called with the primary sb. */
3841                 ASSERT(copy_num == 0);
3842
3843                 /*
3844                  * Drop the page of the primary superblock, so later read will
3845                  * always read from the device.
3846                  */
3847                 invalidate_inode_pages2_range(mapping,
3848                                 bytenr >> PAGE_SHIFT,
3849                                 (bytenr + BTRFS_SUPER_INFO_SIZE) >> PAGE_SHIFT);
3850         }
3851
3852         page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS);
3853         if (IS_ERR(page))
3854                 return ERR_CAST(page);
3855
3856         super = page_address(page);
3857         if (btrfs_super_magic(super) != BTRFS_MAGIC) {
3858                 btrfs_release_disk_super(super);
3859                 return ERR_PTR(-ENODATA);
3860         }
3861
3862         if (btrfs_super_bytenr(super) != bytenr_orig) {
3863                 btrfs_release_disk_super(super);
3864                 return ERR_PTR(-EINVAL);
3865         }
3866
3867         return super;
3868 }
3869
3870
3871 struct btrfs_super_block *btrfs_read_dev_super(struct block_device *bdev)
3872 {
3873         struct btrfs_super_block *super, *latest = NULL;
3874         int i;
3875         u64 transid = 0;
3876
3877         /* we would like to check all the supers, but that would make
3878          * a btrfs mount succeed after a mkfs from a different FS.
3879          * So, we need to add a special mount option to scan for
3880          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
3881          */
3882         for (i = 0; i < 1; i++) {
3883                 super = btrfs_read_dev_one_super(bdev, i, false);
3884                 if (IS_ERR(super))
3885                         continue;
3886
3887                 if (!latest || btrfs_super_generation(super) > transid) {
3888                         if (latest)
3889                                 btrfs_release_disk_super(super);
3890
3891                         latest = super;
3892                         transid = btrfs_super_generation(super);
3893                 }
3894         }
3895
3896         return super;
3897 }
3898
3899 /*
3900  * Write superblock @sb to the @device. Do not wait for completion, all the
3901  * pages we use for writing are locked.
3902  *
3903  * Write @max_mirrors copies of the superblock, where 0 means default that fit
3904  * the expected device size at commit time. Note that max_mirrors must be
3905  * same for write and wait phases.
3906  *
3907  * Return number of errors when page is not found or submission fails.
3908  */
3909 static int write_dev_supers(struct btrfs_device *device,
3910                             struct btrfs_super_block *sb, int max_mirrors)
3911 {
3912         struct btrfs_fs_info *fs_info = device->fs_info;
3913         struct address_space *mapping = device->bdev->bd_inode->i_mapping;
3914         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
3915         int i;
3916         int errors = 0;
3917         int ret;
3918         u64 bytenr, bytenr_orig;
3919
3920         if (max_mirrors == 0)
3921                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
3922
3923         shash->tfm = fs_info->csum_shash;
3924
3925         for (i = 0; i < max_mirrors; i++) {
3926                 struct page *page;
3927                 struct bio *bio;
3928                 struct btrfs_super_block *disk_super;
3929
3930                 bytenr_orig = btrfs_sb_offset(i);
3931                 ret = btrfs_sb_log_location(device, i, WRITE, &bytenr);
3932                 if (ret == -ENOENT) {
3933                         continue;
3934                 } else if (ret < 0) {
3935                         btrfs_err(device->fs_info,
3936                                 "couldn't get super block location for mirror %d",
3937                                 i);
3938                         errors++;
3939                         continue;
3940                 }
3941                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
3942                     device->commit_total_bytes)
3943                         break;
3944
3945                 btrfs_set_super_bytenr(sb, bytenr_orig);
3946
3947                 crypto_shash_digest(shash, (const char *)sb + BTRFS_CSUM_SIZE,
3948                                     BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE,
3949                                     sb->csum);
3950
3951                 page = find_or_create_page(mapping, bytenr >> PAGE_SHIFT,
3952                                            GFP_NOFS);
3953                 if (!page) {
3954                         btrfs_err(device->fs_info,
3955                             "couldn't get super block page for bytenr %llu",
3956                             bytenr);
3957                         errors++;
3958                         continue;
3959                 }
3960
3961                 /* Bump the refcount for wait_dev_supers() */
3962                 get_page(page);
3963
3964                 disk_super = page_address(page);
3965                 memcpy(disk_super, sb, BTRFS_SUPER_INFO_SIZE);
3966
3967                 /*
3968                  * Directly use bios here instead of relying on the page cache
3969                  * to do I/O, so we don't lose the ability to do integrity
3970                  * checking.
3971                  */
3972                 bio = bio_alloc(device->bdev, 1,
3973                                 REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO,
3974                                 GFP_NOFS);
3975                 bio->bi_iter.bi_sector = bytenr >> SECTOR_SHIFT;
3976                 bio->bi_private = device;
3977                 bio->bi_end_io = btrfs_end_super_write;
3978                 __bio_add_page(bio, page, BTRFS_SUPER_INFO_SIZE,
3979                                offset_in_page(bytenr));
3980
3981                 /*
3982                  * We FUA only the first super block.  The others we allow to
3983                  * go down lazy and there's a short window where the on-disk
3984                  * copies might still contain the older version.
3985                  */
3986                 if (i == 0 && !btrfs_test_opt(device->fs_info, NOBARRIER))
3987                         bio->bi_opf |= REQ_FUA;
3988
3989                 btrfsic_check_bio(bio);
3990                 submit_bio(bio);
3991
3992                 if (btrfs_advance_sb_log(device, i))
3993                         errors++;
3994         }
3995         return errors < i ? 0 : -1;
3996 }
3997
3998 /*
3999  * Wait for write completion of superblocks done by write_dev_supers,
4000  * @max_mirrors same for write and wait phases.
4001  *
4002  * Return number of errors when page is not found or not marked up to
4003  * date.
4004  */
4005 static int wait_dev_supers(struct btrfs_device *device, int max_mirrors)
4006 {
4007         int i;
4008         int errors = 0;
4009         bool primary_failed = false;
4010         int ret;
4011         u64 bytenr;
4012
4013         if (max_mirrors == 0)
4014                 max_mirrors = BTRFS_SUPER_MIRROR_MAX;
4015
4016         for (i = 0; i < max_mirrors; i++) {
4017                 struct page *page;
4018
4019                 ret = btrfs_sb_log_location(device, i, READ, &bytenr);
4020                 if (ret == -ENOENT) {
4021                         break;
4022                 } else if (ret < 0) {
4023                         errors++;
4024                         if (i == 0)
4025                                 primary_failed = true;
4026                         continue;
4027                 }
4028                 if (bytenr + BTRFS_SUPER_INFO_SIZE >=
4029                     device->commit_total_bytes)
4030                         break;
4031
4032                 page = find_get_page(device->bdev->bd_inode->i_mapping,
4033                                      bytenr >> PAGE_SHIFT);
4034                 if (!page) {
4035                         errors++;
4036                         if (i == 0)
4037                                 primary_failed = true;
4038                         continue;
4039                 }
4040                 /* Page is submitted locked and unlocked once the IO completes */
4041                 wait_on_page_locked(page);
4042                 if (PageError(page)) {
4043                         errors++;
4044                         if (i == 0)
4045                                 primary_failed = true;
4046                 }
4047
4048                 /* Drop our reference */
4049                 put_page(page);
4050
4051                 /* Drop the reference from the writing run */
4052                 put_page(page);
4053         }
4054
4055         /* log error, force error return */
4056         if (primary_failed) {
4057                 btrfs_err(device->fs_info, "error writing primary super block to device %llu",
4058                           device->devid);
4059                 return -1;
4060         }
4061
4062         return errors < i ? 0 : -1;
4063 }
4064
4065 /*
4066  * endio for the write_dev_flush, this will wake anyone waiting
4067  * for the barrier when it is done
4068  */
4069 static void btrfs_end_empty_barrier(struct bio *bio)
4070 {
4071         bio_uninit(bio);
4072         complete(bio->bi_private);
4073 }
4074
4075 /*
4076  * Submit a flush request to the device if it supports it. Error handling is
4077  * done in the waiting counterpart.
4078  */
4079 static void write_dev_flush(struct btrfs_device *device)
4080 {
4081         struct bio *bio = &device->flush_bio;
4082
4083 #ifndef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4084         /*
4085          * When a disk has write caching disabled, we skip submission of a bio
4086          * with flush and sync requests before writing the superblock, since
4087          * it's not needed. However when the integrity checker is enabled, this
4088          * results in reports that there are metadata blocks referred by a
4089          * superblock that were not properly flushed. So don't skip the bio
4090          * submission only when the integrity checker is enabled for the sake
4091          * of simplicity, since this is a debug tool and not meant for use in
4092          * non-debug builds.
4093          */
4094         if (!bdev_write_cache(device->bdev))
4095                 return;
4096 #endif
4097
4098         bio_init(bio, device->bdev, NULL, 0,
4099                  REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH);
4100         bio->bi_end_io = btrfs_end_empty_barrier;
4101         init_completion(&device->flush_wait);
4102         bio->bi_private = &device->flush_wait;
4103
4104         btrfsic_check_bio(bio);
4105         submit_bio(bio);
4106         set_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
4107 }
4108
4109 /*
4110  * If the flush bio has been submitted by write_dev_flush, wait for it.
4111  */
4112 static blk_status_t wait_dev_flush(struct btrfs_device *device)
4113 {
4114         struct bio *bio = &device->flush_bio;
4115
4116         if (!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state))
4117                 return BLK_STS_OK;
4118
4119         clear_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state);
4120         wait_for_completion_io(&device->flush_wait);
4121
4122         return bio->bi_status;
4123 }
4124
4125 static int check_barrier_error(struct btrfs_fs_info *fs_info)
4126 {
4127         if (!btrfs_check_rw_degradable(fs_info, NULL))
4128                 return -EIO;
4129         return 0;
4130 }
4131
4132 /*
4133  * send an empty flush down to each device in parallel,
4134  * then wait for them
4135  */
4136 static int barrier_all_devices(struct btrfs_fs_info *info)
4137 {
4138         struct list_head *head;
4139         struct btrfs_device *dev;
4140         int errors_wait = 0;
4141         blk_status_t ret;
4142
4143         lockdep_assert_held(&info->fs_devices->device_list_mutex);
4144         /* send down all the barriers */
4145         head = &info->fs_devices->devices;
4146         list_for_each_entry(dev, head, dev_list) {
4147                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
4148                         continue;
4149                 if (!dev->bdev)
4150                         continue;
4151                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4152                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4153                         continue;
4154
4155                 write_dev_flush(dev);
4156                 dev->last_flush_error = BLK_STS_OK;
4157         }
4158
4159         /* wait for all the barriers */
4160         list_for_each_entry(dev, head, dev_list) {
4161                 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state))
4162                         continue;
4163                 if (!dev->bdev) {
4164                         errors_wait++;
4165                         continue;
4166                 }
4167                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4168                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4169                         continue;
4170
4171                 ret = wait_dev_flush(dev);
4172                 if (ret) {
4173                         dev->last_flush_error = ret;
4174                         btrfs_dev_stat_inc_and_print(dev,
4175                                         BTRFS_DEV_STAT_FLUSH_ERRS);
4176                         errors_wait++;
4177                 }
4178         }
4179
4180         if (errors_wait) {
4181                 /*
4182                  * At some point we need the status of all disks
4183                  * to arrive at the volume status. So error checking
4184                  * is being pushed to a separate loop.
4185                  */
4186                 return check_barrier_error(info);
4187         }
4188         return 0;
4189 }
4190
4191 int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags)
4192 {
4193         int raid_type;
4194         int min_tolerated = INT_MAX;
4195
4196         if ((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 ||
4197             (flags & BTRFS_AVAIL_ALLOC_BIT_SINGLE))
4198                 min_tolerated = min_t(int, min_tolerated,
4199                                     btrfs_raid_array[BTRFS_RAID_SINGLE].
4200                                     tolerated_failures);
4201
4202         for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
4203                 if (raid_type == BTRFS_RAID_SINGLE)
4204                         continue;
4205                 if (!(flags & btrfs_raid_array[raid_type].bg_flag))
4206                         continue;
4207                 min_tolerated = min_t(int, min_tolerated,
4208                                     btrfs_raid_array[raid_type].
4209                                     tolerated_failures);
4210         }
4211
4212         if (min_tolerated == INT_MAX) {
4213                 pr_warn("BTRFS: unknown raid flag: %llu", flags);
4214                 min_tolerated = 0;
4215         }
4216
4217         return min_tolerated;
4218 }
4219
4220 int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors)
4221 {
4222         struct list_head *head;
4223         struct btrfs_device *dev;
4224         struct btrfs_super_block *sb;
4225         struct btrfs_dev_item *dev_item;
4226         int ret;
4227         int do_barriers;
4228         int max_errors;
4229         int total_errors = 0;
4230         u64 flags;
4231
4232         do_barriers = !btrfs_test_opt(fs_info, NOBARRIER);
4233
4234         /*
4235          * max_mirrors == 0 indicates we're from commit_transaction,
4236          * not from fsync where the tree roots in fs_info have not
4237          * been consistent on disk.
4238          */
4239         if (max_mirrors == 0)
4240                 backup_super_roots(fs_info);
4241
4242         sb = fs_info->super_for_commit;
4243         dev_item = &sb->dev_item;
4244
4245         mutex_lock(&fs_info->fs_devices->device_list_mutex);
4246         head = &fs_info->fs_devices->devices;
4247         max_errors = btrfs_super_num_devices(fs_info->super_copy) - 1;
4248
4249         if (do_barriers) {
4250                 ret = barrier_all_devices(fs_info);
4251                 if (ret) {
4252                         mutex_unlock(
4253                                 &fs_info->fs_devices->device_list_mutex);
4254                         btrfs_handle_fs_error(fs_info, ret,
4255                                               "errors while submitting device barriers.");
4256                         return ret;
4257                 }
4258         }
4259
4260         list_for_each_entry(dev, head, dev_list) {
4261                 if (!dev->bdev) {
4262                         total_errors++;
4263                         continue;
4264                 }
4265                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4266                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4267                         continue;
4268
4269                 btrfs_set_stack_device_generation(dev_item, 0);
4270                 btrfs_set_stack_device_type(dev_item, dev->type);
4271                 btrfs_set_stack_device_id(dev_item, dev->devid);
4272                 btrfs_set_stack_device_total_bytes(dev_item,
4273                                                    dev->commit_total_bytes);
4274                 btrfs_set_stack_device_bytes_used(dev_item,
4275                                                   dev->commit_bytes_used);
4276                 btrfs_set_stack_device_io_align(dev_item, dev->io_align);
4277                 btrfs_set_stack_device_io_width(dev_item, dev->io_width);
4278                 btrfs_set_stack_device_sector_size(dev_item, dev->sector_size);
4279                 memcpy(dev_item->uuid, dev->uuid, BTRFS_UUID_SIZE);
4280                 memcpy(dev_item->fsid, dev->fs_devices->metadata_uuid,
4281                        BTRFS_FSID_SIZE);
4282
4283                 flags = btrfs_super_flags(sb);
4284                 btrfs_set_super_flags(sb, flags | BTRFS_HEADER_FLAG_WRITTEN);
4285
4286                 ret = btrfs_validate_write_super(fs_info, sb);
4287                 if (ret < 0) {
4288                         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4289                         btrfs_handle_fs_error(fs_info, -EUCLEAN,
4290                                 "unexpected superblock corruption detected");
4291                         return -EUCLEAN;
4292                 }
4293
4294                 ret = write_dev_supers(dev, sb, max_mirrors);
4295                 if (ret)
4296                         total_errors++;
4297         }
4298         if (total_errors > max_errors) {
4299                 btrfs_err(fs_info, "%d errors while writing supers",
4300                           total_errors);
4301                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4302
4303                 /* FUA is masked off if unsupported and can't be the reason */
4304                 btrfs_handle_fs_error(fs_info, -EIO,
4305                                       "%d errors while writing supers",
4306                                       total_errors);
4307                 return -EIO;
4308         }
4309
4310         total_errors = 0;
4311         list_for_each_entry(dev, head, dev_list) {
4312                 if (!dev->bdev)
4313                         continue;
4314                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
4315                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))
4316                         continue;
4317
4318                 ret = wait_dev_supers(dev, max_mirrors);
4319                 if (ret)
4320                         total_errors++;
4321         }
4322         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4323         if (total_errors > max_errors) {
4324                 btrfs_handle_fs_error(fs_info, -EIO,
4325                                       "%d errors while writing supers",
4326                                       total_errors);
4327                 return -EIO;
4328         }
4329         return 0;
4330 }
4331
4332 /* Drop a fs root from the radix tree and free it. */
4333 void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info,
4334                                   struct btrfs_root *root)
4335 {
4336         bool drop_ref = false;
4337
4338         spin_lock(&fs_info->fs_roots_radix_lock);
4339         radix_tree_delete(&fs_info->fs_roots_radix,
4340                           (unsigned long)root->root_key.objectid);
4341         if (test_and_clear_bit(BTRFS_ROOT_IN_RADIX, &root->state))
4342                 drop_ref = true;
4343         spin_unlock(&fs_info->fs_roots_radix_lock);
4344
4345         if (BTRFS_FS_ERROR(fs_info)) {
4346                 ASSERT(root->log_root == NULL);
4347                 if (root->reloc_root) {
4348                         btrfs_put_root(root->reloc_root);
4349                         root->reloc_root = NULL;
4350                 }
4351         }
4352
4353         if (drop_ref)
4354                 btrfs_put_root(root);
4355 }
4356
4357 int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info)
4358 {
4359         u64 root_objectid = 0;
4360         struct btrfs_root *gang[8];
4361         int i = 0;
4362         int err = 0;
4363         unsigned int ret = 0;
4364
4365         while (1) {
4366                 spin_lock(&fs_info->fs_roots_radix_lock);
4367                 ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
4368                                              (void **)gang, root_objectid,
4369                                              ARRAY_SIZE(gang));
4370                 if (!ret) {
4371                         spin_unlock(&fs_info->fs_roots_radix_lock);
4372                         break;
4373                 }
4374                 root_objectid = gang[ret - 1]->root_key.objectid + 1;
4375
4376                 for (i = 0; i < ret; i++) {
4377                         /* Avoid to grab roots in dead_roots */
4378                         if (btrfs_root_refs(&gang[i]->root_item) == 0) {
4379                                 gang[i] = NULL;
4380                                 continue;
4381                         }
4382                         /* grab all the search result for later use */
4383                         gang[i] = btrfs_grab_root(gang[i]);
4384                 }
4385                 spin_unlock(&fs_info->fs_roots_radix_lock);
4386
4387                 for (i = 0; i < ret; i++) {
4388                         if (!gang[i])
4389                                 continue;
4390                         root_objectid = gang[i]->root_key.objectid;
4391                         err = btrfs_orphan_cleanup(gang[i]);
4392                         if (err)
4393                                 break;
4394                         btrfs_put_root(gang[i]);
4395                 }
4396                 root_objectid++;
4397         }
4398
4399         /* release the uncleaned roots due to error */
4400         for (; i < ret; i++) {
4401                 if (gang[i])
4402                         btrfs_put_root(gang[i]);
4403         }
4404         return err;
4405 }
4406
4407 int btrfs_commit_super(struct btrfs_fs_info *fs_info)
4408 {
4409         struct btrfs_root *root = fs_info->tree_root;
4410         struct btrfs_trans_handle *trans;
4411
4412         mutex_lock(&fs_info->cleaner_mutex);
4413         btrfs_run_delayed_iputs(fs_info);
4414         mutex_unlock(&fs_info->cleaner_mutex);
4415         wake_up_process(fs_info->cleaner_kthread);
4416
4417         /* wait until ongoing cleanup work done */
4418         down_write(&fs_info->cleanup_work_sem);
4419         up_write(&fs_info->cleanup_work_sem);
4420
4421         trans = btrfs_join_transaction(root);
4422         if (IS_ERR(trans))
4423                 return PTR_ERR(trans);
4424         return btrfs_commit_transaction(trans);
4425 }
4426
4427 static void warn_about_uncommitted_trans(struct btrfs_fs_info *fs_info)
4428 {
4429         struct btrfs_transaction *trans;
4430         struct btrfs_transaction *tmp;
4431         bool found = false;
4432
4433         if (list_empty(&fs_info->trans_list))
4434                 return;
4435
4436         /*
4437          * This function is only called at the very end of close_ctree(),
4438          * thus no other running transaction, no need to take trans_lock.
4439          */
4440         ASSERT(test_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags));
4441         list_for_each_entry_safe(trans, tmp, &fs_info->trans_list, list) {
4442                 struct extent_state *cached = NULL;
4443                 u64 dirty_bytes = 0;
4444                 u64 cur = 0;
4445                 u64 found_start;
4446                 u64 found_end;
4447
4448                 found = true;
4449                 while (!find_first_extent_bit(&trans->dirty_pages, cur,
4450                         &found_start, &found_end, EXTENT_DIRTY, &cached)) {
4451                         dirty_bytes += found_end + 1 - found_start;
4452                         cur = found_end + 1;
4453                 }
4454                 btrfs_warn(fs_info,
4455         "transaction %llu (with %llu dirty metadata bytes) is not committed",
4456                            trans->transid, dirty_bytes);
4457                 btrfs_cleanup_one_transaction(trans, fs_info);
4458
4459                 if (trans == fs_info->running_transaction)
4460                         fs_info->running_transaction = NULL;
4461                 list_del_init(&trans->list);
4462
4463                 btrfs_put_transaction(trans);
4464                 trace_btrfs_transaction_commit(fs_info);
4465         }
4466         ASSERT(!found);
4467 }
4468
4469 void __cold close_ctree(struct btrfs_fs_info *fs_info)
4470 {
4471         int ret;
4472
4473         set_bit(BTRFS_FS_CLOSING_START, &fs_info->flags);
4474
4475         /*
4476          * If we had UNFINISHED_DROPS we could still be processing them, so
4477          * clear that bit and wake up relocation so it can stop.
4478          * We must do this before stopping the block group reclaim task, because
4479          * at btrfs_relocate_block_group() we wait for this bit, and after the
4480          * wait we stop with -EINTR if btrfs_fs_closing() returns non-zero - we
4481          * have just set BTRFS_FS_CLOSING_START, so btrfs_fs_closing() will
4482          * return 1.
4483          */
4484         btrfs_wake_unfinished_drop(fs_info);
4485
4486         /*
4487          * We may have the reclaim task running and relocating a data block group,
4488          * in which case it may create delayed iputs. So stop it before we park
4489          * the cleaner kthread otherwise we can get new delayed iputs after
4490          * parking the cleaner, and that can make the async reclaim task to hang
4491          * if it's waiting for delayed iputs to complete, since the cleaner is
4492          * parked and can not run delayed iputs - this will make us hang when
4493          * trying to stop the async reclaim task.
4494          */
4495         cancel_work_sync(&fs_info->reclaim_bgs_work);
4496         /*
4497          * We don't want the cleaner to start new transactions, add more delayed
4498          * iputs, etc. while we're closing. We can't use kthread_stop() yet
4499          * because that frees the task_struct, and the transaction kthread might
4500          * still try to wake up the cleaner.
4501          */
4502         kthread_park(fs_info->cleaner_kthread);
4503
4504         /* wait for the qgroup rescan worker to stop */
4505         btrfs_qgroup_wait_for_completion(fs_info, false);
4506
4507         /* wait for the uuid_scan task to finish */
4508         down(&fs_info->uuid_tree_rescan_sem);
4509         /* avoid complains from lockdep et al., set sem back to initial state */
4510         up(&fs_info->uuid_tree_rescan_sem);
4511
4512         /* pause restriper - we want to resume on mount */
4513         btrfs_pause_balance(fs_info);
4514
4515         btrfs_dev_replace_suspend_for_unmount(fs_info);
4516
4517         btrfs_scrub_cancel(fs_info);
4518
4519         /* wait for any defraggers to finish */
4520         wait_event(fs_info->transaction_wait,
4521                    (atomic_read(&fs_info->defrag_running) == 0));
4522
4523         /* clear out the rbtree of defraggable inodes */
4524         btrfs_cleanup_defrag_inodes(fs_info);
4525
4526         /*
4527          * After we parked the cleaner kthread, ordered extents may have
4528          * completed and created new delayed iputs. If one of the async reclaim
4529          * tasks is running and in the RUN_DELAYED_IPUTS flush state, then we
4530          * can hang forever trying to stop it, because if a delayed iput is
4531          * added after it ran btrfs_run_delayed_iputs() and before it called
4532          * btrfs_wait_on_delayed_iputs(), it will hang forever since there is
4533          * no one else to run iputs.
4534          *
4535          * So wait for all ongoing ordered extents to complete and then run
4536          * delayed iputs. This works because once we reach this point no one
4537          * can either create new ordered extents nor create delayed iputs
4538          * through some other means.
4539          *
4540          * Also note that btrfs_wait_ordered_roots() is not safe here, because
4541          * it waits for BTRFS_ORDERED_COMPLETE to be set on an ordered extent,
4542          * but the delayed iput for the respective inode is made only when doing
4543          * the final btrfs_put_ordered_extent() (which must happen at
4544          * btrfs_finish_ordered_io() when we are unmounting).
4545          */
4546         btrfs_flush_workqueue(fs_info->endio_write_workers);
4547         /* Ordered extents for free space inodes. */
4548         btrfs_flush_workqueue(fs_info->endio_freespace_worker);
4549         btrfs_run_delayed_iputs(fs_info);
4550
4551         cancel_work_sync(&fs_info->async_reclaim_work);
4552         cancel_work_sync(&fs_info->async_data_reclaim_work);
4553         cancel_work_sync(&fs_info->preempt_reclaim_work);
4554
4555         /* Cancel or finish ongoing discard work */
4556         btrfs_discard_cleanup(fs_info);
4557
4558         if (!sb_rdonly(fs_info->sb)) {
4559                 /*
4560                  * The cleaner kthread is stopped, so do one final pass over
4561                  * unused block groups.
4562                  */
4563                 btrfs_delete_unused_bgs(fs_info);
4564
4565                 /*
4566                  * There might be existing delayed inode workers still running
4567                  * and holding an empty delayed inode item. We must wait for
4568                  * them to complete first because they can create a transaction.
4569                  * This happens when someone calls btrfs_balance_delayed_items()
4570                  * and then a transaction commit runs the same delayed nodes
4571                  * before any delayed worker has done something with the nodes.
4572                  * We must wait for any worker here and not at transaction
4573                  * commit time since that could cause a deadlock.
4574                  * This is a very rare case.
4575                  */
4576                 btrfs_flush_workqueue(fs_info->delayed_workers);
4577
4578                 ret = btrfs_commit_super(fs_info);
4579                 if (ret)
4580                         btrfs_err(fs_info, "commit super ret %d", ret);
4581         }
4582
4583         if (BTRFS_FS_ERROR(fs_info))
4584                 btrfs_error_commit_super(fs_info);
4585
4586         kthread_stop(fs_info->transaction_kthread);
4587         kthread_stop(fs_info->cleaner_kthread);
4588
4589         ASSERT(list_empty(&fs_info->delayed_iputs));
4590         set_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags);
4591
4592         if (btrfs_check_quota_leak(fs_info)) {
4593                 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4594                 btrfs_err(fs_info, "qgroup reserved space leaked");
4595         }
4596
4597         btrfs_free_qgroup_config(fs_info);
4598         ASSERT(list_empty(&fs_info->delalloc_roots));
4599
4600         if (percpu_counter_sum(&fs_info->delalloc_bytes)) {
4601                 btrfs_info(fs_info, "at unmount delalloc count %lld",
4602                        percpu_counter_sum(&fs_info->delalloc_bytes));
4603         }
4604
4605         if (percpu_counter_sum(&fs_info->ordered_bytes))
4606                 btrfs_info(fs_info, "at unmount dio bytes count %lld",
4607                            percpu_counter_sum(&fs_info->ordered_bytes));
4608
4609         btrfs_sysfs_remove_mounted(fs_info);
4610         btrfs_sysfs_remove_fsid(fs_info->fs_devices);
4611
4612         btrfs_put_block_group_cache(fs_info);
4613
4614         /*
4615          * we must make sure there is not any read request to
4616          * submit after we stopping all workers.
4617          */
4618         invalidate_inode_pages2(fs_info->btree_inode->i_mapping);
4619         btrfs_stop_all_workers(fs_info);
4620
4621         /* We shouldn't have any transaction open at this point */
4622         warn_about_uncommitted_trans(fs_info);
4623
4624         clear_bit(BTRFS_FS_OPEN, &fs_info->flags);
4625         free_root_pointers(fs_info, true);
4626         btrfs_free_fs_roots(fs_info);
4627
4628         /*
4629          * We must free the block groups after dropping the fs_roots as we could
4630          * have had an IO error and have left over tree log blocks that aren't
4631          * cleaned up until the fs roots are freed.  This makes the block group
4632          * accounting appear to be wrong because there's pending reserved bytes,
4633          * so make sure we do the block group cleanup afterwards.
4634          */
4635         btrfs_free_block_groups(fs_info);
4636
4637         iput(fs_info->btree_inode);
4638
4639 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4640         if (btrfs_test_opt(fs_info, CHECK_INTEGRITY))
4641                 btrfsic_unmount(fs_info->fs_devices);
4642 #endif
4643
4644         btrfs_mapping_tree_free(&fs_info->mapping_tree);
4645         btrfs_close_devices(fs_info->fs_devices);
4646 }
4647
4648 int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid,
4649                           int atomic)
4650 {
4651         int ret;
4652         struct inode *btree_inode = buf->pages[0]->mapping->host;
4653
4654         ret = extent_buffer_uptodate(buf);
4655         if (!ret)
4656                 return ret;
4657
4658         ret = verify_parent_transid(&BTRFS_I(btree_inode)->io_tree, buf,
4659                                     parent_transid, atomic);
4660         if (ret == -EAGAIN)
4661                 return ret;
4662         return !ret;
4663 }
4664
4665 void btrfs_mark_buffer_dirty(struct extent_buffer *buf)
4666 {
4667         struct btrfs_fs_info *fs_info = buf->fs_info;
4668         u64 transid = btrfs_header_generation(buf);
4669         int was_dirty;
4670
4671 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4672         /*
4673          * This is a fast path so only do this check if we have sanity tests
4674          * enabled.  Normal people shouldn't be using unmapped buffers as dirty
4675          * outside of the sanity tests.
4676          */
4677         if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &buf->bflags)))
4678                 return;
4679 #endif
4680         btrfs_assert_tree_write_locked(buf);
4681         if (transid != fs_info->generation)
4682                 WARN(1, KERN_CRIT "btrfs transid mismatch buffer %llu, found %llu running %llu\n",
4683                         buf->start, transid, fs_info->generation);
4684         was_dirty = set_extent_buffer_dirty(buf);
4685         if (!was_dirty)
4686                 percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
4687                                          buf->len,
4688                                          fs_info->dirty_metadata_batch);
4689 #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY
4690         /*
4691          * Since btrfs_mark_buffer_dirty() can be called with item pointer set
4692          * but item data not updated.
4693          * So here we should only check item pointers, not item data.
4694          */
4695         if (btrfs_header_level(buf) == 0 &&
4696             btrfs_check_leaf_relaxed(buf)) {
4697                 btrfs_print_leaf(buf);
4698                 ASSERT(0);
4699         }
4700 #endif
4701 }
4702
4703 static void __btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info,
4704                                         int flush_delayed)
4705 {
4706         /*
4707          * looks as though older kernels can get into trouble with
4708          * this code, they end up stuck in balance_dirty_pages forever
4709          */
4710         int ret;
4711
4712         if (current->flags & PF_MEMALLOC)
4713                 return;
4714
4715         if (flush_delayed)
4716                 btrfs_balance_delayed_items(fs_info);
4717
4718         ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes,
4719                                      BTRFS_DIRTY_METADATA_THRESH,
4720                                      fs_info->dirty_metadata_batch);
4721         if (ret > 0) {
4722                 balance_dirty_pages_ratelimited(fs_info->btree_inode->i_mapping);
4723         }
4724 }
4725
4726 void btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info)
4727 {
4728         __btrfs_btree_balance_dirty(fs_info, 1);
4729 }
4730
4731 void btrfs_btree_balance_dirty_nodelay(struct btrfs_fs_info *fs_info)
4732 {
4733         __btrfs_btree_balance_dirty(fs_info, 0);
4734 }
4735
4736 static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info)
4737 {
4738         /* cleanup FS via transaction */
4739         btrfs_cleanup_transaction(fs_info);
4740
4741         mutex_lock(&fs_info->cleaner_mutex);
4742         btrfs_run_delayed_iputs(fs_info);
4743         mutex_unlock(&fs_info->cleaner_mutex);
4744
4745         down_write(&fs_info->cleanup_work_sem);
4746         up_write(&fs_info->cleanup_work_sem);
4747 }
4748
4749 static void btrfs_drop_all_logs(struct btrfs_fs_info *fs_info)
4750 {
4751         struct btrfs_root *gang[8];
4752         u64 root_objectid = 0;
4753         int ret;
4754
4755         spin_lock(&fs_info->fs_roots_radix_lock);
4756         while ((ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
4757                                              (void **)gang, root_objectid,
4758                                              ARRAY_SIZE(gang))) != 0) {
4759                 int i;
4760
4761                 for (i = 0; i < ret; i++)
4762                         gang[i] = btrfs_grab_root(gang[i]);
4763                 spin_unlock(&fs_info->fs_roots_radix_lock);
4764
4765                 for (i = 0; i < ret; i++) {
4766                         if (!gang[i])
4767                                 continue;
4768                         root_objectid = gang[i]->root_key.objectid;
4769                         btrfs_free_log(NULL, gang[i]);
4770                         btrfs_put_root(gang[i]);
4771                 }
4772                 root_objectid++;
4773                 spin_lock(&fs_info->fs_roots_radix_lock);
4774         }
4775         spin_unlock(&fs_info->fs_roots_radix_lock);
4776         btrfs_free_log_root_tree(NULL, fs_info);
4777 }
4778
4779 static void btrfs_destroy_ordered_extents(struct btrfs_root *root)
4780 {
4781         struct btrfs_ordered_extent *ordered;
4782
4783         spin_lock(&root->ordered_extent_lock);
4784         /*
4785          * This will just short circuit the ordered completion stuff which will
4786          * make sure the ordered extent gets properly cleaned up.
4787          */
4788         list_for_each_entry(ordered, &root->ordered_extents,
4789                             root_extent_list)
4790                 set_bit(BTRFS_ORDERED_IOERR, &ordered->flags);
4791         spin_unlock(&root->ordered_extent_lock);
4792 }
4793
4794 static void btrfs_destroy_all_ordered_extents(struct btrfs_fs_info *fs_info)
4795 {
4796         struct btrfs_root *root;
4797         struct list_head splice;
4798
4799         INIT_LIST_HEAD(&splice);
4800
4801         spin_lock(&fs_info->ordered_root_lock);
4802         list_splice_init(&fs_info->ordered_roots, &splice);
4803         while (!list_empty(&splice)) {
4804                 root = list_first_entry(&splice, struct btrfs_root,
4805                                         ordered_root);
4806                 list_move_tail(&root->ordered_root,
4807                                &fs_info->ordered_roots);
4808
4809                 spin_unlock(&fs_info->ordered_root_lock);
4810                 btrfs_destroy_ordered_extents(root);
4811
4812                 cond_resched();
4813                 spin_lock(&fs_info->ordered_root_lock);
4814         }
4815         spin_unlock(&fs_info->ordered_root_lock);
4816
4817         /*
4818          * We need this here because if we've been flipped read-only we won't
4819          * get sync() from the umount, so we need to make sure any ordered
4820          * extents that haven't had their dirty pages IO start writeout yet
4821          * actually get run and error out properly.
4822          */
4823         btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
4824 }
4825
4826 static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans,
4827                                       struct btrfs_fs_info *fs_info)
4828 {
4829         struct rb_node *node;
4830         struct btrfs_delayed_ref_root *delayed_refs;
4831         struct btrfs_delayed_ref_node *ref;
4832         int ret = 0;
4833
4834         delayed_refs = &trans->delayed_refs;
4835
4836         spin_lock(&delayed_refs->lock);
4837         if (atomic_read(&delayed_refs->num_entries) == 0) {
4838                 spin_unlock(&delayed_refs->lock);
4839                 btrfs_debug(fs_info, "delayed_refs has NO entry");
4840                 return ret;
4841         }
4842
4843         while ((node = rb_first_cached(&delayed_refs->href_root)) != NULL) {
4844                 struct btrfs_delayed_ref_head *head;
4845                 struct rb_node *n;
4846                 bool pin_bytes = false;
4847
4848                 head = rb_entry(node, struct btrfs_delayed_ref_head,
4849                                 href_node);
4850                 if (btrfs_delayed_ref_lock(delayed_refs, head))
4851                         continue;
4852
4853                 spin_lock(&head->lock);
4854                 while ((n = rb_first_cached(&head->ref_tree)) != NULL) {
4855                         ref = rb_entry(n, struct btrfs_delayed_ref_node,
4856                                        ref_node);
4857                         ref->in_tree = 0;
4858                         rb_erase_cached(&ref->ref_node, &head->ref_tree);
4859                         RB_CLEAR_NODE(&ref->ref_node);
4860                         if (!list_empty(&ref->add_list))
4861                                 list_del(&ref->add_list);
4862                         atomic_dec(&delayed_refs->num_entries);
4863                         btrfs_put_delayed_ref(ref);
4864                 }
4865                 if (head->must_insert_reserved)
4866                         pin_bytes = true;
4867                 btrfs_free_delayed_extent_op(head->extent_op);
4868                 btrfs_delete_ref_head(delayed_refs, head);
4869                 spin_unlock(&head->lock);
4870                 spin_unlock(&delayed_refs->lock);
4871                 mutex_unlock(&head->mutex);
4872
4873                 if (pin_bytes) {
4874                         struct btrfs_block_group *cache;
4875
4876                         cache = btrfs_lookup_block_group(fs_info, head->bytenr);
4877                         BUG_ON(!cache);
4878
4879                         spin_lock(&cache->space_info->lock);
4880                         spin_lock(&cache->lock);
4881                         cache->pinned += head->num_bytes;
4882                         btrfs_space_info_update_bytes_pinned(fs_info,
4883                                 cache->space_info, head->num_bytes);
4884                         cache->reserved -= head->num_bytes;
4885                         cache->space_info->bytes_reserved -= head->num_bytes;
4886                         spin_unlock(&cache->lock);
4887                         spin_unlock(&cache->space_info->lock);
4888
4889                         btrfs_put_block_group(cache);
4890
4891                         btrfs_error_unpin_extent_range(fs_info, head->bytenr,
4892                                 head->bytenr + head->num_bytes - 1);
4893                 }
4894                 btrfs_cleanup_ref_head_accounting(fs_info, delayed_refs, head);
4895                 btrfs_put_delayed_ref_head(head);
4896                 cond_resched();
4897                 spin_lock(&delayed_refs->lock);
4898         }
4899         btrfs_qgroup_destroy_extent_records(trans);
4900
4901         spin_unlock(&delayed_refs->lock);
4902
4903         return ret;
4904 }
4905
4906 static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root)
4907 {
4908         struct btrfs_inode *btrfs_inode;
4909         struct list_head splice;
4910
4911         INIT_LIST_HEAD(&splice);
4912
4913         spin_lock(&root->delalloc_lock);
4914         list_splice_init(&root->delalloc_inodes, &splice);
4915
4916         while (!list_empty(&splice)) {
4917                 struct inode *inode = NULL;
4918                 btrfs_inode = list_first_entry(&splice, struct btrfs_inode,
4919                                                delalloc_inodes);
4920                 __btrfs_del_delalloc_inode(root, btrfs_inode);
4921                 spin_unlock(&root->delalloc_lock);
4922
4923                 /*
4924                  * Make sure we get a live inode and that it'll not disappear
4925                  * meanwhile.
4926                  */
4927                 inode = igrab(&btrfs_inode->vfs_inode);
4928                 if (inode) {
4929                         invalidate_inode_pages2(inode->i_mapping);
4930                         iput(inode);
4931                 }
4932                 spin_lock(&root->delalloc_lock);
4933         }
4934         spin_unlock(&root->delalloc_lock);
4935 }
4936
4937 static void btrfs_destroy_all_delalloc_inodes(struct btrfs_fs_info *fs_info)
4938 {
4939         struct btrfs_root *root;
4940         struct list_head splice;
4941
4942         INIT_LIST_HEAD(&splice);
4943
4944         spin_lock(&fs_info->delalloc_root_lock);
4945         list_splice_init(&fs_info->delalloc_roots, &splice);
4946         while (!list_empty(&splice)) {
4947                 root = list_first_entry(&splice, struct btrfs_root,
4948                                          delalloc_root);
4949                 root = btrfs_grab_root(root);
4950                 BUG_ON(!root);
4951                 spin_unlock(&fs_info->delalloc_root_lock);
4952
4953                 btrfs_destroy_delalloc_inodes(root);
4954                 btrfs_put_root(root);
4955
4956                 spin_lock(&fs_info->delalloc_root_lock);
4957         }
4958         spin_unlock(&fs_info->delalloc_root_lock);
4959 }
4960
4961 static int btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info,
4962                                         struct extent_io_tree *dirty_pages,
4963                                         int mark)
4964 {
4965         int ret;
4966         struct extent_buffer *eb;
4967         u64 start = 0;
4968         u64 end;
4969
4970         while (1) {
4971                 ret = find_first_extent_bit(dirty_pages, start, &start, &end,
4972                                             mark, NULL);
4973                 if (ret)
4974                         break;
4975
4976                 clear_extent_bits(dirty_pages, start, end, mark);
4977                 while (start <= end) {
4978                         eb = find_extent_buffer(fs_info, start);
4979                         start += fs_info->nodesize;
4980                         if (!eb)
4981                                 continue;
4982
4983                         btrfs_tree_lock(eb);
4984                         wait_on_extent_buffer_writeback(eb);
4985                         btrfs_clear_buffer_dirty(NULL, eb);
4986                         btrfs_tree_unlock(eb);
4987
4988                         free_extent_buffer_stale(eb);
4989                 }
4990         }
4991
4992         return ret;
4993 }
4994
4995 static int btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info,
4996                                        struct extent_io_tree *unpin)
4997 {
4998         u64 start;
4999         u64 end;
5000         int ret;
5001
5002         while (1) {
5003                 struct extent_state *cached_state = NULL;
5004
5005                 /*
5006                  * The btrfs_finish_extent_commit() may get the same range as
5007                  * ours between find_first_extent_bit and clear_extent_dirty.
5008                  * Hence, hold the unused_bg_unpin_mutex to avoid double unpin
5009                  * the same extent range.
5010                  */
5011                 mutex_lock(&fs_info->unused_bg_unpin_mutex);
5012                 ret = find_first_extent_bit(unpin, 0, &start, &end,
5013                                             EXTENT_DIRTY, &cached_state);
5014                 if (ret) {
5015                         mutex_unlock(&fs_info->unused_bg_unpin_mutex);
5016                         break;
5017                 }
5018
5019                 clear_extent_dirty(unpin, start, end, &cached_state);
5020                 free_extent_state(cached_state);
5021                 btrfs_error_unpin_extent_range(fs_info, start, end);
5022                 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
5023                 cond_resched();
5024         }
5025
5026         return 0;
5027 }
5028
5029 static void btrfs_cleanup_bg_io(struct btrfs_block_group *cache)
5030 {
5031         struct inode *inode;
5032
5033         inode = cache->io_ctl.inode;
5034         if (inode) {
5035                 invalidate_inode_pages2(inode->i_mapping);
5036                 BTRFS_I(inode)->generation = 0;
5037                 cache->io_ctl.inode = NULL;
5038                 iput(inode);
5039         }
5040         ASSERT(cache->io_ctl.pages == NULL);
5041         btrfs_put_block_group(cache);
5042 }
5043
5044 void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *cur_trans,
5045                              struct btrfs_fs_info *fs_info)
5046 {
5047         struct btrfs_block_group *cache;
5048
5049         spin_lock(&cur_trans->dirty_bgs_lock);
5050         while (!list_empty(&cur_trans->dirty_bgs)) {
5051                 cache = list_first_entry(&cur_trans->dirty_bgs,
5052                                          struct btrfs_block_group,
5053                                          dirty_list);
5054
5055                 if (!list_empty(&cache->io_list)) {
5056                         spin_unlock(&cur_trans->dirty_bgs_lock);
5057                         list_del_init(&cache->io_list);
5058                         btrfs_cleanup_bg_io(cache);
5059                         spin_lock(&cur_trans->dirty_bgs_lock);
5060                 }
5061
5062                 list_del_init(&cache->dirty_list);
5063                 spin_lock(&cache->lock);
5064                 cache->disk_cache_state = BTRFS_DC_ERROR;
5065                 spin_unlock(&cache->lock);
5066
5067                 spin_unlock(&cur_trans->dirty_bgs_lock);
5068                 btrfs_put_block_group(cache);
5069                 btrfs_delayed_refs_rsv_release(fs_info, 1);
5070                 spin_lock(&cur_trans->dirty_bgs_lock);
5071         }
5072         spin_unlock(&cur_trans->dirty_bgs_lock);
5073
5074         /*
5075          * Refer to the definition of io_bgs member for details why it's safe
5076          * to use it without any locking
5077          */
5078         while (!list_empty(&cur_trans->io_bgs)) {
5079                 cache = list_first_entry(&cur_trans->io_bgs,
5080                                          struct btrfs_block_group,
5081                                          io_list);
5082
5083                 list_del_init(&cache->io_list);
5084                 spin_lock(&cache->lock);
5085                 cache->disk_cache_state = BTRFS_DC_ERROR;
5086                 spin_unlock(&cache->lock);
5087                 btrfs_cleanup_bg_io(cache);
5088         }
5089 }
5090
5091 void btrfs_cleanup_one_transaction(struct btrfs_transaction *cur_trans,
5092                                    struct btrfs_fs_info *fs_info)
5093 {
5094         struct btrfs_device *dev, *tmp;
5095
5096         btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
5097         ASSERT(list_empty(&cur_trans->dirty_bgs));
5098         ASSERT(list_empty(&cur_trans->io_bgs));
5099
5100         list_for_each_entry_safe(dev, tmp, &cur_trans->dev_update_list,
5101                                  post_commit_list) {
5102                 list_del_init(&dev->post_commit_list);
5103         }
5104
5105         btrfs_destroy_delayed_refs(cur_trans, fs_info);
5106
5107         cur_trans->state = TRANS_STATE_COMMIT_START;
5108         wake_up(&fs_info->transaction_blocked_wait);
5109
5110         cur_trans->state = TRANS_STATE_UNBLOCKED;
5111         wake_up(&fs_info->transaction_wait);
5112
5113         btrfs_destroy_delayed_inodes(fs_info);
5114
5115         btrfs_destroy_marked_extents(fs_info, &cur_trans->dirty_pages,
5116                                      EXTENT_DIRTY);
5117         btrfs_destroy_pinned_extent(fs_info, &cur_trans->pinned_extents);
5118
5119         btrfs_free_redirty_list(cur_trans);
5120
5121         cur_trans->state =TRANS_STATE_COMPLETED;
5122         wake_up(&cur_trans->commit_wait);
5123 }
5124
5125 static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info)
5126 {
5127         struct btrfs_transaction *t;
5128
5129         mutex_lock(&fs_info->transaction_kthread_mutex);
5130
5131         spin_lock(&fs_info->trans_lock);
5132         while (!list_empty(&fs_info->trans_list)) {
5133                 t = list_first_entry(&fs_info->trans_list,
5134                                      struct btrfs_transaction, list);
5135                 if (t->state >= TRANS_STATE_COMMIT_START) {
5136                         refcount_inc(&t->use_count);
5137                         spin_unlock(&fs_info->trans_lock);
5138                         btrfs_wait_for_commit(fs_info, t->transid);
5139                         btrfs_put_transaction(t);
5140                         spin_lock(&fs_info->trans_lock);
5141                         continue;
5142                 }
5143                 if (t == fs_info->running_transaction) {
5144                         t->state = TRANS_STATE_COMMIT_DOING;
5145                         spin_unlock(&fs_info->trans_lock);
5146                         /*
5147                          * We wait for 0 num_writers since we don't hold a trans
5148                          * handle open currently for this transaction.
5149                          */
5150                         wait_event(t->writer_wait,
5151                                    atomic_read(&t->num_writers) == 0);
5152                 } else {
5153                         spin_unlock(&fs_info->trans_lock);
5154                 }
5155                 btrfs_cleanup_one_transaction(t, fs_info);
5156
5157                 spin_lock(&fs_info->trans_lock);
5158                 if (t == fs_info->running_transaction)
5159                         fs_info->running_transaction = NULL;
5160                 list_del_init(&t->list);
5161                 spin_unlock(&fs_info->trans_lock);
5162
5163                 btrfs_put_transaction(t);
5164                 trace_btrfs_transaction_commit(fs_info);
5165                 spin_lock(&fs_info->trans_lock);
5166         }
5167         spin_unlock(&fs_info->trans_lock);
5168         btrfs_destroy_all_ordered_extents(fs_info);
5169         btrfs_destroy_delayed_inodes(fs_info);
5170         btrfs_assert_delayed_root_empty(fs_info);
5171         btrfs_destroy_all_delalloc_inodes(fs_info);
5172         btrfs_drop_all_logs(fs_info);
5173         mutex_unlock(&fs_info->transaction_kthread_mutex);
5174
5175         return 0;
5176 }
5177
5178 int btrfs_init_root_free_objectid(struct btrfs_root *root)
5179 {
5180         struct btrfs_path *path;
5181         int ret;
5182         struct extent_buffer *l;
5183         struct btrfs_key search_key;
5184         struct btrfs_key found_key;
5185         int slot;
5186
5187         path = btrfs_alloc_path();
5188         if (!path)
5189                 return -ENOMEM;
5190
5191         search_key.objectid = BTRFS_LAST_FREE_OBJECTID;
5192         search_key.type = -1;
5193         search_key.offset = (u64)-1;
5194         ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0);
5195         if (ret < 0)
5196                 goto error;
5197         BUG_ON(ret == 0); /* Corruption */
5198         if (path->slots[0] > 0) {
5199                 slot = path->slots[0] - 1;
5200                 l = path->nodes[0];
5201                 btrfs_item_key_to_cpu(l, &found_key, slot);
5202                 root->free_objectid = max_t(u64, found_key.objectid + 1,
5203                                             BTRFS_FIRST_FREE_OBJECTID);
5204         } else {
5205                 root->free_objectid = BTRFS_FIRST_FREE_OBJECTID;
5206         }
5207         ret = 0;
5208 error:
5209         btrfs_free_path(path);
5210         return ret;
5211 }
5212
5213 int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid)
5214 {
5215         int ret;
5216         mutex_lock(&root->objectid_mutex);
5217
5218         if (unlikely(root->free_objectid >= BTRFS_LAST_FREE_OBJECTID)) {
5219                 btrfs_warn(root->fs_info,
5220                            "the objectid of root %llu reaches its highest value",
5221                            root->root_key.objectid);
5222                 ret = -ENOSPC;
5223                 goto out;
5224         }
5225
5226         *objectid = root->free_objectid++;
5227         ret = 0;
5228 out:
5229         mutex_unlock(&root->objectid_mutex);
5230         return ret;
5231 }