fs/btrfs/scrub.c

   1 /*
   2  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/blkdev.h>
  20 #include <linux/ratelimit.h>
  21 #include "ctree.h"
  22 #include "volumes.h"
  23 #include "disk-io.h"
  24 #include "ordered-data.h"
  25 #include "transaction.h"
  26 #include "backref.h"
  27 #include "extent_io.h"
  28 #include "dev-replace.h"
  29 #include "check-integrity.h"
  30 #include "rcu-string.h"
  31 #include "raid56.h"
  32
  33 /*
  34  * This is only the first step towards a full-features scrub. It reads all
  35  * extent and super block and verifies the checksums. In case a bad checksum
  36  * is found or the extent cannot be read, good data will be written back if
  37  * any can be found.
  38  *
  39  * Future enhancements:
  40  *  - In case an unrepairable extent is encountered, track which files are
  41  *    affected and report them
  42  *  - track and record media errors, throw out bad devices
  43  *  - add a mode to also read unallocated space
  44  */
  45
  46 struct scrub_block;
  47 struct scrub_ctx;
  48
  49 /*
  50  * the following three values only influence the performance.
  51  * The last one configures the number of parallel and outstanding I/O
  52  * operations. The first two values configure an upper limit for the number
  53  * of (dynamically allocated) pages that are added to a bio.
  54  */
  55 #define SCRUB_PAGES_PER_RD_BIO  32      /* 128k per bio */
  56 #define SCRUB_PAGES_PER_WR_BIO  32      /* 128k per bio */
  57 #define SCRUB_BIOS_PER_SCTX     64      /* 8MB per device in flight */
  58
  59 /*
  60  * the following value times PAGE_SIZE needs to be large enough to match the
  61  * largest node/leaf/sector size that shall be supported.
  62  * Values larger than BTRFS_STRIPE_LEN are not supported.
  63  */
  64 #define SCRUB_MAX_PAGES_PER_BLOCK       16      /* 64k per node/leaf/sector */
  65
  66 struct scrub_recover {
  67         atomic_t                refs;
  68         struct btrfs_bio        *bbio;
  69         u64                     map_length;
  70 };
  71
  72 struct scrub_page {
  73         struct scrub_block      *sblock;
  74         struct page             *page;
  75         struct btrfs_device     *dev;
  76         struct list_head        list;
  77         u64                     flags;  /* extent flags */
  78         u64                     generation;
  79         u64                     logical;
  80         u64                     physical;
  81         u64                     physical_for_dev_replace;
  82         atomic_t                refs;
  83         struct {
  84                 unsigned int    mirror_num:8;
  85                 unsigned int    have_csum:1;
  86                 unsigned int    io_error:1;
  87         };
  88         u8                      csum[BTRFS_CSUM_SIZE];
  89
  90         struct scrub_recover    *recover;
  91 };
  92
  93 struct scrub_bio {
  94         int                     index;
  95         struct scrub_ctx        *sctx;
  96         struct btrfs_device     *dev;
  97         struct bio              *bio;
  98         int                     err;
  99         u64                     logical;
 100         u64                     physical;
 101 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
 102         struct scrub_page       *pagev[SCRUB_PAGES_PER_WR_BIO];
 103 #else
 104         struct scrub_page       *pagev[SCRUB_PAGES_PER_RD_BIO];
 105 #endif
 106         int                     page_count;
 107         int                     next_free;
 108         struct btrfs_work       work;
 109 };
 110
 111 struct scrub_block {
 112         struct scrub_page       *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
 113         int                     page_count;
 114         atomic_t                outstanding_pages;
 115         atomic_t                refs; /* free mem on transition to zero */
 116         struct scrub_ctx        *sctx;
 117         struct scrub_parity     *sparity;
 118         struct {
 119                 unsigned int    header_error:1;
 120                 unsigned int    checksum_error:1;
 121                 unsigned int    no_io_error_seen:1;
 122                 unsigned int    generation_error:1; /* also sets header_error */
 123
 124                 /* The following is for the data used to check parity */
 125                 /* It is for the data with checksum */
 126                 unsigned int    data_corrected:1;
 127         };
 128         struct btrfs_work       work;
 129 };
 130
 131 /* Used for the chunks with parity stripe such RAID5/6 */
 132 struct scrub_parity {
 133         struct scrub_ctx        *sctx;
 134
 135         struct btrfs_device     *scrub_dev;
 136
 137         u64                     logic_start;
 138
 139         u64                     logic_end;
 140
 141         int                     nsectors;
 142
 143         int                     stripe_len;
 144
 145         atomic_t                refs;
 146
 147         struct list_head        spages;
 148
 149         /* Work of parity check and repair */
 150         struct btrfs_work       work;
 151
 152         /* Mark the parity blocks which have data */
 153         unsigned long           *dbitmap;
 154
 155         /*
 156          * Mark the parity blocks which have data, but errors happen when
 157          * read data or check data
 158          */
 159         unsigned long           *ebitmap;
 160
 161         unsigned long           bitmap[0];
 162 };
 163
 164 struct scrub_wr_ctx {
 165         struct scrub_bio *wr_curr_bio;
 166         struct btrfs_device *tgtdev;
 167         int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
 168         atomic_t flush_all_writes;
 169         struct mutex wr_lock;
 170 };
 171
 172 struct scrub_ctx {
 173         struct scrub_bio        *bios[SCRUB_BIOS_PER_SCTX];
 174         struct btrfs_root       *dev_root;
 175         int                     first_free;
 176         int                     curr;
 177         atomic_t                bios_in_flight;
 178         atomic_t                workers_pending;
 179         spinlock_t              list_lock;
 180         wait_queue_head_t       list_wait;
 181         u16                     csum_size;
 182         struct list_head        csum_list;
 183         atomic_t                cancel_req;
 184         int                     readonly;
 185         int                     pages_per_rd_bio;
 186         u32                     sectorsize;
 187         u32                     nodesize;
 188
 189         int                     is_dev_replace;
 190         struct scrub_wr_ctx     wr_ctx;
 191
 192         /*
 193          * statistics
 194          */
 195         struct btrfs_scrub_progress stat;
 196         spinlock_t              stat_lock;
 197
 198         /*
 199          * Use a ref counter to avoid use-after-free issues. Scrub workers
 200          * decrement bios_in_flight and workers_pending and then do a wakeup
 201          * on the list_wait wait queue. We must ensure the main scrub task
 202          * doesn't free the scrub context before or while the workers are
 203          * doing the wakeup() call.
 204          */
 205         atomic_t                refs;
 206 };
 207
 208 struct scrub_fixup_nodatasum {
 209         struct scrub_ctx        *sctx;
 210         struct btrfs_device     *dev;
 211         u64                     logical;
 212         struct btrfs_root       *root;
 213         struct btrfs_work       work;
 214         int                     mirror_num;
 215 };
 216
 217 struct scrub_nocow_inode {
 218         u64                     inum;
 219         u64                     offset;
 220         u64                     root;
 221         struct list_head        list;
 222 };
 223
 224 struct scrub_copy_nocow_ctx {
 225         struct scrub_ctx        *sctx;
 226         u64                     logical;
 227         u64                     len;
 228         int                     mirror_num;
 229         u64                     physical_for_dev_replace;
 230         struct list_head        inodes;
 231         struct btrfs_work       work;
 232 };
 233
 234 struct scrub_warning {
 235         struct btrfs_path       *path;
 236         u64                     extent_item_size;
 237         const char              *errstr;
 238         sector_t                sector;
 239         u64                     logical;
 240         struct btrfs_device     *dev;
 241 };
 242
 243 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
 244 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
 245 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
 246 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
 247 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
 248 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
 249                                      struct scrub_block *sblocks_for_recheck);
 250 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
 251                                 struct scrub_block *sblock, int is_metadata,
 252                                 int have_csum, u8 *csum, u64 generation,
 253                                 u16 csum_size, int retry_failed_mirror);
 254 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
 255                                          struct scrub_block *sblock,
 256                                          int is_metadata, int have_csum,
 257                                          const u8 *csum, u64 generation,
 258                                          u16 csum_size);
 259 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
 260                                              struct scrub_block *sblock_good);
 261 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
 262                                             struct scrub_block *sblock_good,
 263                                             int page_num, int force_write);
 264 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
 265 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
 266                                            int page_num);
 267 static int scrub_checksum_data(struct scrub_block *sblock);
 268 static int scrub_checksum_tree_block(struct scrub_block *sblock);
 269 static int scrub_checksum_super(struct scrub_block *sblock);
 270 static void scrub_block_get(struct scrub_block *sblock);
 271 static void scrub_block_put(struct scrub_block *sblock);
 272 static void scrub_page_get(struct scrub_page *spage);
 273 static void scrub_page_put(struct scrub_page *spage);
 274 static void scrub_parity_get(struct scrub_parity *sparity);
 275 static void scrub_parity_put(struct scrub_parity *sparity);
 276 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
 277                                     struct scrub_page *spage);
 278 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 279                        u64 physical, struct btrfs_device *dev, u64 flags,
 280                        u64 gen, int mirror_num, u8 *csum, int force,
 281                        u64 physical_for_dev_replace);
 282 static void scrub_bio_end_io(struct bio *bio);
 283 static void scrub_bio_end_io_worker(struct btrfs_work *work);
 284 static void scrub_block_complete(struct scrub_block *sblock);
 285 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
 286                                u64 extent_logical, u64 extent_len,
 287                                u64 *extent_physical,
 288                                struct btrfs_device **extent_dev,
 289                                int *extent_mirror_num);
 290 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
 291                               struct scrub_wr_ctx *wr_ctx,
 292                               struct btrfs_fs_info *fs_info,
 293                               struct btrfs_device *dev,
 294                               int is_dev_replace);
 295 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
 296 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
 297                                     struct scrub_page *spage);
 298 static void scrub_wr_submit(struct scrub_ctx *sctx);
 299 static void scrub_wr_bio_end_io(struct bio *bio);
 300 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
 301 static int write_page_nocow(struct scrub_ctx *sctx,
 302                             u64 physical_for_dev_replace, struct page *page);
 303 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
 304                                       struct scrub_copy_nocow_ctx *ctx);
 305 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 306                             int mirror_num, u64 physical_for_dev_replace);
 307 static void copy_nocow_pages_worker(struct btrfs_work *work);
 308 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 309 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 310 static void scrub_put_ctx(struct scrub_ctx *sctx);
 311
 312
 313 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
 314 {
 315         atomic_inc(&sctx->refs);
 316         atomic_inc(&sctx->bios_in_flight);
 317 }
 318
 319 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
 320 {
 321         atomic_dec(&sctx->bios_in_flight);
 322         wake_up(&sctx->list_wait);
 323         scrub_put_ctx(sctx);
 324 }
 325
 326 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 327 {
 328         while (atomic_read(&fs_info->scrub_pause_req)) {
 329                 mutex_unlock(&fs_info->scrub_lock);
 330                 wait_event(fs_info->scrub_pause_wait,
 331                    atomic_read(&fs_info->scrub_pause_req) == 0);
 332                 mutex_lock(&fs_info->scrub_lock);
 333         }
 334 }
 335
 336 static void scrub_pause_on(struct btrfs_fs_info *fs_info)
 337 {
 338         atomic_inc(&fs_info->scrubs_paused);
 339         wake_up(&fs_info->scrub_pause_wait);
 340 }
 341
 342 static void scrub_pause_off(struct btrfs_fs_info *fs_info)
 343 {
 344         mutex_lock(&fs_info->scrub_lock);
 345         __scrub_blocked_if_needed(fs_info);
 346         atomic_dec(&fs_info->scrubs_paused);
 347         mutex_unlock(&fs_info->scrub_lock);
 348
 349         wake_up(&fs_info->scrub_pause_wait);
 350 }
 351
 352 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 353 {
 354         scrub_pause_on(fs_info);
 355         scrub_pause_off(fs_info);
 356 }
 357
 358 /*
 359  * used for workers that require transaction commits (i.e., for the
 360  * NOCOW case)
 361  */
 362 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
 363 {
 364         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 365
 366         atomic_inc(&sctx->refs);
 367         /*
 368          * increment scrubs_running to prevent cancel requests from
 369          * completing as long as a worker is running. we must also
 370          * increment scrubs_paused to prevent deadlocking on pause
 371          * requests used for transactions commits (as the worker uses a
 372          * transaction context). it is safe to regard the worker
 373          * as paused for all matters practical. effectively, we only
 374          * avoid cancellation requests from completing.
 375          */
 376         mutex_lock(&fs_info->scrub_lock);
 377         atomic_inc(&fs_info->scrubs_running);
 378         atomic_inc(&fs_info->scrubs_paused);
 379         mutex_unlock(&fs_info->scrub_lock);
 380
 381         /*
 382          * check if @scrubs_running=@scrubs_paused condition
 383          * inside wait_event() is not an atomic operation.
 384          * which means we may inc/dec @scrub_running/paused
 385          * at any time. Let's wake up @scrub_pause_wait as
 386          * much as we can to let commit transaction blocked less.
 387          */
 388         wake_up(&fs_info->scrub_pause_wait);
 389
 390         atomic_inc(&sctx->workers_pending);
 391 }
 392
 393 /* used for workers that require transaction commits */
 394 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
 395 {
 396         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 397
 398         /*
 399          * see scrub_pending_trans_workers_inc() why we're pretending
 400          * to be paused in the scrub counters
 401          */
 402         mutex_lock(&fs_info->scrub_lock);
 403         atomic_dec(&fs_info->scrubs_running);
 404         atomic_dec(&fs_info->scrubs_paused);
 405         mutex_unlock(&fs_info->scrub_lock);
 406         atomic_dec(&sctx->workers_pending);
 407         wake_up(&fs_info->scrub_pause_wait);
 408         wake_up(&sctx->list_wait);
 409         scrub_put_ctx(sctx);
 410 }
 411
 412 static void scrub_free_csums(struct scrub_ctx *sctx)
 413 {
 414         while (!list_empty(&sctx->csum_list)) {
 415                 struct btrfs_ordered_sum *sum;
 416                 sum = list_first_entry(&sctx->csum_list,
 417                                        struct btrfs_ordered_sum, list);
 418                 list_del(&sum->list);
 419                 kfree(sum);
 420         }
 421 }
 422
 423 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
 424 {
 425         int i;
 426
 427         if (!sctx)
 428                 return;
 429
 430         scrub_free_wr_ctx(&sctx->wr_ctx);
 431
 432         /* this can happen when scrub is cancelled */
 433         if (sctx->curr != -1) {
 434                 struct scrub_bio *sbio = sctx->bios[sctx->curr];
 435
 436                 for (i = 0; i < sbio->page_count; i++) {
 437                         WARN_ON(!sbio->pagev[i]->page);
 438                         scrub_block_put(sbio->pagev[i]->sblock);
 439                 }
 440                 bio_put(sbio->bio);
 441         }
 442
 443         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 444                 struct scrub_bio *sbio = sctx->bios[i];
 445
 446                 if (!sbio)
 447                         break;
 448                 kfree(sbio);
 449         }
 450
 451         scrub_free_csums(sctx);
 452         kfree(sctx);
 453 }
 454
 455 static void scrub_put_ctx(struct scrub_ctx *sctx)
 456 {
 457         if (atomic_dec_and_test(&sctx->refs))
 458                 scrub_free_ctx(sctx);
 459 }
 460
 461 static noinline_for_stack
 462 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
 463 {
 464         struct scrub_ctx *sctx;
 465         int             i;
 466         struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
 467         int ret;
 468
 469         sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
 470         if (!sctx)
 471                 goto nomem;
 472         atomic_set(&sctx->refs, 1);
 473         sctx->is_dev_replace = is_dev_replace;
 474         sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
 475         sctx->curr = -1;
 476         sctx->dev_root = dev->dev_root;
 477         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 478                 struct scrub_bio *sbio;
 479
 480                 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
 481                 if (!sbio)
 482                         goto nomem;
 483                 sctx->bios[i] = sbio;
 484
 485                 sbio->index = i;
 486                 sbio->sctx = sctx;
 487                 sbio->page_count = 0;
 488                 btrfs_init_work(&sbio->work, btrfs_scrub_helper,
 489                                 scrub_bio_end_io_worker, NULL, NULL);
 490
 491                 if (i != SCRUB_BIOS_PER_SCTX - 1)
 492                         sctx->bios[i]->next_free = i + 1;
 493                 else
 494                         sctx->bios[i]->next_free = -1;
 495         }
 496         sctx->first_free = 0;
 497         sctx->nodesize = dev->dev_root->nodesize;
 498         sctx->sectorsize = dev->dev_root->sectorsize;
 499         atomic_set(&sctx->bios_in_flight, 0);
 500         atomic_set(&sctx->workers_pending, 0);
 501         atomic_set(&sctx->cancel_req, 0);
 502         sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
 503         INIT_LIST_HEAD(&sctx->csum_list);
 504
 505         spin_lock_init(&sctx->list_lock);
 506         spin_lock_init(&sctx->stat_lock);
 507         init_waitqueue_head(&sctx->list_wait);
 508
 509         ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
 510                                  fs_info->dev_replace.tgtdev, is_dev_replace);
 511         if (ret) {
 512                 scrub_free_ctx(sctx);
 513                 return ERR_PTR(ret);
 514         }
 515         return sctx;
 516
 517 nomem:
 518         scrub_free_ctx(sctx);
 519         return ERR_PTR(-ENOMEM);
 520 }
 521
 522 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
 523                                      void *warn_ctx)
 524 {
 525         u64 isize;
 526         u32 nlink;
 527         int ret;
 528         int i;
 529         struct extent_buffer *eb;
 530         struct btrfs_inode_item *inode_item;
 531         struct scrub_warning *swarn = warn_ctx;
 532         struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
 533         struct inode_fs_paths *ipath = NULL;
 534         struct btrfs_root *local_root;
 535         struct btrfs_key root_key;
 536         struct btrfs_key key;
 537
 538         root_key.objectid = root;
 539         root_key.type = BTRFS_ROOT_ITEM_KEY;
 540         root_key.offset = (u64)-1;
 541         local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
 542         if (IS_ERR(local_root)) {
 543                 ret = PTR_ERR(local_root);
 544                 goto err;
 545         }
 546
 547         /*
 548          * this makes the path point to (inum INODE_ITEM ioff)
 549          */
 550         key.objectid = inum;
 551         key.type = BTRFS_INODE_ITEM_KEY;
 552         key.offset = 0;
 553
 554         ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
 555         if (ret) {
 556                 btrfs_release_path(swarn->path);
 557                 goto err;
 558         }
 559
 560         eb = swarn->path->nodes[0];
 561         inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
 562                                         struct btrfs_inode_item);
 563         isize = btrfs_inode_size(eb, inode_item);
 564         nlink = btrfs_inode_nlink(eb, inode_item);
 565         btrfs_release_path(swarn->path);
 566
 567         ipath = init_ipath(4096, local_root, swarn->path);
 568         if (IS_ERR(ipath)) {
 569                 ret = PTR_ERR(ipath);
 570                 ipath = NULL;
 571                 goto err;
 572         }
 573         ret = paths_from_inode(inum, ipath);
 574
 575         if (ret < 0)
 576                 goto err;
 577
 578         /*
 579          * we deliberately ignore the bit ipath might have been too small to
 580          * hold all of the paths here
 581          */
 582         for (i = 0; i < ipath->fspath->elem_cnt; ++i)
 583                 btrfs_warn_in_rcu(fs_info, "%s at logical %llu on dev "
 584                         "%s, sector %llu, root %llu, inode %llu, offset %llu, "
 585                         "length %llu, links %u (path: %s)", swarn->errstr,
 586                         swarn->logical, rcu_str_deref(swarn->dev->name),
 587                         (unsigned long long)swarn->sector, root, inum, offset,
 588                         min(isize - offset, (u64)PAGE_SIZE), nlink,
 589                         (char *)(unsigned long)ipath->fspath->val[i]);
 590
 591         free_ipath(ipath);
 592         return 0;
 593
 594 err:
 595         btrfs_warn_in_rcu(fs_info, "%s at logical %llu on dev "
 596                 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
 597                 "resolving failed with ret=%d", swarn->errstr,
 598                 swarn->logical, rcu_str_deref(swarn->dev->name),
 599                 (unsigned long long)swarn->sector, root, inum, offset, ret);
 600
 601         free_ipath(ipath);
 602         return 0;
 603 }
 604
 605 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
 606 {
 607         struct btrfs_device *dev;
 608         struct btrfs_fs_info *fs_info;
 609         struct btrfs_path *path;
 610         struct btrfs_key found_key;
 611         struct extent_buffer *eb;
 612         struct btrfs_extent_item *ei;
 613         struct scrub_warning swarn;
 614         unsigned long ptr = 0;
 615         u64 extent_item_pos;
 616         u64 flags = 0;
 617         u64 ref_root;
 618         u32 item_size;
 619         u8 ref_level;
 620         int ret;
 621
 622         WARN_ON(sblock->page_count < 1);
 623         dev = sblock->pagev[0]->dev;
 624         fs_info = sblock->sctx->dev_root->fs_info;
 625
 626         path = btrfs_alloc_path();
 627         if (!path)
 628                 return;
 629
 630         swarn.sector = (sblock->pagev[0]->physical) >> 9;
 631         swarn.logical = sblock->pagev[0]->logical;
 632         swarn.errstr = errstr;
 633         swarn.dev = NULL;
 634
 635         ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
 636                                   &flags);
 637         if (ret < 0)
 638                 goto out;
 639
 640         extent_item_pos = swarn.logical - found_key.objectid;
 641         swarn.extent_item_size = found_key.offset;
 642
 643         eb = path->nodes[0];
 644         ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
 645         item_size = btrfs_item_size_nr(eb, path->slots[0]);
 646
 647         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 648                 do {
 649                         ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
 650                                                       item_size, &ref_root,
 651                                                       &ref_level);
 652                         btrfs_warn_in_rcu(fs_info,
 653                                 "%s at logical %llu on dev %s, "
 654                                 "sector %llu: metadata %s (level %d) in tree "
 655                                 "%llu", errstr, swarn.logical,
 656                                 rcu_str_deref(dev->name),
 657                                 (unsigned long long)swarn.sector,
 658                                 ref_level ? "node" : "leaf",
 659                                 ret < 0 ? -1 : ref_level,
 660                                 ret < 0 ? -1 : ref_root);
 661                 } while (ret != 1);
 662                 btrfs_release_path(path);
 663         } else {
 664                 btrfs_release_path(path);
 665                 swarn.path = path;
 666                 swarn.dev = dev;
 667                 iterate_extent_inodes(fs_info, found_key.objectid,
 668                                         extent_item_pos, 1,
 669                                         scrub_print_warning_inode, &swarn);
 670         }
 671
 672 out:
 673         btrfs_free_path(path);
 674 }
 675
 676 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
 677 {
 678         struct page *page = NULL;
 679         unsigned long index;
 680         struct scrub_fixup_nodatasum *fixup = fixup_ctx;
 681         int ret;
 682         int corrected = 0;
 683         struct btrfs_key key;
 684         struct inode *inode = NULL;
 685         struct btrfs_fs_info *fs_info;
 686         u64 end = offset + PAGE_SIZE - 1;
 687         struct btrfs_root *local_root;
 688         int srcu_index;
 689
 690         key.objectid = root;
 691         key.type = BTRFS_ROOT_ITEM_KEY;
 692         key.offset = (u64)-1;
 693
 694         fs_info = fixup->root->fs_info;
 695         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
 696
 697         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
 698         if (IS_ERR(local_root)) {
 699                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 700                 return PTR_ERR(local_root);
 701         }
 702
 703         key.type = BTRFS_INODE_ITEM_KEY;
 704         key.objectid = inum;
 705         key.offset = 0;
 706         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
 707         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 708         if (IS_ERR(inode))
 709                 return PTR_ERR(inode);
 710
 711         index = offset >> PAGE_CACHE_SHIFT;
 712
 713         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
 714         if (!page) {
 715                 ret = -ENOMEM;
 716                 goto out;
 717         }
 718
 719         if (PageUptodate(page)) {
 720                 if (PageDirty(page)) {
 721                         /*
 722                          * we need to write the data to the defect sector. the
 723                          * data that was in that sector is not in memory,
 724                          * because the page was modified. we must not write the
 725                          * modified page to that sector.
 726                          *
 727                          * TODO: what could be done here: wait for the delalloc
 728                          *       runner to write out that page (might involve
 729                          *       COW) and see whether the sector is still
 730                          *       referenced afterwards.
 731                          *
 732                          * For the meantime, we'll treat this error
 733                          * incorrectable, although there is a chance that a
 734                          * later scrub will find the bad sector again and that
 735                          * there's no dirty page in memory, then.
 736                          */
 737                         ret = -EIO;
 738                         goto out;
 739                 }
 740                 ret = repair_io_failure(inode, offset, PAGE_SIZE,
 741                                         fixup->logical, page,
 742                                         offset - page_offset(page),
 743                                         fixup->mirror_num);
 744                 unlock_page(page);
 745                 corrected = !ret;
 746         } else {
 747                 /*
 748                  * we need to get good data first. the general readpage path
 749                  * will call repair_io_failure for us, we just have to make
 750                  * sure we read the bad mirror.
 751                  */
 752                 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 753                                         EXTENT_DAMAGED, GFP_NOFS);
 754                 if (ret) {
 755                         /* set_extent_bits should give proper error */
 756                         WARN_ON(ret > 0);
 757                         if (ret > 0)
 758                                 ret = -EFAULT;
 759                         goto out;
 760                 }
 761
 762                 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
 763                                                 btrfs_get_extent,
 764                                                 fixup->mirror_num);
 765                 wait_on_page_locked(page);
 766
 767                 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
 768                                                 end, EXTENT_DAMAGED, 0, NULL);
 769                 if (!corrected)
 770                         clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 771                                                 EXTENT_DAMAGED, GFP_NOFS);
 772         }
 773
 774 out:
 775         if (page)
 776                 put_page(page);
 777
 778         iput(inode);
 779
 780         if (ret < 0)
 781                 return ret;
 782
 783         if (ret == 0 && corrected) {
 784                 /*
 785                  * we only need to call readpage for one of the inodes belonging
 786                  * to this extent. so make iterate_extent_inodes stop
 787                  */
 788                 return 1;
 789         }
 790
 791         return -EIO;
 792 }
 793
 794 static void scrub_fixup_nodatasum(struct btrfs_work *work)
 795 {
 796         int ret;
 797         struct scrub_fixup_nodatasum *fixup;
 798         struct scrub_ctx *sctx;
 799         struct btrfs_trans_handle *trans = NULL;
 800         struct btrfs_path *path;
 801         int uncorrectable = 0;
 802
 803         fixup = container_of(work, struct scrub_fixup_nodatasum, work);
 804         sctx = fixup->sctx;
 805
 806         path = btrfs_alloc_path();
 807         if (!path) {
 808                 spin_lock(&sctx->stat_lock);
 809                 ++sctx->stat.malloc_errors;
 810                 spin_unlock(&sctx->stat_lock);
 811                 uncorrectable = 1;
 812                 goto out;
 813         }
 814
 815         trans = btrfs_join_transaction(fixup->root);
 816         if (IS_ERR(trans)) {
 817                 uncorrectable = 1;
 818                 goto out;
 819         }
 820
 821         /*
 822          * the idea is to trigger a regular read through the standard path. we
 823          * read a page from the (failed) logical address by specifying the
 824          * corresponding copynum of the failed sector. thus, that readpage is
 825          * expected to fail.
 826          * that is the point where on-the-fly error correction will kick in
 827          * (once it's finished) and rewrite the failed sector if a good copy
 828          * can be found.
 829          */
 830         ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
 831                                                 path, scrub_fixup_readpage,
 832                                                 fixup);
 833         if (ret < 0) {
 834                 uncorrectable = 1;
 835                 goto out;
 836         }
 837         WARN_ON(ret != 1);
 838
 839         spin_lock(&sctx->stat_lock);
 840         ++sctx->stat.corrected_errors;
 841         spin_unlock(&sctx->stat_lock);
 842
 843 out:
 844         if (trans && !IS_ERR(trans))
 845                 btrfs_end_transaction(trans, fixup->root);
 846         if (uncorrectable) {
 847                 spin_lock(&sctx->stat_lock);
 848                 ++sctx->stat.uncorrectable_errors;
 849                 spin_unlock(&sctx->stat_lock);
 850                 btrfs_dev_replace_stats_inc(
 851                         &sctx->dev_root->fs_info->dev_replace.
 852                         num_uncorrectable_read_errors);
 853                 btrfs_err_rl_in_rcu(sctx->dev_root->fs_info,
 854                     "unable to fixup (nodatasum) error at logical %llu on dev %s",
 855                         fixup->logical, rcu_str_deref(fixup->dev->name));
 856         }
 857
 858         btrfs_free_path(path);
 859         kfree(fixup);
 860
 861         scrub_pending_trans_workers_dec(sctx);
 862 }
 863
 864 static inline void scrub_get_recover(struct scrub_recover *recover)
 865 {
 866         atomic_inc(&recover->refs);
 867 }
 868
 869 static inline void scrub_put_recover(struct scrub_recover *recover)
 870 {
 871         if (atomic_dec_and_test(&recover->refs)) {
 872                 btrfs_put_bbio(recover->bbio);
 873                 kfree(recover);
 874         }
 875 }
 876
 877 /*
 878  * scrub_handle_errored_block gets called when either verification of the
 879  * pages failed or the bio failed to read, e.g. with EIO. In the latter
 880  * case, this function handles all pages in the bio, even though only one
 881  * may be bad.
 882  * The goal of this function is to repair the errored block by using the
 883  * contents of one of the mirrors.
 884  */
 885 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
 886 {
 887         struct scrub_ctx *sctx = sblock_to_check->sctx;
 888         struct btrfs_device *dev;
 889         struct btrfs_fs_info *fs_info;
 890         u64 length;
 891         u64 logical;
 892         u64 generation;
 893         unsigned int failed_mirror_index;
 894         unsigned int is_metadata;
 895         unsigned int have_csum;
 896         u8 *csum;
 897         struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
 898         struct scrub_block *sblock_bad;
 899         int ret;
 900         int mirror_index;
 901         int page_num;
 902         int success;
 903         static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
 904                                       DEFAULT_RATELIMIT_BURST);
 905
 906         BUG_ON(sblock_to_check->page_count < 1);
 907         fs_info = sctx->dev_root->fs_info;
 908         if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
 909                 /*
 910                  * if we find an error in a super block, we just report it.
 911                  * They will get written with the next transaction commit
 912                  * anyway
 913                  */
 914                 spin_lock(&sctx->stat_lock);
 915                 ++sctx->stat.super_errors;
 916                 spin_unlock(&sctx->stat_lock);
 917                 return 0;
 918         }
 919         length = sblock_to_check->page_count * PAGE_SIZE;
 920         logical = sblock_to_check->pagev[0]->logical;
 921         generation = sblock_to_check->pagev[0]->generation;
 922         BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
 923         failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
 924         is_metadata = !(sblock_to_check->pagev[0]->flags &
 925                         BTRFS_EXTENT_FLAG_DATA);
 926         have_csum = sblock_to_check->pagev[0]->have_csum;
 927         csum = sblock_to_check->pagev[0]->csum;
 928         dev = sblock_to_check->pagev[0]->dev;
 929
 930         if (sctx->is_dev_replace && !is_metadata && !have_csum) {
 931                 sblocks_for_recheck = NULL;
 932                 goto nodatasum_case;
 933         }
 934
 935         /*
 936          * read all mirrors one after the other. This includes to
 937          * re-read the extent or metadata block that failed (that was
 938          * the cause that this fixup code is called) another time,
 939          * page by page this time in order to know which pages
 940          * caused I/O errors and which ones are good (for all mirrors).
 941          * It is the goal to handle the situation when more than one
 942          * mirror contains I/O errors, but the errors do not
 943          * overlap, i.e. the data can be repaired by selecting the
 944          * pages from those mirrors without I/O error on the
 945          * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
 946          * would be that mirror #1 has an I/O error on the first page,
 947          * the second page is good, and mirror #2 has an I/O error on
 948          * the second page, but the first page is good.
 949          * Then the first page of the first mirror can be repaired by
 950          * taking the first page of the second mirror, and the
 951          * second page of the second mirror can be repaired by
 952          * copying the contents of the 2nd page of the 1st mirror.
 953          * One more note: if the pages of one mirror contain I/O
 954          * errors, the checksum cannot be verified. In order to get
 955          * the best data for repairing, the first attempt is to find
 956          * a mirror without I/O errors and with a validated checksum.
 957          * Only if this is not possible, the pages are picked from
 958          * mirrors with I/O errors without considering the checksum.
 959          * If the latter is the case, at the end, the checksum of the
 960          * repaired area is verified in order to correctly maintain
 961          * the statistics.
 962          */
 963
 964         sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
 965                                       sizeof(*sblocks_for_recheck), GFP_NOFS);
 966         if (!sblocks_for_recheck) {
 967                 spin_lock(&sctx->stat_lock);
 968                 sctx->stat.malloc_errors++;
 969                 sctx->stat.read_errors++;
 970                 sctx->stat.uncorrectable_errors++;
 971                 spin_unlock(&sctx->stat_lock);
 972                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 973                 goto out;
 974         }
 975
 976         /* setup the context, map the logical blocks and alloc the pages */
 977         ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
 978         if (ret) {
 979                 spin_lock(&sctx->stat_lock);
 980                 sctx->stat.read_errors++;
 981                 sctx->stat.uncorrectable_errors++;
 982                 spin_unlock(&sctx->stat_lock);
 983                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 984                 goto out;
 985         }
 986         BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
 987         sblock_bad = sblocks_for_recheck + failed_mirror_index;
 988
 989         /* build and submit the bios for the failed mirror, check checksums */
 990         scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
 991                             csum, generation, sctx->csum_size, 1);
 992
 993         if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
 994             sblock_bad->no_io_error_seen) {
 995                 /*
 996                  * the error disappeared after reading page by page, or
 997                  * the area was part of a huge bio and other parts of the
 998                  * bio caused I/O errors, or the block layer merged several
 999                  * read requests into one and the error is caused by a
1000                  * different bio (usually one of the two latter cases is
1001                  * the cause)
1002                  */
1003                 spin_lock(&sctx->stat_lock);
1004                 sctx->stat.unverified_errors++;
1005                 sblock_to_check->data_corrected = 1;
1006                 spin_unlock(&sctx->stat_lock);
1007
1008                 if (sctx->is_dev_replace)
1009                         scrub_write_block_to_dev_replace(sblock_bad);
1010                 goto out;
1011         }
1012
1013         if (!sblock_bad->no_io_error_seen) {
1014                 spin_lock(&sctx->stat_lock);
1015                 sctx->stat.read_errors++;
1016                 spin_unlock(&sctx->stat_lock);
1017                 if (__ratelimit(&_rs))
1018                         scrub_print_warning("i/o error", sblock_to_check);
1019                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1020         } else if (sblock_bad->checksum_error) {
1021                 spin_lock(&sctx->stat_lock);
1022                 sctx->stat.csum_errors++;
1023                 spin_unlock(&sctx->stat_lock);
1024                 if (__ratelimit(&_rs))
1025                         scrub_print_warning("checksum error", sblock_to_check);
1026                 btrfs_dev_stat_inc_and_print(dev,
1027                                              BTRFS_DEV_STAT_CORRUPTION_ERRS);
1028         } else if (sblock_bad->header_error) {
1029                 spin_lock(&sctx->stat_lock);
1030                 sctx->stat.verify_errors++;
1031                 spin_unlock(&sctx->stat_lock);
1032                 if (__ratelimit(&_rs))
1033                         scrub_print_warning("checksum/header error",
1034                                             sblock_to_check);
1035                 if (sblock_bad->generation_error)
1036                         btrfs_dev_stat_inc_and_print(dev,
1037                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1038                 else
1039                         btrfs_dev_stat_inc_and_print(dev,
1040                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1041         }
1042
1043         if (sctx->readonly) {
1044                 ASSERT(!sctx->is_dev_replace);
1045                 goto out;
1046         }
1047
1048         if (!is_metadata && !have_csum) {
1049                 struct scrub_fixup_nodatasum *fixup_nodatasum;
1050
1051                 WARN_ON(sctx->is_dev_replace);
1052
1053 nodatasum_case:
1054
1055                 /*
1056                  * !is_metadata and !have_csum, this means that the data
1057                  * might not be COW'ed, that it might be modified
1058                  * concurrently. The general strategy to work on the
1059                  * commit root does not help in the case when COW is not
1060                  * used.
1061                  */
1062                 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
1063                 if (!fixup_nodatasum)
1064                         goto did_not_correct_error;
1065                 fixup_nodatasum->sctx = sctx;
1066                 fixup_nodatasum->dev = dev;
1067                 fixup_nodatasum->logical = logical;
1068                 fixup_nodatasum->root = fs_info->extent_root;
1069                 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
1070                 scrub_pending_trans_workers_inc(sctx);
1071                 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
1072                                 scrub_fixup_nodatasum, NULL, NULL);
1073                 btrfs_queue_work(fs_info->scrub_workers,
1074                                  &fixup_nodatasum->work);
1075                 goto out;
1076         }
1077
1078         /*
1079          * now build and submit the bios for the other mirrors, check
1080          * checksums.
1081          * First try to pick the mirror which is completely without I/O
1082          * errors and also does not have a checksum error.
1083          * If one is found, and if a checksum is present, the full block
1084          * that is known to contain an error is rewritten. Afterwards
1085          * the block is known to be corrected.
1086          * If a mirror is found which is completely correct, and no
1087          * checksum is present, only those pages are rewritten that had
1088          * an I/O error in the block to be repaired, since it cannot be
1089          * determined, which copy of the other pages is better (and it
1090          * could happen otherwise that a correct page would be
1091          * overwritten by a bad one).
1092          */
1093         for (mirror_index = 0;
1094              mirror_index < BTRFS_MAX_MIRRORS &&
1095              sblocks_for_recheck[mirror_index].page_count > 0;
1096              mirror_index++) {
1097                 struct scrub_block *sblock_other;
1098
1099                 if (mirror_index == failed_mirror_index)
1100                         continue;
1101                 sblock_other = sblocks_for_recheck + mirror_index;
1102
1103                 /* build and submit the bios, check checksums */
1104                 scrub_recheck_block(fs_info, sblock_other, is_metadata,
1105                                     have_csum, csum, generation,
1106                                     sctx->csum_size, 0);
1107
1108                 if (!sblock_other->header_error &&
1109                     !sblock_other->checksum_error &&
1110                     sblock_other->no_io_error_seen) {
1111                         if (sctx->is_dev_replace) {
1112                                 scrub_write_block_to_dev_replace(sblock_other);
1113                                 goto corrected_error;
1114                         } else {
1115                                 ret = scrub_repair_block_from_good_copy(
1116                                                 sblock_bad, sblock_other);
1117                                 if (!ret)
1118                                         goto corrected_error;
1119                         }
1120                 }
1121         }
1122
1123         if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
1124                 goto did_not_correct_error;
1125
1126         /*
1127          * In case of I/O errors in the area that is supposed to be
1128          * repaired, continue by picking good copies of those pages.
1129          * Select the good pages from mirrors to rewrite bad pages from
1130          * the area to fix. Afterwards verify the checksum of the block
1131          * that is supposed to be repaired. This verification step is
1132          * only done for the purpose of statistic counting and for the
1133          * final scrub report, whether errors remain.
1134          * A perfect algorithm could make use of the checksum and try
1135          * all possible combinations of pages from the different mirrors
1136          * until the checksum verification succeeds. For example, when
1137          * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1138          * of mirror #2 is readable but the final checksum test fails,
1139          * then the 2nd page of mirror #3 could be tried, whether now
1140          * the final checksum succeedes. But this would be a rare
1141          * exception and is therefore not implemented. At least it is
1142          * avoided that the good copy is overwritten.
1143          * A more useful improvement would be to pick the sectors
1144          * without I/O error based on sector sizes (512 bytes on legacy
1145          * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1146          * mirror could be repaired by taking 512 byte of a different
1147          * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1148          * area are unreadable.
1149          */
1150         success = 1;
1151         for (page_num = 0; page_num < sblock_bad->page_count;
1152              page_num++) {
1153                 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1154                 struct scrub_block *sblock_other = NULL;
1155
1156                 /* skip no-io-error page in scrub */
1157                 if (!page_bad->io_error && !sctx->is_dev_replace)
1158                         continue;
1159
1160                 /* try to find no-io-error page in mirrors */
1161                 if (page_bad->io_error) {
1162                         for (mirror_index = 0;
1163                              mirror_index < BTRFS_MAX_MIRRORS &&
1164                              sblocks_for_recheck[mirror_index].page_count > 0;
1165                              mirror_index++) {
1166                                 if (!sblocks_for_recheck[mirror_index].
1167                                     pagev[page_num]->io_error) {
1168                                         sblock_other = sblocks_for_recheck +
1169                                                        mirror_index;
1170                                         break;
1171                                 }
1172                         }
1173                         if (!sblock_other)
1174                                 success = 0;
1175                 }
1176
1177                 if (sctx->is_dev_replace) {
1178                         /*
1179                          * did not find a mirror to fetch the page
1180                          * from. scrub_write_page_to_dev_replace()
1181                          * handles this case (page->io_error), by
1182                          * filling the block with zeros before
1183                          * submitting the write request
1184                          */
1185                         if (!sblock_other)
1186                                 sblock_other = sblock_bad;
1187
1188                         if (scrub_write_page_to_dev_replace(sblock_other,
1189                                                             page_num) != 0) {
1190                                 btrfs_dev_replace_stats_inc(
1191                                         &sctx->dev_root->
1192                                         fs_info->dev_replace.
1193                                         num_write_errors);
1194                                 success = 0;
1195                         }
1196                 } else if (sblock_other) {
1197                         ret = scrub_repair_page_from_good_copy(sblock_bad,
1198                                                                sblock_other,
1199                                                                page_num, 0);
1200                         if (0 == ret)
1201                                 page_bad->io_error = 0;
1202                         else
1203                                 success = 0;
1204                 }
1205         }
1206
1207         if (success && !sctx->is_dev_replace) {
1208                 if (is_metadata || have_csum) {
1209                         /*
1210                          * need to verify the checksum now that all
1211                          * sectors on disk are repaired (the write
1212                          * request for data to be repaired is on its way).
1213                          * Just be lazy and use scrub_recheck_block()
1214                          * which re-reads the data before the checksum
1215                          * is verified, but most likely the data comes out
1216                          * of the page cache.
1217                          */
1218                         scrub_recheck_block(fs_info, sblock_bad,
1219                                             is_metadata, have_csum, csum,
1220                                             generation, sctx->csum_size, 1);
1221                         if (!sblock_bad->header_error &&
1222                             !sblock_bad->checksum_error &&
1223                             sblock_bad->no_io_error_seen)
1224                                 goto corrected_error;
1225                         else
1226                                 goto did_not_correct_error;
1227                 } else {
1228 corrected_error:
1229                         spin_lock(&sctx->stat_lock);
1230                         sctx->stat.corrected_errors++;
1231                         sblock_to_check->data_corrected = 1;
1232                         spin_unlock(&sctx->stat_lock);
1233                         btrfs_err_rl_in_rcu(fs_info,
1234                                 "fixed up error at logical %llu on dev %s",
1235                                 logical, rcu_str_deref(dev->name));
1236                 }
1237         } else {
1238 did_not_correct_error:
1239                 spin_lock(&sctx->stat_lock);
1240                 sctx->stat.uncorrectable_errors++;
1241                 spin_unlock(&sctx->stat_lock);
1242                 btrfs_err_rl_in_rcu(fs_info,
1243                         "unable to fixup (regular) error at logical %llu on dev %s",
1244                         logical, rcu_str_deref(dev->name));
1245         }
1246
1247 out:
1248         if (sblocks_for_recheck) {
1249                 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1250                      mirror_index++) {
1251                         struct scrub_block *sblock = sblocks_for_recheck +
1252                                                      mirror_index;
1253                         struct scrub_recover *recover;
1254                         int page_index;
1255
1256                         for (page_index = 0; page_index < sblock->page_count;
1257                              page_index++) {
1258                                 sblock->pagev[page_index]->sblock = NULL;
1259                                 recover = sblock->pagev[page_index]->recover;
1260                                 if (recover) {
1261                                         scrub_put_recover(recover);
1262                                         sblock->pagev[page_index]->recover =
1263                                                                         NULL;
1264                                 }
1265                                 scrub_page_put(sblock->pagev[page_index]);
1266                         }
1267                 }
1268                 kfree(sblocks_for_recheck);
1269         }
1270
1271         return 0;
1272 }
1273
1274 static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
1275 {
1276         if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
1277                 return 2;
1278         else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
1279                 return 3;
1280         else
1281                 return (int)bbio->num_stripes;
1282 }
1283
1284 static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1285                                                  u64 *raid_map,
1286                                                  u64 mapped_length,
1287                                                  int nstripes, int mirror,
1288                                                  int *stripe_index,
1289                                                  u64 *stripe_offset)
1290 {
1291         int i;
1292
1293         if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1294                 /* RAID5/6 */
1295                 for (i = 0; i < nstripes; i++) {
1296                         if (raid_map[i] == RAID6_Q_STRIPE ||
1297                             raid_map[i] == RAID5_P_STRIPE)
1298                                 continue;
1299
1300                         if (logical >= raid_map[i] &&
1301                             logical < raid_map[i] + mapped_length)
1302                                 break;
1303                 }
1304
1305                 *stripe_index = i;
1306                 *stripe_offset = logical - raid_map[i];
1307         } else {
1308                 /* The other RAID type */
1309                 *stripe_index = mirror;
1310                 *stripe_offset = 0;
1311         }
1312 }
1313
1314 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
1315                                      struct scrub_block *sblocks_for_recheck)
1316 {
1317         struct scrub_ctx *sctx = original_sblock->sctx;
1318         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
1319         u64 length = original_sblock->page_count * PAGE_SIZE;
1320         u64 logical = original_sblock->pagev[0]->logical;
1321         u64 generation = original_sblock->pagev[0]->generation;
1322         u64 flags = original_sblock->pagev[0]->flags;
1323         u64 have_csum = original_sblock->pagev[0]->have_csum;
1324         struct scrub_recover *recover;
1325         struct btrfs_bio *bbio;
1326         u64 sublen;
1327         u64 mapped_length;
1328         u64 stripe_offset;
1329         int stripe_index;
1330         int page_index = 0;
1331         int mirror_index;
1332         int nmirrors;
1333         int ret;
1334
1335         /*
1336          * note: the two members refs and outstanding_pages
1337          * are not used (and not set) in the blocks that are used for
1338          * the recheck procedure
1339          */
1340
1341         while (length > 0) {
1342                 sublen = min_t(u64, length, PAGE_SIZE);
1343                 mapped_length = sublen;
1344                 bbio = NULL;
1345
1346                 /*
1347                  * with a length of PAGE_SIZE, each returned stripe
1348                  * represents one mirror
1349                  */
1350                 ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical,
1351                                        &mapped_length, &bbio, 0, 1);
1352                 if (ret || !bbio || mapped_length < sublen) {
1353                         btrfs_put_bbio(bbio);
1354                         return -EIO;
1355                 }
1356
1357                 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1358                 if (!recover) {
1359                         btrfs_put_bbio(bbio);
1360                         return -ENOMEM;
1361                 }
1362
1363                 atomic_set(&recover->refs, 1);
1364                 recover->bbio = bbio;
1365                 recover->map_length = mapped_length;
1366
1367                 BUG_ON(page_index >= SCRUB_PAGES_PER_RD_BIO);
1368
1369                 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
1370
1371                 for (mirror_index = 0; mirror_index < nmirrors;
1372                      mirror_index++) {
1373                         struct scrub_block *sblock;
1374                         struct scrub_page *page;
1375
1376                         sblock = sblocks_for_recheck + mirror_index;
1377                         sblock->sctx = sctx;
1378
1379                         page = kzalloc(sizeof(*page), GFP_NOFS);
1380                         if (!page) {
1381 leave_nomem:
1382                                 spin_lock(&sctx->stat_lock);
1383                                 sctx->stat.malloc_errors++;
1384                                 spin_unlock(&sctx->stat_lock);
1385                                 scrub_put_recover(recover);
1386                                 return -ENOMEM;
1387                         }
1388                         scrub_page_get(page);
1389                         sblock->pagev[page_index] = page;
1390                         page->sblock = sblock;
1391                         page->flags = flags;
1392                         page->generation = generation;
1393                         page->logical = logical;
1394                         page->have_csum = have_csum;
1395                         if (have_csum)
1396                                 memcpy(page->csum,
1397                                        original_sblock->pagev[0]->csum,
1398                                        sctx->csum_size);
1399
1400                         scrub_stripe_index_and_offset(logical,
1401                                                       bbio->map_type,
1402                                                       bbio->raid_map,
1403                                                       mapped_length,
1404                                                       bbio->num_stripes -
1405                                                       bbio->num_tgtdevs,
1406                                                       mirror_index,
1407                                                       &stripe_index,
1408                                                       &stripe_offset);
1409                         page->physical = bbio->stripes[stripe_index].physical +
1410                                          stripe_offset;
1411                         page->dev = bbio->stripes[stripe_index].dev;
1412
1413                         BUG_ON(page_index >= original_sblock->page_count);
1414                         page->physical_for_dev_replace =
1415                                 original_sblock->pagev[page_index]->
1416                                 physical_for_dev_replace;
1417                         /* for missing devices, dev->bdev is NULL */
1418                         page->mirror_num = mirror_index + 1;
1419                         sblock->page_count++;
1420                         page->page = alloc_page(GFP_NOFS);
1421                         if (!page->page)
1422                                 goto leave_nomem;
1423
1424                         scrub_get_recover(recover);
1425                         page->recover = recover;
1426                 }
1427                 scrub_put_recover(recover);
1428                 length -= sublen;
1429                 logical += sublen;
1430                 page_index++;
1431         }
1432
1433         return 0;
1434 }
1435
1436 struct scrub_bio_ret {
1437         struct completion event;
1438         int error;
1439 };
1440
1441 static void scrub_bio_wait_endio(struct bio *bio)
1442 {
1443         struct scrub_bio_ret *ret = bio->bi_private;
1444
1445         ret->error = bio->bi_error;
1446         complete(&ret->event);
1447 }
1448
1449 static inline int scrub_is_page_on_raid56(struct scrub_page *page)
1450 {
1451         return page->recover &&
1452                (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
1453 }
1454
1455 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1456                                         struct bio *bio,
1457                                         struct scrub_page *page)
1458 {
1459         struct scrub_bio_ret done;
1460         int ret;
1461
1462         init_completion(&done.event);
1463         done.error = 0;
1464         bio->bi_iter.bi_sector = page->logical >> 9;
1465         bio->bi_private = &done;
1466         bio->bi_end_io = scrub_bio_wait_endio;
1467
1468         ret = raid56_parity_recover(fs_info->fs_root, bio, page->recover->bbio,
1469                                     page->recover->map_length,
1470                                     page->mirror_num, 0);
1471         if (ret)
1472                 return ret;
1473
1474         wait_for_completion(&done.event);
1475         if (done.error)
1476                 return -EIO;
1477
1478         return 0;
1479 }
1480
1481 /*
1482  * this function will check the on disk data for checksum errors, header
1483  * errors and read I/O errors. If any I/O errors happen, the exact pages
1484  * which are errored are marked as being bad. The goal is to enable scrub
1485  * to take those pages that are not errored from all the mirrors so that
1486  * the pages that are errored in the just handled mirror can be repaired.
1487  */
1488 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1489                                 struct scrub_block *sblock, int is_metadata,
1490                                 int have_csum, u8 *csum, u64 generation,
1491                                 u16 csum_size, int retry_failed_mirror)
1492 {
1493         int page_num;
1494
1495         sblock->no_io_error_seen = 1;
1496         sblock->header_error = 0;
1497         sblock->checksum_error = 0;
1498         sblock->generation_error = 0;
1499
1500         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1501                 struct bio *bio;
1502                 struct scrub_page *page = sblock->pagev[page_num];
1503
1504                 if (page->dev->bdev == NULL) {
1505                         page->io_error = 1;
1506                         sblock->no_io_error_seen = 0;
1507                         continue;
1508                 }
1509
1510                 WARN_ON(!page->page);
1511                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1512                 if (!bio) {
1513                         page->io_error = 1;
1514                         sblock->no_io_error_seen = 0;
1515                         continue;
1516                 }
1517                 bio->bi_bdev = page->dev->bdev;
1518
1519                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1520                 if (!retry_failed_mirror && scrub_is_page_on_raid56(page)) {
1521                         if (scrub_submit_raid56_bio_wait(fs_info, bio, page))
1522                                 sblock->no_io_error_seen = 0;
1523                 } else {
1524                         bio->bi_iter.bi_sector = page->physical >> 9;
1525
1526                         if (btrfsic_submit_bio_wait(READ, bio))
1527                                 sblock->no_io_error_seen = 0;
1528                 }
1529
1530                 bio_put(bio);
1531         }
1532
1533         if (sblock->no_io_error_seen)
1534                 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
1535                                              have_csum, csum, generation,
1536                                              csum_size);
1537
1538         return;
1539 }
1540
1541 static inline int scrub_check_fsid(u8 fsid[],
1542                                    struct scrub_page *spage)
1543 {
1544         struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1545         int ret;
1546
1547         ret = memcmp(fsid, fs_devices->fsid, BTRFS_UUID_SIZE);
1548         return !ret;
1549 }
1550
1551 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
1552                                          struct scrub_block *sblock,
1553                                          int is_metadata, int have_csum,
1554                                          const u8 *csum, u64 generation,
1555                                          u16 csum_size)
1556 {
1557         int page_num;
1558         u8 calculated_csum[BTRFS_CSUM_SIZE];
1559         u32 crc = ~(u32)0;
1560         void *mapped_buffer;
1561
1562         WARN_ON(!sblock->pagev[0]->page);
1563         if (is_metadata) {
1564                 struct btrfs_header *h;
1565
1566                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1567                 h = (struct btrfs_header *)mapped_buffer;
1568
1569                 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h) ||
1570                     !scrub_check_fsid(h->fsid, sblock->pagev[0]) ||
1571                     memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1572                            BTRFS_UUID_SIZE)) {
1573                         sblock->header_error = 1;
1574                 } else if (generation != btrfs_stack_header_generation(h)) {
1575                         sblock->header_error = 1;
1576                         sblock->generation_error = 1;
1577                 }
1578                 csum = h->csum;
1579         } else {
1580                 if (!have_csum)
1581                         return;
1582
1583                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1584         }
1585
1586         for (page_num = 0;;) {
1587                 if (page_num == 0 && is_metadata)
1588                         crc = btrfs_csum_data(
1589                                 ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
1590                                 crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
1591                 else
1592                         crc = btrfs_csum_data(mapped_buffer, crc, PAGE_SIZE);
1593
1594                 kunmap_atomic(mapped_buffer);
1595                 page_num++;
1596                 if (page_num >= sblock->page_count)
1597                         break;
1598                 WARN_ON(!sblock->pagev[page_num]->page);
1599
1600                 mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);
1601         }
1602
1603         btrfs_csum_final(crc, calculated_csum);
1604         if (memcmp(calculated_csum, csum, csum_size))
1605                 sblock->checksum_error = 1;
1606 }
1607
1608 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1609                                              struct scrub_block *sblock_good)
1610 {
1611         int page_num;
1612         int ret = 0;
1613
1614         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1615                 int ret_sub;
1616
1617                 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1618                                                            sblock_good,
1619                                                            page_num, 1);
1620                 if (ret_sub)
1621                         ret = ret_sub;
1622         }
1623
1624         return ret;
1625 }
1626
1627 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1628                                             struct scrub_block *sblock_good,
1629                                             int page_num, int force_write)
1630 {
1631         struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1632         struct scrub_page *page_good = sblock_good->pagev[page_num];
1633
1634         BUG_ON(page_bad->page == NULL);
1635         BUG_ON(page_good->page == NULL);
1636         if (force_write || sblock_bad->header_error ||
1637             sblock_bad->checksum_error || page_bad->io_error) {
1638                 struct bio *bio;
1639                 int ret;
1640
1641                 if (!page_bad->dev->bdev) {
1642                         btrfs_warn_rl(sblock_bad->sctx->dev_root->fs_info,
1643                                 "scrub_repair_page_from_good_copy(bdev == NULL) "
1644                                 "is unexpected");
1645                         return -EIO;
1646                 }
1647
1648                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1649                 if (!bio)
1650                         return -EIO;
1651                 bio->bi_bdev = page_bad->dev->bdev;
1652                 bio->bi_iter.bi_sector = page_bad->physical >> 9;
1653
1654                 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1655                 if (PAGE_SIZE != ret) {
1656                         bio_put(bio);
1657                         return -EIO;
1658                 }
1659
1660                 if (btrfsic_submit_bio_wait(WRITE, bio)) {
1661                         btrfs_dev_stat_inc_and_print(page_bad->dev,
1662                                 BTRFS_DEV_STAT_WRITE_ERRS);
1663                         btrfs_dev_replace_stats_inc(
1664                                 &sblock_bad->sctx->dev_root->fs_info->
1665                                 dev_replace.num_write_errors);
1666                         bio_put(bio);
1667                         return -EIO;
1668                 }
1669                 bio_put(bio);
1670         }
1671
1672         return 0;
1673 }
1674
1675 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1676 {
1677         int page_num;
1678
1679         /*
1680          * This block is used for the check of the parity on the source device,
1681          * so the data needn't be written into the destination device.
1682          */
1683         if (sblock->sparity)
1684                 return;
1685
1686         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1687                 int ret;
1688
1689                 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1690                 if (ret)
1691                         btrfs_dev_replace_stats_inc(
1692                                 &sblock->sctx->dev_root->fs_info->dev_replace.
1693                                 num_write_errors);
1694         }
1695 }
1696
1697 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1698                                            int page_num)
1699 {
1700         struct scrub_page *spage = sblock->pagev[page_num];
1701
1702         BUG_ON(spage->page == NULL);
1703         if (spage->io_error) {
1704                 void *mapped_buffer = kmap_atomic(spage->page);
1705
1706                 memset(mapped_buffer, 0, PAGE_CACHE_SIZE);
1707                 flush_dcache_page(spage->page);
1708                 kunmap_atomic(mapped_buffer);
1709         }
1710         return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1711 }
1712
1713 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1714                                     struct scrub_page *spage)
1715 {
1716         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1717         struct scrub_bio *sbio;
1718         int ret;
1719
1720         mutex_lock(&wr_ctx->wr_lock);
1721 again:
1722         if (!wr_ctx->wr_curr_bio) {
1723                 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1724                                               GFP_NOFS);
1725                 if (!wr_ctx->wr_curr_bio) {
1726                         mutex_unlock(&wr_ctx->wr_lock);
1727                         return -ENOMEM;
1728                 }
1729                 wr_ctx->wr_curr_bio->sctx = sctx;
1730                 wr_ctx->wr_curr_bio->page_count = 0;
1731         }
1732         sbio = wr_ctx->wr_curr_bio;
1733         if (sbio->page_count == 0) {
1734                 struct bio *bio;
1735
1736                 sbio->physical = spage->physical_for_dev_replace;
1737                 sbio->logical = spage->logical;
1738                 sbio->dev = wr_ctx->tgtdev;
1739                 bio = sbio->bio;
1740                 if (!bio) {
1741                         bio = btrfs_io_bio_alloc(GFP_NOFS, wr_ctx->pages_per_wr_bio);
1742                         if (!bio) {
1743                                 mutex_unlock(&wr_ctx->wr_lock);
1744                                 return -ENOMEM;
1745                         }
1746                         sbio->bio = bio;
1747                 }
1748
1749                 bio->bi_private = sbio;
1750                 bio->bi_end_io = scrub_wr_bio_end_io;
1751                 bio->bi_bdev = sbio->dev->bdev;
1752                 bio->bi_iter.bi_sector = sbio->physical >> 9;
1753                 sbio->err = 0;
1754         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1755                    spage->physical_for_dev_replace ||
1756                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1757                    spage->logical) {
1758                 scrub_wr_submit(sctx);
1759                 goto again;
1760         }
1761
1762         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1763         if (ret != PAGE_SIZE) {
1764                 if (sbio->page_count < 1) {
1765                         bio_put(sbio->bio);
1766                         sbio->bio = NULL;
1767                         mutex_unlock(&wr_ctx->wr_lock);
1768                         return -EIO;
1769                 }
1770                 scrub_wr_submit(sctx);
1771                 goto again;
1772         }
1773
1774         sbio->pagev[sbio->page_count] = spage;
1775         scrub_page_get(spage);
1776         sbio->page_count++;
1777         if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1778                 scrub_wr_submit(sctx);
1779         mutex_unlock(&wr_ctx->wr_lock);
1780
1781         return 0;
1782 }
1783
1784 static void scrub_wr_submit(struct scrub_ctx *sctx)
1785 {
1786         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1787         struct scrub_bio *sbio;
1788
1789         if (!wr_ctx->wr_curr_bio)
1790                 return;
1791
1792         sbio = wr_ctx->wr_curr_bio;
1793         wr_ctx->wr_curr_bio = NULL;
1794         WARN_ON(!sbio->bio->bi_bdev);
1795         scrub_pending_bio_inc(sctx);
1796         /* process all writes in a single worker thread. Then the block layer
1797          * orders the requests before sending them to the driver which
1798          * doubled the write performance on spinning disks when measured
1799          * with Linux 3.5 */
1800         btrfsic_submit_bio(WRITE, sbio->bio);
1801 }
1802
1803 static void scrub_wr_bio_end_io(struct bio *bio)
1804 {
1805         struct scrub_bio *sbio = bio->bi_private;
1806         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1807
1808         sbio->err = bio->bi_error;
1809         sbio->bio = bio;
1810
1811         btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
1812                          scrub_wr_bio_end_io_worker, NULL, NULL);
1813         btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
1814 }
1815
1816 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1817 {
1818         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1819         struct scrub_ctx *sctx = sbio->sctx;
1820         int i;
1821
1822         WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1823         if (sbio->err) {
1824                 struct btrfs_dev_replace *dev_replace =
1825                         &sbio->sctx->dev_root->fs_info->dev_replace;
1826
1827                 for (i = 0; i < sbio->page_count; i++) {
1828                         struct scrub_page *spage = sbio->pagev[i];
1829
1830                         spage->io_error = 1;
1831                         btrfs_dev_replace_stats_inc(&dev_replace->
1832                                                     num_write_errors);
1833                 }
1834         }
1835
1836         for (i = 0; i < sbio->page_count; i++)
1837                 scrub_page_put(sbio->pagev[i]);
1838
1839         bio_put(sbio->bio);
1840         kfree(sbio);
1841         scrub_pending_bio_dec(sctx);
1842 }
1843
1844 static int scrub_checksum(struct scrub_block *sblock)
1845 {
1846         u64 flags;
1847         int ret;
1848
1849         WARN_ON(sblock->page_count < 1);
1850         flags = sblock->pagev[0]->flags;
1851         ret = 0;
1852         if (flags & BTRFS_EXTENT_FLAG_DATA)
1853                 ret = scrub_checksum_data(sblock);
1854         else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1855                 ret = scrub_checksum_tree_block(sblock);
1856         else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1857                 (void)scrub_checksum_super(sblock);
1858         else
1859                 WARN_ON(1);
1860         if (ret)
1861                 scrub_handle_errored_block(sblock);
1862
1863         return ret;
1864 }
1865
1866 static int scrub_checksum_data(struct scrub_block *sblock)
1867 {
1868         struct scrub_ctx *sctx = sblock->sctx;
1869         u8 csum[BTRFS_CSUM_SIZE];
1870         u8 *on_disk_csum;
1871         struct page *page;
1872         void *buffer;
1873         u32 crc = ~(u32)0;
1874         int fail = 0;
1875         u64 len;
1876         int index;
1877
1878         BUG_ON(sblock->page_count < 1);
1879         if (!sblock->pagev[0]->have_csum)
1880                 return 0;
1881
1882         on_disk_csum = sblock->pagev[0]->csum;
1883         page = sblock->pagev[0]->page;
1884         buffer = kmap_atomic(page);
1885
1886         len = sctx->sectorsize;
1887         index = 0;
1888         for (;;) {
1889                 u64 l = min_t(u64, len, PAGE_SIZE);
1890
1891                 crc = btrfs_csum_data(buffer, crc, l);
1892                 kunmap_atomic(buffer);
1893                 len -= l;
1894                 if (len == 0)
1895                         break;
1896                 index++;
1897                 BUG_ON(index >= sblock->page_count);
1898                 BUG_ON(!sblock->pagev[index]->page);
1899                 page = sblock->pagev[index]->page;
1900                 buffer = kmap_atomic(page);
1901         }
1902
1903         btrfs_csum_final(crc, csum);
1904         if (memcmp(csum, on_disk_csum, sctx->csum_size))
1905                 fail = 1;
1906
1907         return fail;
1908 }
1909
1910 static int scrub_checksum_tree_block(struct scrub_block *sblock)
1911 {
1912         struct scrub_ctx *sctx = sblock->sctx;
1913         struct btrfs_header *h;
1914         struct btrfs_root *root = sctx->dev_root;
1915         struct btrfs_fs_info *fs_info = root->fs_info;
1916         u8 calculated_csum[BTRFS_CSUM_SIZE];
1917         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1918         struct page *page;
1919         void *mapped_buffer;
1920         u64 mapped_size;
1921         void *p;
1922         u32 crc = ~(u32)0;
1923         int fail = 0;
1924         int crc_fail = 0;
1925         u64 len;
1926         int index;
1927
1928         BUG_ON(sblock->page_count < 1);
1929         page = sblock->pagev[0]->page;
1930         mapped_buffer = kmap_atomic(page);
1931         h = (struct btrfs_header *)mapped_buffer;
1932         memcpy(on_disk_csum, h->csum, sctx->csum_size);
1933
1934         /*
1935          * we don't use the getter functions here, as we
1936          * a) don't have an extent buffer and
1937          * b) the page is already kmapped
1938          */
1939
1940         if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
1941                 ++fail;
1942
1943         if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h))
1944                 ++fail;
1945
1946         if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
1947                 ++fail;
1948
1949         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1950                    BTRFS_UUID_SIZE))
1951                 ++fail;
1952
1953         len = sctx->nodesize - BTRFS_CSUM_SIZE;
1954         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1955         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1956         index = 0;
1957         for (;;) {
1958                 u64 l = min_t(u64, len, mapped_size);
1959
1960                 crc = btrfs_csum_data(p, crc, l);
1961                 kunmap_atomic(mapped_buffer);
1962                 len -= l;
1963                 if (len == 0)
1964                         break;
1965                 index++;
1966                 BUG_ON(index >= sblock->page_count);
1967                 BUG_ON(!sblock->pagev[index]->page);
1968                 page = sblock->pagev[index]->page;
1969                 mapped_buffer = kmap_atomic(page);
1970                 mapped_size = PAGE_SIZE;
1971                 p = mapped_buffer;
1972         }
1973
1974         btrfs_csum_final(crc, calculated_csum);
1975         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1976                 ++crc_fail;
1977
1978         return fail || crc_fail;
1979 }
1980
1981 static int scrub_checksum_super(struct scrub_block *sblock)
1982 {
1983         struct btrfs_super_block *s;
1984         struct scrub_ctx *sctx = sblock->sctx;
1985         u8 calculated_csum[BTRFS_CSUM_SIZE];
1986         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1987         struct page *page;
1988         void *mapped_buffer;
1989         u64 mapped_size;
1990         void *p;
1991         u32 crc = ~(u32)0;
1992         int fail_gen = 0;
1993         int fail_cor = 0;
1994         u64 len;
1995         int index;
1996
1997         BUG_ON(sblock->page_count < 1);
1998         page = sblock->pagev[0]->page;
1999         mapped_buffer = kmap_atomic(page);
2000         s = (struct btrfs_super_block *)mapped_buffer;
2001         memcpy(on_disk_csum, s->csum, sctx->csum_size);
2002
2003         if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
2004                 ++fail_cor;
2005
2006         if (sblock->pagev[0]->generation != btrfs_super_generation(s))
2007                 ++fail_gen;
2008
2009         if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
2010                 ++fail_cor;
2011
2012         len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
2013         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
2014         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
2015         index = 0;
2016         for (;;) {
2017                 u64 l = min_t(u64, len, mapped_size);
2018
2019                 crc = btrfs_csum_data(p, crc, l);
2020                 kunmap_atomic(mapped_buffer);
2021                 len -= l;
2022                 if (len == 0)
2023                         break;
2024                 index++;
2025                 BUG_ON(index >= sblock->page_count);
2026                 BUG_ON(!sblock->pagev[index]->page);
2027                 page = sblock->pagev[index]->page;
2028                 mapped_buffer = kmap_atomic(page);
2029                 mapped_size = PAGE_SIZE;
2030                 p = mapped_buffer;
2031         }
2032
2033         btrfs_csum_final(crc, calculated_csum);
2034         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
2035                 ++fail_cor;
2036
2037         if (fail_cor + fail_gen) {
2038                 /*
2039                  * if we find an error in a super block, we just report it.
2040                  * They will get written with the next transaction commit
2041                  * anyway
2042                  */
2043                 spin_lock(&sctx->stat_lock);
2044                 ++sctx->stat.super_errors;
2045                 spin_unlock(&sctx->stat_lock);
2046                 if (fail_cor)
2047                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2048                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
2049                 else
2050                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2051                                 BTRFS_DEV_STAT_GENERATION_ERRS);
2052         }
2053
2054         return fail_cor + fail_gen;
2055 }
2056
2057 static void scrub_block_get(struct scrub_block *sblock)
2058 {
2059         atomic_inc(&sblock->refs);
2060 }
2061
2062 static void scrub_block_put(struct scrub_block *sblock)
2063 {
2064         if (atomic_dec_and_test(&sblock->refs)) {
2065                 int i;
2066
2067                 if (sblock->sparity)
2068                         scrub_parity_put(sblock->sparity);
2069
2070                 for (i = 0; i < sblock->page_count; i++)
2071                         scrub_page_put(sblock->pagev[i]);
2072                 kfree(sblock);
2073         }
2074 }
2075
2076 static void scrub_page_get(struct scrub_page *spage)
2077 {
2078         atomic_inc(&spage->refs);
2079 }
2080
2081 static void scrub_page_put(struct scrub_page *spage)
2082 {
2083         if (atomic_dec_and_test(&spage->refs)) {
2084                 if (spage->page)
2085                         __free_page(spage->page);
2086                 kfree(spage);
2087         }
2088 }
2089
2090 static void scrub_submit(struct scrub_ctx *sctx)
2091 {
2092         struct scrub_bio *sbio;
2093
2094         if (sctx->curr == -1)
2095                 return;
2096
2097         sbio = sctx->bios[sctx->curr];
2098         sctx->curr = -1;
2099         scrub_pending_bio_inc(sctx);
2100         btrfsic_submit_bio(READ, sbio->bio);
2101 }
2102
2103 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
2104                                     struct scrub_page *spage)
2105 {
2106         struct scrub_block *sblock = spage->sblock;
2107         struct scrub_bio *sbio;
2108         int ret;
2109
2110 again:
2111         /*
2112          * grab a fresh bio or wait for one to become available
2113          */
2114         while (sctx->curr == -1) {
2115                 spin_lock(&sctx->list_lock);
2116                 sctx->curr = sctx->first_free;
2117                 if (sctx->curr != -1) {
2118                         sctx->first_free = sctx->bios[sctx->curr]->next_free;
2119                         sctx->bios[sctx->curr]->next_free = -1;
2120                         sctx->bios[sctx->curr]->page_count = 0;
2121                         spin_unlock(&sctx->list_lock);
2122                 } else {
2123                         spin_unlock(&sctx->list_lock);
2124                         wait_event(sctx->list_wait, sctx->first_free != -1);
2125                 }
2126         }
2127         sbio = sctx->bios[sctx->curr];
2128         if (sbio->page_count == 0) {
2129                 struct bio *bio;
2130
2131                 sbio->physical = spage->physical;
2132                 sbio->logical = spage->logical;
2133                 sbio->dev = spage->dev;
2134                 bio = sbio->bio;
2135                 if (!bio) {
2136                         bio = btrfs_io_bio_alloc(GFP_NOFS, sctx->pages_per_rd_bio);
2137                         if (!bio)
2138                                 return -ENOMEM;
2139                         sbio->bio = bio;
2140                 }
2141
2142                 bio->bi_private = sbio;
2143                 bio->bi_end_io = scrub_bio_end_io;
2144                 bio->bi_bdev = sbio->dev->bdev;
2145                 bio->bi_iter.bi_sector = sbio->physical >> 9;
2146                 sbio->err = 0;
2147         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
2148                    spage->physical ||
2149                    sbio->logical + sbio->page_count * PAGE_SIZE !=
2150                    spage->logical ||
2151                    sbio->dev != spage->dev) {
2152                 scrub_submit(sctx);
2153                 goto again;
2154         }
2155
2156         sbio->pagev[sbio->page_count] = spage;
2157         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
2158         if (ret != PAGE_SIZE) {
2159                 if (sbio->page_count < 1) {
2160                         bio_put(sbio->bio);
2161                         sbio->bio = NULL;
2162                         return -EIO;
2163                 }
2164                 scrub_submit(sctx);
2165                 goto again;
2166         }
2167
2168         scrub_block_get(sblock); /* one for the page added to the bio */
2169         atomic_inc(&sblock->outstanding_pages);
2170         sbio->page_count++;
2171         if (sbio->page_count == sctx->pages_per_rd_bio)
2172                 scrub_submit(sctx);
2173
2174         return 0;
2175 }
2176
2177 static void scrub_missing_raid56_end_io(struct bio *bio)
2178 {
2179         struct scrub_block *sblock = bio->bi_private;
2180         struct btrfs_fs_info *fs_info = sblock->sctx->dev_root->fs_info;
2181
2182         if (bio->bi_error)
2183                 sblock->no_io_error_seen = 0;
2184
2185         btrfs_queue_work(fs_info->scrub_workers, &sblock->work);
2186 }
2187
2188 static void scrub_missing_raid56_worker(struct btrfs_work *work)
2189 {
2190         struct scrub_block *sblock = container_of(work, struct scrub_block, work);
2191         struct scrub_ctx *sctx = sblock->sctx;
2192         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2193         unsigned int is_metadata;
2194         unsigned int have_csum;
2195         u8 *csum;
2196         u64 generation;
2197         u64 logical;
2198         struct btrfs_device *dev;
2199
2200         is_metadata = !(sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA);
2201         have_csum = sblock->pagev[0]->have_csum;
2202         csum = sblock->pagev[0]->csum;
2203         generation = sblock->pagev[0]->generation;
2204         logical = sblock->pagev[0]->logical;
2205         dev = sblock->pagev[0]->dev;
2206
2207         sblock->header_error = 0;
2208         sblock->checksum_error = 0;
2209         sblock->generation_error = 0;
2210         if (sblock->no_io_error_seen) {
2211                 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
2212                                              have_csum, csum, generation,
2213                                              sctx->csum_size);
2214         }
2215
2216         if (!sblock->no_io_error_seen) {
2217                 spin_lock(&sctx->stat_lock);
2218                 sctx->stat.read_errors++;
2219                 spin_unlock(&sctx->stat_lock);
2220                 btrfs_err_rl_in_rcu(fs_info,
2221                         "IO error rebuilding logical %llu for dev %s",
2222                         logical, rcu_str_deref(dev->name));
2223         } else if (sblock->header_error || sblock->checksum_error) {
2224                 spin_lock(&sctx->stat_lock);
2225                 sctx->stat.uncorrectable_errors++;
2226                 spin_unlock(&sctx->stat_lock);
2227                 btrfs_err_rl_in_rcu(fs_info,
2228                         "failed to rebuild valid logical %llu for dev %s",
2229                         logical, rcu_str_deref(dev->name));
2230         } else {
2231                 scrub_write_block_to_dev_replace(sblock);
2232         }
2233
2234         scrub_block_put(sblock);
2235
2236         if (sctx->is_dev_replace &&
2237             atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2238                 mutex_lock(&sctx->wr_ctx.wr_lock);
2239                 scrub_wr_submit(sctx);
2240                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2241         }
2242
2243         scrub_pending_bio_dec(sctx);
2244 }
2245
2246 static void scrub_missing_raid56_pages(struct scrub_block *sblock)
2247 {
2248         struct scrub_ctx *sctx = sblock->sctx;
2249         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2250         u64 length = sblock->page_count * PAGE_SIZE;
2251         u64 logical = sblock->pagev[0]->logical;
2252         struct btrfs_bio *bbio;
2253         struct bio *bio;
2254         struct btrfs_raid_bio *rbio;
2255         int ret;
2256         int i;
2257
2258         ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical, &length,
2259                                &bbio, 0, 1);
2260         if (ret || !bbio || !bbio->raid_map)
2261                 goto bbio_out;
2262
2263         if (WARN_ON(!sctx->is_dev_replace ||
2264                     !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) {
2265                 /*
2266                  * We shouldn't be scrubbing a missing device. Even for dev
2267                  * replace, we should only get here for RAID 5/6. We either
2268                  * managed to mount something with no mirrors remaining or
2269                  * there's a bug in scrub_remap_extent()/btrfs_map_block().
2270                  */
2271                 goto bbio_out;
2272         }
2273
2274         bio = btrfs_io_bio_alloc(GFP_NOFS, 0);
2275         if (!bio)
2276                 goto bbio_out;
2277
2278         bio->bi_iter.bi_sector = logical >> 9;
2279         bio->bi_private = sblock;
2280         bio->bi_end_io = scrub_missing_raid56_end_io;
2281
2282         rbio = raid56_alloc_missing_rbio(sctx->dev_root, bio, bbio, length);
2283         if (!rbio)
2284                 goto rbio_out;
2285
2286         for (i = 0; i < sblock->page_count; i++) {
2287                 struct scrub_page *spage = sblock->pagev[i];
2288
2289                 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2290         }
2291
2292         btrfs_init_work(&sblock->work, btrfs_scrub_helper,
2293                         scrub_missing_raid56_worker, NULL, NULL);
2294         scrub_block_get(sblock);
2295         scrub_pending_bio_inc(sctx);
2296         raid56_submit_missing_rbio(rbio);
2297         return;
2298
2299 rbio_out:
2300         bio_put(bio);
2301 bbio_out:
2302         btrfs_put_bbio(bbio);
2303         spin_lock(&sctx->stat_lock);
2304         sctx->stat.malloc_errors++;
2305         spin_unlock(&sctx->stat_lock);
2306 }
2307
2308 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
2309                        u64 physical, struct btrfs_device *dev, u64 flags,
2310                        u64 gen, int mirror_num, u8 *csum, int force,
2311                        u64 physical_for_dev_replace)
2312 {
2313         struct scrub_block *sblock;
2314         int index;
2315
2316         sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2317         if (!sblock) {
2318                 spin_lock(&sctx->stat_lock);
2319                 sctx->stat.malloc_errors++;
2320                 spin_unlock(&sctx->stat_lock);
2321                 return -ENOMEM;
2322         }
2323
2324         /* one ref inside this function, plus one for each page added to
2325          * a bio later on */
2326         atomic_set(&sblock->refs, 1);
2327         sblock->sctx = sctx;
2328         sblock->no_io_error_seen = 1;
2329
2330         for (index = 0; len > 0; index++) {
2331                 struct scrub_page *spage;
2332                 u64 l = min_t(u64, len, PAGE_SIZE);
2333
2334                 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2335                 if (!spage) {
2336 leave_nomem:
2337                         spin_lock(&sctx->stat_lock);
2338                         sctx->stat.malloc_errors++;
2339                         spin_unlock(&sctx->stat_lock);
2340                         scrub_block_put(sblock);
2341                         return -ENOMEM;
2342                 }
2343                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2344                 scrub_page_get(spage);
2345                 sblock->pagev[index] = spage;
2346                 spage->sblock = sblock;
2347                 spage->dev = dev;
2348                 spage->flags = flags;
2349                 spage->generation = gen;
2350                 spage->logical = logical;
2351                 spage->physical = physical;
2352                 spage->physical_for_dev_replace = physical_for_dev_replace;
2353                 spage->mirror_num = mirror_num;
2354                 if (csum) {
2355                         spage->have_csum = 1;
2356                         memcpy(spage->csum, csum, sctx->csum_size);
2357                 } else {
2358                         spage->have_csum = 0;
2359                 }
2360                 sblock->page_count++;
2361                 spage->page = alloc_page(GFP_NOFS);
2362                 if (!spage->page)
2363                         goto leave_nomem;
2364                 len -= l;
2365                 logical += l;
2366                 physical += l;
2367                 physical_for_dev_replace += l;
2368         }
2369
2370         WARN_ON(sblock->page_count == 0);
2371         if (dev->missing) {
2372                 /*
2373                  * This case should only be hit for RAID 5/6 device replace. See
2374                  * the comment in scrub_missing_raid56_pages() for details.
2375                  */
2376                 scrub_missing_raid56_pages(sblock);
2377         } else {
2378                 for (index = 0; index < sblock->page_count; index++) {
2379                         struct scrub_page *spage = sblock->pagev[index];
2380                         int ret;
2381
2382                         ret = scrub_add_page_to_rd_bio(sctx, spage);
2383                         if (ret) {
2384                                 scrub_block_put(sblock);
2385                                 return ret;
2386                         }
2387                 }
2388
2389                 if (force)
2390                         scrub_submit(sctx);
2391         }
2392
2393         /* last one frees, either here or in bio completion for last page */
2394         scrub_block_put(sblock);
2395         return 0;
2396 }
2397
2398 static void scrub_bio_end_io(struct bio *bio)
2399 {
2400         struct scrub_bio *sbio = bio->bi_private;
2401         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
2402
2403         sbio->err = bio->bi_error;
2404         sbio->bio = bio;
2405
2406         btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2407 }
2408
2409 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2410 {
2411         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2412         struct scrub_ctx *sctx = sbio->sctx;
2413         int i;
2414
2415         BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2416         if (sbio->err) {
2417                 for (i = 0; i < sbio->page_count; i++) {
2418                         struct scrub_page *spage = sbio->pagev[i];
2419
2420                         spage->io_error = 1;
2421                         spage->sblock->no_io_error_seen = 0;
2422                 }
2423         }
2424
2425         /* now complete the scrub_block items that have all pages completed */
2426         for (i = 0; i < sbio->page_count; i++) {
2427                 struct scrub_page *spage = sbio->pagev[i];
2428                 struct scrub_block *sblock = spage->sblock;
2429
2430                 if (atomic_dec_and_test(&sblock->outstanding_pages))
2431                         scrub_block_complete(sblock);
2432                 scrub_block_put(sblock);
2433         }
2434
2435         bio_put(sbio->bio);
2436         sbio->bio = NULL;
2437         spin_lock(&sctx->list_lock);
2438         sbio->next_free = sctx->first_free;
2439         sctx->first_free = sbio->index;
2440         spin_unlock(&sctx->list_lock);
2441
2442         if (sctx->is_dev_replace &&
2443             atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2444                 mutex_lock(&sctx->wr_ctx.wr_lock);
2445                 scrub_wr_submit(sctx);
2446                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2447         }
2448
2449         scrub_pending_bio_dec(sctx);
2450 }
2451
2452 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2453                                        unsigned long *bitmap,
2454                                        u64 start, u64 len)
2455 {
2456         u32 offset;
2457         int nsectors;
2458         int sectorsize = sparity->sctx->dev_root->sectorsize;
2459
2460         if (len >= sparity->stripe_len) {
2461                 bitmap_set(bitmap, 0, sparity->nsectors);
2462                 return;
2463         }
2464
2465         start -= sparity->logic_start;
2466         start = div_u64_rem(start, sparity->stripe_len, &offset);
2467         offset /= sectorsize;
2468         nsectors = (int)len / sectorsize;
2469
2470         if (offset + nsectors <= sparity->nsectors) {
2471                 bitmap_set(bitmap, offset, nsectors);
2472                 return;
2473         }
2474
2475         bitmap_set(bitmap, offset, sparity->nsectors - offset);
2476         bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2477 }
2478
2479 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2480                                                    u64 start, u64 len)
2481 {
2482         __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2483 }
2484
2485 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2486                                                   u64 start, u64 len)
2487 {
2488         __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2489 }
2490
2491 static void scrub_block_complete(struct scrub_block *sblock)
2492 {
2493         int corrupted = 0;
2494
2495         if (!sblock->no_io_error_seen) {
2496                 corrupted = 1;
2497                 scrub_handle_errored_block(sblock);
2498         } else {
2499                 /*
2500                  * if has checksum error, write via repair mechanism in
2501                  * dev replace case, otherwise write here in dev replace
2502                  * case.
2503                  */
2504                 corrupted = scrub_checksum(sblock);
2505                 if (!corrupted && sblock->sctx->is_dev_replace)
2506                         scrub_write_block_to_dev_replace(sblock);
2507         }
2508
2509         if (sblock->sparity && corrupted && !sblock->data_corrected) {
2510                 u64 start = sblock->pagev[0]->logical;
2511                 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
2512                           PAGE_SIZE;
2513
2514                 scrub_parity_mark_sectors_error(sblock->sparity,
2515                                                 start, end - start);
2516         }
2517 }
2518
2519 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u64 len,
2520                            u8 *csum)
2521 {
2522         struct btrfs_ordered_sum *sum = NULL;
2523         unsigned long index;
2524         unsigned long num_sectors;
2525
2526         while (!list_empty(&sctx->csum_list)) {
2527                 sum = list_first_entry(&sctx->csum_list,
2528                                        struct btrfs_ordered_sum, list);
2529                 if (sum->bytenr > logical)
2530                         return 0;
2531                 if (sum->bytenr + sum->len > logical)
2532                         break;
2533
2534                 ++sctx->stat.csum_discards;
2535                 list_del(&sum->list);
2536                 kfree(sum);
2537                 sum = NULL;
2538         }
2539         if (!sum)
2540                 return 0;
2541
2542         index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize;
2543         num_sectors = sum->len / sctx->sectorsize;
2544         memcpy(csum, sum->sums + index, sctx->csum_size);
2545         if (index == num_sectors - 1) {
2546                 list_del(&sum->list);
2547                 kfree(sum);
2548         }
2549         return 1;
2550 }
2551
2552 /* scrub extent tries to collect up to 64 kB for each bio */
2553 static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
2554                         u64 physical, struct btrfs_device *dev, u64 flags,
2555                         u64 gen, int mirror_num, u64 physical_for_dev_replace)
2556 {
2557         int ret;
2558         u8 csum[BTRFS_CSUM_SIZE];
2559         u32 blocksize;
2560
2561         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2562                 blocksize = sctx->sectorsize;
2563                 spin_lock(&sctx->stat_lock);
2564                 sctx->stat.data_extents_scrubbed++;
2565                 sctx->stat.data_bytes_scrubbed += len;
2566                 spin_unlock(&sctx->stat_lock);
2567         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2568                 blocksize = sctx->nodesize;
2569                 spin_lock(&sctx->stat_lock);
2570                 sctx->stat.tree_extents_scrubbed++;
2571                 sctx->stat.tree_bytes_scrubbed += len;
2572                 spin_unlock(&sctx->stat_lock);
2573         } else {
2574                 blocksize = sctx->sectorsize;
2575                 WARN_ON(1);
2576         }
2577
2578         while (len) {
2579                 u64 l = min_t(u64, len, blocksize);
2580                 int have_csum = 0;
2581
2582                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2583                         /* push csums to sbio */
2584                         have_csum = scrub_find_csum(sctx, logical, l, csum);
2585                         if (have_csum == 0)
2586                                 ++sctx->stat.no_csum;
2587                         if (sctx->is_dev_replace && !have_csum) {
2588                                 ret = copy_nocow_pages(sctx, logical, l,
2589                                                        mirror_num,
2590                                                       physical_for_dev_replace);
2591                                 goto behind_scrub_pages;
2592                         }
2593                 }
2594                 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2595                                   mirror_num, have_csum ? csum : NULL, 0,
2596                                   physical_for_dev_replace);
2597 behind_scrub_pages:
2598                 if (ret)
2599                         return ret;
2600                 len -= l;
2601                 logical += l;
2602                 physical += l;
2603                 physical_for_dev_replace += l;
2604         }
2605         return 0;
2606 }
2607
2608 static int scrub_pages_for_parity(struct scrub_parity *sparity,
2609                                   u64 logical, u64 len,
2610                                   u64 physical, struct btrfs_device *dev,
2611                                   u64 flags, u64 gen, int mirror_num, u8 *csum)
2612 {
2613         struct scrub_ctx *sctx = sparity->sctx;
2614         struct scrub_block *sblock;
2615         int index;
2616
2617         sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2618         if (!sblock) {
2619                 spin_lock(&sctx->stat_lock);
2620                 sctx->stat.malloc_errors++;
2621                 spin_unlock(&sctx->stat_lock);
2622                 return -ENOMEM;
2623         }
2624
2625         /* one ref inside this function, plus one for each page added to
2626          * a bio later on */
2627         atomic_set(&sblock->refs, 1);
2628         sblock->sctx = sctx;
2629         sblock->no_io_error_seen = 1;
2630         sblock->sparity = sparity;
2631         scrub_parity_get(sparity);
2632
2633         for (index = 0; len > 0; index++) {
2634                 struct scrub_page *spage;
2635                 u64 l = min_t(u64, len, PAGE_SIZE);
2636
2637                 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2638                 if (!spage) {
2639 leave_nomem:
2640                         spin_lock(&sctx->stat_lock);
2641                         sctx->stat.malloc_errors++;
2642                         spin_unlock(&sctx->stat_lock);
2643                         scrub_block_put(sblock);
2644                         return -ENOMEM;
2645                 }
2646                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2647                 /* For scrub block */
2648                 scrub_page_get(spage);
2649                 sblock->pagev[index] = spage;
2650                 /* For scrub parity */
2651                 scrub_page_get(spage);
2652                 list_add_tail(&spage->list, &sparity->spages);
2653                 spage->sblock = sblock;
2654                 spage->dev = dev;
2655                 spage->flags = flags;
2656                 spage->generation = gen;
2657                 spage->logical = logical;
2658                 spage->physical = physical;
2659                 spage->mirror_num = mirror_num;
2660                 if (csum) {
2661                         spage->have_csum = 1;
2662                         memcpy(spage->csum, csum, sctx->csum_size);
2663                 } else {
2664                         spage->have_csum = 0;
2665                 }
2666                 sblock->page_count++;
2667                 spage->page = alloc_page(GFP_NOFS);
2668                 if (!spage->page)
2669                         goto leave_nomem;
2670                 len -= l;
2671                 logical += l;
2672                 physical += l;
2673         }
2674
2675         WARN_ON(sblock->page_count == 0);
2676         for (index = 0; index < sblock->page_count; index++) {
2677                 struct scrub_page *spage = sblock->pagev[index];
2678                 int ret;
2679
2680                 ret = scrub_add_page_to_rd_bio(sctx, spage);
2681                 if (ret) {
2682                         scrub_block_put(sblock);
2683                         return ret;
2684                 }
2685         }
2686
2687         /* last one frees, either here or in bio completion for last page */
2688         scrub_block_put(sblock);
2689         return 0;
2690 }
2691
2692 static int scrub_extent_for_parity(struct scrub_parity *sparity,
2693                                    u64 logical, u64 len,
2694                                    u64 physical, struct btrfs_device *dev,
2695                                    u64 flags, u64 gen, int mirror_num)
2696 {
2697         struct scrub_ctx *sctx = sparity->sctx;
2698         int ret;
2699         u8 csum[BTRFS_CSUM_SIZE];
2700         u32 blocksize;
2701
2702         if (dev->missing) {
2703                 scrub_parity_mark_sectors_error(sparity, logical, len);
2704                 return 0;
2705         }
2706
2707         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2708                 blocksize = sctx->sectorsize;
2709         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2710                 blocksize = sctx->nodesize;
2711         } else {
2712                 blocksize = sctx->sectorsize;
2713                 WARN_ON(1);
2714         }
2715
2716         while (len) {
2717                 u64 l = min_t(u64, len, blocksize);
2718                 int have_csum = 0;
2719
2720                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2721                         /* push csums to sbio */
2722                         have_csum = scrub_find_csum(sctx, logical, l, csum);
2723                         if (have_csum == 0)
2724                                 goto skip;
2725                 }
2726                 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2727                                              flags, gen, mirror_num,
2728                                              have_csum ? csum : NULL);
2729                 if (ret)
2730                         return ret;
2731 skip:
2732                 len -= l;
2733                 logical += l;
2734                 physical += l;
2735         }
2736         return 0;
2737 }
2738
2739 /*
2740  * Given a physical address, this will calculate it's
2741  * logical offset. if this is a parity stripe, it will return
2742  * the most left data stripe's logical offset.
2743  *
2744  * return 0 if it is a data stripe, 1 means parity stripe.
2745  */
2746 static int get_raid56_logic_offset(u64 physical, int num,
2747                                    struct map_lookup *map, u64 *offset,
2748                                    u64 *stripe_start)
2749 {
2750         int i;
2751         int j = 0;
2752         u64 stripe_nr;
2753         u64 last_offset;
2754         u32 stripe_index;
2755         u32 rot;
2756
2757         last_offset = (physical - map->stripes[num].physical) *
2758                       nr_data_stripes(map);
2759         if (stripe_start)
2760                 *stripe_start = last_offset;
2761
2762         *offset = last_offset;
2763         for (i = 0; i < nr_data_stripes(map); i++) {
2764                 *offset = last_offset + i * map->stripe_len;
2765
2766                 stripe_nr = div_u64(*offset, map->stripe_len);
2767                 stripe_nr = div_u64(stripe_nr, nr_data_stripes(map));
2768
2769                 /* Work out the disk rotation on this stripe-set */
2770                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
2771                 /* calculate which stripe this data locates */
2772                 rot += i;
2773                 stripe_index = rot % map->num_stripes;
2774                 if (stripe_index == num)
2775                         return 0;
2776                 if (stripe_index < num)
2777                         j++;
2778         }
2779         *offset = last_offset + j * map->stripe_len;
2780         return 1;
2781 }
2782
2783 static void scrub_free_parity(struct scrub_parity *sparity)
2784 {
2785         struct scrub_ctx *sctx = sparity->sctx;
2786         struct scrub_page *curr, *next;
2787         int nbits;
2788
2789         nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
2790         if (nbits) {
2791                 spin_lock(&sctx->stat_lock);
2792                 sctx->stat.read_errors += nbits;
2793                 sctx->stat.uncorrectable_errors += nbits;
2794                 spin_unlock(&sctx->stat_lock);
2795         }
2796
2797         list_for_each_entry_safe(curr, next, &sparity->spages, list) {
2798                 list_del_init(&curr->list);
2799                 scrub_page_put(curr);
2800         }
2801
2802         kfree(sparity);
2803 }
2804
2805 static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
2806 {
2807         struct scrub_parity *sparity = container_of(work, struct scrub_parity,
2808                                                     work);
2809         struct scrub_ctx *sctx = sparity->sctx;
2810
2811         scrub_free_parity(sparity);
2812         scrub_pending_bio_dec(sctx);
2813 }
2814
2815 static void scrub_parity_bio_endio(struct bio *bio)
2816 {
2817         struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
2818
2819         if (bio->bi_error)
2820                 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2821                           sparity->nsectors);
2822
2823         bio_put(bio);
2824
2825         btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,
2826                         scrub_parity_bio_endio_worker, NULL, NULL);
2827         btrfs_queue_work(sparity->sctx->dev_root->fs_info->scrub_parity_workers,
2828                          &sparity->work);
2829 }
2830
2831 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
2832 {
2833         struct scrub_ctx *sctx = sparity->sctx;
2834         struct bio *bio;
2835         struct btrfs_raid_bio *rbio;
2836         struct scrub_page *spage;
2837         struct btrfs_bio *bbio = NULL;
2838         u64 length;
2839         int ret;
2840
2841         if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
2842                            sparity->nsectors))
2843                 goto out;
2844
2845         length = sparity->logic_end - sparity->logic_start;
2846         ret = btrfs_map_sblock(sctx->dev_root->fs_info, WRITE,
2847                                sparity->logic_start,
2848                                &length, &bbio, 0, 1);
2849         if (ret || !bbio || !bbio->raid_map)
2850                 goto bbio_out;
2851
2852         bio = btrfs_io_bio_alloc(GFP_NOFS, 0);
2853         if (!bio)
2854                 goto bbio_out;
2855
2856         bio->bi_iter.bi_sector = sparity->logic_start >> 9;
2857         bio->bi_private = sparity;
2858         bio->bi_end_io = scrub_parity_bio_endio;
2859
2860         rbio = raid56_parity_alloc_scrub_rbio(sctx->dev_root, bio, bbio,
2861                                               length, sparity->scrub_dev,
2862                                               sparity->dbitmap,
2863                                               sparity->nsectors);
2864         if (!rbio)
2865                 goto rbio_out;
2866
2867         list_for_each_entry(spage, &sparity->spages, list)
2868                 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2869
2870         scrub_pending_bio_inc(sctx);
2871         raid56_parity_submit_scrub_rbio(rbio);
2872         return;
2873
2874 rbio_out:
2875         bio_put(bio);
2876 bbio_out:
2877         btrfs_put_bbio(bbio);
2878         bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2879                   sparity->nsectors);
2880         spin_lock(&sctx->stat_lock);
2881         sctx->stat.malloc_errors++;
2882         spin_unlock(&sctx->stat_lock);
2883 out:
2884         scrub_free_parity(sparity);
2885 }
2886
2887 static inline int scrub_calc_parity_bitmap_len(int nsectors)
2888 {
2889         return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * (BITS_PER_LONG / 8);
2890 }
2891
2892 static void scrub_parity_get(struct scrub_parity *sparity)
2893 {
2894         atomic_inc(&sparity->refs);
2895 }
2896
2897 static void scrub_parity_put(struct scrub_parity *sparity)
2898 {
2899         if (!atomic_dec_and_test(&sparity->refs))
2900                 return;
2901
2902         scrub_parity_check_and_repair(sparity);
2903 }
2904
2905 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
2906                                                   struct map_lookup *map,
2907                                                   struct btrfs_device *sdev,
2908                                                   struct btrfs_path *path,
2909                                                   u64 logic_start,
2910                                                   u64 logic_end)
2911 {
2912         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2913         struct btrfs_root *root = fs_info->extent_root;
2914         struct btrfs_root *csum_root = fs_info->csum_root;
2915         struct btrfs_extent_item *extent;
2916         struct btrfs_bio *bbio = NULL;
2917         u64 flags;
2918         int ret;
2919         int slot;
2920         struct extent_buffer *l;
2921         struct btrfs_key key;
2922         u64 generation;
2923         u64 extent_logical;
2924         u64 extent_physical;
2925         u64 extent_len;
2926         u64 mapped_length;
2927         struct btrfs_device *extent_dev;
2928         struct scrub_parity *sparity;
2929         int nsectors;
2930         int bitmap_len;
2931         int extent_mirror_num;
2932         int stop_loop = 0;
2933
2934         nsectors = map->stripe_len / root->sectorsize;
2935         bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
2936         sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
2937                           GFP_NOFS);
2938         if (!sparity) {
2939                 spin_lock(&sctx->stat_lock);
2940                 sctx->stat.malloc_errors++;
2941                 spin_unlock(&sctx->stat_lock);
2942                 return -ENOMEM;
2943         }
2944
2945         sparity->stripe_len = map->stripe_len;
2946         sparity->nsectors = nsectors;
2947         sparity->sctx = sctx;
2948         sparity->scrub_dev = sdev;
2949         sparity->logic_start = logic_start;
2950         sparity->logic_end = logic_end;
2951         atomic_set(&sparity->refs, 1);
2952         INIT_LIST_HEAD(&sparity->spages);
2953         sparity->dbitmap = sparity->bitmap;
2954         sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
2955
2956         ret = 0;
2957         while (logic_start < logic_end) {
2958                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2959                         key.type = BTRFS_METADATA_ITEM_KEY;
2960                 else
2961                         key.type = BTRFS_EXTENT_ITEM_KEY;
2962                 key.objectid = logic_start;
2963                 key.offset = (u64)-1;
2964
2965                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2966                 if (ret < 0)
2967                         goto out;
2968
2969                 if (ret > 0) {
2970                         ret = btrfs_previous_extent_item(root, path, 0);
2971                         if (ret < 0)
2972                                 goto out;
2973                         if (ret > 0) {
2974                                 btrfs_release_path(path);
2975                                 ret = btrfs_search_slot(NULL, root, &key,
2976                                                         path, 0, 0);
2977                                 if (ret < 0)
2978                                         goto out;
2979                         }
2980                 }
2981
2982                 stop_loop = 0;
2983                 while (1) {
2984                         u64 bytes;
2985
2986                         l = path->nodes[0];
2987                         slot = path->slots[0];
2988                         if (slot >= btrfs_header_nritems(l)) {
2989                                 ret = btrfs_next_leaf(root, path);
2990                                 if (ret == 0)
2991                                         continue;
2992                                 if (ret < 0)
2993                                         goto out;
2994
2995                                 stop_loop = 1;
2996                                 break;
2997                         }
2998                         btrfs_item_key_to_cpu(l, &key, slot);
2999
3000                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3001                             key.type != BTRFS_METADATA_ITEM_KEY)
3002                                 goto next;
3003
3004                         if (key.type == BTRFS_METADATA_ITEM_KEY)
3005                                 bytes = root->nodesize;
3006                         else
3007                                 bytes = key.offset;
3008
3009                         if (key.objectid + bytes <= logic_start)
3010                                 goto next;
3011
3012                         if (key.objectid >= logic_end) {
3013                                 stop_loop = 1;
3014                                 break;
3015                         }
3016
3017                         while (key.objectid >= logic_start + map->stripe_len)
3018                                 logic_start += map->stripe_len;
3019
3020                         extent = btrfs_item_ptr(l, slot,
3021                                                 struct btrfs_extent_item);
3022                         flags = btrfs_extent_flags(l, extent);
3023                         generation = btrfs_extent_generation(l, extent);
3024
3025                         if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3026                             (key.objectid < logic_start ||
3027                              key.objectid + bytes >
3028                              logic_start + map->stripe_len)) {
3029                                 btrfs_err(fs_info, "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
3030                                           key.objectid, logic_start);
3031                                 spin_lock(&sctx->stat_lock);
3032                                 sctx->stat.uncorrectable_errors++;
3033                                 spin_unlock(&sctx->stat_lock);
3034                                 goto next;
3035                         }
3036 again:
3037                         extent_logical = key.objectid;
3038                         extent_len = bytes;
3039
3040                         if (extent_logical < logic_start) {
3041                                 extent_len -= logic_start - extent_logical;
3042                                 extent_logical = logic_start;
3043                         }
3044
3045                         if (extent_logical + extent_len >
3046                             logic_start + map->stripe_len)
3047                                 extent_len = logic_start + map->stripe_len -
3048                                              extent_logical;
3049
3050                         scrub_parity_mark_sectors_data(sparity, extent_logical,
3051                                                        extent_len);
3052
3053                         mapped_length = extent_len;
3054                         ret = btrfs_map_block(fs_info, READ, extent_logical,
3055                                               &mapped_length, &bbio, 0);
3056                         if (!ret) {
3057                                 if (!bbio || mapped_length < extent_len)
3058                                         ret = -EIO;
3059                         }
3060                         if (ret) {
3061                                 btrfs_put_bbio(bbio);
3062                                 goto out;
3063                         }
3064                         extent_physical = bbio->stripes[0].physical;
3065                         extent_mirror_num = bbio->mirror_num;
3066                         extent_dev = bbio->stripes[0].dev;
3067                         btrfs_put_bbio(bbio);
3068
3069                         ret = btrfs_lookup_csums_range(csum_root,
3070                                                 extent_logical,
3071                                                 extent_logical + extent_len - 1,
3072                                                 &sctx->csum_list, 1);
3073                         if (ret)
3074                                 goto out;
3075
3076                         ret = scrub_extent_for_parity(sparity, extent_logical,
3077                                                       extent_len,
3078                                                       extent_physical,
3079                                                       extent_dev, flags,
3080                                                       generation,
3081                                                       extent_mirror_num);
3082
3083                         scrub_free_csums(sctx);
3084
3085                         if (ret)
3086                                 goto out;
3087
3088                         if (extent_logical + extent_len <
3089                             key.objectid + bytes) {
3090                                 logic_start += map->stripe_len;
3091
3092                                 if (logic_start >= logic_end) {
3093                                         stop_loop = 1;
3094                                         break;
3095                                 }
3096
3097                                 if (logic_start < key.objectid + bytes) {
3098                                         cond_resched();
3099                                         goto again;
3100                                 }
3101                         }
3102 next:
3103                         path->slots[0]++;
3104                 }
3105
3106                 btrfs_release_path(path);
3107
3108                 if (stop_loop)
3109                         break;
3110
3111                 logic_start += map->stripe_len;
3112         }
3113 out:
3114         if (ret < 0)
3115                 scrub_parity_mark_sectors_error(sparity, logic_start,
3116                                                 logic_end - logic_start);
3117         scrub_parity_put(sparity);
3118         scrub_submit(sctx);
3119         mutex_lock(&sctx->wr_ctx.wr_lock);
3120         scrub_wr_submit(sctx);
3121         mutex_unlock(&sctx->wr_ctx.wr_lock);
3122
3123         btrfs_release_path(path);
3124         return ret < 0 ? ret : 0;
3125 }
3126
3127 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
3128                                            struct map_lookup *map,
3129                                            struct btrfs_device *scrub_dev,
3130                                            int num, u64 base, u64 length,
3131                                            int is_dev_replace)
3132 {
3133         struct btrfs_path *path, *ppath;
3134         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3135         struct btrfs_root *root = fs_info->extent_root;
3136         struct btrfs_root *csum_root = fs_info->csum_root;
3137         struct btrfs_extent_item *extent;
3138         struct blk_plug plug;
3139         u64 flags;
3140         int ret;
3141         int slot;
3142         u64 nstripes;
3143         struct extent_buffer *l;
3144         struct btrfs_key key;
3145         u64 physical;
3146         u64 logical;
3147         u64 logic_end;
3148         u64 physical_end;
3149         u64 generation;
3150         int mirror_num;
3151         struct reada_control *reada1;
3152         struct reada_control *reada2;
3153         struct btrfs_key key_start;
3154         struct btrfs_key key_end;
3155         u64 increment = map->stripe_len;
3156         u64 offset;
3157         u64 extent_logical;
3158         u64 extent_physical;
3159         u64 extent_len;
3160         u64 stripe_logical;
3161         u64 stripe_end;
3162         struct btrfs_device *extent_dev;
3163         int extent_mirror_num;
3164         int stop_loop = 0;
3165
3166         physical = map->stripes[num].physical;
3167         offset = 0;
3168         nstripes = div_u64(length, map->stripe_len);
3169         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3170                 offset = map->stripe_len * num;
3171                 increment = map->stripe_len * map->num_stripes;
3172                 mirror_num = 1;
3173         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3174                 int factor = map->num_stripes / map->sub_stripes;
3175                 offset = map->stripe_len * (num / map->sub_stripes);
3176                 increment = map->stripe_len * factor;
3177                 mirror_num = num % map->sub_stripes + 1;
3178         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
3179                 increment = map->stripe_len;
3180                 mirror_num = num % map->num_stripes + 1;
3181         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
3182                 increment = map->stripe_len;
3183                 mirror_num = num % map->num_stripes + 1;
3184         } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3185                 get_raid56_logic_offset(physical, num, map, &offset, NULL);
3186                 increment = map->stripe_len * nr_data_stripes(map);
3187                 mirror_num = 1;
3188         } else {
3189                 increment = map->stripe_len;
3190                 mirror_num = 1;
3191         }
3192
3193         path = btrfs_alloc_path();
3194         if (!path)
3195                 return -ENOMEM;
3196
3197         ppath = btrfs_alloc_path();
3198         if (!ppath) {
3199                 btrfs_free_path(path);
3200                 return -ENOMEM;
3201         }
3202
3203         /*
3204          * work on commit root. The related disk blocks are static as
3205          * long as COW is applied. This means, it is save to rewrite
3206          * them to repair disk errors without any race conditions
3207          */
3208         path->search_commit_root = 1;
3209         path->skip_locking = 1;
3210
3211         ppath->search_commit_root = 1;
3212         ppath->skip_locking = 1;
3213         /*
3214          * trigger the readahead for extent tree csum tree and wait for
3215          * completion. During readahead, the scrub is officially paused
3216          * to not hold off transaction commits
3217          */
3218         logical = base + offset;
3219         physical_end = physical + nstripes * map->stripe_len;
3220         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3221                 get_raid56_logic_offset(physical_end, num,
3222                                         map, &logic_end, NULL);
3223                 logic_end += base;
3224         } else {
3225                 logic_end = logical + increment * nstripes;
3226         }
3227         wait_event(sctx->list_wait,
3228                    atomic_read(&sctx->bios_in_flight) == 0);
3229         scrub_blocked_if_needed(fs_info);
3230
3231         /* FIXME it might be better to start readahead at commit root */
3232         key_start.objectid = logical;
3233         key_start.type = BTRFS_EXTENT_ITEM_KEY;
3234         key_start.offset = (u64)0;
3235         key_end.objectid = logic_end;
3236         key_end.type = BTRFS_METADATA_ITEM_KEY;
3237         key_end.offset = (u64)-1;
3238         reada1 = btrfs_reada_add(root, &key_start, &key_end);
3239
3240         key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3241         key_start.type = BTRFS_EXTENT_CSUM_KEY;
3242         key_start.offset = logical;
3243         key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3244         key_end.type = BTRFS_EXTENT_CSUM_KEY;
3245         key_end.offset = logic_end;
3246         reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);
3247
3248         if (!IS_ERR(reada1))
3249                 btrfs_reada_wait(reada1);
3250         if (!IS_ERR(reada2))
3251                 btrfs_reada_wait(reada2);
3252
3253
3254         /*
3255          * collect all data csums for the stripe to avoid seeking during
3256          * the scrub. This might currently (crc32) end up to be about 1MB
3257          */
3258         blk_start_plug(&plug);
3259
3260         /*
3261          * now find all extents for each stripe and scrub them
3262          */
3263         ret = 0;
3264         while (physical < physical_end) {
3265                 /*
3266                  * canceled?
3267                  */
3268                 if (atomic_read(&fs_info->scrub_cancel_req) ||
3269                     atomic_read(&sctx->cancel_req)) {
3270                         ret = -ECANCELED;
3271                         goto out;
3272                 }
3273                 /*
3274                  * check to see if we have to pause
3275                  */
3276                 if (atomic_read(&fs_info->scrub_pause_req)) {
3277                         /* push queued extents */
3278                         atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
3279                         scrub_submit(sctx);
3280                         mutex_lock(&sctx->wr_ctx.wr_lock);
3281                         scrub_wr_submit(sctx);
3282                         mutex_unlock(&sctx->wr_ctx.wr_lock);
3283                         wait_event(sctx->list_wait,
3284                                    atomic_read(&sctx->bios_in_flight) == 0);
3285                         atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
3286                         scrub_blocked_if_needed(fs_info);
3287                 }
3288
3289                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3290                         ret = get_raid56_logic_offset(physical, num, map,
3291                                                       &logical,
3292                                                       &stripe_logical);
3293                         logical += base;
3294                         if (ret) {
3295                                 /* it is parity strip */
3296                                 stripe_logical += base;
3297                                 stripe_end = stripe_logical + increment;
3298                                 ret = scrub_raid56_parity(sctx, map, scrub_dev,
3299                                                           ppath, stripe_logical,
3300                                                           stripe_end);
3301                                 if (ret)
3302                                         goto out;
3303                                 goto skip;
3304                         }
3305                 }
3306
3307                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3308                         key.type = BTRFS_METADATA_ITEM_KEY;
3309                 else
3310                         key.type = BTRFS_EXTENT_ITEM_KEY;
3311                 key.objectid = logical;
3312                 key.offset = (u64)-1;
3313
3314                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3315                 if (ret < 0)
3316                         goto out;
3317
3318                 if (ret > 0) {
3319                         ret = btrfs_previous_extent_item(root, path, 0);
3320                         if (ret < 0)
3321                                 goto out;
3322                         if (ret > 0) {
3323                                 /* there's no smaller item, so stick with the
3324                                  * larger one */
3325                                 btrfs_release_path(path);
3326                                 ret = btrfs_search_slot(NULL, root, &key,
3327                                                         path, 0, 0);
3328                                 if (ret < 0)
3329                                         goto out;
3330                         }
3331                 }
3332
3333                 stop_loop = 0;
3334                 while (1) {
3335                         u64 bytes;
3336
3337                         l = path->nodes[0];
3338                         slot = path->slots[0];
3339                         if (slot >= btrfs_header_nritems(l)) {
3340                                 ret = btrfs_next_leaf(root, path);
3341                                 if (ret == 0)
3342                                         continue;
3343                                 if (ret < 0)
3344                                         goto out;
3345
3346                                 stop_loop = 1;
3347                                 break;
3348                         }
3349                         btrfs_item_key_to_cpu(l, &key, slot);
3350
3351                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3352                             key.type != BTRFS_METADATA_ITEM_KEY)
3353                                 goto next;
3354
3355                         if (key.type == BTRFS_METADATA_ITEM_KEY)
3356                                 bytes = root->nodesize;
3357                         else
3358                                 bytes = key.offset;
3359
3360                         if (key.objectid + bytes <= logical)
3361                                 goto next;
3362
3363                         if (key.objectid >= logical + map->stripe_len) {
3364                                 /* out of this device extent */
3365                                 if (key.objectid >= logic_end)
3366                                         stop_loop = 1;
3367                                 break;
3368                         }
3369
3370                         extent = btrfs_item_ptr(l, slot,
3371                                                 struct btrfs_extent_item);
3372                         flags = btrfs_extent_flags(l, extent);
3373                         generation = btrfs_extent_generation(l, extent);
3374
3375                         if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3376                             (key.objectid < logical ||
3377                              key.objectid + bytes >
3378                              logical + map->stripe_len)) {
3379                                 btrfs_err(fs_info,
3380                                            "scrub: tree block %llu spanning "
3381                                            "stripes, ignored. logical=%llu",
3382                                        key.objectid, logical);
3383                                 spin_lock(&sctx->stat_lock);
3384                                 sctx->stat.uncorrectable_errors++;
3385                                 spin_unlock(&sctx->stat_lock);
3386                                 goto next;
3387                         }
3388
3389 again:
3390                         extent_logical = key.objectid;
3391                         extent_len = bytes;
3392
3393                         /*
3394                          * trim extent to this stripe
3395                          */
3396                         if (extent_logical < logical) {
3397                                 extent_len -= logical - extent_logical;
3398                                 extent_logical = logical;
3399                         }
3400                         if (extent_logical + extent_len >
3401                             logical + map->stripe_len) {
3402                                 extent_len = logical + map->stripe_len -
3403                                              extent_logical;
3404                         }
3405
3406                         extent_physical = extent_logical - logical + physical;
3407                         extent_dev = scrub_dev;
3408                         extent_mirror_num = mirror_num;
3409                         if (is_dev_replace)
3410                                 scrub_remap_extent(fs_info, extent_logical,
3411                                                    extent_len, &extent_physical,
3412                                                    &extent_dev,
3413                                                    &extent_mirror_num);
3414
3415                         ret = btrfs_lookup_csums_range(csum_root,
3416                                                        extent_logical,
3417                                                        extent_logical +
3418                                                        extent_len - 1,
3419                                                        &sctx->csum_list, 1);
3420                         if (ret)
3421                                 goto out;
3422
3423                         ret = scrub_extent(sctx, extent_logical, extent_len,
3424                                            extent_physical, extent_dev, flags,
3425                                            generation, extent_mirror_num,
3426                                            extent_logical - logical + physical);
3427
3428                         scrub_free_csums(sctx);
3429
3430                         if (ret)
3431                                 goto out;
3432
3433                         if (extent_logical + extent_len <
3434                             key.objectid + bytes) {
3435                                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3436                                         /*
3437                                          * loop until we find next data stripe
3438                                          * or we have finished all stripes.
3439                                          */
3440 loop:
3441                                         physical += map->stripe_len;
3442                                         ret = get_raid56_logic_offset(physical,
3443                                                         num, map, &logical,
3444                                                         &stripe_logical);
3445                                         logical += base;
3446
3447                                         if (ret && physical < physical_end) {
3448                                                 stripe_logical += base;
3449                                                 stripe_end = stripe_logical +
3450                                                                 increment;
3451                                                 ret = scrub_raid56_parity(sctx,
3452                                                         map, scrub_dev, ppath,
3453                                                         stripe_logical,
3454                                                         stripe_end);
3455                                                 if (ret)
3456                                                         goto out;
3457                                                 goto loop;
3458                                         }
3459                                 } else {
3460                                         physical += map->stripe_len;
3461                                         logical += increment;
3462                                 }
3463                                 if (logical < key.objectid + bytes) {
3464                                         cond_resched();
3465                                         goto again;
3466                                 }
3467
3468                                 if (physical >= physical_end) {
3469                                         stop_loop = 1;
3470                                         break;
3471                                 }
3472                         }
3473 next:
3474                         path->slots[0]++;
3475                 }
3476                 btrfs_release_path(path);
3477 skip:
3478                 logical += increment;
3479                 physical += map->stripe_len;
3480                 spin_lock(&sctx->stat_lock);
3481                 if (stop_loop)
3482                         sctx->stat.last_physical = map->stripes[num].physical +
3483                                                    length;
3484                 else
3485                         sctx->stat.last_physical = physical;
3486                 spin_unlock(&sctx->stat_lock);
3487                 if (stop_loop)
3488                         break;
3489         }
3490 out:
3491         /* push queued extents */
3492         scrub_submit(sctx);
3493         mutex_lock(&sctx->wr_ctx.wr_lock);
3494         scrub_wr_submit(sctx);
3495         mutex_unlock(&sctx->wr_ctx.wr_lock);
3496
3497         blk_finish_plug(&plug);
3498         btrfs_free_path(path);
3499         btrfs_free_path(ppath);
3500         return ret < 0 ? ret : 0;
3501 }
3502
3503 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
3504                                           struct btrfs_device *scrub_dev,
3505                                           u64 chunk_offset, u64 length,
3506                                           u64 dev_offset, int is_dev_replace)
3507 {
3508         struct btrfs_mapping_tree *map_tree =
3509                 &sctx->dev_root->fs_info->mapping_tree;
3510         struct map_lookup *map;
3511         struct extent_map *em;
3512         int i;
3513         int ret = 0;
3514
3515         read_lock(&map_tree->map_tree.lock);
3516         em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
3517         read_unlock(&map_tree->map_tree.lock);
3518
3519         if (!em)
3520                 return -EINVAL;
3521
3522         map = (struct map_lookup *)em->bdev;
3523         if (em->start != chunk_offset)
3524                 goto out;
3525
3526         if (em->len < length)
3527                 goto out;
3528
3529         for (i = 0; i < map->num_stripes; ++i) {
3530                 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
3531                     map->stripes[i].physical == dev_offset) {
3532                         ret = scrub_stripe(sctx, map, scrub_dev, i,
3533                                            chunk_offset, length,
3534                                            is_dev_replace);
3535                         if (ret)
3536                                 goto out;
3537                 }
3538         }
3539 out:
3540         free_extent_map(em);
3541
3542         return ret;
3543 }
3544
3545 static noinline_for_stack
3546 int scrub_enumerate_chunks(struct scrub_ctx *sctx,
3547                            struct btrfs_device *scrub_dev, u64 start, u64 end,
3548                            int is_dev_replace)
3549 {
3550         struct btrfs_dev_extent *dev_extent = NULL;
3551         struct btrfs_path *path;
3552         struct btrfs_root *root = sctx->dev_root;
3553         struct btrfs_fs_info *fs_info = root->fs_info;
3554         u64 length;
3555         u64 chunk_offset;
3556         int ret = 0;
3557         int slot;
3558         struct extent_buffer *l;
3559         struct btrfs_key key;
3560         struct btrfs_key found_key;
3561         struct btrfs_block_group_cache *cache;
3562         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
3563
3564         path = btrfs_alloc_path();
3565         if (!path)
3566                 return -ENOMEM;
3567
3568         path->reada = 2;
3569         path->search_commit_root = 1;
3570         path->skip_locking = 1;
3571
3572         key.objectid = scrub_dev->devid;
3573         key.offset = 0ull;
3574         key.type = BTRFS_DEV_EXTENT_KEY;
3575
3576         while (1) {
3577                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3578                 if (ret < 0)
3579                         break;
3580                 if (ret > 0) {
3581                         if (path->slots[0] >=
3582                             btrfs_header_nritems(path->nodes[0])) {
3583                                 ret = btrfs_next_leaf(root, path);
3584                                 if (ret < 0)
3585                                         break;
3586                                 if (ret > 0) {
3587                                         ret = 0;
3588                                         break;
3589                                 }
3590                         } else {
3591                                 ret = 0;
3592                         }
3593                 }
3594
3595                 l = path->nodes[0];
3596                 slot = path->slots[0];
3597
3598                 btrfs_item_key_to_cpu(l, &found_key, slot);
3599
3600                 if (found_key.objectid != scrub_dev->devid)
3601                         break;
3602
3603                 if (found_key.type != BTRFS_DEV_EXTENT_KEY)
3604                         break;
3605
3606                 if (found_key.offset >= end)
3607                         break;
3608
3609                 if (found_key.offset < key.offset)
3610                         break;
3611
3612                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
3613                 length = btrfs_dev_extent_length(l, dev_extent);
3614
3615                 if (found_key.offset + length <= start)
3616                         goto skip;
3617
3618                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
3619
3620                 /*
3621                  * get a reference on the corresponding block group to prevent
3622                  * the chunk from going away while we scrub it
3623                  */
3624                 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3625
3626                 /* some chunks are removed but not committed to disk yet,
3627                  * continue scrubbing */
3628                 if (!cache)
3629                         goto skip;
3630
3631                 /*
3632                  * we need call btrfs_inc_block_group_ro() with scrubs_paused,
3633                  * to avoid deadlock caused by:
3634                  * btrfs_inc_block_group_ro()
3635                  * -> btrfs_wait_for_commit()
3636                  * -> btrfs_commit_transaction()
3637                  * -> btrfs_scrub_pause()
3638                  */
3639                 scrub_pause_on(fs_info);
3640                 ret = btrfs_inc_block_group_ro(root, cache);
3641                 scrub_pause_off(fs_info);
3642                 if (ret) {
3643                         btrfs_put_block_group(cache);
3644                         break;
3645                 }
3646
3647                 dev_replace->cursor_right = found_key.offset + length;
3648                 dev_replace->cursor_left = found_key.offset;
3649                 dev_replace->item_needs_writeback = 1;
3650                 ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length,
3651                                   found_key.offset, is_dev_replace);
3652
3653                 /*
3654                  * flush, submit all pending read and write bios, afterwards
3655                  * wait for them.
3656                  * Note that in the dev replace case, a read request causes
3657                  * write requests that are submitted in the read completion
3658                  * worker. Therefore in the current situation, it is required
3659                  * that all write requests are flushed, so that all read and
3660                  * write requests are really completed when bios_in_flight
3661                  * changes to 0.
3662                  */
3663                 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
3664                 scrub_submit(sctx);
3665                 mutex_lock(&sctx->wr_ctx.wr_lock);
3666                 scrub_wr_submit(sctx);
3667                 mutex_unlock(&sctx->wr_ctx.wr_lock);
3668
3669                 wait_event(sctx->list_wait,
3670                            atomic_read(&sctx->bios_in_flight) == 0);
3671
3672                 scrub_pause_on(fs_info);
3673
3674                 /*
3675                  * must be called before we decrease @scrub_paused.
3676                  * make sure we don't block transaction commit while
3677                  * we are waiting pending workers finished.
3678                  */
3679                 wait_event(sctx->list_wait,
3680                            atomic_read(&sctx->workers_pending) == 0);
3681                 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
3682
3683                 scrub_pause_off(fs_info);
3684
3685                 btrfs_dec_block_group_ro(root, cache);
3686
3687                 btrfs_put_block_group(cache);
3688                 if (ret)
3689                         break;
3690                 if (is_dev_replace &&
3691                     atomic64_read(&dev_replace->num_write_errors) > 0) {
3692                         ret = -EIO;
3693                         break;
3694                 }
3695                 if (sctx->stat.malloc_errors > 0) {
3696                         ret = -ENOMEM;
3697                         break;
3698                 }
3699
3700                 dev_replace->cursor_left = dev_replace->cursor_right;
3701                 dev_replace->item_needs_writeback = 1;
3702 skip:
3703                 key.offset = found_key.offset + length;
3704                 btrfs_release_path(path);
3705         }
3706
3707         btrfs_free_path(path);
3708
3709         return ret;
3710 }
3711
3712 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
3713                                            struct btrfs_device *scrub_dev)
3714 {
3715         int     i;
3716         u64     bytenr;
3717         u64     gen;
3718         int     ret;
3719         struct btrfs_root *root = sctx->dev_root;
3720
3721         if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
3722                 return -EIO;
3723
3724         /* Seed devices of a new filesystem has their own generation. */
3725         if (scrub_dev->fs_devices != root->fs_info->fs_devices)
3726                 gen = scrub_dev->generation;
3727         else
3728                 gen = root->fs_info->last_trans_committed;
3729
3730         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
3731                 bytenr = btrfs_sb_offset(i);
3732                 if (bytenr + BTRFS_SUPER_INFO_SIZE >
3733                     scrub_dev->commit_total_bytes)
3734                         break;
3735
3736                 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
3737                                   scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
3738                                   NULL, 1, bytenr);
3739                 if (ret)
3740                         return ret;
3741         }
3742         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3743
3744         return 0;
3745 }
3746
3747 /*
3748  * get a reference count on fs_info->scrub_workers. start worker if necessary
3749  */
3750 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
3751                                                 int is_dev_replace)
3752 {
3753         unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
3754         int max_active = fs_info->thread_pool_size;
3755
3756         if (fs_info->scrub_workers_refcnt == 0) {
3757                 if (is_dev_replace)
3758                         fs_info->scrub_workers =
3759                                 btrfs_alloc_workqueue("btrfs-scrub", flags,
3760                                                       1, 4);
3761                 else
3762                         fs_info->scrub_workers =
3763                                 btrfs_alloc_workqueue("btrfs-scrub", flags,
3764                                                       max_active, 4);
3765                 if (!fs_info->scrub_workers)
3766                         goto fail_scrub_workers;
3767
3768                 fs_info->scrub_wr_completion_workers =
3769                         btrfs_alloc_workqueue("btrfs-scrubwrc", flags,
3770                                               max_active, 2);
3771                 if (!fs_info->scrub_wr_completion_workers)
3772                         goto fail_scrub_wr_completion_workers;
3773
3774                 fs_info->scrub_nocow_workers =
3775                         btrfs_alloc_workqueue("btrfs-scrubnc", flags, 1, 0);
3776                 if (!fs_info->scrub_nocow_workers)
3777                         goto fail_scrub_nocow_workers;
3778                 fs_info->scrub_parity_workers =
3779                         btrfs_alloc_workqueue("btrfs-scrubparity", flags,
3780                                               max_active, 2);
3781                 if (!fs_info->scrub_parity_workers)
3782                         goto fail_scrub_parity_workers;
3783         }
3784         ++fs_info->scrub_workers_refcnt;
3785         return 0;
3786
3787 fail_scrub_parity_workers:
3788         btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
3789 fail_scrub_nocow_workers:
3790         btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
3791 fail_scrub_wr_completion_workers:
3792         btrfs_destroy_workqueue(fs_info->scrub_workers);
3793 fail_scrub_workers:
3794         return -ENOMEM;
3795 }
3796
3797 static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
3798 {
3799         if (--fs_info->scrub_workers_refcnt == 0) {
3800                 btrfs_destroy_workqueue(fs_info->scrub_workers);
3801                 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
3802                 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
3803                 btrfs_destroy_workqueue(fs_info->scrub_parity_workers);
3804         }
3805         WARN_ON(fs_info->scrub_workers_refcnt < 0);
3806 }
3807
3808 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
3809                     u64 end, struct btrfs_scrub_progress *progress,
3810                     int readonly, int is_dev_replace)
3811 {
3812         struct scrub_ctx *sctx;
3813         int ret;
3814         struct btrfs_device *dev;
3815         struct rcu_string *name;
3816
3817         if (btrfs_fs_closing(fs_info))
3818                 return -EINVAL;
3819
3820         if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {
3821                 /*
3822                  * in this case scrub is unable to calculate the checksum
3823                  * the way scrub is implemented. Do not handle this
3824                  * situation at all because it won't ever happen.
3825                  */
3826                 btrfs_err(fs_info,
3827                            "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
3828                        fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);
3829                 return -EINVAL;
3830         }
3831
3832         if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {
3833                 /* not supported for data w/o checksums */
3834                 btrfs_err(fs_info,
3835                            "scrub: size assumption sectorsize != PAGE_SIZE "
3836                            "(%d != %lu) fails",
3837                        fs_info->chunk_root->sectorsize, PAGE_SIZE);
3838                 return -EINVAL;
3839         }
3840
3841         if (fs_info->chunk_root->nodesize >
3842             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
3843             fs_info->chunk_root->sectorsize >
3844             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
3845                 /*
3846                  * would exhaust the array bounds of pagev member in
3847                  * struct scrub_block
3848                  */
3849                 btrfs_err(fs_info, "scrub: size assumption nodesize and sectorsize "
3850                            "<= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
3851                        fs_info->chunk_root->nodesize,
3852                        SCRUB_MAX_PAGES_PER_BLOCK,
3853                        fs_info->chunk_root->sectorsize,
3854                        SCRUB_MAX_PAGES_PER_BLOCK);
3855                 return -EINVAL;
3856         }
3857
3858
3859         mutex_lock(&fs_info->fs_devices->device_list_mutex);
3860         dev = btrfs_find_device(fs_info, devid, NULL, NULL);
3861         if (!dev || (dev->missing && !is_dev_replace)) {
3862                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3863                 return -ENODEV;
3864         }
3865
3866         if (!is_dev_replace && !readonly && !dev->writeable) {
3867                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3868                 rcu_read_lock();
3869                 name = rcu_dereference(dev->name);
3870                 btrfs_err(fs_info, "scrub: device %s is not writable",
3871                           name->str);
3872                 rcu_read_unlock();
3873                 return -EROFS;
3874         }
3875
3876         mutex_lock(&fs_info->scrub_lock);
3877         if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) {
3878                 mutex_unlock(&fs_info->scrub_lock);
3879                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3880                 return -EIO;
3881         }
3882
3883         btrfs_dev_replace_lock(&fs_info->dev_replace);
3884         if (dev->scrub_device ||
3885             (!is_dev_replace &&
3886              btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
3887                 btrfs_dev_replace_unlock(&fs_info->dev_replace);
3888                 mutex_unlock(&fs_info->scrub_lock);
3889                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3890                 return -EINPROGRESS;
3891         }
3892         btrfs_dev_replace_unlock(&fs_info->dev_replace);
3893
3894         ret = scrub_workers_get(fs_info, is_dev_replace);
3895         if (ret) {
3896                 mutex_unlock(&fs_info->scrub_lock);
3897                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3898                 return ret;
3899         }
3900
3901         sctx = scrub_setup_ctx(dev, is_dev_replace);
3902         if (IS_ERR(sctx)) {
3903                 mutex_unlock(&fs_info->scrub_lock);
3904                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3905                 scrub_workers_put(fs_info);
3906                 return PTR_ERR(sctx);
3907         }
3908         sctx->readonly = readonly;
3909         dev->scrub_device = sctx;
3910         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3911
3912         /*
3913          * checking @scrub_pause_req here, we can avoid
3914          * race between committing transaction and scrubbing.
3915          */
3916         __scrub_blocked_if_needed(fs_info);
3917         atomic_inc(&fs_info->scrubs_running);
3918         mutex_unlock(&fs_info->scrub_lock);
3919
3920         if (!is_dev_replace) {
3921                 /*
3922                  * by holding device list mutex, we can
3923                  * kick off writing super in log tree sync.
3924                  */
3925                 mutex_lock(&fs_info->fs_devices->device_list_mutex);
3926                 ret = scrub_supers(sctx, dev);
3927                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3928         }
3929
3930         if (!ret)
3931                 ret = scrub_enumerate_chunks(sctx, dev, start, end,
3932                                              is_dev_replace);
3933
3934         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3935         atomic_dec(&fs_info->scrubs_running);
3936         wake_up(&fs_info->scrub_pause_wait);
3937
3938         wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
3939
3940         if (progress)
3941                 memcpy(progress, &sctx->stat, sizeof(*progress));
3942
3943         mutex_lock(&fs_info->scrub_lock);
3944         dev->scrub_device = NULL;
3945         scrub_workers_put(fs_info);
3946         mutex_unlock(&fs_info->scrub_lock);
3947
3948         scrub_put_ctx(sctx);
3949
3950         return ret;
3951 }
3952
3953 void btrfs_scrub_pause(struct btrfs_root *root)
3954 {
3955         struct btrfs_fs_info *fs_info = root->fs_info;
3956
3957         mutex_lock(&fs_info->scrub_lock);
3958         atomic_inc(&fs_info->scrub_pause_req);
3959         while (atomic_read(&fs_info->scrubs_paused) !=
3960                atomic_read(&fs_info->scrubs_running)) {
3961                 mutex_unlock(&fs_info->scrub_lock);
3962                 wait_event(fs_info->scrub_pause_wait,
3963                            atomic_read(&fs_info->scrubs_paused) ==
3964                            atomic_read(&fs_info->scrubs_running));
3965                 mutex_lock(&fs_info->scrub_lock);
3966         }
3967         mutex_unlock(&fs_info->scrub_lock);
3968 }
3969
3970 void btrfs_scrub_continue(struct btrfs_root *root)
3971 {
3972         struct btrfs_fs_info *fs_info = root->fs_info;
3973
3974         atomic_dec(&fs_info->scrub_pause_req);
3975         wake_up(&fs_info->scrub_pause_wait);
3976 }
3977
3978 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
3979 {
3980         mutex_lock(&fs_info->scrub_lock);
3981         if (!atomic_read(&fs_info->scrubs_running)) {
3982                 mutex_unlock(&fs_info->scrub_lock);
3983                 return -ENOTCONN;
3984         }
3985
3986         atomic_inc(&fs_info->scrub_cancel_req);
3987         while (atomic_read(&fs_info->scrubs_running)) {
3988                 mutex_unlock(&fs_info->scrub_lock);
3989                 wait_event(fs_info->scrub_pause_wait,
3990                            atomic_read(&fs_info->scrubs_running) == 0);
3991                 mutex_lock(&fs_info->scrub_lock);
3992         }
3993         atomic_dec(&fs_info->scrub_cancel_req);
3994         mutex_unlock(&fs_info->scrub_lock);
3995
3996         return 0;
3997 }
3998
3999 int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
4000                            struct btrfs_device *dev)
4001 {
4002         struct scrub_ctx *sctx;
4003
4004         mutex_lock(&fs_info->scrub_lock);
4005         sctx = dev->scrub_device;
4006         if (!sctx) {
4007                 mutex_unlock(&fs_info->scrub_lock);
4008                 return -ENOTCONN;
4009         }
4010         atomic_inc(&sctx->cancel_req);
4011         while (dev->scrub_device) {
4012                 mutex_unlock(&fs_info->scrub_lock);
4013                 wait_event(fs_info->scrub_pause_wait,
4014                            dev->scrub_device == NULL);
4015                 mutex_lock(&fs_info->scrub_lock);
4016         }
4017         mutex_unlock(&fs_info->scrub_lock);
4018
4019         return 0;
4020 }
4021
4022 int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
4023                          struct btrfs_scrub_progress *progress)
4024 {
4025         struct btrfs_device *dev;
4026         struct scrub_ctx *sctx = NULL;
4027
4028         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
4029         dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);
4030         if (dev)
4031                 sctx = dev->scrub_device;
4032         if (sctx)
4033                 memcpy(progress, &sctx->stat, sizeof(*progress));
4034         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
4035
4036         return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
4037 }
4038
4039 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
4040                                u64 extent_logical, u64 extent_len,
4041                                u64 *extent_physical,
4042                                struct btrfs_device **extent_dev,
4043                                int *extent_mirror_num)
4044 {
4045         u64 mapped_length;
4046         struct btrfs_bio *bbio = NULL;
4047         int ret;
4048
4049         mapped_length = extent_len;
4050         ret = btrfs_map_block(fs_info, READ, extent_logical,
4051                               &mapped_length, &bbio, 0);
4052         if (ret || !bbio || mapped_length < extent_len ||
4053             !bbio->stripes[0].dev->bdev) {
4054                 btrfs_put_bbio(bbio);
4055                 return;
4056         }
4057
4058         *extent_physical = bbio->stripes[0].physical;
4059         *extent_mirror_num = bbio->mirror_num;
4060         *extent_dev = bbio->stripes[0].dev;
4061         btrfs_put_bbio(bbio);
4062 }
4063
4064 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
4065                               struct scrub_wr_ctx *wr_ctx,
4066                               struct btrfs_fs_info *fs_info,
4067                               struct btrfs_device *dev,
4068                               int is_dev_replace)
4069 {
4070         WARN_ON(wr_ctx->wr_curr_bio != NULL);
4071
4072         mutex_init(&wr_ctx->wr_lock);
4073         wr_ctx->wr_curr_bio = NULL;
4074         if (!is_dev_replace)
4075                 return 0;
4076
4077         WARN_ON(!dev->bdev);
4078         wr_ctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO;
4079         wr_ctx->tgtdev = dev;
4080         atomic_set(&wr_ctx->flush_all_writes, 0);
4081         return 0;
4082 }
4083
4084 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx)
4085 {
4086         mutex_lock(&wr_ctx->wr_lock);
4087         kfree(wr_ctx->wr_curr_bio);
4088         wr_ctx->wr_curr_bio = NULL;
4089         mutex_unlock(&wr_ctx->wr_lock);
4090 }
4091
4092 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
4093                             int mirror_num, u64 physical_for_dev_replace)
4094 {
4095         struct scrub_copy_nocow_ctx *nocow_ctx;
4096         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
4097
4098         nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
4099         if (!nocow_ctx) {
4100                 spin_lock(&sctx->stat_lock);
4101                 sctx->stat.malloc_errors++;
4102                 spin_unlock(&sctx->stat_lock);
4103                 return -ENOMEM;
4104         }
4105
4106         scrub_pending_trans_workers_inc(sctx);
4107
4108         nocow_ctx->sctx = sctx;
4109         nocow_ctx->logical = logical;
4110         nocow_ctx->len = len;
4111         nocow_ctx->mirror_num = mirror_num;
4112         nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
4113         btrfs_init_work(&nocow_ctx->work, btrfs_scrubnc_helper,
4114                         copy_nocow_pages_worker, NULL, NULL);
4115         INIT_LIST_HEAD(&nocow_ctx->inodes);
4116         btrfs_queue_work(fs_info->scrub_nocow_workers,
4117                          &nocow_ctx->work);
4118
4119         return 0;
4120 }
4121
4122 static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx)
4123 {
4124         struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
4125         struct scrub_nocow_inode *nocow_inode;
4126
4127         nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS);
4128         if (!nocow_inode)
4129                 return -ENOMEM;
4130         nocow_inode->inum = inum;
4131         nocow_inode->offset = offset;
4132         nocow_inode->root = root;
4133         list_add_tail(&nocow_inode->list, &nocow_ctx->inodes);
4134         return 0;
4135 }
4136
4137 #define COPY_COMPLETE 1
4138
4139 static void copy_nocow_pages_worker(struct btrfs_work *work)
4140 {
4141         struct scrub_copy_nocow_ctx *nocow_ctx =
4142                 container_of(work, struct scrub_copy_nocow_ctx, work);
4143         struct scrub_ctx *sctx = nocow_ctx->sctx;
4144         u64 logical = nocow_ctx->logical;
4145         u64 len = nocow_ctx->len;
4146         int mirror_num = nocow_ctx->mirror_num;
4147         u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
4148         int ret;
4149         struct btrfs_trans_handle *trans = NULL;
4150         struct btrfs_fs_info *fs_info;
4151         struct btrfs_path *path;
4152         struct btrfs_root *root;
4153         int not_written = 0;
4154
4155         fs_info = sctx->dev_root->fs_info;
4156         root = fs_info->extent_root;
4157
4158         path = btrfs_alloc_path();
4159         if (!path) {
4160                 spin_lock(&sctx->stat_lock);
4161                 sctx->stat.malloc_errors++;
4162                 spin_unlock(&sctx->stat_lock);
4163                 not_written = 1;
4164                 goto out;
4165         }
4166
4167         trans = btrfs_join_transaction(root);
4168         if (IS_ERR(trans)) {
4169                 not_written = 1;
4170                 goto out;
4171         }
4172
4173         ret = iterate_inodes_from_logical(logical, fs_info, path,
4174                                           record_inode_for_nocow, nocow_ctx);
4175         if (ret != 0 && ret != -ENOENT) {
4176                 btrfs_warn(fs_info, "iterate_inodes_from_logical() failed: log %llu, "
4177                         "phys %llu, len %llu, mir %u, ret %d",
4178                         logical, physical_for_dev_replace, len, mirror_num,
4179                         ret);
4180                 not_written = 1;
4181                 goto out;
4182         }
4183
4184         btrfs_end_transaction(trans, root);
4185         trans = NULL;
4186         while (!list_empty(&nocow_ctx->inodes)) {
4187                 struct scrub_nocow_inode *entry;
4188                 entry = list_first_entry(&nocow_ctx->inodes,
4189                                          struct scrub_nocow_inode,
4190                                          list);
4191                 list_del_init(&entry->list);
4192                 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset,
4193                                                  entry->root, nocow_ctx);
4194                 kfree(entry);
4195                 if (ret == COPY_COMPLETE) {
4196                         ret = 0;
4197                         break;
4198                 } else if (ret) {
4199                         break;
4200                 }
4201         }
4202 out:
4203         while (!list_empty(&nocow_ctx->inodes)) {
4204                 struct scrub_nocow_inode *entry;
4205                 entry = list_first_entry(&nocow_ctx->inodes,
4206                                          struct scrub_nocow_inode,
4207                                          list);
4208                 list_del_init(&entry->list);
4209                 kfree(entry);
4210         }
4211         if (trans && !IS_ERR(trans))
4212                 btrfs_end_transaction(trans, root);
4213         if (not_written)
4214                 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
4215                                             num_uncorrectable_read_errors);
4216
4217         btrfs_free_path(path);
4218         kfree(nocow_ctx);
4219
4220         scrub_pending_trans_workers_dec(sctx);
4221 }
4222
4223 static int check_extent_to_block(struct inode *inode, u64 start, u64 len,
4224                                  u64 logical)
4225 {
4226         struct extent_state *cached_state = NULL;
4227         struct btrfs_ordered_extent *ordered;
4228         struct extent_io_tree *io_tree;
4229         struct extent_map *em;
4230         u64 lockstart = start, lockend = start + len - 1;
4231         int ret = 0;
4232
4233         io_tree = &BTRFS_I(inode)->io_tree;
4234
4235         lock_extent_bits(io_tree, lockstart, lockend, 0, &cached_state);
4236         ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
4237         if (ordered) {
4238                 btrfs_put_ordered_extent(ordered);
4239                 ret = 1;
4240                 goto out_unlock;
4241         }
4242
4243         em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
4244         if (IS_ERR(em)) {
4245                 ret = PTR_ERR(em);
4246                 goto out_unlock;
4247         }
4248
4249         /*
4250          * This extent does not actually cover the logical extent anymore,
4251          * move on to the next inode.
4252          */
4253         if (em->block_start > logical ||
4254             em->block_start + em->block_len < logical + len) {
4255                 free_extent_map(em);
4256                 ret = 1;
4257                 goto out_unlock;
4258         }
4259         free_extent_map(em);
4260
4261 out_unlock:
4262         unlock_extent_cached(io_tree, lockstart, lockend, &cached_state,
4263                              GFP_NOFS);
4264         return ret;
4265 }
4266
4267 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
4268                                       struct scrub_copy_nocow_ctx *nocow_ctx)
4269 {
4270         struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;
4271         struct btrfs_key key;
4272         struct inode *inode;
4273         struct page *page;
4274         struct btrfs_root *local_root;
4275         struct extent_io_tree *io_tree;
4276         u64 physical_for_dev_replace;
4277         u64 nocow_ctx_logical;
4278         u64 len = nocow_ctx->len;
4279         unsigned long index;
4280         int srcu_index;
4281         int ret = 0;
4282         int err = 0;
4283
4284         key.objectid = root;
4285         key.type = BTRFS_ROOT_ITEM_KEY;
4286         key.offset = (u64)-1;
4287
4288         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
4289
4290         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
4291         if (IS_ERR(local_root)) {
4292                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
4293                 return PTR_ERR(local_root);
4294         }
4295
4296         key.type = BTRFS_INODE_ITEM_KEY;
4297         key.objectid = inum;
4298         key.offset = 0;
4299         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
4300         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
4301         if (IS_ERR(inode))
4302                 return PTR_ERR(inode);
4303
4304         /* Avoid truncate/dio/punch hole.. */
4305         mutex_lock(&inode->i_mutex);
4306         inode_dio_wait(inode);
4307
4308         physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
4309         io_tree = &BTRFS_I(inode)->io_tree;
4310         nocow_ctx_logical = nocow_ctx->logical;
4311
4312         ret = check_extent_to_block(inode, offset, len, nocow_ctx_logical);
4313         if (ret) {
4314                 ret = ret > 0 ? 0 : ret;
4315                 goto out;
4316         }
4317
4318         while (len >= PAGE_CACHE_SIZE) {
4319                 index = offset >> PAGE_CACHE_SHIFT;
4320 again:
4321                 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
4322                 if (!page) {
4323                         btrfs_err(fs_info, "find_or_create_page() failed");
4324                         ret = -ENOMEM;
4325                         goto out;
4326                 }
4327
4328                 if (PageUptodate(page)) {
4329                         if (PageDirty(page))
4330                                 goto next_page;
4331                 } else {
4332                         ClearPageError(page);
4333                         err = extent_read_full_page(io_tree, page,
4334                                                            btrfs_get_extent,
4335                                                            nocow_ctx->mirror_num);
4336                         if (err) {
4337                                 ret = err;
4338                                 goto next_page;
4339                         }
4340
4341                         lock_page(page);
4342                         /*
4343                          * If the page has been remove from the page cache,
4344                          * the data on it is meaningless, because it may be
4345                          * old one, the new data may be written into the new
4346                          * page in the page cache.
4347                          */
4348                         if (page->mapping != inode->i_mapping) {
4349                                 unlock_page(page);
4350                                 page_cache_release(page);
4351                                 goto again;
4352                         }
4353                         if (!PageUptodate(page)) {
4354                                 ret = -EIO;
4355                                 goto next_page;
4356                         }
4357                 }
4358
4359                 ret = check_extent_to_block(inode, offset, len,
4360                                             nocow_ctx_logical);
4361                 if (ret) {
4362                         ret = ret > 0 ? 0 : ret;
4363                         goto next_page;
4364                 }
4365
4366                 err = write_page_nocow(nocow_ctx->sctx,
4367                                        physical_for_dev_replace, page);
4368                 if (err)
4369                         ret = err;
4370 next_page:
4371                 unlock_page(page);
4372                 page_cache_release(page);
4373
4374                 if (ret)
4375                         break;
4376
4377                 offset += PAGE_CACHE_SIZE;
4378                 physical_for_dev_replace += PAGE_CACHE_SIZE;
4379                 nocow_ctx_logical += PAGE_CACHE_SIZE;
4380                 len -= PAGE_CACHE_SIZE;
4381         }
4382         ret = COPY_COMPLETE;
4383 out:
4384         mutex_unlock(&inode->i_mutex);
4385         iput(inode);
4386         return ret;
4387 }
4388
4389 static int write_page_nocow(struct scrub_ctx *sctx,
4390                             u64 physical_for_dev_replace, struct page *page)
4391 {
4392         struct bio *bio;
4393         struct btrfs_device *dev;
4394         int ret;
4395
4396         dev = sctx->wr_ctx.tgtdev;
4397         if (!dev)
4398                 return -EIO;
4399         if (!dev->bdev) {
4400                 btrfs_warn_rl(dev->dev_root->fs_info,
4401                         "scrub write_page_nocow(bdev == NULL) is unexpected");
4402                 return -EIO;
4403         }
4404         bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
4405         if (!bio) {
4406                 spin_lock(&sctx->stat_lock);
4407                 sctx->stat.malloc_errors++;
4408                 spin_unlock(&sctx->stat_lock);
4409                 return -ENOMEM;
4410         }
4411         bio->bi_iter.bi_size = 0;
4412         bio->bi_iter.bi_sector = physical_for_dev_replace >> 9;
4413         bio->bi_bdev = dev->bdev;
4414         ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
4415         if (ret != PAGE_CACHE_SIZE) {
4416 leave_with_eio:
4417                 bio_put(bio);
4418                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
4419                 return -EIO;
4420         }
4421
4422         if (btrfsic_submit_bio_wait(WRITE_SYNC, bio))
4423                 goto leave_with_eio;
4424
4425         bio_put(bio);
4426         return 0;
4427 }