projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
btrfs: update SCRUB_MAX_PAGES_PER_BLOCK
[linux-block.git] / fs / btrfs / scrub.c
CommitLineData
c1d7c514 1// SPDX-License-Identifier: GPL-2.0
a2de733c 2/*
b6bfebc1 3 * Copyright (C) 2011, 2012 STRATO. All rights reserved.
a2de733c
AJ		 4 */
5
a2de733c 6#include <linux/blkdev.h>
558540c1 7#include <linux/ratelimit.h>
de2491fd 8#include <linux/sched/mm.h>
d5178578 9#include <crypto/hash.h>
a2de733c 10#include "ctree.h"
6e80d4f8 11#include "discard.h"
a2de733c
AJ		 12#include "volumes.h"
13#include "disk-io.h"
14#include "ordered-data.h"
0ef8e451 15#include "transaction.h"
558540c1 16#include "backref.h"
5da6fcbc 17#include "extent_io.h"
ff023aac 18#include "dev-replace.h"
21adbd5c 19#include "check-integrity.h"
606686ee 20#include "rcu-string.h"
53b381b3 21#include "raid56.h"
aac0023c 22#include "block-group.h"
12659251 23#include "zoned.h"
a2de733c
AJ		 24
25/*
26 * This is only the first step towards a full-features scrub. It reads all
27 * extent and super block and verifies the checksums. In case a bad checksum
28 * is found or the extent cannot be read, good data will be written back if
29 * any can be found.
30 *
31 * Future enhancements:
a2de733c
AJ		 32 * - In case an unrepairable extent is encountered, track which files are
33 * affected and report them
a2de733c 34 * - track and record media errors, throw out bad devices
a2de733c 35 * - add a mode to also read unallocated space
a2de733c
AJ		 36 */
37
b5d67f64 38struct scrub_block;
d9d181c1 39struct scrub_ctx;
a2de733c 40
ff023aac
SB		 41/*
42 * the following three values only influence the performance.
43 * The last one configures the number of parallel and outstanding I/O
44 * operations. The first two values configure an upper limit for the number
45 * of (dynamically allocated) pages that are added to a bio.
46 */
47#define SCRUB_PAGES_PER_RD_BIO 32 /* 128k per bio */
48#define SCRUB_PAGES_PER_WR_BIO 32 /* 128k per bio */
49#define SCRUB_BIOS_PER_SCTX 64 /* 8MB per device in flight */
7a9e9987
SB		 50
51/*
0bb3acdc 52 * The following value times PAGE_SIZE needs to be large enough to match the
7a9e9987 53 * largest node/leaf/sector size that shall be supported.
7a9e9987 54 */
0bb3acdc 55#define SCRUB_MAX_PAGES_PER_BLOCK (BTRFS_MAX_METADATA_BLOCKSIZE / SZ_4K)
a2de733c 56
af8e2d1d 57struct scrub_recover {
6f615018 58 refcount_t refs;
4c664611 59 struct btrfs_io_context *bioc;
af8e2d1d
MX		 60 u64 map_length;
61};
62
a2de733c 63struct scrub_page {
b5d67f64
SB		 64 struct scrub_block *sblock;
65 struct page *page;
442a4f63 66 struct btrfs_device *dev;
5a6ac9ea 67 struct list_head list;
a2de733c
AJ		 68 u64 flags; /* extent flags */
69 u64 generation;
b5d67f64
SB		 70 u64 logical;
71 u64 physical;
ff023aac 72 u64 physical_for_dev_replace;
57019345 73 atomic_t refs;
2c363954 74 u8 mirror_num;
d08e38b6
CIK		 75 unsigned int have_csum:1;
76 unsigned int io_error:1;
a2de733c 77 u8 csum[BTRFS_CSUM_SIZE];
af8e2d1d
MX		 78
79 struct scrub_recover *recover;
a2de733c
AJ		 80};
81
82struct scrub_bio {
83 int index;
d9d181c1 84 struct scrub_ctx *sctx;
a36cf8b8 85 struct btrfs_device *dev;
a2de733c 86 struct bio *bio;
4e4cbee9 87 blk_status_t status;
a2de733c
AJ		 88 u64 logical;
89 u64 physical;
ff023aac
SB		 90#if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
91 struct scrub_page *pagev[SCRUB_PAGES_PER_WR_BIO];
92#else
93 struct scrub_page *pagev[SCRUB_PAGES_PER_RD_BIO];
94#endif
b5d67f64 95 int page_count;
a2de733c
AJ		 96 int next_free;
97 struct btrfs_work work;
98};
99
b5d67f64 100struct scrub_block {
7a9e9987 101 struct scrub_page *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
b5d67f64
SB		 102 int page_count;
103 atomic_t outstanding_pages;
186debd6 104 refcount_t refs; /* free mem on transition to zero */
d9d181c1 105 struct scrub_ctx *sctx;
5a6ac9ea 106 struct scrub_parity *sparity;
b5d67f64
SB		 107 struct {
108 unsigned int header_error:1;
109 unsigned int checksum_error:1;
110 unsigned int no_io_error_seen:1;
442a4f63 111 unsigned int generation_error:1; /* also sets header_error */
5a6ac9ea
MX		 112
113 /* The following is for the data used to check parity */
114 /* It is for the data with checksum */
115 unsigned int data_corrected:1;
b5d67f64 116 };
73ff61db 117 struct btrfs_work work;
b5d67f64
SB		 118};
119
5a6ac9ea
MX		 120/* Used for the chunks with parity stripe such RAID5/6 */
121struct scrub_parity {
122 struct scrub_ctx *sctx;
123
124 struct btrfs_device *scrub_dev;
125
126 u64 logic_start;
127
128 u64 logic_end;
129
130 int nsectors;
131
fa485d21 132 u32 stripe_len;
5a6ac9ea 133
78a76450 134 refcount_t refs;
5a6ac9ea
MX		 135
136 struct list_head spages;
137
138 /* Work of parity check and repair */
139 struct btrfs_work work;
140
141 /* Mark the parity blocks which have data */
142 unsigned long *dbitmap;
143
144 /*
145 * Mark the parity blocks which have data, but errors happen when
146 * read data or check data
147 */
148 unsigned long *ebitmap;
149
a8753ee3 150 unsigned long bitmap[];
5a6ac9ea
MX		 151};
152
d9d181c1 153struct scrub_ctx {
ff023aac 154 struct scrub_bio *bios[SCRUB_BIOS_PER_SCTX];
fb456252 155 struct btrfs_fs_info *fs_info;
a2de733c
AJ		 156 int first_free;
157 int curr;
b6bfebc1
SB		 158 atomic_t bios_in_flight;
159 atomic_t workers_pending;
a2de733c
AJ		 160 spinlock_t list_lock;
161 wait_queue_head_t list_wait;
a2de733c
AJ		 162 struct list_head csum_list;
163 atomic_t cancel_req;
8628764e 164 int readonly;
ff023aac 165 int pages_per_rd_bio;
63a212ab 166
eb3b5053
DS		 167 /* State of IO submission throttling affecting the associated device */
168 ktime_t throttle_deadline;
169 u64 throttle_sent;
170
63a212ab 171 int is_dev_replace;
de17addc 172 u64 write_pointer;
3fb99303
DS		 173
174 struct scrub_bio *wr_curr_bio;
175 struct mutex wr_lock;
176 int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
3fb99303 177 struct btrfs_device *wr_tgtdev;
2073c4c2 178 bool flush_all_writes;
63a212ab 179
a2de733c
AJ		 180 /*
181 * statistics
182 */
183 struct btrfs_scrub_progress stat;
184 spinlock_t stat_lock;
f55985f4
FM		 185
186 /*
187 * Use a ref counter to avoid use-after-free issues. Scrub workers
188 * decrement bios_in_flight and workers_pending and then do a wakeup
189 * on the list_wait wait queue. We must ensure the main scrub task
190 * doesn't free the scrub context before or while the workers are
191 * doing the wakeup() call.
192 */
99f4cdb1 193 refcount_t refs;
a2de733c
AJ		 194};
195
558540c1
JS		 196struct scrub_warning {
197 struct btrfs_path *path;
198 u64 extent_item_size;
558540c1 199 const char *errstr;
6aa21263 200 u64 physical;
558540c1
JS		 201 u64 logical;
202 struct btrfs_device *dev;
558540c1
JS		 203};
204
0966a7b1
QW		 205struct full_stripe_lock {
206 struct rb_node node;
207 u64 logical;
208 u64 refs;
209 struct mutex mutex;
210};
211
be50a8dd 212static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
ff023aac 213 struct scrub_block *sblocks_for_recheck);
34f5c8e9 214static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
affe4a5a
ZL		 215 struct scrub_block *sblock,
216 int retry_failed_mirror);
ba7cf988 217static void scrub_recheck_block_checksum(struct scrub_block *sblock);
b5d67f64 218static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
114ab50d 219 struct scrub_block *sblock_good);
b5d67f64
SB		 220static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
221 struct scrub_block *sblock_good,
222 int page_num, int force_write);
ff023aac
SB		 223static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
224static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
225 int page_num);
b5d67f64
SB		 226static int scrub_checksum_data(struct scrub_block *sblock);
227static int scrub_checksum_tree_block(struct scrub_block *sblock);
228static int scrub_checksum_super(struct scrub_block *sblock);
b5d67f64 229static void scrub_block_put(struct scrub_block *sblock);
7a9e9987
SB		 230static void scrub_page_get(struct scrub_page *spage);
231static void scrub_page_put(struct scrub_page *spage);
5a6ac9ea
MX		 232static void scrub_parity_get(struct scrub_parity *sparity);
233static void scrub_parity_put(struct scrub_parity *sparity);
fa485d21 234static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u32 len,
a36cf8b8 235 u64 physical, struct btrfs_device *dev, u64 flags,
96e63a45 236 u64 gen, int mirror_num, u8 *csum,
ff023aac 237 u64 physical_for_dev_replace);
4246a0b6 238static void scrub_bio_end_io(struct bio *bio);
b5d67f64
SB		 239static void scrub_bio_end_io_worker(struct btrfs_work *work);
240static void scrub_block_complete(struct scrub_block *sblock);
ff023aac 241static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
fa485d21 242 u64 extent_logical, u32 extent_len,
ff023aac
SB		 243 u64 *extent_physical,
244 struct btrfs_device **extent_dev,
245 int *extent_mirror_num);
ff023aac
SB		 246static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
247 struct scrub_page *spage);
248static void scrub_wr_submit(struct scrub_ctx *sctx);
4246a0b6 249static void scrub_wr_bio_end_io(struct bio *bio);
ff023aac 250static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
f55985f4 251static void scrub_put_ctx(struct scrub_ctx *sctx);
1623edeb 252
261d2dcb 253static inline int scrub_is_page_on_raid56(struct scrub_page *spage)
762221f0 254{
261d2dcb 255 return spage->recover &&
4c664611 256 (spage->recover->bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
762221f0 257}
1623edeb 258
b6bfebc1
SB		 259static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
260{
99f4cdb1 261 refcount_inc(&sctx->refs);
b6bfebc1
SB		 262 atomic_inc(&sctx->bios_in_flight);
263}
264
265static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
266{
267 atomic_dec(&sctx->bios_in_flight);
268 wake_up(&sctx->list_wait);
f55985f4 269 scrub_put_ctx(sctx);
b6bfebc1
SB		 270}
271
cb7ab021 272static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
3cb0929a
WS		 273{
274 while (atomic_read(&fs_info->scrub_pause_req)) {
275 mutex_unlock(&fs_info->scrub_lock);
276 wait_event(fs_info->scrub_pause_wait,
277 atomic_read(&fs_info->scrub_pause_req) == 0);
278 mutex_lock(&fs_info->scrub_lock);
279 }
280}
281
0e22be89 282static void scrub_pause_on(struct btrfs_fs_info *fs_info)
cb7ab021
WS		 283{
284 atomic_inc(&fs_info->scrubs_paused);
285 wake_up(&fs_info->scrub_pause_wait);
0e22be89 286}
cb7ab021 287
0e22be89
Z		 288static void scrub_pause_off(struct btrfs_fs_info *fs_info)
289{
cb7ab021
WS		 290 mutex_lock(&fs_info->scrub_lock);
291 __scrub_blocked_if_needed(fs_info);
292 atomic_dec(&fs_info->scrubs_paused);
293 mutex_unlock(&fs_info->scrub_lock);
294
295 wake_up(&fs_info->scrub_pause_wait);
296}
297
0e22be89
Z		 298static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
299{
300 scrub_pause_on(fs_info);
301 scrub_pause_off(fs_info);
302}
303
0966a7b1
QW		 304/*
305 * Insert new full stripe lock into full stripe locks tree
306 *
307 * Return pointer to existing or newly inserted full_stripe_lock structure if
308 * everything works well.
309 * Return ERR_PTR(-ENOMEM) if we failed to allocate memory
310 *
311 * NOTE: caller must hold full_stripe_locks_root->lock before calling this
312 * function
313 */
314static struct full_stripe_lock *insert_full_stripe_lock(
315 struct btrfs_full_stripe_locks_tree *locks_root,
316 u64 fstripe_logical)
317{
318 struct rb_node **p;
319 struct rb_node *parent = NULL;
320 struct full_stripe_lock *entry;
321 struct full_stripe_lock *ret;
322
a32bf9a3 323 lockdep_assert_held(&locks_root->lock);
0966a7b1
QW		 324
325 p = &locks_root->root.rb_node;
326 while (*p) {
327 parent = *p;
328 entry = rb_entry(parent, struct full_stripe_lock, node);
329 if (fstripe_logical < entry->logical) {
330 p = &(*p)->rb_left;
331 } else if (fstripe_logical > entry->logical) {
332 p = &(*p)->rb_right;
333 } else {
334 entry->refs++;
335 return entry;
336 }
337 }
338
a5fb1142
FM		 339 /*
340 * Insert new lock.
a5fb1142 341 */
0966a7b1
QW		 342 ret = kmalloc(sizeof(*ret), GFP_KERNEL);
343 if (!ret)
344 return ERR_PTR(-ENOMEM);
345 ret->logical = fstripe_logical;
346 ret->refs = 1;
347 mutex_init(&ret->mutex);
348
349 rb_link_node(&ret->node, parent, p);
350 rb_insert_color(&ret->node, &locks_root->root);
351 return ret;
352}
353
354/*
355 * Search for a full stripe lock of a block group
356 *
357 * Return pointer to existing full stripe lock if found
358 * Return NULL if not found
359 */
360static struct full_stripe_lock *search_full_stripe_lock(
361 struct btrfs_full_stripe_locks_tree *locks_root,
362 u64 fstripe_logical)
363{
364 struct rb_node *node;
365 struct full_stripe_lock *entry;
366
a32bf9a3 367 lockdep_assert_held(&locks_root->lock);
0966a7b1
QW		 368
369 node = locks_root->root.rb_node;
370 while (node) {
371 entry = rb_entry(node, struct full_stripe_lock, node);
372 if (fstripe_logical < entry->logical)
373 node = node->rb_left;
374 else if (fstripe_logical > entry->logical)
375 node = node->rb_right;
376 else
377 return entry;
378 }
379 return NULL;
380}
381
382/*
383 * Helper to get full stripe logical from a normal bytenr.
384 *
385 * Caller must ensure @cache is a RAID56 block group.
386 */
32da5386 387static u64 get_full_stripe_logical(struct btrfs_block_group *cache, u64 bytenr)
0966a7b1
QW		 388{
389 u64 ret;
390
391 /*
392 * Due to chunk item size limit, full stripe length should not be
393 * larger than U32_MAX. Just a sanity check here.
394 */
395 WARN_ON_ONCE(cache->full_stripe_len >= U32_MAX);
396
397 /*
398 * round_down() can only handle power of 2, while RAID56 full
399 * stripe length can be 64KiB * n, so we need to manually round down.
400 */
b3470b5d
DS		 401 ret = div64_u64(bytenr - cache->start, cache->full_stripe_len) *
402 cache->full_stripe_len + cache->start;
0966a7b1
QW		 403 return ret;
404}
405
406/*
407 * Lock a full stripe to avoid concurrency of recovery and read
408 *
409 * It's only used for profiles with parities (RAID5/6), for other profiles it
410 * does nothing.
411 *
412 * Return 0 if we locked full stripe covering @bytenr, with a mutex held.
413 * So caller must call unlock_full_stripe() at the same context.
414 *
415 * Return <0 if encounters error.
416 */
417static int lock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
418 bool *locked_ret)
419{
32da5386 420 struct btrfs_block_group *bg_cache;
0966a7b1
QW		 421 struct btrfs_full_stripe_locks_tree *locks_root;
422 struct full_stripe_lock *existing;
423 u64 fstripe_start;
424 int ret = 0;
425
426 *locked_ret = false;
427 bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
428 if (!bg_cache) {
429 ASSERT(0);
430 return -ENOENT;
431 }
432
433 /* Profiles not based on parity don't need full stripe lock */
434 if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
435 goto out;
436 locks_root = &bg_cache->full_stripe_locks_root;
437
438 fstripe_start = get_full_stripe_logical(bg_cache, bytenr);
439
440 /* Now insert the full stripe lock */
441 mutex_lock(&locks_root->lock);
442 existing = insert_full_stripe_lock(locks_root, fstripe_start);
443 mutex_unlock(&locks_root->lock);
444 if (IS_ERR(existing)) {
445 ret = PTR_ERR(existing);
446 goto out;
447 }
448 mutex_lock(&existing->mutex);
449 *locked_ret = true;
450out:
451 btrfs_put_block_group(bg_cache);
452 return ret;
453}
454
455/*
456 * Unlock a full stripe.
457 *
458 * NOTE: Caller must ensure it's the same context calling corresponding
459 * lock_full_stripe().
460 *
461 * Return 0 if we unlock full stripe without problem.
462 * Return <0 for error
463 */
464static int unlock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
465 bool locked)
466{
32da5386 467 struct btrfs_block_group *bg_cache;
0966a7b1
QW		 468 struct btrfs_full_stripe_locks_tree *locks_root;
469 struct full_stripe_lock *fstripe_lock;
470 u64 fstripe_start;
471 bool freeit = false;
472 int ret = 0;
473
474 /* If we didn't acquire full stripe lock, no need to continue */
475 if (!locked)
476 return 0;
477
478 bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
479 if (!bg_cache) {
480 ASSERT(0);
481 return -ENOENT;
482 }
483 if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
484 goto out;
485
486 locks_root = &bg_cache->full_stripe_locks_root;
487 fstripe_start = get_full_stripe_logical(bg_cache, bytenr);
488
489 mutex_lock(&locks_root->lock);
490 fstripe_lock = search_full_stripe_lock(locks_root, fstripe_start);
491 /* Unpaired unlock_full_stripe() detected */
492 if (!fstripe_lock) {
493 WARN_ON(1);
494 ret = -ENOENT;
495 mutex_unlock(&locks_root->lock);
496 goto out;
497 }
498
499 if (fstripe_lock->refs == 0) {
500 WARN_ON(1);
501 btrfs_warn(fs_info, "full stripe lock at %llu refcount underflow",
502 fstripe_lock->logical);
503 } else {
504 fstripe_lock->refs--;
505 }
506
507 if (fstripe_lock->refs == 0) {
508 rb_erase(&fstripe_lock->node, &locks_root->root);
509 freeit = true;
510 }
511 mutex_unlock(&locks_root->lock);
512
513 mutex_unlock(&fstripe_lock->mutex);
514 if (freeit)
515 kfree(fstripe_lock);
516out:
517 btrfs_put_block_group(bg_cache);
518 return ret;
519}
520
d9d181c1 521static void scrub_free_csums(struct scrub_ctx *sctx)
a2de733c 522{
d9d181c1 523 while (!list_empty(&sctx->csum_list)) {
a2de733c 524 struct btrfs_ordered_sum *sum;
d9d181c1 525 sum = list_first_entry(&sctx->csum_list,
a2de733c
AJ		 526 struct btrfs_ordered_sum, list);
527 list_del(&sum->list);
528 kfree(sum);
529 }
530}
531
d9d181c1 532static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
a2de733c
AJ		 533{
534 int i;
a2de733c 535
d9d181c1 536 if (!sctx)
a2de733c
AJ		 537 return;
538
b5d67f64 539 /* this can happen when scrub is cancelled */
d9d181c1
SB		 540 if (sctx->curr != -1) {
541 struct scrub_bio *sbio = sctx->bios[sctx->curr];
b5d67f64
SB		 542
543 for (i = 0; i < sbio->page_count; i++) {
ff023aac 544 WARN_ON(!sbio->pagev[i]->page);
b5d67f64
SB		 545 scrub_block_put(sbio->pagev[i]->sblock);
546 }
547 bio_put(sbio->bio);
548 }
549
ff023aac 550 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
d9d181c1 551 struct scrub_bio *sbio = sctx->bios[i];
a2de733c
AJ		 552
553 if (!sbio)
554 break;
a2de733c
AJ		 555 kfree(sbio);
556 }
557
3fb99303 558 kfree(sctx->wr_curr_bio);
d9d181c1
SB		 559 scrub_free_csums(sctx);
560 kfree(sctx);
a2de733c
AJ		 561}
562
f55985f4
FM		 563static void scrub_put_ctx(struct scrub_ctx *sctx)
564{
99f4cdb1 565 if (refcount_dec_and_test(&sctx->refs))
f55985f4
FM		 566 scrub_free_ctx(sctx);
567}
568
92f7ba43
DS		 569static noinline_for_stack struct scrub_ctx *scrub_setup_ctx(
570 struct btrfs_fs_info *fs_info, int is_dev_replace)
a2de733c 571{
d9d181c1 572 struct scrub_ctx *sctx;
a2de733c 573 int i;
a2de733c 574
58c4e173 575 sctx = kzalloc(sizeof(*sctx), GFP_KERNEL);
d9d181c1 576 if (!sctx)
a2de733c 577 goto nomem;
99f4cdb1 578 refcount_set(&sctx->refs, 1);
63a212ab 579 sctx->is_dev_replace = is_dev_replace;
b54ffb73 580 sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
d9d181c1 581 sctx->curr = -1;
92f7ba43 582 sctx->fs_info = fs_info;
e49be14b 583 INIT_LIST_HEAD(&sctx->csum_list);
ff023aac 584 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
a2de733c
AJ		 585 struct scrub_bio *sbio;
586
58c4e173 587 sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);
a2de733c
AJ		 588 if (!sbio)
589 goto nomem;
d9d181c1 590 sctx->bios[i] = sbio;
a2de733c 591
a2de733c 592 sbio->index = i;
d9d181c1 593 sbio->sctx = sctx;
b5d67f64 594 sbio->page_count = 0;
a0cac0ec
OS		 595 btrfs_init_work(&sbio->work, scrub_bio_end_io_worker, NULL,
596 NULL);
a2de733c 597
ff023aac 598 if (i != SCRUB_BIOS_PER_SCTX - 1)
d9d181c1 599 sctx->bios[i]->next_free = i + 1;
0ef8e451 600 else
d9d181c1
SB		 601 sctx->bios[i]->next_free = -1;
602 }
603 sctx->first_free = 0;
b6bfebc1
SB		 604 atomic_set(&sctx->bios_in_flight, 0);
605 atomic_set(&sctx->workers_pending, 0);
d9d181c1 606 atomic_set(&sctx->cancel_req, 0);
d9d181c1
SB		 607
608 spin_lock_init(&sctx->list_lock);
609 spin_lock_init(&sctx->stat_lock);
610 init_waitqueue_head(&sctx->list_wait);
eb3b5053 611 sctx->throttle_deadline = 0;
ff023aac 612
3fb99303
DS		 613 WARN_ON(sctx->wr_curr_bio != NULL);
614 mutex_init(&sctx->wr_lock);
615 sctx->wr_curr_bio = NULL;
8fcdac3f 616 if (is_dev_replace) {
ded56184 617 WARN_ON(!fs_info->dev_replace.tgtdev);
3fb99303 618 sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO;
ded56184 619 sctx->wr_tgtdev = fs_info->dev_replace.tgtdev;
2073c4c2 620 sctx->flush_all_writes = false;
ff023aac 621 }
8fcdac3f 622
d9d181c1 623 return sctx;
a2de733c
AJ		 624
625nomem:
d9d181c1 626 scrub_free_ctx(sctx);
a2de733c
AJ		 627 return ERR_PTR(-ENOMEM);
628}
629
ff023aac
SB		 630static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
631 void *warn_ctx)
558540c1 632{
558540c1
JS		 633 u32 nlink;
634 int ret;
635 int i;
de2491fd 636 unsigned nofs_flag;
558540c1
JS		 637 struct extent_buffer *eb;
638 struct btrfs_inode_item *inode_item;
ff023aac 639 struct scrub_warning *swarn = warn_ctx;
fb456252 640 struct btrfs_fs_info *fs_info = swarn->dev->fs_info;
558540c1
JS		 641 struct inode_fs_paths *ipath = NULL;
642 struct btrfs_root *local_root;
1d4c08e0 643 struct btrfs_key key;
558540c1 644
56e9357a 645 local_root = btrfs_get_fs_root(fs_info, root, true);
558540c1
JS		 646 if (IS_ERR(local_root)) {
647 ret = PTR_ERR(local_root);
648 goto err;
649 }
650
14692cc1
DS		 651 /*
652 * this makes the path point to (inum INODE_ITEM ioff)
653 */
1d4c08e0
DS		 654 key.objectid = inum;
655 key.type = BTRFS_INODE_ITEM_KEY;
656 key.offset = 0;
657
658 ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
558540c1 659 if (ret) {
00246528 660 btrfs_put_root(local_root);
558540c1
JS		 661 btrfs_release_path(swarn->path);
662 goto err;
663 }
664
665 eb = swarn->path->nodes[0];
666 inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
667 struct btrfs_inode_item);
558540c1
JS		 668 nlink = btrfs_inode_nlink(eb, inode_item);
669 btrfs_release_path(swarn->path);
670
de2491fd
DS		 671 /*
672 * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub
673 * uses GFP_NOFS in this context, so we keep it consistent but it does
674 * not seem to be strictly necessary.
675 */
676 nofs_flag = memalloc_nofs_save();
558540c1 677 ipath = init_ipath(4096, local_root, swarn->path);
de2491fd 678 memalloc_nofs_restore(nofs_flag);
26bdef54 679 if (IS_ERR(ipath)) {
00246528 680 btrfs_put_root(local_root);
26bdef54
DC		 681 ret = PTR_ERR(ipath);
682 ipath = NULL;
683 goto err;
684 }
558540c1
JS		 685 ret = paths_from_inode(inum, ipath);
686
687 if (ret < 0)
688 goto err;
689
690 /*
691 * we deliberately ignore the bit ipath might have been too small to
692 * hold all of the paths here
693 */
694 for (i = 0; i < ipath->fspath->elem_cnt; ++i)
5d163e0e 695 btrfs_warn_in_rcu(fs_info,
8df507cb 696"%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %u, links %u (path: %s)",
5d163e0e
JM		 697 swarn->errstr, swarn->logical,
698 rcu_str_deref(swarn->dev->name),
6aa21263 699 swarn->physical,
5d163e0e 700 root, inum, offset,
8df507cb 701 fs_info->sectorsize, nlink,
5d163e0e 702 (char *)(unsigned long)ipath->fspath->val[i]);
558540c1 703
00246528 704 btrfs_put_root(local_root);
558540c1
JS		 705 free_ipath(ipath);
706 return 0;
707
708err:
5d163e0e 709 btrfs_warn_in_rcu(fs_info,
6aa21263 710 "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d",
5d163e0e
JM		 711 swarn->errstr, swarn->logical,
712 rcu_str_deref(swarn->dev->name),
6aa21263 713 swarn->physical,
5d163e0e 714 root, inum, offset, ret);
558540c1
JS		 715
716 free_ipath(ipath);
717 return 0;
718}
719
b5d67f64 720static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
558540c1 721{
a36cf8b8
SB		 722 struct btrfs_device *dev;
723 struct btrfs_fs_info *fs_info;
558540c1
JS		 724 struct btrfs_path *path;
725 struct btrfs_key found_key;
726 struct extent_buffer *eb;
727 struct btrfs_extent_item *ei;
728 struct scrub_warning swarn;
69917e43
LB		 729 unsigned long ptr = 0;
730 u64 extent_item_pos;
731 u64 flags = 0;
558540c1 732 u64 ref_root;
69917e43 733 u32 item_size;
07c9a8e0 734 u8 ref_level = 0;
69917e43 735 int ret;
558540c1 736
a36cf8b8 737 WARN_ON(sblock->page_count < 1);
7a9e9987 738 dev = sblock->pagev[0]->dev;
fb456252 739 fs_info = sblock->sctx->fs_info;
a36cf8b8 740
558540c1 741 path = btrfs_alloc_path();
8b9456da
DS		 742 if (!path)
743 return;
558540c1 744
6aa21263 745 swarn.physical = sblock->pagev[0]->physical;
7a9e9987 746 swarn.logical = sblock->pagev[0]->logical;
558540c1 747 swarn.errstr = errstr;
a36cf8b8 748 swarn.dev = NULL;
558540c1 749
69917e43
LB		 750 ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
751 &flags);
558540c1
JS		 752 if (ret < 0)
753 goto out;
754
4692cf58 755 extent_item_pos = swarn.logical - found_key.objectid;
558540c1
JS		 756 swarn.extent_item_size = found_key.offset;
757
758 eb = path->nodes[0];
759 ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
3212fa14 760 item_size = btrfs_item_size(eb, path->slots[0]);
558540c1 761
69917e43 762 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
558540c1 763 do {
6eda71d0
LB		 764 ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
765 item_size, &ref_root,
766 &ref_level);
ecaeb14b 767 btrfs_warn_in_rcu(fs_info,
6aa21263 768"%s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu",
5d163e0e 769 errstr, swarn.logical,
606686ee 770 rcu_str_deref(dev->name),
6aa21263 771 swarn.physical,
558540c1
JS		 772 ref_level ? "node" : "leaf",
773 ret < 0 ? -1 : ref_level,
774 ret < 0 ? -1 : ref_root);
775 } while (ret != 1);
d8fe29e9 776 btrfs_release_path(path);
558540c1 777 } else {
d8fe29e9 778 btrfs_release_path(path);
558540c1 779 swarn.path = path;
a36cf8b8 780 swarn.dev = dev;
7a3ae2f8
JS		 781 iterate_extent_inodes(fs_info, found_key.objectid,
782 extent_item_pos, 1,
c995ab3c 783 scrub_print_warning_inode, &swarn, false);
558540c1
JS		 784 }
785
786out:
787 btrfs_free_path(path);
558540c1
JS		 788}
789
af8e2d1d
MX		 790static inline void scrub_get_recover(struct scrub_recover *recover)
791{
6f615018 792 refcount_inc(&recover->refs);
af8e2d1d
MX		 793}
794
e501bfe3
QW		 795static inline void scrub_put_recover(struct btrfs_fs_info *fs_info,
796 struct scrub_recover *recover)
af8e2d1d 797{
6f615018 798 if (refcount_dec_and_test(&recover->refs)) {
e501bfe3 799 btrfs_bio_counter_dec(fs_info);
4c664611 800 btrfs_put_bioc(recover->bioc);
af8e2d1d
MX		 801 kfree(recover);
802 }
803}
804
a2de733c 805/*
b5d67f64
SB		 806 * scrub_handle_errored_block gets called when either verification of the
807 * pages failed or the bio failed to read, e.g. with EIO. In the latter
808 * case, this function handles all pages in the bio, even though only one
809 * may be bad.
810 * The goal of this function is to repair the errored block by using the
811 * contents of one of the mirrors.
a2de733c 812 */
b5d67f64 813static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
a2de733c 814{
d9d181c1 815 struct scrub_ctx *sctx = sblock_to_check->sctx;
a36cf8b8 816 struct btrfs_device *dev;
b5d67f64 817 struct btrfs_fs_info *fs_info;
b5d67f64 818 u64 logical;
b5d67f64
SB		 819 unsigned int failed_mirror_index;
820 unsigned int is_metadata;
821 unsigned int have_csum;
b5d67f64
SB		 822 struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
823 struct scrub_block *sblock_bad;
824 int ret;
825 int mirror_index;
826 int page_num;
827 int success;
28d70e23 828 bool full_stripe_locked;
7c3c7cb9 829 unsigned int nofs_flag;
8bb1cf1b 830 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
b5d67f64
SB		 831 DEFAULT_RATELIMIT_BURST);
832
833 BUG_ON(sblock_to_check->page_count < 1);
fb456252 834 fs_info = sctx->fs_info;
4ded4f63
SB		 835 if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
836 /*
837 * if we find an error in a super block, we just report it.
838 * They will get written with the next transaction commit
839 * anyway
840 */
841 spin_lock(&sctx->stat_lock);
842 ++sctx->stat.super_errors;
843 spin_unlock(&sctx->stat_lock);
844 return 0;
845 }
7a9e9987 846 logical = sblock_to_check->pagev[0]->logical;
7a9e9987
SB		 847 BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
848 failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
849 is_metadata = !(sblock_to_check->pagev[0]->flags &
b5d67f64 850 BTRFS_EXTENT_FLAG_DATA);
7a9e9987 851 have_csum = sblock_to_check->pagev[0]->have_csum;
7a9e9987 852 dev = sblock_to_check->pagev[0]->dev;
13db62b7 853
f7ef5287
NA		 854 if (btrfs_is_zoned(fs_info) && !sctx->is_dev_replace)
855 return btrfs_repair_one_zone(fs_info, logical);
856
7c3c7cb9
FM		 857 /*
858 * We must use GFP_NOFS because the scrub task might be waiting for a
859 * worker task executing this function and in turn a transaction commit
860 * might be waiting the scrub task to pause (which needs to wait for all
861 * the worker tasks to complete before pausing).
862 * We do allocations in the workers through insert_full_stripe_lock()
863 * and scrub_add_page_to_wr_bio(), which happens down the call chain of
864 * this function.
865 */
866 nofs_flag = memalloc_nofs_save();
28d70e23
QW		 867 /*
868 * For RAID5/6, race can happen for a different device scrub thread.
869 * For data corruption, Parity and Data threads will both try
870 * to recovery the data.
871 * Race can lead to doubly added csum error, or even unrecoverable
872 * error.
873 */
874 ret = lock_full_stripe(fs_info, logical, &full_stripe_locked);
875 if (ret < 0) {
7c3c7cb9 876 memalloc_nofs_restore(nofs_flag);
28d70e23
QW		 877 spin_lock(&sctx->stat_lock);
878 if (ret == -ENOMEM)
879 sctx->stat.malloc_errors++;
880 sctx->stat.read_errors++;
881 sctx->stat.uncorrectable_errors++;
882 spin_unlock(&sctx->stat_lock);
883 return ret;
884 }
885
b5d67f64
SB		 886 /*
887 * read all mirrors one after the other. This includes to
888 * re-read the extent or metadata block that failed (that was
889 * the cause that this fixup code is called) another time,
8df507cb 890 * sector by sector this time in order to know which sectors
b5d67f64
SB		 891 * caused I/O errors and which ones are good (for all mirrors).
892 * It is the goal to handle the situation when more than one
893 * mirror contains I/O errors, but the errors do not
894 * overlap, i.e. the data can be repaired by selecting the
8df507cb
QW		 895 * sectors from those mirrors without I/O error on the
896 * particular sectors. One example (with blocks >= 2 * sectorsize)
897 * would be that mirror #1 has an I/O error on the first sector,
898 * the second sector is good, and mirror #2 has an I/O error on
899 * the second sector, but the first sector is good.
900 * Then the first sector of the first mirror can be repaired by
901 * taking the first sector of the second mirror, and the
902 * second sector of the second mirror can be repaired by
903 * copying the contents of the 2nd sector of the 1st mirror.
904 * One more note: if the sectors of one mirror contain I/O
b5d67f64
SB		 905 * errors, the checksum cannot be verified. In order to get
906 * the best data for repairing, the first attempt is to find
907 * a mirror without I/O errors and with a validated checksum.
8df507cb 908 * Only if this is not possible, the sectors are picked from
b5d67f64
SB		 909 * mirrors with I/O errors without considering the checksum.
910 * If the latter is the case, at the end, the checksum of the
911 * repaired area is verified in order to correctly maintain
912 * the statistics.
913 */
914
31e818fe 915 sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
7c3c7cb9 916 sizeof(*sblocks_for_recheck), GFP_KERNEL);
b5d67f64 917 if (!sblocks_for_recheck) {
d9d181c1
SB		 918 spin_lock(&sctx->stat_lock);
919 sctx->stat.malloc_errors++;
920 sctx->stat.read_errors++;
921 sctx->stat.uncorrectable_errors++;
922 spin_unlock(&sctx->stat_lock);
a36cf8b8 923 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
b5d67f64 924 goto out;
a2de733c
AJ		 925 }
926
b5d67f64 927 /* setup the context, map the logical blocks and alloc the pages */
be50a8dd 928 ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
b5d67f64 929 if (ret) {
d9d181c1
SB		 930 spin_lock(&sctx->stat_lock);
931 sctx->stat.read_errors++;
932 sctx->stat.uncorrectable_errors++;
933 spin_unlock(&sctx->stat_lock);
a36cf8b8 934 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
b5d67f64
SB		 935 goto out;
936 }
937 BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
938 sblock_bad = sblocks_for_recheck + failed_mirror_index;
13db62b7 939
b5d67f64 940 /* build and submit the bios for the failed mirror, check checksums */
affe4a5a 941 scrub_recheck_block(fs_info, sblock_bad, 1);
a2de733c 942
b5d67f64
SB		 943 if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
944 sblock_bad->no_io_error_seen) {
945 /*
946 * the error disappeared after reading page by page, or
947 * the area was part of a huge bio and other parts of the
948 * bio caused I/O errors, or the block layer merged several
949 * read requests into one and the error is caused by a
950 * different bio (usually one of the two latter cases is
951 * the cause)
952 */
d9d181c1
SB		 953 spin_lock(&sctx->stat_lock);
954 sctx->stat.unverified_errors++;
5a6ac9ea 955 sblock_to_check->data_corrected = 1;
d9d181c1 956 spin_unlock(&sctx->stat_lock);
a2de733c 957
ff023aac
SB		 958 if (sctx->is_dev_replace)
959 scrub_write_block_to_dev_replace(sblock_bad);
b5d67f64 960 goto out;
a2de733c 961 }
a2de733c 962
b5d67f64 963 if (!sblock_bad->no_io_error_seen) {
d9d181c1
SB		 964 spin_lock(&sctx->stat_lock);
965 sctx->stat.read_errors++;
966 spin_unlock(&sctx->stat_lock);
8bb1cf1b 967 if (__ratelimit(&rs))
b5d67f64 968 scrub_print_warning("i/o error", sblock_to_check);
a36cf8b8 969 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
b5d67f64 970 } else if (sblock_bad->checksum_error) {
d9d181c1
SB		 971 spin_lock(&sctx->stat_lock);
972 sctx->stat.csum_errors++;
973 spin_unlock(&sctx->stat_lock);
8bb1cf1b 974 if (__ratelimit(&rs))
b5d67f64 975 scrub_print_warning("checksum error", sblock_to_check);
a36cf8b8 976 btrfs_dev_stat_inc_and_print(dev,
442a4f63 977 BTRFS_DEV_STAT_CORRUPTION_ERRS);
b5d67f64 978 } else if (sblock_bad->header_error) {
d9d181c1
SB		 979 spin_lock(&sctx->stat_lock);
980 sctx->stat.verify_errors++;
981 spin_unlock(&sctx->stat_lock);
8bb1cf1b 982 if (__ratelimit(&rs))
b5d67f64
SB		 983 scrub_print_warning("checksum/header error",
984 sblock_to_check);
442a4f63 985 if (sblock_bad->generation_error)
a36cf8b8 986 btrfs_dev_stat_inc_and_print(dev,
442a4f63
SB		 987 BTRFS_DEV_STAT_GENERATION_ERRS);
988 else
a36cf8b8 989 btrfs_dev_stat_inc_and_print(dev,
442a4f63 990 BTRFS_DEV_STAT_CORRUPTION_ERRS);
b5d67f64 991 }
a2de733c 992
33ef30ad
ID		 993 if (sctx->readonly) {
994 ASSERT(!sctx->is_dev_replace);
995 goto out;
996 }
a2de733c 997
b5d67f64
SB		 998 /*
999 * now build and submit the bios for the other mirrors, check
cb2ced73
SB		 1000 * checksums.
1001 * First try to pick the mirror which is completely without I/O
b5d67f64
SB		 1002 * errors and also does not have a checksum error.
1003 * If one is found, and if a checksum is present, the full block
1004 * that is known to contain an error is rewritten. Afterwards
1005 * the block is known to be corrected.
1006 * If a mirror is found which is completely correct, and no
1007 * checksum is present, only those pages are rewritten that had
1008 * an I/O error in the block to be repaired, since it cannot be
1009 * determined, which copy of the other pages is better (and it
1010 * could happen otherwise that a correct page would be
1011 * overwritten by a bad one).
1012 */
762221f0 1013 for (mirror_index = 0; ;mirror_index++) {
cb2ced73 1014 struct scrub_block *sblock_other;
b5d67f64 1015
cb2ced73
SB		 1016 if (mirror_index == failed_mirror_index)
1017 continue;
762221f0
LB		 1018
1019 /* raid56's mirror can be more than BTRFS_MAX_MIRRORS */
1020 if (!scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
1021 if (mirror_index >= BTRFS_MAX_MIRRORS)
1022 break;
1023 if (!sblocks_for_recheck[mirror_index].page_count)
1024 break;
1025
1026 sblock_other = sblocks_for_recheck + mirror_index;
1027 } else {
1028 struct scrub_recover *r = sblock_bad->pagev[0]->recover;
4c664611 1029 int max_allowed = r->bioc->num_stripes - r->bioc->num_tgtdevs;
762221f0
LB		 1030
1031 if (mirror_index >= max_allowed)
1032 break;
1033 if (!sblocks_for_recheck[1].page_count)
1034 break;
1035
1036 ASSERT(failed_mirror_index == 0);
1037 sblock_other = sblocks_for_recheck + 1;
1038 sblock_other->pagev[0]->mirror_num = 1 + mirror_index;
1039 }
cb2ced73
SB		 1040
1041 /* build and submit the bios, check checksums */
affe4a5a 1042 scrub_recheck_block(fs_info, sblock_other, 0);
34f5c8e9
SB		 1043
1044 if (!sblock_other->header_error &&
b5d67f64
SB		 1045 !sblock_other->checksum_error &&
1046 sblock_other->no_io_error_seen) {
ff023aac
SB		 1047 if (sctx->is_dev_replace) {
1048 scrub_write_block_to_dev_replace(sblock_other);
114ab50d 1049 goto corrected_error;
ff023aac 1050 } else {
ff023aac 1051 ret = scrub_repair_block_from_good_copy(
114ab50d
ZL		 1052 sblock_bad, sblock_other);
1053 if (!ret)
1054 goto corrected_error;
ff023aac 1055 }
b5d67f64
SB		 1056 }
1057 }
a2de733c 1058
b968fed1
ZL		 1059 if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
1060 goto did_not_correct_error;
ff023aac
SB		 1061
1062 /*
ff023aac 1063 * In case of I/O errors in the area that is supposed to be
8df507cb
QW		 1064 * repaired, continue by picking good copies of those sectors.
1065 * Select the good sectors from mirrors to rewrite bad sectors from
b5d67f64
SB		 1066 * the area to fix. Afterwards verify the checksum of the block
1067 * that is supposed to be repaired. This verification step is
1068 * only done for the purpose of statistic counting and for the
1069 * final scrub report, whether errors remain.
1070 * A perfect algorithm could make use of the checksum and try
8df507cb 1071 * all possible combinations of sectors from the different mirrors
b5d67f64 1072 * until the checksum verification succeeds. For example, when
8df507cb 1073 * the 2nd sector of mirror #1 faces I/O errors, and the 2nd sector
b5d67f64 1074 * of mirror #2 is readable but the final checksum test fails,
8df507cb 1075 * then the 2nd sector of mirror #3 could be tried, whether now
01327610 1076 * the final checksum succeeds. But this would be a rare
b5d67f64
SB		 1077 * exception and is therefore not implemented. At least it is
1078 * avoided that the good copy is overwritten.
1079 * A more useful improvement would be to pick the sectors
1080 * without I/O error based on sector sizes (512 bytes on legacy
8df507cb 1081 * disks) instead of on sectorsize. Then maybe 512 byte of one
b5d67f64 1082 * mirror could be repaired by taking 512 byte of a different
8df507cb 1083 * mirror, even if other 512 byte sectors in the same sectorsize
b5d67f64 1084 * area are unreadable.
a2de733c 1085 */
b5d67f64 1086 success = 1;
b968fed1
ZL		 1087 for (page_num = 0; page_num < sblock_bad->page_count;
1088 page_num++) {
261d2dcb 1089 struct scrub_page *spage_bad = sblock_bad->pagev[page_num];
b968fed1 1090 struct scrub_block *sblock_other = NULL;
b5d67f64 1091
b968fed1 1092 /* skip no-io-error page in scrub */
261d2dcb 1093 if (!spage_bad->io_error && !sctx->is_dev_replace)
a2de733c 1094 continue;
b5d67f64 1095
4759700a
LB		 1096 if (scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
1097 /*
1098 * In case of dev replace, if raid56 rebuild process
1099 * didn't work out correct data, then copy the content
1100 * in sblock_bad to make sure target device is identical
1101 * to source device, instead of writing garbage data in
1102 * sblock_for_recheck array to target device.
1103 */
1104 sblock_other = NULL;
261d2dcb 1105 } else if (spage_bad->io_error) {
4759700a 1106 /* try to find no-io-error page in mirrors */
b968fed1
ZL		 1107 for (mirror_index = 0;
1108 mirror_index < BTRFS_MAX_MIRRORS &&
1109 sblocks_for_recheck[mirror_index].page_count > 0;
1110 mirror_index++) {
1111 if (!sblocks_for_recheck[mirror_index].
1112 pagev[page_num]->io_error) {
1113 sblock_other = sblocks_for_recheck +
1114 mirror_index;
1115 break;
b5d67f64
SB		 1116 }
1117 }
b968fed1
ZL		 1118 if (!sblock_other)
1119 success = 0;
96e36920 1120 }
a2de733c 1121
b968fed1
ZL		 1122 if (sctx->is_dev_replace) {
1123 /*
1124 * did not find a mirror to fetch the page
1125 * from. scrub_write_page_to_dev_replace()
1126 * handles this case (page->io_error), by
1127 * filling the block with zeros before
1128 * submitting the write request
1129 */
1130 if (!sblock_other)
1131 sblock_other = sblock_bad;
1132
1133 if (scrub_write_page_to_dev_replace(sblock_other,
1134 page_num) != 0) {
e37abe97 1135 atomic64_inc(
0b246afa 1136 &fs_info->dev_replace.num_write_errors);
b968fed1
ZL		 1137 success = 0;
1138 }
1139 } else if (sblock_other) {
1140 ret = scrub_repair_page_from_good_copy(sblock_bad,
1141 sblock_other,
1142 page_num, 0);
1143 if (0 == ret)
261d2dcb 1144 spage_bad->io_error = 0;
b968fed1
ZL		 1145 else
1146 success = 0;
b5d67f64 1147 }
a2de733c 1148 }
a2de733c 1149
b968fed1 1150 if (success && !sctx->is_dev_replace) {
b5d67f64
SB		 1151 if (is_metadata || have_csum) {
1152 /*
1153 * need to verify the checksum now that all
1154 * sectors on disk are repaired (the write
1155 * request for data to be repaired is on its way).
1156 * Just be lazy and use scrub_recheck_block()
1157 * which re-reads the data before the checksum
1158 * is verified, but most likely the data comes out
1159 * of the page cache.
1160 */
affe4a5a 1161 scrub_recheck_block(fs_info, sblock_bad, 1);
34f5c8e9 1162 if (!sblock_bad->header_error &&
b5d67f64
SB		 1163 !sblock_bad->checksum_error &&
1164 sblock_bad->no_io_error_seen)
1165 goto corrected_error;
1166 else
1167 goto did_not_correct_error;
1168 } else {
1169corrected_error:
d9d181c1
SB		 1170 spin_lock(&sctx->stat_lock);
1171 sctx->stat.corrected_errors++;
5a6ac9ea 1172 sblock_to_check->data_corrected = 1;
d9d181c1 1173 spin_unlock(&sctx->stat_lock);
b14af3b4
DS		 1174 btrfs_err_rl_in_rcu(fs_info,
1175 "fixed up error at logical %llu on dev %s",
c1c9ff7c 1176 logical, rcu_str_deref(dev->name));
8628764e 1177 }
b5d67f64
SB		 1178 } else {
1179did_not_correct_error:
d9d181c1
SB		 1180 spin_lock(&sctx->stat_lock);
1181 sctx->stat.uncorrectable_errors++;
1182 spin_unlock(&sctx->stat_lock);
b14af3b4
DS		 1183 btrfs_err_rl_in_rcu(fs_info,
1184 "unable to fixup (regular) error at logical %llu on dev %s",
c1c9ff7c 1185 logical, rcu_str_deref(dev->name));
96e36920 1186 }
a2de733c 1187
b5d67f64
SB		 1188out:
1189 if (sblocks_for_recheck) {
1190 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1191 mirror_index++) {
1192 struct scrub_block *sblock = sblocks_for_recheck +
1193 mirror_index;
af8e2d1d 1194 struct scrub_recover *recover;
b5d67f64
SB		 1195 int page_index;
1196
7a9e9987
SB		 1197 for (page_index = 0; page_index < sblock->page_count;
1198 page_index++) {
1199 sblock->pagev[page_index]->sblock = NULL;
af8e2d1d
MX		 1200 recover = sblock->pagev[page_index]->recover;
1201 if (recover) {
e501bfe3 1202 scrub_put_recover(fs_info, recover);
af8e2d1d
MX		 1203 sblock->pagev[page_index]->recover =
1204 NULL;
1205 }
7a9e9987
SB		 1206 scrub_page_put(sblock->pagev[page_index]);
1207 }
b5d67f64
SB		 1208 }
1209 kfree(sblocks_for_recheck);
1210 }
a2de733c 1211
28d70e23 1212 ret = unlock_full_stripe(fs_info, logical, full_stripe_locked);
7c3c7cb9 1213 memalloc_nofs_restore(nofs_flag);
28d70e23
QW		 1214 if (ret < 0)
1215 return ret;
b5d67f64
SB		 1216 return 0;
1217}
a2de733c 1218
4c664611 1219static inline int scrub_nr_raid_mirrors(struct btrfs_io_context *bioc)
af8e2d1d 1220{
4c664611 1221 if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID5)
10f11900 1222 return 2;
4c664611 1223 else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID6)
10f11900
ZL		 1224 return 3;
1225 else
4c664611 1226 return (int)bioc->num_stripes;
af8e2d1d
MX		 1227}
1228
10f11900
ZL		 1229static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1230 u64 *raid_map,
af8e2d1d
MX		 1231 u64 mapped_length,
1232 int nstripes, int mirror,
1233 int *stripe_index,
1234 u64 *stripe_offset)
1235{
1236 int i;
1237
ffe2d203 1238 if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
af8e2d1d
MX		 1239 /* RAID5/6 */
1240 for (i = 0; i < nstripes; i++) {
1241 if (raid_map[i] == RAID6_Q_STRIPE ||
1242 raid_map[i] == RAID5_P_STRIPE)
1243 continue;
1244
1245 if (logical >= raid_map[i] &&
1246 logical < raid_map[i] + mapped_length)
1247 break;
1248 }
1249
1250 *stripe_index = i;
1251 *stripe_offset = logical - raid_map[i];
1252 } else {
1253 /* The other RAID type */
1254 *stripe_index = mirror;
1255 *stripe_offset = 0;
1256 }
1257}
1258
be50a8dd 1259static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
b5d67f64
SB		 1260 struct scrub_block *sblocks_for_recheck)
1261{
be50a8dd 1262 struct scrub_ctx *sctx = original_sblock->sctx;
fb456252 1263 struct btrfs_fs_info *fs_info = sctx->fs_info;
8df507cb 1264 u64 length = original_sblock->page_count * fs_info->sectorsize;
be50a8dd 1265 u64 logical = original_sblock->pagev[0]->logical;
4734b7ed
ZL		 1266 u64 generation = original_sblock->pagev[0]->generation;
1267 u64 flags = original_sblock->pagev[0]->flags;
1268 u64 have_csum = original_sblock->pagev[0]->have_csum;
af8e2d1d 1269 struct scrub_recover *recover;
4c664611 1270 struct btrfs_io_context *bioc;
af8e2d1d
MX		 1271 u64 sublen;
1272 u64 mapped_length;
1273 u64 stripe_offset;
1274 int stripe_index;
be50a8dd 1275 int page_index = 0;
b5d67f64 1276 int mirror_index;
af8e2d1d 1277 int nmirrors;
b5d67f64
SB		 1278 int ret;
1279
1280 /*
57019345 1281 * note: the two members refs and outstanding_pages
b5d67f64
SB		 1282 * are not used (and not set) in the blocks that are used for
1283 * the recheck procedure
1284 */
1285
b5d67f64 1286 while (length > 0) {
8df507cb 1287 sublen = min_t(u64, length, fs_info->sectorsize);
af8e2d1d 1288 mapped_length = sublen;
4c664611 1289 bioc = NULL;
a2de733c 1290
b5d67f64 1291 /*
8df507cb
QW		 1292 * With a length of sectorsize, each returned stripe represents
1293 * one mirror
b5d67f64 1294 */
e501bfe3 1295 btrfs_bio_counter_inc_blocked(fs_info);
cf8cddd3 1296 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
4c664611
QW		 1297 logical, &mapped_length, &bioc);
1298 if (ret || !bioc || mapped_length < sublen) {
1299 btrfs_put_bioc(bioc);
e501bfe3 1300 btrfs_bio_counter_dec(fs_info);
b5d67f64
SB		 1301 return -EIO;
1302 }
a2de733c 1303
af8e2d1d
MX		 1304 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1305 if (!recover) {
4c664611 1306 btrfs_put_bioc(bioc);
e501bfe3 1307 btrfs_bio_counter_dec(fs_info);
af8e2d1d
MX		 1308 return -ENOMEM;
1309 }
1310
6f615018 1311 refcount_set(&recover->refs, 1);
4c664611 1312 recover->bioc = bioc;
af8e2d1d
MX		 1313 recover->map_length = mapped_length;
1314
0bb3acdc 1315 ASSERT(page_index < SCRUB_MAX_PAGES_PER_BLOCK);
af8e2d1d 1316
4c664611 1317 nmirrors = min(scrub_nr_raid_mirrors(bioc), BTRFS_MAX_MIRRORS);
10f11900 1318
af8e2d1d 1319 for (mirror_index = 0; mirror_index < nmirrors;
b5d67f64
SB		 1320 mirror_index++) {
1321 struct scrub_block *sblock;
261d2dcb 1322 struct scrub_page *spage;
b5d67f64 1323
b5d67f64 1324 sblock = sblocks_for_recheck + mirror_index;
7a9e9987 1325 sblock->sctx = sctx;
4734b7ed 1326
261d2dcb
QW		 1327 spage = kzalloc(sizeof(*spage), GFP_NOFS);
1328 if (!spage) {
7a9e9987 1329leave_nomem:
d9d181c1
SB		 1330 spin_lock(&sctx->stat_lock);
1331 sctx->stat.malloc_errors++;
1332 spin_unlock(&sctx->stat_lock);
e501bfe3 1333 scrub_put_recover(fs_info, recover);
b5d67f64
SB		 1334 return -ENOMEM;
1335 }
261d2dcb
QW		 1336 scrub_page_get(spage);
1337 sblock->pagev[page_index] = spage;
1338 spage->sblock = sblock;
1339 spage->flags = flags;
1340 spage->generation = generation;
1341 spage->logical = logical;
1342 spage->have_csum = have_csum;
4734b7ed 1343 if (have_csum)
261d2dcb 1344 memcpy(spage->csum,
4734b7ed 1345 original_sblock->pagev[0]->csum,
2ae0c2d8 1346 sctx->fs_info->csum_size);
af8e2d1d 1347
10f11900 1348 scrub_stripe_index_and_offset(logical,
4c664611
QW		 1349 bioc->map_type,
1350 bioc->raid_map,
af8e2d1d 1351 mapped_length,
4c664611
QW		 1352 bioc->num_stripes -
1353 bioc->num_tgtdevs,
af8e2d1d
MX		 1354 mirror_index,
1355 &stripe_index,
1356 &stripe_offset);
4c664611 1357 spage->physical = bioc->stripes[stripe_index].physical +
af8e2d1d 1358 stripe_offset;
4c664611 1359 spage->dev = bioc->stripes[stripe_index].dev;
af8e2d1d 1360
ff023aac 1361 BUG_ON(page_index >= original_sblock->page_count);
261d2dcb 1362 spage->physical_for_dev_replace =
ff023aac
SB		 1363 original_sblock->pagev[page_index]->
1364 physical_for_dev_replace;
7a9e9987 1365 /* for missing devices, dev->bdev is NULL */
261d2dcb 1366 spage->mirror_num = mirror_index + 1;
b5d67f64 1367 sblock->page_count++;
261d2dcb
QW		 1368 spage->page = alloc_page(GFP_NOFS);
1369 if (!spage->page)
7a9e9987 1370 goto leave_nomem;
af8e2d1d
MX		 1371
1372 scrub_get_recover(recover);
261d2dcb 1373 spage->recover = recover;
b5d67f64 1374 }
e501bfe3 1375 scrub_put_recover(fs_info, recover);
b5d67f64
SB		 1376 length -= sublen;
1377 logical += sublen;
1378 page_index++;
1379 }
1380
1381 return 0;
96e36920
ID		 1382}
1383
4246a0b6 1384static void scrub_bio_wait_endio(struct bio *bio)
af8e2d1d 1385{
b4ff5ad7 1386 complete(bio->bi_private);
af8e2d1d
MX		 1387}
1388
af8e2d1d
MX		 1389static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1390 struct bio *bio,
261d2dcb 1391 struct scrub_page *spage)
af8e2d1d 1392{
b4ff5ad7 1393 DECLARE_COMPLETION_ONSTACK(done);
af8e2d1d 1394 int ret;
762221f0 1395 int mirror_num;
af8e2d1d 1396
261d2dcb 1397 bio->bi_iter.bi_sector = spage->logical >> 9;
af8e2d1d
MX		 1398 bio->bi_private = &done;
1399 bio->bi_end_io = scrub_bio_wait_endio;
1400
261d2dcb 1401 mirror_num = spage->sblock->pagev[0]->mirror_num;
6a258d72 1402 ret = raid56_parity_recover(bio, spage->recover->bioc,
261d2dcb 1403 spage->recover->map_length,
762221f0 1404 mirror_num, 0);
af8e2d1d
MX		 1405 if (ret)
1406 return ret;
1407
b4ff5ad7
LB		 1408 wait_for_completion_io(&done);
1409 return blk_status_to_errno(bio->bi_status);
af8e2d1d
MX		 1410}
1411
6ca1765b
LB		 1412static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info,
1413 struct scrub_block *sblock)
1414{
1415 struct scrub_page *first_page = sblock->pagev[0];
1416 struct bio *bio;
1417 int page_num;
1418
1419 /* All pages in sblock belong to the same stripe on the same device. */
1420 ASSERT(first_page->dev);
1421 if (!first_page->dev->bdev)
1422 goto out;
1423
c3a3b19b 1424 bio = btrfs_bio_alloc(BIO_MAX_VECS);
6ca1765b
LB		 1425 bio_set_dev(bio, first_page->dev->bdev);
1426
1427 for (page_num = 0; page_num < sblock->page_count; page_num++) {
261d2dcb 1428 struct scrub_page *spage = sblock->pagev[page_num];
6ca1765b 1429
261d2dcb
QW		 1430 WARN_ON(!spage->page);
1431 bio_add_page(bio, spage->page, PAGE_SIZE, 0);
6ca1765b
LB		 1432 }
1433
1434 if (scrub_submit_raid56_bio_wait(fs_info, bio, first_page)) {
1435 bio_put(bio);
1436 goto out;
1437 }
1438
1439 bio_put(bio);
1440
1441 scrub_recheck_block_checksum(sblock);
1442
1443 return;
1444out:
1445 for (page_num = 0; page_num < sblock->page_count; page_num++)
1446 sblock->pagev[page_num]->io_error = 1;
1447
1448 sblock->no_io_error_seen = 0;
1449}
1450
b5d67f64
SB		 1451/*
1452 * this function will check the on disk data for checksum errors, header
1453 * errors and read I/O errors. If any I/O errors happen, the exact pages
1454 * which are errored are marked as being bad. The goal is to enable scrub
1455 * to take those pages that are not errored from all the mirrors so that
1456 * the pages that are errored in the just handled mirror can be repaired.
1457 */
34f5c8e9 1458static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
affe4a5a
ZL		 1459 struct scrub_block *sblock,
1460 int retry_failed_mirror)
96e36920 1461{
b5d67f64 1462 int page_num;
96e36920 1463
b5d67f64 1464 sblock->no_io_error_seen = 1;
96e36920 1465
6ca1765b
LB		 1466 /* short cut for raid56 */
1467 if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0]))
1468 return scrub_recheck_block_on_raid56(fs_info, sblock);
1469
b5d67f64
SB		 1470 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1471 struct bio *bio;
261d2dcb 1472 struct scrub_page *spage = sblock->pagev[page_num];
b5d67f64 1473
261d2dcb
QW		 1474 if (spage->dev->bdev == NULL) {
1475 spage->io_error = 1;
ea9947b4
SB		 1476 sblock->no_io_error_seen = 0;
1477 continue;
1478 }
1479
261d2dcb 1480 WARN_ON(!spage->page);
c3a3b19b 1481 bio = btrfs_bio_alloc(1);
261d2dcb 1482 bio_set_dev(bio, spage->dev->bdev);
b5d67f64 1483
8df507cb 1484 bio_add_page(bio, spage->page, fs_info->sectorsize, 0);
261d2dcb 1485 bio->bi_iter.bi_sector = spage->physical >> 9;
6ca1765b 1486 bio->bi_opf = REQ_OP_READ;
af8e2d1d 1487
6ca1765b 1488 if (btrfsic_submit_bio_wait(bio)) {
261d2dcb 1489 spage->io_error = 1;
6ca1765b 1490 sblock->no_io_error_seen = 0;
af8e2d1d 1491 }
33879d45 1492
b5d67f64
SB		 1493 bio_put(bio);
1494 }
96e36920 1495
b5d67f64 1496 if (sblock->no_io_error_seen)
ba7cf988 1497 scrub_recheck_block_checksum(sblock);
a2de733c
AJ		 1498}
1499
17a9be2f
MX		 1500static inline int scrub_check_fsid(u8 fsid[],
1501 struct scrub_page *spage)
1502{
1503 struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1504 int ret;
1505
44880fdc 1506 ret = memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
17a9be2f
MX		 1507 return !ret;
1508}
1509
ba7cf988 1510static void scrub_recheck_block_checksum(struct scrub_block *sblock)
a2de733c 1511{
ba7cf988
ZL		 1512 sblock->header_error = 0;
1513 sblock->checksum_error = 0;
1514 sblock->generation_error = 0;
b5d67f64 1515
ba7cf988
ZL		 1516 if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)
1517 scrub_checksum_data(sblock);
1518 else
1519 scrub_checksum_tree_block(sblock);
a2de733c
AJ		 1520}
1521
b5d67f64 1522static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
114ab50d 1523 struct scrub_block *sblock_good)
b5d67f64
SB		 1524{
1525 int page_num;
1526 int ret = 0;
96e36920 1527
b5d67f64
SB		 1528 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1529 int ret_sub;
96e36920 1530
b5d67f64
SB		 1531 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1532 sblock_good,
114ab50d 1533 page_num, 1);
b5d67f64
SB		 1534 if (ret_sub)
1535 ret = ret_sub;
a2de733c 1536 }
b5d67f64
SB		 1537
1538 return ret;
1539}
1540
1541static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1542 struct scrub_block *sblock_good,
1543 int page_num, int force_write)
1544{
261d2dcb
QW		 1545 struct scrub_page *spage_bad = sblock_bad->pagev[page_num];
1546 struct scrub_page *spage_good = sblock_good->pagev[page_num];
0b246afa 1547 struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info;
8df507cb 1548 const u32 sectorsize = fs_info->sectorsize;
b5d67f64 1549
261d2dcb
QW		 1550 BUG_ON(spage_bad->page == NULL);
1551 BUG_ON(spage_good->page == NULL);
b5d67f64 1552 if (force_write || sblock_bad->header_error ||
261d2dcb 1553 sblock_bad->checksum_error || spage_bad->io_error) {
b5d67f64
SB		 1554 struct bio *bio;
1555 int ret;
b5d67f64 1556
261d2dcb 1557 if (!spage_bad->dev->bdev) {
0b246afa 1558 btrfs_warn_rl(fs_info,
5d163e0e 1559 "scrub_repair_page_from_good_copy(bdev == NULL) is unexpected");
ff023aac
SB		 1560 return -EIO;
1561 }
1562
c3a3b19b 1563 bio = btrfs_bio_alloc(1);
261d2dcb
QW		 1564 bio_set_dev(bio, spage_bad->dev->bdev);
1565 bio->bi_iter.bi_sector = spage_bad->physical >> 9;
ebcc3263 1566 bio->bi_opf = REQ_OP_WRITE;
b5d67f64 1567
8df507cb
QW		 1568 ret = bio_add_page(bio, spage_good->page, sectorsize, 0);
1569 if (ret != sectorsize) {
b5d67f64
SB		 1570 bio_put(bio);
1571 return -EIO;
13db62b7 1572 }
b5d67f64 1573
4e49ea4a 1574 if (btrfsic_submit_bio_wait(bio)) {
261d2dcb 1575 btrfs_dev_stat_inc_and_print(spage_bad->dev,
442a4f63 1576 BTRFS_DEV_STAT_WRITE_ERRS);
e37abe97 1577 atomic64_inc(&fs_info->dev_replace.num_write_errors);
442a4f63
SB		 1578 bio_put(bio);
1579 return -EIO;
1580 }
b5d67f64 1581 bio_put(bio);
a2de733c
AJ		 1582 }
1583
b5d67f64
SB		 1584 return 0;
1585}
1586
ff023aac
SB		 1587static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1588{
0b246afa 1589 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
ff023aac
SB		 1590 int page_num;
1591
5a6ac9ea
MX		 1592 /*
1593 * This block is used for the check of the parity on the source device,
1594 * so the data needn't be written into the destination device.
1595 */
1596 if (sblock->sparity)
1597 return;
1598
ff023aac
SB		 1599 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1600 int ret;
1601
1602 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1603 if (ret)
e37abe97 1604 atomic64_inc(&fs_info->dev_replace.num_write_errors);
ff023aac
SB		 1605 }
1606}
1607
1608static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1609 int page_num)
1610{
1611 struct scrub_page *spage = sblock->pagev[page_num];
1612
1613 BUG_ON(spage->page == NULL);
a8b3a890
DS		 1614 if (spage->io_error)
1615 clear_page(page_address(spage->page));
ff023aac 1616
ff023aac
SB		 1617 return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1618}
1619
de17addc
NA		 1620static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical)
1621{
1622 int ret = 0;
1623 u64 length;
1624
1625 if (!btrfs_is_zoned(sctx->fs_info))
1626 return 0;
1627
7db1c5d1
NA		 1628 if (!btrfs_dev_is_sequential(sctx->wr_tgtdev, physical))
1629 return 0;
1630
de17addc
NA		 1631 if (sctx->write_pointer < physical) {
1632 length = physical - sctx->write_pointer;
1633
1634 ret = btrfs_zoned_issue_zeroout(sctx->wr_tgtdev,
1635 sctx->write_pointer, length);
1636 if (!ret)
1637 sctx->write_pointer = physical;
1638 }
1639 return ret;
1640}
1641
ff023aac
SB		 1642static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1643 struct scrub_page *spage)
1644{
ff023aac
SB		 1645 struct scrub_bio *sbio;
1646 int ret;
8df507cb 1647 const u32 sectorsize = sctx->fs_info->sectorsize;
ff023aac 1648
3fb99303 1649 mutex_lock(&sctx->wr_lock);
ff023aac 1650again:
3fb99303
DS		 1651 if (!sctx->wr_curr_bio) {
1652 sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio),
58c4e173 1653 GFP_KERNEL);
3fb99303
DS		 1654 if (!sctx->wr_curr_bio) {
1655 mutex_unlock(&sctx->wr_lock);
ff023aac
SB		 1656 return -ENOMEM;
1657 }
3fb99303
DS		 1658 sctx->wr_curr_bio->sctx = sctx;
1659 sctx->wr_curr_bio->page_count = 0;
ff023aac 1660 }
3fb99303 1661 sbio = sctx->wr_curr_bio;
ff023aac
SB		 1662 if (sbio->page_count == 0) {
1663 struct bio *bio;
1664
de17addc
NA		 1665 ret = fill_writer_pointer_gap(sctx,
1666 spage->physical_for_dev_replace);
1667 if (ret) {
1668 mutex_unlock(&sctx->wr_lock);
1669 return ret;
1670 }
1671
ff023aac
SB		 1672 sbio->physical = spage->physical_for_dev_replace;
1673 sbio->logical = spage->logical;
3fb99303 1674 sbio->dev = sctx->wr_tgtdev;
ff023aac
SB		 1675 bio = sbio->bio;
1676 if (!bio) {
c3a3b19b 1677 bio = btrfs_bio_alloc(sctx->pages_per_wr_bio);
ff023aac
SB		 1678 sbio->bio = bio;
1679 }
1680
1681 bio->bi_private = sbio;
1682 bio->bi_end_io = scrub_wr_bio_end_io;
74d46992 1683 bio_set_dev(bio, sbio->dev->bdev);
4f024f37 1684 bio->bi_iter.bi_sector = sbio->physical >> 9;
ebcc3263 1685 bio->bi_opf = REQ_OP_WRITE;
4e4cbee9 1686 sbio->status = 0;
8df507cb 1687 } else if (sbio->physical + sbio->page_count * sectorsize !=
ff023aac 1688 spage->physical_for_dev_replace ||
8df507cb 1689 sbio->logical + sbio->page_count * sectorsize !=
ff023aac
SB		 1690 spage->logical) {
1691 scrub_wr_submit(sctx);
1692 goto again;
1693 }
1694
8df507cb
QW		 1695 ret = bio_add_page(sbio->bio, spage->page, sectorsize, 0);
1696 if (ret != sectorsize) {
ff023aac
SB		 1697 if (sbio->page_count < 1) {
1698 bio_put(sbio->bio);
1699 sbio->bio = NULL;
3fb99303 1700 mutex_unlock(&sctx->wr_lock);
ff023aac
SB		 1701 return -EIO;
1702 }
1703 scrub_wr_submit(sctx);
1704 goto again;
1705 }
1706
1707 sbio->pagev[sbio->page_count] = spage;
1708 scrub_page_get(spage);
1709 sbio->page_count++;
3fb99303 1710 if (sbio->page_count == sctx->pages_per_wr_bio)
ff023aac 1711 scrub_wr_submit(sctx);
3fb99303 1712 mutex_unlock(&sctx->wr_lock);
ff023aac
SB		 1713
1714 return 0;
1715}
1716
1717static void scrub_wr_submit(struct scrub_ctx *sctx)
1718{
ff023aac
SB		 1719 struct scrub_bio *sbio;
1720
3fb99303 1721 if (!sctx->wr_curr_bio)
ff023aac
SB		 1722 return;
1723
3fb99303
DS		 1724 sbio = sctx->wr_curr_bio;
1725 sctx->wr_curr_bio = NULL;
309dca30 1726 WARN_ON(!sbio->bio->bi_bdev);
ff023aac
SB		 1727 scrub_pending_bio_inc(sctx);
1728 /* process all writes in a single worker thread. Then the block layer
1729 * orders the requests before sending them to the driver which
1730 * doubled the write performance on spinning disks when measured
1731 * with Linux 3.5 */
4e49ea4a 1732 btrfsic_submit_bio(sbio->bio);
de17addc
NA		 1733
1734 if (btrfs_is_zoned(sctx->fs_info))
8df507cb
QW		 1735 sctx->write_pointer = sbio->physical + sbio->page_count *
1736 sctx->fs_info->sectorsize;
ff023aac
SB		 1737}
1738
4246a0b6 1739static void scrub_wr_bio_end_io(struct bio *bio)
ff023aac
SB		 1740{
1741 struct scrub_bio *sbio = bio->bi_private;
fb456252 1742 struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
ff023aac 1743
4e4cbee9 1744 sbio->status = bio->bi_status;
ff023aac
SB		 1745 sbio->bio = bio;
1746
a0cac0ec 1747 btrfs_init_work(&sbio->work, scrub_wr_bio_end_io_worker, NULL, NULL);
0339ef2f 1748 btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
ff023aac
SB		 1749}
1750
1751static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1752{
1753 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1754 struct scrub_ctx *sctx = sbio->sctx;
1755 int i;
1756
1757 WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
4e4cbee9 1758 if (sbio->status) {
ff023aac 1759 struct btrfs_dev_replace *dev_replace =
fb456252 1760 &sbio->sctx->fs_info->dev_replace;
ff023aac
SB		 1761
1762 for (i = 0; i < sbio->page_count; i++) {
1763 struct scrub_page *spage = sbio->pagev[i];
1764
1765 spage->io_error = 1;
e37abe97 1766 atomic64_inc(&dev_replace->num_write_errors);
ff023aac
SB		 1767 }
1768 }
1769
1770 for (i = 0; i < sbio->page_count; i++)
1771 scrub_page_put(sbio->pagev[i]);
1772
1773 bio_put(sbio->bio);
1774 kfree(sbio);
1775 scrub_pending_bio_dec(sctx);
1776}
1777
1778static int scrub_checksum(struct scrub_block *sblock)
b5d67f64
SB		 1779{
1780 u64 flags;
1781 int ret;
1782
ba7cf988
ZL		 1783 /*
1784 * No need to initialize these stats currently,
1785 * because this function only use return value
1786 * instead of these stats value.
1787 *
1788 * Todo:
1789 * always use stats
1790 */
1791 sblock->header_error = 0;
1792 sblock->generation_error = 0;
1793 sblock->checksum_error = 0;
1794
7a9e9987
SB		 1795 WARN_ON(sblock->page_count < 1);
1796 flags = sblock->pagev[0]->flags;
b5d67f64
SB		 1797 ret = 0;
1798 if (flags & BTRFS_EXTENT_FLAG_DATA)
1799 ret = scrub_checksum_data(sblock);
1800 else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1801 ret = scrub_checksum_tree_block(sblock);
1802 else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1803 (void)scrub_checksum_super(sblock);
1804 else
1805 WARN_ON(1);
1806 if (ret)
1807 scrub_handle_errored_block(sblock);
ff023aac
SB		 1808
1809 return ret;
a2de733c
AJ		 1810}
1811
b5d67f64 1812static int scrub_checksum_data(struct scrub_block *sblock)
a2de733c 1813{
d9d181c1 1814 struct scrub_ctx *sctx = sblock->sctx;
d5178578
JT		 1815 struct btrfs_fs_info *fs_info = sctx->fs_info;
1816 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
a2de733c 1817 u8 csum[BTRFS_CSUM_SIZE];
d41ebef2 1818 struct scrub_page *spage;
b0485252 1819 char *kaddr;
a2de733c 1820
b5d67f64 1821 BUG_ON(sblock->page_count < 1);
d41ebef2
DS		 1822 spage = sblock->pagev[0];
1823 if (!spage->have_csum)
a2de733c
AJ		 1824 return 0;
1825
d41ebef2 1826 kaddr = page_address(spage->page);
b5d67f64 1827
771aba0d
DS		 1828 shash->tfm = fs_info->csum_shash;
1829 crypto_shash_init(shash);
b5d67f64 1830
b29dca44
QW		 1831 /*
1832 * In scrub_pages() and scrub_pages_for_parity() we ensure each spage
1833 * only contains one sector of data.
1834 */
1835 crypto_shash_digest(shash, kaddr, fs_info->sectorsize, csum);
a2de733c 1836
b29dca44
QW		 1837 if (memcmp(csum, spage->csum, fs_info->csum_size))
1838 sblock->checksum_error = 1;
ba7cf988 1839 return sblock->checksum_error;
a2de733c
AJ		 1840}
1841
b5d67f64 1842static int scrub_checksum_tree_block(struct scrub_block *sblock)
a2de733c 1843{
d9d181c1 1844 struct scrub_ctx *sctx = sblock->sctx;
a2de733c 1845 struct btrfs_header *h;
0b246afa 1846 struct btrfs_fs_info *fs_info = sctx->fs_info;
d5178578 1847 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
b5d67f64
SB		 1848 u8 calculated_csum[BTRFS_CSUM_SIZE];
1849 u8 on_disk_csum[BTRFS_CSUM_SIZE];
53f3251d
QW		 1850 /*
1851 * This is done in sectorsize steps even for metadata as there's a
1852 * constraint for nodesize to be aligned to sectorsize. This will need
1853 * to change so we don't misuse data and metadata units like that.
1854 */
1855 const u32 sectorsize = sctx->fs_info->sectorsize;
1856 const int num_sectors = fs_info->nodesize >> fs_info->sectorsize_bits;
521e1022 1857 int i;
100aa5d9 1858 struct scrub_page *spage;
b0485252 1859 char *kaddr;
d5178578 1860
b5d67f64 1861 BUG_ON(sblock->page_count < 1);
53f3251d
QW		 1862
1863 /* Each member in pagev is just one block, not a full page */
1864 ASSERT(sblock->page_count == num_sectors);
1865
100aa5d9
DS		 1866 spage = sblock->pagev[0];
1867 kaddr = page_address(spage->page);
b0485252 1868 h = (struct btrfs_header *)kaddr;
2ae0c2d8 1869 memcpy(on_disk_csum, h->csum, sctx->fs_info->csum_size);
a2de733c
AJ		 1870
1871 /*
1872 * we don't use the getter functions here, as we
1873 * a) don't have an extent buffer and
1874 * b) the page is already kmapped
1875 */
100aa5d9 1876 if (spage->logical != btrfs_stack_header_bytenr(h))
ba7cf988 1877 sblock->header_error = 1;
a2de733c 1878
100aa5d9 1879 if (spage->generation != btrfs_stack_header_generation(h)) {
ba7cf988
ZL		 1880 sblock->header_error = 1;
1881 sblock->generation_error = 1;
1882 }
a2de733c 1883
100aa5d9 1884 if (!scrub_check_fsid(h->fsid, spage))
ba7cf988 1885 sblock->header_error = 1;
a2de733c
AJ		 1886
1887 if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1888 BTRFS_UUID_SIZE))
ba7cf988 1889 sblock->header_error = 1;
a2de733c 1890
521e1022
DS		 1891 shash->tfm = fs_info->csum_shash;
1892 crypto_shash_init(shash);
1893 crypto_shash_update(shash, kaddr + BTRFS_CSUM_SIZE,
53f3251d 1894 sectorsize - BTRFS_CSUM_SIZE);
b5d67f64 1895
53f3251d 1896 for (i = 1; i < num_sectors; i++) {
521e1022 1897 kaddr = page_address(sblock->pagev[i]->page);
53f3251d 1898 crypto_shash_update(shash, kaddr, sectorsize);
b5d67f64
SB		 1899 }
1900
d5178578 1901 crypto_shash_final(shash, calculated_csum);
2ae0c2d8 1902 if (memcmp(calculated_csum, on_disk_csum, sctx->fs_info->csum_size))
ba7cf988 1903 sblock->checksum_error = 1;
a2de733c 1904
ba7cf988 1905 return sblock->header_error || sblock->checksum_error;
a2de733c
AJ		 1906}
1907
b5d67f64 1908static int scrub_checksum_super(struct scrub_block *sblock)
a2de733c
AJ		 1909{
1910 struct btrfs_super_block *s;
d9d181c1 1911 struct scrub_ctx *sctx = sblock->sctx;
d5178578
JT		 1912 struct btrfs_fs_info *fs_info = sctx->fs_info;
1913 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
b5d67f64 1914 u8 calculated_csum[BTRFS_CSUM_SIZE];
c7460541 1915 struct scrub_page *spage;
b0485252 1916 char *kaddr;
442a4f63
SB		 1917 int fail_gen = 0;
1918 int fail_cor = 0;
d5178578 1919
b5d67f64 1920 BUG_ON(sblock->page_count < 1);
c7460541
DS		 1921 spage = sblock->pagev[0];
1922 kaddr = page_address(spage->page);
b0485252 1923 s = (struct btrfs_super_block *)kaddr;
a2de733c 1924
c7460541 1925 if (spage->logical != btrfs_super_bytenr(s))
442a4f63 1926 ++fail_cor;
a2de733c 1927
c7460541 1928 if (spage->generation != btrfs_super_generation(s))
442a4f63 1929 ++fail_gen;
a2de733c 1930
c7460541 1931 if (!scrub_check_fsid(s->fsid, spage))
442a4f63 1932 ++fail_cor;
a2de733c 1933
83cf6d5e
DS		 1934 shash->tfm = fs_info->csum_shash;
1935 crypto_shash_init(shash);
1936 crypto_shash_digest(shash, kaddr + BTRFS_CSUM_SIZE,
1937 BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, calculated_csum);
b5d67f64 1938
2ae0c2d8 1939 if (memcmp(calculated_csum, s->csum, sctx->fs_info->csum_size))
442a4f63 1940 ++fail_cor;
a2de733c 1941
442a4f63 1942 if (fail_cor + fail_gen) {
a2de733c
AJ		 1943 /*
1944 * if we find an error in a super block, we just report it.
1945 * They will get written with the next transaction commit
1946 * anyway
1947 */
d9d181c1
SB		 1948 spin_lock(&sctx->stat_lock);
1949 ++sctx->stat.super_errors;
1950 spin_unlock(&sctx->stat_lock);
442a4f63 1951 if (fail_cor)
c7460541 1952 btrfs_dev_stat_inc_and_print(spage->dev,
442a4f63
SB		 1953 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1954 else
c7460541 1955 btrfs_dev_stat_inc_and_print(spage->dev,
442a4f63 1956 BTRFS_DEV_STAT_GENERATION_ERRS);
a2de733c
AJ		 1957 }
1958
442a4f63 1959 return fail_cor + fail_gen;
a2de733c
AJ		 1960}
1961
b5d67f64
SB		 1962static void scrub_block_get(struct scrub_block *sblock)
1963{
186debd6 1964 refcount_inc(&sblock->refs);
b5d67f64
SB		 1965}
1966
1967static void scrub_block_put(struct scrub_block *sblock)
1968{
186debd6 1969 if (refcount_dec_and_test(&sblock->refs)) {
b5d67f64
SB		 1970 int i;
1971
5a6ac9ea
MX		 1972 if (sblock->sparity)
1973 scrub_parity_put(sblock->sparity);
1974
b5d67f64 1975 for (i = 0; i < sblock->page_count; i++)
7a9e9987 1976 scrub_page_put(sblock->pagev[i]);
b5d67f64
SB		 1977 kfree(sblock);
1978 }
1979}
1980
7a9e9987
SB		 1981static void scrub_page_get(struct scrub_page *spage)
1982{
57019345 1983 atomic_inc(&spage->refs);
7a9e9987
SB		 1984}
1985
1986static void scrub_page_put(struct scrub_page *spage)
1987{
57019345 1988 if (atomic_dec_and_test(&spage->refs)) {
7a9e9987
SB		 1989 if (spage->page)
1990 __free_page(spage->page);
1991 kfree(spage);
1992 }
1993}
1994
eb3b5053
DS		 1995/*
1996 * Throttling of IO submission, bandwidth-limit based, the timeslice is 1
1997 * second. Limit can be set via /sys/fs/UUID/devinfo/devid/scrub_speed_max.
1998 */
1999static void scrub_throttle(struct scrub_ctx *sctx)
2000{
2001 const int time_slice = 1000;
2002 struct scrub_bio *sbio;
2003 struct btrfs_device *device;
2004 s64 delta;
2005 ktime_t now;
2006 u32 div;
2007 u64 bwlimit;
2008
2009 sbio = sctx->bios[sctx->curr];
2010 device = sbio->dev;
2011 bwlimit = READ_ONCE(device->scrub_speed_max);
2012 if (bwlimit == 0)
2013 return;
2014
2015 /*
2016 * Slice is divided into intervals when the IO is submitted, adjust by
2017 * bwlimit and maximum of 64 intervals.
2018 */
2019 div = max_t(u32, 1, (u32)(bwlimit / (16 * 1024 * 1024)));
2020 div = min_t(u32, 64, div);
2021
2022 /* Start new epoch, set deadline */
2023 now = ktime_get();
2024 if (sctx->throttle_deadline == 0) {
2025 sctx->throttle_deadline = ktime_add_ms(now, time_slice / div);
2026 sctx->throttle_sent = 0;
2027 }
2028
2029 /* Still in the time to send? */
2030 if (ktime_before(now, sctx->throttle_deadline)) {
2031 /* If current bio is within the limit, send it */
2032 sctx->throttle_sent += sbio->bio->bi_iter.bi_size;
2033 if (sctx->throttle_sent <= div_u64(bwlimit, div))
2034 return;
2035
2036 /* We're over the limit, sleep until the rest of the slice */
2037 delta = ktime_ms_delta(sctx->throttle_deadline, now);
2038 } else {
2039 /* New request after deadline, start new epoch */
2040 delta = 0;
2041 }
2042
2043 if (delta) {
2044 long timeout;
2045
2046 timeout = div_u64(delta * HZ, 1000);
2047 schedule_timeout_interruptible(timeout);
2048 }
2049
2050 /* Next call will start the deadline period */
2051 sctx->throttle_deadline = 0;
2052}
2053
d9d181c1 2054static void scrub_submit(struct scrub_ctx *sctx)
a2de733c
AJ		 2055{
2056 struct scrub_bio *sbio;
2057
d9d181c1 2058 if (sctx->curr == -1)
1623edeb 2059 return;
a2de733c 2060
eb3b5053
DS		 2061 scrub_throttle(sctx);
2062
d9d181c1
SB		 2063 sbio = sctx->bios[sctx->curr];
2064 sctx->curr = -1;
b6bfebc1 2065 scrub_pending_bio_inc(sctx);
4e49ea4a 2066 btrfsic_submit_bio(sbio->bio);
a2de733c
AJ		 2067}
2068
ff023aac
SB		 2069static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
2070 struct scrub_page *spage)
a2de733c 2071{
b5d67f64 2072 struct scrub_block *sblock = spage->sblock;
a2de733c 2073 struct scrub_bio *sbio;
8df507cb 2074 const u32 sectorsize = sctx->fs_info->sectorsize;
69f4cb52 2075 int ret;
a2de733c
AJ		 2076
2077again:
2078 /*
2079 * grab a fresh bio or wait for one to become available
2080 */
d9d181c1
SB		 2081 while (sctx->curr == -1) {
2082 spin_lock(&sctx->list_lock);
2083 sctx->curr = sctx->first_free;
2084 if (sctx->curr != -1) {
2085 sctx->first_free = sctx->bios[sctx->curr]->next_free;
2086 sctx->bios[sctx->curr]->next_free = -1;
2087 sctx->bios[sctx->curr]->page_count = 0;
2088 spin_unlock(&sctx->list_lock);
a2de733c 2089 } else {
d9d181c1
SB		 2090 spin_unlock(&sctx->list_lock);
2091 wait_event(sctx->list_wait, sctx->first_free != -1);
a2de733c
AJ		 2092 }
2093 }
d9d181c1 2094 sbio = sctx->bios[sctx->curr];
b5d67f64 2095 if (sbio->page_count == 0) {
69f4cb52
AJ		 2096 struct bio *bio;
2097
b5d67f64
SB		 2098 sbio->physical = spage->physical;
2099 sbio->logical = spage->logical;
a36cf8b8 2100 sbio->dev = spage->dev;
b5d67f64
SB		 2101 bio = sbio->bio;
2102 if (!bio) {
c3a3b19b 2103 bio = btrfs_bio_alloc(sctx->pages_per_rd_bio);
b5d67f64
SB		 2104 sbio->bio = bio;
2105 }
69f4cb52
AJ		 2106
2107 bio->bi_private = sbio;
2108 bio->bi_end_io = scrub_bio_end_io;
74d46992 2109 bio_set_dev(bio, sbio->dev->bdev);
4f024f37 2110 bio->bi_iter.bi_sector = sbio->physical >> 9;
ebcc3263 2111 bio->bi_opf = REQ_OP_READ;
4e4cbee9 2112 sbio->status = 0;
8df507cb 2113 } else if (sbio->physical + sbio->page_count * sectorsize !=
b5d67f64 2114 spage->physical ||
8df507cb 2115 sbio->logical + sbio->page_count * sectorsize !=
a36cf8b8
SB		 2116 spage->logical ||
2117 sbio->dev != spage->dev) {
d9d181c1 2118 scrub_submit(sctx);
a2de733c
AJ		 2119 goto again;
2120 }
69f4cb52 2121
b5d67f64 2122 sbio->pagev[sbio->page_count] = spage;
8df507cb
QW		 2123 ret = bio_add_page(sbio->bio, spage->page, sectorsize, 0);
2124 if (ret != sectorsize) {
b5d67f64
SB		 2125 if (sbio->page_count < 1) {
2126 bio_put(sbio->bio);
2127 sbio->bio = NULL;
2128 return -EIO;
2129 }
d9d181c1 2130 scrub_submit(sctx);
69f4cb52
AJ		 2131 goto again;
2132 }
2133
ff023aac 2134 scrub_block_get(sblock); /* one for the page added to the bio */
b5d67f64
SB		 2135 atomic_inc(&sblock->outstanding_pages);
2136 sbio->page_count++;
ff023aac 2137 if (sbio->page_count == sctx->pages_per_rd_bio)
d9d181c1 2138 scrub_submit(sctx);
b5d67f64
SB		 2139
2140 return 0;
2141}
2142
22365979 2143static void scrub_missing_raid56_end_io(struct bio *bio)
73ff61db
OS		 2144{
2145 struct scrub_block *sblock = bio->bi_private;
fb456252 2146 struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
73ff61db 2147
4e4cbee9 2148 if (bio->bi_status)
73ff61db
OS		 2149 sblock->no_io_error_seen = 0;
2150
4673272f
ST		 2151 bio_put(bio);
2152
73ff61db
OS		 2153 btrfs_queue_work(fs_info->scrub_workers, &sblock->work);
2154}
2155
2156static void scrub_missing_raid56_worker(struct btrfs_work *work)
2157{
2158 struct scrub_block *sblock = container_of(work, struct scrub_block, work);
2159 struct scrub_ctx *sctx = sblock->sctx;
0b246afa 2160 struct btrfs_fs_info *fs_info = sctx->fs_info;
73ff61db
OS		 2161 u64 logical;
2162 struct btrfs_device *dev;
2163
73ff61db
OS		 2164 logical = sblock->pagev[0]->logical;
2165 dev = sblock->pagev[0]->dev;
2166
affe4a5a 2167 if (sblock->no_io_error_seen)
ba7cf988 2168 scrub_recheck_block_checksum(sblock);
73ff61db
OS		 2169
2170 if (!sblock->no_io_error_seen) {
2171 spin_lock(&sctx->stat_lock);
2172 sctx->stat.read_errors++;
2173 spin_unlock(&sctx->stat_lock);
0b246afa 2174 btrfs_err_rl_in_rcu(fs_info,
b14af3b4 2175 "IO error rebuilding logical %llu for dev %s",
73ff61db
OS		 2176 logical, rcu_str_deref(dev->name));
2177 } else if (sblock->header_error || sblock->checksum_error) {
2178 spin_lock(&sctx->stat_lock);
2179 sctx->stat.uncorrectable_errors++;
2180 spin_unlock(&sctx->stat_lock);
0b246afa 2181 btrfs_err_rl_in_rcu(fs_info,
b14af3b4 2182 "failed to rebuild valid logical %llu for dev %s",
73ff61db
OS		 2183 logical, rcu_str_deref(dev->name));
2184 } else {
2185 scrub_write_block_to_dev_replace(sblock);
2186 }
2187
2073c4c2 2188 if (sctx->is_dev_replace && sctx->flush_all_writes) {
3fb99303 2189 mutex_lock(&sctx->wr_lock);
73ff61db 2190 scrub_wr_submit(sctx);
3fb99303 2191 mutex_unlock(&sctx->wr_lock);
73ff61db
OS		 2192 }
2193
57d4f0b8 2194 scrub_block_put(sblock);
73ff61db
OS		 2195 scrub_pending_bio_dec(sctx);
2196}
2197
2198static void scrub_missing_raid56_pages(struct scrub_block *sblock)
2199{
2200 struct scrub_ctx *sctx = sblock->sctx;
fb456252 2201 struct btrfs_fs_info *fs_info = sctx->fs_info;
73ff61db
OS		 2202 u64 length = sblock->page_count * PAGE_SIZE;
2203 u64 logical = sblock->pagev[0]->logical;
4c664611 2204 struct btrfs_io_context *bioc = NULL;
73ff61db
OS		 2205 struct bio *bio;
2206 struct btrfs_raid_bio *rbio;
2207 int ret;
2208 int i;
2209
ae6529c3 2210 btrfs_bio_counter_inc_blocked(fs_info);
cf8cddd3 2211 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical,
4c664611
QW		 2212 &length, &bioc);
2213 if (ret || !bioc || !bioc->raid_map)
2214 goto bioc_out;
73ff61db
OS		 2215
2216 if (WARN_ON(!sctx->is_dev_replace ||
4c664611 2217 !(bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) {
73ff61db
OS		 2218 /*
2219 * We shouldn't be scrubbing a missing device. Even for dev
2220 * replace, we should only get here for RAID 5/6. We either
2221 * managed to mount something with no mirrors remaining or
2222 * there's a bug in scrub_remap_extent()/btrfs_map_block().
2223 */
4c664611 2224 goto bioc_out;
73ff61db
OS		 2225 }
2226
c3a3b19b 2227 bio = btrfs_bio_alloc(BIO_MAX_VECS);
73ff61db
OS		 2228 bio->bi_iter.bi_sector = logical >> 9;
2229 bio->bi_private = sblock;
2230 bio->bi_end_io = scrub_missing_raid56_end_io;
2231
6a258d72 2232 rbio = raid56_alloc_missing_rbio(bio, bioc, length);
73ff61db
OS		 2233 if (!rbio)
2234 goto rbio_out;
2235
2236 for (i = 0; i < sblock->page_count; i++) {
2237 struct scrub_page *spage = sblock->pagev[i];
2238
2239 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2240 }
2241
a0cac0ec 2242 btrfs_init_work(&sblock->work, scrub_missing_raid56_worker, NULL, NULL);
73ff61db
OS		 2243 scrub_block_get(sblock);
2244 scrub_pending_bio_inc(sctx);
2245 raid56_submit_missing_rbio(rbio);
2246 return;
2247
2248rbio_out:
2249 bio_put(bio);
4c664611 2250bioc_out:
ae6529c3 2251 btrfs_bio_counter_dec(fs_info);
4c664611 2252 btrfs_put_bioc(bioc);
73ff61db
OS		 2253 spin_lock(&sctx->stat_lock);
2254 sctx->stat.malloc_errors++;
2255 spin_unlock(&sctx->stat_lock);
2256}
2257
fa485d21 2258static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u32 len,
a36cf8b8 2259 u64 physical, struct btrfs_device *dev, u64 flags,
96e63a45 2260 u64 gen, int mirror_num, u8 *csum,
ff023aac 2261 u64 physical_for_dev_replace)
b5d67f64
SB		 2262{
2263 struct scrub_block *sblock;
d0a7a9c0 2264 const u32 sectorsize = sctx->fs_info->sectorsize;
b5d67f64
SB		 2265 int index;
2266
58c4e173 2267 sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
b5d67f64 2268 if (!sblock) {
d9d181c1
SB		 2269 spin_lock(&sctx->stat_lock);
2270 sctx->stat.malloc_errors++;
2271 spin_unlock(&sctx->stat_lock);
b5d67f64 2272 return -ENOMEM;
a2de733c 2273 }
b5d67f64 2274
7a9e9987
SB		 2275 /* one ref inside this function, plus one for each page added to
2276 * a bio later on */
186debd6 2277 refcount_set(&sblock->refs, 1);
d9d181c1 2278 sblock->sctx = sctx;
b5d67f64
SB		 2279 sblock->no_io_error_seen = 1;
2280
2281 for (index = 0; len > 0; index++) {
7a9e9987 2282 struct scrub_page *spage;
d0a7a9c0
QW		 2283 /*
2284 * Here we will allocate one page for one sector to scrub.
2285 * This is fine if PAGE_SIZE == sectorsize, but will cost
2286 * more memory for PAGE_SIZE > sectorsize case.
2287 */
2288 u32 l = min(sectorsize, len);
b5d67f64 2289
58c4e173 2290 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
7a9e9987
SB		 2291 if (!spage) {
2292leave_nomem:
d9d181c1
SB		 2293 spin_lock(&sctx->stat_lock);
2294 sctx->stat.malloc_errors++;
2295 spin_unlock(&sctx->stat_lock);
7a9e9987 2296 scrub_block_put(sblock);
b5d67f64
SB		 2297 return -ENOMEM;
2298 }
0bb3acdc 2299 ASSERT(index < SCRUB_MAX_PAGES_PER_BLOCK);
7a9e9987
SB		 2300 scrub_page_get(spage);
2301 sblock->pagev[index] = spage;
b5d67f64 2302 spage->sblock = sblock;
a36cf8b8 2303 spage->dev = dev;
b5d67f64
SB		 2304 spage->flags = flags;
2305 spage->generation = gen;
2306 spage->logical = logical;
2307 spage->physical = physical;
ff023aac 2308 spage->physical_for_dev_replace = physical_for_dev_replace;
b5d67f64
SB		 2309 spage->mirror_num = mirror_num;
2310 if (csum) {
2311 spage->have_csum = 1;
2ae0c2d8 2312 memcpy(spage->csum, csum, sctx->fs_info->csum_size);
b5d67f64
SB		 2313 } else {
2314 spage->have_csum = 0;
2315 }
2316 sblock->page_count++;
58c4e173 2317 spage->page = alloc_page(GFP_KERNEL);
7a9e9987
SB		 2318 if (!spage->page)
2319 goto leave_nomem;
b5d67f64
SB		 2320 len -= l;
2321 logical += l;
2322 physical += l;
ff023aac 2323 physical_for_dev_replace += l;
b5d67f64
SB		 2324 }
2325
7a9e9987 2326 WARN_ON(sblock->page_count == 0);
e6e674bd 2327 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
73ff61db
OS		 2328 /*
2329 * This case should only be hit for RAID 5/6 device replace. See
2330 * the comment in scrub_missing_raid56_pages() for details.
2331 */
2332 scrub_missing_raid56_pages(sblock);
2333 } else {
2334 for (index = 0; index < sblock->page_count; index++) {
2335 struct scrub_page *spage = sblock->pagev[index];
2336 int ret;
1bc87793 2337
73ff61db
OS		 2338 ret = scrub_add_page_to_rd_bio(sctx, spage);
2339 if (ret) {
2340 scrub_block_put(sblock);
2341 return ret;
2342 }
b5d67f64 2343 }
a2de733c 2344
96e63a45 2345 if (flags & BTRFS_EXTENT_FLAG_SUPER)
73ff61db
OS		 2346 scrub_submit(sctx);
2347 }
a2de733c 2348
b5d67f64
SB		 2349 /* last one frees, either here or in bio completion for last page */
2350 scrub_block_put(sblock);
a2de733c
AJ		 2351 return 0;
2352}
2353
4246a0b6 2354static void scrub_bio_end_io(struct bio *bio)
b5d67f64
SB		 2355{
2356 struct scrub_bio *sbio = bio->bi_private;
fb456252 2357 struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
b5d67f64 2358
4e4cbee9 2359 sbio->status = bio->bi_status;
b5d67f64
SB		 2360 sbio->bio = bio;
2361
0339ef2f 2362 btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
b5d67f64
SB		 2363}
2364
2365static void scrub_bio_end_io_worker(struct btrfs_work *work)
2366{
2367 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
d9d181c1 2368 struct scrub_ctx *sctx = sbio->sctx;
b5d67f64
SB		 2369 int i;
2370
ff023aac 2371 BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
4e4cbee9 2372 if (sbio->status) {
b5d67f64
SB		 2373 for (i = 0; i < sbio->page_count; i++) {
2374 struct scrub_page *spage = sbio->pagev[i];
2375
2376 spage->io_error = 1;
2377 spage->sblock->no_io_error_seen = 0;
2378 }
2379 }
2380
2381 /* now complete the scrub_block items that have all pages completed */
2382 for (i = 0; i < sbio->page_count; i++) {
2383 struct scrub_page *spage = sbio->pagev[i];
2384 struct scrub_block *sblock = spage->sblock;
2385
2386 if (atomic_dec_and_test(&sblock->outstanding_pages))
2387 scrub_block_complete(sblock);
2388 scrub_block_put(sblock);
2389 }
2390
b5d67f64
SB		 2391 bio_put(sbio->bio);
2392 sbio->bio = NULL;
d9d181c1
SB		 2393 spin_lock(&sctx->list_lock);
2394 sbio->next_free = sctx->first_free;
2395 sctx->first_free = sbio->index;
2396 spin_unlock(&sctx->list_lock);
ff023aac 2397
2073c4c2 2398 if (sctx->is_dev_replace && sctx->flush_all_writes) {
3fb99303 2399 mutex_lock(&sctx->wr_lock);
ff023aac 2400 scrub_wr_submit(sctx);
3fb99303 2401 mutex_unlock(&sctx->wr_lock);
ff023aac
SB		 2402 }
2403
b6bfebc1 2404 scrub_pending_bio_dec(sctx);
b5d67f64
SB		 2405}
2406
5a6ac9ea
MX		 2407static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2408 unsigned long *bitmap,
fa485d21 2409 u64 start, u32 len)
5a6ac9ea 2410{
972d7219 2411 u64 offset;
7736b0a4 2412 u32 nsectors;
ab108d99 2413 u32 sectorsize_bits = sparity->sctx->fs_info->sectorsize_bits;
5a6ac9ea
MX		 2414
2415 if (len >= sparity->stripe_len) {
2416 bitmap_set(bitmap, 0, sparity->nsectors);
2417 return;
2418 }
2419
2420 start -= sparity->logic_start;
972d7219 2421 start = div64_u64_rem(start, sparity->stripe_len, &offset);
ab108d99 2422 offset = offset >> sectorsize_bits;
fa485d21 2423 nsectors = len >> sectorsize_bits;
5a6ac9ea
MX		 2424
2425 if (offset + nsectors <= sparity->nsectors) {
2426 bitmap_set(bitmap, offset, nsectors);
2427 return;
2428 }
2429
2430 bitmap_set(bitmap, offset, sparity->nsectors - offset);
2431 bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2432}
2433
2434static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
fa485d21 2435 u64 start, u32 len)
5a6ac9ea
MX		 2436{
2437 __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2438}
2439
2440static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
fa485d21 2441 u64 start, u32 len)
5a6ac9ea
MX		 2442{
2443 __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2444}
2445
b5d67f64
SB		 2446static void scrub_block_complete(struct scrub_block *sblock)
2447{
5a6ac9ea
MX		 2448 int corrupted = 0;
2449
ff023aac 2450 if (!sblock->no_io_error_seen) {
5a6ac9ea 2451 corrupted = 1;
b5d67f64 2452 scrub_handle_errored_block(sblock);
ff023aac
SB		 2453 } else {
2454 /*
2455 * if has checksum error, write via repair mechanism in
2456 * dev replace case, otherwise write here in dev replace
2457 * case.
2458 */
5a6ac9ea
MX		 2459 corrupted = scrub_checksum(sblock);
2460 if (!corrupted && sblock->sctx->is_dev_replace)
ff023aac
SB		 2461 scrub_write_block_to_dev_replace(sblock);
2462 }
5a6ac9ea
MX		 2463
2464 if (sblock->sparity && corrupted && !sblock->data_corrected) {
2465 u64 start = sblock->pagev[0]->logical;
2466 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
8df507cb 2467 sblock->sctx->fs_info->sectorsize;
5a6ac9ea 2468
fa485d21 2469 ASSERT(end - start <= U32_MAX);
5a6ac9ea
MX		 2470 scrub_parity_mark_sectors_error(sblock->sparity,
2471 start, end - start);
2472 }
b5d67f64
SB		 2473}
2474
480a8ec8
QW		 2475static void drop_csum_range(struct scrub_ctx *sctx, struct btrfs_ordered_sum *sum)
2476{
2477 sctx->stat.csum_discards += sum->len >> sctx->fs_info->sectorsize_bits;
2478 list_del(&sum->list);
2479 kfree(sum);
2480}
2481
2482/*
2483 * Find the desired csum for range [logical, logical + sectorsize), and store
2484 * the csum into @csum.
2485 *
2486 * The search source is sctx->csum_list, which is a pre-populated list
1a9fd417 2487 * storing bytenr ordered csum ranges. We're responsible to cleanup any range
480a8ec8
QW		 2488 * that is before @logical.
2489 *
2490 * Return 0 if there is no csum for the range.
2491 * Return 1 if there is csum for the range and copied to @csum.
2492 */
3b5753ec 2493static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum)
a2de733c 2494{
480a8ec8 2495 bool found = false;
a2de733c 2496
d9d181c1 2497 while (!list_empty(&sctx->csum_list)) {
480a8ec8
QW		 2498 struct btrfs_ordered_sum *sum = NULL;
2499 unsigned long index;
2500 unsigned long num_sectors;
2501
d9d181c1 2502 sum = list_first_entry(&sctx->csum_list,
a2de733c 2503 struct btrfs_ordered_sum, list);
480a8ec8 2504 /* The current csum range is beyond our range, no csum found */
a2de733c 2505 if (sum->bytenr > logical)
a2de733c
AJ		 2506 break;
2507
480a8ec8
QW		 2508 /*
2509 * The current sum is before our bytenr, since scrub is always
2510 * done in bytenr order, the csum will never be used anymore,
2511 * clean it up so that later calls won't bother with the range,
2512 * and continue search the next range.
2513 */
2514 if (sum->bytenr + sum->len <= logical) {
2515 drop_csum_range(sctx, sum);
2516 continue;
2517 }
a2de733c 2518
480a8ec8
QW		 2519 /* Now the csum range covers our bytenr, copy the csum */
2520 found = true;
2521 index = (logical - sum->bytenr) >> sctx->fs_info->sectorsize_bits;
2522 num_sectors = sum->len >> sctx->fs_info->sectorsize_bits;
1d1bf92d 2523
480a8ec8
QW		 2524 memcpy(csum, sum->sums + index * sctx->fs_info->csum_size,
2525 sctx->fs_info->csum_size);
2526
2527 /* Cleanup the range if we're at the end of the csum range */
2528 if (index == num_sectors - 1)
2529 drop_csum_range(sctx, sum);
2530 break;
a2de733c 2531 }
480a8ec8
QW		 2532 if (!found)
2533 return 0;
f51a4a18 2534 return 1;
a2de733c
AJ		 2535}
2536
2537/* scrub extent tries to collect up to 64 kB for each bio */
6ca1765b 2538static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map,
fa485d21 2539 u64 logical, u32 len,
a36cf8b8 2540 u64 physical, struct btrfs_device *dev, u64 flags,
ff023aac 2541 u64 gen, int mirror_num, u64 physical_for_dev_replace)
a2de733c
AJ		 2542{
2543 int ret;
2544 u8 csum[BTRFS_CSUM_SIZE];
b5d67f64
SB		 2545 u32 blocksize;
2546
2547 if (flags & BTRFS_EXTENT_FLAG_DATA) {
6ca1765b
LB		 2548 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
2549 blocksize = map->stripe_len;
2550 else
2551 blocksize = sctx->fs_info->sectorsize;
d9d181c1
SB		 2552 spin_lock(&sctx->stat_lock);
2553 sctx->stat.data_extents_scrubbed++;
2554 sctx->stat.data_bytes_scrubbed += len;
2555 spin_unlock(&sctx->stat_lock);
b5d67f64 2556 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
6ca1765b
LB		 2557 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
2558 blocksize = map->stripe_len;
2559 else
2560 blocksize = sctx->fs_info->nodesize;
d9d181c1
SB		 2561 spin_lock(&sctx->stat_lock);
2562 sctx->stat.tree_extents_scrubbed++;
2563 sctx->stat.tree_bytes_scrubbed += len;
2564 spin_unlock(&sctx->stat_lock);
b5d67f64 2565 } else {
25cc1226 2566 blocksize = sctx->fs_info->sectorsize;
ff023aac 2567 WARN_ON(1);
b5d67f64 2568 }
a2de733c
AJ		 2569
2570 while (len) {
fa485d21 2571 u32 l = min(len, blocksize);
a2de733c
AJ		 2572 int have_csum = 0;
2573
2574 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2575 /* push csums to sbio */
3b5753ec 2576 have_csum = scrub_find_csum(sctx, logical, csum);
a2de733c 2577 if (have_csum == 0)
d9d181c1 2578 ++sctx->stat.no_csum;
a2de733c 2579 }
a36cf8b8 2580 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
96e63a45 2581 mirror_num, have_csum ? csum : NULL,
ff023aac 2582 physical_for_dev_replace);
a2de733c
AJ		 2583 if (ret)
2584 return ret;
2585 len -= l;
2586 logical += l;
2587 physical += l;
ff023aac 2588 physical_for_dev_replace += l;
a2de733c
AJ		 2589 }
2590 return 0;
2591}
2592
5a6ac9ea 2593static int scrub_pages_for_parity(struct scrub_parity *sparity,
fa485d21 2594 u64 logical, u32 len,
5a6ac9ea
MX		 2595 u64 physical, struct btrfs_device *dev,
2596 u64 flags, u64 gen, int mirror_num, u8 *csum)
2597{
2598 struct scrub_ctx *sctx = sparity->sctx;
2599 struct scrub_block *sblock;
d0a7a9c0 2600 const u32 sectorsize = sctx->fs_info->sectorsize;
5a6ac9ea
MX		 2601 int index;
2602
d0a7a9c0
QW		 2603 ASSERT(IS_ALIGNED(len, sectorsize));
2604
58c4e173 2605 sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
5a6ac9ea
MX		 2606 if (!sblock) {
2607 spin_lock(&sctx->stat_lock);
2608 sctx->stat.malloc_errors++;
2609 spin_unlock(&sctx->stat_lock);
2610 return -ENOMEM;
2611 }
2612
2613 /* one ref inside this function, plus one for each page added to
2614 * a bio later on */
186debd6 2615 refcount_set(&sblock->refs, 1);
5a6ac9ea
MX		 2616 sblock->sctx = sctx;
2617 sblock->no_io_error_seen = 1;
2618 sblock->sparity = sparity;
2619 scrub_parity_get(sparity);
2620
2621 for (index = 0; len > 0; index++) {
2622 struct scrub_page *spage;
5a6ac9ea 2623
58c4e173 2624 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
5a6ac9ea
MX		 2625 if (!spage) {
2626leave_nomem:
2627 spin_lock(&sctx->stat_lock);
2628 sctx->stat.malloc_errors++;
2629 spin_unlock(&sctx->stat_lock);
2630 scrub_block_put(sblock);
2631 return -ENOMEM;
2632 }
0bb3acdc 2633 ASSERT(index < SCRUB_MAX_PAGES_PER_BLOCK);
5a6ac9ea
MX		 2634 /* For scrub block */
2635 scrub_page_get(spage);
2636 sblock->pagev[index] = spage;
2637 /* For scrub parity */
2638 scrub_page_get(spage);
2639 list_add_tail(&spage->list, &sparity->spages);
2640 spage->sblock = sblock;
2641 spage->dev = dev;
2642 spage->flags = flags;
2643 spage->generation = gen;
2644 spage->logical = logical;
2645 spage->physical = physical;
2646 spage->mirror_num = mirror_num;
2647 if (csum) {
2648 spage->have_csum = 1;
2ae0c2d8 2649 memcpy(spage->csum, csum, sctx->fs_info->csum_size);
5a6ac9ea
MX		 2650 } else {
2651 spage->have_csum = 0;
2652 }
2653 sblock->page_count++;
58c4e173 2654 spage->page = alloc_page(GFP_KERNEL);
5a6ac9ea
MX		 2655 if (!spage->page)
2656 goto leave_nomem;
d0a7a9c0
QW		 2657
2658
2659 /* Iterate over the stripe range in sectorsize steps */
2660 len -= sectorsize;
2661 logical += sectorsize;
2662 physical += sectorsize;
5a6ac9ea
MX		 2663 }
2664
2665 WARN_ON(sblock->page_count == 0);
2666 for (index = 0; index < sblock->page_count; index++) {
2667 struct scrub_page *spage = sblock->pagev[index];
2668 int ret;
2669
2670 ret = scrub_add_page_to_rd_bio(sctx, spage);
2671 if (ret) {
2672 scrub_block_put(sblock);
2673 return ret;
2674 }
2675 }
2676
2677 /* last one frees, either here or in bio completion for last page */
2678 scrub_block_put(sblock);
2679 return 0;
2680}
2681
2682static int scrub_extent_for_parity(struct scrub_parity *sparity,
fa485d21 2683 u64 logical, u32 len,
5a6ac9ea
MX		 2684 u64 physical, struct btrfs_device *dev,
2685 u64 flags, u64 gen, int mirror_num)
2686{
2687 struct scrub_ctx *sctx = sparity->sctx;
2688 int ret;
2689 u8 csum[BTRFS_CSUM_SIZE];
2690 u32 blocksize;
2691
e6e674bd 2692 if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
4a770891
OS		 2693 scrub_parity_mark_sectors_error(sparity, logical, len);
2694 return 0;
2695 }
2696
5a6ac9ea 2697 if (flags & BTRFS_EXTENT_FLAG_DATA) {
6ca1765b 2698 blocksize = sparity->stripe_len;
5a6ac9ea 2699 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
6ca1765b 2700 blocksize = sparity->stripe_len;
5a6ac9ea 2701 } else {
25cc1226 2702 blocksize = sctx->fs_info->sectorsize;
5a6ac9ea
MX		 2703 WARN_ON(1);
2704 }
2705
2706 while (len) {
fa485d21 2707 u32 l = min(len, blocksize);
5a6ac9ea
MX		 2708 int have_csum = 0;
2709
2710 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2711 /* push csums to sbio */
3b5753ec 2712 have_csum = scrub_find_csum(sctx, logical, csum);
5a6ac9ea
MX		 2713 if (have_csum == 0)
2714 goto skip;
2715 }
2716 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2717 flags, gen, mirror_num,
2718 have_csum ? csum : NULL);
5a6ac9ea
MX		 2719 if (ret)
2720 return ret;
6b6d24b3 2721skip:
5a6ac9ea
MX		 2722 len -= l;
2723 logical += l;
2724 physical += l;
2725 }
2726 return 0;
2727}
2728
3b080b25
WS		 2729/*
2730 * Given a physical address, this will calculate it's
2731 * logical offset. if this is a parity stripe, it will return
2732 * the most left data stripe's logical offset.
2733 *
2734 * return 0 if it is a data stripe, 1 means parity stripe.
2735 */
2736static int get_raid56_logic_offset(u64 physical, int num,
5a6ac9ea
MX		 2737 struct map_lookup *map, u64 *offset,
2738 u64 *stripe_start)
3b080b25
WS		 2739{
2740 int i;
2741 int j = 0;
2742 u64 stripe_nr;
2743 u64 last_offset;
9d644a62
DS		 2744 u32 stripe_index;
2745 u32 rot;
cff82672 2746 const int data_stripes = nr_data_stripes(map);
3b080b25 2747
cff82672 2748 last_offset = (physical - map->stripes[num].physical) * data_stripes;
5a6ac9ea
MX		 2749 if (stripe_start)
2750 *stripe_start = last_offset;
2751
3b080b25 2752 *offset = last_offset;
cff82672 2753 for (i = 0; i < data_stripes; i++) {
3b080b25
WS		 2754 *offset = last_offset + i * map->stripe_len;
2755
42c61ab6 2756 stripe_nr = div64_u64(*offset, map->stripe_len);
cff82672 2757 stripe_nr = div_u64(stripe_nr, data_stripes);
3b080b25
WS		 2758
2759 /* Work out the disk rotation on this stripe-set */
47c5713f 2760 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
3b080b25
WS		 2761 /* calculate which stripe this data locates */
2762 rot += i;
e4fbaee2 2763 stripe_index = rot % map->num_stripes;
3b080b25
WS		 2764 if (stripe_index == num)
2765 return 0;
2766 if (stripe_index < num)
2767 j++;
2768 }
2769 *offset = last_offset + j * map->stripe_len;
2770 return 1;
2771}
2772
5a6ac9ea
MX		 2773static void scrub_free_parity(struct scrub_parity *sparity)
2774{
2775 struct scrub_ctx *sctx = sparity->sctx;
2776 struct scrub_page *curr, *next;
2777 int nbits;
2778
2779 nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
2780 if (nbits) {
2781 spin_lock(&sctx->stat_lock);
2782 sctx->stat.read_errors += nbits;
2783 sctx->stat.uncorrectable_errors += nbits;
2784 spin_unlock(&sctx->stat_lock);
2785 }
2786
2787 list_for_each_entry_safe(curr, next, &sparity->spages, list) {
2788 list_del_init(&curr->list);
2789 scrub_page_put(curr);
2790 }
2791
2792 kfree(sparity);
2793}
2794
20b2e302
ZL		 2795static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
2796{
2797 struct scrub_parity *sparity = container_of(work, struct scrub_parity,
2798 work);
2799 struct scrub_ctx *sctx = sparity->sctx;
2800
2801 scrub_free_parity(sparity);
2802 scrub_pending_bio_dec(sctx);
2803}
2804
4246a0b6 2805static void scrub_parity_bio_endio(struct bio *bio)
5a6ac9ea
MX		 2806{
2807 struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
0b246afa 2808 struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;
5a6ac9ea 2809
4e4cbee9 2810 if (bio->bi_status)
5a6ac9ea
MX		 2811 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2812 sparity->nsectors);
2813
5a6ac9ea 2814 bio_put(bio);
20b2e302 2815
a0cac0ec
OS		 2816 btrfs_init_work(&sparity->work, scrub_parity_bio_endio_worker, NULL,
2817 NULL);
0b246afa 2818 btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work);
5a6ac9ea
MX		 2819}
2820
2821static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
2822{
2823 struct scrub_ctx *sctx = sparity->sctx;
0b246afa 2824 struct btrfs_fs_info *fs_info = sctx->fs_info;
5a6ac9ea
MX		 2825 struct bio *bio;
2826 struct btrfs_raid_bio *rbio;
4c664611 2827 struct btrfs_io_context *bioc = NULL;
5a6ac9ea
MX		 2828 u64 length;
2829 int ret;
2830
2831 if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
2832 sparity->nsectors))
2833 goto out;
2834
a0dd59de 2835 length = sparity->logic_end - sparity->logic_start;
ae6529c3
QW		 2836
2837 btrfs_bio_counter_inc_blocked(fs_info);
0b246afa 2838 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start,
4c664611
QW		 2839 &length, &bioc);
2840 if (ret || !bioc || !bioc->raid_map)
2841 goto bioc_out;
5a6ac9ea 2842
c3a3b19b 2843 bio = btrfs_bio_alloc(BIO_MAX_VECS);
5a6ac9ea
MX		 2844 bio->bi_iter.bi_sector = sparity->logic_start >> 9;
2845 bio->bi_private = sparity;
2846 bio->bi_end_io = scrub_parity_bio_endio;
2847
6a258d72
QW		 2848 rbio = raid56_parity_alloc_scrub_rbio(bio, bioc, length,
2849 sparity->scrub_dev,
5a6ac9ea
MX		 2850 sparity->dbitmap,
2851 sparity->nsectors);
2852 if (!rbio)
2853 goto rbio_out;
2854
5a6ac9ea
MX		 2855 scrub_pending_bio_inc(sctx);
2856 raid56_parity_submit_scrub_rbio(rbio);
2857 return;
2858
2859rbio_out:
2860 bio_put(bio);
4c664611 2861bioc_out:
ae6529c3 2862 btrfs_bio_counter_dec(fs_info);
4c664611 2863 btrfs_put_bioc(bioc);
5a6ac9ea
MX		 2864 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2865 sparity->nsectors);
2866 spin_lock(&sctx->stat_lock);
2867 sctx->stat.malloc_errors++;
2868 spin_unlock(&sctx->stat_lock);
2869out:
2870 scrub_free_parity(sparity);
2871}
2872
2873static inline int scrub_calc_parity_bitmap_len(int nsectors)
2874{
bfca9a6d 2875 return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long);
5a6ac9ea
MX		 2876}
2877
2878static void scrub_parity_get(struct scrub_parity *sparity)
2879{
78a76450 2880 refcount_inc(&sparity->refs);
5a6ac9ea
MX		 2881}
2882
2883static void scrub_parity_put(struct scrub_parity *sparity)
2884{
78a76450 2885 if (!refcount_dec_and_test(&sparity->refs))
5a6ac9ea
MX		 2886 return;
2887
2888 scrub_parity_check_and_repair(sparity);
2889}
2890
2891static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
2892 struct map_lookup *map,
2893 struct btrfs_device *sdev,
2894 struct btrfs_path *path,
2895 u64 logic_start,
2896 u64 logic_end)
2897{
fb456252 2898 struct btrfs_fs_info *fs_info = sctx->fs_info;
29cbcf40 2899 struct btrfs_root *root = btrfs_extent_root(fs_info, logic_start);
fc28b25e 2900 struct btrfs_root *csum_root;
5a6ac9ea 2901 struct btrfs_extent_item *extent;
4c664611 2902 struct btrfs_io_context *bioc = NULL;
5a6ac9ea
MX		 2903 u64 flags;
2904 int ret;
2905 int slot;
2906 struct extent_buffer *l;
2907 struct btrfs_key key;
2908 u64 generation;
2909 u64 extent_logical;
2910 u64 extent_physical;
fa485d21
QW		 2911 /* Check the comment in scrub_stripe() for why u32 is enough here */
2912 u32 extent_len;
4a770891 2913 u64 mapped_length;
5a6ac9ea
MX		 2914 struct btrfs_device *extent_dev;
2915 struct scrub_parity *sparity;
2916 int nsectors;
2917 int bitmap_len;
2918 int extent_mirror_num;
2919 int stop_loop = 0;
2920
fa485d21 2921 ASSERT(map->stripe_len <= U32_MAX);
ab108d99 2922 nsectors = map->stripe_len >> fs_info->sectorsize_bits;
5a6ac9ea
MX		 2923 bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
2924 sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
2925 GFP_NOFS);
2926 if (!sparity) {
2927 spin_lock(&sctx->stat_lock);
2928 sctx->stat.malloc_errors++;
2929 spin_unlock(&sctx->stat_lock);
2930 return -ENOMEM;
2931 }
2932
fa485d21 2933 ASSERT(map->stripe_len <= U32_MAX);
5a6ac9ea
MX		 2934 sparity->stripe_len = map->stripe_len;
2935 sparity->nsectors = nsectors;
2936 sparity->sctx = sctx;
2937 sparity->scrub_dev = sdev;
2938 sparity->logic_start = logic_start;
2939 sparity->logic_end = logic_end;
78a76450 2940 refcount_set(&sparity->refs, 1);
5a6ac9ea
MX		 2941 INIT_LIST_HEAD(&sparity->spages);
2942 sparity->dbitmap = sparity->bitmap;
2943 sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
2944
2945 ret = 0;
2946 while (logic_start < logic_end) {
2947 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2948 key.type = BTRFS_METADATA_ITEM_KEY;
2949 else
2950 key.type = BTRFS_EXTENT_ITEM_KEY;
2951 key.objectid = logic_start;
2952 key.offset = (u64)-1;
2953
2954 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2955 if (ret < 0)
2956 goto out;
2957
2958 if (ret > 0) {
2959 ret = btrfs_previous_extent_item(root, path, 0);
2960 if (ret < 0)
2961 goto out;
2962 if (ret > 0) {
2963 btrfs_release_path(path);
2964 ret = btrfs_search_slot(NULL, root, &key,
2965 path, 0, 0);
2966 if (ret < 0)
2967 goto out;
2968 }
2969 }
2970
2971 stop_loop = 0;
2972 while (1) {
2973 u64 bytes;
2974
2975 l = path->nodes[0];
2976 slot = path->slots[0];
2977 if (slot >= btrfs_header_nritems(l)) {
2978 ret = btrfs_next_leaf(root, path);
2979 if (ret == 0)
2980 continue;
2981 if (ret < 0)
2982 goto out;
2983
2984 stop_loop = 1;
2985 break;
2986 }
2987 btrfs_item_key_to_cpu(l, &key, slot);
2988
d7cad238
ZL		 2989 if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2990 key.type != BTRFS_METADATA_ITEM_KEY)
2991 goto next;
2992
5a6ac9ea 2993 if (key.type == BTRFS_METADATA_ITEM_KEY)
0b246afa 2994 bytes = fs_info->nodesize;
5a6ac9ea
MX		 2995 else
2996 bytes = key.offset;
2997
2998 if (key.objectid + bytes <= logic_start)
2999 goto next;
3000
a0dd59de 3001 if (key.objectid >= logic_end) {
5a6ac9ea
MX		 3002 stop_loop = 1;
3003 break;
3004 }
3005
3006 while (key.objectid >= logic_start + map->stripe_len)
3007 logic_start += map->stripe_len;
3008
3009 extent = btrfs_item_ptr(l, slot,
3010 struct btrfs_extent_item);
3011 flags = btrfs_extent_flags(l, extent);
3012 generation = btrfs_extent_generation(l, extent);
3013
a323e813
ZL		 3014 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3015 (key.objectid < logic_start ||
3016 key.objectid + bytes >
3017 logic_start + map->stripe_len)) {
5d163e0e
JM		 3018 btrfs_err(fs_info,
3019 "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
a323e813 3020 key.objectid, logic_start);
9799d2c3
ZL		 3021 spin_lock(&sctx->stat_lock);
3022 sctx->stat.uncorrectable_errors++;
3023 spin_unlock(&sctx->stat_lock);
5a6ac9ea
MX		 3024 goto next;
3025 }
3026again:
3027 extent_logical = key.objectid;
fa485d21 3028 ASSERT(bytes <= U32_MAX);
5a6ac9ea
MX		 3029 extent_len = bytes;
3030
3031 if (extent_logical < logic_start) {
3032 extent_len -= logic_start - extent_logical;
3033 extent_logical = logic_start;
3034 }
3035
3036 if (extent_logical + extent_len >
3037 logic_start + map->stripe_len)
3038 extent_len = logic_start + map->stripe_len -
3039 extent_logical;
3040
3041 scrub_parity_mark_sectors_data(sparity, extent_logical,
3042 extent_len);
3043
4a770891 3044 mapped_length = extent_len;
4c664611 3045 bioc = NULL;
cf8cddd3 3046 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ,
4c664611 3047 extent_logical, &mapped_length, &bioc,
cf8cddd3 3048 0);
4a770891 3049 if (!ret) {
4c664611 3050 if (!bioc || mapped_length < extent_len)
4a770891
OS		 3051 ret = -EIO;
3052 }
3053 if (ret) {
4c664611 3054 btrfs_put_bioc(bioc);
4a770891
OS		 3055 goto out;
3056 }
4c664611
QW		 3057 extent_physical = bioc->stripes[0].physical;
3058 extent_mirror_num = bioc->mirror_num;
3059 extent_dev = bioc->stripes[0].dev;
3060 btrfs_put_bioc(bioc);
5a6ac9ea 3061
fc28b25e 3062 csum_root = btrfs_csum_root(fs_info, extent_logical);
5a6ac9ea
MX		 3063 ret = btrfs_lookup_csums_range(csum_root,
3064 extent_logical,
3065 extent_logical + extent_len - 1,
3066 &sctx->csum_list, 1);
3067 if (ret)
3068 goto out;
3069
3070 ret = scrub_extent_for_parity(sparity, extent_logical,
3071 extent_len,
3072 extent_physical,
3073 extent_dev, flags,
3074 generation,
3075 extent_mirror_num);
6fa96d72
ZL		 3076
3077 scrub_free_csums(sctx);
3078
5a6ac9ea
MX		 3079 if (ret)
3080 goto out;
3081
5a6ac9ea
MX		 3082 if (extent_logical + extent_len <
3083 key.objectid + bytes) {
3084 logic_start += map->stripe_len;
3085
3086 if (logic_start >= logic_end) {
3087 stop_loop = 1;
3088 break;
3089 }
3090
3091 if (logic_start < key.objectid + bytes) {
3092 cond_resched();
3093 goto again;
3094 }
3095 }
3096next:
3097 path->slots[0]++;
3098 }
3099
3100 btrfs_release_path(path);
3101
3102 if (stop_loop)
3103 break;
3104
3105 logic_start += map->stripe_len;
3106 }
3107out:
fa485d21
QW		 3108 if (ret < 0) {
3109 ASSERT(logic_end - logic_start <= U32_MAX);
5a6ac9ea 3110 scrub_parity_mark_sectors_error(sparity, logic_start,
a0dd59de 3111 logic_end - logic_start);
fa485d21 3112 }
5a6ac9ea
MX		 3113 scrub_parity_put(sparity);
3114 scrub_submit(sctx);
3fb99303 3115 mutex_lock(&sctx->wr_lock);
5a6ac9ea 3116 scrub_wr_submit(sctx);
3fb99303 3117 mutex_unlock(&sctx->wr_lock);
5a6ac9ea
MX		 3118
3119 btrfs_release_path(path);
3120 return ret < 0 ? ret : 0;
3121}
3122
de17addc
NA		 3123static void sync_replace_for_zoned(struct scrub_ctx *sctx)
3124{
3125 if (!btrfs_is_zoned(sctx->fs_info))
3126 return;
3127
3128 sctx->flush_all_writes = true;
3129 scrub_submit(sctx);
3130 mutex_lock(&sctx->wr_lock);
3131 scrub_wr_submit(sctx);
3132 mutex_unlock(&sctx->wr_lock);
3133
3134 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3135}
3136
7db1c5d1
NA		 3137static int sync_write_pointer_for_zoned(struct scrub_ctx *sctx, u64 logical,
3138 u64 physical, u64 physical_end)
3139{
3140 struct btrfs_fs_info *fs_info = sctx->fs_info;
3141 int ret = 0;
3142
3143 if (!btrfs_is_zoned(fs_info))
3144 return 0;
3145
3146 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3147
3148 mutex_lock(&sctx->wr_lock);
3149 if (sctx->write_pointer < physical_end) {
3150 ret = btrfs_sync_zone_write_pointer(sctx->wr_tgtdev, logical,
3151 physical,
3152 sctx->write_pointer);
3153 if (ret)
3154 btrfs_err(fs_info,
3155 "zoned: failed to recover write pointer");
3156 }
3157 mutex_unlock(&sctx->wr_lock);
3158 btrfs_dev_clear_zone_empty(sctx->wr_tgtdev, physical);
3159
3160 return ret;
3161}
3162
d9d181c1 3163static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
a36cf8b8
SB		 3164 struct map_lookup *map,
3165 struct btrfs_device *scrub_dev,
2473d24f
FM		 3166 int num, u64 base, u64 length,
3167 struct btrfs_block_group *cache)
a2de733c 3168{
5a6ac9ea 3169 struct btrfs_path *path, *ppath;
fb456252 3170 struct btrfs_fs_info *fs_info = sctx->fs_info;
29cbcf40 3171 struct btrfs_root *root;
fc28b25e 3172 struct btrfs_root *csum_root;
a2de733c 3173 struct btrfs_extent_item *extent;
e7786c3a 3174 struct blk_plug plug;
a2de733c
AJ		 3175 u64 flags;
3176 int ret;
3177 int slot;
a2de733c 3178 u64 nstripes;
a2de733c 3179 struct extent_buffer *l;
a2de733c
AJ		 3180 u64 physical;
3181 u64 logical;
625f1c8d 3182 u64 logic_end;
3b080b25 3183 u64 physical_end;
a2de733c 3184 u64 generation;
e12fa9cd 3185 int mirror_num;
7a26285e
AJ		 3186 struct reada_control *reada1;
3187 struct reada_control *reada2;
e6c11f9a 3188 struct btrfs_key key;
7a26285e 3189 struct btrfs_key key_end;
a2de733c
AJ		 3190 u64 increment = map->stripe_len;
3191 u64 offset;
ff023aac
SB		 3192 u64 extent_logical;
3193 u64 extent_physical;
fa485d21
QW		 3194 /*
3195 * Unlike chunk length, extent length should never go beyond
3196 * BTRFS_MAX_EXTENT_SIZE, thus u32 is enough here.
3197 */
3198 u32 extent_len;
5a6ac9ea
MX		 3199 u64 stripe_logical;
3200 u64 stripe_end;
ff023aac
SB		 3201 struct btrfs_device *extent_dev;
3202 int extent_mirror_num;
3b080b25 3203 int stop_loop = 0;
53b381b3 3204
3b080b25 3205 physical = map->stripes[num].physical;
a2de733c 3206 offset = 0;
42c61ab6 3207 nstripes = div64_u64(length, map->stripe_len);
7735cd75
DS		 3208 mirror_num = 1;
3209 increment = map->stripe_len;
a2de733c
AJ		 3210 if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3211 offset = map->stripe_len * num;
3212 increment = map->stripe_len * map->num_stripes;
a2de733c
AJ		 3213 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3214 int factor = map->num_stripes / map->sub_stripes;
3215 offset = map->stripe_len * (num / map->sub_stripes);
3216 increment = map->stripe_len * factor;
193ea74b 3217 mirror_num = num % map->sub_stripes + 1;
c7369b3f 3218 } else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) {
193ea74b 3219 mirror_num = num % map->num_stripes + 1;
a2de733c 3220 } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
193ea74b 3221 mirror_num = num % map->num_stripes + 1;
ffe2d203 3222 } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5a6ac9ea 3223 get_raid56_logic_offset(physical, num, map, &offset, NULL);
3b080b25 3224 increment = map->stripe_len * nr_data_stripes(map);
a2de733c
AJ		 3225 }
3226
3227 path = btrfs_alloc_path();
3228 if (!path)
3229 return -ENOMEM;
3230
5a6ac9ea
MX		 3231 ppath = btrfs_alloc_path();
3232 if (!ppath) {
379d6854 3233 btrfs_free_path(path);
5a6ac9ea
MX		 3234 return -ENOMEM;
3235 }
3236
b5d67f64
SB		 3237 /*
3238 * work on commit root. The related disk blocks are static as
3239 * long as COW is applied. This means, it is save to rewrite
3240 * them to repair disk errors without any race conditions
3241 */
a2de733c
AJ		 3242 path->search_commit_root = 1;
3243 path->skip_locking = 1;
3244
063c54dc
GH		 3245 ppath->search_commit_root = 1;
3246 ppath->skip_locking = 1;
a2de733c 3247 /*
7a26285e
AJ		 3248 * trigger the readahead for extent tree csum tree and wait for
3249 * completion. During readahead, the scrub is officially paused
3250 * to not hold off transaction commits
a2de733c
AJ		 3251 */
3252 logical = base + offset;
3b080b25 3253 physical_end = physical + nstripes * map->stripe_len;
ffe2d203 3254 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3b080b25 3255 get_raid56_logic_offset(physical_end, num,
5a6ac9ea 3256 map, &logic_end, NULL);
3b080b25
WS		 3257 logic_end += base;
3258 } else {
3259 logic_end = logical + increment * nstripes;
3260 }
d9d181c1 3261 wait_event(sctx->list_wait,
b6bfebc1 3262 atomic_read(&sctx->bios_in_flight) == 0);
cb7ab021 3263 scrub_blocked_if_needed(fs_info);
7a26285e 3264
29cbcf40
JB		 3265 root = btrfs_extent_root(fs_info, logical);
3266
7a26285e 3267 /* FIXME it might be better to start readahead at commit root */
e6c11f9a
DS		 3268 key.objectid = logical;
3269 key.type = BTRFS_EXTENT_ITEM_KEY;
3270 key.offset = (u64)0;
3b080b25 3271 key_end.objectid = logic_end;
3173a18f
JB		 3272 key_end.type = BTRFS_METADATA_ITEM_KEY;
3273 key_end.offset = (u64)-1;
e6c11f9a 3274 reada1 = btrfs_reada_add(root, &key, &key_end);
7a26285e 3275
fc28b25e
JB		 3276 csum_root = btrfs_csum_root(fs_info, logical);
3277
a6889caf
FM		 3278 if (cache->flags & BTRFS_BLOCK_GROUP_DATA) {
3279 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3280 key.type = BTRFS_EXTENT_CSUM_KEY;
3281 key.offset = logical;
3282 key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3283 key_end.type = BTRFS_EXTENT_CSUM_KEY;
3284 key_end.offset = logic_end;
3285 reada2 = btrfs_reada_add(csum_root, &key, &key_end);
3286 } else {
3287 reada2 = NULL;
3288 }
7a26285e
AJ		 3289
3290 if (!IS_ERR(reada1))
3291 btrfs_reada_wait(reada1);
a6889caf 3292 if (!IS_ERR_OR_NULL(reada2))
7a26285e
AJ		 3293 btrfs_reada_wait(reada2);
3294
a2de733c
AJ		 3295
3296 /*
3297 * collect all data csums for the stripe to avoid seeking during
3298 * the scrub. This might currently (crc32) end up to be about 1MB
3299 */
e7786c3a 3300 blk_start_plug(&plug);
a2de733c 3301
de17addc
NA		 3302 if (sctx->is_dev_replace &&
3303 btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) {
3304 mutex_lock(&sctx->wr_lock);
3305 sctx->write_pointer = physical;
3306 mutex_unlock(&sctx->wr_lock);
3307 sctx->flush_all_writes = true;
3308 }
3309
a2de733c
AJ		 3310 /*
3311 * now find all extents for each stripe and scrub them
3312 */
a2de733c 3313 ret = 0;
3b080b25 3314 while (physical < physical_end) {
a2de733c
AJ		 3315 /*
3316 * canceled?
3317 */
3318 if (atomic_read(&fs_info->scrub_cancel_req) ||
d9d181c1 3319 atomic_read(&sctx->cancel_req)) {
a2de733c
AJ		 3320 ret = -ECANCELED;
3321 goto out;
3322 }
3323 /*
3324 * check to see if we have to pause
3325 */
3326 if (atomic_read(&fs_info->scrub_pause_req)) {
3327 /* push queued extents */
2073c4c2 3328 sctx->flush_all_writes = true;
d9d181c1 3329 scrub_submit(sctx);
3fb99303 3330 mutex_lock(&sctx->wr_lock);
ff023aac 3331 scrub_wr_submit(sctx);
3fb99303 3332 mutex_unlock(&sctx->wr_lock);
d9d181c1 3333 wait_event(sctx->list_wait,
b6bfebc1 3334 atomic_read(&sctx->bios_in_flight) == 0);
2073c4c2 3335 sctx->flush_all_writes = false;
3cb0929a 3336 scrub_blocked_if_needed(fs_info);
a2de733c
AJ		 3337 }
3338
f2f66a2f
ZL		 3339 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3340 ret = get_raid56_logic_offset(physical, num, map,
3341 &logical,
3342 &stripe_logical);
3343 logical += base;
3344 if (ret) {
7955323b 3345 /* it is parity strip */
f2f66a2f 3346 stripe_logical += base;
a0dd59de 3347 stripe_end = stripe_logical + increment;
f2f66a2f
ZL		 3348 ret = scrub_raid56_parity(sctx, map, scrub_dev,
3349 ppath, stripe_logical,
3350 stripe_end);
3351 if (ret)
3352 goto out;
3353 goto skip;
3354 }
3355 }
3356
7c76edb7
WS		 3357 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3358 key.type = BTRFS_METADATA_ITEM_KEY;
3359 else
3360 key.type = BTRFS_EXTENT_ITEM_KEY;
a2de733c 3361 key.objectid = logical;
625f1c8d 3362 key.offset = (u64)-1;
a2de733c
AJ		 3363
3364 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3365 if (ret < 0)
3366 goto out;
3173a18f 3367
8c51032f 3368 if (ret > 0) {
ade2e0b3 3369 ret = btrfs_previous_extent_item(root, path, 0);
a2de733c
AJ		 3370 if (ret < 0)
3371 goto out;
8c51032f
AJ		 3372 if (ret > 0) {
3373 /* there's no smaller item, so stick with the
3374 * larger one */
3375 btrfs_release_path(path);
3376 ret = btrfs_search_slot(NULL, root, &key,
3377 path, 0, 0);
3378 if (ret < 0)
3379 goto out;
3380 }
a2de733c
AJ		 3381 }
3382
625f1c8d 3383 stop_loop = 0;
a2de733c 3384 while (1) {
3173a18f
JB		 3385 u64 bytes;
3386
a2de733c
AJ		 3387 l = path->nodes[0];
3388 slot = path->slots[0];
3389 if (slot >= btrfs_header_nritems(l)) {
3390 ret = btrfs_next_leaf(root, path);
3391 if (ret == 0)
3392 continue;
3393 if (ret < 0)
3394 goto out;
3395
625f1c8d 3396 stop_loop = 1;
a2de733c
AJ		 3397 break;
3398 }
3399 btrfs_item_key_to_cpu(l, &key, slot);
3400
d7cad238
ZL		 3401 if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3402 key.type != BTRFS_METADATA_ITEM_KEY)
3403 goto next;
3404
3173a18f 3405 if (key.type == BTRFS_METADATA_ITEM_KEY)
0b246afa 3406 bytes = fs_info->nodesize;
3173a18f
JB		 3407 else
3408 bytes = key.offset;
3409
3410 if (key.objectid + bytes <= logical)
a2de733c
AJ		 3411 goto next;
3412
625f1c8d
LB		 3413 if (key.objectid >= logical + map->stripe_len) {
3414 /* out of this device extent */
3415 if (key.objectid >= logic_end)
3416 stop_loop = 1;
3417 break;
3418 }
a2de733c 3419
2473d24f
FM		 3420 /*
3421 * If our block group was removed in the meanwhile, just
3422 * stop scrubbing since there is no point in continuing.
3423 * Continuing would prevent reusing its device extents
3424 * for new block groups for a long time.
3425 */
3426 spin_lock(&cache->lock);
3427 if (cache->removed) {
3428 spin_unlock(&cache->lock);
3429 ret = 0;
3430 goto out;
3431 }
3432 spin_unlock(&cache->lock);
3433
a2de733c
AJ		 3434 extent = btrfs_item_ptr(l, slot,
3435 struct btrfs_extent_item);
3436 flags = btrfs_extent_flags(l, extent);
3437 generation = btrfs_extent_generation(l, extent);
3438
a323e813
ZL		 3439 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3440 (key.objectid < logical ||
3441 key.objectid + bytes >
3442 logical + map->stripe_len)) {
efe120a0 3443 btrfs_err(fs_info,
5d163e0e 3444 "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
c1c9ff7c 3445 key.objectid, logical);
9799d2c3
ZL		 3446 spin_lock(&sctx->stat_lock);
3447 sctx->stat.uncorrectable_errors++;
3448 spin_unlock(&sctx->stat_lock);
a2de733c
AJ		 3449 goto next;
3450 }
3451
625f1c8d
LB		 3452again:
3453 extent_logical = key.objectid;
fa485d21 3454 ASSERT(bytes <= U32_MAX);
625f1c8d
LB		 3455 extent_len = bytes;
3456
a2de733c
AJ		 3457 /*
3458 * trim extent to this stripe
3459 */
625f1c8d
LB		 3460 if (extent_logical < logical) {
3461 extent_len -= logical - extent_logical;
3462 extent_logical = logical;
a2de733c 3463 }
625f1c8d 3464 if (extent_logical + extent_len >
a2de733c 3465 logical + map->stripe_len) {
625f1c8d
LB		 3466 extent_len = logical + map->stripe_len -
3467 extent_logical;
a2de733c
AJ		 3468 }
3469
625f1c8d 3470 extent_physical = extent_logical - logical + physical;
ff023aac
SB		 3471 extent_dev = scrub_dev;
3472 extent_mirror_num = mirror_num;
32934280 3473 if (sctx->is_dev_replace)
ff023aac
SB		 3474 scrub_remap_extent(fs_info, extent_logical,
3475 extent_len, &extent_physical,
3476 &extent_dev,
3477 &extent_mirror_num);
625f1c8d 3478
89490303
FM		 3479 if (flags & BTRFS_EXTENT_FLAG_DATA) {
3480 ret = btrfs_lookup_csums_range(csum_root,
3481 extent_logical,
3482 extent_logical + extent_len - 1,
3483 &sctx->csum_list, 1);
3484 if (ret)
3485 goto out;
3486 }
625f1c8d 3487
6ca1765b 3488 ret = scrub_extent(sctx, map, extent_logical, extent_len,
ff023aac
SB		 3489 extent_physical, extent_dev, flags,
3490 generation, extent_mirror_num,
115930cb 3491 extent_logical - logical + physical);
6fa96d72
ZL		 3492
3493 scrub_free_csums(sctx);
3494
a2de733c
AJ		 3495 if (ret)
3496 goto out;
3497
de17addc
NA		 3498 if (sctx->is_dev_replace)
3499 sync_replace_for_zoned(sctx);
3500
625f1c8d
LB		 3501 if (extent_logical + extent_len <
3502 key.objectid + bytes) {
ffe2d203 3503 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3b080b25
WS		 3504 /*
3505 * loop until we find next data stripe
3506 * or we have finished all stripes.
3507 */
5a6ac9ea
MX		 3508loop:
3509 physical += map->stripe_len;
3510 ret = get_raid56_logic_offset(physical,
3511 num, map, &logical,
3512 &stripe_logical);
3513 logical += base;
3514
3515 if (ret && physical < physical_end) {
3516 stripe_logical += base;
3517 stripe_end = stripe_logical +
a0dd59de 3518 increment;
5a6ac9ea
MX		 3519 ret = scrub_raid56_parity(sctx,
3520 map, scrub_dev, ppath,
3521 stripe_logical,
3522 stripe_end);
3523 if (ret)
3524 goto out;
3525 goto loop;
3526 }
3b080b25
WS		 3527 } else {
3528 physical += map->stripe_len;
3529 logical += increment;
3530 }
625f1c8d
LB		 3531 if (logical < key.objectid + bytes) {
3532 cond_resched();
3533 goto again;
3534 }
3535
3b080b25 3536 if (physical >= physical_end) {
625f1c8d
LB		 3537 stop_loop = 1;
3538 break;
3539 }
3540 }
a2de733c
AJ		 3541next:
3542 path->slots[0]++;
3543 }
71267333 3544 btrfs_release_path(path);
3b080b25 3545skip:
a2de733c
AJ		 3546 logical += increment;
3547 physical += map->stripe_len;
d9d181c1 3548 spin_lock(&sctx->stat_lock);
625f1c8d
LB		 3549 if (stop_loop)
3550 sctx->stat.last_physical = map->stripes[num].physical +
3551 length;
3552 else
3553 sctx->stat.last_physical = physical;
d9d181c1 3554 spin_unlock(&sctx->stat_lock);
625f1c8d
LB		 3555 if (stop_loop)
3556 break;
a2de733c 3557 }
ff023aac 3558out:
a2de733c 3559 /* push queued extents */
d9d181c1 3560 scrub_submit(sctx);
3fb99303 3561 mutex_lock(&sctx->wr_lock);
ff023aac 3562 scrub_wr_submit(sctx);
3fb99303 3563 mutex_unlock(&sctx->wr_lock);
a2de733c 3564
e7786c3a 3565 blk_finish_plug(&plug);
a2de733c 3566 btrfs_free_path(path);
5a6ac9ea 3567 btrfs_free_path(ppath);
7db1c5d1
NA		 3568
3569 if (sctx->is_dev_replace && ret >= 0) {
3570 int ret2;
3571
3572 ret2 = sync_write_pointer_for_zoned(sctx, base + offset,
3573 map->stripes[num].physical,
3574 physical_end);
3575 if (ret2)
3576 ret = ret2;
3577 }
3578
a2de733c
AJ		 3579 return ret < 0 ? ret : 0;
3580}
3581
d9d181c1 3582static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
a36cf8b8 3583 struct btrfs_device *scrub_dev,
a36cf8b8 3584 u64 chunk_offset, u64 length,
020d5b73 3585 u64 dev_offset,
32da5386 3586 struct btrfs_block_group *cache)
a2de733c 3587{
fb456252 3588 struct btrfs_fs_info *fs_info = sctx->fs_info;
c8bf1b67 3589 struct extent_map_tree *map_tree = &fs_info->mapping_tree;
a2de733c
AJ		 3590 struct map_lookup *map;
3591 struct extent_map *em;
3592 int i;
ff023aac 3593 int ret = 0;
a2de733c 3594
c8bf1b67
DS		 3595 read_lock(&map_tree->lock);
3596 em = lookup_extent_mapping(map_tree, chunk_offset, 1);
3597 read_unlock(&map_tree->lock);
a2de733c 3598
020d5b73
FM		 3599 if (!em) {
3600 /*
3601 * Might have been an unused block group deleted by the cleaner
3602 * kthread or relocation.
3603 */
3604 spin_lock(&cache->lock);
3605 if (!cache->removed)
3606 ret = -EINVAL;
3607 spin_unlock(&cache->lock);
3608
3609 return ret;
3610 }
a2de733c 3611
95617d69 3612 map = em->map_lookup;
a2de733c
AJ		 3613 if (em->start != chunk_offset)
3614 goto out;
3615
3616 if (em->len < length)
3617 goto out;
3618
3619 for (i = 0; i < map->num_stripes; ++i) {
a36cf8b8 3620 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
859acaf1 3621 map->stripes[i].physical == dev_offset) {
a36cf8b8 3622 ret = scrub_stripe(sctx, map, scrub_dev, i,
2473d24f 3623 chunk_offset, length, cache);
a2de733c
AJ		 3624 if (ret)
3625 goto out;
3626 }
3627 }
3628out:
3629 free_extent_map(em);
3630
3631 return ret;
3632}
3633
de17addc
NA		 3634static int finish_extent_writes_for_zoned(struct btrfs_root *root,
3635 struct btrfs_block_group *cache)
3636{
3637 struct btrfs_fs_info *fs_info = cache->fs_info;
3638 struct btrfs_trans_handle *trans;
3639
3640 if (!btrfs_is_zoned(fs_info))
3641 return 0;
3642
3643 btrfs_wait_block_group_reservations(cache);
3644 btrfs_wait_nocow_writers(cache);
3645 btrfs_wait_ordered_roots(fs_info, U64_MAX, cache->start, cache->length);
3646
3647 trans = btrfs_join_transaction(root);
3648 if (IS_ERR(trans))
3649 return PTR_ERR(trans);
3650 return btrfs_commit_transaction(trans);
3651}
3652
a2de733c 3653static noinline_for_stack
a36cf8b8 3654int scrub_enumerate_chunks(struct scrub_ctx *sctx,
32934280 3655 struct btrfs_device *scrub_dev, u64 start, u64 end)
a2de733c
AJ		 3656{
3657 struct btrfs_dev_extent *dev_extent = NULL;
3658 struct btrfs_path *path;
0b246afa
JM		 3659 struct btrfs_fs_info *fs_info = sctx->fs_info;
3660 struct btrfs_root *root = fs_info->dev_root;
a2de733c 3661 u64 length;
a2de733c 3662 u64 chunk_offset;
55e3a601 3663 int ret = 0;
76a8efa1 3664 int ro_set;
a2de733c
AJ		 3665 int slot;
3666 struct extent_buffer *l;
3667 struct btrfs_key key;
3668 struct btrfs_key found_key;
32da5386 3669 struct btrfs_block_group *cache;
ff023aac 3670 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
a2de733c
AJ		 3671
3672 path = btrfs_alloc_path();
3673 if (!path)
3674 return -ENOMEM;
3675
e4058b54 3676 path->reada = READA_FORWARD;
a2de733c
AJ		 3677 path->search_commit_root = 1;
3678 path->skip_locking = 1;
3679
a36cf8b8 3680 key.objectid = scrub_dev->devid;
a2de733c
AJ		 3681 key.offset = 0ull;
3682 key.type = BTRFS_DEV_EXTENT_KEY;
3683
a2de733c
AJ		 3684 while (1) {
3685 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3686 if (ret < 0)
8c51032f
AJ		 3687 break;
3688 if (ret > 0) {
3689 if (path->slots[0] >=
3690 btrfs_header_nritems(path->nodes[0])) {
3691 ret = btrfs_next_leaf(root, path);
55e3a601
Z		 3692 if (ret < 0)
3693 break;
3694 if (ret > 0) {
3695 ret = 0;
8c51032f 3696 break;
55e3a601
Z		 3697 }
3698 } else {
3699 ret = 0;
8c51032f
AJ		 3700 }
3701 }
a2de733c
AJ		 3702
3703 l = path->nodes[0];
3704 slot = path->slots[0];
3705
3706 btrfs_item_key_to_cpu(l, &found_key, slot);
3707
a36cf8b8 3708 if (found_key.objectid != scrub_dev->devid)
a2de733c
AJ		 3709 break;
3710
962a298f 3711 if (found_key.type != BTRFS_DEV_EXTENT_KEY)
a2de733c
AJ		 3712 break;
3713
3714 if (found_key.offset >= end)
3715 break;
3716
3717 if (found_key.offset < key.offset)
3718 break;
3719
3720 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
3721 length = btrfs_dev_extent_length(l, dev_extent);
3722
ced96edc
QW		 3723 if (found_key.offset + length <= start)
3724 goto skip;
a2de733c 3725
a2de733c
AJ		 3726 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
3727
3728 /*
3729 * get a reference on the corresponding block group to prevent
3730 * the chunk from going away while we scrub it
3731 */
3732 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
ced96edc
QW		 3733
3734 /* some chunks are removed but not committed to disk yet,
3735 * continue scrubbing */
3736 if (!cache)
3737 goto skip;
3738
78ce9fc2
NA		 3739 if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) {
3740 spin_lock(&cache->lock);
3741 if (!cache->to_copy) {
3742 spin_unlock(&cache->lock);
0dc16ef4
FM		 3743 btrfs_put_block_group(cache);
3744 goto skip;
78ce9fc2
NA		 3745 }
3746 spin_unlock(&cache->lock);
3747 }
3748
2473d24f
FM		 3749 /*
3750 * Make sure that while we are scrubbing the corresponding block
3751 * group doesn't get its logical address and its device extents
3752 * reused for another block group, which can possibly be of a
3753 * different type and different profile. We do this to prevent
3754 * false error detections and crashes due to bogus attempts to
3755 * repair extents.
3756 */
3757 spin_lock(&cache->lock);
3758 if (cache->removed) {
3759 spin_unlock(&cache->lock);
3760 btrfs_put_block_group(cache);
3761 goto skip;
3762 }
6b7304af 3763 btrfs_freeze_block_group(cache);
2473d24f
FM		 3764 spin_unlock(&cache->lock);
3765
55e3a601
Z		 3766 /*
3767 * we need call btrfs_inc_block_group_ro() with scrubs_paused,
3768 * to avoid deadlock caused by:
3769 * btrfs_inc_block_group_ro()
3770 * -> btrfs_wait_for_commit()
3771 * -> btrfs_commit_transaction()
3772 * -> btrfs_scrub_pause()
3773 */
3774 scrub_pause_on(fs_info);
b12de528
QW		 3775
3776 /*
3777 * Don't do chunk preallocation for scrub.
3778 *
3779 * This is especially important for SYSTEM bgs, or we can hit
3780 * -EFBIG from btrfs_finish_chunk_alloc() like:
3781 * 1. The only SYSTEM bg is marked RO.
3782 * Since SYSTEM bg is small, that's pretty common.
3783 * 2. New SYSTEM bg will be allocated
3784 * Due to regular version will allocate new chunk.
3785 * 3. New SYSTEM bg is empty and will get cleaned up
3786 * Before cleanup really happens, it's marked RO again.
3787 * 4. Empty SYSTEM bg get scrubbed
3788 * We go back to 2.
3789 *
3790 * This can easily boost the amount of SYSTEM chunks if cleaner
3791 * thread can't be triggered fast enough, and use up all space
3792 * of btrfs_super_block::sys_chunk_array
1bbb97b8
QW		 3793 *
3794 * While for dev replace, we need to try our best to mark block
3795 * group RO, to prevent race between:
3796 * - Write duplication
3797 * Contains latest data
3798 * - Scrub copy
3799 * Contains data from commit tree
3800 *
3801 * If target block group is not marked RO, nocow writes can
3802 * be overwritten by scrub copy, causing data corruption.
3803 * So for dev-replace, it's not allowed to continue if a block
3804 * group is not RO.
b12de528 3805 */
1bbb97b8 3806 ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace);
de17addc
NA		 3807 if (!ret && sctx->is_dev_replace) {
3808 ret = finish_extent_writes_for_zoned(root, cache);
3809 if (ret) {
3810 btrfs_dec_block_group_ro(cache);
3811 scrub_pause_off(fs_info);
3812 btrfs_put_block_group(cache);
3813 break;
3814 }
3815 }
3816
76a8efa1
Z		 3817 if (ret == 0) {
3818 ro_set = 1;
1bbb97b8 3819 } else if (ret == -ENOSPC && !sctx->is_dev_replace) {
76a8efa1
Z		 3820 /*
3821 * btrfs_inc_block_group_ro return -ENOSPC when it
3822 * failed in creating new chunk for metadata.
1bbb97b8 3823 * It is not a problem for scrub, because
76a8efa1
Z		 3824 * metadata are always cowed, and our scrub paused
3825 * commit_transactions.
3826 */
3827 ro_set = 0;
195a49ea
FM		 3828 } else if (ret == -ETXTBSY) {
3829 btrfs_warn(fs_info,
3830 "skipping scrub of block group %llu due to active swapfile",
3831 cache->start);
3832 scrub_pause_off(fs_info);
3833 ret = 0;
3834 goto skip_unfreeze;
76a8efa1 3835 } else {
5d163e0e 3836 btrfs_warn(fs_info,
913e1535 3837 "failed setting block group ro: %d", ret);
6b7304af 3838 btrfs_unfreeze_block_group(cache);
55e3a601 3839 btrfs_put_block_group(cache);
1bbb97b8 3840 scrub_pause_off(fs_info);
55e3a601
Z		 3841 break;
3842 }
3843
1bbb97b8
QW		 3844 /*
3845 * Now the target block is marked RO, wait for nocow writes to
3846 * finish before dev-replace.
3847 * COW is fine, as COW never overwrites extents in commit tree.
3848 */
3849 if (sctx->is_dev_replace) {
3850 btrfs_wait_nocow_writers(cache);
3851 btrfs_wait_ordered_roots(fs_info, U64_MAX, cache->start,
3852 cache->length);
3853 }
3854
3855 scrub_pause_off(fs_info);
3ec17a67 3856 down_write(&dev_replace->rwsem);
ff023aac
SB		 3857 dev_replace->cursor_right = found_key.offset + length;
3858 dev_replace->cursor_left = found_key.offset;
3859 dev_replace->item_needs_writeback = 1;
cb5583dd
DS		 3860 up_write(&dev_replace->rwsem);
3861
8c204c96 3862 ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length,
32934280 3863 found_key.offset, cache);
ff023aac
SB		 3864
3865 /*
3866 * flush, submit all pending read and write bios, afterwards
3867 * wait for them.
3868 * Note that in the dev replace case, a read request causes
3869 * write requests that are submitted in the read completion
3870 * worker. Therefore in the current situation, it is required
3871 * that all write requests are flushed, so that all read and
3872 * write requests are really completed when bios_in_flight
3873 * changes to 0.
3874 */
2073c4c2 3875 sctx->flush_all_writes = true;
ff023aac 3876 scrub_submit(sctx);
3fb99303 3877 mutex_lock(&sctx->wr_lock);
ff023aac 3878 scrub_wr_submit(sctx);
3fb99303 3879 mutex_unlock(&sctx->wr_lock);
ff023aac
SB		 3880
3881 wait_event(sctx->list_wait,
3882 atomic_read(&sctx->bios_in_flight) == 0);
b708ce96
Z		 3883
3884 scrub_pause_on(fs_info);
12cf9372
WS		 3885
3886 /*
3887 * must be called before we decrease @scrub_paused.
3888 * make sure we don't block transaction commit while
3889 * we are waiting pending workers finished.
3890 */
ff023aac
SB		 3891 wait_event(sctx->list_wait,
3892 atomic_read(&sctx->workers_pending) == 0);
2073c4c2 3893 sctx->flush_all_writes = false;
12cf9372 3894
b708ce96 3895 scrub_pause_off(fs_info);
ff023aac 3896
78ce9fc2
NA		 3897 if (sctx->is_dev_replace &&
3898 !btrfs_finish_block_group_to_copy(dev_replace->srcdev,
3899 cache, found_key.offset))
3900 ro_set = 0;
3901
3ec17a67 3902 down_write(&dev_replace->rwsem);
1a1a8b73
FM		 3903 dev_replace->cursor_left = dev_replace->cursor_right;
3904 dev_replace->item_needs_writeback = 1;
3ec17a67 3905 up_write(&dev_replace->rwsem);
1a1a8b73 3906
76a8efa1 3907 if (ro_set)
2ff7e61e 3908 btrfs_dec_block_group_ro(cache);
ff023aac 3909
758f2dfc
FM		 3910 /*
3911 * We might have prevented the cleaner kthread from deleting
3912 * this block group if it was already unused because we raced
3913 * and set it to RO mode first. So add it back to the unused
3914 * list, otherwise it might not ever be deleted unless a manual
3915 * balance is triggered or it becomes used and unused again.
3916 */
3917 spin_lock(&cache->lock);
3918 if (!cache->removed && !cache->ro && cache->reserved == 0 &&
bf38be65 3919 cache->used == 0) {
758f2dfc 3920 spin_unlock(&cache->lock);
6e80d4f8
DZ		 3921 if (btrfs_test_opt(fs_info, DISCARD_ASYNC))
3922 btrfs_discard_queue_work(&fs_info->discard_ctl,
3923 cache);
3924 else
3925 btrfs_mark_bg_unused(cache);
758f2dfc
FM		 3926 } else {
3927 spin_unlock(&cache->lock);
3928 }
195a49ea 3929skip_unfreeze:
6b7304af 3930 btrfs_unfreeze_block_group(cache);
a2de733c
AJ		 3931 btrfs_put_block_group(cache);
3932 if (ret)
3933 break;
32934280 3934 if (sctx->is_dev_replace &&
af1be4f8 3935 atomic64_read(&dev_replace->num_write_errors) > 0) {
ff023aac
SB		 3936 ret = -EIO;
3937 break;
3938 }
3939 if (sctx->stat.malloc_errors > 0) {
3940 ret = -ENOMEM;
3941 break;
3942 }
ced96edc 3943skip:
a2de733c 3944 key.offset = found_key.offset + length;
71267333 3945 btrfs_release_path(path);
a2de733c
AJ		 3946 }
3947
a2de733c 3948 btrfs_free_path(path);
8c51032f 3949
55e3a601 3950 return ret;
a2de733c
AJ		 3951}
3952
a36cf8b8
SB		 3953static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
3954 struct btrfs_device *scrub_dev)
a2de733c
AJ		 3955{
3956 int i;
3957 u64 bytenr;
3958 u64 gen;
3959 int ret;
0b246afa 3960 struct btrfs_fs_info *fs_info = sctx->fs_info;
a2de733c 3961
84961539 3962 if (BTRFS_FS_ERROR(fs_info))
fbabd4a3 3963 return -EROFS;
79787eaa 3964
5f546063 3965 /* Seed devices of a new filesystem has their own generation. */
0b246afa 3966 if (scrub_dev->fs_devices != fs_info->fs_devices)
5f546063
MX		 3967 gen = scrub_dev->generation;
3968 else
0b246afa 3969 gen = fs_info->last_trans_committed;
a2de733c
AJ		 3970
3971 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
3972 bytenr = btrfs_sb_offset(i);
935e5cc9
MX		 3973 if (bytenr + BTRFS_SUPER_INFO_SIZE >
3974 scrub_dev->commit_total_bytes)
a2de733c 3975 break;
12659251
NA		 3976 if (!btrfs_check_super_location(scrub_dev, bytenr))
3977 continue;
a2de733c 3978
d9d181c1 3979 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
a36cf8b8 3980 scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
96e63a45 3981 NULL, bytenr);
a2de733c
AJ		 3982 if (ret)
3983 return ret;
3984 }
b6bfebc1 3985 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
a2de733c
AJ		 3986
3987 return 0;
3988}
3989
e89c4a9c
JB		 3990static void scrub_workers_put(struct btrfs_fs_info *fs_info)
3991{
3992 if (refcount_dec_and_mutex_lock(&fs_info->scrub_workers_refcnt,
3993 &fs_info->scrub_lock)) {
3994 struct btrfs_workqueue *scrub_workers = NULL;
3995 struct btrfs_workqueue *scrub_wr_comp = NULL;
3996 struct btrfs_workqueue *scrub_parity = NULL;
3997
3998 scrub_workers = fs_info->scrub_workers;
3999 scrub_wr_comp = fs_info->scrub_wr_completion_workers;
4000 scrub_parity = fs_info->scrub_parity_workers;
4001
4002 fs_info->scrub_workers = NULL;
4003 fs_info->scrub_wr_completion_workers = NULL;
4004 fs_info->scrub_parity_workers = NULL;
4005 mutex_unlock(&fs_info->scrub_lock);
4006
4007 btrfs_destroy_workqueue(scrub_workers);
4008 btrfs_destroy_workqueue(scrub_wr_comp);
4009 btrfs_destroy_workqueue(scrub_parity);
4010 }
4011}
4012
a2de733c
AJ		 4013/*
4014 * get a reference count on fs_info->scrub_workers. start worker if necessary
4015 */
ff023aac
SB		 4016static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
4017 int is_dev_replace)
a2de733c 4018{
e89c4a9c
JB		 4019 struct btrfs_workqueue *scrub_workers = NULL;
4020 struct btrfs_workqueue *scrub_wr_comp = NULL;
4021 struct btrfs_workqueue *scrub_parity = NULL;
6f011058 4022 unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
0339ef2f 4023 int max_active = fs_info->thread_pool_size;
e89c4a9c 4024 int ret = -ENOMEM;
a2de733c 4025
e89c4a9c
JB		 4026 if (refcount_inc_not_zero(&fs_info->scrub_workers_refcnt))
4027 return 0;
eb4318e5 4028
e89c4a9c
JB		 4029 scrub_workers = btrfs_alloc_workqueue(fs_info, "scrub", flags,
4030 is_dev_replace ? 1 : max_active, 4);
4031 if (!scrub_workers)
4032 goto fail_scrub_workers;
e82afc52 4033
e89c4a9c 4034 scrub_wr_comp = btrfs_alloc_workqueue(fs_info, "scrubwrc", flags,
20b2e302 4035 max_active, 2);
e89c4a9c
JB		 4036 if (!scrub_wr_comp)
4037 goto fail_scrub_wr_completion_workers;
ff09c4ca 4038
e89c4a9c
JB		 4039 scrub_parity = btrfs_alloc_workqueue(fs_info, "scrubparity", flags,
4040 max_active, 2);
4041 if (!scrub_parity)
4042 goto fail_scrub_parity_workers;
4043
4044 mutex_lock(&fs_info->scrub_lock);
4045 if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) {
4046 ASSERT(fs_info->scrub_workers == NULL &&
4047 fs_info->scrub_wr_completion_workers == NULL &&
4048 fs_info->scrub_parity_workers == NULL);
4049 fs_info->scrub_workers = scrub_workers;
4050 fs_info->scrub_wr_completion_workers = scrub_wr_comp;
4051 fs_info->scrub_parity_workers = scrub_parity;
ff09c4ca 4052 refcount_set(&fs_info->scrub_workers_refcnt, 1);
e89c4a9c
JB		 4053 mutex_unlock(&fs_info->scrub_lock);
4054 return 0;
632dd772 4055 }
e89c4a9c
JB		 4056 /* Other thread raced in and created the workers for us */
4057 refcount_inc(&fs_info->scrub_workers_refcnt);
4058 mutex_unlock(&fs_info->scrub_lock);
e82afc52 4059
e89c4a9c
JB		 4060 ret = 0;
4061 btrfs_destroy_workqueue(scrub_parity);
e82afc52 4062fail_scrub_parity_workers:
e89c4a9c 4063 btrfs_destroy_workqueue(scrub_wr_comp);
e82afc52 4064fail_scrub_wr_completion_workers:
e89c4a9c 4065 btrfs_destroy_workqueue(scrub_workers);
e82afc52 4066fail_scrub_workers:
e89c4a9c 4067 return ret;
a2de733c
AJ		 4068}
4069
aa1b8cd4
SB		 4070int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
4071 u64 end, struct btrfs_scrub_progress *progress,
63a212ab 4072 int readonly, int is_dev_replace)
a2de733c 4073{
562d7b15 4074 struct btrfs_dev_lookup_args args = { .devid = devid };
d9d181c1 4075 struct scrub_ctx *sctx;
a2de733c
AJ		 4076 int ret;
4077 struct btrfs_device *dev;
a5fb1142 4078 unsigned int nofs_flag;
a2de733c 4079
aa1b8cd4 4080 if (btrfs_fs_closing(fs_info))
6c3abeda 4081 return -EAGAIN;
a2de733c 4082
da17066c 4083 if (fs_info->nodesize > BTRFS_STRIPE_LEN) {
b5d67f64
SB		 4084 /*
4085 * in this case scrub is unable to calculate the checksum
4086 * the way scrub is implemented. Do not handle this
4087 * situation at all because it won't ever happen.
4088 */
efe120a0
FH		 4089 btrfs_err(fs_info,
4090 "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
da17066c
JM		 4091 fs_info->nodesize,
4092 BTRFS_STRIPE_LEN);
b5d67f64
SB		 4093 return -EINVAL;
4094 }
4095
da17066c 4096 if (fs_info->nodesize >
7a9e9987 4097 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
da17066c 4098 fs_info->sectorsize > PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
7a9e9987
SB		 4099 /*
4100 * would exhaust the array bounds of pagev member in
4101 * struct scrub_block
4102 */
5d163e0e
JM		 4103 btrfs_err(fs_info,
4104 "scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
da17066c 4105 fs_info->nodesize,
7a9e9987 4106 SCRUB_MAX_PAGES_PER_BLOCK,
da17066c 4107 fs_info->sectorsize,
7a9e9987
SB		 4108 SCRUB_MAX_PAGES_PER_BLOCK);
4109 return -EINVAL;
4110 }
4111
0e94c4f4
DS		 4112 /* Allocate outside of device_list_mutex */
4113 sctx = scrub_setup_ctx(fs_info, is_dev_replace);
4114 if (IS_ERR(sctx))
4115 return PTR_ERR(sctx);
a2de733c 4116
e89c4a9c
JB		 4117 ret = scrub_workers_get(fs_info, is_dev_replace);
4118 if (ret)
4119 goto out_free_ctx;
4120
aa1b8cd4 4121 mutex_lock(&fs_info->fs_devices->device_list_mutex);
562d7b15 4122 dev = btrfs_find_device(fs_info->fs_devices, &args);
e6e674bd
AJ		 4123 if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) &&
4124 !is_dev_replace)) {
aa1b8cd4 4125 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
0e94c4f4 4126 ret = -ENODEV;
e89c4a9c 4127 goto out;
a2de733c 4128 }
a2de733c 4129
ebbede42
AJ		 4130 if (!is_dev_replace && !readonly &&
4131 !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
5d68da3b 4132 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
a4852cf2
DS		 4133 btrfs_err_in_rcu(fs_info,
4134 "scrub on devid %llu: filesystem on %s is not writable",
4135 devid, rcu_str_deref(dev->name));
0e94c4f4 4136 ret = -EROFS;
e89c4a9c 4137 goto out;
5d68da3b
MX		 4138 }
4139
3b7a016f 4140 mutex_lock(&fs_info->scrub_lock);
e12c9621 4141 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
401e29c1 4142 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state)) {
a2de733c 4143 mutex_unlock(&fs_info->scrub_lock);
aa1b8cd4 4144 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
0e94c4f4 4145 ret = -EIO;
e89c4a9c 4146 goto out;
a2de733c
AJ		 4147 }
4148
cb5583dd 4149 down_read(&fs_info->dev_replace.rwsem);
cadbc0a0 4150 if (dev->scrub_ctx ||
8dabb742
SB		 4151 (!is_dev_replace &&
4152 btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
cb5583dd 4153 up_read(&fs_info->dev_replace.rwsem);
a2de733c 4154 mutex_unlock(&fs_info->scrub_lock);
aa1b8cd4 4155 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
0e94c4f4 4156 ret = -EINPROGRESS;
e89c4a9c 4157 goto out;
a2de733c 4158 }
cb5583dd 4159 up_read(&fs_info->dev_replace.rwsem);
3b7a016f 4160
d9d181c1 4161 sctx->readonly = readonly;
cadbc0a0 4162 dev->scrub_ctx = sctx;
3cb0929a 4163 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
a2de733c 4164
3cb0929a
WS		 4165 /*
4166 * checking @scrub_pause_req here, we can avoid
4167 * race between committing transaction and scrubbing.
4168 */
cb7ab021 4169 __scrub_blocked_if_needed(fs_info);
a2de733c
AJ		 4170 atomic_inc(&fs_info->scrubs_running);
4171 mutex_unlock(&fs_info->scrub_lock);
a2de733c 4172
a5fb1142
FM		 4173 /*
4174 * In order to avoid deadlock with reclaim when there is a transaction
4175 * trying to pause scrub, make sure we use GFP_NOFS for all the
4176 * allocations done at btrfs_scrub_pages() and scrub_pages_for_parity()
4177 * invoked by our callees. The pausing request is done when the
4178 * transaction commit starts, and it blocks the transaction until scrub
4179 * is paused (done at specific points at scrub_stripe() or right above
4180 * before incrementing fs_info->scrubs_running).
4181 */
4182 nofs_flag = memalloc_nofs_save();
ff023aac 4183 if (!is_dev_replace) {
d1e14420 4184 btrfs_info(fs_info, "scrub: started on devid %llu", devid);
9b011adf
WS		 4185 /*
4186 * by holding device list mutex, we can
4187 * kick off writing super in log tree sync.
4188 */
3cb0929a 4189 mutex_lock(&fs_info->fs_devices->device_list_mutex);
ff023aac 4190 ret = scrub_supers(sctx, dev);
3cb0929a 4191 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
ff023aac 4192 }
a2de733c
AJ		 4193
4194 if (!ret)
32934280 4195 ret = scrub_enumerate_chunks(sctx, dev, start, end);
a5fb1142 4196 memalloc_nofs_restore(nofs_flag);
a2de733c 4197
b6bfebc1 4198 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
a2de733c
AJ		 4199 atomic_dec(&fs_info->scrubs_running);
4200 wake_up(&fs_info->scrub_pause_wait);
4201
b6bfebc1 4202 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
0ef8e451 4203
a2de733c 4204 if (progress)
d9d181c1 4205 memcpy(progress, &sctx->stat, sizeof(*progress));
a2de733c 4206
d1e14420
AJ		 4207 if (!is_dev_replace)
4208 btrfs_info(fs_info, "scrub: %s on devid %llu with status: %d",
4209 ret ? "not finished" : "finished", devid, ret);
4210
a2de733c 4211 mutex_lock(&fs_info->scrub_lock);
cadbc0a0 4212 dev->scrub_ctx = NULL;
a2de733c
AJ		 4213 mutex_unlock(&fs_info->scrub_lock);
4214
e89c4a9c 4215 scrub_workers_put(fs_info);
f55985f4 4216 scrub_put_ctx(sctx);
a2de733c 4217
0e94c4f4 4218 return ret;
e89c4a9c
JB		 4219out:
4220 scrub_workers_put(fs_info);
0e94c4f4
DS		 4221out_free_ctx:
4222 scrub_free_ctx(sctx);
4223
a2de733c
AJ		 4224 return ret;
4225}
4226
2ff7e61e 4227void btrfs_scrub_pause(struct btrfs_fs_info *fs_info)
a2de733c 4228{
a2de733c
AJ		 4229 mutex_lock(&fs_info->scrub_lock);
4230 atomic_inc(&fs_info->scrub_pause_req);
4231 while (atomic_read(&fs_info->scrubs_paused) !=
4232 atomic_read(&fs_info->scrubs_running)) {
4233 mutex_unlock(&fs_info->scrub_lock);
4234 wait_event(fs_info->scrub_pause_wait,
4235 atomic_read(&fs_info->scrubs_paused) ==
4236 atomic_read(&fs_info->scrubs_running));
4237 mutex_lock(&fs_info->scrub_lock);
4238 }
4239 mutex_unlock(&fs_info->scrub_lock);
a2de733c
AJ		 4240}
4241
2ff7e61e 4242void btrfs_scrub_continue(struct btrfs_fs_info *fs_info)
a2de733c 4243{
a2de733c
AJ		 4244 atomic_dec(&fs_info->scrub_pause_req);
4245 wake_up(&fs_info->scrub_pause_wait);
a2de733c
AJ		 4246}
4247
aa1b8cd4 4248int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
a2de733c 4249{
a2de733c
AJ		 4250 mutex_lock(&fs_info->scrub_lock);
4251 if (!atomic_read(&fs_info->scrubs_running)) {
4252 mutex_unlock(&fs_info->scrub_lock);
4253 return -ENOTCONN;
4254 }
4255
4256 atomic_inc(&fs_info->scrub_cancel_req);
4257 while (atomic_read(&fs_info->scrubs_running)) {
4258 mutex_unlock(&fs_info->scrub_lock);
4259 wait_event(fs_info->scrub_pause_wait,
4260 atomic_read(&fs_info->scrubs_running) == 0);
4261 mutex_lock(&fs_info->scrub_lock);
4262 }
4263 atomic_dec(&fs_info->scrub_cancel_req);
4264 mutex_unlock(&fs_info->scrub_lock);
4265
4266 return 0;
4267}
4268
163e97ee 4269int btrfs_scrub_cancel_dev(struct btrfs_device *dev)
49b25e05 4270{
163e97ee 4271 struct btrfs_fs_info *fs_info = dev->fs_info;
d9d181c1 4272 struct scrub_ctx *sctx;
a2de733c
AJ		 4273
4274 mutex_lock(&fs_info->scrub_lock);
cadbc0a0 4275 sctx = dev->scrub_ctx;
d9d181c1 4276 if (!sctx) {
a2de733c
AJ		 4277 mutex_unlock(&fs_info->scrub_lock);
4278 return -ENOTCONN;
4279 }
d9d181c1 4280 atomic_inc(&sctx->cancel_req);
cadbc0a0 4281 while (dev->scrub_ctx) {
a2de733c
AJ		 4282 mutex_unlock(&fs_info->scrub_lock);
4283 wait_event(fs_info->scrub_pause_wait,
cadbc0a0 4284 dev->scrub_ctx == NULL);
a2de733c
AJ		 4285 mutex_lock(&fs_info->scrub_lock);
4286 }
4287 mutex_unlock(&fs_info->scrub_lock);
4288
4289 return 0;
4290}
1623edeb 4291
2ff7e61e 4292int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid,
a2de733c
AJ		 4293 struct btrfs_scrub_progress *progress)
4294{
562d7b15 4295 struct btrfs_dev_lookup_args args = { .devid = devid };
a2de733c 4296 struct btrfs_device *dev;
d9d181c1 4297 struct scrub_ctx *sctx = NULL;
a2de733c 4298
0b246afa 4299 mutex_lock(&fs_info->fs_devices->device_list_mutex);
562d7b15 4300 dev = btrfs_find_device(fs_info->fs_devices, &args);
a2de733c 4301 if (dev)
cadbc0a0 4302 sctx = dev->scrub_ctx;
d9d181c1
SB		 4303 if (sctx)
4304 memcpy(progress, &sctx->stat, sizeof(*progress));
0b246afa 4305 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
a2de733c 4306
d9d181c1 4307 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
a2de733c 4308}
ff023aac
SB		 4309
4310static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
fa485d21 4311 u64 extent_logical, u32 extent_len,
ff023aac
SB		 4312 u64 *extent_physical,
4313 struct btrfs_device **extent_dev,
4314 int *extent_mirror_num)
4315{
4316 u64 mapped_length;
4c664611 4317 struct btrfs_io_context *bioc = NULL;
ff023aac
SB		 4318 int ret;
4319
4320 mapped_length = extent_len;
cf8cddd3 4321 ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, extent_logical,
4c664611
QW		 4322 &mapped_length, &bioc, 0);
4323 if (ret || !bioc || mapped_length < extent_len ||
4324 !bioc->stripes[0].dev->bdev) {
4325 btrfs_put_bioc(bioc);
ff023aac
SB		 4326 return;
4327 }
4328
4c664611
QW		 4329 *extent_physical = bioc->stripes[0].physical;
4330 *extent_mirror_num = bioc->mirror_num;
4331 *extent_dev = bioc->stripes[0].dev;
4332 btrfs_put_bioc(bioc);
ff023aac 4333}
Linux 6.x block layer and io_uring tree(s)