projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
raid5: handle io error of batch list
[linux-2.6-block.git] / drivers / md / raid5.c
CommitLineData
1da177e4
LT		 1/*
2 * raid5.c : Multiple Devices driver for Linux
3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
4 * Copyright (C) 1999, 2000 Ingo Molnar
16a53ecc 5 * Copyright (C) 2002, 2003 H. Peter Anvin
1da177e4 6 *
16a53ecc
N		 7 * RAID-4/5/6 management functions.
8 * Thanks to Penguin Computing for making the RAID-6 development possible
9 * by donating a test server!
1da177e4
LT		 10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2, or (at your option)
14 * any later version.
15 *
16 * You should have received a copy of the GNU General Public License
17 * (for example /usr/src/linux/COPYING); if not, write to the Free
18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
ae3c20cc
N		 21/*
22 * BITMAP UNPLUGGING:
23 *
24 * The sequencing for updating the bitmap reliably is a little
25 * subtle (and I got it wrong the first time) so it deserves some
26 * explanation.
27 *
28 * We group bitmap updates into batches. Each batch has a number.
29 * We may write out several batches at once, but that isn't very important.
7c13edc8
N		 30 * conf->seq_write is the number of the last batch successfully written.
31 * conf->seq_flush is the number of the last batch that was closed to
ae3c20cc
N		 32 * new additions.
33 * When we discover that we will need to write to any block in a stripe
34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
7c13edc8 35 * the number of the batch it will be in. This is seq_flush+1.
ae3c20cc
N		 36 * When we are ready to do a write, if that batch hasn't been written yet,
37 * we plug the array and queue the stripe for later.
38 * When an unplug happens, we increment bm_flush, thus closing the current
39 * batch.
40 * When we notice that bm_flush > bm_write, we write out all pending updates
41 * to the bitmap, and advance bm_write to where bm_flush was.
42 * This may occasionally write a bit out twice, but is sure never to
43 * miss any bits.
44 */
1da177e4 45
bff61975 46#include <linux/blkdev.h>
f6705578 47#include <linux/kthread.h>
f701d589 48#include <linux/raid/pq.h>
91c00924 49#include <linux/async_tx.h>
056075c7 50#include <linux/module.h>
07a3b417 51#include <linux/async.h>
bff61975 52#include <linux/seq_file.h>
36d1c647 53#include <linux/cpu.h>
5a0e3ad6 54#include <linux/slab.h>
8bda470e 55#include <linux/ratelimit.h>
851c30c9 56#include <linux/nodemask.h>
46d5b785 57#include <linux/flex_array.h>
a9add5d9
N		 58#include <trace/events/block.h>
59
43b2e5d8 60#include "md.h"
bff61975 61#include "raid5.h"
54071b38 62#include "raid0.h"
ef740c37 63#include "bitmap.h"
72626685 64
851c30c9
SL		 65#define cpu_to_group(cpu) cpu_to_node(cpu)
66#define ANY_GROUP NUMA_NO_NODE
67
8e0e99ba
N		 68static bool devices_handle_discard_safely = false;
69module_param(devices_handle_discard_safely, bool, 0644);
70MODULE_PARM_DESC(devices_handle_discard_safely,
71 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
851c30c9 72static struct workqueue_struct *raid5_wq;
1da177e4
LT		 73/*
74 * Stripe cache
75 */
76
77#define NR_STRIPES 256
78#define STRIPE_SIZE PAGE_SIZE
79#define STRIPE_SHIFT (PAGE_SHIFT - 9)
80#define STRIPE_SECTORS (STRIPE_SIZE>>9)
81#define IO_THRESHOLD 1
8b3e6cdc 82#define BYPASS_THRESHOLD 1
fccddba0 83#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
1da177e4 84#define HASH_MASK (NR_HASH - 1)
bfc90cb0 85#define MAX_STRIPE_BATCH 8
1da177e4 86
d1688a6d 87static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
db298e19
N		 88{
89 int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
90 return &conf->stripe_hashtbl[hash];
91}
1da177e4 92
566c09c5
SL		 93static inline int stripe_hash_locks_hash(sector_t sect)
94{
95 return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
96}
97
98static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
99{
100 spin_lock_irq(conf->hash_locks + hash);
101 spin_lock(&conf->device_lock);
102}
103
104static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
105{
106 spin_unlock(&conf->device_lock);
107 spin_unlock_irq(conf->hash_locks + hash);
108}
109
110static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
111{
112 int i;
113 local_irq_disable();
114 spin_lock(conf->hash_locks);
115 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
116 spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
117 spin_lock(&conf->device_lock);
118}
119
120static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
121{
122 int i;
123 spin_unlock(&conf->device_lock);
124 for (i = NR_STRIPE_HASH_LOCKS; i; i--)
125 spin_unlock(conf->hash_locks + i - 1);
126 local_irq_enable();
127}
128
1da177e4
LT		 129/* bio's attached to a stripe+device for I/O are linked together in bi_sector
130 * order without overlap. There may be several bio's per stripe+device, and
131 * a bio could span several devices.
132 * When walking this list for a particular stripe+device, we must never proceed
133 * beyond a bio that extends past this device, as the next bio might no longer
134 * be valid.
db298e19 135 * This function is used to determine the 'next' bio in the list, given the sector
1da177e4
LT		 136 * of the current stripe+device
137 */
db298e19
N		 138static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
139{
aa8b57aa 140 int sectors = bio_sectors(bio);
4f024f37 141 if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
db298e19
N		 142 return bio->bi_next;
143 else
144 return NULL;
145}
1da177e4 146
960e739d 147/*
5b99c2ff
JA		 148 * We maintain a biased count of active stripes in the bottom 16 bits of
149 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
960e739d 150 */
e7836bd6 151static inline int raid5_bi_processed_stripes(struct bio *bio)
960e739d 152{
e7836bd6
SL		 153 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
154 return (atomic_read(segments) >> 16) & 0xffff;
960e739d
JA		 155}
156
e7836bd6 157static inline int raid5_dec_bi_active_stripes(struct bio *bio)
960e739d 158{
e7836bd6
SL		 159 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
160 return atomic_sub_return(1, segments) & 0xffff;
960e739d
JA		 161}
162
e7836bd6 163static inline void raid5_inc_bi_active_stripes(struct bio *bio)
960e739d 164{
e7836bd6
SL		 165 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
166 atomic_inc(segments);
960e739d
JA		 167}
168
e7836bd6
SL		 169static inline void raid5_set_bi_processed_stripes(struct bio *bio,
170 unsigned int cnt)
960e739d 171{
e7836bd6
SL		 172 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
173 int old, new;
960e739d 174
e7836bd6
SL		 175 do {
176 old = atomic_read(segments);
177 new = (old & 0xffff) | (cnt << 16);
178 } while (atomic_cmpxchg(segments, old, new) != old);
960e739d
JA		 179}
180
e7836bd6 181static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
960e739d 182{
e7836bd6
SL		 183 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
184 atomic_set(segments, cnt);
960e739d
JA		 185}
186
d0dabf7e
N		 187/* Find first data disk in a raid6 stripe */
188static inline int raid6_d0(struct stripe_head *sh)
189{
67cc2b81
N		 190 if (sh->ddf_layout)
191 /* ddf always start from first device */
192 return 0;
193 /* md starts just after Q block */
d0dabf7e
N		 194 if (sh->qd_idx == sh->disks - 1)
195 return 0;
196 else
197 return sh->qd_idx + 1;
198}
16a53ecc
N		 199static inline int raid6_next_disk(int disk, int raid_disks)
200{
201 disk++;
202 return (disk < raid_disks) ? disk : 0;
203}
a4456856 204
d0dabf7e
N		 205/* When walking through the disks in a raid5, starting at raid6_d0,
206 * We need to map each disk to a 'slot', where the data disks are slot
207 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
208 * is raid_disks-1. This help does that mapping.
209 */
67cc2b81
N		 210static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
211 int *count, int syndrome_disks)
d0dabf7e 212{
6629542e 213 int slot = *count;
67cc2b81 214
e4424fee 215 if (sh->ddf_layout)
6629542e 216 (*count)++;
d0dabf7e 217 if (idx == sh->pd_idx)
67cc2b81 218 return syndrome_disks;
d0dabf7e 219 if (idx == sh->qd_idx)
67cc2b81 220 return syndrome_disks + 1;
e4424fee 221 if (!sh->ddf_layout)
6629542e 222 (*count)++;
d0dabf7e
N		 223 return slot;
224}
225
a4456856
DW		 226static void return_io(struct bio *return_bi)
227{
228 struct bio *bi = return_bi;
229 while (bi) {
a4456856
DW		 230
231 return_bi = bi->bi_next;
232 bi->bi_next = NULL;
4f024f37 233 bi->bi_iter.bi_size = 0;
0a82a8d1
LT		 234 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
235 bi, 0);
0e13fe23 236 bio_endio(bi, 0);
a4456856
DW		 237 bi = return_bi;
238 }
239}
240
d1688a6d 241static void print_raid5_conf (struct r5conf *conf);
1da177e4 242
600aa109
DW		 243static int stripe_operations_active(struct stripe_head *sh)
244{
245 return sh->check_state || sh->reconstruct_state ||
246 test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
247 test_bit(STRIPE_COMPUTE_RUN, &sh->state);
248}
249
851c30c9
SL		 250static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
251{
252 struct r5conf *conf = sh->raid_conf;
253 struct r5worker_group *group;
bfc90cb0 254 int thread_cnt;
851c30c9
SL		 255 int i, cpu = sh->cpu;
256
257 if (!cpu_online(cpu)) {
258 cpu = cpumask_any(cpu_online_mask);
259 sh->cpu = cpu;
260 }
261
262 if (list_empty(&sh->lru)) {
263 struct r5worker_group *group;
264 group = conf->worker_groups + cpu_to_group(cpu);
265 list_add_tail(&sh->lru, &group->handle_list);
bfc90cb0
SL		 266 group->stripes_cnt++;
267 sh->group = group;
851c30c9
SL		 268 }
269
270 if (conf->worker_cnt_per_group == 0) {
271 md_wakeup_thread(conf->mddev->thread);
272 return;
273 }
274
275 group = conf->worker_groups + cpu_to_group(sh->cpu);
276
bfc90cb0
SL		 277 group->workers[0].working = true;
278 /* at least one worker should run to avoid race */
279 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
280
281 thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
282 /* wakeup more workers */
283 for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
284 if (group->workers[i].working == false) {
285 group->workers[i].working = true;
286 queue_work_on(sh->cpu, raid5_wq,
287 &group->workers[i].work);
288 thread_cnt--;
289 }
290 }
851c30c9
SL		 291}
292
566c09c5
SL		 293static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
294 struct list_head *temp_inactive_list)
1da177e4 295{
4eb788df
SL		 296 BUG_ON(!list_empty(&sh->lru));
297 BUG_ON(atomic_read(&conf->active_stripes)==0);
298 if (test_bit(STRIPE_HANDLE, &sh->state)) {
299 if (test_bit(STRIPE_DELAYED, &sh->state) &&
ad3ab8b6 300 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4eb788df 301 list_add_tail(&sh->lru, &conf->delayed_list);
ad3ab8b6 302 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
4eb788df
SL		 303 sh->bm_seq - conf->seq_write > 0)
304 list_add_tail(&sh->lru, &conf->bitmap_list);
305 else {
306 clear_bit(STRIPE_DELAYED, &sh->state);
307 clear_bit(STRIPE_BIT_DELAY, &sh->state);
851c30c9
SL		 308 if (conf->worker_cnt_per_group == 0) {
309 list_add_tail(&sh->lru, &conf->handle_list);
310 } else {
311 raid5_wakeup_stripe_thread(sh);
312 return;
313 }
4eb788df
SL		 314 }
315 md_wakeup_thread(conf->mddev->thread);
316 } else {
317 BUG_ON(stripe_operations_active(sh));
318 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
319 if (atomic_dec_return(&conf->preread_active_stripes)
320 < IO_THRESHOLD)
321 md_wakeup_thread(conf->mddev->thread);
322 atomic_dec(&conf->active_stripes);
566c09c5
SL		 323 if (!test_bit(STRIPE_EXPANDING, &sh->state))
324 list_add_tail(&sh->lru, temp_inactive_list);
1da177e4
LT		 325 }
326}
d0dabf7e 327
566c09c5
SL		 328static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
329 struct list_head *temp_inactive_list)
4eb788df
SL		 330{
331 if (atomic_dec_and_test(&sh->count))
566c09c5
SL		 332 do_release_stripe(conf, sh, temp_inactive_list);
333}
334
335/*
336 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
337 *
338 * Be careful: Only one task can add/delete stripes from temp_inactive_list at
339 * given time. Adding stripes only takes device lock, while deleting stripes
340 * only takes hash lock.
341 */
342static void release_inactive_stripe_list(struct r5conf *conf,
343 struct list_head *temp_inactive_list,
344 int hash)
345{
346 int size;
347 bool do_wakeup = false;
348 unsigned long flags;
349
350 if (hash == NR_STRIPE_HASH_LOCKS) {
351 size = NR_STRIPE_HASH_LOCKS;
352 hash = NR_STRIPE_HASH_LOCKS - 1;
353 } else
354 size = 1;
355 while (size) {
356 struct list_head *list = &temp_inactive_list[size - 1];
357
358 /*
359 * We don't hold any lock here yet, get_active_stripe() might
360 * remove stripes from the list
361 */
362 if (!list_empty_careful(list)) {
363 spin_lock_irqsave(conf->hash_locks + hash, flags);
4bda556a
SL		 364 if (list_empty(conf->inactive_list + hash) &&
365 !list_empty(list))
366 atomic_dec(&conf->empty_inactive_list_nr);
566c09c5
SL		 367 list_splice_tail_init(list, conf->inactive_list + hash);
368 do_wakeup = true;
369 spin_unlock_irqrestore(conf->hash_locks + hash, flags);
370 }
371 size--;
372 hash--;
373 }
374
375 if (do_wakeup) {
376 wake_up(&conf->wait_for_stripe);
377 if (conf->retry_read_aligned)
378 md_wakeup_thread(conf->mddev->thread);
379 }
4eb788df
SL		 380}
381
773ca82f 382/* should hold conf->device_lock already */
566c09c5
SL		 383static int release_stripe_list(struct r5conf *conf,
384 struct list_head *temp_inactive_list)
773ca82f
SL		 385{
386 struct stripe_head *sh;
387 int count = 0;
388 struct llist_node *head;
389
390 head = llist_del_all(&conf->released_stripes);
d265d9dc 391 head = llist_reverse_order(head);
773ca82f 392 while (head) {
566c09c5
SL		 393 int hash;
394
773ca82f
SL		 395 sh = llist_entry(head, struct stripe_head, release_list);
396 head = llist_next(head);
397 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
398 smp_mb();
399 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
400 /*
401 * Don't worry the bit is set here, because if the bit is set
402 * again, the count is always > 1. This is true for
403 * STRIPE_ON_UNPLUG_LIST bit too.
404 */
566c09c5
SL		 405 hash = sh->hash_lock_index;
406 __release_stripe(conf, sh, &temp_inactive_list[hash]);
773ca82f
SL		 407 count++;
408 }
409
410 return count;
411}
412
1da177e4
LT		 413static void release_stripe(struct stripe_head *sh)
414{
d1688a6d 415 struct r5conf *conf = sh->raid_conf;
1da177e4 416 unsigned long flags;
566c09c5
SL		 417 struct list_head list;
418 int hash;
773ca82f 419 bool wakeup;
16a53ecc 420
cf170f3f
ES		 421 /* Avoid release_list until the last reference.
422 */
423 if (atomic_add_unless(&sh->count, -1, 1))
424 return;
425
ad4068de 426 if (unlikely(!conf->mddev->thread) ||
427 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
773ca82f
SL		 428 goto slow_path;
429 wakeup = llist_add(&sh->release_list, &conf->released_stripes);
430 if (wakeup)
431 md_wakeup_thread(conf->mddev->thread);
432 return;
433slow_path:
4eb788df 434 local_irq_save(flags);
773ca82f 435 /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
4eb788df 436 if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
566c09c5
SL		 437 INIT_LIST_HEAD(&list);
438 hash = sh->hash_lock_index;
439 do_release_stripe(conf, sh, &list);
4eb788df 440 spin_unlock(&conf->device_lock);
566c09c5 441 release_inactive_stripe_list(conf, &list, hash);
4eb788df
SL		 442 }
443 local_irq_restore(flags);
1da177e4
LT		 444}
445
fccddba0 446static inline void remove_hash(struct stripe_head *sh)
1da177e4 447{
45b4233c
DW		 448 pr_debug("remove_hash(), stripe %llu\n",
449 (unsigned long long)sh->sector);
1da177e4 450
fccddba0 451 hlist_del_init(&sh->hash);
1da177e4
LT		 452}
453
d1688a6d 454static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
1da177e4 455{
fccddba0 456 struct hlist_head *hp = stripe_hash(conf, sh->sector);
1da177e4 457
45b4233c
DW		 458 pr_debug("insert_hash(), stripe %llu\n",
459 (unsigned long long)sh->sector);
1da177e4 460
fccddba0 461 hlist_add_head(&sh->hash, hp);
1da177e4
LT		 462}
463
1da177e4 464/* find an idle stripe, make sure it is unhashed, and return it. */
566c09c5 465static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
1da177e4
LT		 466{
467 struct stripe_head *sh = NULL;
468 struct list_head *first;
469
566c09c5 470 if (list_empty(conf->inactive_list + hash))
1da177e4 471 goto out;
566c09c5 472 first = (conf->inactive_list + hash)->next;
1da177e4
LT		 473 sh = list_entry(first, struct stripe_head, lru);
474 list_del_init(first);
475 remove_hash(sh);
476 atomic_inc(&conf->active_stripes);
566c09c5 477 BUG_ON(hash != sh->hash_lock_index);
4bda556a
SL		 478 if (list_empty(conf->inactive_list + hash))
479 atomic_inc(&conf->empty_inactive_list_nr);
1da177e4
LT		 480out:
481 return sh;
482}
483
e4e11e38 484static void shrink_buffers(struct stripe_head *sh)
1da177e4
LT		 485{
486 struct page *p;
487 int i;
e4e11e38 488 int num = sh->raid_conf->pool_size;
1da177e4 489
e4e11e38 490 for (i = 0; i < num ; i++) {
d592a996 491 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
1da177e4
LT		 492 p = sh->dev[i].page;
493 if (!p)
494 continue;
495 sh->dev[i].page = NULL;
2d1f3b5d 496 put_page(p);
1da177e4
LT		 497 }
498}
499
e4e11e38 500static int grow_buffers(struct stripe_head *sh)
1da177e4
LT		 501{
502 int i;
e4e11e38 503 int num = sh->raid_conf->pool_size;
1da177e4 504
e4e11e38 505 for (i = 0; i < num; i++) {
1da177e4
LT		 506 struct page *page;
507
508 if (!(page = alloc_page(GFP_KERNEL))) {
509 return 1;
510 }
511 sh->dev[i].page = page;
d592a996 512 sh->dev[i].orig_page = page;
1da177e4
LT		 513 }
514 return 0;
515}
516
784052ec 517static void raid5_build_block(struct stripe_head *sh, int i, int previous);
d1688a6d 518static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
911d4ee8 519 struct stripe_head *sh);
1da177e4 520
b5663ba4 521static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
1da177e4 522{
d1688a6d 523 struct r5conf *conf = sh->raid_conf;
566c09c5 524 int i, seq;
1da177e4 525
78bafebd
ES		 526 BUG_ON(atomic_read(&sh->count) != 0);
527 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
600aa109 528 BUG_ON(stripe_operations_active(sh));
59fc630b 529 BUG_ON(sh->batch_head);
d84e0f10 530
45b4233c 531 pr_debug("init_stripe called, stripe %llu\n",
b8e6a15a 532 (unsigned long long)sector);
566c09c5
SL		 533retry:
534 seq = read_seqcount_begin(&conf->gen_lock);
86b42c71 535 sh->generation = conf->generation - previous;
b5663ba4 536 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
1da177e4 537 sh->sector = sector;
911d4ee8 538 stripe_set_idx(sector, conf, previous, sh);
1da177e4
LT		 539 sh->state = 0;
540
7ecaa1e6 541 for (i = sh->disks; i--; ) {
1da177e4
LT		 542 struct r5dev *dev = &sh->dev[i];
543
d84e0f10 544 if (dev->toread || dev->read || dev->towrite || dev->written ||
1da177e4 545 test_bit(R5_LOCKED, &dev->flags)) {
d84e0f10 546 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
1da177e4 547 (unsigned long long)sh->sector, i, dev->toread,
d84e0f10 548 dev->read, dev->towrite, dev->written,
1da177e4 549 test_bit(R5_LOCKED, &dev->flags));
8cfa7b0f 550 WARN_ON(1);
1da177e4
LT		 551 }
552 dev->flags = 0;
784052ec 553 raid5_build_block(sh, i, previous);
1da177e4 554 }
566c09c5
SL		 555 if (read_seqcount_retry(&conf->gen_lock, seq))
556 goto retry;
7a87f434 557 sh->overwrite_disks = 0;
1da177e4 558 insert_hash(conf, sh);
851c30c9 559 sh->cpu = smp_processor_id();
da41ba65 560 set_bit(STRIPE_BATCH_READY, &sh->state);
1da177e4
LT		 561}
562
d1688a6d 563static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
86b42c71 564 short generation)
1da177e4
LT		 565{
566 struct stripe_head *sh;
567
45b4233c 568 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
b67bfe0d 569 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
86b42c71 570 if (sh->sector == sector && sh->generation == generation)
1da177e4 571 return sh;
45b4233c 572 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
1da177e4
LT		 573 return NULL;
574}
575
674806d6
N		 576/*
577 * Need to check if array has failed when deciding whether to:
578 * - start an array
579 * - remove non-faulty devices
580 * - add a spare
581 * - allow a reshape
582 * This determination is simple when no reshape is happening.
583 * However if there is a reshape, we need to carefully check
584 * both the before and after sections.
585 * This is because some failed devices may only affect one
586 * of the two sections, and some non-in_sync devices may
587 * be insync in the section most affected by failed devices.
588 */
908f4fbd 589static int calc_degraded(struct r5conf *conf)
674806d6 590{
908f4fbd 591 int degraded, degraded2;
674806d6 592 int i;
674806d6
N		 593
594 rcu_read_lock();
595 degraded = 0;
596 for (i = 0; i < conf->previous_raid_disks; i++) {
3cb03002 597 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
e5c86471
N		 598 if (rdev && test_bit(Faulty, &rdev->flags))
599 rdev = rcu_dereference(conf->disks[i].replacement);
674806d6
N		 600 if (!rdev || test_bit(Faulty, &rdev->flags))
601 degraded++;
602 else if (test_bit(In_sync, &rdev->flags))
603 ;
604 else
605 /* not in-sync or faulty.
606 * If the reshape increases the number of devices,
607 * this is being recovered by the reshape, so
608 * this 'previous' section is not in_sync.
609 * If the number of devices is being reduced however,
610 * the device can only be part of the array if
611 * we are reverting a reshape, so this section will
612 * be in-sync.
613 */
614 if (conf->raid_disks >= conf->previous_raid_disks)
615 degraded++;
616 }
617 rcu_read_unlock();
908f4fbd
N		 618 if (conf->raid_disks == conf->previous_raid_disks)
619 return degraded;
674806d6 620 rcu_read_lock();
908f4fbd 621 degraded2 = 0;
674806d6 622 for (i = 0; i < conf->raid_disks; i++) {
3cb03002 623 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
e5c86471
N		 624 if (rdev && test_bit(Faulty, &rdev->flags))
625 rdev = rcu_dereference(conf->disks[i].replacement);
674806d6 626 if (!rdev || test_bit(Faulty, &rdev->flags))
908f4fbd 627 degraded2++;
674806d6
N		 628 else if (test_bit(In_sync, &rdev->flags))
629 ;
630 else
631 /* not in-sync or faulty.
632 * If reshape increases the number of devices, this
633 * section has already been recovered, else it
634 * almost certainly hasn't.
635 */
636 if (conf->raid_disks <= conf->previous_raid_disks)
908f4fbd 637 degraded2++;
674806d6
N		 638 }
639 rcu_read_unlock();
908f4fbd
N		 640 if (degraded2 > degraded)
641 return degraded2;
642 return degraded;
643}
644
645static int has_failed(struct r5conf *conf)
646{
647 int degraded;
648
649 if (conf->mddev->reshape_position == MaxSector)
650 return conf->mddev->degraded > conf->max_degraded;
651
652 degraded = calc_degraded(conf);
674806d6
N		 653 if (degraded > conf->max_degraded)
654 return 1;
655 return 0;
656}
657
b5663ba4 658static struct stripe_head *
d1688a6d 659get_active_stripe(struct r5conf *conf, sector_t sector,
a8c906ca 660 int previous, int noblock, int noquiesce)
1da177e4
LT		 661{
662 struct stripe_head *sh;
566c09c5 663 int hash = stripe_hash_locks_hash(sector);
1da177e4 664
45b4233c 665 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
1da177e4 666
566c09c5 667 spin_lock_irq(conf->hash_locks + hash);
1da177e4
LT		 668
669 do {
72626685 670 wait_event_lock_irq(conf->wait_for_stripe,
a8c906ca 671 conf->quiesce == 0 || noquiesce,
566c09c5 672 *(conf->hash_locks + hash));
86b42c71 673 sh = __find_stripe(conf, sector, conf->generation - previous);
1da177e4
LT		 674 if (!sh) {
675 if (!conf->inactive_blocked)
566c09c5 676 sh = get_free_stripe(conf, hash);
1da177e4
LT		 677 if (noblock && sh == NULL)
678 break;
679 if (!sh) {
680 conf->inactive_blocked = 1;
566c09c5
SL		 681 wait_event_lock_irq(
682 conf->wait_for_stripe,
683 !list_empty(conf->inactive_list + hash) &&
684 (atomic_read(&conf->active_stripes)
685 < (conf->max_nr_stripes * 3 / 4)
686 || !conf->inactive_blocked),
687 *(conf->hash_locks + hash));
1da177e4 688 conf->inactive_blocked = 0;
7da9d450 689 } else {
b5663ba4 690 init_stripe(sh, sector, previous);
7da9d450
N		 691 atomic_inc(&sh->count);
692 }
e240c183 693 } else if (!atomic_inc_not_zero(&sh->count)) {
6d183de4 694 spin_lock(&conf->device_lock);
e240c183 695 if (!atomic_read(&sh->count)) {
1da177e4
LT		 696 if (!test_bit(STRIPE_HANDLE, &sh->state))
697 atomic_inc(&conf->active_stripes);
5af9bef7
N		 698 BUG_ON(list_empty(&sh->lru) &&
699 !test_bit(STRIPE_EXPANDING, &sh->state));
16a53ecc 700 list_del_init(&sh->lru);
bfc90cb0
SL		 701 if (sh->group) {
702 sh->group->stripes_cnt--;
703 sh->group = NULL;
704 }
1da177e4 705 }
7da9d450 706 atomic_inc(&sh->count);
6d183de4 707 spin_unlock(&conf->device_lock);
1da177e4
LT		 708 }
709 } while (sh == NULL);
710
566c09c5 711 spin_unlock_irq(conf->hash_locks + hash);
1da177e4
LT		 712 return sh;
713}
714
7a87f434 715static bool is_full_stripe_write(struct stripe_head *sh)
716{
717 BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
718 return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
719}
720
59fc630b 721static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
722{
723 local_irq_disable();
724 if (sh1 > sh2) {
725 spin_lock(&sh2->stripe_lock);
726 spin_lock_nested(&sh1->stripe_lock, 1);
727 } else {
728 spin_lock(&sh1->stripe_lock);
729 spin_lock_nested(&sh2->stripe_lock, 1);
730 }
731}
732
733static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
734{
735 spin_unlock(&sh1->stripe_lock);
736 spin_unlock(&sh2->stripe_lock);
737 local_irq_enable();
738}
739
740/* Only freshly new full stripe normal write stripe can be added to a batch list */
741static bool stripe_can_batch(struct stripe_head *sh)
742{
743 return test_bit(STRIPE_BATCH_READY, &sh->state) &&
744 is_full_stripe_write(sh);
745}
746
747/* we only do back search */
748static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)
749{
750 struct stripe_head *head;
751 sector_t head_sector, tmp_sec;
752 int hash;
753 int dd_idx;
754
755 if (!stripe_can_batch(sh))
756 return;
757 /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
758 tmp_sec = sh->sector;
759 if (!sector_div(tmp_sec, conf->chunk_sectors))
760 return;
761 head_sector = sh->sector - STRIPE_SECTORS;
762
763 hash = stripe_hash_locks_hash(head_sector);
764 spin_lock_irq(conf->hash_locks + hash);
765 head = __find_stripe(conf, head_sector, conf->generation);
766 if (head && !atomic_inc_not_zero(&head->count)) {
767 spin_lock(&conf->device_lock);
768 if (!atomic_read(&head->count)) {
769 if (!test_bit(STRIPE_HANDLE, &head->state))
770 atomic_inc(&conf->active_stripes);
771 BUG_ON(list_empty(&head->lru) &&
772 !test_bit(STRIPE_EXPANDING, &head->state));
773 list_del_init(&head->lru);
774 if (head->group) {
775 head->group->stripes_cnt--;
776 head->group = NULL;
777 }
778 }
779 atomic_inc(&head->count);
780 spin_unlock(&conf->device_lock);
781 }
782 spin_unlock_irq(conf->hash_locks + hash);
783
784 if (!head)
785 return;
786 if (!stripe_can_batch(head))
787 goto out;
788
789 lock_two_stripes(head, sh);
790 /* clear_batch_ready clear the flag */
791 if (!stripe_can_batch(head) || !stripe_can_batch(sh))
792 goto unlock_out;
793
794 if (sh->batch_head)
795 goto unlock_out;
796
797 dd_idx = 0;
798 while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
799 dd_idx++;
800 if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw)
801 goto unlock_out;
802
803 if (head->batch_head) {
804 spin_lock(&head->batch_head->batch_lock);
805 /* This batch list is already running */
806 if (!stripe_can_batch(head)) {
807 spin_unlock(&head->batch_head->batch_lock);
808 goto unlock_out;
809 }
810
811 /*
812 * at this point, head's BATCH_READY could be cleared, but we
813 * can still add the stripe to batch list
814 */
815 list_add(&sh->batch_list, &head->batch_list);
816 spin_unlock(&head->batch_head->batch_lock);
817
818 sh->batch_head = head->batch_head;
819 } else {
820 head->batch_head = head;
821 sh->batch_head = head->batch_head;
822 spin_lock(&head->batch_lock);
823 list_add_tail(&sh->batch_list, &head->batch_list);
824 spin_unlock(&head->batch_lock);
825 }
826
827 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
828 if (atomic_dec_return(&conf->preread_active_stripes)
829 < IO_THRESHOLD)
830 md_wakeup_thread(conf->mddev->thread);
831
832 atomic_inc(&sh->count);
833unlock_out:
834 unlock_two_stripes(head, sh);
835out:
836 release_stripe(head);
837}
838
05616be5
N		 839/* Determine if 'data_offset' or 'new_data_offset' should be used
840 * in this stripe_head.
841 */
842static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
843{
844 sector_t progress = conf->reshape_progress;
845 /* Need a memory barrier to make sure we see the value
846 * of conf->generation, or ->data_offset that was set before
847 * reshape_progress was updated.
848 */
849 smp_rmb();
850 if (progress == MaxSector)
851 return 0;
852 if (sh->generation == conf->generation - 1)
853 return 0;
854 /* We are in a reshape, and this is a new-generation stripe,
855 * so use new_data_offset.
856 */
857 return 1;
858}
859
6712ecf8
N		 860static void
861raid5_end_read_request(struct bio *bi, int error);
862static void
863raid5_end_write_request(struct bio *bi, int error);
91c00924 864
c4e5ac0a 865static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
91c00924 866{
d1688a6d 867 struct r5conf *conf = sh->raid_conf;
91c00924 868 int i, disks = sh->disks;
59fc630b 869 struct stripe_head *head_sh = sh;
91c00924
DW		 870
871 might_sleep();
872
873 for (i = disks; i--; ) {
874 int rw;
9a3e1101 875 int replace_only = 0;
977df362
N		 876 struct bio *bi, *rbi;
877 struct md_rdev *rdev, *rrdev = NULL;
59fc630b 878
879 sh = head_sh;
e9c7469b
TH		 880 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
881 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
882 rw = WRITE_FUA;
883 else
884 rw = WRITE;
9e444768 885 if (test_bit(R5_Discard, &sh->dev[i].flags))
620125f2 886 rw |= REQ_DISCARD;
e9c7469b 887 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
91c00924 888 rw = READ;
9a3e1101
N		 889 else if (test_and_clear_bit(R5_WantReplace,
890 &sh->dev[i].flags)) {
891 rw = WRITE;
892 replace_only = 1;
893 } else
91c00924 894 continue;
bc0934f0
SL		 895 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
896 rw |= REQ_SYNC;
91c00924 897
59fc630b 898again:
91c00924 899 bi = &sh->dev[i].req;
977df362 900 rbi = &sh->dev[i].rreq; /* For writing to replacement */
91c00924 901
91c00924 902 rcu_read_lock();
9a3e1101 903 rrdev = rcu_dereference(conf->disks[i].replacement);
dd054fce
N		 904 smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
905 rdev = rcu_dereference(conf->disks[i].rdev);
906 if (!rdev) {
907 rdev = rrdev;
908 rrdev = NULL;
909 }
9a3e1101
N		 910 if (rw & WRITE) {
911 if (replace_only)
912 rdev = NULL;
dd054fce
N		 913 if (rdev == rrdev)
914 /* We raced and saw duplicates */
915 rrdev = NULL;
9a3e1101 916 } else {
59fc630b 917 if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)
9a3e1101
N		 918 rdev = rrdev;
919 rrdev = NULL;
920 }
977df362 921
91c00924
DW		 922 if (rdev && test_bit(Faulty, &rdev->flags))
923 rdev = NULL;
924 if (rdev)
925 atomic_inc(&rdev->nr_pending);
977df362
N		 926 if (rrdev && test_bit(Faulty, &rrdev->flags))
927 rrdev = NULL;
928 if (rrdev)
929 atomic_inc(&rrdev->nr_pending);
91c00924
DW		 930 rcu_read_unlock();
931
73e92e51 932 /* We have already checked bad blocks for reads. Now
977df362
N		 933 * need to check for writes. We never accept write errors
934 * on the replacement, so we don't to check rrdev.
73e92e51
N		 935 */
936 while ((rw & WRITE) && rdev &&
937 test_bit(WriteErrorSeen, &rdev->flags)) {
938 sector_t first_bad;
939 int bad_sectors;
940 int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
941 &first_bad, &bad_sectors);
942 if (!bad)
943 break;
944
945 if (bad < 0) {
946 set_bit(BlockedBadBlocks, &rdev->flags);
947 if (!conf->mddev->external &&
948 conf->mddev->flags) {
949 /* It is very unlikely, but we might
950 * still need to write out the
951 * bad block log - better give it
952 * a chance*/
953 md_check_recovery(conf->mddev);
954 }
1850753d 955 /*
956 * Because md_wait_for_blocked_rdev
957 * will dec nr_pending, we must
958 * increment it first.
959 */
960 atomic_inc(&rdev->nr_pending);
73e92e51
N		 961 md_wait_for_blocked_rdev(rdev, conf->mddev);
962 } else {
963 /* Acknowledged bad block - skip the write */
964 rdev_dec_pending(rdev, conf->mddev);
965 rdev = NULL;
966 }
967 }
968
91c00924 969 if (rdev) {
9a3e1101
N		 970 if (s->syncing || s->expanding || s->expanded
971 || s->replacing)
91c00924
DW		 972 md_sync_acct(rdev->bdev, STRIPE_SECTORS);
973
2b7497f0
DW		 974 set_bit(STRIPE_IO_STARTED, &sh->state);
975
2f6db2a7 976 bio_reset(bi);
91c00924 977 bi->bi_bdev = rdev->bdev;
2f6db2a7
KO		 978 bi->bi_rw = rw;
979 bi->bi_end_io = (rw & WRITE)
980 ? raid5_end_write_request
981 : raid5_end_read_request;
982 bi->bi_private = sh;
983
91c00924 984 pr_debug("%s: for %llu schedule op %ld on disc %d\n",
e46b272b 985 __func__, (unsigned long long)sh->sector,
91c00924
DW		 986 bi->bi_rw, i);
987 atomic_inc(&sh->count);
59fc630b 988 if (sh != head_sh)
989 atomic_inc(&head_sh->count);
05616be5 990 if (use_new_offset(conf, sh))
4f024f37 991 bi->bi_iter.bi_sector = (sh->sector
05616be5
N		 992 + rdev->new_data_offset);
993 else
4f024f37 994 bi->bi_iter.bi_sector = (sh->sector
05616be5 995 + rdev->data_offset);
59fc630b 996 if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))
e59aa23f 997 bi->bi_rw |= REQ_NOMERGE;
3f9e7c14 998
d592a996
SL		 999 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1000 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1001 sh->dev[i].vec.bv_page = sh->dev[i].page;
4997b72e 1002 bi->bi_vcnt = 1;
91c00924
DW		 1003 bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1004 bi->bi_io_vec[0].bv_offset = 0;
4f024f37 1005 bi->bi_iter.bi_size = STRIPE_SIZE;
37c61ff3
SL		 1006 /*
1007 * If this is discard request, set bi_vcnt 0. We don't
1008 * want to confuse SCSI because SCSI will replace payload
1009 */
1010 if (rw & REQ_DISCARD)
1011 bi->bi_vcnt = 0;
977df362
N		 1012 if (rrdev)
1013 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
e3620a3a
JB		 1014
1015 if (conf->mddev->gendisk)
1016 trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
1017 bi, disk_devt(conf->mddev->gendisk),
1018 sh->dev[i].sector);
91c00924 1019 generic_make_request(bi);
977df362
N		 1020 }
1021 if (rrdev) {
9a3e1101
N		 1022 if (s->syncing || s->expanding || s->expanded
1023 || s->replacing)
977df362
N		 1024 md_sync_acct(rrdev->bdev, STRIPE_SECTORS);
1025
1026 set_bit(STRIPE_IO_STARTED, &sh->state);
1027
2f6db2a7 1028 bio_reset(rbi);
977df362 1029 rbi->bi_bdev = rrdev->bdev;
2f6db2a7
KO		 1030 rbi->bi_rw = rw;
1031 BUG_ON(!(rw & WRITE));
1032 rbi->bi_end_io = raid5_end_write_request;
1033 rbi->bi_private = sh;
1034
977df362
N		 1035 pr_debug("%s: for %llu schedule op %ld on "
1036 "replacement disc %d\n",
1037 __func__, (unsigned long long)sh->sector,
1038 rbi->bi_rw, i);
1039 atomic_inc(&sh->count);
59fc630b 1040 if (sh != head_sh)
1041 atomic_inc(&head_sh->count);
05616be5 1042 if (use_new_offset(conf, sh))
4f024f37 1043 rbi->bi_iter.bi_sector = (sh->sector
05616be5
N		 1044 + rrdev->new_data_offset);
1045 else
4f024f37 1046 rbi->bi_iter.bi_sector = (sh->sector
05616be5 1047 + rrdev->data_offset);
d592a996
SL		 1048 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1049 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1050 sh->dev[i].rvec.bv_page = sh->dev[i].page;
4997b72e 1051 rbi->bi_vcnt = 1;
977df362
N		 1052 rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
1053 rbi->bi_io_vec[0].bv_offset = 0;
4f024f37 1054 rbi->bi_iter.bi_size = STRIPE_SIZE;
37c61ff3
SL		 1055 /*
1056 * If this is discard request, set bi_vcnt 0. We don't
1057 * want to confuse SCSI because SCSI will replace payload
1058 */
1059 if (rw & REQ_DISCARD)
1060 rbi->bi_vcnt = 0;
e3620a3a
JB		 1061 if (conf->mddev->gendisk)
1062 trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
1063 rbi, disk_devt(conf->mddev->gendisk),
1064 sh->dev[i].sector);
977df362
N		 1065 generic_make_request(rbi);
1066 }
1067 if (!rdev && !rrdev) {
b062962e 1068 if (rw & WRITE)
91c00924
DW		 1069 set_bit(STRIPE_DEGRADED, &sh->state);
1070 pr_debug("skip op %ld on disc %d for sector %llu\n",
1071 bi->bi_rw, i, (unsigned long long)sh->sector);
1072 clear_bit(R5_LOCKED, &sh->dev[i].flags);
72ac7330 1073 if (sh->batch_head)
1074 set_bit(STRIPE_BATCH_ERR,
1075 &sh->batch_head->state);
91c00924
DW		 1076 set_bit(STRIPE_HANDLE, &sh->state);
1077 }
59fc630b 1078
1079 if (!head_sh->batch_head)
1080 continue;
1081 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1082 batch_list);
1083 if (sh != head_sh)
1084 goto again;
91c00924
DW		 1085 }
1086}
1087
1088static struct dma_async_tx_descriptor *
d592a996
SL		 1089async_copy_data(int frombio, struct bio *bio, struct page **page,
1090 sector_t sector, struct dma_async_tx_descriptor *tx,
1091 struct stripe_head *sh)
91c00924 1092{
7988613b
KO		 1093 struct bio_vec bvl;
1094 struct bvec_iter iter;
91c00924 1095 struct page *bio_page;
91c00924 1096 int page_offset;
a08abd8c 1097 struct async_submit_ctl submit;
0403e382 1098 enum async_tx_flags flags = 0;
91c00924 1099
4f024f37
KO		 1100 if (bio->bi_iter.bi_sector >= sector)
1101 page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
91c00924 1102 else
4f024f37 1103 page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
a08abd8c 1104
0403e382
DW		 1105 if (frombio)
1106 flags |= ASYNC_TX_FENCE;
1107 init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
1108
7988613b
KO		 1109 bio_for_each_segment(bvl, bio, iter) {
1110 int len = bvl.bv_len;
91c00924
DW		 1111 int clen;
1112 int b_offset = 0;
1113
1114 if (page_offset < 0) {
1115 b_offset = -page_offset;
1116 page_offset += b_offset;
1117 len -= b_offset;
1118 }
1119
1120 if (len > 0 && page_offset + len > STRIPE_SIZE)
1121 clen = STRIPE_SIZE - page_offset;
1122 else
1123 clen = len;
1124
1125 if (clen > 0) {
7988613b
KO		 1126 b_offset += bvl.bv_offset;
1127 bio_page = bvl.bv_page;
d592a996
SL		 1128 if (frombio) {
1129 if (sh->raid_conf->skip_copy &&
1130 b_offset == 0 && page_offset == 0 &&
1131 clen == STRIPE_SIZE)
1132 *page = bio_page;
1133 else
1134 tx = async_memcpy(*page, bio_page, page_offset,
a08abd8c 1135 b_offset, clen, &submit);
d592a996
SL		 1136 } else
1137 tx = async_memcpy(bio_page, *page, b_offset,
a08abd8c 1138 page_offset, clen, &submit);
91c00924 1139 }
a08abd8c
DW		 1140 /* chain the operations */
1141 submit.depend_tx = tx;
1142
91c00924
DW		 1143 if (clen < len) /* hit end of page */
1144 break;
1145 page_offset += len;
1146 }
1147
1148 return tx;
1149}
1150
1151static void ops_complete_biofill(void *stripe_head_ref)
1152{
1153 struct stripe_head *sh = stripe_head_ref;
1154 struct bio *return_bi = NULL;
e4d84909 1155 int i;
91c00924 1156
e46b272b 1157 pr_debug("%s: stripe %llu\n", __func__,
91c00924
DW		 1158 (unsigned long long)sh->sector);
1159
1160 /* clear completed biofills */
1161 for (i = sh->disks; i--; ) {
1162 struct r5dev *dev = &sh->dev[i];
91c00924
DW		 1163
1164 /* acknowledge completion of a biofill operation */
e4d84909
DW		 1165 /* and check if we need to reply to a read request,
1166 * new R5_Wantfill requests are held off until
83de75cc 1167 * !STRIPE_BIOFILL_RUN
e4d84909
DW		 1168 */
1169 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
91c00924 1170 struct bio *rbi, *rbi2;
91c00924 1171
91c00924
DW		 1172 BUG_ON(!dev->read);
1173 rbi = dev->read;
1174 dev->read = NULL;
4f024f37 1175 while (rbi && rbi->bi_iter.bi_sector <
91c00924
DW		 1176 dev->sector + STRIPE_SECTORS) {
1177 rbi2 = r5_next_bio(rbi, dev->sector);
e7836bd6 1178 if (!raid5_dec_bi_active_stripes(rbi)) {
91c00924
DW		 1179 rbi->bi_next = return_bi;
1180 return_bi = rbi;
1181 }
91c00924
DW		 1182 rbi = rbi2;
1183 }
1184 }
1185 }
83de75cc 1186 clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
91c00924
DW		 1187
1188 return_io(return_bi);
1189
e4d84909 1190 set_bit(STRIPE_HANDLE, &sh->state);
91c00924
DW		 1191 release_stripe(sh);
1192}
1193
1194static void ops_run_biofill(struct stripe_head *sh)
1195{
1196 struct dma_async_tx_descriptor *tx = NULL;
a08abd8c 1197 struct async_submit_ctl submit;
91c00924
DW		 1198 int i;
1199
59fc630b 1200 BUG_ON(sh->batch_head);
e46b272b 1201 pr_debug("%s: stripe %llu\n", __func__,
91c00924
DW		 1202 (unsigned long long)sh->sector);
1203
1204 for (i = sh->disks; i--; ) {
1205 struct r5dev *dev = &sh->dev[i];
1206 if (test_bit(R5_Wantfill, &dev->flags)) {
1207 struct bio *rbi;
b17459c0 1208 spin_lock_irq(&sh->stripe_lock);
91c00924
DW		 1209 dev->read = rbi = dev->toread;
1210 dev->toread = NULL;
b17459c0 1211 spin_unlock_irq(&sh->stripe_lock);
4f024f37 1212 while (rbi && rbi->bi_iter.bi_sector <
91c00924 1213 dev->sector + STRIPE_SECTORS) {
d592a996
SL		 1214 tx = async_copy_data(0, rbi, &dev->page,
1215 dev->sector, tx, sh);
91c00924
DW		 1216 rbi = r5_next_bio(rbi, dev->sector);
1217 }
1218 }
1219 }
1220
1221 atomic_inc(&sh->count);
a08abd8c
DW		 1222 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
1223 async_trigger_callback(&submit);
91c00924
DW		 1224}
1225
4e7d2c0a 1226static void mark_target_uptodate(struct stripe_head *sh, int target)
91c00924 1227{
4e7d2c0a 1228 struct r5dev *tgt;
91c00924 1229
4e7d2c0a
DW		 1230 if (target < 0)
1231 return;
91c00924 1232
4e7d2c0a 1233 tgt = &sh->dev[target];
91c00924
DW		 1234 set_bit(R5_UPTODATE, &tgt->flags);
1235 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1236 clear_bit(R5_Wantcompute, &tgt->flags);
4e7d2c0a
DW		 1237}
1238
ac6b53b6 1239static void ops_complete_compute(void *stripe_head_ref)
91c00924
DW		 1240{
1241 struct stripe_head *sh = stripe_head_ref;
91c00924 1242
e46b272b 1243 pr_debug("%s: stripe %llu\n", __func__,
91c00924
DW		 1244 (unsigned long long)sh->sector);
1245
ac6b53b6 1246 /* mark the computed target(s) as uptodate */
4e7d2c0a 1247 mark_target_uptodate(sh, sh->ops.target);
ac6b53b6 1248 mark_target_uptodate(sh, sh->ops.target2);
4e7d2c0a 1249
ecc65c9b
DW		 1250 clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
1251 if (sh->check_state == check_state_compute_run)
1252 sh->check_state = check_state_compute_result;
91c00924
DW		 1253 set_bit(STRIPE_HANDLE, &sh->state);
1254 release_stripe(sh);
1255}
1256
d6f38f31
DW		 1257/* return a pointer to the address conversion region of the scribble buffer */
1258static addr_conv_t *to_addr_conv(struct stripe_head *sh,
46d5b785 1259 struct raid5_percpu *percpu, int i)
d6f38f31 1260{
46d5b785 1261 void *addr;
1262
1263 addr = flex_array_get(percpu->scribble, i);
1264 return addr + sizeof(struct page *) * (sh->disks + 2);
1265}
1266
1267/* return a pointer to the address conversion region of the scribble buffer */
1268static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
1269{
1270 void *addr;
1271
1272 addr = flex_array_get(percpu->scribble, i);
1273 return addr;
d6f38f31
DW		 1274}
1275
1276static struct dma_async_tx_descriptor *
1277ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
91c00924 1278{
91c00924 1279 int disks = sh->disks;
46d5b785 1280 struct page **xor_srcs = to_addr_page(percpu, 0);
91c00924
DW		 1281 int target = sh->ops.target;
1282 struct r5dev *tgt = &sh->dev[target];
1283 struct page *xor_dest = tgt->page;
1284 int count = 0;
1285 struct dma_async_tx_descriptor *tx;
a08abd8c 1286 struct async_submit_ctl submit;
91c00924
DW		 1287 int i;
1288
59fc630b 1289 BUG_ON(sh->batch_head);
1290
91c00924 1291 pr_debug("%s: stripe %llu block: %d\n",
e46b272b 1292 __func__, (unsigned long long)sh->sector, target);
91c00924
DW		 1293 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1294
1295 for (i = disks; i--; )
1296 if (i != target)
1297 xor_srcs[count++] = sh->dev[i].page;
1298
1299 atomic_inc(&sh->count);
1300
0403e382 1301 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
46d5b785 1302 ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
91c00924 1303 if (unlikely(count == 1))
a08abd8c 1304 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
91c00924 1305 else
a08abd8c 1306 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
91c00924 1307
91c00924
DW		 1308 return tx;
1309}
1310
ac6b53b6
DW		 1311/* set_syndrome_sources - populate source buffers for gen_syndrome
1312 * @srcs - (struct page *) array of size sh->disks
1313 * @sh - stripe_head to parse
1314 *
1315 * Populates srcs in proper layout order for the stripe and returns the
1316 * 'count' of sources to be used in a call to async_gen_syndrome. The P
1317 * destination buffer is recorded in srcs[count] and the Q destination
1318 * is recorded in srcs[count+1]].
1319 */
1320static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
1321{
1322 int disks = sh->disks;
1323 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
1324 int d0_idx = raid6_d0(sh);
1325 int count;
1326 int i;
1327
1328 for (i = 0; i < disks; i++)
5dd33c9a 1329 srcs[i] = NULL;
ac6b53b6
DW		 1330
1331 count = 0;
1332 i = d0_idx;
1333 do {
1334 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1335
1336 srcs[slot] = sh->dev[i].page;
1337 i = raid6_next_disk(i, disks);
1338 } while (i != d0_idx);
ac6b53b6 1339
e4424fee 1340 return syndrome_disks;
ac6b53b6
DW		 1341}
1342
1343static struct dma_async_tx_descriptor *
1344ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
1345{
1346 int disks = sh->disks;
46d5b785 1347 struct page **blocks = to_addr_page(percpu, 0);
ac6b53b6
DW		 1348 int target;
1349 int qd_idx = sh->qd_idx;
1350 struct dma_async_tx_descriptor *tx;
1351 struct async_submit_ctl submit;
1352 struct r5dev *tgt;
1353 struct page *dest;
1354 int i;
1355 int count;
1356
59fc630b 1357 BUG_ON(sh->batch_head);
ac6b53b6
DW		 1358 if (sh->ops.target < 0)
1359 target = sh->ops.target2;
1360 else if (sh->ops.target2 < 0)
1361 target = sh->ops.target;
91c00924 1362 else
ac6b53b6
DW		 1363 /* we should only have one valid target */
1364 BUG();
1365 BUG_ON(target < 0);
1366 pr_debug("%s: stripe %llu block: %d\n",
1367 __func__, (unsigned long long)sh->sector, target);
1368
1369 tgt = &sh->dev[target];
1370 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1371 dest = tgt->page;
1372
1373 atomic_inc(&sh->count);
1374
1375 if (target == qd_idx) {
1376 count = set_syndrome_sources(blocks, sh);
1377 blocks[count] = NULL; /* regenerating p is not necessary */
1378 BUG_ON(blocks[count+1] != dest); /* q should already be set */
0403e382
DW		 1379 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1380 ops_complete_compute, sh,
46d5b785 1381 to_addr_conv(sh, percpu, 0));
ac6b53b6
DW		 1382 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
1383 } else {
1384 /* Compute any data- or p-drive using XOR */
1385 count = 0;
1386 for (i = disks; i-- ; ) {
1387 if (i == target || i == qd_idx)
1388 continue;
1389 blocks[count++] = sh->dev[i].page;
1390 }
1391
0403e382
DW		 1392 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1393 NULL, ops_complete_compute, sh,
46d5b785 1394 to_addr_conv(sh, percpu, 0));
ac6b53b6
DW		 1395 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
1396 }
91c00924 1397
91c00924
DW		 1398 return tx;
1399}
1400
ac6b53b6
DW		 1401static struct dma_async_tx_descriptor *
1402ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
1403{
1404 int i, count, disks = sh->disks;
1405 int syndrome_disks = sh->ddf_layout ? disks : disks-2;
1406 int d0_idx = raid6_d0(sh);
1407 int faila = -1, failb = -1;
1408 int target = sh->ops.target;
1409 int target2 = sh->ops.target2;
1410 struct r5dev *tgt = &sh->dev[target];
1411 struct r5dev *tgt2 = &sh->dev[target2];
1412 struct dma_async_tx_descriptor *tx;
46d5b785 1413 struct page **blocks = to_addr_page(percpu, 0);
ac6b53b6
DW		 1414 struct async_submit_ctl submit;
1415
59fc630b 1416 BUG_ON(sh->batch_head);
ac6b53b6
DW		 1417 pr_debug("%s: stripe %llu block1: %d block2: %d\n",
1418 __func__, (unsigned long long)sh->sector, target, target2);
1419 BUG_ON(target < 0 || target2 < 0);
1420 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
1421 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
1422
6c910a78 1423 /* we need to open-code set_syndrome_sources to handle the
ac6b53b6
DW		 1424 * slot number conversion for 'faila' and 'failb'
1425 */
1426 for (i = 0; i < disks ; i++)
5dd33c9a 1427 blocks[i] = NULL;
ac6b53b6
DW		 1428 count = 0;
1429 i = d0_idx;
1430 do {
1431 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1432
1433 blocks[slot] = sh->dev[i].page;
1434
1435 if (i == target)
1436 faila = slot;
1437 if (i == target2)
1438 failb = slot;
1439 i = raid6_next_disk(i, disks);
1440 } while (i != d0_idx);
ac6b53b6
DW		 1441
1442 BUG_ON(faila == failb);
1443 if (failb < faila)
1444 swap(faila, failb);
1445 pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
1446 __func__, (unsigned long long)sh->sector, faila, failb);
1447
1448 atomic_inc(&sh->count);
1449
1450 if (failb == syndrome_disks+1) {
1451 /* Q disk is one of the missing disks */
1452 if (faila == syndrome_disks) {
1453 /* Missing P+Q, just recompute */
0403e382
DW		 1454 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1455 ops_complete_compute, sh,
46d5b785 1456 to_addr_conv(sh, percpu, 0));
e4424fee 1457 return async_gen_syndrome(blocks, 0, syndrome_disks+2,
ac6b53b6
DW		 1458 STRIPE_SIZE, &submit);
1459 } else {
1460 struct page *dest;
1461 int data_target;
1462 int qd_idx = sh->qd_idx;
1463
1464 /* Missing D+Q: recompute D from P, then recompute Q */
1465 if (target == qd_idx)
1466 data_target = target2;
1467 else
1468 data_target = target;
1469
1470 count = 0;
1471 for (i = disks; i-- ; ) {
1472 if (i == data_target || i == qd_idx)
1473 continue;
1474 blocks[count++] = sh->dev[i].page;
1475 }
1476 dest = sh->dev[data_target].page;
0403e382
DW		 1477 init_async_submit(&submit,
1478 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
1479 NULL, NULL, NULL,
46d5b785 1480 to_addr_conv(sh, percpu, 0));
ac6b53b6
DW		 1481 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
1482 &submit);
1483
1484 count = set_syndrome_sources(blocks, sh);
0403e382
DW		 1485 init_async_submit(&submit, ASYNC_TX_FENCE, tx,
1486 ops_complete_compute, sh,
46d5b785 1487 to_addr_conv(sh, percpu, 0));
ac6b53b6
DW		 1488 return async_gen_syndrome(blocks, 0, count+2,
1489 STRIPE_SIZE, &submit);
1490 }
ac6b53b6 1491 } else {
6c910a78
DW		 1492 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
1493 ops_complete_compute, sh,
46d5b785 1494 to_addr_conv(sh, percpu, 0));
6c910a78
DW		 1495 if (failb == syndrome_disks) {
1496 /* We're missing D+P. */
1497 return async_raid6_datap_recov(syndrome_disks+2,
1498 STRIPE_SIZE, faila,
1499 blocks, &submit);
1500 } else {
1501 /* We're missing D+D. */
1502 return async_raid6_2data_recov(syndrome_disks+2,
1503 STRIPE_SIZE, faila, failb,
1504 blocks, &submit);
1505 }
ac6b53b6
DW		 1506 }
1507}
1508
91c00924
DW		 1509static void ops_complete_prexor(void *stripe_head_ref)
1510{
1511 struct stripe_head *sh = stripe_head_ref;
1512
e46b272b 1513 pr_debug("%s: stripe %llu\n", __func__,
91c00924 1514 (unsigned long long)sh->sector);
91c00924
DW		 1515}
1516
1517static struct dma_async_tx_descriptor *
d6f38f31
DW		 1518ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
1519 struct dma_async_tx_descriptor *tx)
91c00924 1520{
91c00924 1521 int disks = sh->disks;
46d5b785 1522 struct page **xor_srcs = to_addr_page(percpu, 0);
91c00924 1523 int count = 0, pd_idx = sh->pd_idx, i;
a08abd8c 1524 struct async_submit_ctl submit;
91c00924
DW		 1525
1526 /* existing parity data subtracted */
1527 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1528
59fc630b 1529 BUG_ON(sh->batch_head);
e46b272b 1530 pr_debug("%s: stripe %llu\n", __func__,
91c00924
DW		 1531 (unsigned long long)sh->sector);
1532
1533 for (i = disks; i--; ) {
1534 struct r5dev *dev = &sh->dev[i];
1535 /* Only process blocks that are known to be uptodate */
d8ee0728 1536 if (test_bit(R5_Wantdrain, &dev->flags))
91c00924
DW		 1537 xor_srcs[count++] = dev->page;
1538 }
1539
0403e382 1540 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
46d5b785 1541 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
a08abd8c 1542 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
91c00924
DW		 1543
1544 return tx;
1545}
1546
1547static struct dma_async_tx_descriptor *
d8ee0728 1548ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
91c00924
DW		 1549{
1550 int disks = sh->disks;
d8ee0728 1551 int i;
59fc630b 1552 struct stripe_head *head_sh = sh;
91c00924 1553
e46b272b 1554 pr_debug("%s: stripe %llu\n", __func__,
91c00924
DW		 1555 (unsigned long long)sh->sector);
1556
1557 for (i = disks; i--; ) {
59fc630b 1558 struct r5dev *dev;
91c00924 1559 struct bio *chosen;
91c00924 1560
59fc630b 1561 sh = head_sh;
1562 if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {
91c00924
DW		 1563 struct bio *wbi;
1564
59fc630b 1565again:
1566 dev = &sh->dev[i];
b17459c0 1567 spin_lock_irq(&sh->stripe_lock);
91c00924
DW		 1568 chosen = dev->towrite;
1569 dev->towrite = NULL;
7a87f434 1570 sh->overwrite_disks = 0;
91c00924
DW		 1571 BUG_ON(dev->written);
1572 wbi = dev->written = chosen;
b17459c0 1573 spin_unlock_irq(&sh->stripe_lock);
d592a996 1574 WARN_ON(dev->page != dev->orig_page);
91c00924 1575
4f024f37 1576 while (wbi && wbi->bi_iter.bi_sector <
91c00924 1577 dev->sector + STRIPE_SECTORS) {
e9c7469b
TH		 1578 if (wbi->bi_rw & REQ_FUA)
1579 set_bit(R5_WantFUA, &dev->flags);
bc0934f0
SL		 1580 if (wbi->bi_rw & REQ_SYNC)
1581 set_bit(R5_SyncIO, &dev->flags);
9e444768 1582 if (wbi->bi_rw & REQ_DISCARD)
620125f2 1583 set_bit(R5_Discard, &dev->flags);
d592a996
SL		 1584 else {
1585 tx = async_copy_data(1, wbi, &dev->page,
1586 dev->sector, tx, sh);
1587 if (dev->page != dev->orig_page) {
1588 set_bit(R5_SkipCopy, &dev->flags);
1589 clear_bit(R5_UPTODATE, &dev->flags);
1590 clear_bit(R5_OVERWRITE, &dev->flags);
1591 }
1592 }
91c00924
DW		 1593 wbi = r5_next_bio(wbi, dev->sector);
1594 }
59fc630b 1595
1596 if (head_sh->batch_head) {
1597 sh = list_first_entry(&sh->batch_list,
1598 struct stripe_head,
1599 batch_list);
1600 if (sh == head_sh)
1601 continue;
1602 goto again;
1603 }
91c00924
DW		 1604 }
1605 }
1606
1607 return tx;
1608}
1609
ac6b53b6 1610static void ops_complete_reconstruct(void *stripe_head_ref)
91c00924
DW		 1611{
1612 struct stripe_head *sh = stripe_head_ref;
ac6b53b6
DW		 1613 int disks = sh->disks;
1614 int pd_idx = sh->pd_idx;
1615 int qd_idx = sh->qd_idx;
1616 int i;
9e444768 1617 bool fua = false, sync = false, discard = false;
91c00924 1618
e46b272b 1619 pr_debug("%s: stripe %llu\n", __func__,
91c00924
DW		 1620 (unsigned long long)sh->sector);
1621
bc0934f0 1622 for (i = disks; i--; ) {
e9c7469b 1623 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
bc0934f0 1624 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
9e444768 1625 discard |= test_bit(R5_Discard, &sh->dev[i].flags);
bc0934f0 1626 }
e9c7469b 1627
91c00924
DW		 1628 for (i = disks; i--; ) {
1629 struct r5dev *dev = &sh->dev[i];
ac6b53b6 1630
e9c7469b 1631 if (dev->written || i == pd_idx || i == qd_idx) {
d592a996 1632 if (!discard && !test_bit(R5_SkipCopy, &dev->flags))
9e444768 1633 set_bit(R5_UPTODATE, &dev->flags);
e9c7469b
TH		 1634 if (fua)
1635 set_bit(R5_WantFUA, &dev->flags);
bc0934f0
SL		 1636 if (sync)
1637 set_bit(R5_SyncIO, &dev->flags);
e9c7469b 1638 }
91c00924
DW		 1639 }
1640
d8ee0728
DW		 1641 if (sh->reconstruct_state == reconstruct_state_drain_run)
1642 sh->reconstruct_state = reconstruct_state_drain_result;
1643 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
1644 sh->reconstruct_state = reconstruct_state_prexor_drain_result;
1645 else {
1646 BUG_ON(sh->reconstruct_state != reconstruct_state_run);
1647 sh->reconstruct_state = reconstruct_state_result;
1648 }
91c00924
DW		 1649
1650 set_bit(STRIPE_HANDLE, &sh->state);
1651 release_stripe(sh);
1652}
1653
1654static void
ac6b53b6
DW		 1655ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
1656 struct dma_async_tx_descriptor *tx)
91c00924 1657{
91c00924 1658 int disks = sh->disks;
59fc630b 1659 struct page **xor_srcs;
a08abd8c 1660 struct async_submit_ctl submit;
59fc630b 1661 int count, pd_idx = sh->pd_idx, i;
91c00924 1662 struct page *xor_dest;
d8ee0728 1663 int prexor = 0;
91c00924 1664 unsigned long flags;
59fc630b 1665 int j = 0;
1666 struct stripe_head *head_sh = sh;
1667 int last_stripe;
91c00924 1668
e46b272b 1669 pr_debug("%s: stripe %llu\n", __func__,
91c00924
DW		 1670 (unsigned long long)sh->sector);
1671
620125f2
SL		 1672 for (i = 0; i < sh->disks; i++) {
1673 if (pd_idx == i)
1674 continue;
1675 if (!test_bit(R5_Discard, &sh->dev[i].flags))
1676 break;
1677 }
1678 if (i >= sh->disks) {
1679 atomic_inc(&sh->count);
620125f2
SL		 1680 set_bit(R5_Discard, &sh->dev[pd_idx].flags);
1681 ops_complete_reconstruct(sh);
1682 return;
1683 }
59fc630b 1684again:
1685 count = 0;
1686 xor_srcs = to_addr_page(percpu, j);
91c00924
DW		 1687 /* check if prexor is active which means only process blocks
1688 * that are part of a read-modify-write (written)
1689 */
59fc630b 1690 if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
d8ee0728 1691 prexor = 1;
91c00924
DW		 1692 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1693 for (i = disks; i--; ) {
1694 struct r5dev *dev = &sh->dev[i];
59fc630b 1695 if (head_sh->dev[i].written)
91c00924
DW		 1696 xor_srcs[count++] = dev->page;
1697 }
1698 } else {
1699 xor_dest = sh->dev[pd_idx].page;
1700 for (i = disks; i--; ) {
1701 struct r5dev *dev = &sh->dev[i];
1702 if (i != pd_idx)
1703 xor_srcs[count++] = dev->page;
1704 }
1705 }
1706
91c00924
DW		 1707 /* 1/ if we prexor'd then the dest is reused as a source
1708 * 2/ if we did not prexor then we are redoing the parity
1709 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
1710 * for the synchronous xor case
1711 */
59fc630b 1712 last_stripe = !head_sh->batch_head ||
1713 list_first_entry(&sh->batch_list,
1714 struct stripe_head, batch_list) == head_sh;
1715 if (last_stripe) {
1716 flags = ASYNC_TX_ACK |
1717 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
1718
1719 atomic_inc(&head_sh->count);
1720 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
1721 to_addr_conv(sh, percpu, j));
1722 } else {
1723 flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
1724 init_async_submit(&submit, flags, tx, NULL, NULL,
1725 to_addr_conv(sh, percpu, j));
1726 }
91c00924 1727
a08abd8c
DW		 1728 if (unlikely(count == 1))
1729 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1730 else
1731 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
59fc630b 1732 if (!last_stripe) {
1733 j++;
1734 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1735 batch_list);
1736 goto again;
1737 }
91c00924
DW		 1738}
1739
ac6b53b6
DW		 1740static void
1741ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
1742 struct dma_async_tx_descriptor *tx)
1743{
1744 struct async_submit_ctl submit;
59fc630b 1745 struct page **blocks;
1746 int count, i, j = 0;
1747 struct stripe_head *head_sh = sh;
1748 int last_stripe;
ac6b53b6
DW		 1749
1750 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
1751
620125f2
SL		 1752 for (i = 0; i < sh->disks; i++) {
1753 if (sh->pd_idx == i || sh->qd_idx == i)
1754 continue;
1755 if (!test_bit(R5_Discard, &sh->dev[i].flags))
1756 break;
1757 }
1758 if (i >= sh->disks) {
1759 atomic_inc(&sh->count);
620125f2
SL		 1760 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
1761 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
1762 ops_complete_reconstruct(sh);
1763 return;
1764 }
1765
59fc630b 1766again:
1767 blocks = to_addr_page(percpu, j);
ac6b53b6 1768 count = set_syndrome_sources(blocks, sh);
59fc630b 1769 last_stripe = !head_sh->batch_head ||
1770 list_first_entry(&sh->batch_list,
1771 struct stripe_head, batch_list) == head_sh;
1772
1773 if (last_stripe) {
1774 atomic_inc(&head_sh->count);
1775 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
1776 head_sh, to_addr_conv(sh, percpu, j));
1777 } else
1778 init_async_submit(&submit, 0, tx, NULL, NULL,
1779 to_addr_conv(sh, percpu, j));
ac6b53b6 1780 async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
59fc630b 1781 if (!last_stripe) {
1782 j++;
1783 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1784 batch_list);
1785 goto again;
1786 }
91c00924
DW		 1787}
1788
1789static void ops_complete_check(void *stripe_head_ref)
1790{
1791 struct stripe_head *sh = stripe_head_ref;
91c00924 1792
e46b272b 1793 pr_debug("%s: stripe %llu\n", __func__,
91c00924
DW		 1794 (unsigned long long)sh->sector);
1795
ecc65c9b 1796 sh->check_state = check_state_check_result;
91c00924
DW		 1797 set_bit(STRIPE_HANDLE, &sh->state);
1798 release_stripe(sh);
1799}
1800
ac6b53b6 1801static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
91c00924 1802{
91c00924 1803 int disks = sh->disks;
ac6b53b6
DW		 1804 int pd_idx = sh->pd_idx;
1805 int qd_idx = sh->qd_idx;
1806 struct page *xor_dest;
46d5b785 1807 struct page **xor_srcs = to_addr_page(percpu, 0);
91c00924 1808 struct dma_async_tx_descriptor *tx;
a08abd8c 1809 struct async_submit_ctl submit;
ac6b53b6
DW		 1810 int count;
1811 int i;
91c00924 1812
e46b272b 1813 pr_debug("%s: stripe %llu\n", __func__,
91c00924
DW		 1814 (unsigned long long)sh->sector);
1815
59fc630b 1816 BUG_ON(sh->batch_head);
ac6b53b6
DW		 1817 count = 0;
1818 xor_dest = sh->dev[pd_idx].page;
1819 xor_srcs[count++] = xor_dest;
91c00924 1820 for (i = disks; i--; ) {
ac6b53b6
DW		 1821 if (i == pd_idx || i == qd_idx)
1822 continue;
1823 xor_srcs[count++] = sh->dev[i].page;
91c00924
DW		 1824 }
1825
d6f38f31 1826 init_async_submit(&submit, 0, NULL, NULL, NULL,
46d5b785 1827 to_addr_conv(sh, percpu, 0));
099f53cb 1828 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
a08abd8c 1829 &sh->ops.zero_sum_result, &submit);
91c00924 1830
91c00924 1831 atomic_inc(&sh->count);
a08abd8c
DW		 1832 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
1833 tx = async_trigger_callback(&submit);
91c00924
DW		 1834}
1835
ac6b53b6
DW		 1836static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
1837{
46d5b785 1838 struct page **srcs = to_addr_page(percpu, 0);
ac6b53b6
DW		 1839 struct async_submit_ctl submit;
1840 int count;
1841
1842 pr_debug("%s: stripe %llu checkp: %d\n", __func__,
1843 (unsigned long long)sh->sector, checkp);
1844
59fc630b 1845 BUG_ON(sh->batch_head);
ac6b53b6
DW		 1846 count = set_syndrome_sources(srcs, sh);
1847 if (!checkp)
1848 srcs[count] = NULL;
91c00924 1849
91c00924 1850 atomic_inc(&sh->count);
ac6b53b6 1851 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
46d5b785 1852 sh, to_addr_conv(sh, percpu, 0));
ac6b53b6
DW		 1853 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
1854 &sh->ops.zero_sum_result, percpu->spare_page, &submit);
91c00924
DW		 1855}
1856
51acbcec 1857static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
91c00924
DW		 1858{
1859 int overlap_clear = 0, i, disks = sh->disks;
1860 struct dma_async_tx_descriptor *tx = NULL;
d1688a6d 1861 struct r5conf *conf = sh->raid_conf;
ac6b53b6 1862 int level = conf->level;
d6f38f31
DW		 1863 struct raid5_percpu *percpu;
1864 unsigned long cpu;
91c00924 1865
d6f38f31
DW		 1866 cpu = get_cpu();
1867 percpu = per_cpu_ptr(conf->percpu, cpu);
83de75cc 1868 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
91c00924
DW		 1869 ops_run_biofill(sh);
1870 overlap_clear++;
1871 }
1872
7b3a871e 1873 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
ac6b53b6
DW		 1874 if (level < 6)
1875 tx = ops_run_compute5(sh, percpu);
1876 else {
1877 if (sh->ops.target2 < 0 || sh->ops.target < 0)
1878 tx = ops_run_compute6_1(sh, percpu);
1879 else
1880 tx = ops_run_compute6_2(sh, percpu);
1881 }
1882 /* terminate the chain if reconstruct is not set to be run */
1883 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
7b3a871e
DW		 1884 async_tx_ack(tx);
1885 }
91c00924 1886
600aa109 1887 if (test_bit(STRIPE_OP_PREXOR, &ops_request))
d6f38f31 1888 tx = ops_run_prexor(sh, percpu, tx);
91c00924 1889
600aa109 1890 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
d8ee0728 1891 tx = ops_run_biodrain(sh, tx);
91c00924
DW		 1892 overlap_clear++;
1893 }
1894
ac6b53b6
DW		 1895 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
1896 if (level < 6)
1897 ops_run_reconstruct5(sh, percpu, tx);
1898 else
1899 ops_run_reconstruct6(sh, percpu, tx);
1900 }
91c00924 1901
ac6b53b6
DW		 1902 if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
1903 if (sh->check_state == check_state_run)
1904 ops_run_check_p(sh, percpu);
1905 else if (sh->check_state == check_state_run_q)
1906 ops_run_check_pq(sh, percpu, 0);
1907 else if (sh->check_state == check_state_run_pq)
1908 ops_run_check_pq(sh, percpu, 1);
1909 else
1910 BUG();
1911 }
91c00924 1912
59fc630b 1913 if (overlap_clear && !sh->batch_head)
91c00924
DW		 1914 for (i = disks; i--; ) {
1915 struct r5dev *dev = &sh->dev[i];
1916 if (test_and_clear_bit(R5_Overlap, &dev->flags))
1917 wake_up(&sh->raid_conf->wait_for_overlap);
1918 }
d6f38f31 1919 put_cpu();
91c00924
DW		 1920}
1921
566c09c5 1922static int grow_one_stripe(struct r5conf *conf, int hash)
1da177e4
LT		 1923{
1924 struct stripe_head *sh;
6ce32846 1925 sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL);
3f294f4f
N		 1926 if (!sh)
1927 return 0;
6ce32846 1928
3f294f4f 1929 sh->raid_conf = conf;
3f294f4f 1930
b17459c0
SL		 1931 spin_lock_init(&sh->stripe_lock);
1932
e4e11e38
N		 1933 if (grow_buffers(sh)) {
1934 shrink_buffers(sh);
3f294f4f
N		 1935 kmem_cache_free(conf->slab_cache, sh);
1936 return 0;
1937 }
566c09c5 1938 sh->hash_lock_index = hash;
3f294f4f
N		 1939 /* we just created an active stripe so... */
1940 atomic_set(&sh->count, 1);
1941 atomic_inc(&conf->active_stripes);
1942 INIT_LIST_HEAD(&sh->lru);
59fc630b 1943
1944 spin_lock_init(&sh->batch_lock);
1945 INIT_LIST_HEAD(&sh->batch_list);
1946 sh->batch_head = NULL;
3f294f4f
N		 1947 release_stripe(sh);
1948 return 1;
1949}
1950
d1688a6d 1951static int grow_stripes(struct r5conf *conf, int num)
3f294f4f 1952{
e18b890b 1953 struct kmem_cache *sc;
5e5e3e78 1954 int devs = max(conf->raid_disks, conf->previous_raid_disks);
566c09c5 1955 int hash;
1da177e4 1956
f4be6b43
N		 1957 if (conf->mddev->gendisk)
1958 sprintf(conf->cache_name[0],
1959 "raid%d-%s", conf->level, mdname(conf->mddev));
1960 else
1961 sprintf(conf->cache_name[0],
1962 "raid%d-%p", conf->level, conf->mddev);
1963 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
1964
ad01c9e3
N		 1965 conf->active_name = 0;
1966 sc = kmem_cache_create(conf->cache_name[conf->active_name],
1da177e4 1967 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
20c2df83 1968 0, 0, NULL);
1da177e4
LT		 1969 if (!sc)
1970 return 1;
1971 conf->slab_cache = sc;
ad01c9e3 1972 conf->pool_size = devs;
566c09c5
SL		 1973 hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
1974 while (num--) {
1975 if (!grow_one_stripe(conf, hash))
1da177e4 1976 return 1;
566c09c5
SL		 1977 conf->max_nr_stripes++;
1978 hash = (hash + 1) % NR_STRIPE_HASH_LOCKS;
1979 }
1da177e4
LT		 1980 return 0;
1981}
29269553 1982
d6f38f31
DW		 1983/**
1984 * scribble_len - return the required size of the scribble region
1985 * @num - total number of disks in the array
1986 *
1987 * The size must be enough to contain:
1988 * 1/ a struct page pointer for each device in the array +2
1989 * 2/ room to convert each entry in (1) to its corresponding dma
1990 * (dma_map_page()) or page (page_address()) address.
1991 *
1992 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
1993 * calculate over all devices (not just the data blocks), using zeros in place
1994 * of the P and Q blocks.
1995 */
46d5b785 1996static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags)
d6f38f31 1997{
46d5b785 1998 struct flex_array *ret;
d6f38f31
DW		 1999 size_t len;
2000
2001 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
46d5b785 2002 ret = flex_array_alloc(len, cnt, flags);
2003 if (!ret)
2004 return NULL;
2005 /* always prealloc all elements, so no locking is required */
2006 if (flex_array_prealloc(ret, 0, cnt, flags)) {
2007 flex_array_free(ret);
2008 return NULL;
2009 }
2010 return ret;
d6f38f31
DW		 2011}
2012
d1688a6d 2013static int resize_stripes(struct r5conf *conf, int newsize)
ad01c9e3
N		 2014{
2015 /* Make all the stripes able to hold 'newsize' devices.
2016 * New slots in each stripe get 'page' set to a new page.
2017 *
2018 * This happens in stages:
2019 * 1/ create a new kmem_cache and allocate the required number of
2020 * stripe_heads.
83f0d77a 2021 * 2/ gather all the old stripe_heads and transfer the pages across
ad01c9e3
N		 2022 * to the new stripe_heads. This will have the side effect of
2023 * freezing the array as once all stripe_heads have been collected,
2024 * no IO will be possible. Old stripe heads are freed once their
2025 * pages have been transferred over, and the old kmem_cache is
2026 * freed when all stripes are done.
2027 * 3/ reallocate conf->disks to be suitable bigger. If this fails,
2028 * we simple return a failre status - no need to clean anything up.
2029 * 4/ allocate new pages for the new slots in the new stripe_heads.
2030 * If this fails, we don't bother trying the shrink the
2031 * stripe_heads down again, we just leave them as they are.
2032 * As each stripe_head is processed the new one is released into
2033 * active service.
2034 *
2035 * Once step2 is started, we cannot afford to wait for a write,
2036 * so we use GFP_NOIO allocations.
2037 */
2038 struct stripe_head *osh, *nsh;
2039 LIST_HEAD(newstripes);
2040 struct disk_info *ndisks;
d6f38f31 2041 unsigned long cpu;
b5470dc5 2042 int err;
e18b890b 2043 struct kmem_cache *sc;
ad01c9e3 2044 int i;
566c09c5 2045 int hash, cnt;
ad01c9e3
N		 2046
2047 if (newsize <= conf->pool_size)
2048 return 0; /* never bother to shrink */
2049
b5470dc5
DW		 2050 err = md_allow_write(conf->mddev);
2051 if (err)
2052 return err;
2a2275d6 2053
ad01c9e3
N		 2054 /* Step 1 */
2055 sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
2056 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
20c2df83 2057 0, 0, NULL);
ad01c9e3
N		 2058 if (!sc)
2059 return -ENOMEM;
2060
2061 for (i = conf->max_nr_stripes; i; i--) {
6ce32846 2062 nsh = kmem_cache_zalloc(sc, GFP_KERNEL);
ad01c9e3
N		 2063 if (!nsh)
2064 break;
2065
ad01c9e3 2066 nsh->raid_conf = conf;
cb13ff69 2067 spin_lock_init(&nsh->stripe_lock);
ad01c9e3
N		 2068
2069 list_add(&nsh->lru, &newstripes);
2070 }
2071 if (i) {
2072 /* didn't get enough, give up */
2073 while (!list_empty(&newstripes)) {
2074 nsh = list_entry(newstripes.next, struct stripe_head, lru);
2075 list_del(&nsh->lru);
2076 kmem_cache_free(sc, nsh);
2077 }
2078 kmem_cache_destroy(sc);
2079 return -ENOMEM;
2080 }
2081 /* Step 2 - Must use GFP_NOIO now.
2082 * OK, we have enough stripes, start collecting inactive
2083 * stripes and copying them over
2084 */
566c09c5
SL		 2085 hash = 0;
2086 cnt = 0;
ad01c9e3 2087 list_for_each_entry(nsh, &newstripes, lru) {
566c09c5
SL		 2088 lock_device_hash_lock(conf, hash);
2089 wait_event_cmd(conf->wait_for_stripe,
2090 !list_empty(conf->inactive_list + hash),
2091 unlock_device_hash_lock(conf, hash),
2092 lock_device_hash_lock(conf, hash));
2093 osh = get_free_stripe(conf, hash);
2094 unlock_device_hash_lock(conf, hash);
ad01c9e3 2095 atomic_set(&nsh->count, 1);
d592a996 2096 for(i=0; i<conf->pool_size; i++) {
ad01c9e3 2097 nsh->dev[i].page = osh->dev[i].page;
d592a996
SL		 2098 nsh->dev[i].orig_page = osh->dev[i].page;
2099 }
ad01c9e3
N		 2100 for( ; i<newsize; i++)
2101 nsh->dev[i].page = NULL;
566c09c5 2102 nsh->hash_lock_index = hash;
ad01c9e3 2103 kmem_cache_free(conf->slab_cache, osh);
566c09c5
SL		 2104 cnt++;
2105 if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
2106 !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
2107 hash++;
2108 cnt = 0;
2109 }
ad01c9e3
N		 2110 }
2111 kmem_cache_destroy(conf->slab_cache);
2112
2113 /* Step 3.
2114 * At this point, we are holding all the stripes so the array
2115 * is completely stalled, so now is a good time to resize
d6f38f31 2116 * conf->disks and the scribble region
ad01c9e3
N		 2117 */
2118 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
2119 if (ndisks) {
2120 for (i=0; i<conf->raid_disks; i++)
2121 ndisks[i] = conf->disks[i];
2122 kfree(conf->disks);
2123 conf->disks = ndisks;
2124 } else
2125 err = -ENOMEM;
2126
d6f38f31 2127 get_online_cpus();
d6f38f31
DW		 2128 for_each_present_cpu(cpu) {
2129 struct raid5_percpu *percpu;
46d5b785 2130 struct flex_array *scribble;
d6f38f31
DW		 2131
2132 percpu = per_cpu_ptr(conf->percpu, cpu);
46d5b785 2133 scribble = scribble_alloc(newsize, conf->chunk_sectors /
2134 STRIPE_SECTORS, GFP_NOIO);
d6f38f31
DW		 2135
2136 if (scribble) {
46d5b785 2137 flex_array_free(percpu->scribble);
d6f38f31
DW		 2138 percpu->scribble = scribble;
2139 } else {
2140 err = -ENOMEM;
2141 break;
2142 }
2143 }
2144 put_online_cpus();
2145
ad01c9e3
N		 2146 /* Step 4, return new stripes to service */
2147 while(!list_empty(&newstripes)) {
2148 nsh = list_entry(newstripes.next, struct stripe_head, lru);
2149 list_del_init(&nsh->lru);
d6f38f31 2150
ad01c9e3
N		 2151 for (i=conf->raid_disks; i < newsize; i++)
2152 if (nsh->dev[i].page == NULL) {
2153 struct page *p = alloc_page(GFP_NOIO);
2154 nsh->dev[i].page = p;
d592a996 2155 nsh->dev[i].orig_page = p;
ad01c9e3
N		 2156 if (!p)
2157 err = -ENOMEM;
2158 }
2159 release_stripe(nsh);
2160 }
2161 /* critical section pass, GFP_NOIO no longer needed */
2162
2163 conf->slab_cache = sc;
2164 conf->active_name = 1-conf->active_name;
2165 conf->pool_size = newsize;
2166 return err;
2167}
1da177e4 2168
566c09c5 2169static int drop_one_stripe(struct r5conf *conf, int hash)
1da177e4
LT		 2170{
2171 struct stripe_head *sh;
2172
566c09c5
SL		 2173 spin_lock_irq(conf->hash_locks + hash);
2174 sh = get_free_stripe(conf, hash);
2175 spin_unlock_irq(conf->hash_locks + hash);
3f294f4f
N		 2176 if (!sh)
2177 return 0;
78bafebd 2178 BUG_ON(atomic_read(&sh->count));
e4e11e38 2179 shrink_buffers(sh);
3f294f4f
N		 2180 kmem_cache_free(conf->slab_cache, sh);
2181 atomic_dec(&conf->active_stripes);
2182 return 1;
2183}
2184
d1688a6d 2185static void shrink_stripes(struct r5conf *conf)
3f294f4f 2186{
566c09c5
SL		 2187 int hash;
2188 for (hash = 0; hash < NR_STRIPE_HASH_LOCKS; hash++)
2189 while (drop_one_stripe(conf, hash))
2190 ;
3f294f4f 2191
29fc7e3e
N		 2192 if (conf->slab_cache)
2193 kmem_cache_destroy(conf->slab_cache);
1da177e4
LT		 2194 conf->slab_cache = NULL;
2195}
2196
6712ecf8 2197static void raid5_end_read_request(struct bio * bi, int error)
1da177e4 2198{
99c0fb5f 2199 struct stripe_head *sh = bi->bi_private;
d1688a6d 2200 struct r5conf *conf = sh->raid_conf;
7ecaa1e6 2201 int disks = sh->disks, i;
1da177e4 2202 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
d6950432 2203 char b[BDEVNAME_SIZE];
dd054fce 2204 struct md_rdev *rdev = NULL;
05616be5 2205 sector_t s;
1da177e4
LT		 2206
2207 for (i=0 ; i<disks; i++)
2208 if (bi == &sh->dev[i].req)
2209 break;
2210
45b4233c
DW		 2211 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
2212 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
1da177e4
LT		 2213 uptodate);
2214 if (i == disks) {
2215 BUG();
6712ecf8 2216 return;
1da177e4 2217 }
14a75d3e 2218 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
dd054fce
N		 2219 /* If replacement finished while this request was outstanding,
2220 * 'replacement' might be NULL already.
2221 * In that case it moved down to 'rdev'.
2222 * rdev is not removed until all requests are finished.
2223 */
14a75d3e 2224 rdev = conf->disks[i].replacement;
dd054fce 2225 if (!rdev)
14a75d3e 2226 rdev = conf->disks[i].rdev;
1da177e4 2227
05616be5
N		 2228 if (use_new_offset(conf, sh))
2229 s = sh->sector + rdev->new_data_offset;
2230 else
2231 s = sh->sector + rdev->data_offset;
1da177e4 2232 if (uptodate) {
1da177e4 2233 set_bit(R5_UPTODATE, &sh->dev[i].flags);
4e5314b5 2234 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
14a75d3e
N		 2235 /* Note that this cannot happen on a
2236 * replacement device. We just fail those on
2237 * any error
2238 */
8bda470e
CD		 2239 printk_ratelimited(
2240 KERN_INFO
2241 "md/raid:%s: read error corrected"
2242 " (%lu sectors at %llu on %s)\n",
2243 mdname(conf->mddev), STRIPE_SECTORS,
05616be5 2244 (unsigned long long)s,
8bda470e 2245 bdevname(rdev->bdev, b));
ddd5115f 2246 atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
4e5314b5
N		 2247 clear_bit(R5_ReadError, &sh->dev[i].flags);
2248 clear_bit(R5_ReWrite, &sh->dev[i].flags);
3f9e7c14 2249 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2250 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2251
14a75d3e
N		 2252 if (atomic_read(&rdev->read_errors))
2253 atomic_set(&rdev->read_errors, 0);
1da177e4 2254 } else {
14a75d3e 2255 const char *bdn = bdevname(rdev->bdev, b);
ba22dcbf 2256 int retry = 0;
2e8ac303 2257 int set_bad = 0;
d6950432 2258
1da177e4 2259 clear_bit(R5_UPTODATE, &sh->dev[i].flags);
d6950432 2260 atomic_inc(&rdev->read_errors);
14a75d3e
N		 2261 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2262 printk_ratelimited(
2263 KERN_WARNING
2264 "md/raid:%s: read error on replacement device "
2265 "(sector %llu on %s).\n",
2266 mdname(conf->mddev),
05616be5 2267 (unsigned long long)s,
14a75d3e 2268 bdn);
2e8ac303 2269 else if (conf->mddev->degraded >= conf->max_degraded) {
2270 set_bad = 1;
8bda470e
CD		 2271 printk_ratelimited(
2272 KERN_WARNING
2273 "md/raid:%s: read error not correctable "
2274 "(sector %llu on %s).\n",
2275 mdname(conf->mddev),
05616be5 2276 (unsigned long long)s,
8bda470e 2277 bdn);
2e8ac303 2278 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
4e5314b5 2279 /* Oh, no!!! */
2e8ac303 2280 set_bad = 1;
8bda470e
CD		 2281 printk_ratelimited(
2282 KERN_WARNING
2283 "md/raid:%s: read error NOT corrected!! "
2284 "(sector %llu on %s).\n",
2285 mdname(conf->mddev),
05616be5 2286 (unsigned long long)s,
8bda470e 2287 bdn);
2e8ac303 2288 } else if (atomic_read(&rdev->read_errors)
ba22dcbf 2289 > conf->max_nr_stripes)
14f8d26b 2290 printk(KERN_WARNING
0c55e022 2291 "md/raid:%s: Too many read errors, failing device %s.\n",
d6950432 2292 mdname(conf->mddev), bdn);
ba22dcbf
N		 2293 else
2294 retry = 1;
edfa1f65
BY		 2295 if (set_bad && test_bit(In_sync, &rdev->flags)
2296 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2297 retry = 1;
ba22dcbf 2298 if (retry)
3f9e7c14 2299 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
2300 set_bit(R5_ReadError, &sh->dev[i].flags);
2301 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2302 } else
2303 set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
ba22dcbf 2304 else {
4e5314b5
N		 2305 clear_bit(R5_ReadError, &sh->dev[i].flags);
2306 clear_bit(R5_ReWrite, &sh->dev[i].flags);
2e8ac303 2307 if (!(set_bad
2308 && test_bit(In_sync, &rdev->flags)
2309 && rdev_set_badblocks(
2310 rdev, sh->sector, STRIPE_SECTORS, 0)))
2311 md_error(conf->mddev, rdev);
ba22dcbf 2312 }
1da177e4 2313 }
14a75d3e 2314 rdev_dec_pending(rdev, conf->mddev);
1da177e4
LT		 2315 clear_bit(R5_LOCKED, &sh->dev[i].flags);
2316 set_bit(STRIPE_HANDLE, &sh->state);
2317 release_stripe(sh);
1da177e4
LT		 2318}
2319
d710e138 2320static void raid5_end_write_request(struct bio *bi, int error)
1da177e4 2321{
99c0fb5f 2322 struct stripe_head *sh = bi->bi_private;
d1688a6d 2323 struct r5conf *conf = sh->raid_conf;
7ecaa1e6 2324 int disks = sh->disks, i;
977df362 2325 struct md_rdev *uninitialized_var(rdev);
1da177e4 2326 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
b84db560
N		 2327 sector_t first_bad;
2328 int bad_sectors;
977df362 2329 int replacement = 0;
1da177e4 2330
977df362
N		 2331 for (i = 0 ; i < disks; i++) {
2332 if (bi == &sh->dev[i].req) {
2333 rdev = conf->disks[i].rdev;
1da177e4 2334 break;
977df362
N		 2335 }
2336 if (bi == &sh->dev[i].rreq) {
2337 rdev = conf->disks[i].replacement;
dd054fce
N		 2338 if (rdev)
2339 replacement = 1;
2340 else
2341 /* rdev was removed and 'replacement'
2342 * replaced it. rdev is not removed
2343 * until all requests are finished.
2344 */
2345 rdev = conf->disks[i].rdev;
977df362
N		 2346 break;
2347 }
2348 }
45b4233c 2349 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
1da177e4
LT		 2350 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
2351 uptodate);
2352 if (i == disks) {
2353 BUG();
6712ecf8 2354 return;
1da177e4
LT		 2355 }
2356
977df362
N		 2357 if (replacement) {
2358 if (!uptodate)
2359 md_error(conf->mddev, rdev);
2360 else if (is_badblock(rdev, sh->sector,
2361 STRIPE_SECTORS,
2362 &first_bad, &bad_sectors))
2363 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2364 } else {
2365 if (!uptodate) {
9f97e4b1 2366 set_bit(STRIPE_DEGRADED, &sh->state);
977df362
N		 2367 set_bit(WriteErrorSeen, &rdev->flags);
2368 set_bit(R5_WriteError, &sh->dev[i].flags);
3a6de292
N		 2369 if (!test_and_set_bit(WantReplacement, &rdev->flags))
2370 set_bit(MD_RECOVERY_NEEDED,
2371 &rdev->mddev->recovery);
977df362
N		 2372 } else if (is_badblock(rdev, sh->sector,
2373 STRIPE_SECTORS,
c0b32972 2374 &first_bad, &bad_sectors)) {
977df362 2375 set_bit(R5_MadeGood, &sh->dev[i].flags);
c0b32972
N		 2376 if (test_bit(R5_ReadError, &sh->dev[i].flags))
2377 /* That was a successful write so make
2378 * sure it looks like we already did
2379 * a re-write.
2380 */
2381 set_bit(R5_ReWrite, &sh->dev[i].flags);
2382 }
977df362
N		 2383 }
2384 rdev_dec_pending(rdev, conf->mddev);
1da177e4 2385
72ac7330 2386 if (sh->batch_head && !uptodate)
2387 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
2388
977df362
N		 2389 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2390 clear_bit(R5_LOCKED, &sh->dev[i].flags);
1da177e4 2391 set_bit(STRIPE_HANDLE, &sh->state);
c04be0aa 2392 release_stripe(sh);
59fc630b 2393
2394 if (sh->batch_head && sh != sh->batch_head)
2395 release_stripe(sh->batch_head);
1da177e4
LT		 2396}
2397
784052ec 2398static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
d592a996 2399
784052ec 2400static void raid5_build_block(struct stripe_head *sh, int i, int previous)
1da177e4
LT		 2401{
2402 struct r5dev *dev = &sh->dev[i];
2403
2404 bio_init(&dev->req);
2405 dev->req.bi_io_vec = &dev->vec;
d592a996 2406 dev->req.bi_max_vecs = 1;
1da177e4
LT		 2407 dev->req.bi_private = sh;
2408
977df362
N		 2409 bio_init(&dev->rreq);
2410 dev->rreq.bi_io_vec = &dev->rvec;
d592a996 2411 dev->rreq.bi_max_vecs = 1;
977df362 2412 dev->rreq.bi_private = sh;
977df362 2413
1da177e4 2414 dev->flags = 0;
784052ec 2415 dev->sector = compute_blocknr(sh, i, previous);
1da177e4
LT		 2416}
2417
fd01b88c 2418static void error(struct mddev *mddev, struct md_rdev *rdev)
1da177e4
LT		 2419{
2420 char b[BDEVNAME_SIZE];
d1688a6d 2421 struct r5conf *conf = mddev->private;
908f4fbd 2422 unsigned long flags;
0c55e022 2423 pr_debug("raid456: error called\n");
1da177e4 2424
908f4fbd
N		 2425 spin_lock_irqsave(&conf->device_lock, flags);
2426 clear_bit(In_sync, &rdev->flags);
2427 mddev->degraded = calc_degraded(conf);
2428 spin_unlock_irqrestore(&conf->device_lock, flags);
2429 set_bit(MD_RECOVERY_INTR, &mddev->recovery);
2430
de393cde 2431 set_bit(Blocked, &rdev->flags);
6f8d0c77
N		 2432 set_bit(Faulty, &rdev->flags);
2433 set_bit(MD_CHANGE_DEVS, &mddev->flags);
2434 printk(KERN_ALERT
2435 "md/raid:%s: Disk failure on %s, disabling device.\n"
2436 "md/raid:%s: Operation continuing on %d devices.\n",
2437 mdname(mddev),
2438 bdevname(rdev->bdev, b),
2439 mdname(mddev),
2440 conf->raid_disks - mddev->degraded);
16a53ecc 2441}
1da177e4
LT		 2442
2443/*
2444 * Input: a 'big' sector number,
2445 * Output: index of the data and parity disk, and the sector # in them.
2446 */
d1688a6d 2447static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
911d4ee8
N		 2448 int previous, int *dd_idx,
2449 struct stripe_head *sh)
1da177e4 2450{
6e3b96ed 2451 sector_t stripe, stripe2;
35f2a591 2452 sector_t chunk_number;
1da177e4 2453 unsigned int chunk_offset;
911d4ee8 2454 int pd_idx, qd_idx;
67cc2b81 2455 int ddf_layout = 0;
1da177e4 2456 sector_t new_sector;
e183eaed
N		 2457 int algorithm = previous ? conf->prev_algo
2458 : conf->algorithm;
09c9e5fa
AN		 2459 int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2460 : conf->chunk_sectors;
112bf897
N		 2461 int raid_disks = previous ? conf->previous_raid_disks
2462 : conf->raid_disks;
2463 int data_disks = raid_disks - conf->max_degraded;
1da177e4
LT		 2464
2465 /* First compute the information on this sector */
2466
2467 /*
2468 * Compute the chunk number and the sector offset inside the chunk
2469 */
2470 chunk_offset = sector_div(r_sector, sectors_per_chunk);
2471 chunk_number = r_sector;
1da177e4
LT		 2472
2473 /*
2474 * Compute the stripe number
2475 */
35f2a591
N		 2476 stripe = chunk_number;
2477 *dd_idx = sector_div(stripe, data_disks);
6e3b96ed 2478 stripe2 = stripe;
1da177e4
LT		 2479 /*
2480 * Select the parity disk based on the user selected algorithm.
2481 */
84789554 2482 pd_idx = qd_idx = -1;
16a53ecc
N		 2483 switch(conf->level) {
2484 case 4:
911d4ee8 2485 pd_idx = data_disks;
16a53ecc
N		 2486 break;
2487 case 5:
e183eaed 2488 switch (algorithm) {
1da177e4 2489 case ALGORITHM_LEFT_ASYMMETRIC:
6e3b96ed 2490 pd_idx = data_disks - sector_div(stripe2, raid_disks);
911d4ee8 2491 if (*dd_idx >= pd_idx)
1da177e4
LT		 2492 (*dd_idx)++;
2493 break;
2494 case ALGORITHM_RIGHT_ASYMMETRIC:
6e3b96ed 2495 pd_idx = sector_div(stripe2, raid_disks);
911d4ee8 2496 if (*dd_idx >= pd_idx)
1da177e4
LT		 2497 (*dd_idx)++;
2498 break;
2499 case ALGORITHM_LEFT_SYMMETRIC:
6e3b96ed 2500 pd_idx = data_disks - sector_div(stripe2, raid_disks);
911d4ee8 2501 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1da177e4
LT		 2502 break;
2503 case ALGORITHM_RIGHT_SYMMETRIC:
6e3b96ed 2504 pd_idx = sector_div(stripe2, raid_disks);
911d4ee8 2505 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1da177e4 2506 break;
99c0fb5f
N		 2507 case ALGORITHM_PARITY_0:
2508 pd_idx = 0;
2509 (*dd_idx)++;
2510 break;
2511 case ALGORITHM_PARITY_N:
2512 pd_idx = data_disks;
2513 break;
1da177e4 2514 default:
99c0fb5f 2515 BUG();
16a53ecc
N		 2516 }
2517 break;
2518 case 6:
2519
e183eaed 2520 switch (algorithm) {
16a53ecc 2521 case ALGORITHM_LEFT_ASYMMETRIC:
6e3b96ed 2522 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
911d4ee8
N		 2523 qd_idx = pd_idx + 1;
2524 if (pd_idx == raid_disks-1) {
99c0fb5f 2525 (*dd_idx)++; /* Q D D D P */
911d4ee8
N		 2526 qd_idx = 0;
2527 } else if (*dd_idx >= pd_idx)
16a53ecc
N		 2528 (*dd_idx) += 2; /* D D P Q D */
2529 break;
2530 case ALGORITHM_RIGHT_ASYMMETRIC:
6e3b96ed 2531 pd_idx = sector_div(stripe2, raid_disks);
911d4ee8
N		 2532 qd_idx = pd_idx + 1;
2533 if (pd_idx == raid_disks-1) {
99c0fb5f 2534 (*dd_idx)++; /* Q D D D P */
911d4ee8
N		 2535 qd_idx = 0;
2536 } else if (*dd_idx >= pd_idx)
16a53ecc
N		 2537 (*dd_idx) += 2; /* D D P Q D */
2538 break;
2539 case ALGORITHM_LEFT_SYMMETRIC:
6e3b96ed 2540 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
911d4ee8
N		 2541 qd_idx = (pd_idx + 1) % raid_disks;
2542 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
16a53ecc
N		 2543 break;
2544 case ALGORITHM_RIGHT_SYMMETRIC:
6e3b96ed 2545 pd_idx = sector_div(stripe2, raid_disks);
911d4ee8
N		 2546 qd_idx = (pd_idx + 1) % raid_disks;
2547 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
16a53ecc 2548 break;
99c0fb5f
N		 2549
2550 case ALGORITHM_PARITY_0:
2551 pd_idx = 0;
2552 qd_idx = 1;
2553 (*dd_idx) += 2;
2554 break;
2555 case ALGORITHM_PARITY_N:
2556 pd_idx = data_disks;
2557 qd_idx = data_disks + 1;
2558 break;
2559
2560 case ALGORITHM_ROTATING_ZERO_RESTART:
2561 /* Exactly the same as RIGHT_ASYMMETRIC, but or
2562 * of blocks for computing Q is different.
2563 */
6e3b96ed 2564 pd_idx = sector_div(stripe2, raid_disks);
99c0fb5f
N		 2565 qd_idx = pd_idx + 1;
2566 if (pd_idx == raid_disks-1) {
2567 (*dd_idx)++; /* Q D D D P */
2568 qd_idx = 0;
2569 } else if (*dd_idx >= pd_idx)
2570 (*dd_idx) += 2; /* D D P Q D */
67cc2b81 2571 ddf_layout = 1;
99c0fb5f
N		 2572 break;
2573
2574 case ALGORITHM_ROTATING_N_RESTART:
2575 /* Same a left_asymmetric, by first stripe is
2576 * D D D P Q rather than
2577 * Q D D D P
2578 */
6e3b96ed
N		 2579 stripe2 += 1;
2580 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
99c0fb5f
N		 2581 qd_idx = pd_idx + 1;
2582 if (pd_idx == raid_disks-1) {
2583 (*dd_idx)++; /* Q D D D P */
2584 qd_idx = 0;
2585 } else if (*dd_idx >= pd_idx)
2586 (*dd_idx) += 2; /* D D P Q D */
67cc2b81 2587 ddf_layout = 1;
99c0fb5f
N		 2588 break;
2589
2590 case ALGORITHM_ROTATING_N_CONTINUE:
2591 /* Same as left_symmetric but Q is before P */
6e3b96ed 2592 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
99c0fb5f
N		 2593 qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
2594 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
67cc2b81 2595 ddf_layout = 1;
99c0fb5f
N		 2596 break;
2597
2598 case ALGORITHM_LEFT_ASYMMETRIC_6:
2599 /* RAID5 left_asymmetric, with Q on last device */
6e3b96ed 2600 pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
99c0fb5f
N		 2601 if (*dd_idx >= pd_idx)
2602 (*dd_idx)++;
2603 qd_idx = raid_disks - 1;
2604 break;
2605
2606 case ALGORITHM_RIGHT_ASYMMETRIC_6:
6e3b96ed 2607 pd_idx = sector_div(stripe2, raid_disks-1);
99c0fb5f
N		 2608 if (*dd_idx >= pd_idx)
2609 (*dd_idx)++;
2610 qd_idx = raid_disks - 1;
2611 break;
2612
2613 case ALGORITHM_LEFT_SYMMETRIC_6:
6e3b96ed 2614 pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
99c0fb5f
N		 2615 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2616 qd_idx = raid_disks - 1;
2617 break;
2618
2619 case ALGORITHM_RIGHT_SYMMETRIC_6:
6e3b96ed 2620 pd_idx = sector_div(stripe2, raid_disks-1);
99c0fb5f
N		 2621 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
2622 qd_idx = raid_disks - 1;
2623 break;
2624
2625 case ALGORITHM_PARITY_0_6:
2626 pd_idx = 0;
2627 (*dd_idx)++;
2628 qd_idx = raid_disks - 1;
2629 break;
2630
16a53ecc 2631 default:
99c0fb5f 2632 BUG();
16a53ecc
N		 2633 }
2634 break;
1da177e4
LT		 2635 }
2636
911d4ee8
N		 2637 if (sh) {
2638 sh->pd_idx = pd_idx;
2639 sh->qd_idx = qd_idx;
67cc2b81 2640 sh->ddf_layout = ddf_layout;
911d4ee8 2641 }
1da177e4
LT		 2642 /*
2643 * Finally, compute the new sector number
2644 */
2645 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
2646 return new_sector;
2647}
2648
784052ec 2649static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
1da177e4 2650{
d1688a6d 2651 struct r5conf *conf = sh->raid_conf;
b875e531
N		 2652 int raid_disks = sh->disks;
2653 int data_disks = raid_disks - conf->max_degraded;
1da177e4 2654 sector_t new_sector = sh->sector, check;
09c9e5fa
AN		 2655 int sectors_per_chunk = previous ? conf->prev_chunk_sectors
2656 : conf->chunk_sectors;
e183eaed
N		 2657 int algorithm = previous ? conf->prev_algo
2658 : conf->algorithm;
1da177e4
LT		 2659 sector_t stripe;
2660 int chunk_offset;
35f2a591
N		 2661 sector_t chunk_number;
2662 int dummy1, dd_idx = i;
1da177e4 2663 sector_t r_sector;
911d4ee8 2664 struct stripe_head sh2;
1da177e4
LT		 2665
2666 chunk_offset = sector_div(new_sector, sectors_per_chunk);
2667 stripe = new_sector;
1da177e4 2668
16a53ecc
N		 2669 if (i == sh->pd_idx)
2670 return 0;
2671 switch(conf->level) {
2672 case 4: break;
2673 case 5:
e183eaed 2674 switch (algorithm) {
1da177e4
LT		 2675 case ALGORITHM_LEFT_ASYMMETRIC:
2676 case ALGORITHM_RIGHT_ASYMMETRIC:
2677 if (i > sh->pd_idx)
2678 i--;
2679 break;
2680 case ALGORITHM_LEFT_SYMMETRIC:
2681 case ALGORITHM_RIGHT_SYMMETRIC:
2682 if (i < sh->pd_idx)
2683 i += raid_disks;
2684 i -= (sh->pd_idx + 1);
2685 break;
99c0fb5f
N		 2686 case ALGORITHM_PARITY_0:
2687 i -= 1;
2688 break;
2689 case ALGORITHM_PARITY_N:
2690 break;
1da177e4 2691 default:
99c0fb5f 2692 BUG();
16a53ecc
N		 2693 }
2694 break;
2695 case 6:
d0dabf7e 2696 if (i == sh->qd_idx)
16a53ecc 2697 return 0; /* It is the Q disk */
e183eaed 2698 switch (algorithm) {
16a53ecc
N		 2699 case ALGORITHM_LEFT_ASYMMETRIC:
2700 case ALGORITHM_RIGHT_ASYMMETRIC:
99c0fb5f
N		 2701 case ALGORITHM_ROTATING_ZERO_RESTART:
2702 case ALGORITHM_ROTATING_N_RESTART:
2703 if (sh->pd_idx == raid_disks-1)
2704 i--; /* Q D D D P */
16a53ecc
N		 2705 else if (i > sh->pd_idx)
2706 i -= 2; /* D D P Q D */
2707 break;
2708 case ALGORITHM_LEFT_SYMMETRIC:
2709 case ALGORITHM_RIGHT_SYMMETRIC:
2710 if (sh->pd_idx == raid_disks-1)
2711 i--; /* Q D D D P */
2712 else {
2713 /* D D P Q D */
2714 if (i < sh->pd_idx)
2715 i += raid_disks;
2716 i -= (sh->pd_idx + 2);
2717 }
2718 break;
99c0fb5f
N		 2719 case ALGORITHM_PARITY_0:
2720 i -= 2;
2721 break;
2722 case ALGORITHM_PARITY_N:
2723 break;
2724 case ALGORITHM_ROTATING_N_CONTINUE:
e4424fee 2725 /* Like left_symmetric, but P is before Q */
99c0fb5f
N		 2726 if (sh->pd_idx == 0)
2727 i--; /* P D D D Q */
e4424fee
N		 2728 else {
2729 /* D D Q P D */
2730 if (i < sh->pd_idx)
2731 i += raid_disks;
2732 i -= (sh->pd_idx + 1);
2733 }
99c0fb5f
N		 2734 break;
2735 case ALGORITHM_LEFT_ASYMMETRIC_6:
2736 case ALGORITHM_RIGHT_ASYMMETRIC_6:
2737 if (i > sh->pd_idx)
2738 i--;
2739 break;
2740 case ALGORITHM_LEFT_SYMMETRIC_6:
2741 case ALGORITHM_RIGHT_SYMMETRIC_6:
2742 if (i < sh->pd_idx)
2743 i += data_disks + 1;
2744 i -= (sh->pd_idx + 1);
2745 break;
2746 case ALGORITHM_PARITY_0_6:
2747 i -= 1;
2748 break;
16a53ecc 2749 default:
99c0fb5f 2750 BUG();
16a53ecc
N		 2751 }
2752 break;
1da177e4
LT		 2753 }
2754
2755 chunk_number = stripe * data_disks + i;
35f2a591 2756 r_sector = chunk_number * sectors_per_chunk + chunk_offset;
1da177e4 2757
112bf897 2758 check = raid5_compute_sector(conf, r_sector,
784052ec 2759 previous, &dummy1, &sh2);
911d4ee8
N		 2760 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
2761 || sh2.qd_idx != sh->qd_idx) {
0c55e022
N		 2762 printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
2763 mdname(conf->mddev));
1da177e4
LT		 2764 return 0;
2765 }
2766 return r_sector;
2767}
2768
600aa109 2769static void
c0f7bddb 2770schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
600aa109 2771 int rcw, int expand)
e33129d8
DW		 2772{
2773 int i, pd_idx = sh->pd_idx, disks = sh->disks;
d1688a6d 2774 struct r5conf *conf = sh->raid_conf;
c0f7bddb 2775 int level = conf->level;
e33129d8
DW		 2776
2777 if (rcw) {
e33129d8
DW		 2778
2779 for (i = disks; i--; ) {
2780 struct r5dev *dev = &sh->dev[i];
2781
2782 if (dev->towrite) {
2783 set_bit(R5_LOCKED, &dev->flags);
d8ee0728 2784 set_bit(R5_Wantdrain, &dev->flags);
e33129d8
DW		 2785 if (!expand)
2786 clear_bit(R5_UPTODATE, &dev->flags);
600aa109 2787 s->locked++;
e33129d8
DW		 2788 }
2789 }
ce7d363a
N		 2790 /* if we are not expanding this is a proper write request, and
2791 * there will be bios with new data to be drained into the
2792 * stripe cache
2793 */
2794 if (!expand) {
2795 if (!s->locked)
2796 /* False alarm, nothing to do */
2797 return;
2798 sh->reconstruct_state = reconstruct_state_drain_run;
2799 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2800 } else
2801 sh->reconstruct_state = reconstruct_state_run;
2802
2803 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2804
c0f7bddb 2805 if (s->locked + conf->max_degraded == disks)
8b3e6cdc 2806 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
c0f7bddb 2807 atomic_inc(&conf->pending_full_writes);
e33129d8 2808 } else {
c0f7bddb 2809 BUG_ON(level == 6);
e33129d8
DW		 2810 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
2811 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
2812
e33129d8
DW		 2813 for (i = disks; i--; ) {
2814 struct r5dev *dev = &sh->dev[i];
2815 if (i == pd_idx)
2816 continue;
2817
e33129d8
DW		 2818 if (dev->towrite &&
2819 (test_bit(R5_UPTODATE, &dev->flags) ||
d8ee0728
DW		 2820 test_bit(R5_Wantcompute, &dev->flags))) {
2821 set_bit(R5_Wantdrain, &dev->flags);
e33129d8
DW		 2822 set_bit(R5_LOCKED, &dev->flags);
2823 clear_bit(R5_UPTODATE, &dev->flags);
600aa109 2824 s->locked++;
e33129d8
DW		 2825 }
2826 }
ce7d363a
N		 2827 if (!s->locked)
2828 /* False alarm - nothing to do */
2829 return;
2830 sh->reconstruct_state = reconstruct_state_prexor_drain_run;
2831 set_bit(STRIPE_OP_PREXOR, &s->ops_request);
2832 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2833 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
e33129d8
DW		 2834 }
2835
c0f7bddb 2836 /* keep the parity disk(s) locked while asynchronous operations
e33129d8
DW		 2837 * are in flight
2838 */
2839 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
2840 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
600aa109 2841 s->locked++;
e33129d8 2842
c0f7bddb
YT		 2843 if (level == 6) {
2844 int qd_idx = sh->qd_idx;
2845 struct r5dev *dev = &sh->dev[qd_idx];
2846
2847 set_bit(R5_LOCKED, &dev->flags);
2848 clear_bit(R5_UPTODATE, &dev->flags);
2849 s->locked++;
2850 }
2851
600aa109 2852 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
e46b272b 2853 __func__, (unsigned long long)sh->sector,
600aa109 2854 s->locked, s->ops_request);
e33129d8 2855}
16a53ecc 2856
1da177e4
LT		 2857/*
2858 * Each stripe/dev can have one or more bion attached.
16a53ecc 2859 * toread/towrite point to the first in a chain.
1da177e4
LT		 2860 * The bi_next chain must be in order.
2861 */
da41ba65 2862static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,
2863 int forwrite, int previous)
1da177e4
LT		 2864{
2865 struct bio **bip;
d1688a6d 2866 struct r5conf *conf = sh->raid_conf;
72626685 2867 int firstwrite=0;
1da177e4 2868
cbe47ec5 2869 pr_debug("adding bi b#%llu to stripe s#%llu\n",
4f024f37 2870 (unsigned long long)bi->bi_iter.bi_sector,
1da177e4
LT		 2871 (unsigned long long)sh->sector);
2872
b17459c0
SL		 2873 /*
2874 * If several bio share a stripe. The bio bi_phys_segments acts as a
2875 * reference count to avoid race. The reference count should already be
2876 * increased before this function is called (for example, in
2877 * make_request()), so other bio sharing this stripe will not free the
2878 * stripe. If a stripe is owned by one stripe, the stripe lock will
2879 * protect it.
2880 */
2881 spin_lock_irq(&sh->stripe_lock);
59fc630b 2882 /* Don't allow new IO added to stripes in batch list */
2883 if (sh->batch_head)
2884 goto overlap;
72626685 2885 if (forwrite) {
1da177e4 2886 bip = &sh->dev[dd_idx].towrite;
7eaf7e8e 2887 if (*bip == NULL)
72626685
N		 2888 firstwrite = 1;
2889 } else
1da177e4 2890 bip = &sh->dev[dd_idx].toread;
4f024f37
KO		 2891 while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
2892 if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
1da177e4
LT		 2893 goto overlap;
2894 bip = & (*bip)->bi_next;
2895 }
4f024f37 2896 if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
1da177e4
LT		 2897 goto overlap;
2898
da41ba65 2899 if (!forwrite || previous)
2900 clear_bit(STRIPE_BATCH_READY, &sh->state);
2901
78bafebd 2902 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
1da177e4
LT		 2903 if (*bip)
2904 bi->bi_next = *bip;
2905 *bip = bi;
e7836bd6 2906 raid5_inc_bi_active_stripes(bi);
72626685 2907
1da177e4
LT		 2908 if (forwrite) {
2909 /* check if page is covered */
2910 sector_t sector = sh->dev[dd_idx].sector;
2911 for (bi=sh->dev[dd_idx].towrite;
2912 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
4f024f37 2913 bi && bi->bi_iter.bi_sector <= sector;
1da177e4 2914 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
f73a1c7d
KO		 2915 if (bio_end_sector(bi) >= sector)
2916 sector = bio_end_sector(bi);
1da177e4
LT		 2917 }
2918 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
7a87f434 2919 if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
2920 sh->overwrite_disks++;
1da177e4 2921 }
cbe47ec5
N		 2922
2923 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
4f024f37 2924 (unsigned long long)(*bip)->bi_iter.bi_sector,
cbe47ec5 2925 (unsigned long long)sh->sector, dd_idx);
b97390ae 2926 spin_unlock_irq(&sh->stripe_lock);
cbe47ec5
N		 2927
2928 if (conf->mddev->bitmap && firstwrite) {
2929 bitmap_startwrite(conf->mddev->bitmap, sh->sector,
2930 STRIPE_SECTORS, 0);
2931 sh->bm_seq = conf->seq_flush+1;
2932 set_bit(STRIPE_BIT_DELAY, &sh->state);
2933 }
59fc630b 2934
2935 if (stripe_can_batch(sh))
2936 stripe_add_to_batch_list(conf, sh);
1da177e4
LT		 2937 return 1;
2938
2939 overlap:
2940 set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
b17459c0 2941 spin_unlock_irq(&sh->stripe_lock);
1da177e4
LT		 2942 return 0;
2943}
2944
d1688a6d 2945static void end_reshape(struct r5conf *conf);
29269553 2946
d1688a6d 2947static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
911d4ee8 2948 struct stripe_head *sh)
ccfcc3c1 2949{
784052ec 2950 int sectors_per_chunk =
09c9e5fa 2951 previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
911d4ee8 2952 int dd_idx;
2d2063ce 2953 int chunk_offset = sector_div(stripe, sectors_per_chunk);
112bf897 2954 int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
2d2063ce 2955
112bf897
N		 2956 raid5_compute_sector(conf,
2957 stripe * (disks - conf->max_degraded)
b875e531 2958 *sectors_per_chunk + chunk_offset,
112bf897 2959 previous,
911d4ee8 2960 &dd_idx, sh);
ccfcc3c1
N		 2961}
2962
a4456856 2963static void
d1688a6d 2964handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
a4456856
DW		 2965 struct stripe_head_state *s, int disks,
2966 struct bio **return_bi)
2967{
2968 int i;
59fc630b 2969 BUG_ON(sh->batch_head);
a4456856
DW		 2970 for (i = disks; i--; ) {
2971 struct bio *bi;
2972 int bitmap_end = 0;
2973
2974 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
3cb03002 2975 struct md_rdev *rdev;
a4456856
DW		 2976 rcu_read_lock();
2977 rdev = rcu_dereference(conf->disks[i].rdev);
2978 if (rdev && test_bit(In_sync, &rdev->flags))
7f0da59b
N		 2979 atomic_inc(&rdev->nr_pending);
2980 else
2981 rdev = NULL;
a4456856 2982 rcu_read_unlock();
7f0da59b
N		 2983 if (rdev) {
2984 if (!rdev_set_badblocks(
2985 rdev,
2986 sh->sector,
2987 STRIPE_SECTORS, 0))
2988 md_error(conf->mddev, rdev);
2989 rdev_dec_pending(rdev, conf->mddev);
2990 }
a4456856 2991 }
b17459c0 2992 spin_lock_irq(&sh->stripe_lock);
a4456856
DW		 2993 /* fail all writes first */
2994 bi = sh->dev[i].towrite;
2995 sh->dev[i].towrite = NULL;
7a87f434 2996 sh->overwrite_disks = 0;
b17459c0 2997 spin_unlock_irq(&sh->stripe_lock);
1ed850f3 2998 if (bi)
a4456856 2999 bitmap_end = 1;
a4456856
DW		 3000
3001 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3002 wake_up(&conf->wait_for_overlap);
3003
4f024f37 3004 while (bi && bi->bi_iter.bi_sector <
a4456856
DW		 3005 sh->dev[i].sector + STRIPE_SECTORS) {
3006 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
3007 clear_bit(BIO_UPTODATE, &bi->bi_flags);
e7836bd6 3008 if (!raid5_dec_bi_active_stripes(bi)) {
a4456856
DW		 3009 md_write_end(conf->mddev);
3010 bi->bi_next = *return_bi;
3011 *return_bi = bi;
3012 }
3013 bi = nextbi;
3014 }
7eaf7e8e
SL		 3015 if (bitmap_end)
3016 bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3017 STRIPE_SECTORS, 0, 0);
3018 bitmap_end = 0;
a4456856
DW		 3019 /* and fail all 'written' */
3020 bi = sh->dev[i].written;
3021 sh->dev[i].written = NULL;
d592a996
SL		 3022 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
3023 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
3024 sh->dev[i].page = sh->dev[i].orig_page;
3025 }
3026
a4456856 3027 if (bi) bitmap_end = 1;
4f024f37 3028 while (bi && bi->bi_iter.bi_sector <
a4456856
DW		 3029 sh->dev[i].sector + STRIPE_SECTORS) {
3030 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
3031 clear_bit(BIO_UPTODATE, &bi->bi_flags);
e7836bd6 3032 if (!raid5_dec_bi_active_stripes(bi)) {
a4456856
DW		 3033 md_write_end(conf->mddev);
3034 bi->bi_next = *return_bi;
3035 *return_bi = bi;
3036 }
3037 bi = bi2;
3038 }
3039
b5e98d65
DW		 3040 /* fail any reads if this device is non-operational and
3041 * the data has not reached the cache yet.
3042 */
3043 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
3044 (!test_bit(R5_Insync, &sh->dev[i].flags) ||
3045 test_bit(R5_ReadError, &sh->dev[i].flags))) {
143c4d05 3046 spin_lock_irq(&sh->stripe_lock);
a4456856
DW		 3047 bi = sh->dev[i].toread;
3048 sh->dev[i].toread = NULL;
143c4d05 3049 spin_unlock_irq(&sh->stripe_lock);
a4456856
DW		 3050 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3051 wake_up(&conf->wait_for_overlap);
4f024f37 3052 while (bi && bi->bi_iter.bi_sector <
a4456856
DW		 3053 sh->dev[i].sector + STRIPE_SECTORS) {
3054 struct bio *nextbi =
3055 r5_next_bio(bi, sh->dev[i].sector);
3056 clear_bit(BIO_UPTODATE, &bi->bi_flags);
e7836bd6 3057 if (!raid5_dec_bi_active_stripes(bi)) {
a4456856
DW		 3058 bi->bi_next = *return_bi;
3059 *return_bi = bi;
3060 }
3061 bi = nextbi;
3062 }
3063 }
a4456856
DW		 3064 if (bitmap_end)
3065 bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3066 STRIPE_SECTORS, 0, 0);
8cfa7b0f
N		 3067 /* If we were in the middle of a write the parity block might
3068 * still be locked - so just clear all R5_LOCKED flags
3069 */
3070 clear_bit(R5_LOCKED, &sh->dev[i].flags);
a4456856
DW		 3071 }
3072
8b3e6cdc
DW		 3073 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
3074 if (atomic_dec_and_test(&conf->pending_full_writes))
3075 md_wakeup_thread(conf->mddev->thread);
a4456856
DW		 3076}
3077
7f0da59b 3078static void
d1688a6d 3079handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
7f0da59b
N		 3080 struct stripe_head_state *s)
3081{
3082 int abort = 0;
3083 int i;
3084
59fc630b 3085 BUG_ON(sh->batch_head);
7f0da59b 3086 clear_bit(STRIPE_SYNCING, &sh->state);
f8dfcffd
N		 3087 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
3088 wake_up(&conf->wait_for_overlap);
7f0da59b 3089 s->syncing = 0;
9a3e1101 3090 s->replacing = 0;
7f0da59b 3091 /* There is nothing more to do for sync/check/repair.
18b9837e
N		 3092 * Don't even need to abort as that is handled elsewhere
3093 * if needed, and not always wanted e.g. if there is a known
3094 * bad block here.
9a3e1101 3095 * For recover/replace we need to record a bad block on all
7f0da59b
N		 3096 * non-sync devices, or abort the recovery
3097 */
18b9837e
N		 3098 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
3099 /* During recovery devices cannot be removed, so
3100 * locking and refcounting of rdevs is not needed
3101 */
3102 for (i = 0; i < conf->raid_disks; i++) {
3103 struct md_rdev *rdev = conf->disks[i].rdev;
3104 if (rdev
3105 && !test_bit(Faulty, &rdev->flags)
3106 && !test_bit(In_sync, &rdev->flags)
3107 && !rdev_set_badblocks(rdev, sh->sector,
3108 STRIPE_SECTORS, 0))
3109 abort = 1;
3110 rdev = conf->disks[i].replacement;
3111 if (rdev
3112 && !test_bit(Faulty, &rdev->flags)
3113 && !test_bit(In_sync, &rdev->flags)
3114 && !rdev_set_badblocks(rdev, sh->sector,
3115 STRIPE_SECTORS, 0))
3116 abort = 1;
3117 }
3118 if (abort)
3119 conf->recovery_disabled =
3120 conf->mddev->recovery_disabled;
7f0da59b 3121 }
18b9837e 3122 md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
7f0da59b
N		 3123}
3124
9a3e1101
N		 3125static int want_replace(struct stripe_head *sh, int disk_idx)
3126{
3127 struct md_rdev *rdev;
3128 int rv = 0;
3129 /* Doing recovery so rcu locking not required */
3130 rdev = sh->raid_conf->disks[disk_idx].replacement;
3131 if (rdev
3132 && !test_bit(Faulty, &rdev->flags)
3133 && !test_bit(In_sync, &rdev->flags)
3134 && (rdev->recovery_offset <= sh->sector
3135 || rdev->mddev->recovery_cp <= sh->sector))
3136 rv = 1;
3137
3138 return rv;
3139}
3140
93b3dbce 3141/* fetch_block - checks the given member device to see if its data needs
1fe797e6
DW		 3142 * to be read or computed to satisfy a request.
3143 *
3144 * Returns 1 when no more member devices need to be checked, otherwise returns
93b3dbce 3145 * 0 to tell the loop in handle_stripe_fill to continue
f38e1219 3146 */
2c58f06e
N		 3147
3148static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
3149 int disk_idx, int disks)
a4456856 3150{
5599becc 3151 struct r5dev *dev = &sh->dev[disk_idx];
f2b3b44d
N		 3152 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
3153 &sh->dev[s->failed_num[1]] };
ea664c82 3154 int i;
5599becc 3155
a79cfe12
N		 3156
3157 if (test_bit(R5_LOCKED, &dev->flags) ||
3158 test_bit(R5_UPTODATE, &dev->flags))
3159 /* No point reading this as we already have it or have
3160 * decided to get it.
3161 */
3162 return 0;
3163
3164 if (dev->toread ||
3165 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)))
3166 /* We need this block to directly satisfy a request */
3167 return 1;
3168
3169 if (s->syncing || s->expanding ||
3170 (s->replacing && want_replace(sh, disk_idx)))
3171 /* When syncing, or expanding we read everything.
3172 * When replacing, we need the replaced block.
3173 */
3174 return 1;
3175
3176 if ((s->failed >= 1 && fdev[0]->toread) ||
3177 (s->failed >= 2 && fdev[1]->toread))
3178 /* If we want to read from a failed device, then
3179 * we need to actually read every other device.
3180 */
3181 return 1;
3182
a9d56950
N		 3183 /* Sometimes neither read-modify-write nor reconstruct-write
3184 * cycles can work. In those cases we read every block we
3185 * can. Then the parity-update is certain to have enough to
3186 * work with.
3187 * This can only be a problem when we need to write something,
3188 * and some device has failed. If either of those tests
3189 * fail we need look no further.
3190 */
3191 if (!s->failed || !s->to_write)
3192 return 0;
3193
3194 if (test_bit(R5_Insync, &dev->flags) &&
3195 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3196 /* Pre-reads at not permitted until after short delay
3197 * to gather multiple requests. However if this
3198 * device is no Insync, the block could only be be computed
3199 * and there is no need to delay that.
3200 */
3201 return 0;
ea664c82
N		 3202
3203 for (i = 0; i < s->failed; i++) {
3204 if (fdev[i]->towrite &&
3205 !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
3206 !test_bit(R5_OVERWRITE, &fdev[i]->flags))
3207 /* If we have a partial write to a failed
3208 * device, then we will need to reconstruct
3209 * the content of that device, so all other
3210 * devices must be read.
3211 */
3212 return 1;
3213 }
3214
3215 /* If we are forced to do a reconstruct-write, either because
3216 * the current RAID6 implementation only supports that, or
3217 * or because parity cannot be trusted and we are currently
3218 * recovering it, there is extra need to be careful.
3219 * If one of the devices that we would need to read, because
3220 * it is not being overwritten (and maybe not written at all)
3221 * is missing/faulty, then we need to read everything we can.
3222 */
3223 if (sh->raid_conf->level != 6 &&
3224 sh->sector < sh->raid_conf->mddev->recovery_cp)
3225 /* reconstruct-write isn't being forced */
3226 return 0;
3227 for (i = 0; i < s->failed; i++) {
3228 if (!test_bit(R5_UPTODATE, &fdev[i]->flags) &&
3229 !test_bit(R5_OVERWRITE, &fdev[i]->flags))
3230 return 1;
3231 }
3232
2c58f06e
N		 3233 return 0;
3234}
3235
3236static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
3237 int disk_idx, int disks)
3238{
3239 struct r5dev *dev = &sh->dev[disk_idx];
3240
3241 /* is the data in this block needed, and can we get it? */
3242 if (need_this_block(sh, s, disk_idx, disks)) {
5599becc
YT		 3243 /* we would like to get this block, possibly by computing it,
3244 * otherwise read it if the backing disk is insync
3245 */
3246 BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
3247 BUG_ON(test_bit(R5_Wantread, &dev->flags));
3248 if ((s->uptodate == disks - 1) &&
f2b3b44d
N		 3249 (s->failed && (disk_idx == s->failed_num[0] ||
3250 disk_idx == s->failed_num[1]))) {
5599becc
YT		 3251 /* have disk failed, and we're requested to fetch it;
3252 * do compute it
a4456856 3253 */
5599becc
YT		 3254 pr_debug("Computing stripe %llu block %d\n",
3255 (unsigned long long)sh->sector, disk_idx);
3256 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3257 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3258 set_bit(R5_Wantcompute, &dev->flags);
3259 sh->ops.target = disk_idx;
3260 sh->ops.target2 = -1; /* no 2nd target */
3261 s->req_compute = 1;
93b3dbce
N		 3262 /* Careful: from this point on 'uptodate' is in the eye
3263 * of raid_run_ops which services 'compute' operations
3264 * before writes. R5_Wantcompute flags a block that will
3265 * be R5_UPTODATE by the time it is needed for a
3266 * subsequent operation.
3267 */
5599becc
YT		 3268 s->uptodate++;
3269 return 1;
3270 } else if (s->uptodate == disks-2 && s->failed >= 2) {
3271 /* Computing 2-failure is *very* expensive; only
3272 * do it if failed >= 2
3273 */
3274 int other;
3275 for (other = disks; other--; ) {
3276 if (other == disk_idx)
3277 continue;
3278 if (!test_bit(R5_UPTODATE,
3279 &sh->dev[other].flags))
3280 break;
a4456856 3281 }
5599becc
YT		 3282 BUG_ON(other < 0);
3283 pr_debug("Computing stripe %llu blocks %d,%d\n",
3284 (unsigned long long)sh->sector,
3285 disk_idx, other);
3286 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3287 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3288 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
3289 set_bit(R5_Wantcompute, &sh->dev[other].flags);
3290 sh->ops.target = disk_idx;
3291 sh->ops.target2 = other;
3292 s->uptodate += 2;
3293 s->req_compute = 1;
3294 return 1;
3295 } else if (test_bit(R5_Insync, &dev->flags)) {
3296 set_bit(R5_LOCKED, &dev->flags);
3297 set_bit(R5_Wantread, &dev->flags);
3298 s->locked++;
3299 pr_debug("Reading block %d (sync=%d)\n",
3300 disk_idx, s->syncing);
a4456856
DW		 3301 }
3302 }
5599becc
YT		 3303
3304 return 0;
3305}
3306
3307/**
93b3dbce 3308 * handle_stripe_fill - read or compute data to satisfy pending requests.
5599becc 3309 */
93b3dbce
N		 3310static void handle_stripe_fill(struct stripe_head *sh,
3311 struct stripe_head_state *s,
3312 int disks)
5599becc
YT		 3313{
3314 int i;
3315
59fc630b 3316 BUG_ON(sh->batch_head);
5599becc
YT		 3317 /* look for blocks to read/compute, skip this if a compute
3318 * is already in flight, or if the stripe contents are in the
3319 * midst of changing due to a write
3320 */
3321 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
3322 !sh->reconstruct_state)
3323 for (i = disks; i--; )
93b3dbce 3324 if (fetch_block(sh, s, i, disks))
5599becc 3325 break;
a4456856
DW		 3326 set_bit(STRIPE_HANDLE, &sh->state);
3327}
3328
1fe797e6 3329/* handle_stripe_clean_event
a4456856
DW		 3330 * any written block on an uptodate or failed drive can be returned.
3331 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
3332 * never LOCKED, so we don't need to test 'failed' directly.
3333 */
d1688a6d 3334static void handle_stripe_clean_event(struct r5conf *conf,
a4456856
DW		 3335 struct stripe_head *sh, int disks, struct bio **return_bi)
3336{
3337 int i;
3338 struct r5dev *dev;
f8dfcffd 3339 int discard_pending = 0;
59fc630b 3340 struct stripe_head *head_sh = sh;
3341 bool do_endio = false;
3342 int wakeup_nr = 0;
a4456856
DW		 3343
3344 for (i = disks; i--; )
3345 if (sh->dev[i].written) {
3346 dev = &sh->dev[i];
3347 if (!test_bit(R5_LOCKED, &dev->flags) &&
9e444768 3348 (test_bit(R5_UPTODATE, &dev->flags) ||
d592a996
SL		 3349 test_bit(R5_Discard, &dev->flags) ||
3350 test_bit(R5_SkipCopy, &dev->flags))) {
a4456856
DW		 3351 /* We can return any write requests */
3352 struct bio *wbi, *wbi2;
45b4233c 3353 pr_debug("Return write for disc %d\n", i);
ca64cae9
N		 3354 if (test_and_clear_bit(R5_Discard, &dev->flags))
3355 clear_bit(R5_UPTODATE, &dev->flags);
d592a996
SL		 3356 if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {
3357 WARN_ON(test_bit(R5_UPTODATE, &dev->flags));
d592a996 3358 }
59fc630b 3359 do_endio = true;
3360
3361returnbi:
3362 dev->page = dev->orig_page;
a4456856
DW		 3363 wbi = dev->written;
3364 dev->written = NULL;
4f024f37 3365 while (wbi && wbi->bi_iter.bi_sector <
a4456856
DW		 3366 dev->sector + STRIPE_SECTORS) {
3367 wbi2 = r5_next_bio(wbi, dev->sector);
e7836bd6 3368 if (!raid5_dec_bi_active_stripes(wbi)) {
a4456856
DW		 3369 md_write_end(conf->mddev);
3370 wbi->bi_next = *return_bi;
3371 *return_bi = wbi;
3372 }
3373 wbi = wbi2;
3374 }
7eaf7e8e
SL		 3375 bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3376 STRIPE_SECTORS,
a4456856 3377 !test_bit(STRIPE_DEGRADED, &sh->state),
7eaf7e8e 3378 0);
59fc630b 3379 if (head_sh->batch_head) {
3380 sh = list_first_entry(&sh->batch_list,
3381 struct stripe_head,
3382 batch_list);
3383 if (sh != head_sh) {
3384 dev = &sh->dev[i];
3385 goto returnbi;
3386 }
3387 }
3388 sh = head_sh;
3389 dev = &sh->dev[i];
f8dfcffd
N		 3390 } else if (test_bit(R5_Discard, &dev->flags))
3391 discard_pending = 1;
d592a996
SL		 3392 WARN_ON(test_bit(R5_SkipCopy, &dev->flags));
3393 WARN_ON(dev->page != dev->orig_page);
f8dfcffd
N		 3394 }
3395 if (!discard_pending &&
3396 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
3397 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
3398 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
3399 if (sh->qd_idx >= 0) {
3400 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
3401 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
3402 }
3403 /* now that discard is done we can proceed with any sync */
3404 clear_bit(STRIPE_DISCARD, &sh->state);
d47648fc
SL		 3405 /*
3406 * SCSI discard will change some bio fields and the stripe has
3407 * no updated data, so remove it from hash list and the stripe
3408 * will be reinitialized
3409 */
3410 spin_lock_irq(&conf->device_lock);
59fc630b 3411unhash:
d47648fc 3412 remove_hash(sh);
59fc630b 3413 if (head_sh->batch_head) {
3414 sh = list_first_entry(&sh->batch_list,
3415 struct stripe_head, batch_list);
3416 if (sh != head_sh)
3417 goto unhash;
3418 }
d47648fc 3419 spin_unlock_irq(&conf->device_lock);
59fc630b 3420 sh = head_sh;
3421
f8dfcffd
N		 3422 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
3423 set_bit(STRIPE_HANDLE, &sh->state);
3424
3425 }
8b3e6cdc
DW		 3426
3427 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
3428 if (atomic_dec_and_test(&conf->pending_full_writes))
3429 md_wakeup_thread(conf->mddev->thread);
59fc630b 3430
3431 if (!head_sh->batch_head || !do_endio)
3432 return;
3433 for (i = 0; i < head_sh->disks; i++) {
3434 if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags))
3435 wakeup_nr++;
3436 }
3437 while (!list_empty(&head_sh->batch_list)) {
3438 int i;
3439 sh = list_first_entry(&head_sh->batch_list,
3440 struct stripe_head, batch_list);
3441 list_del_init(&sh->batch_list);
3442
3443 sh->state = head_sh->state & (~((1 << STRIPE_ACTIVE) |
3444 (1 << STRIPE_PREREAD_ACTIVE)));
3445 sh->check_state = head_sh->check_state;
3446 sh->reconstruct_state = head_sh->reconstruct_state;
3447 for (i = 0; i < sh->disks; i++) {
3448 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3449 wakeup_nr++;
3450 sh->dev[i].flags = head_sh->dev[i].flags;
3451 }
3452
3453 spin_lock_irq(&sh->stripe_lock);
3454 sh->batch_head = NULL;
3455 spin_unlock_irq(&sh->stripe_lock);
3456 release_stripe(sh);
3457 }
3458
3459 spin_lock_irq(&head_sh->stripe_lock);
3460 head_sh->batch_head = NULL;
3461 spin_unlock_irq(&head_sh->stripe_lock);
3462 wake_up_nr(&conf->wait_for_overlap, wakeup_nr);
a4456856
DW		 3463}
3464
d1688a6d 3465static void handle_stripe_dirtying(struct r5conf *conf,
c8ac1803
N		 3466 struct stripe_head *sh,
3467 struct stripe_head_state *s,
3468 int disks)
a4456856
DW		 3469{
3470 int rmw = 0, rcw = 0, i;
a7854487
AL		 3471 sector_t recovery_cp = conf->mddev->recovery_cp;
3472
3473 /* RAID6 requires 'rcw' in current implementation.
3474 * Otherwise, check whether resync is now happening or should start.
3475 * If yes, then the array is dirty (after unclean shutdown or
3476 * initial creation), so parity in some stripes might be inconsistent.
3477 * In this case, we need to always do reconstruct-write, to ensure
3478 * that in case of drive failure or read-error correction, we
3479 * generate correct data from the parity.
3480 */
3481 if (conf->max_degraded == 2 ||
26ac1073
N		 3482 (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
3483 s->failed == 0)) {
a7854487 3484 /* Calculate the real rcw later - for now make it
c8ac1803
N		 3485 * look like rcw is cheaper
3486 */
3487 rcw = 1; rmw = 2;
a7854487
AL		 3488 pr_debug("force RCW max_degraded=%u, recovery_cp=%llu sh->sector=%llu\n",
3489 conf->max_degraded, (unsigned long long)recovery_cp,
3490 (unsigned long long)sh->sector);
c8ac1803 3491 } else for (i = disks; i--; ) {
a4456856
DW		 3492 /* would I have to read this buffer for read_modify_write */
3493 struct r5dev *dev = &sh->dev[i];
3494 if ((dev->towrite || i == sh->pd_idx) &&
3495 !test_bit(R5_LOCKED, &dev->flags) &&
f38e1219
DW		 3496 !(test_bit(R5_UPTODATE, &dev->flags) ||
3497 test_bit(R5_Wantcompute, &dev->flags))) {
a4456856
DW		 3498 if (test_bit(R5_Insync, &dev->flags))
3499 rmw++;
3500 else
3501 rmw += 2*disks; /* cannot read it */
3502 }
3503 /* Would I have to read this buffer for reconstruct_write */
3504 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
3505 !test_bit(R5_LOCKED, &dev->flags) &&
f38e1219
DW		 3506 !(test_bit(R5_UPTODATE, &dev->flags) ||
3507 test_bit(R5_Wantcompute, &dev->flags))) {
67f45548
N		 3508 if (test_bit(R5_Insync, &dev->flags))
3509 rcw++;
a4456856
DW		 3510 else
3511 rcw += 2*disks;
3512 }
3513 }
45b4233c 3514 pr_debug("for sector %llu, rmw=%d rcw=%d\n",
a4456856
DW		 3515 (unsigned long long)sh->sector, rmw, rcw);
3516 set_bit(STRIPE_HANDLE, &sh->state);
a9add5d9 3517 if (rmw < rcw && rmw > 0) {
a4456856 3518 /* prefer read-modify-write, but need to get some data */
e3620a3a
JB		 3519 if (conf->mddev->queue)
3520 blk_add_trace_msg(conf->mddev->queue,
3521 "raid5 rmw %llu %d",
3522 (unsigned long long)sh->sector, rmw);
a4456856
DW		 3523 for (i = disks; i--; ) {
3524 struct r5dev *dev = &sh->dev[i];
3525 if ((dev->towrite || i == sh->pd_idx) &&
3526 !test_bit(R5_LOCKED, &dev->flags) &&
f38e1219
DW		 3527 !(test_bit(R5_UPTODATE, &dev->flags) ||
3528 test_bit(R5_Wantcompute, &dev->flags)) &&
a4456856 3529 test_bit(R5_Insync, &dev->flags)) {
67f45548
N		 3530 if (test_bit(STRIPE_PREREAD_ACTIVE,
3531 &sh->state)) {
3532 pr_debug("Read_old block %d for r-m-w\n",
3533 i);
a4456856
DW		 3534 set_bit(R5_LOCKED, &dev->flags);
3535 set_bit(R5_Wantread, &dev->flags);
3536 s->locked++;
3537 } else {
3538 set_bit(STRIPE_DELAYED, &sh->state);
3539 set_bit(STRIPE_HANDLE, &sh->state);
3540 }
3541 }
3542 }
a9add5d9 3543 }
c8ac1803 3544 if (rcw <= rmw && rcw > 0) {
a4456856 3545 /* want reconstruct write, but need to get some data */
a9add5d9 3546 int qread =0;
c8ac1803 3547 rcw = 0;
a4456856
DW		 3548 for (i = disks; i--; ) {
3549 struct r5dev *dev = &sh->dev[i];
3550 if (!test_bit(R5_OVERWRITE, &dev->flags) &&
c8ac1803 3551 i != sh->pd_idx && i != sh->qd_idx &&
a4456856 3552 !test_bit(R5_LOCKED, &dev->flags) &&
f38e1219 3553 !(test_bit(R5_UPTODATE, &dev->flags) ||
c8ac1803
N		 3554 test_bit(R5_Wantcompute, &dev->flags))) {
3555 rcw++;
67f45548
N		 3556 if (test_bit(R5_Insync, &dev->flags) &&
3557 test_bit(STRIPE_PREREAD_ACTIVE,
3558 &sh->state)) {
45b4233c 3559 pr_debug("Read_old block "
a4456856
DW		 3560 "%d for Reconstruct\n", i);
3561 set_bit(R5_LOCKED, &dev->flags);
3562 set_bit(R5_Wantread, &dev->flags);
3563 s->locked++;
a9add5d9 3564 qread++;
a4456856
DW		 3565 } else {
3566 set_bit(STRIPE_DELAYED, &sh->state);
3567 set_bit(STRIPE_HANDLE, &sh->state);
3568 }
3569 }
3570 }
e3620a3a 3571 if (rcw && conf->mddev->queue)
a9add5d9
N		 3572 blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
3573 (unsigned long long)sh->sector,
3574 rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
c8ac1803 3575 }
b1b02fe9
N		 3576
3577 if (rcw > disks && rmw > disks &&
3578 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3579 set_bit(STRIPE_DELAYED, &sh->state);
3580
a4456856
DW		 3581 /* now if nothing is locked, and if we have enough data,
3582 * we can start a write request
3583 */
f38e1219
DW		 3584 /* since handle_stripe can be called at any time we need to handle the
3585 * case where a compute block operation has been submitted and then a
ac6b53b6
DW		 3586 * subsequent call wants to start a write request. raid_run_ops only
3587 * handles the case where compute block and reconstruct are requested
f38e1219
DW		 3588 * simultaneously. If this is not the case then new writes need to be
3589 * held off until the compute completes.
3590 */
976ea8d4
DW		 3591 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
3592 (s->locked == 0 && (rcw == 0 || rmw == 0) &&
3593 !test_bit(STRIPE_BIT_DELAY, &sh->state)))
c0f7bddb 3594 schedule_reconstruction(sh, s, rcw == 0, 0);
a4456856
DW		 3595}
3596
d1688a6d 3597static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
a4456856
DW		 3598 struct stripe_head_state *s, int disks)
3599{
ecc65c9b 3600 struct r5dev *dev = NULL;
bd2ab670 3601
59fc630b 3602 BUG_ON(sh->batch_head);
a4456856 3603 set_bit(STRIPE_HANDLE, &sh->state);
e89f8962 3604
ecc65c9b
DW		 3605 switch (sh->check_state) {
3606 case check_state_idle:
3607 /* start a new check operation if there are no failures */
bd2ab670 3608 if (s->failed == 0) {
bd2ab670 3609 BUG_ON(s->uptodate != disks);
ecc65c9b
DW		 3610 sh->check_state = check_state_run;
3611 set_bit(STRIPE_OP_CHECK, &s->ops_request);
bd2ab670 3612 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
bd2ab670 3613 s->uptodate--;
ecc65c9b 3614 break;
bd2ab670 3615 }
f2b3b44d 3616 dev = &sh->dev[s->failed_num[0]];
ecc65c9b
DW		 3617 /* fall through */
3618 case check_state_compute_result:
3619 sh->check_state = check_state_idle;
3620 if (!dev)
3621 dev = &sh->dev[sh->pd_idx];
3622
3623 /* check that a write has not made the stripe insync */
3624 if (test_bit(STRIPE_INSYNC, &sh->state))
3625 break;
c8894419 3626
a4456856 3627 /* either failed parity check, or recovery is happening */
a4456856
DW		 3628 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
3629 BUG_ON(s->uptodate != disks);
3630
3631 set_bit(R5_LOCKED, &dev->flags);
ecc65c9b 3632 s->locked++;
a4456856 3633 set_bit(R5_Wantwrite, &dev->flags);
830ea016 3634
a4456856 3635 clear_bit(STRIPE_DEGRADED, &sh->state);
a4456856 3636 set_bit(STRIPE_INSYNC, &sh->state);
ecc65c9b
DW		 3637 break;
3638 case check_state_run:
3639 break; /* we will be called again upon completion */
3640 case check_state_check_result:
3641 sh->check_state = check_state_idle;
3642
3643 /* if a failure occurred during the check operation, leave
3644 * STRIPE_INSYNC not set and let the stripe be handled again
3645 */
3646 if (s->failed)
3647 break;
3648
3649 /* handle a successful check operation, if parity is correct
3650 * we are done. Otherwise update the mismatch count and repair
3651 * parity if !MD_RECOVERY_CHECK
3652 */
ad283ea4 3653 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
ecc65c9b
DW		 3654 /* parity is correct (on disc,
3655 * not in buffer any more)
3656 */
3657 set_bit(STRIPE_INSYNC, &sh->state);
3658 else {
7f7583d4 3659 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
ecc65c9b
DW		 3660 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
3661 /* don't try to repair!! */
3662 set_bit(STRIPE_INSYNC, &sh->state);
3663 else {
3664 sh->check_state = check_state_compute_run;
976ea8d4 3665 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
ecc65c9b
DW		 3666 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3667 set_bit(R5_Wantcompute,
3668 &sh->dev[sh->pd_idx].flags);
3669 sh->ops.target = sh->pd_idx;
ac6b53b6 3670 sh->ops.target2 = -1;
ecc65c9b
DW		 3671 s->uptodate++;
3672 }
3673 }
3674 break;
3675 case check_state_compute_run:
3676 break;
3677 default:
3678 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
3679 __func__, sh->check_state,
3680 (unsigned long long) sh->sector);
3681 BUG();
a4456856
DW		 3682 }
3683}
3684
d1688a6d 3685static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
36d1c647 3686 struct stripe_head_state *s,
f2b3b44d 3687 int disks)
a4456856 3688{
a4456856 3689 int pd_idx = sh->pd_idx;
34e04e87 3690 int qd_idx = sh->qd_idx;
d82dfee0 3691 struct r5dev *dev;
a4456856 3692
59fc630b 3693 BUG_ON(sh->batch_head);
a4456856
DW		 3694 set_bit(STRIPE_HANDLE, &sh->state);
3695
3696 BUG_ON(s->failed > 2);
d82dfee0 3697
a4456856
DW		 3698 /* Want to check and possibly repair P and Q.
3699 * However there could be one 'failed' device, in which
3700 * case we can only check one of them, possibly using the
3701 * other to generate missing data
3702 */
3703
d82dfee0
DW		 3704 switch (sh->check_state) {
3705 case check_state_idle:
3706 /* start a new check operation if there are < 2 failures */
f2b3b44d 3707 if (s->failed == s->q_failed) {
d82dfee0 3708 /* The only possible failed device holds Q, so it
a4456856
DW		 3709 * makes sense to check P (If anything else were failed,
3710 * we would have used P to recreate it).
3711 */
d82dfee0 3712 sh->check_state = check_state_run;
a4456856 3713 }
f2b3b44d 3714 if (!s->q_failed && s->failed < 2) {
d82dfee0 3715 /* Q is not failed, and we didn't use it to generate
a4456856
DW		 3716 * anything, so it makes sense to check it
3717 */
d82dfee0
DW		 3718 if (sh->check_state == check_state_run)
3719 sh->check_state = check_state_run_pq;
3720 else
3721 sh->check_state = check_state_run_q;
a4456856 3722 }
a4456856 3723
d82dfee0
DW		 3724 /* discard potentially stale zero_sum_result */
3725 sh->ops.zero_sum_result = 0;
a4456856 3726
d82dfee0
DW		 3727 if (sh->check_state == check_state_run) {
3728 /* async_xor_zero_sum destroys the contents of P */
3729 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
3730 s->uptodate--;
a4456856 3731 }
d82dfee0
DW		 3732 if (sh->check_state >= check_state_run &&
3733 sh->check_state <= check_state_run_pq) {
3734 /* async_syndrome_zero_sum preserves P and Q, so
3735 * no need to mark them !uptodate here
3736 */
3737 set_bit(STRIPE_OP_CHECK, &s->ops_request);
3738 break;
a4456856
DW		 3739 }
3740
d82dfee0
DW		 3741 /* we have 2-disk failure */
3742 BUG_ON(s->failed != 2);
3743 /* fall through */
3744 case check_state_compute_result:
3745 sh->check_state = check_state_idle;
a4456856 3746
d82dfee0
DW		 3747 /* check that a write has not made the stripe insync */
3748 if (test_bit(STRIPE_INSYNC, &sh->state))
3749 break;
a4456856
DW		 3750
3751 /* now write out any block on a failed drive,
d82dfee0 3752 * or P or Q if they were recomputed
a4456856 3753 */
d82dfee0 3754 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
a4456856 3755 if (s->failed == 2) {
f2b3b44d 3756 dev = &sh->dev[s->failed_num[1]];
a4456856
DW		 3757 s->locked++;
3758 set_bit(R5_LOCKED, &dev->flags);
3759 set_bit(R5_Wantwrite, &dev->flags);
3760 }
3761 if (s->failed >= 1) {
f2b3b44d 3762 dev = &sh->dev[s->failed_num[0]];
a4456856
DW		 3763 s->locked++;
3764 set_bit(R5_LOCKED, &dev->flags);
3765 set_bit(R5_Wantwrite, &dev->flags);
3766 }
d82dfee0 3767 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
a4456856
DW		 3768 dev = &sh->dev[pd_idx];
3769 s->locked++;
3770 set_bit(R5_LOCKED, &dev->flags);
3771 set_bit(R5_Wantwrite, &dev->flags);
3772 }
d82dfee0 3773 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
a4456856
DW		 3774 dev = &sh->dev[qd_idx];
3775 s->locked++;
3776 set_bit(R5_LOCKED, &dev->flags);
3777 set_bit(R5_Wantwrite, &dev->flags);
3778 }
3779 clear_bit(STRIPE_DEGRADED, &sh->state);
3780
3781 set_bit(STRIPE_INSYNC, &sh->state);
d82dfee0
DW		 3782 break;
3783 case check_state_run:
3784 case check_state_run_q:
3785 case check_state_run_pq:
3786 break; /* we will be called again upon completion */
3787 case check_state_check_result:
3788 sh->check_state = check_state_idle;
3789
3790 /* handle a successful check operation, if parity is correct
3791 * we are done. Otherwise update the mismatch count and repair
3792 * parity if !MD_RECOVERY_CHECK
3793 */
3794 if (sh->ops.zero_sum_result == 0) {
3795 /* both parities are correct */
3796 if (!s->failed)
3797 set_bit(STRIPE_INSYNC, &sh->state);
3798 else {
3799 /* in contrast to the raid5 case we can validate
3800 * parity, but still have a failure to write
3801 * back
3802 */
3803 sh->check_state = check_state_compute_result;
3804 /* Returning at this point means that we may go
3805 * off and bring p and/or q uptodate again so
3806 * we make sure to check zero_sum_result again
3807 * to verify if p or q need writeback
3808 */
3809 }
3810 } else {
7f7583d4 3811 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
d82dfee0
DW		 3812 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
3813 /* don't try to repair!! */
3814 set_bit(STRIPE_INSYNC, &sh->state);
3815 else {
3816 int *target = &sh->ops.target;
3817
3818 sh->ops.target = -1;
3819 sh->ops.target2 = -1;
3820 sh->check_state = check_state_compute_run;
3821 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3822 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
3823 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
3824 set_bit(R5_Wantcompute,
3825 &sh->dev[pd_idx].flags);
3826 *target = pd_idx;
3827 target = &sh->ops.target2;
3828 s->uptodate++;
3829 }
3830 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3831 set_bit(R5_Wantcompute,
3832 &sh->dev[qd_idx].flags);
3833 *target = qd_idx;
3834 s->uptodate++;
3835 }
3836 }
3837 }
3838 break;
3839 case check_state_compute_run:
3840 break;
3841 default:
3842 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
3843 __func__, sh->check_state,
3844 (unsigned long long) sh->sector);
3845 BUG();
a4456856
DW		 3846 }
3847}
3848
d1688a6d 3849static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
a4456856
DW		 3850{
3851 int i;
3852
3853 /* We have read all the blocks in this stripe and now we need to
3854 * copy some of them into a target stripe for expand.
3855 */
f0a50d37 3856 struct dma_async_tx_descriptor *tx = NULL;
59fc630b 3857 BUG_ON(sh->batch_head);
a4456856
DW		 3858 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3859 for (i = 0; i < sh->disks; i++)
34e04e87 3860 if (i != sh->pd_idx && i != sh->qd_idx) {
911d4ee8 3861 int dd_idx, j;
a4456856 3862 struct stripe_head *sh2;
a08abd8c 3863 struct async_submit_ctl submit;
a4456856 3864
784052ec 3865 sector_t bn = compute_blocknr(sh, i, 1);
911d4ee8
N		 3866 sector_t s = raid5_compute_sector(conf, bn, 0,
3867 &dd_idx, NULL);
a8c906ca 3868 sh2 = get_active_stripe(conf, s, 0, 1, 1);
a4456856
DW		 3869 if (sh2 == NULL)
3870 /* so far only the early blocks of this stripe
3871 * have been requested. When later blocks
3872 * get requested, we will try again
3873 */
3874 continue;
3875 if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
3876 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
3877 /* must have already done this block */
3878 release_stripe(sh2);
3879 continue;
3880 }
f0a50d37
DW		 3881
3882 /* place all the copies on one channel */
a08abd8c 3883 init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
f0a50d37 3884 tx = async_memcpy(sh2->dev[dd_idx].page,
88ba2aa5 3885 sh->dev[i].page, 0, 0, STRIPE_SIZE,
a08abd8c 3886 &submit);
f0a50d37 3887
a4456856
DW		 3888 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
3889 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
3890 for (j = 0; j < conf->raid_disks; j++)
3891 if (j != sh2->pd_idx &&
86c374ba 3892 j != sh2->qd_idx &&
a4456856
DW		 3893 !test_bit(R5_Expanded, &sh2->dev[j].flags))
3894 break;
3895 if (j == conf->raid_disks) {
3896 set_bit(STRIPE_EXPAND_READY, &sh2->state);
3897 set_bit(STRIPE_HANDLE, &sh2->state);
3898 }
3899 release_stripe(sh2);
f0a50d37 3900
a4456856 3901 }
a2e08551 3902 /* done submitting copies, wait for them to complete */
749586b7 3903 async_tx_quiesce(&tx);
a4456856 3904}
1da177e4
LT		 3905
3906/*
3907 * handle_stripe - do things to a stripe.
3908 *
9a3e1101
N		 3909 * We lock the stripe by setting STRIPE_ACTIVE and then examine the
3910 * state of various bits to see what needs to be done.
1da177e4 3911 * Possible results:
9a3e1101
N		 3912 * return some read requests which now have data
3913 * return some write requests which are safely on storage
1da177e4
LT		 3914 * schedule a read on some buffers
3915 * schedule a write of some buffers
3916 * return confirmation of parity correctness
3917 *
1da177e4 3918 */
a4456856 3919
acfe726b 3920static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
1da177e4 3921{
d1688a6d 3922 struct r5conf *conf = sh->raid_conf;
f416885e 3923 int disks = sh->disks;
474af965
N		 3924 struct r5dev *dev;
3925 int i;
9a3e1101 3926 int do_recovery = 0;
1da177e4 3927
acfe726b
N		 3928 memset(s, 0, sizeof(*s));
3929
acfe726b
N		 3930 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3931 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
3932 s->failed_num[0] = -1;
3933 s->failed_num[1] = -1;
1da177e4 3934
acfe726b 3935 /* Now to look around and see what can be done */
1da177e4 3936 rcu_read_lock();
16a53ecc 3937 for (i=disks; i--; ) {
3cb03002 3938 struct md_rdev *rdev;
31c176ec
N		 3939 sector_t first_bad;
3940 int bad_sectors;
3941 int is_bad = 0;
acfe726b 3942
16a53ecc 3943 dev = &sh->dev[i];
1da177e4 3944
45b4233c 3945 pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
9a3e1101
N		 3946 i, dev->flags,
3947 dev->toread, dev->towrite, dev->written);
6c0069c0
YT		 3948 /* maybe we can reply to a read
3949 *
3950 * new wantfill requests are only permitted while
3951 * ops_complete_biofill is guaranteed to be inactive
3952 */
3953 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
3954 !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
3955 set_bit(R5_Wantfill, &dev->flags);
1da177e4 3956
16a53ecc 3957 /* now count some things */
cc94015a
N		 3958 if (test_bit(R5_LOCKED, &dev->flags))
3959 s->locked++;
3960 if (test_bit(R5_UPTODATE, &dev->flags))
3961 s->uptodate++;
2d6e4ecc 3962 if (test_bit(R5_Wantcompute, &dev->flags)) {
cc94015a
N		 3963 s->compute++;
3964 BUG_ON(s->compute > 2);
2d6e4ecc 3965 }
1da177e4 3966
acfe726b 3967 if (test_bit(R5_Wantfill, &dev->flags))
cc94015a 3968 s->to_fill++;
acfe726b 3969 else if (dev->toread)
cc94015a 3970 s->to_read++;
16a53ecc 3971 if (dev->towrite) {
cc94015a 3972 s->to_write++;
16a53ecc 3973 if (!test_bit(R5_OVERWRITE, &dev->flags))
cc94015a 3974 s->non_overwrite++;
16a53ecc 3975 }
a4456856 3976 if (dev->written)
cc94015a 3977 s->written++;
14a75d3e
N		 3978 /* Prefer to use the replacement for reads, but only
3979 * if it is recovered enough and has no bad blocks.
3980 */
3981 rdev = rcu_dereference(conf->disks[i].replacement);
3982 if (rdev && !test_bit(Faulty, &rdev->flags) &&
3983 rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
3984 !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
3985 &first_bad, &bad_sectors))
3986 set_bit(R5_ReadRepl, &dev->flags);
3987 else {
9a3e1101
N		 3988 if (rdev)
3989 set_bit(R5_NeedReplace, &dev->flags);
14a75d3e
N		 3990 rdev = rcu_dereference(conf->disks[i].rdev);
3991 clear_bit(R5_ReadRepl, &dev->flags);
3992 }
9283d8c5
N		 3993 if (rdev && test_bit(Faulty, &rdev->flags))
3994 rdev = NULL;
31c176ec
N		 3995 if (rdev) {
3996 is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
3997 &first_bad, &bad_sectors);
3998 if (s->blocked_rdev == NULL
3999 && (test_bit(Blocked, &rdev->flags)
4000 || is_bad < 0)) {
4001 if (is_bad < 0)
4002 set_bit(BlockedBadBlocks,
4003 &rdev->flags);
4004 s->blocked_rdev = rdev;
4005 atomic_inc(&rdev->nr_pending);
4006 }
6bfe0b49 4007 }
415e72d0
N		 4008 clear_bit(R5_Insync, &dev->flags);
4009 if (!rdev)
4010 /* Not in-sync */;
31c176ec
N		 4011 else if (is_bad) {
4012 /* also not in-sync */
18b9837e
N		 4013 if (!test_bit(WriteErrorSeen, &rdev->flags) &&
4014 test_bit(R5_UPTODATE, &dev->flags)) {
31c176ec
N		 4015 /* treat as in-sync, but with a read error
4016 * which we can now try to correct
4017 */
4018 set_bit(R5_Insync, &dev->flags);
4019 set_bit(R5_ReadError, &dev->flags);
4020 }
4021 } else if (test_bit(In_sync, &rdev->flags))
415e72d0 4022 set_bit(R5_Insync, &dev->flags);
30d7a483 4023 else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
415e72d0 4024 /* in sync if before recovery_offset */
30d7a483
N		 4025 set_bit(R5_Insync, &dev->flags);
4026 else if (test_bit(R5_UPTODATE, &dev->flags) &&
4027 test_bit(R5_Expanded, &dev->flags))
4028 /* If we've reshaped into here, we assume it is Insync.
4029 * We will shortly update recovery_offset to make
4030 * it official.
4031 */
4032 set_bit(R5_Insync, &dev->flags);
4033
1cc03eb9 4034 if (test_bit(R5_WriteError, &dev->flags)) {
14a75d3e
N		 4035 /* This flag does not apply to '.replacement'
4036 * only to .rdev, so make sure to check that*/
4037 struct md_rdev *rdev2 = rcu_dereference(
4038 conf->disks[i].rdev);
4039 if (rdev2 == rdev)
4040 clear_bit(R5_Insync, &dev->flags);
4041 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
bc2607f3 4042 s->handle_bad_blocks = 1;
14a75d3e 4043 atomic_inc(&rdev2->nr_pending);
bc2607f3
N		 4044 } else
4045 clear_bit(R5_WriteError, &dev->flags);
4046 }
1cc03eb9 4047 if (test_bit(R5_MadeGood, &dev->flags)) {
14a75d3e
N		 4048 /* This flag does not apply to '.replacement'
4049 * only to .rdev, so make sure to check that*/
4050 struct md_rdev *rdev2 = rcu_dereference(
4051 conf->disks[i].rdev);
4052 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
b84db560 4053 s->handle_bad_blocks = 1;
14a75d3e 4054 atomic_inc(&rdev2->nr_pending);
b84db560
N		 4055 } else
4056 clear_bit(R5_MadeGood, &dev->flags);
4057 }
977df362
N		 4058 if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
4059 struct md_rdev *rdev2 = rcu_dereference(
4060 conf->disks[i].replacement);
4061 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
4062 s->handle_bad_blocks = 1;
4063 atomic_inc(&rdev2->nr_pending);
4064 } else
4065 clear_bit(R5_MadeGoodRepl, &dev->flags);
4066 }
415e72d0 4067 if (!test_bit(R5_Insync, &dev->flags)) {
16a53ecc
N		 4068 /* The ReadError flag will just be confusing now */
4069 clear_bit(R5_ReadError, &dev->flags);
4070 clear_bit(R5_ReWrite, &dev->flags);
1da177e4 4071 }
415e72d0
N		 4072 if (test_bit(R5_ReadError, &dev->flags))
4073 clear_bit(R5_Insync, &dev->flags);
4074 if (!test_bit(R5_Insync, &dev->flags)) {
cc94015a
N		 4075 if (s->failed < 2)
4076 s->failed_num[s->failed] = i;
4077 s->failed++;
9a3e1101
N		 4078 if (rdev && !test_bit(Faulty, &rdev->flags))
4079 do_recovery = 1;
415e72d0 4080 }
1da177e4 4081 }
9a3e1101
N		 4082 if (test_bit(STRIPE_SYNCING, &sh->state)) {
4083 /* If there is a failed device being replaced,
4084 * we must be recovering.
4085 * else if we are after recovery_cp, we must be syncing
c6d2e084 4086 * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
9a3e1101
N		 4087 * else we can only be replacing
4088 * sync and recovery both need to read all devices, and so
4089 * use the same flag.
4090 */
4091 if (do_recovery ||
c6d2e084 4092 sh->sector >= conf->mddev->recovery_cp ||
4093 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
9a3e1101
N		 4094 s->syncing = 1;
4095 else
4096 s->replacing = 1;
4097 }
1da177e4 4098 rcu_read_unlock();
cc94015a
N		 4099}
4100
59fc630b 4101static int clear_batch_ready(struct stripe_head *sh)
4102{
4103 struct stripe_head *tmp;
4104 if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
4105 return 0;
4106 spin_lock(&sh->stripe_lock);
4107 if (!sh->batch_head) {
4108 spin_unlock(&sh->stripe_lock);
4109 return 0;
4110 }
4111
4112 /*
4113 * this stripe could be added to a batch list before we check
4114 * BATCH_READY, skips it
4115 */
4116 if (sh->batch_head != sh) {
4117 spin_unlock(&sh->stripe_lock);
4118 return 1;
4119 }
4120 spin_lock(&sh->batch_lock);
4121 list_for_each_entry(tmp, &sh->batch_list, batch_list)
4122 clear_bit(STRIPE_BATCH_READY, &tmp->state);
4123 spin_unlock(&sh->batch_lock);
4124 spin_unlock(&sh->stripe_lock);
4125
4126 /*
4127 * BATCH_READY is cleared, no new stripes can be added.
4128 * batch_list can be accessed without lock
4129 */
4130 return 0;
4131}
4132
72ac7330 4133static void check_break_stripe_batch_list(struct stripe_head *sh)
4134{
4135 struct stripe_head *head_sh, *next;
4136 int i;
4137
4138 if (!test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
4139 return;
4140
4141 head_sh = sh;
4142 do {
4143 sh = list_first_entry(&sh->batch_list,
4144 struct stripe_head, batch_list);
4145 BUG_ON(sh == head_sh);
4146 } while (!test_bit(STRIPE_DEGRADED, &sh->state));
4147
4148 while (sh != head_sh) {
4149 next = list_first_entry(&sh->batch_list,
4150 struct stripe_head, batch_list);
4151 list_del_init(&sh->batch_list);
4152
4153 sh->state = head_sh->state & ~((1 << STRIPE_ACTIVE) |
4154 (1 << STRIPE_PREREAD_ACTIVE) |
4155 (1 << STRIPE_DEGRADED));
4156 sh->check_state = head_sh->check_state;
4157 sh->reconstruct_state = head_sh->reconstruct_state;
4158 for (i = 0; i < sh->disks; i++)
4159 sh->dev[i].flags = head_sh->dev[i].flags &
4160 (~((1 << R5_WriteError) | (1 << R5_Overlap)));
4161
4162 spin_lock_irq(&sh->stripe_lock);
4163 sh->batch_head = NULL;
4164 spin_unlock_irq(&sh->stripe_lock);
4165
4166 set_bit(STRIPE_HANDLE, &sh->state);
4167 release_stripe(sh);
4168
4169 sh = next;
4170 }
4171}
4172
cc94015a
N		 4173static void handle_stripe(struct stripe_head *sh)
4174{
4175 struct stripe_head_state s;
d1688a6d 4176 struct r5conf *conf = sh->raid_conf;
3687c061 4177 int i;
84789554
N		 4178 int prexor;
4179 int disks = sh->disks;
474af965 4180 struct r5dev *pdev, *qdev;
cc94015a
N		 4181
4182 clear_bit(STRIPE_HANDLE, &sh->state);
257a4b42 4183 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
cc94015a
N		 4184 /* already being handled, ensure it gets handled
4185 * again when current action finishes */
4186 set_bit(STRIPE_HANDLE, &sh->state);
4187 return;
4188 }
4189
59fc630b 4190 if (clear_batch_ready(sh) ) {
4191 clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4192 return;
4193 }
4194
72ac7330 4195 check_break_stripe_batch_list(sh);
4196
f8dfcffd
N		 4197 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
4198 spin_lock(&sh->stripe_lock);
4199 /* Cannot process 'sync' concurrently with 'discard' */
4200 if (!test_bit(STRIPE_DISCARD, &sh->state) &&
4201 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
4202 set_bit(STRIPE_SYNCING, &sh->state);
4203 clear_bit(STRIPE_INSYNC, &sh->state);
f94c0b66 4204 clear_bit(STRIPE_REPLACED, &sh->state);
f8dfcffd
N		 4205 }
4206 spin_unlock(&sh->stripe_lock);
cc94015a
N		 4207 }
4208 clear_bit(STRIPE_DELAYED, &sh->state);
4209
4210 pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
4211 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
4212 (unsigned long long)sh->sector, sh->state,
4213 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
4214 sh->check_state, sh->reconstruct_state);
3687c061 4215
acfe726b 4216 analyse_stripe(sh, &s);
c5a31000 4217
bc2607f3
N		 4218 if (s.handle_bad_blocks) {
4219 set_bit(STRIPE_HANDLE, &sh->state);
4220 goto finish;
4221 }
4222
474af965
N		 4223 if (unlikely(s.blocked_rdev)) {
4224 if (s.syncing || s.expanding || s.expanded ||
9a3e1101 4225 s.replacing || s.to_write || s.written) {
474af965
N		 4226 set_bit(STRIPE_HANDLE, &sh->state);
4227 goto finish;
4228 }
4229 /* There is nothing for the blocked_rdev to block */
4230 rdev_dec_pending(s.blocked_rdev, conf->mddev);
4231 s.blocked_rdev = NULL;
4232 }
4233
4234 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
4235 set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
4236 set_bit(STRIPE_BIOFILL_RUN, &sh->state);
4237 }
4238
4239 pr_debug("locked=%d uptodate=%d to_read=%d"
4240 " to_write=%d failed=%d failed_num=%d,%d\n",
4241 s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
4242 s.failed_num[0], s.failed_num[1]);
4243 /* check if the array has lost more than max_degraded devices and,
4244 * if so, some requests might need to be failed.
4245 */
9a3f530f
N		 4246 if (s.failed > conf->max_degraded) {
4247 sh->check_state = 0;
4248 sh->reconstruct_state = 0;
4249 if (s.to_read+s.to_write+s.written)
4250 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
9a3e1101 4251 if (s.syncing + s.replacing)
9a3f530f
N		 4252 handle_failed_sync(conf, sh, &s);
4253 }
474af965 4254
84789554
N		 4255 /* Now we check to see if any write operations have recently
4256 * completed
4257 */
4258 prexor = 0;
4259 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
4260 prexor = 1;
4261 if (sh->reconstruct_state == reconstruct_state_drain_result ||
4262 sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
4263 sh->reconstruct_state = reconstruct_state_idle;
4264
4265 /* All the 'written' buffers and the parity block are ready to
4266 * be written back to disk
4267 */
9e444768
SL		 4268 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
4269 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
84789554 4270 BUG_ON(sh->qd_idx >= 0 &&
9e444768
SL		 4271 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
4272 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
84789554
N		 4273 for (i = disks; i--; ) {
4274 struct r5dev *dev = &sh->dev[i];
4275 if (test_bit(R5_LOCKED, &dev->flags) &&
4276 (i == sh->pd_idx || i == sh->qd_idx ||
4277 dev->written)) {
4278 pr_debug("Writing block %d\n", i);
4279 set_bit(R5_Wantwrite, &dev->flags);
4280 if (prexor)
4281 continue;
9c4bdf69
N		 4282 if (s.failed > 1)
4283 continue;
84789554
N		 4284 if (!test_bit(R5_Insync, &dev->flags) ||
4285 ((i == sh->pd_idx || i == sh->qd_idx) &&
4286 s.failed == 0))
4287 set_bit(STRIPE_INSYNC, &sh->state);
4288 }
4289 }
4290 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4291 s.dec_preread_active = 1;
4292 }
4293
ef5b7c69
N		 4294 /*
4295 * might be able to return some write requests if the parity blocks
4296 * are safe, or on a failed drive
4297 */
4298 pdev = &sh->dev[sh->pd_idx];
4299 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
4300 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
4301 qdev = &sh->dev[sh->qd_idx];
4302 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
4303 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
4304 || conf->level < 6;
4305
4306 if (s.written &&
4307 (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
4308 && !test_bit(R5_LOCKED, &pdev->flags)
4309 && (test_bit(R5_UPTODATE, &pdev->flags) ||
4310 test_bit(R5_Discard, &pdev->flags))))) &&
4311 (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
4312 && !test_bit(R5_LOCKED, &qdev->flags)
4313 && (test_bit(R5_UPTODATE, &qdev->flags) ||
4314 test_bit(R5_Discard, &qdev->flags))))))
4315 handle_stripe_clean_event(conf, sh, disks, &s.return_bi);
4316
4317 /* Now we might consider reading some blocks, either to check/generate
4318 * parity, or to satisfy requests
4319 * or to load a block that is being partially written.
4320 */
4321 if (s.to_read || s.non_overwrite
4322 || (conf->level == 6 && s.to_write && s.failed)
4323 || (s.syncing && (s.uptodate + s.compute < disks))
4324 || s.replacing
4325 || s.expanding)
4326 handle_stripe_fill(sh, &s, disks);
4327
84789554
N		 4328 /* Now to consider new write requests and what else, if anything
4329 * should be read. We do not handle new writes when:
4330 * 1/ A 'write' operation (copy+xor) is already in flight.
4331 * 2/ A 'check' operation is in flight, as it may clobber the parity
4332 * block.
4333 */
4334 if (s.to_write && !sh->reconstruct_state && !sh->check_state)
4335 handle_stripe_dirtying(conf, sh, &s, disks);
4336
4337 /* maybe we need to check and possibly fix the parity for this stripe
4338 * Any reads will already have been scheduled, so we just see if enough
4339 * data is available. The parity check is held off while parity
4340 * dependent operations are in flight.
4341 */
4342 if (sh->check_state ||
4343 (s.syncing && s.locked == 0 &&
4344 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
4345 !test_bit(STRIPE_INSYNC, &sh->state))) {
4346 if (conf->level == 6)
4347 handle_parity_checks6(conf, sh, &s, disks);
4348 else
4349 handle_parity_checks5(conf, sh, &s, disks);
4350 }
c5a31000 4351
f94c0b66
N		 4352 if ((s.replacing || s.syncing) && s.locked == 0
4353 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
4354 && !test_bit(STRIPE_REPLACED, &sh->state)) {
9a3e1101
N		 4355 /* Write out to replacement devices where possible */
4356 for (i = 0; i < conf->raid_disks; i++)
f94c0b66
N		 4357 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
4358 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
9a3e1101
N		 4359 set_bit(R5_WantReplace, &sh->dev[i].flags);
4360 set_bit(R5_LOCKED, &sh->dev[i].flags);
4361 s.locked++;
4362 }
f94c0b66
N		 4363 if (s.replacing)
4364 set_bit(STRIPE_INSYNC, &sh->state);
4365 set_bit(STRIPE_REPLACED, &sh->state);
9a3e1101
N		 4366 }
4367 if ((s.syncing || s.replacing) && s.locked == 0 &&
f94c0b66 4368 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
9a3e1101 4369 test_bit(STRIPE_INSYNC, &sh->state)) {
c5a31000
N		 4370 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
4371 clear_bit(STRIPE_SYNCING, &sh->state);
f8dfcffd
N		 4372 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
4373 wake_up(&conf->wait_for_overlap);
c5a31000
N		 4374 }
4375
4376 /* If the failed drives are just a ReadError, then we might need
4377 * to progress the repair/check process
4378 */
4379 if (s.failed <= conf->max_degraded && !conf->mddev->ro)
4380 for (i = 0; i < s.failed; i++) {
4381 struct r5dev *dev = &sh->dev[s.failed_num[i]];
4382 if (test_bit(R5_ReadError, &dev->flags)
4383 && !test_bit(R5_LOCKED, &dev->flags)
4384 && test_bit(R5_UPTODATE, &dev->flags)
4385 ) {
4386 if (!test_bit(R5_ReWrite, &dev->flags)) {
4387 set_bit(R5_Wantwrite, &dev->flags);
4388 set_bit(R5_ReWrite, &dev->flags);
4389 set_bit(R5_LOCKED, &dev->flags);
4390 s.locked++;
4391 } else {
4392 /* let's read it back */
4393 set_bit(R5_Wantread, &dev->flags);
4394 set_bit(R5_LOCKED, &dev->flags);
4395 s.locked++;
4396 }
4397 }
4398 }
4399
3687c061
N		 4400 /* Finish reconstruct operations initiated by the expansion process */
4401 if (sh->reconstruct_state == reconstruct_state_result) {
4402 struct stripe_head *sh_src
4403 = get_active_stripe(conf, sh->sector, 1, 1, 1);
4404 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
4405 /* sh cannot be written until sh_src has been read.
4406 * so arrange for sh to be delayed a little
4407 */
4408 set_bit(STRIPE_DELAYED, &sh->state);
4409 set_bit(STRIPE_HANDLE, &sh->state);
4410 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
4411 &sh_src->state))
4412 atomic_inc(&conf->preread_active_stripes);
4413 release_stripe(sh_src);
4414 goto finish;
4415 }
4416 if (sh_src)
4417 release_stripe(sh_src);
4418
4419 sh->reconstruct_state = reconstruct_state_idle;
4420 clear_bit(STRIPE_EXPANDING, &sh->state);
4421 for (i = conf->raid_disks; i--; ) {
4422 set_bit(R5_Wantwrite, &sh->dev[i].flags);
4423 set_bit(R5_LOCKED, &sh->dev[i].flags);
4424 s.locked++;
4425 }
4426 }
f416885e 4427
3687c061
N		 4428 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
4429 !sh->reconstruct_state) {
4430 /* Need to write out all blocks after computing parity */
4431 sh->disks = conf->raid_disks;
4432 stripe_set_idx(sh->sector, conf, 0, sh);
4433 schedule_reconstruction(sh, &s, 1, 1);
4434 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
4435 clear_bit(STRIPE_EXPAND_READY, &sh->state);
4436 atomic_dec(&conf->reshape_stripes);
4437 wake_up(&conf->wait_for_overlap);
4438 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
4439 }
4440
4441 if (s.expanding && s.locked == 0 &&
4442 !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
4443 handle_stripe_expansion(conf, sh);
16a53ecc 4444
3687c061 4445finish:
6bfe0b49 4446 /* wait for this device to become unblocked */
5f066c63
N		 4447 if (unlikely(s.blocked_rdev)) {
4448 if (conf->mddev->external)
4449 md_wait_for_blocked_rdev(s.blocked_rdev,
4450 conf->mddev);
4451 else
4452 /* Internal metadata will immediately
4453 * be written by raid5d, so we don't
4454 * need to wait here.
4455 */
4456 rdev_dec_pending(s.blocked_rdev,
4457 conf->mddev);
4458 }
6bfe0b49 4459
bc2607f3
N		 4460 if (s.handle_bad_blocks)
4461 for (i = disks; i--; ) {
3cb03002 4462 struct md_rdev *rdev;
bc2607f3
N		 4463 struct r5dev *dev = &sh->dev[i];
4464 if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
4465 /* We own a safe reference to the rdev */
4466 rdev = conf->disks[i].rdev;
4467 if (!rdev_set_badblocks(rdev, sh->sector,
4468 STRIPE_SECTORS, 0))
4469 md_error(conf->mddev, rdev);
4470 rdev_dec_pending(rdev, conf->mddev);
4471 }
b84db560
N		 4472 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
4473 rdev = conf->disks[i].rdev;
4474 rdev_clear_badblocks(rdev, sh->sector,
c6563a8c 4475 STRIPE_SECTORS, 0);
b84db560
N		 4476 rdev_dec_pending(rdev, conf->mddev);
4477 }
977df362
N		 4478 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
4479 rdev = conf->disks[i].replacement;
dd054fce
N		 4480 if (!rdev)
4481 /* rdev have been moved down */
4482 rdev = conf->disks[i].rdev;
977df362 4483 rdev_clear_badblocks(rdev, sh->sector,
c6563a8c 4484 STRIPE_SECTORS, 0);
977df362
N		 4485 rdev_dec_pending(rdev, conf->mddev);
4486 }
bc2607f3
N		 4487 }
4488
6c0069c0
YT		 4489 if (s.ops_request)
4490 raid_run_ops(sh, s.ops_request);
4491
f0e43bcd 4492 ops_run_io(sh, &s);
16a53ecc 4493
c5709ef6 4494 if (s.dec_preread_active) {
729a1866 4495 /* We delay this until after ops_run_io so that if make_request
e9c7469b 4496 * is waiting on a flush, it won't continue until the writes
729a1866
N		 4497 * have actually been submitted.
4498 */
4499 atomic_dec(&conf->preread_active_stripes);
4500 if (atomic_read(&conf->preread_active_stripes) <
4501 IO_THRESHOLD)
4502 md_wakeup_thread(conf->mddev->thread);
4503 }
4504
c5709ef6 4505 return_io(s.return_bi);
16a53ecc 4506
257a4b42 4507 clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
16a53ecc
N		 4508}
4509
d1688a6d 4510static void raid5_activate_delayed(struct r5conf *conf)
16a53ecc
N		 4511{
4512 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
4513 while (!list_empty(&conf->delayed_list)) {
4514 struct list_head *l = conf->delayed_list.next;
4515 struct stripe_head *sh;
4516 sh = list_entry(l, struct stripe_head, lru);
4517 list_del_init(l);
4518 clear_bit(STRIPE_DELAYED, &sh->state);
4519 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4520 atomic_inc(&conf->preread_active_stripes);
8b3e6cdc 4521 list_add_tail(&sh->lru, &conf->hold_list);
851c30c9 4522 raid5_wakeup_stripe_thread(sh);
16a53ecc 4523 }
482c0834 4524 }
16a53ecc
N		 4525}
4526
566c09c5
SL		 4527static void activate_bit_delay(struct r5conf *conf,
4528 struct list_head *temp_inactive_list)
16a53ecc
N		 4529{
4530 /* device_lock is held */
4531 struct list_head head;
4532 list_add(&head, &conf->bitmap_list);
4533 list_del_init(&conf->bitmap_list);
4534 while (!list_empty(&head)) {
4535 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
566c09c5 4536 int hash;
16a53ecc
N		 4537 list_del_init(&sh->lru);
4538 atomic_inc(&sh->count);
566c09c5
SL		 4539 hash = sh->hash_lock_index;
4540 __release_stripe(conf, sh, &temp_inactive_list[hash]);
16a53ecc
N		 4541 }
4542}
4543
5c675f83 4544static int raid5_congested(struct mddev *mddev, int bits)
f022b2fd 4545{
d1688a6d 4546 struct r5conf *conf = mddev->private;
f022b2fd
N		 4547
4548 /* No difference between reads and writes. Just check
4549 * how busy the stripe_cache is
4550 */
3fa841d7 4551
f022b2fd
N		 4552 if (conf->inactive_blocked)
4553 return 1;
4554 if (conf->quiesce)
4555 return 1;
4bda556a 4556 if (atomic_read(&conf->empty_inactive_list_nr))
f022b2fd
N		 4557 return 1;
4558
4559 return 0;
4560}
4561
23032a0e
RBJ		 4562/* We want read requests to align with chunks where possible,
4563 * but write requests don't need to.
4564 */
64590f45 4565static int raid5_mergeable_bvec(struct mddev *mddev,
cc371e66
AK		 4566 struct bvec_merge_data *bvm,
4567 struct bio_vec *biovec)
23032a0e 4568{
cc371e66 4569 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
23032a0e 4570 int max;
9d8f0363 4571 unsigned int chunk_sectors = mddev->chunk_sectors;
cc371e66 4572 unsigned int bio_sectors = bvm->bi_size >> 9;
23032a0e 4573
cc371e66 4574 if ((bvm->bi_rw & 1) == WRITE)
23032a0e
RBJ		 4575 return biovec->bv_len; /* always allow writes to be mergeable */
4576
664e7c41
AN		 4577 if (mddev->new_chunk_sectors < mddev->chunk_sectors)
4578 chunk_sectors = mddev->new_chunk_sectors;
23032a0e
RBJ		 4579 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
4580 if (max < 0) max = 0;
4581 if (max <= biovec->bv_len && bio_sectors == 0)
4582 return biovec->bv_len;
4583 else
4584 return max;
4585}
4586
fd01b88c 4587static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
f679623f 4588{
4f024f37 4589 sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev);
9d8f0363 4590 unsigned int chunk_sectors = mddev->chunk_sectors;
aa8b57aa 4591 unsigned int bio_sectors = bio_sectors(bio);
f679623f 4592
664e7c41
AN		 4593 if (mddev->new_chunk_sectors < mddev->chunk_sectors)
4594 chunk_sectors = mddev->new_chunk_sectors;
f679623f
RBJ		 4595 return chunk_sectors >=
4596 ((sector & (chunk_sectors - 1)) + bio_sectors);
4597}
4598
46031f9a
RBJ		 4599/*
4600 * add bio to the retry LIFO ( in O(1) ... we are in interrupt )
4601 * later sampled by raid5d.
4602 */
d1688a6d 4603static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
46031f9a
RBJ		 4604{
4605 unsigned long flags;
4606
4607 spin_lock_irqsave(&conf->device_lock, flags);
4608
4609 bi->bi_next = conf->retry_read_aligned_list;
4610 conf->retry_read_aligned_list = bi;
4611
4612 spin_unlock_irqrestore(&conf->device_lock, flags);
4613 md_wakeup_thread(conf->mddev->thread);
4614}
4615
d1688a6d 4616static struct bio *remove_bio_from_retry(struct r5conf *conf)
46031f9a
RBJ		 4617{
4618 struct bio *bi;
4619
4620 bi = conf->retry_read_aligned;
4621 if (bi) {
4622 conf->retry_read_aligned = NULL;
4623 return bi;
4624 }
4625 bi = conf->retry_read_aligned_list;
4626 if(bi) {
387bb173 4627 conf->retry_read_aligned_list = bi->bi_next;
46031f9a 4628 bi->bi_next = NULL;
960e739d
JA		 4629 /*
4630 * this sets the active strip count to 1 and the processed
4631 * strip count to zero (upper 8 bits)
4632 */
e7836bd6 4633 raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
46031f9a
RBJ		 4634 }
4635
4636 return bi;
4637}
4638
f679623f
RBJ		 4639/*
4640 * The "raid5_align_endio" should check if the read succeeded and if it
4641 * did, call bio_endio on the original bio (having bio_put the new bio
4642 * first).
4643 * If the read failed..
4644 */
6712ecf8 4645static void raid5_align_endio(struct bio *bi, int error)
f679623f
RBJ		 4646{
4647 struct bio* raid_bi = bi->bi_private;
fd01b88c 4648 struct mddev *mddev;
d1688a6d 4649 struct r5conf *conf;
46031f9a 4650 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
3cb03002 4651 struct md_rdev *rdev;
46031f9a 4652
f679623f 4653 bio_put(bi);
46031f9a 4654
46031f9a
RBJ		 4655 rdev = (void*)raid_bi->bi_next;
4656 raid_bi->bi_next = NULL;
2b7f2228
N		 4657 mddev = rdev->mddev;
4658 conf = mddev->private;
46031f9a
RBJ		 4659
4660 rdev_dec_pending(rdev, conf->mddev);
4661
4662 if (!error && uptodate) {
0a82a8d1
LT		 4663 trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev),
4664 raid_bi, 0);
6712ecf8 4665 bio_endio(raid_bi, 0);
46031f9a
RBJ		 4666 if (atomic_dec_and_test(&conf->active_aligned_reads))
4667 wake_up(&conf->wait_for_stripe);
6712ecf8 4668 return;
46031f9a
RBJ		 4669 }
4670
45b4233c 4671 pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
46031f9a
RBJ		 4672
4673 add_bio_to_retry(raid_bi, conf);
f679623f
RBJ		 4674}
4675
387bb173
NB		 4676static int bio_fits_rdev(struct bio *bi)
4677{
165125e1 4678 struct request_queue *q = bdev_get_queue(bi->bi_bdev);
387bb173 4679
aa8b57aa 4680 if (bio_sectors(bi) > queue_max_sectors(q))
387bb173
NB		 4681 return 0;
4682 blk_recount_segments(q, bi);
8a78362c 4683 if (bi->bi_phys_segments > queue_max_segments(q))
387bb173
NB		 4684 return 0;
4685
4686 if (q->merge_bvec_fn)
4687 /* it's too hard to apply the merge_bvec_fn at this stage,
4688 * just just give up
4689 */
4690 return 0;
4691
4692 return 1;
4693}
4694
fd01b88c 4695static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
f679623f 4696{
d1688a6d 4697 struct r5conf *conf = mddev->private;
8553fe7e 4698 int dd_idx;
f679623f 4699 struct bio* align_bi;
3cb03002 4700 struct md_rdev *rdev;
671488cc 4701 sector_t end_sector;
f679623f
RBJ		 4702
4703 if (!in_chunk_boundary(mddev, raid_bio)) {
45b4233c 4704 pr_debug("chunk_aligned_read : non aligned\n");
f679623f
RBJ		 4705 return 0;
4706 }
4707 /*
a167f663 4708 * use bio_clone_mddev to make a copy of the bio
f679623f 4709 */
a167f663 4710 align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
f679623f
RBJ		 4711 if (!align_bi)
4712 return 0;
4713 /*
4714 * set bi_end_io to a new function, and set bi_private to the
4715 * original bio.
4716 */
4717 align_bi->bi_end_io = raid5_align_endio;
4718 align_bi->bi_private = raid_bio;
4719 /*
4720 * compute position
4721 */
4f024f37
KO		 4722 align_bi->bi_iter.bi_sector =
4723 raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
4724 0, &dd_idx, NULL);
f679623f 4725
f73a1c7d 4726 end_sector = bio_end_sector(align_bi);
f679623f 4727 rcu_read_lock();
671488cc
N		 4728 rdev = rcu_dereference(conf->disks[dd_idx].replacement);
4729 if (!rdev || test_bit(Faulty, &rdev->flags) ||
4730 rdev->recovery_offset < end_sector) {
4731 rdev = rcu_dereference(conf->disks[dd_idx].rdev);
4732 if (rdev &&
4733 (test_bit(Faulty, &rdev->flags) ||
4734 !(test_bit(In_sync, &rdev->flags) ||
4735 rdev->recovery_offset >= end_sector)))
4736 rdev = NULL;
4737 }
4738 if (rdev) {
31c176ec
N		 4739 sector_t first_bad;
4740 int bad_sectors;
4741
f679623f
RBJ		 4742 atomic_inc(&rdev->nr_pending);
4743 rcu_read_unlock();
46031f9a
RBJ		 4744 raid_bio->bi_next = (void*)rdev;
4745 align_bi->bi_bdev = rdev->bdev;
3fd83717 4746 __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags);
46031f9a 4747
31c176ec 4748 if (!bio_fits_rdev(align_bi) ||
4f024f37
KO		 4749 is_badblock(rdev, align_bi->bi_iter.bi_sector,
4750 bio_sectors(align_bi),
31c176ec
N		 4751 &first_bad, &bad_sectors)) {
4752 /* too big in some way, or has a known bad block */
387bb173
NB		 4753 bio_put(align_bi);
4754 rdev_dec_pending(rdev, mddev);
4755 return 0;
4756 }
4757
6c0544e2 4758 /* No reshape active, so we can trust rdev->data_offset */
4f024f37 4759 align_bi->bi_iter.bi_sector += rdev->data_offset;
6c0544e2 4760
46031f9a
RBJ		 4761 spin_lock_irq(&conf->device_lock);
4762 wait_event_lock_irq(conf->wait_for_stripe,
4763 conf->quiesce == 0,
eed8c02e 4764 conf->device_lock);
46031f9a
RBJ		 4765 atomic_inc(&conf->active_aligned_reads);
4766 spin_unlock_irq(&conf->device_lock);
4767
e3620a3a
JB		 4768 if (mddev->gendisk)
4769 trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
4770 align_bi, disk_devt(mddev->gendisk),
4f024f37 4771 raid_bio->bi_iter.bi_sector);
f679623f
RBJ		 4772 generic_make_request(align_bi);
4773 return 1;
4774 } else {
4775 rcu_read_unlock();
46031f9a 4776 bio_put(align_bi);
f679623f
RBJ		 4777 return 0;
4778 }
4779}
4780
8b3e6cdc
DW		 4781/* __get_priority_stripe - get the next stripe to process
4782 *
4783 * Full stripe writes are allowed to pass preread active stripes up until
4784 * the bypass_threshold is exceeded. In general the bypass_count
4785 * increments when the handle_list is handled before the hold_list; however, it
4786 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
4787 * stripe with in flight i/o. The bypass_count will be reset when the
4788 * head of the hold_list has changed, i.e. the head was promoted to the
4789 * handle_list.
4790 */
851c30c9 4791static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
8b3e6cdc 4792{
851c30c9
SL		 4793 struct stripe_head *sh = NULL, *tmp;
4794 struct list_head *handle_list = NULL;
bfc90cb0 4795 struct r5worker_group *wg = NULL;
851c30c9
SL		 4796
4797 if (conf->worker_cnt_per_group == 0) {
4798 handle_list = &conf->handle_list;
4799 } else if (group != ANY_GROUP) {
4800 handle_list = &conf->worker_groups[group].handle_list;
bfc90cb0 4801 wg = &conf->worker_groups[group];
851c30c9
SL		 4802 } else {
4803 int i;
4804 for (i = 0; i < conf->group_cnt; i++) {
4805 handle_list = &conf->worker_groups[i].handle_list;
bfc90cb0 4806 wg = &conf->worker_groups[i];
851c30c9
SL		 4807 if (!list_empty(handle_list))
4808 break;
4809 }
4810 }
8b3e6cdc
DW		 4811
4812 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
4813 __func__,
851c30c9 4814 list_empty(handle_list) ? "empty" : "busy",
8b3e6cdc
DW		 4815 list_empty(&conf->hold_list) ? "empty" : "busy",
4816 atomic_read(&conf->pending_full_writes), conf->bypass_count);
4817
851c30c9
SL		 4818 if (!list_empty(handle_list)) {
4819 sh = list_entry(handle_list->next, typeof(*sh), lru);
8b3e6cdc
DW		 4820
4821 if (list_empty(&conf->hold_list))
4822 conf->bypass_count = 0;
4823 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
4824 if (conf->hold_list.next == conf->last_hold)
4825 conf->bypass_count++;
4826 else {
4827 conf->last_hold = conf->hold_list.next;
4828 conf->bypass_count -= conf->bypass_threshold;
4829 if (conf->bypass_count < 0)
4830 conf->bypass_count = 0;
4831 }
4832 }
4833 } else if (!list_empty(&conf->hold_list) &&
4834 ((conf->bypass_threshold &&
4835 conf->bypass_count > conf->bypass_threshold) ||
4836 atomic_read(&conf->pending_full_writes) == 0)) {
851c30c9
SL		 4837
4838 list_for_each_entry(tmp, &conf->hold_list, lru) {
4839 if (conf->worker_cnt_per_group == 0 ||
4840 group == ANY_GROUP ||
4841 !cpu_online(tmp->cpu) ||
4842 cpu_to_group(tmp->cpu) == group) {
4843 sh = tmp;
4844 break;
4845 }
4846 }
4847
4848 if (sh) {
4849 conf->bypass_count -= conf->bypass_threshold;
4850 if (conf->bypass_count < 0)
4851 conf->bypass_count = 0;
4852 }
bfc90cb0 4853 wg = NULL;
851c30c9
SL		 4854 }
4855
4856 if (!sh)
8b3e6cdc
DW		 4857 return NULL;
4858
bfc90cb0
SL		 4859 if (wg) {
4860 wg->stripes_cnt--;
4861 sh->group = NULL;
4862 }
8b3e6cdc 4863 list_del_init(&sh->lru);
c7a6d35e 4864 BUG_ON(atomic_inc_return(&sh->count) != 1);
8b3e6cdc
DW		 4865 return sh;
4866}
f679623f 4867
8811b596
SL		 4868struct raid5_plug_cb {
4869 struct blk_plug_cb cb;
4870 struct list_head list;
566c09c5 4871 struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
8811b596
SL		 4872};
4873
4874static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
4875{
4876 struct raid5_plug_cb *cb = container_of(
4877 blk_cb, struct raid5_plug_cb, cb);
4878 struct stripe_head *sh;
4879 struct mddev *mddev = cb->cb.data;
4880 struct r5conf *conf = mddev->private;
a9add5d9 4881 int cnt = 0;
566c09c5 4882 int hash;
8811b596
SL		 4883
4884 if (cb->list.next && !list_empty(&cb->list)) {
4885 spin_lock_irq(&conf->device_lock);
4886 while (!list_empty(&cb->list)) {
4887 sh = list_first_entry(&cb->list, struct stripe_head, lru);
4888 list_del_init(&sh->lru);
4889 /*
4890 * avoid race release_stripe_plug() sees
4891 * STRIPE_ON_UNPLUG_LIST clear but the stripe
4892 * is still in our list
4893 */
4e857c58 4894 smp_mb__before_atomic();
8811b596 4895 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
773ca82f
SL		 4896 /*
4897 * STRIPE_ON_RELEASE_LIST could be set here. In that
4898 * case, the count is always > 1 here
4899 */
566c09c5
SL		 4900 hash = sh->hash_lock_index;
4901 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
a9add5d9 4902 cnt++;
8811b596
SL		 4903 }
4904 spin_unlock_irq(&conf->device_lock);
4905 }
566c09c5
SL		 4906 release_inactive_stripe_list(conf, cb->temp_inactive_list,
4907 NR_STRIPE_HASH_LOCKS);
e3620a3a
JB		 4908 if (mddev->queue)
4909 trace_block_unplug(mddev->queue, cnt, !from_schedule);
8811b596
SL		 4910 kfree(cb);
4911}
4912
4913static void release_stripe_plug(struct mddev *mddev,
4914 struct stripe_head *sh)
4915{
4916 struct blk_plug_cb *blk_cb = blk_check_plugged(
4917 raid5_unplug, mddev,
4918 sizeof(struct raid5_plug_cb));
4919 struct raid5_plug_cb *cb;
4920
4921 if (!blk_cb) {
4922 release_stripe(sh);
4923 return;
4924 }
4925
4926 cb = container_of(blk_cb, struct raid5_plug_cb, cb);
4927
566c09c5
SL		 4928 if (cb->list.next == NULL) {
4929 int i;
8811b596 4930 INIT_LIST_HEAD(&cb->list);
566c09c5
SL		 4931 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
4932 INIT_LIST_HEAD(cb->temp_inactive_list + i);
4933 }
8811b596
SL		 4934
4935 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
4936 list_add_tail(&sh->lru, &cb->list);
4937 else
4938 release_stripe(sh);
4939}
4940
620125f2
SL		 4941static void make_discard_request(struct mddev *mddev, struct bio *bi)
4942{
4943 struct r5conf *conf = mddev->private;
4944 sector_t logical_sector, last_sector;
4945 struct stripe_head *sh;
4946 int remaining;
4947 int stripe_sectors;
4948
4949 if (mddev->reshape_position != MaxSector)
4950 /* Skip discard while reshape is happening */
4951 return;
4952
4f024f37
KO		 4953 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4954 last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
620125f2
SL		 4955
4956 bi->bi_next = NULL;
4957 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
4958
4959 stripe_sectors = conf->chunk_sectors *
4960 (conf->raid_disks - conf->max_degraded);
4961 logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
4962 stripe_sectors);
4963 sector_div(last_sector, stripe_sectors);
4964
4965 logical_sector *= conf->chunk_sectors;
4966 last_sector *= conf->chunk_sectors;
4967
4968 for (; logical_sector < last_sector;
4969 logical_sector += STRIPE_SECTORS) {
4970 DEFINE_WAIT(w);
4971 int d;
4972 again:
4973 sh = get_active_stripe(conf, logical_sector, 0, 0, 0);
4974 prepare_to_wait(&conf->wait_for_overlap, &w,
4975 TASK_UNINTERRUPTIBLE);
f8dfcffd
N		 4976 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
4977 if (test_bit(STRIPE_SYNCING, &sh->state)) {
4978 release_stripe(sh);
4979 schedule();
4980 goto again;
4981 }
4982 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
620125f2
SL		 4983 spin_lock_irq(&sh->stripe_lock);
4984 for (d = 0; d < conf->raid_disks; d++) {
4985 if (d == sh->pd_idx || d == sh->qd_idx)
4986 continue;
4987 if (sh->dev[d].towrite || sh->dev[d].toread) {
4988 set_bit(R5_Overlap, &sh->dev[d].flags);
4989 spin_unlock_irq(&sh->stripe_lock);
4990 release_stripe(sh);
4991 schedule();
4992 goto again;
4993 }
4994 }
f8dfcffd 4995 set_bit(STRIPE_DISCARD, &sh->state);
620125f2 4996 finish_wait(&conf->wait_for_overlap, &w);
7a87f434 4997 sh->overwrite_disks = 0;
620125f2
SL		 4998 for (d = 0; d < conf->raid_disks; d++) {
4999 if (d == sh->pd_idx || d == sh->qd_idx)
5000 continue;
5001 sh->dev[d].towrite = bi;
5002 set_bit(R5_OVERWRITE, &sh->dev[d].flags);
5003 raid5_inc_bi_active_stripes(bi);
7a87f434 5004 sh->overwrite_disks++;
620125f2
SL		 5005 }
5006 spin_unlock_irq(&sh->stripe_lock);
5007 if (conf->mddev->bitmap) {
5008 for (d = 0;
5009 d < conf->raid_disks - conf->max_degraded;
5010 d++)
5011 bitmap_startwrite(mddev->bitmap,
5012 sh->sector,
5013 STRIPE_SECTORS,
5014 0);
5015 sh->bm_seq = conf->seq_flush + 1;
5016 set_bit(STRIPE_BIT_DELAY, &sh->state);
5017 }
5018
5019 set_bit(STRIPE_HANDLE, &sh->state);
5020 clear_bit(STRIPE_DELAYED, &sh->state);
5021 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5022 atomic_inc(&conf->preread_active_stripes);
5023 release_stripe_plug(mddev, sh);
5024 }
5025
5026 remaining = raid5_dec_bi_active_stripes(bi);
5027 if (remaining == 0) {
5028 md_write_end(mddev);
5029 bio_endio(bi, 0);
5030 }
5031}
5032
b4fdcb02 5033static void make_request(struct mddev *mddev, struct bio * bi)
1da177e4 5034{
d1688a6d 5035 struct r5conf *conf = mddev->private;
911d4ee8 5036 int dd_idx;
1da177e4
LT		 5037 sector_t new_sector;
5038 sector_t logical_sector, last_sector;
5039 struct stripe_head *sh;
a362357b 5040 const int rw = bio_data_dir(bi);
49077326 5041 int remaining;
27c0f68f
SL		 5042 DEFINE_WAIT(w);
5043 bool do_prepare;
1da177e4 5044
e9c7469b
TH		 5045 if (unlikely(bi->bi_rw & REQ_FLUSH)) {
5046 md_flush_request(mddev, bi);
5a7bbad2 5047 return;
e5dcdd80
N		 5048 }
5049
3d310eb7 5050 md_write_start(mddev, bi);
06d91a5f 5051
802ba064 5052 if (rw == READ &&
52488615 5053 mddev->reshape_position == MaxSector &&
21a52c6d 5054 chunk_aligned_read(mddev,bi))
5a7bbad2 5055 return;
52488615 5056
620125f2
SL		 5057 if (unlikely(bi->bi_rw & REQ_DISCARD)) {
5058 make_discard_request(mddev, bi);
5059 return;
5060 }
5061
4f024f37 5062 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
f73a1c7d 5063 last_sector = bio_end_sector(bi);
1da177e4
LT		 5064 bi->bi_next = NULL;
5065 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
06d91a5f 5066
27c0f68f 5067 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
1da177e4 5068 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
b5663ba4 5069 int previous;
c46501b2 5070 int seq;
b578d55f 5071
27c0f68f 5072 do_prepare = false;
7ecaa1e6 5073 retry:
c46501b2 5074 seq = read_seqcount_begin(&conf->gen_lock);
b5663ba4 5075 previous = 0;
27c0f68f
SL		 5076 if (do_prepare)
5077 prepare_to_wait(&conf->wait_for_overlap, &w,
5078 TASK_UNINTERRUPTIBLE);
b0f9ec04 5079 if (unlikely(conf->reshape_progress != MaxSector)) {
fef9c61f 5080 /* spinlock is needed as reshape_progress may be
df8e7f76
N		 5081 * 64bit on a 32bit platform, and so it might be
5082 * possible to see a half-updated value
aeb878b0 5083 * Of course reshape_progress could change after
df8e7f76
N		 5084 * the lock is dropped, so once we get a reference
5085 * to the stripe that we think it is, we will have
5086 * to check again.
5087 */
7ecaa1e6 5088 spin_lock_irq(&conf->device_lock);
2c810cdd 5089 if (mddev->reshape_backwards
fef9c61f
N		 5090 ? logical_sector < conf->reshape_progress
5091 : logical_sector >= conf->reshape_progress) {
b5663ba4
N		 5092 previous = 1;
5093 } else {
2c810cdd 5094 if (mddev->reshape_backwards
fef9c61f
N		 5095 ? logical_sector < conf->reshape_safe
5096 : logical_sector >= conf->reshape_safe) {
b578d55f
N		 5097 spin_unlock_irq(&conf->device_lock);
5098 schedule();
27c0f68f 5099 do_prepare = true;
b578d55f
N		 5100 goto retry;
5101 }
5102 }
7ecaa1e6
N		 5103 spin_unlock_irq(&conf->device_lock);
5104 }
16a53ecc 5105
112bf897
N		 5106 new_sector = raid5_compute_sector(conf, logical_sector,
5107 previous,
911d4ee8 5108 &dd_idx, NULL);
0c55e022 5109 pr_debug("raid456: make_request, sector %llu logical %llu\n",
c46501b2 5110 (unsigned long long)new_sector,
1da177e4
LT		 5111 (unsigned long long)logical_sector);
5112
b5663ba4 5113 sh = get_active_stripe(conf, new_sector, previous,
a8c906ca 5114 (bi->bi_rw&RWA_MASK), 0);
1da177e4 5115 if (sh) {
b0f9ec04 5116 if (unlikely(previous)) {
7ecaa1e6 5117 /* expansion might have moved on while waiting for a
df8e7f76
N		 5118 * stripe, so we must do the range check again.
5119 * Expansion could still move past after this
5120 * test, but as we are holding a reference to
5121 * 'sh', we know that if that happens,
5122 * STRIPE_EXPANDING will get set and the expansion
5123 * won't proceed until we finish with the stripe.
7ecaa1e6
N		 5124 */
5125 int must_retry = 0;
5126 spin_lock_irq(&conf->device_lock);
2c810cdd 5127 if (mddev->reshape_backwards
b0f9ec04
N		 5128 ? logical_sector >= conf->reshape_progress
5129 : logical_sector < conf->reshape_progress)
7ecaa1e6
N		 5130 /* mismatch, need to try again */
5131 must_retry = 1;
5132 spin_unlock_irq(&conf->device_lock);
5133 if (must_retry) {
5134 release_stripe(sh);
7a3ab908 5135 schedule();
27c0f68f 5136 do_prepare = true;
7ecaa1e6
N		 5137 goto retry;
5138 }
5139 }
c46501b2
N		 5140 if (read_seqcount_retry(&conf->gen_lock, seq)) {
5141 /* Might have got the wrong stripe_head
5142 * by accident
5143 */
5144 release_stripe(sh);
5145 goto retry;
5146 }
e62e58a5 5147
ffd96e35 5148 if (rw == WRITE &&
a5c308d4 5149 logical_sector >= mddev->suspend_lo &&
e464eafd
N		 5150 logical_sector < mddev->suspend_hi) {
5151 release_stripe(sh);
e62e58a5
N		 5152 /* As the suspend_* range is controlled by
5153 * userspace, we want an interruptible
5154 * wait.
5155 */
5156 flush_signals(current);
5157 prepare_to_wait(&conf->wait_for_overlap,
5158 &w, TASK_INTERRUPTIBLE);
5159 if (logical_sector >= mddev->suspend_lo &&
27c0f68f 5160 logical_sector < mddev->suspend_hi) {
e62e58a5 5161 schedule();
27c0f68f
SL		 5162 do_prepare = true;
5163 }
e464eafd
N		 5164 goto retry;
5165 }
7ecaa1e6
N		 5166
5167 if (test_bit(STRIPE_EXPANDING, &sh->state) ||
da41ba65 5168 !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {
7ecaa1e6
N		 5169 /* Stripe is busy expanding or
5170 * add failed due to overlap. Flush everything
1da177e4
LT		 5171 * and wait a while
5172 */
482c0834 5173 md_wakeup_thread(mddev->thread);
1da177e4
LT		 5174 release_stripe(sh);
5175 schedule();
27c0f68f 5176 do_prepare = true;
1da177e4
LT		 5177 goto retry;
5178 }
6ed3003c
N		 5179 set_bit(STRIPE_HANDLE, &sh->state);
5180 clear_bit(STRIPE_DELAYED, &sh->state);
59fc630b 5181 if ((!sh->batch_head || sh == sh->batch_head) &&
5182 (bi->bi_rw & REQ_SYNC) &&
729a1866
N		 5183 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5184 atomic_inc(&conf->preread_active_stripes);
8811b596 5185 release_stripe_plug(mddev, sh);
1da177e4
LT		 5186 } else {
5187 /* cannot get stripe for read-ahead, just give-up */
5188 clear_bit(BIO_UPTODATE, &bi->bi_flags);
1da177e4
LT		 5189 break;
5190 }
1da177e4 5191 }
27c0f68f 5192 finish_wait(&conf->wait_for_overlap, &w);
7c13edc8 5193
e7836bd6 5194 remaining = raid5_dec_bi_active_stripes(bi);
f6344757 5195 if (remaining == 0) {
1da177e4 5196
16a53ecc 5197 if ( rw == WRITE )
1da177e4 5198 md_write_end(mddev);
6712ecf8 5199
0a82a8d1
LT		 5200 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
5201 bi, 0);
0e13fe23 5202 bio_endio(bi, 0);
1da177e4 5203 }
1da177e4
LT		 5204}
5205
fd01b88c 5206static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
b522adcd 5207
fd01b88c 5208static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
1da177e4 5209{
52c03291
N		 5210 /* reshaping is quite different to recovery/resync so it is
5211 * handled quite separately ... here.
5212 *
5213 * On each call to sync_request, we gather one chunk worth of
5214 * destination stripes and flag them as expanding.
5215 * Then we find all the source stripes and request reads.
5216 * As the reads complete, handle_stripe will copy the data
5217 * into the destination stripe and release that stripe.
5218 */
d1688a6d 5219 struct r5conf *conf = mddev->private;
1da177e4 5220 struct stripe_head *sh;
ccfcc3c1 5221 sector_t first_sector, last_sector;
f416885e
N		 5222 int raid_disks = conf->previous_raid_disks;
5223 int data_disks = raid_disks - conf->max_degraded;
5224 int new_data_disks = conf->raid_disks - conf->max_degraded;
52c03291
N		 5225 int i;
5226 int dd_idx;
c8f517c4 5227 sector_t writepos, readpos, safepos;
ec32a2bd 5228 sector_t stripe_addr;
7a661381 5229 int reshape_sectors;
ab69ae12 5230 struct list_head stripes;
52c03291 5231
fef9c61f
N		 5232 if (sector_nr == 0) {
5233 /* If restarting in the middle, skip the initial sectors */
2c810cdd 5234 if (mddev->reshape_backwards &&
fef9c61f
N		 5235 conf->reshape_progress < raid5_size(mddev, 0, 0)) {
5236 sector_nr = raid5_size(mddev, 0, 0)
5237 - conf->reshape_progress;
2c810cdd 5238 } else if (!mddev->reshape_backwards &&
fef9c61f
N		 5239 conf->reshape_progress > 0)
5240 sector_nr = conf->reshape_progress;
f416885e 5241 sector_div(sector_nr, new_data_disks);
fef9c61f 5242 if (sector_nr) {
8dee7211
N		 5243 mddev->curr_resync_completed = sector_nr;
5244 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
fef9c61f
N		 5245 *skipped = 1;
5246 return sector_nr;
5247 }
52c03291
N		 5248 }
5249
7a661381
N		 5250 /* We need to process a full chunk at a time.
5251 * If old and new chunk sizes differ, we need to process the
5252 * largest of these
5253 */
664e7c41
AN		 5254 if (mddev->new_chunk_sectors > mddev->chunk_sectors)
5255 reshape_sectors = mddev->new_chunk_sectors;
7a661381 5256 else
9d8f0363 5257 reshape_sectors = mddev->chunk_sectors;
7a661381 5258
b5254dd5
N		 5259 /* We update the metadata at least every 10 seconds, or when
5260 * the data about to be copied would over-write the source of
5261 * the data at the front of the range. i.e. one new_stripe
5262 * along from reshape_progress new_maps to after where
5263 * reshape_safe old_maps to
52c03291 5264 */
fef9c61f 5265 writepos = conf->reshape_progress;
f416885e 5266 sector_div(writepos, new_data_disks);
c8f517c4
N		 5267 readpos = conf->reshape_progress;
5268 sector_div(readpos, data_disks);
fef9c61f 5269 safepos = conf->reshape_safe;
f416885e 5270 sector_div(safepos, data_disks);
2c810cdd 5271 if (mddev->reshape_backwards) {
ed37d83e 5272 writepos -= min_t(sector_t, reshape_sectors, writepos);
c8f517c4 5273 readpos += reshape_sectors;
7a661381 5274 safepos += reshape_sectors;
fef9c61f 5275 } else {
7a661381 5276 writepos += reshape_sectors;
ed37d83e
N		 5277 readpos -= min_t(sector_t, reshape_sectors, readpos);
5278 safepos -= min_t(sector_t, reshape_sectors, safepos);
fef9c61f 5279 }
52c03291 5280
b5254dd5
N		 5281 /* Having calculated the 'writepos' possibly use it
5282 * to set 'stripe_addr' which is where we will write to.
5283 */
5284 if (mddev->reshape_backwards) {
5285 BUG_ON(conf->reshape_progress == 0);
5286 stripe_addr = writepos;
5287 BUG_ON((mddev->dev_sectors &
5288 ~((sector_t)reshape_sectors - 1))
5289 - reshape_sectors - stripe_addr
5290 != sector_nr);
5291 } else {
5292 BUG_ON(writepos != sector_nr + reshape_sectors);
5293 stripe_addr = sector_nr;
5294 }
5295
c8f517c4
N		 5296 /* 'writepos' is the most advanced device address we might write.
5297 * 'readpos' is the least advanced device address we might read.
5298 * 'safepos' is the least address recorded in the metadata as having
5299 * been reshaped.
b5254dd5
N		 5300 * If there is a min_offset_diff, these are adjusted either by
5301 * increasing the safepos/readpos if diff is negative, or
5302 * increasing writepos if diff is positive.
5303 * If 'readpos' is then behind 'writepos', there is no way that we can
c8f517c4
N		 5304 * ensure safety in the face of a crash - that must be done by userspace
5305 * making a backup of the data. So in that case there is no particular
5306 * rush to update metadata.
5307 * Otherwise if 'safepos' is behind 'writepos', then we really need to
5308 * update the metadata to advance 'safepos' to match 'readpos' so that
5309 * we can be safe in the event of a crash.
5310 * So we insist on updating metadata if safepos is behind writepos and
5311 * readpos is beyond writepos.
5312 * In any case, update the metadata every 10 seconds.
5313 * Maybe that number should be configurable, but I'm not sure it is
5314 * worth it.... maybe it could be a multiple of safemode_delay???
5315 */
b5254dd5
N		 5316 if (conf->min_offset_diff < 0) {
5317 safepos += -conf->min_offset_diff;
5318 readpos += -conf->min_offset_diff;
5319 } else
5320 writepos += conf->min_offset_diff;
5321
2c810cdd 5322 if ((mddev->reshape_backwards
c8f517c4
N		 5323 ? (safepos > writepos && readpos < writepos)
5324 : (safepos < writepos && readpos > writepos)) ||
5325 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
52c03291
N		 5326 /* Cannot proceed until we've updated the superblock... */
5327 wait_event(conf->wait_for_overlap,
c91abf5a
N		 5328 atomic_read(&conf->reshape_stripes)==0
5329 || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5330 if (atomic_read(&conf->reshape_stripes) != 0)
5331 return 0;
fef9c61f 5332 mddev->reshape_position = conf->reshape_progress;
75d3da43 5333 mddev->curr_resync_completed = sector_nr;
c8f517c4 5334 conf->reshape_checkpoint = jiffies;
850b2b42 5335 set_bit(MD_CHANGE_DEVS, &mddev->flags);
52c03291 5336 md_wakeup_thread(mddev->thread);
850b2b42 5337 wait_event(mddev->sb_wait, mddev->flags == 0 ||
c91abf5a
N		 5338 test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5339 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5340 return 0;
52c03291 5341 spin_lock_irq(&conf->device_lock);
fef9c61f 5342 conf->reshape_safe = mddev->reshape_position;
52c03291
N		 5343 spin_unlock_irq(&conf->device_lock);
5344 wake_up(&conf->wait_for_overlap);
acb180b0 5345 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
52c03291
N		 5346 }
5347
ab69ae12 5348 INIT_LIST_HEAD(&stripes);
7a661381 5349 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
52c03291 5350 int j;
a9f326eb 5351 int skipped_disk = 0;
a8c906ca 5352 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
52c03291
N		 5353 set_bit(STRIPE_EXPANDING, &sh->state);
5354 atomic_inc(&conf->reshape_stripes);
5355 /* If any of this stripe is beyond the end of the old
5356 * array, then we need to zero those blocks
5357 */
5358 for (j=sh->disks; j--;) {
5359 sector_t s;
5360 if (j == sh->pd_idx)
5361 continue;
f416885e 5362 if (conf->level == 6 &&
d0dabf7e 5363 j == sh->qd_idx)
f416885e 5364 continue;
784052ec 5365 s = compute_blocknr(sh, j, 0);
b522adcd 5366 if (s < raid5_size(mddev, 0, 0)) {
a9f326eb 5367 skipped_disk = 1;
52c03291
N		 5368 continue;
5369 }
5370 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
5371 set_bit(R5_Expanded, &sh->dev[j].flags);
5372 set_bit(R5_UPTODATE, &sh->dev[j].flags);
5373 }
a9f326eb 5374 if (!skipped_disk) {
52c03291
N		 5375 set_bit(STRIPE_EXPAND_READY, &sh->state);
5376 set_bit(STRIPE_HANDLE, &sh->state);
5377 }
ab69ae12 5378 list_add(&sh->lru, &stripes);
52c03291
N		 5379 }
5380 spin_lock_irq(&conf->device_lock);
2c810cdd 5381 if (mddev->reshape_backwards)
7a661381 5382 conf->reshape_progress -= reshape_sectors * new_data_disks;
fef9c61f 5383 else
7a661381 5384 conf->reshape_progress += reshape_sectors * new_data_disks;
52c03291
N		 5385 spin_unlock_irq(&conf->device_lock);
5386 /* Ok, those stripe are ready. We can start scheduling
5387 * reads on the source stripes.
5388 * The source stripes are determined by mapping the first and last
5389 * block on the destination stripes.
5390 */
52c03291 5391 first_sector =
ec32a2bd 5392 raid5_compute_sector(conf, stripe_addr*(new_data_disks),
911d4ee8 5393 1, &dd_idx, NULL);
52c03291 5394 last_sector =
0e6e0271 5395 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
09c9e5fa 5396 * new_data_disks - 1),
911d4ee8 5397 1, &dd_idx, NULL);
58c0fed4
AN		 5398 if (last_sector >= mddev->dev_sectors)
5399 last_sector = mddev->dev_sectors - 1;
52c03291 5400 while (first_sector <= last_sector) {
a8c906ca 5401 sh = get_active_stripe(conf, first_sector, 1, 0, 1);
52c03291
N		 5402 set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
5403 set_bit(STRIPE_HANDLE, &sh->state);
5404 release_stripe(sh);
5405 first_sector += STRIPE_SECTORS;
5406 }
ab69ae12
N		 5407 /* Now that the sources are clearly marked, we can release
5408 * the destination stripes
5409 */
5410 while (!list_empty(&stripes)) {
5411 sh = list_entry(stripes.next, struct stripe_head, lru);
5412 list_del_init(&sh->lru);
5413 release_stripe(sh);
5414 }
c6207277
N		 5415 /* If this takes us to the resync_max point where we have to pause,
5416 * then we need to write out the superblock.
5417 */
7a661381 5418 sector_nr += reshape_sectors;
c03f6a19
N		 5419 if ((sector_nr - mddev->curr_resync_completed) * 2
5420 >= mddev->resync_max - mddev->curr_resync_completed) {
c6207277
N		 5421 /* Cannot proceed until we've updated the superblock... */
5422 wait_event(conf->wait_for_overlap,
c91abf5a
N		 5423 atomic_read(&conf->reshape_stripes) == 0
5424 || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5425 if (atomic_read(&conf->reshape_stripes) != 0)
5426 goto ret;
fef9c61f 5427 mddev->reshape_position = conf->reshape_progress;
75d3da43 5428 mddev->curr_resync_completed = sector_nr;
c8f517c4 5429 conf->reshape_checkpoint = jiffies;
c6207277
N		 5430 set_bit(MD_CHANGE_DEVS, &mddev->flags);
5431 md_wakeup_thread(mddev->thread);
5432 wait_event(mddev->sb_wait,
5433 !test_bit(MD_CHANGE_DEVS, &mddev->flags)
c91abf5a
N		 5434 || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
5435 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
5436 goto ret;
c6207277 5437 spin_lock_irq(&conf->device_lock);
fef9c61f 5438 conf->reshape_safe = mddev->reshape_position;
c6207277
N		 5439 spin_unlock_irq(&conf->device_lock);
5440 wake_up(&conf->wait_for_overlap);
acb180b0 5441 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
c6207277 5442 }
c91abf5a 5443ret:
7a661381 5444 return reshape_sectors;
52c03291
N		 5445}
5446
09314799 5447static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
52c03291 5448{
d1688a6d 5449 struct r5conf *conf = mddev->private;
52c03291 5450 struct stripe_head *sh;
58c0fed4 5451 sector_t max_sector = mddev->dev_sectors;
57dab0bd 5452 sector_t sync_blocks;
16a53ecc
N		 5453 int still_degraded = 0;
5454 int i;
1da177e4 5455
72626685 5456 if (sector_nr >= max_sector) {
1da177e4 5457 /* just being told to finish up .. nothing much to do */
cea9c228 5458
29269553
N		 5459 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
5460 end_reshape(conf);
5461 return 0;
5462 }
72626685
N		 5463
5464 if (mddev->curr_resync < max_sector) /* aborted */
5465 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
5466 &sync_blocks, 1);
16a53ecc 5467 else /* completed sync */
72626685
N		 5468 conf->fullsync = 0;
5469 bitmap_close_sync(mddev->bitmap);
5470
1da177e4
LT		 5471 return 0;
5472 }
ccfcc3c1 5473
64bd660b
N		 5474 /* Allow raid5_quiesce to complete */
5475 wait_event(conf->wait_for_overlap, conf->quiesce != 2);
5476
52c03291
N		 5477 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
5478 return reshape_request(mddev, sector_nr, skipped);
f6705578 5479
c6207277
N		 5480 /* No need to check resync_max as we never do more than one
5481 * stripe, and as resync_max will always be on a chunk boundary,
5482 * if the check in md_do_sync didn't fire, there is no chance
5483 * of overstepping resync_max here
5484 */
5485
16a53ecc 5486 /* if there is too many failed drives and we are trying
1da177e4
LT		 5487 * to resync, then assert that we are finished, because there is
5488 * nothing we can do.
5489 */
3285edf1 5490 if (mddev->degraded >= conf->max_degraded &&
16a53ecc 5491 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
58c0fed4 5492 sector_t rv = mddev->dev_sectors - sector_nr;
57afd89f 5493 *skipped = 1;
1da177e4
LT		 5494 return rv;
5495 }
6f608040 5496 if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
5497 !conf->fullsync &&
5498 !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
5499 sync_blocks >= STRIPE_SECTORS) {
72626685
N		 5500 /* we can skip this block, and probably more */
5501 sync_blocks /= STRIPE_SECTORS;
5502 *skipped = 1;
5503 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
5504 }
1da177e4 5505
b47490c9
N		 5506 bitmap_cond_end_sync(mddev->bitmap, sector_nr);
5507
a8c906ca 5508 sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
1da177e4 5509 if (sh == NULL) {
a8c906ca 5510 sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
1da177e4 5511 /* make sure we don't swamp the stripe cache if someone else
16a53ecc 5512 * is trying to get access
1da177e4 5513 */
66c006a5 5514 schedule_timeout_uninterruptible(1);
1da177e4 5515 }
16a53ecc 5516 /* Need to check if array will still be degraded after recovery/resync
16d9cfab
EM		 5517 * Note in case of > 1 drive failures it's possible we're rebuilding
5518 * one drive while leaving another faulty drive in array.
16a53ecc 5519 */
16d9cfab
EM		 5520 rcu_read_lock();
5521 for (i = 0; i < conf->raid_disks; i++) {
5522 struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev);
5523
5524 if (rdev == NULL || test_bit(Faulty, &rdev->flags))
16a53ecc 5525 still_degraded = 1;
16d9cfab
EM		 5526 }
5527 rcu_read_unlock();
16a53ecc
N		 5528
5529 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
5530
83206d66 5531 set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
053f5b65 5532 set_bit(STRIPE_HANDLE, &sh->state);
1da177e4 5533
1da177e4
LT		 5534 release_stripe(sh);
5535
5536 return STRIPE_SECTORS;
5537}
5538
d1688a6d 5539static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
46031f9a
RBJ		 5540{
5541 /* We may not be able to submit a whole bio at once as there
5542 * may not be enough stripe_heads available.
5543 * We cannot pre-allocate enough stripe_heads as we may need
5544 * more than exist in the cache (if we allow ever large chunks).
5545 * So we do one stripe head at a time and record in
5546 * ->bi_hw_segments how many have been done.
5547 *
5548 * We *know* that this entire raid_bio is in one chunk, so
5549 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
5550 */
5551 struct stripe_head *sh;
911d4ee8 5552 int dd_idx;
46031f9a
RBJ		 5553 sector_t sector, logical_sector, last_sector;
5554 int scnt = 0;
5555 int remaining;
5556 int handled = 0;
5557
4f024f37
KO		 5558 logical_sector = raid_bio->bi_iter.bi_sector &
5559 ~((sector_t)STRIPE_SECTORS-1);
112bf897 5560 sector = raid5_compute_sector(conf, logical_sector,
911d4ee8 5561 0, &dd_idx, NULL);
f73a1c7d 5562 last_sector = bio_end_sector(raid_bio);
46031f9a
RBJ		 5563
5564 for (; logical_sector < last_sector;
387bb173
NB		 5565 logical_sector += STRIPE_SECTORS,
5566 sector += STRIPE_SECTORS,
5567 scnt++) {
46031f9a 5568
e7836bd6 5569 if (scnt < raid5_bi_processed_stripes(raid_bio))
46031f9a
RBJ		 5570 /* already done this stripe */
5571 continue;
5572
2844dc32 5573 sh = get_active_stripe(conf, sector, 0, 1, 1);
46031f9a
RBJ		 5574
5575 if (!sh) {
5576 /* failed to get a stripe - must wait */
e7836bd6 5577 raid5_set_bi_processed_stripes(raid_bio, scnt);
46031f9a
RBJ		 5578 conf->retry_read_aligned = raid_bio;
5579 return handled;
5580 }
5581
da41ba65 5582 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
387bb173 5583 release_stripe(sh);
e7836bd6 5584 raid5_set_bi_processed_stripes(raid_bio, scnt);
387bb173
NB		 5585 conf->retry_read_aligned = raid_bio;
5586 return handled;
5587 }
5588
3f9e7c14 5589 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
36d1c647 5590 handle_stripe(sh);
46031f9a
RBJ		 5591 release_stripe(sh);
5592 handled++;
5593 }
e7836bd6 5594 remaining = raid5_dec_bi_active_stripes(raid_bio);
0a82a8d1
LT		 5595 if (remaining == 0) {
5596 trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev),
5597 raid_bio, 0);
0e13fe23 5598 bio_endio(raid_bio, 0);
0a82a8d1 5599 }
46031f9a
RBJ		 5600 if (atomic_dec_and_test(&conf->active_aligned_reads))
5601 wake_up(&conf->wait_for_stripe);
5602 return handled;
5603}
5604
bfc90cb0 5605static int handle_active_stripes(struct r5conf *conf, int group,
566c09c5
SL		 5606 struct r5worker *worker,
5607 struct list_head *temp_inactive_list)
46a06401
SL		 5608{
5609 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
566c09c5
SL		 5610 int i, batch_size = 0, hash;
5611 bool release_inactive = false;
46a06401
SL		 5612
5613 while (batch_size < MAX_STRIPE_BATCH &&
851c30c9 5614 (sh = __get_priority_stripe(conf, group)) != NULL)
46a06401
SL		 5615 batch[batch_size++] = sh;
5616
566c09c5
SL		 5617 if (batch_size == 0) {
5618 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
5619 if (!list_empty(temp_inactive_list + i))
5620 break;
5621 if (i == NR_STRIPE_HASH_LOCKS)
5622 return batch_size;
5623 release_inactive = true;
5624 }
46a06401
SL		 5625 spin_unlock_irq(&conf->device_lock);
5626
566c09c5
SL		 5627 release_inactive_stripe_list(conf, temp_inactive_list,
5628 NR_STRIPE_HASH_LOCKS);
5629
5630 if (release_inactive) {
5631 spin_lock_irq(&conf->device_lock);
5632 return 0;
5633 }
5634
46a06401
SL		 5635 for (i = 0; i < batch_size; i++)
5636 handle_stripe(batch[i]);
5637
5638 cond_resched();
5639
5640 spin_lock_irq(&conf->device_lock);
566c09c5
SL		 5641 for (i = 0; i < batch_size; i++) {
5642 hash = batch[i]->hash_lock_index;
5643 __release_stripe(conf, batch[i], &temp_inactive_list[hash]);
5644 }
46a06401
SL		 5645 return batch_size;
5646}
46031f9a 5647
851c30c9
SL		 5648static void raid5_do_work(struct work_struct *work)
5649{
5650 struct r5worker *worker = container_of(work, struct r5worker, work);
5651 struct r5worker_group *group = worker->group;
5652 struct r5conf *conf = group->conf;
5653 int group_id = group - conf->worker_groups;
5654 int handled;
5655 struct blk_plug plug;
5656
5657 pr_debug("+++ raid5worker active\n");
5658
5659 blk_start_plug(&plug);
5660 handled = 0;
5661 spin_lock_irq(&conf->device_lock);
5662 while (1) {
5663 int batch_size, released;
5664
566c09c5 5665 released = release_stripe_list(conf, worker->temp_inactive_list);
851c30c9 5666
566c09c5
SL		 5667 batch_size = handle_active_stripes(conf, group_id, worker,
5668 worker->temp_inactive_list);
bfc90cb0 5669 worker->working = false;
851c30c9
SL		 5670 if (!batch_size && !released)
5671 break;
5672 handled += batch_size;
5673 }
5674 pr_debug("%d stripes handled\n", handled);
5675
5676 spin_unlock_irq(&conf->device_lock);
5677 blk_finish_plug(&plug);
5678
5679 pr_debug("--- raid5worker inactive\n");
5680}
5681
1da177e4
LT		 5682/*
5683 * This is our raid5 kernel thread.
5684 *
5685 * We scan the hash table for stripes which can be handled now.
5686 * During the scan, completed stripes are saved for us by the interrupt
5687 * handler, so that they will not have to wait for our next wakeup.
5688 */
4ed8731d 5689static void raid5d(struct md_thread *thread)
1da177e4 5690{
4ed8731d 5691 struct mddev *mddev = thread->mddev;
d1688a6d 5692 struct r5conf *conf = mddev->private;
1da177e4 5693 int handled;
e1dfa0a2 5694 struct blk_plug plug;
1da177e4 5695
45b4233c 5696 pr_debug("+++ raid5d active\n");
1da177e4
LT		 5697
5698 md_check_recovery(mddev);
1da177e4 5699
e1dfa0a2 5700 blk_start_plug(&plug);
1da177e4
LT		 5701 handled = 0;
5702 spin_lock_irq(&conf->device_lock);
5703 while (1) {
46031f9a 5704 struct bio *bio;
773ca82f
SL		 5705 int batch_size, released;
5706
566c09c5 5707 released = release_stripe_list(conf, conf->temp_inactive_list);
1da177e4 5708
0021b7bc 5709 if (
7c13edc8
N		 5710 !list_empty(&conf->bitmap_list)) {
5711 /* Now is a good time to flush some bitmap updates */
5712 conf->seq_flush++;
700e432d 5713 spin_unlock_irq(&conf->device_lock);
72626685 5714 bitmap_unplug(mddev->bitmap);
700e432d 5715 spin_lock_irq(&conf->device_lock);
7c13edc8 5716 conf->seq_write = conf->seq_flush;
566c09c5 5717 activate_bit_delay(conf, conf->temp_inactive_list);
72626685 5718 }
0021b7bc 5719 raid5_activate_delayed(conf);
72626685 5720
46031f9a
RBJ		 5721 while ((bio = remove_bio_from_retry(conf))) {
5722 int ok;
5723 spin_unlock_irq(&conf->device_lock);
5724 ok = retry_aligned_read(conf, bio);
5725 spin_lock_irq(&conf->device_lock);
5726 if (!ok)
5727 break;
5728 handled++;
5729 }
5730
566c09c5
SL		 5731 batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
5732 conf->temp_inactive_list);
773ca82f 5733 if (!batch_size && !released)
1da177e4 5734 break;
46a06401 5735 handled += batch_size;
1da177e4 5736
46a06401
SL		 5737 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
5738 spin_unlock_irq(&conf->device_lock);
de393cde 5739 md_check_recovery(mddev);
46a06401
SL		 5740 spin_lock_irq(&conf->device_lock);
5741 }
1da177e4 5742 }
45b4233c 5743 pr_debug("%d stripes handled\n", handled);
1da177e4
LT		 5744
5745 spin_unlock_irq(&conf->device_lock);
5746
c9f21aaf 5747 async_tx_issue_pending_all();
e1dfa0a2 5748 blk_finish_plug(&plug);
1da177e4 5749
45b4233c 5750 pr_debug("--- raid5d inactive\n");
1da177e4
LT		 5751}
5752
3f294f4f 5753static ssize_t
fd01b88c 5754raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
3f294f4f 5755{
7b1485ba
N		 5756 struct r5conf *conf;
5757 int ret = 0;
5758 spin_lock(&mddev->lock);
5759 conf = mddev->private;
96de1e66 5760 if (conf)
7b1485ba
N		 5761 ret = sprintf(page, "%d\n", conf->max_nr_stripes);
5762 spin_unlock(&mddev->lock);
5763 return ret;
3f294f4f
N		 5764}
5765
c41d4ac4 5766int
fd01b88c 5767raid5_set_cache_size(struct mddev *mddev, int size)
3f294f4f 5768{
d1688a6d 5769 struct r5conf *conf = mddev->private;
b5470dc5 5770 int err;
566c09c5 5771 int hash;
b5470dc5 5772
c41d4ac4 5773 if (size <= 16 || size > 32768)
3f294f4f 5774 return -EINVAL;
566c09c5 5775 hash = (conf->max_nr_stripes - 1) % NR_STRIPE_HASH_LOCKS;
c41d4ac4 5776 while (size < conf->max_nr_stripes) {
566c09c5 5777 if (drop_one_stripe(conf, hash))
3f294f4f
N		 5778 conf->max_nr_stripes--;
5779 else
5780 break;
566c09c5
SL		 5781 hash--;
5782 if (hash < 0)
5783 hash = NR_STRIPE_HASH_LOCKS - 1;
3f294f4f 5784 }
b5470dc5
DW		 5785 err = md_allow_write(mddev);
5786 if (err)
5787 return err;
566c09c5 5788 hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
c41d4ac4 5789 while (size > conf->max_nr_stripes) {
566c09c5 5790 if (grow_one_stripe(conf, hash))
3f294f4f
N		 5791 conf->max_nr_stripes++;
5792 else break;
566c09c5 5793 hash = (hash + 1) % NR_STRIPE_HASH_LOCKS;
3f294f4f 5794 }
c41d4ac4
N		 5795 return 0;
5796}
5797EXPORT_SYMBOL(raid5_set_cache_size);
5798
5799static ssize_t
fd01b88c 5800raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
c41d4ac4 5801{
6791875e 5802 struct r5conf *conf;
c41d4ac4
N		 5803 unsigned long new;
5804 int err;
5805
5806 if (len >= PAGE_SIZE)
5807 return -EINVAL;
b29bebd6 5808 if (kstrtoul(page, 10, &new))
c41d4ac4 5809 return -EINVAL;
6791875e 5810 err = mddev_lock(mddev);
c41d4ac4
N		 5811 if (err)
5812 return err;
6791875e
N		 5813 conf = mddev->private;
5814 if (!conf)
5815 err = -ENODEV;
5816 else
5817 err = raid5_set_cache_size(mddev, new);
5818 mddev_unlock(mddev);
5819
5820 return err ?: len;
3f294f4f 5821}
007583c9 5822
96de1e66
N		 5823static struct md_sysfs_entry
5824raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
5825 raid5_show_stripe_cache_size,
5826 raid5_store_stripe_cache_size);
3f294f4f 5827
8b3e6cdc 5828static ssize_t
fd01b88c 5829raid5_show_preread_threshold(struct mddev *mddev, char *page)
8b3e6cdc 5830{
7b1485ba
N		 5831 struct r5conf *conf;
5832 int ret = 0;
5833 spin_lock(&mddev->lock);
5834 conf = mddev->private;
8b3e6cdc 5835 if (conf)
7b1485ba
N		 5836 ret = sprintf(page, "%d\n", conf->bypass_threshold);
5837 spin_unlock(&mddev->lock);
5838 return ret;
8b3e6cdc
DW		 5839}
5840
5841static ssize_t
fd01b88c 5842raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
8b3e6cdc 5843{
6791875e 5844 struct r5conf *conf;
4ef197d8 5845 unsigned long new;
6791875e
N		 5846 int err;
5847
8b3e6cdc
DW		 5848 if (len >= PAGE_SIZE)
5849 return -EINVAL;
b29bebd6 5850 if (kstrtoul(page, 10, &new))
8b3e6cdc 5851 return -EINVAL;
6791875e
N		 5852
5853 err = mddev_lock(mddev);
5854 if (err)
5855 return err;
5856 conf = mddev->private;
5857 if (!conf)
5858 err = -ENODEV;
5859 else if (new > conf->max_nr_stripes)
5860 err = -EINVAL;
5861 else
5862 conf->bypass_threshold = new;
5863 mddev_unlock(mddev);
5864 return err ?: len;
8b3e6cdc
DW		 5865}
5866
5867static struct md_sysfs_entry
5868raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
5869 S_IRUGO | S_IWUSR,
5870 raid5_show_preread_threshold,
5871 raid5_store_preread_threshold);
5872
d592a996
SL		 5873static ssize_t
5874raid5_show_skip_copy(struct mddev *mddev, char *page)
5875{
7b1485ba
N		 5876 struct r5conf *conf;
5877 int ret = 0;
5878 spin_lock(&mddev->lock);
5879 conf = mddev->private;
d592a996 5880 if (conf)
7b1485ba
N		 5881 ret = sprintf(page, "%d\n", conf->skip_copy);
5882 spin_unlock(&mddev->lock);
5883 return ret;
d592a996
SL		 5884}
5885
5886static ssize_t
5887raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)
5888{
6791875e 5889 struct r5conf *conf;
d592a996 5890 unsigned long new;
6791875e
N		 5891 int err;
5892
d592a996
SL		 5893 if (len >= PAGE_SIZE)
5894 return -EINVAL;
d592a996
SL		 5895 if (kstrtoul(page, 10, &new))
5896 return -EINVAL;
5897 new = !!new;
6791875e
N		 5898
5899 err = mddev_lock(mddev);
5900 if (err)
5901 return err;
5902 conf = mddev->private;
5903 if (!conf)
5904 err = -ENODEV;
5905 else if (new != conf->skip_copy) {
5906 mddev_suspend(mddev);
5907 conf->skip_copy = new;
5908 if (new)
5909 mddev->queue->backing_dev_info.capabilities |=
5910 BDI_CAP_STABLE_WRITES;
5911 else
5912 mddev->queue->backing_dev_info.capabilities &=
5913 ~BDI_CAP_STABLE_WRITES;
5914 mddev_resume(mddev);
5915 }
5916 mddev_unlock(mddev);
5917 return err ?: len;
d592a996
SL		 5918}
5919
5920static struct md_sysfs_entry
5921raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,
5922 raid5_show_skip_copy,
5923 raid5_store_skip_copy);
5924
3f294f4f 5925static ssize_t
fd01b88c 5926stripe_cache_active_show(struct mddev *mddev, char *page)
3f294f4f 5927{
d1688a6d 5928 struct r5conf *conf = mddev->private;
96de1e66
N		 5929 if (conf)
5930 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
5931 else
5932 return 0;
3f294f4f
N		 5933}
5934
96de1e66
N		 5935static struct md_sysfs_entry
5936raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
3f294f4f 5937
b721420e
SL		 5938static ssize_t
5939raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
5940{
7b1485ba
N		 5941 struct r5conf *conf;
5942 int ret = 0;
5943 spin_lock(&mddev->lock);
5944 conf = mddev->private;
b721420e 5945 if (conf)
7b1485ba
N		 5946 ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);
5947 spin_unlock(&mddev->lock);
5948 return ret;
b721420e
SL		 5949}
5950
60aaf933 5951static int alloc_thread_groups(struct r5conf *conf, int cnt,
5952 int *group_cnt,
5953 int *worker_cnt_per_group,
5954 struct r5worker_group **worker_groups);
b721420e
SL		 5955static ssize_t
5956raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
5957{
6791875e 5958 struct r5conf *conf;
b721420e
SL		 5959 unsigned long new;
5960 int err;
60aaf933 5961 struct r5worker_group *new_groups, *old_groups;
5962 int group_cnt, worker_cnt_per_group;
b721420e
SL		 5963
5964 if (len >= PAGE_SIZE)
5965 return -EINVAL;
b721420e
SL		 5966 if (kstrtoul(page, 10, &new))
5967 return -EINVAL;
5968
6791875e
N		 5969 err = mddev_lock(mddev);
5970 if (err)
5971 return err;
5972 conf = mddev->private;
5973 if (!conf)
5974 err = -ENODEV;
5975 else if (new != conf->worker_cnt_per_group) {
5976 mddev_suspend(mddev);
b721420e 5977
6791875e
N		 5978 old_groups = conf->worker_groups;
5979 if (old_groups)
5980 flush_workqueue(raid5_wq);
d206dcfa 5981
6791875e
N		 5982 err = alloc_thread_groups(conf, new,
5983 &group_cnt, &worker_cnt_per_group,
5984 &new_groups);
5985 if (!err) {
5986 spin_lock_irq(&conf->device_lock);
5987 conf->group_cnt = group_cnt;
5988 conf->worker_cnt_per_group = worker_cnt_per_group;
5989 conf->worker_groups = new_groups;
5990 spin_unlock_irq(&conf->device_lock);
b721420e 5991
6791875e
N		 5992 if (old_groups)
5993 kfree(old_groups[0].workers);
5994 kfree(old_groups);
5995 }
5996 mddev_resume(mddev);
b721420e 5997 }
6791875e 5998 mddev_unlock(mddev);
b721420e 5999
6791875e 6000 return err ?: len;
b721420e
SL		 6001}
6002
6003static struct md_sysfs_entry
6004raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
6005 raid5_show_group_thread_cnt,
6006 raid5_store_group_thread_cnt);
6007
007583c9 6008static struct attribute *raid5_attrs[] = {
3f294f4f
N		 6009 &raid5_stripecache_size.attr,
6010 &raid5_stripecache_active.attr,
8b3e6cdc 6011 &raid5_preread_bypass_threshold.attr,
b721420e 6012 &raid5_group_thread_cnt.attr,
d592a996 6013 &raid5_skip_copy.attr,
3f294f4f
N		 6014 NULL,
6015};
007583c9
N		 6016static struct attribute_group raid5_attrs_group = {
6017 .name = NULL,
6018 .attrs = raid5_attrs,
3f294f4f
N		 6019};
6020
60aaf933 6021static int alloc_thread_groups(struct r5conf *conf, int cnt,
6022 int *group_cnt,
6023 int *worker_cnt_per_group,
6024 struct r5worker_group **worker_groups)
851c30c9 6025{
566c09c5 6026 int i, j, k;
851c30c9
SL		 6027 ssize_t size;
6028 struct r5worker *workers;
6029
60aaf933 6030 *worker_cnt_per_group = cnt;
851c30c9 6031 if (cnt == 0) {
60aaf933 6032 *group_cnt = 0;
6033 *worker_groups = NULL;
851c30c9
SL		 6034 return 0;
6035 }
60aaf933 6036 *group_cnt = num_possible_nodes();
851c30c9 6037 size = sizeof(struct r5worker) * cnt;
60aaf933 6038 workers = kzalloc(size * *group_cnt, GFP_NOIO);
6039 *worker_groups = kzalloc(sizeof(struct r5worker_group) *
6040 *group_cnt, GFP_NOIO);
6041 if (!*worker_groups || !workers) {
851c30c9 6042 kfree(workers);
60aaf933 6043 kfree(*worker_groups);
851c30c9
SL		 6044 return -ENOMEM;
6045 }
6046
60aaf933 6047 for (i = 0; i < *group_cnt; i++) {
851c30c9
SL		 6048 struct r5worker_group *group;
6049
0c775d52 6050 group = &(*worker_groups)[i];
851c30c9
SL		 6051 INIT_LIST_HEAD(&group->handle_list);
6052 group->conf = conf;
6053 group->workers = workers + i * cnt;
6054
6055 for (j = 0; j < cnt; j++) {
566c09c5
SL		 6056 struct r5worker *worker = group->workers + j;
6057 worker->group = group;
6058 INIT_WORK(&worker->work, raid5_do_work);
6059
6060 for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
6061 INIT_LIST_HEAD(worker->temp_inactive_list + k);
851c30c9
SL		 6062 }
6063 }
6064
6065 return 0;
6066}
6067
6068static void free_thread_groups(struct r5conf *conf)
6069{
6070 if (conf->worker_groups)
6071 kfree(conf->worker_groups[0].workers);
6072 kfree(conf->worker_groups);
6073 conf->worker_groups = NULL;
6074}
6075
80c3a6ce 6076static sector_t
fd01b88c 6077raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
80c3a6ce 6078{
d1688a6d 6079 struct r5conf *conf = mddev->private;
80c3a6ce
DW		 6080
6081 if (!sectors)
6082 sectors = mddev->dev_sectors;
5e5e3e78 6083 if (!raid_disks)
7ec05478 6084 /* size is defined by the smallest of previous and new size */
5e5e3e78 6085 raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
80c3a6ce 6086
9d8f0363 6087 sectors &= ~((sector_t)mddev->chunk_sectors - 1);
664e7c41 6088 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
80c3a6ce
DW		 6089 return sectors * (raid_disks - conf->max_degraded);
6090}
6091
789b5e03
ON		 6092static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
6093{
6094 safe_put_page(percpu->spare_page);
46d5b785 6095 if (percpu->scribble)
6096 flex_array_free(percpu->scribble);
789b5e03
ON		 6097 percpu->spare_page = NULL;
6098 percpu->scribble = NULL;
6099}
6100
6101static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
6102{
6103 if (conf->level == 6 && !percpu->spare_page)
6104 percpu->spare_page = alloc_page(GFP_KERNEL);
6105 if (!percpu->scribble)
46d5b785 6106 percpu->scribble = scribble_alloc(max(conf->raid_disks,
6107 conf->previous_raid_disks), conf->chunk_sectors /
6108 STRIPE_SECTORS, GFP_KERNEL);
789b5e03
ON		 6109
6110 if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) {
6111 free_scratch_buffer(conf, percpu);
6112 return -ENOMEM;
6113 }
6114
6115 return 0;
6116}
6117
d1688a6d 6118static void raid5_free_percpu(struct r5conf *conf)
36d1c647 6119{
36d1c647
DW		 6120 unsigned long cpu;
6121
6122 if (!conf->percpu)
6123 return;
6124
36d1c647
DW		 6125#ifdef CONFIG_HOTPLUG_CPU
6126 unregister_cpu_notifier(&conf->cpu_notify);
6127#endif
789b5e03
ON		 6128
6129 get_online_cpus();
6130 for_each_possible_cpu(cpu)
6131 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
36d1c647
DW		 6132 put_online_cpus();
6133
6134 free_percpu(conf->percpu);
6135}
6136
d1688a6d 6137static void free_conf(struct r5conf *conf)
95fc17aa 6138{
851c30c9 6139 free_thread_groups(conf);
95fc17aa 6140 shrink_stripes(conf);
36d1c647 6141 raid5_free_percpu(conf);
95fc17aa
DW		 6142 kfree(conf->disks);
6143 kfree(conf->stripe_hashtbl);
6144 kfree(conf);
6145}
6146
36d1c647
DW		 6147#ifdef CONFIG_HOTPLUG_CPU
6148static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
6149 void *hcpu)
6150{
d1688a6d 6151 struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
36d1c647
DW		 6152 long cpu = (long)hcpu;
6153 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
6154
6155 switch (action) {
6156 case CPU_UP_PREPARE:
6157 case CPU_UP_PREPARE_FROZEN:
789b5e03 6158 if (alloc_scratch_buffer(conf, percpu)) {
36d1c647
DW		 6159 pr_err("%s: failed memory allocation for cpu%ld\n",
6160 __func__, cpu);
55af6bb5 6161 return notifier_from_errno(-ENOMEM);
36d1c647
DW		 6162 }
6163 break;
6164 case CPU_DEAD:
6165 case CPU_DEAD_FROZEN:
789b5e03 6166 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
36d1c647
DW		 6167 break;
6168 default:
6169 break;
6170 }
6171 return NOTIFY_OK;
6172}
6173#endif
6174
d1688a6d 6175static int raid5_alloc_percpu(struct r5conf *conf)
36d1c647
DW		 6176{
6177 unsigned long cpu;
789b5e03 6178 int err = 0;
36d1c647 6179
789b5e03
ON		 6180 conf->percpu = alloc_percpu(struct raid5_percpu);
6181 if (!conf->percpu)
36d1c647 6182 return -ENOMEM;
789b5e03
ON		 6183
6184#ifdef CONFIG_HOTPLUG_CPU
6185 conf->cpu_notify.notifier_call = raid456_cpu_notify;
6186 conf->cpu_notify.priority = 0;
6187 err = register_cpu_notifier(&conf->cpu_notify);
6188 if (err)
6189 return err;
6190#endif
36d1c647
DW		 6191
6192 get_online_cpus();
36d1c647 6193 for_each_present_cpu(cpu) {
789b5e03
ON		 6194 err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
6195 if (err) {
6196 pr_err("%s: failed memory allocation for cpu%ld\n",
6197 __func__, cpu);
36d1c647
DW		 6198 break;
6199 }
36d1c647 6200 }
36d1c647
DW		 6201 put_online_cpus();
6202
6203 return err;
6204}
6205
d1688a6d 6206static struct r5conf *setup_conf(struct mddev *mddev)
1da177e4 6207{
d1688a6d 6208 struct r5conf *conf;
5e5e3e78 6209 int raid_disk, memory, max_disks;
3cb03002 6210 struct md_rdev *rdev;
1da177e4 6211 struct disk_info *disk;
0232605d 6212 char pers_name[6];
566c09c5 6213 int i;
60aaf933 6214 int group_cnt, worker_cnt_per_group;
6215 struct r5worker_group *new_group;
1da177e4 6216
91adb564
N		 6217 if (mddev->new_level != 5
6218 && mddev->new_level != 4
6219 && mddev->new_level != 6) {
0c55e022 6220 printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
91adb564
N		 6221 mdname(mddev), mddev->new_level);
6222 return ERR_PTR(-EIO);
1da177e4 6223 }
91adb564
N		 6224 if ((mddev->new_level == 5
6225 && !algorithm_valid_raid5(mddev->new_layout)) ||
6226 (mddev->new_level == 6
6227 && !algorithm_valid_raid6(mddev->new_layout))) {
0c55e022 6228 printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
91adb564
N		 6229 mdname(mddev), mddev->new_layout);
6230 return ERR_PTR(-EIO);
99c0fb5f 6231 }
91adb564 6232 if (mddev->new_level == 6 && mddev->raid_disks < 4) {
0c55e022 6233 printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
91adb564
N		 6234 mdname(mddev), mddev->raid_disks);
6235 return ERR_PTR(-EINVAL);
4bbf3771
N		 6236 }
6237
664e7c41
AN		 6238 if (!mddev->new_chunk_sectors ||
6239 (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
6240 !is_power_of_2(mddev->new_chunk_sectors)) {
0c55e022
N		 6241 printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
6242 mdname(mddev), mddev->new_chunk_sectors << 9);
91adb564 6243 return ERR_PTR(-EINVAL);
f6705578
N		 6244 }
6245
d1688a6d 6246 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
91adb564 6247 if (conf == NULL)
1da177e4 6248 goto abort;
851c30c9 6249 /* Don't enable multi-threading by default*/
60aaf933 6250 if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
6251 &new_group)) {
6252 conf->group_cnt = group_cnt;
6253 conf->worker_cnt_per_group = worker_cnt_per_group;
6254 conf->worker_groups = new_group;
6255 } else
851c30c9 6256 goto abort;
f5efd45a 6257 spin_lock_init(&conf->device_lock);
c46501b2 6258 seqcount_init(&conf->gen_lock);
f5efd45a
DW		 6259 init_waitqueue_head(&conf->wait_for_stripe);
6260 init_waitqueue_head(&conf->wait_for_overlap);
6261 INIT_LIST_HEAD(&conf->handle_list);
6262 INIT_LIST_HEAD(&conf->hold_list);
6263 INIT_LIST_HEAD(&conf->delayed_list);
6264 INIT_LIST_HEAD(&conf->bitmap_list);
773ca82f 6265 init_llist_head(&conf->released_stripes);
f5efd45a
DW		 6266 atomic_set(&conf->active_stripes, 0);
6267 atomic_set(&conf->preread_active_stripes, 0);
6268 atomic_set(&conf->active_aligned_reads, 0);
6269 conf->bypass_threshold = BYPASS_THRESHOLD;
d890fa2b 6270 conf->recovery_disabled = mddev->recovery_disabled - 1;
91adb564
N		 6271
6272 conf->raid_disks = mddev->raid_disks;
6273 if (mddev->reshape_position == MaxSector)
6274 conf->previous_raid_disks = mddev->raid_disks;
6275 else
f6705578 6276 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
5e5e3e78 6277 max_disks = max(conf->raid_disks, conf->previous_raid_disks);
f6705578 6278
5e5e3e78 6279 conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
b55e6bfc
N		 6280 GFP_KERNEL);
6281 if (!conf->disks)
6282 goto abort;
9ffae0cf 6283
1da177e4
LT		 6284 conf->mddev = mddev;
6285
fccddba0 6286 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
1da177e4 6287 goto abort;
1da177e4 6288
566c09c5
SL		 6289 /* We init hash_locks[0] separately to that it can be used
6290 * as the reference lock in the spin_lock_nest_lock() call
6291 * in lock_all_device_hash_locks_irq in order to convince
6292 * lockdep that we know what we are doing.
6293 */
6294 spin_lock_init(conf->hash_locks);
6295 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
6296 spin_lock_init(conf->hash_locks + i);
6297
6298 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
6299 INIT_LIST_HEAD(conf->inactive_list + i);
6300
6301 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
6302 INIT_LIST_HEAD(conf->temp_inactive_list + i);
6303
36d1c647 6304 conf->level = mddev->new_level;
46d5b785 6305 conf->chunk_sectors = mddev->new_chunk_sectors;
36d1c647
DW		 6306 if (raid5_alloc_percpu(conf) != 0)
6307 goto abort;
6308
0c55e022 6309 pr_debug("raid456: run(%s) called.\n", mdname(mddev));
1da177e4 6310
dafb20fa 6311 rdev_for_each(rdev, mddev) {
1da177e4 6312 raid_disk = rdev->raid_disk;
5e5e3e78 6313 if (raid_disk >= max_disks
1da177e4
LT		 6314 || raid_disk < 0)
6315 continue;
6316 disk = conf->disks + raid_disk;
6317
17045f52
N		 6318 if (test_bit(Replacement, &rdev->flags)) {
6319 if (disk->replacement)
6320 goto abort;
6321 disk->replacement = rdev;
6322 } else {
6323 if (disk->rdev)
6324 goto abort;
6325 disk->rdev = rdev;
6326 }
1da177e4 6327
b2d444d7 6328 if (test_bit(In_sync, &rdev->flags)) {
1da177e4 6329 char b[BDEVNAME_SIZE];
0c55e022
N		 6330 printk(KERN_INFO "md/raid:%s: device %s operational as raid"
6331 " disk %d\n",
6332 mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
d6b212f4 6333 } else if (rdev->saved_raid_disk != raid_disk)
8c2e870a
NB		 6334 /* Cannot rely on bitmap to complete recovery */
6335 conf->fullsync = 1;
1da177e4
LT		 6336 }
6337
91adb564 6338 conf->level = mddev->new_level;
16a53ecc
N		 6339 if (conf->level == 6)
6340 conf->max_degraded = 2;
6341 else
6342 conf->max_degraded = 1;
91adb564 6343 conf->algorithm = mddev->new_layout;
fef9c61f 6344 conf->reshape_progress = mddev->reshape_position;
e183eaed 6345 if (conf->reshape_progress != MaxSector) {
09c9e5fa 6346 conf->prev_chunk_sectors = mddev->chunk_sectors;
e183eaed
N		 6347 conf->prev_algo = mddev->layout;
6348 }
1da177e4 6349
91adb564 6350 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
5e5e3e78 6351 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
4bda556a 6352 atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
566c09c5 6353 if (grow_stripes(conf, NR_STRIPES)) {
91adb564 6354 printk(KERN_ERR
0c55e022
N		 6355 "md/raid:%s: couldn't allocate %dkB for buffers\n",
6356 mdname(mddev), memory);
91adb564
N		 6357 goto abort;
6358 } else
0c55e022
N		 6359 printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
6360 mdname(mddev), memory);
1da177e4 6361
0232605d
N		 6362 sprintf(pers_name, "raid%d", mddev->new_level);
6363 conf->thread = md_register_thread(raid5d, mddev, pers_name);
91adb564
N		 6364 if (!conf->thread) {
6365 printk(KERN_ERR
0c55e022 6366 "md/raid:%s: couldn't allocate thread.\n",
91adb564 6367 mdname(mddev));
16a53ecc
N		 6368 goto abort;
6369 }
91adb564
N		 6370
6371 return conf;
6372
6373 abort:
6374 if (conf) {
95fc17aa 6375 free_conf(conf);
91adb564
N		 6376 return ERR_PTR(-EIO);
6377 } else
6378 return ERR_PTR(-ENOMEM);
6379}
6380
c148ffdc
N		 6381static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
6382{
6383 switch (algo) {
6384 case ALGORITHM_PARITY_0:
6385 if (raid_disk < max_degraded)
6386 return 1;
6387 break;
6388 case ALGORITHM_PARITY_N:
6389 if (raid_disk >= raid_disks - max_degraded)
6390 return 1;
6391 break;
6392 case ALGORITHM_PARITY_0_6:
f72ffdd6 6393 if (raid_disk == 0 ||
c148ffdc
N		 6394 raid_disk == raid_disks - 1)
6395 return 1;
6396 break;
6397 case ALGORITHM_LEFT_ASYMMETRIC_6:
6398 case ALGORITHM_RIGHT_ASYMMETRIC_6:
6399 case ALGORITHM_LEFT_SYMMETRIC_6:
6400 case ALGORITHM_RIGHT_SYMMETRIC_6:
6401 if (raid_disk == raid_disks - 1)
6402 return 1;
6403 }
6404 return 0;
6405}
6406
fd01b88c 6407static int run(struct mddev *mddev)
91adb564 6408{
d1688a6d 6409 struct r5conf *conf;
9f7c2220 6410 int working_disks = 0;
c148ffdc 6411 int dirty_parity_disks = 0;
3cb03002 6412 struct md_rdev *rdev;
c148ffdc 6413 sector_t reshape_offset = 0;
17045f52 6414 int i;
b5254dd5
N		 6415 long long min_offset_diff = 0;
6416 int first = 1;
91adb564 6417
8c6ac868 6418 if (mddev->recovery_cp != MaxSector)
0c55e022 6419 printk(KERN_NOTICE "md/raid:%s: not clean"
8c6ac868
AN		 6420 " -- starting background reconstruction\n",
6421 mdname(mddev));
b5254dd5
N		 6422
6423 rdev_for_each(rdev, mddev) {
6424 long long diff;
6425 if (rdev->raid_disk < 0)
6426 continue;
6427 diff = (rdev->new_data_offset - rdev->data_offset);
6428 if (first) {
6429 min_offset_diff = diff;
6430 first = 0;
6431 } else if (mddev->reshape_backwards &&
6432 diff < min_offset_diff)
6433 min_offset_diff = diff;
6434 else if (!mddev->reshape_backwards &&
6435 diff > min_offset_diff)
6436 min_offset_diff = diff;
6437 }
6438
91adb564
N		 6439 if (mddev->reshape_position != MaxSector) {
6440 /* Check that we can continue the reshape.
b5254dd5
N		 6441 * Difficulties arise if the stripe we would write to
6442 * next is at or after the stripe we would read from next.
6443 * For a reshape that changes the number of devices, this
6444 * is only possible for a very short time, and mdadm makes
6445 * sure that time appears to have past before assembling
6446 * the array. So we fail if that time hasn't passed.
6447 * For a reshape that keeps the number of devices the same
6448 * mdadm must be monitoring the reshape can keeping the
6449 * critical areas read-only and backed up. It will start
6450 * the array in read-only mode, so we check for that.
91adb564
N		 6451 */
6452 sector_t here_new, here_old;
6453 int old_disks;
18b00334 6454 int max_degraded = (mddev->level == 6 ? 2 : 1);
91adb564 6455
88ce4930 6456 if (mddev->new_level != mddev->level) {
0c55e022 6457 printk(KERN_ERR "md/raid:%s: unsupported reshape "
91adb564
N		 6458 "required - aborting.\n",
6459 mdname(mddev));
6460 return -EINVAL;
6461 }
91adb564
N		 6462 old_disks = mddev->raid_disks - mddev->delta_disks;
6463 /* reshape_position must be on a new-stripe boundary, and one
6464 * further up in new geometry must map after here in old
6465 * geometry.
6466 */
6467 here_new = mddev->reshape_position;
664e7c41 6468 if (sector_div(here_new, mddev->new_chunk_sectors *
91adb564 6469 (mddev->raid_disks - max_degraded))) {
0c55e022
N		 6470 printk(KERN_ERR "md/raid:%s: reshape_position not "
6471 "on a stripe boundary\n", mdname(mddev));
91adb564
N		 6472 return -EINVAL;
6473 }
c148ffdc 6474 reshape_offset = here_new * mddev->new_chunk_sectors;
91adb564
N		 6475 /* here_new is the stripe we will write to */
6476 here_old = mddev->reshape_position;
9d8f0363 6477 sector_div(here_old, mddev->chunk_sectors *
91adb564
N		 6478 (old_disks-max_degraded));
6479 /* here_old is the first stripe that we might need to read
6480 * from */
67ac6011 6481 if (mddev->delta_disks == 0) {
b5254dd5
N		 6482 if ((here_new * mddev->new_chunk_sectors !=
6483 here_old * mddev->chunk_sectors)) {
6484 printk(KERN_ERR "md/raid:%s: reshape position is"
6485 " confused - aborting\n", mdname(mddev));
6486 return -EINVAL;
6487 }
67ac6011 6488 /* We cannot be sure it is safe to start an in-place
b5254dd5 6489 * reshape. It is only safe if user-space is monitoring
67ac6011
N		 6490 * and taking constant backups.
6491 * mdadm always starts a situation like this in
6492 * readonly mode so it can take control before
6493 * allowing any writes. So just check for that.
6494 */
b5254dd5
N		 6495 if (abs(min_offset_diff) >= mddev->chunk_sectors &&
6496 abs(min_offset_diff) >= mddev->new_chunk_sectors)
6497 /* not really in-place - so OK */;
6498 else if (mddev->ro == 0) {
6499 printk(KERN_ERR "md/raid:%s: in-place reshape "
6500 "must be started in read-only mode "
6501 "- aborting\n",
0c55e022 6502 mdname(mddev));
67ac6011
N		 6503 return -EINVAL;
6504 }
2c810cdd 6505 } else if (mddev->reshape_backwards
b5254dd5 6506 ? (here_new * mddev->new_chunk_sectors + min_offset_diff <=
67ac6011
N		 6507 here_old * mddev->chunk_sectors)
6508 : (here_new * mddev->new_chunk_sectors >=
b5254dd5 6509 here_old * mddev->chunk_sectors + (-min_offset_diff))) {
91adb564 6510 /* Reading from the same stripe as writing to - bad */
0c55e022
N		 6511 printk(KERN_ERR "md/raid:%s: reshape_position too early for "
6512 "auto-recovery - aborting.\n",
6513 mdname(mddev));
91adb564
N		 6514 return -EINVAL;
6515 }
0c55e022
N		 6516 printk(KERN_INFO "md/raid:%s: reshape will continue\n",
6517 mdname(mddev));
91adb564
N		 6518 /* OK, we should be able to continue; */
6519 } else {
6520 BUG_ON(mddev->level != mddev->new_level);
6521 BUG_ON(mddev->layout != mddev->new_layout);
664e7c41 6522 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
91adb564 6523 BUG_ON(mddev->delta_disks != 0);
1da177e4 6524 }
91adb564 6525
245f46c2
N		 6526 if (mddev->private == NULL)
6527 conf = setup_conf(mddev);
6528 else
6529 conf = mddev->private;
6530
91adb564
N		 6531 if (IS_ERR(conf))
6532 return PTR_ERR(conf);
6533
b5254dd5 6534 conf->min_offset_diff = min_offset_diff;
91adb564
N		 6535 mddev->thread = conf->thread;
6536 conf->thread = NULL;
6537 mddev->private = conf;
6538
17045f52
N		 6539 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
6540 i++) {
6541 rdev = conf->disks[i].rdev;
6542 if (!rdev && conf->disks[i].replacement) {
6543 /* The replacement is all we have yet */
6544 rdev = conf->disks[i].replacement;
6545 conf->disks[i].replacement = NULL;
6546 clear_bit(Replacement, &rdev->flags);
6547 conf->disks[i].rdev = rdev;
6548 }
6549 if (!rdev)
c148ffdc 6550 continue;
17045f52
N		 6551 if (conf->disks[i].replacement &&
6552 conf->reshape_progress != MaxSector) {
6553 /* replacements and reshape simply do not mix. */
6554 printk(KERN_ERR "md: cannot handle concurrent "
6555 "replacement and reshape.\n");
6556 goto abort;
6557 }
2f115882 6558 if (test_bit(In_sync, &rdev->flags)) {
91adb564 6559 working_disks++;
2f115882
N		 6560 continue;
6561 }
c148ffdc
N		 6562 /* This disc is not fully in-sync. However if it
6563 * just stored parity (beyond the recovery_offset),
6564 * when we don't need to be concerned about the
6565 * array being dirty.
6566 * When reshape goes 'backwards', we never have
6567 * partially completed devices, so we only need
6568 * to worry about reshape going forwards.
6569 */
6570 /* Hack because v0.91 doesn't store recovery_offset properly. */
6571 if (mddev->major_version == 0 &&
6572 mddev->minor_version > 90)
6573 rdev->recovery_offset = reshape_offset;
5026d7a9 6574
c148ffdc
N		 6575 if (rdev->recovery_offset < reshape_offset) {
6576 /* We need to check old and new layout */
6577 if (!only_parity(rdev->raid_disk,
6578 conf->algorithm,
6579 conf->raid_disks,
6580 conf->max_degraded))
6581 continue;
6582 }
6583 if (!only_parity(rdev->raid_disk,
6584 conf->prev_algo,
6585 conf->previous_raid_disks,
6586 conf->max_degraded))
6587 continue;
6588 dirty_parity_disks++;
6589 }
91adb564 6590
17045f52
N		 6591 /*
6592 * 0 for a fully functional array, 1 or 2 for a degraded array.
6593 */
908f4fbd 6594 mddev->degraded = calc_degraded(conf);
91adb564 6595
674806d6 6596 if (has_failed(conf)) {
0c55e022 6597 printk(KERN_ERR "md/raid:%s: not enough operational devices"
1da177e4 6598 " (%d/%d failed)\n",
02c2de8c 6599 mdname(mddev), mddev->degraded, conf->raid_disks);
1da177e4
LT		 6600 goto abort;
6601 }
6602
91adb564 6603 /* device size must be a multiple of chunk size */
9d8f0363 6604 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
91adb564
N		 6605 mddev->resync_max_sectors = mddev->dev_sectors;
6606
c148ffdc 6607 if (mddev->degraded > dirty_parity_disks &&
1da177e4 6608 mddev->recovery_cp != MaxSector) {
6ff8d8ec
N		 6609 if (mddev->ok_start_degraded)
6610 printk(KERN_WARNING
0c55e022
N		 6611 "md/raid:%s: starting dirty degraded array"
6612 " - data corruption possible.\n",
6ff8d8ec
N		 6613 mdname(mddev));
6614 else {
6615 printk(KERN_ERR
0c55e022 6616 "md/raid:%s: cannot start dirty degraded array.\n",
6ff8d8ec
N		 6617 mdname(mddev));
6618 goto abort;
6619 }
1da177e4
LT		 6620 }
6621
1da177e4 6622 if (mddev->degraded == 0)
0c55e022
N		 6623 printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
6624 " devices, algorithm %d\n", mdname(mddev), conf->level,
e183eaed
N		 6625 mddev->raid_disks-mddev->degraded, mddev->raid_disks,
6626 mddev->new_layout);
1da177e4 6627 else
0c55e022
N		 6628 printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
6629 " out of %d devices, algorithm %d\n",
6630 mdname(mddev), conf->level,
6631 mddev->raid_disks - mddev->degraded,
6632 mddev->raid_disks, mddev->new_layout);
1da177e4
LT		 6633
6634 print_raid5_conf(conf);
6635
fef9c61f 6636 if (conf->reshape_progress != MaxSector) {
fef9c61f 6637 conf->reshape_safe = conf->reshape_progress;
f6705578
N		 6638 atomic_set(&conf->reshape_stripes, 0);
6639 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
6640 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
6641 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
6642 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
6643 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
0da3c619 6644 "reshape");
f6705578
N		 6645 }
6646
1da177e4 6647 /* Ok, everything is just fine now */
a64c876f
N		 6648 if (mddev->to_remove == &raid5_attrs_group)
6649 mddev->to_remove = NULL;
00bcb4ac
N		 6650 else if (mddev->kobj.sd &&
6651 sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
5e55e2f5 6652 printk(KERN_WARNING
4a5add49 6653 "raid5: failed to create sysfs attributes for %s\n",
5e55e2f5 6654 mdname(mddev));
4a5add49 6655 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
7a5febe9 6656
4a5add49 6657 if (mddev->queue) {
9f7c2220 6658 int chunk_size;
620125f2 6659 bool discard_supported = true;
4a5add49
N		 6660 /* read-ahead size must cover two whole stripes, which
6661 * is 2 * (datadisks) * chunksize where 'n' is the
6662 * number of raid devices
6663 */
6664 int data_disks = conf->previous_raid_disks - conf->max_degraded;
6665 int stripe = data_disks *
6666 ((mddev->chunk_sectors << 9) / PAGE_SIZE);
6667 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
6668 mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
91adb564 6669
9f7c2220
N		 6670 chunk_size = mddev->chunk_sectors << 9;
6671 blk_queue_io_min(mddev->queue, chunk_size);
6672 blk_queue_io_opt(mddev->queue, chunk_size *
6673 (conf->raid_disks - conf->max_degraded));
c78afc62 6674 mddev->queue->limits.raid_partial_stripes_expensive = 1;
620125f2
SL		 6675 /*
6676 * We can only discard a whole stripe. It doesn't make sense to
6677 * discard data disk but write parity disk
6678 */
6679 stripe = stripe * PAGE_SIZE;
4ac6875e
N		 6680 /* Round up to power of 2, as discard handling
6681 * currently assumes that */
6682 while ((stripe-1) & stripe)
6683 stripe = (stripe | (stripe-1)) + 1;
620125f2
SL		 6684 mddev->queue->limits.discard_alignment = stripe;
6685 mddev->queue->limits.discard_granularity = stripe;
6686 /*
6687 * unaligned part of discard request will be ignored, so can't
8e0e99ba 6688 * guarantee discard_zeroes_data
620125f2
SL		 6689 */
6690 mddev->queue->limits.discard_zeroes_data = 0;
8f6c2e4b 6691
5026d7a9
PA		 6692 blk_queue_max_write_same_sectors(mddev->queue, 0);
6693
05616be5 6694 rdev_for_each(rdev, mddev) {
9f7c2220
N		 6695 disk_stack_limits(mddev->gendisk, rdev->bdev,
6696 rdev->data_offset << 9);
05616be5
N		 6697 disk_stack_limits(mddev->gendisk, rdev->bdev,
6698 rdev->new_data_offset << 9);
620125f2
SL		 6699 /*
6700 * discard_zeroes_data is required, otherwise data
6701 * could be lost. Consider a scenario: discard a stripe
6702 * (the stripe could be inconsistent if
6703 * discard_zeroes_data is 0); write one disk of the
6704 * stripe (the stripe could be inconsistent again
6705 * depending on which disks are used to calculate
6706 * parity); the disk is broken; The stripe data of this
6707 * disk is lost.
6708 */
6709 if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) ||
6710 !bdev_get_queue(rdev->bdev)->
6711 limits.discard_zeroes_data)
6712 discard_supported = false;
8e0e99ba
N		 6713 /* Unfortunately, discard_zeroes_data is not currently
6714 * a guarantee - just a hint. So we only allow DISCARD
6715 * if the sysadmin has confirmed that only safe devices
6716 * are in use by setting a module parameter.
6717 */
6718 if (!devices_handle_discard_safely) {
6719 if (discard_supported) {
6720 pr_info("md/raid456: discard support disabled due to uncertainty.\n");
6721 pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n");
6722 }
6723 discard_supported = false;
6724 }
05616be5 6725 }
620125f2
SL		 6726
6727 if (discard_supported &&
6728 mddev->queue->limits.max_discard_sectors >= stripe &&
6729 mddev->queue->limits.discard_granularity >= stripe)
6730 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
6731 mddev->queue);
6732 else
6733 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
6734 mddev->queue);
9f7c2220 6735 }
23032a0e 6736
1da177e4
LT		 6737 return 0;
6738abort:
01f96c0a 6739 md_unregister_thread(&mddev->thread);
e4f869d9
N		 6740 print_raid5_conf(conf);
6741 free_conf(conf);
1da177e4 6742 mddev->private = NULL;
0c55e022 6743 printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
1da177e4
LT		 6744 return -EIO;
6745}
6746
afa0f557 6747static void raid5_free(struct mddev *mddev, void *priv)
1da177e4 6748{
afa0f557 6749 struct r5conf *conf = priv;
1da177e4 6750
95fc17aa 6751 free_conf(conf);
a64c876f 6752 mddev->to_remove = &raid5_attrs_group;
1da177e4
LT		 6753}
6754
fd01b88c 6755static void status(struct seq_file *seq, struct mddev *mddev)
1da177e4 6756{
d1688a6d 6757 struct r5conf *conf = mddev->private;
1da177e4
LT		 6758 int i;
6759
9d8f0363
AN		 6760 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
6761 mddev->chunk_sectors / 2, mddev->layout);
02c2de8c 6762 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
1da177e4
LT		 6763 for (i = 0; i < conf->raid_disks; i++)
6764 seq_printf (seq, "%s",
6765 conf->disks[i].rdev &&
b2d444d7 6766 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
1da177e4 6767 seq_printf (seq, "]");
1da177e4
LT		 6768}
6769
d1688a6d 6770static void print_raid5_conf (struct r5conf *conf)
1da177e4
LT		 6771{
6772 int i;
6773 struct disk_info *tmp;
6774
0c55e022 6775 printk(KERN_DEBUG "RAID conf printout:\n");
1da177e4
LT		 6776 if (!conf) {
6777 printk("(conf==NULL)\n");
6778 return;
6779 }
0c55e022
N		 6780 printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
6781 conf->raid_disks,
6782 conf->raid_disks - conf->mddev->degraded);
1da177e4
LT		 6783
6784 for (i = 0; i < conf->raid_disks; i++) {
6785 char b[BDEVNAME_SIZE];
6786 tmp = conf->disks + i;
6787 if (tmp->rdev)
0c55e022
N		 6788 printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
6789 i, !test_bit(Faulty, &tmp->rdev->flags),
6790 bdevname(tmp->rdev->bdev, b));
1da177e4
LT		 6791 }
6792}
6793
fd01b88c 6794static int raid5_spare_active(struct mddev *mddev)
1da177e4
LT		 6795{
6796 int i;
d1688a6d 6797 struct r5conf *conf = mddev->private;
1da177e4 6798 struct disk_info *tmp;
6b965620
N		 6799 int count = 0;
6800 unsigned long flags;
1da177e4
LT		 6801
6802 for (i = 0; i < conf->raid_disks; i++) {
6803 tmp = conf->disks + i;
dd054fce
N		 6804 if (tmp->replacement
6805 && tmp->replacement->recovery_offset == MaxSector
6806 && !test_bit(Faulty, &tmp->replacement->flags)
6807 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
6808 /* Replacement has just become active. */
6809 if (!tmp->rdev
6810 || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
6811 count++;
6812 if (tmp->rdev) {
6813 /* Replaced device not technically faulty,
6814 * but we need to be sure it gets removed
6815 * and never re-added.
6816 */
6817 set_bit(Faulty, &tmp->rdev->flags);
6818 sysfs_notify_dirent_safe(
6819 tmp->rdev->sysfs_state);
6820 }
6821 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
6822 } else if (tmp->rdev
70fffd0b 6823 && tmp->rdev->recovery_offset == MaxSector
b2d444d7 6824 && !test_bit(Faulty, &tmp->rdev->flags)
c04be0aa 6825 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
6b965620 6826 count++;
43c73ca4 6827 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
1da177e4
LT		 6828 }
6829 }
6b965620 6830 spin_lock_irqsave(&conf->device_lock, flags);
908f4fbd 6831 mddev->degraded = calc_degraded(conf);
6b965620 6832 spin_unlock_irqrestore(&conf->device_lock, flags);
1da177e4 6833 print_raid5_conf(conf);
6b965620 6834 return count;
1da177e4
LT		 6835}
6836
b8321b68 6837static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
1da177e4 6838{
d1688a6d 6839 struct r5conf *conf = mddev->private;
1da177e4 6840 int err = 0;
b8321b68 6841 int number = rdev->raid_disk;
657e3e4d 6842 struct md_rdev **rdevp;
1da177e4
LT		 6843 struct disk_info *p = conf->disks + number;
6844
6845 print_raid5_conf(conf);
657e3e4d
N		 6846 if (rdev == p->rdev)
6847 rdevp = &p->rdev;
6848 else if (rdev == p->replacement)
6849 rdevp = &p->replacement;
6850 else
6851 return 0;
6852
6853 if (number >= conf->raid_disks &&
6854 conf->reshape_progress == MaxSector)
6855 clear_bit(In_sync, &rdev->flags);
6856
6857 if (test_bit(In_sync, &rdev->flags) ||
6858 atomic_read(&rdev->nr_pending)) {
6859 err = -EBUSY;
6860 goto abort;
6861 }
6862 /* Only remove non-faulty devices if recovery
6863 * isn't possible.
6864 */
6865 if (!test_bit(Faulty, &rdev->flags) &&
6866 mddev->recovery_disabled != conf->recovery_disabled &&
6867 !has_failed(conf) &&
dd054fce 6868 (!p->replacement || p->replacement == rdev) &&
657e3e4d
N		 6869 number < conf->raid_disks) {
6870 err = -EBUSY;
6871 goto abort;
6872 }
6873 *rdevp = NULL;
6874 synchronize_rcu();
6875 if (atomic_read(&rdev->nr_pending)) {
6876 /* lost the race, try later */
6877 err = -EBUSY;
6878 *rdevp = rdev;
dd054fce
N		 6879 } else if (p->replacement) {
6880 /* We must have just cleared 'rdev' */
6881 p->rdev = p->replacement;
6882 clear_bit(Replacement, &p->replacement->flags);
6883 smp_mb(); /* Make sure other CPUs may see both as identical
6884 * but will never see neither - if they are careful
6885 */
6886 p->replacement = NULL;
6887 clear_bit(WantReplacement, &rdev->flags);
6888 } else
6889 /* We might have just removed the Replacement as faulty-
6890 * clear the bit just in case
6891 */
6892 clear_bit(WantReplacement, &rdev->flags);
1da177e4
LT		 6893abort:
6894
6895 print_raid5_conf(conf);
6896 return err;
6897}
6898
fd01b88c 6899static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
1da177e4 6900{
d1688a6d 6901 struct r5conf *conf = mddev->private;
199050ea 6902 int err = -EEXIST;
1da177e4
LT		 6903 int disk;
6904 struct disk_info *p;
6c2fce2e
NB		 6905 int first = 0;
6906 int last = conf->raid_disks - 1;
1da177e4 6907
7f0da59b
N		 6908 if (mddev->recovery_disabled == conf->recovery_disabled)
6909 return -EBUSY;
6910
dc10c643 6911 if (rdev->saved_raid_disk < 0 && has_failed(conf))
1da177e4 6912 /* no point adding a device */
199050ea 6913 return -EINVAL;
1da177e4 6914
6c2fce2e
NB		 6915 if (rdev->raid_disk >= 0)
6916 first = last = rdev->raid_disk;
1da177e4
LT		 6917
6918 /*
16a53ecc
N		 6919 * find the disk ... but prefer rdev->saved_raid_disk
6920 * if possible.
1da177e4 6921 */
16a53ecc 6922 if (rdev->saved_raid_disk >= 0 &&
6c2fce2e 6923 rdev->saved_raid_disk >= first &&
16a53ecc 6924 conf->disks[rdev->saved_raid_disk].rdev == NULL)
5cfb22a1
N		 6925 first = rdev->saved_raid_disk;
6926
6927 for (disk = first; disk <= last; disk++) {
7bfec5f3
N		 6928 p = conf->disks + disk;
6929 if (p->rdev == NULL) {
b2d444d7 6930 clear_bit(In_sync, &rdev->flags);
1da177e4 6931 rdev->raid_disk = disk;
199050ea 6932 err = 0;
72626685
N		 6933 if (rdev->saved_raid_disk != disk)
6934 conf->fullsync = 1;
d6065f7b 6935 rcu_assign_pointer(p->rdev, rdev);
5cfb22a1 6936 goto out;
1da177e4 6937 }
5cfb22a1
N		 6938 }
6939 for (disk = first; disk <= last; disk++) {
6940 p = conf->disks + disk;
7bfec5f3
N		 6941 if (test_bit(WantReplacement, &p->rdev->flags) &&
6942 p->replacement == NULL) {
6943 clear_bit(In_sync, &rdev->flags);
6944 set_bit(Replacement, &rdev->flags);
6945 rdev->raid_disk = disk;
6946 err = 0;
6947 conf->fullsync = 1;
6948 rcu_assign_pointer(p->replacement, rdev);
6949 break;
6950 }
6951 }
5cfb22a1 6952out:
1da177e4 6953 print_raid5_conf(conf);
199050ea 6954 return err;
1da177e4
LT		 6955}
6956
fd01b88c 6957static int raid5_resize(struct mddev *mddev, sector_t sectors)
1da177e4
LT		 6958{
6959 /* no resync is happening, and there is enough space
6960 * on all devices, so we can resize.
6961 * We need to make sure resync covers any new space.
6962 * If the array is shrinking we should possibly wait until
6963 * any io in the removed space completes, but it hardly seems
6964 * worth it.
6965 */
a4a6125a 6966 sector_t newsize;
9d8f0363 6967 sectors &= ~((sector_t)mddev->chunk_sectors - 1);
a4a6125a
N		 6968 newsize = raid5_size(mddev, sectors, mddev->raid_disks);
6969 if (mddev->external_size &&
6970 mddev->array_sectors > newsize)
b522adcd 6971 return -EINVAL;
a4a6125a
N		 6972 if (mddev->bitmap) {
6973 int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0);
6974 if (ret)
6975 return ret;
6976 }
6977 md_set_array_sectors(mddev, newsize);
f233ea5c 6978 set_capacity(mddev->gendisk, mddev->array_sectors);
449aad3e 6979 revalidate_disk(mddev->gendisk);
b098636c
N		 6980 if (sectors > mddev->dev_sectors &&
6981 mddev->recovery_cp > mddev->dev_sectors) {
58c0fed4 6982 mddev->recovery_cp = mddev->dev_sectors;
1da177e4
LT		 6983 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
6984 }
58c0fed4 6985 mddev->dev_sectors = sectors;
4b5c7ae8 6986 mddev->resync_max_sectors = sectors;
1da177e4
LT		 6987 return 0;
6988}
6989
fd01b88c 6990static int check_stripe_cache(struct mddev *mddev)
01ee22b4
N		 6991{
6992 /* Can only proceed if there are plenty of stripe_heads.
6993 * We need a minimum of one full stripe,, and for sensible progress
6994 * it is best to have about 4 times that.
6995 * If we require 4 times, then the default 256 4K stripe_heads will
6996 * allow for chunk sizes up to 256K, which is probably OK.
6997 * If the chunk size is greater, user-space should request more
6998 * stripe_heads first.
6999 */
d1688a6d 7000 struct r5conf *conf = mddev->private;
01ee22b4
N		 7001 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
7002 > conf->max_nr_stripes ||
7003 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
7004 > conf->max_nr_stripes) {
0c55e022
N		 7005 printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n",
7006 mdname(mddev),
01ee22b4
N		 7007 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
7008 / STRIPE_SIZE)*4);
7009 return 0;
7010 }
7011 return 1;
7012}
7013
fd01b88c 7014static int check_reshape(struct mddev *mddev)
29269553 7015{
d1688a6d 7016 struct r5conf *conf = mddev->private;
29269553 7017
88ce4930
N		 7018 if (mddev->delta_disks == 0 &&
7019 mddev->new_layout == mddev->layout &&
664e7c41 7020 mddev->new_chunk_sectors == mddev->chunk_sectors)
50ac168a 7021 return 0; /* nothing to do */
674806d6 7022 if (has_failed(conf))
ec32a2bd 7023 return -EINVAL;
fdcfbbb6 7024 if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
ec32a2bd
N		 7025 /* We might be able to shrink, but the devices must
7026 * be made bigger first.
7027 * For raid6, 4 is the minimum size.
7028 * Otherwise 2 is the minimum
7029 */
7030 int min = 2;
7031 if (mddev->level == 6)
7032 min = 4;
7033 if (mddev->raid_disks + mddev->delta_disks < min)
7034 return -EINVAL;
7035 }
29269553 7036
01ee22b4 7037 if (!check_stripe_cache(mddev))
29269553 7038 return -ENOSPC;
29269553 7039
e56108d6
N		 7040 return resize_stripes(conf, (conf->previous_raid_disks
7041 + mddev->delta_disks));
63c70c4f
N		 7042}
7043
fd01b88c 7044static int raid5_start_reshape(struct mddev *mddev)
63c70c4f 7045{
d1688a6d 7046 struct r5conf *conf = mddev->private;
3cb03002 7047 struct md_rdev *rdev;
63c70c4f 7048 int spares = 0;
c04be0aa 7049 unsigned long flags;
63c70c4f 7050
f416885e 7051 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
63c70c4f
N		 7052 return -EBUSY;
7053
01ee22b4
N		 7054 if (!check_stripe_cache(mddev))
7055 return -ENOSPC;
7056
30b67645
N		 7057 if (has_failed(conf))
7058 return -EINVAL;
7059
c6563a8c 7060 rdev_for_each(rdev, mddev) {
469518a3
N		 7061 if (!test_bit(In_sync, &rdev->flags)
7062 && !test_bit(Faulty, &rdev->flags))
29269553 7063 spares++;
c6563a8c 7064 }
63c70c4f 7065
f416885e 7066 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
29269553
N		 7067 /* Not enough devices even to make a degraded array
7068 * of that size
7069 */
7070 return -EINVAL;
7071
ec32a2bd
N		 7072 /* Refuse to reduce size of the array. Any reductions in
7073 * array size must be through explicit setting of array_size
7074 * attribute.
7075 */
7076 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
7077 < mddev->array_sectors) {
0c55e022 7078 printk(KERN_ERR "md/raid:%s: array size must be reduced "
ec32a2bd
N		 7079 "before number of disks\n", mdname(mddev));
7080 return -EINVAL;
7081 }
7082
f6705578 7083 atomic_set(&conf->reshape_stripes, 0);
29269553 7084 spin_lock_irq(&conf->device_lock);
c46501b2 7085 write_seqcount_begin(&conf->gen_lock);
29269553 7086 conf->previous_raid_disks = conf->raid_disks;
63c70c4f 7087 conf->raid_disks += mddev->delta_disks;
09c9e5fa
AN		 7088 conf->prev_chunk_sectors = conf->chunk_sectors;
7089 conf->chunk_sectors = mddev->new_chunk_sectors;
88ce4930
N		 7090 conf->prev_algo = conf->algorithm;
7091 conf->algorithm = mddev->new_layout;
05616be5
N		 7092 conf->generation++;
7093 /* Code that selects data_offset needs to see the generation update
7094 * if reshape_progress has been set - so a memory barrier needed.
7095 */
7096 smp_mb();
2c810cdd 7097 if (mddev->reshape_backwards)
fef9c61f
N		 7098 conf->reshape_progress = raid5_size(mddev, 0, 0);
7099 else
7100 conf->reshape_progress = 0;
7101 conf->reshape_safe = conf->reshape_progress;
c46501b2 7102 write_seqcount_end(&conf->gen_lock);
29269553
N		 7103 spin_unlock_irq(&conf->device_lock);
7104
4d77e3ba
N		 7105 /* Now make sure any requests that proceeded on the assumption
7106 * the reshape wasn't running - like Discard or Read - have
7107 * completed.
7108 */
7109 mddev_suspend(mddev);
7110 mddev_resume(mddev);
7111
29269553
N		 7112 /* Add some new drives, as many as will fit.
7113 * We know there are enough to make the newly sized array work.
3424bf6a
N		 7114 * Don't add devices if we are reducing the number of
7115 * devices in the array. This is because it is not possible
7116 * to correctly record the "partially reconstructed" state of
7117 * such devices during the reshape and confusion could result.
29269553 7118 */
87a8dec9 7119 if (mddev->delta_disks >= 0) {
dafb20fa 7120 rdev_for_each(rdev, mddev)
87a8dec9
N		 7121 if (rdev->raid_disk < 0 &&
7122 !test_bit(Faulty, &rdev->flags)) {
7123 if (raid5_add_disk(mddev, rdev) == 0) {
87a8dec9 7124 if (rdev->raid_disk
9d4c7d87 7125 >= conf->previous_raid_disks)
87a8dec9 7126 set_bit(In_sync, &rdev->flags);
9d4c7d87 7127 else
87a8dec9 7128 rdev->recovery_offset = 0;
36fad858
NK		 7129
7130 if (sysfs_link_rdev(mddev, rdev))
87a8dec9 7131 /* Failure here is OK */;
50da0840 7132 }
87a8dec9
N		 7133 } else if (rdev->raid_disk >= conf->previous_raid_disks
7134 && !test_bit(Faulty, &rdev->flags)) {
7135 /* This is a spare that was manually added */
7136 set_bit(In_sync, &rdev->flags);
87a8dec9 7137 }
29269553 7138
87a8dec9
N		 7139 /* When a reshape changes the number of devices,
7140 * ->degraded is measured against the larger of the
7141 * pre and post number of devices.
7142 */
ec32a2bd 7143 spin_lock_irqsave(&conf->device_lock, flags);
908f4fbd 7144 mddev->degraded = calc_degraded(conf);
ec32a2bd
N		 7145 spin_unlock_irqrestore(&conf->device_lock, flags);
7146 }
63c70c4f 7147 mddev->raid_disks = conf->raid_disks;
e516402c 7148 mddev->reshape_position = conf->reshape_progress;
850b2b42 7149 set_bit(MD_CHANGE_DEVS, &mddev->flags);
f6705578 7150
29269553
N		 7151 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
7152 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
7153 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
7154 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
7155 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
0da3c619 7156 "reshape");
29269553
N		 7157 if (!mddev->sync_thread) {
7158 mddev->recovery = 0;
7159 spin_lock_irq(&conf->device_lock);
ba8805b9 7160 write_seqcount_begin(&conf->gen_lock);
29269553 7161 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
ba8805b9
N		 7162 mddev->new_chunk_sectors =
7163 conf->chunk_sectors = conf->prev_chunk_sectors;
7164 mddev->new_layout = conf->algorithm = conf->prev_algo;
05616be5
N		 7165 rdev_for_each(rdev, mddev)
7166 rdev->new_data_offset = rdev->data_offset;
7167 smp_wmb();
ba8805b9 7168 conf->generation --;
fef9c61f 7169 conf->reshape_progress = MaxSector;
1e3fa9bd 7170 mddev->reshape_position = MaxSector;
ba8805b9 7171 write_seqcount_end(&conf->gen_lock);
29269553
N		 7172 spin_unlock_irq(&conf->device_lock);
7173 return -EAGAIN;
7174 }
c8f517c4 7175 conf->reshape_checkpoint = jiffies;
29269553
N		 7176 md_wakeup_thread(mddev->sync_thread);
7177 md_new_event(mddev);
7178 return 0;
7179}
29269553 7180
ec32a2bd
N		 7181/* This is called from the reshape thread and should make any
7182 * changes needed in 'conf'
7183 */
d1688a6d 7184static void end_reshape(struct r5conf *conf)
29269553 7185{
29269553 7186
f6705578 7187 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
05616be5 7188 struct md_rdev *rdev;
f6705578 7189
f6705578 7190 spin_lock_irq(&conf->device_lock);
cea9c228 7191 conf->previous_raid_disks = conf->raid_disks;
05616be5
N		 7192 rdev_for_each(rdev, conf->mddev)
7193 rdev->data_offset = rdev->new_data_offset;
7194 smp_wmb();
fef9c61f 7195 conf->reshape_progress = MaxSector;
f6705578 7196 spin_unlock_irq(&conf->device_lock);
b0f9ec04 7197 wake_up(&conf->wait_for_overlap);
16a53ecc
N		 7198
7199 /* read-ahead size must cover two whole stripes, which is
7200 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
7201 */
4a5add49 7202 if (conf->mddev->queue) {
cea9c228 7203 int data_disks = conf->raid_disks - conf->max_degraded;
09c9e5fa 7204 int stripe = data_disks * ((conf->chunk_sectors << 9)
cea9c228 7205 / PAGE_SIZE);
16a53ecc
N		 7206 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
7207 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
7208 }
29269553 7209 }
29269553
N		 7210}
7211
ec32a2bd
N		 7212/* This is called from the raid5d thread with mddev_lock held.
7213 * It makes config changes to the device.
7214 */
fd01b88c 7215static void raid5_finish_reshape(struct mddev *mddev)
cea9c228 7216{
d1688a6d 7217 struct r5conf *conf = mddev->private;
cea9c228
N		 7218
7219 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
7220
ec32a2bd
N		 7221 if (mddev->delta_disks > 0) {
7222 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
7223 set_capacity(mddev->gendisk, mddev->array_sectors);
449aad3e 7224 revalidate_disk(mddev->gendisk);
ec32a2bd
N		 7225 } else {
7226 int d;
908f4fbd
N		 7227 spin_lock_irq(&conf->device_lock);
7228 mddev->degraded = calc_degraded(conf);
7229 spin_unlock_irq(&conf->device_lock);
ec32a2bd
N		 7230 for (d = conf->raid_disks ;
7231 d < conf->raid_disks - mddev->delta_disks;
1a67dde0 7232 d++) {
3cb03002 7233 struct md_rdev *rdev = conf->disks[d].rdev;
da7613b8
N		 7234 if (rdev)
7235 clear_bit(In_sync, &rdev->flags);
7236 rdev = conf->disks[d].replacement;
7237 if (rdev)
7238 clear_bit(In_sync, &rdev->flags);
1a67dde0 7239 }
cea9c228 7240 }
88ce4930 7241 mddev->layout = conf->algorithm;
09c9e5fa 7242 mddev->chunk_sectors = conf->chunk_sectors;
ec32a2bd
N		 7243 mddev->reshape_position = MaxSector;
7244 mddev->delta_disks = 0;
2c810cdd 7245 mddev->reshape_backwards = 0;
cea9c228
N		 7246 }
7247}
7248
fd01b88c 7249static void raid5_quiesce(struct mddev *mddev, int state)
72626685 7250{
d1688a6d 7251 struct r5conf *conf = mddev->private;
72626685
N		 7252
7253 switch(state) {
e464eafd
N		 7254 case 2: /* resume for a suspend */
7255 wake_up(&conf->wait_for_overlap);
7256 break;
7257
72626685 7258 case 1: /* stop all writes */
566c09c5 7259 lock_all_device_hash_locks_irq(conf);
64bd660b
N		 7260 /* '2' tells resync/reshape to pause so that all
7261 * active stripes can drain
7262 */
7263 conf->quiesce = 2;
566c09c5 7264 wait_event_cmd(conf->wait_for_stripe,
46031f9a
RBJ		 7265 atomic_read(&conf->active_stripes) == 0 &&
7266 atomic_read(&conf->active_aligned_reads) == 0,
566c09c5
SL		 7267 unlock_all_device_hash_locks_irq(conf),
7268 lock_all_device_hash_locks_irq(conf));
64bd660b 7269 conf->quiesce = 1;
566c09c5 7270 unlock_all_device_hash_locks_irq(conf);
64bd660b
N		 7271 /* allow reshape to continue */
7272 wake_up(&conf->wait_for_overlap);
72626685
N		 7273 break;
7274
7275 case 0: /* re-enable writes */
566c09c5 7276 lock_all_device_hash_locks_irq(conf);
72626685
N		 7277 conf->quiesce = 0;
7278 wake_up(&conf->wait_for_stripe);
e464eafd 7279 wake_up(&conf->wait_for_overlap);
566c09c5 7280 unlock_all_device_hash_locks_irq(conf);
72626685
N		 7281 break;
7282 }
72626685 7283}
b15c2e57 7284
fd01b88c 7285static void *raid45_takeover_raid0(struct mddev *mddev, int level)
54071b38 7286{
e373ab10 7287 struct r0conf *raid0_conf = mddev->private;
d76c8420 7288 sector_t sectors;
54071b38 7289
f1b29bca 7290 /* for raid0 takeover only one zone is supported */
e373ab10 7291 if (raid0_conf->nr_strip_zones > 1) {
0c55e022
N		 7292 printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
7293 mdname(mddev));
f1b29bca
DW		 7294 return ERR_PTR(-EINVAL);
7295 }
7296
e373ab10
N		 7297 sectors = raid0_conf->strip_zone[0].zone_end;
7298 sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
3b71bd93 7299 mddev->dev_sectors = sectors;
f1b29bca 7300 mddev->new_level = level;
54071b38
TM		 7301 mddev->new_layout = ALGORITHM_PARITY_N;
7302 mddev->new_chunk_sectors = mddev->chunk_sectors;
7303 mddev->raid_disks += 1;
7304 mddev->delta_disks = 1;
7305 /* make sure it will be not marked as dirty */
7306 mddev->recovery_cp = MaxSector;
7307
7308 return setup_conf(mddev);
7309}
7310
fd01b88c 7311static void *raid5_takeover_raid1(struct mddev *mddev)
d562b0c4
N		 7312{
7313 int chunksect;
7314
7315 if (mddev->raid_disks != 2 ||
7316 mddev->degraded > 1)
7317 return ERR_PTR(-EINVAL);
7318
7319 /* Should check if there are write-behind devices? */
7320
7321 chunksect = 64*2; /* 64K by default */
7322
7323 /* The array must be an exact multiple of chunksize */
7324 while (chunksect && (mddev->array_sectors & (chunksect-1)))
7325 chunksect >>= 1;
7326
7327 if ((chunksect<<9) < STRIPE_SIZE)
7328 /* array size does not allow a suitable chunk size */
7329 return ERR_PTR(-EINVAL);
7330
7331 mddev->new_level = 5;
7332 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
664e7c41 7333 mddev->new_chunk_sectors = chunksect;
d562b0c4
N		 7334
7335 return setup_conf(mddev);
7336}
7337
fd01b88c 7338static void *raid5_takeover_raid6(struct mddev *mddev)
fc9739c6
N		 7339{
7340 int new_layout;
7341
7342 switch (mddev->layout) {
7343 case ALGORITHM_LEFT_ASYMMETRIC_6:
7344 new_layout = ALGORITHM_LEFT_ASYMMETRIC;
7345 break;
7346 case ALGORITHM_RIGHT_ASYMMETRIC_6:
7347 new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
7348 break;
7349 case ALGORITHM_LEFT_SYMMETRIC_6:
7350 new_layout = ALGORITHM_LEFT_SYMMETRIC;
7351 break;
7352 case ALGORITHM_RIGHT_SYMMETRIC_6:
7353 new_layout = ALGORITHM_RIGHT_SYMMETRIC;
7354 break;
7355 case ALGORITHM_PARITY_0_6:
7356 new_layout = ALGORITHM_PARITY_0;
7357 break;
7358 case ALGORITHM_PARITY_N:
7359 new_layout = ALGORITHM_PARITY_N;
7360 break;
7361 default:
7362 return ERR_PTR(-EINVAL);
7363 }
7364 mddev->new_level = 5;
7365 mddev->new_layout = new_layout;
7366 mddev->delta_disks = -1;
7367 mddev->raid_disks -= 1;
7368 return setup_conf(mddev);
7369}
7370
fd01b88c 7371static int raid5_check_reshape(struct mddev *mddev)
b3546035 7372{
88ce4930
N		 7373 /* For a 2-drive array, the layout and chunk size can be changed
7374 * immediately as not restriping is needed.
7375 * For larger arrays we record the new value - after validation
7376 * to be used by a reshape pass.
b3546035 7377 */
d1688a6d 7378 struct r5conf *conf = mddev->private;
597a711b 7379 int new_chunk = mddev->new_chunk_sectors;
b3546035 7380
597a711b 7381 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
b3546035
N		 7382 return -EINVAL;
7383 if (new_chunk > 0) {
0ba459d2 7384 if (!is_power_of_2(new_chunk))
b3546035 7385 return -EINVAL;
597a711b 7386 if (new_chunk < (PAGE_SIZE>>9))
b3546035 7387 return -EINVAL;
597a711b 7388 if (mddev->array_sectors & (new_chunk-1))
b3546035
N		 7389 /* not factor of array size */
7390 return -EINVAL;
7391 }
7392
7393 /* They look valid */
7394
88ce4930 7395 if (mddev->raid_disks == 2) {
597a711b
N		 7396 /* can make the change immediately */
7397 if (mddev->new_layout >= 0) {
7398 conf->algorithm = mddev->new_layout;
7399 mddev->layout = mddev->new_layout;
88ce4930
N		 7400 }
7401 if (new_chunk > 0) {
597a711b
N		 7402 conf->chunk_sectors = new_chunk ;
7403 mddev->chunk_sectors = new_chunk;
88ce4930
N		 7404 }
7405 set_bit(MD_CHANGE_DEVS, &mddev->flags);
7406 md_wakeup_thread(mddev->thread);
b3546035 7407 }
50ac168a 7408 return check_reshape(mddev);
88ce4930
N		 7409}
7410
fd01b88c 7411static int raid6_check_reshape(struct mddev *mddev)
88ce4930 7412{
597a711b 7413 int new_chunk = mddev->new_chunk_sectors;
50ac168a 7414
597a711b 7415 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
88ce4930 7416 return -EINVAL;
b3546035 7417 if (new_chunk > 0) {
0ba459d2 7418 if (!is_power_of_2(new_chunk))
88ce4930 7419 return -EINVAL;
597a711b 7420 if (new_chunk < (PAGE_SIZE >> 9))
88ce4930 7421 return -EINVAL;
597a711b 7422 if (mddev->array_sectors & (new_chunk-1))
88ce4930
N		 7423 /* not factor of array size */
7424 return -EINVAL;
b3546035 7425 }
88ce4930
N		 7426
7427 /* They look valid */
50ac168a 7428 return check_reshape(mddev);
b3546035
N		 7429}
7430
fd01b88c 7431static void *raid5_takeover(struct mddev *mddev)
d562b0c4
N		 7432{
7433 /* raid5 can take over:
f1b29bca 7434 * raid0 - if there is only one strip zone - make it a raid4 layout
d562b0c4
N		 7435 * raid1 - if there are two drives. We need to know the chunk size
7436 * raid4 - trivial - just use a raid4 layout.
7437 * raid6 - Providing it is a *_6 layout
d562b0c4 7438 */
f1b29bca
DW		 7439 if (mddev->level == 0)
7440 return raid45_takeover_raid0(mddev, 5);
d562b0c4
N		 7441 if (mddev->level == 1)
7442 return raid5_takeover_raid1(mddev);
e9d4758f
N		 7443 if (mddev->level == 4) {
7444 mddev->new_layout = ALGORITHM_PARITY_N;
7445 mddev->new_level = 5;
7446 return setup_conf(mddev);
7447 }
fc9739c6
N		 7448 if (mddev->level == 6)
7449 return raid5_takeover_raid6(mddev);
d562b0c4
N		 7450
7451 return ERR_PTR(-EINVAL);
7452}
7453
fd01b88c 7454static void *raid4_takeover(struct mddev *mddev)
a78d38a1 7455{
f1b29bca
DW		 7456 /* raid4 can take over:
7457 * raid0 - if there is only one strip zone
7458 * raid5 - if layout is right
a78d38a1 7459 */
f1b29bca
DW		 7460 if (mddev->level == 0)
7461 return raid45_takeover_raid0(mddev, 4);
a78d38a1
N		 7462 if (mddev->level == 5 &&
7463 mddev->layout == ALGORITHM_PARITY_N) {
7464 mddev->new_layout = 0;
7465 mddev->new_level = 4;
7466 return setup_conf(mddev);
7467 }
7468 return ERR_PTR(-EINVAL);
7469}
d562b0c4 7470
84fc4b56 7471static struct md_personality raid5_personality;
245f46c2 7472
fd01b88c 7473static void *raid6_takeover(struct mddev *mddev)
245f46c2
N		 7474{
7475 /* Currently can only take over a raid5. We map the
7476 * personality to an equivalent raid6 personality
7477 * with the Q block at the end.
7478 */
7479 int new_layout;
7480
7481 if (mddev->pers != &raid5_personality)
7482 return ERR_PTR(-EINVAL);
7483 if (mddev->degraded > 1)
7484 return ERR_PTR(-EINVAL);
7485 if (mddev->raid_disks > 253)
7486 return ERR_PTR(-EINVAL);
7487 if (mddev->raid_disks < 3)
7488 return ERR_PTR(-EINVAL);
7489
7490 switch (mddev->layout) {
7491 case ALGORITHM_LEFT_ASYMMETRIC:
7492 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
7493 break;
7494 case ALGORITHM_RIGHT_ASYMMETRIC:
7495 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
7496 break;
7497 case ALGORITHM_LEFT_SYMMETRIC:
7498 new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
7499 break;
7500 case ALGORITHM_RIGHT_SYMMETRIC:
7501 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
7502 break;
7503 case ALGORITHM_PARITY_0:
7504 new_layout = ALGORITHM_PARITY_0_6;
7505 break;
7506 case ALGORITHM_PARITY_N:
7507 new_layout = ALGORITHM_PARITY_N;
7508 break;
7509 default:
7510 return ERR_PTR(-EINVAL);
7511 }
7512 mddev->new_level = 6;
7513 mddev->new_layout = new_layout;
7514 mddev->delta_disks = 1;
7515 mddev->raid_disks += 1;
7516 return setup_conf(mddev);
7517}
7518
84fc4b56 7519static struct md_personality raid6_personality =
16a53ecc
N		 7520{
7521 .name = "raid6",
7522 .level = 6,
7523 .owner = THIS_MODULE,
7524 .make_request = make_request,
7525 .run = run,
afa0f557 7526 .free = raid5_free,
16a53ecc
N		 7527 .status = status,
7528 .error_handler = error,
7529 .hot_add_disk = raid5_add_disk,
7530 .hot_remove_disk= raid5_remove_disk,
7531 .spare_active = raid5_spare_active,
7532 .sync_request = sync_request,
7533 .resize = raid5_resize,
80c3a6ce 7534 .size = raid5_size,
50ac168a 7535 .check_reshape = raid6_check_reshape,
f416885e 7536 .start_reshape = raid5_start_reshape,
cea9c228 7537 .finish_reshape = raid5_finish_reshape,
16a53ecc 7538 .quiesce = raid5_quiesce,
245f46c2 7539 .takeover = raid6_takeover,
5c675f83 7540 .congested = raid5_congested,
64590f45 7541 .mergeable_bvec = raid5_mergeable_bvec,
16a53ecc 7542};
84fc4b56 7543static struct md_personality raid5_personality =
1da177e4
LT		 7544{
7545 .name = "raid5",
2604b703 7546 .level = 5,
1da177e4
LT		 7547 .owner = THIS_MODULE,
7548 .make_request = make_request,
7549 .run = run,
afa0f557 7550 .free = raid5_free,
1da177e4
LT		 7551 .status = status,
7552 .error_handler = error,
7553 .hot_add_disk = raid5_add_disk,
7554 .hot_remove_disk= raid5_remove_disk,
7555 .spare_active = raid5_spare_active,
7556 .sync_request = sync_request,
7557 .resize = raid5_resize,
80c3a6ce 7558 .size = raid5_size,
63c70c4f
N		 7559 .check_reshape = raid5_check_reshape,
7560 .start_reshape = raid5_start_reshape,
cea9c228 7561 .finish_reshape = raid5_finish_reshape,
72626685 7562 .quiesce = raid5_quiesce,
d562b0c4 7563 .takeover = raid5_takeover,
5c675f83 7564 .congested = raid5_congested,
64590f45 7565 .mergeable_bvec = raid5_mergeable_bvec,
1da177e4
LT		 7566};
7567
84fc4b56 7568static struct md_personality raid4_personality =
1da177e4 7569{
2604b703
N		 7570 .name = "raid4",
7571 .level = 4,
7572 .owner = THIS_MODULE,
7573 .make_request = make_request,
7574 .run = run,
afa0f557 7575 .free = raid5_free,
2604b703
N		 7576 .status = status,
7577 .error_handler = error,
7578 .hot_add_disk = raid5_add_disk,
7579 .hot_remove_disk= raid5_remove_disk,
7580 .spare_active = raid5_spare_active,
7581 .sync_request = sync_request,
7582 .resize = raid5_resize,
80c3a6ce 7583 .size = raid5_size,
3d37890b
N		 7584 .check_reshape = raid5_check_reshape,
7585 .start_reshape = raid5_start_reshape,
cea9c228 7586 .finish_reshape = raid5_finish_reshape,
2604b703 7587 .quiesce = raid5_quiesce,
a78d38a1 7588 .takeover = raid4_takeover,
5c675f83 7589 .congested = raid5_congested,
64590f45 7590 .mergeable_bvec = raid5_mergeable_bvec,
2604b703
N		 7591};
7592
7593static int __init raid5_init(void)
7594{
851c30c9
SL		 7595 raid5_wq = alloc_workqueue("raid5wq",
7596 WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
7597 if (!raid5_wq)
7598 return -ENOMEM;
16a53ecc 7599 register_md_personality(&raid6_personality);
2604b703
N		 7600 register_md_personality(&raid5_personality);
7601 register_md_personality(&raid4_personality);
7602 return 0;
1da177e4
LT		 7603}
7604
2604b703 7605static void raid5_exit(void)
1da177e4 7606{
16a53ecc 7607 unregister_md_personality(&raid6_personality);
2604b703
N		 7608 unregister_md_personality(&raid5_personality);
7609 unregister_md_personality(&raid4_personality);
851c30c9 7610 destroy_workqueue(raid5_wq);
1da177e4
LT		 7611}
7612
7613module_init(raid5_init);
7614module_exit(raid5_exit);
7615MODULE_LICENSE("GPL");
0efb9e61 7616MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
1da177e4 7617MODULE_ALIAS("md-personality-4"); /* RAID5 */
d9d166c2
N		 7618MODULE_ALIAS("md-raid5");
7619MODULE_ALIAS("md-raid4");
2604b703
N		 7620MODULE_ALIAS("md-level-5");
7621MODULE_ALIAS("md-level-4");
16a53ecc
N		 7622MODULE_ALIAS("md-personality-8"); /* RAID6 */
7623MODULE_ALIAS("md-raid6");
7624MODULE_ALIAS("md-level-6");
7625
7626/* This used to be two separate modules, they were: */
7627MODULE_ALIAS("raid5");
7628MODULE_ALIAS("raid6");
Linux 6.x block layer and io_uring tree(s)