1 // SPDX-License-Identifier: GPL-2.0-only
3 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
5 * bitmap_create - sets up the bitmap structure
6 * bitmap_destroy - destroys the bitmap structure
8 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
9 * - added disk storage for bitmap
10 * - changes to allow various bitmap chunk sizes
16 * flush after percent set rather than just time based. (maybe both).
19 #include <linux/blkdev.h>
20 #include <linux/module.h>
21 #include <linux/errno.h>
22 #include <linux/slab.h>
23 #include <linux/init.h>
24 #include <linux/timer.h>
25 #include <linux/sched.h>
26 #include <linux/list.h>
27 #include <linux/file.h>
28 #include <linux/mount.h>
29 #include <linux/buffer_head.h>
30 #include <linux/seq_file.h>
31 #include <trace/events/block.h>
33 #include "md-bitmap.h"
35 static inline char *bmname(struct bitmap *bitmap)
37 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
41 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
43 * 1) check to see if this page is allocated, if it's not then try to alloc
44 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
45 * page pointer directly as a counter
47 * if we find our page, we increment the page's refcount so that it stays
48 * allocated while we're using it
50 static int md_bitmap_checkpage(struct bitmap_counts *bitmap,
51 unsigned long page, int create, int no_hijack)
52 __releases(bitmap->lock)
53 __acquires(bitmap->lock)
55 unsigned char *mappage;
57 WARN_ON_ONCE(page >= bitmap->pages);
58 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
61 if (bitmap->bp[page].map) /* page is already allocated, just return */
67 /* this page has not been allocated yet */
69 spin_unlock_irq(&bitmap->lock);
70 /* It is possible that this is being called inside a
71 * prepare_to_wait/finish_wait loop from raid5c:make_request().
72 * In general it is not permitted to sleep in that context as it
73 * can cause the loop to spin freely.
74 * That doesn't apply here as we can only reach this point
76 * When this function completes, either bp[page].map or
77 * bp[page].hijacked. In either case, this function will
78 * abort before getting to this point again. So there is
79 * no risk of a free-spin, and so it is safe to assert
80 * that sleeping here is allowed.
82 sched_annotate_sleep();
83 mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
84 spin_lock_irq(&bitmap->lock);
86 if (mappage == NULL) {
87 pr_debug("md/bitmap: map page allocation failed, hijacking\n");
88 /* We don't support hijack for cluster raid */
91 /* failed - set the hijacked flag so that we can use the
92 * pointer as a counter */
93 if (!bitmap->bp[page].map)
94 bitmap->bp[page].hijacked = 1;
95 } else if (bitmap->bp[page].map ||
96 bitmap->bp[page].hijacked) {
97 /* somebody beat us to getting the page */
101 /* no page was in place and we have one, so install it */
103 bitmap->bp[page].map = mappage;
104 bitmap->missing_pages--;
109 /* if page is completely empty, put it back on the free list, or dealloc it */
110 /* if page was hijacked, unmark the flag so it might get alloced next time */
111 /* Note: lock should be held when calling this */
112 static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
116 if (bitmap->bp[page].count) /* page is still busy */
119 /* page is no longer in use, it can be released */
121 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
122 bitmap->bp[page].hijacked = 0;
123 bitmap->bp[page].map = NULL;
125 /* normal case, free the page */
126 ptr = bitmap->bp[page].map;
127 bitmap->bp[page].map = NULL;
128 bitmap->missing_pages++;
134 * bitmap file handling - read and write the bitmap file and its superblock
138 * basic page I/O operations
141 /* IO operations when bitmap is stored near all superblocks */
143 /* choose a good rdev and read the page from there */
144 static int read_sb_page(struct mddev *mddev, loff_t offset,
145 struct page *page, unsigned long index, int size)
148 sector_t sector = mddev->bitmap_info.offset + offset +
149 index * (PAGE_SIZE / SECTOR_SIZE);
150 struct md_rdev *rdev;
152 rdev_for_each(rdev, mddev) {
153 u32 iosize = roundup(size, bdev_logical_block_size(rdev->bdev));
155 if (!test_bit(In_sync, &rdev->flags) ||
156 test_bit(Faulty, &rdev->flags) ||
157 test_bit(Bitmap_sync, &rdev->flags))
160 if (sync_page_io(rdev, sector, iosize, page, REQ_OP_READ, true))
166 static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
168 /* Iterate the disks of an mddev, using rcu to protect access to the
169 * linked list, and raising the refcount of devices we return to ensure
170 * they don't disappear while in use.
171 * As devices are only added or removed when raid_disk is < 0 and
172 * nr_pending is 0 and In_sync is clear, the entries we return will
173 * still be in the same position on the list when we re-enter
174 * list_for_each_entry_continue_rcu.
176 * Note that if entered with 'rdev == NULL' to start at the
177 * beginning, we temporarily assign 'rdev' to an address which
178 * isn't really an rdev, but which can be used by
179 * list_for_each_entry_continue_rcu() to find the first entry.
183 /* start at the beginning */
184 rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
186 /* release the previous rdev and start from there. */
187 rdev_dec_pending(rdev, mddev);
189 list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
190 if (rdev->raid_disk >= 0 &&
191 !test_bit(Faulty, &rdev->flags)) {
192 /* this is a usable devices */
193 atomic_inc(&rdev->nr_pending);
202 static unsigned int optimal_io_size(struct block_device *bdev,
203 unsigned int last_page_size,
204 unsigned int io_size)
206 if (bdev_io_opt(bdev) > bdev_logical_block_size(bdev))
207 return roundup(last_page_size, bdev_io_opt(bdev));
211 static unsigned int bitmap_io_size(unsigned int io_size, unsigned int opt_size,
212 loff_t start, loff_t boundary)
214 if (io_size != opt_size &&
215 start + opt_size / SECTOR_SIZE <= boundary)
217 if (start + io_size / SECTOR_SIZE <= boundary)
220 /* Overflows boundary */
224 static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap,
225 unsigned long pg_index, struct page *page)
227 struct block_device *bdev;
228 struct mddev *mddev = bitmap->mddev;
229 struct bitmap_storage *store = &bitmap->storage;
230 loff_t sboff, offset = mddev->bitmap_info.offset;
231 sector_t ps = pg_index * PAGE_SIZE / SECTOR_SIZE;
232 unsigned int size = PAGE_SIZE;
233 unsigned int opt_size = PAGE_SIZE;
236 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
237 /* we compare length (page numbers), not page offset. */
238 if ((pg_index - store->sb_index) == store->file_pages - 1) {
239 unsigned int last_page_size = store->bytes & (PAGE_SIZE - 1);
241 if (last_page_size == 0)
242 last_page_size = PAGE_SIZE;
243 size = roundup(last_page_size, bdev_logical_block_size(bdev));
244 opt_size = optimal_io_size(bdev, last_page_size, size);
247 sboff = rdev->sb_start + offset;
248 doff = rdev->data_offset;
250 /* Just make sure we aren't corrupting data or metadata */
251 if (mddev->external) {
252 /* Bitmap could be anywhere. */
253 if (sboff + ps > doff &&
254 sboff < (doff + mddev->dev_sectors + PAGE_SIZE / SECTOR_SIZE))
256 } else if (offset < 0) {
257 /* DATA BITMAP METADATA */
258 size = bitmap_io_size(size, opt_size, offset + ps, 0);
260 /* bitmap runs in to metadata */
263 if (doff + mddev->dev_sectors > sboff)
264 /* data runs in to bitmap */
266 } else if (rdev->sb_start < rdev->data_offset) {
267 /* METADATA BITMAP DATA */
268 size = bitmap_io_size(size, opt_size, sboff + ps, doff);
270 /* bitmap runs in to data */
273 /* DATA METADATA BITMAP - no problems */
276 md_super_write(mddev, rdev, sboff + ps, (int) size, page);
280 static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index,
281 struct page *page, bool wait)
283 struct mddev *mddev = bitmap->mddev;
286 struct md_rdev *rdev = NULL;
288 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
289 if (__write_sb_page(rdev, bitmap, pg_index, page) < 0) {
290 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
294 } while (wait && md_super_wait(mddev) < 0);
297 static void md_bitmap_file_kick(struct bitmap *bitmap);
299 #ifdef CONFIG_MD_BITMAP_FILE
300 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
302 struct buffer_head *bh = page_buffers(page);
304 while (bh && bh->b_blocknr) {
305 atomic_inc(&bitmap->pending_writes);
306 set_buffer_locked(bh);
307 set_buffer_mapped(bh);
308 submit_bh(REQ_OP_WRITE | REQ_SYNC, bh);
309 bh = bh->b_this_page;
313 wait_event(bitmap->write_wait,
314 atomic_read(&bitmap->pending_writes) == 0);
317 static void end_bitmap_write(struct buffer_head *bh, int uptodate)
319 struct bitmap *bitmap = bh->b_private;
322 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
323 if (atomic_dec_and_test(&bitmap->pending_writes))
324 wake_up(&bitmap->write_wait);
327 static void free_buffers(struct page *page)
329 struct buffer_head *bh;
331 if (!PagePrivate(page))
334 bh = page_buffers(page);
336 struct buffer_head *next = bh->b_this_page;
337 free_buffer_head(bh);
340 detach_page_private(page);
344 /* read a page from a file.
345 * We both read the page, and attach buffers to the page to record the
346 * address of each block (using bmap). These addresses will be used
347 * to write the block later, completely bypassing the filesystem.
348 * This usage is similar to how swap files are handled, and allows us
349 * to write to a file with no concerns of memory allocation failing.
351 static int read_file_page(struct file *file, unsigned long index,
352 struct bitmap *bitmap, unsigned long count, struct page *page)
355 struct inode *inode = file_inode(file);
356 struct buffer_head *bh;
357 sector_t block, blk_cur;
358 unsigned long blocksize = i_blocksize(inode);
360 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
361 (unsigned long long)index << PAGE_SHIFT);
363 bh = alloc_page_buffers(page, blocksize, false);
368 attach_page_private(page, bh);
369 blk_cur = index << (PAGE_SHIFT - inode->i_blkbits);
376 ret = bmap(inode, &block);
383 bh->b_blocknr = block;
384 bh->b_bdev = inode->i_sb->s_bdev;
385 if (count < blocksize)
390 bh->b_end_io = end_bitmap_write;
391 bh->b_private = bitmap;
392 atomic_inc(&bitmap->pending_writes);
393 set_buffer_locked(bh);
394 set_buffer_mapped(bh);
395 submit_bh(REQ_OP_READ, bh);
398 bh = bh->b_this_page;
401 wait_event(bitmap->write_wait,
402 atomic_read(&bitmap->pending_writes)==0);
403 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
407 pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
409 (unsigned long long)index << PAGE_SHIFT,
413 #else /* CONFIG_MD_BITMAP_FILE */
414 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
417 static int read_file_page(struct file *file, unsigned long index,
418 struct bitmap *bitmap, unsigned long count, struct page *page)
422 static void free_buffers(struct page *page)
426 #endif /* CONFIG_MD_BITMAP_FILE */
429 * bitmap file superblock operations
433 * write out a page to a file
435 static void filemap_write_page(struct bitmap *bitmap, unsigned long pg_index,
438 struct bitmap_storage *store = &bitmap->storage;
439 struct page *page = store->filemap[pg_index];
441 if (mddev_is_clustered(bitmap->mddev)) {
442 /* go to node bitmap area starting point */
443 pg_index += store->sb_index;
447 write_file_page(bitmap, page, wait);
449 write_sb_page(bitmap, pg_index, page, wait);
453 * md_bitmap_wait_writes() should be called before writing any bitmap
454 * blocks, to ensure previous writes, particularly from
455 * md_bitmap_daemon_work(), have completed.
457 static void md_bitmap_wait_writes(struct bitmap *bitmap)
459 if (bitmap->storage.file)
460 wait_event(bitmap->write_wait,
461 atomic_read(&bitmap->pending_writes)==0);
463 /* Note that we ignore the return value. The writes
464 * might have failed, but that would just mean that
465 * some bits which should be cleared haven't been,
466 * which is safe. The relevant bitmap blocks will
467 * probably get written again, but there is no great
468 * loss if they aren't.
470 md_super_wait(bitmap->mddev);
474 /* update the event counter and sync the superblock to disk */
475 void md_bitmap_update_sb(struct bitmap *bitmap)
479 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
481 if (bitmap->mddev->bitmap_info.external)
483 if (!bitmap->storage.sb_page) /* no superblock */
485 sb = kmap_atomic(bitmap->storage.sb_page);
486 sb->events = cpu_to_le64(bitmap->mddev->events);
487 if (bitmap->mddev->events < bitmap->events_cleared)
488 /* rocking back to read-only */
489 bitmap->events_cleared = bitmap->mddev->events;
490 sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
492 * clear BITMAP_WRITE_ERROR bit to protect against the case that
493 * a bitmap write error occurred but the later writes succeeded.
495 sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR));
496 /* Just in case these have been changed via sysfs: */
497 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
498 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
499 /* This might have been changed by a reshape */
500 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
501 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
502 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
503 sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
507 if (bitmap->storage.file)
508 write_file_page(bitmap, bitmap->storage.sb_page, 1);
510 write_sb_page(bitmap, bitmap->storage.sb_index,
511 bitmap->storage.sb_page, 1);
513 EXPORT_SYMBOL(md_bitmap_update_sb);
515 /* print out the bitmap file superblock */
516 void md_bitmap_print_sb(struct bitmap *bitmap)
520 if (!bitmap || !bitmap->storage.sb_page)
522 sb = kmap_atomic(bitmap->storage.sb_page);
523 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
524 pr_debug(" magic: %08x\n", le32_to_cpu(sb->magic));
525 pr_debug(" version: %u\n", le32_to_cpu(sb->version));
526 pr_debug(" uuid: %08x.%08x.%08x.%08x\n",
527 le32_to_cpu(*(__le32 *)(sb->uuid+0)),
528 le32_to_cpu(*(__le32 *)(sb->uuid+4)),
529 le32_to_cpu(*(__le32 *)(sb->uuid+8)),
530 le32_to_cpu(*(__le32 *)(sb->uuid+12)));
531 pr_debug(" events: %llu\n",
532 (unsigned long long) le64_to_cpu(sb->events));
533 pr_debug("events cleared: %llu\n",
534 (unsigned long long) le64_to_cpu(sb->events_cleared));
535 pr_debug(" state: %08x\n", le32_to_cpu(sb->state));
536 pr_debug(" chunksize: %u B\n", le32_to_cpu(sb->chunksize));
537 pr_debug(" daemon sleep: %us\n", le32_to_cpu(sb->daemon_sleep));
538 pr_debug(" sync size: %llu KB\n",
539 (unsigned long long)le64_to_cpu(sb->sync_size)/2);
540 pr_debug("max write behind: %u\n", le32_to_cpu(sb->write_behind));
548 * This function is somewhat the reverse of bitmap_read_sb. bitmap_read_sb
549 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
550 * This function verifies 'bitmap_info' and populates the on-disk bitmap
551 * structure, which is to be written to disk.
553 * Returns: 0 on success, -Exxx on error
555 static int md_bitmap_new_disk_sb(struct bitmap *bitmap)
558 unsigned long chunksize, daemon_sleep, write_behind;
560 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
561 if (bitmap->storage.sb_page == NULL)
563 bitmap->storage.sb_index = 0;
565 sb = kmap_atomic(bitmap->storage.sb_page);
567 sb->magic = cpu_to_le32(BITMAP_MAGIC);
568 sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
570 chunksize = bitmap->mddev->bitmap_info.chunksize;
572 if (!is_power_of_2(chunksize)) {
574 pr_warn("bitmap chunksize not a power of 2\n");
577 sb->chunksize = cpu_to_le32(chunksize);
579 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
580 if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
581 pr_debug("Choosing daemon_sleep default (5 sec)\n");
582 daemon_sleep = 5 * HZ;
584 sb->daemon_sleep = cpu_to_le32(daemon_sleep);
585 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
588 * FIXME: write_behind for RAID1. If not specified, what
589 * is a good choice? We choose COUNTER_MAX / 2 arbitrarily.
591 write_behind = bitmap->mddev->bitmap_info.max_write_behind;
592 if (write_behind > COUNTER_MAX)
593 write_behind = COUNTER_MAX / 2;
594 sb->write_behind = cpu_to_le32(write_behind);
595 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
597 /* keep the array size field of the bitmap superblock up to date */
598 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
600 memcpy(sb->uuid, bitmap->mddev->uuid, 16);
602 set_bit(BITMAP_STALE, &bitmap->flags);
603 sb->state = cpu_to_le32(bitmap->flags);
604 bitmap->events_cleared = bitmap->mddev->events;
605 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
606 bitmap->mddev->bitmap_info.nodes = 0;
613 /* read the superblock from the bitmap file and initialize some bitmap fields */
614 static int md_bitmap_read_sb(struct bitmap *bitmap)
618 unsigned long chunksize, daemon_sleep, write_behind;
619 unsigned long long events;
621 unsigned long sectors_reserved = 0;
623 struct page *sb_page;
626 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
627 chunksize = 128 * 1024 * 1024;
628 daemon_sleep = 5 * HZ;
630 set_bit(BITMAP_STALE, &bitmap->flags);
634 /* page 0 is the superblock, read it... */
635 sb_page = alloc_page(GFP_KERNEL);
638 bitmap->storage.sb_page = sb_page;
641 /* If cluster_slot is set, the cluster is setup */
642 if (bitmap->cluster_slot >= 0) {
643 sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
645 bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks,
646 (bitmap->mddev->bitmap_info.chunksize >> 9));
648 bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
650 bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
651 offset = bitmap->cluster_slot * (bm_blocks << 3);
652 pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
653 bitmap->cluster_slot, offset);
656 if (bitmap->storage.file) {
657 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
658 int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
660 err = read_file_page(bitmap->storage.file, 0,
661 bitmap, bytes, sb_page);
663 err = read_sb_page(bitmap->mddev, offset, sb_page, 0,
664 sizeof(bitmap_super_t));
670 sb = kmap_atomic(sb_page);
672 chunksize = le32_to_cpu(sb->chunksize);
673 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
674 write_behind = le32_to_cpu(sb->write_behind);
675 sectors_reserved = le32_to_cpu(sb->sectors_reserved);
677 /* verify that the bitmap-specific fields are valid */
678 if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
679 reason = "bad magic";
680 else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
681 le32_to_cpu(sb->version) > BITMAP_MAJOR_CLUSTERED)
682 reason = "unrecognized superblock version";
683 else if (chunksize < 512)
684 reason = "bitmap chunksize too small";
685 else if (!is_power_of_2(chunksize))
686 reason = "bitmap chunksize not a power of 2";
687 else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
688 reason = "daemon sleep period out of range";
689 else if (write_behind > COUNTER_MAX)
690 reason = "write-behind limit out of range (0 - 16383)";
692 pr_warn("%s: invalid bitmap file superblock: %s\n",
693 bmname(bitmap), reason);
698 * Setup nodes/clustername only if bitmap version is
701 if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) {
702 nodes = le32_to_cpu(sb->nodes);
703 strscpy(bitmap->mddev->bitmap_info.cluster_name,
704 sb->cluster_name, 64);
707 /* keep the array size field of the bitmap superblock up to date */
708 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
710 if (bitmap->mddev->persistent) {
712 * We have a persistent array superblock, so compare the
713 * bitmap's UUID and event counter to the mddev's
715 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
716 pr_warn("%s: bitmap superblock UUID mismatch\n",
720 events = le64_to_cpu(sb->events);
721 if (!nodes && (events < bitmap->mddev->events)) {
722 pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
723 bmname(bitmap), events,
724 (unsigned long long) bitmap->mddev->events);
725 set_bit(BITMAP_STALE, &bitmap->flags);
729 /* assign fields using values from superblock */
730 bitmap->flags |= le32_to_cpu(sb->state);
731 if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
732 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
733 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
738 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
739 /* Assigning chunksize is required for "re_read" */
740 bitmap->mddev->bitmap_info.chunksize = chunksize;
741 err = md_setup_cluster(bitmap->mddev, nodes);
743 pr_warn("%s: Could not setup cluster service (%d)\n",
744 bmname(bitmap), err);
747 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
753 if (test_bit(BITMAP_STALE, &bitmap->flags))
754 bitmap->events_cleared = bitmap->mddev->events;
755 bitmap->mddev->bitmap_info.chunksize = chunksize;
756 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
757 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
758 bitmap->mddev->bitmap_info.nodes = nodes;
759 if (bitmap->mddev->bitmap_info.space == 0 ||
760 bitmap->mddev->bitmap_info.space > sectors_reserved)
761 bitmap->mddev->bitmap_info.space = sectors_reserved;
763 md_bitmap_print_sb(bitmap);
764 if (bitmap->cluster_slot < 0)
765 md_cluster_stop(bitmap->mddev);
771 * general bitmap file operations
777 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
778 * file a page at a time. There's a superblock at the start of the file.
780 /* calculate the index of the page that contains this bit */
781 static inline unsigned long file_page_index(struct bitmap_storage *store,
785 chunk += sizeof(bitmap_super_t) << 3;
786 return chunk >> PAGE_BIT_SHIFT;
789 /* calculate the (bit) offset of this bit within a page */
790 static inline unsigned long file_page_offset(struct bitmap_storage *store,
794 chunk += sizeof(bitmap_super_t) << 3;
795 return chunk & (PAGE_BITS - 1);
799 * return a pointer to the page in the filemap that contains the given bit
802 static inline struct page *filemap_get_page(struct bitmap_storage *store,
805 if (file_page_index(store, chunk) >= store->file_pages)
807 return store->filemap[file_page_index(store, chunk)];
810 static int md_bitmap_storage_alloc(struct bitmap_storage *store,
811 unsigned long chunks, int with_super,
814 int pnum, offset = 0;
815 unsigned long num_pages;
818 bytes = DIV_ROUND_UP(chunks, 8);
820 bytes += sizeof(bitmap_super_t);
822 num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
823 offset = slot_number * num_pages;
825 store->filemap = kmalloc_array(num_pages, sizeof(struct page *),
830 if (with_super && !store->sb_page) {
831 store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
832 if (store->sb_page == NULL)
837 if (store->sb_page) {
838 store->filemap[0] = store->sb_page;
840 store->sb_index = offset;
843 for ( ; pnum < num_pages; pnum++) {
844 store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
845 if (!store->filemap[pnum]) {
846 store->file_pages = pnum;
850 store->file_pages = pnum;
852 /* We need 4 bits per page, rounded up to a multiple
853 * of sizeof(unsigned long) */
854 store->filemap_attr = kzalloc(
855 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
857 if (!store->filemap_attr)
860 store->bytes = bytes;
865 static void md_bitmap_file_unmap(struct bitmap_storage *store)
867 struct file *file = store->file;
868 struct page *sb_page = store->sb_page;
869 struct page **map = store->filemap;
870 int pages = store->file_pages;
873 if (map[pages] != sb_page) /* 0 is sb_page, release it below */
874 free_buffers(map[pages]);
876 kfree(store->filemap_attr);
879 free_buffers(sb_page);
882 struct inode *inode = file_inode(file);
883 invalidate_mapping_pages(inode->i_mapping, 0, -1);
889 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
890 * then it is no longer reliable, so we stop using it and we mark the file
891 * as failed in the superblock
893 static void md_bitmap_file_kick(struct bitmap *bitmap)
895 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
896 md_bitmap_update_sb(bitmap);
898 if (bitmap->storage.file) {
899 pr_warn("%s: kicking failed bitmap file %pD4 from array!\n",
900 bmname(bitmap), bitmap->storage.file);
903 pr_warn("%s: disabling internal bitmap due to errors\n",
908 enum bitmap_page_attr {
909 BITMAP_PAGE_DIRTY = 0, /* there are set bits that need to be synced */
910 BITMAP_PAGE_PENDING = 1, /* there are bits that are being cleaned.
911 * i.e. counter is 1 or 2. */
912 BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
915 static inline void set_page_attr(struct bitmap *bitmap, int pnum,
916 enum bitmap_page_attr attr)
918 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
921 static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
922 enum bitmap_page_attr attr)
924 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
927 static inline int test_page_attr(struct bitmap *bitmap, int pnum,
928 enum bitmap_page_attr attr)
930 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
933 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
934 enum bitmap_page_attr attr)
936 return test_and_clear_bit((pnum<<2) + attr,
937 bitmap->storage.filemap_attr);
940 * bitmap_file_set_bit -- called before performing a write to the md device
941 * to set (and eventually sync) a particular bit in the bitmap file
943 * we set the bit immediately, then we record the page number so that
944 * when an unplug occurs, we can flush the dirty pages out to disk
946 static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
951 unsigned long chunk = block >> bitmap->counts.chunkshift;
952 struct bitmap_storage *store = &bitmap->storage;
953 unsigned long index = file_page_index(store, chunk);
954 unsigned long node_offset = 0;
956 index += store->sb_index;
957 if (mddev_is_clustered(bitmap->mddev))
958 node_offset = bitmap->cluster_slot * store->file_pages;
960 page = filemap_get_page(&bitmap->storage, chunk);
963 bit = file_page_offset(&bitmap->storage, chunk);
966 kaddr = kmap_atomic(page);
967 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
970 set_bit_le(bit, kaddr);
971 kunmap_atomic(kaddr);
972 pr_debug("set file bit %lu page %lu\n", bit, index);
973 /* record page number so it gets flushed to disk when unplug occurs */
974 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_DIRTY);
977 static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
982 unsigned long chunk = block >> bitmap->counts.chunkshift;
983 struct bitmap_storage *store = &bitmap->storage;
984 unsigned long index = file_page_index(store, chunk);
985 unsigned long node_offset = 0;
987 index += store->sb_index;
988 if (mddev_is_clustered(bitmap->mddev))
989 node_offset = bitmap->cluster_slot * store->file_pages;
991 page = filemap_get_page(&bitmap->storage, chunk);
994 bit = file_page_offset(&bitmap->storage, chunk);
995 paddr = kmap_atomic(page);
996 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
997 clear_bit(bit, paddr);
999 clear_bit_le(bit, paddr);
1000 kunmap_atomic(paddr);
1001 if (!test_page_attr(bitmap, index - node_offset, BITMAP_PAGE_NEEDWRITE)) {
1002 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_PENDING);
1003 bitmap->allclean = 0;
1007 static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
1012 unsigned long chunk = block >> bitmap->counts.chunkshift;
1015 page = filemap_get_page(&bitmap->storage, chunk);
1018 bit = file_page_offset(&bitmap->storage, chunk);
1019 paddr = kmap_atomic(page);
1020 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1021 set = test_bit(bit, paddr);
1023 set = test_bit_le(bit, paddr);
1024 kunmap_atomic(paddr);
1028 /* this gets called when the md device is ready to unplug its underlying
1029 * (slave) device queues -- before we let any writes go down, we need to
1030 * sync the dirty pages of the bitmap file to disk */
1031 void md_bitmap_unplug(struct bitmap *bitmap)
1034 int dirty, need_write;
1037 if (!md_bitmap_enabled(bitmap))
1040 /* look at each page to see if there are any set bits that need to be
1041 * flushed out to disk */
1042 for (i = 0; i < bitmap->storage.file_pages; i++) {
1043 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1044 need_write = test_and_clear_page_attr(bitmap, i,
1045 BITMAP_PAGE_NEEDWRITE);
1046 if (dirty || need_write) {
1048 md_bitmap_wait_writes(bitmap);
1049 mddev_add_trace_msg(bitmap->mddev,
1050 "md bitmap_unplug");
1052 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
1053 filemap_write_page(bitmap, i, false);
1058 md_bitmap_wait_writes(bitmap);
1060 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1061 md_bitmap_file_kick(bitmap);
1063 EXPORT_SYMBOL(md_bitmap_unplug);
1065 struct bitmap_unplug_work {
1066 struct work_struct work;
1067 struct bitmap *bitmap;
1068 struct completion *done;
1071 static void md_bitmap_unplug_fn(struct work_struct *work)
1073 struct bitmap_unplug_work *unplug_work =
1074 container_of(work, struct bitmap_unplug_work, work);
1076 md_bitmap_unplug(unplug_work->bitmap);
1077 complete(unplug_work->done);
1080 void md_bitmap_unplug_async(struct bitmap *bitmap)
1082 DECLARE_COMPLETION_ONSTACK(done);
1083 struct bitmap_unplug_work unplug_work;
1085 INIT_WORK_ONSTACK(&unplug_work.work, md_bitmap_unplug_fn);
1086 unplug_work.bitmap = bitmap;
1087 unplug_work.done = &done;
1089 queue_work(md_bitmap_wq, &unplug_work.work);
1090 wait_for_completion(&done);
1092 EXPORT_SYMBOL(md_bitmap_unplug_async);
1094 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1097 * Initialize the in-memory bitmap from the on-disk bitmap and set up the memory
1098 * mapping of the bitmap file.
1100 * Special case: If there's no bitmap file, or if the bitmap file had been
1101 * previously kicked from the array, we mark all the bits as 1's in order to
1102 * cause a full resync.
1104 * We ignore all bits for sectors that end earlier than 'start'.
1105 * This is used when reading an out-of-date bitmap.
1107 static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1109 bool outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1110 struct mddev *mddev = bitmap->mddev;
1111 unsigned long chunks = bitmap->counts.chunks;
1112 struct bitmap_storage *store = &bitmap->storage;
1113 struct file *file = store->file;
1114 unsigned long node_offset = 0;
1115 unsigned long bit_cnt = 0;
1119 if (!file && !mddev->bitmap_info.offset) {
1120 /* No permanent bitmap - fill with '1s'. */
1121 store->filemap = NULL;
1122 store->file_pages = 0;
1123 for (i = 0; i < chunks ; i++) {
1124 /* if the disk bit is set, set the memory bit */
1125 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1127 md_bitmap_set_memory_bits(bitmap,
1128 (sector_t)i << bitmap->counts.chunkshift,
1134 if (file && i_size_read(file->f_mapping->host) < store->bytes) {
1135 pr_warn("%s: bitmap file too short %lu < %lu\n",
1137 (unsigned long) i_size_read(file->f_mapping->host),
1143 if (mddev_is_clustered(mddev))
1144 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1146 for (i = 0; i < store->file_pages; i++) {
1147 struct page *page = store->filemap[i];
1150 /* unmap the old page, we're done with it */
1151 if (i == store->file_pages - 1)
1152 count = store->bytes - i * PAGE_SIZE;
1157 ret = read_file_page(file, i, bitmap, count, page);
1159 ret = read_sb_page(mddev, 0, page, i + node_offset,
1166 pr_warn("%s: bitmap file is out of date, doing full recovery\n",
1169 for (i = 0; i < store->file_pages; i++) {
1170 struct page *page = store->filemap[i];
1171 unsigned long offset = 0;
1174 if (i == 0 && !mddev->bitmap_info.external)
1175 offset = sizeof(bitmap_super_t);
1178 * If the bitmap is out of date, dirty the whole page
1181 paddr = kmap_atomic(page);
1182 memset(paddr + offset, 0xff, PAGE_SIZE - offset);
1183 kunmap_atomic(paddr);
1185 filemap_write_page(bitmap, i, true);
1186 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) {
1193 for (i = 0; i < chunks; i++) {
1194 struct page *page = filemap_get_page(&bitmap->storage, i);
1195 unsigned long bit = file_page_offset(&bitmap->storage, i);
1199 paddr = kmap_atomic(page);
1200 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1201 was_set = test_bit(bit, paddr);
1203 was_set = test_bit_le(bit, paddr);
1204 kunmap_atomic(paddr);
1207 /* if the disk bit is set, set the memory bit */
1208 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1210 md_bitmap_set_memory_bits(bitmap,
1211 (sector_t)i << bitmap->counts.chunkshift,
1217 pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
1218 bmname(bitmap), store->file_pages,
1224 pr_warn("%s: bitmap initialisation failed: %d\n",
1225 bmname(bitmap), ret);
1229 void md_bitmap_write_all(struct bitmap *bitmap)
1231 /* We don't actually write all bitmap blocks here,
1232 * just flag them as needing to be written
1236 if (!bitmap || !bitmap->storage.filemap)
1238 if (bitmap->storage.file)
1239 /* Only one copy, so nothing needed */
1242 for (i = 0; i < bitmap->storage.file_pages; i++)
1243 set_page_attr(bitmap, i,
1244 BITMAP_PAGE_NEEDWRITE);
1245 bitmap->allclean = 0;
1248 static void md_bitmap_count_page(struct bitmap_counts *bitmap,
1249 sector_t offset, int inc)
1251 sector_t chunk = offset >> bitmap->chunkshift;
1252 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1253 bitmap->bp[page].count += inc;
1254 md_bitmap_checkfree(bitmap, page);
1257 static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1259 sector_t chunk = offset >> bitmap->chunkshift;
1260 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1261 struct bitmap_page *bp = &bitmap->bp[page];
1267 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1268 sector_t offset, sector_t *blocks,
1271 static void mddev_set_timeout(struct mddev *mddev, unsigned long timeout,
1274 struct md_thread *thread;
1277 thread = rcu_dereference(mddev->thread);
1282 if (force || thread->timeout < MAX_SCHEDULE_TIMEOUT)
1283 thread->timeout = timeout;
1290 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1293 void md_bitmap_daemon_work(struct mddev *mddev)
1295 struct bitmap *bitmap;
1297 unsigned long nextpage;
1299 struct bitmap_counts *counts;
1301 /* Use a mutex to guard daemon_work against
1304 mutex_lock(&mddev->bitmap_info.mutex);
1305 bitmap = mddev->bitmap;
1306 if (bitmap == NULL) {
1307 mutex_unlock(&mddev->bitmap_info.mutex);
1310 if (time_before(jiffies, bitmap->daemon_lastrun
1311 + mddev->bitmap_info.daemon_sleep))
1314 bitmap->daemon_lastrun = jiffies;
1315 if (bitmap->allclean) {
1316 mddev_set_timeout(mddev, MAX_SCHEDULE_TIMEOUT, true);
1319 bitmap->allclean = 1;
1321 mddev_add_trace_msg(bitmap->mddev, "md bitmap_daemon_work");
1323 /* Any file-page which is PENDING now needs to be written.
1324 * So set NEEDWRITE now, then after we make any last-minute changes
1327 for (j = 0; j < bitmap->storage.file_pages; j++)
1328 if (test_and_clear_page_attr(bitmap, j,
1329 BITMAP_PAGE_PENDING))
1330 set_page_attr(bitmap, j,
1331 BITMAP_PAGE_NEEDWRITE);
1333 if (bitmap->need_sync &&
1334 mddev->bitmap_info.external == 0) {
1335 /* Arrange for superblock update as well as
1338 bitmap->need_sync = 0;
1339 if (bitmap->storage.filemap) {
1340 sb = kmap_atomic(bitmap->storage.sb_page);
1341 sb->events_cleared =
1342 cpu_to_le64(bitmap->events_cleared);
1344 set_page_attr(bitmap, 0,
1345 BITMAP_PAGE_NEEDWRITE);
1348 /* Now look at the bitmap counters and if any are '2' or '1',
1349 * decrement and handle accordingly.
1351 counts = &bitmap->counts;
1352 spin_lock_irq(&counts->lock);
1354 for (j = 0; j < counts->chunks; j++) {
1355 bitmap_counter_t *bmc;
1356 sector_t block = (sector_t)j << counts->chunkshift;
1358 if (j == nextpage) {
1359 nextpage += PAGE_COUNTER_RATIO;
1360 if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1361 j |= PAGE_COUNTER_MASK;
1364 counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1367 bmc = md_bitmap_get_counter(counts, block, &blocks, 0);
1369 j |= PAGE_COUNTER_MASK;
1372 if (*bmc == 1 && !bitmap->need_sync) {
1373 /* We can clear the bit */
1375 md_bitmap_count_page(counts, block, -1);
1376 md_bitmap_file_clear_bit(bitmap, block);
1377 } else if (*bmc && *bmc <= 2) {
1379 md_bitmap_set_pending(counts, block);
1380 bitmap->allclean = 0;
1383 spin_unlock_irq(&counts->lock);
1385 md_bitmap_wait_writes(bitmap);
1386 /* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1387 * DIRTY pages need to be written by bitmap_unplug so it can wait
1389 * If we find any DIRTY page we stop there and let bitmap_unplug
1390 * handle all the rest. This is important in the case where
1391 * the first blocking holds the superblock and it has been updated.
1392 * We mustn't write any other blocks before the superblock.
1395 j < bitmap->storage.file_pages
1396 && !test_bit(BITMAP_STALE, &bitmap->flags);
1398 if (test_page_attr(bitmap, j,
1400 /* bitmap_unplug will handle the rest */
1402 if (bitmap->storage.filemap &&
1403 test_and_clear_page_attr(bitmap, j,
1404 BITMAP_PAGE_NEEDWRITE))
1405 filemap_write_page(bitmap, j, false);
1409 if (bitmap->allclean == 0)
1410 mddev_set_timeout(mddev, mddev->bitmap_info.daemon_sleep, true);
1411 mutex_unlock(&mddev->bitmap_info.mutex);
1414 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1415 sector_t offset, sector_t *blocks,
1417 __releases(bitmap->lock)
1418 __acquires(bitmap->lock)
1420 /* If 'create', we might release the lock and reclaim it.
1421 * The lock must have been taken with interrupts enabled.
1422 * If !create, we don't release the lock.
1424 sector_t chunk = offset >> bitmap->chunkshift;
1425 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1426 unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1430 if (page >= bitmap->pages) {
1432 * This can happen if bitmap_start_sync goes beyond
1433 * End-of-device while looking for a whole page or
1434 * user set a huge number to sysfs bitmap_set_bits.
1438 err = md_bitmap_checkpage(bitmap, page, create, 0);
1440 if (bitmap->bp[page].hijacked ||
1441 bitmap->bp[page].map == NULL)
1442 csize = ((sector_t)1) << (bitmap->chunkshift +
1443 PAGE_COUNTER_SHIFT);
1445 csize = ((sector_t)1) << bitmap->chunkshift;
1446 *blocks = csize - (offset & (csize - 1));
1451 /* now locked ... */
1453 if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1454 /* should we use the first or second counter field
1455 * of the hijacked pointer? */
1456 int hi = (pageoff > PAGE_COUNTER_MASK);
1457 return &((bitmap_counter_t *)
1458 &bitmap->bp[page].map)[hi];
1459 } else /* page is allocated */
1460 return (bitmap_counter_t *)
1461 &(bitmap->bp[page].map[pageoff]);
1464 int md_bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1471 atomic_inc(&bitmap->behind_writes);
1472 bw = atomic_read(&bitmap->behind_writes);
1473 if (bw > bitmap->behind_writes_used)
1474 bitmap->behind_writes_used = bw;
1476 pr_debug("inc write-behind count %d/%lu\n",
1477 bw, bitmap->mddev->bitmap_info.max_write_behind);
1482 bitmap_counter_t *bmc;
1484 spin_lock_irq(&bitmap->counts.lock);
1485 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1487 spin_unlock_irq(&bitmap->counts.lock);
1491 if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1492 DEFINE_WAIT(__wait);
1493 /* note that it is safe to do the prepare_to_wait
1494 * after the test as long as we do it before dropping
1497 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1498 TASK_UNINTERRUPTIBLE);
1499 spin_unlock_irq(&bitmap->counts.lock);
1501 finish_wait(&bitmap->overflow_wait, &__wait);
1507 md_bitmap_file_set_bit(bitmap, offset);
1508 md_bitmap_count_page(&bitmap->counts, offset, 1);
1516 spin_unlock_irq(&bitmap->counts.lock);
1519 if (sectors > blocks)
1526 EXPORT_SYMBOL(md_bitmap_startwrite);
1528 void md_bitmap_endwrite(struct bitmap *bitmap, sector_t offset,
1529 unsigned long sectors, int success, int behind)
1534 if (atomic_dec_and_test(&bitmap->behind_writes))
1535 wake_up(&bitmap->behind_wait);
1536 pr_debug("dec write-behind count %d/%lu\n",
1537 atomic_read(&bitmap->behind_writes),
1538 bitmap->mddev->bitmap_info.max_write_behind);
1543 unsigned long flags;
1544 bitmap_counter_t *bmc;
1546 spin_lock_irqsave(&bitmap->counts.lock, flags);
1547 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1549 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1553 if (success && !bitmap->mddev->degraded &&
1554 bitmap->events_cleared < bitmap->mddev->events) {
1555 bitmap->events_cleared = bitmap->mddev->events;
1556 bitmap->need_sync = 1;
1557 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1560 if (!success && !NEEDED(*bmc))
1561 *bmc |= NEEDED_MASK;
1563 if (COUNTER(*bmc) == COUNTER_MAX)
1564 wake_up(&bitmap->overflow_wait);
1568 md_bitmap_set_pending(&bitmap->counts, offset);
1569 bitmap->allclean = 0;
1571 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1573 if (sectors > blocks)
1579 EXPORT_SYMBOL(md_bitmap_endwrite);
1581 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1584 bitmap_counter_t *bmc;
1586 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1588 return 1; /* always resync if no bitmap */
1590 spin_lock_irq(&bitmap->counts.lock);
1591 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1597 else if (NEEDED(*bmc)) {
1599 if (!degraded) { /* don't set/clear bits if degraded */
1600 *bmc |= RESYNC_MASK;
1601 *bmc &= ~NEEDED_MASK;
1605 spin_unlock_irq(&bitmap->counts.lock);
1609 int md_bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1612 /* bitmap_start_sync must always report on multiples of whole
1613 * pages, otherwise resync (which is very PAGE_SIZE based) will
1615 * So call __bitmap_start_sync repeatedly (if needed) until
1616 * At least PAGE_SIZE>>9 blocks are covered.
1617 * Return the 'or' of the result.
1623 while (*blocks < (PAGE_SIZE>>9)) {
1624 rv |= __bitmap_start_sync(bitmap, offset,
1625 &blocks1, degraded);
1631 EXPORT_SYMBOL(md_bitmap_start_sync);
1633 void md_bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1635 bitmap_counter_t *bmc;
1636 unsigned long flags;
1638 if (bitmap == NULL) {
1642 spin_lock_irqsave(&bitmap->counts.lock, flags);
1643 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1648 *bmc &= ~RESYNC_MASK;
1650 if (!NEEDED(*bmc) && aborted)
1651 *bmc |= NEEDED_MASK;
1654 md_bitmap_set_pending(&bitmap->counts, offset);
1655 bitmap->allclean = 0;
1660 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1662 EXPORT_SYMBOL(md_bitmap_end_sync);
1664 void md_bitmap_close_sync(struct bitmap *bitmap)
1666 /* Sync has finished, and any bitmap chunks that weren't synced
1667 * properly have been aborted. It remains to us to clear the
1668 * RESYNC bit wherever it is still on
1670 sector_t sector = 0;
1674 while (sector < bitmap->mddev->resync_max_sectors) {
1675 md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1679 EXPORT_SYMBOL(md_bitmap_close_sync);
1681 void md_bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
1689 bitmap->last_end_sync = jiffies;
1692 if (!force && time_before(jiffies, (bitmap->last_end_sync
1693 + bitmap->mddev->bitmap_info.daemon_sleep)))
1695 wait_event(bitmap->mddev->recovery_wait,
1696 atomic_read(&bitmap->mddev->recovery_active) == 0);
1698 bitmap->mddev->curr_resync_completed = sector;
1699 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
1700 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1702 while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1703 md_bitmap_end_sync(bitmap, s, &blocks, 0);
1706 bitmap->last_end_sync = jiffies;
1707 sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed);
1709 EXPORT_SYMBOL(md_bitmap_cond_end_sync);
1711 void md_bitmap_sync_with_cluster(struct mddev *mddev,
1712 sector_t old_lo, sector_t old_hi,
1713 sector_t new_lo, sector_t new_hi)
1715 struct bitmap *bitmap = mddev->bitmap;
1716 sector_t sector, blocks = 0;
1718 for (sector = old_lo; sector < new_lo; ) {
1719 md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1722 WARN((blocks > new_lo) && old_lo, "alignment is not correct for lo\n");
1724 for (sector = old_hi; sector < new_hi; ) {
1725 md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1728 WARN((blocks > new_hi) && old_hi, "alignment is not correct for hi\n");
1730 EXPORT_SYMBOL(md_bitmap_sync_with_cluster);
1732 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1734 /* For each chunk covered by any of these sectors, set the
1735 * counter to 2 and possibly set resync_needed. They should all
1736 * be 0 at this point
1740 bitmap_counter_t *bmc;
1741 spin_lock_irq(&bitmap->counts.lock);
1742 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1744 spin_unlock_irq(&bitmap->counts.lock);
1749 md_bitmap_count_page(&bitmap->counts, offset, 1);
1750 md_bitmap_set_pending(&bitmap->counts, offset);
1751 bitmap->allclean = 0;
1754 *bmc |= NEEDED_MASK;
1755 spin_unlock_irq(&bitmap->counts.lock);
1758 /* dirty the memory and file bits for bitmap chunks "s" to "e" */
1759 void md_bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1761 unsigned long chunk;
1763 for (chunk = s; chunk <= e; chunk++) {
1764 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1765 md_bitmap_set_memory_bits(bitmap, sec, 1);
1766 md_bitmap_file_set_bit(bitmap, sec);
1767 if (sec < bitmap->mddev->recovery_cp)
1768 /* We are asserting that the array is dirty,
1769 * so move the recovery_cp address back so
1770 * that it is obvious that it is dirty
1772 bitmap->mddev->recovery_cp = sec;
1777 * flush out any pending updates
1779 void md_bitmap_flush(struct mddev *mddev)
1781 struct bitmap *bitmap = mddev->bitmap;
1784 if (!bitmap) /* there was no bitmap */
1787 /* run the daemon_work three time to ensure everything is flushed
1790 sleep = mddev->bitmap_info.daemon_sleep * 2;
1791 bitmap->daemon_lastrun -= sleep;
1792 md_bitmap_daemon_work(mddev);
1793 bitmap->daemon_lastrun -= sleep;
1794 md_bitmap_daemon_work(mddev);
1795 bitmap->daemon_lastrun -= sleep;
1796 md_bitmap_daemon_work(mddev);
1797 if (mddev->bitmap_info.external)
1798 md_super_wait(mddev);
1799 md_bitmap_update_sb(bitmap);
1803 * free memory that was allocated
1805 void md_bitmap_free(struct bitmap *bitmap)
1807 unsigned long k, pages;
1808 struct bitmap_page *bp;
1810 if (!bitmap) /* there was no bitmap */
1813 if (bitmap->sysfs_can_clear)
1814 sysfs_put(bitmap->sysfs_can_clear);
1816 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
1817 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
1818 md_cluster_stop(bitmap->mddev);
1820 /* Shouldn't be needed - but just in case.... */
1821 wait_event(bitmap->write_wait,
1822 atomic_read(&bitmap->pending_writes) == 0);
1824 /* release the bitmap file */
1825 md_bitmap_file_unmap(&bitmap->storage);
1827 bp = bitmap->counts.bp;
1828 pages = bitmap->counts.pages;
1830 /* free all allocated memory */
1832 if (bp) /* deallocate the page memory */
1833 for (k = 0; k < pages; k++)
1834 if (bp[k].map && !bp[k].hijacked)
1839 EXPORT_SYMBOL(md_bitmap_free);
1841 void md_bitmap_wait_behind_writes(struct mddev *mddev)
1843 struct bitmap *bitmap = mddev->bitmap;
1845 /* wait for behind writes to complete */
1846 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
1847 pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
1849 /* need to kick something here to make sure I/O goes? */
1850 wait_event(bitmap->behind_wait,
1851 atomic_read(&bitmap->behind_writes) == 0);
1855 void md_bitmap_destroy(struct mddev *mddev)
1857 struct bitmap *bitmap = mddev->bitmap;
1859 if (!bitmap) /* there was no bitmap */
1862 md_bitmap_wait_behind_writes(mddev);
1863 if (!mddev->serialize_policy)
1864 mddev_destroy_serial_pool(mddev, NULL);
1866 mutex_lock(&mddev->bitmap_info.mutex);
1867 spin_lock(&mddev->lock);
1868 mddev->bitmap = NULL; /* disconnect from the md device */
1869 spin_unlock(&mddev->lock);
1870 mutex_unlock(&mddev->bitmap_info.mutex);
1871 mddev_set_timeout(mddev, MAX_SCHEDULE_TIMEOUT, true);
1873 md_bitmap_free(bitmap);
1877 * initialize the bitmap structure
1878 * if this returns an error, bitmap_destroy must be called to do clean up
1879 * once mddev->bitmap is set
1881 struct bitmap *md_bitmap_create(struct mddev *mddev, int slot)
1883 struct bitmap *bitmap;
1884 sector_t blocks = mddev->resync_max_sectors;
1885 struct file *file = mddev->bitmap_info.file;
1887 struct kernfs_node *bm = NULL;
1889 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1891 BUG_ON(file && mddev->bitmap_info.offset);
1893 if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
1894 pr_notice("md/raid:%s: array with journal cannot have bitmap\n",
1896 return ERR_PTR(-EBUSY);
1899 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1901 return ERR_PTR(-ENOMEM);
1903 spin_lock_init(&bitmap->counts.lock);
1904 atomic_set(&bitmap->pending_writes, 0);
1905 init_waitqueue_head(&bitmap->write_wait);
1906 init_waitqueue_head(&bitmap->overflow_wait);
1907 init_waitqueue_head(&bitmap->behind_wait);
1909 bitmap->mddev = mddev;
1910 bitmap->cluster_slot = slot;
1913 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1915 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1918 bitmap->sysfs_can_clear = NULL;
1920 bitmap->storage.file = file;
1923 /* As future accesses to this file will use bmap,
1924 * and bypass the page cache, we must sync the file
1929 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1930 if (!mddev->bitmap_info.external) {
1932 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
1933 * instructing us to create a new on-disk bitmap instance.
1935 if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
1936 err = md_bitmap_new_disk_sb(bitmap);
1938 err = md_bitmap_read_sb(bitmap);
1941 if (mddev->bitmap_info.chunksize == 0 ||
1942 mddev->bitmap_info.daemon_sleep == 0)
1943 /* chunksize and time_base need to be
1950 bitmap->daemon_lastrun = jiffies;
1951 err = md_bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1955 pr_debug("created bitmap (%lu pages) for device %s\n",
1956 bitmap->counts.pages, bmname(bitmap));
1958 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1964 md_bitmap_free(bitmap);
1965 return ERR_PTR(err);
1968 int md_bitmap_load(struct mddev *mddev)
1972 sector_t sector = 0;
1973 struct bitmap *bitmap = mddev->bitmap;
1974 struct md_rdev *rdev;
1979 rdev_for_each(rdev, mddev)
1980 mddev_create_serial_pool(mddev, rdev);
1982 if (mddev_is_clustered(mddev))
1983 md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);
1985 /* Clear out old bitmap info first: Either there is none, or we
1986 * are resuming after someone else has possibly changed things,
1987 * so we should forget old cached info.
1988 * All chunks should be clean, but some might need_sync.
1990 while (sector < mddev->resync_max_sectors) {
1992 md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1995 md_bitmap_close_sync(bitmap);
1997 if (mddev->degraded == 0
1998 || bitmap->events_cleared == mddev->events)
1999 /* no need to keep dirty bits to optimise a
2000 * re-add of a missing device */
2001 start = mddev->recovery_cp;
2003 mutex_lock(&mddev->bitmap_info.mutex);
2004 err = md_bitmap_init_from_disk(bitmap, start);
2005 mutex_unlock(&mddev->bitmap_info.mutex);
2009 clear_bit(BITMAP_STALE, &bitmap->flags);
2011 /* Kick recovery in case any bits were set */
2012 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
2014 mddev_set_timeout(mddev, mddev->bitmap_info.daemon_sleep, true);
2015 md_wakeup_thread(mddev->thread);
2017 md_bitmap_update_sb(bitmap);
2019 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
2024 EXPORT_SYMBOL_GPL(md_bitmap_load);
2026 /* caller need to free returned bitmap with md_bitmap_free() */
2027 struct bitmap *get_bitmap_from_slot(struct mddev *mddev, int slot)
2030 struct bitmap *bitmap;
2032 bitmap = md_bitmap_create(mddev, slot);
2033 if (IS_ERR(bitmap)) {
2034 rv = PTR_ERR(bitmap);
2038 rv = md_bitmap_init_from_disk(bitmap, 0);
2040 md_bitmap_free(bitmap);
2046 EXPORT_SYMBOL(get_bitmap_from_slot);
2048 /* Loads the bitmap associated with slot and copies the resync information
2051 int md_bitmap_copy_from_slot(struct mddev *mddev, int slot,
2052 sector_t *low, sector_t *high, bool clear_bits)
2055 sector_t block, lo = 0, hi = 0;
2056 struct bitmap_counts *counts;
2057 struct bitmap *bitmap;
2059 bitmap = get_bitmap_from_slot(mddev, slot);
2060 if (IS_ERR(bitmap)) {
2061 pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
2065 counts = &bitmap->counts;
2066 for (j = 0; j < counts->chunks; j++) {
2067 block = (sector_t)j << counts->chunkshift;
2068 if (md_bitmap_file_test_bit(bitmap, block)) {
2072 md_bitmap_file_clear_bit(bitmap, block);
2073 md_bitmap_set_memory_bits(mddev->bitmap, block, 1);
2074 md_bitmap_file_set_bit(mddev->bitmap, block);
2079 md_bitmap_update_sb(bitmap);
2080 /* BITMAP_PAGE_PENDING is set, but bitmap_unplug needs
2081 * BITMAP_PAGE_DIRTY or _NEEDWRITE to write ... */
2082 for (i = 0; i < bitmap->storage.file_pages; i++)
2083 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING))
2084 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
2085 md_bitmap_unplug(bitmap);
2087 md_bitmap_unplug(mddev->bitmap);
2090 md_bitmap_free(bitmap);
2094 EXPORT_SYMBOL_GPL(md_bitmap_copy_from_slot);
2097 void md_bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
2099 unsigned long chunk_kb;
2100 struct bitmap_counts *counts;
2105 counts = &bitmap->counts;
2107 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
2108 seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
2110 counts->pages - counts->missing_pages,
2112 (counts->pages - counts->missing_pages)
2113 << (PAGE_SHIFT - 10),
2114 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
2115 chunk_kb ? "KB" : "B");
2116 if (bitmap->storage.file) {
2117 seq_printf(seq, ", file: ");
2118 seq_file_path(seq, bitmap->storage.file, " \t\n");
2121 seq_printf(seq, "\n");
2124 int md_bitmap_resize(struct bitmap *bitmap, sector_t blocks,
2125 int chunksize, int init)
2127 /* If chunk_size is 0, choose an appropriate chunk size.
2128 * Then possibly allocate new storage space.
2129 * Then quiesce, copy bits, replace bitmap, and re-start
2131 * This function is called both to set up the initial bitmap
2132 * and to resize the bitmap while the array is active.
2133 * If this happens as a result of the array being resized,
2134 * chunksize will be zero, and we need to choose a suitable
2135 * chunksize, otherwise we use what we are given.
2137 struct bitmap_storage store;
2138 struct bitmap_counts old_counts;
2139 unsigned long chunks;
2141 sector_t old_blocks, new_blocks;
2145 struct bitmap_page *new_bp;
2147 if (bitmap->storage.file && !init) {
2148 pr_info("md: cannot resize file-based bitmap\n");
2152 if (chunksize == 0) {
2153 /* If there is enough space, leave the chunk size unchanged,
2154 * else increase by factor of two until there is enough space.
2157 long space = bitmap->mddev->bitmap_info.space;
2160 /* We don't know how much space there is, so limit
2161 * to current size - in sectors.
2163 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
2164 if (!bitmap->mddev->bitmap_info.external)
2165 bytes += sizeof(bitmap_super_t);
2166 space = DIV_ROUND_UP(bytes, 512);
2167 bitmap->mddev->bitmap_info.space = space;
2169 chunkshift = bitmap->counts.chunkshift;
2172 /* 'chunkshift' is shift from block size to chunk size */
2174 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2175 bytes = DIV_ROUND_UP(chunks, 8);
2176 if (!bitmap->mddev->bitmap_info.external)
2177 bytes += sizeof(bitmap_super_t);
2178 } while (bytes > (space << 9) && (chunkshift + BITMAP_BLOCK_SHIFT) <
2179 (BITS_PER_BYTE * sizeof(((bitmap_super_t *)0)->chunksize) - 1));
2181 chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
2183 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2184 memset(&store, 0, sizeof(store));
2185 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
2186 ret = md_bitmap_storage_alloc(&store, chunks,
2187 !bitmap->mddev->bitmap_info.external,
2188 mddev_is_clustered(bitmap->mddev)
2189 ? bitmap->cluster_slot : 0);
2191 md_bitmap_file_unmap(&store);
2195 pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
2197 new_bp = kcalloc(pages, sizeof(*new_bp), GFP_KERNEL);
2200 md_bitmap_file_unmap(&store);
2205 bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2207 store.file = bitmap->storage.file;
2208 bitmap->storage.file = NULL;
2210 if (store.sb_page && bitmap->storage.sb_page)
2211 memcpy(page_address(store.sb_page),
2212 page_address(bitmap->storage.sb_page),
2213 sizeof(bitmap_super_t));
2214 spin_lock_irq(&bitmap->counts.lock);
2215 md_bitmap_file_unmap(&bitmap->storage);
2216 bitmap->storage = store;
2218 old_counts = bitmap->counts;
2219 bitmap->counts.bp = new_bp;
2220 bitmap->counts.pages = pages;
2221 bitmap->counts.missing_pages = pages;
2222 bitmap->counts.chunkshift = chunkshift;
2223 bitmap->counts.chunks = chunks;
2224 bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift +
2225 BITMAP_BLOCK_SHIFT);
2227 blocks = min(old_counts.chunks << old_counts.chunkshift,
2228 chunks << chunkshift);
2230 /* For cluster raid, need to pre-allocate bitmap */
2231 if (mddev_is_clustered(bitmap->mddev)) {
2233 for (page = 0; page < pages; page++) {
2234 ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1);
2238 /* deallocate the page memory */
2239 for (k = 0; k < page; k++) {
2240 kfree(new_bp[k].map);
2244 /* restore some fields from old_counts */
2245 bitmap->counts.bp = old_counts.bp;
2246 bitmap->counts.pages = old_counts.pages;
2247 bitmap->counts.missing_pages = old_counts.pages;
2248 bitmap->counts.chunkshift = old_counts.chunkshift;
2249 bitmap->counts.chunks = old_counts.chunks;
2250 bitmap->mddev->bitmap_info.chunksize =
2251 1UL << (old_counts.chunkshift + BITMAP_BLOCK_SHIFT);
2252 blocks = old_counts.chunks << old_counts.chunkshift;
2253 pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
2256 bitmap->counts.bp[page].count += 1;
2260 for (block = 0; block < blocks; ) {
2261 bitmap_counter_t *bmc_old, *bmc_new;
2264 bmc_old = md_bitmap_get_counter(&old_counts, block, &old_blocks, 0);
2265 set = bmc_old && NEEDED(*bmc_old);
2268 bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2270 if (*bmc_new == 0) {
2271 /* need to set on-disk bits too. */
2272 sector_t end = block + new_blocks;
2273 sector_t start = block >> chunkshift;
2275 start <<= chunkshift;
2276 while (start < end) {
2277 md_bitmap_file_set_bit(bitmap, block);
2278 start += 1 << chunkshift;
2281 md_bitmap_count_page(&bitmap->counts, block, 1);
2282 md_bitmap_set_pending(&bitmap->counts, block);
2284 *bmc_new |= NEEDED_MASK;
2286 if (new_blocks < old_blocks)
2287 old_blocks = new_blocks;
2289 block += old_blocks;
2292 if (bitmap->counts.bp != old_counts.bp) {
2294 for (k = 0; k < old_counts.pages; k++)
2295 if (!old_counts.bp[k].hijacked)
2296 kfree(old_counts.bp[k].map);
2297 kfree(old_counts.bp);
2302 while (block < (chunks << chunkshift)) {
2303 bitmap_counter_t *bmc;
2304 bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2306 /* new space. It needs to be resynced, so
2307 * we set NEEDED_MASK.
2310 *bmc = NEEDED_MASK | 2;
2311 md_bitmap_count_page(&bitmap->counts, block, 1);
2312 md_bitmap_set_pending(&bitmap->counts, block);
2315 block += new_blocks;
2317 for (i = 0; i < bitmap->storage.file_pages; i++)
2318 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
2320 spin_unlock_irq(&bitmap->counts.lock);
2323 md_bitmap_unplug(bitmap);
2324 bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2330 EXPORT_SYMBOL_GPL(md_bitmap_resize);
2333 location_show(struct mddev *mddev, char *page)
2336 if (mddev->bitmap_info.file)
2337 len = sprintf(page, "file");
2338 else if (mddev->bitmap_info.offset)
2339 len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
2341 len = sprintf(page, "none");
2342 len += sprintf(page+len, "\n");
2347 location_store(struct mddev *mddev, const char *buf, size_t len)
2351 rv = mddev_suspend_and_lock(mddev);
2356 if (mddev->recovery || mddev->sync_thread) {
2362 if (mddev->bitmap || mddev->bitmap_info.file ||
2363 mddev->bitmap_info.offset) {
2364 /* bitmap already configured. Only option is to clear it */
2365 if (strncmp(buf, "none", 4) != 0) {
2370 md_bitmap_destroy(mddev);
2371 mddev->bitmap_info.offset = 0;
2372 if (mddev->bitmap_info.file) {
2373 struct file *f = mddev->bitmap_info.file;
2374 mddev->bitmap_info.file = NULL;
2378 /* No bitmap, OK to set a location */
2380 struct bitmap *bitmap;
2382 if (strncmp(buf, "none", 4) == 0)
2383 /* nothing to be done */;
2384 else if (strncmp(buf, "file:", 5) == 0) {
2385 /* Not supported yet */
2390 rv = kstrtoll(buf+1, 10, &offset);
2392 rv = kstrtoll(buf, 10, &offset);
2399 if (mddev->bitmap_info.external == 0 &&
2400 mddev->major_version == 0 &&
2401 offset != mddev->bitmap_info.default_offset) {
2406 mddev->bitmap_info.offset = offset;
2407 bitmap = md_bitmap_create(mddev, -1);
2408 if (IS_ERR(bitmap)) {
2409 rv = PTR_ERR(bitmap);
2413 mddev->bitmap = bitmap;
2414 rv = md_bitmap_load(mddev);
2416 mddev->bitmap_info.offset = 0;
2417 md_bitmap_destroy(mddev);
2422 if (!mddev->external) {
2423 /* Ensure new bitmap info is stored in
2424 * metadata promptly.
2426 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2427 md_wakeup_thread(mddev->thread);
2431 mddev_unlock_and_resume(mddev);
2437 static struct md_sysfs_entry bitmap_location =
2438 __ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2440 /* 'bitmap/space' is the space available at 'location' for the
2441 * bitmap. This allows the kernel to know when it is safe to
2442 * resize the bitmap to match a resized array.
2445 space_show(struct mddev *mddev, char *page)
2447 return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2451 space_store(struct mddev *mddev, const char *buf, size_t len)
2453 unsigned long sectors;
2456 rv = kstrtoul(buf, 10, §ors);
2463 if (mddev->bitmap &&
2464 sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2465 return -EFBIG; /* Bitmap is too big for this small space */
2467 /* could make sure it isn't too big, but that isn't really
2468 * needed - user-space should be careful.
2470 mddev->bitmap_info.space = sectors;
2474 static struct md_sysfs_entry bitmap_space =
2475 __ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2478 timeout_show(struct mddev *mddev, char *page)
2481 unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2482 unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2484 len = sprintf(page, "%lu", secs);
2486 len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2487 len += sprintf(page+len, "\n");
2492 timeout_store(struct mddev *mddev, const char *buf, size_t len)
2494 /* timeout can be set at any time */
2495 unsigned long timeout;
2496 int rv = strict_strtoul_scaled(buf, &timeout, 4);
2500 /* just to make sure we don't overflow... */
2501 if (timeout >= LONG_MAX / HZ)
2504 timeout = timeout * HZ / 10000;
2506 if (timeout >= MAX_SCHEDULE_TIMEOUT)
2507 timeout = MAX_SCHEDULE_TIMEOUT-1;
2511 mddev->bitmap_info.daemon_sleep = timeout;
2512 mddev_set_timeout(mddev, timeout, false);
2513 md_wakeup_thread(mddev->thread);
2518 static struct md_sysfs_entry bitmap_timeout =
2519 __ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2522 backlog_show(struct mddev *mddev, char *page)
2524 return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2528 backlog_store(struct mddev *mddev, const char *buf, size_t len)
2530 unsigned long backlog;
2531 unsigned long old_mwb = mddev->bitmap_info.max_write_behind;
2532 struct md_rdev *rdev;
2533 bool has_write_mostly = false;
2534 int rv = kstrtoul(buf, 10, &backlog);
2537 if (backlog > COUNTER_MAX)
2540 rv = mddev_suspend_and_lock(mddev);
2545 * Without write mostly device, it doesn't make sense to set
2546 * backlog for max_write_behind.
2548 rdev_for_each(rdev, mddev) {
2549 if (test_bit(WriteMostly, &rdev->flags)) {
2550 has_write_mostly = true;
2554 if (!has_write_mostly) {
2555 pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n",
2557 mddev_unlock(mddev);
2561 mddev->bitmap_info.max_write_behind = backlog;
2562 if (!backlog && mddev->serial_info_pool) {
2563 /* serial_info_pool is not needed if backlog is zero */
2564 if (!mddev->serialize_policy)
2565 mddev_destroy_serial_pool(mddev, NULL);
2566 } else if (backlog && !mddev->serial_info_pool) {
2567 /* serial_info_pool is needed since backlog is not zero */
2568 rdev_for_each(rdev, mddev)
2569 mddev_create_serial_pool(mddev, rdev);
2571 if (old_mwb != backlog)
2572 md_bitmap_update_sb(mddev->bitmap);
2574 mddev_unlock_and_resume(mddev);
2578 static struct md_sysfs_entry bitmap_backlog =
2579 __ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2582 chunksize_show(struct mddev *mddev, char *page)
2584 return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2588 chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2590 /* Can only be changed when no bitmap is active */
2592 unsigned long csize;
2595 rv = kstrtoul(buf, 10, &csize);
2599 !is_power_of_2(csize))
2601 if (BITS_PER_LONG > 32 && csize >= (1ULL << (BITS_PER_BYTE *
2602 sizeof(((bitmap_super_t *)0)->chunksize))))
2604 mddev->bitmap_info.chunksize = csize;
2608 static struct md_sysfs_entry bitmap_chunksize =
2609 __ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2611 static ssize_t metadata_show(struct mddev *mddev, char *page)
2613 if (mddev_is_clustered(mddev))
2614 return sprintf(page, "clustered\n");
2615 return sprintf(page, "%s\n", (mddev->bitmap_info.external
2616 ? "external" : "internal"));
2619 static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2621 if (mddev->bitmap ||
2622 mddev->bitmap_info.file ||
2623 mddev->bitmap_info.offset)
2625 if (strncmp(buf, "external", 8) == 0)
2626 mddev->bitmap_info.external = 1;
2627 else if ((strncmp(buf, "internal", 8) == 0) ||
2628 (strncmp(buf, "clustered", 9) == 0))
2629 mddev->bitmap_info.external = 0;
2635 static struct md_sysfs_entry bitmap_metadata =
2636 __ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2638 static ssize_t can_clear_show(struct mddev *mddev, char *page)
2641 spin_lock(&mddev->lock);
2643 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2646 len = sprintf(page, "\n");
2647 spin_unlock(&mddev->lock);
2651 static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2653 if (mddev->bitmap == NULL)
2655 if (strncmp(buf, "false", 5) == 0)
2656 mddev->bitmap->need_sync = 1;
2657 else if (strncmp(buf, "true", 4) == 0) {
2658 if (mddev->degraded)
2660 mddev->bitmap->need_sync = 0;
2666 static struct md_sysfs_entry bitmap_can_clear =
2667 __ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2670 behind_writes_used_show(struct mddev *mddev, char *page)
2673 spin_lock(&mddev->lock);
2674 if (mddev->bitmap == NULL)
2675 ret = sprintf(page, "0\n");
2677 ret = sprintf(page, "%lu\n",
2678 mddev->bitmap->behind_writes_used);
2679 spin_unlock(&mddev->lock);
2684 behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2687 mddev->bitmap->behind_writes_used = 0;
2691 static struct md_sysfs_entry max_backlog_used =
2692 __ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2693 behind_writes_used_show, behind_writes_used_reset);
2695 static struct attribute *md_bitmap_attrs[] = {
2696 &bitmap_location.attr,
2698 &bitmap_timeout.attr,
2699 &bitmap_backlog.attr,
2700 &bitmap_chunksize.attr,
2701 &bitmap_metadata.attr,
2702 &bitmap_can_clear.attr,
2703 &max_backlog_used.attr,
2706 const struct attribute_group md_bitmap_group = {
2708 .attrs = md_bitmap_attrs,