Merge tag 'for-6.0/dm-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/device...

[linux-2.6-block.git] / drivers / md / dm-writecache.c

diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c
index d74c5a7a0ab496f5a1154548de8cacc64e5ecc6b..96a003eb732341812322332c673cd4986ffbf97e 100644 (file)
--- a/drivers/md/dm-writecache.c
+++ b/drivers/md/dm-writecache.c
@@ -22,7 +22,7 @@
 
 #define HIGH_WATERMARK                 50
 #define LOW_WATERMARK                  45
-#define MAX_WRITEBACK_JOBS             0
+#define MAX_WRITEBACK_JOBS             min(0x10000000 / PAGE_SIZE, totalram_pages() / 16)
 #define ENDIO_LATENCY                  16
 #define WRITEBACK_LATENCY              64
 #define AUTOCOMMIT_BLOCKS_SSD          65536
@@ -523,8 +523,7 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
 
                region.sector += wc->start_sector;
                atomic_inc(&endio.count);
-               req.bi_op = REQ_OP_WRITE;
-               req.bi_op_flags = REQ_SYNC;
+               req.bi_opf = REQ_OP_WRITE | REQ_SYNC;
                req.mem.type = DM_IO_VMA;
                req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY;
                req.client = wc->dm_io;
@@ -562,8 +561,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc)
 
        region.sector += wc->start_sector;
 
-       req.bi_op = REQ_OP_WRITE;
-       req.bi_op_flags = REQ_SYNC | REQ_FUA;
+       req.bi_opf = REQ_OP_WRITE | REQ_SYNC | REQ_FUA;
        req.mem.type = DM_IO_VMA;
        req.mem.ptr.vma = (char *)wc->memory_map;
        req.client = wc->dm_io;
@@ -592,8 +590,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
        region.bdev = dev->bdev;
        region.sector = 0;
        region.count = 0;
-       req.bi_op = REQ_OP_WRITE;
-       req.bi_op_flags = REQ_PREFLUSH;
+       req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
        req.mem.type = DM_IO_KMEM;
        req.mem.ptr.addr = NULL;
        req.client = wc->dm_io;
@@ -981,8 +978,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors
        region.bdev = wc->ssd_dev->bdev;
        region.sector = wc->start_sector;
        region.count = n_sectors;
-       req.bi_op = REQ_OP_READ;
-       req.bi_op_flags = REQ_SYNC;
+       req.bi_opf = REQ_OP_READ | REQ_SYNC;
        req.mem.type = DM_IO_VMA;
        req.mem.ptr.vma = (char *)wc->memory_map;
        req.client = wc->dm_io;
@@ -1329,8 +1325,8 @@ enum wc_map_op {
        WC_MAP_ERROR,
 };
 
-static enum wc_map_op writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio,
-                                                 struct wc_entry *e)
+static void writecache_map_remap_origin(struct dm_writecache *wc, struct bio *bio,
+                                       struct wc_entry *e)
 {
        if (e) {
                sector_t next_boundary =
@@ -1338,8 +1334,6 @@ static enum wc_map_op writecache_map_remap_origin(struct dm_writecache *wc, stru
                if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT)
                        dm_accept_partial_bio(bio, next_boundary);
        }
-
-       return WC_MAP_REMAP_ORIGIN;
 }
 
 static enum wc_map_op writecache_map_read(struct dm_writecache *wc, struct bio *bio)
@@ -1366,14 +1360,16 @@ read_next_block:
                        map_op = WC_MAP_REMAP;
                }
        } else {
-               map_op = writecache_map_remap_origin(wc, bio, e);
+               writecache_map_remap_origin(wc, bio, e);
+               wc->stats.reads += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
+               map_op = WC_MAP_REMAP_ORIGIN;
        }
 
        return map_op;
 }
 
-static enum wc_map_op writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio,
-                                             struct wc_entry *e, bool search_used)
+static void writecache_bio_copy_ssd(struct dm_writecache *wc, struct bio *bio,
+                                   struct wc_entry *e, bool search_used)
 {
        unsigned bio_size = wc->block_size;
        sector_t start_cache_sec = cache_sector(wc, e);
@@ -1413,14 +1409,15 @@ static enum wc_map_op writecache_bio_copy_ssd(struct dm_writecache *wc, struct b
        bio->bi_iter.bi_sector = start_cache_sec;
        dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
 
+       wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
+       wc->stats.writes_allocate += (bio->bi_iter.bi_size - wc->block_size) >> wc->block_size_bits;
+
        if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
                wc->uncommitted_blocks = 0;
                queue_work(wc->writeback_wq, &wc->flush_work);
        } else {
                writecache_schedule_autocommit(wc);
        }
-
-       return WC_MAP_REMAP;
 }
 
 static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio *bio)
@@ -1430,9 +1427,10 @@ static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio
        do {
                bool found_entry = false;
                bool search_used = false;
-               wc->stats.writes++;
-               if (writecache_has_error(wc))
+               if (writecache_has_error(wc)) {
+                       wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
                        return WC_MAP_ERROR;
+               }
                e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
                if (e) {
                        if (!writecache_entry_is_committed(wc, e)) {
@@ -1456,9 +1454,11 @@ static enum wc_map_op writecache_map_write(struct dm_writecache *wc, struct bio
                if (unlikely(!e)) {
                        if (!WC_MODE_PMEM(wc) && !found_entry) {
 direct_write:
-                               wc->stats.writes_around++;
                                e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
-                               return writecache_map_remap_origin(wc, bio, e);
+                               writecache_map_remap_origin(wc, bio, e);
+                               wc->stats.writes_around += bio->bi_iter.bi_size >> wc->block_size_bits;
+                               wc->stats.writes += bio->bi_iter.bi_size >> wc->block_size_bits;
+                               return WC_MAP_REMAP_ORIGIN;
                        }
                        wc->stats.writes_blocked_on_freelist++;
                        writecache_wait_on_freelist(wc);
@@ -1469,10 +1469,13 @@ direct_write:
                wc->uncommitted_blocks++;
                wc->stats.writes_allocate++;
 bio_copy:
-               if (WC_MODE_PMEM(wc))
+               if (WC_MODE_PMEM(wc)) {
                        bio_copy_block(wc, bio, memory_data(wc, e));
-               else
-                       return writecache_bio_copy_ssd(wc, bio, e, search_used);
+                       wc->stats.writes++;
+               } else {
+                       writecache_bio_copy_ssd(wc, bio, e, search_used);
+                       return WC_MAP_REMAP;
+               }
        } while (bio->bi_iter.bi_size);
 
        if (unlikely(bio->bi_opf & REQ_FUA || wc->uncommitted_blocks >= wc->autocommit_blocks))
@@ -1507,7 +1510,7 @@ static enum wc_map_op writecache_map_flush(struct dm_writecache *wc, struct bio
 
 static enum wc_map_op writecache_map_discard(struct dm_writecache *wc, struct bio *bio)
 {
-       wc->stats.discards++;
+       wc->stats.discards += bio->bi_iter.bi_size >> wc->block_size_bits;
 
        if (writecache_has_error(wc))
                return WC_MAP_ERROR;
@@ -1591,7 +1594,8 @@ done:
 
        default:
                BUG();
-               return -1;
+               wc_unlock(wc);
+               return DM_MAPIO_KILL;
        }
 }