block: support uncached IO

Add a bio helper for deferring uncached writes to workqueue, like what
is done for bios that contain folios that need marking as dirty. The
read side of block fops is already fine, only the write needs special
handling to avoid IRQ context for uncached write completions.

Enable uncached buffered IO to block devices by setting FOP_UNCACHED.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bdev.c b/block/bdev.c
index 738e3c8457e7f430332e6852e2b1baa2883af326..6344a1050ae62adb1919c918c41f0c887cc2a50a 100644 (file)
--- a/block/bdev.c
+++ b/block/bdev.c
@@ -439,6 +439,9 @@ struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)
                iput(inode);
                return NULL;
        }
+       bdev->uncached_list = NULL;
+       INIT_WORK(&bdev->uncached_work, bio_uncache_work);
+       spin_lock_init(&bdev->uncached_lock);
        bdev->bd_disk = disk;
        return bdev;
 }

diff --git a/block/bio.c b/block/bio.c
index 699a78c85c75660fed2463759e2491bb12c63453..d93b47c869867c3c29c03001a2e17be2ccab1055 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -1522,8 +1522,43 @@ EXPORT_SYMBOL_GPL(bio_set_pages_dirty);
 static void bio_dirty_fn(struct work_struct *work);
 
 static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
-static DEFINE_SPINLOCK(bio_dirty_lock);
 static struct bio *bio_dirty_list;
+static DEFINE_SPINLOCK(bio_dirty_lock);
+
+void bio_uncache_work(struct work_struct *work)
+{
+       struct block_device *bdev;
+       struct bio *bio, *next;
+
+       bdev = container_of(work, struct block_device, uncached_work);
+       spin_lock_irq(&bdev->uncached_lock);
+       next = bdev->uncached_list;
+       bdev->uncached_list = NULL;
+       spin_unlock_irq(&bdev->uncached_lock);
+
+       while ((bio = next) != NULL) {
+               struct buffer_head *bh = bio->bi_private;
+
+               next = bio->bi_next;
+               bh->b_end_io(bh, !bio->bi_status);
+               bio_put(bio);
+       }
+}
+
+void bio_reap_uncached_write(struct bio *bio)
+{
+       struct block_device *bdev = bio->bi_bdev;
+       unsigned long flags;
+       bool was_empty;
+
+       spin_lock_irqsave(&bdev->uncached_lock, flags);
+       bio->bi_next = bdev->uncached_list;
+       bdev->uncached_list = bio;
+       was_empty = !bio->bi_next;
+       spin_unlock_irqrestore(&bdev->uncached_lock, flags);
+       if (was_empty)
+               schedule_work(&bdev->uncached_work);
+}
 
 /*
  * This runs in process context

diff --git a/block/fops.c b/block/fops.c
index 13a67940d0408d60130dd66604a5281f5ef06324..f617c66633acd318b1dab9b57131f032e561200c 100644 (file)
--- a/block/fops.c
+++ b/block/fops.c
@@ -872,7 +872,7 @@ const struct file_operations def_blk_fops = {
        .splice_write   = iter_file_splice_write,
        .fallocate      = blkdev_fallocate,
        .uring_cmd      = blkdev_uring_cmd,
-       .fop_flags      = FOP_BUFFER_RASYNC,
+       .fop_flags      = FOP_BUFFER_RASYNC | FOP_UNCACHED,
 };
 
 static __init int blkdev_init(void)

diff --git a/fs/buffer.c b/fs/buffer.c
index cc8452f6025167630bfe42d8f6652c74adfbdedd..940c536529c16f6679460085fc89e8731f948375 100644 (file)
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -2763,8 +2763,13 @@ static void end_bio_bh_io_sync(struct bio *bio)
        if (unlikely(bio_flagged(bio, BIO_QUIET)))
                set_bit(BH_Quiet, &bh->b_state);
 
-       bh->b_end_io(bh, !bio->bi_status);
-       bio_put(bio);
+       if (op_is_write(bio_op(bio)) &&
+           folio_test_uncached(page_folio(bh->b_page))) {
+               bio_reap_uncached_write(bio);
+       } else {
+               bh->b_end_io(bh, !bio->bi_status);
+               bio_put(bio);
+       }
 }
 
 static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh,

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 60830a6a59394f546d3e87d3fd93517038b24a42..71731551a5a4dc0c19b1de027703233e18457f77 100644 (file)
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -427,6 +427,8 @@ void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter);
 void __bio_release_pages(struct bio *bio, bool mark_dirty);
 extern void bio_set_pages_dirty(struct bio *bio);
 extern void bio_check_pages_dirty(struct bio *bio);
+void bio_uncache_work(struct work_struct *work);
+void bio_reap_uncached_write(struct bio *bio);
 
 extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
                               struct bio *src, struct bvec_iter *src_iter);

diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index dce7615c35e7e34b4091b30f095b6b1655b0eb39..44a536ca98f43a26098a4a0d583eab0fa4a988ff 100644 (file)
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -74,6 +74,14 @@ struct block_device {
 #ifdef CONFIG_SECURITY
        void                    *bd_security;
 #endif
+
+       /*
+        * For punting of uncached buffered writes to a workqueue context
+        */
+       struct bio              *uncached_list;
+       spinlock_t              uncached_lock;
+       struct work_struct      uncached_work;
+
        /*
         * keep this out-of-line as it's both big and not needed in the fast
         * path