block: add REQ_HIPRI and inherit it from IOCB_HIPRI

We use IOCB_HIPRI to poll for IO in the caller instead of scheduling.
This information is not available for (or after) IO submission. The
driver may make different queue choices based on the type of IO, so
make the fact that we will poll for this IO known to the lower layers
as well.

Reviewed-by: Hannes Reinecke <hare@suse.com>
Reviewed-by: Keith Busch <keith.busch@intel.com>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/block_dev.c b/fs/block_dev.c
index a80b4f0ee7c4f172d19b4df1c8c330e4c0738557..c039abfb205203e1cf250d49e8a2c5887a23b310 100644 (file)
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -232,6 +232,8 @@ __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
                bio.bi_opf = dio_bio_write_op(iocb);
                task_io_account_write(ret);
        }
+       if (iocb->ki_flags & IOCB_HIPRI)
+               bio.bi_opf |= REQ_HIPRI;
 
        qc = submit_bio(&bio);
        for (;;) {

diff --git a/fs/direct-io.c b/fs/direct-io.c
index 722d17c88edb9016ac910072f4f758605089ec83..ea07d5a34317dcf2a644a5942d4fa8736c7a23b1 100644 (file)
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -1265,6 +1265,8 @@ do_blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
        } else {
                dio->op = REQ_OP_READ;
        }
+       if (iocb->ki_flags & IOCB_HIPRI)
+               dio->op_flags |= REQ_HIPRI;
 
        /*
         * For AIO O_(D)SYNC writes we need to defer completions to a workqueue

diff --git a/fs/iomap.c b/fs/iomap.c
index 64ce240217a18985dd0510fd968b257dce214cd4..f61d13dfdf09583403e8402962ce373d68c03d19 100644 (file)
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -1553,6 +1553,7 @@ iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos,
                unsigned len)
 {
        struct page *page = ZERO_PAGE(0);
+       int flags = REQ_SYNC | REQ_IDLE;
        struct bio *bio;
 
        bio = bio_alloc(GFP_KERNEL, 1);
@@ -1561,9 +1562,12 @@ iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos,
        bio->bi_private = dio;
        bio->bi_end_io = iomap_dio_bio_end_io;
 
+       if (dio->iocb->ki_flags & IOCB_HIPRI)
+               flags |= REQ_HIPRI;
+
        get_page(page);
        __bio_add_page(bio, page, len, 0);
-       bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC | REQ_IDLE);
+       bio_set_op_attrs(bio, REQ_OP_WRITE, flags);
 
        atomic_inc(&dio->ref);
        return submit_bio(bio);
@@ -1662,6 +1666,9 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length,
                                bio_set_pages_dirty(bio);
                }
 
+               if (dio->iocb->ki_flags & IOCB_HIPRI)
+                       bio->bi_opf |= REQ_HIPRI;
+
                iov_iter_advance(dio->submit.iter, n);
 
                dio->size += n;

diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 1dcf652ba0aa3e989a7b93e78e4303aa920a1dab..dbdbfbd6a9878ea9e1280adf915335906f3921b8 100644 (file)
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -323,6 +323,8 @@ enum req_flag_bits {
        /* command specific flags for REQ_OP_WRITE_ZEROES: */
        __REQ_NOUNMAP,          /* do not free blocks when zeroing */
 
+       __REQ_HIPRI,
+
        /* for driver use */
        __REQ_DRV,
        __REQ_SWAP,             /* swapping request. */
@@ -343,8 +345,8 @@ enum req_flag_bits {
 #define REQ_RAHEAD             (1ULL << __REQ_RAHEAD)
 #define REQ_BACKGROUND         (1ULL << __REQ_BACKGROUND)
 #define REQ_NOWAIT             (1ULL << __REQ_NOWAIT)
-
 #define REQ_NOUNMAP            (1ULL << __REQ_NOUNMAP)
+#define REQ_HIPRI              (1ULL << __REQ_HIPRI)
 
 #define REQ_DRV                        (1ULL << __REQ_DRV)
 #define REQ_SWAP               (1ULL << __REQ_SWAP)