summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJens Axboe <axboe@kernel.dk>2021-08-18 19:35:25 -0600
committerJens Axboe <axboe@kernel.dk>2021-08-18 21:22:10 -0600
commitfacf46c9a78c920229e4ad242910cd9c4688ced8 (patch)
tree9d9bd12a3d9c10b7423d98eebaace1acdef25b1e
parentb5599bb5cd57ea6b2f240bc01df14359689ae538 (diff)
block: add support for passing back a bio for non-polled dio
We support cached bio allocations and frees from polled IO, since it's trivial to do when we know IRQs are not going to be involved. However, some issuers of non-polled dio always complete from task context. That means we can support bio caching for that as well. Add a IOCB_PUT_CACHE flag that allows to pass back a bio through the kiocb, passing ownership of that bio to the kiocb completion handling. Signed-off-by: Jens Axboe <axboe@kernel.dk>
-rw-r--r--fs/block_dev.c20
-rw-r--r--include/linux/fs.h2
2 files changed, 19 insertions, 3 deletions
diff --git a/fs/block_dev.c b/fs/block_dev.c
index 3c7fb7106713..2953c4119ac5 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -331,6 +331,7 @@ static void blkdev_bio_end_io(struct bio *bio)
{
struct blkdev_dio *dio = bio->bi_private;
bool should_dirty = dio->should_dirty;
+ bool free_bio = true;
if (bio->bi_status && !dio->bio.bi_status)
dio->bio.bi_status = bio->bi_status;
@@ -346,8 +347,20 @@ static void blkdev_bio_end_io(struct bio *bio)
} else {
ret = blk_status_to_errno(dio->bio.bi_status);
}
-
- dio->iocb->ki_complete(iocb, ret, 0);
+ /*
+ * If IRQ driven and not using multi-bio, pass
+ * ownership of bio to issuer for task-based free. Then
+ * we can participate in the cached bio allocations, if
+ * the caller marked the kiocb IOCB_ALLOC_CACHE.
+ */
+ if (!should_dirty && !dio->multi_bio &&
+ (iocb->ki_flags & (IOCB_ALLOC_CACHE|IOCB_HIPRI)) ==
+ IOCB_ALLOC_CACHE) {
+ iocb->ki_flags |= IOCB_PUT_CACHE;
+ iocb->private = bio;
+ free_bio = false;
+ }
+ iocb->ki_complete(iocb, ret, 0);
if (dio->multi_bio)
bio_put(&dio->bio);
} else {
@@ -362,7 +375,8 @@ static void blkdev_bio_end_io(struct bio *bio)
bio_check_pages_dirty(bio);
} else {
bio_release_pages(bio, false);
- bio_put(bio);
+ if (free_bio)
+ bio_put(bio);
}
}
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 0dcc5de779c9..79297b13a4bd 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -321,6 +321,8 @@ enum rw_hint {
#define IOCB_NOIO (1 << 20)
/* can use bio alloc cache */
#define IOCB_ALLOC_CACHE (1 << 21)
+/* bio ownership (and put) passed back to caller */
+#define IOCB_PUT_CACHE (1 << 22)
struct kiocb {
struct file *ki_filp;