io_uring: support bio caching for non-polled IO

Mark the kiocb with IOCB_ALLOC_CACHE even for non-polled IO, in case
the lower layer participates in per-cpu bio caching. If it does, then
IOCB_PUT_CACHE will be set upon kiocb->ki_complete() invocation,
passing ownership to io_uring.

io_uring doesn't complete even IRQ based requests from IRQ context,
so we can safely put the bio when we run the actual io_kiocb completion.

This provides a 5-10% boost in IOPS with IRQ driven IO.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index a3628418bb84001f76bb6008eb559c6161cbb1e2..d8ef3abf1aa4b3f23a5211cd1e9f43ac7233e253 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2533,7 +2533,12 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res)
 static void io_req_task_complete(struct io_kiocb *req)
 {
        int cflags = 0;
+#ifdef CONFIG_BLOCK
+       struct kiocb *kiocb = &req->rw.kiocb;
 
+       if (kiocb->ki_flags & IOCB_PUT_CACHE)
+               bio_put(kiocb->private);
+#endif
        if (req->flags & REQ_F_BUFFER_SELECTED)
                cflags = io_put_rw_kbuf(req);
        __io_req_complete(req, 0, req->result, cflags);
@@ -2741,6 +2746,13 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
        } else {
                if (kiocb->ki_flags & IOCB_HIPRI)
                        return -EINVAL;
+               /*
+                * IRQ driven IO can participate in the bio alloc cache, since
+                * we don't complete from IRQ anymore. This requires the caller
+                * to pass back ownership of the bio before calling ki_complete,
+                * and then ki_complete will put it from a safe context.
+                */
+               kiocb->ki_flags |= IOCB_ALLOC_CACHE;
                kiocb->ki_complete = io_complete_rw;
        }