From 89ad7923d6702d0c8fc2b0ec97998b395f2fa03a Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 18 Aug 2021 19:38:58 -0600 Subject: [PATCH] io_uring: support bio caching for non-polled IO Mark the kiocb with IOCB_ALLOC_CACHE even for non-polled IO, in case the lower layer participates in per-cpu bio caching. If it does, then IOCB_PUT_CACHE will be set upon kiocb->ki_complete() invocation, passing ownership to io_uring. io_uring doesn't complete even IRQ based requests from IRQ context, so we can safely put the bio when we run the actual io_kiocb completion. This provides a 5-10% boost in IOPS with IRQ driven IO. Signed-off-by: Jens Axboe --- fs/io_uring.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index a3628418bb84..d8ef3abf1aa4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2533,7 +2533,12 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) static void io_req_task_complete(struct io_kiocb *req) { int cflags = 0; +#ifdef CONFIG_BLOCK + struct kiocb *kiocb = &req->rw.kiocb; + if (kiocb->ki_flags & IOCB_PUT_CACHE) + bio_put(kiocb->private); +#endif if (req->flags & REQ_F_BUFFER_SELECTED) cflags = io_put_rw_kbuf(req); __io_req_complete(req, 0, req->result, cflags); @@ -2741,6 +2746,13 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) } else { if (kiocb->ki_flags & IOCB_HIPRI) return -EINVAL; + /* + * IRQ driven IO can participate in the bio alloc cache, since + * we don't complete from IRQ anymore. This requires the caller + * to pass back ownership of the bio before calling ki_complete, + * and then ki_complete will put it from a safe context. + */ + kiocb->ki_flags |= IOCB_ALLOC_CACHE; kiocb->ki_complete = io_complete_rw; } -- 2.25.1