io_uring: refactor req allocation

Follow the io_get_sqe pattern returning the result via a pointer
and hide request cache refill inside io_alloc_req().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/8c37c2e8a3cb5e4cd6a8ae3b91371227a92708a6.1674484266.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index c1f27626c20526fd65571b3766266d559e8d2cc4..2b046a80f75b139295e4823b123269f2790306cd 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2415,9 +2415,8 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
                const struct io_uring_sqe *sqe;
                struct io_kiocb *req;
 
-               if (unlikely(!io_alloc_req_refill(ctx)))
+               if (unlikely(!io_alloc_req(ctx, &req)))
                        break;
-               req = io_alloc_req(ctx);
                if (unlikely(!io_get_sqe(ctx, &sqe))) {
                        io_req_add_to_cache(req, ctx);
                        break;
@@ -2736,14 +2735,14 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx)
 
 static void io_req_caches_free(struct io_ring_ctx *ctx)
 {
+       struct io_kiocb *req;
        int nr = 0;
 
        mutex_lock(&ctx->uring_lock);
        io_flush_cached_locked_reqs(ctx, &ctx->submit_state);
 
        while (!io_req_cache_empty(ctx)) {
-               struct io_kiocb *req = io_alloc_req(ctx);
-
+               req = io_extract_req(ctx);
                kmem_cache_free(req_cachep, req);
                nr++;
        }

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 9270156288aade58db3963477cab9020c5a64699..f90816aac8961463c98e3f0ad7d3a67c56b019f2 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -341,16 +341,9 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx)
        return !ctx->submit_state.free_list.next;
 }
 
-static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx)
-{
-       if (unlikely(io_req_cache_empty(ctx)))
-               return __io_alloc_req_refill(ctx);
-       return true;
-}
-
 extern struct kmem_cache *req_cachep;
 
-static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
+static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx)
 {
        struct io_kiocb *req;
 
@@ -360,6 +353,16 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
        return req;
 }
 
+static inline bool io_alloc_req(struct io_ring_ctx *ctx, struct io_kiocb **req)
+{
+       if (unlikely(io_req_cache_empty(ctx))) {
+               if (!__io_alloc_req_refill(ctx))
+                       return false;
+       }
+       *req = io_extract_req(ctx);
+       return true;
+}
+
 static inline bool io_allowed_defer_tw_run(struct io_ring_ctx *ctx)
 {
        return likely(ctx->submitter_task == current);

diff --git a/io_uring/notif.c b/io_uring/notif.c
index c4bb793ebf0ec1b019fee3df17164190409c4368..09dfd0832d19f9bc022d68919bea10c37a55dd43 100644 (file)
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -68,9 +68,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
        struct io_kiocb *notif;
        struct io_notif_data *nd;
 
-       if (unlikely(!io_alloc_req_refill(ctx)))
+       if (unlikely(!io_alloc_req(ctx, &notif)))
                return NULL;
-       notif = io_alloc_req(ctx);
        notif->opcode = IORING_OP_NOP;
        notif->flags = 0;
        notif->file = NULL;