io_uring/kbuf: remove legacy kbuf caching

Remove all struct io_buffer caches. It makes it a fair bit simpler.
Apart from from killing a bunch of lines and juggling between lists,
__io_put_kbuf_list() doesn't need ->completion_lock locking now.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/18287217466ee2576ea0b1e72daccf7b22c7e856.1738724373.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 3def525a1da3753de262cad8ade7d59fb67d7139..e2fef264ff8b8efceb1cc66abd7fca362286ce84 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -360,7 +360,6 @@ struct io_ring_ctx {
 
        spinlock_t              completion_lock;
 
-       struct list_head        io_buffers_comp;
        struct list_head        cq_overflow_list;
 
        struct hlist_head       waitid_list;
@@ -379,8 +378,6 @@ struct io_ring_ctx {
        unsigned int            file_alloc_start;
        unsigned int            file_alloc_end;
 
-       struct list_head        io_buffers_cache;
-
        /* Keep this last, we don't need it for the fast path */
        struct wait_queue_head          poll_wq;
        struct io_restriction           restrictions;

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index ed7c9081352a48b3ae4bad13d663668e07b9f352..969caaccce9d8cc28cc5ee12bff249a41f38f413 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -323,7 +323,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
        init_waitqueue_head(&ctx->sqo_sq_wait);
        INIT_LIST_HEAD(&ctx->sqd_list);
        INIT_LIST_HEAD(&ctx->cq_overflow_list);
-       INIT_LIST_HEAD(&ctx->io_buffers_cache);
        ret = io_alloc_cache_init(&ctx->apoll_cache, IO_POLL_ALLOC_CACHE_MAX,
                            sizeof(struct async_poll), 0);
        ret |= io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
@@ -348,7 +347,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
        spin_lock_init(&ctx->completion_lock);
        raw_spin_lock_init(&ctx->timeout_lock);
        INIT_WQ_LIST(&ctx->iopoll_list);
-       INIT_LIST_HEAD(&ctx->io_buffers_comp);
        INIT_LIST_HEAD(&ctx->defer_list);
        INIT_LIST_HEAD(&ctx->timeout_list);
        INIT_LIST_HEAD(&ctx->ltimeout_list);

diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index 3a43af9f7061f24fb01a7ba2b10ad9fa8fbdf680..caf5b9bb2aecc529e493af59c69718a1253a60b3 100644 (file)
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -70,9 +70,7 @@ bool io_kbuf_recycle_legacy(struct io_kiocb *req, unsigned issue_flags)
 
 void __io_put_kbuf(struct io_kiocb *req, int len, unsigned issue_flags)
 {
-       spin_lock(&req->ctx->completion_lock);
        __io_put_kbuf_list(req, len);
-       spin_unlock(&req->ctx->completion_lock);
 }
 
 static void __user *io_provided_buffer_select(struct io_kiocb *req, size_t *len,
@@ -345,7 +343,9 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx,
                struct io_buffer *nxt;
 
                nxt = list_first_entry(&bl->buf_list, struct io_buffer, list);
-               list_move(&nxt->list, &ctx->io_buffers_cache);
+               list_del(&nxt->list);
+               kfree(nxt);
+
                if (++i == nbufs)
                        return i;
                cond_resched();
@@ -363,8 +363,6 @@ static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
 void io_destroy_buffers(struct io_ring_ctx *ctx)
 {
        struct io_buffer_list *bl;
-       struct list_head *item, *tmp;
-       struct io_buffer *buf;
 
        while (1) {
                unsigned long index = 0;
@@ -378,19 +376,6 @@ void io_destroy_buffers(struct io_ring_ctx *ctx)
                        break;
                io_put_bl(ctx, bl);
        }
-
-       /*
-        * Move deferred locked entries to cache before pruning
-        */
-       spin_lock(&ctx->completion_lock);
-       if (!list_empty(&ctx->io_buffers_comp))
-               list_splice_init(&ctx->io_buffers_comp, &ctx->io_buffers_cache);
-       spin_unlock(&ctx->completion_lock);
-
-       list_for_each_safe(item, tmp, &ctx->io_buffers_cache) {
-               buf = list_entry(item, struct io_buffer, list);
-               kfree(buf);
-       }
 }
 
 static void io_destroy_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl)
@@ -479,33 +464,6 @@ int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe
        return 0;
 }
 
-static int io_refill_buffer_cache(struct io_ring_ctx *ctx)
-{
-       struct io_buffer *buf;
-
-       /*
-        * Completions that don't happen inline (eg not under uring_lock) will
-        * add to ->io_buffers_comp. If we don't have any free buffers, check
-        * the completion list and splice those entries first.
-        */
-       if (!list_empty_careful(&ctx->io_buffers_comp)) {
-               spin_lock(&ctx->completion_lock);
-               if (!list_empty(&ctx->io_buffers_comp)) {
-                       list_splice_init(&ctx->io_buffers_comp,
-                                               &ctx->io_buffers_cache);
-                       spin_unlock(&ctx->completion_lock);
-                       return 0;
-               }
-               spin_unlock(&ctx->completion_lock);
-       }
-
-       buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);
-       if (!buf)
-               return -ENOMEM;
-       list_add_tail(&buf->list, &ctx->io_buffers_cache);
-       return 0;
-}
-
 static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
                          struct io_buffer_list *bl)
 {
@@ -514,12 +472,11 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf,
        int i, bid = pbuf->bid;
 
        for (i = 0; i < pbuf->nbufs; i++) {
-               if (list_empty(&ctx->io_buffers_cache) &&
-                   io_refill_buffer_cache(ctx))
+               buf = kmalloc(sizeof(*buf), GFP_KERNEL_ACCOUNT);
+               if (!buf)
                        break;
-               buf = list_first_entry(&ctx->io_buffers_cache, struct io_buffer,
-                                       list);
-               list_move_tail(&buf->list, &bl->buf_list);
+
+               list_add_tail(&buf->list, &bl->buf_list);
                buf->addr = addr;
                buf->len = min_t(__u32, pbuf->len, MAX_RW_COUNT);
                buf->bid = bid;

diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h
index 1f2877064829804b7e02af437a359db57d196a8a..c0b9636c5c4ae10671908b44747a0e5558c62e42 100644 (file)
--- a/io_uring/kbuf.h
+++ b/io_uring/kbuf.h
@@ -166,8 +166,9 @@ static inline void __io_put_kbuf_list(struct io_kiocb *req, int len)
                __io_put_kbuf_ring(req, len, 1);
        } else {
                req->buf_index = req->kbuf->bgid;
-               list_add(&req->kbuf->list, &req->ctx->io_buffers_comp);
                req->flags &= ~REQ_F_BUFFER_SELECTED;
+               kfree(req->kbuf);
+               req->kbuf = NULL;
        }
 }
 
@@ -176,10 +177,8 @@ static inline void io_kbuf_drop(struct io_kiocb *req)
        if (!(req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)))
                return;
 
-       spin_lock(&req->ctx->completion_lock);
        /* len == 0 is fine here, non-ring will always drop all of it */
        __io_put_kbuf_list(req, 0);
-       spin_unlock(&req->ctx->completion_lock);
 }
 
 static inline unsigned int __io_put_kbufs(struct io_kiocb *req, int len,