io_uring/rsrc: add custom limit for node caching

The number of entries in the rsrc node cache is limited to 512, which
still seems unnecessarily large. Add per cache thresholds and set to
to 32 for the rsrc node cache.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/d0cd538b944dac0bf878e276fc0199f21e6bccea.1680576071.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 5d772e36e7fc686ab7130fb53eace4b45548bb4f..4a6ce03a490393055293b5c2e14f77656b268409 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -190,6 +190,7 @@ struct io_ev_fd {
 struct io_alloc_cache {
        struct io_wq_work_node  list;
        unsigned int            nr_cached;
+       unsigned int            max_cached;
        size_t                  elem_size;
 };
 

diff --git a/io_uring/alloc_cache.h b/io_uring/alloc_cache.h
index 2fbecaa3a1ba4053d9b53e1b0a686a240a6b9c4e..851a527afb5e1626ef86903a03eb21b85aa7c6c9 100644 (file)
--- a/io_uring/alloc_cache.h
+++ b/io_uring/alloc_cache.h
@@ -13,7 +13,7 @@ struct io_cache_entry {
 static inline bool io_alloc_cache_put(struct io_alloc_cache *cache,
                                      struct io_cache_entry *entry)
 {
-       if (cache->nr_cached < IO_ALLOC_CACHE_MAX) {
+       if (cache->nr_cached < cache->max_cached) {
                cache->nr_cached++;
                wq_stack_add_head(&entry->node, &cache->list);
                /* KASAN poisons object */
@@ -38,10 +38,12 @@ static inline struct io_cache_entry *io_alloc_cache_get(struct io_alloc_cache *c
        return NULL;
 }
 
-static inline void io_alloc_cache_init(struct io_alloc_cache *cache, size_t size)
+static inline void io_alloc_cache_init(struct io_alloc_cache *cache,
+                                      unsigned max_nr, size_t size)
 {
        cache->list.next = NULL;
        cache->nr_cached = 0;
+       cache->max_cached = max_nr;
        cache->elem_size = size;
 }
 

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index da36fa1eeac90fa6cffb6803bf0d423e538b1097..ae90d2753e0db6897d8a98b9f60de92833229ba3 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -310,9 +310,12 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
        INIT_LIST_HEAD(&ctx->sqd_list);
        INIT_LIST_HEAD(&ctx->cq_overflow_list);
        INIT_LIST_HEAD(&ctx->io_buffers_cache);
-       io_alloc_cache_init(&ctx->rsrc_node_cache, sizeof(struct io_rsrc_node));
-       io_alloc_cache_init(&ctx->apoll_cache, sizeof(struct async_poll));
-       io_alloc_cache_init(&ctx->netmsg_cache, sizeof(struct io_async_msghdr));
+       io_alloc_cache_init(&ctx->rsrc_node_cache, IO_NODE_ALLOC_CACHE_MAX,
+                           sizeof(struct io_rsrc_node));
+       io_alloc_cache_init(&ctx->apoll_cache, IO_ALLOC_CACHE_MAX,
+                           sizeof(struct async_poll));
+       io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
+                           sizeof(struct io_async_msghdr));
        init_completion(&ctx->ref_comp);
        xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
        mutex_init(&ctx->uring_lock);

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 7ab9b2b2e7570027576fb148e208266aa6106c8e..8729f2fee2567c0f37301f5ca006553ff83a8ed5 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -6,6 +6,8 @@
 
 #include "alloc_cache.h"
 
+#define IO_NODE_ALLOC_CACHE_MAX 32
+
 #define IO_RSRC_TAG_TABLE_SHIFT        (PAGE_SHIFT - 3)
 #define IO_RSRC_TAG_TABLE_MAX  (1U << IO_RSRC_TAG_TABLE_SHIFT)
 #define IO_RSRC_TAG_TABLE_MASK (IO_RSRC_TAG_TABLE_MAX - 1)