io_uring/rsrc: add last-lookup cache hit to io_rsrc_node_lookup()

This avoids array_index_nospec() for repeated lookups on the same node,
which can be quite common (and costly). If a cached node is removed from
the given table, it'll get cleared in the cache as well.
io_reset_rsrc_node() takes care of that, which is used in the spots
that's replacing a node.

Note: need to double check this is 100% safe wrt speculation, but I
believe it should be as we're not using the passed in value to index
any arrays (directly).

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 77fd508d043ae6b29e2b142071bdc14b6dc7d3ed..c283179b0c89de0ec8da733a7d0d43e0d0a3ed99 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -57,6 +57,8 @@ struct io_wq_work {
 
 struct io_rsrc_data {
        unsigned int                    nr;
+       unsigned int                    last_index;
+       struct io_rsrc_node             *last_node;
        struct io_rsrc_node             **nodes;
 };
 

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 150a14284e5cb080cb3ded133534e388a7f96473..555769dc248788107e6c6ee9fdcac06bb9a32be4 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -151,6 +151,8 @@ __cold void io_rsrc_data_free(struct io_rsrc_data *data)
                if (data->nodes[data->nr])
                        io_put_rsrc_node(data->nodes[data->nr]);
        }
+       data->last_node = NULL;
+       data->last_index = -1U;
        kvfree(data->nodes);
        data->nodes = NULL;
        data->nr = 0;
@@ -162,6 +164,7 @@ __cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr)
                                        GFP_KERNEL_ACCOUNT | __GFP_ZERO);
        if (data->nodes) {
                data->nr = nr;
+               data->last_index = -1U;
                return 0;
        }
        return -ENOMEM;

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index abd214f303f581192a2e11fbe6c6119aa3a2f325..5026a1c1068708669c88e770afe4b88fbbaed208 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -74,8 +74,16 @@ extern const struct io_rsrc_node empty_node;
 static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data,
                                                       int index)
 {
-       if (index < data->nr)
-               return data->nodes[array_index_nospec(index, data->nr)];
+       if (index < data->nr) {
+               if (index != data->last_index) {
+                       index = array_index_nospec(index, data->nr);
+                       if (data->nodes[index]) {
+                               data->last_index = index;
+                               data->last_node = data->nodes[index];
+                       }
+               }
+               return data->last_node;
+       }
        return NULL;
 }
 
@@ -89,8 +97,14 @@ static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index)
 {
        struct io_rsrc_node *node = data->nodes[index];
 
-       if (!node)
+       if (!node) {
+               WARN_ON_ONCE(index == data->last_index);
                return false;
+       }
+       if (index == data->last_index) {
+               data->last_node = NULL;
+               data->last_index = -1U;
+       }
        io_put_rsrc_node(node);
        data->nodes[index] = NULL;
        return true;