io_uring/rsrc: add last-lookup cache hit to io_rsrc_node_lookup()

This avoids array_index_nospec() for repeated lookups on the same node,
which can be quite common (and costly). When the node sees its last
put, it'll clear the cache as well.

Note: need to double check this is 100% safe wrt speculation, but I
believe it should be as we're not using the passed in value to index
any values (directly).

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 77fd508d043ae6b29e2b142071bdc14b6dc7d3ed..c283179b0c89de0ec8da733a7d0d43e0d0a3ed99 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -57,6 +57,8 @@ struct io_wq_work {
 
 struct io_rsrc_data {
        unsigned int                    nr;
+       unsigned int                    last_index;
+       struct io_rsrc_node             *last_node;
        struct io_rsrc_node             **nodes;
 };
 

diff --git a/io_uring/filetable.c b/io_uring/filetable.c
index 7b6e4df7cef9f952964e6baaba38078cabc5fb0c..78e77e2017bedf5dfcb960e114e31a110aff958c 100644 (file)
--- a/io_uring/filetable.c
+++ b/io_uring/filetable.c
@@ -74,7 +74,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
 
        old_node = io_rsrc_node_lookup(&ctx->file_table.data, &slot_index);
        if (old_node)
-               io_put_rsrc_node(old_node);
+               io_reset_rsrc_node(&ctx->file_table.data, slot_index);
        else
                io_file_bitmap_set(&ctx->file_table, slot_index);
 
@@ -134,8 +134,7 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, int offset)
        node = io_rsrc_node_lookup(&ctx->file_table.data, &offset);
        if (!node)
                return -EBADF;
-       io_put_rsrc_node(node);
-       ctx->file_table.data.nodes[offset] = NULL;
+       io_reset_rsrc_node(&ctx->file_table.data, offset);
        io_file_bitmap_clear(&ctx->file_table, offset);
        return 0;
 }

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index f2242f3866fb7d9d3c0f632989fb21fe33065110..ab53893dde44f7233dddfeabf1c49a4336c75423 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -166,6 +166,7 @@ __cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr)
                                        GFP_KERNEL_ACCOUNT | __GFP_ZERO);
        if (data->nodes) {
                data->nr = nr;
+               data->last_index = -1U;
                return 0;
        }
        return -ENOMEM;
@@ -204,8 +205,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
                index = up->offset + done;
                node = io_rsrc_node_lookup(&ctx->file_table.data, &index);
                if (node) {
-                       io_put_rsrc_node(node);
-                       ctx->file_table.data.nodes[index] = NULL;
+                       io_reset_rsrc_node(&ctx->file_table.data, index);
                        io_file_bitmap_clear(&ctx->file_table, index);
                }
                if (fd != -1) {
@@ -286,7 +286,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
                        break;
                }
                if (ctx->buf_table.nodes[i])
-                       io_put_rsrc_node(ctx->buf_table.nodes[i]);
+                       io_reset_rsrc_node(&ctx->buf_table, i);
 
                ctx->buf_table.nodes[i] = node;
                if (tag)
@@ -476,10 +476,12 @@ void io_free_rsrc_node(struct io_rsrc_node *node)
 
        switch (node->type) {
        case IORING_RSRC_FILE:
+               io_rsrc_cache_clear(&ctx->file_table.data, node);
                if (io_slot_file(node))
                        fput(io_slot_file(node));
                break;
        case IORING_RSRC_BUFFER:
+               io_rsrc_cache_clear(&ctx->buf_table, node);
                if (node->buf)
                        io_buffer_unmap(node->ctx, node);
                break;

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 5897306bcc35b1bc43042575c92aa9c90dcb84ae..617b4077dfd359d48c390f6b0133eb9936cb1495 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -78,18 +78,44 @@ static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data
                                                       int *index)
 {
        if (*index < data->nr) {
-               *index = array_index_nospec(*index, data->nr);
-               return data->nodes[*index];
+               if (*index != data->last_index) {
+                       *index = array_index_nospec(*index, data->nr);
+                       if (data->nodes[*index]) {
+                               data->last_index = *index;
+                               data->last_node = data->nodes[*index];
+                       }
+               }
+               return data->last_node;
        }
        return NULL;
 }
 
+static inline void io_rsrc_cache_clear(struct io_rsrc_data *data,
+                                      struct io_rsrc_node *node)
+{
+       if (data->last_node == node) {
+               data->last_node = NULL;
+               data->last_index = -1U;
+       }
+}
+
 static inline void io_put_rsrc_node(struct io_rsrc_node *node)
 {
        if (node != &empty_node && !--node->refs)
                io_free_rsrc_node(node);
 }
 
+static inline void io_reset_rsrc_node(struct io_rsrc_data *data, int index)
+{
+       struct io_rsrc_node *node = data->nodes[index];
+
+       if (node) {
+               io_rsrc_cache_clear(data, node);
+               io_put_rsrc_node(node);
+               data->nodes[index] = NULL;
+       }
+}
+
 static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)
 {
        if (req->rsrc_nodes[IORING_RSRC_FILE] != rsrc_empty_node) {