From 6a660351179109255ae971bb9883d20335a71d24 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 27 Oct 2024 09:19:51 -0600 Subject: [PATCH] io_uring/rsrc: add last-lookup cache hit to io_rsrc_node_lookup() This avoids array_index_nospec() for repeated lookups on the same node, which can be quite common (and costly). If a cached node is removed from the given table, it'll get cleared in the cache as well. io_reset_rsrc_node() takes care of that, which is used in the spots that's replacing a node. Note: need to double check this is 100% safe wrt speculation, but I believe it should be as we're not using the passed in value to index any arrays (directly). Signed-off-by: Jens Axboe --- include/linux/io_uring_types.h | 2 ++ io_uring/rsrc.c | 3 +++ io_uring/rsrc.h | 20 +++++++++++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 77fd508d043a..c283179b0c89 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -57,6 +57,8 @@ struct io_wq_work { struct io_rsrc_data { unsigned int nr; + unsigned int last_index; + struct io_rsrc_node *last_node; struct io_rsrc_node **nodes; }; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 150a14284e5c..555769dc2487 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -151,6 +151,8 @@ __cold void io_rsrc_data_free(struct io_rsrc_data *data) if (data->nodes[data->nr]) io_put_rsrc_node(data->nodes[data->nr]); } + data->last_node = NULL; + data->last_index = -1U; kvfree(data->nodes); data->nodes = NULL; data->nr = 0; @@ -162,6 +164,7 @@ __cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr) GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (data->nodes) { data->nr = nr; + data->last_index = -1U; return 0; } return -ENOMEM; diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index abd214f303f5..5026a1c10687 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -74,8 +74,16 @@ extern const struct io_rsrc_node empty_node; static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data, int index) { - if (index < data->nr) - return data->nodes[array_index_nospec(index, data->nr)]; + if (index < data->nr) { + if (index != data->last_index) { + index = array_index_nospec(index, data->nr); + if (data->nodes[index]) { + data->last_index = index; + data->last_node = data->nodes[index]; + } + } + return data->last_node; + } return NULL; } @@ -89,8 +97,14 @@ static inline bool io_reset_rsrc_node(struct io_rsrc_data *data, int index) { struct io_rsrc_node *node = data->nodes[index]; - if (!node) + if (!node) { + WARN_ON_ONCE(index == data->last_index); return false; + } + if (index == data->last_index) { + data->last_node = NULL; + data->last_index = -1U; + } io_put_rsrc_node(node); data->nodes[index] = NULL; return true; -- 2.25.1