From: Jens Axboe <axboe@kernel.dk>
Date: Sun, 27 Oct 2024 15:19:51 +0000 (-0600)
Subject: io_uring/rsrc: add last-lookup cache hit to io_rsrc_node_lookup()
X-Git-Url: https://git.kernel.dk/?a=commitdiff_plain;h=refs%2Fheads%2Fio_uring-rsrc.1;p=linux-2.6-block.git

io_uring/rsrc: add last-lookup cache hit to io_rsrc_node_lookup()

This avoids array_index_nospec() for repeated lookups on the same node,
which can be quite common (and costly). When the node sees its last
put, it'll clear the cache as well.

Note: need to double check this is 100% safe wrt speculation, but I
believe it should be as we're not using the passed in value to index
any values (directly).

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 77fd508d043a..c283179b0c89 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -57,6 +57,8 @@ struct io_wq_work {
 
 struct io_rsrc_data {
 	unsigned int			nr;
+	unsigned int			last_index;
+	struct io_rsrc_node		*last_node;
 	struct io_rsrc_node		**nodes;
 };
 
diff --git a/io_uring/filetable.c b/io_uring/filetable.c
index 7b6e4df7cef9..78e77e2017be 100644
--- a/io_uring/filetable.c
+++ b/io_uring/filetable.c
@@ -74,7 +74,7 @@ static int io_install_fixed_file(struct io_ring_ctx *ctx, struct file *file,
 
 	old_node = io_rsrc_node_lookup(&ctx->file_table.data, &slot_index);
 	if (old_node)
-		io_put_rsrc_node(old_node);
+		io_reset_rsrc_node(&ctx->file_table.data, slot_index);
 	else
 		io_file_bitmap_set(&ctx->file_table, slot_index);
 
@@ -134,8 +134,7 @@ int io_fixed_fd_remove(struct io_ring_ctx *ctx, int offset)
 	node = io_rsrc_node_lookup(&ctx->file_table.data, &offset);
 	if (!node)
 		return -EBADF;
-	io_put_rsrc_node(node);
-	ctx->file_table.data.nodes[offset] = NULL;
+	io_reset_rsrc_node(&ctx->file_table.data, offset);
 	io_file_bitmap_clear(&ctx->file_table, offset);
 	return 0;
 }
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index f2242f3866fb..ab53893dde44 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -166,6 +166,7 @@ __cold int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr)
 					GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 	if (data->nodes) {
 		data->nr = nr;
+		data->last_index = -1U;
 		return 0;
 	}
 	return -ENOMEM;
@@ -204,8 +205,7 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
 		index = up->offset + done;
 		node = io_rsrc_node_lookup(&ctx->file_table.data, &index);
 		if (node) {
-			io_put_rsrc_node(node);
-			ctx->file_table.data.nodes[index] = NULL;
+			io_reset_rsrc_node(&ctx->file_table.data, index);
 			io_file_bitmap_clear(&ctx->file_table, index);
 		}
 		if (fd != -1) {
@@ -286,7 +286,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 			break;
 		}
 		if (ctx->buf_table.nodes[i])
-			io_put_rsrc_node(ctx->buf_table.nodes[i]);
+			io_reset_rsrc_node(&ctx->buf_table, i);
 
 		ctx->buf_table.nodes[i] = node;
 		if (tag)
@@ -476,10 +476,12 @@ void io_free_rsrc_node(struct io_rsrc_node *node)
 
 	switch (node->type) {
 	case IORING_RSRC_FILE:
+		io_rsrc_cache_clear(&ctx->file_table.data, node);
 		if (io_slot_file(node))
 			fput(io_slot_file(node));
 		break;
 	case IORING_RSRC_BUFFER:
+		io_rsrc_cache_clear(&ctx->buf_table, node);
 		if (node->buf)
 			io_buffer_unmap(node->ctx, node);
 		break;
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 5897306bcc35..617b4077dfd3 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -78,18 +78,44 @@ static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data
 						       int *index)
 {
 	if (*index < data->nr) {
-		*index = array_index_nospec(*index, data->nr);
-		return data->nodes[*index];
+		if (*index != data->last_index) {
+			*index = array_index_nospec(*index, data->nr);
+			if (data->nodes[*index]) {
+				data->last_index = *index;
+				data->last_node = data->nodes[*index];
+			}
+		}
+		return data->last_node;
 	}
 	return NULL;
 }
 
+static inline void io_rsrc_cache_clear(struct io_rsrc_data *data,
+				       struct io_rsrc_node *node)
+{
+	if (data->last_node == node) {
+		data->last_node = NULL;
+		data->last_index = -1U;
+	}
+}
+
 static inline void io_put_rsrc_node(struct io_rsrc_node *node)
 {
 	if (node != &empty_node && !--node->refs)
 		io_free_rsrc_node(node);
 }
 
+static inline void io_reset_rsrc_node(struct io_rsrc_data *data, int index)
+{
+	struct io_rsrc_node *node = data->nodes[index];
+
+	if (node) {
+		io_rsrc_cache_clear(data, node);
+		io_put_rsrc_node(node);
+		data->nodes[index] = NULL;
+	}
+}
+
 static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)
 {
 	if (req->rsrc_nodes[IORING_RSRC_FILE] != rsrc_empty_node) {