io_uring/rsrc: add an empty io_rsrc_node for sparse buffer entries

Rather than allocate an io_rsrc_node for an empty/sparse buffer entry,
add a const entry that can be used for that. This just needs checking
for writing the tag, and the put check needs to check for that sparse
node rather than NULL for validity.

This avoids allocating rsrc nodes for sparse buffer entries.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 094788cca47fd3bcb5e476319780ad53095b3fa9..9282d5fa45d321c1f8cee202fafbac68eac6d56a 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2032,8 +2032,8 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
        req->flags = (__force io_req_flags_t) sqe_flags;
        req->cqe.user_data = READ_ONCE(sqe->user_data);
        req->file = NULL;
-       req->rsrc_nodes[IORING_RSRC_FILE] = NULL;
-       req->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
+       req->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node;
+       req->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node;
        req->task = current;
        req->cancel_seq_set = false;
 

diff --git a/io_uring/notif.c b/io_uring/notif.c
index 4f02e969cf08c0eba24422ade878a4b059a67807..44bf21c0f8103ece7b7f8f46693772f62959cda1 100644 (file)
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -117,8 +117,8 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
        notif->file = NULL;
        notif->task = current;
        io_get_task_refs(1);
-       notif->rsrc_nodes[IORING_RSRC_FILE] = NULL;
-       notif->rsrc_nodes[IORING_RSRC_BUFFER] = NULL;
+       notif->rsrc_nodes[IORING_RSRC_FILE] = rsrc_empty_node;
+       notif->rsrc_nodes[IORING_RSRC_BUFFER] = rsrc_empty_node;
 
        nd = io_notif_to_data(notif);
        nd->zc_report = false;

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 8f8147dd714ce0cf0de7452b2e3b615a41a8698b..69a9cd82460d01463c72d10348b03d144cdc2cde 100644 (file)
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -38,6 +38,11 @@ static const struct io_mapped_ubuf dummy_ubuf = {
        .len = UINT_MAX,
 };
 
+const struct io_rsrc_node empty_node = {
+       .type = IORING_RSRC_BUFFER,
+       .buf = (struct io_mapped_ubuf *) &dummy_ubuf,
+};
+
 int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
 {
        unsigned long page_limit, cur_pages, new_pages;
@@ -144,7 +149,8 @@ static void io_rsrc_data_free(struct io_rsrc_data *data)
        for (i = 0; i < data->nr; i++) {
                struct io_rsrc_node *node = data->nodes[i];
 
-               io_put_rsrc_node(node);
+               if (node)
+                       io_put_rsrc_node(node);
        }
        kvfree(data->nodes);
        kfree(data);
@@ -229,7 +235,8 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
                                break;
                        }
                        ctx->file_table.nodes[i] = node;
-                       node->tag = tag;
+                       if (tag)
+                               node->tag = tag;
                        io_fixed_file_set(node, file);
                        io_file_bitmap_set(&ctx->file_table, i);
                }
@@ -281,10 +288,12 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
                        err = PTR_ERR(node);
                        break;
                }
-               io_put_rsrc_node(ctx->user_bufs[i]);
+               if (ctx->user_bufs[i])
+                       io_put_rsrc_node(ctx->user_bufs[i]);
 
                ctx->user_bufs[i] = node;
-               node->tag = tag;
+               if (tag)
+                       node->tag = tag;
                if (ctx->compat)
                        user_data += sizeof(struct compat_iovec);
                else
@@ -600,8 +609,10 @@ static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
        lockdep_assert_held(&ctx->uring_lock);
 
        for (i = 0; i < ctx->nr_user_bufs; i++) {
-               io_put_rsrc_node(ctx->user_bufs[i]);
-               ctx->user_bufs[i] = NULL;
+               if (ctx->user_bufs[i]) {
+                       io_put_rsrc_node(ctx->user_bufs[i]);
+                       ctx->user_bufs[i] = NULL;
+               }
        }
        kvfree(ctx->user_bufs);
        ctx->user_bufs = NULL;
@@ -799,11 +810,6 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
                return ERR_PTR(-ENOMEM);
        node->buf = NULL;
 
-       if (!iov->iov_base) {
-               node->buf = (struct io_mapped_ubuf *) &dummy_ubuf;
-               return node;
-       }
-
        ret = -ENOMEM;
        pages = io_pin_pages((unsigned long) iov->iov_base, iov->iov_len,
                                &nr_pages);
@@ -927,7 +933,8 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
                        ret = PTR_ERR(node);
                        break;
                }
-               node->tag = tag;
+               if (tag)
+                       node->tag = tag;
                ctx->user_bufs[i] = node;
        }
 
@@ -1028,18 +1035,18 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
                goto out_free_data;
 
        for (i = 0; i < nbufs; i++) {
-               struct io_mapped_ubuf *imu = src_ctx->user_bufs[i]->buf;
+               struct io_rsrc_node *src_node = src_ctx->user_bufs[i];
                struct io_rsrc_node *dst_node;
 
-               dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
-               if (!dst_node)
-                       goto out_put_free;
-
-               if (imu == &dummy_ubuf) {
-                       dst_node->buf = (struct io_mapped_ubuf *) &dummy_ubuf;
+               if (src_node == rsrc_empty_node) {
+                       dst_node = rsrc_empty_node;
                } else {
-                       refcount_inc(&imu->refs);
-                       dst_node->buf = imu;
+                       dst_node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER);
+                       if (!dst_node)
+                               goto out_put_free;
+
+                       refcount_inc(&src_node->buf->refs);
+                       dst_node->buf = src_node->buf;
                }
                user_bufs[i] = dst_node;
        }

diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 20a316854238dc1b8bd544a247a0187b3a8dacf1..323c3e78b864a00f8f9cda5f2885652070764360 100644 (file)
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -70,9 +70,12 @@ int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
 int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
                        unsigned int size, unsigned int type);
 
+extern const struct io_rsrc_node empty_node;
+#define rsrc_empty_node        (struct io_rsrc_node *) &empty_node
+
 static inline void io_put_rsrc_node(struct io_rsrc_node *node)
 {
-       if (node && !--node->refs)
+       if (node != rsrc_empty_node && !--node->refs)
                io_free_rsrc_node(node);
 }
 
@@ -85,8 +88,10 @@ static inline void io_req_put_rsrc_nodes(struct io_kiocb *req)
 static inline void io_req_assign_rsrc_node(struct io_kiocb *req,
                                           struct io_rsrc_node *node)
 {
-       node->refs++;
-       req->rsrc_nodes[node->type] = node;
+       if (node != rsrc_empty_node) {
+               node->refs++;
+               req->rsrc_nodes[node->type] = node;
+       }
 }
 
 int io_files_update(struct io_kiocb *req, unsigned int issue_flags);

diff --git a/io_uring/splice.c b/io_uring/splice.c
index a0b4e0435b8b2c3d82fe821472f2c3401d836f08..f78afb575ae6c805cad0e99e07b1b5398d949089 100644 (file)
--- a/io_uring/splice.c
+++ b/io_uring/splice.c
@@ -35,7 +35,7 @@ static int __io_splice_prep(struct io_kiocb *req,
        if (unlikely(sp->flags & ~valid_flags))
                return -EINVAL;
        sp->splice_fd_in = READ_ONCE(sqe->splice_fd_in);
-       sp->rsrc_node = NULL;
+       sp->rsrc_node = rsrc_empty_node;
        req->flags |= REQ_F_FORCE_ASYNC;
        return 0;
 }