io_uring/memmap: implement kernel allocated regions

Allow the kernel to allocate memory for a region. That's the classical
way SQ/CQ are allocated. It's not yet useful to user space as there
is no way to mmap it, which is why it's explicitly disabled in
io_register_mem_region().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/7b8c40e6542546bbf93f4842a9a42a7373b81e0d.1732886067.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/memmap.c b/io_uring/memmap.c
index a37ccb167258c5ffc63c4e33af4954f36ad7316a..0908a71bf57ee1082ba51d8f80db25b983a45a58 100644 (file)
--- a/io_uring/memmap.c
+++ b/io_uring/memmap.c
@@ -273,6 +273,39 @@ static int io_region_pin_pages(struct io_ring_ctx *ctx,
        return 0;
 }
 
+static int io_region_allocate_pages(struct io_ring_ctx *ctx,
+                                   struct io_mapped_region *mr,
+                                   struct io_uring_region_desc *reg)
+{
+       gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN;
+       unsigned long size = mr->nr_pages << PAGE_SHIFT;
+       unsigned long nr_allocated;
+       struct page **pages;
+       void *p;
+
+       pages = kvmalloc_array(mr->nr_pages, sizeof(*pages), gfp);
+       if (!pages)
+               return -ENOMEM;
+
+       p = io_mem_alloc_compound(pages, mr->nr_pages, size, gfp);
+       if (!IS_ERR(p)) {
+               mr->flags |= IO_REGION_F_SINGLE_REF;
+               mr->pages = pages;
+               return 0;
+       }
+
+       nr_allocated = alloc_pages_bulk_array_node(gfp, NUMA_NO_NODE,
+                                                  mr->nr_pages, pages);
+       if (nr_allocated != mr->nr_pages) {
+               if (nr_allocated)
+                       release_pages(pages, nr_allocated);
+               kvfree(pages);
+               return -ENOMEM;
+       }
+       mr->pages = pages;
+       return 0;
+}
+
 int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
                     struct io_uring_region_desc *reg)
 {
@@ -283,9 +316,10 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
                return -EFAULT;
        if (memchr_inv(&reg->__resv, 0, sizeof(reg->__resv)))
                return -EINVAL;
-       if (reg->flags != IORING_MEM_REGION_TYPE_USER)
+       if (reg->flags & ~IORING_MEM_REGION_TYPE_USER)
                return -EINVAL;
-       if (!reg->user_addr)
+       /* user_addr should be set IFF it's a user memory backed region */
+       if ((reg->flags & IORING_MEM_REGION_TYPE_USER) != !!reg->user_addr)
                return -EFAULT;
        if (!reg->size || reg->mmap_offset || reg->id)
                return -EINVAL;
@@ -304,7 +338,10 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
        }
        mr->nr_pages = nr_pages;
 
-       ret = io_region_pin_pages(ctx, mr, reg);
+       if (reg->flags & IORING_MEM_REGION_TYPE_USER)
+               ret = io_region_pin_pages(ctx, mr, reg);
+       else
+               ret = io_region_allocate_pages(ctx, mr, reg);
        if (ret)
                goto out_free;
 

diff --git a/io_uring/register.c b/io_uring/register.c
index 77743e3f67518468bce8360eb4460a93d0bf1e01..47992bc1ffcb98a9e4f71f055c59c2015695e66b 100644 (file)
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -589,6 +589,8 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg)
        if (copy_from_user(&rd, rd_uptr, sizeof(rd)))
                return -EFAULT;
 
+       if (!(rd.flags & IORING_MEM_REGION_TYPE_USER))
+               return -EINVAL;
        if (memchr_inv(&reg.__resv, 0, sizeof(reg.__resv)))
                return -EINVAL;
        if (reg.flags & ~IORING_MEM_REGION_REG_WAIT_ARG)