engines:io_uring: enable support for separate metadata buffer

[fio.git] / engines / io_uring.c

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 30d9ccd79803b96a7496b941341de6f3f85b001a..4916e3b052c6dccb89290ab9a8ca5e132c5125ae 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -59,6 +59,7 @@ struct ioring_data {
        int ring_fd;
 
        struct io_u **io_u_index;
+       char *md_buf;
 
        int *fds;
 
@@ -95,6 +96,7 @@ struct ioring_options {
        unsigned int uncached;
        unsigned int nowait;
        unsigned int force_async;
+       unsigned int md_per_io_size;
        enum uring_cmd_type cmd_type;
 };
 
@@ -217,6 +219,16 @@ static struct fio_option options[] = {
                .group  = FIO_OPT_G_IOURING,
        },
        CMDPRIO_OPTIONS(struct ioring_options, FIO_OPT_G_IOURING),
+       {
+               .name   = "md_per_io_size",
+               .lname  = "Separate Metadata Buffer Size per I/O",
+               .type   = FIO_OPT_INT,
+               .off1   = offsetof(struct ioring_options, md_per_io_size),
+               .def    = "0",
+               .help   = "Size of separate metadata buffer per I/O (Default: 0)",
+               .category = FIO_OPT_C_ENGINE,
+               .group  = FIO_OPT_G_IOURING,
+       },
        {
                .name   = NULL,
        },
@@ -631,6 +643,7 @@ static void fio_ioring_cleanup(struct thread_data *td)
 
                fio_cmdprio_cleanup(&ld->cmdprio);
                free(ld->io_u_index);
+               free(ld->md_buf);
                free(ld->iovecs);
                free(ld->fds);
                free(ld->dsm);
@@ -1016,6 +1029,7 @@ static int fio_ioring_init(struct thread_data *td)
 {
        struct ioring_options *o = td->eo;
        struct ioring_data *ld;
+       unsigned long long md_size;
        int ret;
 
        /* sqthread submission requires registered files */
@@ -1036,6 +1050,28 @@ static int fio_ioring_init(struct thread_data *td)
 
        /* io_u index */
        ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *));
+
+       /*
+        * metadata buffer for nvme command.
+        * We are only supporting iomem=malloc / mem=malloc as of now.
+        */
+       if (!strcmp(td->io_ops->name, "io_uring_cmd") &&
+           (o->cmd_type == FIO_URING_CMD_NVME) && o->md_per_io_size) {
+               md_size = (unsigned long long) o->md_per_io_size
+                               * (unsigned long long) td->o.iodepth;
+               md_size += page_mask + td->o.mem_align;
+               if (td->o.mem_align && td->o.mem_align > page_size)
+                       md_size += td->o.mem_align - page_size;
+               if (td->o.mem_type == MEM_MALLOC) {
+                       ld->md_buf = malloc(md_size);
+                       if (!ld->md_buf)
+                               return 1;
+               } else {
+                       log_err("fio: Only iomem=malloc or mem=malloc is supported\n");
+                       return 1;
+               }
+       }
+
        ld->iovecs = calloc(td->o.iodepth, sizeof(struct iovec));
 
        td->io_ops_data = ld;
@@ -1062,8 +1098,17 @@ static int fio_ioring_init(struct thread_data *td)
 static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
 {
        struct ioring_data *ld = td->io_ops_data;
+       struct ioring_options *o = td->eo;
+       char *p;
 
        ld->io_u_index[io_u->index] = io_u;
+
+       if (!strcmp(td->io_ops->name, "io_uring_cmd")) {
+               p = PTR_ALIGN(ld->md_buf, page_mask) + td->o.mem_align;
+               p += o->md_per_io_size * io_u->index;
+               io_u->mmap_data = p;
+       }
+
        return 0;
 }
 
@@ -1117,6 +1162,15 @@ static int fio_ioring_cmd_open_file(struct thread_data *td, struct fio_file *f)
                                td_verror(td, EINVAL, "fio_ioring_cmd_open_file");
                                return 1;
                        }
+                       if (data->ms && !data->lba_ext && ddir != DDIR_TRIM &&
+                           (o->md_per_io_size < ((td->o.max_bs[ddir] / data->lba_size) *
+                                                 data->ms))) {
+                               log_err("%s: md_per_io_size should be at least %llu bytes\n",
+                                       f->file_name,
+                                       ((td->o.max_bs[ddir] / data->lba_size) * data->ms));
+                               td_verror(td, EINVAL, "fio_ioring_cmd_open_file");
+                               return 1;
+                       }
                 }
        }
        if (!ld || !o->registerfiles)