engines/io_uring: use its own option group

[fio.git] / engines / io_uring.c

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 5b3509a98d734bd3fe50157b81a3aa1a358b9547..10cfe9f23e2476ffed8500ce13e46d2ad2e8ed90 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -50,6 +50,8 @@ struct ioring_data {
 
        struct io_u **io_u_index;
 
+       int *fds;
+
        struct io_sq_ring sq_ring;
        struct io_uring_sqe *sqes;
        struct iovec *iovecs;
@@ -69,6 +71,7 @@ struct ioring_options {
        void *pad;
        unsigned int hipri;
        unsigned int fixedbufs;
+       unsigned int registerfiles;
        unsigned int sqpoll_thread;
        unsigned int sqpoll_set;
        unsigned int sqpoll_cpu;
@@ -91,7 +94,7 @@ static struct fio_option options[] = {
                .off1   = offsetof(struct ioring_options, hipri),
                .help   = "Use polled IO completions",
                .category = FIO_OPT_C_ENGINE,
-               .group  = FIO_OPT_G_LIBAIO,
+               .group  = FIO_OPT_G_IOURING,
        },
        {
                .name   = "fixedbufs",
@@ -100,7 +103,16 @@ static struct fio_option options[] = {
                .off1   = offsetof(struct ioring_options, fixedbufs),
                .help   = "Pre map IO buffers",
                .category = FIO_OPT_C_ENGINE,
-               .group  = FIO_OPT_G_LIBAIO,
+               .group  = FIO_OPT_G_IOURING,
+       },
+       {
+               .name   = "registerfiles",
+               .lname  = "Register file set",
+               .type   = FIO_OPT_STR_SET,
+               .off1   = offsetof(struct ioring_options, registerfiles),
+               .help   = "Pre-open/register files",
+               .category = FIO_OPT_C_ENGINE,
+               .group  = FIO_OPT_G_IOURING,
        },
        {
                .name   = "sqthread_poll",
@@ -109,7 +121,7 @@ static struct fio_option options[] = {
                .off1   = offsetof(struct ioring_options, sqpoll_thread),
                .help   = "Offload submission/completion to kernel thread",
                .category = FIO_OPT_C_ENGINE,
-               .group  = FIO_OPT_G_LIBAIO,
+               .group  = FIO_OPT_G_IOURING,
        },
        {
                .name   = "sqthread_poll_cpu",
@@ -118,7 +130,7 @@ static struct fio_option options[] = {
                .cb     = fio_ioring_sqpoll_cb,
                .help   = "What CPU to run SQ thread polling on",
                .category = FIO_OPT_C_ENGINE,
-               .group  = FIO_OPT_G_LIBAIO,
+               .group  = FIO_OPT_G_IOURING,
        },
        {
                .name   = NULL,
@@ -140,8 +152,13 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
        struct io_uring_sqe *sqe;
 
        sqe = &ld->sqes[io_u->index];
-       sqe->fd = f->fd;
-       sqe->flags = 0;
+       if (o->registerfiles) {
+               sqe->fd = f->engine_pos;
+               sqe->flags = IOSQE_FIXED_FILE;
+       } else {
+               sqe->fd = f->fd;
+               sqe->flags = 0;
+       }
        sqe->ioprio = 0;
        sqe->buf_index = 0;
 
@@ -233,6 +250,8 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min,
                r = fio_ioring_cqring_reap(td, events, max);
                if (r) {
                        events += r;
+                       if (actual_min != 0)
+                               actual_min -= r;
                        continue;
                }
 
@@ -386,6 +405,7 @@ static void fio_ioring_cleanup(struct thread_data *td)
 
                free(ld->io_u_index);
                free(ld->iovecs);
+               free(ld->fds);
                free(ld);
        }
 }
@@ -474,9 +494,50 @@ static int fio_ioring_queue_init(struct thread_data *td)
        return fio_ioring_mmap(ld, &p);
 }
 
+static int fio_ioring_register_files(struct thread_data *td)
+{
+       struct ioring_data *ld = td->io_ops_data;
+       struct fio_file *f;
+       unsigned int i;
+       int ret;
+
+       ld->fds = calloc(td->o.nr_files, sizeof(int));
+
+       for_each_file(td, f, i) {
+               ret = generic_open_file(td, f);
+               if (ret)
+                       goto err;
+               ld->fds[i] = f->fd;
+               f->engine_pos = i;
+       }
+
+       ret = syscall(__NR_sys_io_uring_register, ld->ring_fd,
+                       IORING_REGISTER_FILES, ld->fds, td->o.nr_files);
+       if (ret) {
+err:
+               free(ld->fds);
+               ld->fds = NULL;
+       }
+
+       /*
+        * Pretend the file is closed again, and really close it if we hit
+        * an error.
+        */
+       for_each_file(td, f, i) {
+               if (ret) {
+                       int fio_unused ret2;
+                       ret2 = generic_close_file(td, f);
+               } else
+                       f->fd = -1;
+       }
+
+       return ret;
+}
+
 static int fio_ioring_post_init(struct thread_data *td)
 {
        struct ioring_data *ld = td->io_ops_data;
+       struct ioring_options *o = td->eo;
        struct io_u *io_u;
        int err, i;
 
@@ -494,6 +555,14 @@ static int fio_ioring_post_init(struct thread_data *td)
                return 1;
        }
 
+       if (o->registerfiles) {
+               err = fio_ioring_register_files(td);
+               if (err) {
+                       td_verror(td, errno, "ioring_register_files");
+                       return 1;
+               }
+       }
+
        return 0;
 }
 
@@ -504,8 +573,19 @@ static unsigned roundup_pow2(unsigned depth)
 
 static int fio_ioring_init(struct thread_data *td)
 {
+       struct ioring_options *o = td->eo;
        struct ioring_data *ld;
 
+       /* sqthread submission requires registered files */
+       if (o->sqpoll_thread)
+               o->registerfiles = 1;
+
+       if (o->registerfiles && td->o.nr_files != td->o.open_files) {
+               log_err("fio: io_uring registered files require nr_files to "
+                       "be identical to open_files\n");
+               return 1;
+       }
+
        ld = calloc(1, sizeof(*ld));
 
        /* ring depth must be a power-of-2 */
@@ -528,9 +608,33 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
        return 0;
 }
 
+static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f)
+{
+       struct ioring_data *ld = td->io_ops_data;
+       struct ioring_options *o = td->eo;
+
+       if (!o->registerfiles)
+               return generic_open_file(td, f);
+
+       f->fd = ld->fds[f->engine_pos];
+       return 0;
+}
+
+static int fio_ioring_close_file(struct thread_data *td, struct fio_file *f)
+{
+       struct ioring_options *o = td->eo;
+
+       if (!o->registerfiles)
+               return generic_close_file(td, f);
+
+       f->fd = -1;
+       return 0;
+}
+
 static struct ioengine_ops ioengine = {
        .name                   = "io_uring",
        .version                = FIO_IOOPS_VERSION,
+       .flags                  = FIO_ASYNCIO_SYNC_TRIM,
        .init                   = fio_ioring_init,
        .post_init              = fio_ioring_post_init,
        .io_u_init              = fio_ioring_io_u_init,
@@ -540,8 +644,8 @@ static struct ioengine_ops ioengine = {
        .getevents              = fio_ioring_getevents,
        .event                  = fio_ioring_event,
        .cleanup                = fio_ioring_cleanup,
-       .open_file              = generic_open_file,
-       .close_file             = generic_close_file,
+       .open_file              = fio_ioring_open_file,
+       .close_file             = fio_ioring_close_file,
        .get_file_size          = generic_get_file_size,
        .options                = options,
        .option_struct_size     = sizeof(struct ioring_options),