engines/io_uring: add support for registered files

This feature is exposed as a separate option, like fixedbufs, and
provides a way for fio to register a set of files with the kernel.
This improves IO efficiency.

It is also a requirement to be able to use sqthread_poll, as that
feature requires fixed files on the kernel side.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/HOWTO b/HOWTO
index 4201e2e946f0a4594eb04f778eee575122696536..6b449e97029b674525983ac158e5b40881f1b733 100644 (file)
--- a/HOWTO
+++ b/HOWTO
@@ -2033,6 +2033,12 @@ with the caveat that when used on the command line, they must come after the
        map and release for each IO. This is more efficient, and reduces the
        IO latency as well.
 
        map and release for each IO. This is more efficient, and reduces the
        IO latency as well.
 

+.. option:: registerfiles : [io_uring]
+       With this option, fio registers the set of files being used with the
+       kernel. This avoids the overhead of managing file counts in the kernel,
+       making the submission and completion part more lightweight. Required
+       for the below :option:`sqthread_poll` option.
+

 .. option:: sqthread_poll : [io_uring]
 
        Normally fio will submit IO by issuing a system call to notify the
 .. option:: sqthread_poll : [io_uring]
 
        Normally fio will submit IO by issuing a system call to notify the

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 9bcfec1726b05402bbc0875b2a53bf3edfc13e52..53bd39c8d739c1f91da4db8ef8b1845a49b5ae0d 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -50,6 +50,8 @@ struct ioring_data {
 
        struct io_u **io_u_index;
 
 
        struct io_u **io_u_index;
 

+       int *fds;
+

        struct io_sq_ring sq_ring;
        struct io_uring_sqe *sqes;
        struct iovec *iovecs;
        struct io_sq_ring sq_ring;
        struct io_uring_sqe *sqes;
        struct iovec *iovecs;


@@ -69,6 +71,7 @@ struct ioring_options {
        void *pad;
        unsigned int hipri;
        unsigned int fixedbufs;
        void *pad;
        unsigned int hipri;
        unsigned int fixedbufs;

+       unsigned int registerfiles;

        unsigned int sqpoll_thread;
        unsigned int sqpoll_set;
        unsigned int sqpoll_cpu;
        unsigned int sqpoll_thread;
        unsigned int sqpoll_set;
        unsigned int sqpoll_cpu;


@@ -102,6 +105,15 @@ static struct fio_option options[] = {
                .category = FIO_OPT_C_ENGINE,
                .group  = FIO_OPT_G_LIBAIO,
        },
                .category = FIO_OPT_C_ENGINE,
                .group  = FIO_OPT_G_LIBAIO,
        },

+       {
+               .name   = "registerfiles",
+               .lname  = "Register file set",
+               .type   = FIO_OPT_STR_SET,
+               .off1   = offsetof(struct ioring_options, registerfiles),
+               .help   = "Pre-open/register files",
+               .category = FIO_OPT_C_ENGINE,
+               .group  = FIO_OPT_G_LIBAIO,
+       },

        {
                .name   = "sqthread_poll",
                .lname  = "Kernel SQ thread polling",
        {
                .name   = "sqthread_poll",
                .lname  = "Kernel SQ thread polling",


@@ -140,8 +152,13 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
        struct io_uring_sqe *sqe;
 
        sqe = &ld->sqes[io_u->index];
        struct io_uring_sqe *sqe;
 
        sqe = &ld->sqes[io_u->index];

-       sqe->fd = f->fd;
-       sqe->flags = 0;
+       if (o->registerfiles) {
+               sqe->fd = f->engine_pos;
+               sqe->flags = IOSQE_FIXED_FILE;
+       } else {
+               sqe->fd = f->fd;
+               sqe->flags = 0;
+       }

        sqe->ioprio = 0;
        sqe->buf_index = 0;
 
        sqe->ioprio = 0;
        sqe->buf_index = 0;
 


@@ -388,6 +405,7 @@ static void fio_ioring_cleanup(struct thread_data *td)
 
                free(ld->io_u_index);
                free(ld->iovecs);
 
                free(ld->io_u_index);
                free(ld->iovecs);

+               free(ld->fds);

                free(ld);
        }
 }
                free(ld);
        }
 }


@@ -476,9 +494,50 @@ static int fio_ioring_queue_init(struct thread_data *td)
        return fio_ioring_mmap(ld, &p);
 }
 
        return fio_ioring_mmap(ld, &p);
 }
 

+static int fio_ioring_register_files(struct thread_data *td)
+{
+       struct ioring_data *ld = td->io_ops_data;
+       struct fio_file *f;
+       unsigned int i;
+       int ret;
+
+       ld->fds = calloc(td->o.nr_files, sizeof(int));
+
+       for_each_file(td, f, i) {
+               ret = generic_open_file(td, f);
+               if (ret)
+                       goto err;
+               ld->fds[i] = f->fd;
+               f->engine_pos = i;
+       }
+
+       ret = syscall(__NR_sys_io_uring_register, ld->ring_fd,
+                       IORING_REGISTER_FILES, ld->fds, td->o.nr_files);
+       if (ret) {
+err:
+               free(ld->fds);
+               ld->fds = NULL;
+       }
+
+       /*
+        * Pretend the file is closed again, and really close it if we hit
+        * an error.
+        */
+       for_each_file(td, f, i) {
+               if (ret) {
+                       int fio_unused ret2;
+                       ret2 = generic_close_file(td, f);
+               } else
+                       f->fd = -1;
+       }
+
+       return ret;
+}
+

 static int fio_ioring_post_init(struct thread_data *td)
 {
        struct ioring_data *ld = td->io_ops_data;
 static int fio_ioring_post_init(struct thread_data *td)
 {
        struct ioring_data *ld = td->io_ops_data;

+       struct ioring_options *o = td->eo;

        struct io_u *io_u;
        int err, i;
 
        struct io_u *io_u;
        int err, i;
 


@@ -496,6 +555,14 @@ static int fio_ioring_post_init(struct thread_data *td)
                return 1;
        }
 
                return 1;
        }
 

+       if (o->registerfiles) {
+               err = fio_ioring_register_files(td);
+               if (err) {
+                       td_verror(td, errno, "ioring_register_files");
+                       return 1;
+               }
+       }
+

        return 0;
 }
 
        return 0;
 }
 


@@ -506,8 +573,19 @@ static unsigned roundup_pow2(unsigned depth)
 
 static int fio_ioring_init(struct thread_data *td)
 {
 
 static int fio_ioring_init(struct thread_data *td)
 {

+       struct ioring_options *o = td->eo;

        struct ioring_data *ld;
 
        struct ioring_data *ld;
 

+       /* sqthread submission requires registered files */
+       if (o->sqpoll_thread)
+               o->registerfiles = 1;
+
+       if (o->registerfiles && td->o.nr_files != td->o.open_files) {
+               log_err("fio: io_uring registered files require nr_files to "
+                       "be identical to open_files\n");
+               return 1;
+       }
+

        ld = calloc(1, sizeof(*ld));
 
        /* ring depth must be a power-of-2 */
        ld = calloc(1, sizeof(*ld));
 
        /* ring depth must be a power-of-2 */


@@ -530,6 +608,29 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
        return 0;
 }
 
        return 0;
 }
 

+static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f)
+{
+       struct ioring_data *ld = td->io_ops_data;
+       struct ioring_options *o = td->eo;
+
+       if (!o->registerfiles)
+               return generic_open_file(td, f);
+
+       f->fd = ld->fds[f->engine_pos];
+       return 0;
+}
+
+static int fio_ioring_close_file(struct thread_data *td, struct fio_file *f)
+{
+       struct ioring_options *o = td->eo;
+
+       if (!o->registerfiles)
+               return generic_close_file(td, f);
+
+       f->fd = -1;
+       return 0;
+}
+

 static struct ioengine_ops ioengine = {
        .name                   = "io_uring",
        .version                = FIO_IOOPS_VERSION,
 static struct ioengine_ops ioengine = {
        .name                   = "io_uring",
        .version                = FIO_IOOPS_VERSION,


@@ -543,8 +644,8 @@ static struct ioengine_ops ioengine = {
        .getevents              = fio_ioring_getevents,
        .event                  = fio_ioring_event,
        .cleanup                = fio_ioring_cleanup,
        .getevents              = fio_ioring_getevents,
        .event                  = fio_ioring_event,
        .cleanup                = fio_ioring_cleanup,

-       .open_file              = generic_open_file,
-       .close_file             = generic_close_file,
+       .open_file              = fio_ioring_open_file,
+       .close_file             = fio_ioring_close_file,

        .get_file_size          = generic_get_file_size,
        .options                = options,
        .option_struct_size     = sizeof(struct ioring_options),
        .get_file_size          = generic_get_file_size,
        .options                = options,
        .option_struct_size     = sizeof(struct ioring_options),

diff --git a/fio.1 b/fio.1
index 3e872bcea7ce332ab3c07060c0b24d06bc51b930..e0283f7f93ce8030ea3f82a21a7222fbcee1e6f1 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -1791,6 +1791,12 @@ release them when IO is done. If this option is set, the pages are pre-mapped
 before IO is started. This eliminates the need to map and release for each IO.
 This is more efficient, and reduces the IO latency as well.
 .TP
 before IO is started. This eliminates the need to map and release for each IO.
 This is more efficient, and reduces the IO latency as well.
 .TP

+.BI (io_uring)registerfiles
+With this option, fio registers the set of files being used with the kernel.
+This avoids the overhead of managing file counts in the kernel, making the
+submission and completion part more lightweight. Required for the below
+sqthread_poll option.
+.TP

 .BI (io_uring)sqthread_poll
 Normally fio will submit IO by issuing a system call to notify the kernel of
 available items in the SQ ring. If this option is set, the act of submitting IO
 .BI (io_uring)sqthread_poll
 Normally fio will submit IO by issuing a system call to notify the kernel of
 available items in the SQ ring. If this option is set, the act of submitting IO