Merge branch 'master' of https://github.com/DevriesL/fio

[fio.git] / engines / io_uring.c

diff --git a/engines/io_uring.c b/engines/io_uring.c
index ec8cb18a354ee9f7ea6a29d31744a9a71836b24a..b962e8041b6f8d113669b4b2a31224a68d19aa0f 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -66,6 +66,7 @@ struct ioring_data {
        unsigned iodepth;
        bool ioprio_class_set;
        bool ioprio_set;
+       int prepped;
 
        struct ioring_mmap mmap[3];
 };
@@ -82,6 +83,7 @@ struct ioring_options {
        unsigned int nonvectored;
        unsigned int uncached;
        unsigned int nowait;
+       unsigned int force_async;
 };
 
 static const int ddir_to_op[2][2] = {
@@ -197,6 +199,15 @@ static struct fio_option options[] = {
                .category = FIO_OPT_C_ENGINE,
                .group  = FIO_OPT_G_IOURING,
        },
+       {
+               .name   = "force_async",
+               .lname  = "Force async",
+               .type   = FIO_OPT_INT,
+               .off1   = offsetof(struct ioring_options, force_async),
+               .help   = "Set IOSQE_ASYNC every N requests",
+               .category = FIO_OPT_C_ENGINE,
+               .group  = FIO_OPT_G_IOURING,
+       },
        {
                .name   = NULL,
        },
@@ -218,9 +229,6 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
 
        sqe = &ld->sqes[io_u->index];
 
-       /* zero out fields not used in this submission */
-       memset(sqe, 0, sizeof(*sqe));
-
        if (o->registerfiles) {
                sqe->fd = f->engine_pos;
                sqe->flags = IOSQE_FIXED_FILE;
@@ -262,19 +270,29 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
                if (ld->ioprio_set)
                        sqe->ioprio |= td->o.ioprio;
                sqe->off = io_u->offset;
+               sqe->rw_flags = 0;
        } else if (ddir_sync(io_u->ddir)) {
+               sqe->ioprio = 0;
                if (io_u->ddir == DDIR_SYNC_FILE_RANGE) {
                        sqe->off = f->first_write;
                        sqe->len = f->last_write - f->first_write;
                        sqe->sync_range_flags = td->o.sync_file_range;
                        sqe->opcode = IORING_OP_SYNC_FILE_RANGE;
                } else {
+                       sqe->off = 0;
+                       sqe->addr = 0;
+                       sqe->len = 0;
                        if (io_u->ddir == DDIR_DATASYNC)
                                sqe->fsync_flags |= IORING_FSYNC_DATASYNC;
                        sqe->opcode = IORING_OP_FSYNC;
                }
        }
 
+       if (o->force_async && ++ld->prepped == o->force_async) {
+               ld->prepped = 0;
+               sqe->flags |= IOSQE_ASYNC;
+       }
+
        sqe->user_data = (unsigned long) io_u;
        return 0;
 }
@@ -444,9 +462,10 @@ static int fio_ioring_commit(struct thread_data *td)
         */
        if (o->sqpoll_thread) {
                struct io_sq_ring *ring = &ld->sq_ring;
+               unsigned flags;
 
-               read_barrier();
-               if (*ring->flags & IORING_SQ_NEED_WAKEUP)
+               flags = atomic_load_acquire(ring->flags);
+               if (flags & IORING_SQ_NEED_WAKEUP)
                        io_uring_enter(ld, ld->queued, 0,
                                        IORING_ENTER_SQ_WAKEUP);
                ld->queued = 0;
@@ -488,7 +507,7 @@ static void fio_ioring_unmap(struct ioring_data *ld)
 {
        int i;
 
-       for (i = 0; i < ARRAY_SIZE(ld->mmap); i++)
+       for (i = 0; i < FIO_ARRAY_SIZE(ld->mmap); i++)
                munmap(ld->mmap[i].ptr, ld->mmap[i].len);
        close(ld->ring_fd);
 }
@@ -677,10 +696,21 @@ static int fio_ioring_post_init(struct thread_data *td)
 
        err = fio_ioring_queue_init(td);
        if (err) {
-               td_verror(td, errno, "io_queue_init");
+               int init_err = errno;
+
+               if (init_err == ENOSYS)
+                       log_err("fio: your kernel doesn't support io_uring\n");
+               td_verror(td, init_err, "io_queue_init");
                return 1;
        }
 
+       for (i = 0; i < td->o.iodepth; i++) {
+               struct io_uring_sqe *sqe;
+
+               sqe = &ld->sqes[i];
+               memset(sqe, 0, sizeof(*sqe));
+       }
+
        if (o->registerfiles) {
                err = fio_ioring_register_files(td);
                if (err) {
@@ -698,6 +728,12 @@ static int fio_ioring_init(struct thread_data *td)
        struct ioring_data *ld;
        struct thread_options *to = &td->o;
 
+       if (to->io_submit_mode == IO_MODE_OFFLOAD) {
+               log_err("fio: io_submit_mode=offload is not compatible (or "
+                       "useful) with io_uring\n");
+               return 1;
+       }
+
        /* sqthread submission requires registered files */
        if (o->sqpoll_thread)
                o->registerfiles = 1;
@@ -774,7 +810,7 @@ static int fio_ioring_close_file(struct thread_data *td, struct fio_file *f)
 static struct ioengine_ops ioengine = {
        .name                   = "io_uring",
        .version                = FIO_IOOPS_VERSION,
-       .flags                  = FIO_ASYNCIO_SYNC_TRIM,
+       .flags                  = FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD,
        .init                   = fio_ioring_init,
        .post_init              = fio_ioring_post_init,
        .io_u_init              = fio_ioring_io_u_init,