engines/io_uring: set COOP_TASKRUN for ring setup

[fio.git] / engines / io_uring.c

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 5a5406d4bf4ded1d1f7a4b6d3b04aa47d019d018..a51468f5b0dd528c3a27b738e9ad9821d4860ca8 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -402,45 +402,41 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u)
        struct ioring_data *ld = td->io_ops_data;
        struct ioring_options *o = td->eo;
        struct fio_file *f = io_u->file;
+       struct nvme_uring_cmd *cmd;
        struct io_uring_sqe *sqe;
-       int ret;
 
-       /* nvme_uring_cmd case */
-       if (o->cmd_type == FIO_URING_CMD_NVME) {
-               struct nvme_uring_cmd *cmd;
+       /* only supports nvme_uring_cmd */
+       if (o->cmd_type != FIO_URING_CMD_NVME)
+               return -EINVAL;
 
-               sqe = &ld->sqes[(io_u->index) << 1];
+       sqe = &ld->sqes[(io_u->index) << 1];
 
-               if (o->registerfiles) {
-                       sqe->fd = f->engine_pos;
-                       sqe->flags = IOSQE_FIXED_FILE;
-               } else {
-                       sqe->fd = f->fd;
-               }
-               sqe->rw_flags = 0;
-               if (!td->o.odirect && o->uncached)
-                       sqe->rw_flags |= RWF_UNCACHED;
-               if (o->nowait)
-                       sqe->rw_flags |= RWF_NOWAIT;
-
-               sqe->opcode = IORING_OP_URING_CMD;
-               sqe->user_data = (unsigned long) io_u;
-               if (o->nonvectored)
-                       sqe->cmd_op = NVME_URING_CMD_IO;
-               else
-                       sqe->cmd_op = NVME_URING_CMD_IO_VEC;
-               if (o->force_async && ++ld->prepped == o->force_async) {
-                       ld->prepped = 0;
-                       sqe->flags |= IOSQE_ASYNC;
-               }
-
-               cmd = (struct nvme_uring_cmd *)sqe->cmd;
-               ret = fio_nvme_uring_cmd_prep(cmd, io_u,
-                               o->nonvectored ? NULL : &ld->iovecs[io_u->index]);
+       if (o->registerfiles) {
+               sqe->fd = f->engine_pos;
+               sqe->flags = IOSQE_FIXED_FILE;
+       } else {
+               sqe->fd = f->fd;
+       }
+       sqe->rw_flags = 0;
+       if (!td->o.odirect && o->uncached)
+               sqe->rw_flags |= RWF_UNCACHED;
+       if (o->nowait)
+               sqe->rw_flags |= RWF_NOWAIT;
 
-               return ret;
+       sqe->opcode = IORING_OP_URING_CMD;
+       sqe->user_data = (unsigned long) io_u;
+       if (o->nonvectored)
+               sqe->cmd_op = NVME_URING_CMD_IO;
+       else
+               sqe->cmd_op = NVME_URING_CMD_IO_VEC;
+       if (o->force_async && ++ld->prepped == o->force_async) {
+               ld->prepped = 0;
+               sqe->flags |= IOSQE_ASYNC;
        }
-       return -EINVAL;
+
+       cmd = (struct nvme_uring_cmd *)sqe->cmd;
+       return fio_nvme_uring_cmd_prep(cmd, io_u,
+                       o->nonvectored ? NULL : &ld->iovecs[io_u->index]);
 }
 
 static struct io_u *fio_ioring_event(struct thread_data *td, int event)
@@ -536,6 +532,7 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min,
                        if (r < 0) {
                                if (errno == EAGAIN || errno == EINTR)
                                        continue;
+                               r = -errno;
                                td_verror(td, errno, "io_uring_enter");
                                break;
                        }
@@ -612,6 +609,12 @@ static void fio_ioring_queued(struct thread_data *td, int start, int nr)
 
                start++;
        }
+
+       /*
+        * only used for iolog
+        */
+       if (td->o.read_iolog_file)
+               memcpy(&td->last_issue, &now, sizeof(now));
 }
 
 static int fio_ioring_commit(struct thread_data *td)
@@ -663,6 +666,7 @@ static int fio_ioring_commit(struct thread_data *td)
                                usleep(1);
                                continue;
                        }
+                       ret = -errno;
                        td_verror(td, errno, "io_uring_enter submit");
                        break;
                }
@@ -805,9 +809,19 @@ static int fio_ioring_queue_init(struct thread_data *td)
        p.flags |= IORING_SETUP_CQSIZE;
        p.cq_entries = depth;
 
+       /*
+        * Setup COOP_TASKRUN as we don't need to get IPI interrupted for
+        * completing IO operations.
+        */
+       p.flags |= IORING_SETUP_COOP_TASKRUN;
+
 retry:
        ret = syscall(__NR_io_uring_setup, depth, &p);
        if (ret < 0) {
+               if (errno == EINVAL && p.flags & IORING_SETUP_COOP_TASKRUN) {
+                       p.flags &= ~IORING_SETUP_COOP_TASKRUN;
+                       goto retry;
+               }
                if (errno == EINVAL && p.flags & IORING_SETUP_CQSIZE) {
                        p.flags &= ~IORING_SETUP_CQSIZE;
                        goto retry;
@@ -1195,7 +1209,8 @@ static int fio_ioring_cmd_get_max_open_zones(struct thread_data *td,
 static struct ioengine_ops ioengine_uring = {
        .name                   = "io_uring",
        .version                = FIO_IOOPS_VERSION,
-       .flags                  = FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD,
+       .flags                  = FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD |
+                                       FIO_ASYNCIO_SETS_ISSUE_TIME,
        .init                   = fio_ioring_init,
        .post_init              = fio_ioring_post_init,
        .io_u_init              = fio_ioring_io_u_init,
@@ -1215,7 +1230,9 @@ static struct ioengine_ops ioengine_uring = {
 static struct ioengine_ops ioengine_uring_cmd = {
        .name                   = "io_uring_cmd",
        .version                = FIO_IOOPS_VERSION,
-       .flags                  = FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD | FIO_MEMALIGN | FIO_RAWIO,
+       .flags                  = FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD |
+                                       FIO_MEMALIGN | FIO_RAWIO |
+                                       FIO_ASYNCIO_SETS_ISSUE_TIME,
        .init                   = fio_ioring_init,
        .post_init              = fio_ioring_cmd_post_init,
        .io_u_init              = fio_ioring_io_u_init,