projects / fio.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
io_uring: fix possible infinite loop
[fio.git] / engines / io_uring.c
diff --git a/engines/io_uring.c b/engines/io_uring.c
index 39359af909c7b199677dcadf784778435873b28c..a5e77d8f52fcfe29e1985824f953b0b594e64f85 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -21,7 +21,7 @@
 #ifdef ARCH_HAVE_IOURING
 
 #include "../lib/types.h"
-#include "../os/io_uring.h"
+#include "../os/linux/io_uring.h"
 
 struct io_sq_ring {
        unsigned *head;
@@ -48,7 +48,6 @@ struct ioring_mmap {
 struct ioring_data {
        int ring_fd;
 
-       struct io_u **io_us;
        struct io_u **io_u_index;
 
        struct io_sq_ring sq_ring;
@@ -63,9 +62,6 @@ struct ioring_data {
        int cq_ring_off;
        unsigned iodepth;
 
-       uint64_t cachehit;
-       uint64_t cachemiss;
-
        struct ioring_mmap mmap[3];
 };
 
@@ -73,6 +69,7 @@ struct ioring_options {
        void *pad;
        unsigned int hipri;
        unsigned int fixedbufs;
+       unsigned int sqpoll_thread;
        unsigned int sqpoll_set;
        unsigned int sqpoll_cpu;
 };
@@ -107,10 +104,19 @@ static struct fio_option options[] = {
        },
        {
                .name   = "sqthread_poll",
-               .lname  = "Kernel SQ thread should poll",
+               .lname  = "Kernel SQ thread polling",
+               .type   = FIO_OPT_INT,
+               .off1   = offsetof(struct ioring_options, sqpoll_thread),
+               .help   = "Offload submission/completion to kernel thread",
+               .category = FIO_OPT_C_ENGINE,
+               .group  = FIO_OPT_G_LIBAIO,
+       },
+       {
+               .name   = "sqthread_poll_cpu",
+               .lname  = "SQ Thread Poll CPU",
                .type   = FIO_OPT_INT,
                .cb     = fio_ioring_sqpoll_cb,
-               .help   = "Offload submission to kernel thread",
+               .help   = "What CPU to run SQ thread polling on",
                .category = FIO_OPT_C_ENGINE,
                .group  = FIO_OPT_G_LIBAIO,
        },
@@ -123,7 +129,7 @@ static int io_uring_enter(struct ioring_data *ld, unsigned int to_submit,
                         unsigned int min_complete, unsigned int flags)
 {
        return syscall(__NR_sys_io_uring_enter, ld->ring_fd, to_submit,
-                       min_complete, flags);
+                       min_complete, flags, NULL, 0);
 }
 
 static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
@@ -140,25 +146,31 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
        sqe->buf_index = 0;
 
        if (io_u->ddir == DDIR_READ || io_u->ddir == DDIR_WRITE) {
-               if (io_u->ddir == DDIR_READ)
-                       sqe->opcode = IORING_OP_READV;
-               else
-                       sqe->opcode = IORING_OP_WRITEV;
-
                if (o->fixedbufs) {
-                       sqe->flags |= IOSQE_FIXED_BUFFER;
-                       sqe->addr = io_u->xfer_buf;
+                       if (io_u->ddir == DDIR_READ)
+                               sqe->opcode = IORING_OP_READ_FIXED;
+                       else
+                               sqe->opcode = IORING_OP_WRITE_FIXED;
+                       sqe->addr = (unsigned long) io_u->xfer_buf;
                        sqe->len = io_u->xfer_buflen;
                        sqe->buf_index = io_u->index;
                } else {
-                       sqe->addr = &ld->iovecs[io_u->index];
+                       if (io_u->ddir == DDIR_READ)
+                               sqe->opcode = IORING_OP_READV;
+                       else
+                               sqe->opcode = IORING_OP_WRITEV;
+                       sqe->addr = (unsigned long) &ld->iovecs[io_u->index];
                        sqe->len = 1;
                }
                sqe->off = io_u->offset;
-       } else if (ddir_sync(io_u->ddir))
+       } else if (ddir_sync(io_u->ddir)) {
+               sqe->fsync_flags = 0;
+               if (io_u->ddir == DDIR_DATASYNC)
+                       sqe->fsync_flags |= IORING_FSYNC_DATASYNC;
                sqe->opcode = IORING_OP_FSYNC;
+       }
 
-       sqe->data = (unsigned long) io_u;
+       sqe->user_data = (unsigned long) io_u;
        return 0;
 }
 
@@ -172,7 +184,7 @@ static struct io_u *fio_ioring_event(struct thread_data *td, int event)
        index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
 
        cqe = &ld->cq_ring.cqes[index];
-       io_u = (struct io_u *) cqe->data;
+       io_u = (struct io_u *) (uintptr_t) cqe->user_data;
 
        if (cqe->res != io_u->xfer_buflen) {
                if (cqe->res > io_u->xfer_buflen)
@@ -182,13 +194,6 @@ static struct io_u *fio_ioring_event(struct thread_data *td, int event)
        } else
                io_u->error = 0;
 
-       if (io_u->ddir == DDIR_READ) {
-               if (cqe->flags & IOCQE_FLAG_CACHEHIT)
-                       ld->cachehit++;
-               else
-                       ld->cachemiss++;
-       }
-
        return io_u;
 }
 
@@ -228,10 +233,12 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min,
                r = fio_ioring_cqring_reap(td, events, max);
                if (r) {
                        events += r;
+                       if (actual_min != 0)
+                               actual_min -= r;
                        continue;
                }
 
-               if (!o->sqpoll_set) {
+               if (!o->sqpoll_thread) {
                        r = io_uring_enter(ld, 0, actual_min,
                                                IORING_ENTER_GETEVENTS);
                        if (r < 0) {
@@ -274,6 +281,8 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td,
        if (next_tail == *ring->head)
                return FIO_Q_BUSY;
 
+       /* ensure sqe stores are ordered with tail update */
+       write_barrier();
        ring->array[tail & ld->sq_ring_mask] = io_u->index;
        *ring->tail = next_tail;
        write_barrier();
@@ -313,13 +322,18 @@ static int fio_ioring_commit(struct thread_data *td)
        if (!ld->queued)
                return 0;
 
-       /* Nothing to do */
-       if (o->sqpoll_set) {
+       /*
+        * Kernel side does submission. just need to check if the ring is
+        * flagged as needing a kick, if so, call io_uring_enter(). This
+        * only happens if we've been idle too long.
+        */
+       if (o->sqpoll_thread) {
                struct io_sq_ring *ring = &ld->sq_ring;
 
                read_barrier();
                if (*ring->flags & IORING_SQ_NEED_WAKEUP)
-                       io_uring_enter(ld, ld->queued, 0, 0);
+                       io_uring_enter(ld, ld->queued, 0,
+                                       IORING_ENTER_SQ_WAKEUP);
                ld->queued = 0;
                return 0;
        }
@@ -369,14 +383,10 @@ static void fio_ioring_cleanup(struct thread_data *td)
        struct ioring_data *ld = td->io_ops_data;
 
        if (ld) {
-               td->ts.cachehit += ld->cachehit;
-               td->ts.cachemiss += ld->cachemiss;
-
                if (!(td->flags & TD_F_CHILD))
                        fio_ioring_unmap(ld);
 
                free(ld->io_u_index);
-               free(ld->io_us);
                free(ld->iovecs);
                free(ld);
        }
@@ -434,18 +444,12 @@ static int fio_ioring_queue_init(struct thread_data *td)
 
        if (o->hipri)
                p.flags |= IORING_SETUP_IOPOLL;
-       if (o->sqpoll_set) {
-               p.flags |= IORING_SETUP_SQPOLL | IORING_SETUP_SQ_AFF;
-               p.sq_thread_cpu = o->sqpoll_cpu;
-       }
-
-       if (o->fixedbufs) {
-               struct rlimit rlim = {
-                       .rlim_cur = RLIM_INFINITY,
-                       .rlim_max = RLIM_INFINITY,
-               };
-
-               setrlimit(RLIMIT_MEMLOCK, &rlim);
+       if (o->sqpoll_thread) {
+               p.flags |= IORING_SETUP_SQPOLL;
+               if (o->sqpoll_set) {
+                       p.flags |= IORING_SETUP_SQ_AFF;
+                       p.sq_thread_cpu = o->sqpoll_cpu;
+               }
        }
 
        ret = syscall(__NR_sys_io_uring_setup, depth, &p);
@@ -455,13 +459,16 @@ static int fio_ioring_queue_init(struct thread_data *td)
        ld->ring_fd = ret;
 
        if (o->fixedbufs) {
-               struct io_uring_register_buffers reg = {
-                       .iovecs = ld->iovecs,
-                       .nr_iovecs = depth
+               struct rlimit rlim = {
+                       .rlim_cur = RLIM_INFINITY,
+                       .rlim_max = RLIM_INFINITY,
                };
 
+               if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0)
+                       return -1;
+
                ret = syscall(__NR_sys_io_uring_register, ld->ring_fd,
-                               IORING_REGISTER_BUFFERS, &reg);
+                               IORING_REGISTER_BUFFERS, ld->iovecs, depth);
                if (ret < 0)
                        return ret;
        }
@@ -509,7 +516,6 @@ static int fio_ioring_init(struct thread_data *td)
 
        /* io_u index */
        ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *));
-       ld->io_us = calloc(td->o.iodepth, sizeof(struct io_u *));
        ld->iovecs = calloc(td->o.iodepth, sizeof(struct iovec));
 
        td->io_ops_data = ld;
fio - Flexible IO Tester