t/io_uring: make bs and polled IO configurable at runtime

[fio.git] / engines / io_uring.c

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 5e59f975c3c529ab735961763c49270ec63eb111..cd0810f47f57d2e4dfcb5fe5c237c6b5dde768fa 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -63,6 +63,8 @@ struct ioring_data {
        int queued;
        int cq_ring_off;
        unsigned iodepth;
+       bool ioprio_class_set;
+       bool ioprio_set;
 
        struct ioring_mmap mmap[3];
 };
@@ -78,6 +80,7 @@ struct ioring_options {
        unsigned int sqpoll_cpu;
        unsigned int nonvectored;
        unsigned int uncached;
+       unsigned int nowait;
 };
 
 static const int ddir_to_op[2][2] = {
@@ -183,6 +186,15 @@ static struct fio_option options[] = {
                .category = FIO_OPT_C_ENGINE,
                .group  = FIO_OPT_G_IOURING,
        },
+       {
+               .name   = "nowait",
+               .lname  = "RWF_NOWAIT",
+               .type   = FIO_OPT_BOOL,
+               .off1   = offsetof(struct ioring_options, nowait),
+               .help   = "Use RWF_NOWAIT for reads/writes",
+               .category = FIO_OPT_C_ENGINE,
+               .group  = FIO_OPT_G_IOURING,
+       },
        {
                .name   = NULL,
        },
@@ -233,9 +245,11 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
                }
                if (!td->o.odirect && o->uncached)
                        sqe->rw_flags = RWF_UNCACHED;
-               if (fio_option_is_set(&td->o, ioprio_class))
+               if (o->nowait)
+                       sqe->rw_flags |= RWF_NOWAIT;
+               if (ld->ioprio_class_set)
                        sqe->ioprio = td->o.ioprio_class << 13;
-               if (fio_option_is_set(&td->o, ioprio))
+               if (ld->ioprio_set)
                        sqe->ioprio |= td->o.ioprio;
                sqe->off = io_u->offset;
        } else if (ddir_sync(io_u->ddir)) {
@@ -287,15 +301,13 @@ static int fio_ioring_cqring_reap(struct thread_data *td, unsigned int events,
 
        head = *ring->head;
        do {
-               read_barrier();
-               if (head == *ring->tail)
+               if (head == atomic_load_acquire(ring->tail))
                        break;
                reaped++;
                head++;
        } while (reaped + events < max);
 
-       *ring->head = head;
-       write_barrier();
+       atomic_store_release(ring->head, head);
        return reaped;
 }
 
@@ -370,17 +382,13 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td,
 
        tail = *ring->tail;
        next_tail = tail + 1;
-       read_barrier();
-       if (next_tail == *ring->head)
+       if (next_tail == atomic_load_acquire(ring->head))
                return FIO_Q_BUSY;
 
-       /* ensure sqe stores are ordered with tail update */
-       write_barrier();
        if (o->cmdprio_percentage)
                fio_ioring_prio_prep(td, io_u);
        ring->array[tail & ld->sq_ring_mask] = io_u->index;
-       *ring->tail = next_tail;
-       write_barrier();
+       atomic_store_release(ring->tail, next_tail);
 
        ld->queued++;
        return FIO_Q_QUEUED;
@@ -687,6 +695,12 @@ static int fio_ioring_init(struct thread_data *td)
                td_verror(td, EINVAL, "fio_io_uring_init");
                return 1;
        }
+
+       if (fio_option_is_set(&td->o, ioprio_class))
+               ld->ioprio_class_set = true;
+       if (fio_option_is_set(&td->o, ioprio))
+               ld->ioprio_set = true;
+
        return 0;
 }