Add roundup_pow2() as a generic helper

[fio.git] / engines / io_uring.c

diff --git a/engines/io_uring.c b/engines/io_uring.c
index f1ffc7128ea60fcf334c55e6fdb34f2420c9d6b1..0ccd23184fab3b17a9c68c93a9c30375b5673986 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -17,6 +17,7 @@
 #include "../optgroup.h"
 #include "../lib/memalign.h"
 #include "../lib/fls.h"
+#include "../lib/roundup.h"
 
 #ifdef ARCH_HAVE_IOURING
 
@@ -63,6 +64,8 @@ struct ioring_data {
        int queued;
        int cq_ring_off;
        unsigned iodepth;
+       bool ioprio_class_set;
+       bool ioprio_set;
 
        struct ioring_mmap mmap[3];
 };
@@ -78,6 +81,7 @@ struct ioring_options {
        unsigned int sqpoll_cpu;
        unsigned int nonvectored;
        unsigned int uncached;
+       unsigned int nowait;
 };
 
 static const int ddir_to_op[2][2] = {
@@ -183,6 +187,15 @@ static struct fio_option options[] = {
                .category = FIO_OPT_C_ENGINE,
                .group  = FIO_OPT_G_IOURING,
        },
+       {
+               .name   = "nowait",
+               .lname  = "RWF_NOWAIT",
+               .type   = FIO_OPT_BOOL,
+               .off1   = offsetof(struct ioring_options, nowait),
+               .help   = "Use RWF_NOWAIT for reads/writes",
+               .category = FIO_OPT_C_ENGINE,
+               .group  = FIO_OPT_G_IOURING,
+       },
        {
                .name   = NULL,
        },
@@ -191,7 +204,7 @@ static struct fio_option options[] = {
 static int io_uring_enter(struct ioring_data *ld, unsigned int to_submit,
                         unsigned int min_complete, unsigned int flags)
 {
-       return syscall(__NR_sys_io_uring_enter, ld->ring_fd, to_submit,
+       return syscall(__NR_io_uring_enter, ld->ring_fd, to_submit,
                        min_complete, flags, NULL, 0);
 }
 
@@ -233,9 +246,11 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
                }
                if (!td->o.odirect && o->uncached)
                        sqe->rw_flags = RWF_UNCACHED;
-               if (fio_option_is_set(&td->o, ioprio_class))
+               if (o->nowait)
+                       sqe->rw_flags |= RWF_NOWAIT;
+               if (ld->ioprio_class_set)
                        sqe->ioprio = td->o.ioprio_class << 13;
-               if (fio_option_is_set(&td->o, ioprio))
+               if (ld->ioprio_set)
                        sqe->ioprio |= td->o.ioprio;
                sqe->off = io_u->offset;
        } else if (ddir_sync(io_u->ddir)) {
@@ -287,15 +302,15 @@ static int fio_ioring_cqring_reap(struct thread_data *td, unsigned int events,
 
        head = *ring->head;
        do {
-               read_barrier();
-               if (head == *ring->tail)
+               if (head == atomic_load_acquire(ring->tail))
                        break;
                reaped++;
                head++;
        } while (reaped + events < max);
 
-       *ring->head = head;
-       write_barrier();
+       if (reaped)
+               atomic_store_release(ring->head, head);
+
        return reaped;
 }
 
@@ -370,17 +385,13 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td,
 
        tail = *ring->tail;
        next_tail = tail + 1;
-       read_barrier();
-       if (next_tail == *ring->head)
+       if (next_tail == atomic_load_acquire(ring->head))
                return FIO_Q_BUSY;
 
-       /* ensure sqe stores are ordered with tail update */
-       write_barrier();
        if (o->cmdprio_percentage)
                fio_ioring_prio_prep(td, io_u);
        ring->array[tail & ld->sq_ring_mask] = io_u->index;
-       *ring->tail = next_tail;
-       write_barrier();
+       atomic_store_release(ring->tail, next_tail);
 
        ld->queued++;
        return FIO_Q_QUEUED;
@@ -548,7 +559,7 @@ static int fio_ioring_queue_init(struct thread_data *td)
                }
        }
 
-       ret = syscall(__NR_sys_io_uring_setup, depth, &p);
+       ret = syscall(__NR_io_uring_setup, depth, &p);
        if (ret < 0)
                return ret;
 
@@ -563,7 +574,7 @@ static int fio_ioring_queue_init(struct thread_data *td)
                if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0)
                        return -1;
 
-               ret = syscall(__NR_sys_io_uring_register, ld->ring_fd,
+               ret = syscall(__NR_io_uring_register, ld->ring_fd,
                                IORING_REGISTER_BUFFERS, ld->iovecs, depth);
                if (ret < 0)
                        return ret;
@@ -589,7 +600,7 @@ static int fio_ioring_register_files(struct thread_data *td)
                f->engine_pos = i;
        }
 
-       ret = syscall(__NR_sys_io_uring_register, ld->ring_fd,
+       ret = syscall(__NR_io_uring_register, ld->ring_fd,
                        IORING_REGISTER_FILES, ld->fds, td->o.nr_files);
        if (ret) {
 err:
@@ -644,11 +655,6 @@ static int fio_ioring_post_init(struct thread_data *td)
        return 0;
 }
 
-static unsigned roundup_pow2(unsigned depth)
-{
-       return 1UL << __fls(depth - 1);
-}
-
 static int fio_ioring_init(struct thread_data *td)
 {
        struct ioring_options *o = td->eo;
@@ -687,6 +693,12 @@ static int fio_ioring_init(struct thread_data *td)
                td_verror(td, EINVAL, "fio_io_uring_init");
                return 1;
        }
+
+       if (fio_option_is_set(&td->o, ioprio_class))
+               ld->ioprio_class_set = true;
+       if (fio_option_is_set(&td->o, ioprio))
+               ld->ioprio_set = true;
+
        return 0;
 }