aioring: remove IOCB_FLAG_HIPRI

[fio.git] / engines / aioring.c

diff --git a/engines/aioring.c b/engines/aioring.c
index f4973dea70557fc9486f9f6c6f2969d404fd8a2d..f836009d29013ba02dc41319d8bd5f4a9bfe1b0f 100644 (file)
--- a/engines/aioring.c
+++ b/engines/aioring.c
@@ -17,13 +17,10 @@
 #include "../lib/pow2.h"
 #include "../optgroup.h"
 #include "../lib/memalign.h"
+#include "../lib/fls.h"
 
 #ifdef ARCH_HAVE_AIORING
 
-#ifndef IOCB_FLAG_HIPRI
-#define IOCB_FLAG_HIPRI        (1 << 2)
-#endif
-
 /*
  * io_setup2(2) flags
  */
@@ -50,11 +47,8 @@
 /*
  * io_ring_enter(2) flags
  */
-#ifndef IORING_FLAG_SUBMIT
-#define IORING_FLAG_SUBMIT     (1 << 0)
-#endif
 #ifndef IORING_FLAG_GETEVENTS
-#define IORING_FLAG_GETEVENTS  (1 << 1)
+#define IORING_FLAG_GETEVENTS  (1 << 0)
 #endif
 
 typedef uint64_t u64;
@@ -99,12 +93,15 @@ struct aioring_data {
 
        struct aio_sq_ring *sq_ring;
        struct iocb *iocbs;
+       unsigned sq_ring_mask;
 
        struct aio_cq_ring *cq_ring;
        struct io_event *events;
+       unsigned cq_ring_mask;
 
        int queued;
        int cq_ring_off;
+       unsigned iodepth;
 
        uint64_t cachehit;
        uint64_t cachemiss;
@@ -192,7 +189,6 @@ static int fio_aioring_prep(struct thread_data *td, struct io_u *io_u)
 {
        struct aioring_data *ld = td->io_ops_data;
        struct fio_file *f = io_u->file;
-       struct aioring_options *o = td->eo;
        struct iocb *iocb;
 
        iocb = &ld->iocbs[io_u->index];
@@ -207,10 +203,7 @@ static int fio_aioring_prep(struct thread_data *td, struct io_u *io_u)
                iocb->u.c.buf = io_u->xfer_buf;
                iocb->u.c.nbytes = io_u->xfer_buflen;
                iocb->u.c.offset = io_u->offset;
-               if (o->hipri)
-                       iocb->u.c.flags |= IOCB_FLAG_HIPRI;
-               else
-                       iocb->u.c.flags = 0;
+               iocb->u.c.flags = 0;
        } else if (ddir_sync(io_u->ddir))
                io_prep_fsync(iocb, f->fd);
 
@@ -223,11 +216,9 @@ static struct io_u *fio_aioring_event(struct thread_data *td, int event)
        struct aioring_data *ld = td->io_ops_data;
        struct io_event *ev;
        struct io_u *io_u;
-       int index;
+       unsigned index;
 
-       index = event + ld->cq_ring_off;
-       if (index >= ld->cq_ring->nr_events)
-               index -= ld->cq_ring->nr_events;
+       index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
 
        ev = &ld->cq_ring->events[index];
        io_u = ev->data;
@@ -264,8 +255,6 @@ static int fio_aioring_cqring_reap(struct thread_data *td, unsigned int events,
                        break;
                reaped++;
                head++;
-               if (head == ring->nr_events)
-                       head = 0;
        } while (reaped + events < max);
 
        ring->head = head;
@@ -280,7 +269,8 @@ static int fio_aioring_getevents(struct thread_data *td, unsigned int min,
        unsigned actual_min = td->o.iodepth_batch_complete_min == 0 ? 0 : min;
        struct aioring_options *o = td->eo;
        struct aio_cq_ring *ring = ld->cq_ring;
-       int r, events = 0;
+       unsigned events = 0;
+       int r;
 
        ld->cq_ring_off = ring->head;
        do {
@@ -314,7 +304,7 @@ static enum fio_q_status fio_aioring_queue(struct thread_data *td,
 
        fio_ro_check(td, io_u);
 
-       if (ld->queued == td->o.iodepth)
+       if (ld->queued == ld->iodepth)
                return FIO_Q_BUSY;
 
        if (io_u->ddir == DDIR_TRIM) {
@@ -329,13 +319,11 @@ static enum fio_q_status fio_aioring_queue(struct thread_data *td,
 
        tail = ring->tail;
        next_tail = tail + 1;
-       if (next_tail == ring->nr_events)
-               next_tail = 0;
        read_barrier();
        if (next_tail == ring->head)
                return FIO_Q_BUSY;
 
-       ring->array[tail] = io_u->index;
+       ring->array[tail & ld->sq_ring_mask] = io_u->index;
        ring->tail = next_tail;
        write_barrier();
 
@@ -354,15 +342,13 @@ static void fio_aioring_queued(struct thread_data *td, int start, int nr)
        fio_gettime(&now, NULL);
 
        while (nr--) {
-               int index = ld->sq_ring->array[start];
-               struct io_u *io_u = io_u = ld->io_u_index[index];
+               int index = ld->sq_ring->array[start & ld->sq_ring_mask];
+               struct io_u *io_u = ld->io_u_index[index];
 
                memcpy(&io_u->issue_time, &now, sizeof(now));
                io_u_queued(td, io_u);
 
                start++;
-               if (start == ld->sq_ring->nr_events)
-                       start = 0;
        }
 }
 
@@ -380,17 +366,16 @@ static int fio_aioring_commit(struct thread_data *td)
                struct aio_sq_ring *ring = ld->sq_ring;
 
                if (ring->kflags & IORING_SQ_NEED_WAKEUP)
-                       io_ring_enter(ld->aio_ctx, ld->queued, 0, IORING_FLAG_SUBMIT);
+                       io_ring_enter(ld->aio_ctx, ld->queued, 0, 0);
                ld->queued = 0;
                return 0;
        }
 
        do {
-               int start = ld->sq_ring->head;
+               unsigned start = ld->sq_ring->head;
                long nr = ld->queued;
 
-               ret = io_ring_enter(ld->aio_ctx, nr, 0, IORING_FLAG_SUBMIT |
-                                               IORING_FLAG_GETEVENTS);
+               ret = io_ring_enter(ld->aio_ctx, nr, 0, IORING_FLAG_GETEVENTS);
                if (ret > 0) {
                        fio_aioring_queued(td, start, ret);
                        io_u_mark_submit(td, ret);
@@ -432,6 +417,11 @@ static size_t aioring_sq_size(struct thread_data *td)
        return sizeof(struct aio_sq_ring) + td->o.iodepth * sizeof(u32);
 }
 
+static unsigned roundup_pow2(unsigned depth)
+{
+       return 1UL << __fls(depth - 1);
+}
+
 static void fio_aioring_cleanup(struct thread_data *td)
 {
        struct aioring_data *ld = td->io_ops_data;
@@ -440,9 +430,6 @@ static void fio_aioring_cleanup(struct thread_data *td)
                td->ts.cachehit += ld->cachehit;
                td->ts.cachemiss += ld->cachemiss;
 
-               /* Bump depth to match init depth */
-               td->o.iodepth++;
-
                /*
                 * Work-around to avoid huge RCU stalls at exit time. If we
                 * don't do this here, then it'll be torn down by exit_aio().
@@ -508,20 +495,16 @@ static int fio_aioring_post_init(struct thread_data *td)
                        iocb = &ld->iocbs[i];
                        iocb->u.c.buf = io_u->buf;
                        iocb->u.c.nbytes = td_max_bs(td);
-
-                       if (o->hipri)
-                               iocb->u.c.flags |= IOCB_FLAG_HIPRI;
                }
        }
 
        err = fio_aioring_queue_init(td);
+
        if (err) {
-               td_verror(td, -err, "io_queue_init");
+               td_verror(td, errno, "io_queue_init");
                return 1;
        }
 
-       /* Adjust depth back again */
-       td->o.iodepth--;
        return 0;
 }
 
@@ -529,11 +512,12 @@ static int fio_aioring_init(struct thread_data *td)
 {
        struct aioring_data *ld;
 
-       /* ring needs an extra entry, add one to achieve QD set */
-       td->o.iodepth++;
-
        ld = calloc(1, sizeof(*ld));
 
+       /* ring depth must be a power-of-2 */
+       ld->iodepth = td->o.iodepth;
+       td->o.iodepth = roundup_pow2(td->o.iodepth);
+
        /* io_u index */
        ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *));
        ld->io_us = calloc(td->o.iodepth, sizeof(struct io_u *));
@@ -545,10 +529,12 @@ static int fio_aioring_init(struct thread_data *td)
        memset(ld->sq_ring, 0, aioring_sq_size(td));
        ld->sq_ring->nr_events = td->o.iodepth;
        ld->sq_ring->iocbs = (u64) (uintptr_t) ld->iocbs;
+       ld->sq_ring_mask = td->o.iodepth - 1;
 
        ld->cq_ring = fio_memalign(page_size, aioring_cq_size(td), false);
        memset(ld->cq_ring, 0, aioring_cq_size(td));
        ld->cq_ring->nr_events = td->o.iodepth * 2;
+       ld->cq_ring_mask = (2 * td->o.iodepth) - 1;
 
        td->io_ops_data = ld;
        return 0;