engines/aioring: enable IOCTX_FLAG_SQPOLL

With this flag set, we don't have to do any system calls for
polled IO.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/engines/aioring.c b/engines/aioring.c
index 51e0e1c13d8d6acb524e54762810f2b49715b590..cb13b4158b2bc117e995b770e2c8bf21a87cc657 100644 (file)
--- a/engines/aioring.c
+++ b/engines/aioring.c
@@ -42,6 +42,10 @@
 #ifndef IOCTX_FLAG_SQWQ
 #define IOCTX_FLAG_SQWQ                (1 << 4)
 #endif
+#ifndef IOCTX_FLAG_SQPOLL
+#define IOCTX_FLAG_SQPOLL      (1 << 5)
+#endif
+
 
 /*
  * io_ring_enter(2) flags
@@ -104,6 +108,7 @@ struct aioring_options {
        unsigned int fixedbufs;
        unsigned int sqthread;
        unsigned int sqthread_set;
+       unsigned int sqthread_poll;
        unsigned int sqwq;
 };
 
@@ -145,6 +150,15 @@ static struct fio_option options[] = {
                .category = FIO_OPT_C_ENGINE,
                .group  = FIO_OPT_G_LIBAIO,
        },
+       {
+               .name   = "sqthread_poll",
+               .lname  = "Kernel SQ thread should poll",
+               .type   = FIO_OPT_STR_SET,
+               .off1   = offsetof(struct aioring_options, sqthread_poll),
+               .help   = "Used with sqthread, enables kernel side polling",
+               .category = FIO_OPT_C_ENGINE,
+               .group  = FIO_OPT_G_LIBAIO,
+       },
        {
                .name   = "sqwq",
                .lname  = "Offload submission to kernel workqueue",
@@ -159,8 +173,6 @@ static struct fio_option options[] = {
        },
 };
 
-static int fio_aioring_commit(struct thread_data *td);
-
 static int io_ring_enter(io_context_t ctx, unsigned int to_submit,
                         unsigned int min_complete, unsigned int flags)
 {
@@ -257,6 +269,7 @@ static int fio_aioring_getevents(struct thread_data *td, unsigned int min,
 {
        struct aioring_data *ld = td->io_ops_data;
        unsigned actual_min = td->o.iodepth_batch_complete_min == 0 ? 0 : min;
+       struct aioring_options *o = td->eo;
        struct aio_cq_ring *ring = ld->cq_ring;
        int r, events = 0;
 
@@ -268,13 +281,15 @@ static int fio_aioring_getevents(struct thread_data *td, unsigned int min,
                        continue;
                }
 
-               r = io_ring_enter(ld->aio_ctx, 0, actual_min,
-                                       IORING_FLAG_GETEVENTS);
-               if (r < 0) {
-                       if (errno == EAGAIN)
-                               continue;
-                       td_verror(td, errno, "io_ring_enter get");
-                       break;
+               if (!o->sqthread_poll) {
+                       r = io_ring_enter(ld->aio_ctx, 0, actual_min,
+                                               IORING_FLAG_GETEVENTS);
+                       if (r < 0) {
+                               if (errno == EAGAIN)
+                                       continue;
+                               td_verror(td, errno, "io_ring_enter get");
+                               break;
+                       }
                }
        } while (events < min);
 
@@ -345,11 +360,18 @@ static void fio_aioring_queued(struct thread_data *td, int start, int nr)
 static int fio_aioring_commit(struct thread_data *td)
 {
        struct aioring_data *ld = td->io_ops_data;
+       struct aioring_options *o = td->eo;
        int ret;
 
        if (!ld->queued)
                return 0;
 
+       /* Nothing to do */
+       if (o->sqthread_poll) {
+               ld->queued = 0;
+               return 0;
+       }
+
        do {
                int start = ld->sq_ring->head;
                long nr = ld->queued;
@@ -434,6 +456,8 @@ static int fio_aioring_queue_init(struct thread_data *td)
        if (o->sqthread_set) {
                ld->sq_ring->sq_thread_cpu = o->sqthread;
                flags |= IOCTX_FLAG_SQTHREAD;
+               if (o->sqthread_poll)
+                       flags |= IOCTX_FLAG_SQPOLL;
        } else if (o->sqwq)
                flags |= IOCTX_FLAG_SQWQ;