t/io_uring: fixes

[fio.git] / t / io_uring.c

diff --git a/t/io_uring.c b/t/io_uring.c
index 8d3f3a9ba198086dd1145d48ef01185c9849054f..0545a1dd680c64f00aaa68c31d55839235058c39 100644 (file)
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -22,7 +22,7 @@
 
 #include "../arch/arch.h"
 #include "../lib/types.h"
-#include "../os/io_uring.h"
+#include "../os/linux/io_uring.h"
 
 #define barrier()      __asm__ __volatile__("": : :"memory")
 
@@ -47,8 +47,8 @@ struct io_cq_ring {
 
 #define DEPTH                  128
 
-#define BATCH_SUBMIT           64
-#define BATCH_COMPLETE         64
+#define BATCH_SUBMIT           32
+#define BATCH_COMPLETE         32
 
 #define BS                     4096
 
@@ -90,6 +90,7 @@ static volatile int finish;
 
 static int polled = 1;         /* use IO polling */
 static int fixedbufs = 1;      /* use fixed user buffers */
+static int register_files = 1; /* use fixed files */
 static int buffered = 0;       /* use buffered IO, not O_DIRECT */
 static int sq_thread_poll = 0; /* use kernel submission/poller thread */
 static int sq_thread_cpu = -1; /* pin above thread to this CPU */
@@ -171,6 +172,7 @@ static void init_io(struct submitter *s, unsigned index)
                        s->cur_file++;
                        if (s->cur_file == s->nr_files)
                                s->cur_file = 0;
+                       f = &s->files[s->cur_file];
                }
        }
        f->pending_ios++;
@@ -178,7 +180,13 @@ static void init_io(struct submitter *s, unsigned index)
        lrand48_r(&s->rand, &r);
        offset = (r % (f->max_blocks - 1)) * BS;
 
-       sqe->flags = IOSQE_FIXED_FILE;
+       if (register_files) {
+               sqe->flags = IOSQE_FIXED_FILE;
+               sqe->fd = f->fixed_fd;
+       } else {
+               sqe->flags = 0;
+               sqe->fd = f->real_fd;
+       }
        if (fixedbufs) {
                sqe->opcode = IORING_OP_READ_FIXED;
                sqe->addr = s->iovecs[index].iov_base;
@@ -191,7 +199,6 @@ static void init_io(struct submitter *s, unsigned index)
                sqe->buf_index = 0;
        }
        sqe->ioprio = 0;
-       sqe->fd = f->fixed_fd;
        sqe->off = offset;
        sqe->user_data = (unsigned long) f;
 }
@@ -261,7 +268,7 @@ static int reap_events(struct submitter *s)
                        break;
                cqe = &ring->cqes[head & cq_ring_mask];
                if (!do_nop) {
-                       f = (struct file *) cqe->user_data;
+                       f = (struct file *) (uintptr_t) cqe->user_data;
                        f->pending_ios--;
                        if (cqe->res != BS) {
                                printf("io: unexpected ret=%d\n", cqe->res);
@@ -304,7 +311,7 @@ static void *submitter_fn(void *data)
 submit_more:
                to_submit = prepped;
 submit:
-               if (s->inflight + BATCH_SUBMIT < DEPTH)
+               if (to_submit && (s->inflight + to_submit <= DEPTH))
                        to_wait = 0;
                else
                        to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);
@@ -331,9 +338,10 @@ submit:
                do {
                        int r;
                        r = reap_events(s);
-                       if (r == -1)
+                       if (r == -1) {
+                               s->finish = 1;
                                break;
-                       else if (r > 0)
+                       } else if (r > 0)
                                this_reap += r;
                } while (sq_thread_poll && this_reap < to_wait);
                s->reaps += this_reap;
@@ -399,7 +407,7 @@ static int setup_ring(struct submitter *s)
 
        memset(&p, 0, sizeof(p));
 
-       if (polled)
+       if (polled && !do_nop)
                p.flags |= IORING_SETUP_IOPOLL;
        if (sq_thread_poll) {
                p.flags |= IORING_SETUP_SQPOLL;
@@ -424,10 +432,12 @@ static int setup_ring(struct submitter *s)
                }
        }
 
-       ret = io_uring_register_files(s);
-       if (ret < 0) {
-               perror("io_uring_register_files");
-               return 1;
+       if (register_files) {
+               ret = io_uring_register_files(s);
+               if (ret < 0) {
+                       perror("io_uring_register_files");
+                       return 1;
+               }
        }
 
        ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(__u32),
@@ -460,12 +470,30 @@ static int setup_ring(struct submitter *s)
        return 0;
 }
 
+static void file_depths(char *buf)
+{
+       struct submitter *s = &submitters[0];
+       char *p;
+       int i;
+
+       buf[0] = '\0';
+       p = buf;
+       for (i = 0; i < s->nr_files; i++) {
+               struct file *f = &s->files[i];
+
+               if (i + 1 == s->nr_files)
+                       p += sprintf(p, "%d", f->pending_ios);
+               else
+                       p += sprintf(p, "%d, ", f->pending_ios);
+       }
+}
+
 int main(int argc, char *argv[])
 {
        struct submitter *s = &submitters[0];
        unsigned long done, calls, reap, cache_hit, cache_miss;
        int err, i, flags, fd;
-       struct rlimit rlim;
+       char *fdepths;
        void *ret;
 
        if (!do_nop && argc < 2) {
@@ -502,11 +530,15 @@ int main(int argc, char *argv[])
                i++;
        }
 
-       rlim.rlim_cur = RLIM_INFINITY;
-       rlim.rlim_max = RLIM_INFINITY;
-       if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
-               perror("setrlimit");
-               return 1;
+       if (fixedbufs) {
+               struct rlimit rlim;
+
+               rlim.rlim_cur = RLIM_INFINITY;
+               rlim.rlim_max = RLIM_INFINITY;
+               if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) {
+                       perror("setrlimit");
+                       return 1;
+               }
        }
 
        arm_sig_int();
@@ -532,6 +564,7 @@ int main(int argc, char *argv[])
 
        pthread_create(&s->thread, NULL, submitter_fn, s);
 
+       fdepths = malloc(8 * s->nr_files);
        cache_hit = cache_miss = reap = calls = done = 0;
        do {
                unsigned long this_done = 0;
@@ -561,9 +594,10 @@ int main(int argc, char *argv[])
                        ipc = (this_reap - reap) / (this_call - calls);
                } else
                        rpc = ipc = -1;
-               printf("IOPS=%lu, IOS/call=%ld/%ld, inflight=%u (head=%u tail=%u), Cachehit=%0.2f%%\n",
+               file_depths(fdepths);
+               printf("IOPS=%lu, IOS/call=%ld/%ld, inflight=%u (%s), Cachehit=%0.2f%%\n",
                                this_done - done, rpc, ipc, s->inflight,
-                               *s->cq_ring.head, *s->cq_ring.tail, hit);
+                               fdepths, hit);
                done = this_done;
                calls = this_call;
                reap = this_reap;
@@ -573,5 +607,6 @@ int main(int argc, char *argv[])
 
        pthread_join(s->thread, &ret);
        close(s->ring_fd);
+       free(fdepths);
        return 0;
 }