engines/libblkio: Add option libblkio_force_enable_completion_eventfd

[fio.git] / t / io_uring.c

diff --git a/t/io_uring.c b/t/io_uring.c
index 1746e59a1195bbdcf254cd88d771144530af3176..edbacee3fc9fe93b39c65e00044b5f76ae237a5f 100644 (file)
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -449,6 +449,8 @@ static int io_uring_register_files(struct submitter *s)
 
 static int io_uring_setup(unsigned entries, struct io_uring_params *p)
 {
+       int ret;
+
        /*
         * Clamp CQ ring size at our SQ ring size, we don't need more entries
         * than that.
@@ -456,7 +458,28 @@ static int io_uring_setup(unsigned entries, struct io_uring_params *p)
        p->flags |= IORING_SETUP_CQSIZE;
        p->cq_entries = entries;
 
-       return syscall(__NR_io_uring_setup, entries, p);
+       p->flags |= IORING_SETUP_COOP_TASKRUN;
+       p->flags |= IORING_SETUP_SINGLE_ISSUER;
+       p->flags |= IORING_SETUP_DEFER_TASKRUN;
+retry:
+       ret = syscall(__NR_io_uring_setup, entries, p);
+       if (!ret)
+               return 0;
+
+       if (errno == EINVAL && p->flags & IORING_SETUP_COOP_TASKRUN) {
+               p->flags &= ~IORING_SETUP_COOP_TASKRUN;
+               goto retry;
+       }
+       if (errno == EINVAL && p->flags & IORING_SETUP_SINGLE_ISSUER) {
+               p->flags &= ~IORING_SETUP_SINGLE_ISSUER;
+               goto retry;
+       }
+       if (errno == EINVAL && p->flags & IORING_SETUP_DEFER_TASKRUN) {
+               p->flags &= ~IORING_SETUP_DEFER_TASKRUN;
+               goto retry;
+       }
+
+       return ret;
 }
 
 static void io_uring_probe(int fd)
@@ -627,6 +650,10 @@ static void init_io_pt(struct submitter *s, unsigned index)
        cmd->cdw12 = nlb;
        cmd->addr = (unsigned long) s->iovecs[index].iov_base;
        cmd->data_len = bs;
+       if (fixedbufs) {
+               sqe->uring_cmd_flags = IORING_URING_CMD_FIXED;
+               sqe->buf_index = index;
+       }
        cmd->nsid = f->nsid;
        cmd->opcode = 2;
 }
@@ -634,12 +661,17 @@ static void init_io_pt(struct submitter *s, unsigned index)
 static int prep_more_ios_uring(struct submitter *s, int max_ios)
 {
        struct io_sq_ring *ring = &s->sq_ring;
-       unsigned index, tail, next_tail, prepped = 0;
+       unsigned head, index, tail, next_tail, prepped = 0;
+
+       if (sq_thread_poll)
+               head = atomic_load_acquire(ring->head);
+       else
+               head = *ring->head;
 
        next_tail = tail = *ring->tail;
        do {
                next_tail++;
-               if (next_tail == atomic_load_acquire(ring->head))
+               if (next_tail == head)
                        break;
 
                index = tail & sq_ring_mask;
@@ -647,7 +679,6 @@ static int prep_more_ios_uring(struct submitter *s, int max_ios)
                        init_io_pt(s, index);
                else
                        init_io(s, index);
-               ring->array[index] = index;
                prepped++;
                tail = next_tail;
        } while (prepped < max_ios);
@@ -714,7 +745,6 @@ static int reap_events_uring(struct submitter *s)
        do {
                struct file *f;
 
-               read_barrier();
                if (head == atomic_load_acquire(ring->tail))
                        break;
                cqe = &ring->cqes[head & cq_ring_mask];
@@ -769,7 +799,6 @@ static int reap_events_uring_pt(struct submitter *s)
        do {
                struct file *f;
 
-               read_barrier();
                if (head == atomic_load_acquire(ring->tail))
                        break;
                index = head & cq_ring_mask;
@@ -833,7 +862,10 @@ static int detect_node(struct submitter *s, const char *name)
        char str[128];
        int ret, fd, node;
 
-       sprintf(str, "/sys/block/%s/device/numa_node", base);
+       if (pt)
+               sprintf(str, "/sys/class/nvme-generic/%s/device/numa_node", base);
+       else
+               sprintf(str, "/sys/block/%s/device/numa_node", base);
        fd = open(str, O_RDONLY);
        if (fd < 0)
                return -1;
@@ -885,7 +917,7 @@ static int setup_ring(struct submitter *s)
        struct io_sq_ring *sring = &s->sq_ring;
        struct io_cq_ring *cring = &s->cq_ring;
        struct io_uring_params p;
-       int ret, fd;
+       int ret, fd, i;
        void *ptr;
        size_t len;
 
@@ -980,6 +1012,10 @@ static int setup_ring(struct submitter *s)
        cring->ring_entries = ptr + p.cq_off.ring_entries;
        cring->cqes = ptr + p.cq_off.cqes;
        cq_ring_mask = *cring->ring_mask;
+
+       for (i = 0; i < p.sq_entries; i++)
+               sring->array[i] = i;
+
        return 0;
 }