Update to newer io_uring API

[fio.git] / engines / io_uring.c

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 55f48edac22069f43675845b28f68f10a1862774..475ead79cb8613e1b0c9ef6d823394338ccf134b 100644 (file)
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -37,7 +37,7 @@ struct io_cq_ring {
        unsigned *tail;
        unsigned *ring_mask;
        unsigned *ring_entries;
-       struct io_uring_event *events;
+       struct io_uring_cqe *cqes;
 };
 
 struct ioring_mmap {
@@ -52,7 +52,7 @@ struct ioring_data {
        struct io_u **io_u_index;
 
        struct io_sq_ring sq_ring;
-       struct io_uring_iocb *iocbs;
+       struct io_uring_sqe *sqes;
        struct iovec *iovecs;
        unsigned sq_ring_mask;
 
@@ -151,56 +151,60 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
        struct ioring_data *ld = td->io_ops_data;
        struct ioring_options *o = td->eo;
        struct fio_file *f = io_u->file;
-       struct io_uring_iocb *iocb;
+       struct io_uring_sqe *sqe;
 
-       iocb = &ld->iocbs[io_u->index];
-       iocb->fd = f->fd;
-       iocb->flags = 0;
-       iocb->ioprio = 0;
+       sqe = &ld->sqes[io_u->index];
+       sqe->fd = f->fd;
+       sqe->flags = 0;
+       sqe->ioprio = 0;
 
        if (io_u->ddir == DDIR_READ || io_u->ddir == DDIR_WRITE) {
-               if (io_u->ddir == DDIR_READ) {
-                       if (o->fixedbufs)
-                               iocb->opcode = IORING_OP_READ_FIXED;
+               if (o->fixedbufs) {
+                       if (io_u->ddir == DDIR_READ)
+                               sqe->opcode = IORING_OP_READ_FIXED;
                        else
-                               iocb->opcode = IORING_OP_READ;
+                               sqe->opcode = IORING_OP_WRITE_FIXED;
+                       sqe->addr = io_u->xfer_buf;
+                       sqe->len = io_u->xfer_buflen;
+                       sqe->index = io_u->index;
                } else {
-                       if (o->fixedbufs)
-                               iocb->opcode = IORING_OP_WRITE_FIXED;
+                       if (io_u->ddir == DDIR_READ)
+                               sqe->opcode = IORING_OP_READV;
                        else
-                               iocb->opcode = IORING_OP_WRITE;
+                               sqe->opcode = IORING_OP_WRITEV;
+                       sqe->addr = &ld->iovecs[io_u->index];
+                       sqe->len = 1;
                }
-               iocb->off = io_u->offset;
-               iocb->addr = io_u->xfer_buf;
-               iocb->len = io_u->xfer_buflen;
+               sqe->off = io_u->offset;
        } else if (ddir_sync(io_u->ddir))
-               iocb->opcode = IORING_OP_FSYNC;
+               sqe->opcode = IORING_OP_FSYNC;
 
+       sqe->data = (unsigned long) io_u;
        return 0;
 }
 
 static struct io_u *fio_ioring_event(struct thread_data *td, int event)
 {
        struct ioring_data *ld = td->io_ops_data;
-       struct io_uring_event *ev;
+       struct io_uring_cqe *cqe;
        struct io_u *io_u;
        unsigned index;
 
        index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
 
-       ev = &ld->cq_ring.events[index];
-       io_u = ld->io_u_index[ev->index];
+       cqe = &ld->cq_ring.cqes[index];
+       io_u = (struct io_u *) cqe->data;
 
-       if (ev->res != io_u->xfer_buflen) {
-               if (ev->res > io_u->xfer_buflen)
-                       io_u->error = -ev->res;
+       if (cqe->res != io_u->xfer_buflen) {
+               if (cqe->res > io_u->xfer_buflen)
+                       io_u->error = -cqe->res;
                else
-                       io_u->resid = io_u->xfer_buflen - ev->res;
+                       io_u->resid = io_u->xfer_buflen - cqe->res;
        } else
                io_u->error = 0;
 
        if (io_u->ddir == DDIR_READ) {
-               if (ev->flags & IOEV_FLAG_CACHEHIT)
+               if (cqe->flags & IOCQE_FLAG_CACHEHIT)
                        ld->cachehit++;
                else
                        ld->cachemiss++;
@@ -417,14 +421,14 @@ static int fio_ioring_mmap(struct ioring_data *ld, struct io_uring_params *p)
        sring->array = ptr + p->sq_off.array;
        ld->sq_ring_mask = *sring->ring_mask;
 
-       ld->mmap[1].len = p->sq_entries * sizeof(struct io_uring_iocb);
-       ld->iocbs = mmap(0, ld->mmap[1].len, PROT_READ | PROT_WRITE,
+       ld->mmap[1].len = p->sq_entries * sizeof(struct io_uring_sqe);
+       ld->sqes = mmap(0, ld->mmap[1].len, PROT_READ | PROT_WRITE,
                                MAP_SHARED | MAP_POPULATE, ld->ring_fd,
-                               IORING_OFF_IOCB);
-       ld->mmap[1].ptr = ld->iocbs;
+                               IORING_OFF_SQES);
+       ld->mmap[1].ptr = ld->sqes;
 
-       ld->mmap[2].len = p->cq_off.events +
-                               p->cq_entries * sizeof(struct io_uring_event);
+       ld->mmap[2].len = p->cq_off.cqes +
+                               p->cq_entries * sizeof(struct io_uring_cqe);
        ptr = mmap(0, ld->mmap[2].len, PROT_READ | PROT_WRITE,
                        MAP_SHARED | MAP_POPULATE, ld->ring_fd,
                        IORING_OFF_CQ_RING);
@@ -433,7 +437,7 @@ static int fio_ioring_mmap(struct ioring_data *ld, struct io_uring_params *p)
        cring->tail = ptr + p->cq_off.tail;
        cring->ring_mask = ptr + p->cq_off.ring_mask;
        cring->ring_entries = ptr + p->cq_off.ring_entries;
-       cring->events = ptr + p->cq_off.events;
+       cring->cqes = ptr + p->cq_off.cqes;
        ld->cq_ring_mask = *cring->ring_mask;
        return 0;
 }
@@ -443,6 +447,7 @@ static int fio_ioring_queue_init(struct thread_data *td)
        struct ioring_data *ld = td->io_ops_data;
        struct ioring_options *o = td->eo;
        int depth = td->o.iodepth;
+       struct iovec *vecs = NULL;
        struct io_uring_params p;
        int ret;
 
@@ -466,10 +471,10 @@ static int fio_ioring_queue_init(struct thread_data *td)
                };
 
                setrlimit(RLIMIT_MEMLOCK, &rlim);
-               p.flags |= IORING_SETUP_FIXEDBUFS;
+               vecs = ld->iovecs;
        }
 
-       ret = syscall(__NR_sys_io_uring_setup, depth, ld->iovecs, &p);
+       ret = syscall(__NR_sys_io_uring_setup, depth, vecs, &p);
        if (ret < 0)
                return ret;
 
@@ -480,20 +485,15 @@ static int fio_ioring_queue_init(struct thread_data *td)
 static int fio_ioring_post_init(struct thread_data *td)
 {
        struct ioring_data *ld = td->io_ops_data;
-       struct ioring_options *o = td->eo;
        struct io_u *io_u;
-       int err;
-
-       if (o->fixedbufs) {
-               int i;
+       int err, i;
 
-               for (i = 0; i < td->o.iodepth; i++) {
-                       struct iovec *iov = &ld->iovecs[i];
+       for (i = 0; i < td->o.iodepth; i++) {
+               struct iovec *iov = &ld->iovecs[i];
 
-                       io_u = ld->io_u_index[i];
-                       iov->iov_base = io_u->buf;
-                       iov->iov_len = td_max_bs(td);
-               }
+               io_u = ld->io_u_index[i];
+               iov->iov_base = io_u->buf;
+               iov->iov_len = td_max_bs(td);
        }
 
        err = fio_ioring_queue_init(td);
@@ -523,7 +523,6 @@ static int fio_ioring_init(struct thread_data *td)
        /* io_u index */
        ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *));
        ld->io_us = calloc(td->o.iodepth, sizeof(struct io_u *));
-
        ld->iovecs = calloc(td->o.iodepth, sizeof(struct iovec));
 
        td->io_ops_data = ld;