X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=engines%2Fio_uring.c;h=b962e8041b6f8d113669b4b2a31224a68d19aa0f;hb=6b8cadb66c62394420a39b46af1a2967b916c829;hp=15a4d4753639ce3bf6ef73b1d752a491fc520129;hpb=e2239016d4839aeb54b5da61f94baf0d518233fb;p=fio.git diff --git a/engines/io_uring.c b/engines/io_uring.c index 15a4d475..b962e804 100644 --- a/engines/io_uring.c +++ b/engines/io_uring.c @@ -17,10 +17,12 @@ #include "../optgroup.h" #include "../lib/memalign.h" #include "../lib/fls.h" +#include "../lib/roundup.h" #ifdef ARCH_HAVE_IOURING -#include "../os/io_uring.h" +#include "../lib/types.h" +#include "../os/linux/io_uring.h" struct io_sq_ring { unsigned *head; @@ -36,7 +38,7 @@ struct io_cq_ring { unsigned *tail; unsigned *ring_mask; unsigned *ring_entries; - struct io_uring_event *events; + struct io_uring_cqe *cqes; }; struct ioring_mmap { @@ -47,11 +49,12 @@ struct ioring_mmap { struct ioring_data { int ring_fd; - struct io_u **io_us; struct io_u **io_u_index; + int *fds; + struct io_sq_ring sq_ring; - struct io_uring_iocb *iocbs; + struct io_uring_sqe *sqes; struct iovec *iovecs; unsigned sq_ring_mask; @@ -61,9 +64,9 @@ struct ioring_data { int queued; int cq_ring_off; unsigned iodepth; - - uint64_t cachehit; - uint64_t cachemiss; + bool ioprio_class_set; + bool ioprio_set; + int prepped; struct ioring_mmap mmap[3]; }; @@ -71,19 +74,34 @@ struct ioring_data { struct ioring_options { void *pad; unsigned int hipri; + unsigned int cmdprio_percentage; unsigned int fixedbufs; - unsigned int sqthread; - unsigned int sqthread_set; - unsigned int sqthread_poll; - unsigned int sqwq; + unsigned int registerfiles; + unsigned int sqpoll_thread; + unsigned int sqpoll_set; + unsigned int sqpoll_cpu; + unsigned int nonvectored; + unsigned int uncached; + unsigned int nowait; + unsigned int force_async; }; -static int fio_ioring_sqthread_cb(void *data, unsigned long long *val) +static const int ddir_to_op[2][2] = { + { IORING_OP_READV, IORING_OP_READ }, + { IORING_OP_WRITEV, IORING_OP_WRITE } +}; + +static const int fixed_ddir_to_op[2] = { + IORING_OP_READ_FIXED, + IORING_OP_WRITE_FIXED +}; + +static int fio_ioring_sqpoll_cb(void *data, unsigned long long *val) { struct ioring_options *o = data; - o->sqthread = *val; - o->sqthread_set = 1; + o->sqpoll_cpu = *val; + o->sqpoll_set = 1; return 0; } @@ -95,8 +113,28 @@ static struct fio_option options[] = { .off1 = offsetof(struct ioring_options, hipri), .help = "Use polled IO completions", .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, + .group = FIO_OPT_G_IOURING, + }, +#ifdef FIO_HAVE_IOPRIO_CLASS + { + .name = "cmdprio_percentage", + .lname = "high priority percentage", + .type = FIO_OPT_INT, + .off1 = offsetof(struct ioring_options, cmdprio_percentage), + .minval = 1, + .maxval = 100, + .help = "Send high priority I/O this percentage of the time", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_IOURING, }, +#else + { + .name = "cmdprio_percentage", + .lname = "high priority percentage", + .type = FIO_OPT_UNSUPPORTED, + .help = "Your platform does not support I/O priority classes", + }, +#endif { .name = "fixedbufs", .lname = "Fixed (pre-mapped) IO buffers", @@ -104,34 +142,71 @@ static struct fio_option options[] = { .off1 = offsetof(struct ioring_options, fixedbufs), .help = "Pre map IO buffers", .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, + .group = FIO_OPT_G_IOURING, }, { - .name = "sqthread", - .lname = "Use kernel SQ thread on this CPU", - .type = FIO_OPT_INT, - .cb = fio_ioring_sqthread_cb, - .help = "Offload submission to kernel thread", + .name = "registerfiles", + .lname = "Register file set", + .type = FIO_OPT_STR_SET, + .off1 = offsetof(struct ioring_options, registerfiles), + .help = "Pre-open/register files", .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, + .group = FIO_OPT_G_IOURING, }, { .name = "sqthread_poll", - .lname = "Kernel SQ thread should poll", - .type = FIO_OPT_STR_SET, - .off1 = offsetof(struct ioring_options, sqthread_poll), - .help = "Used with sqthread, enables kernel side polling", + .lname = "Kernel SQ thread polling", + .type = FIO_OPT_INT, + .off1 = offsetof(struct ioring_options, sqpoll_thread), + .help = "Offload submission/completion to kernel thread", .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, + .group = FIO_OPT_G_IOURING, }, { - .name = "sqwq", - .lname = "Offload submission to kernel workqueue", - .type = FIO_OPT_STR_SET, - .off1 = offsetof(struct ioring_options, sqwq), - .help = "Offload submission to kernel workqueue", + .name = "sqthread_poll_cpu", + .lname = "SQ Thread Poll CPU", + .type = FIO_OPT_INT, + .cb = fio_ioring_sqpoll_cb, + .help = "What CPU to run SQ thread polling on", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_IOURING, + }, + { + .name = "nonvectored", + .lname = "Non-vectored", + .type = FIO_OPT_INT, + .off1 = offsetof(struct ioring_options, nonvectored), + .def = "-1", + .help = "Use non-vectored read/write commands", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_IOURING, + }, + { + .name = "uncached", + .lname = "Uncached", + .type = FIO_OPT_INT, + .off1 = offsetof(struct ioring_options, uncached), + .help = "Use RWF_UNCACHED for buffered read/writes", .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, + .group = FIO_OPT_G_IOURING, + }, + { + .name = "nowait", + .lname = "RWF_NOWAIT", + .type = FIO_OPT_BOOL, + .off1 = offsetof(struct ioring_options, nowait), + .help = "Use RWF_NOWAIT for reads/writes", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_IOURING, + }, + { + .name = "force_async", + .lname = "Force async", + .type = FIO_OPT_INT, + .off1 = offsetof(struct ioring_options, force_async), + .help = "Set IOSQE_ASYNC every N requests", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_IOURING, }, { .name = NULL, @@ -141,62 +216,107 @@ static struct fio_option options[] = { static int io_uring_enter(struct ioring_data *ld, unsigned int to_submit, unsigned int min_complete, unsigned int flags) { - return syscall(__NR_sys_io_uring_enter, ld->ring_fd, to_submit, - min_complete, flags); + return syscall(__NR_io_uring_enter, ld->ring_fd, to_submit, + min_complete, flags, NULL, 0); } static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) { struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; struct fio_file *f = io_u->file; - struct io_uring_iocb *iocb; + struct io_uring_sqe *sqe; - iocb = &ld->iocbs[io_u->index]; - iocb->fd = f->fd; - iocb->flags = 0; - iocb->ioprio = 0; + sqe = &ld->sqes[io_u->index]; + + if (o->registerfiles) { + sqe->fd = f->engine_pos; + sqe->flags = IOSQE_FIXED_FILE; + } else { + sqe->fd = f->fd; + } if (io_u->ddir == DDIR_READ || io_u->ddir == DDIR_WRITE) { - if (io_u->ddir == DDIR_READ) - iocb->opcode = IORING_OP_READ; - else - iocb->opcode = IORING_OP_WRITE; - iocb->off = io_u->offset; - iocb->addr = io_u->xfer_buf; - iocb->len = io_u->xfer_buflen; - } else if (ddir_sync(io_u->ddir)) - iocb->opcode = IORING_OP_FSYNC; + if (o->fixedbufs) { + sqe->opcode = fixed_ddir_to_op[io_u->ddir]; + sqe->addr = (unsigned long) io_u->xfer_buf; + sqe->len = io_u->xfer_buflen; + sqe->buf_index = io_u->index; + } else { + struct iovec *iov = &ld->iovecs[io_u->index]; + + /* + * Update based on actual io_u, requeue could have + * adjusted these + */ + iov->iov_base = io_u->xfer_buf; + iov->iov_len = io_u->xfer_buflen; + + sqe->opcode = ddir_to_op[io_u->ddir][!!o->nonvectored]; + if (o->nonvectored) { + sqe->addr = (unsigned long) iov->iov_base; + sqe->len = iov->iov_len; + } else { + sqe->addr = (unsigned long) iov; + sqe->len = 1; + } + } + if (!td->o.odirect && o->uncached) + sqe->rw_flags = RWF_UNCACHED; + if (o->nowait) + sqe->rw_flags |= RWF_NOWAIT; + if (ld->ioprio_class_set) + sqe->ioprio = td->o.ioprio_class << 13; + if (ld->ioprio_set) + sqe->ioprio |= td->o.ioprio; + sqe->off = io_u->offset; + sqe->rw_flags = 0; + } else if (ddir_sync(io_u->ddir)) { + sqe->ioprio = 0; + if (io_u->ddir == DDIR_SYNC_FILE_RANGE) { + sqe->off = f->first_write; + sqe->len = f->last_write - f->first_write; + sqe->sync_range_flags = td->o.sync_file_range; + sqe->opcode = IORING_OP_SYNC_FILE_RANGE; + } else { + sqe->off = 0; + sqe->addr = 0; + sqe->len = 0; + if (io_u->ddir == DDIR_DATASYNC) + sqe->fsync_flags |= IORING_FSYNC_DATASYNC; + sqe->opcode = IORING_OP_FSYNC; + } + } + if (o->force_async && ++ld->prepped == o->force_async) { + ld->prepped = 0; + sqe->flags |= IOSQE_ASYNC; + } + + sqe->user_data = (unsigned long) io_u; return 0; } static struct io_u *fio_ioring_event(struct thread_data *td, int event) { struct ioring_data *ld = td->io_ops_data; - struct io_uring_event *ev; + struct io_uring_cqe *cqe; struct io_u *io_u; unsigned index; index = (event + ld->cq_ring_off) & ld->cq_ring_mask; - ev = &ld->cq_ring.events[index]; - io_u = ld->io_u_index[ev->index]; + cqe = &ld->cq_ring.cqes[index]; + io_u = (struct io_u *) (uintptr_t) cqe->user_data; - if (ev->res != io_u->xfer_buflen) { - if (ev->res > io_u->xfer_buflen) - io_u->error = -ev->res; + if (cqe->res != io_u->xfer_buflen) { + if (cqe->res > io_u->xfer_buflen) + io_u->error = -cqe->res; else - io_u->resid = io_u->xfer_buflen - ev->res; + io_u->resid = io_u->xfer_buflen - cqe->res; } else io_u->error = 0; - if (io_u->ddir == DDIR_READ) { - if (ev->flags & IOEV_FLAG_CACHEHIT) - ld->cachehit++; - else - ld->cachemiss++; - } - return io_u; } @@ -209,15 +329,15 @@ static int fio_ioring_cqring_reap(struct thread_data *td, unsigned int events, head = *ring->head; do { - read_barrier(); - if (head == *ring->tail) + if (head == atomic_load_acquire(ring->tail)) break; reaped++; head++; } while (reaped + events < max); - *ring->head = head; - write_barrier(); + if (reaped) + atomic_store_release(ring->head, head); + return reaped; } @@ -236,14 +356,16 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min, r = fio_ioring_cqring_reap(td, events, max); if (r) { events += r; + if (actual_min != 0) + actual_min -= r; continue; } - if (!o->sqthread_poll) { + if (!o->sqpoll_thread) { r = io_uring_enter(ld, 0, actual_min, IORING_ENTER_GETEVENTS); if (r < 0) { - if (errno == EAGAIN) + if (errno == EAGAIN || errno == EINTR) continue; td_verror(td, errno, "io_uring_enter"); break; @@ -254,11 +376,23 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min, return r < 0 ? r : events; } +static void fio_ioring_prio_prep(struct thread_data *td, struct io_u *io_u) +{ + struct ioring_options *o = td->eo; + struct ioring_data *ld = td->io_ops_data; + if (rand_between(&td->prio_state, 0, 99) < o->cmdprio_percentage) { + ld->sqes[io_u->index].ioprio = IOPRIO_CLASS_RT << IOPRIO_CLASS_SHIFT; + io_u->flags |= IO_U_F_PRIORITY; + } + return; +} + static enum fio_q_status fio_ioring_queue(struct thread_data *td, struct io_u *io_u) { struct ioring_data *ld = td->io_ops_data; struct io_sq_ring *ring = &ld->sq_ring; + struct ioring_options *o = td->eo; unsigned tail, next_tail; fio_ro_check(td, io_u); @@ -278,13 +412,13 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td, tail = *ring->tail; next_tail = tail + 1; - read_barrier(); - if (next_tail == *ring->head) + if (next_tail == atomic_load_acquire(ring->head)) return FIO_Q_BUSY; + if (o->cmdprio_percentage) + fio_ioring_prio_prep(td, io_u); ring->array[tail & ld->sq_ring_mask] = io_u->index; - *ring->tail = next_tail; - write_barrier(); + atomic_store_release(ring->tail, next_tail); ld->queued++; return FIO_Q_QUEUED; @@ -321,12 +455,19 @@ static int fio_ioring_commit(struct thread_data *td) if (!ld->queued) return 0; - /* Nothing to do */ - if (o->sqthread_poll) { + /* + * Kernel side does submission. just need to check if the ring is + * flagged as needing a kick, if so, call io_uring_enter(). This + * only happens if we've been idle too long. + */ + if (o->sqpoll_thread) { struct io_sq_ring *ring = &ld->sq_ring; + unsigned flags; - if (*ring->flags & IORING_SQ_NEED_WAKEUP) - io_uring_enter(ld, ld->queued, 0, 0); + flags = atomic_load_acquire(ring->flags); + if (flags & IORING_SQ_NEED_WAKEUP) + io_uring_enter(ld, ld->queued, 0, + IORING_ENTER_SQ_WAKEUP); ld->queued = 0; return 0; } @@ -346,7 +487,7 @@ static int fio_ioring_commit(struct thread_data *td) io_u_mark_submit(td, ret); continue; } else { - if (errno == EAGAIN) { + if (errno == EAGAIN || errno == EINTR) { ret = fio_ioring_cqring_reap(td, 0, ld->queued); if (ret) continue; @@ -366,7 +507,7 @@ static void fio_ioring_unmap(struct ioring_data *ld) { int i; - for (i = 0; i < ARRAY_SIZE(ld->mmap); i++) + for (i = 0; i < FIO_ARRAY_SIZE(ld->mmap); i++) munmap(ld->mmap[i].ptr, ld->mmap[i].len); close(ld->ring_fd); } @@ -376,15 +517,12 @@ static void fio_ioring_cleanup(struct thread_data *td) struct ioring_data *ld = td->io_ops_data; if (ld) { - td->ts.cachehit += ld->cachehit; - td->ts.cachemiss += ld->cachemiss; - if (!(td->flags & TD_F_CHILD)) fio_ioring_unmap(ld); free(ld->io_u_index); - free(ld->io_us); free(ld->iovecs); + free(ld->fds); free(ld); } } @@ -408,14 +546,14 @@ static int fio_ioring_mmap(struct ioring_data *ld, struct io_uring_params *p) sring->array = ptr + p->sq_off.array; ld->sq_ring_mask = *sring->ring_mask; - ld->mmap[1].len = p->sq_entries * sizeof(struct io_uring_iocb); - ld->iocbs = mmap(0, ld->mmap[1].len, PROT_READ | PROT_WRITE, + ld->mmap[1].len = p->sq_entries * sizeof(struct io_uring_sqe); + ld->sqes = mmap(0, ld->mmap[1].len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, ld->ring_fd, - IORING_OFF_IOCB); - ld->mmap[1].ptr = ld->iocbs; + IORING_OFF_SQES); + ld->mmap[1].ptr = ld->sqes; - ld->mmap[2].len = p->cq_off.events + - p->cq_entries * sizeof(struct io_uring_event); + ld->mmap[2].len = p->cq_off.cqes + + p->cq_entries * sizeof(struct io_uring_cqe); ptr = mmap(0, ld->mmap[2].len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, ld->ring_fd, IORING_OFF_CQ_RING); @@ -424,11 +562,45 @@ static int fio_ioring_mmap(struct ioring_data *ld, struct io_uring_params *p) cring->tail = ptr + p->cq_off.tail; cring->ring_mask = ptr + p->cq_off.ring_mask; cring->ring_entries = ptr + p->cq_off.ring_entries; - cring->events = ptr + p->cq_off.events; + cring->cqes = ptr + p->cq_off.cqes; ld->cq_ring_mask = *cring->ring_mask; return 0; } +static void fio_ioring_probe(struct thread_data *td) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + struct io_uring_probe *p; + int ret; + + /* already set by user, don't touch */ + if (o->nonvectored != -1) + return; + + /* default to off, as that's always safe */ + o->nonvectored = 0; + + p = malloc(sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); + if (!p) + return; + + memset(p, 0, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); + ret = syscall(__NR_io_uring_register, ld->ring_fd, + IORING_REGISTER_PROBE, p, 256); + if (ret < 0) + goto out; + + if (IORING_OP_WRITE > p->ops_len) + goto out; + + if ((p->ops[IORING_OP_READ].flags & IO_URING_OP_SUPPORTED) && + (p->ops[IORING_OP_WRITE].flags & IO_URING_OP_SUPPORTED)) + o->nonvectored = 1; +out: + free(p); +} + static int fio_ioring_queue_init(struct thread_data *td) { struct ioring_data *ld = td->io_ops_data; @@ -441,69 +613,136 @@ static int fio_ioring_queue_init(struct thread_data *td) if (o->hipri) p.flags |= IORING_SETUP_IOPOLL; - if (o->sqthread_set) { - p.sq_thread_cpu = o->sqthread; - p.flags |= IORING_SETUP_SQTHREAD; - if (o->sqthread_poll) - p.flags |= IORING_SETUP_SQPOLL; - } - if (o->sqwq) - p.flags |= IORING_SETUP_SQWQ; - - if (o->fixedbufs) { - struct rlimit rlim = { - .rlim_cur = RLIM_INFINITY, - .rlim_max = RLIM_INFINITY, - }; - - setrlimit(RLIMIT_MEMLOCK, &rlim); - p.flags |= IORING_SETUP_FIXEDBUFS; + if (o->sqpoll_thread) { + p.flags |= IORING_SETUP_SQPOLL; + if (o->sqpoll_set) { + p.flags |= IORING_SETUP_SQ_AFF; + p.sq_thread_cpu = o->sqpoll_cpu; + } } - ret = syscall(__NR_sys_io_uring_setup, depth, ld->iovecs, &p); + ret = syscall(__NR_io_uring_setup, depth, &p); if (ret < 0) return ret; ld->ring_fd = ret; + + fio_ioring_probe(td); + + if (o->fixedbufs) { + ret = syscall(__NR_io_uring_register, ld->ring_fd, + IORING_REGISTER_BUFFERS, ld->iovecs, depth); + if (ret < 0) + return ret; + } + return fio_ioring_mmap(ld, &p); } +static int fio_ioring_register_files(struct thread_data *td) +{ + struct ioring_data *ld = td->io_ops_data; + struct fio_file *f; + unsigned int i; + int ret; + + ld->fds = calloc(td->o.nr_files, sizeof(int)); + + for_each_file(td, f, i) { + ret = generic_open_file(td, f); + if (ret) + goto err; + ld->fds[i] = f->fd; + f->engine_pos = i; + } + + ret = syscall(__NR_io_uring_register, ld->ring_fd, + IORING_REGISTER_FILES, ld->fds, td->o.nr_files); + if (ret) { +err: + free(ld->fds); + ld->fds = NULL; + } + + /* + * Pretend the file is closed again, and really close it if we hit + * an error. + */ + for_each_file(td, f, i) { + if (ret) { + int fio_unused ret2; + ret2 = generic_close_file(td, f); + } else + f->fd = -1; + } + + return ret; +} + static int fio_ioring_post_init(struct thread_data *td) { struct ioring_data *ld = td->io_ops_data; struct ioring_options *o = td->eo; struct io_u *io_u; - int err; + int err, i; - if (o->fixedbufs) { - int i; - - for (i = 0; i < td->o.iodepth; i++) { - struct iovec *iov = &ld->iovecs[i]; + for (i = 0; i < td->o.iodepth; i++) { + struct iovec *iov = &ld->iovecs[i]; - io_u = ld->io_u_index[i]; - iov->iov_base = io_u->buf; - iov->iov_len = td_max_bs(td); - } + io_u = ld->io_u_index[i]; + iov->iov_base = io_u->buf; + iov->iov_len = td_max_bs(td); } err = fio_ioring_queue_init(td); if (err) { - td_verror(td, errno, "io_queue_init"); + int init_err = errno; + + if (init_err == ENOSYS) + log_err("fio: your kernel doesn't support io_uring\n"); + td_verror(td, init_err, "io_queue_init"); return 1; } - return 0; -} + for (i = 0; i < td->o.iodepth; i++) { + struct io_uring_sqe *sqe; -static unsigned roundup_pow2(unsigned depth) -{ - return 1UL << __fls(depth - 1); + sqe = &ld->sqes[i]; + memset(sqe, 0, sizeof(*sqe)); + } + + if (o->registerfiles) { + err = fio_ioring_register_files(td); + if (err) { + td_verror(td, errno, "ioring_register_files"); + return 1; + } + } + + return 0; } static int fio_ioring_init(struct thread_data *td) { + struct ioring_options *o = td->eo; struct ioring_data *ld; + struct thread_options *to = &td->o; + + if (to->io_submit_mode == IO_MODE_OFFLOAD) { + log_err("fio: io_submit_mode=offload is not compatible (or " + "useful) with io_uring\n"); + return 1; + } + + /* sqthread submission requires registered files */ + if (o->sqpoll_thread) + o->registerfiles = 1; + + if (o->registerfiles && td->o.nr_files != td->o.open_files) { + log_err("fio: io_uring registered files require nr_files to " + "be identical to open_files\n"); + return 1; + } ld = calloc(1, sizeof(*ld)); @@ -513,11 +752,26 @@ static int fio_ioring_init(struct thread_data *td) /* io_u index */ ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *)); - ld->io_us = calloc(td->o.iodepth, sizeof(struct io_u *)); - ld->iovecs = calloc(td->o.iodepth, sizeof(struct iovec)); td->io_ops_data = ld; + + /* + * Check for option conflicts + */ + if ((fio_option_is_set(to, ioprio) || fio_option_is_set(to, ioprio_class)) && + o->cmdprio_percentage != 0) { + log_err("%s: cmdprio_percentage option and mutually exclusive " + "prio or prioclass option is set, exiting\n", to->name); + td_verror(td, EINVAL, "fio_io_uring_init"); + return 1; + } + + if (fio_option_is_set(&td->o, ioprio_class)) + ld->ioprio_class_set = true; + if (fio_option_is_set(&td->o, ioprio)) + ld->ioprio_set = true; + return 0; } @@ -529,9 +783,34 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u) return 0; } +static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + + if (!ld || !o->registerfiles) + return generic_open_file(td, f); + + f->fd = ld->fds[f->engine_pos]; + return 0; +} + +static int fio_ioring_close_file(struct thread_data *td, struct fio_file *f) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + + if (!ld || !o->registerfiles) + return generic_close_file(td, f); + + f->fd = -1; + return 0; +} + static struct ioengine_ops ioengine = { .name = "io_uring", .version = FIO_IOOPS_VERSION, + .flags = FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD, .init = fio_ioring_init, .post_init = fio_ioring_post_init, .io_u_init = fio_ioring_io_u_init, @@ -541,8 +820,8 @@ static struct ioengine_ops ioengine = { .getevents = fio_ioring_getevents, .event = fio_ioring_event, .cleanup = fio_ioring_cleanup, - .open_file = generic_open_file, - .close_file = generic_close_file, + .open_file = fio_ioring_open_file, + .close_file = fio_ioring_close_file, .get_file_size = generic_get_file_size, .options = options, .option_struct_size = sizeof(struct ioring_options),