X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=engines%2Fio_uring.c;h=e1abf6888fd50c886b87ecc29cf25519c8bbba9d;hb=79012fece0a0ecb02ad6a3322913980efdb1d726;hp=0ccd23184fab3b17a9c68c93a9c30375b5673986;hpb=6d975f2c8c3061b1d39c121d509d866890afd162;p=fio.git diff --git a/engines/io_uring.c b/engines/io_uring.c index 0ccd2318..e1abf688 100644 --- a/engines/io_uring.c +++ b/engines/io_uring.c @@ -23,6 +23,15 @@ #include "../lib/types.h" #include "../os/linux/io_uring.h" +#include "cmdprio.h" +#include "zbd.h" +#include "nvme.h" + +#include + +enum uring_cmd_type { + FIO_URING_CMD_NVME = 1, +}; struct io_sq_ring { unsigned *head; @@ -64,16 +73,19 @@ struct ioring_data { int queued; int cq_ring_off; unsigned iodepth; - bool ioprio_class_set; - bool ioprio_set; + int prepped; struct ioring_mmap mmap[3]; + + struct cmdprio cmdprio; + + struct nvme_dsm_range *dsm; }; struct ioring_options { - void *pad; + struct thread_data *td; unsigned int hipri; - unsigned int cmdprio_percentage; + struct cmdprio_options cmdprio_options; unsigned int fixedbufs; unsigned int registerfiles; unsigned int sqpoll_thread; @@ -82,6 +94,8 @@ struct ioring_options { unsigned int nonvectored; unsigned int uncached; unsigned int nowait; + unsigned int force_async; + enum uring_cmd_type cmd_type; }; static const int ddir_to_op[2][2] = { @@ -113,26 +127,6 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_IOURING, }, -#ifdef FIO_HAVE_IOPRIO_CLASS - { - .name = "cmdprio_percentage", - .lname = "high priority percentage", - .type = FIO_OPT_INT, - .off1 = offsetof(struct ioring_options, cmdprio_percentage), - .minval = 1, - .maxval = 100, - .help = "Send high priority I/O this percentage of the time", - .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_IOURING, - }, -#else - { - .name = "cmdprio_percentage", - .lname = "high priority percentage", - .type = FIO_OPT_UNSUPPORTED, - .help = "Your platform does not support I/O priority classes", - }, -#endif { .name = "fixedbufs", .lname = "Fixed (pre-mapped) IO buffers", @@ -154,7 +148,7 @@ static struct fio_option options[] = { { .name = "sqthread_poll", .lname = "Kernel SQ thread polling", - .type = FIO_OPT_INT, + .type = FIO_OPT_STR_SET, .off1 = offsetof(struct ioring_options, sqpoll_thread), .help = "Offload submission/completion to kernel thread", .category = FIO_OPT_C_ENGINE, @@ -174,6 +168,7 @@ static struct fio_option options[] = { .lname = "Non-vectored", .type = FIO_OPT_INT, .off1 = offsetof(struct ioring_options, nonvectored), + .def = "-1", .help = "Use non-vectored read/write commands", .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_IOURING, @@ -196,6 +191,32 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_IOURING, }, + { + .name = "force_async", + .lname = "Force async", + .type = FIO_OPT_INT, + .off1 = offsetof(struct ioring_options, force_async), + .help = "Set IOSQE_ASYNC every N requests", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_IOURING, + }, + { + .name = "cmd_type", + .lname = "Uring cmd type", + .type = FIO_OPT_STR, + .off1 = offsetof(struct ioring_options, cmd_type), + .help = "Specify uring-cmd type", + .def = "nvme", + .posval = { + { .ival = "nvme", + .oval = FIO_URING_CMD_NVME, + .help = "Issue nvme-uring-cmd", + }, + }, + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_IOURING, + }, + CMDPRIO_OPTIONS(struct ioring_options, FIO_OPT_G_IOURING), { .name = NULL, }, @@ -204,8 +225,13 @@ static struct fio_option options[] = { static int io_uring_enter(struct ioring_data *ld, unsigned int to_submit, unsigned int min_complete, unsigned int flags) { +#ifdef FIO_ARCH_HAS_SYSCALL + return __do_syscall6(__NR_io_uring_enter, ld->ring_fd, to_submit, + min_complete, flags, NULL, 0); +#else return syscall(__NR_io_uring_enter, ld->ring_fd, to_submit, min_complete, flags, NULL, 0); +#endif } static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) @@ -217,14 +243,12 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) sqe = &ld->sqes[io_u->index]; - /* zero out fields not used in this submission */ - memset(sqe, 0, sizeof(*sqe)); - if (o->registerfiles) { sqe->fd = f->engine_pos; sqe->flags = IOSQE_FIXED_FILE; } else { sqe->fd = f->fd; + sqe->flags = 0; } if (io_u->ddir == DDIR_READ || io_u->ddir == DDIR_WRITE) { @@ -234,42 +258,118 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) sqe->len = io_u->xfer_buflen; sqe->buf_index = io_u->index; } else { + struct iovec *iov = &ld->iovecs[io_u->index]; + + /* + * Update based on actual io_u, requeue could have + * adjusted these + */ + iov->iov_base = io_u->xfer_buf; + iov->iov_len = io_u->xfer_buflen; + sqe->opcode = ddir_to_op[io_u->ddir][!!o->nonvectored]; if (o->nonvectored) { - sqe->addr = (unsigned long) - ld->iovecs[io_u->index].iov_base; - sqe->len = ld->iovecs[io_u->index].iov_len; + sqe->addr = (unsigned long) iov->iov_base; + sqe->len = iov->iov_len; } else { - sqe->addr = (unsigned long) &ld->iovecs[io_u->index]; + sqe->addr = (unsigned long) iov; sqe->len = 1; } } + sqe->rw_flags = 0; if (!td->o.odirect && o->uncached) - sqe->rw_flags = RWF_UNCACHED; + sqe->rw_flags |= RWF_UNCACHED; if (o->nowait) sqe->rw_flags |= RWF_NOWAIT; - if (ld->ioprio_class_set) - sqe->ioprio = td->o.ioprio_class << 13; - if (ld->ioprio_set) - sqe->ioprio |= td->o.ioprio; + + /* + * Since io_uring can have a submission context (sqthread_poll) + * that is different from the process context, we cannot rely on + * the IO priority set by ioprio_set() (options prio, prioclass, + * and priohint) to be inherited. + * td->ioprio will have the value of the "default prio", so set + * this unconditionally. This value might get overridden by + * fio_ioring_cmdprio_prep() if the option cmdprio_percentage or + * cmdprio_bssplit is used. + */ + sqe->ioprio = td->ioprio; sqe->off = io_u->offset; } else if (ddir_sync(io_u->ddir)) { + sqe->ioprio = 0; if (io_u->ddir == DDIR_SYNC_FILE_RANGE) { sqe->off = f->first_write; sqe->len = f->last_write - f->first_write; sqe->sync_range_flags = td->o.sync_file_range; sqe->opcode = IORING_OP_SYNC_FILE_RANGE; } else { + sqe->off = 0; + sqe->addr = 0; + sqe->len = 0; if (io_u->ddir == DDIR_DATASYNC) sqe->fsync_flags |= IORING_FSYNC_DATASYNC; sqe->opcode = IORING_OP_FSYNC; } } + if (o->force_async && ++ld->prepped == o->force_async) { + ld->prepped = 0; + sqe->flags |= IOSQE_ASYNC; + } + sqe->user_data = (unsigned long) io_u; return 0; } +static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + struct fio_file *f = io_u->file; + struct nvme_uring_cmd *cmd; + struct io_uring_sqe *sqe; + + /* only supports nvme_uring_cmd */ + if (o->cmd_type != FIO_URING_CMD_NVME) + return -EINVAL; + + if (io_u->ddir == DDIR_TRIM && td->io_ops->flags & FIO_ASYNCIO_SYNC_TRIM) + return 0; + + sqe = &ld->sqes[(io_u->index) << 1]; + + if (o->registerfiles) { + sqe->fd = f->engine_pos; + sqe->flags = IOSQE_FIXED_FILE; + } else { + sqe->fd = f->fd; + } + sqe->rw_flags = 0; + if (!td->o.odirect && o->uncached) + sqe->rw_flags |= RWF_UNCACHED; + if (o->nowait) + sqe->rw_flags |= RWF_NOWAIT; + + sqe->opcode = IORING_OP_URING_CMD; + sqe->user_data = (unsigned long) io_u; + if (o->nonvectored) + sqe->cmd_op = NVME_URING_CMD_IO; + else + sqe->cmd_op = NVME_URING_CMD_IO_VEC; + if (o->force_async && ++ld->prepped == o->force_async) { + ld->prepped = 0; + sqe->flags |= IOSQE_ASYNC; + } + if (o->fixedbufs) { + sqe->uring_cmd_flags = IORING_URING_CMD_FIXED; + sqe->buf_index = io_u->index; + } + + cmd = (struct nvme_uring_cmd *)sqe->cmd; + return fio_nvme_uring_cmd_prep(cmd, io_u, + o->nonvectored ? NULL : &ld->iovecs[io_u->index], + &ld->dsm[io_u->index]); +} + static struct io_u *fio_ioring_event(struct thread_data *td, int event) { struct ioring_data *ld = td->io_ops_data; @@ -293,6 +393,29 @@ static struct io_u *fio_ioring_event(struct thread_data *td, int event) return io_u; } +static struct io_u *fio_ioring_cmd_event(struct thread_data *td, int event) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + struct io_uring_cqe *cqe; + struct io_u *io_u; + unsigned index; + + index = (event + ld->cq_ring_off) & ld->cq_ring_mask; + if (o->cmd_type == FIO_URING_CMD_NVME) + index <<= 1; + + cqe = &ld->cq_ring.cqes[index]; + io_u = (struct io_u *) (uintptr_t) cqe->user_data; + + if (cqe->res != 0) + io_u->error = -cqe->res; + else + io_u->error = 0; + + return io_u; +} + static int fio_ioring_cqring_reap(struct thread_data *td, unsigned int events, unsigned int max) { @@ -329,6 +452,7 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min, r = fio_ioring_cqring_reap(td, events, max); if (r) { events += r; + max -= r; if (actual_min != 0) actual_min -= r; continue; @@ -340,6 +464,7 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min, if (r < 0) { if (errno == EAGAIN || errno == EINTR) continue; + r = -errno; td_verror(td, errno, "io_uring_enter"); break; } @@ -349,15 +474,14 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min, return r < 0 ? r : events; } -static void fio_ioring_prio_prep(struct thread_data *td, struct io_u *io_u) +static inline void fio_ioring_cmdprio_prep(struct thread_data *td, + struct io_u *io_u) { - struct ioring_options *o = td->eo; struct ioring_data *ld = td->io_ops_data; - if (rand_between(&td->prio_state, 0, 99) < o->cmdprio_percentage) { - ld->sqes[io_u->index].ioprio = IOPRIO_CLASS_RT << IOPRIO_CLASS_SHIFT; - io_u->flags |= IO_U_F_PRIORITY; - } - return; + struct cmdprio *cmdprio = &ld->cmdprio; + + if (fio_cmdprio_set_ioprio(td, cmdprio, io_u)) + ld->sqes[io_u->index].ioprio = io_u->ioprio; } static enum fio_q_status fio_ioring_queue(struct thread_data *td, @@ -365,7 +489,6 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td, { struct ioring_data *ld = td->io_ops_data; struct io_sq_ring *ring = &ld->sq_ring; - struct ioring_options *o = td->eo; unsigned tail, next_tail; fio_ro_check(td, io_u); @@ -373,11 +496,12 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td, if (ld->queued == ld->iodepth) return FIO_Q_BUSY; - if (io_u->ddir == DDIR_TRIM) { + if (io_u->ddir == DDIR_TRIM && td->io_ops->flags & FIO_ASYNCIO_SYNC_TRIM) { if (ld->queued) return FIO_Q_BUSY; do_io_u_trim(td, io_u); + io_u_mark_submit(td, 1); io_u_mark_complete(td, 1); return FIO_Q_COMPLETED; @@ -388,8 +512,9 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td, if (next_tail == atomic_load_acquire(ring->head)) return FIO_Q_BUSY; - if (o->cmdprio_percentage) - fio_ioring_prio_prep(td, io_u); + if (ld->cmdprio.mode != CMDPRIO_MODE_NONE) + fio_ioring_cmdprio_prep(td, io_u); + ring->array[tail & ld->sq_ring_mask] = io_u->index; atomic_store_release(ring->tail, next_tail); @@ -417,6 +542,12 @@ static void fio_ioring_queued(struct thread_data *td, int start, int nr) start++; } + + /* + * only used for iolog + */ + if (td->o.read_iolog_file) + memcpy(&td->last_issue, &now, sizeof(now)); } static int fio_ioring_commit(struct thread_data *td) @@ -435,11 +566,16 @@ static int fio_ioring_commit(struct thread_data *td) */ if (o->sqpoll_thread) { struct io_sq_ring *ring = &ld->sq_ring; + unsigned start = *ld->sq_ring.tail - ld->queued; + unsigned flags; - read_barrier(); - if (*ring->flags & IORING_SQ_NEED_WAKEUP) + flags = atomic_load_acquire(ring->flags); + if (flags & IORING_SQ_NEED_WAKEUP) io_uring_enter(ld, ld->queued, 0, IORING_ENTER_SQ_WAKEUP); + fio_ioring_queued(td, start, ld->queued); + io_u_mark_submit(td, ld->queued); + ld->queued = 0; return 0; } @@ -467,6 +603,7 @@ static int fio_ioring_commit(struct thread_data *td) usleep(1); continue; } + ret = -errno; td_verror(td, errno, "io_uring_enter submit"); break; } @@ -479,7 +616,7 @@ static void fio_ioring_unmap(struct ioring_data *ld) { int i; - for (i = 0; i < ARRAY_SIZE(ld->mmap); i++) + for (i = 0; i < FIO_ARRAY_SIZE(ld->mmap); i++) munmap(ld->mmap[i].ptr, ld->mmap[i].len); close(ld->ring_fd); } @@ -492,9 +629,11 @@ static void fio_ioring_cleanup(struct thread_data *td) if (!(td->flags & TD_F_CHILD)) fio_ioring_unmap(ld); + fio_cmdprio_cleanup(&ld->cmdprio); free(ld->io_u_index); free(ld->iovecs); free(ld->fds); + free(ld->dsm); free(ld); } } @@ -518,14 +657,22 @@ static int fio_ioring_mmap(struct ioring_data *ld, struct io_uring_params *p) sring->array = ptr + p->sq_off.array; ld->sq_ring_mask = *sring->ring_mask; - ld->mmap[1].len = p->sq_entries * sizeof(struct io_uring_sqe); + if (p->flags & IORING_SETUP_SQE128) + ld->mmap[1].len = 2 * p->sq_entries * sizeof(struct io_uring_sqe); + else + ld->mmap[1].len = p->sq_entries * sizeof(struct io_uring_sqe); ld->sqes = mmap(0, ld->mmap[1].len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, ld->ring_fd, IORING_OFF_SQES); ld->mmap[1].ptr = ld->sqes; - ld->mmap[2].len = p->cq_off.cqes + - p->cq_entries * sizeof(struct io_uring_cqe); + if (p->flags & IORING_SETUP_CQE32) { + ld->mmap[2].len = p->cq_off.cqes + + 2 * p->cq_entries * sizeof(struct io_uring_cqe); + } else { + ld->mmap[2].len = p->cq_off.cqes + + p->cq_entries * sizeof(struct io_uring_cqe); + } ptr = mmap(0, ld->mmap[2].len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, ld->ring_fd, IORING_OFF_CQ_RING); @@ -539,6 +686,39 @@ static int fio_ioring_mmap(struct ioring_data *ld, struct io_uring_params *p) return 0; } +static void fio_ioring_probe(struct thread_data *td) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + struct io_uring_probe *p; + int ret; + + /* already set by user, don't touch */ + if (o->nonvectored != -1) + return; + + /* default to off, as that's always safe */ + o->nonvectored = 0; + + p = calloc(1, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); + if (!p) + return; + + ret = syscall(__NR_io_uring_register, ld->ring_fd, + IORING_REGISTER_PROBE, p, 256); + if (ret < 0) + goto out; + + if (IORING_OP_WRITE > p->ops_len) + goto out; + + if ((p->ops[IORING_OP_READ].flags & IO_URING_OP_SUPPORTED) && + (p->ops[IORING_OP_WRITE].flags & IO_URING_OP_SUPPORTED)) + o->nonvectored = 1; +out: + free(p); +} + static int fio_ioring_queue_init(struct thread_data *td) { struct ioring_data *ld = td->io_ops_data; @@ -557,23 +737,143 @@ static int fio_ioring_queue_init(struct thread_data *td) p.flags |= IORING_SETUP_SQ_AFF; p.sq_thread_cpu = o->sqpoll_cpu; } + + /* + * Submission latency for sqpoll_thread is just the time it + * takes to fill in the SQ ring entries, and any syscall if + * IORING_SQ_NEED_WAKEUP is set, we don't need to log that time + * separately. + */ + td->o.disable_slat = 1; } + /* + * Clamp CQ ring size at our SQ ring size, we don't need more entries + * than that. + */ + p.flags |= IORING_SETUP_CQSIZE; + p.cq_entries = depth; + + /* + * Setup COOP_TASKRUN as we don't need to get IPI interrupted for + * completing IO operations. + */ + p.flags |= IORING_SETUP_COOP_TASKRUN; + + /* + * io_uring is always a single issuer, and we can defer task_work + * runs until we reap events. + */ + p.flags |= IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN; + +retry: ret = syscall(__NR_io_uring_setup, depth, &p); - if (ret < 0) + if (ret < 0) { + if (errno == EINVAL && p.flags & IORING_SETUP_DEFER_TASKRUN) { + p.flags &= ~IORING_SETUP_DEFER_TASKRUN; + p.flags &= ~IORING_SETUP_SINGLE_ISSUER; + goto retry; + } + if (errno == EINVAL && p.flags & IORING_SETUP_COOP_TASKRUN) { + p.flags &= ~IORING_SETUP_COOP_TASKRUN; + goto retry; + } + if (errno == EINVAL && p.flags & IORING_SETUP_CQSIZE) { + p.flags &= ~IORING_SETUP_CQSIZE; + goto retry; + } return ret; + } ld->ring_fd = ret; + fio_ioring_probe(td); + if (o->fixedbufs) { - struct rlimit rlim = { - .rlim_cur = RLIM_INFINITY, - .rlim_max = RLIM_INFINITY, - }; + ret = syscall(__NR_io_uring_register, ld->ring_fd, + IORING_REGISTER_BUFFERS, ld->iovecs, depth); + if (ret < 0) + return ret; + } + + return fio_ioring_mmap(ld, &p); +} + +static int fio_ioring_cmd_queue_init(struct thread_data *td) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + int depth = td->o.iodepth; + struct io_uring_params p; + int ret; + + memset(&p, 0, sizeof(p)); + + if (o->hipri) + p.flags |= IORING_SETUP_IOPOLL; + if (o->sqpoll_thread) { + p.flags |= IORING_SETUP_SQPOLL; + if (o->sqpoll_set) { + p.flags |= IORING_SETUP_SQ_AFF; + p.sq_thread_cpu = o->sqpoll_cpu; + } + + /* + * Submission latency for sqpoll_thread is just the time it + * takes to fill in the SQ ring entries, and any syscall if + * IORING_SQ_NEED_WAKEUP is set, we don't need to log that time + * separately. + */ + td->o.disable_slat = 1; + } + if (o->cmd_type == FIO_URING_CMD_NVME) { + p.flags |= IORING_SETUP_SQE128; + p.flags |= IORING_SETUP_CQE32; + } + + /* + * Clamp CQ ring size at our SQ ring size, we don't need more entries + * than that. + */ + p.flags |= IORING_SETUP_CQSIZE; + p.cq_entries = depth; + + /* + * Setup COOP_TASKRUN as we don't need to get IPI interrupted for + * completing IO operations. + */ + p.flags |= IORING_SETUP_COOP_TASKRUN; + + /* + * io_uring is always a single issuer, and we can defer task_work + * runs until we reap events. + */ + p.flags |= IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN; + +retry: + ret = syscall(__NR_io_uring_setup, depth, &p); + if (ret < 0) { + if (errno == EINVAL && p.flags & IORING_SETUP_DEFER_TASKRUN) { + p.flags &= ~IORING_SETUP_DEFER_TASKRUN; + p.flags &= ~IORING_SETUP_SINGLE_ISSUER; + goto retry; + } + if (errno == EINVAL && p.flags & IORING_SETUP_COOP_TASKRUN) { + p.flags &= ~IORING_SETUP_COOP_TASKRUN; + goto retry; + } + if (errno == EINVAL && p.flags & IORING_SETUP_CQSIZE) { + p.flags &= ~IORING_SETUP_CQSIZE; + goto retry; + } + return ret; + } + + ld->ring_fd = ret; - if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) - return -1; + fio_ioring_probe(td); + if (o->fixedbufs) { ret = syscall(__NR_io_uring_register, ld->ring_fd, IORING_REGISTER_BUFFERS, ld->iovecs, depth); if (ret < 0) @@ -640,10 +940,67 @@ static int fio_ioring_post_init(struct thread_data *td) err = fio_ioring_queue_init(td); if (err) { - td_verror(td, errno, "io_queue_init"); + int init_err = errno; + + if (init_err == ENOSYS) + log_err("fio: your kernel doesn't support io_uring\n"); + td_verror(td, init_err, "io_queue_init"); return 1; } + for (i = 0; i < td->o.iodepth; i++) { + struct io_uring_sqe *sqe; + + sqe = &ld->sqes[i]; + memset(sqe, 0, sizeof(*sqe)); + } + + if (o->registerfiles) { + err = fio_ioring_register_files(td); + if (err) { + td_verror(td, errno, "ioring_register_files"); + return 1; + } + } + + return 0; +} + +static int fio_ioring_cmd_post_init(struct thread_data *td) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + struct io_u *io_u; + int err, i; + + for (i = 0; i < td->o.iodepth; i++) { + struct iovec *iov = &ld->iovecs[i]; + + io_u = ld->io_u_index[i]; + iov->iov_base = io_u->buf; + iov->iov_len = td_max_bs(td); + } + + err = fio_ioring_cmd_queue_init(td); + if (err) { + int init_err = errno; + + td_verror(td, init_err, "io_queue_init"); + return 1; + } + + for (i = 0; i < td->o.iodepth; i++) { + struct io_uring_sqe *sqe; + + if (o->cmd_type == FIO_URING_CMD_NVME) { + sqe = &ld->sqes[i << 1]; + memset(sqe, 0, 2 * sizeof(*sqe)); + } else { + sqe = &ld->sqes[i]; + memset(sqe, 0, sizeof(*sqe)); + } + } + if (o->registerfiles) { err = fio_ioring_register_files(td); if (err) { @@ -659,7 +1016,7 @@ static int fio_ioring_init(struct thread_data *td) { struct ioring_options *o = td->eo; struct ioring_data *ld; - struct thread_options *to = &td->o; + int ret; /* sqthread submission requires registered files */ if (o->sqpoll_thread) @@ -683,21 +1040,21 @@ static int fio_ioring_init(struct thread_data *td) td->io_ops_data = ld; - /* - * Check for option conflicts - */ - if ((fio_option_is_set(to, ioprio) || fio_option_is_set(to, ioprio_class)) && - o->cmdprio_percentage != 0) { - log_err("%s: cmdprio_percentage option and mutually exclusive " - "prio or prioclass option is set, exiting\n", to->name); - td_verror(td, EINVAL, "fio_io_uring_init"); + ret = fio_cmdprio_init(td, &ld->cmdprio, &o->cmdprio_options); + if (ret) { + td_verror(td, EINVAL, "fio_ioring_init"); return 1; } - if (fio_option_is_set(&td->o, ioprio_class)) - ld->ioprio_class_set = true; - if (fio_option_is_set(&td->o, ioprio)) - ld->ioprio_set = true; + /* + * For io_uring_cmd, trims are async operations unless we are operating + * in zbd mode where trim means zone reset. + */ + if (!strcmp(td->io_ops->name, "io_uring_cmd") && td_trim(td) && + td->o.zone_mode == ZONE_MODE_ZBD) + td->io_ops->flags |= FIO_ASYNCIO_SYNC_TRIM; + else + ld->dsm = calloc(ld->iodepth, sizeof(*ld->dsm)); return 0; } @@ -722,6 +1079,57 @@ static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f) return 0; } +static int fio_ioring_cmd_open_file(struct thread_data *td, struct fio_file *f) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + + if (o->cmd_type == FIO_URING_CMD_NVME) { + struct nvme_data *data = NULL; + unsigned int nsid, lba_size = 0; + __u32 ms = 0; + __u64 nlba = 0; + int ret; + + /* Store the namespace-id and lba size. */ + data = FILE_ENG_DATA(f); + if (data == NULL) { + ret = fio_nvme_get_info(f, &nsid, &lba_size, &ms, &nlba); + if (ret) + return ret; + + data = calloc(1, sizeof(struct nvme_data)); + data->nsid = nsid; + if (ms) + data->lba_ext = lba_size + ms; + else + data->lba_shift = ilog2(lba_size); + + FILE_SET_ENG_DATA(f, data); + } + + assert(data->lba_shift < 32); + lba_size = data->lba_ext ? data->lba_ext : (1U << data->lba_shift); + + for_each_rw_ddir(ddir) { + if (td->o.min_bs[ddir] % lba_size || + td->o.max_bs[ddir] % lba_size) { + if (data->lba_ext) + log_err("block size must be a multiple of " + "(LBA data size + Metadata size)\n"); + else + log_err("block size must be a multiple of LBA data size\n"); + return 1; + } + } + } + if (!ld || !o->registerfiles) + return generic_open_file(td, f); + + f->fd = ld->fds[f->engine_pos]; + return 0; +} + static int fio_ioring_close_file(struct thread_data *td, struct fio_file *f) { struct ioring_data *ld = td->io_ops_data; @@ -734,10 +1142,116 @@ static int fio_ioring_close_file(struct thread_data *td, struct fio_file *f) return 0; } -static struct ioengine_ops ioengine = { +static int fio_ioring_cmd_close_file(struct thread_data *td, + struct fio_file *f) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + + if (o->cmd_type == FIO_URING_CMD_NVME) { + struct nvme_data *data = FILE_ENG_DATA(f); + + FILE_SET_ENG_DATA(f, NULL); + free(data); + } + if (!ld || !o->registerfiles) + return generic_close_file(td, f); + + f->fd = -1; + return 0; +} + +static int fio_ioring_cmd_get_file_size(struct thread_data *td, + struct fio_file *f) +{ + struct ioring_options *o = td->eo; + + if (fio_file_size_known(f)) + return 0; + + if (o->cmd_type == FIO_URING_CMD_NVME) { + struct nvme_data *data = NULL; + unsigned int nsid, lba_size = 0; + __u32 ms = 0; + __u64 nlba = 0; + int ret; + + ret = fio_nvme_get_info(f, &nsid, &lba_size, &ms, &nlba); + if (ret) + return ret; + + data = calloc(1, sizeof(struct nvme_data)); + data->nsid = nsid; + if (ms) + data->lba_ext = lba_size + ms; + else + data->lba_shift = ilog2(lba_size); + + f->real_file_size = lba_size * nlba; + fio_file_set_size_known(f); + + FILE_SET_ENG_DATA(f, data); + return 0; + } + return generic_get_file_size(td, f); +} + +static int fio_ioring_cmd_get_zoned_model(struct thread_data *td, + struct fio_file *f, + enum zbd_zoned_model *model) +{ + return fio_nvme_get_zoned_model(td, f, model); +} + +static int fio_ioring_cmd_report_zones(struct thread_data *td, + struct fio_file *f, uint64_t offset, + struct zbd_zone *zbdz, + unsigned int nr_zones) +{ + return fio_nvme_report_zones(td, f, offset, zbdz, nr_zones); +} + +static int fio_ioring_cmd_reset_wp(struct thread_data *td, struct fio_file *f, + uint64_t offset, uint64_t length) +{ + return fio_nvme_reset_wp(td, f, offset, length); +} + +static int fio_ioring_cmd_get_max_open_zones(struct thread_data *td, + struct fio_file *f, + unsigned int *max_open_zones) +{ + return fio_nvme_get_max_open_zones(td, f, max_open_zones); +} + +static int fio_ioring_cmd_fetch_ruhs(struct thread_data *td, struct fio_file *f, + struct fio_ruhs_info *fruhs_info) +{ + struct nvme_fdp_ruh_status *ruhs; + int bytes, ret, i; + + bytes = sizeof(*ruhs) + FDP_MAX_RUHS * sizeof(struct nvme_fdp_ruh_status_desc); + ruhs = scalloc(1, bytes); + if (!ruhs) + return -ENOMEM; + + ret = fio_nvme_iomgmt_ruhs(td, f, ruhs, bytes); + if (ret) + goto free; + + fruhs_info->nr_ruhs = le16_to_cpu(ruhs->nruhsd); + for (i = 0; i < fruhs_info->nr_ruhs; i++) + fruhs_info->plis[i] = le16_to_cpu(ruhs->ruhss[i].pid); +free: + sfree(ruhs); + return ret; +} + +static struct ioengine_ops ioengine_uring = { .name = "io_uring", .version = FIO_IOOPS_VERSION, - .flags = FIO_ASYNCIO_SYNC_TRIM, + .flags = FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD | + FIO_ASYNCIO_SETS_ISSUE_TIME, .init = fio_ioring_init, .post_init = fio_ioring_post_init, .io_u_init = fio_ioring_io_u_init, @@ -754,13 +1268,41 @@ static struct ioengine_ops ioengine = { .option_struct_size = sizeof(struct ioring_options), }; +static struct ioengine_ops ioengine_uring_cmd = { + .name = "io_uring_cmd", + .version = FIO_IOOPS_VERSION, + .flags = FIO_NO_OFFLOAD | FIO_MEMALIGN | FIO_RAWIO | + FIO_ASYNCIO_SETS_ISSUE_TIME, + .init = fio_ioring_init, + .post_init = fio_ioring_cmd_post_init, + .io_u_init = fio_ioring_io_u_init, + .prep = fio_ioring_cmd_prep, + .queue = fio_ioring_queue, + .commit = fio_ioring_commit, + .getevents = fio_ioring_getevents, + .event = fio_ioring_cmd_event, + .cleanup = fio_ioring_cleanup, + .open_file = fio_ioring_cmd_open_file, + .close_file = fio_ioring_cmd_close_file, + .get_file_size = fio_ioring_cmd_get_file_size, + .get_zoned_model = fio_ioring_cmd_get_zoned_model, + .report_zones = fio_ioring_cmd_report_zones, + .reset_wp = fio_ioring_cmd_reset_wp, + .get_max_open_zones = fio_ioring_cmd_get_max_open_zones, + .options = options, + .option_struct_size = sizeof(struct ioring_options), + .fdp_fetch_ruhs = fio_ioring_cmd_fetch_ruhs, +}; + static void fio_init fio_ioring_register(void) { - register_ioengine(&ioengine); + register_ioengine(&ioengine_uring); + register_ioengine(&ioengine_uring_cmd); } static void fio_exit fio_ioring_unregister(void) { - unregister_ioengine(&ioengine); + unregister_ioengine(&ioengine_uring); + unregister_ioengine(&ioengine_uring_cmd); } #endif