X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=engines%2Fio_uring.c;h=e1abf6888fd50c886b87ecc29cf25519c8bbba9d;hb=79012fece0a0ecb02ad6a3322913980efdb1d726;hp=f10a45933ff5bb1877b072538547ee2e637a69d5;hpb=d86ac3e9f4c703b7d7c9add96e69f2d02affdc65;p=fio.git diff --git a/engines/io_uring.c b/engines/io_uring.c index f10a4593..e1abf688 100644 --- a/engines/io_uring.c +++ b/engines/io_uring.c @@ -78,6 +78,8 @@ struct ioring_data { struct ioring_mmap mmap[3]; struct cmdprio cmdprio; + + struct nvme_dsm_range *dsm; }; struct ioring_options { @@ -125,87 +127,6 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_IOURING, }, -#ifdef FIO_HAVE_IOPRIO_CLASS - { - .name = "cmdprio_percentage", - .lname = "high priority percentage", - .type = FIO_OPT_INT, - .off1 = offsetof(struct ioring_options, - cmdprio_options.percentage[DDIR_READ]), - .off2 = offsetof(struct ioring_options, - cmdprio_options.percentage[DDIR_WRITE]), - .minval = 0, - .maxval = 100, - .help = "Send high priority I/O this percentage of the time", - .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_IOURING, - }, - { - .name = "cmdprio_class", - .lname = "Asynchronous I/O priority class", - .type = FIO_OPT_INT, - .off1 = offsetof(struct ioring_options, - cmdprio_options.class[DDIR_READ]), - .off2 = offsetof(struct ioring_options, - cmdprio_options.class[DDIR_WRITE]), - .help = "Set asynchronous IO priority class", - .minval = IOPRIO_MIN_PRIO_CLASS + 1, - .maxval = IOPRIO_MAX_PRIO_CLASS, - .interval = 1, - .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_IOURING, - }, - { - .name = "cmdprio", - .lname = "Asynchronous I/O priority level", - .type = FIO_OPT_INT, - .off1 = offsetof(struct ioring_options, - cmdprio_options.level[DDIR_READ]), - .off2 = offsetof(struct ioring_options, - cmdprio_options.level[DDIR_WRITE]), - .help = "Set asynchronous IO priority level", - .minval = IOPRIO_MIN_PRIO, - .maxval = IOPRIO_MAX_PRIO, - .interval = 1, - .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_IOURING, - }, - { - .name = "cmdprio_bssplit", - .lname = "Priority percentage block size split", - .type = FIO_OPT_STR_STORE, - .off1 = offsetof(struct ioring_options, - cmdprio_options.bssplit_str), - .help = "Set priority percentages for different block sizes", - .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_IOURING, - }, -#else - { - .name = "cmdprio_percentage", - .lname = "high priority percentage", - .type = FIO_OPT_UNSUPPORTED, - .help = "Your platform does not support I/O priority classes", - }, - { - .name = "cmdprio_class", - .lname = "Asynchronous I/O priority class", - .type = FIO_OPT_UNSUPPORTED, - .help = "Your platform does not support I/O priority classes", - }, - { - .name = "cmdprio", - .lname = "Asynchronous I/O priority level", - .type = FIO_OPT_UNSUPPORTED, - .help = "Your platform does not support I/O priority classes", - }, - { - .name = "cmdprio_bssplit", - .lname = "Priority percentage block size split", - .type = FIO_OPT_UNSUPPORTED, - .help = "Your platform does not support I/O priority classes", - }, -#endif { .name = "fixedbufs", .lname = "Fixed (pre-mapped) IO buffers", @@ -295,6 +216,7 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_IOURING, }, + CMDPRIO_OPTIONS(struct ioring_options, FIO_OPT_G_IOURING), { .name = NULL, }, @@ -363,8 +285,8 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) /* * Since io_uring can have a submission context (sqthread_poll) * that is different from the process context, we cannot rely on - * the IO priority set by ioprio_set() (option prio/prioclass) - * to be inherited. + * the IO priority set by ioprio_set() (options prio, prioclass, + * and priohint) to be inherited. * td->ioprio will have the value of the "default prio", so set * this unconditionally. This value might get overridden by * fio_ioring_cmdprio_prep() if the option cmdprio_percentage or @@ -410,7 +332,7 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u) if (o->cmd_type != FIO_URING_CMD_NVME) return -EINVAL; - if (io_u->ddir == DDIR_TRIM) + if (io_u->ddir == DDIR_TRIM && td->io_ops->flags & FIO_ASYNCIO_SYNC_TRIM) return 0; sqe = &ld->sqes[(io_u->index) << 1]; @@ -444,7 +366,8 @@ static int fio_ioring_cmd_prep(struct thread_data *td, struct io_u *io_u) cmd = (struct nvme_uring_cmd *)sqe->cmd; return fio_nvme_uring_cmd_prep(cmd, io_u, - o->nonvectored ? NULL : &ld->iovecs[io_u->index]); + o->nonvectored ? NULL : &ld->iovecs[io_u->index], + &ld->dsm[io_u->index]); } static struct io_u *fio_ioring_event(struct thread_data *td, int event) @@ -529,6 +452,7 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min, r = fio_ioring_cqring_reap(td, events, max); if (r) { events += r; + max -= r; if (actual_min != 0) actual_min -= r; continue; @@ -560,27 +484,6 @@ static inline void fio_ioring_cmdprio_prep(struct thread_data *td, ld->sqes[io_u->index].ioprio = io_u->ioprio; } -static int fio_ioring_cmd_io_u_trim(const struct thread_data *td, - struct io_u *io_u) -{ - struct fio_file *f = io_u->file; - int ret; - - if (td->o.zone_mode == ZONE_MODE_ZBD) { - ret = zbd_do_io_u_trim(td, io_u); - if (ret == io_u_completed) - return io_u->xfer_buflen; - if (ret) - goto err; - } - - return fio_nvme_trim(td, f, io_u->offset, io_u->xfer_buflen); - -err: - io_u->error = ret; - return 0; -} - static enum fio_q_status fio_ioring_queue(struct thread_data *td, struct io_u *io_u) { @@ -593,14 +496,11 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td, if (ld->queued == ld->iodepth) return FIO_Q_BUSY; - if (io_u->ddir == DDIR_TRIM) { + if (io_u->ddir == DDIR_TRIM && td->io_ops->flags & FIO_ASYNCIO_SYNC_TRIM) { if (ld->queued) return FIO_Q_BUSY; - if (!strcmp(td->io_ops->name, "io_uring_cmd")) - fio_ioring_cmd_io_u_trim(td, io_u); - else - do_io_u_trim(td, io_u); + do_io_u_trim(td, io_u); io_u_mark_submit(td, 1); io_u_mark_complete(td, 1); @@ -666,7 +566,7 @@ static int fio_ioring_commit(struct thread_data *td) */ if (o->sqpoll_thread) { struct io_sq_ring *ring = &ld->sq_ring; - unsigned start = *ld->sq_ring.head; + unsigned start = *ld->sq_ring.tail - ld->queued; unsigned flags; flags = atomic_load_acquire(ring->flags); @@ -733,6 +633,7 @@ static void fio_ioring_cleanup(struct thread_data *td) free(ld->io_u_index); free(ld->iovecs); free(ld->fds); + free(ld->dsm); free(ld); } } @@ -799,11 +700,10 @@ static void fio_ioring_probe(struct thread_data *td) /* default to off, as that's always safe */ o->nonvectored = 0; - p = malloc(sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); + p = calloc(1, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); if (!p) return; - memset(p, 0, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op)); ret = syscall(__NR_io_uring_register, ld->ring_fd, IORING_REGISTER_PROBE, p, 256); if (ret < 0) @@ -1146,6 +1046,16 @@ static int fio_ioring_init(struct thread_data *td) return 1; } + /* + * For io_uring_cmd, trims are async operations unless we are operating + * in zbd mode where trim means zone reset. + */ + if (!strcmp(td->io_ops->name, "io_uring_cmd") && td_trim(td) && + td->o.zone_mode == ZONE_MODE_ZBD) + td->io_ops->flags |= FIO_ASYNCIO_SYNC_TRIM; + else + ld->dsm = calloc(ld->iodepth, sizeof(*ld->dsm)); + return 0; } @@ -1177,22 +1087,41 @@ static int fio_ioring_cmd_open_file(struct thread_data *td, struct fio_file *f) if (o->cmd_type == FIO_URING_CMD_NVME) { struct nvme_data *data = NULL; unsigned int nsid, lba_size = 0; + __u32 ms = 0; __u64 nlba = 0; int ret; /* Store the namespace-id and lba size. */ data = FILE_ENG_DATA(f); if (data == NULL) { - ret = fio_nvme_get_info(f, &nsid, &lba_size, &nlba); + ret = fio_nvme_get_info(f, &nsid, &lba_size, &ms, &nlba); if (ret) return ret; data = calloc(1, sizeof(struct nvme_data)); data->nsid = nsid; - data->lba_shift = ilog2(lba_size); + if (ms) + data->lba_ext = lba_size + ms; + else + data->lba_shift = ilog2(lba_size); FILE_SET_ENG_DATA(f, data); } + + assert(data->lba_shift < 32); + lba_size = data->lba_ext ? data->lba_ext : (1U << data->lba_shift); + + for_each_rw_ddir(ddir) { + if (td->o.min_bs[ddir] % lba_size || + td->o.max_bs[ddir] % lba_size) { + if (data->lba_ext) + log_err("block size must be a multiple of " + "(LBA data size + Metadata size)\n"); + else + log_err("block size must be a multiple of LBA data size\n"); + return 1; + } + } } if (!ld || !o->registerfiles) return generic_open_file(td, f); @@ -1243,16 +1172,20 @@ static int fio_ioring_cmd_get_file_size(struct thread_data *td, if (o->cmd_type == FIO_URING_CMD_NVME) { struct nvme_data *data = NULL; unsigned int nsid, lba_size = 0; + __u32 ms = 0; __u64 nlba = 0; int ret; - ret = fio_nvme_get_info(f, &nsid, &lba_size, &nlba); + ret = fio_nvme_get_info(f, &nsid, &lba_size, &ms, &nlba); if (ret) return ret; data = calloc(1, sizeof(struct nvme_data)); data->nsid = nsid; - data->lba_shift = ilog2(lba_size); + if (ms) + data->lba_ext = lba_size + ms; + else + data->lba_shift = ilog2(lba_size); f->real_file_size = lba_size * nlba; fio_file_set_size_known(f); @@ -1297,7 +1230,7 @@ static int fio_ioring_cmd_fetch_ruhs(struct thread_data *td, struct fio_file *f, struct nvme_fdp_ruh_status *ruhs; int bytes, ret, i; - bytes = sizeof(*ruhs) + 128 * sizeof(struct nvme_fdp_ruh_status_desc); + bytes = sizeof(*ruhs) + FDP_MAX_RUHS * sizeof(struct nvme_fdp_ruh_status_desc); ruhs = scalloc(1, bytes); if (!ruhs) return -ENOMEM; @@ -1338,8 +1271,7 @@ static struct ioengine_ops ioengine_uring = { static struct ioengine_ops ioengine_uring_cmd = { .name = "io_uring_cmd", .version = FIO_IOOPS_VERSION, - .flags = FIO_ASYNCIO_SYNC_TRIM | FIO_NO_OFFLOAD | - FIO_MEMALIGN | FIO_RAWIO | + .flags = FIO_NO_OFFLOAD | FIO_MEMALIGN | FIO_RAWIO | FIO_ASYNCIO_SETS_ISSUE_TIME, .init = fio_ioring_init, .post_init = fio_ioring_cmd_post_init,