From e5f3b613876d6fd026ccf532f89fd21b841ab99d Mon Sep 17 00:00:00 2001 From: Ankit Kumar Date: Wed, 12 Jul 2023 15:50:41 +0530 Subject: [PATCH] engines/xnvme: add support for fdp Add FDP support to xnvme I/O engine. This support can be used only with nvme-ns generic character device (/dev/ngXnY). The available backends are --xnvme_async=io_uring_cmd and --xnvme_sync=nvme. Add a xnvme-fdp config example file. Update the minimum required xnvme version to 0.7.0 Signed-off-by: Ankit Kumar Signed-off-by: Vincent Fu --- HOWTO.rst | 6 ++-- configure | 2 +- engines/xnvme.c | 78 +++++++++++++++++++++++++++++++++++++++++- examples/xnvme-fdp.fio | 36 +++++++++++++++++++ fio.1 | 6 ++-- 5 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 examples/xnvme-fdp.fio diff --git a/HOWTO.rst b/HOWTO.rst index 24789f41..b047877e 100644 --- a/HOWTO.rst +++ b/HOWTO.rst @@ -2431,11 +2431,11 @@ with the caveat that when used on the command line, they must come after the For direct I/O, requests will only succeed if cache invalidation isn't required, file blocks are fully allocated and the disk request could be issued immediately. -.. option:: fdp=bool : [io_uring_cmd] +.. option:: fdp=bool : [io_uring_cmd] [xnvme] Enable Flexible Data Placement mode for write commands. -.. option:: fdp_pli_select=str : [io_uring_cmd] +.. option:: fdp_pli_select=str : [io_uring_cmd] [xnvme] Defines how fio decides which placement ID to use next. The following types are defined: @@ -2450,7 +2450,7 @@ with the caveat that when used on the command line, they must come after the The available placement ID index/indices is defined by the option :option:`fdp_pli`. -.. option:: fdp_pli=str : [io_uring_cmd] +.. option:: fdp_pli=str : [io_uring_cmd] [xnvme] Select which Placement ID Index/Indicies this job is allowed to use for writes. By default, the job will cycle through all available Placement diff --git a/configure b/configure index 74416fd4..6c938251 100755 --- a/configure +++ b/configure @@ -2651,7 +2651,7 @@ fi ########################################## # Check if we have xnvme if test "$xnvme" != "no" ; then - if check_min_lib_version xnvme 0.2.0; then + if check_min_lib_version xnvme 0.7.0; then xnvme="yes" xnvme_cflags=$(pkg-config --cflags xnvme) xnvme_libs=$(pkg-config --libs xnvme) diff --git a/engines/xnvme.c b/engines/xnvme.c index bb92a121..ce7b2bdd 100644 --- a/engines/xnvme.c +++ b/engines/xnvme.c @@ -16,6 +16,7 @@ #include #include "fio.h" #include "zbd_types.h" +#include "fdp.h" #include "optgroup.h" static pthread_mutex_t g_serialize = PTHREAD_MUTEX_INITIALIZER; @@ -509,6 +510,7 @@ static enum fio_q_status xnvme_fioe_queue(struct thread_data *td, struct io_u *i uint16_t nlb; int err; bool vectored_io = ((struct xnvme_fioe_options *)td->eo)->xnvme_iovec; + uint32_t dir = io_u->dtype; fio_ro_check(td, io_u); @@ -524,6 +526,10 @@ static enum fio_q_status xnvme_fioe_queue(struct thread_data *td, struct io_u *i ctx->cmd.common.nsid = nsid; ctx->cmd.nvm.slba = slba; ctx->cmd.nvm.nlb = nlb; + if (dir) { + ctx->cmd.nvm.dtype = io_u->dtype; + ctx->cmd.nvm.cdw13.dspec = io_u->dspec; + } switch (io_u->ddir) { case DDIR_READ: @@ -947,6 +953,72 @@ exit: return err; } +static int xnvme_fioe_fetch_ruhs(struct thread_data *td, struct fio_file *f, + struct fio_ruhs_info *fruhs_info) +{ + struct xnvme_opts opts = xnvme_opts_from_fioe(td); + struct xnvme_dev *dev; + struct xnvme_spec_ruhs *ruhs; + struct xnvme_cmd_ctx ctx; + uint32_t ruhs_nbytes; + uint32_t nsid; + int err = 0, err_lock; + + if (f->filetype != FIO_TYPE_CHAR) { + log_err("ioeng->fdp_ruhs(): ignoring filetype: %d\n", f->filetype); + return -EINVAL; + } + + err = pthread_mutex_lock(&g_serialize); + if (err) { + log_err("ioeng->fdp_ruhs(): pthread_mutex_lock(), err(%d)\n", err); + return -err; + } + + dev = xnvme_dev_open(f->file_name, &opts); + if (!dev) { + log_err("ioeng->fdp_ruhs(): xnvme_dev_open(%s) failed, errno: %d\n", + f->file_name, errno); + err = -errno; + goto exit; + } + + ruhs_nbytes = sizeof(*ruhs) + (FDP_MAX_RUHS * sizeof(struct xnvme_spec_ruhs_desc)); + ruhs = xnvme_buf_alloc(dev, ruhs_nbytes); + if (!ruhs) { + err = -errno; + goto exit; + } + memset(ruhs, 0, ruhs_nbytes); + + ctx = xnvme_cmd_ctx_from_dev(dev); + nsid = xnvme_dev_get_nsid(dev); + + err = xnvme_nvm_mgmt_recv(&ctx, nsid, XNVME_SPEC_IO_MGMT_RECV_RUHS, 0, ruhs, ruhs_nbytes); + + if (err || xnvme_cmd_ctx_cpl_status(&ctx)) { + err = err ? err : -EIO; + log_err("ioeng->fdp_ruhs(): err(%d), sc(%d)", err, ctx.cpl.status.sc); + goto free_buffer; + } + + fruhs_info->nr_ruhs = ruhs->nruhsd; + for (uint32_t idx = 0; idx < fruhs_info->nr_ruhs; ++idx) { + fruhs_info->plis[idx] = le16_to_cpu(ruhs->desc[idx].pi); + } + +free_buffer: + xnvme_buf_free(dev, ruhs); +exit: + xnvme_dev_close(dev); + + err_lock = pthread_mutex_unlock(&g_serialize); + if (err_lock) + log_err("ioeng->fdp_ruhs(): pthread_mutex_unlock(), err(%d)\n", err_lock); + + return err; +} + static int xnvme_fioe_get_file_size(struct thread_data *td, struct fio_file *f) { struct xnvme_opts opts = xnvme_opts_from_fioe(td); @@ -971,7 +1043,9 @@ static int xnvme_fioe_get_file_size(struct thread_data *td, struct fio_file *f) f->real_file_size = xnvme_dev_get_geo(dev)->tbytes; fio_file_set_size_known(f); - f->filetype = FIO_TYPE_BLOCK; + + if (td->o.zone_mode == ZONE_MODE_ZBD) + f->filetype = FIO_TYPE_BLOCK; exit: xnvme_dev_close(dev); @@ -1011,6 +1085,8 @@ FIO_STATIC struct ioengine_ops ioengine = { .get_zoned_model = xnvme_fioe_get_zoned_model, .report_zones = xnvme_fioe_report_zones, .reset_wp = xnvme_fioe_reset_wp, + + .fdp_fetch_ruhs = xnvme_fioe_fetch_ruhs, }; static void fio_init fio_xnvme_register(void) diff --git a/examples/xnvme-fdp.fio b/examples/xnvme-fdp.fio new file mode 100644 index 00000000..86fbe0d3 --- /dev/null +++ b/examples/xnvme-fdp.fio @@ -0,0 +1,36 @@ +; README +; +; This job-file is intended to be used either as: +; +; # Use the xNVMe io-engine engine io_uring_cmd async. impl. +; fio examples/xnvme-fdp.fio \ +; --section=default \ +; --ioengine=xnvme \ +; --xnvme_async=io_uring_cmd \ +; --filename=/dev/ng0n1 +; +; # Use the xNVMe io-engine engine with nvme sync. impl. +; fio examples/xnvme-fdp.fio \ +; --section=default \ +; --ioengine=xnvme \ +; --xnvme_sync=nvme \ +; --filename=/dev/ng0n1 +; +; FIO_BS="512" FIO_RW="read" FIO_IODEPTH=16 fio examples/xnvme-fdp.fio \ +; --section=override --ioengine=xnvme --xnvme_sync=nvme --filename=/dev/ng0n1 +; +[global] +rw=randwrite +size=2M +iodepth=1 +bs=4K +thread=1 +fdp=1 +fdp_pli=4,5 + +[default] + +[override] +rw=${FIO_RW} +iodepth=${FIO_IODEPTH} +bs=${FIO_BS} diff --git a/fio.1 b/fio.1 index 0257513b..86cb2af6 100644 --- a/fio.1 +++ b/fio.1 @@ -2192,10 +2192,10 @@ cached data. Currently the RWF_NOWAIT flag does not supported for cached write. For direct I/O, requests will only succeed if cache invalidation isn't required, file blocks are fully allocated and the disk request could be issued immediately. .TP -.BI (io_uring_cmd)fdp \fR=\fPbool +.BI (io_uring_cmd,xnvme)fdp \fR=\fPbool Enable Flexible Data Placement mode for write commands. .TP -.BI (io_uring_cmd)fdp_pli_select \fR=\fPstr +.BI (io_uring_cmd,xnvme)fdp_pli_select \fR=\fPstr Defines how fio decides which placement ID to use next. The following types are defined: .RS @@ -2211,7 +2211,7 @@ Round robin over available placement IDs. This is the default. The available placement ID index/indices is defined by \fBfdp_pli\fR option. .RE .TP -.BI (io_uring_cmd)fdp_pli \fR=\fPstr +.BI (io_uring_cmd,xnvme)fdp_pli \fR=\fPstr Select which Placement ID Index/Indicies this job is allowed to use for writes. By default, the job will cycle through all available Placement IDs, so use this to isolate these identifiers to specific jobs. If you want fio to use placement -- 2.25.1