4 * IO engine using the xNVMe C API.
6 * See: http://xnvme.io/
8 * SPDX-License-Identifier: Apache-2.0
13 #include <libxnvme_libconf.h>
14 #include <libxnvme_nvm.h>
15 #include <libxnvme_znd.h>
16 #include <libxnvme_spec_fs.h>
18 #include "zbd_types.h"
22 static pthread_mutex_t g_serialize = PTHREAD_MUTEX_INITIALIZER;
24 struct xnvme_fioe_fwrap {
25 /* fio file representation */
26 struct fio_file *fio_file;
28 /* xNVMe device handle */
29 struct xnvme_dev *dev;
30 /* xNVMe device geometry */
31 const struct xnvme_geo *geo;
33 struct xnvme_queue *queue;
40 XNVME_STATIC_ASSERT(sizeof(struct xnvme_fioe_fwrap) == 64, "Incorrect size")
42 struct xnvme_fioe_data {
43 /* I/O completion queue */
46 /* # of iocq entries; incremented via getevents()/cb_pool() */
50 * # of errors; incremented when observed on completion via
51 * getevents()/cb_pool()
55 /* Controller which device/file to select */
59 /* Number of devices/files for which open() has been called */
61 /* Number of devices/files allocated in files[] */
68 struct xnvme_fioe_fwrap files[];
70 XNVME_STATIC_ASSERT(sizeof(struct xnvme_fioe_data) == 64, "Incorrect size")
72 struct xnvme_fioe_options {
75 unsigned int sqpoll_thread;
76 unsigned int xnvme_dev_nsid;
77 unsigned int xnvme_iovec;
83 char *xnvme_dev_subnqn;
86 static struct fio_option options[] = {
89 .lname = "High Priority",
90 .type = FIO_OPT_STR_SET,
91 .off1 = offsetof(struct xnvme_fioe_options, hipri),
92 .help = "Use polled IO completions",
93 .category = FIO_OPT_C_ENGINE,
94 .group = FIO_OPT_G_XNVME,
97 .name = "sqthread_poll",
98 .lname = "Kernel SQ thread polling",
99 .type = FIO_OPT_STR_SET,
100 .off1 = offsetof(struct xnvme_fioe_options, sqpoll_thread),
101 .help = "Offload submission/completion to kernel thread",
102 .category = FIO_OPT_C_ENGINE,
103 .group = FIO_OPT_G_XNVME,
107 .lname = "xNVMe Backend",
108 .type = FIO_OPT_STR_STORE,
109 .off1 = offsetof(struct xnvme_fioe_options, xnvme_be),
110 .help = "Select xNVMe backend [spdk,linux,fbsd]",
111 .category = FIO_OPT_C_ENGINE,
112 .group = FIO_OPT_G_XNVME,
116 .lname = "xNVMe Memory Backend",
117 .type = FIO_OPT_STR_STORE,
118 .off1 = offsetof(struct xnvme_fioe_options, xnvme_mem),
119 .help = "Select xNVMe memory backend",
120 .category = FIO_OPT_C_ENGINE,
121 .group = FIO_OPT_G_XNVME,
124 .name = "xnvme_async",
125 .lname = "xNVMe Asynchronous command-interface",
126 .type = FIO_OPT_STR_STORE,
127 .off1 = offsetof(struct xnvme_fioe_options, xnvme_async),
128 .help = "Select xNVMe async. interface: "
129 "[emu,thrpool,io_uring,io_uring_cmd,libaio,posix,vfio,nil]",
130 .category = FIO_OPT_C_ENGINE,
131 .group = FIO_OPT_G_XNVME,
134 .name = "xnvme_sync",
135 .lname = "xNVMe Synchronous. command-interface",
136 .type = FIO_OPT_STR_STORE,
137 .off1 = offsetof(struct xnvme_fioe_options, xnvme_sync),
138 .help = "Select xNVMe sync. interface: [nvme,psync,block]",
139 .category = FIO_OPT_C_ENGINE,
140 .group = FIO_OPT_G_XNVME,
143 .name = "xnvme_admin",
144 .lname = "xNVMe Admin command-interface",
145 .type = FIO_OPT_STR_STORE,
146 .off1 = offsetof(struct xnvme_fioe_options, xnvme_admin),
147 .help = "Select xNVMe admin. cmd-interface: [nvme,block]",
148 .category = FIO_OPT_C_ENGINE,
149 .group = FIO_OPT_G_XNVME,
152 .name = "xnvme_dev_nsid",
153 .lname = "xNVMe Namespace-Identifier, for user-space NVMe driver",
155 .off1 = offsetof(struct xnvme_fioe_options, xnvme_dev_nsid),
156 .help = "xNVMe Namespace-Identifier, for user-space NVMe driver",
157 .category = FIO_OPT_C_ENGINE,
158 .group = FIO_OPT_G_XNVME,
161 .name = "xnvme_dev_subnqn",
162 .lname = "Subsystem nqn for Fabrics",
163 .type = FIO_OPT_STR_STORE,
164 .off1 = offsetof(struct xnvme_fioe_options, xnvme_dev_subnqn),
165 .help = "Subsystem NQN for Fabrics",
166 .category = FIO_OPT_C_ENGINE,
167 .group = FIO_OPT_G_XNVME,
170 .name = "xnvme_iovec",
171 .lname = "Vectored IOs",
172 .type = FIO_OPT_STR_SET,
173 .off1 = offsetof(struct xnvme_fioe_options, xnvme_iovec),
174 .help = "Send vectored IOs",
175 .category = FIO_OPT_C_ENGINE,
176 .group = FIO_OPT_G_XNVME,
184 static void cb_pool(struct xnvme_cmd_ctx *ctx, void *cb_arg)
186 struct io_u *io_u = cb_arg;
187 struct xnvme_fioe_data *xd = io_u->mmap_data;
189 if (xnvme_cmd_ctx_cpl_status(ctx)) {
190 xnvme_cmd_ctx_pr(ctx, XNVME_PR_DEF);
195 xd->iocq[xd->completed++] = io_u;
196 xnvme_queue_put_cmd_ctx(ctx->async.queue, ctx);
199 static struct xnvme_opts xnvme_opts_from_fioe(struct thread_data *td)
201 struct xnvme_fioe_options *o = td->eo;
202 struct xnvme_opts opts = xnvme_opts_default();
204 opts.nsid = o->xnvme_dev_nsid;
205 opts.subnqn = o->xnvme_dev_subnqn;
206 opts.be = o->xnvme_be;
207 opts.mem = o->xnvme_mem;
208 opts.async = o->xnvme_async;
209 opts.sync = o->xnvme_sync;
210 opts.admin = o->xnvme_admin;
212 opts.poll_io = o->hipri;
213 opts.poll_sq = o->sqpoll_thread;
215 opts.direct = td->o.odirect;
220 static void _dev_close(struct thread_data *td, struct xnvme_fioe_fwrap *fwrap)
223 xnvme_queue_term(fwrap->queue);
225 xnvme_dev_close(fwrap->dev);
227 memset(fwrap, 0, sizeof(*fwrap));
230 static void xnvme_fioe_cleanup(struct thread_data *td)
232 struct xnvme_fioe_data *xd = NULL;
235 if (!td->io_ops_data)
238 xd = td->io_ops_data;
240 err = pthread_mutex_lock(&g_serialize);
242 log_err("ioeng->cleanup(): pthread_mutex_lock(), err(%d)\n", err);
243 /* NOTE: not returning here */
245 for (uint64_t i = 0; i < xd->nallocated; ++i)
246 _dev_close(td, &xd->files[i]);
249 err = pthread_mutex_unlock(&g_serialize);
251 log_err("ioeng->cleanup(): pthread_mutex_unlock(), err(%d)\n", err);
257 td->io_ops_data = NULL;
261 * Helper function setting up device handles as addressed by the naming
262 * convention of the given `fio_file` filename.
264 * Checks thread-options for explicit control of asynchronous implementation via
265 * the ``--xnvme_async={thrpool,emu,posix,io_uring,libaio,nil}``.
267 static int _dev_open(struct thread_data *td, struct fio_file *f)
269 struct xnvme_opts opts = xnvme_opts_from_fioe(td);
270 struct xnvme_fioe_data *xd = td->io_ops_data;
271 struct xnvme_fioe_fwrap *fwrap;
275 if (f->fileno > (int)xd->nallocated) {
276 log_err("ioeng->_dev_open(%s): invalid assumption\n", f->file_name);
280 fwrap = &xd->files[f->fileno];
282 err = pthread_mutex_lock(&g_serialize);
284 log_err("ioeng->_dev_open(%s): pthread_mutex_lock(), err(%d)\n", f->file_name,
289 fwrap->dev = xnvme_dev_open(f->file_name, &opts);
291 log_err("ioeng->_dev_open(%s): xnvme_dev_open(), err(%d)\n", f->file_name, errno);
294 fwrap->geo = xnvme_dev_get_geo(fwrap->dev);
296 if (xnvme_queue_init(fwrap->dev, td->o.iodepth, flags, &(fwrap->queue))) {
297 log_err("ioeng->_dev_open(%s): xnvme_queue_init(), err(?)\n", f->file_name);
300 xnvme_queue_set_cb(fwrap->queue, cb_pool, NULL);
302 fwrap->ssw = xnvme_dev_get_ssw(fwrap->dev);
303 fwrap->lba_nbytes = fwrap->geo->lba_nbytes;
306 fwrap->fio_file->filetype = FIO_TYPE_BLOCK;
307 fwrap->fio_file->real_file_size = fwrap->geo->tbytes;
308 fio_file_set_size_known(fwrap->fio_file);
310 err = pthread_mutex_unlock(&g_serialize);
312 log_err("ioeng->_dev_open(%s): pthread_mutex_unlock(), err(%d)\n", f->file_name,
318 xnvme_queue_term(fwrap->queue);
319 xnvme_dev_close(fwrap->dev);
321 err = pthread_mutex_unlock(&g_serialize);
323 log_err("ioeng->_dev_open(%s): pthread_mutex_unlock(), err(%d)\n", f->file_name,
329 static int xnvme_fioe_init(struct thread_data *td)
331 struct xnvme_fioe_data *xd = NULL;
335 if (!td->o.use_thread) {
336 log_err("ioeng->init(): --thread=1 is required\n");
340 /* Allocate xd and iocq */
341 xd = calloc(1, sizeof(*xd) + sizeof(*xd->files) * td->o.nr_files);
343 log_err("ioeng->init(): !calloc(), err(%d)\n", errno);
347 xd->iocq = calloc(td->o.iodepth, sizeof(struct io_u *));
350 log_err("ioeng->init(): !calloc(xd->iocq), err(%d)\n", errno);
354 xd->iovec = calloc(td->o.iodepth, sizeof(*xd->iovec));
358 log_err("ioeng->init(): !calloc(xd->iovec), err(%d)\n", errno);
363 td->io_ops_data = xd;
365 for_each_file(td, f, i)
367 if (_dev_open(td, f)) {
369 * Note: We are not freeing xd, iocq and iovec. This
370 * will be done as part of cleanup routine.
372 log_err("ioeng->init(): failed; _dev_open(%s)\n", f->file_name);
379 if (xd->nallocated != td->o.nr_files) {
380 log_err("ioeng->init(): failed; nallocated != td->o.nr_files\n");
387 /* NOTE: using the first device for buffer-allocators) */
388 static int xnvme_fioe_iomem_alloc(struct thread_data *td, size_t total_mem)
390 struct xnvme_fioe_data *xd = td->io_ops_data;
391 struct xnvme_fioe_fwrap *fwrap = &xd->files[0];
394 log_err("ioeng->iomem_alloc(): failed; no dev-handle\n");
398 td->orig_buffer = xnvme_buf_alloc(fwrap->dev, total_mem);
400 return td->orig_buffer == NULL;
403 /* NOTE: using the first device for buffer-allocators) */
404 static void xnvme_fioe_iomem_free(struct thread_data *td)
406 struct xnvme_fioe_data *xd = NULL;
407 struct xnvme_fioe_fwrap *fwrap = NULL;
409 if (!td->io_ops_data)
412 xd = td->io_ops_data;
413 fwrap = &xd->files[0];
416 log_err("ioeng->iomem_free(): failed no dev-handle\n");
420 xnvme_buf_free(fwrap->dev, td->orig_buffer);
423 static int xnvme_fioe_io_u_init(struct thread_data *td, struct io_u *io_u)
425 io_u->mmap_data = td->io_ops_data;
430 static void xnvme_fioe_io_u_free(struct thread_data *td, struct io_u *io_u)
432 io_u->mmap_data = NULL;
435 static struct io_u *xnvme_fioe_event(struct thread_data *td, int event)
437 struct xnvme_fioe_data *xd = td->io_ops_data;
440 assert((unsigned)event < xd->completed);
442 return xd->iocq[event];
445 static int xnvme_fioe_getevents(struct thread_data *td, unsigned int min, unsigned int max,
446 const struct timespec *t)
448 struct xnvme_fioe_data *xd = td->io_ops_data;
449 struct xnvme_fioe_fwrap *fwrap = NULL;
450 int nfiles = xd->nallocated;
453 if (xd->prev != -1 && ++xd->prev < nfiles) {
454 fwrap = &xd->files[xd->prev];
460 if (fwrap == NULL || xd->cur == nfiles) {
461 fwrap = &xd->files[0];
465 while (fwrap != NULL && xd->cur < nfiles && err >= 0) {
466 err = xnvme_queue_poke(fwrap->queue, max - xd->completed);
475 log_err("ioeng->getevents(): unhandled IO error\n");
480 if (xd->completed >= min) {
482 return xd->completed;
485 fwrap = &xd->files[xd->cur];
500 return xd->completed;
503 static enum fio_q_status xnvme_fioe_queue(struct thread_data *td, struct io_u *io_u)
505 struct xnvme_fioe_data *xd = td->io_ops_data;
506 struct xnvme_fioe_fwrap *fwrap;
507 struct xnvme_cmd_ctx *ctx;
512 bool vectored_io = ((struct xnvme_fioe_options *)td->eo)->xnvme_iovec;
513 uint32_t dir = io_u->dtype;
515 fio_ro_check(td, io_u);
517 fwrap = &xd->files[io_u->file->fileno];
518 nsid = xnvme_dev_get_nsid(fwrap->dev);
520 slba = io_u->offset >> fwrap->ssw;
521 nlb = (io_u->xfer_buflen >> fwrap->ssw) - 1;
523 ctx = xnvme_queue_get_cmd_ctx(fwrap->queue);
524 ctx->async.cb_arg = io_u;
526 ctx->cmd.common.nsid = nsid;
527 ctx->cmd.nvm.slba = slba;
528 ctx->cmd.nvm.nlb = nlb;
530 ctx->cmd.nvm.dtype = io_u->dtype;
531 ctx->cmd.nvm.cdw13.dspec = io_u->dspec;
534 switch (io_u->ddir) {
536 ctx->cmd.common.opcode = XNVME_SPEC_NVM_OPC_READ;
540 ctx->cmd.common.opcode = XNVME_SPEC_NVM_OPC_WRITE;
544 log_err("ioeng->queue(): ENOSYS: %u\n", io_u->ddir);
545 xnvme_queue_put_cmd_ctx(ctx->async.queue, ctx);
547 io_u->error = ENOSYS;
549 return FIO_Q_COMPLETED;
553 xd->iovec[io_u->index].iov_base = io_u->xfer_buf;
554 xd->iovec[io_u->index].iov_len = io_u->xfer_buflen;
556 err = xnvme_cmd_passv(ctx, &xd->iovec[io_u->index], 1, io_u->xfer_buflen, NULL, 0,
559 err = xnvme_cmd_pass(ctx, io_u->xfer_buf, io_u->xfer_buflen, NULL, 0);
567 xnvme_queue_put_cmd_ctx(ctx->async.queue, ctx);
571 log_err("ioeng->queue(): err: '%d'\n", err);
573 xnvme_queue_put_cmd_ctx(ctx->async.queue, ctx);
575 io_u->error = abs(err);
577 return FIO_Q_COMPLETED;
581 static int xnvme_fioe_close(struct thread_data *td, struct fio_file *f)
583 struct xnvme_fioe_data *xd = td->io_ops_data;
585 dprint(FD_FILE, "xnvme close %s -- nopen: %ld\n", f->file_name, xd->nopen);
592 static int xnvme_fioe_open(struct thread_data *td, struct fio_file *f)
594 struct xnvme_fioe_data *xd = td->io_ops_data;
596 dprint(FD_FILE, "xnvme open %s -- nopen: %ld\n", f->file_name, xd->nopen);
598 if (f->fileno > (int)xd->nallocated) {
599 log_err("ioeng->open(): f->fileno > xd->nallocated; invalid assumption\n");
602 if (xd->files[f->fileno].fio_file != f) {
603 log_err("ioeng->open(): fio_file != f; invalid assumption\n");
612 static int xnvme_fioe_invalidate(struct thread_data *td, struct fio_file *f)
614 /* Consider only doing this with be:spdk */
618 static int xnvme_fioe_get_max_open_zones(struct thread_data *td, struct fio_file *f,
619 unsigned int *max_open_zones)
621 struct xnvme_opts opts = xnvme_opts_from_fioe(td);
622 struct xnvme_dev *dev;
623 const struct xnvme_spec_znd_idfy_ns *zns;
624 int err = 0, err_lock;
626 if (f->filetype != FIO_TYPE_FILE && f->filetype != FIO_TYPE_BLOCK &&
627 f->filetype != FIO_TYPE_CHAR) {
628 log_info("ioeng->get_max_open_zoned(): ignoring filetype: %d\n", f->filetype);
631 err_lock = pthread_mutex_lock(&g_serialize);
633 log_err("ioeng->get_max_open_zones(): pthread_mutex_lock(), err(%d)\n", err_lock);
637 dev = xnvme_dev_open(f->file_name, &opts);
639 log_err("ioeng->get_max_open_zones(): xnvme_dev_open(), err(%d)\n", err_lock);
643 if (xnvme_dev_get_geo(dev)->type != XNVME_GEO_ZONED) {
649 zns = (void *)xnvme_dev_get_ns_css(dev);
651 log_err("ioeng->get_max_open_zones(): xnvme_dev_get_ns_css(), err(%d)\n", errno);
657 * intentional overflow as the value is zero-based and NVMe
658 * defines 0xFFFFFFFF as unlimited thus overflowing to 0 which
659 * is how fio indicates unlimited and otherwise just converting
662 *max_open_zones = zns->mor + 1;
665 xnvme_dev_close(dev);
666 err_lock = pthread_mutex_unlock(&g_serialize);
668 log_err("ioeng->get_max_open_zones(): pthread_mutex_unlock(), err(%d)\n",
675 * Currently, this function is called before of I/O engine initialization, so,
676 * we cannot consult the file-wrapping done when 'fioe' initializes.
677 * Instead we just open based on the given filename.
679 * TODO: unify the different setup methods, consider keeping the handle around,
680 * and consider how to support the --be option in this usecase
682 static int xnvme_fioe_get_zoned_model(struct thread_data *td, struct fio_file *f,
683 enum zbd_zoned_model *model)
685 struct xnvme_opts opts = xnvme_opts_from_fioe(td);
686 struct xnvme_dev *dev;
687 int err = 0, err_lock;
689 if (f->filetype != FIO_TYPE_FILE && f->filetype != FIO_TYPE_BLOCK &&
690 f->filetype != FIO_TYPE_CHAR) {
691 log_info("ioeng->get_zoned_model(): ignoring filetype: %d\n", f->filetype);
695 err = pthread_mutex_lock(&g_serialize);
697 log_err("ioeng->get_zoned_model(): pthread_mutex_lock(), err(%d)\n", err);
701 dev = xnvme_dev_open(f->file_name, &opts);
703 log_err("ioeng->get_zoned_model(): xnvme_dev_open(%s) failed, errno: %d\n",
704 f->file_name, errno);
709 switch (xnvme_dev_get_geo(dev)->type) {
710 case XNVME_GEO_UNKNOWN:
711 dprint(FD_ZBD, "%s: got 'unknown', assigning ZBD_NONE\n", f->file_name);
715 case XNVME_GEO_CONVENTIONAL:
716 dprint(FD_ZBD, "%s: got 'conventional', assigning ZBD_NONE\n", f->file_name);
720 case XNVME_GEO_ZONED:
721 dprint(FD_ZBD, "%s: got 'zoned', assigning ZBD_HOST_MANAGED\n", f->file_name);
722 *model = ZBD_HOST_MANAGED;
726 dprint(FD_ZBD, "%s: hit-default, assigning ZBD_NONE\n", f->file_name);
734 xnvme_dev_close(dev);
736 err_lock = pthread_mutex_unlock(&g_serialize);
738 log_err("ioeng->get_zoned_model(): pthread_mutex_unlock(), err(%d)\n", err_lock);
744 * Fills the given ``zbdz`` with at most ``nr_zones`` zone-descriptors.
746 * The implementation converts the NVMe Zoned Command Set log-pages for Zone
747 * descriptors into the Linux Kernel Zoned Block Report format.
749 * NOTE: This function is called before I/O engine initialization, that is,
750 * before ``_dev_open`` has been called and file-wrapping is setup. Thus is has
751 * to do the ``_dev_open`` itself, and shut it down again once it is done
752 * retrieving the log-pages and converting them to the report format.
754 * TODO: unify the different setup methods, consider keeping the handle around,
755 * and consider how to support the --async option in this usecase
757 static int xnvme_fioe_report_zones(struct thread_data *td, struct fio_file *f, uint64_t offset,
758 struct zbd_zone *zbdz, unsigned int nr_zones)
760 struct xnvme_opts opts = xnvme_opts_from_fioe(td);
761 const struct xnvme_spec_znd_idfy_lbafe *lbafe = NULL;
762 struct xnvme_dev *dev = NULL;
763 const struct xnvme_geo *geo = NULL;
764 struct xnvme_znd_report *rprt = NULL;
767 unsigned int limit = 0;
768 int err = 0, err_lock;
770 dprint(FD_ZBD, "%s: report_zones() offset: %zu, nr_zones: %u\n", f->file_name, offset,
773 err = pthread_mutex_lock(&g_serialize);
775 log_err("ioeng->report_zones(%s): pthread_mutex_lock(), err(%d)\n", f->file_name,
780 dev = xnvme_dev_open(f->file_name, &opts);
782 log_err("ioeng->report_zones(%s): xnvme_dev_open(), err(%d)\n", f->file_name,
787 geo = xnvme_dev_get_geo(dev);
788 ssw = xnvme_dev_get_ssw(dev);
789 lbafe = xnvme_znd_dev_get_lbafe(dev);
791 limit = nr_zones > geo->nzone ? geo->nzone : nr_zones;
793 dprint(FD_ZBD, "%s: limit: %u\n", f->file_name, limit);
795 slba = ((offset >> ssw) / geo->nsect) * geo->nsect;
797 rprt = xnvme_znd_report_from_dev(dev, slba, limit, 0);
799 log_err("ioeng->report_zones(%s): xnvme_znd_report_from_dev(), err(%d)\n",
800 f->file_name, errno);
804 if (rprt->nentries != limit) {
805 log_err("ioeng->report_zones(%s): nentries != nr_zones\n", f->file_name);
809 if (offset > geo->tbytes) {
810 log_err("ioeng->report_zones(%s): out-of-bounds\n", f->file_name);
814 /* Transform the zone-report */
815 for (uint32_t idx = 0; idx < rprt->nentries; ++idx) {
816 struct xnvme_spec_znd_descr *descr = XNVME_ZND_REPORT_DESCR(rprt, idx);
818 zbdz[idx].start = descr->zslba << ssw;
819 zbdz[idx].len = lbafe->zsze << ssw;
820 zbdz[idx].capacity = descr->zcap << ssw;
821 zbdz[idx].wp = descr->wp << ssw;
824 case XNVME_SPEC_ZND_TYPE_SEQWR:
825 zbdz[idx].type = ZBD_ZONE_TYPE_SWR;
829 log_err("ioeng->report_zones(%s): invalid type for zone at offset(%zu)\n",
830 f->file_name, zbdz[idx].start);
836 case XNVME_SPEC_ZND_STATE_EMPTY:
837 zbdz[idx].cond = ZBD_ZONE_COND_EMPTY;
839 case XNVME_SPEC_ZND_STATE_IOPEN:
840 zbdz[idx].cond = ZBD_ZONE_COND_IMP_OPEN;
842 case XNVME_SPEC_ZND_STATE_EOPEN:
843 zbdz[idx].cond = ZBD_ZONE_COND_EXP_OPEN;
845 case XNVME_SPEC_ZND_STATE_CLOSED:
846 zbdz[idx].cond = ZBD_ZONE_COND_CLOSED;
848 case XNVME_SPEC_ZND_STATE_FULL:
849 zbdz[idx].cond = ZBD_ZONE_COND_FULL;
852 case XNVME_SPEC_ZND_STATE_RONLY:
853 case XNVME_SPEC_ZND_STATE_OFFLINE:
855 zbdz[idx].cond = ZBD_ZONE_COND_OFFLINE;
861 xnvme_buf_virt_free(rprt);
863 xnvme_dev_close(dev);
865 err_lock = pthread_mutex_unlock(&g_serialize);
867 log_err("ioeng->report_zones(): pthread_mutex_unlock(), err: %d\n", err_lock);
869 dprint(FD_ZBD, "err: %d, nr_zones: %d\n", err, (int)nr_zones);
871 return err ? err : (int)limit;
875 * NOTE: This function may get called before I/O engine initialization, that is,
876 * before ``_dev_open`` has been called and file-wrapping is setup. In such
877 * case it has to do ``_dev_open`` itself, and shut it down again once it is
878 * done resetting write pointer of zones.
880 static int xnvme_fioe_reset_wp(struct thread_data *td, struct fio_file *f, uint64_t offset,
883 struct xnvme_opts opts = xnvme_opts_from_fioe(td);
884 struct xnvme_fioe_data *xd = NULL;
885 struct xnvme_fioe_fwrap *fwrap = NULL;
886 struct xnvme_dev *dev = NULL;
887 const struct xnvme_geo *geo = NULL;
888 uint64_t first, last;
891 int err = 0, err_lock;
893 if (td->io_ops_data) {
894 xd = td->io_ops_data;
895 fwrap = &xd->files[f->fileno];
904 err = pthread_mutex_lock(&g_serialize);
906 log_err("ioeng->reset_wp(): pthread_mutex_lock(), err(%d)\n", err);
910 dev = xnvme_dev_open(f->file_name, &opts);
912 log_err("ioeng->reset_wp(): xnvme_dev_open(%s) failed, errno(%d)\n",
913 f->file_name, errno);
916 geo = xnvme_dev_get_geo(dev);
917 ssw = xnvme_dev_get_ssw(dev);
920 nsid = xnvme_dev_get_nsid(dev);
922 first = ((offset >> ssw) / geo->nsect) * geo->nsect;
923 last = (((offset + length) >> ssw) / geo->nsect) * geo->nsect;
924 dprint(FD_ZBD, "first: 0x%lx, last: 0x%lx\n", first, last);
926 for (uint64_t zslba = first; zslba < last; zslba += geo->nsect) {
927 struct xnvme_cmd_ctx ctx = xnvme_cmd_ctx_from_dev(dev);
929 if (zslba >= (geo->nsect * geo->nzone)) {
930 log_err("ioeng->reset_wp(): out-of-bounds\n");
935 err = xnvme_znd_mgmt_send(&ctx, nsid, zslba, false,
936 XNVME_SPEC_ZND_CMD_MGMT_SEND_RESET, 0x0, NULL);
937 if (err || xnvme_cmd_ctx_cpl_status(&ctx)) {
938 err = err ? err : -EIO;
939 log_err("ioeng->reset_wp(): err(%d), sc(%d)", err, ctx.cpl.status.sc);
945 if (!td->io_ops_data) {
946 xnvme_dev_close(dev);
948 err_lock = pthread_mutex_unlock(&g_serialize);
950 log_err("ioeng->reset_wp(): pthread_mutex_unlock(), err(%d)\n", err_lock);
956 static int xnvme_fioe_fetch_ruhs(struct thread_data *td, struct fio_file *f,
957 struct fio_ruhs_info *fruhs_info)
959 struct xnvme_opts opts = xnvme_opts_from_fioe(td);
960 struct xnvme_dev *dev;
961 struct xnvme_spec_ruhs *ruhs;
962 struct xnvme_cmd_ctx ctx;
963 uint32_t ruhs_nbytes;
965 int err = 0, err_lock;
967 if (f->filetype != FIO_TYPE_CHAR) {
968 log_err("ioeng->fdp_ruhs(): ignoring filetype: %d\n", f->filetype);
972 err = pthread_mutex_lock(&g_serialize);
974 log_err("ioeng->fdp_ruhs(): pthread_mutex_lock(), err(%d)\n", err);
978 dev = xnvme_dev_open(f->file_name, &opts);
980 log_err("ioeng->fdp_ruhs(): xnvme_dev_open(%s) failed, errno: %d\n",
981 f->file_name, errno);
986 ruhs_nbytes = sizeof(*ruhs) + (FDP_MAX_RUHS * sizeof(struct xnvme_spec_ruhs_desc));
987 ruhs = xnvme_buf_alloc(dev, ruhs_nbytes);
992 memset(ruhs, 0, ruhs_nbytes);
994 ctx = xnvme_cmd_ctx_from_dev(dev);
995 nsid = xnvme_dev_get_nsid(dev);
997 err = xnvme_nvm_mgmt_recv(&ctx, nsid, XNVME_SPEC_IO_MGMT_RECV_RUHS, 0, ruhs, ruhs_nbytes);
999 if (err || xnvme_cmd_ctx_cpl_status(&ctx)) {
1000 err = err ? err : -EIO;
1001 log_err("ioeng->fdp_ruhs(): err(%d), sc(%d)", err, ctx.cpl.status.sc);
1005 fruhs_info->nr_ruhs = ruhs->nruhsd;
1006 for (uint32_t idx = 0; idx < fruhs_info->nr_ruhs; ++idx) {
1007 fruhs_info->plis[idx] = le16_to_cpu(ruhs->desc[idx].pi);
1011 xnvme_buf_free(dev, ruhs);
1013 xnvme_dev_close(dev);
1015 err_lock = pthread_mutex_unlock(&g_serialize);
1017 log_err("ioeng->fdp_ruhs(): pthread_mutex_unlock(), err(%d)\n", err_lock);
1022 static int xnvme_fioe_get_file_size(struct thread_data *td, struct fio_file *f)
1024 struct xnvme_opts opts = xnvme_opts_from_fioe(td);
1025 struct xnvme_dev *dev;
1028 if (fio_file_size_known(f))
1031 ret = pthread_mutex_lock(&g_serialize);
1033 log_err("ioeng->reset_wp(): pthread_mutex_lock(), err(%d)\n", ret);
1037 dev = xnvme_dev_open(f->file_name, &opts);
1039 log_err("%s: failed retrieving device handle, errno: %d\n", f->file_name, errno);
1044 f->real_file_size = xnvme_dev_get_geo(dev)->tbytes;
1045 fio_file_set_size_known(f);
1047 if (td->o.zone_mode == ZONE_MODE_ZBD)
1048 f->filetype = FIO_TYPE_BLOCK;
1051 xnvme_dev_close(dev);
1052 err = pthread_mutex_unlock(&g_serialize);
1054 log_err("ioeng->reset_wp(): pthread_mutex_unlock(), err(%d)\n", err);
1059 FIO_STATIC struct ioengine_ops ioengine = {
1061 .version = FIO_IOOPS_VERSION,
1063 .option_struct_size = sizeof(struct xnvme_fioe_options),
1064 .flags = FIO_DISKLESSIO | FIO_NODISKUTIL | FIO_NOEXTEND | FIO_MEMALIGN | FIO_RAWIO,
1066 .cleanup = xnvme_fioe_cleanup,
1067 .init = xnvme_fioe_init,
1069 .iomem_free = xnvme_fioe_iomem_free,
1070 .iomem_alloc = xnvme_fioe_iomem_alloc,
1072 .io_u_free = xnvme_fioe_io_u_free,
1073 .io_u_init = xnvme_fioe_io_u_init,
1075 .event = xnvme_fioe_event,
1076 .getevents = xnvme_fioe_getevents,
1077 .queue = xnvme_fioe_queue,
1079 .close_file = xnvme_fioe_close,
1080 .open_file = xnvme_fioe_open,
1081 .get_file_size = xnvme_fioe_get_file_size,
1083 .invalidate = xnvme_fioe_invalidate,
1084 .get_max_open_zones = xnvme_fioe_get_max_open_zones,
1085 .get_zoned_model = xnvme_fioe_get_zoned_model,
1086 .report_zones = xnvme_fioe_report_zones,
1087 .reset_wp = xnvme_fioe_reset_wp,
1089 .fdp_fetch_ruhs = xnvme_fioe_fetch_ruhs,
1092 static void fio_init fio_xnvme_register(void)
1094 register_ioengine(&ioengine);
1097 static void fio_exit fio_xnvme_unregister(void)
1099 unregister_ioengine(&ioengine);