[fio.git] / engines / nvme.c

// SPDX-License-Identifier: GPL-2.0
/*
 * nvme structure declarations and helper functions for the
 * io_uring_cmd engine.
 */

#include "nvme.h"
#include "../crc/crc-t10dif.h"

static inline __u64 get_slba(struct nvme_data *data, struct io_u *io_u)
{
	if (data->lba_ext)
		return io_u->offset / data->lba_ext;
	else
		return io_u->offset >> data->lba_shift;
}

static inline __u32 get_nlb(struct nvme_data *data, struct io_u *io_u)
{
	if (data->lba_ext)
		return io_u->xfer_buflen / data->lba_ext - 1;
	else
		return (io_u->xfer_buflen >> data->lba_shift) - 1;
}

static void fio_nvme_generate_pi_16b_guard(struct nvme_data *data,
					   struct io_u *io_u,
					   struct nvme_cmd_ext_io_opts *opts)
{
	struct nvme_pi_data *pi_data = io_u->engine_data;
	struct nvme_16b_guard_pif *pi;
	unsigned char *buf = io_u->xfer_buf;
	unsigned char *md_buf = io_u->mmap_data;
	__u64 slba = get_slba(data, io_u);
	__u32 nlb = get_nlb(data, io_u) + 1;
	__u32 lba_num = 0;
	__u16 guard = 0;

	if (data->pi_loc) {
		if (data->lba_ext)
			pi_data->interval = data->lba_ext - data->ms;
		else
			pi_data->interval = 0;
	} else {
		if (data->lba_ext)
			pi_data->interval = data->lba_ext - sizeof(struct nvme_16b_guard_pif);
		else
			pi_data->interval = data->ms - sizeof(struct nvme_16b_guard_pif);
	}

	if (io_u->ddir != DDIR_WRITE)
		return;

	while (lba_num < nlb) {
		if (data->lba_ext)
			pi = (struct nvme_16b_guard_pif *)(buf + pi_data->interval);
		else
			pi = (struct nvme_16b_guard_pif *)(md_buf + pi_data->interval);

		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
			if (data->lba_ext) {
				guard = fio_crc_t10dif(0, buf, pi_data->interval);
			} else {
				guard = fio_crc_t10dif(0, buf, data->lba_size);
				guard = fio_crc_t10dif(guard, md_buf, pi_data->interval);
			}
			pi->guard = cpu_to_be16(guard);
		}

		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
			pi->apptag = cpu_to_be16(pi_data->apptag);

		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
			switch (data->pi_type) {
			case NVME_NS_DPS_PI_TYPE1:
			case NVME_NS_DPS_PI_TYPE2:
				pi->srtag = cpu_to_be32((__u32)slba + lba_num);
				break;
			case NVME_NS_DPS_PI_TYPE3:
				break;
			}
		}
		if (data->lba_ext) {
			buf += data->lba_ext;
		} else {
			buf += data->lba_size;
			md_buf += data->ms;
		}
		lba_num++;
	}
}

static int fio_nvme_verify_pi_16b_guard(struct nvme_data *data,
					struct io_u *io_u)
{
	struct nvme_pi_data *pi_data = io_u->engine_data;
	struct nvme_16b_guard_pif *pi;
	struct fio_file *f = io_u->file;
	unsigned char *buf = io_u->xfer_buf;
	unsigned char *md_buf = io_u->mmap_data;
	__u64 slba = get_slba(data, io_u);
	__u32 nlb = get_nlb(data, io_u) + 1;
	__u32 lba_num = 0;
	__u16 unmask_app, unmask_app_exp, guard = 0;

	while (lba_num < nlb) {
		if (data->lba_ext)
			pi = (struct nvme_16b_guard_pif *)(buf + pi_data->interval);
		else
			pi = (struct nvme_16b_guard_pif *)(md_buf + pi_data->interval);

		if (data->pi_type == NVME_NS_DPS_PI_TYPE3) {
			if (pi->apptag == NVME_PI_APP_DISABLE &&
			    pi->srtag == NVME_PI_REF_DISABLE)
				goto next;
		} else if (data->pi_type == NVME_NS_DPS_PI_TYPE1 ||
			   data->pi_type == NVME_NS_DPS_PI_TYPE2) {
			if (pi->apptag == NVME_PI_APP_DISABLE)
				goto next;
		}

		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
			if (data->lba_ext) {
				guard = fio_crc_t10dif(0, buf, pi_data->interval);
			} else {
				guard = fio_crc_t10dif(0, buf, data->lba_size);
				guard = fio_crc_t10dif(guard, md_buf, pi_data->interval);
			}
			if (be16_to_cpu(pi->guard) != guard) {
				log_err("%s: Guard compare error: LBA: %llu Expected=%x, Actual=%x\n",
					f->file_name, (unsigned long long)slba,
					guard, be16_to_cpu(pi->guard));
				return -EIO;
			}
		}

		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_APP) {
			unmask_app = be16_to_cpu(pi->apptag) & pi_data->apptag_mask;
			unmask_app_exp = pi_data->apptag & pi_data->apptag_mask;
			if (unmask_app != unmask_app_exp) {
				log_err("%s: APPTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
					f->file_name, (unsigned long long)slba,
					unmask_app_exp, unmask_app);
				return -EIO;
			}
		}

		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
			switch (data->pi_type) {
			case NVME_NS_DPS_PI_TYPE1:
			case NVME_NS_DPS_PI_TYPE2:
				if (be32_to_cpu(pi->srtag) !=
				    ((__u32)slba + lba_num)) {
					log_err("%s: REFTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
						f->file_name, (unsigned long long)slba,
						(__u32)slba + lba_num,
						be32_to_cpu(pi->srtag));
					return -EIO;
				}
				break;
			case NVME_NS_DPS_PI_TYPE3:
				break;
			}
		}
next:
		if (data->lba_ext) {
			buf += data->lba_ext;
		} else {
			buf += data->lba_size;
			md_buf += data->ms;
		}
		lba_num++;
	}

	return 0;
}

void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
				  struct nvme_dsm_range *dsm)
{
	struct nvme_data *data = FILE_ENG_DATA(io_u->file);

	cmd->opcode = nvme_cmd_dsm;
	cmd->nsid = data->nsid;
	cmd->cdw10 = 0;
	cmd->cdw11 = NVME_ATTRIBUTE_DEALLOCATE;
	cmd->addr = (__u64) (uintptr_t) dsm;
	cmd->data_len = sizeof(*dsm);

	dsm->slba = get_slba(data, io_u);
	/* nlb is a 1-based value for deallocate */
	dsm->nlb = get_nlb(data, io_u) + 1;
}

int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
			    struct iovec *iov, struct nvme_dsm_range *dsm)
{
	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
	__u64 slba;
	__u32 nlb;

	memset(cmd, 0, sizeof(struct nvme_uring_cmd));

	switch (io_u->ddir) {
	case DDIR_READ:
		cmd->opcode = nvme_cmd_read;
		break;
	case DDIR_WRITE:
		cmd->opcode = nvme_cmd_write;
		break;
	case DDIR_TRIM:
		fio_nvme_uring_cmd_trim_prep(cmd, io_u, dsm);
		return 0;
	default:
		return -ENOTSUP;
	}

	slba = get_slba(data, io_u);
	nlb = get_nlb(data, io_u);

	/* cdw10 and cdw11 represent starting lba */
	cmd->cdw10 = slba & 0xffffffff;
	cmd->cdw11 = slba >> 32;
	/* cdw12 represent number of lba's for read/write */
	cmd->cdw12 = nlb | (io_u->dtype << 20);
	cmd->cdw13 = io_u->dspec << 16;
	if (iov) {
		iov->iov_base = io_u->xfer_buf;
		iov->iov_len = io_u->xfer_buflen;
		cmd->addr = (__u64)(uintptr_t)iov;
		cmd->data_len = 1;
	} else {
		cmd->addr = (__u64)(uintptr_t)io_u->xfer_buf;
		cmd->data_len = io_u->xfer_buflen;
	}
	if (data->lba_shift && data->ms) {
		cmd->metadata = (__u64)(uintptr_t)io_u->mmap_data;
		cmd->metadata_len = (nlb + 1) * data->ms;
	}
	cmd->nsid = data->nsid;
	return 0;
}

void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
		      struct nvme_cmd_ext_io_opts *opts)
{
	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
	__u64 slba;

	slba = get_slba(data, io_u);
	cmd->cdw12 |= opts->io_flags;

	if (data->pi_type && !(opts->io_flags & NVME_IO_PRINFO_PRACT)) {
		if (data->guard_type == NVME_NVM_NS_16B_GUARD)
			fio_nvme_generate_pi_16b_guard(data, io_u, opts);
	}

	switch (data->pi_type) {
	case NVME_NS_DPS_PI_TYPE1:
	case NVME_NS_DPS_PI_TYPE2:
		switch (data->guard_type) {
		case NVME_NVM_NS_16B_GUARD:
			cmd->cdw14 = (__u32)slba;
			break;
		case NVME_NVM_NS_64B_GUARD:
			cmd->cdw14 = (__u32)slba;
			cmd->cdw3 = ((slba >> 32) & 0xffff);
			break;
		default:
			break;
		}
		cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
		break;
	case NVME_NS_DPS_PI_TYPE3:
		cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
		break;
	case NVME_NS_DPS_PI_NONE:
		break;
	}
}

int fio_nvme_pi_verify(struct nvme_data *data, struct io_u *io_u)
{
	int ret = 0;

	switch (data->guard_type) {
	case NVME_NVM_NS_16B_GUARD:
		ret = fio_nvme_verify_pi_16b_guard(data, io_u);
		break;
	default:
		break;
	}

	return ret;
}

static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
			 enum nvme_csi csi, void *data)
{
	struct nvme_passthru_cmd cmd = {
		.opcode         = nvme_admin_identify,
		.nsid           = nsid,
		.addr           = (__u64)(uintptr_t)data,
		.data_len       = NVME_IDENTIFY_DATA_SIZE,
		.cdw10          = cns,
		.cdw11          = csi << NVME_IDENTIFY_CSI_SHIFT,
		.timeout_ms     = NVME_DEFAULT_IOCTL_TIMEOUT,
	};

	return ioctl(fd, NVME_IOCTL_ADMIN_CMD, &cmd);
}

int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
		      struct nvme_data *data)
{
	struct nvme_id_ns ns;
	struct nvme_id_ctrl ctrl;
	struct nvme_nvm_id_ns nvm_ns;
	int namespace_id;
	int fd, err;
	__u32 format_idx, elbaf;

	if (f->filetype != FIO_TYPE_CHAR) {
		log_err("ioengine io_uring_cmd only works with nvme ns "
			"generic char devices (/dev/ngXnY)\n");
		return 1;
	}

	fd = open(f->file_name, O_RDONLY);
	if (fd < 0)
		return -errno;

	namespace_id = ioctl(fd, NVME_IOCTL_ID);
	if (namespace_id < 0) {
		err = -errno;
		log_err("%s: failed to fetch namespace-id\n", f->file_name);
		goto out;
	}

	err = nvme_identify(fd, 0, NVME_IDENTIFY_CNS_CTRL, NVME_CSI_NVM, &ctrl);
	if (err) {
		log_err("%s: failed to fetch identify ctrl\n", f->file_name);
		goto out;
	}

	/*
	 * Identify namespace to get namespace-id, namespace size in LBA's
	 * and LBA data size.
	 */
	err = nvme_identify(fd, namespace_id, NVME_IDENTIFY_CNS_NS,
				NVME_CSI_NVM, &ns);
	if (err) {
		log_err("%s: failed to fetch identify namespace\n",
			f->file_name);
		goto out;
	}

	data->nsid = namespace_id;

	/*
	 * 16 or 64 as maximum number of supported LBA formats.
	 * From flbas bit 0-3 indicates lsb and bit 5-6 indicates msb
	 * of the format index used to format the namespace.
	 */
	if (ns.nlbaf < 16)
		format_idx = ns.flbas & 0xf;
	else
		format_idx = (ns.flbas & 0xf) + (((ns.flbas >> 5) & 0x3) << 4);

	data->lba_size = 1 << ns.lbaf[format_idx].ds;
	data->ms = le16_to_cpu(ns.lbaf[format_idx].ms);

	/* Check for end to end data protection support */
	if (data->ms && (ns.dps & NVME_NS_DPS_PI_MASK))
		data->pi_type = (ns.dps & NVME_NS_DPS_PI_MASK);

	if (!data->pi_type)
		goto check_elba;

	if (ctrl.ctratt & NVME_CTRL_CTRATT_ELBAS) {
		err = nvme_identify(fd, namespace_id, NVME_IDENTIFY_CNS_CSI_NS,
					NVME_CSI_NVM, &nvm_ns);
		if (err) {
			log_err("%s: failed to fetch identify nvm namespace\n",
				f->file_name);
			goto out;
		}

		elbaf = le32_to_cpu(nvm_ns.elbaf[format_idx]);

		/* Currently we don't support storage tags */
		if (elbaf & NVME_ID_NS_NVM_STS_MASK) {
			log_err("%s: Storage tag not supported\n",
				f->file_name);
			err = -ENOTSUP;
			goto out;
		}

		data->guard_type = (elbaf >> NVME_ID_NS_NVM_GUARD_SHIFT) &
				NVME_ID_NS_NVM_GUARD_MASK;

		/* No 32 bit guard, as storage tag is mandatory for it */
		switch (data->guard_type) {
		case NVME_NVM_NS_16B_GUARD:
			data->pi_size = sizeof(struct nvme_16b_guard_pif);
			break;
		case NVME_NVM_NS_64B_GUARD:
			data->pi_size = sizeof(struct nvme_64b_guard_pif);
			break;
		default:
			break;
		}
	} else {
		data->guard_type = NVME_NVM_NS_16B_GUARD;
		data->pi_size = sizeof(struct nvme_16b_guard_pif);
	}

	/*
	 * when PRACT bit is set to 1, and metadata size is equal to protection
	 * information size, controller inserts and removes PI for write and
	 * read commands respectively.
	 */
	if (pi_act && data->ms == data->pi_size)
		data->ms = 0;

	data->pi_loc = (ns.dps & NVME_NS_DPS_PI_FIRST);

check_elba:
	/*
	 * Bit 4 for flbas indicates if metadata is transferred at the end of
	 * logical block creating an extended LBA.
	 */
	if (data->ms && ((ns.flbas >> 4) & 0x1))
		data->lba_ext = data->lba_size + data->ms;
	else
		data->lba_shift = ilog2(data->lba_size);

	*nlba = ns.nsze;

out:
	close(fd);
	return err;
}

int fio_nvme_get_zoned_model(struct thread_data *td, struct fio_file *f,
			     enum zbd_zoned_model *model)
{
	struct nvme_data *data = FILE_ENG_DATA(f);
	struct nvme_id_ns ns;
	struct nvme_passthru_cmd cmd;
	int fd, ret = 0;

	if (f->filetype != FIO_TYPE_CHAR)
		return -EINVAL;

	/* File is not yet opened */
	fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
	if (fd < 0)
		return -errno;

	/* Using nvme_id_ns for data as sizes are same */
	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_CTRL,
				NVME_CSI_ZNS, &ns);
	if (ret) {
		*model = ZBD_NONE;
		goto out;
	}

	memset(&cmd, 0, sizeof(struct nvme_passthru_cmd));

	/* Using nvme_id_ns for data as sizes are same */
	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
				NVME_CSI_ZNS, &ns);
	if (ret) {
		*model = ZBD_NONE;
		goto out;
	}

	*model = ZBD_HOST_MANAGED;
out:
	close(fd);
	return 0;
}

static int nvme_report_zones(int fd, __u32 nsid, __u64 slba, __u32 zras_feat,
			     __u32 data_len, void *data)
{
	struct nvme_passthru_cmd cmd = {
		.opcode         = nvme_zns_cmd_mgmt_recv,
		.nsid           = nsid,
		.addr           = (__u64)(uintptr_t)data,
		.data_len       = data_len,
		.cdw10          = slba & 0xffffffff,
		.cdw11          = slba >> 32,
		.cdw12		= (data_len >> 2) - 1,
		.cdw13		= NVME_ZNS_ZRA_REPORT_ZONES | zras_feat,
		.timeout_ms     = NVME_DEFAULT_IOCTL_TIMEOUT,
	};

	return ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
}

int fio_nvme_report_zones(struct thread_data *td, struct fio_file *f,
			  uint64_t offset, struct zbd_zone *zbdz,
			  unsigned int nr_zones)
{
	struct nvme_data *data = FILE_ENG_DATA(f);
	struct nvme_zone_report *zr;
	struct nvme_zns_id_ns zns_ns;
	struct nvme_id_ns ns;
	unsigned int i = 0, j, zones_fetched = 0;
	unsigned int max_zones, zones_chunks = 1024;
	int fd, ret = 0;
	__u32 zr_len;
	__u64 zlen;

	/* File is not yet opened */
	fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
	if (fd < 0)
		return -errno;

	zones_fetched = 0;
	zr_len = sizeof(*zr) + (zones_chunks * sizeof(struct nvme_zns_desc));
	zr = calloc(1, zr_len);
	if (!zr) {
		close(fd);
		return -ENOMEM;
	}

	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_NS,
				NVME_CSI_NVM, &ns);
	if (ret) {
		log_err("%s: nvme_identify_ns failed, err=%d\n", f->file_name,
			ret);
		goto out;
	}

	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
				NVME_CSI_ZNS, &zns_ns);
	if (ret) {
		log_err("%s: nvme_zns_identify_ns failed, err=%d\n",
			f->file_name, ret);
		goto out;
	}
	zlen = zns_ns.lbafe[ns.flbas & 0x0f].zsze << data->lba_shift;

	max_zones = (f->real_file_size - offset) / zlen;
	if (max_zones < nr_zones)
		nr_zones = max_zones;

	if (nr_zones < zones_chunks)
		zones_chunks = nr_zones;

	while (zones_fetched < nr_zones) {
		if (zones_fetched + zones_chunks >= nr_zones) {
			zones_chunks = nr_zones - zones_fetched;
			zr_len = sizeof(*zr) + (zones_chunks * sizeof(struct nvme_zns_desc));
		}
		ret = nvme_report_zones(fd, data->nsid, offset >> data->lba_shift,
					NVME_ZNS_ZRAS_FEAT_ERZ, zr_len, (void *)zr);
		if (ret) {
			log_err("%s: nvme_zns_report_zones failed, err=%d\n",
				f->file_name, ret);
			goto out;
		}

		/* Transform the zone-report */
		for (j = 0; j < zr->nr_zones; j++, i++) {
			struct nvme_zns_desc *desc = (struct nvme_zns_desc *)&(zr->entries[j]);

			zbdz[i].start = desc->zslba << data->lba_shift;
			zbdz[i].len = zlen;
			zbdz[i].wp = desc->wp << data->lba_shift;
			zbdz[i].capacity = desc->zcap << data->lba_shift;

			/* Zone Type is stored in first 4 bits. */
			switch (desc->zt & 0x0f) {
			case NVME_ZONE_TYPE_SEQWRITE_REQ:
				zbdz[i].type = ZBD_ZONE_TYPE_SWR;
				break;
			default:
				log_err("%s: invalid type for zone at offset %llu.\n",
					f->file_name, (unsigned long long) desc->zslba);
				ret = -EIO;
				goto out;
			}

			/* Zone State is stored in last 4 bits. */
			switch (desc->zs >> 4) {
			case NVME_ZNS_ZS_EMPTY:
				zbdz[i].cond = ZBD_ZONE_COND_EMPTY;
				break;
			case NVME_ZNS_ZS_IMPL_OPEN:
				zbdz[i].cond = ZBD_ZONE_COND_IMP_OPEN;
				break;
			case NVME_ZNS_ZS_EXPL_OPEN:
				zbdz[i].cond = ZBD_ZONE_COND_EXP_OPEN;
				break;
			case NVME_ZNS_ZS_CLOSED:
				zbdz[i].cond = ZBD_ZONE_COND_CLOSED;
				break;
			case NVME_ZNS_ZS_FULL:
				zbdz[i].cond = ZBD_ZONE_COND_FULL;
				break;
			case NVME_ZNS_ZS_READ_ONLY:
			case NVME_ZNS_ZS_OFFLINE:
			default:
				/* Treat all these conditions as offline (don't use!) */
				zbdz[i].cond = ZBD_ZONE_COND_OFFLINE;
				zbdz[i].wp = zbdz[i].start;
			}
		}
		zones_fetched += zr->nr_zones;
		offset += zr->nr_zones * zlen;
	}

	ret = zones_fetched;
out:
	free(zr);
	close(fd);

	return ret;
}

int fio_nvme_reset_wp(struct thread_data *td, struct fio_file *f,
		      uint64_t offset, uint64_t length)
{
	struct nvme_data *data = FILE_ENG_DATA(f);
	unsigned int nr_zones;
	unsigned long long zslba;
	int i, fd, ret = 0;

	/* If the file is not yet opened, open it for this function. */
	fd = f->fd;
	if (fd < 0) {
		fd = open(f->file_name, O_RDWR | O_LARGEFILE);
		if (fd < 0)
			return -errno;
	}

	zslba = offset >> data->lba_shift;
	nr_zones = (length + td->o.zone_size - 1) / td->o.zone_size;

	for (i = 0; i < nr_zones; i++, zslba += (td->o.zone_size >> data->lba_shift)) {
		struct nvme_passthru_cmd cmd = {
			.opcode         = nvme_zns_cmd_mgmt_send,
			.nsid           = data->nsid,
			.cdw10          = zslba & 0xffffffff,
			.cdw11          = zslba >> 32,
			.cdw13          = NVME_ZNS_ZSA_RESET,
			.addr           = (__u64)(uintptr_t)NULL,
			.data_len       = 0,
			.timeout_ms     = NVME_DEFAULT_IOCTL_TIMEOUT,
		};

		ret = ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
	}

	if (f->fd < 0)
		close(fd);
	return -ret;
}

int fio_nvme_get_max_open_zones(struct thread_data *td, struct fio_file *f,
				unsigned int *max_open_zones)
{
	struct nvme_data *data = FILE_ENG_DATA(f);
	struct nvme_zns_id_ns zns_ns;
	int fd, ret = 0;

	fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
	if (fd < 0)
		return -errno;

	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
				NVME_CSI_ZNS, &zns_ns);
	if (ret) {
		log_err("%s: nvme_zns_identify_ns failed, err=%d\n",
			f->file_name, ret);
		goto out;
	}

	*max_open_zones = zns_ns.mor + 1;
out:
	close(fd);
	return ret;
}

static inline int nvme_fdp_reclaim_unit_handle_status(int fd, __u32 nsid,
						      __u32 data_len, void *data)
{
	struct nvme_passthru_cmd cmd = {
		.opcode		= nvme_cmd_io_mgmt_recv,
		.nsid		= nsid,
		.addr		= (__u64)(uintptr_t)data,
		.data_len 	= data_len,
		.cdw10		= 1,
		.cdw11		= (data_len >> 2) - 1,
	};

	return ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
}

int fio_nvme_iomgmt_ruhs(struct thread_data *td, struct fio_file *f,
			 struct nvme_fdp_ruh_status *ruhs, __u32 bytes)
{
	struct nvme_data *data = FILE_ENG_DATA(f);
	int fd, ret;

	fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
	if (fd < 0)
		return -errno;

	ret = nvme_fdp_reclaim_unit_handle_status(fd, data->nsid, bytes, ruhs);
	if (ret) {
		log_err("%s: nvme_fdp_reclaim_unit_handle_status failed, err=%d\n",
			f->file_name, ret);
		errno = ENOTSUP;
	} else
		errno = 0;

	ret = -errno;
	close(fd);
	return ret;
}
Commit	Line	Data
ca59c41c	1	// SPDX-License-Identifier: GPL-2.0
b3d5e3fd AK	2	/*
	3	* nvme structure declarations and helper functions for the
	4	* io_uring_cmd engine.
	5	*/
	6
	7	#include "nvme.h"
5163f35e	8	#include "../crc/crc-t10dif.h"
b3d5e3fd	9
4885a6eb VF	10	static inline __u64 get_slba(struct nvme_data data, struct io_u io_u)
	11	{
	12	if (data->lba_ext)
	13	return io_u->offset / data->lba_ext;
	14	else
	15	return io_u->offset >> data->lba_shift;
	16	}
	17
	18	static inline __u32 get_nlb(struct nvme_data data, struct io_u io_u)
	19	{
	20	if (data->lba_ext)
	21	return io_u->xfer_buflen / data->lba_ext - 1;
	22	else
	23	return (io_u->xfer_buflen >> data->lba_shift) - 1;
	24	}
	25
5163f35e AK	26	static void fio_nvme_generate_pi_16b_guard(struct nvme_data *data,
	27	struct io_u *io_u,
	28	struct nvme_cmd_ext_io_opts *opts)
	29	{
	30	struct nvme_pi_data *pi_data = io_u->engine_data;
	31	struct nvme_16b_guard_pif *pi;
	32	unsigned char *buf = io_u->xfer_buf;
	33	unsigned char *md_buf = io_u->mmap_data;
	34	__u64 slba = get_slba(data, io_u);
	35	__u32 nlb = get_nlb(data, io_u) + 1;
	36	__u32 lba_num = 0;
	37	__u16 guard = 0;
	38
	39	if (data->pi_loc) {
	40	if (data->lba_ext)
	41	pi_data->interval = data->lba_ext - data->ms;
	42	else
	43	pi_data->interval = 0;
	44	} else {
	45	if (data->lba_ext)
	46	pi_data->interval = data->lba_ext - sizeof(struct nvme_16b_guard_pif);
	47	else
	48	pi_data->interval = data->ms - sizeof(struct nvme_16b_guard_pif);
	49	}
	50
	51	if (io_u->ddir != DDIR_WRITE)
	52	return;
	53
	54	while (lba_num < nlb) {
	55	if (data->lba_ext)
	56	pi = (struct nvme_16b_guard_pif *)(buf + pi_data->interval);
	57	else
	58	pi = (struct nvme_16b_guard_pif *)(md_buf + pi_data->interval);
	59
	60	if (opts->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
	61	if (data->lba_ext) {
	62	guard = fio_crc_t10dif(0, buf, pi_data->interval);
	63	} else {
	64	guard = fio_crc_t10dif(0, buf, data->lba_size);
	65	guard = fio_crc_t10dif(guard, md_buf, pi_data->interval);
	66	}
	67	pi->guard = cpu_to_be16(guard);
	68	}
	69
	70	if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
	71	pi->apptag = cpu_to_be16(pi_data->apptag);
	72
	73	if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
	74	switch (data->pi_type) {
	75	case NVME_NS_DPS_PI_TYPE1:
	76	case NVME_NS_DPS_PI_TYPE2:
	77	pi->srtag = cpu_to_be32((__u32)slba + lba_num);
	78	break;
	79	case NVME_NS_DPS_PI_TYPE3:
	80	break;
	81	}
	82	}
	83	if (data->lba_ext) {
	84	buf += data->lba_ext;
	85	} else {
	86	buf += data->lba_size;
	87	md_buf += data->ms;
	88	}
	89	lba_num++;
90	}
91	}
92
93	static int fio_nvme_verify_pi_16b_guard(struct nvme_data *data,
94	struct io_u *io_u)
95	{
96	struct nvme_pi_data *pi_data = io_u->engine_data;
97	struct nvme_16b_guard_pif *pi;
98	struct fio_file *f = io_u->file;
99	unsigned char *buf = io_u->xfer_buf;
100	unsigned char *md_buf = io_u->mmap_data;
101	__u64 slba = get_slba(data, io_u);
102	__u32 nlb = get_nlb(data, io_u) + 1;
103	__u32 lba_num = 0;
104	__u16 unmask_app, unmask_app_exp, guard = 0;
105
106	while (lba_num < nlb) {
107	if (data->lba_ext)
108	pi = (struct nvme_16b_guard_pif *)(buf + pi_data->interval);
109	else
110	pi = (struct nvme_16b_guard_pif *)(md_buf + pi_data->interval);
111
112	if (data->pi_type == NVME_NS_DPS_PI_TYPE3) {
113	if (pi->apptag == NVME_PI_APP_DISABLE &&
114	pi->srtag == NVME_PI_REF_DISABLE)
115	goto next;
116	} else if (data->pi_type == NVME_NS_DPS_PI_TYPE1 \|\|
117	data->pi_type == NVME_NS_DPS_PI_TYPE2) {
118	if (pi->apptag == NVME_PI_APP_DISABLE)
119	goto next;
120	}
121
122	if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
123	if (data->lba_ext) {
124	guard = fio_crc_t10dif(0, buf, pi_data->interval);
125	} else {
126	guard = fio_crc_t10dif(0, buf, data->lba_size);
127	guard = fio_crc_t10dif(guard, md_buf, pi_data->interval);
128	}
129	if (be16_to_cpu(pi->guard) != guard) {
130	log_err("%s: Guard compare error: LBA: %llu Expected=%x, Actual=%x\n",
131	f->file_name, (unsigned long long)slba,
132	guard, be16_to_cpu(pi->guard));
133	return -EIO;
134	}
135	}
136
137	if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_APP) {
138	unmask_app = be16_to_cpu(pi->apptag) & pi_data->apptag_mask;
139	unmask_app_exp = pi_data->apptag & pi_data->apptag_mask;
140	if (unmask_app != unmask_app_exp) {
141	log_err("%s: APPTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
142	f->file_name, (unsigned long long)slba,
143	unmask_app_exp, unmask_app);
144	return -EIO;
145	}
146	}
147
148	if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
149	switch (data->pi_type) {
150	case NVME_NS_DPS_PI_TYPE1:
151	case NVME_NS_DPS_PI_TYPE2:
152	if (be32_to_cpu(pi->srtag) !=
153	((__u32)slba + lba_num)) {
154	log_err("%s: REFTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
155	f->file_name, (unsigned long long)slba,
156	(__u32)slba + lba_num,
157	be32_to_cpu(pi->srtag));
158	return -EIO;
159	}
160	break;
161	case NVME_NS_DPS_PI_TYPE3:
162	break;
163	}
164	}
165	next:
166	if (data->lba_ext) {
167	buf += data->lba_ext;
168	} else {
169	buf += data->lba_size;
170	md_buf += data->ms;
171	}
172	lba_num++;
173	}
174
175	return 0;
176	}
177
4885a6eb VF	178	void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd cmd, struct io_u io_u,
	179	struct nvme_dsm_range *dsm)
	180	{
	181	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
	182
	183	cmd->opcode = nvme_cmd_dsm;
	184	cmd->nsid = data->nsid;
	185	cmd->cdw10 = 0;
	186	cmd->cdw11 = NVME_ATTRIBUTE_DEALLOCATE;
	187	cmd->addr = (__u64) (uintptr_t) dsm;
	188	cmd->data_len = sizeof(*dsm);
	189
	190	dsm->slba = get_slba(data, io_u);
	191	/* nlb is a 1-based value for deallocate */
	192	dsm->nlb = get_nlb(data, io_u) + 1;
	193	}
	194
b3d5e3fd	195	int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd cmd, struct io_u io_u,
4885a6eb	196	struct iovec iov, struct nvme_dsm_range dsm)
b3d5e3fd AK	197	{
	198	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
	199	__u64 slba;
	200	__u32 nlb;
	201
	202	memset(cmd, 0, sizeof(struct nvme_uring_cmd));
	203
4885a6eb VF	204	switch (io_u->ddir) {
4885a6eb VF	205	case DDIR_READ:
b3d5e3fd	206	cmd->opcode = nvme_cmd_read;
4885a6eb VF	207	break;
4885a6eb VF	208	case DDIR_WRITE:
b3d5e3fd	209	cmd->opcode = nvme_cmd_write;
4885a6eb VF	210	break;
	211	case DDIR_TRIM:
	212	fio_nvme_uring_cmd_trim_prep(cmd, io_u, dsm);
	213	return 0;
	214	default:
b3d5e3fd	215	return -ENOTSUP;
345fa8fd	216	}
b3d5e3fd	217
4885a6eb VF	218	slba = get_slba(data, io_u);
	219	nlb = get_nlb(data, io_u);
	220
b3d5e3fd AK	221	/* cdw10 and cdw11 represent starting lba */
	222	cmd->cdw10 = slba & 0xffffffff;
	223	cmd->cdw11 = slba >> 32;
	224	/* cdw12 represent number of lba's for read/write */
a7e8aae0 KB	225	cmd->cdw12 = nlb \| (io_u->dtype << 20);
a7e8aae0 KB	226	cmd->cdw13 = io_u->dspec << 16;
b3d5e3fd AK	227	if (iov) {
	228	iov->iov_base = io_u->xfer_buf;
	229	iov->iov_len = io_u->xfer_buflen;
	230	cmd->addr = (__u64)(uintptr_t)iov;
	231	cmd->data_len = 1;
	232	} else {
	233	cmd->addr = (__u64)(uintptr_t)io_u->xfer_buf;
	234	cmd->data_len = io_u->xfer_buflen;
	235	}
2d6451c9 AK	236	if (data->lba_shift && data->ms) {
	237	cmd->metadata = (__u64)(uintptr_t)io_u->mmap_data;
	238	cmd->metadata_len = (nlb + 1) * data->ms;
	239	}
b3d5e3fd AK	240	cmd->nsid = data->nsid;
	241	return 0;
	242	}
	243
3ee8311a AK	244	void fio_nvme_pi_fill(struct nvme_uring_cmd cmd, struct io_u io_u,
	245	struct nvme_cmd_ext_io_opts *opts)
	246	{
	247	struct nvme_data *data = FILE_ENG_DATA(io_u->file);
	248	__u64 slba;
	249
	250	slba = get_slba(data, io_u);
	251	cmd->cdw12 \|= opts->io_flags;
	252
5163f35e AK	253	if (data->pi_type && !(opts->io_flags & NVME_IO_PRINFO_PRACT)) {
	254	if (data->guard_type == NVME_NVM_NS_16B_GUARD)
	255	fio_nvme_generate_pi_16b_guard(data, io_u, opts);
	256	}
	257
3ee8311a AK	258	switch (data->pi_type) {
	259	case NVME_NS_DPS_PI_TYPE1:
	260	case NVME_NS_DPS_PI_TYPE2:
	261	switch (data->guard_type) {
	262	case NVME_NVM_NS_16B_GUARD:
	263	cmd->cdw14 = (__u32)slba;
	264	break;
	265	case NVME_NVM_NS_64B_GUARD:
	266	cmd->cdw14 = (__u32)slba;
	267	cmd->cdw3 = ((slba >> 32) & 0xffff);
	268	break;
	269	default:
	270	break;
	271	}
	272	cmd->cdw15 = (opts->apptag_mask << 16 \| opts->apptag);
	273	break;
	274	case NVME_NS_DPS_PI_TYPE3:
	275	cmd->cdw15 = (opts->apptag_mask << 16 \| opts->apptag);
	276	break;
	277	case NVME_NS_DPS_PI_NONE:
	278	break;
	279	}
	280	}
	281
5163f35e AK	282	int fio_nvme_pi_verify(struct nvme_data data, struct io_u io_u)
	283	{
	284	int ret = 0;
	285
	286	switch (data->guard_type) {
	287	case NVME_NVM_NS_16B_GUARD:
	288	ret = fio_nvme_verify_pi_16b_guard(data, io_u);
	289	break;
	290	default:
	291	break;
	292	}
	293
	294	return ret;
	295	}
	296
b3d5e3fd AK	297	static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
	298	enum nvme_csi csi, void *data)
	299	{
	300	struct nvme_passthru_cmd cmd = {
	301	.opcode = nvme_admin_identify,
	302	.nsid = nsid,
	303	.addr = (__u64)(uintptr_t)data,
	304	.data_len = NVME_IDENTIFY_DATA_SIZE,
	305	.cdw10 = cns,
	306	.cdw11 = csi << NVME_IDENTIFY_CSI_SHIFT,
	307	.timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
	308	};
	309
	310	return ioctl(fd, NVME_IOCTL_ADMIN_CMD, &cmd);
	311	}
	312
3ee8311a AK	313	int fio_nvme_get_info(struct fio_file f, __u64 nlba, __u32 pi_act,
3ee8311a AK	314	struct nvme_data *data)
b3d5e3fd AK	315	{
b3d5e3fd AK	316	struct nvme_id_ns ns;
3ee8311a AK	317	struct nvme_id_ctrl ctrl;
3ee8311a AK	318	struct nvme_nvm_id_ns nvm_ns;
37a0881f	319	int namespace_id;
b3d5e3fd	320	int fd, err;
3ee8311a	321	__u32 format_idx, elbaf;
b3d5e3fd AK	322
	323	if (f->filetype != FIO_TYPE_CHAR) {
	324	log_err("ioengine io_uring_cmd only works with nvme ns "
	325	"generic char devices (/dev/ngXnY)\n");
	326	return 1;
	327	}
	328
	329	fd = open(f->file_name, O_RDONLY);
	330	if (fd < 0)
	331	return -errno;
	332
	333	namespace_id = ioctl(fd, NVME_IOCTL_ID);
	334	if (namespace_id < 0) {
af10f514	335	err = -errno;
345fa8fd AK	336	log_err("%s: failed to fetch namespace-id\n", f->file_name);
345fa8fd AK	337	goto out;
b3d5e3fd AK	338	}
b3d5e3fd AK	339
3ee8311a AK	340	err = nvme_identify(fd, 0, NVME_IDENTIFY_CNS_CTRL, NVME_CSI_NVM, &ctrl);
	341	if (err) {
	342	log_err("%s: failed to fetch identify ctrl\n", f->file_name);
	343	goto out;
	344	}
	345
b3d5e3fd AK	346	/*
	347	* Identify namespace to get namespace-id, namespace size in LBA's
	348	* and LBA data size.
	349	*/
	350	err = nvme_identify(fd, namespace_id, NVME_IDENTIFY_CNS_NS,
	351	NVME_CSI_NVM, &ns);
	352	if (err) {
345fa8fd AK	353	log_err("%s: failed to fetch identify namespace\n",
345fa8fd AK	354	f->file_name);
3ee8311a	355	goto out;
b3d5e3fd AK	356	}
b3d5e3fd AK	357
e7e5023b	358	data->nsid = namespace_id;
01a7d384 AK	359
	360	/*
	361	* 16 or 64 as maximum number of supported LBA formats.
	362	* From flbas bit 0-3 indicates lsb and bit 5-6 indicates msb
	363	* of the format index used to format the namespace.
	364	*/
	365	if (ns.nlbaf < 16)
	366	format_idx = ns.flbas & 0xf;
	367	else
	368	format_idx = (ns.flbas & 0xf) + (((ns.flbas >> 5) & 0x3) << 4);
	369
e7e5023b	370	data->lba_size = 1 << ns.lbaf[format_idx].ds;
2d6451c9	371	data->ms = le16_to_cpu(ns.lbaf[format_idx].ms);
345fa8fd	372
3ee8311a AK	373	/* Check for end to end data protection support */
	374	if (data->ms && (ns.dps & NVME_NS_DPS_PI_MASK))
	375	data->pi_type = (ns.dps & NVME_NS_DPS_PI_MASK);
	376
	377	if (!data->pi_type)
	378	goto check_elba;
	379
	380	if (ctrl.ctratt & NVME_CTRL_CTRATT_ELBAS) {
	381	err = nvme_identify(fd, namespace_id, NVME_IDENTIFY_CNS_CSI_NS,
	382	NVME_CSI_NVM, &nvm_ns);
	383	if (err) {
	384	log_err("%s: failed to fetch identify nvm namespace\n",
	385	f->file_name);
	386	goto out;
	387	}
	388
	389	elbaf = le32_to_cpu(nvm_ns.elbaf[format_idx]);
	390
	391	/* Currently we don't support storage tags */
	392	if (elbaf & NVME_ID_NS_NVM_STS_MASK) {
	393	log_err("%s: Storage tag not supported\n",
	394	f->file_name);
	395	err = -ENOTSUP;
	396	goto out;
	397	}
	398
	399	data->guard_type = (elbaf >> NVME_ID_NS_NVM_GUARD_SHIFT) &
	400	NVME_ID_NS_NVM_GUARD_MASK;
	401
	402	/* No 32 bit guard, as storage tag is mandatory for it */
	403	switch (data->guard_type) {
	404	case NVME_NVM_NS_16B_GUARD:
	405	data->pi_size = sizeof(struct nvme_16b_guard_pif);
	406	break;
	407	case NVME_NVM_NS_64B_GUARD:
	408	data->pi_size = sizeof(struct nvme_64b_guard_pif);
	409	break;
	410	default:
	411	break;
	412	}
	413	} else {
	414	data->guard_type = NVME_NVM_NS_16B_GUARD;
	415	data->pi_size = sizeof(struct nvme_16b_guard_pif);
	416	}
	417
	418	/*
	419	* when PRACT bit is set to 1, and metadata size is equal to protection
	420	* information size, controller inserts and removes PI for write and
	421	* read commands respectively.
	422	*/
	423	if (pi_act && data->ms == data->pi_size)
	424	data->ms = 0;
	425
	426	data->pi_loc = (ns.dps & NVME_NS_DPS_PI_FIRST);
	427
	428	check_elba:
345fa8fd	429	/*
345fa8fd AK	430	* Bit 4 for flbas indicates if metadata is transferred at the end of
	431	* logical block creating an extended LBA.
	432	*/
2d6451c9	433	if (data->ms && ((ns.flbas >> 4) & 0x1))
e7e5023b AK	434	data->lba_ext = data->lba_size + data->ms;
	435	else
	436	data->lba_shift = ilog2(data->lba_size);
	437
b3d5e3fd AK	438	*nlba = ns.nsze;
b3d5e3fd AK	439
345fa8fd	440	out:
b3d5e3fd	441	close(fd);
345fa8fd	442	return err;
b3d5e3fd	443	}
3d05e0ff AK	444
	445	int fio_nvme_get_zoned_model(struct thread_data td, struct fio_file f,
	446	enum zbd_zoned_model *model)
	447	{
	448	struct nvme_data *data = FILE_ENG_DATA(f);
	449	struct nvme_id_ns ns;
	450	struct nvme_passthru_cmd cmd;
	451	int fd, ret = 0;
	452
	453	if (f->filetype != FIO_TYPE_CHAR)
	454	return -EINVAL;
	455
	456	/* File is not yet opened */
	457	fd = open(f->file_name, O_RDONLY \| O_LARGEFILE);
	458	if (fd < 0)
	459	return -errno;
	460
	461	/* Using nvme_id_ns for data as sizes are same */
	462	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_CTRL,
	463	NVME_CSI_ZNS, &ns);
	464	if (ret) {
	465	*model = ZBD_NONE;
	466	goto out;
	467	}
	468
	469	memset(&cmd, 0, sizeof(struct nvme_passthru_cmd));
	470
	471	/* Using nvme_id_ns for data as sizes are same */
	472	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
	473	NVME_CSI_ZNS, &ns);
	474	if (ret) {
	475	*model = ZBD_NONE;
	476	goto out;
	477	}
	478
	479	*model = ZBD_HOST_MANAGED;
	480	out:
	481	close(fd);
	482	return 0;
	483	}
	484
	485	static int nvme_report_zones(int fd, __u32 nsid, __u64 slba, __u32 zras_feat,
	486	__u32 data_len, void *data)
	487	{
	488	struct nvme_passthru_cmd cmd = {
	489	.opcode = nvme_zns_cmd_mgmt_recv,
	490	.nsid = nsid,
	491	.addr = (__u64)(uintptr_t)data,
	492	.data_len = data_len,
	493	.cdw10 = slba & 0xffffffff,
	494	.cdw11 = slba >> 32,
	495	.cdw12 = (data_len >> 2) - 1,
	496	.cdw13 = NVME_ZNS_ZRA_REPORT_ZONES \| zras_feat,
	497	.timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
	498	};
	499
	500	return ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
	501	}
	502
	503	int fio_nvme_report_zones(struct thread_data td, struct fio_file f,
	504	uint64_t offset, struct zbd_zone *zbdz,
	505	unsigned int nr_zones)
	506	{
	507	struct nvme_data *data = FILE_ENG_DATA(f);
508	struct nvme_zone_report *zr;
509	struct nvme_zns_id_ns zns_ns;
510	struct nvme_id_ns ns;
511	unsigned int i = 0, j, zones_fetched = 0;
512	unsigned int max_zones, zones_chunks = 1024;
513	int fd, ret = 0;
514	__u32 zr_len;
515	__u64 zlen;
516
517	/* File is not yet opened */
518	fd = open(f->file_name, O_RDONLY \| O_LARGEFILE);
519	if (fd < 0)
520	return -errno;
521
522	zones_fetched = 0;
523	zr_len = sizeof(zr) + (zones_chunks sizeof(struct nvme_zns_desc));
524	zr = calloc(1, zr_len);
3efcb23f JA	525	if (!zr) {
3efcb23f JA	526	close(fd);
3d05e0ff	527	return -ENOMEM;
3efcb23f	528	}
3d05e0ff AK	529
	530	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_NS,
	531	NVME_CSI_NVM, &ns);
	532	if (ret) {
	533	log_err("%s: nvme_identify_ns failed, err=%d\n", f->file_name,
	534	ret);
	535	goto out;
	536	}
	537
	538	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
	539	NVME_CSI_ZNS, &zns_ns);
	540	if (ret) {
	541	log_err("%s: nvme_zns_identify_ns failed, err=%d\n",
	542	f->file_name, ret);
	543	goto out;
	544	}
	545	zlen = zns_ns.lbafe[ns.flbas & 0x0f].zsze << data->lba_shift;
	546
	547	max_zones = (f->real_file_size - offset) / zlen;
	548	if (max_zones < nr_zones)
	549	nr_zones = max_zones;
	550
	551	if (nr_zones < zones_chunks)
	552	zones_chunks = nr_zones;
	553
	554	while (zones_fetched < nr_zones) {
	555	if (zones_fetched + zones_chunks >= nr_zones) {
	556	zones_chunks = nr_zones - zones_fetched;
	557	zr_len = sizeof(zr) + (zones_chunks sizeof(struct nvme_zns_desc));
	558	}
	559	ret = nvme_report_zones(fd, data->nsid, offset >> data->lba_shift,
	560	NVME_ZNS_ZRAS_FEAT_ERZ, zr_len, (void *)zr);
	561	if (ret) {
	562	log_err("%s: nvme_zns_report_zones failed, err=%d\n",
	563	f->file_name, ret);
	564	goto out;
	565	}
	566
	567	/* Transform the zone-report */
	568	for (j = 0; j < zr->nr_zones; j++, i++) {
	569	struct nvme_zns_desc desc = (struct nvme_zns_desc )&(zr->entries[j]);
	570
	571	zbdz[i].start = desc->zslba << data->lba_shift;
	572	zbdz[i].len = zlen;
	573	zbdz[i].wp = desc->wp << data->lba_shift;
	574	zbdz[i].capacity = desc->zcap << data->lba_shift;
	575
	576	/* Zone Type is stored in first 4 bits. */
	577	switch (desc->zt & 0x0f) {
	578	case NVME_ZONE_TYPE_SEQWRITE_REQ:
	579	zbdz[i].type = ZBD_ZONE_TYPE_SWR;
	580	break;
	581	default:
	582	log_err("%s: invalid type for zone at offset %llu.\n",
2fa0ab21	583	f->file_name, (unsigned long long) desc->zslba);
3d05e0ff AK	584	ret = -EIO;
	585	goto out;
	586	}
	587
	588	/* Zone State is stored in last 4 bits. */
	589	switch (desc->zs >> 4) {
	590	case NVME_ZNS_ZS_EMPTY:
	591	zbdz[i].cond = ZBD_ZONE_COND_EMPTY;
	592	break;
	593	case NVME_ZNS_ZS_IMPL_OPEN:
	594	zbdz[i].cond = ZBD_ZONE_COND_IMP_OPEN;
	595	break;
	596	case NVME_ZNS_ZS_EXPL_OPEN:
	597	zbdz[i].cond = ZBD_ZONE_COND_EXP_OPEN;
	598	break;
	599	case NVME_ZNS_ZS_CLOSED:
	600	zbdz[i].cond = ZBD_ZONE_COND_CLOSED;
	601	break;
	602	case NVME_ZNS_ZS_FULL:
	603	zbdz[i].cond = ZBD_ZONE_COND_FULL;
	604	break;
	605	case NVME_ZNS_ZS_READ_ONLY:
	606	case NVME_ZNS_ZS_OFFLINE:
	607	default:
	608	/* Treat all these conditions as offline (don't use!) */
	609	zbdz[i].cond = ZBD_ZONE_COND_OFFLINE;
	610	zbdz[i].wp = zbdz[i].start;
	611	}
	612	}
	613	zones_fetched += zr->nr_zones;
	614	offset += zr->nr_zones * zlen;
	615	}
	616
	617	ret = zones_fetched;
	618	out:
	619	free(zr);
	620	close(fd);
	621
	622	return ret;
	623	}
	624
	625	int fio_nvme_reset_wp(struct thread_data td, struct fio_file f,
	626	uint64_t offset, uint64_t length)
	627	{
	628	struct nvme_data *data = FILE_ENG_DATA(f);
	629	unsigned int nr_zones;
	630	unsigned long long zslba;
	631	int i, fd, ret = 0;
	632
	633	/* If the file is not yet opened, open it for this function. */
	634	fd = f->fd;
	635	if (fd < 0) {
	636	fd = open(f->file_name, O_RDWR \| O_LARGEFILE);
	637	if (fd < 0)
	638	return -errno;
	639	}
	640
	641	zslba = offset >> data->lba_shift;
	642	nr_zones = (length + td->o.zone_size - 1) / td->o.zone_size;
	643
	644	for (i = 0; i < nr_zones; i++, zslba += (td->o.zone_size >> data->lba_shift)) {
	645	struct nvme_passthru_cmd cmd = {
	646	.opcode = nvme_zns_cmd_mgmt_send,
	647	.nsid = data->nsid,
648	.cdw10 = zslba & 0xffffffff,
649	.cdw11 = zslba >> 32,
650	.cdw13 = NVME_ZNS_ZSA_RESET,
651	.addr = (__u64)(uintptr_t)NULL,
652	.data_len = 0,
653	.timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
654	};
655
656	ret = ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
657	}
658
659	if (f->fd < 0)
660	close(fd);
661	return -ret;
662	}
663
664	int fio_nvme_get_max_open_zones(struct thread_data td, struct fio_file f,
665	unsigned int *max_open_zones)
666	{
667	struct nvme_data *data = FILE_ENG_DATA(f);
668	struct nvme_zns_id_ns zns_ns;
669	int fd, ret = 0;
670
671	fd = open(f->file_name, O_RDONLY \| O_LARGEFILE);
672	if (fd < 0)
673	return -errno;
674
675	ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
676	NVME_CSI_ZNS, &zns_ns);
677	if (ret) {
678	log_err("%s: nvme_zns_identify_ns failed, err=%d\n",
679	f->file_name, ret);
680	goto out;
681	}
682
683	*max_open_zones = zns_ns.mor + 1;
684	out:
685	close(fd);
686	return ret;
687	}
a7e8aae0 KB	688
	689	static inline int nvme_fdp_reclaim_unit_handle_status(int fd, __u32 nsid,
	690	__u32 data_len, void *data)
	691	{
	692	struct nvme_passthru_cmd cmd = {
	693	.opcode = nvme_cmd_io_mgmt_recv,
	694	.nsid = nsid,
	695	.addr = (__u64)(uintptr_t)data,
	696	.data_len = data_len,
	697	.cdw10 = 1,
	698	.cdw11 = (data_len >> 2) - 1,
	699	};
	700
	701	return ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
	702	}
	703
	704	int fio_nvme_iomgmt_ruhs(struct thread_data td, struct fio_file f,
	705	struct nvme_fdp_ruh_status *ruhs, __u32 bytes)
	706	{
	707	struct nvme_data *data = FILE_ENG_DATA(f);
	708	int fd, ret;
	709
	710	fd = open(f->file_name, O_RDONLY \| O_LARGEFILE);
	711	if (fd < 0)
	712	return -errno;
	713
	714	ret = nvme_fdp_reclaim_unit_handle_status(fd, data->nsid, bytes, ruhs);
	715	if (ret) {
	716	log_err("%s: nvme_fdp_reclaim_unit_handle_status failed, err=%d\n",
	717	f->file_name, ret);
	718	errno = ENOTSUP;
	719	} else
	720	errno = 0;
	721
af10f514	722	ret = -errno;
a7e8aae0	723	close(fd);
af10f514	724	return ret;
a7e8aae0	725	}