#include "nvme.h"
#include "../crc/crc-t10dif.h"
+#include "../crc/crc64.h"
static inline __u64 get_slba(struct nvme_data *data, struct io_u *io_u)
{
return 0;
}
+static void fio_nvme_generate_pi_64b_guard(struct nvme_data *data,
+ struct io_u *io_u,
+ struct nvme_cmd_ext_io_opts *opts)
+{
+ struct nvme_pi_data *pi_data = io_u->engine_data;
+ struct nvme_64b_guard_pif *pi;
+ unsigned char *buf = io_u->xfer_buf;
+ unsigned char *md_buf = io_u->mmap_data;
+ uint64_t guard = 0;
+ __u64 slba = get_slba(data, io_u);
+ __u32 nlb = get_nlb(data, io_u) + 1;
+ __u32 lba_num = 0;
+
+ if (data->pi_loc) {
+ if (data->lba_ext)
+ pi_data->interval = data->lba_ext - data->ms;
+ else
+ pi_data->interval = 0;
+ } else {
+ if (data->lba_ext)
+ pi_data->interval = data->lba_ext - sizeof(struct nvme_64b_guard_pif);
+ else
+ pi_data->interval = data->ms - sizeof(struct nvme_64b_guard_pif);
+ }
+
+ if (io_u->ddir != DDIR_WRITE)
+ return;
+
+ while (lba_num < nlb) {
+ if (data->lba_ext)
+ pi = (struct nvme_64b_guard_pif *)(buf + pi_data->interval);
+ else
+ pi = (struct nvme_64b_guard_pif *)(md_buf + pi_data->interval);
+
+ if (opts->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
+ if (data->lba_ext) {
+ guard = fio_crc64_nvme(0, buf, pi_data->interval);
+ } else {
+ guard = fio_crc64_nvme(0, buf, data->lba_size);
+ guard = fio_crc64_nvme(guard, md_buf, pi_data->interval);
+ }
+ pi->guard = cpu_to_be64(guard);
+ }
+
+ if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
+ pi->apptag = cpu_to_be16(pi_data->apptag);
+
+ if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
+ switch (data->pi_type) {
+ case NVME_NS_DPS_PI_TYPE1:
+ case NVME_NS_DPS_PI_TYPE2:
+ put_unaligned_be48(slba + lba_num, pi->srtag);
+ break;
+ case NVME_NS_DPS_PI_TYPE3:
+ break;
+ }
+ }
+ if (data->lba_ext) {
+ buf += data->lba_ext;
+ } else {
+ buf += data->lba_size;
+ md_buf += data->ms;
+ }
+ lba_num++;
+ }
+}
+
+static int fio_nvme_verify_pi_64b_guard(struct nvme_data *data,
+ struct io_u *io_u)
+{
+ struct nvme_pi_data *pi_data = io_u->engine_data;
+ struct nvme_64b_guard_pif *pi;
+ struct fio_file *f = io_u->file;
+ unsigned char *buf = io_u->xfer_buf;
+ unsigned char *md_buf = io_u->mmap_data;
+ __u64 slba = get_slba(data, io_u);
+ __u64 ref, ref_exp, guard = 0;
+ __u32 nlb = get_nlb(data, io_u) + 1;
+ __u32 lba_num = 0;
+ __u16 unmask_app, unmask_app_exp;
+
+ while (lba_num < nlb) {
+ if (data->lba_ext)
+ pi = (struct nvme_64b_guard_pif *)(buf + pi_data->interval);
+ else
+ pi = (struct nvme_64b_guard_pif *)(md_buf + pi_data->interval);
+
+ if (data->pi_type == NVME_NS_DPS_PI_TYPE3) {
+ if (pi->apptag == NVME_PI_APP_DISABLE &&
+ fio_nvme_pi_ref_escape(pi->srtag))
+ goto next;
+ } else if (data->pi_type == NVME_NS_DPS_PI_TYPE1 ||
+ data->pi_type == NVME_NS_DPS_PI_TYPE2) {
+ if (pi->apptag == NVME_PI_APP_DISABLE)
+ goto next;
+ }
+
+ if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
+ if (data->lba_ext) {
+ guard = fio_crc64_nvme(0, buf, pi_data->interval);
+ } else {
+ guard = fio_crc64_nvme(0, buf, data->lba_size);
+ guard = fio_crc64_nvme(guard, md_buf, pi_data->interval);
+ }
+ if (be64_to_cpu((uint64_t)pi->guard) != guard) {
+ log_err("%s: Guard compare error: LBA: %llu Expected=%llx, Actual=%llx\n",
+ f->file_name, (unsigned long long)slba,
+ guard, be64_to_cpu((uint64_t)pi->guard));
+ return -EIO;
+ }
+ }
+
+ if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_APP) {
+ unmask_app = be16_to_cpu(pi->apptag) & pi_data->apptag_mask;
+ unmask_app_exp = pi_data->apptag & pi_data->apptag_mask;
+ if (unmask_app != unmask_app_exp) {
+ log_err("%s: APPTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
+ f->file_name, (unsigned long long)slba,
+ unmask_app_exp, unmask_app);
+ return -EIO;
+ }
+ }
+
+ if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
+ switch (data->pi_type) {
+ case NVME_NS_DPS_PI_TYPE1:
+ case NVME_NS_DPS_PI_TYPE2:
+ ref = get_unaligned_be48(pi->srtag);
+ ref_exp = (slba + lba_num) & ((1ULL << 48) - 1);
+ if (ref != ref_exp) {
+ log_err("%s: REFTAG compare error: LBA: %llu Expected=%llx, Actual=%llx\n",
+ f->file_name, (unsigned long long)slba,
+ ref_exp, ref);
+ return -EIO;
+ }
+ break;
+ case NVME_NS_DPS_PI_TYPE3:
+ break;
+ }
+ }
+next:
+ if (data->lba_ext) {
+ buf += data->lba_ext;
+ } else {
+ buf += data->lba_size;
+ md_buf += data->ms;
+ }
+ lba_num++;
+ }
+
+ return 0;
+}
void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
struct nvme_dsm_range *dsm)
{
if (data->pi_type && !(opts->io_flags & NVME_IO_PRINFO_PRACT)) {
if (data->guard_type == NVME_NVM_NS_16B_GUARD)
fio_nvme_generate_pi_16b_guard(data, io_u, opts);
+ else if (data->guard_type == NVME_NVM_NS_64B_GUARD)
+ fio_nvme_generate_pi_64b_guard(data, io_u, opts);
}
switch (data->pi_type) {
case NVME_NVM_NS_16B_GUARD:
ret = fio_nvme_verify_pi_16b_guard(data, io_u);
break;
+ case NVME_NVM_NS_64B_GUARD:
+ ret = fio_nvme_verify_pi_64b_guard(data, io_u);
+ break;
default:
break;
}