projects / fio.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'enable-dataplacement-while-replaying-io' of https://github.com/parkvibe...
[fio.git] / engines / nvme.c
CommitLineData
ca59c41c 1// SPDX-License-Identifier: GPL-2.0
b3d5e3fd
AK		 2/*
3 * nvme structure declarations and helper functions for the
4 * io_uring_cmd engine.
5 */
6
7#include "nvme.h"
5163f35e 8#include "../crc/crc-t10dif.h"
08371767 9#include "../crc/crc64.h"
b3d5e3fd 10
0d610785 11static inline __u64 get_slba(struct nvme_data *data, __u64 offset)
4885a6eb
VF		 12{
13 if (data->lba_ext)
0d610785
AK		 14 return offset / data->lba_ext;
15
16 return offset >> data->lba_shift;
4885a6eb
VF		 17}
18
0d610785 19static inline __u32 get_nlb(struct nvme_data *data, __u64 len)
4885a6eb
VF		 20{
21 if (data->lba_ext)
0d610785
AK		 22 return len / data->lba_ext - 1;
23
24 return (len >> data->lba_shift) - 1;
4885a6eb
VF		 25}
26
5163f35e
AK		 27static void fio_nvme_generate_pi_16b_guard(struct nvme_data *data,
28 struct io_u *io_u,
29 struct nvme_cmd_ext_io_opts *opts)
30{
31 struct nvme_pi_data *pi_data = io_u->engine_data;
32 struct nvme_16b_guard_pif *pi;
33 unsigned char *buf = io_u->xfer_buf;
34 unsigned char *md_buf = io_u->mmap_data;
0d610785
AK		 35 __u64 slba = get_slba(data, io_u->offset);
36 __u32 nlb = get_nlb(data, io_u->xfer_buflen) + 1;
5163f35e
AK		 37 __u32 lba_num = 0;
38 __u16 guard = 0;
39
40 if (data->pi_loc) {
41 if (data->lba_ext)
42 pi_data->interval = data->lba_ext - data->ms;
43 else
44 pi_data->interval = 0;
45 } else {
46 if (data->lba_ext)
47 pi_data->interval = data->lba_ext - sizeof(struct nvme_16b_guard_pif);
48 else
49 pi_data->interval = data->ms - sizeof(struct nvme_16b_guard_pif);
50 }
51
52 if (io_u->ddir != DDIR_WRITE)
53 return;
54
55 while (lba_num < nlb) {
56 if (data->lba_ext)
57 pi = (struct nvme_16b_guard_pif *)(buf + pi_data->interval);
58 else
59 pi = (struct nvme_16b_guard_pif *)(md_buf + pi_data->interval);
60
61 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
62 if (data->lba_ext) {
63 guard = fio_crc_t10dif(0, buf, pi_data->interval);
64 } else {
65 guard = fio_crc_t10dif(0, buf, data->lba_size);
66 guard = fio_crc_t10dif(guard, md_buf, pi_data->interval);
67 }
68 pi->guard = cpu_to_be16(guard);
69 }
70
71 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
72 pi->apptag = cpu_to_be16(pi_data->apptag);
73
74 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
75 switch (data->pi_type) {
76 case NVME_NS_DPS_PI_TYPE1:
77 case NVME_NS_DPS_PI_TYPE2:
78 pi->srtag = cpu_to_be32((__u32)slba + lba_num);
79 break;
80 case NVME_NS_DPS_PI_TYPE3:
81 break;
82 }
83 }
84 if (data->lba_ext) {
85 buf += data->lba_ext;
86 } else {
87 buf += data->lba_size;
88 md_buf += data->ms;
89 }
90 lba_num++;
91 }
92}
93
94static int fio_nvme_verify_pi_16b_guard(struct nvme_data *data,
95 struct io_u *io_u)
96{
97 struct nvme_pi_data *pi_data = io_u->engine_data;
98 struct nvme_16b_guard_pif *pi;
99 struct fio_file *f = io_u->file;
100 unsigned char *buf = io_u->xfer_buf;
101 unsigned char *md_buf = io_u->mmap_data;
0d610785
AK		 102 __u64 slba = get_slba(data, io_u->offset);
103 __u32 nlb = get_nlb(data, io_u->xfer_buflen) + 1;
5163f35e
AK		 104 __u32 lba_num = 0;
105 __u16 unmask_app, unmask_app_exp, guard = 0;
106
107 while (lba_num < nlb) {
108 if (data->lba_ext)
109 pi = (struct nvme_16b_guard_pif *)(buf + pi_data->interval);
110 else
111 pi = (struct nvme_16b_guard_pif *)(md_buf + pi_data->interval);
112
113 if (data->pi_type == NVME_NS_DPS_PI_TYPE3) {
114 if (pi->apptag == NVME_PI_APP_DISABLE &&
115 pi->srtag == NVME_PI_REF_DISABLE)
116 goto next;
117 } else if (data->pi_type == NVME_NS_DPS_PI_TYPE1 ||
118 data->pi_type == NVME_NS_DPS_PI_TYPE2) {
119 if (pi->apptag == NVME_PI_APP_DISABLE)
120 goto next;
121 }
122
123 if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
124 if (data->lba_ext) {
125 guard = fio_crc_t10dif(0, buf, pi_data->interval);
126 } else {
127 guard = fio_crc_t10dif(0, buf, data->lba_size);
128 guard = fio_crc_t10dif(guard, md_buf, pi_data->interval);
129 }
130 if (be16_to_cpu(pi->guard) != guard) {
131 log_err("%s: Guard compare error: LBA: %llu Expected=%x, Actual=%x\n",
132 f->file_name, (unsigned long long)slba,
133 guard, be16_to_cpu(pi->guard));
134 return -EIO;
135 }
136 }
137
138 if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_APP) {
139 unmask_app = be16_to_cpu(pi->apptag) & pi_data->apptag_mask;
140 unmask_app_exp = pi_data->apptag & pi_data->apptag_mask;
141 if (unmask_app != unmask_app_exp) {
142 log_err("%s: APPTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
143 f->file_name, (unsigned long long)slba,
144 unmask_app_exp, unmask_app);
145 return -EIO;
146 }
147 }
148
149 if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
150 switch (data->pi_type) {
151 case NVME_NS_DPS_PI_TYPE1:
152 case NVME_NS_DPS_PI_TYPE2:
153 if (be32_to_cpu(pi->srtag) !=
154 ((__u32)slba + lba_num)) {
155 log_err("%s: REFTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
156 f->file_name, (unsigned long long)slba,
157 (__u32)slba + lba_num,
158 be32_to_cpu(pi->srtag));
159 return -EIO;
160 }
161 break;
162 case NVME_NS_DPS_PI_TYPE3:
163 break;
164 }
165 }
166next:
167 if (data->lba_ext) {
168 buf += data->lba_ext;
169 } else {
170 buf += data->lba_size;
171 md_buf += data->ms;
172 }
173 lba_num++;
174 }
175
176 return 0;
177}
178
08371767
AK		 179static void fio_nvme_generate_pi_64b_guard(struct nvme_data *data,
180 struct io_u *io_u,
181 struct nvme_cmd_ext_io_opts *opts)
182{
183 struct nvme_pi_data *pi_data = io_u->engine_data;
184 struct nvme_64b_guard_pif *pi;
185 unsigned char *buf = io_u->xfer_buf;
186 unsigned char *md_buf = io_u->mmap_data;
187 uint64_t guard = 0;
0d610785
AK		 188 __u64 slba = get_slba(data, io_u->offset);
189 __u32 nlb = get_nlb(data, io_u->xfer_buflen) + 1;
08371767
AK		 190 __u32 lba_num = 0;
191
192 if (data->pi_loc) {
193 if (data->lba_ext)
194 pi_data->interval = data->lba_ext - data->ms;
195 else
196 pi_data->interval = 0;
197 } else {
198 if (data->lba_ext)
199 pi_data->interval = data->lba_ext - sizeof(struct nvme_64b_guard_pif);
200 else
201 pi_data->interval = data->ms - sizeof(struct nvme_64b_guard_pif);
202 }
203
204 if (io_u->ddir != DDIR_WRITE)
205 return;
206
207 while (lba_num < nlb) {
208 if (data->lba_ext)
209 pi = (struct nvme_64b_guard_pif *)(buf + pi_data->interval);
210 else
211 pi = (struct nvme_64b_guard_pif *)(md_buf + pi_data->interval);
212
213 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
214 if (data->lba_ext) {
215 guard = fio_crc64_nvme(0, buf, pi_data->interval);
216 } else {
217 guard = fio_crc64_nvme(0, buf, data->lba_size);
218 guard = fio_crc64_nvme(guard, md_buf, pi_data->interval);
219 }
220 pi->guard = cpu_to_be64(guard);
221 }
222
223 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
224 pi->apptag = cpu_to_be16(pi_data->apptag);
225
226 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
227 switch (data->pi_type) {
228 case NVME_NS_DPS_PI_TYPE1:
229 case NVME_NS_DPS_PI_TYPE2:
230 put_unaligned_be48(slba + lba_num, pi->srtag);
231 break;
232 case NVME_NS_DPS_PI_TYPE3:
233 break;
234 }
235 }
236 if (data->lba_ext) {
237 buf += data->lba_ext;
238 } else {
239 buf += data->lba_size;
240 md_buf += data->ms;
241 }
242 lba_num++;
243 }
244}
245
246static int fio_nvme_verify_pi_64b_guard(struct nvme_data *data,
247 struct io_u *io_u)
248{
249 struct nvme_pi_data *pi_data = io_u->engine_data;
250 struct nvme_64b_guard_pif *pi;
251 struct fio_file *f = io_u->file;
252 unsigned char *buf = io_u->xfer_buf;
253 unsigned char *md_buf = io_u->mmap_data;
0d610785 254 __u64 slba = get_slba(data, io_u->offset);
08371767 255 __u64 ref, ref_exp, guard = 0;
0d610785 256 __u32 nlb = get_nlb(data, io_u->xfer_buflen) + 1;
08371767
AK		 257 __u32 lba_num = 0;
258 __u16 unmask_app, unmask_app_exp;
259
260 while (lba_num < nlb) {
261 if (data->lba_ext)
262 pi = (struct nvme_64b_guard_pif *)(buf + pi_data->interval);
263 else
264 pi = (struct nvme_64b_guard_pif *)(md_buf + pi_data->interval);
265
266 if (data->pi_type == NVME_NS_DPS_PI_TYPE3) {
267 if (pi->apptag == NVME_PI_APP_DISABLE &&
268 fio_nvme_pi_ref_escape(pi->srtag))
269 goto next;
270 } else if (data->pi_type == NVME_NS_DPS_PI_TYPE1 ||
271 data->pi_type == NVME_NS_DPS_PI_TYPE2) {
272 if (pi->apptag == NVME_PI_APP_DISABLE)
273 goto next;
274 }
275
276 if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
277 if (data->lba_ext) {
278 guard = fio_crc64_nvme(0, buf, pi_data->interval);
279 } else {
280 guard = fio_crc64_nvme(0, buf, data->lba_size);
281 guard = fio_crc64_nvme(guard, md_buf, pi_data->interval);
282 }
283 if (be64_to_cpu((uint64_t)pi->guard) != guard) {
284 log_err("%s: Guard compare error: LBA: %llu Expected=%llx, Actual=%llx\n",
285 f->file_name, (unsigned long long)slba,
286 guard, be64_to_cpu((uint64_t)pi->guard));
287 return -EIO;
288 }
289 }
290
291 if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_APP) {
292 unmask_app = be16_to_cpu(pi->apptag) & pi_data->apptag_mask;
293 unmask_app_exp = pi_data->apptag & pi_data->apptag_mask;
294 if (unmask_app != unmask_app_exp) {
295 log_err("%s: APPTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
296 f->file_name, (unsigned long long)slba,
297 unmask_app_exp, unmask_app);
298 return -EIO;
299 }
300 }
301
302 if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
303 switch (data->pi_type) {
304 case NVME_NS_DPS_PI_TYPE1:
305 case NVME_NS_DPS_PI_TYPE2:
306 ref = get_unaligned_be48(pi->srtag);
307 ref_exp = (slba + lba_num) & ((1ULL << 48) - 1);
308 if (ref != ref_exp) {
309 log_err("%s: REFTAG compare error: LBA: %llu Expected=%llx, Actual=%llx\n",
310 f->file_name, (unsigned long long)slba,
311 ref_exp, ref);
312 return -EIO;
313 }
314 break;
315 case NVME_NS_DPS_PI_TYPE3:
316 break;
317 }
318 }
319next:
320 if (data->lba_ext) {
321 buf += data->lba_ext;
322 } else {
323 buf += data->lba_size;
324 md_buf += data->ms;
325 }
326 lba_num++;
327 }
328
329 return 0;
330}
4885a6eb 331void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
5d4ee0de 332 struct nvme_dsm *dsm)
4885a6eb
VF		 333{
334 struct nvme_data *data = FILE_ENG_DATA(io_u->file);
5d4ee0de
AK		 335 struct trim_range *range;
336 uint8_t *buf_point;
337 int i;
4885a6eb
VF		 338
339 cmd->opcode = nvme_cmd_dsm;
340 cmd->nsid = data->nsid;
4885a6eb 341 cmd->cdw11 = NVME_ATTRIBUTE_DEALLOCATE;
5d4ee0de
AK		 342 cmd->addr = (__u64) (uintptr_t) (&dsm->range[0]);
343
344 if (dsm->nr_ranges == 1) {
345 dsm->range[0].slba = get_slba(data, io_u->offset);
346 /* nlb is a 1-based value for deallocate */
347 dsm->range[0].nlb = get_nlb(data, io_u->xfer_buflen) + 1;
348 cmd->cdw10 = 0;
349 cmd->data_len = sizeof(struct nvme_dsm_range);
350 } else {
351 buf_point = io_u->xfer_buf;
352 for (i = 0; i < io_u->number_trim; i++) {
353 range = (struct trim_range *)buf_point;
354 dsm->range[i].slba = get_slba(data, range->start);
355 /* nlb is a 1-based value for deallocate */
356 dsm->range[i].nlb = get_nlb(data, range->len) + 1;
357 buf_point += sizeof(struct trim_range);
358 }
359 cmd->cdw10 = io_u->number_trim - 1;
360 cmd->data_len = io_u->number_trim * sizeof(struct nvme_dsm_range);
361 }
4885a6eb
VF		 362}
363
b3d5e3fd 364int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
55e14d73
MI		 365 struct iovec *iov, struct nvme_dsm *dsm,
366 unsigned int cdw12_flags)
b3d5e3fd
AK		 367{
368 struct nvme_data *data = FILE_ENG_DATA(io_u->file);
369 __u64 slba;
370 __u32 nlb;
371
372 memset(cmd, 0, sizeof(struct nvme_uring_cmd));
373
4885a6eb
VF		 374 switch (io_u->ddir) {
375 case DDIR_READ:
b3d5e3fd 376 cmd->opcode = nvme_cmd_read;
4885a6eb
VF		 377 break;
378 case DDIR_WRITE:
b3d5e3fd 379 cmd->opcode = nvme_cmd_write;
4885a6eb
VF		 380 break;
381 case DDIR_TRIM:
382 fio_nvme_uring_cmd_trim_prep(cmd, io_u, dsm);
383 return 0;
384 default:
b3d5e3fd 385 return -ENOTSUP;
345fa8fd 386 }
b3d5e3fd 387
0d610785
AK		 388 slba = get_slba(data, io_u->offset);
389 nlb = get_nlb(data, io_u->xfer_buflen);
4885a6eb 390
b3d5e3fd
AK		 391 /* cdw10 and cdw11 represent starting lba */
392 cmd->cdw10 = slba & 0xffffffff;
393 cmd->cdw11 = slba >> 32;
394 /* cdw12 represent number of lba's for read/write */
55e14d73 395 cmd->cdw12 = nlb | (io_u->dtype << 20) | cdw12_flags;
a7e8aae0 396 cmd->cdw13 = io_u->dspec << 16;
b3d5e3fd
AK		 397 if (iov) {
398 iov->iov_base = io_u->xfer_buf;
399 iov->iov_len = io_u->xfer_buflen;
400 cmd->addr = (__u64)(uintptr_t)iov;
401 cmd->data_len = 1;
402 } else {
403 cmd->addr = (__u64)(uintptr_t)io_u->xfer_buf;
404 cmd->data_len = io_u->xfer_buflen;
405 }
2d6451c9
AK		 406 if (data->lba_shift && data->ms) {
407 cmd->metadata = (__u64)(uintptr_t)io_u->mmap_data;
408 cmd->metadata_len = (nlb + 1) * data->ms;
409 }
b3d5e3fd
AK		 410 cmd->nsid = data->nsid;
411 return 0;
412}
413
3ee8311a
AK		 414void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
415 struct nvme_cmd_ext_io_opts *opts)
416{
417 struct nvme_data *data = FILE_ENG_DATA(io_u->file);
418 __u64 slba;
419
0d610785 420 slba = get_slba(data, io_u->offset);
3ee8311a
AK		 421 cmd->cdw12 |= opts->io_flags;
422
5163f35e
AK		 423 if (data->pi_type && !(opts->io_flags & NVME_IO_PRINFO_PRACT)) {
424 if (data->guard_type == NVME_NVM_NS_16B_GUARD)
425 fio_nvme_generate_pi_16b_guard(data, io_u, opts);
08371767
AK		 426 else if (data->guard_type == NVME_NVM_NS_64B_GUARD)
427 fio_nvme_generate_pi_64b_guard(data, io_u, opts);
5163f35e
AK		 428 }
429
3ee8311a
AK		 430 switch (data->pi_type) {
431 case NVME_NS_DPS_PI_TYPE1:
432 case NVME_NS_DPS_PI_TYPE2:
433 switch (data->guard_type) {
434 case NVME_NVM_NS_16B_GUARD:
e4a9812d
AK		 435 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF)
436 cmd->cdw14 = (__u32)slba;
3ee8311a
AK		 437 break;
438 case NVME_NVM_NS_64B_GUARD:
e4a9812d
AK		 439 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
440 cmd->cdw14 = (__u32)slba;
441 cmd->cdw3 = ((slba >> 32) & 0xffff);
442 }
3ee8311a
AK		 443 break;
444 default:
445 break;
446 }
e4a9812d
AK		 447 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
448 cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
3ee8311a
AK		 449 break;
450 case NVME_NS_DPS_PI_TYPE3:
e4a9812d
AK		 451 if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
452 cmd->cdw15 = (opts->apptag_mask << 16 | opts->apptag);
3ee8311a
AK		 453 break;
454 case NVME_NS_DPS_PI_NONE:
455 break;
456 }
457}
458
5163f35e
AK		 459int fio_nvme_pi_verify(struct nvme_data *data, struct io_u *io_u)
460{
461 int ret = 0;
462
463 switch (data->guard_type) {
464 case NVME_NVM_NS_16B_GUARD:
465 ret = fio_nvme_verify_pi_16b_guard(data, io_u);
466 break;
08371767
AK		 467 case NVME_NVM_NS_64B_GUARD:
468 ret = fio_nvme_verify_pi_64b_guard(data, io_u);
469 break;
5163f35e
AK		 470 default:
471 break;
472 }
473
474 return ret;
475}
476
b3d5e3fd
AK		 477static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
478 enum nvme_csi csi, void *data)
479{
480 struct nvme_passthru_cmd cmd = {
481 .opcode = nvme_admin_identify,
482 .nsid = nsid,
483 .addr = (__u64)(uintptr_t)data,
484 .data_len = NVME_IDENTIFY_DATA_SIZE,
485 .cdw10 = cns,
486 .cdw11 = csi << NVME_IDENTIFY_CSI_SHIFT,
487 .timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
488 };
489
490 return ioctl(fd, NVME_IOCTL_ADMIN_CMD, &cmd);
491}
492
3ee8311a
AK		 493int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
494 struct nvme_data *data)
b3d5e3fd
AK		 495{
496 struct nvme_id_ns ns;
3ee8311a
AK		 497 struct nvme_id_ctrl ctrl;
498 struct nvme_nvm_id_ns nvm_ns;
37a0881f 499 int namespace_id;
b3d5e3fd 500 int fd, err;
3ee8311a 501 __u32 format_idx, elbaf;
b3d5e3fd
AK		 502
503 if (f->filetype != FIO_TYPE_CHAR) {
504 log_err("ioengine io_uring_cmd only works with nvme ns "
505 "generic char devices (/dev/ngXnY)\n");
506 return 1;
507 }
508
509 fd = open(f->file_name, O_RDONLY);
510 if (fd < 0)
511 return -errno;
512
513 namespace_id = ioctl(fd, NVME_IOCTL_ID);
514 if (namespace_id < 0) {
af10f514 515 err = -errno;
345fa8fd
AK		 516 log_err("%s: failed to fetch namespace-id\n", f->file_name);
517 goto out;
b3d5e3fd
AK		 518 }
519
3ee8311a
AK		 520 err = nvme_identify(fd, 0, NVME_IDENTIFY_CNS_CTRL, NVME_CSI_NVM, &ctrl);
521 if (err) {
522 log_err("%s: failed to fetch identify ctrl\n", f->file_name);
523 goto out;
524 }
525
b3d5e3fd
AK		 526 /*
527 * Identify namespace to get namespace-id, namespace size in LBA's
528 * and LBA data size.
529 */
530 err = nvme_identify(fd, namespace_id, NVME_IDENTIFY_CNS_NS,
531 NVME_CSI_NVM, &ns);
532 if (err) {
345fa8fd
AK		 533 log_err("%s: failed to fetch identify namespace\n",
534 f->file_name);
3ee8311a 535 goto out;
b3d5e3fd
AK		 536 }
537
e7e5023b 538 data->nsid = namespace_id;
01a7d384
AK		 539
540 /*
541 * 16 or 64 as maximum number of supported LBA formats.
542 * From flbas bit 0-3 indicates lsb and bit 5-6 indicates msb
543 * of the format index used to format the namespace.
544 */
545 if (ns.nlbaf < 16)
546 format_idx = ns.flbas & 0xf;
547 else
548 format_idx = (ns.flbas & 0xf) + (((ns.flbas >> 5) & 0x3) << 4);
549
e7e5023b 550 data->lba_size = 1 << ns.lbaf[format_idx].ds;
2d6451c9 551 data->ms = le16_to_cpu(ns.lbaf[format_idx].ms);
345fa8fd 552
3ee8311a
AK		 553 /* Check for end to end data protection support */
554 if (data->ms && (ns.dps & NVME_NS_DPS_PI_MASK))
555 data->pi_type = (ns.dps & NVME_NS_DPS_PI_MASK);
556
557 if (!data->pi_type)
558 goto check_elba;
559
560 if (ctrl.ctratt & NVME_CTRL_CTRATT_ELBAS) {
561 err = nvme_identify(fd, namespace_id, NVME_IDENTIFY_CNS_CSI_NS,
562 NVME_CSI_NVM, &nvm_ns);
563 if (err) {
564 log_err("%s: failed to fetch identify nvm namespace\n",
565 f->file_name);
566 goto out;
567 }
568
569 elbaf = le32_to_cpu(nvm_ns.elbaf[format_idx]);
570
571 /* Currently we don't support storage tags */
572 if (elbaf & NVME_ID_NS_NVM_STS_MASK) {
573 log_err("%s: Storage tag not supported\n",
574 f->file_name);
575 err = -ENOTSUP;
576 goto out;
577 }
578
579 data->guard_type = (elbaf >> NVME_ID_NS_NVM_GUARD_SHIFT) &
580 NVME_ID_NS_NVM_GUARD_MASK;
581
582 /* No 32 bit guard, as storage tag is mandatory for it */
583 switch (data->guard_type) {
584 case NVME_NVM_NS_16B_GUARD:
585 data->pi_size = sizeof(struct nvme_16b_guard_pif);
586 break;
587 case NVME_NVM_NS_64B_GUARD:
588 data->pi_size = sizeof(struct nvme_64b_guard_pif);
589 break;
590 default:
591 break;
592 }
593 } else {
594 data->guard_type = NVME_NVM_NS_16B_GUARD;
595 data->pi_size = sizeof(struct nvme_16b_guard_pif);
596 }
597
598 /*
599 * when PRACT bit is set to 1, and metadata size is equal to protection
600 * information size, controller inserts and removes PI for write and
601 * read commands respectively.
602 */
603 if (pi_act && data->ms == data->pi_size)
604 data->ms = 0;
605
606 data->pi_loc = (ns.dps & NVME_NS_DPS_PI_FIRST);
607
608check_elba:
345fa8fd 609 /*
345fa8fd
AK		 610 * Bit 4 for flbas indicates if metadata is transferred at the end of
611 * logical block creating an extended LBA.
612 */
2d6451c9 613 if (data->ms && ((ns.flbas >> 4) & 0x1))
e7e5023b
AK		 614 data->lba_ext = data->lba_size + data->ms;
615 else
616 data->lba_shift = ilog2(data->lba_size);
617
b3d5e3fd
AK		 618 *nlba = ns.nsze;
619
345fa8fd 620out:
b3d5e3fd 621 close(fd);
345fa8fd 622 return err;
b3d5e3fd 623}
3d05e0ff
AK		 624
625int fio_nvme_get_zoned_model(struct thread_data *td, struct fio_file *f,
626 enum zbd_zoned_model *model)
627{
628 struct nvme_data *data = FILE_ENG_DATA(f);
629 struct nvme_id_ns ns;
630 struct nvme_passthru_cmd cmd;
631 int fd, ret = 0;
632
633 if (f->filetype != FIO_TYPE_CHAR)
634 return -EINVAL;
635
636 /* File is not yet opened */
637 fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
638 if (fd < 0)
639 return -errno;
640
641 /* Using nvme_id_ns for data as sizes are same */
642 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_CTRL,
643 NVME_CSI_ZNS, &ns);
644 if (ret) {
645 *model = ZBD_NONE;
646 goto out;
647 }
648
649 memset(&cmd, 0, sizeof(struct nvme_passthru_cmd));
650
651 /* Using nvme_id_ns for data as sizes are same */
652 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
653 NVME_CSI_ZNS, &ns);
654 if (ret) {
655 *model = ZBD_NONE;
656 goto out;
657 }
658
659 *model = ZBD_HOST_MANAGED;
660out:
661 close(fd);
662 return 0;
663}
664
665static int nvme_report_zones(int fd, __u32 nsid, __u64 slba, __u32 zras_feat,
666 __u32 data_len, void *data)
667{
668 struct nvme_passthru_cmd cmd = {
669 .opcode = nvme_zns_cmd_mgmt_recv,
670 .nsid = nsid,
671 .addr = (__u64)(uintptr_t)data,
672 .data_len = data_len,
673 .cdw10 = slba & 0xffffffff,
674 .cdw11 = slba >> 32,
675 .cdw12 = (data_len >> 2) - 1,
676 .cdw13 = NVME_ZNS_ZRA_REPORT_ZONES | zras_feat,
677 .timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
678 };
679
680 return ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
681}
682
683int fio_nvme_report_zones(struct thread_data *td, struct fio_file *f,
684 uint64_t offset, struct zbd_zone *zbdz,
685 unsigned int nr_zones)
686{
687 struct nvme_data *data = FILE_ENG_DATA(f);
688 struct nvme_zone_report *zr;
689 struct nvme_zns_id_ns zns_ns;
690 struct nvme_id_ns ns;
691 unsigned int i = 0, j, zones_fetched = 0;
692 unsigned int max_zones, zones_chunks = 1024;
693 int fd, ret = 0;
694 __u32 zr_len;
695 __u64 zlen;
696
697 /* File is not yet opened */
698 fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
699 if (fd < 0)
700 return -errno;
701
702 zones_fetched = 0;
703 zr_len = sizeof(*zr) + (zones_chunks * sizeof(struct nvme_zns_desc));
704 zr = calloc(1, zr_len);
3efcb23f
JA		 705 if (!zr) {
706 close(fd);
3d05e0ff 707 return -ENOMEM;
3efcb23f 708 }
3d05e0ff
AK		 709
710 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_NS,
711 NVME_CSI_NVM, &ns);
712 if (ret) {
713 log_err("%s: nvme_identify_ns failed, err=%d\n", f->file_name,
714 ret);
715 goto out;
716 }
717
718 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
719 NVME_CSI_ZNS, &zns_ns);
720 if (ret) {
721 log_err("%s: nvme_zns_identify_ns failed, err=%d\n",
722 f->file_name, ret);
723 goto out;
724 }
725 zlen = zns_ns.lbafe[ns.flbas & 0x0f].zsze << data->lba_shift;
726
727 max_zones = (f->real_file_size - offset) / zlen;
728 if (max_zones < nr_zones)
729 nr_zones = max_zones;
730
731 if (nr_zones < zones_chunks)
732 zones_chunks = nr_zones;
733
734 while (zones_fetched < nr_zones) {
735 if (zones_fetched + zones_chunks >= nr_zones) {
736 zones_chunks = nr_zones - zones_fetched;
737 zr_len = sizeof(*zr) + (zones_chunks * sizeof(struct nvme_zns_desc));
738 }
739 ret = nvme_report_zones(fd, data->nsid, offset >> data->lba_shift,
740 NVME_ZNS_ZRAS_FEAT_ERZ, zr_len, (void *)zr);
741 if (ret) {
742 log_err("%s: nvme_zns_report_zones failed, err=%d\n",
743 f->file_name, ret);
744 goto out;
745 }
746
747 /* Transform the zone-report */
748 for (j = 0; j < zr->nr_zones; j++, i++) {
749 struct nvme_zns_desc *desc = (struct nvme_zns_desc *)&(zr->entries[j]);
750
751 zbdz[i].start = desc->zslba << data->lba_shift;
752 zbdz[i].len = zlen;
753 zbdz[i].wp = desc->wp << data->lba_shift;
754 zbdz[i].capacity = desc->zcap << data->lba_shift;
755
756 /* Zone Type is stored in first 4 bits. */
757 switch (desc->zt & 0x0f) {
758 case NVME_ZONE_TYPE_SEQWRITE_REQ:
759 zbdz[i].type = ZBD_ZONE_TYPE_SWR;
760 break;
761 default:
762 log_err("%s: invalid type for zone at offset %llu.\n",
2fa0ab21 763 f->file_name, (unsigned long long) desc->zslba);
3d05e0ff
AK		 764 ret = -EIO;
765 goto out;
766 }
767
768 /* Zone State is stored in last 4 bits. */
769 switch (desc->zs >> 4) {
770 case NVME_ZNS_ZS_EMPTY:
771 zbdz[i].cond = ZBD_ZONE_COND_EMPTY;
772 break;
773 case NVME_ZNS_ZS_IMPL_OPEN:
774 zbdz[i].cond = ZBD_ZONE_COND_IMP_OPEN;
775 break;
776 case NVME_ZNS_ZS_EXPL_OPEN:
777 zbdz[i].cond = ZBD_ZONE_COND_EXP_OPEN;
778 break;
779 case NVME_ZNS_ZS_CLOSED:
780 zbdz[i].cond = ZBD_ZONE_COND_CLOSED;
781 break;
782 case NVME_ZNS_ZS_FULL:
783 zbdz[i].cond = ZBD_ZONE_COND_FULL;
784 break;
785 case NVME_ZNS_ZS_READ_ONLY:
786 case NVME_ZNS_ZS_OFFLINE:
787 default:
788 /* Treat all these conditions as offline (don't use!) */
789 zbdz[i].cond = ZBD_ZONE_COND_OFFLINE;
790 zbdz[i].wp = zbdz[i].start;
791 }
792 }
793 zones_fetched += zr->nr_zones;
794 offset += zr->nr_zones * zlen;
795 }
796
797 ret = zones_fetched;
798out:
799 free(zr);
800 close(fd);
801
802 return ret;
803}
804
805int fio_nvme_reset_wp(struct thread_data *td, struct fio_file *f,
806 uint64_t offset, uint64_t length)
807{
808 struct nvme_data *data = FILE_ENG_DATA(f);
809 unsigned int nr_zones;
810 unsigned long long zslba;
811 int i, fd, ret = 0;
812
813 /* If the file is not yet opened, open it for this function. */
814 fd = f->fd;
815 if (fd < 0) {
816 fd = open(f->file_name, O_RDWR | O_LARGEFILE);
817 if (fd < 0)
818 return -errno;
819 }
820
821 zslba = offset >> data->lba_shift;
822 nr_zones = (length + td->o.zone_size - 1) / td->o.zone_size;
823
824 for (i = 0; i < nr_zones; i++, zslba += (td->o.zone_size >> data->lba_shift)) {
825 struct nvme_passthru_cmd cmd = {
826 .opcode = nvme_zns_cmd_mgmt_send,
827 .nsid = data->nsid,
828 .cdw10 = zslba & 0xffffffff,
829 .cdw11 = zslba >> 32,
830 .cdw13 = NVME_ZNS_ZSA_RESET,
831 .addr = (__u64)(uintptr_t)NULL,
832 .data_len = 0,
833 .timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
834 };
835
836 ret = ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
837 }
838
839 if (f->fd < 0)
840 close(fd);
841 return -ret;
842}
843
844int fio_nvme_get_max_open_zones(struct thread_data *td, struct fio_file *f,
845 unsigned int *max_open_zones)
846{
847 struct nvme_data *data = FILE_ENG_DATA(f);
848 struct nvme_zns_id_ns zns_ns;
849 int fd, ret = 0;
850
851 fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
852 if (fd < 0)
853 return -errno;
854
855 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
856 NVME_CSI_ZNS, &zns_ns);
857 if (ret) {
858 log_err("%s: nvme_zns_identify_ns failed, err=%d\n",
859 f->file_name, ret);
860 goto out;
861 }
862
863 *max_open_zones = zns_ns.mor + 1;
864out:
865 close(fd);
866 return ret;
867}
a7e8aae0
KB		 868
869static inline int nvme_fdp_reclaim_unit_handle_status(int fd, __u32 nsid,
870 __u32 data_len, void *data)
871{
872 struct nvme_passthru_cmd cmd = {
873 .opcode = nvme_cmd_io_mgmt_recv,
874 .nsid = nsid,
875 .addr = (__u64)(uintptr_t)data,
876 .data_len = data_len,
877 .cdw10 = 1,
878 .cdw11 = (data_len >> 2) - 1,
879 };
880
881 return ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
882}
883
884int fio_nvme_iomgmt_ruhs(struct thread_data *td, struct fio_file *f,
885 struct nvme_fdp_ruh_status *ruhs, __u32 bytes)
886{
887 struct nvme_data *data = FILE_ENG_DATA(f);
888 int fd, ret;
889
890 fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
891 if (fd < 0)
892 return -errno;
893
894 ret = nvme_fdp_reclaim_unit_handle_status(fd, data->nsid, bytes, ruhs);
895 if (ret) {
896 log_err("%s: nvme_fdp_reclaim_unit_handle_status failed, err=%d\n",
897 f->file_name, ret);
898 errno = ENOTSUP;
899 } else
900 errno = 0;
901
af10f514 902 ret = -errno;
a7e8aae0 903 close(fd);
af10f514 904 return ret;
a7e8aae0 905}
fio - Flexible IO Tester