Read stats for backlog verifies not reported for time-expired workloads
[fio.git] / engines / nvme.c
CommitLineData
b3d5e3fd
AK
1/*
2 * nvme structure declarations and helper functions for the
3 * io_uring_cmd engine.
4 */
5
6#include "nvme.h"
7
8int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
9 struct iovec *iov)
10{
11 struct nvme_data *data = FILE_ENG_DATA(io_u->file);
12 __u64 slba;
13 __u32 nlb;
14
15 memset(cmd, 0, sizeof(struct nvme_uring_cmd));
16
17 if (io_u->ddir == DDIR_READ)
18 cmd->opcode = nvme_cmd_read;
19 else if (io_u->ddir == DDIR_WRITE)
20 cmd->opcode = nvme_cmd_write;
21 else
22 return -ENOTSUP;
23
24 slba = io_u->offset >> data->lba_shift;
25 nlb = (io_u->xfer_buflen >> data->lba_shift) - 1;
26
27 /* cdw10 and cdw11 represent starting lba */
28 cmd->cdw10 = slba & 0xffffffff;
29 cmd->cdw11 = slba >> 32;
30 /* cdw12 represent number of lba's for read/write */
31 cmd->cdw12 = nlb;
32 if (iov) {
33 iov->iov_base = io_u->xfer_buf;
34 iov->iov_len = io_u->xfer_buflen;
35 cmd->addr = (__u64)(uintptr_t)iov;
36 cmd->data_len = 1;
37 } else {
38 cmd->addr = (__u64)(uintptr_t)io_u->xfer_buf;
39 cmd->data_len = io_u->xfer_buflen;
40 }
41 cmd->nsid = data->nsid;
42 return 0;
43}
44
45static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
46 enum nvme_csi csi, void *data)
47{
48 struct nvme_passthru_cmd cmd = {
49 .opcode = nvme_admin_identify,
50 .nsid = nsid,
51 .addr = (__u64)(uintptr_t)data,
52 .data_len = NVME_IDENTIFY_DATA_SIZE,
53 .cdw10 = cns,
54 .cdw11 = csi << NVME_IDENTIFY_CSI_SHIFT,
55 .timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
56 };
57
58 return ioctl(fd, NVME_IOCTL_ADMIN_CMD, &cmd);
59}
60
61int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
62 __u64 *nlba)
63{
64 struct nvme_id_ns ns;
37a0881f 65 int namespace_id;
b3d5e3fd
AK
66 int fd, err;
67
68 if (f->filetype != FIO_TYPE_CHAR) {
69 log_err("ioengine io_uring_cmd only works with nvme ns "
70 "generic char devices (/dev/ngXnY)\n");
71 return 1;
72 }
73
74 fd = open(f->file_name, O_RDONLY);
75 if (fd < 0)
76 return -errno;
77
78 namespace_id = ioctl(fd, NVME_IOCTL_ID);
79 if (namespace_id < 0) {
80 log_err("failed to fetch namespace-id");
81 close(fd);
82 return -errno;
83 }
84
85 /*
86 * Identify namespace to get namespace-id, namespace size in LBA's
87 * and LBA data size.
88 */
89 err = nvme_identify(fd, namespace_id, NVME_IDENTIFY_CNS_NS,
90 NVME_CSI_NVM, &ns);
91 if (err) {
92 log_err("failed to fetch identify namespace\n");
93 close(fd);
94 return err;
95 }
96
97 *nsid = namespace_id;
98 *lba_sz = 1 << ns.lbaf[(ns.flbas & 0x0f)].ds;
99 *nlba = ns.nsze;
100
101 close(fd);
102 return 0;
103}
3d05e0ff
AK
104
105int fio_nvme_get_zoned_model(struct thread_data *td, struct fio_file *f,
106 enum zbd_zoned_model *model)
107{
108 struct nvme_data *data = FILE_ENG_DATA(f);
109 struct nvme_id_ns ns;
110 struct nvme_passthru_cmd cmd;
111 int fd, ret = 0;
112
113 if (f->filetype != FIO_TYPE_CHAR)
114 return -EINVAL;
115
116 /* File is not yet opened */
117 fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
118 if (fd < 0)
119 return -errno;
120
121 /* Using nvme_id_ns for data as sizes are same */
122 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_CTRL,
123 NVME_CSI_ZNS, &ns);
124 if (ret) {
125 *model = ZBD_NONE;
126 goto out;
127 }
128
129 memset(&cmd, 0, sizeof(struct nvme_passthru_cmd));
130
131 /* Using nvme_id_ns for data as sizes are same */
132 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
133 NVME_CSI_ZNS, &ns);
134 if (ret) {
135 *model = ZBD_NONE;
136 goto out;
137 }
138
139 *model = ZBD_HOST_MANAGED;
140out:
141 close(fd);
142 return 0;
143}
144
145static int nvme_report_zones(int fd, __u32 nsid, __u64 slba, __u32 zras_feat,
146 __u32 data_len, void *data)
147{
148 struct nvme_passthru_cmd cmd = {
149 .opcode = nvme_zns_cmd_mgmt_recv,
150 .nsid = nsid,
151 .addr = (__u64)(uintptr_t)data,
152 .data_len = data_len,
153 .cdw10 = slba & 0xffffffff,
154 .cdw11 = slba >> 32,
155 .cdw12 = (data_len >> 2) - 1,
156 .cdw13 = NVME_ZNS_ZRA_REPORT_ZONES | zras_feat,
157 .timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
158 };
159
160 return ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
161}
162
163int fio_nvme_report_zones(struct thread_data *td, struct fio_file *f,
164 uint64_t offset, struct zbd_zone *zbdz,
165 unsigned int nr_zones)
166{
167 struct nvme_data *data = FILE_ENG_DATA(f);
168 struct nvme_zone_report *zr;
169 struct nvme_zns_id_ns zns_ns;
170 struct nvme_id_ns ns;
171 unsigned int i = 0, j, zones_fetched = 0;
172 unsigned int max_zones, zones_chunks = 1024;
173 int fd, ret = 0;
174 __u32 zr_len;
175 __u64 zlen;
176
177 /* File is not yet opened */
178 fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
179 if (fd < 0)
180 return -errno;
181
182 zones_fetched = 0;
183 zr_len = sizeof(*zr) + (zones_chunks * sizeof(struct nvme_zns_desc));
184 zr = calloc(1, zr_len);
3efcb23f
JA
185 if (!zr) {
186 close(fd);
3d05e0ff 187 return -ENOMEM;
3efcb23f 188 }
3d05e0ff
AK
189
190 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_NS,
191 NVME_CSI_NVM, &ns);
192 if (ret) {
193 log_err("%s: nvme_identify_ns failed, err=%d\n", f->file_name,
194 ret);
195 goto out;
196 }
197
198 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
199 NVME_CSI_ZNS, &zns_ns);
200 if (ret) {
201 log_err("%s: nvme_zns_identify_ns failed, err=%d\n",
202 f->file_name, ret);
203 goto out;
204 }
205 zlen = zns_ns.lbafe[ns.flbas & 0x0f].zsze << data->lba_shift;
206
207 max_zones = (f->real_file_size - offset) / zlen;
208 if (max_zones < nr_zones)
209 nr_zones = max_zones;
210
211 if (nr_zones < zones_chunks)
212 zones_chunks = nr_zones;
213
214 while (zones_fetched < nr_zones) {
215 if (zones_fetched + zones_chunks >= nr_zones) {
216 zones_chunks = nr_zones - zones_fetched;
217 zr_len = sizeof(*zr) + (zones_chunks * sizeof(struct nvme_zns_desc));
218 }
219 ret = nvme_report_zones(fd, data->nsid, offset >> data->lba_shift,
220 NVME_ZNS_ZRAS_FEAT_ERZ, zr_len, (void *)zr);
221 if (ret) {
222 log_err("%s: nvme_zns_report_zones failed, err=%d\n",
223 f->file_name, ret);
224 goto out;
225 }
226
227 /* Transform the zone-report */
228 for (j = 0; j < zr->nr_zones; j++, i++) {
229 struct nvme_zns_desc *desc = (struct nvme_zns_desc *)&(zr->entries[j]);
230
231 zbdz[i].start = desc->zslba << data->lba_shift;
232 zbdz[i].len = zlen;
233 zbdz[i].wp = desc->wp << data->lba_shift;
234 zbdz[i].capacity = desc->zcap << data->lba_shift;
235
236 /* Zone Type is stored in first 4 bits. */
237 switch (desc->zt & 0x0f) {
238 case NVME_ZONE_TYPE_SEQWRITE_REQ:
239 zbdz[i].type = ZBD_ZONE_TYPE_SWR;
240 break;
241 default:
242 log_err("%s: invalid type for zone at offset %llu.\n",
243 f->file_name, desc->zslba);
244 ret = -EIO;
245 goto out;
246 }
247
248 /* Zone State is stored in last 4 bits. */
249 switch (desc->zs >> 4) {
250 case NVME_ZNS_ZS_EMPTY:
251 zbdz[i].cond = ZBD_ZONE_COND_EMPTY;
252 break;
253 case NVME_ZNS_ZS_IMPL_OPEN:
254 zbdz[i].cond = ZBD_ZONE_COND_IMP_OPEN;
255 break;
256 case NVME_ZNS_ZS_EXPL_OPEN:
257 zbdz[i].cond = ZBD_ZONE_COND_EXP_OPEN;
258 break;
259 case NVME_ZNS_ZS_CLOSED:
260 zbdz[i].cond = ZBD_ZONE_COND_CLOSED;
261 break;
262 case NVME_ZNS_ZS_FULL:
263 zbdz[i].cond = ZBD_ZONE_COND_FULL;
264 break;
265 case NVME_ZNS_ZS_READ_ONLY:
266 case NVME_ZNS_ZS_OFFLINE:
267 default:
268 /* Treat all these conditions as offline (don't use!) */
269 zbdz[i].cond = ZBD_ZONE_COND_OFFLINE;
270 zbdz[i].wp = zbdz[i].start;
271 }
272 }
273 zones_fetched += zr->nr_zones;
274 offset += zr->nr_zones * zlen;
275 }
276
277 ret = zones_fetched;
278out:
279 free(zr);
280 close(fd);
281
282 return ret;
283}
284
285int fio_nvme_reset_wp(struct thread_data *td, struct fio_file *f,
286 uint64_t offset, uint64_t length)
287{
288 struct nvme_data *data = FILE_ENG_DATA(f);
289 unsigned int nr_zones;
290 unsigned long long zslba;
291 int i, fd, ret = 0;
292
293 /* If the file is not yet opened, open it for this function. */
294 fd = f->fd;
295 if (fd < 0) {
296 fd = open(f->file_name, O_RDWR | O_LARGEFILE);
297 if (fd < 0)
298 return -errno;
299 }
300
301 zslba = offset >> data->lba_shift;
302 nr_zones = (length + td->o.zone_size - 1) / td->o.zone_size;
303
304 for (i = 0; i < nr_zones; i++, zslba += (td->o.zone_size >> data->lba_shift)) {
305 struct nvme_passthru_cmd cmd = {
306 .opcode = nvme_zns_cmd_mgmt_send,
307 .nsid = data->nsid,
308 .cdw10 = zslba & 0xffffffff,
309 .cdw11 = zslba >> 32,
310 .cdw13 = NVME_ZNS_ZSA_RESET,
311 .addr = (__u64)(uintptr_t)NULL,
312 .data_len = 0,
313 .timeout_ms = NVME_DEFAULT_IOCTL_TIMEOUT,
314 };
315
316 ret = ioctl(fd, NVME_IOCTL_IO_CMD, &cmd);
317 }
318
319 if (f->fd < 0)
320 close(fd);
321 return -ret;
322}
323
324int fio_nvme_get_max_open_zones(struct thread_data *td, struct fio_file *f,
325 unsigned int *max_open_zones)
326{
327 struct nvme_data *data = FILE_ENG_DATA(f);
328 struct nvme_zns_id_ns zns_ns;
329 int fd, ret = 0;
330
331 fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
332 if (fd < 0)
333 return -errno;
334
335 ret = nvme_identify(fd, data->nsid, NVME_IDENTIFY_CNS_CSI_NS,
336 NVME_CSI_ZNS, &zns_ns);
337 if (ret) {
338 log_err("%s: nvme_zns_identify_ns failed, err=%d\n",
339 f->file_name, ret);
340 goto out;
341 }
342
343 *max_open_zones = zns_ns.mor + 1;
344out:
345 close(fd);
346 return ret;
347}