glusterfs: update for new API
[fio.git] / zbd.c
CommitLineData
bfbdd35b
BVA
1/*
2 * Copyright (C) 2018 Western Digital Corporation or its affiliates.
3 *
4 * This file is released under the GPL.
5 */
6
7#include <errno.h>
8#include <string.h>
9#include <stdlib.h>
10#include <dirent.h>
11#include <fcntl.h>
12#include <sys/ioctl.h>
13#include <sys/stat.h>
14#include <unistd.h>
15#include <linux/blkzoned.h>
16#include "file.h"
17#include "fio.h"
18#include "lib/pow2.h"
19#include "log.h"
20#include "smalloc.h"
21#include "verify.h"
22#include "zbd.h"
23
24/**
25 * zbd_zone_idx - convert an offset into a zone number
26 * @f: file pointer.
27 * @offset: offset in bytes. If this offset is in the first zone_size bytes
28 * past the disk size then the index of the sentinel is returned.
29 */
30static uint32_t zbd_zone_idx(const struct fio_file *f, uint64_t offset)
31{
32 uint32_t zone_idx;
33
cd775e06 34 if (f->zbd_info->zone_size_log2 > 0)
bfbdd35b
BVA
35 zone_idx = offset >> f->zbd_info->zone_size_log2;
36 else
ee3696bd 37 zone_idx = offset / f->zbd_info->zone_size;
bfbdd35b
BVA
38
39 return min(zone_idx, f->zbd_info->nr_zones);
40}
41
42/**
43 * zbd_zone_full - verify whether a minimum number of bytes remain in a zone
44 * @f: file pointer.
45 * @z: zone info pointer.
46 * @required: minimum number of bytes that must remain in a zone.
47 *
48 * The caller must hold z->mutex.
49 */
50static bool zbd_zone_full(const struct fio_file *f, struct fio_zone_info *z,
51 uint64_t required)
52{
53 assert((required & 511) == 0);
54
55 return z->type == BLK_ZONE_TYPE_SEQWRITE_REQ &&
ee3696bd 56 z->wp + required > z->start + f->zbd_info->zone_size;
bfbdd35b
BVA
57}
58
59static bool is_valid_offset(const struct fio_file *f, uint64_t offset)
60{
61 return (uint64_t)(offset - f->file_offset) < f->io_size;
62}
63
64/* Verify whether direct I/O is used for all host-managed zoned drives. */
65static bool zbd_using_direct_io(void)
66{
67 struct thread_data *td;
68 struct fio_file *f;
69 int i, j;
70
71 for_each_td(td, i) {
72 if (td->o.odirect || !(td->o.td_ddir & TD_DDIR_WRITE))
73 continue;
74 for_each_file(td, f, j) {
75 if (f->zbd_info &&
76 f->zbd_info->model == ZBD_DM_HOST_MANAGED)
77 return false;
78 }
79 }
80
81 return true;
82}
83
84/* Whether or not the I/O range for f includes one or more sequential zones */
85static bool zbd_is_seq_job(struct fio_file *f)
86{
87 uint32_t zone_idx, zone_idx_b, zone_idx_e;
88
89 assert(f->zbd_info);
90 if (f->io_size == 0)
91 return false;
92 zone_idx_b = zbd_zone_idx(f, f->file_offset);
93 zone_idx_e = zbd_zone_idx(f, f->file_offset + f->io_size - 1);
94 for (zone_idx = zone_idx_b; zone_idx <= zone_idx_e; zone_idx++)
95 if (f->zbd_info->zone_info[zone_idx].type ==
96 BLK_ZONE_TYPE_SEQWRITE_REQ)
97 return true;
98
99 return false;
100}
101
102/*
103 * Verify whether offset and size parameters are aligned with zone boundaries.
104 */
105static bool zbd_verify_sizes(void)
106{
107 const struct fio_zone_info *z;
108 struct thread_data *td;
109 struct fio_file *f;
110 uint64_t new_offset, new_end;
111 uint32_t zone_idx;
112 int i, j;
113
114 for_each_td(td, i) {
115 for_each_file(td, f, j) {
116 if (!f->zbd_info)
117 continue;
118 if (f->file_offset >= f->real_file_size)
119 continue;
120 if (!zbd_is_seq_job(f))
121 continue;
122 zone_idx = zbd_zone_idx(f, f->file_offset);
123 z = &f->zbd_info->zone_info[zone_idx];
ee3696bd
DLM
124 if (f->file_offset != z->start) {
125 new_offset = (z+1)->start;
bfbdd35b
BVA
126 if (new_offset >= f->file_offset + f->io_size) {
127 log_info("%s: io_size must be at least one zone\n",
128 f->file_name);
129 return false;
130 }
a0c84dd4
JA
131 log_info("%s: rounded up offset from %llu to %llu\n",
132 f->file_name, (unsigned long long) f->file_offset,
133 (unsigned long long) new_offset);
bfbdd35b
BVA
134 f->io_size -= (new_offset - f->file_offset);
135 f->file_offset = new_offset;
136 }
137 zone_idx = zbd_zone_idx(f, f->file_offset + f->io_size);
138 z = &f->zbd_info->zone_info[zone_idx];
ee3696bd 139 new_end = z->start;
bfbdd35b
BVA
140 if (f->file_offset + f->io_size != new_end) {
141 if (new_end <= f->file_offset) {
142 log_info("%s: io_size must be at least one zone\n",
143 f->file_name);
144 return false;
145 }
a0c84dd4
JA
146 log_info("%s: rounded down io_size from %llu to %llu\n",
147 f->file_name, (unsigned long long) f->io_size,
148 (unsigned long long) new_end - f->file_offset);
bfbdd35b
BVA
149 f->io_size = new_end - f->file_offset;
150 }
151 }
152 }
153
154 return true;
155}
156
157static bool zbd_verify_bs(void)
158{
159 struct thread_data *td;
160 struct fio_file *f;
161 uint32_t zone_size;
162 int i, j, k;
163
164 for_each_td(td, i) {
165 for_each_file(td, f, j) {
166 if (!f->zbd_info)
167 continue;
168 zone_size = f->zbd_info->zone_size;
169 for (k = 0; k < ARRAY_SIZE(td->o.bs); k++) {
170 if (td->o.verify != VERIFY_NONE &&
ee3696bd 171 zone_size % td->o.bs[k] != 0) {
bfbdd35b
BVA
172 log_info("%s: block size %llu is not a divisor of the zone size %d\n",
173 f->file_name, td->o.bs[k],
ee3696bd 174 zone_size);
bfbdd35b
BVA
175 return false;
176 }
177 }
178 }
179 }
180 return true;
181}
182
183/*
184 * Read zone information into @buf starting from sector @start_sector.
185 * @fd is a file descriptor that refers to a block device and @bufsz is the
186 * size of @buf.
187 *
188 * Returns 0 upon success and a negative error code upon failure.
7f125e7f
DLM
189 * If the zone report is empty, always assume an error (device problem) and
190 * return -EIO.
bfbdd35b
BVA
191 */
192static int read_zone_info(int fd, uint64_t start_sector,
193 void *buf, unsigned int bufsz)
194{
195 struct blk_zone_report *hdr = buf;
7f125e7f 196 int ret;
bfbdd35b
BVA
197
198 if (bufsz < sizeof(*hdr))
199 return -EINVAL;
200
201 memset(hdr, 0, sizeof(*hdr));
202
203 hdr->nr_zones = (bufsz - sizeof(*hdr)) / sizeof(struct blk_zone);
204 hdr->sector = start_sector;
7f125e7f
DLM
205 ret = ioctl(fd, BLKREPORTZONE, hdr);
206 if (ret)
207 return -errno;
208 if (!hdr->nr_zones)
209 return -EIO;
210 return 0;
bfbdd35b
BVA
211}
212
213/*
214 * Read up to 255 characters from the first line of a file. Strip the trailing
215 * newline.
216 */
217static char *read_file(const char *path)
218{
219 char line[256], *p = line;
220 FILE *f;
221
222 f = fopen(path, "rb");
223 if (!f)
224 return NULL;
225 if (!fgets(line, sizeof(line), f))
226 line[0] = '\0';
227 strsep(&p, "\n");
228 fclose(f);
229
230 return strdup(line);
231}
232
233static enum blk_zoned_model get_zbd_model(const char *file_name)
234{
235 enum blk_zoned_model model = ZBD_DM_NONE;
236 char *zoned_attr_path = NULL;
237 char *model_str = NULL;
238 struct stat statbuf;
63e3e40d
SK
239 char *sys_devno_path = NULL;
240 char *part_attr_path = NULL;
241 char *part_str = NULL;
242 char sys_path[PATH_MAX];
243 ssize_t sz;
244 char *delim = NULL;
bfbdd35b
BVA
245
246 if (stat(file_name, &statbuf) < 0)
247 goto out;
63e3e40d
SK
248
249 if (asprintf(&sys_devno_path, "/sys/dev/block/%d:%d",
bfbdd35b
BVA
250 major(statbuf.st_rdev), minor(statbuf.st_rdev)) < 0)
251 goto out;
63e3e40d
SK
252
253 sz = readlink(sys_devno_path, sys_path, sizeof(sys_path) - 1);
254 if (sz < 0)
255 goto out;
256 sys_path[sz] = '\0';
257
258 /*
259 * If the device is a partition device, cut the device name in the
260 * canonical sysfs path to obtain the sysfs path of the holder device.
261 * e.g.: /sys/devices/.../sda/sda1 -> /sys/devices/.../sda
262 */
263 if (asprintf(&part_attr_path, "/sys/dev/block/%s/partition",
264 sys_path) < 0)
265 goto out;
266 part_str = read_file(part_attr_path);
267 if (part_str && *part_str == '1') {
268 delim = strrchr(sys_path, '/');
269 if (!delim)
270 goto out;
271 *delim = '\0';
272 }
273
274 if (asprintf(&zoned_attr_path,
275 "/sys/dev/block/%s/queue/zoned", sys_path) < 0)
276 goto out;
277
bfbdd35b
BVA
278 model_str = read_file(zoned_attr_path);
279 if (!model_str)
280 goto out;
281 dprint(FD_ZBD, "%s: zbd model string: %s\n", file_name, model_str);
282 if (strcmp(model_str, "host-aware") == 0)
283 model = ZBD_DM_HOST_AWARE;
284 else if (strcmp(model_str, "host-managed") == 0)
285 model = ZBD_DM_HOST_MANAGED;
286
287out:
288 free(model_str);
289 free(zoned_attr_path);
63e3e40d
SK
290 free(part_str);
291 free(part_attr_path);
292 free(sys_devno_path);
bfbdd35b
BVA
293 return model;
294}
295
296static int ilog2(uint64_t i)
297{
298 int log = -1;
299
300 while (i) {
301 i >>= 1;
302 log++;
303 }
304 return log;
305}
306
307/*
308 * Initialize f->zbd_info for devices that are not zoned block devices. This
309 * allows to execute a ZBD workload against a non-ZBD device.
310 */
311static int init_zone_info(struct thread_data *td, struct fio_file *f)
312{
313 uint32_t nr_zones;
314 struct fio_zone_info *p;
315 uint64_t zone_size;
316 struct zoned_block_device_info *zbd_info = NULL;
317 pthread_mutexattr_t attr;
318 int i;
319
ee3696bd 320 zone_size = td->o.zone_size;
bfbdd35b 321 assert(zone_size);
ee3696bd 322 nr_zones = (f->real_file_size + zone_size - 1) / zone_size;
bfbdd35b
BVA
323 zbd_info = scalloc(1, sizeof(*zbd_info) +
324 (nr_zones + 1) * sizeof(zbd_info->zone_info[0]));
325 if (!zbd_info)
326 return -ENOMEM;
327
328 pthread_mutexattr_init(&attr);
329 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
330 pthread_mutexattr_setpshared(&attr, true);
331 pthread_mutex_init(&zbd_info->mutex, &attr);
332 zbd_info->refcount = 1;
333 p = &zbd_info->zone_info[0];
334 for (i = 0; i < nr_zones; i++, p++) {
335 pthread_mutex_init(&p->mutex, &attr);
336 p->start = i * zone_size;
337 p->wp = p->start + zone_size;
338 p->type = BLK_ZONE_TYPE_SEQWRITE_REQ;
339 p->cond = BLK_ZONE_COND_EMPTY;
340 }
341 /* a sentinel */
342 p->start = nr_zones * zone_size;
343
344 f->zbd_info = zbd_info;
345 f->zbd_info->zone_size = zone_size;
346 f->zbd_info->zone_size_log2 = is_power_of_2(zone_size) ?
ee3696bd 347 ilog2(zone_size) : -1;
bfbdd35b
BVA
348 f->zbd_info->nr_zones = nr_zones;
349 pthread_mutexattr_destroy(&attr);
350 return 0;
351}
352
353/*
354 * Parse the BLKREPORTZONE output and store it in f->zbd_info. Must be called
355 * only for devices that support this ioctl, namely zoned block devices.
356 */
357static int parse_zone_info(struct thread_data *td, struct fio_file *f)
358{
359 const unsigned int bufsz = sizeof(struct blk_zone_report) +
360 4096 * sizeof(struct blk_zone);
361 uint32_t nr_zones;
362 struct blk_zone_report *hdr;
363 const struct blk_zone *z;
364 struct fio_zone_info *p;
365 uint64_t zone_size, start_sector;
366 struct zoned_block_device_info *zbd_info = NULL;
367 pthread_mutexattr_t attr;
368 void *buf;
369 int fd, i, j, ret = 0;
370
371 pthread_mutexattr_init(&attr);
372 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
373 pthread_mutexattr_setpshared(&attr, true);
374
375 buf = malloc(bufsz);
376 if (!buf)
377 goto out;
378
379 fd = open(f->file_name, O_RDONLY | O_LARGEFILE);
380 if (fd < 0) {
381 ret = -errno;
382 goto free;
383 }
384
385 ret = read_zone_info(fd, 0, buf, bufsz);
386 if (ret < 0) {
387 log_info("fio: BLKREPORTZONE(%lu) failed for %s (%d).\n",
388 0UL, f->file_name, -ret);
389 goto close;
390 }
391 hdr = buf;
392 if (hdr->nr_zones < 1) {
393 log_info("fio: %s has invalid zone information.\n",
394 f->file_name);
395 goto close;
396 }
397 z = (void *)(hdr + 1);
ee3696bd
DLM
398 zone_size = z->len << 9;
399 nr_zones = (f->real_file_size + zone_size - 1) / zone_size;
bfbdd35b
BVA
400
401 if (td->o.zone_size == 0) {
ee3696bd
DLM
402 td->o.zone_size = zone_size;
403 } else if (td->o.zone_size != zone_size) {
a0c84dd4
JA
404 log_info("fio: %s job parameter zonesize %llu does not match disk zone size %llu.\n",
405 f->file_name, (unsigned long long) td->o.zone_size,
406 (unsigned long long) zone_size);
bfbdd35b
BVA
407 ret = -EINVAL;
408 goto close;
409 }
410
a0c84dd4
JA
411 dprint(FD_ZBD, "Device %s has %d zones of size %llu KB\n", f->file_name,
412 nr_zones, (unsigned long long) zone_size / 1024);
bfbdd35b
BVA
413
414 zbd_info = scalloc(1, sizeof(*zbd_info) +
415 (nr_zones + 1) * sizeof(zbd_info->zone_info[0]));
416 ret = -ENOMEM;
417 if (!zbd_info)
418 goto close;
419 pthread_mutex_init(&zbd_info->mutex, &attr);
420 zbd_info->refcount = 1;
421 p = &zbd_info->zone_info[0];
422 for (start_sector = 0, j = 0; j < nr_zones;) {
423 z = (void *)(hdr + 1);
424 for (i = 0; i < hdr->nr_zones; i++, j++, z++, p++) {
425 pthread_mutex_init(&p->mutex, &attr);
ee3696bd 426 p->start = z->start << 9;
bfbdd35b
BVA
427 switch (z->cond) {
428 case BLK_ZONE_COND_NOT_WP:
bfbdd35b 429 case BLK_ZONE_COND_FULL:
ee3696bd 430 p->wp = p->start + zone_size;
bfbdd35b
BVA
431 break;
432 default:
433 assert(z->start <= z->wp);
ee3696bd
DLM
434 assert(z->wp <= z->start + (zone_size >> 9));
435 p->wp = z->wp << 9;
bfbdd35b
BVA
436 break;
437 }
438 p->type = z->type;
439 p->cond = z->cond;
440 if (j > 0 && p->start != p[-1].start + zone_size) {
441 log_info("%s: invalid zone data\n",
442 f->file_name);
443 ret = -EINVAL;
444 goto close;
445 }
446 }
447 z--;
448 start_sector = z->start + z->len;
449 if (j >= nr_zones)
450 break;
451 ret = read_zone_info(fd, start_sector, buf, bufsz);
452 if (ret < 0) {
a0c84dd4
JA
453 log_info("fio: BLKREPORTZONE(%llu) failed for %s (%d).\n",
454 (unsigned long long) start_sector, f->file_name, -ret);
bfbdd35b
BVA
455 goto close;
456 }
457 }
458 /* a sentinel */
ee3696bd 459 zbd_info->zone_info[nr_zones].start = start_sector << 9;
bfbdd35b
BVA
460
461 f->zbd_info = zbd_info;
462 f->zbd_info->zone_size = zone_size;
463 f->zbd_info->zone_size_log2 = is_power_of_2(zone_size) ?
ee3696bd 464 ilog2(zone_size) : -1;
bfbdd35b
BVA
465 f->zbd_info->nr_zones = nr_zones;
466 zbd_info = NULL;
467 ret = 0;
468
469close:
470 sfree(zbd_info);
471 close(fd);
472free:
473 free(buf);
474out:
475 pthread_mutexattr_destroy(&attr);
476 return ret;
477}
478
479/*
480 * Allocate zone information and store it into f->zbd_info if zonemode=zbd.
481 *
482 * Returns 0 upon success and a negative error code upon failure.
483 */
484int zbd_create_zone_info(struct thread_data *td, struct fio_file *f)
485{
486 enum blk_zoned_model zbd_model;
487 int ret = 0;
488
489 assert(td->o.zone_mode == ZONE_MODE_ZBD);
490
491 zbd_model = get_zbd_model(f->file_name);
492 switch (zbd_model) {
493 case ZBD_DM_HOST_AWARE:
494 case ZBD_DM_HOST_MANAGED:
495 ret = parse_zone_info(td, f);
496 break;
497 case ZBD_DM_NONE:
498 ret = init_zone_info(td, f);
499 break;
500 }
501 if (ret == 0)
502 f->zbd_info->model = zbd_model;
503 return ret;
504}
505
506void zbd_free_zone_info(struct fio_file *f)
507{
508 uint32_t refcount;
509
510 if (!f->zbd_info)
511 return;
512
513 pthread_mutex_lock(&f->zbd_info->mutex);
514 refcount = --f->zbd_info->refcount;
515 pthread_mutex_unlock(&f->zbd_info->mutex);
516
517 assert((int32_t)refcount >= 0);
518 if (refcount == 0)
519 sfree(f->zbd_info);
520 f->zbd_info = NULL;
521}
522
523/*
524 * Initialize f->zbd_info.
525 *
526 * Returns 0 upon success and a negative error code upon failure.
527 *
528 * Note: this function can only work correctly if it is called before the first
529 * fio fork() call.
530 */
531static int zbd_init_zone_info(struct thread_data *td, struct fio_file *file)
532{
533 struct thread_data *td2;
534 struct fio_file *f2;
535 int i, j, ret;
536
537 for_each_td(td2, i) {
538 for_each_file(td2, f2, j) {
539 if (td2 == td && f2 == file)
540 continue;
541 if (!f2->zbd_info ||
542 strcmp(f2->file_name, file->file_name) != 0)
543 continue;
544 file->zbd_info = f2->zbd_info;
545 file->zbd_info->refcount++;
546 return 0;
547 }
548 }
549
550 ret = zbd_create_zone_info(td, file);
551 if (ret < 0)
552 td_verror(td, -ret, "BLKREPORTZONE failed");
553 return ret;
554}
555
556int zbd_init(struct thread_data *td)
557{
558 struct fio_file *f;
559 int i;
560
561 for_each_file(td, f, i) {
562 if (f->filetype != FIO_TYPE_BLOCK)
563 continue;
564 if (td->o.zone_size && td->o.zone_size < 512) {
565 log_err("%s: zone size must be at least 512 bytes for --zonemode=zbd\n\n",
566 f->file_name);
567 return 1;
568 }
569 if (td->o.zone_size == 0 &&
570 get_zbd_model(f->file_name) == ZBD_DM_NONE) {
571 log_err("%s: Specifying the zone size is mandatory for regular block devices with --zonemode=zbd\n\n",
572 f->file_name);
573 return 1;
574 }
575 zbd_init_zone_info(td, f);
576 }
577
578 if (!zbd_using_direct_io()) {
579 log_err("Using direct I/O is mandatory for writing to ZBD drives\n\n");
580 return 1;
581 }
582
583 if (!zbd_verify_sizes())
584 return 1;
585
586 if (!zbd_verify_bs())
587 return 1;
588
589 return 0;
590}
591
592/**
593 * zbd_reset_range - reset zones for a range of sectors
594 * @td: FIO thread data.
595 * @f: Fio file for which to reset zones
596 * @sector: Starting sector in units of 512 bytes
597 * @nr_sectors: Number of sectors in units of 512 bytes
598 *
599 * Returns 0 upon success and a negative error code upon failure.
600 */
601static int zbd_reset_range(struct thread_data *td, const struct fio_file *f,
ee3696bd 602 uint64_t offset, uint64_t length)
bfbdd35b
BVA
603{
604 struct blk_zone_range zr = {
ee3696bd
DLM
605 .sector = offset >> 9,
606 .nr_sectors = length >> 9,
bfbdd35b
BVA
607 };
608 uint32_t zone_idx_b, zone_idx_e;
609 struct fio_zone_info *zb, *ze, *z;
610 int ret = 0;
611
612 assert(f->fd != -1);
ee3696bd 613 assert(is_valid_offset(f, offset + length - 1));
bfbdd35b
BVA
614 switch (f->zbd_info->model) {
615 case ZBD_DM_HOST_AWARE:
616 case ZBD_DM_HOST_MANAGED:
617 ret = ioctl(f->fd, BLKRESETZONE, &zr);
618 if (ret < 0) {
619 td_verror(td, errno, "resetting wp failed");
620 log_err("%s: resetting wp for %llu sectors at sector %llu failed (%d).\n",
621 f->file_name, zr.nr_sectors, zr.sector, errno);
622 return ret;
623 }
624 break;
625 case ZBD_DM_NONE:
626 break;
627 }
628
ee3696bd 629 zone_idx_b = zbd_zone_idx(f, offset);
bfbdd35b 630 zb = &f->zbd_info->zone_info[zone_idx_b];
ee3696bd 631 zone_idx_e = zbd_zone_idx(f, offset + length);
bfbdd35b
BVA
632 ze = &f->zbd_info->zone_info[zone_idx_e];
633 for (z = zb; z < ze; z++) {
634 pthread_mutex_lock(&z->mutex);
a7c2b6fc
BVA
635 pthread_mutex_lock(&f->zbd_info->mutex);
636 f->zbd_info->sectors_with_data -= z->wp - z->start;
637 pthread_mutex_unlock(&f->zbd_info->mutex);
bfbdd35b
BVA
638 z->wp = z->start;
639 z->verify_block = 0;
640 pthread_mutex_unlock(&z->mutex);
641 }
642
fd5d733f
BVA
643 td->ts.nr_zone_resets += ze - zb;
644
bfbdd35b
BVA
645 return ret;
646}
647
a0c84dd4
JA
648static unsigned int zbd_zone_nr(struct zoned_block_device_info *zbd_info,
649 struct fio_zone_info *zone)
650{
089ddd95 651 return zone - zbd_info->zone_info;
a0c84dd4
JA
652}
653
bfbdd35b
BVA
654/**
655 * zbd_reset_zone - reset the write pointer of a single zone
656 * @td: FIO thread data.
657 * @f: FIO file associated with the disk for which to reset a write pointer.
658 * @z: Zone to reset.
659 *
660 * Returns 0 upon success and a negative error code upon failure.
661 */
662static int zbd_reset_zone(struct thread_data *td, const struct fio_file *f,
663 struct fio_zone_info *z)
664{
a0c84dd4
JA
665 dprint(FD_ZBD, "%s: resetting wp of zone %u.\n", f->file_name,
666 zbd_zone_nr(f->zbd_info, z));
d60be7d5
DLM
667
668 return zbd_reset_range(td, f, z->start, (z+1)->start - z->start);
bfbdd35b
BVA
669}
670
671/*
672 * Reset a range of zones. Returns 0 upon success and 1 upon failure.
673 * @td: fio thread data.
674 * @f: fio file for which to reset zones
675 * @zb: first zone to reset.
676 * @ze: first zone not to reset.
677 * @all_zones: whether to reset all zones or only those zones for which the
678 * write pointer is not a multiple of td->o.min_bs[DDIR_WRITE].
679 */
680static int zbd_reset_zones(struct thread_data *td, struct fio_file *f,
681 struct fio_zone_info *const zb,
682 struct fio_zone_info *const ze, bool all_zones)
683{
684 struct fio_zone_info *z, *start_z = ze;
ee3696bd 685 const uint32_t min_bs = td->o.min_bs[DDIR_WRITE];
bfbdd35b
BVA
686 bool reset_wp;
687 int res = 0;
688
a0c84dd4
JA
689 dprint(FD_ZBD, "%s: examining zones %u .. %u\n", f->file_name,
690 zbd_zone_nr(f->zbd_info, zb), zbd_zone_nr(f->zbd_info, ze));
bfbdd35b
BVA
691 assert(f->fd != -1);
692 for (z = zb; z < ze; z++) {
693 pthread_mutex_lock(&z->mutex);
694 switch (z->type) {
695 case BLK_ZONE_TYPE_SEQWRITE_REQ:
696 reset_wp = all_zones ? z->wp != z->start :
697 (td->o.td_ddir & TD_DDIR_WRITE) &&
698 z->wp % min_bs != 0;
699 if (start_z == ze && reset_wp) {
700 start_z = z;
701 } else if (start_z < ze && !reset_wp) {
702 dprint(FD_ZBD,
a0c84dd4 703 "%s: resetting zones %u .. %u\n",
bfbdd35b 704 f->file_name,
a0c84dd4
JA
705 zbd_zone_nr(f->zbd_info, start_z),
706 zbd_zone_nr(f->zbd_info, z));
bfbdd35b
BVA
707 if (zbd_reset_range(td, f, start_z->start,
708 z->start - start_z->start) < 0)
709 res = 1;
710 start_z = ze;
711 }
712 break;
713 default:
714 if (start_z == ze)
715 break;
a0c84dd4
JA
716 dprint(FD_ZBD, "%s: resetting zones %u .. %u\n",
717 f->file_name, zbd_zone_nr(f->zbd_info, start_z),
718 zbd_zone_nr(f->zbd_info, z));
bfbdd35b
BVA
719 if (zbd_reset_range(td, f, start_z->start,
720 z->start - start_z->start) < 0)
721 res = 1;
722 start_z = ze;
723 break;
724 }
725 }
726 if (start_z < ze) {
a0c84dd4
JA
727 dprint(FD_ZBD, "%s: resetting zones %u .. %u\n", f->file_name,
728 zbd_zone_nr(f->zbd_info, start_z),
729 zbd_zone_nr(f->zbd_info, z));
bfbdd35b
BVA
730 if (zbd_reset_range(td, f, start_z->start,
731 z->start - start_z->start) < 0)
732 res = 1;
733 }
734 for (z = zb; z < ze; z++)
735 pthread_mutex_unlock(&z->mutex);
736
737 return res;
738}
739
a7c2b6fc
BVA
740/*
741 * Reset zbd_info.write_cnt, the counter that counts down towards the next
742 * zone reset.
743 */
744static void zbd_reset_write_cnt(const struct thread_data *td,
745 const struct fio_file *f)
746{
747 assert(0 <= td->o.zrf.u.f && td->o.zrf.u.f <= 1);
748
749 pthread_mutex_lock(&f->zbd_info->mutex);
750 f->zbd_info->write_cnt = td->o.zrf.u.f ?
751 min(1.0 / td->o.zrf.u.f, 0.0 + UINT_MAX) : UINT_MAX;
752 pthread_mutex_unlock(&f->zbd_info->mutex);
753}
754
755static bool zbd_dec_and_reset_write_cnt(const struct thread_data *td,
756 const struct fio_file *f)
757{
758 uint32_t write_cnt = 0;
759
760 pthread_mutex_lock(&f->zbd_info->mutex);
761 assert(f->zbd_info->write_cnt);
762 if (f->zbd_info->write_cnt)
763 write_cnt = --f->zbd_info->write_cnt;
764 if (write_cnt == 0)
765 zbd_reset_write_cnt(td, f);
766 pthread_mutex_unlock(&f->zbd_info->mutex);
767
768 return write_cnt == 0;
769}
770
91d25131
BVA
771enum swd_action {
772 CHECK_SWD,
773 SET_SWD,
774};
775
776/* Calculate the number of sectors with data (swd) and perform action 'a' */
777static uint64_t zbd_process_swd(const struct fio_file *f, enum swd_action a)
615555bb 778{
615555bb
BVA
779 struct fio_zone_info *zb, *ze, *z;
780 uint64_t swd = 0;
781
782 zb = &f->zbd_info->zone_info[zbd_zone_idx(f, f->file_offset)];
783 ze = &f->zbd_info->zone_info[zbd_zone_idx(f, f->file_offset +
784 f->io_size)];
785 for (z = zb; z < ze; z++) {
786 pthread_mutex_lock(&z->mutex);
787 swd += z->wp - z->start;
788 }
789 pthread_mutex_lock(&f->zbd_info->mutex);
91d25131
BVA
790 switch (a) {
791 case CHECK_SWD:
792 assert(f->zbd_info->sectors_with_data == swd);
793 break;
794 case SET_SWD:
795 f->zbd_info->sectors_with_data = swd;
796 break;
797 }
615555bb
BVA
798 pthread_mutex_unlock(&f->zbd_info->mutex);
799 for (z = zb; z < ze; z++)
800 pthread_mutex_unlock(&z->mutex);
91d25131
BVA
801
802 return swd;
803}
804
805/*
806 * The swd check is useful for debugging but takes too much time to leave
807 * it enabled all the time. Hence it is disabled by default.
808 */
809static const bool enable_check_swd = false;
810
811/* Check whether the value of zbd_info.sectors_with_data is correct. */
812static void zbd_check_swd(const struct fio_file *f)
813{
814 if (!enable_check_swd)
815 return;
816
817 zbd_process_swd(f, CHECK_SWD);
818}
819
820static void zbd_init_swd(struct fio_file *f)
821{
822 uint64_t swd;
823
824 swd = zbd_process_swd(f, SET_SWD);
825 dprint(FD_ZBD, "%s(%s): swd = %" PRIu64 "\n", __func__, f->file_name,
826 swd);
615555bb
BVA
827}
828
bfbdd35b
BVA
829void zbd_file_reset(struct thread_data *td, struct fio_file *f)
830{
91d25131 831 struct fio_zone_info *zb, *ze;
bfbdd35b
BVA
832 uint32_t zone_idx_e;
833
834 if (!f->zbd_info)
835 return;
836
837 zb = &f->zbd_info->zone_info[zbd_zone_idx(f, f->file_offset)];
838 zone_idx_e = zbd_zone_idx(f, f->file_offset + f->io_size);
839 ze = &f->zbd_info->zone_info[zone_idx_e];
91d25131 840 zbd_init_swd(f);
bfbdd35b
BVA
841 /*
842 * If data verification is enabled reset the affected zones before
843 * writing any data to avoid that a zone reset has to be issued while
844 * writing data, which causes data loss.
845 */
846 zbd_reset_zones(td, f, zb, ze, td->o.verify != VERIFY_NONE &&
847 (td->o.td_ddir & TD_DDIR_WRITE) &&
848 td->runstate != TD_VERIFYING);
a7c2b6fc 849 zbd_reset_write_cnt(td, f);
bfbdd35b
BVA
850}
851
59b07544
BVA
852/* The caller must hold f->zbd_info->mutex. */
853static bool is_zone_open(const struct thread_data *td, const struct fio_file *f,
854 unsigned int zone_idx)
855{
856 struct zoned_block_device_info *zbdi = f->zbd_info;
857 int i;
858
859 assert(td->o.max_open_zones <= ARRAY_SIZE(zbdi->open_zones));
860 assert(zbdi->num_open_zones <= td->o.max_open_zones);
861
862 for (i = 0; i < zbdi->num_open_zones; i++)
863 if (zbdi->open_zones[i] == zone_idx)
864 return true;
865
866 return false;
867}
868
869/*
870 * Open a ZBD zone if it was not yet open. Returns true if either the zone was
871 * already open or if opening a new zone is allowed. Returns false if the zone
872 * was not yet open and opening a new zone would cause the zone limit to be
873 * exceeded.
874 */
875static bool zbd_open_zone(struct thread_data *td, const struct io_u *io_u,
876 uint32_t zone_idx)
877{
878 const uint32_t min_bs = td->o.min_bs[DDIR_WRITE];
879 const struct fio_file *f = io_u->file;
880 struct fio_zone_info *z = &f->zbd_info->zone_info[zone_idx];
881 bool res = true;
882
883 if (z->cond == BLK_ZONE_COND_OFFLINE)
884 return false;
885
886 /*
887 * Skip full zones with data verification enabled because resetting a
888 * zone causes data loss and hence causes verification to fail.
889 */
890 if (td->o.verify != VERIFY_NONE && zbd_zone_full(f, z, min_bs))
891 return false;
892
893 /* Zero means no limit */
894 if (!td->o.max_open_zones)
895 return true;
896
897 pthread_mutex_lock(&f->zbd_info->mutex);
898 if (is_zone_open(td, f, zone_idx))
899 goto out;
900 res = false;
901 if (f->zbd_info->num_open_zones >= td->o.max_open_zones)
902 goto out;
903 dprint(FD_ZBD, "%s: opening zone %d\n", f->file_name, zone_idx);
904 f->zbd_info->open_zones[f->zbd_info->num_open_zones++] = zone_idx;
905 z->open = 1;
906 res = true;
907
908out:
909 pthread_mutex_unlock(&f->zbd_info->mutex);
910 return res;
911}
912
913/* The caller must hold f->zbd_info->mutex */
914static void zbd_close_zone(struct thread_data *td, const struct fio_file *f,
915 unsigned int open_zone_idx)
916{
917 uint32_t zone_idx;
918
919 assert(open_zone_idx < f->zbd_info->num_open_zones);
920 zone_idx = f->zbd_info->open_zones[open_zone_idx];
921 memmove(f->zbd_info->open_zones + open_zone_idx,
922 f->zbd_info->open_zones + open_zone_idx + 1,
923 (FIO_MAX_OPEN_ZBD_ZONES - (open_zone_idx + 1)) *
924 sizeof(f->zbd_info->open_zones[0]));
925 f->zbd_info->num_open_zones--;
926 f->zbd_info->zone_info[zone_idx].open = 0;
927}
928
929/*
930 * Modify the offset of an I/O unit that does not refer to an open zone such
931 * that it refers to an open zone. Close an open zone and open a new zone if
932 * necessary. This algorithm can only work correctly if all write pointers are
933 * a multiple of the fio block size. The caller must neither hold z->mutex
934 * nor f->zbd_info->mutex. Returns with z->mutex held upon success.
935 */
936struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td,
937 struct io_u *io_u)
938{
939 const uint32_t min_bs = td->o.min_bs[io_u->ddir];
940 const struct fio_file *f = io_u->file;
941 struct fio_zone_info *z;
942 unsigned int open_zone_idx = -1;
943 uint32_t zone_idx, new_zone_idx;
944 int i;
945
946 assert(is_valid_offset(f, io_u->offset));
947
948 if (td->o.max_open_zones) {
949 /*
950 * This statement accesses f->zbd_info->open_zones[] on purpose
951 * without locking.
952 */
953 zone_idx = f->zbd_info->open_zones[(io_u->offset -
954 f->file_offset) *
955 f->zbd_info->num_open_zones / f->io_size];
956 } else {
957 zone_idx = zbd_zone_idx(f, io_u->offset);
958 }
959 dprint(FD_ZBD, "%s(%s): starting from zone %d (offset %lld, buflen %lld)\n",
960 __func__, f->file_name, zone_idx, io_u->offset, io_u->buflen);
961
962 /*
963 * Since z->mutex is the outer lock and f->zbd_info->mutex the inner
964 * lock it can happen that the state of the zone with index zone_idx
965 * has changed after 'z' has been assigned and before f->zbd_info->mutex
966 * has been obtained. Hence the loop.
967 */
968 for (;;) {
969 z = &f->zbd_info->zone_info[zone_idx];
970
971 pthread_mutex_lock(&z->mutex);
972 pthread_mutex_lock(&f->zbd_info->mutex);
973 if (td->o.max_open_zones == 0)
974 goto examine_zone;
975 if (f->zbd_info->num_open_zones == 0) {
976 pthread_mutex_unlock(&f->zbd_info->mutex);
977 pthread_mutex_unlock(&z->mutex);
978 dprint(FD_ZBD, "%s(%s): no zones are open\n",
979 __func__, f->file_name);
980 return NULL;
981 }
982 open_zone_idx = (io_u->offset - f->file_offset) *
983 f->zbd_info->num_open_zones / f->io_size;
984 assert(open_zone_idx < f->zbd_info->num_open_zones);
985 new_zone_idx = f->zbd_info->open_zones[open_zone_idx];
986 if (new_zone_idx == zone_idx)
987 break;
988 zone_idx = new_zone_idx;
989 pthread_mutex_unlock(&f->zbd_info->mutex);
990 pthread_mutex_unlock(&z->mutex);
991 }
992
993 /* Both z->mutex and f->zbd_info->mutex are held. */
994
995examine_zone:
ee3696bd 996 if (z->wp + min_bs <= (z+1)->start) {
59b07544
BVA
997 pthread_mutex_unlock(&f->zbd_info->mutex);
998 goto out;
999 }
1000 dprint(FD_ZBD, "%s(%s): closing zone %d\n", __func__, f->file_name,
1001 zone_idx);
1002 if (td->o.max_open_zones)
1003 zbd_close_zone(td, f, open_zone_idx);
1004 pthread_mutex_unlock(&f->zbd_info->mutex);
1005
1006 /* Only z->mutex is held. */
1007
1008 /* Zone 'z' is full, so try to open a new zone. */
1009 for (i = f->io_size / f->zbd_info->zone_size; i > 0; i--) {
1010 zone_idx++;
1011 pthread_mutex_unlock(&z->mutex);
1012 z++;
ee3696bd 1013 if (!is_valid_offset(f, z->start)) {
59b07544
BVA
1014 /* Wrap-around. */
1015 zone_idx = zbd_zone_idx(f, f->file_offset);
1016 z = &f->zbd_info->zone_info[zone_idx];
1017 }
ee3696bd 1018 assert(is_valid_offset(f, z->start));
59b07544
BVA
1019 pthread_mutex_lock(&z->mutex);
1020 if (z->open)
1021 continue;
1022 if (zbd_open_zone(td, io_u, zone_idx))
1023 goto out;
1024 }
1025
1026 /* Only z->mutex is held. */
1027
1028 /* Check whether the write fits in any of the already opened zones. */
1029 pthread_mutex_lock(&f->zbd_info->mutex);
1030 for (i = 0; i < f->zbd_info->num_open_zones; i++) {
1031 zone_idx = f->zbd_info->open_zones[i];
1032 pthread_mutex_unlock(&f->zbd_info->mutex);
1033 pthread_mutex_unlock(&z->mutex);
1034
1035 z = &f->zbd_info->zone_info[zone_idx];
1036
1037 pthread_mutex_lock(&z->mutex);
ee3696bd 1038 if (z->wp + min_bs <= (z+1)->start)
59b07544
BVA
1039 goto out;
1040 pthread_mutex_lock(&f->zbd_info->mutex);
1041 }
1042 pthread_mutex_unlock(&f->zbd_info->mutex);
1043 pthread_mutex_unlock(&z->mutex);
1044 dprint(FD_ZBD, "%s(%s): did not open another zone\n", __func__,
1045 f->file_name);
1046 return NULL;
1047
1048out:
1049 dprint(FD_ZBD, "%s(%s): returning zone %d\n", __func__, f->file_name,
1050 zone_idx);
ee3696bd 1051 io_u->offset = z->start;
59b07544
BVA
1052 return z;
1053}
1054
bfbdd35b 1055/* The caller must hold z->mutex. */
59b07544
BVA
1056static struct fio_zone_info *zbd_replay_write_order(struct thread_data *td,
1057 struct io_u *io_u,
1058 struct fio_zone_info *z)
bfbdd35b
BVA
1059{
1060 const struct fio_file *f = io_u->file;
1061 const uint32_t min_bs = td->o.min_bs[DDIR_WRITE];
1062
59b07544
BVA
1063 if (!zbd_open_zone(td, io_u, z - f->zbd_info->zone_info)) {
1064 pthread_mutex_unlock(&z->mutex);
1065 z = zbd_convert_to_open_zone(td, io_u);
1066 assert(z);
1067 }
1068
bfbdd35b 1069 if (z->verify_block * min_bs >= f->zbd_info->zone_size)
a0c84dd4
JA
1070 log_err("%s: %d * %d >= %llu\n", f->file_name, z->verify_block,
1071 min_bs, (unsigned long long) f->zbd_info->zone_size);
ee3696bd 1072 io_u->offset = z->start + z->verify_block++ * min_bs;
59b07544 1073 return z;
bfbdd35b
BVA
1074}
1075
1076/*
1077 * Find another zone for which @io_u fits below the write pointer. Start
1078 * searching in zones @zb + 1 .. @zl and continue searching in zones
1079 * @zf .. @zb - 1.
1080 *
1081 * Either returns NULL or returns a zone pointer and holds the mutex for that
1082 * zone.
1083 */
1084static struct fio_zone_info *
1085zbd_find_zone(struct thread_data *td, struct io_u *io_u,
1086 struct fio_zone_info *zb, struct fio_zone_info *zl)
1087{
1088 const uint32_t min_bs = td->o.min_bs[io_u->ddir];
1089 const struct fio_file *f = io_u->file;
1090 struct fio_zone_info *z1, *z2;
1091 const struct fio_zone_info *const zf =
1092 &f->zbd_info->zone_info[zbd_zone_idx(f, f->file_offset)];
1093
1094 /*
1095 * Skip to the next non-empty zone in case of sequential I/O and to
1096 * the nearest non-empty zone in case of random I/O.
1097 */
1098 for (z1 = zb + 1, z2 = zb - 1; z1 < zl || z2 >= zf; z1++, z2--) {
1099 if (z1 < zl && z1->cond != BLK_ZONE_COND_OFFLINE) {
1100 pthread_mutex_lock(&z1->mutex);
ee3696bd 1101 if (z1->start + min_bs <= z1->wp)
bfbdd35b
BVA
1102 return z1;
1103 pthread_mutex_unlock(&z1->mutex);
1104 } else if (!td_random(td)) {
1105 break;
1106 }
1107 if (td_random(td) && z2 >= zf &&
1108 z2->cond != BLK_ZONE_COND_OFFLINE) {
1109 pthread_mutex_lock(&z2->mutex);
ee3696bd 1110 if (z2->start + min_bs <= z2->wp)
bfbdd35b
BVA
1111 return z2;
1112 pthread_mutex_unlock(&z2->mutex);
1113 }
1114 }
1115 dprint(FD_ZBD, "%s: adjusting random read offset failed\n",
1116 f->file_name);
1117 return NULL;
1118}
1119
bfbdd35b 1120/**
d9ed3e63 1121 * zbd_queue_io - update the write pointer of a sequential zone
bfbdd35b 1122 * @io_u: I/O unit
d9ed3e63
DLM
1123 * @success: Whether or not the I/O unit has been queued successfully
1124 * @q: queueing status (busy, completed or queued).
bfbdd35b 1125 *
d9ed3e63
DLM
1126 * For write and trim operations, update the write pointer of the I/O unit
1127 * target zone.
bfbdd35b 1128 */
d9ed3e63 1129static void zbd_queue_io(struct io_u *io_u, int q, bool success)
bfbdd35b 1130{
d9ed3e63
DLM
1131 const struct fio_file *f = io_u->file;
1132 struct zoned_block_device_info *zbd_info = f->zbd_info;
bfbdd35b
BVA
1133 struct fio_zone_info *z;
1134 uint32_t zone_idx;
d9ed3e63 1135 uint64_t zone_end;
bfbdd35b 1136
bfbdd35b
BVA
1137 if (!zbd_info)
1138 return;
1139
d9ed3e63 1140 zone_idx = zbd_zone_idx(f, io_u->offset);
bfbdd35b 1141 assert(zone_idx < zbd_info->nr_zones);
d9ed3e63
DLM
1142 z = &zbd_info->zone_info[zone_idx];
1143
bfbdd35b
BVA
1144 if (z->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
1145 return;
d9ed3e63 1146
bfbdd35b
BVA
1147 if (!success)
1148 goto unlock;
d9ed3e63
DLM
1149
1150 dprint(FD_ZBD,
1151 "%s: queued I/O (%lld, %llu) for zone %u\n",
1152 f->file_name, io_u->offset, io_u->buflen, zone_idx);
1153
bfbdd35b
BVA
1154 switch (io_u->ddir) {
1155 case DDIR_WRITE:
d9ed3e63
DLM
1156 zone_end = min((uint64_t)(io_u->offset + io_u->buflen),
1157 (z + 1)->start);
a7c2b6fc
BVA
1158 pthread_mutex_lock(&zbd_info->mutex);
1159 /*
1160 * z->wp > zone_end means that one or more I/O errors
1161 * have occurred.
1162 */
1163 if (z->wp <= zone_end)
1164 zbd_info->sectors_with_data += zone_end - z->wp;
1165 pthread_mutex_unlock(&zbd_info->mutex);
bfbdd35b
BVA
1166 z->wp = zone_end;
1167 break;
1168 case DDIR_TRIM:
1169 assert(z->wp == z->start);
1170 break;
1171 default:
1172 break;
1173 }
d9ed3e63 1174
bfbdd35b 1175unlock:
d9ed3e63
DLM
1176 if (!success || q != FIO_Q_QUEUED) {
1177 /* BUSY or COMPLETED: unlock the zone */
1178 pthread_mutex_unlock(&z->mutex);
1179 io_u->zbd_put_io = NULL;
1180 }
1181}
1182
1183/**
1184 * zbd_put_io - Unlock an I/O unit target zone lock
1185 * @io_u: I/O unit
1186 */
1187static void zbd_put_io(const struct io_u *io_u)
1188{
1189 const struct fio_file *f = io_u->file;
1190 struct zoned_block_device_info *zbd_info = f->zbd_info;
1191 struct fio_zone_info *z;
1192 uint32_t zone_idx;
1193
1194 if (!zbd_info)
1195 return;
615555bb 1196
d9ed3e63
DLM
1197 zone_idx = zbd_zone_idx(f, io_u->offset);
1198 assert(zone_idx < zbd_info->nr_zones);
1199 z = &zbd_info->zone_info[zone_idx];
1200
1201 if (z->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
1202 return;
1203
1204 dprint(FD_ZBD,
1205 "%s: terminate I/O (%lld, %llu) for zone %u\n",
1206 f->file_name, io_u->offset, io_u->buflen, zone_idx);
1207
1208 assert(pthread_mutex_unlock(&z->mutex) == 0);
1209 zbd_check_swd(f);
bfbdd35b
BVA
1210}
1211
1212bool zbd_unaligned_write(int error_code)
1213{
1214 switch (error_code) {
1215 case EIO:
1216 case EREMOTEIO:
1217 return true;
1218 }
1219 return false;
1220}
1221
1222/**
1223 * zbd_adjust_block - adjust the offset and length as necessary for ZBD drives
1224 * @td: FIO thread data.
1225 * @io_u: FIO I/O unit.
1226 *
1227 * Locking strategy: returns with z->mutex locked if and only if z refers
1228 * to a sequential zone and if io_u_accept is returned. z is the zone that
1229 * corresponds to io_u->offset at the end of this function.
1230 */
1231enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u)
1232{
1233 const struct fio_file *f = io_u->file;
1234 uint32_t zone_idx_b;
de65f7b7 1235 struct fio_zone_info *zb, *zl, *orig_zb;
bfbdd35b
BVA
1236 uint32_t orig_len = io_u->buflen;
1237 uint32_t min_bs = td->o.min_bs[io_u->ddir];
1238 uint64_t new_len;
1239 int64_t range;
1240
1241 if (!f->zbd_info)
1242 return io_u_accept;
1243
1244 assert(is_valid_offset(f, io_u->offset));
1245 assert(io_u->buflen);
1246 zone_idx_b = zbd_zone_idx(f, io_u->offset);
1247 zb = &f->zbd_info->zone_info[zone_idx_b];
de65f7b7 1248 orig_zb = zb;
bfbdd35b
BVA
1249
1250 /* Accept the I/O offset for conventional zones. */
1251 if (zb->type == BLK_ZONE_TYPE_CONVENTIONAL)
1252 return io_u_accept;
1253
1254 /*
1255 * Accept the I/O offset for reads if reading beyond the write pointer
1256 * is enabled.
1257 */
1258 if (zb->cond != BLK_ZONE_COND_OFFLINE &&
1259 io_u->ddir == DDIR_READ && td->o.read_beyond_wp)
1260 return io_u_accept;
1261
615555bb
BVA
1262 zbd_check_swd(f);
1263
6f0c6085
DLM
1264 /*
1265 * Lock the io_u target zone. The zone will be unlocked if io_u offset
1266 * is changed or when io_u completes and zbd_put_io() executed.
1267 * To avoid multiple jobs doing asynchronous I/Os from deadlocking each
1268 * other waiting for zone locks when building an io_u batch, first
1269 * only trylock the zone. If the zone is already locked by another job,
1270 * process the currently queued I/Os so that I/O progress is made and
1271 * zones unlocked.
1272 */
1273 if (pthread_mutex_trylock(&zb->mutex) != 0) {
1274 if (!td_ioengine_flagged(td, FIO_SYNCIO))
1275 io_u_quiesce(td);
1276 pthread_mutex_lock(&zb->mutex);
1277 }
1278
bfbdd35b
BVA
1279 switch (io_u->ddir) {
1280 case DDIR_READ:
1281 if (td->runstate == TD_VERIFYING) {
59b07544 1282 zb = zbd_replay_write_order(td, io_u, zb);
bfbdd35b
BVA
1283 goto accept;
1284 }
1285 /*
de65f7b7
DLM
1286 * Check that there is enough written data in the zone to do an
1287 * I/O of at least min_bs B. If there isn't, find a new zone for
1288 * the I/O.
bfbdd35b
BVA
1289 */
1290 range = zb->cond != BLK_ZONE_COND_OFFLINE ?
ee3696bd 1291 zb->wp - zb->start : 0;
de65f7b7 1292 if (range < min_bs ||
ee3696bd 1293 ((!td_random(td)) && (io_u->offset + min_bs > zb->wp))) {
bfbdd35b
BVA
1294 pthread_mutex_unlock(&zb->mutex);
1295 zl = &f->zbd_info->zone_info[zbd_zone_idx(f,
1296 f->file_offset + f->io_size)];
1297 zb = zbd_find_zone(td, io_u, zb, zl);
1298 if (!zb) {
1299 dprint(FD_ZBD,
1300 "%s: zbd_find_zone(%lld, %llu) failed\n",
1301 f->file_name, io_u->offset,
1302 io_u->buflen);
1303 goto eof;
1304 }
de65f7b7
DLM
1305 /*
1306 * zbd_find_zone() returned a zone with a range of at
1307 * least min_bs.
1308 */
ee3696bd 1309 range = zb->wp - zb->start;
de65f7b7
DLM
1310 assert(range >= min_bs);
1311
1312 if (!td_random(td))
ee3696bd 1313 io_u->offset = zb->start;
bfbdd35b 1314 }
de65f7b7
DLM
1315 /*
1316 * Make sure the I/O is within the zone valid data range while
1317 * maximizing the I/O size and preserving randomness.
1318 */
1319 if (range <= io_u->buflen)
ee3696bd 1320 io_u->offset = zb->start;
de65f7b7 1321 else if (td_random(td))
ee3696bd
DLM
1322 io_u->offset = zb->start +
1323 ((io_u->offset - orig_zb->start) %
de65f7b7
DLM
1324 (range - io_u->buflen)) / min_bs * min_bs;
1325 /*
1326 * Make sure the I/O does not cross over the zone wp position.
1327 */
1328 new_len = min((unsigned long long)io_u->buflen,
ee3696bd 1329 (unsigned long long)(zb->wp - io_u->offset));
de65f7b7
DLM
1330 new_len = new_len / min_bs * min_bs;
1331 if (new_len < io_u->buflen) {
1332 io_u->buflen = new_len;
1333 dprint(FD_IO, "Changed length from %u into %llu\n",
1334 orig_len, io_u->buflen);
bfbdd35b 1335 }
ee3696bd
DLM
1336 assert(zb->start <= io_u->offset);
1337 assert(io_u->offset + io_u->buflen <= zb->wp);
bfbdd35b
BVA
1338 goto accept;
1339 case DDIR_WRITE:
ee3696bd 1340 if (io_u->buflen > f->zbd_info->zone_size)
bfbdd35b 1341 goto eof;
59b07544
BVA
1342 if (!zbd_open_zone(td, io_u, zone_idx_b)) {
1343 pthread_mutex_unlock(&zb->mutex);
1344 zb = zbd_convert_to_open_zone(td, io_u);
1345 if (!zb)
1346 goto eof;
1347 zone_idx_b = zb - f->zbd_info->zone_info;
1348 }
a7c2b6fc
BVA
1349 /* Check whether the zone reset threshold has been exceeded */
1350 if (td->o.zrf.u.f) {
ee3696bd 1351 if (f->zbd_info->sectors_with_data >=
a7c2b6fc
BVA
1352 f->io_size * td->o.zrt.u.f &&
1353 zbd_dec_and_reset_write_cnt(td, f)) {
1354 zb->reset_zone = 1;
1355 }
1356 }
bfbdd35b
BVA
1357 /* Reset the zone pointer if necessary */
1358 if (zb->reset_zone || zbd_zone_full(f, zb, min_bs)) {
1359 assert(td->o.verify == VERIFY_NONE);
1360 /*
1361 * Since previous write requests may have been submitted
1362 * asynchronously and since we will submit the zone
1363 * reset synchronously, wait until previously submitted
1364 * write requests have completed before issuing a
1365 * zone reset.
1366 */
1367 io_u_quiesce(td);
1368 zb->reset_zone = 0;
1369 if (zbd_reset_zone(td, f, zb) < 0)
1370 goto eof;
1371 }
1372 /* Make writes occur at the write pointer */
1373 assert(!zbd_zone_full(f, zb, min_bs));
ee3696bd 1374 io_u->offset = zb->wp;
bfbdd35b
BVA
1375 if (!is_valid_offset(f, io_u->offset)) {
1376 dprint(FD_ZBD, "Dropped request with offset %llu\n",
1377 io_u->offset);
1378 goto eof;
1379 }
1380 /*
1381 * Make sure that the buflen is a multiple of the minimal
1382 * block size. Give up if shrinking would make the request too
1383 * small.
1384 */
1385 new_len = min((unsigned long long)io_u->buflen,
ee3696bd 1386 (zb + 1)->start - io_u->offset);
bfbdd35b
BVA
1387 new_len = new_len / min_bs * min_bs;
1388 if (new_len == io_u->buflen)
1389 goto accept;
1390 if (new_len >= min_bs) {
1391 io_u->buflen = new_len;
1392 dprint(FD_IO, "Changed length from %u into %llu\n",
1393 orig_len, io_u->buflen);
1394 goto accept;
1395 }
1396 log_err("Zone remainder %lld smaller than minimum block size %d\n",
ee3696bd 1397 ((zb + 1)->start - io_u->offset),
bfbdd35b
BVA
1398 min_bs);
1399 goto eof;
1400 case DDIR_TRIM:
1401 /* fall-through */
1402 case DDIR_SYNC:
1403 case DDIR_DATASYNC:
1404 case DDIR_SYNC_FILE_RANGE:
1405 case DDIR_WAIT:
1406 case DDIR_LAST:
1407 case DDIR_INVAL:
1408 goto accept;
1409 }
1410
1411 assert(false);
1412
1413accept:
1414 assert(zb);
1415 assert(zb->cond != BLK_ZONE_COND_OFFLINE);
d9ed3e63
DLM
1416 assert(!io_u->zbd_queue_io);
1417 assert(!io_u->zbd_put_io);
1418 io_u->zbd_queue_io = zbd_queue_io;
1419 io_u->zbd_put_io = zbd_put_io;
bfbdd35b
BVA
1420 return io_u_accept;
1421
1422eof:
1423 if (zb)
1424 pthread_mutex_unlock(&zb->mutex);
1425 return io_u_eof;
1426}
fd5d733f
BVA
1427
1428/* Return a string with ZBD statistics */
1429char *zbd_write_status(const struct thread_stat *ts)
1430{
1431 char *res;
1432
a0c84dd4 1433 if (asprintf(&res, "; %llu zone resets", (unsigned long long) ts->nr_zone_resets) < 0)
fd5d733f
BVA
1434 return NULL;
1435 return res;
1436}