X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=zbd.c;h=b1fd6b4bb0ae23cda7dba13a3c0bf3577139600f;hp=70afdd825f7d6b59d5f6de02645fd7e0a7352c48;hb=2b3d4a6a924e0aa82654d3b96fb134085af7a98a;hpb=b5a0f7ce303d5fa5cca51af0652b7f7cf9bc5d61 diff --git a/zbd.c b/zbd.c index 70afdd82..b1fd6b4b 100644 --- a/zbd.c +++ b/zbd.c @@ -27,19 +27,20 @@ static bool is_valid_offset(const struct fio_file *f, uint64_t offset) return (uint64_t)(offset - f->file_offset) < f->io_size; } -static inline unsigned int zbd_zone_nr(const struct fio_file *f, - struct fio_zone_info *zone) +static inline unsigned int zbd_zone_idx(const struct fio_file *f, + struct fio_zone_info *zone) { return zone - f->zbd_info->zone_info; } /** - * zbd_zone_idx - convert an offset into a zone number + * zbd_offset_to_zone_idx - convert an offset into a zone number * @f: file pointer. * @offset: offset in bytes. If this offset is in the first zone_size bytes * past the disk size then the index of the sentinel is returned. */ -static uint32_t zbd_zone_idx(const struct fio_file *f, uint64_t offset) +static unsigned int zbd_offset_to_zone_idx(const struct fio_file *f, + uint64_t offset) { uint32_t zone_idx; @@ -122,10 +123,16 @@ static inline void zone_unlock(struct fio_zone_info *z) assert(!ret); } -static inline struct fio_zone_info *get_zone(const struct fio_file *f, - unsigned int zone_nr) +static inline struct fio_zone_info *zbd_get_zone(const struct fio_file *f, + unsigned int zone_idx) { - return &f->zbd_info->zone_info[zone_nr]; + return &f->zbd_info->zone_info[zone_idx]; +} + +static inline struct fio_zone_info * +zbd_offset_to_zone(const struct fio_file *f, uint64_t offset) +{ + return zbd_get_zone(f, zbd_offset_to_zone_idx(f, offset)); } /** @@ -252,8 +259,9 @@ static int zbd_reset_zone(struct thread_data *td, struct fio_file *f, assert(is_valid_offset(f, offset + length - 1)); - dprint(FD_ZBD, "%s: resetting wp of zone %u.\n", f->file_name, - zbd_zone_nr(f, z)); + dprint(FD_ZBD, "%s: resetting wp of zone %u.\n", + f->file_name, zbd_zone_idx(f, z)); + switch (f->zbd_info->model) { case ZBD_HOST_AWARE: case ZBD_HOST_MANAGED: @@ -269,6 +277,7 @@ static int zbd_reset_zone(struct thread_data *td, struct fio_file *f, f->zbd_info->sectors_with_data -= data_in_zone; f->zbd_info->wp_sectors_with_data -= data_in_zone; pthread_mutex_unlock(&f->zbd_info->mutex); + z->wp = z->start; z->verify_block = 0; @@ -286,25 +295,31 @@ static int zbd_reset_zone(struct thread_data *td, struct fio_file *f, * The caller must hold f->zbd_info->mutex. */ static void zbd_close_zone(struct thread_data *td, const struct fio_file *f, - unsigned int zone_idx) + struct fio_zone_info *z) { - uint32_t open_zone_idx = 0; + uint32_t ozi; - for (; open_zone_idx < f->zbd_info->num_open_zones; open_zone_idx++) { - if (f->zbd_info->open_zones[open_zone_idx] == zone_idx) + if (!z->open) + return; + + for (ozi = 0; ozi < f->zbd_info->num_open_zones; ozi++) { + if (zbd_get_zone(f, f->zbd_info->open_zones[ozi]) == z) break; } - if (open_zone_idx == f->zbd_info->num_open_zones) + if (ozi == f->zbd_info->num_open_zones) return; - dprint(FD_ZBD, "%s: closing zone %d\n", f->file_name, zone_idx); - memmove(f->zbd_info->open_zones + open_zone_idx, - f->zbd_info->open_zones + open_zone_idx + 1, - (ZBD_MAX_OPEN_ZONES - (open_zone_idx + 1)) * + dprint(FD_ZBD, "%s: closing zone %u\n", + f->file_name, zbd_zone_idx(f, z)); + + memmove(f->zbd_info->open_zones + ozi, + f->zbd_info->open_zones + ozi + 1, + (ZBD_MAX_OPEN_ZONES - (ozi + 1)) * sizeof(f->zbd_info->open_zones[0])); + f->zbd_info->num_open_zones--; td->num_open_zones--; - get_zone(f, zone_idx)->open = 0; + z->open = 0; } /** @@ -326,23 +341,25 @@ static int zbd_reset_zones(struct thread_data *td, struct fio_file *f, assert(min_bs); - dprint(FD_ZBD, "%s: examining zones %u .. %u\n", f->file_name, - zbd_zone_nr(f, zb), zbd_zone_nr(f, ze)); - for (z = zb; z < ze; z++) { - uint32_t nz = zbd_zone_nr(f, z); + dprint(FD_ZBD, "%s: examining zones %u .. %u\n", + f->file_name, zbd_zone_idx(f, zb), zbd_zone_idx(f, ze)); + for (z = zb; z < ze; z++) { if (!z->has_wp) continue; + zone_lock(td, f, z); pthread_mutex_lock(&f->zbd_info->mutex); - zbd_close_zone(td, f, nz); + zbd_close_zone(td, f, z); pthread_mutex_unlock(&f->zbd_info->mutex); + if (z->wp != z->start) { dprint(FD_ZBD, "%s: resetting zone %u\n", - f->file_name, zbd_zone_nr(f, z)); + f->file_name, zbd_zone_idx(f, z)); if (zbd_reset_zone(td, f, z) < 0) res = 1; } + zone_unlock(z); } @@ -390,11 +407,11 @@ static int zbd_get_max_open_zones(struct thread_data *td, struct fio_file *f, * to be exceeded. */ static bool zbd_open_zone(struct thread_data *td, const struct fio_file *f, - uint32_t zone_idx) + struct fio_zone_info *z) { const uint64_t min_bs = td->o.min_bs[DDIR_WRITE]; struct zoned_block_device_info *zbdi = f->zbd_info; - struct fio_zone_info *z = get_zone(f, zone_idx); + uint32_t zone_idx = zbd_zone_idx(f, z); bool res = true; if (z->cond == ZBD_ZONE_COND_OFFLINE) @@ -427,6 +444,7 @@ static bool zbd_open_zone(struct thread_data *td, const struct fio_file *f, res = false; goto out; } + res = false; /* Zero means no limit */ if (td->o.job_max_open_zones > 0 && @@ -434,7 +452,10 @@ static bool zbd_open_zone(struct thread_data *td, const struct fio_file *f, goto out; if (zbdi->num_open_zones >= zbdi->max_open_zones) goto out; - dprint(FD_ZBD, "%s: opening zone %d\n", f->file_name, zone_idx); + + dprint(FD_ZBD, "%s: opening zone %u\n", + f->file_name, zone_idx); + zbdi->open_zones[zbdi->num_open_zones++] = zone_idx; td->num_open_zones++; z->open = 1; @@ -471,90 +492,106 @@ static bool zbd_is_seq_job(struct fio_file *f) uint32_t zone_idx, zone_idx_b, zone_idx_e; assert(f->zbd_info); + if (f->io_size == 0) return false; - zone_idx_b = zbd_zone_idx(f, f->file_offset); - zone_idx_e = zbd_zone_idx(f, f->file_offset + f->io_size - 1); + + zone_idx_b = zbd_offset_to_zone_idx(f, f->file_offset); + zone_idx_e = + zbd_offset_to_zone_idx(f, f->file_offset + f->io_size - 1); for (zone_idx = zone_idx_b; zone_idx <= zone_idx_e; zone_idx++) - if (get_zone(f, zone_idx)->has_wp) + if (zbd_get_zone(f, zone_idx)->has_wp) return true; return false; } +/* + * Verify whether the file offset and size parameters are aligned with zone + * boundaries. If the file offset is not aligned, align it down to the start of + * the zone containing the start offset and align up the file io_size parameter. + */ +static bool zbd_zone_align_file_sizes(struct thread_data *td, + struct fio_file *f) +{ + const struct fio_zone_info *z; + uint64_t new_offset, new_end; + + if (!f->zbd_info) + return true; + if (f->file_offset >= f->real_file_size) + return true; + if (!zbd_is_seq_job(f)) + return true; + + if (!td->o.zone_size) { + td->o.zone_size = f->zbd_info->zone_size; + if (!td->o.zone_size) { + log_err("%s: invalid 0 zone size\n", + f->file_name); + return false; + } + } else if (td->o.zone_size != f->zbd_info->zone_size) { + log_err("%s: zonesize %llu does not match the device zone size %"PRIu64".\n", + f->file_name, td->o.zone_size, + f->zbd_info->zone_size); + return false; + } + + if (td->o.zone_skip % td->o.zone_size) { + log_err("%s: zoneskip %llu is not a multiple of the device zone size %llu.\n", + f->file_name, td->o.zone_skip, + td->o.zone_size); + return false; + } + + z = zbd_offset_to_zone(f, f->file_offset); + if ((f->file_offset != z->start) && + (td->o.td_ddir != TD_DDIR_READ)) { + new_offset = zbd_zone_end(z); + if (new_offset >= f->file_offset + f->io_size) { + log_info("%s: io_size must be at least one zone\n", + f->file_name); + return false; + } + log_info("%s: rounded up offset from %"PRIu64" to %"PRIu64"\n", + f->file_name, f->file_offset, + new_offset); + f->io_size -= (new_offset - f->file_offset); + f->file_offset = new_offset; + } + + z = zbd_offset_to_zone(f, f->file_offset + f->io_size); + new_end = z->start; + if ((td->o.td_ddir != TD_DDIR_READ) && + (f->file_offset + f->io_size != new_end)) { + if (new_end <= f->file_offset) { + log_info("%s: io_size must be at least one zone\n", + f->file_name); + return false; + } + log_info("%s: rounded down io_size from %"PRIu64" to %"PRIu64"\n", + f->file_name, f->io_size, + new_end - f->file_offset); + f->io_size = new_end - f->file_offset; + } + + return true; +} + /* * Verify whether offset and size parameters are aligned with zone boundaries. */ static bool zbd_verify_sizes(void) { - const struct fio_zone_info *z; struct thread_data *td; struct fio_file *f; - uint64_t new_offset, new_end; - uint32_t zone_idx; int i, j; for_each_td(td, i) { for_each_file(td, f, j) { - if (!f->zbd_info) - continue; - if (f->file_offset >= f->real_file_size) - continue; - if (!zbd_is_seq_job(f)) - continue; - - if (!td->o.zone_size) { - td->o.zone_size = f->zbd_info->zone_size; - if (!td->o.zone_size) { - log_err("%s: invalid 0 zone size\n", - f->file_name); - return false; - } - } else if (td->o.zone_size != f->zbd_info->zone_size) { - log_err("%s: job parameter zonesize %llu does not match disk zone size %"PRIu64".\n", - f->file_name, td->o.zone_size, - f->zbd_info->zone_size); + if (!zbd_zone_align_file_sizes(td, f)) return false; - } - - if (td->o.zone_skip % td->o.zone_size) { - log_err("%s: zoneskip %llu is not a multiple of the device zone size %llu.\n", - f->file_name, td->o.zone_skip, - td->o.zone_size); - return false; - } - - zone_idx = zbd_zone_idx(f, f->file_offset); - z = get_zone(f, zone_idx); - if ((f->file_offset != z->start) && - (td->o.td_ddir != TD_DDIR_READ)) { - new_offset = zbd_zone_end(z); - if (new_offset >= f->file_offset + f->io_size) { - log_info("%s: io_size must be at least one zone\n", - f->file_name); - return false; - } - log_info("%s: rounded up offset from %"PRIu64" to %"PRIu64"\n", - f->file_name, f->file_offset, - new_offset); - f->io_size -= (new_offset - f->file_offset); - f->file_offset = new_offset; - } - zone_idx = zbd_zone_idx(f, f->file_offset + f->io_size); - z = get_zone(f, zone_idx); - new_end = z->start; - if ((td->o.td_ddir != TD_DDIR_READ) && - (f->file_offset + f->io_size != new_end)) { - if (new_end <= f->file_offset) { - log_info("%s: io_size must be at least one zone\n", - f->file_name); - return false; - } - log_info("%s: rounded down io_size from %"PRIu64" to %"PRIu64"\n", - f->file_name, f->io_size, - new_end - f->file_offset); - f->io_size = new_end - f->file_offset; - } } } @@ -579,6 +616,7 @@ static bool zbd_verify_bs(void) if (!f->zbd_info) continue; + zone_size = f->zbd_info->zone_size; if (td_trim(td) && td->o.bs[DDIR_TRIM] != zone_size) { log_info("%s: trim block size %llu is not the zone size %"PRIu64"\n", @@ -723,8 +761,8 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f) goto out; } - dprint(FD_ZBD, "Device %s has %d zones of size %"PRIu64" KB\n", f->file_name, - nr_zones, zone_size / 1024); + dprint(FD_ZBD, "Device %s has %d zones of size %"PRIu64" KB\n", + f->file_name, nr_zones, zone_size / 1024); zbd_info = scalloc(1, sizeof(*zbd_info) + (nr_zones + 1) * sizeof(zbd_info->zone_info[0])); @@ -740,6 +778,7 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f) PTHREAD_MUTEX_RECURSIVE); p->start = z->start; p->capacity = z->capacity; + switch (z->cond) { case ZBD_ZONE_COND_NOT_WP: case ZBD_ZONE_COND_FULL: @@ -773,6 +812,7 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f) offset = z->start + z->len; if (j >= nr_zones) break; + nrz = zbd_report_zones(td, f, offset, zones, min((uint32_t)(nr_zones - j), ZBD_REPORT_MAX_ZONES)); @@ -840,7 +880,8 @@ out: /* Ensure that the limit is not larger than FIO's internal limit */ if (zbd->max_open_zones > ZBD_MAX_OPEN_ZONES) { td_verror(td, EINVAL, "'max_open_zones' value is too large"); - log_err("'max_open_zones' value is larger than %u\n", ZBD_MAX_OPEN_ZONES); + log_err("'max_open_zones' value is larger than %u\n", + ZBD_MAX_OPEN_ZONES); return -EINVAL; } @@ -942,6 +983,7 @@ static int zbd_init_zone_info(struct thread_data *td, struct fio_file *file) ret = zbd_create_zone_info(td, file); if (ret < 0) td_verror(td, -ret, "zbd_create_zone_info() failed"); + return ret; } @@ -954,6 +996,7 @@ int zbd_init_files(struct thread_data *td) if (zbd_init_zone_info(td, f)) return 1; } + return 0; } @@ -964,27 +1007,24 @@ void zbd_recalc_options_with_zone_granularity(struct thread_data *td) for_each_file(td, f, i) { struct zoned_block_device_info *zbd = f->zbd_info; - // zonemode=strided doesn't get per-file zone size. - uint64_t zone_size = zbd ? zbd->zone_size : td->o.zone_size; + uint64_t zone_size; + /* zonemode=strided doesn't get per-file zone size. */ + zone_size = zbd ? zbd->zone_size : td->o.zone_size; if (zone_size == 0) continue; - if (td->o.size_nz > 0) { + if (td->o.size_nz > 0) td->o.size = td->o.size_nz * zone_size; - } - if (td->o.io_size_nz > 0) { + if (td->o.io_size_nz > 0) td->o.io_size = td->o.io_size_nz * zone_size; - } - if (td->o.start_offset_nz > 0) { + if (td->o.start_offset_nz > 0) td->o.start_offset = td->o.start_offset_nz * zone_size; - } - if (td->o.offset_increment_nz > 0) { - td->o.offset_increment = td->o.offset_increment_nz * zone_size; - } - if (td->o.zone_skip_nz > 0) { + if (td->o.offset_increment_nz > 0) + td->o.offset_increment = + td->o.offset_increment_nz * zone_size; + if (td->o.zone_skip_nz > 0) td->o.zone_skip = td->o.zone_skip_nz * zone_size; - } } } @@ -1011,8 +1051,9 @@ int zbd_setup_files(struct thread_data *td) assert(zbd); - f->min_zone = zbd_zone_idx(f, f->file_offset); - f->max_zone = zbd_zone_idx(f, f->file_offset + f->io_size); + f->min_zone = zbd_offset_to_zone_idx(f, f->file_offset); + f->max_zone = + zbd_offset_to_zone_idx(f, f->file_offset + f->io_size); /* * When all zones in the I/O range are conventional, io_size @@ -1052,7 +1093,7 @@ int zbd_setup_files(struct thread_data *td) if (z->cond != ZBD_ZONE_COND_IMP_OPEN && z->cond != ZBD_ZONE_COND_EXP_OPEN) continue; - if (zbd_open_zone(td, f, zi)) + if (zbd_open_zone(td, f, z)) continue; /* * If the number of open zones exceeds specified limits, @@ -1118,8 +1159,8 @@ static uint64_t zbd_process_swd(struct thread_data *td, uint64_t swd = 0; uint64_t wp_swd = 0; - zb = get_zone(f, f->min_zone); - ze = get_zone(f, f->max_zone); + zb = zbd_get_zone(f, f->min_zone); + ze = zbd_get_zone(f, f->max_zone); for (z = zb; z < ze; z++) { if (z->has_wp) { zone_lock(td, f, z); @@ -1127,6 +1168,7 @@ static uint64_t zbd_process_swd(struct thread_data *td, } swd += z->wp - z->start; } + pthread_mutex_lock(&f->zbd_info->mutex); switch (a) { case CHECK_SWD: @@ -1139,6 +1181,7 @@ static uint64_t zbd_process_swd(struct thread_data *td, break; } pthread_mutex_unlock(&f->zbd_info->mutex); + for (z = zb; z < ze; z++) if (z->has_wp) zone_unlock(z); @@ -1169,11 +1212,13 @@ void zbd_file_reset(struct thread_data *td, struct fio_file *f) if (!f->zbd_info || !td_write(td)) return; - zb = get_zone(f, f->min_zone); - ze = get_zone(f, f->max_zone); + zb = zbd_get_zone(f, f->min_zone); + ze = zbd_get_zone(f, f->max_zone); swd = zbd_process_swd(td, f, SET_SWD); - dprint(FD_ZBD, "%s(%s): swd = %" PRIu64 "\n", __func__, f->file_name, - swd); + + dprint(FD_ZBD, "%s(%s): swd = %" PRIu64 "\n", + __func__, f->file_name, swd); + /* * If data verification is enabled reset the affected zones before * writing any data to avoid that a zone reset has to be issued while @@ -1188,8 +1233,8 @@ void zbd_file_reset(struct thread_data *td, struct fio_file *f) static uint32_t pick_random_zone_idx(const struct fio_file *f, const struct io_u *io_u) { - return (io_u->offset - f->file_offset) * f->zbd_info->num_open_zones / - f->io_size; + return (io_u->offset - f->file_offset) * + f->zbd_info->num_open_zones / f->io_size; } static bool any_io_in_flight(void) @@ -1236,13 +1281,15 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, */ zone_idx = zbdi->open_zones[pick_random_zone_idx(f, io_u)]; } else { - zone_idx = zbd_zone_idx(f, io_u->offset); + zone_idx = zbd_offset_to_zone_idx(f, io_u->offset); } if (zone_idx < f->min_zone) zone_idx = f->min_zone; else if (zone_idx >= f->max_zone) zone_idx = f->max_zone - 1; - dprint(FD_ZBD, "%s(%s): starting from zone %d (offset %lld, buflen %lld)\n", + + dprint(FD_ZBD, + "%s(%s): starting from zone %d (offset %lld, buflen %lld)\n", __func__, f->file_name, zone_idx, io_u->offset, io_u->buflen); /* @@ -1254,13 +1301,16 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, for (;;) { uint32_t tmp_idx; - z = get_zone(f, zone_idx); + z = zbd_get_zone(f, zone_idx); if (z->has_wp) zone_lock(td, f, z); + pthread_mutex_lock(&zbdi->mutex); + if (z->has_wp) { if (z->cond != ZBD_ZONE_COND_OFFLINE && - zbdi->max_open_zones == 0 && td->o.job_max_open_zones == 0) + zbdi->max_open_zones == 0 && + td->o.job_max_open_zones == 0) goto examine_zone; if (zbdi->num_open_zones == 0) { dprint(FD_ZBD, "%s(%s): no zones are open\n", @@ -1270,14 +1320,15 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, } /* - * List of opened zones is per-device, shared across all threads. - * Start with quasi-random candidate zone. - * Ignore zones which don't belong to thread's offset/size area. + * List of opened zones is per-device, shared across all + * threads. Start with quasi-random candidate zone. Ignore + * zones which don't belong to thread's offset/size area. */ open_zone_idx = pick_random_zone_idx(f, io_u); assert(!open_zone_idx || open_zone_idx < zbdi->num_open_zones); tmp_idx = open_zone_idx; + for (i = 0; i < zbdi->num_open_zones; i++) { uint32_t tmpz; @@ -1294,9 +1345,12 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, dprint(FD_ZBD, "%s(%s): no candidate zone\n", __func__, f->file_name); + pthread_mutex_unlock(&zbdi->mutex); + if (z->has_wp) zone_unlock(z); + return NULL; found_candidate_zone: @@ -1304,7 +1358,9 @@ found_candidate_zone: if (new_zone_idx == zone_idx) break; zone_idx = new_zone_idx; + pthread_mutex_unlock(&zbdi->mutex); + if (z->has_wp) zone_unlock(z); } @@ -1335,7 +1391,8 @@ open_other_zone: * zone close before opening a new zone. */ if (wait_zone_close) { - dprint(FD_ZBD, "%s(%s): quiesce to allow open zones to close\n", + dprint(FD_ZBD, + "%s(%s): quiesce to allow open zones to close\n", __func__, f->file_name); io_u_quiesce(td); } @@ -1350,7 +1407,7 @@ retry: if (!is_valid_offset(f, z->start)) { /* Wrap-around. */ zone_idx = f->min_zone; - z = get_zone(f, zone_idx); + z = zbd_get_zone(f, zone_idx); } assert(is_valid_offset(f, z->start)); if (!z->has_wp) @@ -1358,7 +1415,7 @@ retry: zone_lock(td, f, z); if (z->open) continue; - if (zbd_open_zone(td, f, zone_idx)) + if (zbd_open_zone(td, f, z)) goto out; } @@ -1373,7 +1430,7 @@ retry: pthread_mutex_unlock(&zbdi->mutex); zone_unlock(z); - z = get_zone(f, zone_idx); + z = zbd_get_zone(f, zone_idx); zone_lock(td, f, z); if (z->wp + min_bs <= zbd_zone_capacity_end(z)) @@ -1388,7 +1445,8 @@ retry: */ in_flight = any_io_in_flight(); if (in_flight || should_retry) { - dprint(FD_ZBD, "%s(%s): wait zone close and retry open zones\n", + dprint(FD_ZBD, + "%s(%s): wait zone close and retry open zones\n", __func__, f->file_name); pthread_mutex_unlock(&zbdi->mutex); zone_unlock(z); @@ -1399,17 +1457,22 @@ retry: } pthread_mutex_unlock(&zbdi->mutex); + zone_unlock(z); - dprint(FD_ZBD, "%s(%s): did not open another zone\n", __func__, - f->file_name); + + dprint(FD_ZBD, "%s(%s): did not open another zone\n", + __func__, f->file_name); + return NULL; out: - dprint(FD_ZBD, "%s(%s): returning zone %d\n", __func__, f->file_name, - zone_idx); + dprint(FD_ZBD, "%s(%s): returning zone %d\n", + __func__, f->file_name, zone_idx); + io_u->offset = z->start; assert(z->has_wp); assert(z->cond != ZBD_ZONE_COND_OFFLINE); + return z; } @@ -1421,25 +1484,27 @@ static struct fio_zone_info *zbd_replay_write_order(struct thread_data *td, const struct fio_file *f = io_u->file; const uint64_t min_bs = td->o.min_bs[DDIR_WRITE]; - if (!zbd_open_zone(td, f, zbd_zone_nr(f, z))) { + if (!zbd_open_zone(td, f, z)) { zone_unlock(z); z = zbd_convert_to_open_zone(td, io_u); assert(z); } if (z->verify_block * min_bs >= z->capacity) { - log_err("%s: %d * %"PRIu64" >= %"PRIu64"\n", f->file_name, z->verify_block, - min_bs, z->capacity); + log_err("%s: %d * %"PRIu64" >= %"PRIu64"\n", + f->file_name, z->verify_block, min_bs, z->capacity); /* * If the assertion below fails during a test run, adding * "--experimental_verify=1" to the command line may help. */ assert(false); } + io_u->offset = z->start + z->verify_block * min_bs; if (io_u->offset + io_u->buflen >= zbd_zone_capacity_end(z)) { - log_err("%s: %llu + %llu >= %"PRIu64"\n", f->file_name, io_u->offset, - io_u->buflen, zbd_zone_capacity_end(z)); + log_err("%s: %llu + %llu >= %"PRIu64"\n", + f->file_name, io_u->offset, io_u->buflen, + zbd_zone_capacity_end(z)); assert(false); } z->verify_block += io_u->buflen / min_bs; @@ -1460,7 +1525,7 @@ zbd_find_zone(struct thread_data *td, struct io_u *io_u, uint64_t min_bytes, { struct fio_file *f = io_u->file; struct fio_zone_info *z1, *z2; - const struct fio_zone_info *const zf = get_zone(f, f->min_zone); + const struct fio_zone_info *const zf = zbd_get_zone(f, f->min_zone); /* * Skip to the next non-empty zone in case of sequential I/O and to @@ -1477,6 +1542,7 @@ zbd_find_zone(struct thread_data *td, struct io_u *io_u, uint64_t min_bytes, } else if (!td_random(td)) { break; } + if (td_random(td) && z2 >= zf && z2->cond != ZBD_ZONE_COND_OFFLINE) { if (z2->has_wp) @@ -1487,8 +1553,11 @@ zbd_find_zone(struct thread_data *td, struct io_u *io_u, uint64_t min_bytes, zone_unlock(z2); } } - dprint(FD_ZBD, "%s: no zone has %"PRIu64" bytes of readable data\n", + + dprint(FD_ZBD, + "%s: no zone has %"PRIu64" bytes of readable data\n", f->file_name, min_bytes); + return NULL; } @@ -1509,7 +1578,7 @@ static void zbd_end_zone_io(struct thread_data *td, const struct io_u *io_u, if (io_u->ddir == DDIR_WRITE && io_u->offset + io_u->buflen >= zbd_zone_capacity_end(z)) { pthread_mutex_lock(&f->zbd_info->mutex); - zbd_close_zone(td, f, zbd_zone_nr(f, z)); + zbd_close_zone(td, f, z); pthread_mutex_unlock(&f->zbd_info->mutex); } } @@ -1529,15 +1598,11 @@ static void zbd_queue_io(struct thread_data *td, struct io_u *io_u, int q, const struct fio_file *f = io_u->file; struct zoned_block_device_info *zbd_info = f->zbd_info; struct fio_zone_info *z; - uint32_t zone_idx; uint64_t zone_end; assert(zbd_info); - zone_idx = zbd_zone_idx(f, io_u->offset); - assert(zone_idx < zbd_info->nr_zones); - z = get_zone(f, zone_idx); - + z = zbd_offset_to_zone(f, io_u->offset); assert(z->has_wp); if (!success) @@ -1545,17 +1610,18 @@ static void zbd_queue_io(struct thread_data *td, struct io_u *io_u, int q, dprint(FD_ZBD, "%s: queued I/O (%lld, %llu) for zone %u\n", - f->file_name, io_u->offset, io_u->buflen, zone_idx); + f->file_name, io_u->offset, io_u->buflen, zbd_zone_idx(f, z)); switch (io_u->ddir) { case DDIR_WRITE: zone_end = min((uint64_t)(io_u->offset + io_u->buflen), zbd_zone_capacity_end(z)); - pthread_mutex_lock(&zbd_info->mutex); + /* * z->wp > zone_end means that one or more I/O errors * have occurred. */ + pthread_mutex_lock(&zbd_info->mutex); if (z->wp <= zone_end) { zbd_info->sectors_with_data += zone_end - z->wp; zbd_info->wp_sectors_with_data += zone_end - z->wp; @@ -1587,19 +1653,15 @@ static void zbd_put_io(struct thread_data *td, const struct io_u *io_u) const struct fio_file *f = io_u->file; struct zoned_block_device_info *zbd_info = f->zbd_info; struct fio_zone_info *z; - uint32_t zone_idx; assert(zbd_info); - zone_idx = zbd_zone_idx(f, io_u->offset); - assert(zone_idx < zbd_info->nr_zones); - z = get_zone(f, zone_idx); - + z = zbd_offset_to_zone(f, io_u->offset); assert(z->has_wp); dprint(FD_ZBD, "%s: terminate I/O (%lld, %llu) for zone %u\n", - f->file_name, io_u->offset, io_u->buflen, zone_idx); + f->file_name, io_u->offset, io_u->buflen, zbd_zone_idx(f, z)); zbd_end_zone_io(td, io_u, z); @@ -1641,28 +1703,26 @@ void setup_zbd_zone_mode(struct thread_data *td, struct io_u *io_u) struct fio_file *f = io_u->file; enum fio_ddir ddir = io_u->ddir; struct fio_zone_info *z; - uint32_t zone_idx; assert(td->o.zone_mode == ZONE_MODE_ZBD); assert(td->o.zone_size); assert(f->zbd_info); - zone_idx = zbd_zone_idx(f, f->last_pos[ddir]); - z = get_zone(f, zone_idx); + z = zbd_offset_to_zone(f, f->last_pos[ddir]); /* * When the zone capacity is smaller than the zone size and the I/O is * sequential write, skip to zone end if the latest position is at the * zone capacity limit. */ - if (z->capacity < f->zbd_info->zone_size && !td_random(td) && - ddir == DDIR_WRITE && + if (z->capacity < f->zbd_info->zone_size && + !td_random(td) && ddir == DDIR_WRITE && f->last_pos[ddir] >= zbd_zone_capacity_end(z)) { dprint(FD_ZBD, "%s: Jump from zone capacity limit to zone end:" " (%"PRIu64" -> %"PRIu64") for zone %u (%"PRIu64")\n", f->file_name, f->last_pos[ddir], - zbd_zone_end(z), zone_idx, z->capacity); + zbd_zone_end(z), zbd_zone_idx(f, z), z->capacity); td->io_skip_bytes += zbd_zone_end(z) - f->last_pos[ddir]; f->last_pos[ddir] = zbd_zone_end(z); } @@ -1743,7 +1803,6 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) { struct fio_file *f = io_u->file; struct zoned_block_device_info *zbdi = f->zbd_info; - uint32_t zone_idx_b; struct fio_zone_info *zb, *zl, *orig_zb; uint32_t orig_len = io_u->buflen; uint64_t min_bs = td->o.min_bs[io_u->ddir]; @@ -1754,14 +1813,15 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) assert(min_bs); assert(is_valid_offset(f, io_u->offset)); assert(io_u->buflen); - zone_idx_b = zbd_zone_idx(f, io_u->offset); - zb = get_zone(f, zone_idx_b); + + zb = zbd_offset_to_zone(f, io_u->offset); orig_zb = zb; if (!zb->has_wp) { /* Accept non-write I/Os for conventional zones. */ if (io_u->ddir != DDIR_WRITE) return io_u_accept; + /* * Make sure that writes to conventional zones * don't cross over to any sequential zones. @@ -1775,12 +1835,16 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) "%s: off=%llu + min_bs=%"PRIu64" > next zone %"PRIu64"\n", f->file_name, io_u->offset, min_bs, (zb + 1)->start); - io_u->offset = zb->start + (zb + 1)->start - io_u->offset; - new_len = min(io_u->buflen, (zb + 1)->start - io_u->offset); + io_u->offset = + zb->start + (zb + 1)->start - io_u->offset; + new_len = min(io_u->buflen, + (zb + 1)->start - io_u->offset); } else { new_len = (zb + 1)->start - io_u->offset; } + io_u->buflen = new_len / min_bs * min_bs; + return io_u_accept; } @@ -1802,6 +1866,7 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) zb = zbd_replay_write_order(td, io_u, zb); goto accept; } + /* * Check that there is enough written data in the zone to do an * I/O of at least min_bs B. If there isn't, find a new zone for @@ -1812,7 +1877,7 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) if (range < min_bs || ((!td_random(td)) && (io_u->offset + min_bs > zb->wp))) { zone_unlock(zb); - zl = get_zone(f, f->max_zone); + zl = zbd_get_zone(f, f->max_zone); zb = zbd_find_zone(td, io_u, min_bs, zb, zl); if (!zb) { dprint(FD_ZBD, @@ -1831,6 +1896,7 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) if (!td_random(td)) io_u->offset = zb->start; } + /* * Make sure the I/O is within the zone valid data range while * maximizing the I/O size and preserving randomness. @@ -1841,12 +1907,14 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) io_u->offset = zb->start + ((io_u->offset - orig_zb->start) % (range - io_u->buflen)) / min_bs * min_bs; + /* * When zbd_find_zone() returns a conventional zone, * we can simply accept the new i/o offset here. */ if (!zb->has_wp) return io_u_accept; + /* * Make sure the I/O does not cross over the zone wp position. */ @@ -1858,9 +1926,12 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) dprint(FD_IO, "Changed length from %u into %llu\n", orig_len, io_u->buflen); } + assert(zb->start <= io_u->offset); assert(io_u->offset + io_u->buflen <= zb->wp); + goto accept; + case DDIR_WRITE: if (io_u->buflen > zbdi->zone_size) { td_verror(td, EINVAL, "I/O buflen exceeds zone size"); @@ -1869,7 +1940,8 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) f->file_name, io_u->buflen, zbdi->zone_size); goto eof; } - if (!zbd_open_zone(td, f, zone_idx_b)) { + + if (!zbd_open_zone(td, f, zb)) { zone_unlock(zb); zb = zbd_convert_to_open_zone(td, io_u); if (!zb) { @@ -1878,14 +1950,14 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) goto eof; } } + /* Check whether the zone reset threshold has been exceeded */ if (td->o.zrf.u.f) { - if (zbdi->wp_sectors_with_data >= - f->io_size * td->o.zrt.u.f && - zbd_dec_and_reset_write_cnt(td, f)) { + if (zbdi->wp_sectors_with_data >= f->io_size * td->o.zrt.u.f && + zbd_dec_and_reset_write_cnt(td, f)) zb->reset_zone = 1; - } } + /* Reset the zone pointer if necessary */ if (zb->reset_zone || zbd_zone_full(f, zb, min_bs)) { assert(td->o.verify == VERIFY_NONE); @@ -1908,6 +1980,7 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) goto eof; } } + /* Make writes occur at the write pointer */ assert(!zbd_zone_full(f, zb, min_bs)); io_u->offset = zb->wp; @@ -1917,6 +1990,7 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) f->file_name, io_u->offset); goto eof; } + /* * Make sure that the buflen is a multiple of the minimal * block size. Give up if shrinking would make the request too @@ -1933,10 +2007,13 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) orig_len, io_u->buflen); goto accept; } + td_verror(td, EIO, "zone remainder too small"); log_err("zone remainder %lld smaller than min block size %"PRIu64"\n", (zbd_zone_capacity_end(zb) - io_u->offset), min_bs); + goto eof; + case DDIR_TRIM: /* Check random trim targets a non-empty zone */ if (!td_random(td) || zb->wp > zb->start) @@ -1944,7 +2021,7 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) /* Find out a non-empty zone to trim */ zone_unlock(zb); - zl = get_zone(f, f->max_zone); + zl = zbd_get_zone(f, f->max_zone); zb = zbd_find_zone(td, io_u, 1, zb, zl); if (zb) { io_u->offset = zb->start; @@ -1952,7 +2029,9 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) f->file_name, io_u->offset); goto accept; } + goto eof; + case DDIR_SYNC: /* fall-through */ case DDIR_DATASYNC: @@ -1970,19 +2049,23 @@ accept: assert(zb->cond != ZBD_ZONE_COND_OFFLINE); assert(!io_u->zbd_queue_io); assert(!io_u->zbd_put_io); + io_u->zbd_queue_io = zbd_queue_io; io_u->zbd_put_io = zbd_put_io; + /* * Since we return with the zone lock still held, * add an annotation to let Coverity know that it * is intentional. */ /* coverity[missing_unlock] */ + return io_u_accept; eof: if (zb && zb->has_wp) zone_unlock(zb); + return io_u_eof; } @@ -2010,17 +2093,15 @@ int zbd_do_io_u_trim(const struct thread_data *td, struct io_u *io_u) { struct fio_file *f = io_u->file; struct fio_zone_info *z; - uint32_t zone_idx; int ret; - zone_idx = zbd_zone_idx(f, io_u->offset); - z = get_zone(f, zone_idx); - + z = zbd_offset_to_zone(f, io_u->offset); if (!z->has_wp) return 0; if (io_u->offset != z->start) { - log_err("Trim offset not at zone start (%lld)\n", io_u->offset); + log_err("Trim offset not at zone start (%lld)\n", + io_u->offset); return -EINVAL; }