X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=zbd.c;h=cf2cded93cff4371263106d3e7c0c9c02a6640c0;hb=0123751ca55fe462e8888c8d7b8c7def14b74ab3;hp=2a324d3415ee72aaf12b894fc9d49ce0ede2d744;hpb=6c5b11d3d31fc6a9ae6ba8602ffc03e729476f57;p=fio.git diff --git a/zbd.c b/zbd.c index 2a324d34..cf2cded9 100644 --- a/zbd.c +++ b/zbd.c @@ -7,11 +7,11 @@ #include #include #include -#include #include #include #include +#include "os/os.h" #include "file.h" #include "fio.h" #include "lib/pow2.h" @@ -19,6 +19,7 @@ #include "oslib/asprintf.h" #include "smalloc.h" #include "verify.h" +#include "pshared.h" #include "zbd.h" /** @@ -156,8 +157,14 @@ static bool zbd_zone_full(const struct fio_file *f, struct fio_zone_info *z, z->wp + required > z->start + f->zbd_info->zone_size; } -static void zone_lock(struct thread_data *td, struct fio_zone_info *z) +static void zone_lock(struct thread_data *td, struct fio_file *f, struct fio_zone_info *z) { + struct zoned_block_device_info *zbd = f->zbd_info; + uint32_t nz = z - zbd->zone_info; + + /* A thread should never lock zones outside its working area. */ + assert(f->min_zone <= nz && nz < f->max_zone); + /* * Lock the io_u target zone. The zone will be unlocked if io_u offset * is changed or when io_u completes and zbd_put_io() executed. @@ -262,7 +269,8 @@ static bool zbd_verify_sizes(void) zone_idx = zbd_zone_idx(f, f->file_offset); z = &f->zbd_info->zone_info[zone_idx]; - if (f->file_offset != z->start) { + if ((f->file_offset != z->start) && + (td->o.td_ddir != TD_DDIR_READ)) { new_offset = (z+1)->start; if (new_offset >= f->file_offset + f->io_size) { log_info("%s: io_size must be at least one zone\n", @@ -278,7 +286,8 @@ static bool zbd_verify_sizes(void) zone_idx = zbd_zone_idx(f, f->file_offset + f->io_size); z = &f->zbd_info->zone_info[zone_idx]; new_end = z->start; - if (f->file_offset + f->io_size != new_end) { + if ((td->o.td_ddir != TD_DDIR_READ) && + (f->file_offset + f->io_size != new_end)) { if (new_end <= f->file_offset) { log_info("%s: io_size must be at least one zone\n", f->file_name); @@ -289,6 +298,9 @@ static bool zbd_verify_sizes(void) (unsigned long long) new_end - f->file_offset); f->io_size = new_end - f->file_offset; } + + f->min_zone = zbd_zone_idx(f, f->file_offset); + f->max_zone = zbd_zone_idx(f, f->file_offset + f->io_size); } } @@ -342,7 +354,6 @@ static int init_zone_info(struct thread_data *td, struct fio_file *f) struct fio_zone_info *p; uint64_t zone_size = td->o.zone_size; struct zoned_block_device_info *zbd_info = NULL; - pthread_mutexattr_t attr; int i; if (zone_size == 0) { @@ -363,16 +374,14 @@ static int init_zone_info(struct thread_data *td, struct fio_file *f) if (!zbd_info) return -ENOMEM; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutexattr_setpshared(&attr, true); - pthread_mutex_init(&zbd_info->mutex, &attr); + mutex_init_pshared(&zbd_info->mutex); zbd_info->refcount = 1; p = &zbd_info->zone_info[0]; for (i = 0; i < nr_zones; i++, p++) { - pthread_mutex_init(&p->mutex, &attr); + mutex_init_pshared_with_type(&p->mutex, + PTHREAD_MUTEX_RECURSIVE); p->start = i * zone_size; - p->wp = p->start + zone_size; + p->wp = p->start; p->type = ZBD_ZONE_TYPE_SWR; p->cond = ZBD_ZONE_COND_EMPTY; } @@ -384,7 +393,6 @@ static int init_zone_info(struct thread_data *td, struct fio_file *f) f->zbd_info->zone_size_log2 = is_power_of_2(zone_size) ? ilog2(zone_size) : 0; f->zbd_info->nr_zones = nr_zones; - pthread_mutexattr_destroy(&attr); return 0; } @@ -404,13 +412,8 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f) struct fio_zone_info *p; uint64_t zone_size, offset; struct zoned_block_device_info *zbd_info = NULL; - pthread_mutexattr_t attr; int i, j, ret = 0; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutexattr_setpshared(&attr, true); - zones = calloc(ZBD_REPORT_MAX_ZONES, sizeof(struct zbd_zone)); if (!zones) goto out; @@ -444,13 +447,14 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f) ret = -ENOMEM; if (!zbd_info) goto out; - pthread_mutex_init(&zbd_info->mutex, &attr); + mutex_init_pshared(&zbd_info->mutex); zbd_info->refcount = 1; p = &zbd_info->zone_info[0]; for (offset = 0, j = 0; j < nr_zones;) { z = &zones[0]; for (i = 0; i < nrz; i++, j++, z++, p++) { - pthread_mutex_init(&p->mutex, &attr); + mutex_init_pshared_with_type(&p->mutex, + PTHREAD_MUTEX_RECURSIVE); p->start = z->start; switch (z->cond) { case ZBD_ZONE_COND_NOT_WP: @@ -501,7 +505,6 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f) out: sfree(zbd_info); free(zones); - pthread_mutexattr_destroy(&attr); return ret; } @@ -537,8 +540,10 @@ static int zbd_create_zone_info(struct thread_data *td, struct fio_file *f) return -EINVAL; } - if (ret == 0) + if (ret == 0) { f->zbd_info->model = zbd_model; + f->zbd_info->max_open_zones = td->o.max_open_zones; + } return ret; } @@ -546,8 +551,7 @@ void zbd_free_zone_info(struct fio_file *f) { uint32_t refcount; - if (!f->zbd_info) - return; + assert(f->zbd_info); pthread_mutex_lock(&f->zbd_info->mutex); refcount = --f->zbd_info->refcount; @@ -592,7 +596,7 @@ static int zbd_init_zone_info(struct thread_data *td, struct fio_file *file) return ret; } -int zbd_init(struct thread_data *td) +int zbd_setup_files(struct thread_data *td) { struct fio_file *f; int i; @@ -613,6 +617,25 @@ int zbd_init(struct thread_data *td) if (!zbd_verify_bs()) return 1; + for_each_file(td, f, i) { + struct zoned_block_device_info *zbd = f->zbd_info; + + if (!zbd) + continue; + + zbd->max_open_zones = zbd->max_open_zones ?: ZBD_MAX_OPEN_ZONES; + + if (td->o.max_open_zones > 0 && + zbd->max_open_zones != td->o.max_open_zones) { + log_err("Different 'max_open_zones' values\n"); + return 1; + } + if (zbd->max_open_zones > ZBD_MAX_OPEN_ZONES) { + log_err("'max_open_zones' value is limited by %u\n", ZBD_MAX_OPEN_ZONES); + return 1; + } + } + return 0; } @@ -687,6 +710,23 @@ static int zbd_reset_zone(struct thread_data *td, struct fio_file *f, return zbd_reset_range(td, f, z->start, (z+1)->start - z->start); } +/* The caller must hold f->zbd_info->mutex */ +static void zbd_close_zone(struct thread_data *td, const struct fio_file *f, + unsigned int open_zone_idx) +{ + uint32_t zone_idx; + + assert(open_zone_idx < f->zbd_info->num_open_zones); + zone_idx = f->zbd_info->open_zones[open_zone_idx]; + memmove(f->zbd_info->open_zones + open_zone_idx, + f->zbd_info->open_zones + open_zone_idx + 1, + (ZBD_MAX_OPEN_ZONES - (open_zone_idx + 1)) * + sizeof(f->zbd_info->open_zones[0])); + f->zbd_info->num_open_zones--; + td->num_open_zones--; + f->zbd_info->zone_info[zone_idx].open = 0; +} + /* * Reset a range of zones. Returns 0 upon success and 1 upon failure. * @td: fio thread data. @@ -705,15 +745,30 @@ static int zbd_reset_zones(struct thread_data *td, struct fio_file *f, bool reset_wp; int res = 0; + assert(min_bs); + dprint(FD_ZBD, "%s: examining zones %u .. %u\n", f->file_name, zbd_zone_nr(f->zbd_info, zb), zbd_zone_nr(f->zbd_info, ze)); for (z = zb; z < ze; z++) { + uint32_t nz = z - f->zbd_info->zone_info; + if (!zbd_zone_swr(z)) continue; - zone_lock(td, z); - reset_wp = all_zones ? z->wp != z->start : - (td->o.td_ddir & TD_DDIR_WRITE) && - z->wp % min_bs != 0; + zone_lock(td, f, z); + if (all_zones) { + unsigned int i; + + pthread_mutex_lock(&f->zbd_info->mutex); + for (i = 0; i < f->zbd_info->num_open_zones; i++) { + if (f->zbd_info->open_zones[i] == nz) + zbd_close_zone(td, f, i); + } + pthread_mutex_unlock(&f->zbd_info->mutex); + + reset_wp = z->wp != z->start; + } else { + reset_wp = z->wp % min_bs != 0; + } if (reset_wp) { dprint(FD_ZBD, "%s: resetting zone %u\n", f->file_name, @@ -731,14 +786,20 @@ static int zbd_reset_zones(struct thread_data *td, struct fio_file *f, * Reset zbd_info.write_cnt, the counter that counts down towards the next * zone reset. */ -static void zbd_reset_write_cnt(const struct thread_data *td, - const struct fio_file *f) +static void _zbd_reset_write_cnt(const struct thread_data *td, + const struct fio_file *f) { assert(0 <= td->o.zrf.u.f && td->o.zrf.u.f <= 1); - pthread_mutex_lock(&f->zbd_info->mutex); f->zbd_info->write_cnt = td->o.zrf.u.f ? min(1.0 / td->o.zrf.u.f, 0.0 + UINT_MAX) : UINT_MAX; +} + +static void zbd_reset_write_cnt(const struct thread_data *td, + const struct fio_file *f) +{ + pthread_mutex_lock(&f->zbd_info->mutex); + _zbd_reset_write_cnt(td, f); pthread_mutex_unlock(&f->zbd_info->mutex); } @@ -752,7 +813,7 @@ static bool zbd_dec_and_reset_write_cnt(const struct thread_data *td, if (f->zbd_info->write_cnt) write_cnt = --f->zbd_info->write_cnt; if (write_cnt == 0) - zbd_reset_write_cnt(td, f); + _zbd_reset_write_cnt(td, f); pthread_mutex_unlock(&f->zbd_info->mutex); return write_cnt == 0; @@ -822,14 +883,12 @@ static void zbd_init_swd(struct fio_file *f) void zbd_file_reset(struct thread_data *td, struct fio_file *f) { struct fio_zone_info *zb, *ze; - uint32_t zone_idx_e; - if (!f->zbd_info) + if (!f->zbd_info || !td_write(td)) return; - zb = &f->zbd_info->zone_info[zbd_zone_idx(f, f->file_offset)]; - zone_idx_e = zbd_zone_idx(f, f->file_offset + f->io_size); - ze = &f->zbd_info->zone_info[zone_idx_e]; + zb = &f->zbd_info->zone_info[f->min_zone]; + ze = &f->zbd_info->zone_info[f->max_zone]; zbd_init_swd(f); /* * If data verification is enabled reset the affected zones before @@ -837,7 +896,6 @@ void zbd_file_reset(struct thread_data *td, struct fio_file *f) * writing data, which causes data loss. */ zbd_reset_zones(td, f, zb, ze, td->o.verify != VERIFY_NONE && - (td->o.td_ddir & TD_DDIR_WRITE) && td->runstate != TD_VERIFYING); zbd_reset_write_cnt(td, f); } @@ -849,8 +907,9 @@ static bool is_zone_open(const struct thread_data *td, const struct fio_file *f, struct zoned_block_device_info *zbdi = f->zbd_info; int i; - assert(td->o.max_open_zones <= ARRAY_SIZE(zbdi->open_zones)); - assert(zbdi->num_open_zones <= td->o.max_open_zones); + assert(td->o.job_max_open_zones == 0 || td->num_open_zones <= td->o.job_max_open_zones); + assert(td->o.job_max_open_zones <= zbdi->max_open_zones); + assert(zbdi->num_open_zones <= zbdi->max_open_zones); for (i = 0; i < zbdi->num_open_zones; i++) if (zbdi->open_zones[i] == zone_idx) @@ -883,18 +942,19 @@ static bool zbd_open_zone(struct thread_data *td, const struct io_u *io_u, if (td->o.verify != VERIFY_NONE && zbd_zone_full(f, z, min_bs)) return false; - /* Zero means no limit */ - if (!td->o.max_open_zones) - return true; - pthread_mutex_lock(&f->zbd_info->mutex); if (is_zone_open(td, f, zone_idx)) goto out; res = false; - if (f->zbd_info->num_open_zones >= td->o.max_open_zones) + /* Zero means no limit */ + if (td->o.job_max_open_zones > 0 && + td->num_open_zones >= td->o.job_max_open_zones) + goto out; + if (f->zbd_info->num_open_zones >= f->zbd_info->max_open_zones) goto out; dprint(FD_ZBD, "%s: opening zone %d\n", f->file_name, zone_idx); f->zbd_info->open_zones[f->zbd_info->num_open_zones++] = zone_idx; + td->num_open_zones++; z->open = 1; res = true; @@ -903,22 +963,6 @@ out: return res; } -/* The caller must hold f->zbd_info->mutex */ -static void zbd_close_zone(struct thread_data *td, const struct fio_file *f, - unsigned int open_zone_idx) -{ - uint32_t zone_idx; - - assert(open_zone_idx < f->zbd_info->num_open_zones); - zone_idx = f->zbd_info->open_zones[open_zone_idx]; - memmove(f->zbd_info->open_zones + open_zone_idx, - f->zbd_info->open_zones + open_zone_idx + 1, - (ZBD_MAX_OPEN_ZONES - (open_zone_idx + 1)) * - sizeof(f->zbd_info->open_zones[0])); - f->zbd_info->num_open_zones--; - f->zbd_info->zone_info[zone_idx].open = 0; -} - /* Anything goes as long as it is not a constant. */ static uint32_t pick_random_zone_idx(const struct fio_file *f, const struct io_u *io_u) @@ -937,7 +981,7 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, struct io_u *io_u) { const uint32_t min_bs = td->o.min_bs[io_u->ddir]; - const struct fio_file *f = io_u->file; + struct fio_file *f = io_u->file; struct fio_zone_info *z; unsigned int open_zone_idx = -1; uint32_t zone_idx, new_zone_idx; @@ -945,7 +989,7 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, assert(is_valid_offset(f, io_u->offset)); - if (td->o.max_open_zones) { + if (td->o.max_open_zones || td->o.job_max_open_zones) { /* * This statement accesses f->zbd_info->open_zones[] on purpose * without locking. @@ -954,6 +998,10 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, } else { zone_idx = zbd_zone_idx(f, io_u->offset); } + if (zone_idx < f->min_zone) + zone_idx = f->min_zone; + else if (zone_idx >= f->max_zone) + zone_idx = f->max_zone - 1; dprint(FD_ZBD, "%s(%s): starting from zone %d (offset %lld, buflen %lld)\n", __func__, f->file_name, zone_idx, io_u->offset, io_u->buflen); @@ -968,9 +1016,9 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, z = &f->zbd_info->zone_info[zone_idx]; - zone_lock(td, z); + zone_lock(td, f, z); pthread_mutex_lock(&f->zbd_info->mutex); - if (td->o.max_open_zones == 0) + if (td->o.max_open_zones == 0 && td->o.job_max_open_zones == 0) goto examine_zone; if (f->zbd_info->num_open_zones == 0) { pthread_mutex_unlock(&f->zbd_info->mutex); @@ -994,8 +1042,7 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, if (tmp_idx >= f->zbd_info->num_open_zones) tmp_idx = 0; tmpz = f->zbd_info->open_zones[tmp_idx]; - - if (is_valid_offset(f, f->zbd_info->zone_info[tmpz].start)) { + if (f->min_zone <= tmpz && tmpz < f->max_zone) { open_zone_idx = tmp_idx; goto found_candidate_zone; } @@ -1005,6 +1052,8 @@ static struct fio_zone_info *zbd_convert_to_open_zone(struct thread_data *td, dprint(FD_ZBD, "%s(%s): no candidate zone\n", __func__, f->file_name); + pthread_mutex_unlock(&f->zbd_info->mutex); + pthread_mutex_unlock(&z->mutex); return NULL; found_candidate_zone: @@ -1025,7 +1074,7 @@ examine_zone: } dprint(FD_ZBD, "%s(%s): closing zone %d\n", __func__, f->file_name, zone_idx); - if (td->o.max_open_zones) + if (td->o.max_open_zones || td->o.job_max_open_zones) zbd_close_zone(td, f, open_zone_idx); pthread_mutex_unlock(&f->zbd_info->mutex); @@ -1038,11 +1087,11 @@ examine_zone: z++; if (!is_valid_offset(f, z->start)) { /* Wrap-around. */ - zone_idx = zbd_zone_idx(f, f->file_offset); + zone_idx = f->min_zone; z = &f->zbd_info->zone_info[zone_idx]; } assert(is_valid_offset(f, z->start)); - zone_lock(td, z); + zone_lock(td, f, z); if (z->open) continue; if (zbd_open_zone(td, io_u, zone_idx)) @@ -1055,12 +1104,14 @@ examine_zone: pthread_mutex_lock(&f->zbd_info->mutex); for (i = 0; i < f->zbd_info->num_open_zones; i++) { zone_idx = f->zbd_info->open_zones[i]; + if (zone_idx < f->min_zone || zone_idx >= f->max_zone) + continue; pthread_mutex_unlock(&f->zbd_info->mutex); pthread_mutex_unlock(&z->mutex); z = &f->zbd_info->zone_info[zone_idx]; - zone_lock(td, z); + zone_lock(td, f, z); if (z->wp + min_bs <= (z+1)->start) goto out; pthread_mutex_lock(&f->zbd_info->mutex); @@ -1112,7 +1163,7 @@ zbd_find_zone(struct thread_data *td, struct io_u *io_u, struct fio_zone_info *zb, struct fio_zone_info *zl) { const uint32_t min_bs = td->o.min_bs[io_u->ddir]; - const struct fio_file *f = io_u->file; + struct fio_file *f = io_u->file; struct fio_zone_info *z1, *z2; const struct fio_zone_info *const zf = &f->zbd_info->zone_info[zbd_zone_idx(f, f->file_offset)]; @@ -1123,7 +1174,7 @@ zbd_find_zone(struct thread_data *td, struct io_u *io_u, */ for (z1 = zb + 1, z2 = zb - 1; z1 < zl || z2 >= zf; z1++, z2--) { if (z1 < zl && z1->cond != ZBD_ZONE_COND_OFFLINE) { - pthread_mutex_lock(&z1->mutex); + zone_lock(td, f, z1); if (z1->start + min_bs <= z1->wp) return z1; pthread_mutex_unlock(&z1->mutex); @@ -1132,7 +1183,7 @@ zbd_find_zone(struct thread_data *td, struct io_u *io_u, } if (td_random(td) && z2 >= zf && z2->cond != ZBD_ZONE_COND_OFFLINE) { - pthread_mutex_lock(&z2->mutex); + zone_lock(td, f, z2); if (z2->start + min_bs <= z2->wp) return z2; pthread_mutex_unlock(&z2->mutex); @@ -1216,6 +1267,7 @@ static void zbd_put_io(const struct io_u *io_u) struct zoned_block_device_info *zbd_info = f->zbd_info; struct fio_zone_info *z; uint32_t zone_idx; + int ret; if (!zbd_info) return; @@ -1231,10 +1283,18 @@ static void zbd_put_io(const struct io_u *io_u) "%s: terminate I/O (%lld, %llu) for zone %u\n", f->file_name, io_u->offset, io_u->buflen, zone_idx); - assert(pthread_mutex_unlock(&z->mutex) == 0); + ret = pthread_mutex_unlock(&z->mutex); + assert(ret == 0); zbd_check_swd(f); } +/* + * Windows and MacOS do not define this. + */ +#ifndef EREMOTEIO +#define EREMOTEIO 121 /* POSIX value */ +#endif + bool zbd_unaligned_write(int error_code) { switch (error_code) { @@ -1304,6 +1364,34 @@ void setup_zbd_zone_mode(struct thread_data *td, struct io_u *io_u) } } +/** + * zbd_adjust_ddir - Adjust an I/O direction for zonemode=zbd. + * + * @td: FIO thread data. + * @io_u: FIO I/O unit. + * @ddir: I/O direction before adjustment. + * + * Return adjusted I/O direction. + */ +enum fio_ddir zbd_adjust_ddir(struct thread_data *td, struct io_u *io_u, + enum fio_ddir ddir) +{ + /* + * In case read direction is chosen for the first random I/O, fio with + * zonemode=zbd stops because no data can be read from zoned block + * devices with all empty zones. Overwrite the first I/O direction as + * write to make sure data to read exists. + */ + if (ddir != DDIR_READ || !td_rw(td)) + return ddir; + + if (io_u->file->zbd_info->sectors_with_data || + td->o.read_beyond_wp) + return DDIR_READ; + + return DDIR_WRITE; +} + /** * zbd_adjust_block - adjust the offset and length as necessary for ZBD drives * @td: FIO thread data. @@ -1326,6 +1414,7 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) if (!f->zbd_info) return io_u_accept; + assert(min_bs); assert(is_valid_offset(f, io_u->offset)); assert(io_u->buflen); zone_idx_b = zbd_zone_idx(f, io_u->offset); @@ -1346,12 +1435,13 @@ enum io_u_action zbd_adjust_block(struct thread_data *td, struct io_u *io_u) zbd_check_swd(f); - zone_lock(td, zb); + zone_lock(td, f, zb); switch (io_u->ddir) { case DDIR_READ: if (td->runstate == TD_VERIFYING) { - zb = zbd_replay_write_order(td, io_u, zb); + if (td_write(td)) + zb = zbd_replay_write_order(td, io_u, zb); goto accept; } /*