X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=io_u.c;h=42d98ebfe1d6a0dc85193348a77de95667ff117f;hp=8d3491281ddebd665b9cc294b87b324e7bd1d4b7;hb=395feabb53806d5cff2e0b73b6c94048f05b5aae;hpb=19ddc35b9b97be5af371bb65e93a4864d1dce7b6 diff --git a/io_u.c b/io_u.c index 8d349128..42d98ebf 100644 --- a/io_u.c +++ b/io_u.c @@ -20,7 +20,7 @@ struct io_completion_data { int error; /* output */ uint64_t bytes_done[DDIR_RWDIR_CNT]; /* output */ - struct timeval time; /* output */ + struct timespec time; /* output */ }; /* @@ -37,7 +37,7 @@ static bool random_map_free(struct fio_file *f, const uint64_t block) */ static void mark_random_map(struct thread_data *td, struct io_u *io_u) { - unsigned int min_bs = td->o.rw_min_bs; + unsigned int min_bs = td->o.min_bs[io_u->ddir]; struct fio_file *f = io_u->file; unsigned int nr_blocks; uint64_t block; @@ -62,6 +62,7 @@ static uint64_t last_block(struct thread_data *td, struct fio_file *f, /* * Hmm, should we make sure that ->io_size <= ->real_file_size? + * -> not for now since there is code assuming it could go either. */ max_size = f->io_size; if (max_size > f->real_file_size) @@ -86,24 +87,19 @@ struct rand_off { }; static int __get_next_rand_offset(struct thread_data *td, struct fio_file *f, - enum fio_ddir ddir, uint64_t *b) + enum fio_ddir ddir, uint64_t *b, + uint64_t lastb) { uint64_t r; if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE || td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE64) { - uint64_t frand_max, lastb; - lastb = last_block(td, f, ddir); - if (!lastb) - return 1; - - frand_max = rand_max(&td->random_state); r = __rand(&td->random_state); dprint(FD_RANDOM, "off rand %llu\n", (unsigned long long) r); - *b = lastb * (r / ((uint64_t) frand_max + 1.0)); + *b = lastb * (r / (rand_max(&td->random_state) + 1.0)); } else { uint64_t off = 0; @@ -161,6 +157,130 @@ static int __get_next_rand_offset_gauss(struct thread_data *td, return 0; } +static int __get_next_rand_offset_zoned_abs(struct thread_data *td, + struct fio_file *f, + enum fio_ddir ddir, uint64_t *b) +{ + struct zone_split_index *zsi; + uint64_t lastb, send, stotal; + static int warned; + unsigned int v; + + lastb = last_block(td, f, ddir); + if (!lastb) + return 1; + + if (!td->o.zone_split_nr[ddir]) { +bail: + return __get_next_rand_offset(td, f, ddir, b, lastb); + } + + /* + * Generate a value, v, between 1 and 100, both inclusive + */ + v = rand32_between(&td->zone_state, 1, 100); + + /* + * Find our generated table. 'send' is the end block of this zone, + * 'stotal' is our start offset. + */ + zsi = &td->zone_state_index[ddir][v - 1]; + stotal = zsi->size_prev / td->o.ba[ddir]; + send = zsi->size / td->o.ba[ddir]; + + /* + * Should never happen + */ + if (send == -1U) { + if (!warned) { + log_err("fio: bug in zoned generation\n"); + warned = 1; + } + goto bail; + } else if (send > lastb) { + /* + * This happens if the user specifies ranges that exceed + * the file/device size. We can't handle that gracefully, + * so error and exit. + */ + log_err("fio: zoned_abs sizes exceed file size\n"); + return 1; + } + + /* + * Generate index from 0..send-stotal + */ + if (__get_next_rand_offset(td, f, ddir, b, send - stotal) == 1) + return 1; + + *b += stotal; + return 0; +} + +static int __get_next_rand_offset_zoned(struct thread_data *td, + struct fio_file *f, enum fio_ddir ddir, + uint64_t *b) +{ + unsigned int v, send, stotal; + uint64_t offset, lastb; + static int warned; + struct zone_split_index *zsi; + + lastb = last_block(td, f, ddir); + if (!lastb) + return 1; + + if (!td->o.zone_split_nr[ddir]) { +bail: + return __get_next_rand_offset(td, f, ddir, b, lastb); + } + + /* + * Generate a value, v, between 1 and 100, both inclusive + */ + v = rand32_between(&td->zone_state, 1, 100); + + zsi = &td->zone_state_index[ddir][v - 1]; + stotal = zsi->size_perc_prev; + send = zsi->size_perc; + + /* + * Should never happen + */ + if (send == -1U) { + if (!warned) { + log_err("fio: bug in zoned generation\n"); + warned = 1; + } + goto bail; + } + + /* + * 'send' is some percentage below or equal to 100 that + * marks the end of the current IO range. 'stotal' marks + * the start, in percent. + */ + if (stotal) + offset = stotal * lastb / 100ULL; + else + offset = 0; + + lastb = lastb * (send - stotal) / 100ULL; + + /* + * Generate index from 0..send-of-lastb + */ + if (__get_next_rand_offset(td, f, ddir, b, lastb) == 1) + return 1; + + /* + * Add our start offset, if any + */ + if (offset) + *b += offset; + + return 0; +} static int flist_cmp(void *data, struct flist_head *a, struct flist_head *b) { @@ -173,14 +293,24 @@ static int flist_cmp(void *data, struct flist_head *a, struct flist_head *b) static int get_off_from_method(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir, uint64_t *b) { - if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) - return __get_next_rand_offset(td, f, ddir, b); - else if (td->o.random_distribution == FIO_RAND_DIST_ZIPF) + if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) { + uint64_t lastb; + + lastb = last_block(td, f, ddir); + if (!lastb) + return 1; + + return __get_next_rand_offset(td, f, ddir, b, lastb); + } else if (td->o.random_distribution == FIO_RAND_DIST_ZIPF) return __get_next_rand_offset_zipf(td, f, ddir, b); else if (td->o.random_distribution == FIO_RAND_DIST_PARETO) return __get_next_rand_offset_pareto(td, f, ddir, b); else if (td->o.random_distribution == FIO_RAND_DIST_GAUSS) return __get_next_rand_offset_gauss(td, f, ddir, b); + else if (td->o.random_distribution == FIO_RAND_DIST_ZONED) + return __get_next_rand_offset_zoned(td, f, ddir, b); + else if (td->o.random_distribution == FIO_RAND_DIST_ZONED_ABS) + return __get_next_rand_offset_zoned_abs(td, f, ddir, b); log_err("fio: unknown random distribution: %d\n", td->o.random_distribution); return 1; @@ -207,16 +337,12 @@ static inline bool should_sort_io(struct thread_data *td) static bool should_do_random(struct thread_data *td, enum fio_ddir ddir) { - uint64_t frand_max; unsigned int v; - unsigned long r; if (td->o.perc_rand[ddir] == 100) return true; - frand_max = rand_max(&td->seq_rand_state[ddir]); - r = __rand(&td->seq_rand_state[ddir]); - v = 1 + (int) (100.0 * (r / (frand_max + 1.0))); + v = rand32_between(&td->seq_rand_state[ddir], 1, 100); return v <= td->o.perc_rand[ddir]; } @@ -259,16 +385,29 @@ fetch: goto fetch; } +static void loop_cache_invalidate(struct thread_data *td, struct fio_file *f) +{ + struct thread_options *o = &td->o; + + if (o->invalidate_cache && !o->odirect) { + int fio_unused ret; + + ret = file_invalidate_cache(td, f); + } +} + static int get_next_rand_block(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir, uint64_t *b) { if (!get_next_rand_offset(td, f, ddir, b)) return 0; - if (td->o.time_based) { + if (td->o.time_based || + (td->o.file_service_type & __FIO_FSERVICE_NONUNIFORM)) { fio_file_reset(td, f); if (!get_next_rand_offset(td, f, ddir, b)) return 0; + loop_cache_invalidate(td, f); } dprint(FD_IO, "%s: rand offset failed, last=%llu, size=%llu\n", @@ -284,22 +423,25 @@ static int get_next_seq_offset(struct thread_data *td, struct fio_file *f, assert(ddir_rw(ddir)); + /* + * If we reach the end for a time based run, reset us back to 0 + * and invalidate the cache, if we need to. + */ if (f->last_pos[ddir] >= f->io_size + get_start_offset(td, f) && o->time_based) { - struct thread_options *o = &td->o; - uint64_t io_size = f->io_size + (f->io_size % o->min_bs[ddir]); - - if (io_size > f->last_pos[ddir]) - f->last_pos[ddir] = 0; - else - f->last_pos[ddir] = f->last_pos[ddir] - io_size; + f->last_pos[ddir] = f->file_offset; + loop_cache_invalidate(td, f); } if (f->last_pos[ddir] < f->real_file_size) { uint64_t pos; - if (f->last_pos[ddir] == f->file_offset && o->ddir_seq_add < 0) - f->last_pos[ddir] = f->real_file_size; + if (f->last_pos[ddir] == f->file_offset && o->ddir_seq_add < 0) { + if (f->real_file_size > f->io_size) + f->last_pos[ddir] = f->io_size; + else + f->last_pos[ddir] = f->real_file_size; + } pos = f->last_pos[ddir] - f->file_offset; if (pos && o->ddir_seq_add) { @@ -308,10 +450,21 @@ static int get_next_seq_offset(struct thread_data *td, struct fio_file *f, /* * If we reach beyond the end of the file * with holed IO, wrap around to the - * beginning again. + * beginning again. If we're doing backwards IO, + * wrap to the end. */ - if (pos >= f->real_file_size) - pos = f->file_offset; + if (pos >= f->real_file_size) { + if (o->ddir_seq_add > 0) + pos = f->file_offset; + else { + if (f->real_file_size > f->io_size) + pos = f->io_size; + else + pos = f->real_file_size; + + pos += o->ddir_seq_add; + } + } } *offset = pos; @@ -340,7 +493,7 @@ static int get_next_block(struct thread_data *td, struct io_u *io_u, *is_random = 1; } else { *is_random = 0; - io_u_set(io_u, IO_U_F_BUSY_OK); + io_u_set(td, io_u, IO_U_F_BUSY_OK); ret = get_next_seq_offset(td, f, ddir, &offset); if (ret) ret = get_next_rand_block(td, f, ddir, &b); @@ -350,7 +503,7 @@ static int get_next_block(struct thread_data *td, struct io_u *io_u, ret = get_next_seq_offset(td, f, ddir, &offset); } } else { - io_u_set(io_u, IO_U_F_BUSY_OK); + io_u_set(td, io_u, IO_U_F_BUSY_OK); *is_random = 0; if (td->o.rw_seq == RW_SEQ_SEQ) { @@ -452,8 +605,8 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, int ddir = io_u->ddir; unsigned int buflen = 0; unsigned int minbs, maxbs; - uint64_t frand_max; - unsigned long r; + uint64_t frand_max, r; + bool power_2; assert(ddir_rw(ddir)); @@ -472,9 +625,9 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, if (!io_u_fits(td, io_u, minbs)) return 0; - frand_max = rand_max(&td->bsrange_state); + frand_max = rand_max(&td->bsrange_state[ddir]); do { - r = __rand(&td->bsrange_state); + r = __rand(&td->bsrange_state[ddir]); if (!td->o.bssplit_nr[ddir]) { buflen = 1 + (unsigned int) ((double) maxbs * @@ -482,7 +635,7 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, if (buflen < minbs) buflen = minbs; } else { - long perc = 0; + long long perc = 0; unsigned int i; for (i = 0; i < td->o.bssplit_nr[ddir]; i++) { @@ -490,19 +643,19 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, buflen = bsp->bs; perc += bsp->perc; - if ((r <= ((frand_max / 100L) * perc)) && + if (!perc) + break; + if ((r / perc <= frand_max / 100ULL) && io_u_fits(td, io_u, buflen)) break; } } - if (td->o.verify != VERIFY_NONE) - buflen = (buflen + td->o.verify_interval - 1) & - ~(td->o.verify_interval - 1); - - if (!td->o.bs_unaligned && is_power_of_2(minbs)) + power_2 = is_power_of_2(minbs); + if (!td->o.bs_unaligned && power_2) buflen &= ~(minbs - 1); - + else if (!td->o.bs_unaligned && !power_2) + buflen -= buflen % minbs; } while (!io_u_fits(td, io_u, buflen)); return buflen; @@ -536,12 +689,9 @@ static void set_rwmix_bytes(struct thread_data *td) static inline enum fio_ddir get_rand_ddir(struct thread_data *td) { - uint64_t frand_max = rand_max(&td->rwmix_state); unsigned int v; - unsigned long r; - r = __rand(&td->rwmix_state); - v = 1 + (int) (100.0 * (r / (frand_max + 1.0))); + v = rand32_between(&td->rwmix_state, 1, 100); if (v <= td->o.rwmix[DDIR_READ]) return DDIR_READ; @@ -569,20 +719,24 @@ int io_u_quiesce(struct thread_data *td) } while (td->io_u_in_flight) { - int fio_unused ret; + int ret; ret = io_u_queued_complete(td, 1); if (ret > 0) completed += ret; } + if (td->flags & TD_F_REGROW_LOGS) + regrow_logs(td); + return completed; } static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir) { enum fio_ddir odir = ddir ^ 1; - long usec, now; + uint64_t usec; + uint64_t now; assert(ddir_rw(ddir)); now = utime_since_now(&td->start); @@ -605,11 +759,11 @@ static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir) return odir; /* - * Both directions are ahead of rate. sleep the min - * switch if necissary + * Both directions are ahead of rate. sleep the min, + * switch if necessary */ if (td->rate_next_io_time[ddir] <= - td->rate_next_io_time[odir]) { + td->rate_next_io_time[odir]) { usec = td->rate_next_io_time[ddir] - now; } else { usec = td->rate_next_io_time[odir] - now; @@ -621,8 +775,7 @@ static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir) if (td->o.io_submit_mode == IO_MODE_INLINE) io_u_quiesce(td); - usec = usec_sleep(td, usec); - + usec_sleep(td, usec); return ddir; } @@ -636,28 +789,22 @@ static enum fio_ddir get_rw_ddir(struct thread_data *td) enum fio_ddir ddir; /* - * see if it's time to fsync + * See if it's time to fsync/fdatasync/sync_file_range first, + * and if not then move on to check regular I/Os. */ - if (td->o.fsync_blocks && - !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks) && - td->io_issues[DDIR_WRITE] && should_fsync(td)) - return DDIR_SYNC; - - /* - * see if it's time to fdatasync - */ - if (td->o.fdatasync_blocks && - !(td->io_issues[DDIR_WRITE] % td->o.fdatasync_blocks) && - td->io_issues[DDIR_WRITE] && should_fsync(td)) - return DDIR_DATASYNC; - - /* - * see if it's time to sync_file_range - */ - if (td->sync_file_range_nr && - !(td->io_issues[DDIR_WRITE] % td->sync_file_range_nr) && - td->io_issues[DDIR_WRITE] && should_fsync(td)) - return DDIR_SYNC_FILE_RANGE; + if (should_fsync(td)) { + if (td->o.fsync_blocks && td->io_issues[DDIR_WRITE] && + !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks)) + return DDIR_SYNC; + + if (td->o.fdatasync_blocks && td->io_issues[DDIR_WRITE] && + !(td->io_issues[DDIR_WRITE] % td->o.fdatasync_blocks)) + return DDIR_DATASYNC; + + if (td->sync_file_range_nr && td->io_issues[DDIR_WRITE] && + !(td->io_issues[DDIR_WRITE] % td->sync_file_range_nr)) + return DDIR_SYNC_FILE_RANGE; + } if (td_rw(td)) { /* @@ -681,8 +828,10 @@ static enum fio_ddir get_rw_ddir(struct thread_data *td) ddir = DDIR_READ; else if (td_write(td)) ddir = DDIR_WRITE; - else + else if (td_trim(td)) ddir = DDIR_TRIM; + else + ddir = DDIR_INVAL; td->rwmix_ddir = rate_ddir(td, ddir); return td->rwmix_ddir; @@ -702,11 +851,11 @@ static void set_rw_ddir(struct thread_data *td, struct io_u *io_u) io_u->ddir = io_u->acct_ddir = ddir; - if (io_u->ddir == DDIR_WRITE && (td->io_ops->flags & FIO_BARRIER) && + if (io_u->ddir == DDIR_WRITE && td_ioengine_flagged(td, FIO_BARRIER) && td->o.barrier_blocks && !(td->io_issues[DDIR_WRITE] % td->o.barrier_blocks) && td->io_issues[DDIR_WRITE]) - io_u_set(io_u, IO_U_F_BARRIER); + io_u_set(td, io_u, IO_U_F_BARRIER); } void put_file_log(struct thread_data *td, struct fio_file *f) @@ -728,7 +877,7 @@ void put_io_u(struct thread_data *td, struct io_u *io_u) put_file_log(td, io_u->file); io_u->file = NULL; - io_u_set(io_u, IO_U_F_FREE); + io_u_set(td, io_u, IO_U_F_FREE); if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { td->cur_depth--; @@ -741,7 +890,7 @@ void put_io_u(struct thread_data *td, struct io_u *io_u) void clear_io_u(struct thread_data *td, struct io_u *io_u) { - io_u_clear(io_u, IO_U_F_FLIGHT); + io_u_clear(td, io_u, IO_U_F_FLIGHT); put_io_u(td, io_u); } @@ -757,11 +906,11 @@ void requeue_io_u(struct thread_data *td, struct io_u **io_u) td_io_u_lock(td); - io_u_set(__io_u, IO_U_F_FREE); + io_u_set(td, __io_u, IO_U_F_FREE); if ((__io_u->flags & IO_U_F_FLIGHT) && ddir_rw(ddir)) td->io_issues[ddir]--; - io_u_clear(__io_u, IO_U_F_FLIGHT); + io_u_clear(td, __io_u, IO_U_F_FLIGHT); if (__io_u->flags & IO_U_F_IN_CUR_DEPTH) { td->cur_depth--; assert(!(td->flags & TD_F_CHILD)); @@ -777,7 +926,7 @@ static int fill_io_u(struct thread_data *td, struct io_u *io_u) { unsigned int is_random; - if (td->io_ops->flags & FIO_NOIO) + if (td_ioengine_flagged(td, FIO_NOIO)) goto out; set_rw_ddir(td, io_u); @@ -822,8 +971,9 @@ static int fill_io_u(struct thread_data *td, struct io_u *io_u) } if (io_u->offset + io_u->buflen > io_u->file->real_file_size) { - dprint(FD_IO, "io_u %p, offset too large\n", io_u); - dprint(FD_IO, " off=%llu/%lu > %llu\n", + dprint(FD_IO, "io_u %p, offset + buflen exceeds file size\n", + io_u); + dprint(FD_IO, " offset=%llu/buflen=%lu > %llu\n", (unsigned long long) io_u->offset, io_u->buflen, (unsigned long long) io_u->file->real_file_size); return 1; @@ -911,11 +1061,52 @@ void io_u_mark_depth(struct thread_data *td, unsigned int nr) td->ts.io_u_map[idx] += nr; } -static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec) +static void io_u_mark_lat_nsec(struct thread_data *td, unsigned long long nsec) +{ + int idx = 0; + + assert(nsec < 1000); + + switch (nsec) { + case 750 ... 999: + idx = 9; + break; + case 500 ... 749: + idx = 8; + break; + case 250 ... 499: + idx = 7; + break; + case 100 ... 249: + idx = 6; + break; + case 50 ... 99: + idx = 5; + break; + case 20 ... 49: + idx = 4; + break; + case 10 ... 19: + idx = 3; + break; + case 4 ... 9: + idx = 2; + break; + case 2 ... 3: + idx = 1; + case 0 ... 1: + break; + } + + assert(idx < FIO_IO_U_LAT_N_NR); + td->ts.io_u_lat_n[idx]++; +} + +static void io_u_mark_lat_usec(struct thread_data *td, unsigned long long usec) { int idx = 0; - assert(usec < 1000); + assert(usec < 1000 && usec >= 1); switch (usec) { case 750 ... 999: @@ -952,10 +1143,12 @@ static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec) td->ts.io_u_lat_u[idx]++; } -static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec) +static void io_u_mark_lat_msec(struct thread_data *td, unsigned long long msec) { int idx = 0; + assert(msec >= 1); + switch (msec) { default: idx = 11; @@ -997,12 +1190,42 @@ static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec) td->ts.io_u_lat_m[idx]++; } -static void io_u_mark_latency(struct thread_data *td, unsigned long usec) +static void io_u_mark_latency(struct thread_data *td, unsigned long long nsec) { - if (usec < 1000) - io_u_mark_lat_usec(td, usec); + if (nsec < 1000) + io_u_mark_lat_nsec(td, nsec); + else if (nsec < 1000000) + io_u_mark_lat_usec(td, nsec / 1000); else - io_u_mark_lat_msec(td, usec / 1000); + io_u_mark_lat_msec(td, nsec / 1000000); +} + +static unsigned int __get_next_fileno_rand(struct thread_data *td) +{ + unsigned long fileno; + + if (td->o.file_service_type == FIO_FSERVICE_RANDOM) { + uint64_t frand_max = rand_max(&td->next_file_state); + unsigned long r; + + r = __rand(&td->next_file_state); + return (unsigned int) ((double) td->o.nr_files + * (r / (frand_max + 1.0))); + } + + if (td->o.file_service_type == FIO_FSERVICE_ZIPF) + fileno = zipf_next(&td->next_file_zipf); + else if (td->o.file_service_type == FIO_FSERVICE_PARETO) + fileno = pareto_next(&td->next_file_zipf); + else if (td->o.file_service_type == FIO_FSERVICE_GAUSS) + fileno = gauss_next(&td->next_file_gauss); + else { + log_err("fio: bad file service type: %d\n", td->o.file_service_type); + assert(0); + return 0; + } + + return fileno >> FIO_FSERVICE_SHIFT; } /* @@ -1012,17 +1235,13 @@ static struct fio_file *get_next_file_rand(struct thread_data *td, enum fio_file_flags goodf, enum fio_file_flags badf) { - uint64_t frand_max = rand_max(&td->next_file_state); struct fio_file *f; int fno; do { int opened = 0; - unsigned long r; - r = __rand(&td->next_file_state); - fno = (unsigned int) ((double) td->o.nr_files - * (r / (frand_max + 1.0))); + fno = __get_next_fileno_rand(td); f = td->files[fno]; if (fio_file_done(f)) @@ -1175,20 +1394,24 @@ static long set_io_u_file(struct thread_data *td, struct io_u *io_u) put_file_log(td, f); td_io_close_file(td, f); io_u->file = NULL; - fio_file_set_done(f); - td->nr_done_files++; - dprint(FD_FILE, "%s: is done (%d of %d)\n", f->file_name, + if (td->o.file_service_type & __FIO_FSERVICE_NONUNIFORM) + fio_file_reset(td, f); + else { + fio_file_set_done(f); + td->nr_done_files++; + dprint(FD_FILE, "%s: is done (%d of %d)\n", f->file_name, td->nr_done_files, td->o.nr_files); + } } while (1); return 0; } static void lat_fatal(struct thread_data *td, struct io_completion_data *icd, - unsigned long tusec, unsigned long max_usec) + unsigned long long tnsec, unsigned long long max_nsec) { if (!td->error) - log_err("fio: latency of %lu usec exceeds specified max (%lu usec)\n", tusec, max_usec); + log_err("fio: latency of %llu nsec exceeds specified max (%llu nsec)\n", tnsec, max_nsec); td_verror(td, ETIMEDOUT, "max latency exceeded"); icd->error = ETIMEDOUT; } @@ -1363,7 +1586,7 @@ again: if (io_u) { assert(io_u->flags & IO_U_F_FREE); - io_u_clear(io_u, IO_U_F_FREE | IO_U_F_NO_FILE_PUT | + io_u_clear(td, io_u, IO_U_F_FREE | IO_U_F_NO_FILE_PUT | IO_U_F_TRIMMED | IO_U_F_BARRIER | IO_U_F_VER_LIST); @@ -1371,7 +1594,7 @@ again: io_u->acct_ddir = -1; td->cur_depth++; assert(!(td->flags & TD_F_CHILD)); - io_u_set(io_u, IO_U_F_IN_CUR_DEPTH); + io_u_set(td, io_u, IO_U_F_IN_CUR_DEPTH); io_u->ipo = NULL; } else if (td_async_processing(td)) { /* @@ -1391,22 +1614,19 @@ static bool check_get_trim(struct thread_data *td, struct io_u *io_u) { if (!(td->flags & TD_F_TRIM_BACKLOG)) return false; + if (!td->trim_entries) + return false; - if (td->trim_entries) { - int get_trim = 0; - - if (td->trim_batch) { - td->trim_batch--; - get_trim = 1; - } else if (!(td->io_hist_len % td->o.trim_backlog) && - td->last_ddir != DDIR_READ) { - td->trim_batch = td->o.trim_batch; - if (!td->trim_batch) - td->trim_batch = td->o.trim_backlog; - get_trim = 1; - } - - if (get_trim && !get_next_trim(td, io_u)) + if (td->trim_batch) { + td->trim_batch--; + if (get_next_trim(td, io_u)) + return true; + } else if (!(td->io_hist_len % td->o.trim_backlog) && + td->last_ddir != DDIR_READ) { + td->trim_batch = td->o.trim_batch; + if (!td->trim_batch) + td->trim_batch = td->o.trim_backlog; + if (get_next_trim(td, io_u)) return true; } @@ -1448,32 +1668,40 @@ static bool check_get_verify(struct thread_data *td, struct io_u *io_u) */ static void small_content_scramble(struct io_u *io_u) { - unsigned int i, nr_blocks = io_u->buflen / 512; - uint64_t boffset; + unsigned int i, nr_blocks = io_u->buflen >> 9; unsigned int offset; - void *p, *end; + uint64_t boffset, *iptr; + char *p; if (!nr_blocks) return; p = io_u->xfer_buf; boffset = io_u->offset; - io_u->buf_filled_len = 0; + + if (io_u->buf_filled_len) + io_u->buf_filled_len = 0; + + /* + * Generate random index between 0..7. We do chunks of 512b, if + * we assume a cacheline is 64 bytes, then we have 8 of those. + * Scramble content within the blocks in the same cacheline to + * speed things up. + */ + offset = (io_u->start_time.tv_nsec ^ boffset) & 7; for (i = 0; i < nr_blocks; i++) { /* - * Fill the byte offset into a "random" start offset of - * the buffer, given by the product of the usec time - * and the actual offset. + * Fill offset into start of cacheline, time into end + * of cacheline */ - offset = (io_u->start_time.tv_usec ^ boffset) & 511; - offset &= ~(sizeof(uint64_t) - 1); - if (offset >= 512 - sizeof(uint64_t)) - offset -= sizeof(uint64_t); - memcpy(p + offset, &boffset, sizeof(boffset)); - - end = p + 512 - sizeof(io_u->start_time); - memcpy(end, &io_u->start_time, sizeof(io_u->start_time)); + iptr = (void *) p + (offset << 6); + *iptr = boffset; + + iptr = (void *) p + 64 - 2 * sizeof(uint64_t); + iptr[0] = io_u->start_time.tv_sec; + iptr[1] = io_u->start_time.tv_nsec; + p += 512; boffset += 512; } @@ -1528,7 +1756,7 @@ struct io_u *get_io_u(struct thread_data *td) assert(fio_file_open(f)); if (ddir_rw(io_u->ddir)) { - if (!io_u->buflen && !(td->io_ops->flags & FIO_NOIO)) { + if (!io_u->buflen && !td_ioengine_flagged(td, FIO_NOIO)) { dprint(FD_IO, "get_io_u: zero buflen on %p\n", io_u); goto err_put; } @@ -1568,8 +1796,10 @@ out: if (!td_io_prep(td, io_u)) { if (!td->o.disable_lat) fio_gettime(&io_u->start_time, NULL); + if (do_scramble) small_content_scramble(io_u); + return io_u; } err_put: @@ -1607,7 +1837,7 @@ void io_u_log_error(struct thread_data *td, struct io_u *io_u) { __io_u_log_error(td, io_u); if (td->parent) - __io_u_log_error(td, io_u); + __io_u_log_error(td->parent, io_u); } static inline bool gtod_reduce(struct thread_data *td) @@ -1621,45 +1851,50 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, const enum fio_ddir idx, unsigned int bytes) { const int no_reduce = !gtod_reduce(td); - unsigned long lusec = 0; + unsigned long long llnsec = 0; if (td->parent) td = td->parent; + if (!td->o.stats || td_ioengine_flagged(td, FIO_NOSTATS)) + return; + if (no_reduce) - lusec = utime_since(&io_u->issue_time, &icd->time); + llnsec = ntime_since(&io_u->issue_time, &icd->time); if (!td->o.disable_lat) { - unsigned long tusec; + unsigned long long tnsec; - tusec = utime_since(&io_u->start_time, &icd->time); - add_lat_sample(td, idx, tusec, bytes, io_u->offset); + tnsec = ntime_since(&io_u->start_time, &icd->time); + add_lat_sample(td, idx, tnsec, bytes, io_u->offset); if (td->flags & TD_F_PROFILE_OPS) { struct prof_io_ops *ops = &td->prof_io_ops; if (ops->io_u_lat) - icd->error = ops->io_u_lat(td, tusec); + icd->error = ops->io_u_lat(td, tnsec); } - if (td->o.max_latency && tusec > td->o.max_latency) - lat_fatal(td, icd, tusec, td->o.max_latency); - if (td->o.latency_target && tusec > td->o.latency_target) { + if (td->o.max_latency && tnsec > td->o.max_latency) + lat_fatal(td, icd, tnsec, td->o.max_latency); + if (td->o.latency_target && tnsec > td->o.latency_target) { if (lat_target_failed(td)) - lat_fatal(td, icd, tusec, td->o.latency_target); + lat_fatal(td, icd, tnsec, td->o.latency_target); } } - if (!td->o.disable_clat) { - add_clat_sample(td, idx, lusec, bytes, io_u->offset); - io_u_mark_latency(td, lusec); - } + if (ddir_rw(idx)) { + if (!td->o.disable_clat) { + add_clat_sample(td, idx, llnsec, bytes, io_u->offset); + io_u_mark_latency(td, llnsec); + } - if (!td->o.disable_bw) - add_bw_sample(td, idx, bytes, &icd->time); + if (!td->o.disable_bw && per_unit_log(td->bw_log)) + add_bw_sample(td, io_u, bytes, llnsec); - if (no_reduce) - add_iops_sample(td, idx, bytes, &icd->time); + if (no_reduce && per_unit_log(td->iops_log)) + add_iops_sample(td, io_u, bytes); + } if (td->ts.nr_block_infos && io_u->ddir == DDIR_TRIM) { uint32_t *info = io_u_block_info(td, io_u); @@ -1675,6 +1910,28 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, } } +static void file_log_write_comp(const struct thread_data *td, struct fio_file *f, + uint64_t offset, unsigned int bytes) +{ + int idx; + + if (!f) + return; + + if (f->first_write == -1ULL || offset < f->first_write) + f->first_write = offset; + if (f->last_write == -1ULL || ((offset + bytes) > f->last_write)) + f->last_write = offset + bytes; + + if (!f->last_write_comp) + return; + + idx = f->last_write_idx++; + f->last_write_comp[idx] = offset; + if (f->last_write_idx == td->o.iodepth) + f->last_write_idx = 0; +} + static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, struct io_completion_data *icd) { @@ -1685,7 +1942,7 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, dprint_io_u(io_u, "io complete"); assert(io_u->flags & IO_U_F_FLIGHT); - io_u_clear(io_u, IO_U_F_FLIGHT | IO_U_F_BUSY_OK); + io_u_clear(td, io_u, IO_U_F_FLIGHT | IO_U_F_BUSY_OK); /* * Mark IO ok to verify @@ -1719,30 +1976,16 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, int ret; td->io_blocks[ddir]++; - td->this_io_blocks[ddir]++; td->io_bytes[ddir] += bytes; - if (!(io_u->flags & IO_U_F_VER_LIST)) + if (!(io_u->flags & IO_U_F_VER_LIST)) { + td->this_io_blocks[ddir]++; td->this_io_bytes[ddir] += bytes; - - if (ddir == DDIR_WRITE) { - if (f) { - if (f->first_write == -1ULL || - io_u->offset < f->first_write) - f->first_write = io_u->offset; - if (f->last_write == -1ULL || - ((io_u->offset + bytes) > f->last_write)) - f->last_write = io_u->offset + bytes; - } - if (td->last_write_comp) { - int idx = td->last_write_idx++; - - td->last_write_comp[idx] = io_u->offset; - if (td->last_write_idx == td->o.iodepth) - td->last_write_idx = 0; - } } + if (ddir == DDIR_WRITE) + file_log_write_comp(td, f, io_u->offset, bytes); + if (ramp_time_over(td) && (td->runstate == TD_RUNNING || td->runstate == TD_VERIFYING)) account_io_completion(td, io_u, icd, ddir, bytes); @@ -1788,7 +2031,7 @@ static void init_icd(struct thread_data *td, struct io_completion_data *icd, icd->nr = nr; icd->error = 0; - for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) + for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) icd->bytes_done[ddir] = 0; } @@ -1827,7 +2070,7 @@ int io_u_sync_complete(struct thread_data *td, struct io_u *io_u) return -1; } - for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) + for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) td->bytes_done[ddir] += icd.bytes_done[ddir]; return 0; @@ -1843,7 +2086,7 @@ int io_u_queued_complete(struct thread_data *td, int min_evts) int ret, ddir; struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, }; - dprint(FD_IO, "io_u_queued_completed: min=%d\n", min_evts); + dprint(FD_IO, "io_u_queued_complete: min=%d\n", min_evts); if (!min_evts) tvp = &ts; @@ -1866,7 +2109,7 @@ int io_u_queued_complete(struct thread_data *td, int min_evts) return -1; } - for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) + for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) td->bytes_done[ddir] += icd.bytes_done[ddir]; return ret; @@ -1877,10 +2120,10 @@ int io_u_queued_complete(struct thread_data *td, int min_evts) */ void io_u_queued(struct thread_data *td, struct io_u *io_u) { - if (!td->o.disable_slat) { + if (!td->o.disable_slat && ramp_time_over(td) && td->o.stats) { unsigned long slat_time; - slat_time = utime_since(&io_u->start_time, &io_u->issue_time); + slat_time = ntime_since(&io_u->start_time, &io_u->issue_time); if (td->parent) td = td->parent; @@ -1895,9 +2138,7 @@ void io_u_queued(struct thread_data *td, struct io_u *io_u) */ static struct frand_state *get_buf_state(struct thread_data *td) { - uint64_t frand_max; unsigned int v; - unsigned long r; if (!td->o.dedupe_percentage) return &td->buf_state; @@ -1906,9 +2147,7 @@ static struct frand_state *get_buf_state(struct thread_data *td) return &td->buf_state; } - frand_max = rand_max(&td->dedupe_state); - r = __rand(&td->dedupe_state); - v = 1 + (int) (100.0 * (r / (frand_max + 1.0))); + v = rand32_between(&td->dedupe_state, 1, 100); if (v <= td->o.dedupe_percentage) return &td->buf_state_prev; @@ -1929,6 +2168,9 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write, { struct thread_options *o = &td->o; + if (o->mem_type == MEM_CUDA_MALLOC) + return; + if (o->compress_percentage || o->dedupe_percentage) { unsigned int perc = td->o.compress_percentage; struct frand_state *rs; @@ -1974,3 +2216,61 @@ void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u, io_u->buf_filled_len = 0; fill_io_buffer(td, io_u->buf, min_write, max_bs); } + +static int do_sync_file_range(const struct thread_data *td, + struct fio_file *f) +{ + off64_t offset, nbytes; + + offset = f->first_write; + nbytes = f->last_write - f->first_write; + + if (!nbytes) + return 0; + + return sync_file_range(f->fd, offset, nbytes, td->o.sync_file_range); +} + +int do_io_u_sync(const struct thread_data *td, struct io_u *io_u) +{ + int ret; + + if (io_u->ddir == DDIR_SYNC) { + ret = fsync(io_u->file->fd); + } else if (io_u->ddir == DDIR_DATASYNC) { +#ifdef CONFIG_FDATASYNC + ret = fdatasync(io_u->file->fd); +#else + ret = io_u->xfer_buflen; + io_u->error = EINVAL; +#endif + } else if (io_u->ddir == DDIR_SYNC_FILE_RANGE) + ret = do_sync_file_range(td, io_u->file); + else { + ret = io_u->xfer_buflen; + io_u->error = EINVAL; + } + + if (ret < 0) + io_u->error = errno; + + return ret; +} + +int do_io_u_trim(const struct thread_data *td, struct io_u *io_u) +{ +#ifndef FIO_HAVE_TRIM + io_u->error = EINVAL; + return 0; +#else + struct fio_file *f = io_u->file; + int ret; + + ret = os_trim(f, io_u->offset, io_u->xfer_buflen); + if (!ret) + return io_u->xfer_buflen; + + io_u->error = ret; + return 0; +#endif +}