X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=io_u.c;h=9f10206b95e8b1e80d14504c35159284cf238802;hp=efbcea9eb15bb47dca4e0f802f85ad86197665de;hb=50a8ce864e2c5bee7c44935b39b357aa8071615b;hpb=03a7c2512c4166a62e99dcc689bc7f53afe96205 diff --git a/io_u.c b/io_u.c index efbcea9e..9f10206b 100644 --- a/io_u.c +++ b/io_u.c @@ -12,6 +12,7 @@ #include "lib/rand.h" #include "lib/axmap.h" #include "err.h" +#include "lib/pow2.h" struct io_completion_data { int nr; /* input */ @@ -68,6 +69,9 @@ static uint64_t last_block(struct thread_data *td, struct fio_file *f, if (td->o.zone_range) max_size = td->o.zone_range; + if (td->o.min_bs[ddir] > td->o.ba[ddir]) + max_size -= td->o.min_bs[ddir] - td->o.ba[ddir]; + max_blocks = max_size / (uint64_t) td->o.ba[ddir]; if (!max_blocks) return 0; @@ -85,21 +89,25 @@ static int __get_next_rand_offset(struct thread_data *td, struct fio_file *f, { uint64_t r; - if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE) { - uint64_t lastb; + if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE || + td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE64) { + uint64_t frand_max, lastb; lastb = last_block(td, f, ddir); if (!lastb) return 1; + frand_max = rand_max(&td->random_state); r = __rand(&td->random_state); dprint(FD_RANDOM, "off rand %llu\n", (unsigned long long) r); - *b = lastb * (r / ((uint64_t) FRAND_MAX + 1.0)); + *b = lastb * (r / ((uint64_t) frand_max + 1.0)); } else { uint64_t off = 0; + assert(fio_file_lfsr(f)); + if (lfsr_next(&f->lfsr, &off)) return 1; @@ -144,6 +152,15 @@ static int __get_next_rand_offset_pareto(struct thread_data *td, return 0; } +static int __get_next_rand_offset_gauss(struct thread_data *td, + struct fio_file *f, enum fio_ddir ddir, + uint64_t *b) +{ + *b = gauss_next(&f->gauss); + return 0; +} + + static int flist_cmp(void *data, struct flist_head *a, struct flist_head *b) { struct rand_off *r1 = flist_entry(a, struct rand_off, list); @@ -161,6 +178,8 @@ static int get_off_from_method(struct thread_data *td, struct fio_file *f, return __get_next_rand_offset_zipf(td, f, ddir, b); else if (td->o.random_distribution == FIO_RAND_DIST_PARETO) return __get_next_rand_offset_pareto(td, f, ddir, b); + else if (td->o.random_distribution == FIO_RAND_DIST_GAUSS) + return __get_next_rand_offset_gauss(td, f, ddir, b); log_err("fio: unknown random distribution: %d\n", td->o.random_distribution); return 1; @@ -178,7 +197,8 @@ static inline int should_sort_io(struct thread_data *td) return 0; if (td->runstate != TD_VERIFYING) return 0; - if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE) + if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE || + td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE64) return 0; return 1; @@ -186,14 +206,16 @@ static inline int should_sort_io(struct thread_data *td) static int should_do_random(struct thread_data *td, enum fio_ddir ddir) { + uint64_t frand_max; unsigned int v; unsigned long r; if (td->o.perc_rand[ddir] == 100) return 1; + frand_max = rand_max(&td->seq_rand_state[ddir]); r = __rand(&td->seq_rand_state[ddir]); - v = 1 + (int) (100.0 * (r / (FRAND_MAX + 1.0))); + v = 1 + (int) (100.0 * (r / (frand_max + 1.0))); return v <= td->o.perc_rand[ddir]; } @@ -310,7 +332,7 @@ static int get_next_block(struct thread_data *td, struct io_u *io_u, *is_random = 1; } else { *is_random = 0; - io_u->flags |= IO_U_F_BUSY_OK; + io_u_set(io_u, IO_U_F_BUSY_OK); ret = get_next_seq_offset(td, f, ddir, &offset); if (ret) ret = get_next_rand_block(td, f, ddir, &b); @@ -320,7 +342,7 @@ static int get_next_block(struct thread_data *td, struct io_u *io_u, ret = get_next_seq_offset(td, f, ddir, &offset); } } else { - io_u->flags |= IO_U_F_BUSY_OK; + io_u_set(io_u, IO_U_F_BUSY_OK); *is_random = 0; if (td->o.rw_seq == RW_SEQ_SEQ) { @@ -422,6 +444,7 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, int ddir = io_u->ddir; unsigned int buflen = 0; unsigned int minbs, maxbs; + uint64_t frand_max; unsigned long r; assert(ddir_rw(ddir)); @@ -441,12 +464,13 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, if (!io_u_fits(td, io_u, minbs)) return 0; + frand_max = rand_max(&td->bsrange_state); do { r = __rand(&td->bsrange_state); if (!td->o.bssplit_nr[ddir]) { buflen = 1 + (unsigned int) ((double) maxbs * - (r / (FRAND_MAX + 1.0))); + (r / (frand_max + 1.0))); if (buflen < minbs) buflen = minbs; } else { @@ -458,7 +482,7 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, buflen = bsp->bs; perc += bsp->perc; - if ((r <= ((FRAND_MAX / 100L) * perc)) && + if ((r <= ((frand_max / 100L) * perc)) && io_u_fits(td, io_u, buflen)) break; } @@ -469,7 +493,7 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, ~(td->o.verify_interval - 1); if (!td->o.bs_unaligned && is_power_of_2(minbs)) - buflen = (buflen + minbs - 1) & ~(minbs - 1); + buflen &= ~(minbs - 1); } while (!io_u_fits(td, io_u, buflen)); @@ -504,11 +528,12 @@ static void set_rwmix_bytes(struct thread_data *td) static inline enum fio_ddir get_rand_ddir(struct thread_data *td) { + uint64_t frand_max = rand_max(&td->rwmix_state); unsigned int v; unsigned long r; r = __rand(&td->rwmix_state); - v = 1 + (int) (100.0 * (r / (FRAND_MAX + 1.0))); + v = 1 + (int) (100.0 * (r / (frand_max + 1.0))); if (v <= td->o.rwmix[DDIR_READ]) return DDIR_READ; @@ -527,63 +552,62 @@ void io_u_quiesce(struct thread_data *td) * io's that have been actually submitted to an async engine, * and cur_depth is meaningless for sync engines. */ + if (td->io_u_queued || td->cur_depth) { + int fio_unused ret; + + ret = td_io_commit(td); + } + while (td->io_u_in_flight) { int fio_unused ret; - ret = io_u_queued_complete(td, 1, NULL); + ret = io_u_queued_complete(td, 1); } } static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir) { enum fio_ddir odir = ddir ^ 1; - struct timeval t; - long usec; + long usec, now; assert(ddir_rw(ddir)); + now = utime_since_now(&td->start); - if (td->rate_pending_usleep[ddir] <= 0) + /* + * if rate_next_io_time is in the past, need to catch up to rate + */ + if (td->rate_next_io_time[ddir] <= now) return ddir; /* - * We have too much pending sleep in this direction. See if we + * We are ahead of rate in this direction. See if we * should switch. */ if (td_rw(td) && td->o.rwmix[odir]) { /* - * Other direction does not have too much pending, switch + * Other direction is behind rate, switch */ - if (td->rate_pending_usleep[odir] < 100000) + if (td->rate_next_io_time[odir] <= now) return odir; /* - * Both directions have pending sleep. Sleep the minimum time - * and deduct from both. + * Both directions are ahead of rate. sleep the min + * switch if necissary */ - if (td->rate_pending_usleep[ddir] <= - td->rate_pending_usleep[odir]) { - usec = td->rate_pending_usleep[ddir]; + if (td->rate_next_io_time[ddir] <= + td->rate_next_io_time[odir]) { + usec = td->rate_next_io_time[ddir] - now; } else { - usec = td->rate_pending_usleep[odir]; + usec = td->rate_next_io_time[odir] - now; ddir = odir; } } else - usec = td->rate_pending_usleep[ddir]; - - io_u_quiesce(td); - - fio_gettime(&t, NULL); - usec_sleep(td, usec); - usec = utime_since_now(&t); + usec = td->rate_next_io_time[ddir] - now; - td->rate_pending_usleep[ddir] -= usec; + if (td->o.io_submit_mode == IO_MODE_INLINE) + io_u_quiesce(td); - odir = ddir ^ 1; - if (td_rw(td) && __should_check_rate(td, odir)) - td->rate_pending_usleep[odir] -= usec; - - if (ddir == DDIR_TRIM) - return DDIR_TRIM; + usec = usec_sleep(td, usec); return ddir; } @@ -652,13 +676,23 @@ static enum fio_ddir get_rw_ddir(struct thread_data *td) static void set_rw_ddir(struct thread_data *td, struct io_u *io_u) { - io_u->ddir = io_u->acct_ddir = get_rw_ddir(td); + enum fio_ddir ddir = get_rw_ddir(td); + + if (td_trimwrite(td)) { + struct fio_file *f = io_u->file; + if (f->last_pos[DDIR_WRITE] == f->last_pos[DDIR_TRIM]) + ddir = DDIR_TRIM; + else + ddir = DDIR_WRITE; + } + + io_u->ddir = io_u->acct_ddir = ddir; if (io_u->ddir == DDIR_WRITE && (td->io_ops->flags & FIO_BARRIER) && td->o.barrier_blocks && !(td->io_issues[DDIR_WRITE] % td->o.barrier_blocks) && td->io_issues[DDIR_WRITE]) - io_u->flags |= IO_U_F_BARRIER; + io_u_set(io_u, IO_U_F_BARRIER); } void put_file_log(struct thread_data *td, struct fio_file *f) @@ -671,16 +705,21 @@ void put_file_log(struct thread_data *td, struct fio_file *f) void put_io_u(struct thread_data *td, struct io_u *io_u) { + if (td->parent) + td = td->parent; + td_io_u_lock(td); if (io_u->file && !(io_u->flags & IO_U_F_NO_FILE_PUT)) put_file_log(td, io_u->file); io_u->file = NULL; - io_u->flags |= IO_U_F_FREE; + io_u_set(io_u, IO_U_F_FREE); - if (io_u->flags & IO_U_F_IN_CUR_DEPTH) + if (io_u->flags & IO_U_F_IN_CUR_DEPTH) { td->cur_depth--; + assert(!(td->flags & TD_F_CHILD)); + } io_u_qpush(&td->io_u_freelist, io_u); td_io_u_unlock(td); td_io_u_free_notify(td); @@ -688,7 +727,7 @@ void put_io_u(struct thread_data *td, struct io_u *io_u) void clear_io_u(struct thread_data *td, struct io_u *io_u) { - io_u->flags &= ~IO_U_F_FLIGHT; + io_u_clear(io_u, IO_U_F_FLIGHT); put_io_u(td, io_u); } @@ -699,18 +738,24 @@ void requeue_io_u(struct thread_data *td, struct io_u **io_u) dprint(FD_IO, "requeue %p\n", __io_u); + if (td->parent) + td = td->parent; + td_io_u_lock(td); - __io_u->flags |= IO_U_F_FREE; + io_u_set(__io_u, IO_U_F_FREE); if ((__io_u->flags & IO_U_F_FLIGHT) && ddir_rw(ddir)) td->io_issues[ddir]--; - __io_u->flags &= ~IO_U_F_FLIGHT; - if (__io_u->flags & IO_U_F_IN_CUR_DEPTH) + io_u_clear(__io_u, IO_U_F_FLIGHT); + if (__io_u->flags & IO_U_F_IN_CUR_DEPTH) { td->cur_depth--; + assert(!(td->flags & TD_F_CHILD)); + } io_u_rpush(&td->io_u_requeues, __io_u); td_io_u_unlock(td); + td_io_u_free_notify(td); *io_u = NULL; } @@ -953,6 +998,7 @@ static struct fio_file *get_next_file_rand(struct thread_data *td, enum fio_file_flags goodf, enum fio_file_flags badf) { + uint64_t frand_max = rand_max(&td->next_file_state); struct fio_file *f; int fno; @@ -962,7 +1008,7 @@ static struct fio_file *get_next_file_rand(struct thread_data *td, r = __rand(&td->next_file_state); fno = (unsigned int) ((double) td->o.nr_files - * (r / (FRAND_MAX + 1.0))); + * (r / (frand_max + 1.0))); f = td->files[fno]; if (fio_file_done(f)) @@ -1303,21 +1349,23 @@ again: if (io_u) { assert(io_u->flags & IO_U_F_FREE); - io_u->flags &= ~(IO_U_F_FREE | IO_U_F_NO_FILE_PUT | + io_u_clear(io_u, IO_U_F_FREE | IO_U_F_NO_FILE_PUT | IO_U_F_TRIMMED | IO_U_F_BARRIER | IO_U_F_VER_LIST); io_u->error = 0; io_u->acct_ddir = -1; td->cur_depth++; - io_u->flags |= IO_U_F_IN_CUR_DEPTH; + assert(!(td->flags & TD_F_CHILD)); + io_u_set(io_u, IO_U_F_IN_CUR_DEPTH); io_u->ipo = NULL; - } else if (td->o.verify_async) { + } else if (td_async_processing(td)) { /* * We ran out, wait for async verify threads to finish and * return one */ - pthread_cond_wait(&td->free_cond, &td->io_u_lock); + assert(!(td->flags & TD_F_CHILD)); + assert(!pthread_cond_wait(&td->free_cond, &td->io_u_lock)); goto again; } @@ -1516,7 +1564,7 @@ err_put: return ERR_PTR(ret); } -void io_u_log_error(struct thread_data *td, struct io_u *io_u) +static void __io_u_log_error(struct thread_data *td, struct io_u *io_u) { enum error_type_bit eb = td_error_type(io_u->ddir, io_u->error); @@ -1534,6 +1582,13 @@ void io_u_log_error(struct thread_data *td, struct io_u *io_u) td_verror(td, io_u->error, "io_u error"); } +void io_u_log_error(struct thread_data *td, struct io_u *io_u) +{ + __io_u_log_error(td, io_u); + if (td->parent) + __io_u_log_error(td, io_u); +} + static inline int gtod_reduce(struct thread_data *td) { return td->o.disable_clat && td->o.disable_lat && td->o.disable_slat @@ -1544,9 +1599,13 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, struct io_completion_data *icd, const enum fio_ddir idx, unsigned int bytes) { + const int no_reduce = !gtod_reduce(td); unsigned long lusec = 0; - if (!gtod_reduce(td)) + if (td->parent) + td = td->parent; + + if (no_reduce) lusec = utime_since(&io_u->issue_time, &icd->time); if (!td->o.disable_lat) { @@ -1578,19 +1637,21 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, if (!td->o.disable_bw) add_bw_sample(td, idx, bytes, &icd->time); - if (!gtod_reduce(td)) + if (no_reduce) add_iops_sample(td, idx, bytes, &icd->time); -} -static long long usec_for_io(struct thread_data *td, enum fio_ddir ddir) -{ - uint64_t secs, remainder, bps, bytes; - - bytes = td->this_io_bytes[ddir]; - bps = td->rate_bps[ddir]; - secs = bytes / bps; - remainder = bytes % bps; - return remainder * 1000000 / bps + secs * 1000000; + if (td->ts.nr_block_infos && io_u->ddir == DDIR_TRIM) { + uint32_t *info = io_u_block_info(td, io_u); + if (BLOCK_INFO_STATE(*info) < BLOCK_STATE_TRIM_FAILURE) { + if (io_u->ddir == DDIR_TRIM) { + *info = BLOCK_INFO(BLOCK_STATE_TRIMMED, + BLOCK_INFO_TRIMS(*info) + 1); + } else if (io_u->ddir == DDIR_WRITE) { + *info = BLOCK_INFO_SET_STATE(BLOCK_STATE_WRITTEN, + *info); + } + } + } } static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, @@ -1602,9 +1663,8 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, dprint_io_u(io_u, "io complete"); - td_io_u_lock(td); assert(io_u->flags & IO_U_F_FLIGHT); - io_u->flags &= ~(IO_U_F_FLIGHT | IO_U_F_BUSY_OK); + io_u_clear(io_u, IO_U_F_FLIGHT | IO_U_F_BUSY_OK); /* * Mark IO ok to verify @@ -1621,8 +1681,6 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, } } - td_io_u_unlock(td); - if (ddir_sync(ddir)) { td->last_was_sync = 1; if (f) { @@ -1637,7 +1695,6 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, if (!io_u->error && ddir_rw(ddir)) { unsigned int bytes = io_u->buflen - io_u->resid; - const enum fio_ddir oddir = ddir ^ 1; int ret; td->io_blocks[ddir]++; @@ -1666,22 +1723,9 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, } if (ramp_time_over(td) && (td->runstate == TD_RUNNING || - td->runstate == TD_VERIFYING)) { + td->runstate == TD_VERIFYING)) account_io_completion(td, io_u, icd, ddir, bytes); - if (__should_check_rate(td, ddir)) { - td->rate_pending_usleep[ddir] = - (usec_for_io(td, ddir) - - utime_since_now(&td->start)); - } - if (ddir != DDIR_TRIM && - __should_check_rate(td, oddir)) { - td->rate_pending_usleep[oddir] = - (usec_for_io(td, oddir) - - utime_since_now(&td->start)); - } - } - icd->bytes_done[ddir] += bytes; if (io_u->end_io) { @@ -1746,10 +1790,10 @@ static void ios_completed(struct thread_data *td, /* * Complete a single io_u for the sync engines. */ -int io_u_sync_complete(struct thread_data *td, struct io_u *io_u, - uint64_t *bytes) +int io_u_sync_complete(struct thread_data *td, struct io_u *io_u) { struct io_completion_data icd; + int ddir; init_icd(td, &icd, 1); io_completed(td, &io_u, &icd); @@ -1762,12 +1806,8 @@ int io_u_sync_complete(struct thread_data *td, struct io_u *io_u, return -1; } - if (bytes) { - int ddir; - - for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) - bytes[ddir] += icd.bytes_done[ddir]; - } + for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) + td->bytes_done[ddir] += icd.bytes_done[ddir]; return 0; } @@ -1775,12 +1815,11 @@ int io_u_sync_complete(struct thread_data *td, struct io_u *io_u, /* * Called to complete min_events number of io for the async engines. */ -int io_u_queued_complete(struct thread_data *td, int min_evts, - uint64_t *bytes) +int io_u_queued_complete(struct thread_data *td, int min_evts) { struct io_completion_data icd; struct timespec *tvp = NULL; - int ret; + int ret, ddir; struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, }; dprint(FD_IO, "io_u_queued_completed: min=%d\n", min_evts); @@ -1804,12 +1843,8 @@ int io_u_queued_complete(struct thread_data *td, int min_evts, return -1; } - if (bytes) { - int ddir; - - for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) - bytes[ddir] += icd.bytes_done[ddir]; - } + for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) + td->bytes_done[ddir] += icd.bytes_done[ddir]; return 0; } @@ -1823,6 +1858,10 @@ void io_u_queued(struct thread_data *td, struct io_u *io_u) unsigned long slat_time; slat_time = utime_since(&io_u->start_time, &io_u->issue_time); + + if (td->parent) + td = td->parent; + add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen, io_u->offset); } @@ -1833,16 +1872,20 @@ void io_u_queued(struct thread_data *td, struct io_u *io_u) */ static struct frand_state *get_buf_state(struct thread_data *td) { + uint64_t frand_max; unsigned int v; unsigned long r; if (!td->o.dedupe_percentage) return &td->buf_state; - else if (td->o.dedupe_percentage == 100) - return &td->buf_state_prev; + else if (td->o.dedupe_percentage == 100) { + frand_copy(&td->buf_state_prev, &td->buf_state); + return &td->buf_state; + } + frand_max = rand_max(&td->dedupe_state); r = __rand(&td->dedupe_state); - v = 1 + (int) (100.0 * (r / (FRAND_MAX + 1.0))); + v = 1 + (int) (100.0 * (r / (frand_max + 1.0))); if (v <= td->o.dedupe_percentage) return &td->buf_state_prev; @@ -1852,7 +1895,9 @@ static struct frand_state *get_buf_state(struct thread_data *td) static void save_buf_state(struct thread_data *td, struct frand_state *rs) { - if (rs == &td->buf_state) + if (td->o.dedupe_percentage == 100) + frand_copy(rs, &td->buf_state_prev); + else if (rs == &td->buf_state) frand_copy(&td->buf_state_prev, rs); } @@ -1861,7 +1906,7 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write, { struct thread_options *o = &td->o; - if (o->compress_percentage) { + if (o->compress_percentage || o->dedupe_percentage) { unsigned int perc = td->o.compress_percentage; struct frand_state *rs; unsigned int left = max_bs; @@ -1890,8 +1935,10 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write, } while (left); } else if (o->buffer_pattern_bytes) fill_buffer_pattern(td, buf, max_bs); - else + else if (o->zero_buffers) memset(buf, 0, max_bs); + else + fill_random_buf(get_buf_state(td), buf, max_bs); } /*