X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=io_u.c;h=6ec04fa30607ba755835bf96034bff1edcf34bd5;hp=cb8fc4ad606154a89b0ce8d0c99e36cae32deda2;hb=59466396a8f7d37ca7884aaacf591cb9a894a336;hpb=12bb8569a2cff7c3905e5a3a132069f47a861d12 diff --git a/io_u.c b/io_u.c index cb8fc4ad..6ec04fa3 100644 --- a/io_u.c +++ b/io_u.c @@ -20,7 +20,7 @@ struct io_completion_data { int error; /* output */ uint64_t bytes_done[DDIR_RWDIR_CNT]; /* output */ - struct timeval time; /* output */ + struct timespec time; /* output */ }; /* @@ -37,7 +37,7 @@ static bool random_map_free(struct fio_file *f, const uint64_t block) */ static void mark_random_map(struct thread_data *td, struct io_u *io_u) { - unsigned int min_bs = td->o.rw_min_bs; + unsigned int min_bs = td->o.min_bs[io_u->ddir]; struct fio_file *f = io_u->file; unsigned int nr_blocks; uint64_t block; @@ -157,6 +157,80 @@ static int __get_next_rand_offset_gauss(struct thread_data *td, return 0; } +static int __get_next_rand_offset_zoned_abs(struct thread_data *td, + struct fio_file *f, + enum fio_ddir ddir, uint64_t *b) +{ + struct zone_split_index *zsi; + uint64_t offset, lastb; + uint64_t send, stotal; + static int warned; + unsigned int v; + + lastb = last_block(td, f, ddir); + if (!lastb) + return 1; + + if (!td->o.zone_split_nr[ddir]) { +bail: + return __get_next_rand_offset(td, f, ddir, b, lastb); + } + + /* + * Generate a value, v, between 1 and 100, both inclusive + */ + v = rand32_between(&td->zone_state, 1, 100); + + zsi = &td->zone_state_index[ddir][v - 1]; + stotal = zsi->size_prev / td->o.ba[ddir]; + send = zsi->size / td->o.ba[ddir]; + + /* + * Should never happen + */ + if (send == -1U) { + if (!warned) { + log_err("fio: bug in zoned generation\n"); + warned = 1; + } + goto bail; + } else if (send > lastb) { + /* + * This happens if the user specifies ranges that exceed + * the file/device size. We can't handle that gracefully, + * so error and exit. + */ + log_err("fio: zoned_abs sizes exceed file size\n"); + return 1; + } + + /* + * 'send' is some percentage below or equal to 100 that + * marks the end of the current IO range. 'stotal' marks + * the start, in percent. + */ + if (stotal) + offset = stotal; + else + offset = 0; + + lastb = send - stotal; + + /* + * Generate index from 0..send-of-lastb + */ + if (__get_next_rand_offset(td, f, ddir, b, lastb) == 1) + return 1; + + /* + * Add our start offset, if any + */ + if (offset) + *b += offset; + + return 0; +} + static int __get_next_rand_offset_zoned(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir, uint64_t *b) @@ -249,6 +323,8 @@ static int get_off_from_method(struct thread_data *td, struct fio_file *f, return __get_next_rand_offset_gauss(td, f, ddir, b); else if (td->o.random_distribution == FIO_RAND_DIST_ZONED) return __get_next_rand_offset_zoned(td, f, ddir, b); + else if (td->o.random_distribution == FIO_RAND_DIST_ZONED_ABS) + return __get_next_rand_offset_zoned_abs(td, f, ddir, b); log_err("fio: unknown random distribution: %d\n", td->o.random_distribution); return 1; @@ -323,6 +399,17 @@ fetch: goto fetch; } +static void loop_cache_invalidate(struct thread_data *td, struct fio_file *f) +{ + struct thread_options *o = &td->o; + + if (o->invalidate_cache && !o->odirect) { + int fio_unused ret; + + ret = file_invalidate_cache(td, f); + } +} + static int get_next_rand_block(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir, uint64_t *b) { @@ -334,6 +421,7 @@ static int get_next_rand_block(struct thread_data *td, struct fio_file *f, fio_file_reset(td, f); if (!get_next_rand_offset(td, f, ddir, b)) return 0; + loop_cache_invalidate(td, f); } dprint(FD_IO, "%s: rand offset failed, last=%llu, size=%llu\n", @@ -349,15 +437,14 @@ static int get_next_seq_offset(struct thread_data *td, struct fio_file *f, assert(ddir_rw(ddir)); + /* + * If we reach the end for a time based run, reset us back to 0 + * and invalidate the cache, if we need to. + */ if (f->last_pos[ddir] >= f->io_size + get_start_offset(td, f) && o->time_based) { - struct thread_options *o = &td->o; - uint64_t io_size = f->io_size + (f->io_size % o->min_bs[ddir]); - - if (io_size > f->last_pos[ddir]) - f->last_pos[ddir] = 0; - else - f->last_pos[ddir] = f->last_pos[ddir] - io_size; + f->last_pos[ddir] = f->file_offset; + loop_cache_invalidate(td, f); } if (f->last_pos[ddir] < f->real_file_size) { @@ -533,6 +620,7 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, unsigned int buflen = 0; unsigned int minbs, maxbs; uint64_t frand_max, r; + bool power_2; assert(ddir_rw(ddir)); @@ -551,9 +639,9 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, if (!io_u_fits(td, io_u, minbs)) return 0; - frand_max = rand_max(&td->bsrange_state); + frand_max = rand_max(&td->bsrange_state[ddir]); do { - r = __rand(&td->bsrange_state); + r = __rand(&td->bsrange_state[ddir]); if (!td->o.bssplit_nr[ddir]) { buflen = 1 + (unsigned int) ((double) maxbs * @@ -577,9 +665,11 @@ static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u, } } - if (!td->o.bs_unaligned && is_power_of_2(minbs)) + power_2 = is_power_of_2(minbs); + if (!td->o.bs_unaligned && power_2) buflen &= ~(minbs - 1); - + else if (!td->o.bs_unaligned && !power_2) + buflen -= buflen % minbs; } while (!io_u_fits(td, io_u, buflen)); return buflen; @@ -659,7 +749,7 @@ int io_u_quiesce(struct thread_data *td) static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir) { enum fio_ddir odir = ddir ^ 1; - long usec; + uint64_t usec; uint64_t now; assert(ddir_rw(ddir)); @@ -714,28 +804,22 @@ static enum fio_ddir get_rw_ddir(struct thread_data *td) enum fio_ddir ddir; /* - * see if it's time to fsync - */ - if (td->o.fsync_blocks && - !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks) && - td->io_issues[DDIR_WRITE] && should_fsync(td)) - return DDIR_SYNC; - - /* - * see if it's time to fdatasync - */ - if (td->o.fdatasync_blocks && - !(td->io_issues[DDIR_WRITE] % td->o.fdatasync_blocks) && - td->io_issues[DDIR_WRITE] && should_fsync(td)) - return DDIR_DATASYNC; - - /* - * see if it's time to sync_file_range + * See if it's time to fsync/fdatasync/sync_file_range first, + * and if not then move on to check regular I/Os. */ - if (td->sync_file_range_nr && - !(td->io_issues[DDIR_WRITE] % td->sync_file_range_nr) && - td->io_issues[DDIR_WRITE] && should_fsync(td)) - return DDIR_SYNC_FILE_RANGE; + if (should_fsync(td)) { + if (td->o.fsync_blocks && td->io_issues[DDIR_WRITE] && + !(td->io_issues[DDIR_WRITE] % td->o.fsync_blocks)) + return DDIR_SYNC; + + if (td->o.fdatasync_blocks && td->io_issues[DDIR_WRITE] && + !(td->io_issues[DDIR_WRITE] % td->o.fdatasync_blocks)) + return DDIR_DATASYNC; + + if (td->sync_file_range_nr && td->io_issues[DDIR_WRITE] && + !(td->io_issues[DDIR_WRITE] % td->sync_file_range_nr)) + return DDIR_SYNC_FILE_RANGE; + } if (td_rw(td)) { /* @@ -902,8 +986,9 @@ static int fill_io_u(struct thread_data *td, struct io_u *io_u) } if (io_u->offset + io_u->buflen > io_u->file->real_file_size) { - dprint(FD_IO, "io_u %p, offset too large\n", io_u); - dprint(FD_IO, " off=%llu/%lu > %llu\n", + dprint(FD_IO, "io_u %p, offset + buflen exceeds file size\n", + io_u); + dprint(FD_IO, " offset=%llu/buflen=%lu > %llu\n", (unsigned long long) io_u->offset, io_u->buflen, (unsigned long long) io_u->file->real_file_size); return 1; @@ -991,11 +1076,52 @@ void io_u_mark_depth(struct thread_data *td, unsigned int nr) td->ts.io_u_map[idx] += nr; } -static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec) +static void io_u_mark_lat_nsec(struct thread_data *td, unsigned long long nsec) { int idx = 0; - assert(usec < 1000); + assert(nsec < 1000); + + switch (nsec) { + case 750 ... 999: + idx = 9; + break; + case 500 ... 749: + idx = 8; + break; + case 250 ... 499: + idx = 7; + break; + case 100 ... 249: + idx = 6; + break; + case 50 ... 99: + idx = 5; + break; + case 20 ... 49: + idx = 4; + break; + case 10 ... 19: + idx = 3; + break; + case 4 ... 9: + idx = 2; + break; + case 2 ... 3: + idx = 1; + case 0 ... 1: + break; + } + + assert(idx < FIO_IO_U_LAT_N_NR); + td->ts.io_u_lat_n[idx]++; +} + +static void io_u_mark_lat_usec(struct thread_data *td, unsigned long long usec) +{ + int idx = 0; + + assert(usec < 1000 && usec >= 1); switch (usec) { case 750 ... 999: @@ -1032,10 +1158,12 @@ static void io_u_mark_lat_usec(struct thread_data *td, unsigned long usec) td->ts.io_u_lat_u[idx]++; } -static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec) +static void io_u_mark_lat_msec(struct thread_data *td, unsigned long long msec) { int idx = 0; + assert(msec >= 1); + switch (msec) { default: idx = 11; @@ -1077,12 +1205,14 @@ static void io_u_mark_lat_msec(struct thread_data *td, unsigned long msec) td->ts.io_u_lat_m[idx]++; } -static void io_u_mark_latency(struct thread_data *td, unsigned long usec) +static void io_u_mark_latency(struct thread_data *td, unsigned long long nsec) { - if (usec < 1000) - io_u_mark_lat_usec(td, usec); + if (nsec < 1000) + io_u_mark_lat_nsec(td, nsec); + else if (nsec < 1000000) + io_u_mark_lat_usec(td, nsec / 1000); else - io_u_mark_lat_msec(td, usec / 1000); + io_u_mark_lat_msec(td, nsec / 1000000); } static unsigned int __get_next_fileno_rand(struct thread_data *td) @@ -1559,7 +1689,7 @@ static void small_content_scramble(struct io_u *io_u) unsigned int i, nr_blocks = io_u->buflen / 512; uint64_t boffset; unsigned int offset; - void *p, *end; + char *p, *end; if (!nr_blocks) return; @@ -1574,7 +1704,7 @@ static void small_content_scramble(struct io_u *io_u) * the buffer, given by the product of the usec time * and the actual offset. */ - offset = (io_u->start_time.tv_usec ^ boffset) & 511; + offset = ((io_u->start_time.tv_nsec/1000) ^ boffset) & 511; offset &= ~(sizeof(uint64_t) - 1); if (offset >= 512 - sizeof(uint64_t)) offset -= sizeof(uint64_t); @@ -1676,8 +1806,10 @@ out: if (!td_io_prep(td, io_u)) { if (!td->o.disable_lat) fio_gettime(&io_u->start_time, NULL); + if (do_scramble) small_content_scramble(io_u); + return io_u; } err_put: @@ -1729,43 +1861,46 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, const enum fio_ddir idx, unsigned int bytes) { const int no_reduce = !gtod_reduce(td); - unsigned long lusec = 0; + unsigned long long llnsec = 0; if (td->parent) td = td->parent; + if (!td->o.stats || td_ioengine_flagged(td, FIO_NOSTATS)) + return; + if (no_reduce) - lusec = utime_since(&io_u->issue_time, &icd->time); + llnsec = ntime_since(&io_u->issue_time, &icd->time); if (!td->o.disable_lat) { - unsigned long tusec; + unsigned long long tnsec; - tusec = utime_since(&io_u->start_time, &icd->time); - add_lat_sample(td, idx, tusec, bytes, io_u->offset); + tnsec = ntime_since(&io_u->start_time, &icd->time); + add_lat_sample(td, idx, tnsec, bytes, io_u->offset); if (td->flags & TD_F_PROFILE_OPS) { struct prof_io_ops *ops = &td->prof_io_ops; if (ops->io_u_lat) - icd->error = ops->io_u_lat(td, tusec); + icd->error = ops->io_u_lat(td, tnsec/1000); } - if (td->o.max_latency && tusec > td->o.max_latency) - lat_fatal(td, icd, tusec, td->o.max_latency); - if (td->o.latency_target && tusec > td->o.latency_target) { + if (td->o.max_latency && tnsec/1000 > td->o.max_latency) + lat_fatal(td, icd, tnsec/1000, td->o.max_latency); + if (td->o.latency_target && tnsec/1000 > td->o.latency_target) { if (lat_target_failed(td)) - lat_fatal(td, icd, tusec, td->o.latency_target); + lat_fatal(td, icd, tnsec/1000, td->o.latency_target); } } if (ddir_rw(idx)) { if (!td->o.disable_clat) { - add_clat_sample(td, idx, lusec, bytes, io_u->offset); - io_u_mark_latency(td, lusec); + add_clat_sample(td, idx, llnsec, bytes, io_u->offset); + io_u_mark_latency(td, llnsec); } if (!td->o.disable_bw && per_unit_log(td->bw_log)) - add_bw_sample(td, io_u, bytes, lusec); + add_bw_sample(td, io_u, bytes, llnsec); if (no_reduce && per_unit_log(td->iops_log)) add_iops_sample(td, io_u, bytes); @@ -1994,10 +2129,10 @@ int io_u_queued_complete(struct thread_data *td, int min_evts) */ void io_u_queued(struct thread_data *td, struct io_u *io_u) { - if (!td->o.disable_slat && ramp_time_over(td)) { + if (!td->o.disable_slat && ramp_time_over(td) && td->o.stats) { unsigned long slat_time; - slat_time = utime_since(&io_u->start_time, &io_u->issue_time); + slat_time = ntime_since(&io_u->start_time, &io_u->issue_time); if (td->parent) td = td->parent; @@ -2042,6 +2177,9 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write, { struct thread_options *o = &td->o; + if (o->mem_type == MEM_CUDA_MALLOC) + return; + if (o->compress_percentage || o->dedupe_percentage) { unsigned int perc = td->o.compress_percentage; struct frand_state *rs; @@ -2087,3 +2225,61 @@ void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u, io_u->buf_filled_len = 0; fill_io_buffer(td, io_u->buf, min_write, max_bs); } + +static int do_sync_file_range(const struct thread_data *td, + struct fio_file *f) +{ + off64_t offset, nbytes; + + offset = f->first_write; + nbytes = f->last_write - f->first_write; + + if (!nbytes) + return 0; + + return sync_file_range(f->fd, offset, nbytes, td->o.sync_file_range); +} + +int do_io_u_sync(const struct thread_data *td, struct io_u *io_u) +{ + int ret; + + if (io_u->ddir == DDIR_SYNC) { + ret = fsync(io_u->file->fd); + } else if (io_u->ddir == DDIR_DATASYNC) { +#ifdef CONFIG_FDATASYNC + ret = fdatasync(io_u->file->fd); +#else + ret = io_u->xfer_buflen; + io_u->error = EINVAL; +#endif + } else if (io_u->ddir == DDIR_SYNC_FILE_RANGE) + ret = do_sync_file_range(td, io_u->file); + else { + ret = io_u->xfer_buflen; + io_u->error = EINVAL; + } + + if (ret < 0) + io_u->error = errno; + + return ret; +} + +int do_io_u_trim(const struct thread_data *td, struct io_u *io_u) +{ +#ifndef FIO_HAVE_TRIM + io_u->error = EINVAL; + return 0; +#else + struct fio_file *f = io_u->file; + int ret; + + ret = os_trim(f, io_u->offset, io_u->xfer_buflen); + if (!ret) + return io_u->xfer_buflen; + + io_u->error = ret; + return 0; +#endif +}