X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=io_u.c;h=af3b415160609d9dbb5317ef27c4d67d004be23b;hp=4b0b5a7a11bdb002f72d552c89ee4ce8c4fd6b4e;hb=5c94b00876437a27f1761e47437166780c3c3b93;hpb=71b84caab6ef5ddbc7893e3268887b8ff0516f75 diff --git a/io_u.c b/io_u.c index 4b0b5a7a..af3b4151 100644 --- a/io_u.c +++ b/io_u.c @@ -223,7 +223,7 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f, if (!flist_empty(&td->next_rand_list)) { struct rand_off *r; fetch: - r = flist_entry(td->next_rand_list.next, struct rand_off, list); + r = flist_first_entry(&td->next_rand_list, struct rand_off, list); flist_del(&r->list); *b = r->off; free(r); @@ -688,10 +688,10 @@ void put_io_u(struct thread_data *td, struct io_u *io_u) { td_io_u_lock(td); - if (io_u->file && !(io_u->flags & IO_U_F_FREE_DEF)) + if (io_u->file && !(io_u->flags & IO_U_F_NO_FILE_PUT)) put_file_log(td, io_u->file); + io_u->file = NULL; - io_u->flags &= ~IO_U_F_FREE_DEF; io_u->flags |= IO_U_F_FREE; if (io_u->flags & IO_U_F_IN_CUR_DEPTH) @@ -1100,7 +1100,7 @@ out: static struct fio_file *get_next_file(struct thread_data *td) { - if (!(td->flags & TD_F_PROFILE_OPS)) { + if (td->flags & TD_F_PROFILE_OPS) { struct prof_io_ops *ops = &td->prof_io_ops; if (ops->get_next_file) @@ -1313,9 +1313,9 @@ again: if (io_u) { assert(io_u->flags & IO_U_F_FREE); - io_u->flags &= ~(IO_U_F_FREE | IO_U_F_FREE_DEF); - io_u->flags &= ~(IO_U_F_TRIMMED | IO_U_F_BARRIER); - io_u->flags &= ~IO_U_F_VER_LIST; + io_u->flags &= ~(IO_U_F_FREE | IO_U_F_NO_FILE_PUT | + IO_U_F_TRIMMED | IO_U_F_BARRIER | + IO_U_F_VER_LIST); io_u->error = 0; io_u->acct_ddir = -1; @@ -1528,21 +1528,16 @@ err_put: void io_u_log_error(struct thread_data *td, struct io_u *io_u) { enum error_type_bit eb = td_error_type(io_u->ddir, io_u->error); - const char *msg[] = { "read", "write", "sync", "datasync", - "sync_file_range", "wait", "trim" }; if (td_non_fatal_error(td, eb, io_u->error) && !td->o.error_dump) return; - log_err("fio: io_u error"); - - if (io_u->file) - log_err(" on file %s", io_u->file->file_name); - - log_err(": %s\n", strerror(io_u->error)); - - log_err(" %s offset=%llu, buflen=%lu\n", msg[io_u->ddir], - io_u->offset, io_u->xfer_buflen); + log_err("fio: io_u error%s%s: %s: %s offset=%llu, buflen=%lu\n", + io_u->file ? " on file " : "", + io_u->file ? io_u->file->file_name : "", + strerror(io_u->error), + io_ddir_name(io_u->ddir), + io_u->offset, io_u->xfer_buflen); if (!td->error) td_verror(td, io_u->error, "io_u error"); @@ -1567,7 +1562,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, unsigned long tusec; tusec = utime_since(&io_u->start_time, &icd->time); - add_lat_sample(td, idx, tusec, bytes); + add_lat_sample(td, idx, tusec, bytes, io_u->offset); if (td->flags & TD_F_PROFILE_OPS) { struct prof_io_ops *ops = &td->prof_io_ops; @@ -1585,7 +1580,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, } if (!td->o.disable_clat) { - add_clat_sample(td, idx, lusec, bytes); + add_clat_sample(td, idx, lusec, bytes, io_u->offset); io_u_mark_latency(td, lusec); } @@ -1607,10 +1602,12 @@ static long long usec_for_io(struct thread_data *td, enum fio_ddir ddir) return remainder * 1000000 / bps + secs * 1000000; } -static void io_completed(struct thread_data *td, struct io_u *io_u, +static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, struct io_completion_data *icd) { - struct fio_file *f; + struct io_u *io_u = *io_u_ptr; + enum fio_ddir ddir = io_u->ddir; + struct fio_file *f = io_u->file; dprint_io_u(io_u, "io complete"); @@ -1622,15 +1619,21 @@ static void io_completed(struct thread_data *td, struct io_u *io_u, * Mark IO ok to verify */ if (io_u->ipo) { - io_u->ipo->flags &= ~IP_F_IN_FLIGHT; - write_barrier(); + /* + * Remove errored entry from the verification list + */ + if (io_u->error) + unlog_io_piece(td, io_u); + else { + io_u->ipo->flags &= ~IP_F_IN_FLIGHT; + write_barrier(); + } } td_io_u_unlock(td); - if (ddir_sync(io_u->ddir)) { + if (ddir_sync(ddir)) { td->last_was_sync = 1; - f = io_u->file; if (f) { f->first_write = -1ULL; f->last_write = -1ULL; @@ -1639,52 +1642,51 @@ static void io_completed(struct thread_data *td, struct io_u *io_u, } td->last_was_sync = 0; - td->last_ddir = io_u->ddir; + td->last_ddir = ddir; - if (!io_u->error && ddir_rw(io_u->ddir)) { + if (!io_u->error && ddir_rw(ddir)) { unsigned int bytes = io_u->buflen - io_u->resid; - const enum fio_ddir idx = io_u->ddir; - const enum fio_ddir odx = io_u->ddir ^ 1; + const enum fio_ddir oddir = ddir ^ 1; int ret; - td->io_blocks[idx]++; - td->this_io_blocks[idx]++; - td->io_bytes[idx] += bytes; + td->io_blocks[ddir]++; + td->this_io_blocks[ddir]++; + td->io_bytes[ddir] += bytes; if (!(io_u->flags & IO_U_F_VER_LIST)) - td->this_io_bytes[idx] += bytes; - - if (idx == DDIR_WRITE) { - f = io_u->file; - if (f) { - if (f->first_write == -1ULL || - io_u->offset < f->first_write) - f->first_write = io_u->offset; - if (f->last_write == -1ULL || - ((io_u->offset + bytes) > f->last_write)) - f->last_write = io_u->offset + bytes; - } + td->this_io_bytes[ddir] += bytes; + + if (ddir == DDIR_WRITE && f) { + if (f->first_write == -1ULL || + io_u->offset < f->first_write) + f->first_write = io_u->offset; + if (f->last_write == -1ULL || + ((io_u->offset + bytes) > f->last_write)) + f->last_write = io_u->offset + bytes; } if (ramp_time_over(td) && (td->runstate == TD_RUNNING || td->runstate == TD_VERIFYING)) { - account_io_completion(td, io_u, icd, idx, bytes); + account_io_completion(td, io_u, icd, ddir, bytes); - if (__should_check_rate(td, idx)) { - td->rate_pending_usleep[idx] = - (usec_for_io(td, idx) - + if (__should_check_rate(td, ddir)) { + td->rate_pending_usleep[ddir] = + (usec_for_io(td, ddir) - utime_since_now(&td->start)); } - if (idx != DDIR_TRIM && __should_check_rate(td, odx)) - td->rate_pending_usleep[odx] = - (usec_for_io(td, odx) - + if (ddir != DDIR_TRIM && + __should_check_rate(td, oddir)) { + td->rate_pending_usleep[oddir] = + (usec_for_io(td, oddir) - utime_since_now(&td->start)); + } } - icd->bytes_done[idx] += bytes; + icd->bytes_done[ddir] += bytes; if (io_u->end_io) { - ret = io_u->end_io(td, io_u); + ret = io_u->end_io(td, io_u_ptr); + io_u = *io_u_ptr; if (ret && !icd->error) icd->error = ret; } @@ -1693,9 +1695,11 @@ static void io_completed(struct thread_data *td, struct io_u *io_u, io_u_log_error(td, io_u); } if (icd->error) { - enum error_type_bit eb = td_error_type(io_u->ddir, icd->error); + enum error_type_bit eb = td_error_type(ddir, icd->error); + if (!td_non_fatal_error(td, eb, icd->error)) return; + /* * If there is a non_fatal error, then add to the error count * and clear all the errors. @@ -1703,7 +1707,8 @@ static void io_completed(struct thread_data *td, struct io_u *io_u, update_error_count(td, icd->error); td_clear_error(td); icd->error = 0; - io_u->error = 0; + if (io_u) + io_u->error = 0; } } @@ -1731,9 +1736,9 @@ static void ios_completed(struct thread_data *td, for (i = 0; i < icd->nr; i++) { io_u = td->io_ops->event(td, i); - io_completed(td, io_u, icd); + io_completed(td, &io_u, icd); - if (!(io_u->flags & IO_U_F_FREE_DEF)) + if (io_u) put_io_u(td, io_u); } } @@ -1747,9 +1752,9 @@ int io_u_sync_complete(struct thread_data *td, struct io_u *io_u, struct io_completion_data icd; init_icd(td, &icd, 1); - io_completed(td, io_u, &icd); + io_completed(td, &io_u, &icd); - if (!(io_u->flags & IO_U_F_FREE_DEF)) + if (io_u) put_io_u(td, io_u); if (icd.error) { @@ -1782,6 +1787,8 @@ int io_u_queued_complete(struct thread_data *td, int min_evts, if (!min_evts) tvp = &ts; + else if (min_evts > td->cur_depth) + min_evts = td->cur_depth; ret = td_io_getevents(td, min_evts, td->o.iodepth_batch_complete, tvp); if (ret < 0) { @@ -1816,10 +1823,37 @@ void io_u_queued(struct thread_data *td, struct io_u *io_u) unsigned long slat_time; slat_time = utime_since(&io_u->start_time, &io_u->issue_time); - add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen); + add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen, + io_u->offset); } } +/* + * See if we should reuse the last seed, if dedupe is enabled + */ +static struct frand_state *get_buf_state(struct thread_data *td) +{ + unsigned int v; + unsigned long r; + + if (!td->o.dedupe_percentage) + return &td->buf_state; + + r = __rand(&td->dedupe_state); + v = 1 + (int) (100.0 * (r / (FRAND_MAX + 1.0))); + + if (v <= td->o.dedupe_percentage) + return &td->buf_state_prev; + + return &td->buf_state; +} + +static void save_buf_state(struct thread_data *td, struct frand_state *rs) +{ + if (rs == &td->buf_state) + frand_copy(&td->buf_state_prev, rs); +} + void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write, unsigned int max_bs) { @@ -1827,6 +1861,9 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write, fill_buffer_pattern(td, buf, max_bs); else if (!td->o.zero_buffers) { unsigned int perc = td->o.compress_percentage; + struct frand_state *rs; + + rs = get_buf_state(td); if (perc) { unsigned int seg = min_write; @@ -1835,10 +1872,12 @@ void fill_io_buffer(struct thread_data *td, void *buf, unsigned int min_write, if (!seg) seg = min_write; - fill_random_buf_percentage(&td->buf_state, buf, - perc, seg, max_bs); - } else - fill_random_buf(&td->buf_state, buf, max_bs); + fill_random_buf_percentage(rs, buf, perc, seg,max_bs); + save_buf_state(td, rs); + } else { + fill_random_buf(rs, buf, max_bs); + save_buf_state(td, rs); + } } else memset(buf, 0, max_bs); }