X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=io_u.c;h=1e9018c4f6ee85bc5362aea312b9b2ec15a15978;hp=a6d1f1d04c593c72c3f4d6f01bbb78b10b525f0e;hb=b0f65863844b4de92d10fefaabde80ea5bc3e5cc;hpb=de605666880852ba9d1a0469723126b09782c520 diff --git a/io_u.c b/io_u.c index a6d1f1d0..1e9018c4 100644 --- a/io_u.c +++ b/io_u.c @@ -7,12 +7,6 @@ #include "fio.h" #include "hash.h" -#include "lib/ffz.h" - -/* - * Change this define to play with the timeout handling - */ -#undef FIO_USE_TIMEOUT struct io_completion_data { int nr; /* input */ @@ -44,40 +38,52 @@ static void mark_random_map(struct thread_data *td, struct io_u *io_u) unsigned int min_bs = td->o.rw_min_bs; struct fio_file *f = io_u->file; unsigned long long block; - unsigned int blocks; - unsigned int nr_blocks; + unsigned int blocks, nr_blocks; block = (io_u->offset - f->file_offset) / (unsigned long long) min_bs; - blocks = 0; nr_blocks = (io_u->buflen + min_bs - 1) / min_bs; + blocks = 0; - while (blocks < nr_blocks) { + while (nr_blocks) { + unsigned int this_blocks, mask; unsigned int idx, bit; /* * If we have a mixed random workload, we may * encounter blocks we already did IO to. */ - if ((td->o.ddir_nr == 1) && !random_map_free(f, block)) + if ((td->o.ddir_nr == 1) && !random_map_free(f, block)) { + if (!blocks) + blocks = 1; break; + } idx = RAND_MAP_IDX(f, block); bit = RAND_MAP_BIT(f, block); fio_assert(td, idx < f->num_maps); - f->file_map[idx] |= (1 << bit); - block++; - blocks++; + this_blocks = nr_blocks; + if (this_blocks + bit > BLOCKS_PER_MAP) + this_blocks = BLOCKS_PER_MAP - bit; + + if (this_blocks == BLOCKS_PER_MAP) + mask = -1U; + else + mask = ((1U << this_blocks) - 1) << bit; + + f->file_map[idx] |= mask; + nr_blocks -= this_blocks; + blocks += this_blocks; + block += this_blocks; } if ((blocks * min_bs) < io_u->buflen) io_u->buflen = blocks * min_bs; } -static inline unsigned long long last_block(struct thread_data *td, - struct fio_file *f, - enum fio_ddir ddir) +static unsigned long long last_block(struct thread_data *td, struct fio_file *f, + enum fio_ddir ddir) { unsigned long long max_blocks; unsigned long long max_size; @@ -89,7 +95,7 @@ static inline unsigned long long last_block(struct thread_data *td, if (max_size > f->real_file_size) max_size = f->real_file_size; - max_blocks = max_size / (unsigned long long) td->o.min_bs[ddir]; + max_blocks = max_size / (unsigned long long) td->o.ba[ddir]; if (!max_blocks) return 0; @@ -107,7 +113,8 @@ static int get_next_free_block(struct thread_data *td, struct fio_file *f, i = f->last_free_lookup; *b = (i * BLOCKS_PER_MAP); - while ((*b) * min_bs < f->real_file_size) { + while ((*b) * min_bs < f->real_file_size && + (*b) * min_bs < f->io_size) { if (f->file_map[i] != (unsigned int) -1) { *b += ffz(f->file_map[i]); if (*b > last_block(td, f, ddir)) @@ -134,7 +141,7 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f, r = os_random_long(&td->random_state); dprint(FD_RANDOM, "off rand %llu\n", r); *b = (last_block(td, f, ddir) - 1) - * (r / ((unsigned long long) RAND_MAX + 1.0)); + * (r / ((unsigned long long) OS_RAND_MAX + 1.0)); /* * if we are not maintaining a random map, we are done. @@ -159,7 +166,8 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f, */ loops = 10; do { - f->last_free_lookup = (f->num_maps - 1) * (r / (RAND_MAX+1.0)); + f->last_free_lookup = (f->num_maps - 1) * + (r / (OS_RAND_MAX + 1.0)); if (!get_next_free_block(td, f, ddir, b)) return 0; @@ -187,18 +195,25 @@ static int get_next_offset(struct thread_data *td, struct io_u *io_u) if (td_random(td) && (td->o.ddir_nr && !--td->ddir_nr)) { td->ddir_nr = td->o.ddir_nr; - if (get_next_rand_offset(td, f, ddir, &b)) + if (get_next_rand_offset(td, f, ddir, &b)) { + dprint(FD_IO, "%s: getting rand offset failed\n", + f->file_name); return 1; + } } else { if (f->last_pos >= f->real_file_size) { if (!td_random(td) || - get_next_rand_offset(td, f, ddir, &b)) + get_next_rand_offset(td, f, ddir, &b)) { + dprint(FD_IO, "%s: pos %llu > size %llu\n", + f->file_name, f->last_pos, + f->real_file_size); return 1; + } } else b = (f->last_pos - f->file_offset) / td->o.min_bs[ddir]; } - io_u->offset = b * td->o.min_bs[ddir]; + io_u->offset = b * td->o.ba[ddir]; if (io_u->offset >= f->io_size) { dprint(FD_IO, "get_next_offset: offset %llu >= io_size %llu\n", io_u->offset, f->io_size); @@ -215,43 +230,51 @@ static int get_next_offset(struct thread_data *td, struct io_u *io_u) return 0; } +static inline int is_power_of_2(unsigned int val) +{ + return (val != 0 && ((val & (val - 1)) == 0)); +} + static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u) { const int ddir = io_u->ddir; - unsigned int buflen = buflen; /* silence dumb gcc warning */ + unsigned int uninitialized_var(buflen); + unsigned int minbs, maxbs; long r; - if (td->o.min_bs[ddir] == td->o.max_bs[ddir]) - buflen = td->o.min_bs[ddir]; + minbs = td->o.min_bs[ddir]; + maxbs = td->o.max_bs[ddir]; + + if (minbs == maxbs) + buflen = minbs; else { r = os_random_long(&td->bsrange_state); - if (!td->o.bssplit_nr) { - buflen = (unsigned int) - (1 + (double) (td->o.max_bs[ddir] - 1) - * r / (RAND_MAX + 1.0)); + if (!td->o.bssplit_nr[ddir]) { + buflen = 1 + (unsigned int) ((double) maxbs * + (r / (OS_RAND_MAX + 1.0))); + if (buflen < minbs) + buflen = minbs; } else { long perc = 0; unsigned int i; - for (i = 0; i < td->o.bssplit_nr; i++) { - struct bssplit *bsp = &td->o.bssplit[i]; + for (i = 0; i < td->o.bssplit_nr[ddir]; i++) { + struct bssplit *bsp = &td->o.bssplit[ddir][i]; buflen = bsp->bs; perc += bsp->perc; - if (r <= ((LONG_MAX / 100L) * perc)) + if (r <= ((OS_RAND_MAX / 100L) * perc)) break; } } - if (!td->o.bs_unaligned) { - buflen = (buflen + td->o.min_bs[ddir] - 1) - & ~(td->o.min_bs[ddir] - 1); - } + if (!td->o.bs_unaligned && is_power_of_2(minbs)) + buflen = (buflen + minbs - 1) & ~(minbs - 1); } if (io_u->offset + buflen > io_u->file->real_file_size) { dprint(FD_IO, "lower buflen %u -> %u (ddir=%d)\n", buflen, - td->o.min_bs[ddir], ddir); - buflen = td->o.min_bs[ddir]; + minbs, ddir); + buflen = minbs; } return buflen; @@ -276,7 +299,7 @@ static inline enum fio_ddir get_rand_ddir(struct thread_data *td) long r; r = os_random_long(&td->rwmix_state); - v = 1 + (int) (100.0 * (r / (RAND_MAX + 1.0))); + v = 1 + (int) (100.0 * (r / (OS_RAND_MAX + 1.0))); if (v <= td->o.rwmix[DDIR_READ]) return DDIR_READ; @@ -307,8 +330,10 @@ static enum fio_ddir get_rw_ddir(struct thread_data *td) max_bytes = td->this_io_bytes[ddir]; if (max_bytes >= (td->o.size * td->o.rwmix[ddir] / 100)) { - if (!td->rw_end_set[ddir]) + if (!td->rw_end_set[ddir]) { td->rw_end_set[ddir] = 1; + fio_gettime(&td->rw_end[ddir], NULL); + } ddir ^= 1; } @@ -342,8 +367,8 @@ void put_io_u(struct thread_data *td, struct io_u *io_u) put_file_log(td, io_u->file); io_u->file = NULL; - list_del(&io_u->list); - list_add(&io_u->list, &td->io_u_freelist); + flist_del(&io_u->list); + flist_add(&io_u->list, &td->io_u_freelist); td->cur_depth--; } @@ -359,8 +384,8 @@ void requeue_io_u(struct thread_data *td, struct io_u **io_u) __io_u->flags &= ~IO_U_F_FLIGHT; - list_del(&__io_u->list); - list_add_tail(&__io_u->list, &td->io_u_requeues); + flist_del(&__io_u->list); + flist_add_tail(&__io_u->list, &td->io_u_requeues); td->cur_depth--; *io_u = NULL; } @@ -604,17 +629,29 @@ static struct fio_file *get_next_file_rand(struct thread_data *td, int goodf, do { long r = os_random_long(&td->next_file_state); + int opened = 0; fno = (unsigned int) ((double) td->o.nr_files - * (r / (RAND_MAX + 1.0))); + * (r / (OS_RAND_MAX + 1.0))); f = td->files[fno]; if (f->flags & FIO_FILE_DONE) continue; + if (!(f->flags & FIO_FILE_OPEN)) { + int err; + + err = td_io_open_file(td, f); + if (err) + continue; + opened = 1; + } + if ((!goodf || (f->flags & goodf)) && !(f->flags & badf)) { dprint(FD_FILE, "get_next_file_rand: %p\n", f); return f; } + if (opened) + td_io_close_file(td, f); } while (1); } @@ -628,20 +665,40 @@ static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf, struct fio_file *f; do { + int opened = 0; + f = td->files[td->next_file]; td->next_file++; if (td->next_file >= td->o.nr_files) td->next_file = 0; + dprint(FD_FILE, "trying file %s %x\n", f->file_name, f->flags); if (f->flags & FIO_FILE_DONE) { f = NULL; continue; } + if (!(f->flags & FIO_FILE_OPEN)) { + int err; + + err = td_io_open_file(td, f); + if (err) { + dprint(FD_FILE, "error %d on open of %s\n", + err, f->file_name); + f = NULL; + continue; + } + opened = 1; + } + + dprint(FD_FILE, "goodf=%x, badf=%x, ff=%x\n", goodf, badf, f->flags); if ((!goodf || (f->flags & goodf)) && !(f->flags & badf)) break; + if (opened) + td_io_close_file(td, f); + f = NULL; } while (td->next_file != old_next_file); @@ -655,7 +712,7 @@ static struct fio_file *get_next_file(struct thread_data *td) assert(td->o.nr_files <= td->files_index); - if (!td->nr_open_files || td->nr_done_files >= td->o.nr_files) { + if (td->nr_done_files >= td->o.nr_files) { dprint(FD_FILE, "get_next_file: nr_open=%d, nr_done=%d," " nr_files=%d\n", td->nr_open_files, td->nr_done_files, @@ -664,10 +721,15 @@ static struct fio_file *get_next_file(struct thread_data *td) } f = td->file_service_file; - if (f && (f->flags & FIO_FILE_OPEN) && td->file_service_left--) - goto out; + if (f && (f->flags & FIO_FILE_OPEN) && !(f->flags & FIO_FILE_CLOSING)) { + if (td->o.file_service_type == FIO_FSERVICE_SEQ) + goto out; + if (td->file_service_left--) + goto out; + } - if (td->o.file_service_type == FIO_FSERVICE_RR) + if (td->o.file_service_type == FIO_FSERVICE_RR || + td->o.file_service_type == FIO_FSERVICE_SEQ) f = get_next_file_rr(td, FIO_FILE_OPEN, FIO_FILE_CLOSING); else f = get_next_file_rand(td, FIO_FILE_OPEN, FIO_FILE_CLOSING); @@ -675,22 +737,7 @@ static struct fio_file *get_next_file(struct thread_data *td) td->file_service_file = f; td->file_service_left = td->file_service_nr - 1; out: - dprint(FD_FILE, "get_next_file: %p\n", f); - return f; -} - -static struct fio_file *find_next_new_file(struct thread_data *td) -{ - struct fio_file *f; - - if (!td->nr_open_files || td->nr_done_files >= td->o.nr_files) - return NULL; - - if (td->o.file_service_type == FIO_FSERVICE_RR) - f = get_next_file_rr(td, 0, FIO_FILE_OPEN); - else - f = get_next_file_rand(td, 0, FIO_FILE_OPEN); - + dprint(FD_FILE, "get_next_file: %p [%s]\n", f, f->file_name); return f; } @@ -703,45 +750,18 @@ static int set_io_u_file(struct thread_data *td, struct io_u *io_u) if (!f) return 1; -set_file: io_u->file = f; get_file(f); if (!fill_io_u(td, io_u)) break; - /* - * optimization to prevent close/open of the same file. This - * way we preserve queueing etc. - */ - if (td->o.nr_files == 1 && td->o.time_based) { - put_file_log(td, f); - fio_file_reset(f); - goto set_file; - } - - /* - * td_io_close() does a put_file() as well, so no need to - * do that here. - */ - io_u->file = NULL; + put_file_log(td, f); td_io_close_file(td, f); + io_u->file = NULL; f->flags |= FIO_FILE_DONE; td->nr_done_files++; - - /* - * probably not the right place to do this, but see - * if we need to open a new file - */ - if (td->nr_open_files < td->o.open_files && - td->o.open_files != td->o.nr_files) { - f = find_next_new_file(td); - - if (!f || td_io_open_file(td, f)) - return 1; - - goto set_file; - } + dprint(FD_FILE, "%s: is done (%d of %d)\n", f->file_name, td->nr_done_files, td->o.nr_files); } while (1); return 0; @@ -752,10 +772,10 @@ struct io_u *__get_io_u(struct thread_data *td) { struct io_u *io_u = NULL; - if (!list_empty(&td->io_u_requeues)) - io_u = list_entry(td->io_u_requeues.next, struct io_u, list); + if (!flist_empty(&td->io_u_requeues)) + io_u = flist_entry(td->io_u_requeues.next, struct io_u, list); else if (!queue_full(td)) { - io_u = list_entry(td->io_u_freelist.next, struct io_u, list); + io_u = flist_entry(td->io_u_freelist.next, struct io_u, list); io_u->buflen = 0; io_u->resid = 0; @@ -768,8 +788,8 @@ struct io_u *__get_io_u(struct thread_data *td) io_u->flags &= ~IO_U_F_FREE; io_u->error = 0; - list_del(&io_u->list); - list_add(&io_u->list, &td->io_u_busylist); + flist_del(&io_u->list); + flist_add(&io_u->list, &td->io_u_busylist); td->cur_depth++; } @@ -819,7 +839,7 @@ struct io_u *get_io_u(struct thread_data *td) f->last_pos = io_u->offset + io_u->buflen; - if (td->o.verify != VERIFY_NONE) + if (td->o.verify != VERIFY_NONE && io_u->ddir == DDIR_WRITE) populate_verify_io_u(td, io_u); else if (td->o.refill_buffers && io_u->ddir == DDIR_WRITE) io_u_fill_buffer(td, io_u, io_u->xfer_buflen); @@ -834,7 +854,8 @@ struct io_u *get_io_u(struct thread_data *td) out: if (!td_io_prep(td, io_u)) { - fio_gettime(&io_u->start_time, NULL); + if (!td->o.disable_slat) + fio_gettime(&io_u->start_time, NULL); return io_u; } err_put: @@ -864,7 +885,11 @@ void io_u_log_error(struct thread_data *td, struct io_u *io_u) static void io_completed(struct thread_data *td, struct io_u *io_u, struct io_completion_data *icd) { - unsigned long usec; + /* + * Older gcc's are too dumb to realize that usec is always used + * initialized, silence that warning. + */ + unsigned long uninitialized_var(usec); dprint_io_u(io_u, "io complete"); @@ -887,11 +912,18 @@ static void io_completed(struct thread_data *td, struct io_u *io_u, td->io_bytes[idx] += bytes; td->this_io_bytes[idx] += bytes; - usec = utime_since(&io_u->issue_time, &icd->time); + if (ramp_time_over(td)) { + if (!td->o.disable_clat || !td->o.disable_bw) + usec = utime_since(&io_u->issue_time, + &icd->time); - add_clat_sample(td, idx, usec); - add_bw_sample(td, idx, &icd->time); - io_u_mark_latency(td, usec); + if (!td->o.disable_clat) { + add_clat_sample(td, idx, usec, bytes); + io_u_mark_latency(td, usec); + } + if (!td->o.disable_bw) + add_bw_sample(td, idx, bytes, &icd->time); + } if (td_write(td) && idx == DDIR_WRITE && td->o.do_verify && @@ -911,9 +943,11 @@ static void io_completed(struct thread_data *td, struct io_u *io_u, } } -static void init_icd(struct io_completion_data *icd, int nr) +static void init_icd(struct thread_data *td, struct io_completion_data *icd, + int nr) { - fio_gettime(&icd->time, NULL); + if (!td->o.disable_clat || !td->o.disable_bw) + fio_gettime(&icd->time, NULL); icd->nr = nr; @@ -942,7 +976,7 @@ long io_u_sync_complete(struct thread_data *td, struct io_u *io_u) { struct io_completion_data icd; - init_icd(&icd, 1); + init_icd(td, &icd, 1); io_completed(td, io_u, &icd); put_io_u(td, io_u); @@ -956,26 +990,26 @@ long io_u_sync_complete(struct thread_data *td, struct io_u *io_u) /* * Called to complete min_events number of io for the async engines. */ -long io_u_queued_complete(struct thread_data *td, int min_events) +long io_u_queued_complete(struct thread_data *td, int min_evts) { struct io_completion_data icd; struct timespec *tvp = NULL; int ret; struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, }; - dprint(FD_IO, "io_u_queued_completed: min=%d\n", min_events); + dprint(FD_IO, "io_u_queued_completed: min=%d\n", min_evts); - if (!min_events) + if (!min_evts) tvp = &ts; - ret = td_io_getevents(td, min_events, td->cur_depth, tvp); + ret = td_io_getevents(td, min_evts, td->o.iodepth_batch_complete, tvp); if (ret < 0) { td_verror(td, -ret, "td_io_getevents"); return ret; } else if (!ret) return ret; - init_icd(&icd, ret); + init_icd(td, &icd, ret); ios_completed(td, &icd); if (!icd.error) return icd.bytes_done[0] + icd.bytes_done[1]; @@ -989,10 +1023,12 @@ long io_u_queued_complete(struct thread_data *td, int min_events) */ void io_u_queued(struct thread_data *td, struct io_u *io_u) { - unsigned long slat_time; + if (!td->o.disable_slat) { + unsigned long slat_time; - slat_time = utime_since(&io_u->start_time, &io_u->issue_time); - add_slat_sample(td, io_u->ddir, slat_time); + slat_time = utime_since(&io_u->start_time, &io_u->issue_time); + add_slat_sample(td, io_u->ddir, io_u->xfer_buflen, slat_time); + } } /* @@ -1011,86 +1047,3 @@ void io_u_fill_buffer(struct thread_data *td, struct io_u *io_u, } else memset(ptr, 0, max_bs); } - -#ifdef FIO_USE_TIMEOUT -void io_u_set_timeout(struct thread_data *td) -{ - assert(td->cur_depth); - - td->timer.it_interval.tv_sec = 0; - td->timer.it_interval.tv_usec = 0; - td->timer.it_value.tv_sec = IO_U_TIMEOUT + IO_U_TIMEOUT_INC; - td->timer.it_value.tv_usec = 0; - setitimer(ITIMER_REAL, &td->timer, NULL); - fio_gettime(&td->timeout_end, NULL); -} - -static void io_u_dump(struct io_u *io_u) -{ - unsigned long t_start = mtime_since_now(&io_u->start_time); - unsigned long t_issue = mtime_since_now(&io_u->issue_time); - - log_err("io_u=%p, t_start=%lu, t_issue=%lu\n", io_u, t_start, t_issue); - log_err(" buf=%p/%p, len=%lu/%lu, offset=%llu\n", io_u->buf, - io_u->xfer_buf, io_u->buflen, - io_u->xfer_buflen, - io_u->offset); - log_err(" ddir=%d, fname=%s\n", io_u->ddir, io_u->file->file_name); -} -#else -void io_u_set_timeout(struct thread_data fio_unused *td) -{ -} -#endif - -#ifdef FIO_USE_TIMEOUT -static void io_u_timeout_handler(int fio_unused sig) -{ - struct thread_data *td, *__td; - pid_t pid = getpid(); - struct list_head *entry; - struct io_u *io_u; - int i; - - log_err("fio: io_u timeout\n"); - - /* - * TLS would be nice... - */ - td = NULL; - for_each_td(__td, i) { - if (__td->pid == pid) { - td = __td; - break; - } - } - - if (!td) { - log_err("fio: io_u timeout, can't find job\n"); - exit(1); - } - - if (!td->cur_depth) { - log_err("fio: timeout without pending work?\n"); - return; - } - - log_err("fio: io_u timeout: job=%s, pid=%d\n", td->o.name, td->pid); - - list_for_each(entry, &td->io_u_busylist) { - io_u = list_entry(entry, struct io_u, list); - - io_u_dump(io_u); - } - - td_verror(td, ETIMEDOUT, "io_u timeout"); - exit(1); -} -#endif - -void io_u_init_timeout(void) -{ -#ifdef FIO_USE_TIMEOUT - signal(SIGALRM, io_u_timeout_handler); -#endif -}