X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=fio.c;h=ad2282d7516fe7fc0033b3bc721a12b16d370fe1;hp=37a425e85ee6fde71ec7aa537b0f6947687cc128;hb=5bfc35d7abe2582dc54127ca1d6e03792c9d62f5;hpb=6c28f0bb768f30673a7c15d9ab493ed2dfe4d5e3 diff --git a/fio.c b/fio.c index 37a425e8..ad2282d7 100644 --- a/fio.c +++ b/fio.c @@ -54,19 +54,21 @@ unsigned long done_secs = 0; static struct fio_mutex *startup_mutex; static volatile int fio_abort; static int exit_value; +static struct itimerval itimer; +static pthread_t gtod_thread; struct io_log *agg_io_log[2]; #define TERMINATE_ALL (-1) #define JOB_START_TIMEOUT (5 * 1000) -static inline void td_set_runstate(struct thread_data *td, int runstate) +void td_set_runstate(struct thread_data *td, int runstate) { if (td->runstate == runstate) return; - dprint(FD_PROCESS, "pid=%d: runstate %d -> %d\n", td->pid, td->runstate, - runstate); + dprint(FD_PROCESS, "pid=%d: runstate %d -> %d\n", (int) td->pid, + td->runstate, runstate); td->runstate = runstate; } @@ -80,7 +82,7 @@ static void terminate_threads(int group_id) for_each_td(td, i) { if (group_id == TERMINATE_ALL || groupid == td->groupid) { dprint(FD_PROCESS, "setting terminate on %s/%d\n", - td->o.name, td->pid); + td->o.name, (int) td->pid); td->terminate = 1; td->o.start_delay = 0; @@ -99,25 +101,87 @@ static void terminate_threads(int group_id) } } -static void sig_handler(int sig) +static void status_timer_arm(void) { - if (!threads) - return; + itimer.it_value.tv_sec = 0; + itimer.it_value.tv_usec = DISK_UTIL_MSEC * 1000; + setitimer(ITIMER_REAL, &itimer, NULL); +} - switch (sig) { - case SIGALRM: +static void sig_alrm(int fio_unused sig) +{ + if (threads) { update_io_ticks(); - disk_util_timer_arm(); print_thread_status(); - break; - default: + status_timer_arm(); + } +} + +/* + * Happens on thread runs with ctrl-c, ignore our own SIGQUIT + */ +static void sig_quit(int sig) +{ +} + +static void sig_int(int sig) +{ + if (threads) { printf("\nfio: terminating on signal %d\n", sig); fflush(stdout); terminate_threads(TERMINATE_ALL); - break; } } +static void sig_ill(int fio_unused sig) +{ + if (!threads) + return; + + log_err("fio: illegal instruction. your cpu does not support " + "the sse4.2 instruction for crc32c\n"); + terminate_threads(TERMINATE_ALL); + exit(4); +} + +static void set_sig_handlers(void) +{ + struct sigaction act; + + memset(&act, 0, sizeof(act)); + act.sa_handler = sig_alrm; + act.sa_flags = SA_RESTART; + sigaction(SIGALRM, &act, NULL); + + memset(&act, 0, sizeof(act)); + act.sa_handler = sig_int; + act.sa_flags = SA_RESTART; + sigaction(SIGINT, &act, NULL); + + memset(&act, 0, sizeof(act)); + act.sa_handler = sig_ill; + act.sa_flags = SA_RESTART; + sigaction(SIGILL, &act, NULL); + + memset(&act, 0, sizeof(act)); + act.sa_handler = sig_quit; + act.sa_flags = SA_RESTART; + sigaction(SIGQUIT, &act, NULL); +} + +static inline int should_check_rate(struct thread_data *td) +{ + struct thread_options *o = &td->o; + + /* + * If some rate setting was given, we need to check it + */ + if (o->rate || o->ratemin || o->rate_iops || o->rate_iops_min) + return 1; + + return 0; +} + /* * Check if we are above the minimum rate given. */ @@ -128,12 +192,6 @@ static int check_min_rate(struct thread_data *td, struct timeval *now) unsigned long spent; unsigned long rate; - /* - * No minimum rate set, always ok - */ - if (!td->o.ratemin && !td->o.rate_iops_min) - return 0; - /* * allow a 2 second settle period in the beginning */ @@ -218,7 +276,7 @@ static inline int runtime_exceeded(struct thread_data *td, struct timeval *t) */ static void cleanup_pending_aio(struct thread_data *td) { - struct list_head *entry, *n; + struct flist_head *entry, *n; struct io_u *io_u; int r; @@ -233,8 +291,8 @@ static void cleanup_pending_aio(struct thread_data *td) * now cancel remaining active events */ if (td->io_ops->cancel) { - list_for_each_safe(entry, n, &td->io_u_busylist) { - io_u = list_entry(entry, struct io_u, list); + flist_for_each_safe(entry, n, &td->io_u_busylist) { + io_u = flist_entry(entry, struct io_u, list); /* * if the io_u isn't in flight, then that generally @@ -302,6 +360,12 @@ requeue: return 0; } +static inline void update_tv_cache(struct thread_data *td) +{ + if ((++td->tv_cache_nr & td->tv_cache_mask) == td->tv_cache_mask) + fio_gettime(&td->tv_cache, NULL); +} + /* * The main verify engine. Runs over the writes we previously submitted, * reads the blocks back in, and checks the crc/md5 of the data. @@ -333,13 +397,15 @@ static void do_verify(struct thread_data *td) io_u = NULL; while (!td->terminate) { - int ret2; + int ret2, full; io_u = __get_io_u(td); if (!io_u) break; - if (runtime_exceeded(td, &io_u->start_time)) { + update_tv_cache(td); + + if (runtime_exceeded(td, &td->tv_cache)) { put_io_u(td, io_u); td->terminate = 1; break; @@ -370,7 +436,7 @@ static void do_verify(struct thread_data *td) * zero read, fail */ if (!bytes) { - td_verror(td, ENODATA, "full resid"); + td_verror(td, EIO, "full resid"); put_io_u(td, io_u); break; } @@ -413,19 +479,25 @@ sync_done: * if we can queue more, do so. but check if there are * completed io_u's first. */ - min_events = 0; - if (queue_full(td) || ret == FIO_Q_BUSY) { - if (td->cur_depth >= td->o.iodepth_low) - min_events = td->cur_depth - td->o.iodepth_low; - if (!min_events) + full = queue_full(td) || ret == FIO_Q_BUSY; + if (full || !td->o.iodepth_batch_complete) { + min_events = td->o.iodepth_batch_complete; + if (full && !min_events) min_events = 1; - } - /* - * Reap required number of io units, if any, and do the - * verification on them through the callback handler - */ - if (io_u_queued_complete(td, min_events) < 0) + do { + /* + * Reap required number of io units, if any, + * and do the verification on them through + * the callback handler + */ + if (io_u_queued_complete(td, min_events) < 0) { + ret = -1; + break; + } + } while (full && (td->cur_depth > td->o.iodepth_low)); + } + if (ret < 0) break; } @@ -446,19 +518,21 @@ sync_done: */ static void do_io(struct thread_data *td) { - struct timeval s; unsigned long usec; unsigned int i; int ret = 0; - td_set_runstate(td, TD_RUNNING); + if (in_ramp_time(td)) + td_set_runstate(td, TD_RAMP); + else + td_set_runstate(td, TD_RUNNING); while ((td->this_io_bytes[0] + td->this_io_bytes[1]) < td->o.size) { struct timeval comp_time; long bytes_done = 0; int min_evts = 0; struct io_u *io_u; - int ret2; + int ret2, full; if (td->terminate) break; @@ -467,9 +541,9 @@ static void do_io(struct thread_data *td) if (!io_u) break; - memcpy(&s, &io_u->start_time, sizeof(s)); + update_tv_cache(td); - if (runtime_exceeded(td, &s)) { + if (runtime_exceeded(td, &td->tv_cache)) { put_io_u(td, io_u); td->terminate = 1; break; @@ -482,7 +556,9 @@ static void do_io(struct thread_data *td) if (td->o.verify != VERIFY_NONE && io_u->ddir == DDIR_READ) { io_u->end_io = verify_io_u; td_set_runstate(td, TD_VERIFYING); - } else + } else if (in_ramp_time(td)) + td_set_runstate(td, TD_RAMP); + else td_set_runstate(td, TD_RUNNING); ret = td_io_queue(td, io_u); @@ -498,7 +574,7 @@ static void do_io(struct thread_data *td) * zero read, fail */ if (!bytes) { - td_verror(td, ENODATA, "full resid"); + td_verror(td, EIO, "full resid"); put_io_u(td, io_u); break; } @@ -515,7 +591,9 @@ static void do_io(struct thread_data *td) requeue_io_u(td, &io_u); } else { sync_done: - fio_gettime(&comp_time, NULL); + if (should_check_rate(td)) + fio_gettime(&comp_time, NULL); + bytes_done = io_u_sync_complete(td, io_u); if (bytes_done < 0) ret = bytes_done; @@ -548,18 +626,26 @@ sync_done: /* * See if we need to complete some commands */ - if (queue_full(td) || ret == FIO_Q_BUSY) { - min_evts = 0; - if (td->cur_depth >= td->o.iodepth_low) - min_evts = td->cur_depth - td->o.iodepth_low; - if (!min_evts) + full = queue_full(td) || ret == FIO_Q_BUSY; + if (full || !td->o.iodepth_batch_complete) { + min_evts = td->o.iodepth_batch_complete; + if (full && !min_evts) min_evts = 1; - fio_gettime(&comp_time, NULL); - bytes_done = io_u_queued_complete(td, min_evts); - if (bytes_done < 0) - break; + + if (should_check_rate(td)) + fio_gettime(&comp_time, NULL); + + do { + ret = io_u_queued_complete(td, min_evts); + if (ret <= 0) + break; + + bytes_done += ret; + } while (full && (td->cur_depth > td->o.iodepth_low)); } + if (ret < 0) + break; if (!bytes_done) continue; @@ -568,15 +654,17 @@ sync_done: * of completions except the very first one which may look * a little bursty */ - usec = utime_since(&s, &comp_time); + if (!in_ramp_time(td) && should_check_rate(td)) { + usec = utime_since(&td->tv_cache, &comp_time); - rate_throttle(td, usec, bytes_done); + rate_throttle(td, usec, bytes_done); - if (check_min_rate(td, &comp_time)) { - if (exitall_on_terminate) - terminate_threads(td->groupid); - td_verror(td, ENODATA, "check_min_rate"); - break; + if (check_min_rate(td, &comp_time)) { + if (exitall_on_terminate) + terminate_threads(td->groupid); + td_verror(td, EIO, "check_min_rate"); + break; + } } if (td->o.thinktime) { @@ -587,7 +675,7 @@ sync_done: int left; if (td->o.thinktime_spin) - __usec_sleep(td->o.thinktime_spin); + usec_spin(td->o.thinktime_spin); left = td->o.thinktime - td->o.thinktime_spin; if (left) @@ -628,13 +716,13 @@ sync_done: static void cleanup_io_u(struct thread_data *td) { - struct list_head *entry, *n; + struct flist_head *entry, *n; struct io_u *io_u; - list_for_each_safe(entry, n, &td->io_u_freelist) { - io_u = list_entry(entry, struct io_u, list); + flist_for_each_safe(entry, n, &td->io_u_freelist) { + io_u = flist_entry(entry, struct io_u, list); - list_del(&io_u->list); + flist_del(&io_u->list); free(io_u); } @@ -678,7 +766,7 @@ static int init_io_u(struct thread_data *td) return 1; io_u = malloc(sizeof(*io_u)); memset(io_u, 0, sizeof(*io_u)); - INIT_LIST_HEAD(&io_u->list); + INIT_FLIST_HEAD(&io_u->list); if (!(td->io_ops->flags & FIO_NOIO)) { io_u->buf = p + max_bs * i; @@ -689,11 +777,9 @@ static int init_io_u(struct thread_data *td) io_u->index = i; io_u->flags = IO_U_F_FREE; - list_add(&io_u->list, &td->io_u_freelist); + flist_add(&io_u->list, &td->io_u_freelist); } - io_u_init_timeout(); - return 0; } @@ -774,12 +860,8 @@ static int keep_running(struct thread_data *td) return 0; } -static int clear_io_state(struct thread_data *td) +static void reset_io_counters(struct thread_data *td) { - struct fio_file *f; - unsigned int i; - int ret; - td->ts.stat_io_bytes[0] = td->ts.stat_io_bytes[1] = 0; td->this_io_bytes[0] = td->this_io_bytes[1] = 0; td->zone_bytes = 0; @@ -795,17 +877,41 @@ static int clear_io_state(struct thread_data *td) if (td->o.time_based || td->o.loops) td->nr_done_files = 0; - close_files(td); + /* + * Set the same seed to get repeatable runs + */ + td_fill_rand_seeds(td); +} - ret = 0; - for_each_file(td, f, i) { - f->flags &= ~FIO_FILE_DONE; - ret = td_io_open_file(td, f); - if (ret) - break; +void reset_all_stats(struct thread_data *td) +{ + struct timeval tv; + int i; + + reset_io_counters(td); + + for (i = 0; i < 2; i++) { + td->io_bytes[i] = 0; + td->io_blocks[i] = 0; + td->io_issues[i] = 0; + td->ts.total_io_u[i] = 0; } + + fio_gettime(&tv, NULL); + memcpy(&td->epoch, &tv, sizeof(tv)); + memcpy(&td->start, &tv, sizeof(tv)); +} - return ret; +static void clear_io_state(struct thread_data *td) +{ + struct fio_file *f; + unsigned int i; + + reset_io_counters(td); + + close_files(td); + for_each_file(td, f, i) + f->flags &= ~FIO_FILE_DONE; } /* @@ -823,18 +929,21 @@ static void *thread_main(void *data) td->pid = getpid(); - dprint(FD_PROCESS, "jobs pid=%d started\n", td->pid); + dprint(FD_PROCESS, "jobs pid=%d started\n", (int) td->pid); - INIT_LIST_HEAD(&td->io_u_freelist); - INIT_LIST_HEAD(&td->io_u_busylist); - INIT_LIST_HEAD(&td->io_u_requeues); - INIT_LIST_HEAD(&td->io_log_list); - INIT_LIST_HEAD(&td->io_hist_list); + INIT_FLIST_HEAD(&td->io_u_freelist); + INIT_FLIST_HEAD(&td->io_u_busylist); + INIT_FLIST_HEAD(&td->io_u_requeues); + INIT_FLIST_HEAD(&td->io_log_list); + INIT_FLIST_HEAD(&td->io_hist_list); td->io_hist_tree = RB_ROOT; td_set_runstate(td, TD_INITIALIZED); + dprint(FD_MUTEX, "up startup_mutex\n"); fio_mutex_up(startup_mutex); + dprint(FD_MUTEX, "wait on td->mutex\n"); fio_mutex_down(td->mutex); + dprint(FD_MUTEX, "done waiting on td->mutex\n"); /* * the ->mutex mutex is now no longer used, close it to avoid @@ -857,6 +966,18 @@ static void *thread_main(void *data) goto err; } + /* + * If we have a gettimeofday() thread, make sure we exclude that + * thread from this job + */ + if (td->o.gtod_cpu) { + fio_cpu_clear(&td->o.cpumask, td->o.gtod_cpu); + if (fio_setaffinity(td) == -1) { + td_verror(td, errno, "cpu_set_affinity"); + goto err; + } + } + if (td->ioprio_set) { if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) { td_verror(td, errno, "ioprio_set"); @@ -878,9 +999,6 @@ static void *thread_main(void *data) if (td_io_init(td)) goto err; - if (open_files(td)) - goto err; - if (init_random_map(td)) goto err; @@ -890,7 +1008,6 @@ static void *thread_main(void *data) } fio_gettime(&td->epoch, NULL); - memcpy(&td->timeout_end, &td->epoch, sizeof(td->epoch)); getrusage(RUSAGE_SELF, &td->ts.ru_start); runtime[0] = runtime[1] = 0; @@ -898,13 +1015,14 @@ static void *thread_main(void *data) while (keep_running(td)) { fio_gettime(&td->start, NULL); memcpy(&td->ts.stat_sample_time, &td->start, sizeof(td->start)); + memcpy(&td->tv_cache, &td->start, sizeof(td->start)); if (td->o.ratemin) memcpy(&td->lastrate, &td->ts.stat_sample_time, sizeof(td->lastrate)); - if (clear_state && clear_io_state(td)) - break; + if (clear_state) + clear_io_state(td); prune_io_piece_log(td); @@ -939,8 +1057,7 @@ static void *thread_main(void *data) (td->io_ops->flags & FIO_UNIDIR)) continue; - if (clear_io_state(td)) - break; + clear_io_state(td); fio_gettime(&td->start, NULL); @@ -959,12 +1076,27 @@ static void *thread_main(void *data) td->ts.io_bytes[0] = td->io_bytes[0]; td->ts.io_bytes[1] = td->io_bytes[1]; - if (td->ts.bw_log) - finish_log(td, td->ts.bw_log, "bw"); - if (td->ts.slat_log) - finish_log(td, td->ts.slat_log, "slat"); - if (td->ts.clat_log) - finish_log(td, td->ts.clat_log, "clat"); + if (td->ts.bw_log) { + if (td->o.bw_log_file) { + finish_log_named(td, td->ts.bw_log, + td->o.bw_log_file, "bw"); + } else + finish_log(td, td->ts.bw_log, "bw"); + } + if (td->ts.slat_log) { + if (td->o.lat_log_file) { + finish_log_named(td, td->ts.slat_log, + td->o.lat_log_file, "clat"); + } else + finish_log(td, td->ts.slat_log, "slat"); + } + if (td->ts.clat_log) { + if (td->o.lat_log_file) { + finish_log_named(td, td->ts.clat_log, + td->o.lat_log_file, "clat"); + } else + finish_log(td, td->ts.clat_log, "clat"); + } if (td->o.exec_postrun) { if (system(td->o.exec_postrun) < 0) log_err("fio: postrun %s failed\n", td->o.exec_postrun); @@ -975,12 +1107,18 @@ static void *thread_main(void *data) err: if (td->error) - printf("fio: pid=%d, err=%d/%s\n", td->pid, td->error, + printf("fio: pid=%d, err=%d/%s\n", (int) td->pid, td->error, td->verror); close_and_free_files(td); close_ioengine(td); cleanup_io_u(td); + if (td->o.cpumask_set) { + int ret = fio_cpuset_exit(&td->o.cpumask); + + td_verror(td, ret, "fio_cpuset_exit"); + } + /* * do this very late, it will log file closing as well */ @@ -1063,8 +1201,8 @@ static void reap_threads(int *nr_running, int *t_rate, int *m_rate) ret = waitpid(td->pid, &status, flags); if (ret < 0) { if (errno == ECHILD) { - log_err("fio: pid=%d disappeared %d\n", td->pid, - td->runstate); + log_err("fio: pid=%d disappeared %d\n", + (int) td->pid, td->runstate); td_set_runstate(td, TD_REAPED); goto reaped; } @@ -1075,7 +1213,7 @@ static void reap_threads(int *nr_running, int *t_rate, int *m_rate) if (sig != SIGQUIT) log_err("fio: pid=%d, got signal=%d\n", - td->pid, sig); + (int) td->pid, sig); td_set_runstate(td, TD_REAPED); goto reaped; } @@ -1110,6 +1248,46 @@ reaped: terminate_threads(TERMINATE_ALL); } +static void *gtod_thread_main(void *data) +{ + fio_mutex_up(startup_mutex); + + /* + * As long as we have jobs around, update the clock. It would be nice + * to have some way of NOT hammering that CPU with gettimeofday(), + * but I'm not sure what to use outside of a simple CPU nop to relax + * it - we don't want to lose precision. + */ + while (threads) { + fio_gtod_update(); + nop; + } + + return NULL; +} + +static int fio_start_gtod_thread(void) +{ + int ret; + + ret = pthread_create(>od_thread, NULL, gtod_thread_main, NULL); + if (ret) { + log_err("Can't create gtod thread: %s\n", strerror(ret)); + return 1; + } + + ret = pthread_detach(gtod_thread); + if (ret) { + log_err("Can't detatch gtod thread: %s\n", strerror(ret)); + return 1; + } + + dprint(FD_MUTEX, "wait on startup_mutex\n"); + fio_mutex_down(startup_mutex); + dprint(FD_MUTEX, "done waiting on startup_mutex\n"); + return 0; +} + /* * Main function for kicking off and reaping jobs, as needed. */ @@ -1122,6 +1300,9 @@ static void run_threads(void) if (fio_pin_memory()) return; + if (fio_gtod_offload && fio_start_gtod_thread()) + return; + if (!terse_output) { printf("Starting "); if (nr_thread) @@ -1137,8 +1318,7 @@ static void run_threads(void) fflush(stdout); } - signal(SIGINT, sig_handler); - signal(SIGALRM, sig_handler); + set_sig_handlers(); todo = thread_number; nr_running = 0; @@ -1161,8 +1341,8 @@ static void run_threads(void) if (setup_files(td)) { exit_value++; if (td->error) - log_err("fio: pid=%d, err=%d/%s\n", td->pid, - td->error, td->verror); + log_err("fio: pid=%d, err=%d/%s\n", + (int) td->pid, td->error, td->verror); td_set_runstate(td, TD_REAPED); todo--; } else { @@ -1226,15 +1406,21 @@ static void run_threads(void) nr_started++; if (td->o.use_thread) { + int ret; + dprint(FD_PROCESS, "will pthread_create\n"); - if (pthread_create(&td->thread, NULL, - thread_main, td)) { - perror("pthread_create"); + ret = pthread_create(&td->thread, NULL, + thread_main, td); + if (ret) { + log_err("pthread_create: %s\n", + strerror(ret)); nr_started--; break; } - if (pthread_detach(td->thread) < 0) - perror("pthread_detach"); + ret = pthread_detach(td->thread); + if (ret) + log_err("pthread_detach: %s", + strerror(ret)); } else { pid_t pid; dprint(FD_PROCESS, "will fork\n"); @@ -1246,7 +1432,9 @@ static void run_threads(void) } else if (i == fio_debug_jobno) *fio_debug_jobp = pid; } + dprint(FD_MUTEX, "wait on startup_mutex\n"); fio_mutex_down(startup_mutex); + dprint(FD_MUTEX, "done waiting on startup_mutex\n"); } /* @@ -1295,7 +1483,10 @@ static void run_threads(void) if (td->runstate != TD_INITIALIZED) continue; - td_set_runstate(td, TD_RUNNING); + if (in_ramp_time(td)) + td_set_runstate(td, TD_RAMP); + else + td_set_runstate(td, TD_RUNNING); nr_running++; nr_started--; m_rate += td->o.ratemin; @@ -1356,7 +1547,7 @@ int main(int argc, char *argv[]) set_genesis_time(); - disk_util_timer_arm(); + status_timer_arm(); run_threads();