X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=backend.c;h=d19d870f8143d985882841af68a0a94c056f7e35;hp=ee75566af84cd6f07a8a4e824ba79e9788667d1b;hb=4a995ddae8b12d20efc2367ad4e1c9b159f96eed;hpb=c89318761586dbd77b8f31ce0ed68ff4fc086c89 diff --git a/backend.c b/backend.c index ee75566a..d19d870f 100644 --- a/backend.c +++ b/backend.c @@ -53,9 +53,13 @@ #include "lib/getrusage.h" #include "idletime.h" #include "err.h" +#include "lib/tp.h" + +static pthread_t helper_thread; +static pthread_mutex_t helper_lock; +pthread_cond_t helper_cond; +int helper_do_stat = 0; -static pthread_t disk_util_thread; -static struct fio_mutex *disk_thread_mutex; static struct fio_mutex *startup_mutex; static struct flist_head *cgroup_list; static char *cgroup_mnt; @@ -72,7 +76,7 @@ unsigned int stat_number = 0; int shm_id = 0; int temp_stall_ts; unsigned long done_secs = 0; -volatile int disk_util_exit = 0; +volatile int helper_exit = 0; #define PAGE_ALIGN(buf) \ (char *) (((uintptr_t) (buf) + page_mask) & ~page_mask) @@ -86,7 +90,7 @@ static void sig_int(int sig) fio_server_got_signal(sig); else { log_info("\nfio: terminating on signal %d\n", sig); - fflush(stdout); + log_info_flush(); exit_value = 128; } @@ -390,7 +394,7 @@ static int break_on_this_error(struct thread_data *td, enum fio_ddir ddir, * fill_device option is set. */ td_clear_error(td); - td->terminate = 1; + fio_mark_td_terminate(td); return 1; } else { /* @@ -459,7 +463,7 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes) if (runtime_exceeded(td, &td->tv_cache)) { __update_tv_cache(td); if (runtime_exceeded(td, &td->tv_cache)) { - td->terminate = 1; + fio_mark_td_terminate(td); break; } } @@ -713,7 +717,7 @@ static uint64_t do_io(struct thread_data *td) if (runtime_exceeded(td, &td->tv_cache)) { __update_tv_cache(td); if (runtime_exceeded(td, &td->tv_cache)) { - td->terminate = 1; + fio_mark_td_terminate(td); break; } } @@ -921,7 +925,7 @@ reap: if (td->o.fill_device && td->error == ENOSPC) { td->error = 0; - td->terminate = 1; + fio_mark_td_terminate(td); } if (!td->error) { struct fio_file *f; @@ -1443,6 +1447,9 @@ static void *thread_main(void *data) goto err; } + if (td->flags & TD_F_COMPRESS_LOG) + tp_init(&td->tp_data); + fio_verify_init(td); fio_gettime(&td->epoch, NULL); @@ -1524,6 +1531,9 @@ static void *thread_main(void *data) fio_writeout_logs(td); + if (td->flags & TD_F_COMPRESS_LOG) + tp_exit(&td->tp_data); + if (o->exec_postrun) exec_string(o, o->exec_postrun, (const char *)"postrun"); @@ -1555,13 +1565,17 @@ err: if (o->write_iolog_file) write_iolog_close(td); - fio_mutex_remove(td->rusage_sem); - td->rusage_sem = NULL; - fio_mutex_remove(td->mutex); td->mutex = NULL; td_set_runstate(td, TD_EXITED); + + /* + * Do this last after setting our runstate to exited, so we + * know that the stat thread is signaled. + */ + check_update_rusage(td); + return (void *) (uintptr_t) td->error; } @@ -1596,6 +1610,13 @@ static int fork_main(int shmid, int offset) return (int) (uintptr_t) ret; } +static void dump_td_info(struct thread_data *td) +{ + log_err("fio: job '%s' hasn't exited in %lu seconds, it appears to " + "be stuck. Doing forceful exit of this job.\n", td->o.name, + (unsigned long) time_since_now(&td->terminate_time)); +} + /* * Run over the job map and reap the threads that have exited, if any. */ @@ -1673,6 +1694,17 @@ static void reap_threads(unsigned int *nr_running, unsigned int *t_rate, } } + /* + * If the job is stuck, do a forceful timeout of it and + * move on. + */ + if (td->terminate && + time_since_now(&td->terminate_time) >= FIO_REAP_TIMEOUT) { + dump_td_info(td); + td_set_runstate(td, TD_REAPED); + goto reaped; + } + /* * thread is not dead, continue */ @@ -1738,7 +1770,7 @@ static void run_threads(void) nr_process > 1 ? "es" : ""); } log_info("\n"); - fflush(stdout); + log_info_flush(); } todo = thread_number; @@ -1945,57 +1977,70 @@ static void run_threads(void) update_io_ticks(); } -void wait_for_disk_thread_exit(void) +static void wait_for_helper_thread_exit(void) { - fio_mutex_down(disk_thread_mutex); + void *ret; + + helper_exit = 1; + pthread_cond_signal(&helper_cond); + pthread_join(helper_thread, &ret); } static void free_disk_util(void) { - disk_util_start_exit(); - wait_for_disk_thread_exit(); disk_util_prune_entries(); + + pthread_cond_destroy(&helper_cond); } -static void *disk_thread_main(void *data) +static void *helper_thread_main(void *data) { int ret = 0; fio_mutex_up(startup_mutex); - while (threads && !ret) { - usleep(DISK_UTIL_MSEC * 1000); - if (!threads) - break; + while (!ret) { + uint64_t sec = DISK_UTIL_MSEC / 1000; + uint64_t nsec = (DISK_UTIL_MSEC % 1000) * 1000000; + struct timespec ts; + struct timeval tv; + + gettimeofday(&tv, NULL); + ts.tv_sec = tv.tv_sec + sec; + ts.tv_nsec = (tv.tv_usec * 1000) + nsec; + if (ts.tv_nsec > 1000000000ULL) { + ts.tv_nsec -= 1000000000ULL; + ts.tv_sec++; + } + + pthread_cond_timedwait(&helper_cond, &helper_lock, &ts); + ret = update_io_ticks(); + if (helper_do_stat) { + helper_do_stat = 0; + __show_running_run_stats(); + } + if (!is_backend) print_thread_status(); } - fio_mutex_up(disk_thread_mutex); return NULL; } -static int create_disk_util_thread(void) +static int create_helper_thread(void) { int ret; setup_disk_util(); - disk_thread_mutex = fio_mutex_init(FIO_MUTEX_LOCKED); - - ret = pthread_create(&disk_util_thread, NULL, disk_thread_main, NULL); - if (ret) { - fio_mutex_remove(disk_thread_mutex); - log_err("Can't create disk util thread: %s\n", strerror(ret)); - return 1; - } + pthread_cond_init(&helper_cond, NULL); + pthread_mutex_init(&helper_lock, NULL); - ret = pthread_detach(disk_util_thread); + ret = pthread_create(&helper_thread, NULL, helper_thread_main, NULL); if (ret) { - fio_mutex_remove(disk_thread_mutex); - log_err("Can't detatch disk util thread: %s\n", strerror(ret)); + log_err("Can't create helper thread: %s\n", strerror(ret)); return 1; } @@ -2020,9 +2065,13 @@ int fio_backend(void) return 0; if (write_bw_log) { - setup_log(&agg_io_log[DDIR_READ], 0, IO_LOG_TYPE_BW); - setup_log(&agg_io_log[DDIR_WRITE], 0, IO_LOG_TYPE_BW); - setup_log(&agg_io_log[DDIR_TRIM], 0, IO_LOG_TYPE_BW); + struct log_params p = { + .log_type = IO_LOG_TYPE_BW, + }; + + setup_log(&agg_io_log[DDIR_READ], &p, "agg-read_bw.log"); + setup_log(&agg_io_log[DDIR_WRITE], &p, "agg-write_bw.log"); + setup_log(&agg_io_log[DDIR_TRIM], &p, "agg-trim_bw.log"); } startup_mutex = fio_mutex_init(FIO_MUTEX_LOCKED); @@ -2031,26 +2080,34 @@ int fio_backend(void) set_genesis_time(); stat_init(); - create_disk_util_thread(); + create_helper_thread(); cgroup_list = smalloc(sizeof(*cgroup_list)); INIT_FLIST_HEAD(cgroup_list); run_threads(); + wait_for_helper_thread_exit(); + if (!fio_abort) { - show_run_stats(); + __show_run_stats(); if (write_bw_log) { - __finish_log(agg_io_log[DDIR_READ], "agg-read_bw.log"); - __finish_log(agg_io_log[DDIR_WRITE], - "agg-write_bw.log"); - __finish_log(agg_io_log[DDIR_TRIM], - "agg-write_bw.log"); + int i; + + for (i = 0; i < DDIR_RWDIR_CNT; i++) { + struct io_log *log = agg_io_log[i]; + + flush_log(log); + free_log(log); + } } } - for_each_td(td, i) + for_each_td(td, i) { fio_options_free(td); + fio_mutex_remove(td->rusage_sem); + td->rusage_sem = NULL; + } free_disk_util(); cgroup_kill(cgroup_list); @@ -2058,7 +2115,6 @@ int fio_backend(void) sfree(cgroup_mnt); fio_mutex_remove(startup_mutex); - fio_mutex_remove(disk_thread_mutex); stat_exit(); return exit_value; }