Improve thread reap handling

[fio.git] / fio.c

diff --git a/fio.c b/fio.c
index 9594f80582266ce8a8a5794ea9c52fb02a6adb1d..0f72fd92c15c883e75603a19b2a4fc89e915ba6f 100644 (file)
--- a/fio.c
+++ b/fio.c
@@ -160,9 +160,19 @@ static void cleanup_pending_aio(struct thread_data *td)
                list_for_each_safe(entry, n, &td->io_u_busylist) {
                        io_u = list_entry(entry, struct io_u, list);
 
-                       r = td->io_ops->cancel(td, io_u);
-                       if (!r)
+                       /*
+                        * if the io_u isn't in flight, then that generally
+                        * means someone leaked an io_u. complain but fix
+                        * it up, so we don't stall here.
+                        */
+                       if ((io_u->flags & IO_U_F_FLIGHT) == 0) {
+                               log_err("fio: non-busy IO on busy list\n");
                                put_io_u(td, io_u);
+                       } else {
+                               r = td->io_ops->cancel(td, io_u);
+                               if (!r)
+                                       put_io_u(td, io_u);
+                       }
                }
        }
 
@@ -553,9 +563,12 @@ static int init_io_u(struct thread_data *td)
                        fill_rand_buf(io_u, max_bs);
 
                io_u->index = i;
+               io_u->flags = IO_U_F_FREE;
                list_add(&io_u->list, &td->io_u_freelist);
        }
 
+       io_u_init_timeout();
+
        return 0;
 }
 
@@ -704,6 +717,7 @@ static void *thread_main(void *data)
        }
 
        fio_gettime(&td->epoch, NULL);
+       memcpy(&td->timeout_end, &td->epoch, sizeof(td->epoch));
        getrusage(RUSAGE_SELF, &td->ts.ru_start);
 
        runtime[0] = runtime[1] = 0;
@@ -810,6 +824,8 @@ static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
         */
        pending = cputhreads = 0;
        for_each_td(td, i) {
+               int flags;
+
                /*
                 * ->io_ops is NULL for a thread that has closed its
                 * io engine
@@ -817,55 +833,57 @@ static void reap_threads(int *nr_running, int *t_rate, int *m_rate)
                if (td->io_ops && td->io_ops->flags & FIO_CPUIO)
                        cputhreads++;
 
-               if (td->runstate < TD_EXITED) {
-                       /*
-                        * check if someone quit or got killed in an unusual way
-                        */
-                       ret = waitpid(td->pid, &status, WNOHANG);
-                       if (ret < 0)
-                               perror("waitpid");
-                       else if ((ret == td->pid) && WIFSIGNALED(status)) {
+               if (!td->pid || td->runstate == TD_REAPED)
+                       continue;
+
+               flags = WNOHANG;
+               if (td->runstate == TD_EXITED)
+                       flags = 0;
+
+               /*
+                * check if someone quit or got killed in an unusual way
+                */
+               ret = waitpid(td->pid, &status, flags);
+               if (ret < 0) {
+                       if (errno == ECHILD) {
+                               log_err("fio: pid=%d disappeared %d\n", td->pid, td->runstate);
+                               td_set_runstate(td, TD_REAPED);
+                               goto reaped;
+                       }
+                       perror("waitpid");
+               } else if (ret == td->pid) {
+                       if (WIFSIGNALED(status)) {
                                int sig = WTERMSIG(status);
 
                                log_err("fio: pid=%d, got signal=%d\n", td->pid, sig);
                                td_set_runstate(td, TD_REAPED);
                                goto reaped;
                        }
-               }
-
-               if (td->runstate != TD_EXITED) {
-                       if (td->runstate < TD_RUNNING)
-                               pending++;
-
-                       continue;
-               }
-
-               if (td->error)
-                       exit_value++;
-
-               td_set_runstate(td, TD_REAPED);
-
-               if (td->use_thread) {
-                       long ret;
-
-                       if (pthread_join(td->thread, (void *) &ret))
-                               perror("thread_join");
-               } else {
-                       int status;
-
-                       ret = waitpid(td->pid, &status, 0);
-                       if (ret < 0)
-                               perror("waitpid");
-                       else if (WIFEXITED(status) && WEXITSTATUS(status)) {
-                               if (!exit_value)
-                                       exit_value++;
+                       if (WIFEXITED(status)) {
+                               if (WEXITSTATUS(status) && !td->error)
+                                       td->error = WEXITSTATUS(status);
+                               if (td->use_thread) {
+                                       long ret;
+
+                                       if (pthread_join(td->thread, (void *) &ret))
+                                               perror("pthread_join");
+                               }
+                               td_set_runstate(td, TD_REAPED);
+                               goto reaped;
                        }
                }
 
+               /*
+                * thread is not dead, continue
+                */
+               continue;
 reaped:
                (*nr_running)--;
                (*m_rate) -= td->ratemin;
                (*t_rate) -= td->rate;
+
+               if (td->error)
+                       exit_value++;
        }
 
        if (*nr_running == cputhreads && !pending)
@@ -921,6 +939,8 @@ static void run_threads(void)
                init_disk_util(td);
        }
 
+       set_genesis_time();
+
        while (todo) {
                struct thread_data *map[MAX_JOBS];
                struct timeval this_start;
@@ -1079,6 +1099,8 @@ int main(int argc, char *argv[])
                setup_log(&agg_io_log[DDIR_WRITE]);
        }
 
+       set_genesis_time();
+
        disk_util_timer_arm();
 
        run_threads();