From: Jens Axboe <axboe@fb.com>
Date: Wed, 9 Jul 2014 06:46:12 +0000 (+0200)
Subject: Allow threads 60 seconds to exit before being forceful
X-Git-Tag: fio-2.1.11~7
X-Git-Url: https://git.kernel.dk/?a=commitdiff_plain;h=cba5460c70058e4612f73ded4ac960cbc6ce1a30;p=fio.git

Allow threads 60 seconds to exit before being forceful

Right now the fio status thread can sit forever waiting for jobs
to exit, when ctrl-c (or another signal) is sent. Be a bit more
brutal and force quit jobs if they haven't exited on their own
in 60 seconds. That should be long enough to ensure that they
are stuck in some way.

Signed-off-by: Jens Axboe <axboe@fb.com>
---

diff --git a/backend.c b/backend.c
index 68540abb..d9ece4c7 100644
--- a/backend.c
+++ b/backend.c
@@ -1603,6 +1603,13 @@ static int fork_main(int shmid, int offset)
 	return (int) (uintptr_t) ret;
 }
 
+static void dump_td_info(struct thread_data *td)
+{
+	log_err("fio: job '%s' hasn't exited in %lu seconds, it appears to "
+		"be stuck. Doing forceful exit of this job.\n", td->o.name,
+			(unsigned long) time_since_now(&td->terminate_time));
+}
+
 /*
  * Run over the job map and reap the threads that have exited, if any.
  */
@@ -1680,6 +1687,17 @@ static void reap_threads(unsigned int *nr_running, unsigned int *t_rate,
 			}
 		}
 
+		/*
+		 * If the job is stuck, do a forceful timeout of it and
+		 * move on.
+		 */
+		if (td->terminate &&
+		    time_since_now(&td->terminate_time) >= FIO_REAP_TIMEOUT) {
+			dump_td_info(td);
+			td_set_runstate(td, TD_REAPED);
+			goto reaped;
+		}
+
 		/*
 		 * thread is not dead, continue
 		 */
diff --git a/fio.h b/fio.h
index df0d0200..199610c7 100644
--- a/fio.h
+++ b/fio.h
@@ -254,6 +254,7 @@ struct thread_data {
 	struct timeval epoch;	/* time job was started */
 	struct timeval last_issue;
 	struct timeval tv_cache;
+	struct timeval terminate_time;
 	unsigned int tv_cache_nr;
 	unsigned int tv_cache_mask;
 	unsigned int ramp_time_over;
@@ -486,6 +487,12 @@ extern void td_set_runstate(struct thread_data *, int);
 extern int td_bump_runstate(struct thread_data *, int);
 extern void td_restore_runstate(struct thread_data *, int);
 
+/*
+ * Allow 60 seconds for a job to quit on its own, otherwise reap with
+ * a vengeance.
+ */
+#define FIO_REAP_TIMEOUT	60
+
 #define TERMINATE_ALL		(-1)
 extern void fio_terminate_threads(int);
 
diff --git a/libfio.c b/libfio.c
index 8af11297..9245688a 100644
--- a/libfio.c
+++ b/libfio.c
@@ -199,8 +199,13 @@ void fio_terminate_threads(int group_id)
 		if (group_id == TERMINATE_ALL || groupid == td->groupid) {
 			dprint(FD_PROCESS, "setting terminate on %s/%d\n",
 						td->o.name, (int) td->pid);
+
+			if (td->terminate)
+				continue;
+
 			td->terminate = 1;
 			td->o.start_delay = 0;
+			fio_gettime(&td->terminate_time, NULL);
 
 			/*
 			 * if the thread is running, just let it exit