From: Jens Axboe <axboe@fb.com>
Date: Tue, 15 Dec 2015 01:51:22 +0000 (-0700)
Subject: client: make SEND_ETA timeout non-fatal
X-Git-Tag: fio-2.3~33^2~1
X-Git-Url: https://git.kernel.dk/?p=fio.git;a=commitdiff_plain;h=a64c13a449ddd63bf7b501d85571360f3afd8980

client: make SEND_ETA timeout non-fatal

If we fail a single ETA, then just soldier on. If we fail 5 in a row,
then give up permanently.

Also bring the command timeout back to 5s per command.

Signed-off-by: Jens Axboe <axboe@fb.com>
---

diff --git a/client.c b/client.c
index 2cba8a03..35cbf68c 100644
--- a/client.c
+++ b/client.c
@@ -1159,6 +1159,7 @@ static void handle_eta(struct fio_client *client, struct fio_net_cmd *cmd)
 
 	client->eta_in_flight = NULL;
 	flist_del_init(&client->eta_list);
+	client->eta_timeouts = 0;
 
 	if (client->ops->jobs_eta)
 		client->ops->jobs_eta(client, je);
@@ -1588,6 +1589,34 @@ static void request_client_etas(struct client_ops *ops)
 	dprint(FD_NET, "client: requested eta tag %p\n", eta);
 }
 
+/*
+ * A single SEND_ETA timeout isn't fatal. Attempt to recover.
+ */
+static int handle_cmd_timeout(struct fio_client *client,
+			      struct fio_net_cmd_reply *reply)
+{
+	if (reply->opcode != FIO_NET_CMD_SEND_ETA)
+		return 1;
+
+	log_info("client <%s>: timeout on SEND_ETA\n", client->hostname);
+	flist_del(&reply->list);
+	free(reply);
+
+	flist_del_init(&client->eta_list);
+	if (client->eta_in_flight) {
+		fio_client_dec_jobs_eta(client->eta_in_flight, client->ops->eta);
+		client->eta_in_flight = NULL;
+	}
+
+	/*
+	 * If we fail 5 in a row, give up...
+	 */
+	if (client->eta_timeouts++ > 5)
+		return 1;
+
+	return 0;
+}
+
 static int client_check_cmd_timeout(struct fio_client *client,
 				    struct timeval *now)
 {
@@ -1601,6 +1630,9 @@ static int client_check_cmd_timeout(struct fio_client *client,
 		if (mtime_since(&reply->tv, now) < FIO_NET_CLIENT_TIMEOUT)
 			continue;
 
+		if (!handle_cmd_timeout(client, reply))
+			continue;
+
 		log_err("fio: client %s, timeout on cmd %s\n", client->hostname,
 						fio_server_op(reply->opcode));
 		flist_del(&reply->list);
diff --git a/client.h b/client.h
index cfb0b4d4..46e30edc 100644
--- a/client.h
+++ b/client.h
@@ -60,6 +60,7 @@ struct fio_client {
 
 	struct flist_head eta_list;
 	struct client_eta *eta_in_flight;
+	unsigned int eta_timeouts;
 
 	struct flist_head cmd_list;
 
diff --git a/server.h b/server.h
index 6747bf02..d73ce1d6 100644
--- a/server.h
+++ b/server.h
@@ -74,7 +74,7 @@ enum {
 
 	FIO_NET_NAME_MAX		= 256,
 
-	FIO_NET_CLIENT_TIMEOUT		= 30000,
+	FIO_NET_CLIENT_TIMEOUT		= 5000,
 
 	FIO_PROBE_FLAG_ZLIB		= 1UL << 0,
 };