From e916b390684ec1ca6247f98138fa9c1682701d29 Mon Sep 17 00:00:00 2001
From: Jens Axboe <jens.axboe@oracle.com>
Date: Tue, 20 Feb 2007 14:37:26 +0100
Subject: [PATCH] Add low water mark for queuing depth

Current fio will attempt to keep the queue full at all times,
but sometimes that's not what you want. Add iodepth_low to
indicate a low water mark for queuing depth, so that when we
see a FIO_Q_BUSY or run out of free requests, let the queue
drain down to the iodepth_low setting before building it up
again.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---
 HOWTO  |  7 +++++++
 fio.c  | 12 ++++++++++--
 fio.h  |  1 +
 init.c | 14 ++++++++++++++
 4 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/HOWTO b/HOWTO
index ac15fc80..ea98810a 100644
--- a/HOWTO
+++ b/HOWTO
@@ -294,6 +294,13 @@ iodepth=int	This defines how many io units to keep in flight against
 		job, can be overridden with a larger value for higher
 		concurrency.
 
+iodepth_low=int	The low water mark indicating when to start filling
+		the queue again. Defaults to the same as iodepth, meaning
+		that fio will attempt to keep the queue full at all times.
+		If iodepth is set to eg 16 and iodepth_low is set to 4, then
+		after fio has filled the queue of 16 requests, it will let
+		the depth drain down to 4 before starting to fill it again.
+
 direct=bool	If value is true, use non-buffered io. This is usually
 		O_DIRECT.
 
diff --git a/fio.c b/fio.c
index e4fec4f6..1c2748e0 100644
--- a/fio.c
+++ b/fio.c
@@ -287,9 +287,13 @@ requeue:
 		 * completed io_u's first.
 		 */
 		min_events = 0;
-		if (queue_full(td) || ret == FIO_Q_BUSY)
+		if (queue_full(td) || ret == FIO_Q_BUSY) {
 			min_events = 1;
 
+			if (td->cur_depth > td->iodepth_low)
+				min_events = td->cur_depth - td->iodepth_low;
+		}
+
 		/*
 		 * Reap required number of io units, if any, and do the
 		 * verification on them through the callback handler
@@ -411,9 +415,13 @@ requeue:
 		 */
 		if (ret == FIO_Q_QUEUED || ret == FIO_Q_BUSY) {
 			min_evts = 0;
-			if (queue_full(td) || ret == FIO_Q_BUSY)
+			if (queue_full(td) || ret == FIO_Q_BUSY) {
 				min_evts = 1;
 
+				if (td->cur_depth > td->iodepth_low)
+					min_evts = td->cur_depth - td->iodepth_low;
+			}
+
 			fio_gettime(&comp_time, NULL);
 			bytes_done = io_u_queued_complete(td, min_evts, NULL);
 			if (bytes_done < 0)
diff --git a/fio.h b/fio.h
index 38de5e3d..282ccf0d 100644
--- a/fio.h
+++ b/fio.h
@@ -325,6 +325,7 @@ struct thread_data {
 	unsigned int stonewall;
 	unsigned int numjobs;
 	unsigned int iodepth;
+	unsigned int iodepth_low;
 	os_cpu_mask_t cpumask;
 	unsigned int iolog;
 	unsigned int read_iolog;
diff --git a/init.c b/init.c
index 6dc221ea..5630c9fb 100644
--- a/init.c
+++ b/init.c
@@ -90,6 +90,12 @@ static struct fio_option options[] = {
 		.help	= "Amount of IO buffers to keep in flight",
 		.def	= "1",
 	},
+	{
+		.name	= "iodepth_low",
+		.type	= FIO_OPT_INT,
+		.off1	= td_var_offset(iodepth_low),
+		.help	= "Low water mark for queuing depth",
+	},
 	{
 		.name	= "size",
 		.type	= FIO_OPT_STR_VAL,
@@ -644,6 +650,14 @@ static void fixup_options(struct thread_data *td)
 	 */
 	if (td->thinktime_spin > td->thinktime)
 		td->thinktime_spin = td->thinktime;
+
+	/*
+	 * The low water mark cannot be bigger than the iodepth
+	 */
+	if (td->iodepth_low > td->iodepth || !td->iodepth_low)
+		td->iodepth_low = td->iodepth;
+
+	printf("io depth %d/%d\n", td->iodepth_low, td->iodepth);
 }
 
 /*
-- 
2.25.1