[fio.git] / tools / genfio

#!/bin/bash
#
#  Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
#  Author: Erwan Velu  <erwan@enovance.com>
#
#  The license below covers all files distributed with fio unless otherwise
#  noted in the file itself.
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License version 2 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

BLK_SIZE=
BLOCK_SIZE=4k
SEQ=-1
TEMPLATE=/tmp/template.fio
OUTFILE=
DISKS=
PRINTABLE_DISKS=
RUNTIME=300
ETA=0
MODES="write,randwrite,read,randread"
SHORT_HOSTNAME=
CACHED_IO="FALSE"
PREFIX=""
PREFIX_FILENAME=""
IODEPTH=1

show_help() {
	PROG=$(basename $0)
	echo "usage of $PROG:"
	cat << EOF
-h				: Show this help & exit
-c				: Enable cached-based IOs
					Disabled by default
-a				: Run sequential test then parallel one
					Disabled by default
-s				: Run sequential test (default value)
					one test after another then one disk after another
					Disabled by default
-p				: Run parallel test
					one test after anoter but all disks at the same time
					Enabled by default
-D iodepth			: Run with the specified iodepth
					Default is $IODEPTH
-d disk1[,disk2,disk3,..]	: Run the tests on the selected disks
					Separated each disk with a comma
-r seconds			: Time in seconds per benchmark
					0 means till the end of the device
					Default is $RUNTIME seconds
-b blocksize[,blocksize1, ...]  : The blocksizes to test under fio format (4k, 1m, ...)
					Separated each blocksize with a comma
					Default is $BLOCK_SIZE
-m mode1,[mode2,mode3, ...]     : Define the fio IO profile to use like read, write, randread, randwrite
					Default is "$MODES"
-x prefix			: Add a prefix to the fio filename
					Useful to let a context associated with the file
					If the prefix features a / (slash), prefix will be considered as a directory
-A cmd_to_run			: System command to run after each job (exec_postrun in fio)
-B cmd_to_run			: System command to run before each job (exec_prerun in fio)

Example:

$PROG -d /dev/sdb,/dev/sdc,/dev/sdd,/dev/sde -a -b 4k,128k,1m -r 100 -a -x dellr720-day2/

	Will generate an fio file that will run
		- a sequential bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 4k with write,randwrite,read,randread tests
			ETA ~ 4 tests * 4 disks * 100 seconds
		- a sequential bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 128k with write,randwrite,read,randread tests
			ETA ~ 4 tests * 4 disks * 100 seconds
		- a sequential bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 1m with write,randwrite,read,randread tests
			ETA ~ 4 tests * 4 disks * 100 seconds
		- a parallel bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 4k with write,randwrite,read,randread tests
			ETA ~ 4 tests * 100 seconds
		- a parallel bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 128k with write,randwrite,read,randread tests
			ETA ~ 4 tests * 100 seconds
		- a parallel bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 1m with write,randwrite,read,randread tests
			ETA ~ 4 tests * 100 seconds

Generating dellr720-day2/localhost-4k,128k,1m-all-write,randwrite,read,randread-sdb,sdc,sdd,sde.fio
Estimated Time = 6000 seconds : 1 hour 40 minutes
EOF
}

finish_template() {
echo "iodepth=$IODEPTH" >> $TEMPLATE

if [ "$RUNTIME" != "0" ]; then
	echo "runtime=$RUNTIME" >> $TEMPLATE
	echo "time_based" >> $TEMPLATE
fi

if [ "$CACHED_IO" = "FALSE" ]; then
	echo "direct=1" >> $TEMPLATE
fi
}


diskname_to_printable() {
COUNT=0
for disk in $(echo $@ | tr "," " "); do
	R=$(basename $disk | sed 's|/|_|g')
	COUNT=$(($COUNT + 1))
	if [ $COUNT -eq 1 ]; then
		P="$R"
	else
		P="$P,$R"
	fi
done
echo $P
}

gen_template() {
cat >$TEMPLATE << EOF
[global]
ioengine=libaio
invalidate=1
ramp_time=5
EOF
}

gen_seq_suite() {
TYPE=$1
disk=$2
PRINTABLE_DISK=$(diskname_to_printable $disk)
cat >> $OUTFILE << EOF
[$TYPE-$PRINTABLE_DISK-$BLK_SIZE-seq]
stonewall
bs=$BLK_SIZE
filename=$disk
rw=$TYPE
write_bw_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-seq.results
write_iops_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-seq.results
EOF
ETA=$(($ETA + $RUNTIME))
}

gen_seq_fio() {
for disk in $(echo $DISKS | tr "," " "); do
	for mode in $(echo $MODES | tr "," " "); do
		gen_seq_suite "$mode" "$disk"
	done
done
}


gen_para_suite() {
TYPE=$1
NEED_WALL=$2
D=0
for disk in $(echo $DISKS | tr "," " "); do
    PRINTABLE_DISK=$(diskname_to_printable $disk)
    cat >> $OUTFILE << EOF
[$TYPE-$PRINTABLE_DISK-$BLK_SIZE-para]
bs=$BLK_SIZE
EOF

if [ "$D" = 0 ]; then
    echo "stonewall" >> $OUTFILE
    D=1
fi

cat >> $OUTFILE << EOF
filename=$disk
rw=$TYPE
write_bw_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-para.results
write_iops_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-para.results
EOF
done

ETA=$(($ETA + $RUNTIME))
echo >> $OUTFILE
}

gen_para_fio() {
for mode in $(echo $MODES | tr "," " "); do
	gen_para_suite "$mode"
done
}

gen_fio() {
case $SEQ in
	2)
		gen_seq_fio
		gen_para_fio
	;;
	1)
		gen_seq_fio
	;;
	0)
		gen_para_fio
	;;
esac
}

parse_cmdline() {
while getopts "hacpsd:b:r:m:x:D:A:B:" opt; do
  case $opt in
    h)
	show_help
	exit 0
	;;
    b)
	BLOCK_SIZE=$OPTARG
	;;
    c)
	CACHED_IO="TRUE"
	;;
    s)
	if [ "$SEQ" = "-1" ]; then
		SEQ=1
	fi
      ;;
    x)
	PREFIX=$OPTARG
	echo "$PREFIX" | grep -q "/"
	if [ "$?" -eq 0 ]; then
		mkdir -p $PREFIX
		# No need to keep the prefix for the log files
		# we do have a directory for that
		PREFIX_FILENAME=""
	else
		# We need to keep the prefix for the log files
		PREFIX_FILENAME=$PREFIX
	fi
	;;
    r)
	RUNTIME=$OPTARG
      ;;
    p)
	if [ "$SEQ" = "-1" ]; then
		SEQ=0
	fi
      ;;
    m)
        MODES=$OPTARG;
      ;;
    d)
 	DISKS=$OPTARG
	PRINTABLE_DISKS=$(diskname_to_printable "$DISKS")
      ;;
    D)
	IODEPTH=$OPTARG
      ;;
    a)
	SEQ=2
      ;;
    B)
	echo "exec_prerun=$OPTARG" >> $TEMPLATE
      ;;
    A)
	echo "exec_postrun=$OPTARG" >> $TEMPLATE
      ;;
    \?)
      echo "Invalid option: -$OPTARG" >&2
      ;;
  esac
done

if [ "$SEQ" = "-1" ]; then
	SEQ=0
fi

SHORT_HOSTNAME=$(hostname -s)
case $SEQ in
	2)
		OUTFILE=${PREFIX}$SHORT_HOSTNAME-$BLOCK_SIZE-all-$MODES-$PRINTABLE_DISKS.fio
	;;

	1)
		OUTFILE=${PREFIX}$SHORT_HOSTNAME-$BLOCK_SIZE-sequential-$MODES-$PRINTABLE_DISKS.fio
	;;
	0)
		OUTFILE=${PREFIX}$SHORT_HOSTNAME-$BLOCK_SIZE-parallel-$MODES-$PRINTABLE_DISKS.fio
	;;
esac

if [ -z "$DISKS" ]; then
	echo "Missing DISKS !"
	echo "Please read the help !"
	show_help
	exit 1
fi

}

check_mode_order() {
FOUND_WRITE="NO"
CAUSE="You are reading data before writing them          "

# If no write occurs, let's show a different message
echo $MODES | grep -q "write"
if [ "$?" -ne 0 ]; then
	CAUSE="You are reading data while never wrote them before"
fi

for mode in $(echo $MODES | tr "," " "); do
	echo $mode | grep -q write
	if [ "$?" -eq 0 ]; then
		FOUND_WRITE="YES"
	fi
	echo $mode | grep -q "read"
	if [ "$?" -eq 0 ]; then
		if [ "$FOUND_WRITE" = "NO" ]; then
			echo "###############################################################"
			echo "# Warning : $CAUSE#"
			echo "# On some storage devices, this could lead to invalid results #"
			echo "#                                                             #"
			echo "# Press Ctrl-C to adjust pattern order if you have doubts     #"
			echo "# Or Wait 5 seconds before the file will be created           #"
			echo "###############################################################"
			sleep 5
			# No need to try showing the message more than one time
			return
		fi
	fi
done
}


########## MAIN
gen_template
parse_cmdline "$@"
finish_template
check_mode_order

echo "Generating $OUTFILE"
cp -f $TEMPLATE $OUTFILE
echo >> $OUTFILE

for BLK_SIZE in $(echo $BLOCK_SIZE | tr "," " "); do
	gen_fio
done
ETA_H=$(($ETA / 3600))
ETA_M=$((($ETA - ($ETA_H*3600)) / 60))
if [ "$ETA" = "0" ]; then
	echo "Cannot estimate ETA as RUNTIME=0"
else
	echo "Estimated Time = $ETA seconds : $ETA_H hour $ETA_M minutes"
fi
Commit	Line	Data
	1	#!/bin/bash
	2	#
	3	# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
	4	# Author: Erwan Velu <erwan@enovance.com>
	5	#
	6	# The license below covers all files distributed with fio unless otherwise
	7	# noted in the file itself.
	8	#
	9	# This program is free software; you can redistribute it and/or modify
	10	# it under the terms of the GNU General Public License version 2 as
	11	# published by the Free Software Foundation.
	12	#
	13	# This program is distributed in the hope that it will be useful,
	14	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	16	# GNU General Public License for more details.
	17	#
	18	# You should have received a copy of the GNU General Public License
	19	# along with this program; if not, write to the Free Software
	20	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	21
	22	BLK_SIZE=
	23	BLOCK_SIZE=4k
	24	SEQ=-1
	25	TEMPLATE=/tmp/template.fio
	26	OUTFILE=
	27	DISKS=
	28	PRINTABLE_DISKS=
	29	RUNTIME=300
	30	ETA=0
	31	MODES="write,randwrite,read,randread"
	32	SHORT_HOSTNAME=
	33	CACHED_IO="FALSE"
	34	PREFIX=""
	35	PREFIX_FILENAME=""
	36	IODEPTH=1
	37
	38	show_help() {
	39	PROG=$(basename $0)
	40	echo "usage of $PROG:"
	41	cat << EOF
	42	-h : Show this help & exit
	43	-c : Enable cached-based IOs
	44	Disabled by default
	45	-a : Run sequential test then parallel one
	46	Disabled by default
	47	-s : Run sequential test (default value)
	48	one test after another then one disk after another
	49	Disabled by default
	50	-p : Run parallel test
	51	one test after anoter but all disks at the same time
	52	Enabled by default
	53	-D iodepth : Run with the specified iodepth
	54	Default is $IODEPTH
	55	-d disk1[,disk2,disk3,..] : Run the tests on the selected disks
	56	Separated each disk with a comma
	57	-r seconds : Time in seconds per benchmark
	58	0 means till the end of the device
	59	Default is $RUNTIME seconds
	60	-b blocksize[,blocksize1, ...] : The blocksizes to test under fio format (4k, 1m, ...)
	61	Separated each blocksize with a comma
	62	Default is $BLOCK_SIZE
	63	-m mode1,[mode2,mode3, ...] : Define the fio IO profile to use like read, write, randread, randwrite
	64	Default is "$MODES"
	65	-x prefix : Add a prefix to the fio filename
	66	Useful to let a context associated with the file
	67	If the prefix features a / (slash), prefix will be considered as a directory
	68	-A cmd_to_run : System command to run after each job (exec_postrun in fio)
	69	-B cmd_to_run : System command to run before each job (exec_prerun in fio)
	70
	71	Example:
	72
	73	$PROG -d /dev/sdb,/dev/sdc,/dev/sdd,/dev/sde -a -b 4k,128k,1m -r 100 -a -x dellr720-day2/
	74
	75	Will generate an fio file that will run
	76	- a sequential bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 4k with write,randwrite,read,randread tests
	77	ETA ~ 4 tests * 4 disks * 100 seconds
	78	- a sequential bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 128k with write,randwrite,read,randread tests
	79	ETA ~ 4 tests * 4 disks * 100 seconds
	80	- a sequential bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 1m with write,randwrite,read,randread tests
	81	ETA ~ 4 tests * 4 disks * 100 seconds
	82	- a parallel bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 4k with write,randwrite,read,randread tests
	83	ETA ~ 4 tests * 100 seconds
	84	- a parallel bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 128k with write,randwrite,read,randread tests
	85	ETA ~ 4 tests * 100 seconds
	86	- a parallel bench on /dev/sdb /dev/sdc /dev/sdd /dev/sde for block size = 1m with write,randwrite,read,randread tests
	87	ETA ~ 4 tests * 100 seconds
	88
	89	Generating dellr720-day2/localhost-4k,128k,1m-all-write,randwrite,read,randread-sdb,sdc,sdd,sde.fio
	90	Estimated Time = 6000 seconds : 1 hour 40 minutes
	91	EOF
	92	}
	93
	94	finish_template() {
	95	echo "iodepth=$IODEPTH" >> $TEMPLATE
	96
	97	if [ "$RUNTIME" != "0" ]; then
	98	echo "runtime=$RUNTIME" >> $TEMPLATE
	99	echo "time_based" >> $TEMPLATE
	100	fi
	101
	102	if [ "$CACHED_IO" = "FALSE" ]; then
	103	echo "direct=1" >> $TEMPLATE
	104	fi
	105	}
	106
	107
	108	diskname_to_printable() {
	109	COUNT=0
	110	for disk in $(echo $@ \| tr "," " "); do
	111	R=$(basename $disk \| sed 's\|/\|_\|g')
	112	COUNT=$(($COUNT + 1))
	113	if [ $COUNT -eq 1 ]; then
	114	P="$R"
	115	else
	116	P="$P,$R"
	117	fi
	118	done
	119	echo $P
	120	}
	121
	122	gen_template() {
	123	cat >$TEMPLATE << EOF
	124	[global]
	125	ioengine=libaio
	126	invalidate=1
	127	ramp_time=5
	128	EOF
	129	}
	130
	131	gen_seq_suite() {
	132	TYPE=$1
	133	disk=$2
	134	PRINTABLE_DISK=$(diskname_to_printable $disk)
	135	cat >> $OUTFILE << EOF
	136	[$TYPE-$PRINTABLE_DISK-$BLK_SIZE-seq]
	137	stonewall
	138	bs=$BLK_SIZE
	139	filename=$disk
	140	rw=$TYPE
	141	write_bw_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-seq.results
	142	write_iops_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-seq.results
	143	EOF
	144	ETA=$(($ETA + $RUNTIME))
	145	}
	146
	147	gen_seq_fio() {
	148	for disk in $(echo $DISKS \| tr "," " "); do
	149	for mode in $(echo $MODES \| tr "," " "); do
	150	gen_seq_suite "$mode" "$disk"
	151	done
	152	done
	153	}
	154
	155
	156	gen_para_suite() {
	157	TYPE=$1
	158	NEED_WALL=$2
	159	D=0
	160	for disk in $(echo $DISKS \| tr "," " "); do
	161	PRINTABLE_DISK=$(diskname_to_printable $disk)
	162	cat >> $OUTFILE << EOF
	163	[$TYPE-$PRINTABLE_DISK-$BLK_SIZE-para]
	164	bs=$BLK_SIZE
	165	EOF
	166
	167	if [ "$D" = 0 ]; then
	168	echo "stonewall" >> $OUTFILE
	169	D=1
	170	fi
	171
	172	cat >> $OUTFILE << EOF
	173	filename=$disk
	174	rw=$TYPE
	175	write_bw_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-para.results
	176	write_iops_log=${PREFIX_FILENAME}$SHORT_HOSTNAME-$BLK_SIZE-$PRINTABLE_DISK-$TYPE-para.results
	177	EOF
	178	done
	179
	180	ETA=$(($ETA + $RUNTIME))
	181	echo >> $OUTFILE
	182	}
	183
	184	gen_para_fio() {
	185	for mode in $(echo $MODES \| tr "," " "); do
	186	gen_para_suite "$mode"
	187	done
	188	}
	189
	190	gen_fio() {
	191	case $SEQ in
	192	2)
	193	gen_seq_fio
	194	gen_para_fio
	195	;;
	196	1)
	197	gen_seq_fio
	198	;;
	199	0)
	200	gen_para_fio
	201	;;
	202	esac
	203	}
	204
	205	parse_cmdline() {
	206	while getopts "hacpsd:b:r:m:x:D:A:B:" opt; do
	207	case $opt in
	208	h)
	209	show_help
	210	exit 0
	211	;;
	212	b)
	213	BLOCK_SIZE=$OPTARG
	214	;;
	215	c)
	216	CACHED_IO="TRUE"
	217	;;
	218	s)
	219	if [ "$SEQ" = "-1" ]; then
	220	SEQ=1
	221	fi
	222	;;
	223	x)
	224	PREFIX=$OPTARG
	225	echo "$PREFIX" \| grep -q "/"
	226	if [ "$?" -eq 0 ]; then
	227	mkdir -p $PREFIX
	228	# No need to keep the prefix for the log files
	229	# we do have a directory for that
	230	PREFIX_FILENAME=""
	231	else
	232	# We need to keep the prefix for the log files
	233	PREFIX_FILENAME=$PREFIX
	234	fi
	235	;;
	236	r)
	237	RUNTIME=$OPTARG
	238	;;
	239	p)
	240	if [ "$SEQ" = "-1" ]; then
	241	SEQ=0
	242	fi
	243	;;
	244	m)
	245	MODES=$OPTARG;
	246	;;
	247	d)
	248	DISKS=$OPTARG
	249	PRINTABLE_DISKS=$(diskname_to_printable "$DISKS")
	250	;;
	251	D)
	252	IODEPTH=$OPTARG
	253	;;
	254	a)
	255	SEQ=2
	256	;;
	257	B)
	258	echo "exec_prerun=$OPTARG" >> $TEMPLATE
	259	;;
	260	A)
	261	echo "exec_postrun=$OPTARG" >> $TEMPLATE
	262	;;
	263	\?)
	264	echo "Invalid option: -$OPTARG" >&2
	265	;;
	266	esac
	267	done
	268
	269	if [ "$SEQ" = "-1" ]; then
	270	SEQ=0
	271	fi
	272
	273	SHORT_HOSTNAME=$(hostname -s)
	274	case $SEQ in
	275	2)
	276	OUTFILE=${PREFIX}$SHORT_HOSTNAME-$BLOCK_SIZE-all-$MODES-$PRINTABLE_DISKS.fio
	277	;;
	278
	279	1)
	280	OUTFILE=${PREFIX}$SHORT_HOSTNAME-$BLOCK_SIZE-sequential-$MODES-$PRINTABLE_DISKS.fio
	281	;;
	282	0)
	283	OUTFILE=${PREFIX}$SHORT_HOSTNAME-$BLOCK_SIZE-parallel-$MODES-$PRINTABLE_DISKS.fio
	284	;;
	285	esac
	286
	287	if [ -z "$DISKS" ]; then
	288	echo "Missing DISKS !"
	289	echo "Please read the help !"
	290	show_help
	291	exit 1
	292	fi
	293
	294	}
	295
	296	check_mode_order() {
	297	FOUND_WRITE="NO"
	298	CAUSE="You are reading data before writing them "
	299
	300	# If no write occurs, let's show a different message
	301	echo $MODES \| grep -q "write"
	302	if [ "$?" -ne 0 ]; then
	303	CAUSE="You are reading data while never wrote them before"
	304	fi
	305
	306	for mode in $(echo $MODES \| tr "," " "); do
	307	echo $mode \| grep -q write
	308	if [ "$?" -eq 0 ]; then
	309	FOUND_WRITE="YES"
	310	fi
	311	echo $mode \| grep -q "read"
	312	if [ "$?" -eq 0 ]; then
	313	if [ "$FOUND_WRITE" = "NO" ]; then
	314	echo "###############################################################"
	315	echo "# Warning : $CAUSE#"
	316	echo "# On some storage devices, this could lead to invalid results #"
	317	echo "# #"
	318	echo "# Press Ctrl-C to adjust pattern order if you have doubts #"
	319	echo "# Or Wait 5 seconds before the file will be created #"
	320	echo "###############################################################"
	321	sleep 5
	322	# No need to try showing the message more than one time
	323	return
	324	fi
	325	fi
	326	done
	327	}
	328
	329
	330	########## MAIN
	331	gen_template
	332	parse_cmdline "$@"
	333	finish_template
	334	check_mode_order
	335
	336	echo "Generating $OUTFILE"
	337	cp -f $TEMPLATE $OUTFILE
	338	echo >> $OUTFILE
	339
	340	for BLK_SIZE in $(echo $BLOCK_SIZE \| tr "," " "); do
	341	gen_fio
	342	done
	343	ETA_H=$(($ETA / 3600))
	344	ETA_M=$((($ETA - ($ETA_H*3600)) / 60))
	345	if [ "$ETA" = "0" ]; then
	346	echo "Cannot estimate ETA as RUNTIME=0"
	347	else
	348	echo "Estimated Time = $ETA seconds : $ETA_H hour $ETA_M minutes"
	349	fi