[fio.git] / t / steadystate_tests.py

#!/usr/bin/python2.7
# Note: this script is python2 and python 3 compatible.
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# On Windows this script works under Cygwin but not from cmd.exe
# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

from __future__ import absolute_import
from __future__ import print_function
import os
import sys
import json
import uuid
import pprint
import argparse
import subprocess
from scipy import stats
from six.moves import range

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio',
                        help='path to fio executable')
    parser.add_argument('--read',
                        help='target for read testing')
    parser.add_argument('--write',
                        help='target for write testing')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = list(range(len(data)))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = m / mean * 100
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args'])
        if test['output'] in output.decode():
            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
        else:
            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': False, 'timeout': 20, 'numjobs': 2},
              {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
            ]

    if args.read == None:
        if os.name == 'posix':
            args.read = '/dev/zero'
            extra = [ "--size=134217728" ]  # 128 MiB
        else:
            print("ERROR: file for read testing must be specified on non-posix systems")
            sys.exit(1)
    else:
        extra = []

    jobnum = 0
    for job in reads:

        tf = uuid.uuid4().hex
        parameters = [ "--name=job{0}".format(jobnum) ]
        parameters.extend(extra)
        parameters.extend([ "--thread",
                            "--output-format=json",
                            "--output={0}".format(tf),
                            "--filename={0}".format(args.read),
                            "--rw=randrw",
                            "--rwmixread=100",
                            "--stonewall",
                            "--group_reporting",
                            "--numjobs={0}".format(job['numjobs']),
                            "--time_based",
                            "--runtime={0}".format(job['timeout']) ])
        if job['s']:
           if job['iops']:
               ss = 'iops'
           else:
               ss = 'bw'
           if job['slope']:
               ss += "_slope"
           ss += ":" + str(job['ss_limit'])
           if job['pct']:
               ss += '%'
           parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                               '--ss={0}'.format(ss),
                               '--ss_ramp={0}'.format(job['ss_ramp']) ])

        output = subprocess.call([args.fio] + parameters)
        with open(tf, 'r') as source:
            jsondata = json.loads(source.read())
        os.remove(tf)

        for jsonjob in jsondata['jobs']:
            line = "job {0}".format(jsonjob['job options']['name'])
            if job['s']:
                if jsonjob['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
                    actual = jsonjob['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = job['timeout'] * 1000
                    actual = jsonjob['read']['runtime']
                    if abs(expected - actual) > 10:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
            else:
                expected = job['timeout'] * 1000
                actual = jsonjob['read']['runtime']
                if abs(expected - actual) < 10:
                    result = 'PASSED '
                else:
                    result = 'FAILED '
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print(line)
            if 'steadystate' in jsonjob:
                pp.pprint(jsonjob['steadystate'])
        jobnum += 1
Commit	Line	Data
60023ade	1	#!/usr/bin/python2.7
5eac3b00	2	# Note: this script is python2 and python 3 compatible.
16e56d25 VF	3	#
	4	# steadystate_tests.py
	5	#
	6	# Test option parsing and functonality for fio's steady state detection feature.
	7	#
a72f40bb	8	# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
16e56d25 VF	9	#
	10	# REQUIREMENTS
	11	# Python 2.6+
	12	# SciPy
	13	#
	14	# KNOWN ISSUES
	15	# only option parsing and read tests are carried out
32df42c9 VF	16	# On Windows this script works under Cygwin but not from cmd.exe
32df42c9 VF	17	# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
16e56d25 VF	18	# min runtime:
	19	# if ss attained: min runtime = ss_dur + ss_ramp
	20	# if not attained: runtime = timeout
	21
5eac3b00 BD	22	from __future__ import absolute_import
5eac3b00 BD	23	from __future__ import print_function
16e56d25	24	import os
39f5379e	25	import sys
16e56d25	26	import json
32df42c9	27	import uuid
dd4e8700	28	import pprint
16e56d25 VF	29	import argparse
	30	import subprocess
	31	from scipy import stats
5eac3b00	32	from six.moves import range
16e56d25 VF	33
	34	def parse_args():
	35	parser = argparse.ArgumentParser()
	36	parser.add_argument('fio',
5eac3b00	37	help='path to fio executable')
39f5379e	38	parser.add_argument('--read',
16e56d25	39	help='target for read testing')
39f5379e	40	parser.add_argument('--write',
16e56d25 VF	41	help='target for write testing')
	42	args = parser.parse_args()
	43
	44	return args
	45
	46
	47	def check(data, iops, slope, pct, limit, dur, criterion):
ba8fb6f6 VF	48	measurement = 'iops' if iops else 'bw'
ba8fb6f6 VF	49	data = data[measurement]
16e56d25 VF	50	mean = sum(data) / len(data)
16e56d25 VF	51	if slope:
5eac3b00	52	x = list(range(len(data)))
16e56d25 VF	53	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	54	m = abs(m)
	55	if pct:
4001d829 VF	56	target = m / mean * 100
4001d829 VF	57	criterion = criterion[:-1]
16e56d25 VF	58	else:
	59	target = m
	60	else:
	61	maxdev = 0
	62	for x in data:
	63	maxdev = max(abs(mean-x), maxdev)
	64	if pct:
4001d829 VF	65	target = maxdev / mean * 100
4001d829 VF	66	criterion = criterion[:-1]
16e56d25 VF	67	else:
	68	target = maxdev
	69
4001d829	70	criterion = float(criterion)
ba8fb6f6	71	return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
16e56d25 VF	72
	73
	74	if __name__ == '__main__':
	75	args = parse_args()
	76
dd4e8700 VF	77	pp = pprint.PrettyPrinter(indent=4)
dd4e8700 VF	78
16e56d25 VF	79	#
	80	# test option parsing
	81	#
bfc4884e	82	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
16e56d25	83	'output': "set steady state IOPS threshold to 10.000000" },
bfc4884e	84	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
16e56d25	85	'output': "set steady state threshold to 10.000000%" },
bfc4884e	86	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
16e56d25	87	'output': "set steady state threshold to 0.100000%" },
bfc4884e	88	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
16e56d25	89	'output': "set steady state threshold to 10.000000%" },
bfc4884e	90	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
16e56d25	91	'output': "set steady state threshold to 0.100000%" },
bfc4884e	92	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
16e56d25 VF	93	'output': "set steady state BW threshold to 12" },
	94	]
	95	for test in parsing:
5eac3b00 BD	96	output = subprocess.check_output([args.fio] + test['args'])
	97	if test['output'] in output.decode():
	98	print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
16e56d25	99	else:
5eac3b00	100	print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
16e56d25 VF	101
	102	#
	103	# test some read workloads
	104	#
	105	# if ss active and attained,
	106	# check that runtime is less than job time
	107	# check criteria
	108	# how to check ramp time?
	109	#
	110	# if ss inactive
	111	# check that runtime is what was specified
	112	#
32df42c9 VF	113	reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	114	{'s': False, 'timeout': 20, 'numjobs': 2},
	115	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	116	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
16e56d25 VF	117	]
16e56d25 VF	118
32df42c9 VF	119	if args.read == None:
	120	if os.name == 'posix':
	121	args.read = '/dev/zero'
2b9136a3	122	extra = [ "--size=134217728" ] # 128 MiB
32df42c9	123	else:
5eac3b00	124	print("ERROR: file for read testing must be specified on non-posix systems")
32df42c9 VF	125	sys.exit(1)
	126	else:
	127	extra = []
	128
	129	jobnum = 0
	130	for job in reads:
	131
	132	tf = uuid.uuid4().hex
	133	parameters = [ "--name=job{0}".format(jobnum) ]
	134	parameters.extend(extra)
	135	parameters.extend([ "--thread",
	136	"--output-format=json",
	137	"--output={0}".format(tf),
	138	"--filename={0}".format(args.read),
	139	"--rw=randrw",
	140	"--rwmixread=100",
	141	"--stonewall",
	142	"--group_reporting",
	143	"--numjobs={0}".format(job['numjobs']),
	144	"--time_based",
	145	"--runtime={0}".format(job['timeout']) ])
	146	if job['s']:
	147	if job['iops']:
	148	ss = 'iops'
	149	else:
	150	ss = 'bw'
	151	if job['slope']:
	152	ss += "_slope"
	153	ss += ":" + str(job['ss_limit'])
	154	if job['pct']:
	155	ss += '%'
	156	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	157	'--ss={0}'.format(ss),
	158	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	159
	160	output = subprocess.call([args.fio] + parameters)
	161	with open(tf, 'r') as source:
16e56d25	162	jsondata = json.loads(source.read())
32df42c9 VF	163	os.remove(tf)
	164
	165	for jsonjob in jsondata['jobs']:
	166	line = "job {0}".format(jsonjob['job options']['name'])
	167	if job['s']:
	168	if jsonjob['steadystate']['attained'] == 1:
16e56d25	169	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
32df42c9 VF	170	mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
32df42c9 VF	171	actual = jsonjob['read']['runtime']
16e56d25	172	if mintime > actual:
32df42c9	173	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
16e56d25	174	else:
32df42c9 VF	175	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	176	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	177	iops=job['iops'],
	178	slope=job['slope'],
	179	pct=job['pct'],
	180	limit=job['ss_limit'],
	181	dur=job['ss_dur'],
	182	criterion=jsonjob['steadystate']['criterion'])
16e56d25	183	if not objsame:
32df42c9	184	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
16e56d25 VF	185	else:
16e56d25 VF	186	if met:
32df42c9	187	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
16e56d25	188	else:
32df42c9	189	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
16e56d25 VF	190	else:
16e56d25 VF	191	# check runtime, confirm criterion calculation, and confirm that criterion was not met
32df42c9 VF	192	expected = job['timeout'] * 1000
32df42c9 VF	193	actual = jsonjob['read']['runtime']
16e56d25 VF	194	if abs(expected - actual) > 10:
	195	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
	196	else:
32df42c9 VF	197	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	198	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	199	iops=job['iops'],
	200	slope=job['slope'],
	201	pct=job['pct'],
	202	limit=job['ss_limit'],
	203	dur=job['ss_dur'],
	204	criterion=jsonjob['steadystate']['criterion'])
16e56d25	205	if not objsame:
32df42c9 VF	206	if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
32df42c9 VF	207	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
16e56d25	208	else:
32df42c9	209	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
16e56d25 VF	210	else:
16e56d25 VF	211	if met:
32df42c9	212	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
16e56d25	213	else:
32df42c9	214	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
16e56d25	215	else:
32df42c9 VF	216	expected = job['timeout'] * 1000
32df42c9 VF	217	actual = jsonjob['read']['runtime']
16e56d25 VF	218	if abs(expected - actual) < 10:
	219	result = 'PASSED '
	220	else:
	221	result = 'FAILED '
	222	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
5eac3b00	223	print(line)
32df42c9 VF	224	if 'steadystate' in jsonjob:
	225	pp.pprint(jsonjob['steadystate'])
	226	jobnum += 1