[fio.git] / unit_tests / steadystate_tests.py

#!/usr/bin/python
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that
#     it does not match what fio reports
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

import os
import sys
import json
import pprint
import tempfile
import platform
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio',
                        help='path to fio executable');
    parser.add_argument('--read',
                        help='target for read testing')
    parser.add_argument('--write',
                        help='target for write testing')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = range(len(data))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = m / mean * 100
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args']);
        if test['output'] in output:
            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
        else:
            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
                {'s': False, 'timeout': 20, 'numjobs': 2},
                {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
                {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ],
            ]

    accum = []
    suitenum = 0
    for suite in reads:
        jobnum = 0
        for job in suite:
            if args.read == None:
                if os.name == 'posix':
                    args.read = '/dev/zero'
                    parameters = [ "--size=128M" ]
                else:
                    print "ERROR: file for read testing must be specified on non-posix systems"
                    sys.exit(1)
            else:
                parameters = []

            parameters.extend([ "--name=job{0}".format(jobnum),
                                "--thread",
                                "--filename={0}".format(args.read),
                                "--rw=randrw", "--rwmixread=100", "--stonewall",
                                "--group_reporting", "--numjobs={0}".format(job['numjobs']),
                                "--time_based", "--runtime={0}".format(job['timeout']) ])
            if job['s']:
               if job['iops']:
                   ss = 'iops'
               else:
                   ss = 'bw'
               if job['slope']:
                   ss += "_slope"
               ss += ":" + str(job['ss_limit'])
               if job['pct']:
                   ss += '%'
               parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                                   '--ss={0}'.format(ss),
                                   '--ss_ramp={0}'.format(job['ss_ramp']) ])
            accum.extend(parameters)
            jobnum += 1

        tf = tempfile.NamedTemporaryFile(delete=False)
	tf.close()
        output = subprocess.check_output([args.fio,
                                          "--output-format=json",
                                          "--output={0}".format(tf.name)] + accum)
        with open(tf.name, 'r') as source:
            jsondata = json.loads(source.read())
        os.remove(tf.name)
        jobnum = 0
        for job in jsondata['jobs']:
            line = "suite {0}, {1}".format(suitenum, job['job options']['name'])
            if suite[jobnum]['s']:
                if job['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000
                    actual = job['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                        objsame, met, mean, target = check(data=job['steadystate']['data'],
                            iops=suite[jobnum]['iops'],
                            slope=suite[jobnum]['slope'],
                            pct=suite[jobnum]['pct'],
                            limit=suite[jobnum]['ss_limit'],
                            dur=suite[jobnum]['ss_dur'],
                            criterion=job['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, suite[jobnum]['ss_limit'])
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = suite[jobnum]['timeout'] * 1000
                    actual = job['read']['runtime']
                    if abs(expected - actual) > 10:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                        objsame, met, mean, target = check(data=job['steadystate']['data'],
                            iops=suite[jobnum]['iops'],
                            slope=suite[jobnum]['slope'],
                            pct=suite[jobnum]['pct'],
                            limit=suite[jobnum]['ss_limit'],
                            dur=suite[jobnum]['ss_dur'],
                            criterion=job['steadystate']['criterion'])
                        if not objsame:
                            if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(job['steadystate']['criterion'])
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, suite[jobnum]['ss_limit'])
            else:
                expected = suite[jobnum]['timeout'] * 1000
                actual = job['read']['runtime']
                if abs(expected - actual) < 10:
                    result = 'PASSED '
                else:
                    result = 'FAILED '
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print line
            if 'steadystate' in job:
                pp.pprint(job['steadystate'])
            jobnum += 1
        suitenum += 1
Commit	Line	Data
16e56d25 VF	1	#!/usr/bin/python
	2	#
	3	# steadystate_tests.py
	4	#
	5	# Test option parsing and functonality for fio's steady state detection feature.
	6	#
	7	# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
	8	#
	9	# REQUIREMENTS
	10	# Python 2.6+
	11	# SciPy
	12	#
	13	# KNOWN ISSUES
	14	# only option parsing and read tests are carried out
	15	# the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that
	16	# it does not match what fio reports
	17	# min runtime:
	18	# if ss attained: min runtime = ss_dur + ss_ramp
	19	# if not attained: runtime = timeout
	20
	21	import os
39f5379e	22	import sys
16e56d25	23	import json
dd4e8700	24	import pprint
16e56d25	25	import tempfile
39f5379e	26	import platform
16e56d25 VF	27	import argparse
	28	import subprocess
	29	from scipy import stats
	30
	31	def parse_args():
	32	parser = argparse.ArgumentParser()
	33	parser.add_argument('fio',
	34	help='path to fio executable');
39f5379e	35	parser.add_argument('--read',
16e56d25	36	help='target for read testing')
39f5379e	37	parser.add_argument('--write',
16e56d25 VF	38	help='target for write testing')
	39	args = parser.parse_args()
	40
	41	return args
	42
	43
	44	def check(data, iops, slope, pct, limit, dur, criterion):
ba8fb6f6 VF	45	measurement = 'iops' if iops else 'bw'
ba8fb6f6 VF	46	data = data[measurement]
16e56d25 VF	47	mean = sum(data) / len(data)
	48	if slope:
	49	x = range(len(data))
	50	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	51	m = abs(m)
	52	if pct:
4001d829 VF	53	target = m / mean * 100
4001d829 VF	54	criterion = criterion[:-1]
16e56d25 VF	55	else:
	56	target = m
	57	else:
	58	maxdev = 0
	59	for x in data:
	60	maxdev = max(abs(mean-x), maxdev)
	61	if pct:
4001d829 VF	62	target = maxdev / mean * 100
4001d829 VF	63	criterion = criterion[:-1]
16e56d25 VF	64	else:
	65	target = maxdev
	66
4001d829	67	criterion = float(criterion)
ba8fb6f6	68	return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
16e56d25 VF	69
	70
	71	if __name__ == '__main__':
	72	args = parse_args()
	73
dd4e8700 VF	74	pp = pprint.PrettyPrinter(indent=4)
dd4e8700 VF	75
16e56d25 VF	76	#
	77	# test option parsing
	78	#
bfc4884e	79	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
16e56d25	80	'output': "set steady state IOPS threshold to 10.000000" },
bfc4884e	81	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
16e56d25	82	'output': "set steady state threshold to 10.000000%" },
bfc4884e	83	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
16e56d25	84	'output': "set steady state threshold to 0.100000%" },
bfc4884e	85	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
16e56d25	86	'output': "set steady state threshold to 10.000000%" },
bfc4884e	87	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
16e56d25	88	'output': "set steady state threshold to 0.100000%" },
bfc4884e	89	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
16e56d25 VF	90	'output': "set steady state BW threshold to 12" },
	91	]
	92	for test in parsing:
	93	output = subprocess.check_output([args.fio] + test['args']);
	94	if test['output'] in output:
	95	print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
	96	else:
	97	print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
	98
	99	#
	100	# test some read workloads
	101	#
	102	# if ss active and attained,
	103	# check that runtime is less than job time
	104	# check criteria
	105	# how to check ramp time?
	106	#
	107	# if ss inactive
	108	# check that runtime is what was specified
	109	#
	110	reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	111	{'s': False, 'timeout': 20, 'numjobs': 2},
	112	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	113	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ],
	114	]
	115
	116	accum = []
	117	suitenum = 0
	118	for suite in reads:
	119	jobnum = 0
	120	for job in suite:
39f5379e VF	121	if args.read == None:
	122	if os.name == 'posix':
	123	args.read = '/dev/zero'
	124	parameters = [ "--size=128M" ]
	125	else:
	126	print "ERROR: file for read testing must be specified on non-posix systems"
	127	sys.exit(1)
	128	else:
	129	parameters = []
	130
	131	parameters.extend([ "--name=job{0}".format(jobnum),
	132	"--thread",
	133	"--filename={0}".format(args.read),
	134	"--rw=randrw", "--rwmixread=100", "--stonewall",
	135	"--group_reporting", "--numjobs={0}".format(job['numjobs']),
	136	"--time_based", "--runtime={0}".format(job['timeout']) ])
16e56d25 VF	137	if job['s']:
	138	if job['iops']:
	139	ss = 'iops'
	140	else:
	141	ss = 'bw'
	142	if job['slope']:
	143	ss += "_slope"
	144	ss += ":" + str(job['ss_limit'])
	145	if job['pct']:
	146	ss += '%'
	147	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	148	'--ss={0}'.format(ss),
	149	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	150	accum.extend(parameters)
	151	jobnum += 1
	152
	153	tf = tempfile.NamedTemporaryFile(delete=False)
	154	tf.close()
bfc4884e VF	155	output = subprocess.check_output([args.fio,
bfc4884e VF	156	"--output-format=json",
16e56d25 VF	157	"--output={0}".format(tf.name)] + accum)
	158	with open(tf.name, 'r') as source:
	159	jsondata = json.loads(source.read())
	160	os.remove(tf.name)
	161	jobnum = 0
	162	for job in jsondata['jobs']:
	163	line = "suite {0}, {1}".format(suitenum, job['job options']['name'])
	164	if suite[jobnum]['s']:
	165	if job['steadystate']['attained'] == 1:
	166	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
	167	mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000
	168	actual = job['read']['runtime']
	169	if mintime > actual:
	170	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
	171	else:
	172	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
	173	objsame, met, mean, target = check(data=job['steadystate']['data'],
	174	iops=suite[jobnum]['iops'],
	175	slope=suite[jobnum]['slope'],
	176	pct=suite[jobnum]['pct'],
	177	limit=suite[jobnum]['ss_limit'],
	178	dur=suite[jobnum]['ss_dur'],
	179	criterion=job['steadystate']['criterion'])
	180	if not objsame:
dd4e8700	181	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
16e56d25 VF	182	else:
16e56d25 VF	183	if met:
dd4e8700	184	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, suite[jobnum]['ss_limit'])
16e56d25	185	else:
dd4e8700	186	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
16e56d25 VF	187	else:
	188	# check runtime, confirm criterion calculation, and confirm that criterion was not met
	189	expected = suite[jobnum]['timeout'] * 1000
	190	actual = job['read']['runtime']
	191	if abs(expected - actual) > 10:
	192	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
	193	else:
	194	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
	195	objsame, met, mean, target = check(data=job['steadystate']['data'],
	196	iops=suite[jobnum]['iops'],
	197	slope=suite[jobnum]['slope'],
	198	pct=suite[jobnum]['pct'],
	199	limit=suite[jobnum]['ss_limit'],
	200	dur=suite[jobnum]['ss_dur'],
	201	criterion=job['steadystate']['criterion'])
	202	if not objsame:
	203	if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
dd4e8700	204	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
16e56d25	205	else:
dd4e8700	206	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(job['steadystate']['criterion'])
16e56d25 VF	207	else:
16e56d25 VF	208	if met:
dd4e8700	209	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
16e56d25	210	else:
dd4e8700	211	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, suite[jobnum]['ss_limit'])
16e56d25 VF	212	else:
	213	expected = suite[jobnum]['timeout'] * 1000
	214	actual = job['read']['runtime']
	215	if abs(expected - actual) < 10:
	216	result = 'PASSED '
	217	else:
	218	result = 'FAILED '
	219	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
	220	print line
dd4e8700 VF	221	if 'steadystate' in job:
dd4e8700 VF	222	pp.pprint(job['steadystate'])
16e56d25 VF	223	jobnum += 1
16e56d25 VF	224	suitenum += 1