[fio.git] / unit_tests / steadystate_tests.py

#!/usr/bin/python
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that
#     it does not match what fio reports
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

import os
import json
import pprint
import tempfile
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio',
                        help='path to fio executable');
    parser.add_argument('read',
                        help='target for read testing')
    parser.add_argument('write',
                        help='target for write testing')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = range(len(data))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = m / mean * 100
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args']);
        if test['output'] in output:
            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
        else:
            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
                {'s': False, 'timeout': 20, 'numjobs': 2},
                {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
                {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ],
            ]

    accum = []
    suitenum = 0
    for suite in reads:
        jobnum = 0
        for job in suite:
            parameters = [ "--name=job{0}".format(jobnum),
                           "--thread",
                           "--filename={0}".format(args.read),
                           "--rw=randrw", "--rwmixread=100", "--stonewall",
                           "--group_reporting", "--numjobs={0}".format(job['numjobs']),
                           "--time_based", "--runtime={0}".format(job['timeout']) ]
            if job['s']:
               if job['iops']:
                   ss = 'iops'
               else:
                   ss = 'bw'
               if job['slope']:
                   ss += "_slope"
               ss += ":" + str(job['ss_limit'])
               if job['pct']:
                   ss += '%'
               parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                                   '--ss={0}'.format(ss),
                                   '--ss_ramp={0}'.format(job['ss_ramp']) ])
            accum.extend(parameters)
            jobnum += 1

        tf = tempfile.NamedTemporaryFile(delete=False)
	tf.close()
        output = subprocess.check_output([args.fio,
                                          "--output-format=json",
                                          "--output={0}".format(tf.name)] + accum)
        with open(tf.name, 'r') as source:
            jsondata = json.loads(source.read())
        os.remove(tf.name)
        jobnum = 0
        for job in jsondata['jobs']:
            line = "suite {0}, {1}".format(suitenum, job['job options']['name'])
            if suite[jobnum]['s']:
                if job['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000
                    actual = job['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                        objsame, met, mean, target = check(data=job['steadystate']['data'],
                            iops=suite[jobnum]['iops'],
                            slope=suite[jobnum]['slope'],
                            pct=suite[jobnum]['pct'],
                            limit=suite[jobnum]['ss_limit'],
                            dur=suite[jobnum]['ss_dur'],
                            criterion=job['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, suite[jobnum]['ss_limit'])
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = suite[jobnum]['timeout'] * 1000
                    actual = job['read']['runtime']
                    if abs(expected - actual) > 10:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                        objsame, met, mean, target = check(data=job['steadystate']['data'],
                            iops=suite[jobnum]['iops'],
                            slope=suite[jobnum]['slope'],
                            pct=suite[jobnum]['pct'],
                            limit=suite[jobnum]['ss_limit'],
                            dur=suite[jobnum]['ss_dur'],
                            criterion=job['steadystate']['criterion'])
                        if not objsame:
                            if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(job['steadystate']['criterion'])
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, suite[jobnum]['ss_limit'])
            else:
                expected = suite[jobnum]['timeout'] * 1000
                actual = job['read']['runtime']
                if abs(expected - actual) < 10:
                    result = 'PASSED '
                else:
                    result = 'FAILED '
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print line
            if 'steadystate' in job:
                pp.pprint(job['steadystate'])
            jobnum += 1
        suitenum += 1
Commit	Line	Data
16e56d25 VF	1	#!/usr/bin/python
	2	#
	3	# steadystate_tests.py
	4	#
	5	# Test option parsing and functonality for fio's steady state detection feature.
	6	#
	7	# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
	8	#
	9	# REQUIREMENTS
	10	# Python 2.6+
	11	# SciPy
	12	#
	13	# KNOWN ISSUES
	14	# only option parsing and read tests are carried out
	15	# the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that
	16	# it does not match what fio reports
	17	# min runtime:
	18	# if ss attained: min runtime = ss_dur + ss_ramp
	19	# if not attained: runtime = timeout
	20
	21	import os
	22	import json
dd4e8700	23	import pprint
16e56d25 VF	24	import tempfile
	25	import argparse
	26	import subprocess
	27	from scipy import stats
	28
	29	def parse_args():
	30	parser = argparse.ArgumentParser()
	31	parser.add_argument('fio',
	32	help='path to fio executable');
	33	parser.add_argument('read',
	34	help='target for read testing')
	35	parser.add_argument('write',
	36	help='target for write testing')
	37	args = parser.parse_args()
	38
	39	return args
	40
	41
	42	def check(data, iops, slope, pct, limit, dur, criterion):
ba8fb6f6 VF	43	measurement = 'iops' if iops else 'bw'
ba8fb6f6 VF	44	data = data[measurement]
16e56d25 VF	45	mean = sum(data) / len(data)
	46	if slope:
	47	x = range(len(data))
	48	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	49	m = abs(m)
	50	if pct:
4001d829 VF	51	target = m / mean * 100
4001d829 VF	52	criterion = criterion[:-1]
16e56d25 VF	53	else:
	54	target = m
	55	else:
	56	maxdev = 0
	57	for x in data:
	58	maxdev = max(abs(mean-x), maxdev)
	59	if pct:
4001d829 VF	60	target = maxdev / mean * 100
4001d829 VF	61	criterion = criterion[:-1]
16e56d25 VF	62	else:
	63	target = maxdev
	64
4001d829	65	criterion = float(criterion)
ba8fb6f6	66	return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
16e56d25 VF	67
	68
	69	if __name__ == '__main__':
	70	args = parse_args()
	71
dd4e8700 VF	72	pp = pprint.PrettyPrinter(indent=4)
dd4e8700 VF	73
16e56d25 VF	74	#
	75	# test option parsing
	76	#
bfc4884e	77	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
16e56d25	78	'output': "set steady state IOPS threshold to 10.000000" },
bfc4884e	79	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
16e56d25	80	'output': "set steady state threshold to 10.000000%" },
bfc4884e	81	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
16e56d25	82	'output': "set steady state threshold to 0.100000%" },
bfc4884e	83	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
16e56d25	84	'output': "set steady state threshold to 10.000000%" },
bfc4884e	85	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
16e56d25	86	'output': "set steady state threshold to 0.100000%" },
bfc4884e	87	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
16e56d25 VF	88	'output': "set steady state BW threshold to 12" },
	89	]
	90	for test in parsing:
	91	output = subprocess.check_output([args.fio] + test['args']);
	92	if test['output'] in output:
	93	print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
	94	else:
	95	print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
	96
	97	#
	98	# test some read workloads
	99	#
	100	# if ss active and attained,
	101	# check that runtime is less than job time
	102	# check criteria
	103	# how to check ramp time?
	104	#
	105	# if ss inactive
	106	# check that runtime is what was specified
	107	#
	108	reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	109	{'s': False, 'timeout': 20, 'numjobs': 2},
	110	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	111	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ],
	112	]
	113
	114	accum = []
	115	suitenum = 0
	116	for suite in reads:
	117	jobnum = 0
	118	for job in suite:
	119	parameters = [ "--name=job{0}".format(jobnum),
	120	"--thread",
	121	"--filename={0}".format(args.read),
	122	"--rw=randrw", "--rwmixread=100", "--stonewall",
	123	"--group_reporting", "--numjobs={0}".format(job['numjobs']),
	124	"--time_based", "--runtime={0}".format(job['timeout']) ]
	125	if job['s']:
	126	if job['iops']:
	127	ss = 'iops'
	128	else:
	129	ss = 'bw'
	130	if job['slope']:
	131	ss += "_slope"
	132	ss += ":" + str(job['ss_limit'])
	133	if job['pct']:
	134	ss += '%'
	135	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	136	'--ss={0}'.format(ss),
	137	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	138	accum.extend(parameters)
	139	jobnum += 1
	140
	141	tf = tempfile.NamedTemporaryFile(delete=False)
	142	tf.close()
bfc4884e VF	143	output = subprocess.check_output([args.fio,
bfc4884e VF	144	"--output-format=json",
16e56d25 VF	145	"--output={0}".format(tf.name)] + accum)
	146	with open(tf.name, 'r') as source:
	147	jsondata = json.loads(source.read())
	148	os.remove(tf.name)
	149	jobnum = 0
	150	for job in jsondata['jobs']:
	151	line = "suite {0}, {1}".format(suitenum, job['job options']['name'])
	152	if suite[jobnum]['s']:
	153	if job['steadystate']['attained'] == 1:
	154	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
	155	mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000
	156	actual = job['read']['runtime']
	157	if mintime > actual:
	158	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
	159	else:
	160	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
	161	objsame, met, mean, target = check(data=job['steadystate']['data'],
	162	iops=suite[jobnum]['iops'],
	163	slope=suite[jobnum]['slope'],
	164	pct=suite[jobnum]['pct'],
	165	limit=suite[jobnum]['ss_limit'],
	166	dur=suite[jobnum]['ss_dur'],
	167	criterion=job['steadystate']['criterion'])
	168	if not objsame:
dd4e8700	169	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
16e56d25 VF	170	else:
16e56d25 VF	171	if met:
dd4e8700	172	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, suite[jobnum]['ss_limit'])
16e56d25	173	else:
dd4e8700	174	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
16e56d25 VF	175	else:
	176	# check runtime, confirm criterion calculation, and confirm that criterion was not met
	177	expected = suite[jobnum]['timeout'] * 1000
	178	actual = job['read']['runtime']
	179	if abs(expected - actual) > 10:
	180	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
	181	else:
	182	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
	183	objsame, met, mean, target = check(data=job['steadystate']['data'],
	184	iops=suite[jobnum]['iops'],
	185	slope=suite[jobnum]['slope'],
	186	pct=suite[jobnum]['pct'],
	187	limit=suite[jobnum]['ss_limit'],
	188	dur=suite[jobnum]['ss_dur'],
	189	criterion=job['steadystate']['criterion'])
	190	if not objsame:
	191	if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
dd4e8700	192	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
16e56d25	193	else:
dd4e8700	194	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(job['steadystate']['criterion'])
16e56d25 VF	195	else:
16e56d25 VF	196	if met:
dd4e8700	197	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
16e56d25	198	else:
dd4e8700	199	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, suite[jobnum]['ss_limit'])
16e56d25 VF	200	else:
	201	expected = suite[jobnum]['timeout'] * 1000
	202	actual = job['read']['runtime']
	203	if abs(expected - actual) < 10:
	204	result = 'PASSED '
	205	else:
	206	result = 'FAILED '
	207	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
	208	print line
dd4e8700 VF	209	if 'steadystate' in job:
dd4e8700 VF	210	pp.pprint(job['steadystate'])
16e56d25 VF	211	jobnum += 1
16e56d25 VF	212	suitenum += 1