[fio.git] / unit_tests / steadystate_tests.py

#!/usr/bin/python
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that
#     it does not match what fio reports
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

import os
import sys
import json
import pprint
import tempfile
import platform
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio',
                        help='path to fio executable');
    parser.add_argument('--read',
                        help='target for read testing')
    parser.add_argument('--write',
                        help='target for write testing')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = range(len(data))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = m / mean * 100
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args']);
        if test['output'] in output:
            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
        else:
            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
                {'s': False, 'timeout': 20, 'numjobs': 2},
                {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
                {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ],
            ]

    accum = []
    suitenum = 0
    for suite in reads:
        jobnum = 0
        for job in suite:
            if args.read == None:
                if os.name == 'posix':
                    args.read = '/dev/zero'
                    parameters = [ "--size=128M" ]
                else:
                    print "ERROR: file for read testing must be specified on non-posix systems"
                    sys.exit(1)
            else:
                parameters = []

            parameters.extend([ "--name=job{0}".format(jobnum),
                                "--thread",
                                "--filename={0}".format(args.read),
                                "--rw=randrw", "--rwmixread=100", "--stonewall",
                                "--group_reporting", "--numjobs={0}".format(job['numjobs']),
                                "--time_based", "--runtime={0}".format(job['timeout']) ])
            if job['s']:
               if job['iops']:
                   ss = 'iops'
               else:
                   ss = 'bw'
               if job['slope']:
                   ss += "_slope"
               ss += ":" + str(job['ss_limit'])
               if job['pct']:
                   ss += '%'
               parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                                   '--ss={0}'.format(ss),
                                   '--ss_ramp={0}'.format(job['ss_ramp']) ])
            accum.extend(parameters)
            jobnum += 1

        tf = tempfile.NamedTemporaryFile(delete=False)
	tf.close()
        output = subprocess.check_output([args.fio,
                                          "--output-format=json",
                                          "--output={0}".format(tf.name)] + accum)
        with open(tf.name, 'r') as source:
            jsondata = json.loads(source.read())
        os.remove(tf.name)
        jobnum = 0
        for job in jsondata['jobs']:
            line = "suite {0}, {1}".format(suitenum, job['job options']['name'])
            if suite[jobnum]['s']:
                if job['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000
                    actual = job['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                        objsame, met, mean, target = check(data=job['steadystate']['data'],
                            iops=suite[jobnum]['iops'],
                            slope=suite[jobnum]['slope'],
                            pct=suite[jobnum]['pct'],
                            limit=suite[jobnum]['ss_limit'],
                            dur=suite[jobnum]['ss_dur'],
                            criterion=job['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, suite[jobnum]['ss_limit'])
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = suite[jobnum]['timeout'] * 1000
                    actual = job['read']['runtime']
                    if abs(expected - actual) > 10:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                        objsame, met, mean, target = check(data=job['steadystate']['data'],
                            iops=suite[jobnum]['iops'],
                            slope=suite[jobnum]['slope'],
                            pct=suite[jobnum]['pct'],
                            limit=suite[jobnum]['ss_limit'],
                            dur=suite[jobnum]['ss_dur'],
                            criterion=job['steadystate']['criterion'])
                        if not objsame:
                            if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(job['steadystate']['criterion'])
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, suite[jobnum]['ss_limit'])
            else:
                expected = suite[jobnum]['timeout'] * 1000
                actual = job['read']['runtime']
                if abs(expected - actual) < 10:
                    result = 'PASSED '
                else:
                    result = 'FAILED '
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print line
            if 'steadystate' in job:
                pp.pprint(job['steadystate'])
            jobnum += 1
        suitenum += 1
Commit	Line	Data
	1	#!/usr/bin/python
	2	#
	3	# steadystate_tests.py
	4	#
	5	# Test option parsing and functonality for fio's steady state detection feature.
	6	#
	7	# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
	8	#
	9	# REQUIREMENTS
	10	# Python 2.6+
	11	# SciPy
	12	#
	13	# KNOWN ISSUES
	14	# only option parsing and read tests are carried out
	15	# the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that
	16	# it does not match what fio reports
	17	# min runtime:
	18	# if ss attained: min runtime = ss_dur + ss_ramp
	19	# if not attained: runtime = timeout
	20
	21	import os
	22	import sys
	23	import json
	24	import pprint
	25	import tempfile
	26	import platform
	27	import argparse
	28	import subprocess
	29	from scipy import stats
	30
	31	def parse_args():
	32	parser = argparse.ArgumentParser()
	33	parser.add_argument('fio',
	34	help='path to fio executable');
	35	parser.add_argument('--read',
	36	help='target for read testing')
	37	parser.add_argument('--write',
	38	help='target for write testing')
	39	args = parser.parse_args()
	40
	41	return args
	42
	43
	44	def check(data, iops, slope, pct, limit, dur, criterion):
	45	measurement = 'iops' if iops else 'bw'
	46	data = data[measurement]
	47	mean = sum(data) / len(data)
	48	if slope:
	49	x = range(len(data))
	50	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	51	m = abs(m)
	52	if pct:
	53	target = m / mean * 100
	54	criterion = criterion[:-1]
	55	else:
	56	target = m
	57	else:
	58	maxdev = 0
	59	for x in data:
	60	maxdev = max(abs(mean-x), maxdev)
	61	if pct:
	62	target = maxdev / mean * 100
	63	criterion = criterion[:-1]
	64	else:
	65	target = maxdev
	66
	67	criterion = float(criterion)
	68	return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
	69
	70
	71	if __name__ == '__main__':
	72	args = parse_args()
	73
	74	pp = pprint.PrettyPrinter(indent=4)
	75
	76	#
	77	# test option parsing
	78	#
	79	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
	80	'output': "set steady state IOPS threshold to 10.000000" },
	81	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
	82	'output': "set steady state threshold to 10.000000%" },
	83	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
	84	'output': "set steady state threshold to 0.100000%" },
	85	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
	86	'output': "set steady state threshold to 10.000000%" },
	87	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
	88	'output': "set steady state threshold to 0.100000%" },
	89	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
	90	'output': "set steady state BW threshold to 12" },
	91	]
	92	for test in parsing:
	93	output = subprocess.check_output([args.fio] + test['args']);
	94	if test['output'] in output:
	95	print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
	96	else:
	97	print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
	98
	99	#
	100	# test some read workloads
	101	#
	102	# if ss active and attained,
	103	# check that runtime is less than job time
	104	# check criteria
	105	# how to check ramp time?
	106	#
	107	# if ss inactive
	108	# check that runtime is what was specified
	109	#
	110	reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	111	{'s': False, 'timeout': 20, 'numjobs': 2},
	112	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	113	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ],
	114	]
	115
	116	accum = []
	117	suitenum = 0
	118	for suite in reads:
	119	jobnum = 0
	120	for job in suite:
	121	if args.read == None:
	122	if os.name == 'posix':
	123	args.read = '/dev/zero'
	124	parameters = [ "--size=128M" ]
	125	else:
	126	print "ERROR: file for read testing must be specified on non-posix systems"
	127	sys.exit(1)
	128	else:
	129	parameters = []
	130
	131	parameters.extend([ "--name=job{0}".format(jobnum),
	132	"--thread",
	133	"--filename={0}".format(args.read),
	134	"--rw=randrw", "--rwmixread=100", "--stonewall",
	135	"--group_reporting", "--numjobs={0}".format(job['numjobs']),
	136	"--time_based", "--runtime={0}".format(job['timeout']) ])
	137	if job['s']:
	138	if job['iops']:
	139	ss = 'iops'
	140	else:
	141	ss = 'bw'
	142	if job['slope']:
	143	ss += "_slope"
	144	ss += ":" + str(job['ss_limit'])
	145	if job['pct']:
	146	ss += '%'
	147	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	148	'--ss={0}'.format(ss),
	149	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	150	accum.extend(parameters)
	151	jobnum += 1
	152
	153	tf = tempfile.NamedTemporaryFile(delete=False)
	154	tf.close()
	155	output = subprocess.check_output([args.fio,
	156	"--output-format=json",
	157	"--output={0}".format(tf.name)] + accum)
	158	with open(tf.name, 'r') as source:
	159	jsondata = json.loads(source.read())
	160	os.remove(tf.name)
	161	jobnum = 0
	162	for job in jsondata['jobs']:
	163	line = "suite {0}, {1}".format(suitenum, job['job options']['name'])
	164	if suite[jobnum]['s']:
	165	if job['steadystate']['attained'] == 1:
	166	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
	167	mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000
	168	actual = job['read']['runtime']
	169	if mintime > actual:
	170	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
	171	else:
	172	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
	173	objsame, met, mean, target = check(data=job['steadystate']['data'],
	174	iops=suite[jobnum]['iops'],
	175	slope=suite[jobnum]['slope'],
	176	pct=suite[jobnum]['pct'],
	177	limit=suite[jobnum]['ss_limit'],
	178	dur=suite[jobnum]['ss_dur'],
	179	criterion=job['steadystate']['criterion'])
	180	if not objsame:
	181	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
	182	else:
	183	if met:
	184	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, suite[jobnum]['ss_limit'])
	185	else:
	186	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
	187	else:
	188	# check runtime, confirm criterion calculation, and confirm that criterion was not met
	189	expected = suite[jobnum]['timeout'] * 1000
	190	actual = job['read']['runtime']
	191	if abs(expected - actual) > 10:
	192	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
	193	else:
	194	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
	195	objsame, met, mean, target = check(data=job['steadystate']['data'],
	196	iops=suite[jobnum]['iops'],
	197	slope=suite[jobnum]['slope'],
	198	pct=suite[jobnum]['pct'],
	199	limit=suite[jobnum]['ss_limit'],
	200	dur=suite[jobnum]['ss_dur'],
	201	criterion=job['steadystate']['criterion'])
	202	if not objsame:
	203	if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
	204	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
	205	else:
	206	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(job['steadystate']['criterion'])
	207	else:
	208	if met:
	209	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
	210	else:
	211	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, suite[jobnum]['ss_limit'])
	212	else:
	213	expected = suite[jobnum]['timeout'] * 1000
	214	actual = job['read']['runtime']
	215	if abs(expected - actual) < 10:
	216	result = 'PASSED '
	217	else:
	218	result = 'FAILED '
	219	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
	220	print line
	221	if 'steadystate' in job:
	222	pp.pprint(job['steadystate'])
	223	jobnum += 1
	224	suitenum += 1