[fio.git] / unit_tests / steadystate_tests.py

#!/usr/bin/python
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# On Windows this script works under Cygwin but not from cmd.exe
# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

import os
import sys
import json
import uuid
import pprint
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio',
                        help='path to fio executable');
    parser.add_argument('--read',
                        help='target for read testing')
    parser.add_argument('--write',
                        help='target for write testing')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = range(len(data))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = m / mean * 100
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args']);
        if test['output'] in output:
            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
        else:
            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': False, 'timeout': 20, 'numjobs': 2},
              {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
            ]

    if args.read == None:
        if os.name == 'posix':
            args.read = '/dev/zero'
            extra = [ "--size=134217728" ]  # 128 MiB
        else:
            print "ERROR: file for read testing must be specified on non-posix systems"
            sys.exit(1)
    else:
        extra = []

    jobnum = 0
    for job in reads:

        tf = uuid.uuid4().hex
        parameters = [ "--name=job{0}".format(jobnum) ]
        parameters.extend(extra)
        parameters.extend([ "--thread",
                            "--output-format=json",
                            "--output={0}".format(tf),
                            "--filename={0}".format(args.read),
                            "--rw=randrw",
                            "--rwmixread=100",
                            "--stonewall",
                            "--group_reporting",
                            "--numjobs={0}".format(job['numjobs']),
                            "--time_based",
                            "--runtime={0}".format(job['timeout']) ])
        if job['s']:
           if job['iops']:
               ss = 'iops'
           else:
               ss = 'bw'
           if job['slope']:
               ss += "_slope"
           ss += ":" + str(job['ss_limit'])
           if job['pct']:
               ss += '%'
           parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                               '--ss={0}'.format(ss),
                               '--ss_ramp={0}'.format(job['ss_ramp']) ])

        output = subprocess.call([args.fio] + parameters)
        with open(tf, 'r') as source:
            jsondata = json.loads(source.read())
        os.remove(tf)

        for jsonjob in jsondata['jobs']:
            line = "job {0}".format(jsonjob['job options']['name'])
            if job['s']:
                if jsonjob['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
                    actual = jsonjob['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = job['timeout'] * 1000
                    actual = jsonjob['read']['runtime']
                    if abs(expected - actual) > 10:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
            else:
                expected = job['timeout'] * 1000
                actual = jsonjob['read']['runtime']
                if abs(expected - actual) < 10:
                    result = 'PASSED '
                else:
                    result = 'FAILED '
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print line
            if 'steadystate' in jsonjob:
                pp.pprint(jsonjob['steadystate'])
        jobnum += 1
Commit	Line	Data
16e56d25 VF	1	#!/usr/bin/python
	2	#
	3	# steadystate_tests.py
	4	#
	5	# Test option parsing and functonality for fio's steady state detection feature.
	6	#
	7	# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
	8	#
	9	# REQUIREMENTS
	10	# Python 2.6+
	11	# SciPy
	12	#
	13	# KNOWN ISSUES
	14	# only option parsing and read tests are carried out
32df42c9 VF	15	# On Windows this script works under Cygwin but not from cmd.exe
32df42c9 VF	16	# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
16e56d25 VF	17	# min runtime:
	18	# if ss attained: min runtime = ss_dur + ss_ramp
	19	# if not attained: runtime = timeout
	20
	21	import os
39f5379e	22	import sys
16e56d25	23	import json
32df42c9	24	import uuid
dd4e8700	25	import pprint
16e56d25 VF	26	import argparse
	27	import subprocess
	28	from scipy import stats
	29
	30	def parse_args():
	31	parser = argparse.ArgumentParser()
	32	parser.add_argument('fio',
	33	help='path to fio executable');
39f5379e	34	parser.add_argument('--read',
16e56d25	35	help='target for read testing')
39f5379e	36	parser.add_argument('--write',
16e56d25 VF	37	help='target for write testing')
	38	args = parser.parse_args()
	39
	40	return args
	41
	42
	43	def check(data, iops, slope, pct, limit, dur, criterion):
ba8fb6f6 VF	44	measurement = 'iops' if iops else 'bw'
ba8fb6f6 VF	45	data = data[measurement]
16e56d25 VF	46	mean = sum(data) / len(data)
	47	if slope:
	48	x = range(len(data))
	49	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	50	m = abs(m)
	51	if pct:
4001d829 VF	52	target = m / mean * 100
4001d829 VF	53	criterion = criterion[:-1]
16e56d25 VF	54	else:
	55	target = m
	56	else:
	57	maxdev = 0
	58	for x in data:
	59	maxdev = max(abs(mean-x), maxdev)
	60	if pct:
4001d829 VF	61	target = maxdev / mean * 100
4001d829 VF	62	criterion = criterion[:-1]
16e56d25 VF	63	else:
	64	target = maxdev
	65
4001d829	66	criterion = float(criterion)
ba8fb6f6	67	return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
16e56d25 VF	68
	69
	70	if __name__ == '__main__':
	71	args = parse_args()
	72
dd4e8700 VF	73	pp = pprint.PrettyPrinter(indent=4)
dd4e8700 VF	74
16e56d25 VF	75	#
	76	# test option parsing
	77	#
bfc4884e	78	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
16e56d25	79	'output': "set steady state IOPS threshold to 10.000000" },
bfc4884e	80	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
16e56d25	81	'output': "set steady state threshold to 10.000000%" },
bfc4884e	82	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
16e56d25	83	'output': "set steady state threshold to 0.100000%" },
bfc4884e	84	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
16e56d25	85	'output': "set steady state threshold to 10.000000%" },
bfc4884e	86	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
16e56d25	87	'output': "set steady state threshold to 0.100000%" },
bfc4884e	88	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
16e56d25 VF	89	'output': "set steady state BW threshold to 12" },
	90	]
	91	for test in parsing:
	92	output = subprocess.check_output([args.fio] + test['args']);
	93	if test['output'] in output:
	94	print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
	95	else:
	96	print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
	97
	98	#
	99	# test some read workloads
	100	#
	101	# if ss active and attained,
	102	# check that runtime is less than job time
	103	# check criteria
	104	# how to check ramp time?
	105	#
	106	# if ss inactive
	107	# check that runtime is what was specified
	108	#
32df42c9 VF	109	reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	110	{'s': False, 'timeout': 20, 'numjobs': 2},
	111	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	112	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
16e56d25 VF	113	]
16e56d25 VF	114
32df42c9 VF	115	if args.read == None:
	116	if os.name == 'posix':
	117	args.read = '/dev/zero'
2b9136a3	118	extra = [ "--size=134217728" ] # 128 MiB
32df42c9 VF	119	else:
	120	print "ERROR: file for read testing must be specified on non-posix systems"
	121	sys.exit(1)
	122	else:
	123	extra = []
	124
	125	jobnum = 0
	126	for job in reads:
	127
	128	tf = uuid.uuid4().hex
	129	parameters = [ "--name=job{0}".format(jobnum) ]
	130	parameters.extend(extra)
	131	parameters.extend([ "--thread",
	132	"--output-format=json",
	133	"--output={0}".format(tf),
	134	"--filename={0}".format(args.read),
	135	"--rw=randrw",
	136	"--rwmixread=100",
	137	"--stonewall",
	138	"--group_reporting",
	139	"--numjobs={0}".format(job['numjobs']),
	140	"--time_based",
	141	"--runtime={0}".format(job['timeout']) ])
	142	if job['s']:
	143	if job['iops']:
	144	ss = 'iops'
	145	else:
	146	ss = 'bw'
	147	if job['slope']:
	148	ss += "_slope"
	149	ss += ":" + str(job['ss_limit'])
	150	if job['pct']:
	151	ss += '%'
	152	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	153	'--ss={0}'.format(ss),
	154	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	155
	156	output = subprocess.call([args.fio] + parameters)
	157	with open(tf, 'r') as source:
16e56d25	158	jsondata = json.loads(source.read())
32df42c9 VF	159	os.remove(tf)
	160
	161	for jsonjob in jsondata['jobs']:
	162	line = "job {0}".format(jsonjob['job options']['name'])
	163	if job['s']:
	164	if jsonjob['steadystate']['attained'] == 1:
16e56d25	165	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
32df42c9 VF	166	mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
32df42c9 VF	167	actual = jsonjob['read']['runtime']
16e56d25	168	if mintime > actual:
32df42c9	169	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
16e56d25	170	else:
32df42c9 VF	171	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	172	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	173	iops=job['iops'],
	174	slope=job['slope'],
	175	pct=job['pct'],
	176	limit=job['ss_limit'],
	177	dur=job['ss_dur'],
	178	criterion=jsonjob['steadystate']['criterion'])
16e56d25	179	if not objsame:
32df42c9	180	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
16e56d25 VF	181	else:
16e56d25 VF	182	if met:
32df42c9	183	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
16e56d25	184	else:
32df42c9	185	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
16e56d25 VF	186	else:
16e56d25 VF	187	# check runtime, confirm criterion calculation, and confirm that criterion was not met
32df42c9 VF	188	expected = job['timeout'] * 1000
32df42c9 VF	189	actual = jsonjob['read']['runtime']
16e56d25 VF	190	if abs(expected - actual) > 10:
	191	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
	192	else:
32df42c9 VF	193	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	194	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	195	iops=job['iops'],
	196	slope=job['slope'],
	197	pct=job['pct'],
	198	limit=job['ss_limit'],
	199	dur=job['ss_dur'],
	200	criterion=jsonjob['steadystate']['criterion'])
16e56d25	201	if not objsame:
32df42c9 VF	202	if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
32df42c9 VF	203	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
16e56d25	204	else:
32df42c9	205	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
16e56d25 VF	206	else:
16e56d25 VF	207	if met:
32df42c9	208	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
16e56d25	209	else:
32df42c9	210	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
16e56d25	211	else:
32df42c9 VF	212	expected = job['timeout'] * 1000
32df42c9 VF	213	actual = jsonjob['read']['runtime']
16e56d25 VF	214	if abs(expected - actual) < 10:
	215	result = 'PASSED '
	216	else:
	217	result = 'FAILED '
	218	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
	219	print line
32df42c9 VF	220	if 'steadystate' in jsonjob:
	221	pp.pprint(jsonjob['steadystate'])
	222	jobnum += 1