[fio.git] / t / steadystate_tests.py

#!/usr/bin/env python
# Note: this script is python2 and python3 compatible.
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# On Windows this script works under Cygwin but not from cmd.exe
# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

from __future__ import absolute_import
from __future__ import print_function
import os
import sys
import json
import pprint
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio', help='path to fio executable')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = list(range(len(data)))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = (m / mean * 100) if mean != 0 else 0
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    if criterion == 0.0:
        objsame = False
    else:
        objsame = abs(target - criterion) / criterion < 0.005
    return (objsame, target < limit, mean, target)


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

    passed = 0
    failed = 0

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args'])
        if test['output'] in output.decode():
            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
            passed = passed + 1
        else:
            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
            failed = failed + 1

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': False, 'timeout': 20, 'numjobs': 2},
              {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
            ]

    jobnum = 0
    for job in reads:

        tf = "steadystate_job{0}.json".format(jobnum)
        parameters = [ "--name=job{0}".format(jobnum) ]
        parameters.extend([ "--thread",
                            "--output-format=json",
                            "--output={0}".format(tf),
                            "--ioengine=null",
                            "--size=1G",
                            "--rw=randrw",
                            "--rwmixread=100",
                            "--stonewall",
                            "--group_reporting",
                            "--numjobs={0}".format(job['numjobs']),
                            "--time_based",
                            "--runtime={0}".format(job['timeout']) ])
        if job['s']:
           if job['iops']:
               ss = 'iops'
           else:
               ss = 'bw'
           if job['slope']:
               ss += "_slope"
           ss += ":" + str(job['ss_limit'])
           if job['pct']:
               ss += '%'
           parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                               '--ss={0}'.format(ss),
                               '--ss_ramp={0}'.format(job['ss_ramp']) ])

        output = subprocess.call([args.fio] + parameters)
        with open(tf, 'r') as source:
            jsondata = json.loads(source.read())
            source.close()

        for jsonjob in jsondata['jobs']:
            line = "{0}".format(jsonjob['job options']['name'])
            if job['s']:
                if jsonjob['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
                    actual = jsonjob['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
                        failed = failed + 1
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                            failed = failed + 1
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = job['timeout'] * 1000
                    actual = jsonjob['read']['runtime']
                    if abs(expected - actual) > 10:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
                                passed = passed + 1
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
            else:
                expected = job['timeout'] * 1000
                actual = jsonjob['read']['runtime']
                if abs(expected - actual) < 10:
                    result = 'PASSED '
                    passed = passed + 1
                else:
                    result = 'FAILED '
                    failed = failed + 1
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print(line)
            if 'steadystate' in jsonjob:
                pp.pprint(jsonjob['steadystate'])
        jobnum += 1

    print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
    sys.exit(failed)
Commit	Line	Data
	1	#!/usr/bin/env python
	2	# Note: this script is python2 and python3 compatible.
	3	#
	4	# steadystate_tests.py
	5	#
	6	# Test option parsing and functonality for fio's steady state detection feature.
	7	#
	8	# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
	9	#
	10	# REQUIREMENTS
	11	# Python 2.6+
	12	# SciPy
	13	#
	14	# KNOWN ISSUES
	15	# only option parsing and read tests are carried out
	16	# On Windows this script works under Cygwin but not from cmd.exe
	17	# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
	18	# min runtime:
	19	# if ss attained: min runtime = ss_dur + ss_ramp
	20	# if not attained: runtime = timeout
	21
	22	from __future__ import absolute_import
	23	from __future__ import print_function
	24	import os
	25	import sys
	26	import json
	27	import pprint
	28	import argparse
	29	import subprocess
	30	from scipy import stats
	31
	32	def parse_args():
	33	parser = argparse.ArgumentParser()
	34	parser.add_argument('fio', help='path to fio executable')
	35	args = parser.parse_args()
	36
	37	return args
	38
	39
	40	def check(data, iops, slope, pct, limit, dur, criterion):
	41	measurement = 'iops' if iops else 'bw'
	42	data = data[measurement]
	43	mean = sum(data) / len(data)
	44	if slope:
	45	x = list(range(len(data)))
	46	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	47	m = abs(m)
	48	if pct:
	49	target = (m / mean * 100) if mean != 0 else 0
	50	criterion = criterion[:-1]
	51	else:
	52	target = m
	53	else:
	54	maxdev = 0
	55	for x in data:
	56	maxdev = max(abs(mean-x), maxdev)
	57	if pct:
	58	target = maxdev / mean * 100
	59	criterion = criterion[:-1]
	60	else:
	61	target = maxdev
	62
	63	criterion = float(criterion)
	64	if criterion == 0.0:
	65	objsame = False
	66	else:
	67	objsame = abs(target - criterion) / criterion < 0.005
	68	return (objsame, target < limit, mean, target)
	69
	70
	71	if __name__ == '__main__':
	72	args = parse_args()
	73
	74	pp = pprint.PrettyPrinter(indent=4)
	75
	76	passed = 0
	77	failed = 0
	78
	79	#
	80	# test option parsing
	81	#
	82	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
	83	'output': "set steady state IOPS threshold to 10.000000" },
	84	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
	85	'output': "set steady state threshold to 10.000000%" },
	86	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
	87	'output': "set steady state threshold to 0.100000%" },
	88	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
	89	'output': "set steady state threshold to 10.000000%" },
	90	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
	91	'output': "set steady state threshold to 0.100000%" },
	92	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
	93	'output': "set steady state BW threshold to 12" },
	94	]
	95	for test in parsing:
	96	output = subprocess.check_output([args.fio] + test['args'])
	97	if test['output'] in output.decode():
	98	print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
	99	passed = passed + 1
	100	else:
	101	print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
	102	failed = failed + 1
	103
	104	#
	105	# test some read workloads
	106	#
	107	# if ss active and attained,
	108	# check that runtime is less than job time
	109	# check criteria
	110	# how to check ramp time?
	111	#
	112	# if ss inactive
	113	# check that runtime is what was specified
	114	#
	115	reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	116	{'s': False, 'timeout': 20, 'numjobs': 2},
	117	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	118	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	119	]
	120
	121	jobnum = 0
	122	for job in reads:
	123
	124	tf = "steadystate_job{0}.json".format(jobnum)
	125	parameters = [ "--name=job{0}".format(jobnum) ]
	126	parameters.extend([ "--thread",
	127	"--output-format=json",
	128	"--output={0}".format(tf),
	129	"--ioengine=null",
	130	"--size=1G",
	131	"--rw=randrw",
	132	"--rwmixread=100",
	133	"--stonewall",
	134	"--group_reporting",
	135	"--numjobs={0}".format(job['numjobs']),
	136	"--time_based",
	137	"--runtime={0}".format(job['timeout']) ])
	138	if job['s']:
	139	if job['iops']:
	140	ss = 'iops'
	141	else:
	142	ss = 'bw'
	143	if job['slope']:
	144	ss += "_slope"
	145	ss += ":" + str(job['ss_limit'])
	146	if job['pct']:
	147	ss += '%'
	148	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	149	'--ss={0}'.format(ss),
	150	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	151
	152	output = subprocess.call([args.fio] + parameters)
	153	with open(tf, 'r') as source:
	154	jsondata = json.loads(source.read())
	155	source.close()
	156
	157	for jsonjob in jsondata['jobs']:
	158	line = "{0}".format(jsonjob['job options']['name'])
	159	if job['s']:
	160	if jsonjob['steadystate']['attained'] == 1:
	161	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
	162	mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
	163	actual = jsonjob['read']['runtime']
	164	if mintime > actual:
	165	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
	166	failed = failed + 1
	167	else:
	168	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	169	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	170	iops=job['iops'],
	171	slope=job['slope'],
	172	pct=job['pct'],
	173	limit=job['ss_limit'],
	174	dur=job['ss_dur'],
	175	criterion=jsonjob['steadystate']['criterion'])
	176	if not objsame:
	177	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
	178	failed = failed + 1
	179	else:
	180	if met:
	181	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
	182	passed = passed + 1
	183	else:
	184	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
	185	failed = failed + 1
	186	else:
	187	# check runtime, confirm criterion calculation, and confirm that criterion was not met
	188	expected = job['timeout'] * 1000
	189	actual = jsonjob['read']['runtime']
	190	if abs(expected - actual) > 10:
	191	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
	192	else:
	193	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	194	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	195	iops=job['iops'],
	196	slope=job['slope'],
	197	pct=job['pct'],
	198	limit=job['ss_limit'],
	199	dur=job['ss_dur'],
	200	criterion=jsonjob['steadystate']['criterion'])
	201	if not objsame:
	202	if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
	203	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
	204	failed = failed + 1
	205	else:
	206	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
	207	passed = passed + 1
	208	else:
	209	if met:
	210	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
	211	failed = failed + 1
	212	else:
	213	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
	214	passed = passed + 1
	215	else:
	216	expected = job['timeout'] * 1000
	217	actual = jsonjob['read']['runtime']
	218	if abs(expected - actual) < 10:
	219	result = 'PASSED '
	220	passed = passed + 1
	221	else:
	222	result = 'FAILED '
	223	failed = failed + 1
	224	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
	225	print(line)
	226	if 'steadystate' in jsonjob:
	227	pp.pprint(jsonjob['steadystate'])
	228	jobnum += 1
	229
	230	print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
	231	sys.exit(failed)