[fio.git] / t / steadystate_tests.py

#!/usr/bin/env python
# Note: this script is python2 and python3 compatible.
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# On Windows this script works under Cygwin but not from cmd.exe
# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

from __future__ import absolute_import
from __future__ import print_function
import os
import sys
import json
import pprint
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio', help='path to fio executable')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = list(range(len(data)))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = (m / mean * 100) if mean != 0 else 0
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    if criterion == 0.0:
        objsame = False
    else:
        objsame = abs(target - criterion) / criterion < 0.005
    return (objsame, target < limit, mean, target)


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

    passed = 0
    failed = 0

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args'])
        if test['output'] in output.decode():
            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
            passed = passed + 1
        else:
            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
            failed = failed + 1

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': False, 'timeout': 20, 'numjobs': 2},
              {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
            ]

    jobnum = 0
    for job in reads:

        tf = "steadystate_job{0}.json".format(jobnum)
        parameters = [ "--name=job{0}".format(jobnum) ]
        parameters.extend([ "--thread",
                            "--output-format=json",
                            "--output={0}".format(tf),
                            "--ioengine=null",
                            "--size=1G",
                            "--rw=randrw",
                            "--rwmixread=100",
                            "--stonewall",
                            "--group_reporting",
                            "--numjobs={0}".format(job['numjobs']),
                            "--time_based",
                            "--runtime={0}".format(job['timeout']) ])
        if job['s']:
           if job['iops']:
               ss = 'iops'
           else:
               ss = 'bw'
           if job['slope']:
               ss += "_slope"
           ss += ":" + str(job['ss_limit'])
           if job['pct']:
               ss += '%'
           parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                               '--ss={0}'.format(ss),
                               '--ss_ramp={0}'.format(job['ss_ramp']) ])

        output = subprocess.call([args.fio] + parameters)
        with open(tf, 'r') as source:
            jsondata = json.loads(source.read())
            source.close()

        for jsonjob in jsondata['jobs']:
            line = "{0}".format(jsonjob['job options']['name'])
            if job['s']:
                if jsonjob['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
                    actual = jsonjob['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
                        failed = failed + 1
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                            failed = failed + 1
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = job['timeout'] * 1000
                    actual = jsonjob['read']['runtime']
                    if abs(expected - actual) > 50:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
                                passed = passed + 1
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
            else:
                expected = job['timeout'] * 1000
                actual = jsonjob['read']['runtime']
                if abs(expected - actual) > 50:
                    result = 'FAILED '
                    failed = failed + 1
                else:
                    result = 'PASSED '
                    passed = passed + 1
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print(line)
            if 'steadystate' in jsonjob:
                pp.pprint(jsonjob['steadystate'])
        jobnum += 1

    print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
    sys.exit(failed)
Commit	Line	Data
cd22f801 VF	1	#!/usr/bin/env python
cd22f801 VF	2	# Note: this script is python2 and python3 compatible.
16e56d25 VF	3	#
	4	# steadystate_tests.py
	5	#
	6	# Test option parsing and functonality for fio's steady state detection feature.
	7	#
a72f40bb	8	# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
16e56d25 VF	9	#
	10	# REQUIREMENTS
	11	# Python 2.6+
	12	# SciPy
	13	#
	14	# KNOWN ISSUES
	15	# only option parsing and read tests are carried out
32df42c9 VF	16	# On Windows this script works under Cygwin but not from cmd.exe
32df42c9 VF	17	# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
16e56d25 VF	18	# min runtime:
	19	# if ss attained: min runtime = ss_dur + ss_ramp
	20	# if not attained: runtime = timeout
	21
5eac3b00 BD	22	from __future__ import absolute_import
5eac3b00 BD	23	from __future__ import print_function
16e56d25	24	import os
39f5379e	25	import sys
16e56d25	26	import json
dd4e8700	27	import pprint
16e56d25 VF	28	import argparse
	29	import subprocess
	30	from scipy import stats
	31
	32	def parse_args():
	33	parser = argparse.ArgumentParser()
82d0ad06	34	parser.add_argument('fio', help='path to fio executable')
16e56d25 VF	35	args = parser.parse_args()
	36
	37	return args
	38
	39
	40	def check(data, iops, slope, pct, limit, dur, criterion):
ba8fb6f6 VF	41	measurement = 'iops' if iops else 'bw'
ba8fb6f6 VF	42	data = data[measurement]
16e56d25 VF	43	mean = sum(data) / len(data)
16e56d25 VF	44	if slope:
5eac3b00	45	x = list(range(len(data)))
16e56d25 VF	46	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	47	m = abs(m)
	48	if pct:
cd22f801	49	target = (m / mean * 100) if mean != 0 else 0
4001d829	50	criterion = criterion[:-1]
16e56d25 VF	51	else:
	52	target = m
	53	else:
	54	maxdev = 0
	55	for x in data:
	56	maxdev = max(abs(mean-x), maxdev)
	57	if pct:
4001d829 VF	58	target = maxdev / mean * 100
4001d829 VF	59	criterion = criterion[:-1]
16e56d25 VF	60	else:
	61	target = maxdev
	62
4001d829	63	criterion = float(criterion)
cd22f801 VF	64	if criterion == 0.0:
	65	objsame = False
	66	else:
	67	objsame = abs(target - criterion) / criterion < 0.005
	68	return (objsame, target < limit, mean, target)
16e56d25 VF	69
	70
	71	if __name__ == '__main__':
	72	args = parse_args()
	73
dd4e8700 VF	74	pp = pprint.PrettyPrinter(indent=4)
dd4e8700 VF	75
cd22f801 VF	76	passed = 0
	77	failed = 0
	78
16e56d25 VF	79	#
	80	# test option parsing
	81	#
bfc4884e	82	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
16e56d25	83	'output': "set steady state IOPS threshold to 10.000000" },
bfc4884e	84	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
16e56d25	85	'output': "set steady state threshold to 10.000000%" },
bfc4884e	86	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
16e56d25	87	'output': "set steady state threshold to 0.100000%" },
bfc4884e	88	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
16e56d25	89	'output': "set steady state threshold to 10.000000%" },
bfc4884e	90	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
16e56d25	91	'output': "set steady state threshold to 0.100000%" },
bfc4884e	92	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
16e56d25 VF	93	'output': "set steady state BW threshold to 12" },
	94	]
	95	for test in parsing:
5eac3b00 BD	96	output = subprocess.check_output([args.fio] + test['args'])
	97	if test['output'] in output.decode():
	98	print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
cd22f801	99	passed = passed + 1
16e56d25	100	else:
5eac3b00	101	print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
cd22f801	102	failed = failed + 1
16e56d25 VF	103
	104	#
	105	# test some read workloads
	106	#
	107	# if ss active and attained,
	108	# check that runtime is less than job time
	109	# check criteria
	110	# how to check ramp time?
	111	#
	112	# if ss inactive
	113	# check that runtime is what was specified
	114	#
32df42c9 VF	115	reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	116	{'s': False, 'timeout': 20, 'numjobs': 2},
	117	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	118	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
16e56d25 VF	119	]
16e56d25 VF	120
32df42c9 VF	121	jobnum = 0
	122	for job in reads:
	123
cd22f801	124	tf = "steadystate_job{0}.json".format(jobnum)
32df42c9	125	parameters = [ "--name=job{0}".format(jobnum) ]
32df42c9 VF	126	parameters.extend([ "--thread",
	127	"--output-format=json",
	128	"--output={0}".format(tf),
82d0ad06 VF	129	"--ioengine=null",
82d0ad06 VF	130	"--size=1G",
32df42c9 VF	131	"--rw=randrw",
	132	"--rwmixread=100",
	133	"--stonewall",
	134	"--group_reporting",
	135	"--numjobs={0}".format(job['numjobs']),
	136	"--time_based",
	137	"--runtime={0}".format(job['timeout']) ])
	138	if job['s']:
	139	if job['iops']:
	140	ss = 'iops'
	141	else:
	142	ss = 'bw'
	143	if job['slope']:
	144	ss += "_slope"
	145	ss += ":" + str(job['ss_limit'])
	146	if job['pct']:
	147	ss += '%'
	148	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	149	'--ss={0}'.format(ss),
	150	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	151
	152	output = subprocess.call([args.fio] + parameters)
	153	with open(tf, 'r') as source:
16e56d25	154	jsondata = json.loads(source.read())
cd22f801	155	source.close()
32df42c9 VF	156
32df42c9 VF	157	for jsonjob in jsondata['jobs']:
cd22f801	158	line = "{0}".format(jsonjob['job options']['name'])
32df42c9 VF	159	if job['s']:
32df42c9 VF	160	if jsonjob['steadystate']['attained'] == 1:
16e56d25	161	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
32df42c9 VF	162	mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
32df42c9 VF	163	actual = jsonjob['read']['runtime']
16e56d25	164	if mintime > actual:
32df42c9	165	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
cd22f801	166	failed = failed + 1
16e56d25	167	else:
32df42c9 VF	168	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	169	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	170	iops=job['iops'],
	171	slope=job['slope'],
	172	pct=job['pct'],
	173	limit=job['ss_limit'],
	174	dur=job['ss_dur'],
	175	criterion=jsonjob['steadystate']['criterion'])
16e56d25	176	if not objsame:
32df42c9	177	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
cd22f801	178	failed = failed + 1
16e56d25 VF	179	else:
16e56d25 VF	180	if met:
32df42c9	181	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
cd22f801	182	passed = passed + 1
16e56d25	183	else:
32df42c9	184	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
cd22f801	185	failed = failed + 1
16e56d25 VF	186	else:
16e56d25 VF	187	# check runtime, confirm criterion calculation, and confirm that criterion was not met
32df42c9 VF	188	expected = job['timeout'] * 1000
32df42c9 VF	189	actual = jsonjob['read']['runtime']
3a0be784	190	if abs(expected - actual) > 50:
16e56d25 VF	191	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
16e56d25 VF	192	else:
32df42c9 VF	193	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	194	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	195	iops=job['iops'],
	196	slope=job['slope'],
	197	pct=job['pct'],
	198	limit=job['ss_limit'],
	199	dur=job['ss_dur'],
	200	criterion=jsonjob['steadystate']['criterion'])
16e56d25	201	if not objsame:
32df42c9 VF	202	if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
32df42c9 VF	203	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
cd22f801	204	failed = failed + 1
16e56d25	205	else:
32df42c9	206	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
cd22f801	207	passed = passed + 1
16e56d25 VF	208	else:
16e56d25 VF	209	if met:
32df42c9	210	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
cd22f801	211	failed = failed + 1
16e56d25	212	else:
32df42c9	213	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
cd22f801	214	passed = passed + 1
16e56d25	215	else:
32df42c9 VF	216	expected = job['timeout'] * 1000
32df42c9 VF	217	actual = jsonjob['read']['runtime']
3a0be784	218	if abs(expected - actual) > 50:
16e56d25	219	result = 'FAILED '
cd22f801	220	failed = failed + 1
3a0be784 VF	221	else:
	222	result = 'PASSED '
	223	passed = passed + 1
16e56d25	224	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
5eac3b00	225	print(line)
32df42c9 VF	226	if 'steadystate' in jsonjob:
	227	pp.pprint(jsonjob['steadystate'])
	228	jobnum += 1
cd22f801 VF	229
	230	print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
	231	sys.exit(failed)