[fio.git] / t / steadystate_tests.py

#!/usr/bin/env python
# Note: this script is python2 and python3 compatible.
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# On Windows this script works under Cygwin but not from cmd.exe
# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

from __future__ import absolute_import
from __future__ import print_function
import os
import sys
import json
import pprint
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio',
                        help='path to fio executable')
    parser.add_argument('--read',
                        help='target for read testing')
    parser.add_argument('--write',
                        help='target for write testing')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = list(range(len(data)))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = (m / mean * 100) if mean != 0 else 0
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    if criterion == 0.0:
        objsame = False
    else:
        objsame = abs(target - criterion) / criterion < 0.005
    return (objsame, target < limit, mean, target)


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

    passed = 0
    failed = 0

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args'])
        if test['output'] in output.decode():
            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
            passed = passed + 1
        else:
            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
            failed = failed + 1

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': False, 'timeout': 20, 'numjobs': 2},
              {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
            ]

    if args.read == None:
        if os.name == 'posix':
            args.read = '/dev/zero'
            extra = [ "--size=128M" ]
        else:
            print("ERROR: file for read testing must be specified on non-posix systems")
            sys.exit(1)
    else:
        extra = []

    jobnum = 0
    for job in reads:

        tf = "steadystate_job{0}.json".format(jobnum)
        parameters = [ "--name=job{0}".format(jobnum) ]
        parameters.extend(extra)
        parameters.extend([ "--thread",
                            "--output-format=json",
                            "--output={0}".format(tf),
                            "--filename={0}".format(args.read),
                            "--rw=randrw",
                            "--rwmixread=100",
                            "--stonewall",
                            "--group_reporting",
                            "--numjobs={0}".format(job['numjobs']),
                            "--time_based",
                            "--runtime={0}".format(job['timeout']) ])
        if job['s']:
           if job['iops']:
               ss = 'iops'
           else:
               ss = 'bw'
           if job['slope']:
               ss += "_slope"
           ss += ":" + str(job['ss_limit'])
           if job['pct']:
               ss += '%'
           parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                               '--ss={0}'.format(ss),
                               '--ss_ramp={0}'.format(job['ss_ramp']) ])

        output = subprocess.call([args.fio] + parameters)
        with open(tf, 'r') as source:
            jsondata = json.loads(source.read())
            source.close()

        for jsonjob in jsondata['jobs']:
            line = "{0}".format(jsonjob['job options']['name'])
            if job['s']:
                if jsonjob['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
                    actual = jsonjob['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
                        failed = failed + 1
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                            failed = failed + 1
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = job['timeout'] * 1000
                    actual = jsonjob['read']['runtime']
                    if abs(expected - actual) > 10:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
                                passed = passed + 1
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
            else:
                expected = job['timeout'] * 1000
                actual = jsonjob['read']['runtime']
                if abs(expected - actual) < 10:
                    result = 'PASSED '
                    passed = passed + 1
                else:
                    result = 'FAILED '
                    failed = failed + 1
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print(line)
            if 'steadystate' in jsonjob:
                pp.pprint(jsonjob['steadystate'])
        jobnum += 1

    print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
    sys.exit(failed)
Commit	Line	Data
cd22f801 VF	1	#!/usr/bin/env python
cd22f801 VF	2	# Note: this script is python2 and python3 compatible.
16e56d25 VF	3	#
	4	# steadystate_tests.py
	5	#
	6	# Test option parsing and functonality for fio's steady state detection feature.
	7	#
a72f40bb	8	# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
16e56d25 VF	9	#
	10	# REQUIREMENTS
	11	# Python 2.6+
	12	# SciPy
	13	#
	14	# KNOWN ISSUES
	15	# only option parsing and read tests are carried out
32df42c9 VF	16	# On Windows this script works under Cygwin but not from cmd.exe
32df42c9 VF	17	# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
16e56d25 VF	18	# min runtime:
	19	# if ss attained: min runtime = ss_dur + ss_ramp
	20	# if not attained: runtime = timeout
	21
5eac3b00 BD	22	from __future__ import absolute_import
5eac3b00 BD	23	from __future__ import print_function
16e56d25	24	import os
39f5379e	25	import sys
16e56d25	26	import json
dd4e8700	27	import pprint
16e56d25 VF	28	import argparse
	29	import subprocess
	30	from scipy import stats
	31
	32	def parse_args():
	33	parser = argparse.ArgumentParser()
	34	parser.add_argument('fio',
5eac3b00	35	help='path to fio executable')
39f5379e	36	parser.add_argument('--read',
16e56d25	37	help='target for read testing')
39f5379e	38	parser.add_argument('--write',
16e56d25 VF	39	help='target for write testing')
	40	args = parser.parse_args()
	41
	42	return args
	43
	44
	45	def check(data, iops, slope, pct, limit, dur, criterion):
ba8fb6f6 VF	46	measurement = 'iops' if iops else 'bw'
ba8fb6f6 VF	47	data = data[measurement]
16e56d25 VF	48	mean = sum(data) / len(data)
16e56d25 VF	49	if slope:
5eac3b00	50	x = list(range(len(data)))
16e56d25 VF	51	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	52	m = abs(m)
	53	if pct:
cd22f801	54	target = (m / mean * 100) if mean != 0 else 0
4001d829	55	criterion = criterion[:-1]
16e56d25 VF	56	else:
	57	target = m
	58	else:
	59	maxdev = 0
	60	for x in data:
	61	maxdev = max(abs(mean-x), maxdev)
	62	if pct:
4001d829 VF	63	target = maxdev / mean * 100
4001d829 VF	64	criterion = criterion[:-1]
16e56d25 VF	65	else:
	66	target = maxdev
	67
4001d829	68	criterion = float(criterion)
cd22f801 VF	69	if criterion == 0.0:
	70	objsame = False
	71	else:
	72	objsame = abs(target - criterion) / criterion < 0.005
	73	return (objsame, target < limit, mean, target)
16e56d25 VF	74
	75
	76	if __name__ == '__main__':
	77	args = parse_args()
	78
dd4e8700 VF	79	pp = pprint.PrettyPrinter(indent=4)
dd4e8700 VF	80
cd22f801 VF	81	passed = 0
	82	failed = 0
	83
16e56d25 VF	84	#
	85	# test option parsing
	86	#
bfc4884e	87	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
16e56d25	88	'output': "set steady state IOPS threshold to 10.000000" },
bfc4884e	89	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
16e56d25	90	'output': "set steady state threshold to 10.000000%" },
bfc4884e	91	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
16e56d25	92	'output': "set steady state threshold to 0.100000%" },
bfc4884e	93	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
16e56d25	94	'output': "set steady state threshold to 10.000000%" },
bfc4884e	95	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
16e56d25	96	'output': "set steady state threshold to 0.100000%" },
bfc4884e	97	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
16e56d25 VF	98	'output': "set steady state BW threshold to 12" },
	99	]
	100	for test in parsing:
5eac3b00 BD	101	output = subprocess.check_output([args.fio] + test['args'])
	102	if test['output'] in output.decode():
	103	print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
cd22f801	104	passed = passed + 1
16e56d25	105	else:
5eac3b00	106	print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
cd22f801	107	failed = failed + 1
16e56d25 VF	108
	109	#
	110	# test some read workloads
	111	#
	112	# if ss active and attained,
	113	# check that runtime is less than job time
	114	# check criteria
	115	# how to check ramp time?
	116	#
	117	# if ss inactive
	118	# check that runtime is what was specified
	119	#
32df42c9 VF	120	reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	121	{'s': False, 'timeout': 20, 'numjobs': 2},
	122	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	123	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
16e56d25 VF	124	]
16e56d25 VF	125
32df42c9 VF	126	if args.read == None:
	127	if os.name == 'posix':
	128	args.read = '/dev/zero'
cd22f801	129	extra = [ "--size=128M" ]
32df42c9	130	else:
5eac3b00	131	print("ERROR: file for read testing must be specified on non-posix systems")
32df42c9 VF	132	sys.exit(1)
	133	else:
	134	extra = []
	135
	136	jobnum = 0
	137	for job in reads:
	138
cd22f801	139	tf = "steadystate_job{0}.json".format(jobnum)
32df42c9 VF	140	parameters = [ "--name=job{0}".format(jobnum) ]
	141	parameters.extend(extra)
	142	parameters.extend([ "--thread",
	143	"--output-format=json",
	144	"--output={0}".format(tf),
	145	"--filename={0}".format(args.read),
	146	"--rw=randrw",
	147	"--rwmixread=100",
	148	"--stonewall",
	149	"--group_reporting",
	150	"--numjobs={0}".format(job['numjobs']),
	151	"--time_based",
	152	"--runtime={0}".format(job['timeout']) ])
	153	if job['s']:
	154	if job['iops']:
	155	ss = 'iops'
	156	else:
	157	ss = 'bw'
	158	if job['slope']:
	159	ss += "_slope"
	160	ss += ":" + str(job['ss_limit'])
	161	if job['pct']:
	162	ss += '%'
	163	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	164	'--ss={0}'.format(ss),
	165	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	166
	167	output = subprocess.call([args.fio] + parameters)
	168	with open(tf, 'r') as source:
16e56d25	169	jsondata = json.loads(source.read())
cd22f801	170	source.close()
32df42c9 VF	171
32df42c9 VF	172	for jsonjob in jsondata['jobs']:
cd22f801	173	line = "{0}".format(jsonjob['job options']['name'])
32df42c9 VF	174	if job['s']:
32df42c9 VF	175	if jsonjob['steadystate']['attained'] == 1:
16e56d25	176	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
32df42c9 VF	177	mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
32df42c9 VF	178	actual = jsonjob['read']['runtime']
16e56d25	179	if mintime > actual:
32df42c9	180	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
cd22f801	181	failed = failed + 1
16e56d25	182	else:
32df42c9 VF	183	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	184	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	185	iops=job['iops'],
	186	slope=job['slope'],
	187	pct=job['pct'],
	188	limit=job['ss_limit'],
	189	dur=job['ss_dur'],
	190	criterion=jsonjob['steadystate']['criterion'])
16e56d25	191	if not objsame:
32df42c9	192	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
cd22f801	193	failed = failed + 1
16e56d25 VF	194	else:
16e56d25 VF	195	if met:
32df42c9	196	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
cd22f801	197	passed = passed + 1
16e56d25	198	else:
32df42c9	199	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
cd22f801	200	failed = failed + 1
16e56d25 VF	201	else:
16e56d25 VF	202	# check runtime, confirm criterion calculation, and confirm that criterion was not met
32df42c9 VF	203	expected = job['timeout'] * 1000
32df42c9 VF	204	actual = jsonjob['read']['runtime']
16e56d25 VF	205	if abs(expected - actual) > 10:
	206	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
	207	else:
32df42c9 VF	208	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	209	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	210	iops=job['iops'],
	211	slope=job['slope'],
	212	pct=job['pct'],
	213	limit=job['ss_limit'],
	214	dur=job['ss_dur'],
	215	criterion=jsonjob['steadystate']['criterion'])
16e56d25	216	if not objsame:
32df42c9 VF	217	if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
32df42c9 VF	218	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
cd22f801	219	failed = failed + 1
16e56d25	220	else:
32df42c9	221	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
cd22f801	222	passed = passed + 1
16e56d25 VF	223	else:
16e56d25 VF	224	if met:
32df42c9	225	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
cd22f801	226	failed = failed + 1
16e56d25	227	else:
32df42c9	228	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
cd22f801	229	passed = passed + 1
16e56d25	230	else:
32df42c9 VF	231	expected = job['timeout'] * 1000
32df42c9 VF	232	actual = jsonjob['read']['runtime']
16e56d25 VF	233	if abs(expected - actual) < 10:
16e56d25 VF	234	result = 'PASSED '
cd22f801	235	passed = passed + 1
16e56d25 VF	236	else:
16e56d25 VF	237	result = 'FAILED '
cd22f801	238	failed = failed + 1
16e56d25	239	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
5eac3b00	240	print(line)
32df42c9 VF	241	if 'steadystate' in jsonjob:
	242	pp.pprint(jsonjob['steadystate'])
	243	jobnum += 1
cd22f801 VF	244
	245	print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
	246	sys.exit(failed)