[fio.git] / t / steadystate_tests.py

#!/usr/bin/env python3
#
# steadystate_tests.py
#
# Test option parsing and functionality for fio's steady state detection feature.
#
# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# On Windows this script works under Cygwin but not from cmd.exe
# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

from __future__ import absolute_import
from __future__ import print_function
import os
import sys
import json
import pprint
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio', help='path to fio executable')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = list(range(len(data)))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = (m / mean * 100) if mean != 0 else 0
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    if criterion == 0.0:
        objsame = False
    else:
        objsame = abs(target - criterion) / criterion < 0.005
    return (objsame, target < limit, mean, target)


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

    passed = 0
    failed = 0

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args'])
        if test['output'] in output.decode():
            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
            passed = passed + 1
        else:
            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
            failed = failed + 1

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': False, 'timeout': 20, 'numjobs': 2},
              {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
            ]

    jobnum = 0
    for job in reads:

        tf = "steadystate_job{0}.json".format(jobnum)
        parameters = [ "--max-jobs=16", "--name=job{0}".format(jobnum) ]
        parameters.extend([ "--thread",
                            "--output-format=json",
                            "--output={0}".format(tf),
                            "--ioengine=null",
                            "--size=1G",
                            "--rw=randrw",
                            "--rwmixread=100",
                            "--stonewall",
                            "--group_reporting",
                            "--numjobs={0}".format(job['numjobs']),
                            "--time_based",
                            "--runtime={0}".format(job['timeout']) ])
        if job['s']:
           if job['iops']:
               ss = 'iops'
           else:
               ss = 'bw'
           if job['slope']:
               ss += "_slope"
           ss += ":" + str(job['ss_limit'])
           if job['pct']:
               ss += '%'
           parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                               '--ss={0}'.format(ss),
                               '--ss_ramp={0}'.format(job['ss_ramp']) ])

        output = subprocess.call([args.fio] + parameters)
        with open(tf, 'r') as source:
            jsondata = json.loads(source.read())
            source.close()

        for jsonjob in jsondata['jobs']:
            line = "{0}".format(jsonjob['job options']['name'])
            if job['s']:
                if jsonjob['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
                    actual = jsonjob['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
                        failed = failed + 1
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                            failed = failed + 1
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = job['timeout'] * 1000
                    actual = jsonjob['read']['runtime']
                    if abs(expected - actual) > 50:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
                                passed = passed + 1
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
            else:
                expected = job['timeout'] * 1000
                actual = jsonjob['read']['runtime']
                if abs(expected - actual) > 50:
                    result = 'FAILED '
                    failed = failed + 1
                else:
                    result = 'PASSED '
                    passed = passed + 1
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print(line)
            if 'steadystate' in jsonjob:
                pp.pprint(jsonjob['steadystate'])
        jobnum += 1

    print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
    sys.exit(failed)
Commit	Line	Data
8c00c383	1	#!/usr/bin/env python3
16e56d25 VF	2	#
	3	# steadystate_tests.py
	4	#
fc002f14	5	# Test option parsing and functionality for fio's steady state detection feature.
16e56d25	6	#
a72f40bb	7	# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
16e56d25 VF	8	#
	9	# REQUIREMENTS
	10	# Python 2.6+
	11	# SciPy
	12	#
	13	# KNOWN ISSUES
	14	# only option parsing and read tests are carried out
32df42c9 VF	15	# On Windows this script works under Cygwin but not from cmd.exe
32df42c9 VF	16	# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
16e56d25 VF	17	# min runtime:
	18	# if ss attained: min runtime = ss_dur + ss_ramp
	19	# if not attained: runtime = timeout
	20
5eac3b00 BD	21	from __future__ import absolute_import
5eac3b00 BD	22	from __future__ import print_function
16e56d25	23	import os
39f5379e	24	import sys
16e56d25	25	import json
dd4e8700	26	import pprint
16e56d25 VF	27	import argparse
	28	import subprocess
	29	from scipy import stats
	30
	31	def parse_args():
	32	parser = argparse.ArgumentParser()
82d0ad06	33	parser.add_argument('fio', help='path to fio executable')
16e56d25 VF	34	args = parser.parse_args()
	35
	36	return args
	37
	38
	39	def check(data, iops, slope, pct, limit, dur, criterion):
ba8fb6f6 VF	40	measurement = 'iops' if iops else 'bw'
ba8fb6f6 VF	41	data = data[measurement]
16e56d25 VF	42	mean = sum(data) / len(data)
16e56d25 VF	43	if slope:
5eac3b00	44	x = list(range(len(data)))
16e56d25 VF	45	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	46	m = abs(m)
	47	if pct:
cd22f801	48	target = (m / mean * 100) if mean != 0 else 0
4001d829	49	criterion = criterion[:-1]
16e56d25 VF	50	else:
	51	target = m
	52	else:
	53	maxdev = 0
	54	for x in data:
	55	maxdev = max(abs(mean-x), maxdev)
	56	if pct:
4001d829 VF	57	target = maxdev / mean * 100
4001d829 VF	58	criterion = criterion[:-1]
16e56d25 VF	59	else:
	60	target = maxdev
	61
4001d829	62	criterion = float(criterion)
cd22f801 VF	63	if criterion == 0.0:
	64	objsame = False
	65	else:
	66	objsame = abs(target - criterion) / criterion < 0.005
	67	return (objsame, target < limit, mean, target)
16e56d25 VF	68
	69
	70	if __name__ == '__main__':
	71	args = parse_args()
	72
dd4e8700 VF	73	pp = pprint.PrettyPrinter(indent=4)
dd4e8700 VF	74
cd22f801 VF	75	passed = 0
	76	failed = 0
	77
16e56d25 VF	78	#
	79	# test option parsing
	80	#
bfc4884e	81	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
16e56d25	82	'output': "set steady state IOPS threshold to 10.000000" },
bfc4884e	83	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
16e56d25	84	'output': "set steady state threshold to 10.000000%" },
bfc4884e	85	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
16e56d25	86	'output': "set steady state threshold to 0.100000%" },
bfc4884e	87	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
16e56d25	88	'output': "set steady state threshold to 10.000000%" },
bfc4884e	89	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
16e56d25	90	'output': "set steady state threshold to 0.100000%" },
bfc4884e	91	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
16e56d25 VF	92	'output': "set steady state BW threshold to 12" },
	93	]
	94	for test in parsing:
5eac3b00 BD	95	output = subprocess.check_output([args.fio] + test['args'])
	96	if test['output'] in output.decode():
	97	print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
cd22f801	98	passed = passed + 1
16e56d25	99	else:
5eac3b00	100	print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
cd22f801	101	failed = failed + 1
16e56d25 VF	102
	103	#
	104	# test some read workloads
	105	#
	106	# if ss active and attained,
	107	# check that runtime is less than job time
	108	# check criteria
	109	# how to check ramp time?
	110	#
	111	# if ss inactive
	112	# check that runtime is what was specified
	113	#
32df42c9 VF	114	reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	115	{'s': False, 'timeout': 20, 'numjobs': 2},
	116	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	117	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
16e56d25 VF	118	]
16e56d25 VF	119
32df42c9 VF	120	jobnum = 0
	121	for job in reads:
	122
cd22f801	123	tf = "steadystate_job{0}.json".format(jobnum)
771dbb52	124	parameters = [ "--max-jobs=16", "--name=job{0}".format(jobnum) ]
32df42c9 VF	125	parameters.extend([ "--thread",
	126	"--output-format=json",
	127	"--output={0}".format(tf),
82d0ad06 VF	128	"--ioengine=null",
82d0ad06 VF	129	"--size=1G",
32df42c9 VF	130	"--rw=randrw",
	131	"--rwmixread=100",
	132	"--stonewall",
	133	"--group_reporting",
	134	"--numjobs={0}".format(job['numjobs']),
	135	"--time_based",
	136	"--runtime={0}".format(job['timeout']) ])
	137	if job['s']:
	138	if job['iops']:
	139	ss = 'iops'
	140	else:
	141	ss = 'bw'
	142	if job['slope']:
	143	ss += "_slope"
	144	ss += ":" + str(job['ss_limit'])
	145	if job['pct']:
	146	ss += '%'
	147	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	148	'--ss={0}'.format(ss),
	149	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	150
	151	output = subprocess.call([args.fio] + parameters)
	152	with open(tf, 'r') as source:
16e56d25	153	jsondata = json.loads(source.read())
cd22f801	154	source.close()
32df42c9 VF	155
32df42c9 VF	156	for jsonjob in jsondata['jobs']:
cd22f801	157	line = "{0}".format(jsonjob['job options']['name'])
32df42c9 VF	158	if job['s']:
32df42c9 VF	159	if jsonjob['steadystate']['attained'] == 1:
16e56d25	160	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
32df42c9 VF	161	mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
32df42c9 VF	162	actual = jsonjob['read']['runtime']
16e56d25	163	if mintime > actual:
32df42c9	164	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
cd22f801	165	failed = failed + 1
16e56d25	166	else:
32df42c9 VF	167	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	168	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	169	iops=job['iops'],
	170	slope=job['slope'],
	171	pct=job['pct'],
	172	limit=job['ss_limit'],
	173	dur=job['ss_dur'],
	174	criterion=jsonjob['steadystate']['criterion'])
16e56d25	175	if not objsame:
32df42c9	176	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
cd22f801	177	failed = failed + 1
16e56d25 VF	178	else:
16e56d25 VF	179	if met:
32df42c9	180	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
cd22f801	181	passed = passed + 1
16e56d25	182	else:
32df42c9	183	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
cd22f801	184	failed = failed + 1
16e56d25 VF	185	else:
16e56d25 VF	186	# check runtime, confirm criterion calculation, and confirm that criterion was not met
32df42c9 VF	187	expected = job['timeout'] * 1000
32df42c9 VF	188	actual = jsonjob['read']['runtime']
3a0be784	189	if abs(expected - actual) > 50:
16e56d25 VF	190	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
16e56d25 VF	191	else:
32df42c9 VF	192	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	193	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	194	iops=job['iops'],
	195	slope=job['slope'],
	196	pct=job['pct'],
	197	limit=job['ss_limit'],
	198	dur=job['ss_dur'],
	199	criterion=jsonjob['steadystate']['criterion'])
16e56d25	200	if not objsame:
32df42c9 VF	201	if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
32df42c9 VF	202	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
cd22f801	203	failed = failed + 1
16e56d25	204	else:
32df42c9	205	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
cd22f801	206	passed = passed + 1
16e56d25 VF	207	else:
16e56d25 VF	208	if met:
32df42c9	209	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
cd22f801	210	failed = failed + 1
16e56d25	211	else:
32df42c9	212	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
cd22f801	213	passed = passed + 1
16e56d25	214	else:
32df42c9 VF	215	expected = job['timeout'] * 1000
32df42c9 VF	216	actual = jsonjob['read']['runtime']
3a0be784	217	if abs(expected - actual) > 50:
16e56d25	218	result = 'FAILED '
cd22f801	219	failed = failed + 1
3a0be784 VF	220	else:
	221	result = 'PASSED '
	222	passed = passed + 1
16e56d25	223	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
5eac3b00	224	print(line)
32df42c9 VF	225	if 'steadystate' in jsonjob:
	226	pp.pprint(jsonjob['steadystate'])
	227	jobnum += 1
cd22f801 VF	228
	229	print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
	230	sys.exit(failed)