[fio.git] / t / steadystate_tests.py

#!/usr/bin/env python3
#
# steadystate_tests.py
#
# Test option parsing and functionality for fio's steady state detection feature.
#
# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# On Windows this script works under Cygwin but not from cmd.exe
# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

from __future__ import absolute_import
from __future__ import print_function
import os
import sys
import json
import pprint
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio', help='path to fio executable')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = list(range(len(data)))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = (m / mean * 100) if mean != 0 else 0
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    if criterion == 0.0:
        objsame = False
    else:
        objsame = abs(target - criterion) / criterion < 0.005
    return (objsame, target < limit, mean, target)


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

    passed = 0
    failed = 0

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args'])
        if test['output'] in output.decode():
            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
            passed = passed + 1
        else:
            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
            failed = failed + 1

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': False, 'timeout': 20, 'numjobs': 2},
              {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True, 'ss_interval': 5},
            ]

    jobnum = 0
    for job in reads:

        tf = "steadystate_job{0}.json".format(jobnum)
        parameters = [ "--max-jobs=16", "--name=job{0}".format(jobnum) ]
        parameters.extend([ "--thread",
                            "--output-format=json",
                            "--output={0}".format(tf),
                            "--ioengine=null",
                            "--size=1G",
                            "--rw=randrw",
                            "--rwmixread=100",
                            "--stonewall",
                            "--group_reporting",
                            "--numjobs={0}".format(job['numjobs']),
                            "--time_based",
                            "--runtime={0}".format(job['timeout']) ])
        if job['s']:
           if job['iops']:
               ss = 'iops'
           else:
               ss = 'bw'
           if job['slope']:
               ss += "_slope"
           ss += ":" + str(job['ss_limit'])
           if job['pct']:
               ss += '%'
           parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                               '--ss={0}'.format(ss),
                               '--ss_ramp={0}'.format(job['ss_ramp']) ])

        output = subprocess.call([args.fio] + parameters)
        with open(tf, 'r') as source:
            jsondata = json.loads(source.read())
            source.close()

        for jsonjob in jsondata['jobs']:
            line = "{0}".format(jsonjob['job options']['name'])
            if job['s']:
                if jsonjob['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
                    actual = jsonjob['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
                        failed = failed + 1
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                            failed = failed + 1
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = job['timeout'] * 1000
                    actual = jsonjob['read']['runtime']
                    if abs(expected - actual) > 50:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
                                passed = passed + 1
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                                failed = failed + 1
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
                                passed = passed + 1
            else:
                expected = job['timeout'] * 1000
                actual = jsonjob['read']['runtime']
                if abs(expected - actual) > 50:
                    result = 'FAILED '
                    failed = failed + 1
                else:
                    result = 'PASSED '
                    passed = passed + 1
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print(line)
            if 'steadystate' in jsonjob:
                pp.pprint(jsonjob['steadystate'])
        jobnum += 1

    print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
    sys.exit(failed)
Commit	Line	Data
8c00c383	1	#!/usr/bin/env python3
16e56d25 VF	2	#
	3	# steadystate_tests.py
	4	#
fc002f14	5	# Test option parsing and functionality for fio's steady state detection feature.
16e56d25	6	#
a72f40bb	7	# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
16e56d25 VF	8	#
	9	# REQUIREMENTS
	10	# Python 2.6+
	11	# SciPy
	12	#
	13	# KNOWN ISSUES
	14	# only option parsing and read tests are carried out
32df42c9 VF	15	# On Windows this script works under Cygwin but not from cmd.exe
32df42c9 VF	16	# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
16e56d25 VF	17	# min runtime:
	18	# if ss attained: min runtime = ss_dur + ss_ramp
	19	# if not attained: runtime = timeout
	20
5eac3b00 BD	21	from __future__ import absolute_import
5eac3b00 BD	22	from __future__ import print_function
16e56d25	23	import os
39f5379e	24	import sys
16e56d25	25	import json
dd4e8700	26	import pprint
16e56d25 VF	27	import argparse
	28	import subprocess
	29	from scipy import stats
	30
	31	def parse_args():
	32	parser = argparse.ArgumentParser()
82d0ad06	33	parser.add_argument('fio', help='path to fio executable')
16e56d25 VF	34	args = parser.parse_args()
	35
	36	return args
	37
	38
	39	def check(data, iops, slope, pct, limit, dur, criterion):
ba8fb6f6 VF	40	measurement = 'iops' if iops else 'bw'
ba8fb6f6 VF	41	data = data[measurement]
16e56d25 VF	42	mean = sum(data) / len(data)
16e56d25 VF	43	if slope:
5eac3b00	44	x = list(range(len(data)))
16e56d25 VF	45	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	46	m = abs(m)
	47	if pct:
cd22f801	48	target = (m / mean * 100) if mean != 0 else 0
4001d829	49	criterion = criterion[:-1]
16e56d25 VF	50	else:
	51	target = m
	52	else:
	53	maxdev = 0
	54	for x in data:
	55	maxdev = max(abs(mean-x), maxdev)
	56	if pct:
4001d829 VF	57	target = maxdev / mean * 100
4001d829 VF	58	criterion = criterion[:-1]
16e56d25 VF	59	else:
	60	target = maxdev
	61
4001d829	62	criterion = float(criterion)
cd22f801 VF	63	if criterion == 0.0:
	64	objsame = False
	65	else:
	66	objsame = abs(target - criterion) / criterion < 0.005
	67	return (objsame, target < limit, mean, target)
16e56d25 VF	68
	69
	70	if __name__ == '__main__':
	71	args = parse_args()
	72
dd4e8700 VF	73	pp = pprint.PrettyPrinter(indent=4)
dd4e8700 VF	74
cd22f801 VF	75	passed = 0
	76	failed = 0
	77
16e56d25 VF	78	#
	79	# test option parsing
	80	#
bfc4884e	81	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
16e56d25	82	'output': "set steady state IOPS threshold to 10.000000" },
bfc4884e	83	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
16e56d25	84	'output': "set steady state threshold to 10.000000%" },
bfc4884e	85	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
16e56d25	86	'output': "set steady state threshold to 0.100000%" },
bfc4884e	87	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
16e56d25	88	'output': "set steady state threshold to 10.000000%" },
bfc4884e	89	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
16e56d25	90	'output': "set steady state threshold to 0.100000%" },
bfc4884e	91	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
16e56d25 VF	92	'output': "set steady state BW threshold to 12" },
	93	]
	94	for test in parsing:
5eac3b00 BD	95	output = subprocess.check_output([args.fio] + test['args'])
	96	if test['output'] in output.decode():
	97	print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
cd22f801	98	passed = passed + 1
16e56d25	99	else:
5eac3b00	100	print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
cd22f801	101	failed = failed + 1
16e56d25 VF	102
	103	#
	104	# test some read workloads
	105	#
	106	# if ss active and attained,
	107	# check that runtime is less than job time
	108	# check criteria
	109	# how to check ramp time?
	110	#
	111	# if ss inactive
	112	# check that runtime is what was specified
	113	#
32df42c9 VF	114	reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	115	{'s': False, 'timeout': 20, 'numjobs': 2},
	116	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	117	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
90e678ba	118	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True, 'ss_interval': 5},
16e56d25 VF	119	]
16e56d25 VF	120
32df42c9 VF	121	jobnum = 0
	122	for job in reads:
	123
cd22f801	124	tf = "steadystate_job{0}.json".format(jobnum)
771dbb52	125	parameters = [ "--max-jobs=16", "--name=job{0}".format(jobnum) ]
32df42c9 VF	126	parameters.extend([ "--thread",
	127	"--output-format=json",
	128	"--output={0}".format(tf),
82d0ad06 VF	129	"--ioengine=null",
82d0ad06 VF	130	"--size=1G",
32df42c9 VF	131	"--rw=randrw",
	132	"--rwmixread=100",
	133	"--stonewall",
	134	"--group_reporting",
	135	"--numjobs={0}".format(job['numjobs']),
	136	"--time_based",
	137	"--runtime={0}".format(job['timeout']) ])
	138	if job['s']:
	139	if job['iops']:
	140	ss = 'iops'
	141	else:
	142	ss = 'bw'
	143	if job['slope']:
	144	ss += "_slope"
	145	ss += ":" + str(job['ss_limit'])
	146	if job['pct']:
	147	ss += '%'
	148	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	149	'--ss={0}'.format(ss),
	150	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	151
	152	output = subprocess.call([args.fio] + parameters)
	153	with open(tf, 'r') as source:
16e56d25	154	jsondata = json.loads(source.read())
cd22f801	155	source.close()
32df42c9 VF	156
32df42c9 VF	157	for jsonjob in jsondata['jobs']:
cd22f801	158	line = "{0}".format(jsonjob['job options']['name'])
32df42c9 VF	159	if job['s']:
32df42c9 VF	160	if jsonjob['steadystate']['attained'] == 1:
16e56d25	161	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
32df42c9 VF	162	mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
32df42c9 VF	163	actual = jsonjob['read']['runtime']
16e56d25	164	if mintime > actual:
32df42c9	165	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
cd22f801	166	failed = failed + 1
16e56d25	167	else:
32df42c9 VF	168	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	169	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	170	iops=job['iops'],
	171	slope=job['slope'],
	172	pct=job['pct'],
	173	limit=job['ss_limit'],
	174	dur=job['ss_dur'],
	175	criterion=jsonjob['steadystate']['criterion'])
16e56d25	176	if not objsame:
32df42c9	177	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
cd22f801	178	failed = failed + 1
16e56d25 VF	179	else:
16e56d25 VF	180	if met:
32df42c9	181	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
cd22f801	182	passed = passed + 1
16e56d25	183	else:
32df42c9	184	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
cd22f801	185	failed = failed + 1
16e56d25 VF	186	else:
16e56d25 VF	187	# check runtime, confirm criterion calculation, and confirm that criterion was not met
32df42c9 VF	188	expected = job['timeout'] * 1000
32df42c9 VF	189	actual = jsonjob['read']['runtime']
3a0be784	190	if abs(expected - actual) > 50:
16e56d25 VF	191	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
16e56d25 VF	192	else:
32df42c9 VF	193	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	194	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	195	iops=job['iops'],
	196	slope=job['slope'],
	197	pct=job['pct'],
	198	limit=job['ss_limit'],
	199	dur=job['ss_dur'],
	200	criterion=jsonjob['steadystate']['criterion'])
16e56d25	201	if not objsame:
32df42c9 VF	202	if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
32df42c9 VF	203	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
cd22f801	204	failed = failed + 1
16e56d25	205	else:
32df42c9	206	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
cd22f801	207	passed = passed + 1
16e56d25 VF	208	else:
16e56d25 VF	209	if met:
32df42c9	210	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
cd22f801	211	failed = failed + 1
16e56d25	212	else:
32df42c9	213	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
cd22f801	214	passed = passed + 1
16e56d25	215	else:
32df42c9 VF	216	expected = job['timeout'] * 1000
32df42c9 VF	217	actual = jsonjob['read']['runtime']
3a0be784	218	if abs(expected - actual) > 50:
16e56d25	219	result = 'FAILED '
cd22f801	220	failed = failed + 1
3a0be784 VF	221	else:
	222	result = 'PASSED '
	223	passed = passed + 1
16e56d25	224	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
5eac3b00	225	print(line)
32df42c9 VF	226	if 'steadystate' in jsonjob:
	227	pp.pprint(jsonjob['steadystate'])
	228	jobnum += 1
cd22f801 VF	229
	230	print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed))
	231	sys.exit(failed)