[fio.git] / t / steadystate_tests.py

#!/usr/bin/python2.7
# Note: this script is python2 and python 3 compatible.
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# On Windows this script works under Cygwin but not from cmd.exe
# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

from __future__ import absolute_import
from __future__ import print_function
import os
import sys
import json
import uuid
import pprint
import argparse
import subprocess
from scipy import stats
from six.moves import range

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio',
                        help='path to fio executable')
    parser.add_argument('--read',
                        help='target for read testing')
    parser.add_argument('--write',
                        help='target for write testing')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    measurement = 'iops' if iops else 'bw'
    data = data[measurement]
    mean = sum(data) / len(data)
    if slope:
        x = list(range(len(data)))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = m / mean * 100
            criterion = criterion[:-1]
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
            criterion = criterion[:-1]
        else:
            target = maxdev

    criterion = float(criterion)
    return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target


if __name__ == '__main__':
    args = parse_args()

    pp = pprint.PrettyPrinter(indent=4)

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args'])
        if test['output'] in output.decode():
            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
        else:
            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': False, 'timeout': 20, 'numjobs': 2},
              {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
              {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
            ]

    if args.read == None:
        if os.name == 'posix':
            args.read = '/dev/zero'
            extra = [ "--size=134217728" ]  # 128 MiB
        else:
            print("ERROR: file for read testing must be specified on non-posix systems")
            sys.exit(1)
    else:
        extra = []

    jobnum = 0
    for job in reads:

        tf = uuid.uuid4().hex
        parameters = [ "--name=job{0}".format(jobnum) ]
        parameters.extend(extra)
        parameters.extend([ "--thread",
                            "--output-format=json",
                            "--output={0}".format(tf),
                            "--filename={0}".format(args.read),
                            "--rw=randrw",
                            "--rwmixread=100",
                            "--stonewall",
                            "--group_reporting",
                            "--numjobs={0}".format(job['numjobs']),
                            "--time_based",
                            "--runtime={0}".format(job['timeout']) ])
        if job['s']:
           if job['iops']:
               ss = 'iops'
           else:
               ss = 'bw'
           if job['slope']:
               ss += "_slope"
           ss += ":" + str(job['ss_limit'])
           if job['pct']:
               ss += '%'
           parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                               '--ss={0}'.format(ss),
                               '--ss_ramp={0}'.format(job['ss_ramp']) ])

        output = subprocess.call([args.fio] + parameters)
        with open(tf, 'r') as source:
            jsondata = json.loads(source.read())
        os.remove(tf)

        for jsonjob in jsondata['jobs']:
            line = "job {0}".format(jsonjob['job options']['name'])
            if job['s']:
                if jsonjob['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
                    actual = jsonjob['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = job['timeout'] * 1000
                    actual = jsonjob['read']['runtime']
                    if abs(expected - actual) > 10:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
                        objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
                            iops=job['iops'],
                            slope=job['slope'],
                            pct=job['pct'],
                            limit=job['ss_limit'],
                            dur=job['ss_dur'],
                            criterion=jsonjob['steadystate']['criterion'])
                        if not objsame:
                            if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
            else:
                expected = job['timeout'] * 1000
                actual = jsonjob['read']['runtime']
                if abs(expected - actual) < 10:
                    result = 'PASSED '
                else:
                    result = 'FAILED '
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print(line)
            if 'steadystate' in jsonjob:
                pp.pprint(jsonjob['steadystate'])
        jobnum += 1
Commit	Line	Data
	1	#!/usr/bin/python2.7
	2	# Note: this script is python2 and python 3 compatible.
	3	#
	4	# steadystate_tests.py
	5	#
	6	# Test option parsing and functonality for fio's steady state detection feature.
	7	#
	8	# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
	9	#
	10	# REQUIREMENTS
	11	# Python 2.6+
	12	# SciPy
	13	#
	14	# KNOWN ISSUES
	15	# only option parsing and read tests are carried out
	16	# On Windows this script works under Cygwin but not from cmd.exe
	17	# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
	18	# min runtime:
	19	# if ss attained: min runtime = ss_dur + ss_ramp
	20	# if not attained: runtime = timeout
	21
	22	from __future__ import absolute_import
	23	from __future__ import print_function
	24	import os
	25	import sys
	26	import json
	27	import uuid
	28	import pprint
	29	import argparse
	30	import subprocess
	31	from scipy import stats
	32	from six.moves import range
	33
	34	def parse_args():
	35	parser = argparse.ArgumentParser()
	36	parser.add_argument('fio',
	37	help='path to fio executable')
	38	parser.add_argument('--read',
	39	help='target for read testing')
	40	parser.add_argument('--write',
	41	help='target for write testing')
	42	args = parser.parse_args()
	43
	44	return args
	45
	46
	47	def check(data, iops, slope, pct, limit, dur, criterion):
	48	measurement = 'iops' if iops else 'bw'
	49	data = data[measurement]
	50	mean = sum(data) / len(data)
	51	if slope:
	52	x = list(range(len(data)))
	53	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	54	m = abs(m)
	55	if pct:
	56	target = m / mean * 100
	57	criterion = criterion[:-1]
	58	else:
	59	target = m
	60	else:
	61	maxdev = 0
	62	for x in data:
	63	maxdev = max(abs(mean-x), maxdev)
	64	if pct:
	65	target = maxdev / mean * 100
	66	criterion = criterion[:-1]
	67	else:
	68	target = maxdev
	69
	70	criterion = float(criterion)
	71	return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
	72
	73
	74	if __name__ == '__main__':
	75	args = parse_args()
	76
	77	pp = pprint.PrettyPrinter(indent=4)
	78
	79	#
	80	# test option parsing
	81	#
	82	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
	83	'output': "set steady state IOPS threshold to 10.000000" },
	84	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
	85	'output': "set steady state threshold to 10.000000%" },
	86	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
	87	'output': "set steady state threshold to 0.100000%" },
	88	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
	89	'output': "set steady state threshold to 10.000000%" },
	90	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
	91	'output': "set steady state threshold to 0.100000%" },
	92	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
	93	'output': "set steady state BW threshold to 12" },
	94	]
	95	for test in parsing:
	96	output = subprocess.check_output([args.fio] + test['args'])
	97	if test['output'] in output.decode():
	98	print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
	99	else:
	100	print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
	101
	102	#
	103	# test some read workloads
	104	#
	105	# if ss active and attained,
	106	# check that runtime is less than job time
	107	# check criteria
	108	# how to check ramp time?
	109	#
	110	# if ss inactive
	111	# check that runtime is what was specified
	112	#
	113	reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	114	{'s': False, 'timeout': 20, 'numjobs': 2},
	115	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	116	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
	117	]
	118
	119	if args.read == None:
	120	if os.name == 'posix':
	121	args.read = '/dev/zero'
	122	extra = [ "--size=134217728" ] # 128 MiB
	123	else:
	124	print("ERROR: file for read testing must be specified on non-posix systems")
	125	sys.exit(1)
	126	else:
	127	extra = []
	128
	129	jobnum = 0
	130	for job in reads:
	131
	132	tf = uuid.uuid4().hex
	133	parameters = [ "--name=job{0}".format(jobnum) ]
	134	parameters.extend(extra)
	135	parameters.extend([ "--thread",
	136	"--output-format=json",
	137	"--output={0}".format(tf),
	138	"--filename={0}".format(args.read),
	139	"--rw=randrw",
	140	"--rwmixread=100",
	141	"--stonewall",
	142	"--group_reporting",
	143	"--numjobs={0}".format(job['numjobs']),
	144	"--time_based",
	145	"--runtime={0}".format(job['timeout']) ])
	146	if job['s']:
	147	if job['iops']:
	148	ss = 'iops'
	149	else:
	150	ss = 'bw'
	151	if job['slope']:
	152	ss += "_slope"
	153	ss += ":" + str(job['ss_limit'])
	154	if job['pct']:
	155	ss += '%'
	156	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
	157	'--ss={0}'.format(ss),
	158	'--ss_ramp={0}'.format(job['ss_ramp']) ])
	159
	160	output = subprocess.call([args.fio] + parameters)
	161	with open(tf, 'r') as source:
	162	jsondata = json.loads(source.read())
	163	os.remove(tf)
	164
	165	for jsonjob in jsondata['jobs']:
	166	line = "job {0}".format(jsonjob['job options']['name'])
	167	if job['s']:
	168	if jsonjob['steadystate']['attained'] == 1:
	169	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
	170	mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
	171	actual = jsonjob['read']['runtime']
	172	if mintime > actual:
	173	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
	174	else:
	175	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	176	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	177	iops=job['iops'],
	178	slope=job['slope'],
	179	pct=job['pct'],
	180	limit=job['ss_limit'],
	181	dur=job['ss_dur'],
	182	criterion=jsonjob['steadystate']['criterion'])
	183	if not objsame:
	184	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
	185	else:
	186	if met:
	187	line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
	188	else:
	189	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
	190	else:
	191	# check runtime, confirm criterion calculation, and confirm that criterion was not met
	192	expected = job['timeout'] * 1000
	193	actual = jsonjob['read']['runtime']
	194	if abs(expected - actual) > 10:
	195	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
	196	else:
	197	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
	198	objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
	199	iops=job['iops'],
	200	slope=job['slope'],
	201	pct=job['pct'],
	202	limit=job['ss_limit'],
	203	dur=job['ss_dur'],
	204	criterion=jsonjob['steadystate']['criterion'])
	205	if not objsame:
	206	if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
	207	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
	208	else:
	209	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
	210	else:
	211	if met:
	212	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
	213	else:
	214	line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
	215	else:
	216	expected = job['timeout'] * 1000
	217	actual = jsonjob['read']['runtime']
	218	if abs(expected - actual) < 10:
	219	result = 'PASSED '
	220	else:
	221	result = 'FAILED '
	222	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
	223	print(line)
	224	if 'steadystate' in jsonjob:
	225	pp.pprint(jsonjob['steadystate'])
	226	jobnum += 1