[fio.git] / unit_tests / steadystate_tests.py

#!/usr/bin/python
#
# steadystate_tests.py
#
# Test option parsing and functonality for fio's steady state detection feature.
#
# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
#
# REQUIREMENTS
# Python 2.6+
# SciPy
#
# KNOWN ISSUES
# only option parsing and read tests are carried out
# the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that
#     it does not match what fio reports
# min runtime:
# if ss attained: min runtime = ss_dur + ss_ramp
# if not attained: runtime = timeout

import os
import json
import tempfile
import argparse
import subprocess
from scipy import stats

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('fio',
                        help='path to fio executable');
    parser.add_argument('read',
                        help='target for read testing')
    parser.add_argument('write',
                        help='target for write testing')
    args = parser.parse_args()

    return args


def check(data, iops, slope, pct, limit, dur, criterion):
    mean = sum(data) / len(data)
    if slope:
        x = range(len(data))
        m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
        m = abs(m)
        if pct:
            target = m / mean * 100
        else:
            target = m
    else:
        maxdev = 0
        for x in data:
            maxdev = max(abs(mean-x), maxdev)
        if pct:
            target = maxdev / mean * 100
        else:
            target = maxdev

    return (abs(target - criterion) / criterion < 0.001), target < limit, mean, target


if __name__ == '__main__':
    args = parse_args()

#
# test option parsing
#
    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"], 
                  'output': "set steady state IOPS threshold to 10.000000" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"], 
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"], 
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"], 
                  'output': "set steady state threshold to 10.000000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"], 
                  'output': "set steady state threshold to 0.100000%" },
                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"], 
                  'output': "set steady state BW threshold to 12" },
              ]
    for test in parsing:
        output = subprocess.check_output([args.fio] + test['args']);
        if test['output'] in output:
            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
        else:
            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])

#
# test some read workloads
#
# if ss active and attained,
#   check that runtime is less than job time
#   check criteria
#   how to check ramp time?
#
# if ss inactive
#   check that runtime is what was specified
#
    reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
                {'s': False, 'timeout': 20, 'numjobs': 2},
                {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
                {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ],
            ]

    accum = []
    suitenum = 0
    for suite in reads:
        jobnum = 0
        for job in suite:
            parameters = [ "--name=job{0}".format(jobnum),
                           "--thread",
                           "--filename={0}".format(args.read),
                           "--rw=randrw", "--rwmixread=100", "--stonewall",
                           "--group_reporting", "--numjobs={0}".format(job['numjobs']),
                           "--time_based", "--runtime={0}".format(job['timeout']) ]
            if job['s']:
               if job['iops']:
                   ss = 'iops'
               else:
                   ss = 'bw'
               if job['slope']:
                   ss += "_slope"
               ss += ":" + str(job['ss_limit'])
               if job['pct']:
                   ss += '%'
               parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
                                   '--ss={0}'.format(ss),
                                   '--ss_ramp={0}'.format(job['ss_ramp']) ])
            accum.extend(parameters)
            jobnum += 1

        tf = tempfile.NamedTemporaryFile(delete=False)
	tf.close()
        output = subprocess.check_output([args.fio, 
                                          "--output-format=json", 
                                          "--output={0}".format(tf.name)] + accum)
        with open(tf.name, 'r') as source:
            jsondata = json.loads(source.read())
        os.remove(tf.name)
        jobnum = 0
        for job in jsondata['jobs']:
            line = "suite {0}, {1}".format(suitenum, job['job options']['name'])
            if suite[jobnum]['s']:
                if job['steadystate']['attained'] == 1:
                    # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
                    mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000
                    actual = job['read']['runtime']
                    if mintime > actual:
                        line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                    else:
                        line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                        objsame, met, mean, target = check(data=job['steadystate']['data'],
                            iops=suite[jobnum]['iops'],
                            slope=suite[jobnum]['slope'],
                            pct=suite[jobnum]['pct'],
                            limit=suite[jobnum]['ss_limit'],
                            dur=suite[jobnum]['ss_dur'],
                            criterion=job['steadystate']['criterion'])
                        if not objsame:
                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']['data'])
                        else:
                            if met:
                                line = 'PASSED ' + line + ' target {0} < limit {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
                            else:
                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
                    
                else:
                    # check runtime, confirm criterion calculation, and confirm that criterion was not met
                    expected = suite[jobnum]['timeout'] * 1000
                    actual = job['read']['runtime']
                    if abs(expected - actual) > 10:
                        line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
                    else:
                        line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
                        objsame, met, mean, target = check(data=job['steadystate']['data'],
                            iops=suite[jobnum]['iops'],
                            slope=suite[jobnum]['slope'],
                            pct=suite[jobnum]['pct'],
                            limit=suite[jobnum]['ss_limit'],
                            dur=suite[jobnum]['ss_dur'],
                            criterion=job['steadystate']['criterion'])
                        if not objsame:
                            if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']['data'])
                            else:
                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed, data: {1} '.format(job['steadystate']['criterion'], job['steadystate']['data'])
                        else:
                            if met:
                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
                            else:
                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
            else:
                expected = suite[jobnum]['timeout'] * 1000
                actual = job['read']['runtime']
                if abs(expected - actual) < 10:
                    result = 'PASSED '
                else:
                    result = 'FAILED '
                line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
            print line
            jobnum += 1
        suitenum += 1
Commit	Line	Data
16e56d25 VF	1	#!/usr/bin/python
	2	#
	3	# steadystate_tests.py
	4	#
	5	# Test option parsing and functonality for fio's steady state detection feature.
	6	#
	7	# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
	8	#
	9	# REQUIREMENTS
	10	# Python 2.6+
	11	# SciPy
	12	#
	13	# KNOWN ISSUES
	14	# only option parsing and read tests are carried out
	15	# the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that
	16	# it does not match what fio reports
	17	# min runtime:
	18	# if ss attained: min runtime = ss_dur + ss_ramp
	19	# if not attained: runtime = timeout
	20
	21	import os
	22	import json
	23	import tempfile
	24	import argparse
	25	import subprocess
	26	from scipy import stats
	27
	28	def parse_args():
	29	parser = argparse.ArgumentParser()
	30	parser.add_argument('fio',
	31	help='path to fio executable');
	32	parser.add_argument('read',
	33	help='target for read testing')
	34	parser.add_argument('write',
	35	help='target for write testing')
	36	args = parser.parse_args()
	37
	38	return args
	39
	40
	41	def check(data, iops, slope, pct, limit, dur, criterion):
	42	mean = sum(data) / len(data)
	43	if slope:
	44	x = range(len(data))
	45	m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
	46	m = abs(m)
	47	if pct:
	48	target = m / mean * 100
	49	else:
	50	target = m
	51	else:
	52	maxdev = 0
	53	for x in data:
	54	maxdev = max(abs(mean-x), maxdev)
	55	if pct:
	56	target = maxdev / mean * 100
	57	else:
	58	target = maxdev
	59
	60	return (abs(target - criterion) / criterion < 0.001), target < limit, mean, target
	61
	62
	63	if __name__ == '__main__':
	64	args = parse_args()
65
66	#
67	# test option parsing
68	#
69	parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
70	'output': "set steady state IOPS threshold to 10.000000" },
71	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
72	'output': "set steady state threshold to 10.000000%" },
73	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
74	'output': "set steady state threshold to 0.100000%" },
75	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
76	'output': "set steady state threshold to 10.000000%" },
77	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
78	'output': "set steady state threshold to 0.100000%" },
79	{ 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
80	'output': "set steady state BW threshold to 12" },
81	]
82	for test in parsing:
83	output = subprocess.check_output([args.fio] + test['args']);
84	if test['output'] in output:
85	print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
86	else:
87	print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
88
89	#
90	# test some read workloads
91	#
92	# if ss active and attained,
93	# check that runtime is less than job time
94	# check criteria
95	# how to check ramp time?
96	#
97	# if ss inactive
98	# check that runtime is what was specified
99	#
100	reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
101	{'s': False, 'timeout': 20, 'numjobs': 2},
102	{'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
103	{'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ],
104	]
105
106	accum = []
107	suitenum = 0
108	for suite in reads:
109	jobnum = 0
110	for job in suite:
111	parameters = [ "--name=job{0}".format(jobnum),
112	"--thread",
113	"--filename={0}".format(args.read),
114	"--rw=randrw", "--rwmixread=100", "--stonewall",
115	"--group_reporting", "--numjobs={0}".format(job['numjobs']),
116	"--time_based", "--runtime={0}".format(job['timeout']) ]
117	if job['s']:
118	if job['iops']:
119	ss = 'iops'
120	else:
121	ss = 'bw'
122	if job['slope']:
123	ss += "_slope"
124	ss += ":" + str(job['ss_limit'])
125	if job['pct']:
126	ss += '%'
127	parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
128	'--ss={0}'.format(ss),
129	'--ss_ramp={0}'.format(job['ss_ramp']) ])
130	accum.extend(parameters)
131	jobnum += 1
132
133	tf = tempfile.NamedTemporaryFile(delete=False)
134	tf.close()
135	output = subprocess.check_output([args.fio,
136	"--output-format=json",
137	"--output={0}".format(tf.name)] + accum)
138	with open(tf.name, 'r') as source:
139	jsondata = json.loads(source.read())
140	os.remove(tf.name)
141	jobnum = 0
142	for job in jsondata['jobs']:
143	line = "suite {0}, {1}".format(suitenum, job['job options']['name'])
144	if suite[jobnum]['s']:
145	if job['steadystate']['attained'] == 1:
146	# check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
147	mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000
148	actual = job['read']['runtime']
149	if mintime > actual:
150	line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
151	else:
152	line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
153	objsame, met, mean, target = check(data=job['steadystate']['data'],
154	iops=suite[jobnum]['iops'],
155	slope=suite[jobnum]['slope'],
156	pct=suite[jobnum]['pct'],
157	limit=suite[jobnum]['ss_limit'],
158	dur=suite[jobnum]['ss_dur'],
159	criterion=job['steadystate']['criterion'])
160	if not objsame:
161	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']['data'])
162	else:
163	if met:
164	line = 'PASSED ' + line + ' target {0} < limit {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
165	else:
166	line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
167
168	else:
169	# check runtime, confirm criterion calculation, and confirm that criterion was not met
170	expected = suite[jobnum]['timeout'] * 1000
171	actual = job['read']['runtime']
172	if abs(expected - actual) > 10:
173	line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
174	else:
175	line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
176	objsame, met, mean, target = check(data=job['steadystate']['data'],
177	iops=suite[jobnum]['iops'],
178	slope=suite[jobnum]['slope'],
179	pct=suite[jobnum]['pct'],
180	limit=suite[jobnum]['ss_limit'],
181	dur=suite[jobnum]['ss_dur'],
182	criterion=job['steadystate']['criterion'])
183	if not objsame:
184	if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
185	line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']['data'])
186	else:
187	line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed, data: {1} '.format(job['steadystate']['criterion'], job['steadystate']['data'])
188	else:
189	if met:
190	line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
191	else:
192	line = 'PASSED ' + line + ' criterion {0} > threshold {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
193	else:
194	expected = suite[jobnum]['timeout'] * 1000
195	actual = job['read']['runtime']
196	if abs(expected - actual) < 10:
197	result = 'PASSED '
198	else:
199	result = 'FAILED '
200	line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
201	print line
202	jobnum += 1
203	suitenum += 1