5 # Test option parsing and functonality for fio's steady state detection feature.
7 # steadystate_tests.py ./fio file-for-read-testing file-for-write-testing
14 # only option parsing and read tests are carried out
15 # the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that
16 # it does not match what fio reports
18 # if ss attained: min runtime = ss_dur + ss_ramp
19 # if not attained: runtime = timeout
29 from scipy import stats
32 parser = argparse.ArgumentParser()
33 parser.add_argument('fio',
34 help='path to fio executable');
35 parser.add_argument('--read',
36 help='target for read testing')
37 parser.add_argument('--write',
38 help='target for write testing')
39 args = parser.parse_args()
44 def check(data, iops, slope, pct, limit, dur, criterion):
45 measurement = 'iops' if iops else 'bw'
46 data = data[measurement]
47 mean = sum(data) / len(data)
50 m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
53 target = m / mean * 100
54 criterion = criterion[:-1]
60 maxdev = max(abs(mean-x), maxdev)
62 target = maxdev / mean * 100
63 criterion = criterion[:-1]
67 criterion = float(criterion)
68 return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
71 if __name__ == '__main__':
74 pp = pprint.PrettyPrinter(indent=4)
79 parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
80 'output': "set steady state IOPS threshold to 10.000000" },
81 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
82 'output': "set steady state threshold to 10.000000%" },
83 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
84 'output': "set steady state threshold to 0.100000%" },
85 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
86 'output': "set steady state threshold to 10.000000%" },
87 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
88 'output': "set steady state threshold to 0.100000%" },
89 { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
90 'output': "set steady state BW threshold to 12" },
93 output = subprocess.check_output([args.fio] + test['args']);
94 if test['output'] in output:
95 print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
97 print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
100 # test some read workloads
102 # if ss active and attained,
103 # check that runtime is less than job time
105 # how to check ramp time?
108 # check that runtime is what was specified
110 reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
111 {'s': False, 'timeout': 20, 'numjobs': 2},
112 {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
113 {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ],
121 if args.read == None:
122 if os.name == 'posix':
123 args.read = '/dev/zero'
124 parameters = [ "--size=128M" ]
126 print "ERROR: file for read testing must be specified on non-posix systems"
131 parameters.extend([ "--name=job{0}".format(jobnum),
133 "--filename={0}".format(args.read),
134 "--rw=randrw", "--rwmixread=100", "--stonewall",
135 "--group_reporting", "--numjobs={0}".format(job['numjobs']),
136 "--time_based", "--runtime={0}".format(job['timeout']) ])
144 ss += ":" + str(job['ss_limit'])
147 parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
148 '--ss={0}'.format(ss),
149 '--ss_ramp={0}'.format(job['ss_ramp']) ])
150 accum.extend(parameters)
153 tf = tempfile.NamedTemporaryFile(delete=False)
155 output = subprocess.check_output([args.fio,
156 "--output-format=json",
157 "--output={0}".format(tf.name)] + accum)
158 with open(tf.name, 'r') as source:
159 jsondata = json.loads(source.read())
162 for job in jsondata['jobs']:
163 line = "suite {0}, {1}".format(suitenum, job['job options']['name'])
164 if suite[jobnum]['s']:
165 if job['steadystate']['attained'] == 1:
166 # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
167 mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000
168 actual = job['read']['runtime']
170 line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
172 line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
173 objsame, met, mean, target = check(data=job['steadystate']['data'],
174 iops=suite[jobnum]['iops'],
175 slope=suite[jobnum]['slope'],
176 pct=suite[jobnum]['pct'],
177 limit=suite[jobnum]['ss_limit'],
178 dur=suite[jobnum]['ss_dur'],
179 criterion=job['steadystate']['criterion'])
181 line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
184 line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, suite[jobnum]['ss_limit'])
186 line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
188 # check runtime, confirm criterion calculation, and confirm that criterion was not met
189 expected = suite[jobnum]['timeout'] * 1000
190 actual = job['read']['runtime']
191 if abs(expected - actual) > 10:
192 line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
194 line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp'])
195 objsame, met, mean, target = check(data=job['steadystate']['data'],
196 iops=suite[jobnum]['iops'],
197 slope=suite[jobnum]['slope'],
198 pct=suite[jobnum]['pct'],
199 limit=suite[jobnum]['ss_limit'],
200 dur=suite[jobnum]['ss_dur'],
201 criterion=job['steadystate']['criterion'])
203 if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
204 line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
206 line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(job['steadystate']['criterion'])
209 line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
211 line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, suite[jobnum]['ss_limit'])
213 expected = suite[jobnum]['timeout'] * 1000
214 actual = job['read']['runtime']
215 if abs(expected - actual) < 10:
219 line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
221 if 'steadystate' in job:
222 pp.pprint(job['steadystate'])