Commit | Line | Data |
---|---|---|
cd22f801 VF |
1 | #!/usr/bin/env python |
2 | # Note: this script is python2 and python3 compatible. | |
16e56d25 VF |
3 | # |
4 | # steadystate_tests.py | |
5 | # | |
6 | # Test option parsing and functonality for fio's steady state detection feature. | |
7 | # | |
a72f40bb | 8 | # steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio |
16e56d25 VF |
9 | # |
10 | # REQUIREMENTS | |
11 | # Python 2.6+ | |
12 | # SciPy | |
13 | # | |
14 | # KNOWN ISSUES | |
15 | # only option parsing and read tests are carried out | |
32df42c9 VF |
16 | # On Windows this script works under Cygwin but not from cmd.exe |
17 | # On Windows I encounter frequent fio problems generating JSON output (nothing to decode) | |
16e56d25 VF |
18 | # min runtime: |
19 | # if ss attained: min runtime = ss_dur + ss_ramp | |
20 | # if not attained: runtime = timeout | |
21 | ||
5eac3b00 BD |
22 | from __future__ import absolute_import |
23 | from __future__ import print_function | |
16e56d25 | 24 | import os |
39f5379e | 25 | import sys |
16e56d25 | 26 | import json |
dd4e8700 | 27 | import pprint |
16e56d25 VF |
28 | import argparse |
29 | import subprocess | |
30 | from scipy import stats | |
31 | ||
32 | def parse_args(): | |
33 | parser = argparse.ArgumentParser() | |
34 | parser.add_argument('fio', | |
5eac3b00 | 35 | help='path to fio executable') |
39f5379e | 36 | parser.add_argument('--read', |
16e56d25 | 37 | help='target for read testing') |
39f5379e | 38 | parser.add_argument('--write', |
16e56d25 VF |
39 | help='target for write testing') |
40 | args = parser.parse_args() | |
41 | ||
42 | return args | |
43 | ||
44 | ||
45 | def check(data, iops, slope, pct, limit, dur, criterion): | |
ba8fb6f6 VF |
46 | measurement = 'iops' if iops else 'bw' |
47 | data = data[measurement] | |
16e56d25 VF |
48 | mean = sum(data) / len(data) |
49 | if slope: | |
5eac3b00 | 50 | x = list(range(len(data))) |
16e56d25 VF |
51 | m, intercept, r_value, p_value, std_err = stats.linregress(x,data) |
52 | m = abs(m) | |
53 | if pct: | |
cd22f801 | 54 | target = (m / mean * 100) if mean != 0 else 0 |
4001d829 | 55 | criterion = criterion[:-1] |
16e56d25 VF |
56 | else: |
57 | target = m | |
58 | else: | |
59 | maxdev = 0 | |
60 | for x in data: | |
61 | maxdev = max(abs(mean-x), maxdev) | |
62 | if pct: | |
4001d829 VF |
63 | target = maxdev / mean * 100 |
64 | criterion = criterion[:-1] | |
16e56d25 VF |
65 | else: |
66 | target = maxdev | |
67 | ||
4001d829 | 68 | criterion = float(criterion) |
cd22f801 VF |
69 | if criterion == 0.0: |
70 | objsame = False | |
71 | else: | |
72 | objsame = abs(target - criterion) / criterion < 0.005 | |
73 | return (objsame, target < limit, mean, target) | |
16e56d25 VF |
74 | |
75 | ||
76 | if __name__ == '__main__': | |
77 | args = parse_args() | |
78 | ||
dd4e8700 VF |
79 | pp = pprint.PrettyPrinter(indent=4) |
80 | ||
cd22f801 VF |
81 | passed = 0 |
82 | failed = 0 | |
83 | ||
16e56d25 VF |
84 | # |
85 | # test option parsing | |
86 | # | |
bfc4884e | 87 | parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"], |
16e56d25 | 88 | 'output': "set steady state IOPS threshold to 10.000000" }, |
bfc4884e | 89 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"], |
16e56d25 | 90 | 'output': "set steady state threshold to 10.000000%" }, |
bfc4884e | 91 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"], |
16e56d25 | 92 | 'output': "set steady state threshold to 0.100000%" }, |
bfc4884e | 93 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"], |
16e56d25 | 94 | 'output': "set steady state threshold to 10.000000%" }, |
bfc4884e | 95 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"], |
16e56d25 | 96 | 'output': "set steady state threshold to 0.100000%" }, |
bfc4884e | 97 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"], |
16e56d25 VF |
98 | 'output': "set steady state BW threshold to 12" }, |
99 | ] | |
100 | for test in parsing: | |
5eac3b00 BD |
101 | output = subprocess.check_output([args.fio] + test['args']) |
102 | if test['output'] in output.decode(): | |
103 | print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])) | |
cd22f801 | 104 | passed = passed + 1 |
16e56d25 | 105 | else: |
5eac3b00 | 106 | print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])) |
cd22f801 | 107 | failed = failed + 1 |
16e56d25 VF |
108 | |
109 | # | |
110 | # test some read workloads | |
111 | # | |
112 | # if ss active and attained, | |
113 | # check that runtime is less than job time | |
114 | # check criteria | |
115 | # how to check ramp time? | |
116 | # | |
117 | # if ss inactive | |
118 | # check that runtime is what was specified | |
119 | # | |
32df42c9 VF |
120 | reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
121 | {'s': False, 'timeout': 20, 'numjobs': 2}, | |
122 | {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, | |
123 | {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, | |
16e56d25 VF |
124 | ] |
125 | ||
32df42c9 VF |
126 | if args.read == None: |
127 | if os.name == 'posix': | |
128 | args.read = '/dev/zero' | |
cd22f801 | 129 | extra = [ "--size=128M" ] |
32df42c9 | 130 | else: |
5eac3b00 | 131 | print("ERROR: file for read testing must be specified on non-posix systems") |
32df42c9 VF |
132 | sys.exit(1) |
133 | else: | |
134 | extra = [] | |
135 | ||
136 | jobnum = 0 | |
137 | for job in reads: | |
138 | ||
cd22f801 | 139 | tf = "steadystate_job{0}.json".format(jobnum) |
32df42c9 VF |
140 | parameters = [ "--name=job{0}".format(jobnum) ] |
141 | parameters.extend(extra) | |
142 | parameters.extend([ "--thread", | |
143 | "--output-format=json", | |
144 | "--output={0}".format(tf), | |
145 | "--filename={0}".format(args.read), | |
146 | "--rw=randrw", | |
147 | "--rwmixread=100", | |
148 | "--stonewall", | |
149 | "--group_reporting", | |
150 | "--numjobs={0}".format(job['numjobs']), | |
151 | "--time_based", | |
152 | "--runtime={0}".format(job['timeout']) ]) | |
153 | if job['s']: | |
154 | if job['iops']: | |
155 | ss = 'iops' | |
156 | else: | |
157 | ss = 'bw' | |
158 | if job['slope']: | |
159 | ss += "_slope" | |
160 | ss += ":" + str(job['ss_limit']) | |
161 | if job['pct']: | |
162 | ss += '%' | |
163 | parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']), | |
164 | '--ss={0}'.format(ss), | |
165 | '--ss_ramp={0}'.format(job['ss_ramp']) ]) | |
166 | ||
167 | output = subprocess.call([args.fio] + parameters) | |
168 | with open(tf, 'r') as source: | |
16e56d25 | 169 | jsondata = json.loads(source.read()) |
cd22f801 | 170 | source.close() |
32df42c9 VF |
171 | |
172 | for jsonjob in jsondata['jobs']: | |
cd22f801 | 173 | line = "{0}".format(jsonjob['job options']['name']) |
32df42c9 VF |
174 | if job['s']: |
175 | if jsonjob['steadystate']['attained'] == 1: | |
16e56d25 | 176 | # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit |
32df42c9 VF |
177 | mintime = (job['ss_dur'] + job['ss_ramp']) * 1000 |
178 | actual = jsonjob['read']['runtime'] | |
16e56d25 | 179 | if mintime > actual: |
32df42c9 | 180 | line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp']) |
cd22f801 | 181 | failed = failed + 1 |
16e56d25 | 182 | else: |
32df42c9 VF |
183 | line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) |
184 | objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], | |
185 | iops=job['iops'], | |
186 | slope=job['slope'], | |
187 | pct=job['pct'], | |
188 | limit=job['ss_limit'], | |
189 | dur=job['ss_dur'], | |
190 | criterion=jsonjob['steadystate']['criterion']) | |
16e56d25 | 191 | if not objsame: |
32df42c9 | 192 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) |
cd22f801 | 193 | failed = failed + 1 |
16e56d25 VF |
194 | else: |
195 | if met: | |
32df42c9 | 196 | line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit']) |
cd22f801 | 197 | passed = passed + 1 |
16e56d25 | 198 | else: |
32df42c9 | 199 | line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit']) |
cd22f801 | 200 | failed = failed + 1 |
16e56d25 VF |
201 | else: |
202 | # check runtime, confirm criterion calculation, and confirm that criterion was not met | |
32df42c9 VF |
203 | expected = job['timeout'] * 1000 |
204 | actual = jsonjob['read']['runtime'] | |
16e56d25 VF |
205 | if abs(expected - actual) > 10: |
206 | line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual) | |
207 | else: | |
32df42c9 VF |
208 | line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) |
209 | objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], | |
210 | iops=job['iops'], | |
211 | slope=job['slope'], | |
212 | pct=job['pct'], | |
213 | limit=job['ss_limit'], | |
214 | dur=job['ss_dur'], | |
215 | criterion=jsonjob['steadystate']['criterion']) | |
16e56d25 | 216 | if not objsame: |
32df42c9 VF |
217 | if actual > (job['ss_dur'] + job['ss_ramp'])*1000: |
218 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) | |
cd22f801 | 219 | failed = failed + 1 |
16e56d25 | 220 | else: |
32df42c9 | 221 | line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion']) |
cd22f801 | 222 | passed = passed + 1 |
16e56d25 VF |
223 | else: |
224 | if met: | |
32df42c9 | 225 | line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit']) |
cd22f801 | 226 | failed = failed + 1 |
16e56d25 | 227 | else: |
32df42c9 | 228 | line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit']) |
cd22f801 | 229 | passed = passed + 1 |
16e56d25 | 230 | else: |
32df42c9 VF |
231 | expected = job['timeout'] * 1000 |
232 | actual = jsonjob['read']['runtime'] | |
16e56d25 VF |
233 | if abs(expected - actual) < 10: |
234 | result = 'PASSED ' | |
cd22f801 | 235 | passed = passed + 1 |
16e56d25 VF |
236 | else: |
237 | result = 'FAILED ' | |
cd22f801 | 238 | failed = failed + 1 |
16e56d25 | 239 | line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual) |
5eac3b00 | 240 | print(line) |
32df42c9 VF |
241 | if 'steadystate' in jsonjob: |
242 | pp.pprint(jsonjob['steadystate']) | |
243 | jobnum += 1 | |
cd22f801 VF |
244 | |
245 | print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed)) | |
246 | sys.exit(failed) |