Commit | Line | Data |
---|---|---|
16e56d25 VF |
1 | #!/usr/bin/python |
2 | # | |
3 | # steadystate_tests.py | |
4 | # | |
5 | # Test option parsing and functonality for fio's steady state detection feature. | |
6 | # | |
7 | # steadystate_tests.py ./fio file-for-read-testing file-for-write-testing | |
8 | # | |
9 | # REQUIREMENTS | |
10 | # Python 2.6+ | |
11 | # SciPy | |
12 | # | |
13 | # KNOWN ISSUES | |
14 | # only option parsing and read tests are carried out | |
15 | # the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that | |
16 | # it does not match what fio reports | |
17 | # min runtime: | |
18 | # if ss attained: min runtime = ss_dur + ss_ramp | |
19 | # if not attained: runtime = timeout | |
20 | ||
21 | import os | |
22 | import json | |
23 | import tempfile | |
24 | import argparse | |
25 | import subprocess | |
26 | from scipy import stats | |
27 | ||
28 | def parse_args(): | |
29 | parser = argparse.ArgumentParser() | |
30 | parser.add_argument('fio', | |
31 | help='path to fio executable'); | |
32 | parser.add_argument('read', | |
33 | help='target for read testing') | |
34 | parser.add_argument('write', | |
35 | help='target for write testing') | |
36 | args = parser.parse_args() | |
37 | ||
38 | return args | |
39 | ||
40 | ||
41 | def check(data, iops, slope, pct, limit, dur, criterion): | |
ba8fb6f6 VF |
42 | measurement = 'iops' if iops else 'bw' |
43 | data = data[measurement] | |
16e56d25 VF |
44 | mean = sum(data) / len(data) |
45 | if slope: | |
46 | x = range(len(data)) | |
47 | m, intercept, r_value, p_value, std_err = stats.linregress(x,data) | |
48 | m = abs(m) | |
49 | if pct: | |
ba8fb6f6 | 50 | target = m / mean |
16e56d25 VF |
51 | else: |
52 | target = m | |
53 | else: | |
54 | maxdev = 0 | |
55 | for x in data: | |
56 | maxdev = max(abs(mean-x), maxdev) | |
57 | if pct: | |
ba8fb6f6 | 58 | target = maxdev / mean |
16e56d25 VF |
59 | else: |
60 | target = maxdev | |
61 | ||
ba8fb6f6 | 62 | return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target |
16e56d25 VF |
63 | |
64 | ||
65 | if __name__ == '__main__': | |
66 | args = parse_args() | |
67 | ||
68 | # | |
69 | # test option parsing | |
70 | # | |
71 | parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"], | |
72 | 'output': "set steady state IOPS threshold to 10.000000" }, | |
73 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"], | |
74 | 'output': "set steady state threshold to 10.000000%" }, | |
75 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"], | |
76 | 'output': "set steady state threshold to 0.100000%" }, | |
77 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"], | |
78 | 'output': "set steady state threshold to 10.000000%" }, | |
79 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"], | |
80 | 'output': "set steady state threshold to 0.100000%" }, | |
81 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"], | |
82 | 'output': "set steady state BW threshold to 12" }, | |
83 | ] | |
84 | for test in parsing: | |
85 | output = subprocess.check_output([args.fio] + test['args']); | |
86 | if test['output'] in output: | |
87 | print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args']) | |
88 | else: | |
89 | print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']) | |
90 | ||
91 | # | |
92 | # test some read workloads | |
93 | # | |
94 | # if ss active and attained, | |
95 | # check that runtime is less than job time | |
96 | # check criteria | |
97 | # how to check ramp time? | |
98 | # | |
99 | # if ss inactive | |
100 | # check that runtime is what was specified | |
101 | # | |
102 | reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True}, | |
103 | {'s': False, 'timeout': 20, 'numjobs': 2}, | |
104 | {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, | |
105 | {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ], | |
106 | ] | |
107 | ||
108 | accum = [] | |
109 | suitenum = 0 | |
110 | for suite in reads: | |
111 | jobnum = 0 | |
112 | for job in suite: | |
113 | parameters = [ "--name=job{0}".format(jobnum), | |
114 | "--thread", | |
115 | "--filename={0}".format(args.read), | |
116 | "--rw=randrw", "--rwmixread=100", "--stonewall", | |
117 | "--group_reporting", "--numjobs={0}".format(job['numjobs']), | |
118 | "--time_based", "--runtime={0}".format(job['timeout']) ] | |
119 | if job['s']: | |
120 | if job['iops']: | |
121 | ss = 'iops' | |
122 | else: | |
123 | ss = 'bw' | |
124 | if job['slope']: | |
125 | ss += "_slope" | |
126 | ss += ":" + str(job['ss_limit']) | |
127 | if job['pct']: | |
128 | ss += '%' | |
129 | parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']), | |
130 | '--ss={0}'.format(ss), | |
131 | '--ss_ramp={0}'.format(job['ss_ramp']) ]) | |
132 | accum.extend(parameters) | |
133 | jobnum += 1 | |
134 | ||
135 | tf = tempfile.NamedTemporaryFile(delete=False) | |
136 | tf.close() | |
137 | output = subprocess.check_output([args.fio, | |
138 | "--output-format=json", | |
139 | "--output={0}".format(tf.name)] + accum) | |
140 | with open(tf.name, 'r') as source: | |
141 | jsondata = json.loads(source.read()) | |
142 | os.remove(tf.name) | |
143 | jobnum = 0 | |
144 | for job in jsondata['jobs']: | |
145 | line = "suite {0}, {1}".format(suitenum, job['job options']['name']) | |
146 | if suite[jobnum]['s']: | |
147 | if job['steadystate']['attained'] == 1: | |
148 | # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit | |
149 | mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000 | |
150 | actual = job['read']['runtime'] | |
151 | if mintime > actual: | |
152 | line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp']) | |
153 | else: | |
154 | line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp']) | |
155 | objsame, met, mean, target = check(data=job['steadystate']['data'], | |
156 | iops=suite[jobnum]['iops'], | |
157 | slope=suite[jobnum]['slope'], | |
158 | pct=suite[jobnum]['pct'], | |
159 | limit=suite[jobnum]['ss_limit'], | |
160 | dur=suite[jobnum]['ss_dur'], | |
161 | criterion=job['steadystate']['criterion']) | |
162 | if not objsame: | |
ba8fb6f6 | 163 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']) |
16e56d25 VF |
164 | else: |
165 | if met: | |
ba8fb6f6 | 166 | line = 'PASSED ' + line + ' target {0} < limit {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']) |
16e56d25 | 167 | else: |
ba8fb6f6 | 168 | line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']) |
16e56d25 VF |
169 | |
170 | else: | |
171 | # check runtime, confirm criterion calculation, and confirm that criterion was not met | |
172 | expected = suite[jobnum]['timeout'] * 1000 | |
173 | actual = job['read']['runtime'] | |
174 | if abs(expected - actual) > 10: | |
175 | line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual) | |
176 | else: | |
177 | line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp']) | |
178 | objsame, met, mean, target = check(data=job['steadystate']['data'], | |
179 | iops=suite[jobnum]['iops'], | |
180 | slope=suite[jobnum]['slope'], | |
181 | pct=suite[jobnum]['pct'], | |
182 | limit=suite[jobnum]['ss_limit'], | |
183 | dur=suite[jobnum]['ss_dur'], | |
184 | criterion=job['steadystate']['criterion']) | |
185 | if not objsame: | |
186 | if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000: | |
ba8fb6f6 | 187 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']) |
16e56d25 | 188 | else: |
ba8fb6f6 | 189 | line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed, data: {1} '.format(job['steadystate']['criterion'], job['steadystate']) |
16e56d25 VF |
190 | else: |
191 | if met: | |
ba8fb6f6 | 192 | line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']) |
16e56d25 | 193 | else: |
ba8fb6f6 | 194 | line = 'PASSED ' + line + ' criterion {0} > threshold {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']) |
16e56d25 VF |
195 | else: |
196 | expected = suite[jobnum]['timeout'] * 1000 | |
197 | actual = job['read']['runtime'] | |
198 | if abs(expected - actual) < 10: | |
199 | result = 'PASSED ' | |
200 | else: | |
201 | result = 'FAILED ' | |
202 | line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual) | |
203 | print line | |
204 | jobnum += 1 | |
205 | suitenum += 1 |