| 1 | #!/usr/bin/env python3 |
| 2 | # |
| 3 | # steadystate_tests.py |
| 4 | # |
| 5 | # Test option parsing and functonality for fio's steady state detection feature. |
| 6 | # |
| 7 | # steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio |
| 8 | # |
| 9 | # REQUIREMENTS |
| 10 | # Python 2.6+ |
| 11 | # SciPy |
| 12 | # |
| 13 | # KNOWN ISSUES |
| 14 | # only option parsing and read tests are carried out |
| 15 | # On Windows this script works under Cygwin but not from cmd.exe |
| 16 | # On Windows I encounter frequent fio problems generating JSON output (nothing to decode) |
| 17 | # min runtime: |
| 18 | # if ss attained: min runtime = ss_dur + ss_ramp |
| 19 | # if not attained: runtime = timeout |
| 20 | |
| 21 | from __future__ import absolute_import |
| 22 | from __future__ import print_function |
| 23 | import os |
| 24 | import sys |
| 25 | import json |
| 26 | import pprint |
| 27 | import argparse |
| 28 | import subprocess |
| 29 | from scipy import stats |
| 30 | |
| 31 | def parse_args(): |
| 32 | parser = argparse.ArgumentParser() |
| 33 | parser.add_argument('fio', help='path to fio executable') |
| 34 | args = parser.parse_args() |
| 35 | |
| 36 | return args |
| 37 | |
| 38 | |
| 39 | def check(data, iops, slope, pct, limit, dur, criterion): |
| 40 | measurement = 'iops' if iops else 'bw' |
| 41 | data = data[measurement] |
| 42 | mean = sum(data) / len(data) |
| 43 | if slope: |
| 44 | x = list(range(len(data))) |
| 45 | m, intercept, r_value, p_value, std_err = stats.linregress(x,data) |
| 46 | m = abs(m) |
| 47 | if pct: |
| 48 | target = (m / mean * 100) if mean != 0 else 0 |
| 49 | criterion = criterion[:-1] |
| 50 | else: |
| 51 | target = m |
| 52 | else: |
| 53 | maxdev = 0 |
| 54 | for x in data: |
| 55 | maxdev = max(abs(mean-x), maxdev) |
| 56 | if pct: |
| 57 | target = maxdev / mean * 100 |
| 58 | criterion = criterion[:-1] |
| 59 | else: |
| 60 | target = maxdev |
| 61 | |
| 62 | criterion = float(criterion) |
| 63 | if criterion == 0.0: |
| 64 | objsame = False |
| 65 | else: |
| 66 | objsame = abs(target - criterion) / criterion < 0.005 |
| 67 | return (objsame, target < limit, mean, target) |
| 68 | |
| 69 | |
| 70 | if __name__ == '__main__': |
| 71 | args = parse_args() |
| 72 | |
| 73 | pp = pprint.PrettyPrinter(indent=4) |
| 74 | |
| 75 | passed = 0 |
| 76 | failed = 0 |
| 77 | |
| 78 | # |
| 79 | # test option parsing |
| 80 | # |
| 81 | parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"], |
| 82 | 'output': "set steady state IOPS threshold to 10.000000" }, |
| 83 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"], |
| 84 | 'output': "set steady state threshold to 10.000000%" }, |
| 85 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"], |
| 86 | 'output': "set steady state threshold to 0.100000%" }, |
| 87 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"], |
| 88 | 'output': "set steady state threshold to 10.000000%" }, |
| 89 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"], |
| 90 | 'output': "set steady state threshold to 0.100000%" }, |
| 91 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"], |
| 92 | 'output': "set steady state BW threshold to 12" }, |
| 93 | ] |
| 94 | for test in parsing: |
| 95 | output = subprocess.check_output([args.fio] + test['args']) |
| 96 | if test['output'] in output.decode(): |
| 97 | print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])) |
| 98 | passed = passed + 1 |
| 99 | else: |
| 100 | print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])) |
| 101 | failed = failed + 1 |
| 102 | |
| 103 | # |
| 104 | # test some read workloads |
| 105 | # |
| 106 | # if ss active and attained, |
| 107 | # check that runtime is less than job time |
| 108 | # check criteria |
| 109 | # how to check ramp time? |
| 110 | # |
| 111 | # if ss inactive |
| 112 | # check that runtime is what was specified |
| 113 | # |
| 114 | reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 115 | {'s': False, 'timeout': 20, 'numjobs': 2}, |
| 116 | {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 117 | {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 118 | ] |
| 119 | |
| 120 | jobnum = 0 |
| 121 | for job in reads: |
| 122 | |
| 123 | tf = "steadystate_job{0}.json".format(jobnum) |
| 124 | parameters = [ "--max-jobs=16", "--name=job{0}".format(jobnum) ] |
| 125 | parameters.extend([ "--thread", |
| 126 | "--output-format=json", |
| 127 | "--output={0}".format(tf), |
| 128 | "--ioengine=null", |
| 129 | "--size=1G", |
| 130 | "--rw=randrw", |
| 131 | "--rwmixread=100", |
| 132 | "--stonewall", |
| 133 | "--group_reporting", |
| 134 | "--numjobs={0}".format(job['numjobs']), |
| 135 | "--time_based", |
| 136 | "--runtime={0}".format(job['timeout']) ]) |
| 137 | if job['s']: |
| 138 | if job['iops']: |
| 139 | ss = 'iops' |
| 140 | else: |
| 141 | ss = 'bw' |
| 142 | if job['slope']: |
| 143 | ss += "_slope" |
| 144 | ss += ":" + str(job['ss_limit']) |
| 145 | if job['pct']: |
| 146 | ss += '%' |
| 147 | parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']), |
| 148 | '--ss={0}'.format(ss), |
| 149 | '--ss_ramp={0}'.format(job['ss_ramp']) ]) |
| 150 | |
| 151 | output = subprocess.call([args.fio] + parameters) |
| 152 | with open(tf, 'r') as source: |
| 153 | jsondata = json.loads(source.read()) |
| 154 | source.close() |
| 155 | |
| 156 | for jsonjob in jsondata['jobs']: |
| 157 | line = "{0}".format(jsonjob['job options']['name']) |
| 158 | if job['s']: |
| 159 | if jsonjob['steadystate']['attained'] == 1: |
| 160 | # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit |
| 161 | mintime = (job['ss_dur'] + job['ss_ramp']) * 1000 |
| 162 | actual = jsonjob['read']['runtime'] |
| 163 | if mintime > actual: |
| 164 | line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp']) |
| 165 | failed = failed + 1 |
| 166 | else: |
| 167 | line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) |
| 168 | objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], |
| 169 | iops=job['iops'], |
| 170 | slope=job['slope'], |
| 171 | pct=job['pct'], |
| 172 | limit=job['ss_limit'], |
| 173 | dur=job['ss_dur'], |
| 174 | criterion=jsonjob['steadystate']['criterion']) |
| 175 | if not objsame: |
| 176 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) |
| 177 | failed = failed + 1 |
| 178 | else: |
| 179 | if met: |
| 180 | line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit']) |
| 181 | passed = passed + 1 |
| 182 | else: |
| 183 | line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit']) |
| 184 | failed = failed + 1 |
| 185 | else: |
| 186 | # check runtime, confirm criterion calculation, and confirm that criterion was not met |
| 187 | expected = job['timeout'] * 1000 |
| 188 | actual = jsonjob['read']['runtime'] |
| 189 | if abs(expected - actual) > 50: |
| 190 | line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual) |
| 191 | else: |
| 192 | line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) |
| 193 | objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], |
| 194 | iops=job['iops'], |
| 195 | slope=job['slope'], |
| 196 | pct=job['pct'], |
| 197 | limit=job['ss_limit'], |
| 198 | dur=job['ss_dur'], |
| 199 | criterion=jsonjob['steadystate']['criterion']) |
| 200 | if not objsame: |
| 201 | if actual > (job['ss_dur'] + job['ss_ramp'])*1000: |
| 202 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) |
| 203 | failed = failed + 1 |
| 204 | else: |
| 205 | line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion']) |
| 206 | passed = passed + 1 |
| 207 | else: |
| 208 | if met: |
| 209 | line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit']) |
| 210 | failed = failed + 1 |
| 211 | else: |
| 212 | line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit']) |
| 213 | passed = passed + 1 |
| 214 | else: |
| 215 | expected = job['timeout'] * 1000 |
| 216 | actual = jsonjob['read']['runtime'] |
| 217 | if abs(expected - actual) > 50: |
| 218 | result = 'FAILED ' |
| 219 | failed = failed + 1 |
| 220 | else: |
| 221 | result = 'PASSED ' |
| 222 | passed = passed + 1 |
| 223 | line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual) |
| 224 | print(line) |
| 225 | if 'steadystate' in jsonjob: |
| 226 | pp.pprint(jsonjob['steadystate']) |
| 227 | jobnum += 1 |
| 228 | |
| 229 | print("{0} test(s) PASSED, {1} test(s) FAILED".format(passed,failed)) |
| 230 | sys.exit(failed) |