| 1 | #!/usr/bin/python |
| 2 | # |
| 3 | # steadystate_tests.py |
| 4 | # |
| 5 | # Test option parsing and functonality for fio's steady state detection feature. |
| 6 | # |
| 7 | # steadystate_tests.py ./fio file-for-read-testing file-for-write-testing |
| 8 | # |
| 9 | # REQUIREMENTS |
| 10 | # Python 2.6+ |
| 11 | # SciPy |
| 12 | # |
| 13 | # KNOWN ISSUES |
| 14 | # only option parsing and read tests are carried out |
| 15 | # the read test fails when ss_ramp > timeout because it tries to calculate the stopping criterion and finds that |
| 16 | # it does not match what fio reports |
| 17 | # min runtime: |
| 18 | # if ss attained: min runtime = ss_dur + ss_ramp |
| 19 | # if not attained: runtime = timeout |
| 20 | |
| 21 | import os |
| 22 | import json |
| 23 | import tempfile |
| 24 | import argparse |
| 25 | import subprocess |
| 26 | from scipy import stats |
| 27 | |
| 28 | def parse_args(): |
| 29 | parser = argparse.ArgumentParser() |
| 30 | parser.add_argument('fio', |
| 31 | help='path to fio executable'); |
| 32 | parser.add_argument('read', |
| 33 | help='target for read testing') |
| 34 | parser.add_argument('write', |
| 35 | help='target for write testing') |
| 36 | args = parser.parse_args() |
| 37 | |
| 38 | return args |
| 39 | |
| 40 | |
| 41 | def check(data, iops, slope, pct, limit, dur, criterion): |
| 42 | measurement = 'iops' if iops else 'bw' |
| 43 | data = data[measurement] |
| 44 | mean = sum(data) / len(data) |
| 45 | if slope: |
| 46 | x = range(len(data)) |
| 47 | m, intercept, r_value, p_value, std_err = stats.linregress(x,data) |
| 48 | m = abs(m) |
| 49 | if pct: |
| 50 | target = m / mean * 100 |
| 51 | criterion = criterion[:-1] |
| 52 | else: |
| 53 | target = m |
| 54 | else: |
| 55 | maxdev = 0 |
| 56 | for x in data: |
| 57 | maxdev = max(abs(mean-x), maxdev) |
| 58 | if pct: |
| 59 | target = maxdev / mean * 100 |
| 60 | criterion = criterion[:-1] |
| 61 | else: |
| 62 | target = maxdev |
| 63 | |
| 64 | criterion = float(criterion) |
| 65 | return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target |
| 66 | |
| 67 | |
| 68 | if __name__ == '__main__': |
| 69 | args = parse_args() |
| 70 | |
| 71 | # |
| 72 | # test option parsing |
| 73 | # |
| 74 | parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"], |
| 75 | 'output': "set steady state IOPS threshold to 10.000000" }, |
| 76 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"], |
| 77 | 'output': "set steady state threshold to 10.000000%" }, |
| 78 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"], |
| 79 | 'output': "set steady state threshold to 0.100000%" }, |
| 80 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"], |
| 81 | 'output': "set steady state threshold to 10.000000%" }, |
| 82 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"], |
| 83 | 'output': "set steady state threshold to 0.100000%" }, |
| 84 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"], |
| 85 | 'output': "set steady state BW threshold to 12" }, |
| 86 | ] |
| 87 | for test in parsing: |
| 88 | output = subprocess.check_output([args.fio] + test['args']); |
| 89 | if test['output'] in output: |
| 90 | print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args']) |
| 91 | else: |
| 92 | print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']) |
| 93 | |
| 94 | # |
| 95 | # test some read workloads |
| 96 | # |
| 97 | # if ss active and attained, |
| 98 | # check that runtime is less than job time |
| 99 | # check criteria |
| 100 | # how to check ramp time? |
| 101 | # |
| 102 | # if ss inactive |
| 103 | # check that runtime is what was specified |
| 104 | # |
| 105 | reads = [ [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 106 | {'s': False, 'timeout': 20, 'numjobs': 2}, |
| 107 | {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 108 | {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True} ], |
| 109 | ] |
| 110 | |
| 111 | accum = [] |
| 112 | suitenum = 0 |
| 113 | for suite in reads: |
| 114 | jobnum = 0 |
| 115 | for job in suite: |
| 116 | parameters = [ "--name=job{0}".format(jobnum), |
| 117 | "--thread", |
| 118 | "--filename={0}".format(args.read), |
| 119 | "--rw=randrw", "--rwmixread=100", "--stonewall", |
| 120 | "--group_reporting", "--numjobs={0}".format(job['numjobs']), |
| 121 | "--time_based", "--runtime={0}".format(job['timeout']) ] |
| 122 | if job['s']: |
| 123 | if job['iops']: |
| 124 | ss = 'iops' |
| 125 | else: |
| 126 | ss = 'bw' |
| 127 | if job['slope']: |
| 128 | ss += "_slope" |
| 129 | ss += ":" + str(job['ss_limit']) |
| 130 | if job['pct']: |
| 131 | ss += '%' |
| 132 | parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']), |
| 133 | '--ss={0}'.format(ss), |
| 134 | '--ss_ramp={0}'.format(job['ss_ramp']) ]) |
| 135 | accum.extend(parameters) |
| 136 | jobnum += 1 |
| 137 | |
| 138 | tf = tempfile.NamedTemporaryFile(delete=False) |
| 139 | tf.close() |
| 140 | output = subprocess.check_output([args.fio, |
| 141 | "--output-format=json", |
| 142 | "--output={0}".format(tf.name)] + accum) |
| 143 | with open(tf.name, 'r') as source: |
| 144 | jsondata = json.loads(source.read()) |
| 145 | os.remove(tf.name) |
| 146 | jobnum = 0 |
| 147 | for job in jsondata['jobs']: |
| 148 | line = "suite {0}, {1}".format(suitenum, job['job options']['name']) |
| 149 | if suite[jobnum]['s']: |
| 150 | if job['steadystate']['attained'] == 1: |
| 151 | # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit |
| 152 | mintime = (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp']) * 1000 |
| 153 | actual = job['read']['runtime'] |
| 154 | if mintime > actual: |
| 155 | line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp']) |
| 156 | else: |
| 157 | line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp']) |
| 158 | objsame, met, mean, target = check(data=job['steadystate']['data'], |
| 159 | iops=suite[jobnum]['iops'], |
| 160 | slope=suite[jobnum]['slope'], |
| 161 | pct=suite[jobnum]['pct'], |
| 162 | limit=suite[jobnum]['ss_limit'], |
| 163 | dur=suite[jobnum]['ss_dur'], |
| 164 | criterion=job['steadystate']['criterion']) |
| 165 | if not objsame: |
| 166 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']) |
| 167 | else: |
| 168 | if met: |
| 169 | line = 'PASSED ' + line + ' target {0} < limit {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']) |
| 170 | else: |
| 171 | line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']) |
| 172 | else: |
| 173 | # check runtime, confirm criterion calculation, and confirm that criterion was not met |
| 174 | expected = suite[jobnum]['timeout'] * 1000 |
| 175 | actual = job['read']['runtime'] |
| 176 | if abs(expected - actual) > 10: |
| 177 | line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual) |
| 178 | else: |
| 179 | line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, suite[jobnum]['ss_dur'], suite[jobnum]['ss_ramp']) |
| 180 | objsame, met, mean, target = check(data=job['steadystate']['data'], |
| 181 | iops=suite[jobnum]['iops'], |
| 182 | slope=suite[jobnum]['slope'], |
| 183 | pct=suite[jobnum]['pct'], |
| 184 | limit=suite[jobnum]['ss_limit'], |
| 185 | dur=suite[jobnum]['ss_dur'], |
| 186 | criterion=job['steadystate']['criterion']) |
| 187 | if not objsame: |
| 188 | if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000: |
| 189 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']) |
| 190 | else: |
| 191 | line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed, data: {1} '.format(job['steadystate']['criterion'], job['steadystate']) |
| 192 | else: |
| 193 | if met: |
| 194 | line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']) |
| 195 | else: |
| 196 | line = 'PASSED ' + line + ' criterion {0} > threshold {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']) |
| 197 | else: |
| 198 | expected = suite[jobnum]['timeout'] * 1000 |
| 199 | actual = job['read']['runtime'] |
| 200 | if abs(expected - actual) < 10: |
| 201 | result = 'PASSED ' |
| 202 | else: |
| 203 | result = 'FAILED ' |
| 204 | line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual) |
| 205 | print line |
| 206 | jobnum += 1 |
| 207 | suitenum += 1 |