| 1 | #!/usr/bin/python |
| 2 | # |
| 3 | # steadystate_tests.py |
| 4 | # |
| 5 | # Test option parsing and functonality for fio's steady state detection feature. |
| 6 | # |
| 7 | # steadystate_tests.py ./fio file-for-read-testing file-for-write-testing |
| 8 | # |
| 9 | # REQUIREMENTS |
| 10 | # Python 2.6+ |
| 11 | # SciPy |
| 12 | # |
| 13 | # KNOWN ISSUES |
| 14 | # only option parsing and read tests are carried out |
| 15 | # On Windows this script works under Cygwin but not from cmd.exe |
| 16 | # On Windows I encounter frequent fio problems generating JSON output (nothing to decode) |
| 17 | # min runtime: |
| 18 | # if ss attained: min runtime = ss_dur + ss_ramp |
| 19 | # if not attained: runtime = timeout |
| 20 | |
| 21 | import os |
| 22 | import sys |
| 23 | import json |
| 24 | import uuid |
| 25 | import pprint |
| 26 | import argparse |
| 27 | import subprocess |
| 28 | from scipy import stats |
| 29 | |
| 30 | def parse_args(): |
| 31 | parser = argparse.ArgumentParser() |
| 32 | parser.add_argument('fio', |
| 33 | help='path to fio executable'); |
| 34 | parser.add_argument('--read', |
| 35 | help='target for read testing') |
| 36 | parser.add_argument('--write', |
| 37 | help='target for write testing') |
| 38 | args = parser.parse_args() |
| 39 | |
| 40 | return args |
| 41 | |
| 42 | |
| 43 | def check(data, iops, slope, pct, limit, dur, criterion): |
| 44 | measurement = 'iops' if iops else 'bw' |
| 45 | data = data[measurement] |
| 46 | mean = sum(data) / len(data) |
| 47 | if slope: |
| 48 | x = range(len(data)) |
| 49 | m, intercept, r_value, p_value, std_err = stats.linregress(x,data) |
| 50 | m = abs(m) |
| 51 | if pct: |
| 52 | target = m / mean * 100 |
| 53 | criterion = criterion[:-1] |
| 54 | else: |
| 55 | target = m |
| 56 | else: |
| 57 | maxdev = 0 |
| 58 | for x in data: |
| 59 | maxdev = max(abs(mean-x), maxdev) |
| 60 | if pct: |
| 61 | target = maxdev / mean * 100 |
| 62 | criterion = criterion[:-1] |
| 63 | else: |
| 64 | target = maxdev |
| 65 | |
| 66 | criterion = float(criterion) |
| 67 | return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target |
| 68 | |
| 69 | |
| 70 | if __name__ == '__main__': |
| 71 | args = parse_args() |
| 72 | |
| 73 | pp = pprint.PrettyPrinter(indent=4) |
| 74 | |
| 75 | # |
| 76 | # test option parsing |
| 77 | # |
| 78 | parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"], |
| 79 | 'output': "set steady state IOPS threshold to 10.000000" }, |
| 80 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"], |
| 81 | 'output': "set steady state threshold to 10.000000%" }, |
| 82 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"], |
| 83 | 'output': "set steady state threshold to 0.100000%" }, |
| 84 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"], |
| 85 | 'output': "set steady state threshold to 10.000000%" }, |
| 86 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"], |
| 87 | 'output': "set steady state threshold to 0.100000%" }, |
| 88 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"], |
| 89 | 'output': "set steady state BW threshold to 12" }, |
| 90 | ] |
| 91 | for test in parsing: |
| 92 | output = subprocess.check_output([args.fio] + test['args']); |
| 93 | if test['output'] in output: |
| 94 | print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args']) |
| 95 | else: |
| 96 | print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']) |
| 97 | |
| 98 | # |
| 99 | # test some read workloads |
| 100 | # |
| 101 | # if ss active and attained, |
| 102 | # check that runtime is less than job time |
| 103 | # check criteria |
| 104 | # how to check ramp time? |
| 105 | # |
| 106 | # if ss inactive |
| 107 | # check that runtime is what was specified |
| 108 | # |
| 109 | reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 110 | {'s': False, 'timeout': 20, 'numjobs': 2}, |
| 111 | {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 112 | {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 113 | ] |
| 114 | |
| 115 | if args.read == None: |
| 116 | if os.name == 'posix': |
| 117 | args.read = '/dev/zero' |
| 118 | extra = [ "--size=134217728" ] # 128 MiB |
| 119 | else: |
| 120 | print "ERROR: file for read testing must be specified on non-posix systems" |
| 121 | sys.exit(1) |
| 122 | else: |
| 123 | extra = [] |
| 124 | |
| 125 | jobnum = 0 |
| 126 | for job in reads: |
| 127 | |
| 128 | tf = uuid.uuid4().hex |
| 129 | parameters = [ "--name=job{0}".format(jobnum) ] |
| 130 | parameters.extend(extra) |
| 131 | parameters.extend([ "--thread", |
| 132 | "--output-format=json", |
| 133 | "--output={0}".format(tf), |
| 134 | "--filename={0}".format(args.read), |
| 135 | "--rw=randrw", |
| 136 | "--rwmixread=100", |
| 137 | "--stonewall", |
| 138 | "--group_reporting", |
| 139 | "--numjobs={0}".format(job['numjobs']), |
| 140 | "--time_based", |
| 141 | "--runtime={0}".format(job['timeout']) ]) |
| 142 | if job['s']: |
| 143 | if job['iops']: |
| 144 | ss = 'iops' |
| 145 | else: |
| 146 | ss = 'bw' |
| 147 | if job['slope']: |
| 148 | ss += "_slope" |
| 149 | ss += ":" + str(job['ss_limit']) |
| 150 | if job['pct']: |
| 151 | ss += '%' |
| 152 | parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']), |
| 153 | '--ss={0}'.format(ss), |
| 154 | '--ss_ramp={0}'.format(job['ss_ramp']) ]) |
| 155 | |
| 156 | output = subprocess.call([args.fio] + parameters) |
| 157 | with open(tf, 'r') as source: |
| 158 | jsondata = json.loads(source.read()) |
| 159 | os.remove(tf) |
| 160 | |
| 161 | for jsonjob in jsondata['jobs']: |
| 162 | line = "job {0}".format(jsonjob['job options']['name']) |
| 163 | if job['s']: |
| 164 | if jsonjob['steadystate']['attained'] == 1: |
| 165 | # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit |
| 166 | mintime = (job['ss_dur'] + job['ss_ramp']) * 1000 |
| 167 | actual = jsonjob['read']['runtime'] |
| 168 | if mintime > actual: |
| 169 | line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp']) |
| 170 | else: |
| 171 | line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) |
| 172 | objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], |
| 173 | iops=job['iops'], |
| 174 | slope=job['slope'], |
| 175 | pct=job['pct'], |
| 176 | limit=job['ss_limit'], |
| 177 | dur=job['ss_dur'], |
| 178 | criterion=jsonjob['steadystate']['criterion']) |
| 179 | if not objsame: |
| 180 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) |
| 181 | else: |
| 182 | if met: |
| 183 | line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit']) |
| 184 | else: |
| 185 | line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit']) |
| 186 | else: |
| 187 | # check runtime, confirm criterion calculation, and confirm that criterion was not met |
| 188 | expected = job['timeout'] * 1000 |
| 189 | actual = jsonjob['read']['runtime'] |
| 190 | if abs(expected - actual) > 10: |
| 191 | line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual) |
| 192 | else: |
| 193 | line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) |
| 194 | objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], |
| 195 | iops=job['iops'], |
| 196 | slope=job['slope'], |
| 197 | pct=job['pct'], |
| 198 | limit=job['ss_limit'], |
| 199 | dur=job['ss_dur'], |
| 200 | criterion=jsonjob['steadystate']['criterion']) |
| 201 | if not objsame: |
| 202 | if actual > (job['ss_dur'] + job['ss_ramp'])*1000: |
| 203 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) |
| 204 | else: |
| 205 | line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion']) |
| 206 | else: |
| 207 | if met: |
| 208 | line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit']) |
| 209 | else: |
| 210 | line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit']) |
| 211 | else: |
| 212 | expected = job['timeout'] * 1000 |
| 213 | actual = jsonjob['read']['runtime'] |
| 214 | if abs(expected - actual) < 10: |
| 215 | result = 'PASSED ' |
| 216 | else: |
| 217 | result = 'FAILED ' |
| 218 | line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual) |
| 219 | print line |
| 220 | if 'steadystate' in jsonjob: |
| 221 | pp.pprint(jsonjob['steadystate']) |
| 222 | jobnum += 1 |