| 1 | #!/usr/bin/python2.7 |
| 2 | # Note: this script is python2 and python 3 compatible. |
| 3 | # |
| 4 | # steadystate_tests.py |
| 5 | # |
| 6 | # Test option parsing and functonality for fio's steady state detection feature. |
| 7 | # |
| 8 | # steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio |
| 9 | # |
| 10 | # REQUIREMENTS |
| 11 | # Python 2.6+ |
| 12 | # SciPy |
| 13 | # |
| 14 | # KNOWN ISSUES |
| 15 | # only option parsing and read tests are carried out |
| 16 | # On Windows this script works under Cygwin but not from cmd.exe |
| 17 | # On Windows I encounter frequent fio problems generating JSON output (nothing to decode) |
| 18 | # min runtime: |
| 19 | # if ss attained: min runtime = ss_dur + ss_ramp |
| 20 | # if not attained: runtime = timeout |
| 21 | |
| 22 | from __future__ import absolute_import |
| 23 | from __future__ import print_function |
| 24 | import os |
| 25 | import sys |
| 26 | import json |
| 27 | import uuid |
| 28 | import pprint |
| 29 | import argparse |
| 30 | import subprocess |
| 31 | from scipy import stats |
| 32 | from six.moves import range |
| 33 | |
| 34 | def parse_args(): |
| 35 | parser = argparse.ArgumentParser() |
| 36 | parser.add_argument('fio', |
| 37 | help='path to fio executable') |
| 38 | parser.add_argument('--read', |
| 39 | help='target for read testing') |
| 40 | parser.add_argument('--write', |
| 41 | help='target for write testing') |
| 42 | args = parser.parse_args() |
| 43 | |
| 44 | return args |
| 45 | |
| 46 | |
| 47 | def check(data, iops, slope, pct, limit, dur, criterion): |
| 48 | measurement = 'iops' if iops else 'bw' |
| 49 | data = data[measurement] |
| 50 | mean = sum(data) / len(data) |
| 51 | if slope: |
| 52 | x = list(range(len(data))) |
| 53 | m, intercept, r_value, p_value, std_err = stats.linregress(x,data) |
| 54 | m = abs(m) |
| 55 | if pct: |
| 56 | target = m / mean * 100 |
| 57 | criterion = criterion[:-1] |
| 58 | else: |
| 59 | target = m |
| 60 | else: |
| 61 | maxdev = 0 |
| 62 | for x in data: |
| 63 | maxdev = max(abs(mean-x), maxdev) |
| 64 | if pct: |
| 65 | target = maxdev / mean * 100 |
| 66 | criterion = criterion[:-1] |
| 67 | else: |
| 68 | target = maxdev |
| 69 | |
| 70 | criterion = float(criterion) |
| 71 | return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target |
| 72 | |
| 73 | |
| 74 | if __name__ == '__main__': |
| 75 | args = parse_args() |
| 76 | |
| 77 | pp = pprint.PrettyPrinter(indent=4) |
| 78 | |
| 79 | # |
| 80 | # test option parsing |
| 81 | # |
| 82 | parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"], |
| 83 | 'output': "set steady state IOPS threshold to 10.000000" }, |
| 84 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"], |
| 85 | 'output': "set steady state threshold to 10.000000%" }, |
| 86 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"], |
| 87 | 'output': "set steady state threshold to 0.100000%" }, |
| 88 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"], |
| 89 | 'output': "set steady state threshold to 10.000000%" }, |
| 90 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"], |
| 91 | 'output': "set steady state threshold to 0.100000%" }, |
| 92 | { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"], |
| 93 | 'output': "set steady state BW threshold to 12" }, |
| 94 | ] |
| 95 | for test in parsing: |
| 96 | output = subprocess.check_output([args.fio] + test['args']) |
| 97 | if test['output'] in output.decode(): |
| 98 | print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])) |
| 99 | else: |
| 100 | print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])) |
| 101 | |
| 102 | # |
| 103 | # test some read workloads |
| 104 | # |
| 105 | # if ss active and attained, |
| 106 | # check that runtime is less than job time |
| 107 | # check criteria |
| 108 | # how to check ramp time? |
| 109 | # |
| 110 | # if ss inactive |
| 111 | # check that runtime is what was specified |
| 112 | # |
| 113 | reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 114 | {'s': False, 'timeout': 20, 'numjobs': 2}, |
| 115 | {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 116 | {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True}, |
| 117 | ] |
| 118 | |
| 119 | if args.read == None: |
| 120 | if os.name == 'posix': |
| 121 | args.read = '/dev/zero' |
| 122 | extra = [ "--size=134217728" ] # 128 MiB |
| 123 | else: |
| 124 | print("ERROR: file for read testing must be specified on non-posix systems") |
| 125 | sys.exit(1) |
| 126 | else: |
| 127 | extra = [] |
| 128 | |
| 129 | jobnum = 0 |
| 130 | for job in reads: |
| 131 | |
| 132 | tf = uuid.uuid4().hex |
| 133 | parameters = [ "--name=job{0}".format(jobnum) ] |
| 134 | parameters.extend(extra) |
| 135 | parameters.extend([ "--thread", |
| 136 | "--output-format=json", |
| 137 | "--output={0}".format(tf), |
| 138 | "--filename={0}".format(args.read), |
| 139 | "--rw=randrw", |
| 140 | "--rwmixread=100", |
| 141 | "--stonewall", |
| 142 | "--group_reporting", |
| 143 | "--numjobs={0}".format(job['numjobs']), |
| 144 | "--time_based", |
| 145 | "--runtime={0}".format(job['timeout']) ]) |
| 146 | if job['s']: |
| 147 | if job['iops']: |
| 148 | ss = 'iops' |
| 149 | else: |
| 150 | ss = 'bw' |
| 151 | if job['slope']: |
| 152 | ss += "_slope" |
| 153 | ss += ":" + str(job['ss_limit']) |
| 154 | if job['pct']: |
| 155 | ss += '%' |
| 156 | parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']), |
| 157 | '--ss={0}'.format(ss), |
| 158 | '--ss_ramp={0}'.format(job['ss_ramp']) ]) |
| 159 | |
| 160 | output = subprocess.call([args.fio] + parameters) |
| 161 | with open(tf, 'r') as source: |
| 162 | jsondata = json.loads(source.read()) |
| 163 | os.remove(tf) |
| 164 | |
| 165 | for jsonjob in jsondata['jobs']: |
| 166 | line = "job {0}".format(jsonjob['job options']['name']) |
| 167 | if job['s']: |
| 168 | if jsonjob['steadystate']['attained'] == 1: |
| 169 | # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit |
| 170 | mintime = (job['ss_dur'] + job['ss_ramp']) * 1000 |
| 171 | actual = jsonjob['read']['runtime'] |
| 172 | if mintime > actual: |
| 173 | line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp']) |
| 174 | else: |
| 175 | line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) |
| 176 | objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], |
| 177 | iops=job['iops'], |
| 178 | slope=job['slope'], |
| 179 | pct=job['pct'], |
| 180 | limit=job['ss_limit'], |
| 181 | dur=job['ss_dur'], |
| 182 | criterion=jsonjob['steadystate']['criterion']) |
| 183 | if not objsame: |
| 184 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) |
| 185 | else: |
| 186 | if met: |
| 187 | line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit']) |
| 188 | else: |
| 189 | line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit']) |
| 190 | else: |
| 191 | # check runtime, confirm criterion calculation, and confirm that criterion was not met |
| 192 | expected = job['timeout'] * 1000 |
| 193 | actual = jsonjob['read']['runtime'] |
| 194 | if abs(expected - actual) > 10: |
| 195 | line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual) |
| 196 | else: |
| 197 | line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp']) |
| 198 | objsame, met, mean, target = check(data=jsonjob['steadystate']['data'], |
| 199 | iops=job['iops'], |
| 200 | slope=job['slope'], |
| 201 | pct=job['pct'], |
| 202 | limit=job['ss_limit'], |
| 203 | dur=job['ss_dur'], |
| 204 | criterion=jsonjob['steadystate']['criterion']) |
| 205 | if not objsame: |
| 206 | if actual > (job['ss_dur'] + job['ss_ramp'])*1000: |
| 207 | line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target) |
| 208 | else: |
| 209 | line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion']) |
| 210 | else: |
| 211 | if met: |
| 212 | line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit']) |
| 213 | else: |
| 214 | line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit']) |
| 215 | else: |
| 216 | expected = job['timeout'] * 1000 |
| 217 | actual = jsonjob['read']['runtime'] |
| 218 | if abs(expected - actual) < 10: |
| 219 | result = 'PASSED ' |
| 220 | else: |
| 221 | result = 'FAILED ' |
| 222 | line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual) |
| 223 | print(line) |
| 224 | if 'steadystate' in jsonjob: |
| 225 | pp.pprint(jsonjob['steadystate']) |
| 226 | jobnum += 1 |