--- /dev/null
+#!/usr/bin/python2.7
+# Note: this script is python2 and python 3 compatible.
+#
+# steadystate_tests.py
+#
+# Test option parsing and functonality for fio's steady state detection feature.
+#
+# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
+#
+# REQUIREMENTS
+# Python 2.6+
+# SciPy
+#
+# KNOWN ISSUES
+# only option parsing and read tests are carried out
+# On Windows this script works under Cygwin but not from cmd.exe
+# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
+# min runtime:
+# if ss attained: min runtime = ss_dur + ss_ramp
+# if not attained: runtime = timeout
+
+from __future__ import absolute_import
+from __future__ import print_function
+import os
+import sys
+import json
+import uuid
+import pprint
+import argparse
+import subprocess
+from scipy import stats
+from six.moves import range
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('fio',
+ help='path to fio executable')
+ parser.add_argument('--read',
+ help='target for read testing')
+ parser.add_argument('--write',
+ help='target for write testing')
+ args = parser.parse_args()
+
+ return args
+
+
+def check(data, iops, slope, pct, limit, dur, criterion):
+ measurement = 'iops' if iops else 'bw'
+ data = data[measurement]
+ mean = sum(data) / len(data)
+ if slope:
+ x = list(range(len(data)))
+ m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
+ m = abs(m)
+ if pct:
+ target = m / mean * 100
+ criterion = criterion[:-1]
+ else:
+ target = m
+ else:
+ maxdev = 0
+ for x in data:
+ maxdev = max(abs(mean-x), maxdev)
+ if pct:
+ target = maxdev / mean * 100
+ criterion = criterion[:-1]
+ else:
+ target = maxdev
+
+ criterion = float(criterion)
+ return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
+
+
+if __name__ == '__main__':
+ args = parse_args()
+
+ pp = pprint.PrettyPrinter(indent=4)
+
+#
+# test option parsing
+#
+ parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
+ 'output': "set steady state IOPS threshold to 10.000000" },
+ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
+ 'output': "set steady state threshold to 10.000000%" },
+ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
+ 'output': "set steady state threshold to 0.100000%" },
+ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
+ 'output': "set steady state threshold to 10.000000%" },
+ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
+ 'output': "set steady state threshold to 0.100000%" },
+ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
+ 'output': "set steady state BW threshold to 12" },
+ ]
+ for test in parsing:
+ output = subprocess.check_output([args.fio] + test['args'])
+ if test['output'] in output.decode():
+ print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
+ else:
+ print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
+
+#
+# test some read workloads
+#
+# if ss active and attained,
+# check that runtime is less than job time
+# check criteria
+# how to check ramp time?
+#
+# if ss inactive
+# check that runtime is what was specified
+#
+ reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
+ {'s': False, 'timeout': 20, 'numjobs': 2},
+ {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
+ {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
+ ]
+
+ if args.read == None:
+ if os.name == 'posix':
+ args.read = '/dev/zero'
+ extra = [ "--size=134217728" ] # 128 MiB
+ else:
+ print("ERROR: file for read testing must be specified on non-posix systems")
+ sys.exit(1)
+ else:
+ extra = []
+
+ jobnum = 0
+ for job in reads:
+
+ tf = uuid.uuid4().hex
+ parameters = [ "--name=job{0}".format(jobnum) ]
+ parameters.extend(extra)
+ parameters.extend([ "--thread",
+ "--output-format=json",
+ "--output={0}".format(tf),
+ "--filename={0}".format(args.read),
+ "--rw=randrw",
+ "--rwmixread=100",
+ "--stonewall",
+ "--group_reporting",
+ "--numjobs={0}".format(job['numjobs']),
+ "--time_based",
+ "--runtime={0}".format(job['timeout']) ])
+ if job['s']:
+ if job['iops']:
+ ss = 'iops'
+ else:
+ ss = 'bw'
+ if job['slope']:
+ ss += "_slope"
+ ss += ":" + str(job['ss_limit'])
+ if job['pct']:
+ ss += '%'
+ parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
+ '--ss={0}'.format(ss),
+ '--ss_ramp={0}'.format(job['ss_ramp']) ])
+
+ output = subprocess.call([args.fio] + parameters)
+ with open(tf, 'r') as source:
+ jsondata = json.loads(source.read())
+ os.remove(tf)
+
+ for jsonjob in jsondata['jobs']:
+ line = "job {0}".format(jsonjob['job options']['name'])
+ if job['s']:
+ if jsonjob['steadystate']['attained'] == 1:
+ # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
+ mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
+ actual = jsonjob['read']['runtime']
+ if mintime > actual:
+ line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
+ else:
+ line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
+ objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
+ iops=job['iops'],
+ slope=job['slope'],
+ pct=job['pct'],
+ limit=job['ss_limit'],
+ dur=job['ss_dur'],
+ criterion=jsonjob['steadystate']['criterion'])
+ if not objsame:
+ line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
+ else:
+ if met:
+ line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
+ else:
+ line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
+ else:
+ # check runtime, confirm criterion calculation, and confirm that criterion was not met
+ expected = job['timeout'] * 1000
+ actual = jsonjob['read']['runtime']
+ if abs(expected - actual) > 10:
+ line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
+ else:
+ line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
+ objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
+ iops=job['iops'],
+ slope=job['slope'],
+ pct=job['pct'],
+ limit=job['ss_limit'],
+ dur=job['ss_dur'],
+ criterion=jsonjob['steadystate']['criterion'])
+ if not objsame:
+ if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
+ line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
+ else:
+ line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
+ else:
+ if met:
+ line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
+ else:
+ line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
+ else:
+ expected = job['timeout'] * 1000
+ actual = jsonjob['read']['runtime']
+ if abs(expected - actual) < 10:
+ result = 'PASSED '
+ else:
+ result = 'FAILED '
+ line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
+ print(line)
+ if 'steadystate' in jsonjob:
+ pp.pprint(jsonjob['steadystate'])
+ jobnum += 1
+++ /dev/null
-#!/usr/bin/python2.7
-# Note: this script is python2 and python 3 compatible.
-#
-# steadystate_tests.py
-#
-# Test option parsing and functonality for fio's steady state detection feature.
-#
-# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio
-#
-# REQUIREMENTS
-# Python 2.6+
-# SciPy
-#
-# KNOWN ISSUES
-# only option parsing and read tests are carried out
-# On Windows this script works under Cygwin but not from cmd.exe
-# On Windows I encounter frequent fio problems generating JSON output (nothing to decode)
-# min runtime:
-# if ss attained: min runtime = ss_dur + ss_ramp
-# if not attained: runtime = timeout
-
-from __future__ import absolute_import
-from __future__ import print_function
-import os
-import sys
-import json
-import uuid
-import pprint
-import argparse
-import subprocess
-from scipy import stats
-from six.moves import range
-
-def parse_args():
- parser = argparse.ArgumentParser()
- parser.add_argument('fio',
- help='path to fio executable')
- parser.add_argument('--read',
- help='target for read testing')
- parser.add_argument('--write',
- help='target for write testing')
- args = parser.parse_args()
-
- return args
-
-
-def check(data, iops, slope, pct, limit, dur, criterion):
- measurement = 'iops' if iops else 'bw'
- data = data[measurement]
- mean = sum(data) / len(data)
- if slope:
- x = list(range(len(data)))
- m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
- m = abs(m)
- if pct:
- target = m / mean * 100
- criterion = criterion[:-1]
- else:
- target = m
- else:
- maxdev = 0
- for x in data:
- maxdev = max(abs(mean-x), maxdev)
- if pct:
- target = maxdev / mean * 100
- criterion = criterion[:-1]
- else:
- target = maxdev
-
- criterion = float(criterion)
- return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
-
-
-if __name__ == '__main__':
- args = parse_args()
-
- pp = pprint.PrettyPrinter(indent=4)
-
-#
-# test option parsing
-#
- parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
- 'output': "set steady state IOPS threshold to 10.000000" },
- { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
- 'output': "set steady state threshold to 10.000000%" },
- { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
- 'output': "set steady state threshold to 0.100000%" },
- { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
- 'output': "set steady state threshold to 10.000000%" },
- { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
- 'output': "set steady state threshold to 0.100000%" },
- { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
- 'output': "set steady state BW threshold to 12" },
- ]
- for test in parsing:
- output = subprocess.check_output([args.fio] + test['args'])
- if test['output'] in output.decode():
- print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
- else:
- print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
-
-#
-# test some read workloads
-#
-# if ss active and attained,
-# check that runtime is less than job time
-# check criteria
-# how to check ramp time?
-#
-# if ss inactive
-# check that runtime is what was specified
-#
- reads = [ {'s': True, 'timeout': 100, 'numjobs': 1, 'ss_dur': 5, 'ss_ramp': 3, 'iops': True, 'slope': True, 'ss_limit': 0.1, 'pct': True},
- {'s': False, 'timeout': 20, 'numjobs': 2},
- {'s': True, 'timeout': 100, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 5, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
- {'s': True, 'timeout': 10, 'numjobs': 3, 'ss_dur': 10, 'ss_ramp': 500, 'iops': False, 'slope': True, 'ss_limit': 0.1, 'pct': True},
- ]
-
- if args.read == None:
- if os.name == 'posix':
- args.read = '/dev/zero'
- extra = [ "--size=134217728" ] # 128 MiB
- else:
- print("ERROR: file for read testing must be specified on non-posix systems")
- sys.exit(1)
- else:
- extra = []
-
- jobnum = 0
- for job in reads:
-
- tf = uuid.uuid4().hex
- parameters = [ "--name=job{0}".format(jobnum) ]
- parameters.extend(extra)
- parameters.extend([ "--thread",
- "--output-format=json",
- "--output={0}".format(tf),
- "--filename={0}".format(args.read),
- "--rw=randrw",
- "--rwmixread=100",
- "--stonewall",
- "--group_reporting",
- "--numjobs={0}".format(job['numjobs']),
- "--time_based",
- "--runtime={0}".format(job['timeout']) ])
- if job['s']:
- if job['iops']:
- ss = 'iops'
- else:
- ss = 'bw'
- if job['slope']:
- ss += "_slope"
- ss += ":" + str(job['ss_limit'])
- if job['pct']:
- ss += '%'
- parameters.extend([ '--ss_dur={0}'.format(job['ss_dur']),
- '--ss={0}'.format(ss),
- '--ss_ramp={0}'.format(job['ss_ramp']) ])
-
- output = subprocess.call([args.fio] + parameters)
- with open(tf, 'r') as source:
- jsondata = json.loads(source.read())
- os.remove(tf)
-
- for jsonjob in jsondata['jobs']:
- line = "job {0}".format(jsonjob['job options']['name'])
- if job['s']:
- if jsonjob['steadystate']['attained'] == 1:
- # check runtime >= ss_dur + ss_ramp, check criterion, check criterion < limit
- mintime = (job['ss_dur'] + job['ss_ramp']) * 1000
- actual = jsonjob['read']['runtime']
- if mintime > actual:
- line = 'FAILED ' + line + ' ss attained, runtime {0} < ss_dur {1} + ss_ramp {2}'.format(actual, job['ss_dur'], job['ss_ramp'])
- else:
- line = line + ' ss attained, runtime {0} > ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
- objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
- iops=job['iops'],
- slope=job['slope'],
- pct=job['pct'],
- limit=job['ss_limit'],
- dur=job['ss_dur'],
- criterion=jsonjob['steadystate']['criterion'])
- if not objsame:
- line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
- else:
- if met:
- line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, job['ss_limit'])
- else:
- line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, job['ss_limit'])
- else:
- # check runtime, confirm criterion calculation, and confirm that criterion was not met
- expected = job['timeout'] * 1000
- actual = jsonjob['read']['runtime']
- if abs(expected - actual) > 10:
- line = 'FAILED ' + line + ' ss not attained, expected runtime {0} != actual runtime {1}'.format(expected, actual)
- else:
- line = line + ' ss not attained, runtime {0} != ss_dur {1} + ss_ramp {2},'.format(actual, job['ss_dur'], job['ss_ramp'])
- objsame, met, mean, target = check(data=jsonjob['steadystate']['data'],
- iops=job['iops'],
- slope=job['slope'],
- pct=job['pct'],
- limit=job['ss_limit'],
- dur=job['ss_dur'],
- criterion=jsonjob['steadystate']['criterion'])
- if not objsame:
- if actual > (job['ss_dur'] + job['ss_ramp'])*1000:
- line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(jsonjob['steadystate']['criterion'], target)
- else:
- line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(jsonjob['steadystate']['criterion'])
- else:
- if met:
- line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, job['ss_limit'])
- else:
- line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, job['ss_limit'])
- else:
- expected = job['timeout'] * 1000
- actual = jsonjob['read']['runtime']
- if abs(expected - actual) < 10:
- result = 'PASSED '
- else:
- result = 'FAILED '
- line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
- print(line)
- if 'steadystate' in jsonjob:
- pp.pprint(jsonjob['steadystate'])
- jobnum += 1
projects / fio.git / commitdiff