steadystate: improve output of test script

[fio.git] / unit_tests / steadystate_tests.py

diff --git a/unit_tests/steadystate_tests.py b/unit_tests/steadystate_tests.py
index 02b2b0d63d729b1abcac8a0c2c6bec0dee3bd9fc..368d69b798ae803df157d83c83441ac5cde40492 100755 (executable)
--- a/unit_tests/steadystate_tests.py
+++ b/unit_tests/steadystate_tests.py
@@ -20,6 +20,7 @@
 
 import os
 import json
+import pprint
 import tempfile
 import argparse
 import subprocess
@@ -39,6 +40,8 @@ def parse_args():
 
 
 def check(data, iops, slope, pct, limit, dur, criterion):
+    measurement = 'iops' if iops else 'bw'
+    data = data[measurement]
     mean = sum(data) / len(data)
     if slope:
         x = range(len(data))
@@ -46,6 +49,7 @@ def check(data, iops, slope, pct, limit, dur, criterion):
         m = abs(m)
         if pct:
             target = m / mean * 100
+            criterion = criterion[:-1]
         else:
             target = m
     else:
@@ -54,29 +58,33 @@ def check(data, iops, slope, pct, limit, dur, criterion):
             maxdev = max(abs(mean-x), maxdev)
         if pct:
             target = maxdev / mean * 100
+            criterion = criterion[:-1]
         else:
             target = maxdev
 
-    return (abs(target - criterion) / criterion < 0.001), target < limit, mean, target
+    criterion = float(criterion)
+    return (abs(target - criterion) / criterion < 0.005), target < limit, mean, target
 
 
 if __name__ == '__main__':
     args = parse_args()
 
+    pp = pprint.PrettyPrinter(indent=4)
+
 #
 # test option parsing
 #
-    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"], 
+    parsing = [ { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10", "--ss_ramp=5"],
                   'output': "set steady state IOPS threshold to 10.000000" },
-                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"], 
+                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:10%", "--ss_ramp=5"],
                   'output': "set steady state threshold to 10.000000%" },
-                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"], 
+                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=iops:.1%", "--ss_ramp=5"],
                   'output': "set steady state threshold to 0.100000%" },
-                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"], 
+                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:10%", "--ss_ramp=5"],
                   'output': "set steady state threshold to 10.000000%" },
-                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"], 
+                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:.1%", "--ss_ramp=5"],
                   'output': "set steady state threshold to 0.100000%" },
-                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"], 
+                { 'args': ["--parse-only", "--debug=parse", "--ss_dur=10s", "--ss=bw:12", "--ss_ramp=5"],
                   'output': "set steady state BW threshold to 12" },
               ]
     for test in parsing:
@@ -132,8 +140,8 @@ if __name__ == '__main__':
 
         tf = tempfile.NamedTemporaryFile(delete=False)
        tf.close()
-        output = subprocess.check_output([args.fio, 
-                                          "--output-format=json", 
+        output = subprocess.check_output([args.fio,
+                                          "--output-format=json",
                                           "--output={0}".format(tf.name)] + accum)
         with open(tf.name, 'r') as source:
             jsondata = json.loads(source.read())
@@ -158,13 +166,12 @@ if __name__ == '__main__':
                             dur=suite[jobnum]['ss_dur'],
                             criterion=job['steadystate']['criterion'])
                         if not objsame:
-                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']['data'])
+                            line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
                         else:
                             if met:
-                                line = 'PASSED ' + line + ' target {0} < limit {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
+                                line = 'PASSED ' + line + ' target {0} < limit {1}'.format(target, suite[jobnum]['ss_limit'])
                             else:
-                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
-                    
+                                line = 'FAILED ' + line + ' target {0} < limit {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
                 else:
                     # check runtime, confirm criterion calculation, and confirm that criterion was not met
                     expected = suite[jobnum]['timeout'] * 1000
@@ -182,14 +189,14 @@ if __name__ == '__main__':
                             criterion=job['steadystate']['criterion'])
                         if not objsame:
                             if actual > (suite[jobnum]['ss_dur'] + suite[jobnum]['ss_ramp'])*1000:
-                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1}, data: {2} '.format(job['steadystate']['criterion'], target, job['steadystate']['data'])
+                                line = 'FAILED ' + line + ' fio criterion {0} != calculated criterion {1} '.format(job['steadystate']['criterion'], target)
                             else:
-                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed, data: {1} '.format(job['steadystate']['criterion'], job['steadystate']['data'])
+                                line = 'PASSED ' + line + ' fio criterion {0} == 0.0 since ss_dur + ss_ramp has not elapsed '.format(job['steadystate']['criterion'])
                         else:
                             if met:
-                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained, data: {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
+                                line = 'FAILED ' + line + ' target {0} < threshold {1} but fio reports ss not attained '.format(target, suite[jobnum]['ss_limit'])
                             else:
-                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}, data {2}'.format(target, suite[jobnum]['ss_limit'], job['steadystate']['data'])
+                                line = 'PASSED ' + line + ' criterion {0} > threshold {1}'.format(target, suite[jobnum]['ss_limit'])
             else:
                 expected = suite[jobnum]['timeout'] * 1000
                 actual = job['read']['runtime']
@@ -199,5 +206,7 @@ if __name__ == '__main__':
                     result = 'FAILED '
                 line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
             print line
+            if 'steadystate' in job:
+                pp.pprint(job['steadystate'])
             jobnum += 1
         suitenum += 1