Merge branch 'nvme/support-sync-fua-for-iouring-v2' of https://github.com/minwooim/fio

[fio.git] / tools / hist / fio-histo-log-pctiles.py

diff --git a/tools/hist/fio-histo-log-pctiles.py b/tools/hist/fio-histo-log-pctiles.py
index 81f6de6e1a1e20842d4c83fc87594f9fe0911a6c..b5d167de2295fa45354c920be63b3f3f92dfebcd 100755 (executable)
--- a/tools/hist/fio-histo-log-pctiles.py
+++ b/tools/hist/fio-histo-log-pctiles.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # module to parse fio histogram log files, not using pandas
 # runs in python v2 or v3
@@ -24,7 +24,13 @@
 import sys, os, math, copy, time
 from copy import deepcopy
 import argparse
-import unittest2
+from functools import reduce
+
+unittest2_imported = True
+try:
+    import unittest2
+except ImportError:
+    unittest2_imported = False
 
 msec_per_sec = 1000
 nsec_per_usec = 1000
@@ -66,6 +72,8 @@ def parse_hist_file(logfn, buckets_per_interval, log_hist_msec):
     with open(logfn, 'r') as f:
         records = [ l.strip() for l in f.readlines() ]
     intervals = []
+    last_time_ms = -1
+    last_direction = -1
     for k, r in enumerate(records):
         if r == '':
             continue
@@ -75,7 +83,7 @@ def parse_hist_file(logfn, buckets_per_interval, log_hist_msec):
         except ValueError as e:
             raise FioHistoLogExc('non-integer value %s' % exception_suffix(k+1, logfn))
 
-        neg_ints = list(filter( lambda tk : tk < 0, int_tokens ))
+        neg_ints = list([tk for tk in int_tokens if tk < 0])
         if len(neg_ints) > 0:
             raise FioHistoLogExc('negative integer value %s' % exception_suffix(k+1, logfn))
 
@@ -104,6 +112,15 @@ def parse_hist_file(logfn, buckets_per_interval, log_hist_msec):
         if len(buckets) != buckets_per_interval:
             raise FioHistoLogExc('%d buckets per interval but %d expected in %s' % 
                     (len(buckets), buckets_per_interval, exception_suffix(k+1, logfn)))
+
+        # hack to filter out records with the same timestamp
+        # we should not have to do this if fio logs histogram records correctly
+
+        if time_ms == last_time_ms and direction == last_direction:
+            continue
+        last_time_ms = time_ms
+        last_direction = direction
+
         intervals.append((time_ms, direction, bsz, buckets))
     if len(intervals) == 0:
         raise FioHistoLogExc('no records in %s' % logfn)
@@ -256,6 +273,13 @@ def add_to_histo_from( target, source ):
     for b in range(0, len(source)):
         target[b] += source[b]
 
+
+# calculate total samples in the histogram buckets
+
+def get_samples(buckets):
+    return reduce( lambda x,y: x + y, buckets)
+
+
 # compute percentiles
 # inputs:
 #   buckets: histogram bucket array 
@@ -437,14 +461,24 @@ def compute_percentiles_from_logs():
     # calculate percentiles across aggregate histogram for all threads
     # print CSV header just like fiologparser_hist does
 
-    header = 'msec-since-start, '
+    header = 'msec-since-start, samples, '
     for p in args.pctiles_wanted:
-        header += '%3.1f, ' % p
+        if p == 0.:
+            next_pctile_header = 'min'
+        elif p == 100.:
+            next_pctile_header = 'max'
+        elif p == 50.:
+            next_pctile_header = 'median'
+        else:
+            next_pctile_header = '%3.1f' % p
+        header += '%s, ' % next_pctile_header
+
     print('time (millisec), percentiles in increasing order with values in ' + args.output_unit)
     print(header)
 
     for (t_msec, all_threads_histo_t) in all_threads_histograms:
-        record = '%8d, ' % t_msec
+        samples = get_samples(all_threads_histo_t)
+        record = '%8d, %8d, ' % (t_msec, samples)
         pct = get_pctiles(all_threads_histo_t, args.pctiles_wanted, bucket_times)
         if not pct:
             for w in args.pctiles_wanted:
@@ -460,14 +494,14 @@ def compute_percentiles_from_logs():
 #end of MAIN PROGRAM
 
 
-
 ##### below are unit tests ##############
 
-import tempfile, shutil
-from os.path import join
-should_not_get_here = False
+if unittest2_imported:
+  import tempfile, shutil
+  from os.path import join
+  should_not_get_here = False
 
-class Test(unittest2.TestCase):
+  class Test(unittest2.TestCase):
     tempdir = None
 
     # a little less typing please
@@ -714,7 +748,7 @@ class Test(unittest2.TestCase):
     def test_e2_get_pctiles_highest_pct(self):
         fio_v3_bucket_count = 29 * 64
         with open(self.fn, 'w') as f:
-            # make a empty fio v3 histogram
+            # make an empty fio v3 histogram
             buckets = [ 0 for j in range(0, fio_v3_bucket_count) ]
             # add one I/O request to last bucket
             buckets[-1] = 1
@@ -732,7 +766,10 @@ class Test(unittest2.TestCase):
 
 if __name__ == '__main__':
     if os.getenv('UNITTEST'):
-        sys.exit(unittest2.main())
+        if unittest2_imported:
+            sys.exit(unittest2.main())
+        else:
+            raise Exception('you must install unittest2 module to run unit test')
     else:
         compute_percentiles_from_logs()