XGitUrl: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=tools%2Fhist%2Ffiologparser_hist.py;h=778cc007601196aaa76803309314956f0ea75113;hp=ce98d2ecd75581bc66e8c3de3b94da91690fe1c1;hb=d1f6fcadb7cb28a5e57a5e573395fe2deb3cfd7b;hpb=3c746856fd63f3f7624a34ea70e226a290d9cc01
diff git a/tools/hist/fiologparser_hist.py b/tools/hist/fiologparser_hist.py
index ce98d2ec..778cc007 100755
 a/tools/hist/fiologparser_hist.py
+++ b/tools/hist/fiologparser_hist.py
@@ 11,113 +11,6 @@
4000, 39, 1152, 1546.962, 1545.785, 1627.192, 1640.019, 1691.204, 1744
...
 Notes:

 * endtimes are calculated to be uniform increments of the interval value given,
 regardless of when histogram samples are reported. Of note:

 * Intervals with no samples are omitted. In the example above this means
 "no statistics from 2 to 3 seconds" and "39 samples influenced the statistics
 of the interval from 3 to 4 seconds".

 * Intervals with a single sample will have the same value for all statistics

 * The number of samples is unweighted, corresponding to the total number of samples
 which have any effect whatsoever on the interval.

 * Min statistics are computed using value of the lower boundary of the first bin
 (in increasing bin order) with nonzero samples in it. Similarly for max,
 we take the upper boundary of the last bin with nonzero samples in it.
 This is semantically identical to taking the 0th and 100th percentiles with a
 50% binwidth buffer (because percentiles are computed using midpoints of
 the bins). This enforces the following nice properties:

 * min <= 50th <= 90th <= 95th <= 99th <= max

 * min and max are strict lower and upper bounds on the actual
 min / max seen by fio (and reported in *_clat.* with averaging turned off).

 * Average statistics use a standard weighted arithmetic mean.

 * Percentile statistics are computed using the weighted percentile method as
 described here: https://en.wikipedia.org/wiki/Percentile#Weighted_percentile
 See weights() method for details on how weights are computed for individual
 samples. In process_interval() we further multiply by the height of each bin
 to get weighted histograms.

 * We convert files given on the command line, assumed to be fio histogram files,
 onthefly into their corresponding differenced files i.e. noncumulative histograms
 because fio outputs cumulative histograms, but we want histograms corresponding
 to individual time intervals. An individual histogram file can contain the cumulative
 histograms for multiple different r/w directions (notably when rw=randrw). This
 is accounted for by tracking each r/w direction separately. In the statistics
 reported we ultimately merge *all* histograms (regardless of r/w direction).

 * The value of *_GROUP_NR in stat.h (and *_BITS) determines how many latency bins
 fio outputs when histogramming is enabled. Namely for the current default of
 GROUP_NR=19, we get 1,216 bins with a maximum latency of approximately 17
 seconds. For certain applications this may not be sufficient. With GROUP_NR=24
 we have 1,536 bins, giving us a maximum latency of 541 seconds (~ 9 minutes). If
 you expect your application to experience latencies greater than 17 seconds,
 you will need to recompile fio with a larger GROUP_NR, e.g. with:

 sed i.bak 's/^#define FIO_IO_U_PLAT_GROUP_NR 19\n/#define FIO_IO_U_PLAT_GROUP_NR 24/g' stat.h
 make fio

 Quick reference table for the max latency corresponding to a sampling of
 values for GROUP_NR:

 GROUP_NR  # bins  max latency bin value
 19  1216  16.9 sec
 20  1280  33.8 sec
 21  1344  67.6 sec
 22  1408  2 min, 15 sec
 23  1472  4 min, 32 sec
 24  1536  9 min, 4 sec
 25  1600  18 min, 8 sec
 26  1664  36 min, 16 sec

 * At present this program automatically detects the number of histogram bins in
 the log files, and adjusts the bin latency values accordingly. In particular if
 you use the log_hist_coarseness parameter of fio, you get output files with
 a number of bins according to the following table (note that the first
 row is identical to the table above):

 coarse \ GROUP_NR
 19 20 21 22 23 24 25 26
 
 0 [[ 1216, 1280, 1344, 1408, 1472, 1536, 1600, 1664],
 1 [ 608, 640, 672, 704, 736, 768, 800, 832],
 2 [ 304, 320, 336, 352, 368, 384, 400, 416],
 3 [ 152, 160, 168, 176, 184, 192, 200, 208],
 4 [ 76, 80, 84, 88, 92, 96, 100, 104],
 5 [ 38, 40, 42, 44, 46, 48, 50, 52],
 6 [ 19, 20, 21, 22, 23, 24, 25, 26],
 7 [ N/A, 10, N/A, 11, N/A, 12, N/A, 13],
 8 [ N/A, 5, N/A, N/A, N/A, 6, N/A, N/A]]

 For other values of GROUP_NR and coarseness, this table can be computed like this:

 bins = [1216,1280,1344,1408,1472,1536,1600,1664]
 max_coarse = 8
 fncn = lambda z: list(map(lambda x: z/2**x if z % 2**x == 0 else nan, range(max_coarse + 1)))
 np.transpose(list(map(fncn, bins)))

 Also note that you can achieve the same downsampling / log file size reduction
 by preprocessing (before inputting into this script) with half_bins.py.

 * If you have not adjusted GROUP_NR for your (high latency) application, then you
 will see the percentiles computed by this tool max out at the max latency bin
 value as in the first table above, and in this plot (where GROUP_NR=19 and thus we see
 a max latency of ~16.7 seconds in the red line):

 https://www.cronburg.com/fio/max_latency_bin_value_bug.png

 * Motivation for, design decisions, and the implementation process are
 described in further detail here:

 https://www.cronburg.com/fio/cloudlatencyproblemmeasurement/

@author Karl Cronburg
"""
import os
@@ 188,23 +81,8 @@ __HIST_COLUMNS = 1216
__NON_HIST_COLUMNS = 3
__TOTAL_COLUMNS = __HIST_COLUMNS + __NON_HIST_COLUMNS
def sequential_diffs(head_row, times, rws, hists):
 """ Take the difference of sequential (in time) histograms with the same
 r/w direction, returning a new array of differenced histograms. """
 result = np.empty(shape=(0, __HIST_COLUMNS))
 result_times = np.empty(shape=(1, 0))
 for i in range(8):
 idx = np.where(rws == i)
 diff = np.diff(np.append(head_row[i], hists[idx], axis=0), axis=0).astype(int)
 result = np.append(diff, result, axis=0)
 result_times = np.append(times[idx], result_times)
 idx = np.argsort(result_times)
 return result[idx]

def read_chunk(head_row, rdr, sz):
 """ Read the next chunk of size sz from the given reader, computing the
 differences across neighboring histogram samples.
 """
+def read_chunk(rdr, sz):
+ """ Read the next chunk of size sz from the given reader. """
try:
""" StopIteration occurs when the pandas reader is empty, and AttributeError
occurs if rdr is None due to the file being empty. """
@@ 212,32 +90,20 @@ def read_chunk(head_row, rdr, sz):
except (StopIteration, AttributeError):
return None
 """ Extract array of just the times, and histograms matrix without times column.
 Then, take the sequential difference of each of the rows in the histogram
 matrix. This is necessary because fio outputs *cumulative* histograms as
 opposed to histograms with counts just for a particular interval. """
+ """ Extract array of just the times, and histograms matrix without times column. """
times, rws, szs = new_arr[:,0], new_arr[:,1], new_arr[:,2]
hists = new_arr[:,__NON_HIST_COLUMNS:]
 hists_diff = sequential_diffs(head_row, times, rws, hists)
times = times.reshape((len(times),1))
 arr = np.append(times, hists_diff, axis=1)
+ arr = np.append(times, hists, axis=1)
 """ hists[1] will be the row we need to start our differencing with the
 next time we call read_chunk() on the same rdr """
 return arr, hists[1]
+ return arr
def get_min(fps, arrs):
""" Find the file with the current first row with the smallest start time """
 return min([fp for fp in fps if not arrs[fp] is None], key=lambda fp: arrs.get(fp)[0][0][0])
+ return min([fp for fp in fps if not arrs[fp] is None], key=lambda fp: arrs.get(fp)[0][0])
def histogram_generator(ctx, fps, sz):
 """ head_row for a particular file keeps track of the last (cumulative)
 histogram we read so that we have a reference point to subtract off
 when computing sequential differences. """
 head_row = np.zeros(shape=(1, __HIST_COLUMNS))
 head_rows = {fp: {i: head_row for i in range(8)} for fp in fps}

# Create a chunked pandas reader for each of the files:
rdrs = {}
for fp in fps:
@@ 245,13 +111,13 @@ def histogram_generator(ctx, fps, sz):
rdrs[fp] = pandas.read_csv(fp, dtype=int, header=None, chunksize=sz)
except ValueError as e:
if e.message == 'No columns to parse from file':
 if not ctx.nowarn: sys.stderr.write("WARNING: Empty input file encountered.\n")
+ if ctx.warn: sys.stderr.write("WARNING: Empty input file encountered.\n")
rdrs[fp] = None
else:
raise(e)
 # Initial histograms and corresponding head_rows:
 arrs = {fp: read_chunk(head_rows[fp], rdr, sz) for fp,rdr in rdrs.items()}
+ # Initial histograms from disk:
+ arrs = {fp: read_chunk(rdr, sz) for fp,rdr in rdrs.items()}
while True:
try:
@@ 259,13 +125,12 @@ def histogram_generator(ctx, fps, sz):
fp = get_min(fps, arrs)
except ValueError:
return
 arr, head_row = arrs[fp]
+ arr = arrs[fp]
yield np.insert(arr[0], 1, fps.index(fp))
 arrs[fp] = arr[1:], head_row
 head_rows[fp] = head_row
+ arrs[fp] = arr[1:]
 if arrs[fp][0].shape[0] == 0:
 arrs[fp] = read_chunk(head_rows[fp], rdrs[fp], sz)
+ if arrs[fp].shape[0] == 0:
+ arrs[fp] = read_chunk(rdrs[fp], sz)
def _plat_idx_to_val(idx, edge=0.5, FIO_IO_U_PLAT_BITS=6, FIO_IO_U_PLAT_VAL=64):
""" Taken from fio's stat.c for calculating the latency value of a bin
@@ 471,11 +336,11 @@ if __name__ == '__main__':
type=int,
help='number of decimal places to print floats to')
 arg('nowarn',
 dest='nowarn',
 action='store_false',
 default=True,
 help='do not print any warning messages to stderr')
+ arg('warn',
+ dest='warn',
+ action='store_true',
+ default=False,
+ help='print warning messages to stderr')
arg('group_nr',
default=19,