5 # This tool lets you parse multiple fio log files and look at interaval
6 # statistics even when samples are non-uniform. For instance:
8 # fiologparser.py -s *bw*
10 # to see per-interval sums for all bandwidth logs or:
12 # fiologparser.py -a *clat*
14 # to see per-interval average completion latency.
21 parser = argparse.ArgumentParser()
22 parser.add_argument('-i', '--interval', required=False, type=int, default=1000, help='interval of time in seconds.')
23 parser.add_argument('-d', '--divisor', required=False, type=int, default=1, help='divide the results by this value.')
24 parser.add_argument('-f', '--full', dest='full', action='store_true', default=False, help='print full output.')
25 parser.add_argument('-A', '--all', dest='allstats', action='store_true', default=False,
26 help='print all stats for each interval.')
27 parser.add_argument('-a', '--average', dest='average', action='store_true', default=False, help='print the average for each interval.')
28 parser.add_argument('-s', '--sum', dest='sum', action='store_true', default=False, help='print the sum for each interval.')
29 parser.add_argument("FILE", help="collectl log output files to parse", nargs="+")
30 args = parser.parse_args()
34 def get_ftime(series):
37 if ftime == 0 or ts.last.end < ftime:
41 def print_full(ctx, series):
42 ftime = get_ftime(series)
46 while (start < ftime):
47 end = ftime if ftime < end else end
48 results = [ts.get_value(start, end) for ts in series]
49 print "%s, %s" % (end, ', '.join(["%0.3f" % i for i in results]))
53 def print_sums(ctx, series):
54 ftime = get_ftime(series)
58 while (start < ftime):
59 end = ftime if ftime < end else end
60 results = [ts.get_value(start, end) for ts in series]
61 print "%s, %0.3f" % (end, sum(results))
65 def print_averages(ctx, series):
66 ftime = get_ftime(series)
70 while (start < ftime):
71 end = ftime if ftime < end else end
72 results = [ts.get_value(start, end) for ts in series]
73 print "%s, %0.3f" % (end, float(sum(results))/len(results))
77 # FIXME: this routine is computationally inefficient
78 # and has O(N^2) behavior
79 # it would be better to make one pass through samples
80 # to segment them into a series of time intervals, and
81 # then compute stats on each time interval instead.
82 # to debug this routine, use
83 # # sort -n -t ',' -k 2 small.log
85 # Sometimes scipy interpolates between two values to get a percentile
87 def my_extend( vlist, val ):
91 array_collapser = lambda vlist, val: my_extend(vlist, val)
93 def print_all_stats(ctx, series):
94 ftime = get_ftime(series)
97 print('start-time, samples, min, avg, median, 90%, 95%, 99%, max')
98 while (start < ftime): # for each time interval
99 end = ftime if ftime < end else end
100 sample_arrays = [ s.get_samples(start, end) for s in series ]
101 samplevalue_arrays = []
102 for sample_array in sample_arrays:
103 samplevalue_arrays.append(
104 [ sample.value for sample in sample_array ] )
105 #print('samplevalue_arrays len: %d' % len(samplevalue_arrays))
106 #print('samplevalue_arrays elements len: ' + \
107 #str(map( lambda l: len(l), samplevalue_arrays)))
108 # collapse list of lists of sample values into list of sample values
109 samplevalues = reduce( array_collapser, samplevalue_arrays, [] )
110 #print('samplevalues: ' + str(sorted(samplevalues)))
111 # compute all stats and print them
112 myarray = scipy.fromiter(samplevalues, float)
113 mymin = scipy.amin(myarray)
114 myavg = scipy.average(myarray)
115 mymedian = scipy.median(myarray)
116 my90th = scipy.percentile(myarray, 90)
117 my95th = scipy.percentile(myarray, 95)
118 my99th = scipy.percentile(myarray, 99)
119 mymax = scipy.amax(myarray)
120 print( '%f, %d, %f, %f, %f, %f, %f, %f, %f' % (
121 start, len(samplevalues),
122 mymin, myavg, mymedian, my90th, my95th, my99th, mymax))
124 # advance to next interval
125 start += ctx.interval
129 def print_default(ctx, series):
130 ftime = get_ftime(series)
136 while (start < ftime):
137 end = ftime if ftime < end else end
138 results = [ts.get_value(start, end) for ts in series]
139 averages.append(sum(results))
140 weights.append(end-start)
141 start += ctx.interval
145 for i in xrange(0, len(averages)):
146 total += averages[i]*weights[i]
147 print '%0.3f' % (total/sum(weights))
150 def __init__(self, ctx, fn):
156 def read_data(self, fn):
160 (time, value, foo, bar) = line.rstrip('\r\n').rsplit(', ')
161 self.add_sample(p_time, int(time), int(value))
164 def add_sample(self, start, end, value):
165 sample = Sample(ctx, start, end, value)
166 if not self.last or self.last.end < end:
168 self.samples.append(sample)
170 def get_samples(self, start, end):
172 for s in self.samples:
173 if s.start >= start and s.end <= end:
174 sample_list.append(s)
177 def get_value(self, start, end):
179 for sample in self.samples:
180 value += sample.get_contribution(start, end)
184 def __init__(self, ctx, start, end, value):
190 def get_contribution(self, start, end):
191 # short circuit if not within the bound
192 if (end < self.start or start > self.end):
195 sbound = self.start if start < self.start else start
196 ebound = self.end if end > self.end else end
197 ratio = float(ebound-sbound) / (end-start)
198 return self.value*ratio/ctx.divisor
201 if __name__ == '__main__':
205 series.append(TimeSeries(ctx, fn))
207 print_sums(ctx, series)
209 print_averages(ctx, series)
211 print_full(ctx, series)
213 print_all_stats(ctx, series)
215 print_default(ctx, series)