| 1 | #!/bin/python |
| 2 | # |
| 3 | # fiologparser.py |
| 4 | # |
| 5 | # This tool lets you parse multiple fio log files and look at interaval |
| 6 | # statistics even when samples are non-uniform. For instance: |
| 7 | # |
| 8 | # fiologparser.py -s *bw* |
| 9 | # |
| 10 | # to see per-interval sums for all bandwidth logs or: |
| 11 | # |
| 12 | # fiologparser.py -a *clat* |
| 13 | # |
| 14 | # to see per-interval average completion latency. |
| 15 | |
| 16 | import argparse |
| 17 | |
| 18 | def parse_args(): |
| 19 | parser = argparse.ArgumentParser() |
| 20 | parser.add_argument('-i', '--interval', required=False, type=int, default=1000, help='interval of time in seconds.') |
| 21 | parser.add_argument('-d', '--divisor', required=False, type=int, default=1, help='divide the results by this value.') |
| 22 | parser.add_argument('-f', '--full', dest='full', action='store_true', default=False, help='print full output.') |
| 23 | parser.add_argument('-a', '--average', dest='average', action='store_true', default=False, help='print the average for each interval.') |
| 24 | parser.add_argument('-s', '--sum', dest='sum', action='store_true', default=False, help='print the sum for each interval.') |
| 25 | parser.add_argument("FILE", help="collectl log output files to parse", nargs="+") |
| 26 | args = parser.parse_args() |
| 27 | |
| 28 | return args |
| 29 | |
| 30 | def get_ftime(series): |
| 31 | ftime = 0 |
| 32 | for ts in series: |
| 33 | if ftime == 0 or ts.last.end < ftime: |
| 34 | ftime = ts.last.end |
| 35 | return ftime |
| 36 | |
| 37 | def print_full(ctx, series): |
| 38 | ftime = get_ftime(series) |
| 39 | start = 0 |
| 40 | end = ctx.interval |
| 41 | |
| 42 | while (start < ftime): |
| 43 | end = ftime if ftime < end else end |
| 44 | results = [ts.get_value(start, end) for ts in series] |
| 45 | print "%s, %s" % (end, ', '.join(["%0.3f" % i for i in results])) |
| 46 | start += ctx.interval |
| 47 | end += ctx.interval |
| 48 | |
| 49 | def print_sums(ctx, series): |
| 50 | ftime = get_ftime(series) |
| 51 | start = 0 |
| 52 | end = ctx.interval |
| 53 | |
| 54 | while (start < ftime): |
| 55 | end = ftime if ftime < end else end |
| 56 | results = [ts.get_value(start, end) for ts in series] |
| 57 | print "%s, %0.3f" % (end, sum(results)) |
| 58 | start += ctx.interval |
| 59 | end += ctx.interval |
| 60 | |
| 61 | def print_averages(ctx, series): |
| 62 | ftime = get_ftime(series) |
| 63 | start = 0 |
| 64 | end = ctx.interval |
| 65 | |
| 66 | while (start < ftime): |
| 67 | end = ftime if ftime < end else end |
| 68 | results = [ts.get_value(start, end) for ts in series] |
| 69 | print "%s, %0.3f" % (end, float(sum(results))/len(results)) |
| 70 | start += ctx.interval |
| 71 | end += ctx.interval |
| 72 | |
| 73 | |
| 74 | def print_default(ctx, series): |
| 75 | ftime = get_ftime(series) |
| 76 | start = 0 |
| 77 | end = ctx.interval |
| 78 | averages = [] |
| 79 | weights = [] |
| 80 | |
| 81 | while (start < ftime): |
| 82 | end = ftime if ftime < end else end |
| 83 | results = [ts.get_value(start, end) for ts in series] |
| 84 | averages.append(sum(results)) |
| 85 | weights.append(end-start) |
| 86 | start += ctx.interval |
| 87 | end += ctx.interval |
| 88 | |
| 89 | total = 0 |
| 90 | for i in xrange(0, len(averages)): |
| 91 | total += averages[i]*weights[i] |
| 92 | print '%0.3f' % (total/sum(weights)) |
| 93 | |
| 94 | class TimeSeries(): |
| 95 | def __init__(self, ctx, fn): |
| 96 | self.ctx = ctx |
| 97 | self.last = None |
| 98 | self.samples = [] |
| 99 | self.read_data(fn) |
| 100 | |
| 101 | def read_data(self, fn): |
| 102 | f = open(fn, 'r') |
| 103 | p_time = 0 |
| 104 | for line in f: |
| 105 | (time, value, foo, bar) = line.rstrip('\r\n').rsplit(', ') |
| 106 | self.add_sample(p_time, int(time), int(value)) |
| 107 | p_time = int(time) |
| 108 | |
| 109 | def add_sample(self, start, end, value): |
| 110 | sample = Sample(ctx, start, end, value) |
| 111 | if not self.last or self.last.end < end: |
| 112 | self.last = sample |
| 113 | self.samples.append(sample) |
| 114 | |
| 115 | def get_value(self, start, end): |
| 116 | value = 0 |
| 117 | for sample in self.samples: |
| 118 | value += sample.get_contribution(start, end) |
| 119 | return value |
| 120 | |
| 121 | class Sample(): |
| 122 | def __init__(self, ctx, start, end, value): |
| 123 | self.ctx = ctx |
| 124 | self.start = start |
| 125 | self.end = end |
| 126 | self.value = value |
| 127 | |
| 128 | def get_contribution(self, start, end): |
| 129 | # short circuit if not within the bound |
| 130 | if (end < self.start or start > self.end): |
| 131 | return 0 |
| 132 | |
| 133 | sbound = self.start if start < self.start else start |
| 134 | ebound = self.end if end > self.end else end |
| 135 | ratio = float(ebound-sbound) / (end-start) |
| 136 | return self.value*ratio/ctx.divisor |
| 137 | |
| 138 | |
| 139 | if __name__ == '__main__': |
| 140 | ctx = parse_args() |
| 141 | series = [] |
| 142 | for fn in ctx.FILE: |
| 143 | series.append(TimeSeries(ctx, fn)) |
| 144 | if ctx.sum: |
| 145 | print_sums(ctx, series) |
| 146 | elif ctx.average: |
| 147 | print_averages(ctx, series) |
| 148 | elif ctx.full: |
| 149 | print_full(ctx, series) |
| 150 | else: |
| 151 | print_default(ctx, series) |
| 152 | |