Commit | Line | Data |
---|---|---|
60023ade | 1 | #!/usr/bin/python2.7 |
afe8d310 VF |
2 | # |
3 | # fio_jsonplus_clat2csv | |
4 | # | |
5 | # This script converts fio's json+ completion latency data to CSV format. | |
6 | # | |
7 | # For example: | |
8 | # | |
9 | # Run the following fio jobs: | |
10 | # ../fio --output=fio-jsonplus.output --output-format=json+ --name=test1 | |
11 | # --ioengine=null --time_based --runtime=5s --size=1G --rw=randrw | |
12 | # --name=test2 --ioengine=null --time_based --runtime=3s --size=1G | |
13 | # --rw=read --name=test3 --ioengine=null --time_based --runtime=4s | |
14 | # --size=8G --rw=write | |
15 | # | |
16 | # Then run: | |
17 | # fio_jsonplus_clat2csv fio-jsonplus.output fio-latency.csv | |
18 | # | |
19 | # You will end up with the following 3 files | |
20 | # | |
21 | # -rw-r--r-- 1 root root 6467 Jun 27 14:57 fio-latency_job0.csv | |
22 | # -rw-r--r-- 1 root root 3985 Jun 27 14:57 fio-latency_job1.csv | |
23 | # -rw-r--r-- 1 root root 4490 Jun 27 14:57 fio-latency_job2.csv | |
24 | # | |
25 | # fio-latency_job0.csv will look something like: | |
26 | # | |
27 | # clat_nsec, read_count, read_cumulative, read_percentile, write_count, | |
28 | # write_cumulative, write_percentile, trim_count, trim_cumulative, | |
29 | # trim_percentile, | |
30 | # 25, 1, 1, 1.50870705013e-07, , , , , , , | |
31 | # 26, 12, 13, 1.96131916517e-06, 947, 947, 0.000142955890032, , , , | |
32 | # 27, 843677, 843690, 0.127288105112, 838347, 839294, 0.126696959629, , , , | |
33 | # 28, 1877982, 2721672, 0.410620573454, 1870189, 2709483, 0.409014312345, , , , | |
34 | # 29, 4471, 2726143, 0.411295116376, 7718, 2717201, 0.410179395301, , , , | |
35 | # 30, 2142885, 4869028, 0.734593687087, 2138164, 4855365, 0.732949340025, , , , | |
36 | # ... | |
37 | # 2544, , , , 2, 6624404, 0.999997433738, , , , | |
38 | # 2576, 3, 6628178, 0.99999788781, 4, 6624408, 0.999998037564, , , , | |
39 | # 2608, 4, 6628182, 0.999998491293, 4, 6624412, 0.999998641391, , , , | |
40 | # 2640, 3, 6628185, 0.999998943905, 2, 6624414, 0.999998943304, , , , | |
41 | # 2672, 1, 6628186, 0.999999094776, 3, 6624417, 0.999999396174, , , , | |
42 | # 2736, 1, 6628187, 0.999999245646, 1, 6624418, 0.99999954713, , , , | |
43 | # 2768, 2, 6628189, 0.999999547388, 1, 6624419, 0.999999698087, , , , | |
44 | # 2800, , , , 1, 6624420, 0.999999849043, , , , | |
45 | # 2832, 1, 6628190, 0.999999698259, , , , , , , | |
46 | # 4192, 1, 6628191, 0.999999849129, , , , , , , | |
47 | # 5792, , , , 1, 6624421, 1.0, , , , | |
48 | # 10304, 1, 6628192, 1.0, , , , , , , | |
49 | # | |
50 | # The first line says that you had one read IO with 25ns clat, | |
51 | # the cumulative number of read IOs at or below 25ns is 1, and | |
52 | # 25ns is the 0.00001509th percentile for read latency | |
53 | # | |
54 | # The job had 2 write IOs complete in 2544ns, | |
55 | # 6624404 write IOs completed in 2544ns or less, | |
56 | # and this represents the 99.99974th percentile for write latency | |
57 | # | |
58 | # The last line says that one read IO had 10304ns clat, | |
59 | # 6628192 read IOs had 10304ns or shorter clat, and | |
60 | # 10304ns is the 100th percentile for read latency | |
61 | # | |
62 | ||
63 | import os | |
64 | import json | |
65 | import argparse | |
66 | ||
67 | ||
68 | def parse_args(): | |
69 | parser = argparse.ArgumentParser() | |
70 | parser.add_argument('source', | |
71 | help='fio json+ output file containing completion ' | |
72 | 'latency data') | |
73 | parser.add_argument('dest', | |
74 | help='destination file stub for latency data in CSV ' | |
75 | 'format. job number will be appended to filename') | |
76 | args = parser.parse_args() | |
77 | ||
78 | return args | |
79 | ||
80 | ||
81 | def percentile(idx, run_total): | |
82 | total = run_total[len(run_total)-1] | |
83 | if total == 0: | |
84 | return 0 | |
85 | ||
86 | return float(run_total[idx]) / total | |
87 | ||
88 | ||
89 | def more_lines(indices, bins): | |
90 | for key, value in indices.iteritems(): | |
91 | if value < len(bins[key]): | |
92 | return True | |
93 | ||
94 | return False | |
95 | ||
96 | ||
97 | def main(): | |
98 | args = parse_args() | |
99 | ||
100 | with open(args.source, 'r') as source: | |
101 | jsondata = json.loads(source.read()) | |
102 | ||
103 | for jobnum in range(0, len(jsondata['jobs'])): | |
104 | bins = {} | |
105 | run_total = {} | |
106 | ddir_set = set(['read', 'write', 'trim']) | |
107 | ||
108 | prev_ddir = None | |
109 | for ddir in ddir_set: | |
24cab44e VF |
110 | if 'bins' in jsondata['jobs'][jobnum][ddir]['clat_ns']: |
111 | bins_loc = 'clat_ns' | |
112 | elif 'bins' in jsondata['jobs'][jobnum][ddir]['lat_ns']: | |
113 | bins_loc = 'lat_ns' | |
114 | else: | |
115 | raise RuntimeError("Latency bins not found. " | |
116 | "Are you sure you are using json+ output?") | |
117 | ||
afe8d310 | 118 | bins[ddir] = [[int(key), value] for key, value in |
24cab44e | 119 | jsondata['jobs'][jobnum][ddir][bins_loc] |
afe8d310 VF |
120 | ['bins'].iteritems()] |
121 | bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0]) | |
122 | ||
123 | run_total[ddir] = [0 for x in range(0, len(bins[ddir]))] | |
124 | if len(bins[ddir]) > 0: | |
125 | run_total[ddir][0] = bins[ddir][0][1] | |
126 | for x in range(1, len(bins[ddir])): | |
127 | run_total[ddir][x] = run_total[ddir][x-1] + \ | |
128 | bins[ddir][x][1] | |
129 | ||
130 | stub, ext = os.path.splitext(args.dest) | |
131 | outfile = stub + '_job' + str(jobnum) + ext | |
132 | ||
133 | with open(outfile, 'w') as output: | |
24cab44e | 134 | output.write("{0}ec, ".format(bins_loc)) |
afe8d310 VF |
135 | ddir_list = list(ddir_set) |
136 | for ddir in ddir_list: | |
137 | output.write("{0}_count, {0}_cumulative, {0}_percentile, ". | |
138 | format(ddir)) | |
139 | output.write("\n") | |
140 | ||
141 | # | |
142 | # Have a counter for each ddir | |
143 | # In each round, pick the shortest remaining duration | |
144 | # and output a line with any values for that duration | |
145 | # | |
146 | indices = {x: 0 for x in ddir_list} | |
147 | while more_lines(indices, bins): | |
148 | min_lat = 17112760320 | |
149 | for ddir in ddir_list: | |
150 | if indices[ddir] < len(bins[ddir]): | |
151 | min_lat = min(bins[ddir][indices[ddir]][0], min_lat) | |
152 | ||
153 | output.write("{0}, ".format(min_lat)) | |
154 | ||
155 | for ddir in ddir_list: | |
156 | if indices[ddir] < len(bins[ddir]) and \ | |
157 | min_lat == bins[ddir][indices[ddir]][0]: | |
158 | count = bins[ddir][indices[ddir]][1] | |
159 | cumulative = run_total[ddir][indices[ddir]] | |
160 | ptile = percentile(indices[ddir], run_total[ddir]) | |
161 | output.write("{0}, {1}, {2}, ".format(count, | |
162 | cumulative, ptile)) | |
163 | indices[ddir] += 1 | |
164 | else: | |
165 | output.write(", , , ") | |
166 | output.write("\n") | |
167 | ||
168 | print "{0} generated".format(outfile) | |
169 | ||
170 | ||
171 | if __name__ == '__main__': | |
172 | main() |