Commit | Line | Data |
---|---|---|
60023ade | 1 | #!/usr/bin/python2.7 |
50d2a375 | 2 | # Note: this script is python2 and python3 compatible. |
afe8d310 VF |
3 | # |
4 | # fio_jsonplus_clat2csv | |
5 | # | |
6 | # This script converts fio's json+ completion latency data to CSV format. | |
7 | # | |
8 | # For example: | |
9 | # | |
10 | # Run the following fio jobs: | |
11 | # ../fio --output=fio-jsonplus.output --output-format=json+ --name=test1 | |
12 | # --ioengine=null --time_based --runtime=5s --size=1G --rw=randrw | |
13 | # --name=test2 --ioengine=null --time_based --runtime=3s --size=1G | |
14 | # --rw=read --name=test3 --ioengine=null --time_based --runtime=4s | |
15 | # --size=8G --rw=write | |
16 | # | |
17 | # Then run: | |
18 | # fio_jsonplus_clat2csv fio-jsonplus.output fio-latency.csv | |
19 | # | |
20 | # You will end up with the following 3 files | |
21 | # | |
22 | # -rw-r--r-- 1 root root 6467 Jun 27 14:57 fio-latency_job0.csv | |
23 | # -rw-r--r-- 1 root root 3985 Jun 27 14:57 fio-latency_job1.csv | |
24 | # -rw-r--r-- 1 root root 4490 Jun 27 14:57 fio-latency_job2.csv | |
25 | # | |
26 | # fio-latency_job0.csv will look something like: | |
27 | # | |
28 | # clat_nsec, read_count, read_cumulative, read_percentile, write_count, | |
29 | # write_cumulative, write_percentile, trim_count, trim_cumulative, | |
30 | # trim_percentile, | |
31 | # 25, 1, 1, 1.50870705013e-07, , , , , , , | |
32 | # 26, 12, 13, 1.96131916517e-06, 947, 947, 0.000142955890032, , , , | |
33 | # 27, 843677, 843690, 0.127288105112, 838347, 839294, 0.126696959629, , , , | |
34 | # 28, 1877982, 2721672, 0.410620573454, 1870189, 2709483, 0.409014312345, , , , | |
35 | # 29, 4471, 2726143, 0.411295116376, 7718, 2717201, 0.410179395301, , , , | |
36 | # 30, 2142885, 4869028, 0.734593687087, 2138164, 4855365, 0.732949340025, , , , | |
37 | # ... | |
38 | # 2544, , , , 2, 6624404, 0.999997433738, , , , | |
39 | # 2576, 3, 6628178, 0.99999788781, 4, 6624408, 0.999998037564, , , , | |
40 | # 2608, 4, 6628182, 0.999998491293, 4, 6624412, 0.999998641391, , , , | |
41 | # 2640, 3, 6628185, 0.999998943905, 2, 6624414, 0.999998943304, , , , | |
42 | # 2672, 1, 6628186, 0.999999094776, 3, 6624417, 0.999999396174, , , , | |
43 | # 2736, 1, 6628187, 0.999999245646, 1, 6624418, 0.99999954713, , , , | |
44 | # 2768, 2, 6628189, 0.999999547388, 1, 6624419, 0.999999698087, , , , | |
45 | # 2800, , , , 1, 6624420, 0.999999849043, , , , | |
46 | # 2832, 1, 6628190, 0.999999698259, , , , , , , | |
47 | # 4192, 1, 6628191, 0.999999849129, , , , , , , | |
48 | # 5792, , , , 1, 6624421, 1.0, , , , | |
49 | # 10304, 1, 6628192, 1.0, , , , , , , | |
50 | # | |
51 | # The first line says that you had one read IO with 25ns clat, | |
52 | # the cumulative number of read IOs at or below 25ns is 1, and | |
53 | # 25ns is the 0.00001509th percentile for read latency | |
54 | # | |
55 | # The job had 2 write IOs complete in 2544ns, | |
56 | # 6624404 write IOs completed in 2544ns or less, | |
57 | # and this represents the 99.99974th percentile for write latency | |
58 | # | |
59 | # The last line says that one read IO had 10304ns clat, | |
60 | # 6628192 read IOs had 10304ns or shorter clat, and | |
61 | # 10304ns is the 100th percentile for read latency | |
62 | # | |
63 | ||
50d2a375 BD |
64 | from __future__ import absolute_import |
65 | from __future__ import print_function | |
afe8d310 VF |
66 | import os |
67 | import json | |
68 | import argparse | |
50d2a375 BD |
69 | import six |
70 | from six.moves import range | |
afe8d310 VF |
71 | |
72 | ||
73 | def parse_args(): | |
74 | parser = argparse.ArgumentParser() | |
75 | parser.add_argument('source', | |
76 | help='fio json+ output file containing completion ' | |
77 | 'latency data') | |
78 | parser.add_argument('dest', | |
79 | help='destination file stub for latency data in CSV ' | |
80 | 'format. job number will be appended to filename') | |
81 | args = parser.parse_args() | |
82 | ||
83 | return args | |
84 | ||
85 | ||
86 | def percentile(idx, run_total): | |
87 | total = run_total[len(run_total)-1] | |
88 | if total == 0: | |
89 | return 0 | |
90 | ||
91 | return float(run_total[idx]) / total | |
92 | ||
93 | ||
94 | def more_lines(indices, bins): | |
50d2a375 | 95 | for key, value in six.iteritems(indices): |
afe8d310 VF |
96 | if value < len(bins[key]): |
97 | return True | |
98 | ||
99 | return False | |
100 | ||
101 | ||
102 | def main(): | |
103 | args = parse_args() | |
104 | ||
105 | with open(args.source, 'r') as source: | |
106 | jsondata = json.loads(source.read()) | |
107 | ||
108 | for jobnum in range(0, len(jsondata['jobs'])): | |
109 | bins = {} | |
110 | run_total = {} | |
111 | ddir_set = set(['read', 'write', 'trim']) | |
112 | ||
113 | prev_ddir = None | |
114 | for ddir in ddir_set: | |
24cab44e VF |
115 | if 'bins' in jsondata['jobs'][jobnum][ddir]['clat_ns']: |
116 | bins_loc = 'clat_ns' | |
117 | elif 'bins' in jsondata['jobs'][jobnum][ddir]['lat_ns']: | |
118 | bins_loc = 'lat_ns' | |
119 | else: | |
120 | raise RuntimeError("Latency bins not found. " | |
121 | "Are you sure you are using json+ output?") | |
122 | ||
afe8d310 | 123 | bins[ddir] = [[int(key), value] for key, value in |
50d2a375 BD |
124 | six.iteritems(jsondata['jobs'][jobnum][ddir][bins_loc] |
125 | ['bins'])] | |
afe8d310 VF |
126 | bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0]) |
127 | ||
128 | run_total[ddir] = [0 for x in range(0, len(bins[ddir]))] | |
129 | if len(bins[ddir]) > 0: | |
130 | run_total[ddir][0] = bins[ddir][0][1] | |
131 | for x in range(1, len(bins[ddir])): | |
132 | run_total[ddir][x] = run_total[ddir][x-1] + \ | |
133 | bins[ddir][x][1] | |
134 | ||
135 | stub, ext = os.path.splitext(args.dest) | |
136 | outfile = stub + '_job' + str(jobnum) + ext | |
137 | ||
138 | with open(outfile, 'w') as output: | |
24cab44e | 139 | output.write("{0}ec, ".format(bins_loc)) |
afe8d310 VF |
140 | ddir_list = list(ddir_set) |
141 | for ddir in ddir_list: | |
142 | output.write("{0}_count, {0}_cumulative, {0}_percentile, ". | |
143 | format(ddir)) | |
144 | output.write("\n") | |
145 | ||
146 | # | |
147 | # Have a counter for each ddir | |
148 | # In each round, pick the shortest remaining duration | |
149 | # and output a line with any values for that duration | |
150 | # | |
151 | indices = {x: 0 for x in ddir_list} | |
152 | while more_lines(indices, bins): | |
153 | min_lat = 17112760320 | |
154 | for ddir in ddir_list: | |
155 | if indices[ddir] < len(bins[ddir]): | |
156 | min_lat = min(bins[ddir][indices[ddir]][0], min_lat) | |
157 | ||
158 | output.write("{0}, ".format(min_lat)) | |
159 | ||
160 | for ddir in ddir_list: | |
161 | if indices[ddir] < len(bins[ddir]) and \ | |
162 | min_lat == bins[ddir][indices[ddir]][0]: | |
163 | count = bins[ddir][indices[ddir]][1] | |
164 | cumulative = run_total[ddir][indices[ddir]] | |
165 | ptile = percentile(indices[ddir], run_total[ddir]) | |
166 | output.write("{0}, {1}, {2}, ".format(count, | |
167 | cumulative, ptile)) | |
168 | indices[ddir] += 1 | |
169 | else: | |
170 | output.write(", , , ") | |
171 | output.write("\n") | |
172 | ||
50d2a375 | 173 | print("{0} generated".format(outfile)) |
afe8d310 VF |
174 | |
175 | ||
176 | if __name__ == '__main__': | |
177 | main() |