Commit | Line | Data |
---|---|---|
afe8d310 VF |
1 | #!/usr/bin/python |
2 | # | |
3 | # fio_jsonplus_clat2csv | |
4 | # | |
5 | # This script converts fio's json+ completion latency data to CSV format. | |
6 | # | |
7 | # For example: | |
8 | # | |
9 | # Run the following fio jobs: | |
10 | # ../fio --output=fio-jsonplus.output --output-format=json+ --name=test1 | |
11 | # --ioengine=null --time_based --runtime=5s --size=1G --rw=randrw | |
12 | # --name=test2 --ioengine=null --time_based --runtime=3s --size=1G | |
13 | # --rw=read --name=test3 --ioengine=null --time_based --runtime=4s | |
14 | # --size=8G --rw=write | |
15 | # | |
16 | # Then run: | |
17 | # fio_jsonplus_clat2csv fio-jsonplus.output fio-latency.csv | |
18 | # | |
19 | # You will end up with the following 3 files | |
20 | # | |
21 | # -rw-r--r-- 1 root root 6467 Jun 27 14:57 fio-latency_job0.csv | |
22 | # -rw-r--r-- 1 root root 3985 Jun 27 14:57 fio-latency_job1.csv | |
23 | # -rw-r--r-- 1 root root 4490 Jun 27 14:57 fio-latency_job2.csv | |
24 | # | |
25 | # fio-latency_job0.csv will look something like: | |
26 | # | |
27 | # clat_nsec, read_count, read_cumulative, read_percentile, write_count, | |
28 | # write_cumulative, write_percentile, trim_count, trim_cumulative, | |
29 | # trim_percentile, | |
30 | # 25, 1, 1, 1.50870705013e-07, , , , , , , | |
31 | # 26, 12, 13, 1.96131916517e-06, 947, 947, 0.000142955890032, , , , | |
32 | # 27, 843677, 843690, 0.127288105112, 838347, 839294, 0.126696959629, , , , | |
33 | # 28, 1877982, 2721672, 0.410620573454, 1870189, 2709483, 0.409014312345, , , , | |
34 | # 29, 4471, 2726143, 0.411295116376, 7718, 2717201, 0.410179395301, , , , | |
35 | # 30, 2142885, 4869028, 0.734593687087, 2138164, 4855365, 0.732949340025, , , , | |
36 | # ... | |
37 | # 2544, , , , 2, 6624404, 0.999997433738, , , , | |
38 | # 2576, 3, 6628178, 0.99999788781, 4, 6624408, 0.999998037564, , , , | |
39 | # 2608, 4, 6628182, 0.999998491293, 4, 6624412, 0.999998641391, , , , | |
40 | # 2640, 3, 6628185, 0.999998943905, 2, 6624414, 0.999998943304, , , , | |
41 | # 2672, 1, 6628186, 0.999999094776, 3, 6624417, 0.999999396174, , , , | |
42 | # 2736, 1, 6628187, 0.999999245646, 1, 6624418, 0.99999954713, , , , | |
43 | # 2768, 2, 6628189, 0.999999547388, 1, 6624419, 0.999999698087, , , , | |
44 | # 2800, , , , 1, 6624420, 0.999999849043, , , , | |
45 | # 2832, 1, 6628190, 0.999999698259, , , , , , , | |
46 | # 4192, 1, 6628191, 0.999999849129, , , , , , , | |
47 | # 5792, , , , 1, 6624421, 1.0, , , , | |
48 | # 10304, 1, 6628192, 1.0, , , , , , , | |
49 | # | |
50 | # The first line says that you had one read IO with 25ns clat, | |
51 | # the cumulative number of read IOs at or below 25ns is 1, and | |
52 | # 25ns is the 0.00001509th percentile for read latency | |
53 | # | |
54 | # The job had 2 write IOs complete in 2544ns, | |
55 | # 6624404 write IOs completed in 2544ns or less, | |
56 | # and this represents the 99.99974th percentile for write latency | |
57 | # | |
58 | # The last line says that one read IO had 10304ns clat, | |
59 | # 6628192 read IOs had 10304ns or shorter clat, and | |
60 | # 10304ns is the 100th percentile for read latency | |
61 | # | |
62 | ||
63 | import os | |
64 | import json | |
65 | import argparse | |
66 | ||
67 | ||
68 | def parse_args(): | |
69 | parser = argparse.ArgumentParser() | |
70 | parser.add_argument('source', | |
71 | help='fio json+ output file containing completion ' | |
72 | 'latency data') | |
73 | parser.add_argument('dest', | |
74 | help='destination file stub for latency data in CSV ' | |
75 | 'format. job number will be appended to filename') | |
76 | args = parser.parse_args() | |
77 | ||
78 | return args | |
79 | ||
80 | ||
81 | def percentile(idx, run_total): | |
82 | total = run_total[len(run_total)-1] | |
83 | if total == 0: | |
84 | return 0 | |
85 | ||
86 | return float(run_total[idx]) / total | |
87 | ||
88 | ||
89 | def more_lines(indices, bins): | |
90 | for key, value in indices.iteritems(): | |
91 | if value < len(bins[key]): | |
92 | return True | |
93 | ||
94 | return False | |
95 | ||
96 | ||
97 | def main(): | |
98 | args = parse_args() | |
99 | ||
100 | with open(args.source, 'r') as source: | |
101 | jsondata = json.loads(source.read()) | |
102 | ||
103 | for jobnum in range(0, len(jsondata['jobs'])): | |
104 | bins = {} | |
105 | run_total = {} | |
106 | ddir_set = set(['read', 'write', 'trim']) | |
107 | ||
108 | prev_ddir = None | |
109 | for ddir in ddir_set: | |
110 | bins[ddir] = [[int(key), value] for key, value in | |
111 | jsondata['jobs'][jobnum][ddir]['clat_ns'] | |
112 | ['bins'].iteritems()] | |
113 | bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0]) | |
114 | ||
115 | run_total[ddir] = [0 for x in range(0, len(bins[ddir]))] | |
116 | if len(bins[ddir]) > 0: | |
117 | run_total[ddir][0] = bins[ddir][0][1] | |
118 | for x in range(1, len(bins[ddir])): | |
119 | run_total[ddir][x] = run_total[ddir][x-1] + \ | |
120 | bins[ddir][x][1] | |
121 | ||
122 | stub, ext = os.path.splitext(args.dest) | |
123 | outfile = stub + '_job' + str(jobnum) + ext | |
124 | ||
125 | with open(outfile, 'w') as output: | |
126 | output.write("clat_nsec, ") | |
127 | ddir_list = list(ddir_set) | |
128 | for ddir in ddir_list: | |
129 | output.write("{0}_count, {0}_cumulative, {0}_percentile, ". | |
130 | format(ddir)) | |
131 | output.write("\n") | |
132 | ||
133 | # | |
134 | # Have a counter for each ddir | |
135 | # In each round, pick the shortest remaining duration | |
136 | # and output a line with any values for that duration | |
137 | # | |
138 | indices = {x: 0 for x in ddir_list} | |
139 | while more_lines(indices, bins): | |
140 | min_lat = 17112760320 | |
141 | for ddir in ddir_list: | |
142 | if indices[ddir] < len(bins[ddir]): | |
143 | min_lat = min(bins[ddir][indices[ddir]][0], min_lat) | |
144 | ||
145 | output.write("{0}, ".format(min_lat)) | |
146 | ||
147 | for ddir in ddir_list: | |
148 | if indices[ddir] < len(bins[ddir]) and \ | |
149 | min_lat == bins[ddir][indices[ddir]][0]: | |
150 | count = bins[ddir][indices[ddir]][1] | |
151 | cumulative = run_total[ddir][indices[ddir]] | |
152 | ptile = percentile(indices[ddir], run_total[ddir]) | |
153 | output.write("{0}, {1}, {2}, ".format(count, | |
154 | cumulative, ptile)) | |
155 | indices[ddir] += 1 | |
156 | else: | |
157 | output.write(", , , ") | |
158 | output.write("\n") | |
159 | ||
160 | print "{0} generated".format(outfile) | |
161 | ||
162 | ||
163 | if __name__ == '__main__': | |
164 | main() |