| 1 | #!/usr/bin/python |
| 2 | # |
| 3 | # fio_jsonplus_clat2csv |
| 4 | # |
| 5 | # This script converts fio's json+ completion latency data to CSV format. |
| 6 | # |
| 7 | # For example: |
| 8 | # |
| 9 | # Run the following fio jobs: |
| 10 | # ../fio --output=fio-jsonplus.output --output-format=json+ --name=test1 |
| 11 | # --ioengine=null --time_based --runtime=5s --size=1G --rw=randrw |
| 12 | # --name=test2 --ioengine=null --time_based --runtime=3s --size=1G |
| 13 | # --rw=read --name=test3 --ioengine=null --time_based --runtime=4s |
| 14 | # --size=8G --rw=write |
| 15 | # |
| 16 | # Then run: |
| 17 | # fio_jsonplus_clat2csv fio-jsonplus.output fio-latency.csv |
| 18 | # |
| 19 | # You will end up with the following 3 files |
| 20 | # |
| 21 | # -rw-r--r-- 1 root root 6467 Jun 27 14:57 fio-latency_job0.csv |
| 22 | # -rw-r--r-- 1 root root 3985 Jun 27 14:57 fio-latency_job1.csv |
| 23 | # -rw-r--r-- 1 root root 4490 Jun 27 14:57 fio-latency_job2.csv |
| 24 | # |
| 25 | # fio-latency_job0.csv will look something like: |
| 26 | # |
| 27 | # clat_nsec, read_count, read_cumulative, read_percentile, write_count, |
| 28 | # write_cumulative, write_percentile, trim_count, trim_cumulative, |
| 29 | # trim_percentile, |
| 30 | # 25, 1, 1, 1.50870705013e-07, , , , , , , |
| 31 | # 26, 12, 13, 1.96131916517e-06, 947, 947, 0.000142955890032, , , , |
| 32 | # 27, 843677, 843690, 0.127288105112, 838347, 839294, 0.126696959629, , , , |
| 33 | # 28, 1877982, 2721672, 0.410620573454, 1870189, 2709483, 0.409014312345, , , , |
| 34 | # 29, 4471, 2726143, 0.411295116376, 7718, 2717201, 0.410179395301, , , , |
| 35 | # 30, 2142885, 4869028, 0.734593687087, 2138164, 4855365, 0.732949340025, , , , |
| 36 | # ... |
| 37 | # 2544, , , , 2, 6624404, 0.999997433738, , , , |
| 38 | # 2576, 3, 6628178, 0.99999788781, 4, 6624408, 0.999998037564, , , , |
| 39 | # 2608, 4, 6628182, 0.999998491293, 4, 6624412, 0.999998641391, , , , |
| 40 | # 2640, 3, 6628185, 0.999998943905, 2, 6624414, 0.999998943304, , , , |
| 41 | # 2672, 1, 6628186, 0.999999094776, 3, 6624417, 0.999999396174, , , , |
| 42 | # 2736, 1, 6628187, 0.999999245646, 1, 6624418, 0.99999954713, , , , |
| 43 | # 2768, 2, 6628189, 0.999999547388, 1, 6624419, 0.999999698087, , , , |
| 44 | # 2800, , , , 1, 6624420, 0.999999849043, , , , |
| 45 | # 2832, 1, 6628190, 0.999999698259, , , , , , , |
| 46 | # 4192, 1, 6628191, 0.999999849129, , , , , , , |
| 47 | # 5792, , , , 1, 6624421, 1.0, , , , |
| 48 | # 10304, 1, 6628192, 1.0, , , , , , , |
| 49 | # |
| 50 | # The first line says that you had one read IO with 25ns clat, |
| 51 | # the cumulative number of read IOs at or below 25ns is 1, and |
| 52 | # 25ns is the 0.00001509th percentile for read latency |
| 53 | # |
| 54 | # The job had 2 write IOs complete in 2544ns, |
| 55 | # 6624404 write IOs completed in 2544ns or less, |
| 56 | # and this represents the 99.99974th percentile for write latency |
| 57 | # |
| 58 | # The last line says that one read IO had 10304ns clat, |
| 59 | # 6628192 read IOs had 10304ns or shorter clat, and |
| 60 | # 10304ns is the 100th percentile for read latency |
| 61 | # |
| 62 | |
| 63 | import os |
| 64 | import json |
| 65 | import argparse |
| 66 | |
| 67 | |
| 68 | def parse_args(): |
| 69 | parser = argparse.ArgumentParser() |
| 70 | parser.add_argument('source', |
| 71 | help='fio json+ output file containing completion ' |
| 72 | 'latency data') |
| 73 | parser.add_argument('dest', |
| 74 | help='destination file stub for latency data in CSV ' |
| 75 | 'format. job number will be appended to filename') |
| 76 | args = parser.parse_args() |
| 77 | |
| 78 | return args |
| 79 | |
| 80 | |
| 81 | def percentile(idx, run_total): |
| 82 | total = run_total[len(run_total)-1] |
| 83 | if total == 0: |
| 84 | return 0 |
| 85 | |
| 86 | return float(run_total[idx]) / total |
| 87 | |
| 88 | |
| 89 | def more_lines(indices, bins): |
| 90 | for key, value in indices.iteritems(): |
| 91 | if value < len(bins[key]): |
| 92 | return True |
| 93 | |
| 94 | return False |
| 95 | |
| 96 | |
| 97 | def main(): |
| 98 | args = parse_args() |
| 99 | |
| 100 | with open(args.source, 'r') as source: |
| 101 | jsondata = json.loads(source.read()) |
| 102 | |
| 103 | for jobnum in range(0, len(jsondata['jobs'])): |
| 104 | bins = {} |
| 105 | run_total = {} |
| 106 | ddir_set = set(['read', 'write', 'trim']) |
| 107 | |
| 108 | prev_ddir = None |
| 109 | for ddir in ddir_set: |
| 110 | bins[ddir] = [[int(key), value] for key, value in |
| 111 | jsondata['jobs'][jobnum][ddir]['clat_ns'] |
| 112 | ['bins'].iteritems()] |
| 113 | bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0]) |
| 114 | |
| 115 | run_total[ddir] = [0 for x in range(0, len(bins[ddir]))] |
| 116 | if len(bins[ddir]) > 0: |
| 117 | run_total[ddir][0] = bins[ddir][0][1] |
| 118 | for x in range(1, len(bins[ddir])): |
| 119 | run_total[ddir][x] = run_total[ddir][x-1] + \ |
| 120 | bins[ddir][x][1] |
| 121 | |
| 122 | stub, ext = os.path.splitext(args.dest) |
| 123 | outfile = stub + '_job' + str(jobnum) + ext |
| 124 | |
| 125 | with open(outfile, 'w') as output: |
| 126 | output.write("clat_nsec, ") |
| 127 | ddir_list = list(ddir_set) |
| 128 | for ddir in ddir_list: |
| 129 | output.write("{0}_count, {0}_cumulative, {0}_percentile, ". |
| 130 | format(ddir)) |
| 131 | output.write("\n") |
| 132 | |
| 133 | # |
| 134 | # Have a counter for each ddir |
| 135 | # In each round, pick the shortest remaining duration |
| 136 | # and output a line with any values for that duration |
| 137 | # |
| 138 | indices = {x: 0 for x in ddir_list} |
| 139 | while more_lines(indices, bins): |
| 140 | min_lat = 17112760320 |
| 141 | for ddir in ddir_list: |
| 142 | if indices[ddir] < len(bins[ddir]): |
| 143 | min_lat = min(bins[ddir][indices[ddir]][0], min_lat) |
| 144 | |
| 145 | output.write("{0}, ".format(min_lat)) |
| 146 | |
| 147 | for ddir in ddir_list: |
| 148 | if indices[ddir] < len(bins[ddir]) and \ |
| 149 | min_lat == bins[ddir][indices[ddir]][0]: |
| 150 | count = bins[ddir][indices[ddir]][1] |
| 151 | cumulative = run_total[ddir][indices[ddir]] |
| 152 | ptile = percentile(indices[ddir], run_total[ddir]) |
| 153 | output.write("{0}, {1}, {2}, ".format(count, |
| 154 | cumulative, ptile)) |
| 155 | indices[ddir] += 1 |
| 156 | else: |
| 157 | output.write(", , , ") |
| 158 | output.write("\n") |
| 159 | |
| 160 | print "{0} generated".format(outfile) |
| 161 | |
| 162 | |
| 163 | if __name__ == '__main__': |
| 164 | main() |