3 # Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
4 # Author: Erwan Velu <erwan@enovance.com>
6 # The license below covers all files distributed with fio unless otherwise
7 # noted in the file itself.
9 # This program is free software; you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License version 2 as
11 # published by the Free Software Foundation.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License
19 # along with this program; if not, write to the Free Software
20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
29 def find_file(path, pattern):
31 # For all the local files
32 for file in os.listdir(path):
33 # If the file math the regexp
34 if fnmatch.fnmatch(file, pattern):
35 # Let's consider this file
36 fio_data_file.append(file)
40 def generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,mode,disk_perf):
42 f.write("call \'graph3D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\'\n" % (title,gnuplot_output_filename,gnuplot_output_filename,mode))
45 # Let's create a temporary file for each selected fio file
46 for file in fio_data_file:
47 tmp_filename = "gnuplot_temp_file.%d" % pos
48 png_file=file.replace('.log','')
49 raw_filename = "%s-2Draw" % (png_file)
50 smooth_filename = "%s-2Dsmooth" % (png_file)
51 trend_filename = "%s-2Dtrend" % (png_file)
52 avg = average(disk_perf[pos])
53 f.write("call \'graph2D.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' \'%s\' \'%s\' \'%f\'\n" % (title,tmp_filename,raw_filename,mode,smooth_filename,trend_filename,avg))
58 def generate_gnuplot_math_script(title,gnuplot_output_filename,mode,average):
60 f.write("call \'math.gpm\' \'%s' \'%s\' \'\' \'%s\' \'%s\' %s\n" % (title,gnuplot_output_filename,gnuplot_output_filename,mode,average))
63 def compute_aggregated_file(fio_data_file, gnuplot_output_filename):
66 # Let's create a temporary file for each selected fio file
67 for file in fio_data_file:
68 tmp_filename = "gnuplot_temp_file.%d" % pos
69 temp_files.append(open(tmp_filename,'r'))
72 f = open(gnuplot_output_filename, "w")
74 # Let's add some information
75 for tempfile in temp_files:
76 f.write("# Disk%d was coming from %s\n" % (index,fio_data_file[index]))
77 f.write(tempfile.read())
83 def average(s): return sum(s) * 1.0 / len(s)
85 def compute_temp_file(fio_data_file,disk_perf):
89 for file in fio_data_file:
90 files.append(open(file))
92 tmp_filename = "gnuplot_temp_file.%d" % pos
93 temp_outfile.append(open(tmp_filename,'w'))
100 s=file.readline().replace(',',' ').split()
104 current_line.append(s);
106 if shall_break == True:
112 for line in current_line:
113 time, perf, x, block_size = line
115 blk_size=int(block_size)
117 # We ignore the first 500msec as it doesn't seems to be part of the real benchmark
118 # Time < 500 usually reports BW=0 breaking the min computing
119 if ((int(time)) > 500):
120 disk_perf[index].append(int(perf))
124 # If we reach this point, it means that all the traces are coherent
125 for p in enumerate(perfs):
126 temp_outfile[p[0]].write("%s %.2f %s\n" % (p[0], float(float(time)/1000), p[1]))
130 for file in temp_outfile:
134 def compute_math(fio_data_file, title,gnuplot_output_filename,mode,disk_perf):
137 average_file=open(gnuplot_output_filename+'.average', 'w')
138 min_file=open(gnuplot_output_filename+'.min', 'w')
139 max_file=open(gnuplot_output_filename+'.max', 'w')
140 stddev_file=open(gnuplot_output_filename+'.stddev', 'w')
141 global_file=open(gnuplot_output_filename+'.global','w')
143 min_file.write('DiskName %s\n' % mode)
144 max_file.write('DiskName %s\n'% mode)
145 average_file.write('DiskName %s\n'% mode)
146 stddev_file.write('DiskName %s\n'% mode )
147 for disk in xrange(len(fio_data_file)):
148 # print disk_perf[disk]
149 min_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
150 max_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
151 average_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
152 stddev_file.write("# Disk%d was coming from %s\n" % (disk,fio_data_file[disk]))
153 avg = average(disk_perf[disk])
154 variance = map(lambda x: (x - avg)**2, disk_perf[disk])
155 standard_deviation = math.sqrt(average(variance))
156 # print "Disk%d [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (disk,min(disk_perf[disk]),max(disk_perf[disk]),avg, standard_deviation)
157 average_file.write('%d %d\n' % (disk, avg))
158 stddev_file.write('%d %d\n' % (disk, standard_deviation))
159 local_min=min(disk_perf[disk])
160 local_max=max(disk_perf[disk])
161 min_file.write('%d %d\n' % (disk, local_min))
162 max_file.write('%d %d\n' % (disk, local_max))
163 global_min.append(int(local_min))
164 global_max.append(int(local_max))
166 global_disk_perf = sum(disk_perf, [])
167 avg = average(global_disk_perf)
168 variance = map(lambda x: (x - avg)**2, global_disk_perf)
169 standard_deviation = math.sqrt(average(variance))
171 global_file.write('min=%.2f\n' % min(global_disk_perf))
172 global_file.write('max=%.2f\n' % max(global_disk_perf))
173 global_file.write('avg=%.2f\n' % avg)
174 global_file.write('stddev=%.2f\n' % standard_deviation)
175 global_file.write('values_count=%d\n' % len(global_disk_perf))
176 global_file.write('disks_count=%d\n' % len(fio_data_file))
177 #print "Global [ min=%.2f max=%.2f avg=%.2f stddev=%.2f \n" % (min(global_disk_perf),max(global_disk_perf),avg, standard_deviation)
189 generate_gnuplot_math_script("Average values of "+title,gnuplot_output_filename+'.average',mode,int(avg))
190 generate_gnuplot_math_script("Min values of "+title,gnuplot_output_filename+'.min',mode,average(global_min))
191 generate_gnuplot_math_script("Max values of "+title,gnuplot_output_filename+'.max',mode,average(global_max))
192 generate_gnuplot_math_script("Standard Deviation of "+title,gnuplot_output_filename+'.stddev',mode,int(standard_deviation))
194 def parse_global_files(fio_data_file, global_search):
197 for file in fio_data_file:
202 # Let's read the complete file
205 # We do split the name from the value
206 name,value=f.readline().split("=")
210 # If we ended the file
212 # Let's process what we have
216 # disks_count is not global_search item
217 # As we need it for some computation, let's save it
218 if name=="disks_count":
219 disks_count=int(value)
221 # Let's catch the searched item
222 if global_search in name:
223 search_value=float(value)
225 # Let's process the avg value by estimated the global bandwidth per file
226 # We keep the biggest in memory for reporting
227 if global_search == "avg":
228 if (disks_count > 0) and (search_value != -1):
229 result=disks_count*search_value
230 if (result > max_result):
233 # Let's print the avg output
234 if global_search == "avg":
235 print "Biggest aggregated value of %s was %2.f in file %s\n" % (global_search, max_result, max_file)
237 print "Global search %s is not yet implemented\n" % global_search
239 def render_gnuplot():
240 print "Running gnuplot Rendering\n"
242 os.system("gnuplot mymath")
243 os.system("gnuplot mygraph")
245 print "Could not run gnuplot on mymath or mygraph !\n"
249 print 'fio2gnuplot.py -ghbio -t <title> -o <outputfile> -p <pattern>'
251 print '-h --help : Print this help'
252 print '-p <pattern> or --pattern <pattern> : A pattern in regexp to select fio input files'
253 print '-b or --bandwidth : A predefined pattern for selecting *_bw.log files'
254 print '-i or --iops : A predefined pattern for selecting *_iops.log files'
255 print '-g or --gnuplot : Render gnuplot traces before exiting'
256 print '-o or --outputfile <file> : The basename for gnuplot traces'
257 print ' - Basename is set with the pattern if defined'
258 print '-t or --title <title> : The title of the gnuplot traces'
259 print ' - Title is set with the block size detected in fio traces'
260 print '-G or --Global <type> : Search for <type> in .global files match by a pattern'
261 print ' - Available types are : min, max, avg, stddev'
262 print ' - The .global extension is added automatically to the pattern'
267 pattern_set_by_user=False
269 gnuplot_output_filename='result'
276 opts, args = getopt.getopt(argv[1:],"ghbio:t:p:G:")
277 except getopt.GetoptError:
281 for opt, arg in opts:
282 if opt in ("-b", "--bandwidth"):
284 elif opt in ("-i", "--iops"):
286 elif opt in ("-p", "--pattern"):
287 pattern_set_by_user=True
289 pattern=pattern.replace('\\','')
290 elif opt in ("-o", "--outputfile"):
291 gnuplot_output_filename=arg
292 elif opt in ("-t", "--title"):
294 elif opt in ("-g", "--gnuplot"):
296 elif opt in ("-G", "--Global"):
299 elif opt in ("-h", "--help"):
303 # Adding .global extension to the file
304 if parse_global==True:
305 if not gnuplot_output_filename.endswith('.global'):
306 pattern = pattern+'.global'
308 fio_data_file=find_file('.',pattern)
309 if len(fio_data_file) == 0:
310 print "No log file found with pattern %s!" % pattern
313 fio_data_file=sorted(fio_data_file, key=str.lower)
314 for file in fio_data_file:
315 print 'Selected %s' % file
316 if "_bw.log" in file :
317 mode="Bandwidth (KB/sec)"
318 if "_iops.log" in file :
319 mode="IO per Seconds (IO/sec)"
320 if (title == 'No title') and (mode != 'unknown'):
321 if "Bandwidth" in mode:
322 title='Bandwidth benchmark with %d fio results' % len(fio_data_file)
324 title='IO benchmark with %d fio results' % len(fio_data_file)
326 #We need to adjust the output filename regarding the pattern required by the user
327 if (pattern_set_by_user == True):
328 gnuplot_output_filename=pattern
329 # As we do have some regexp in the pattern, let's make this simpliest
330 # We do remove the simpliest parts of the expression to get a clear file name
331 gnuplot_output_filename=gnuplot_output_filename.replace('-*-','-')
332 gnuplot_output_filename=gnuplot_output_filename.replace('*','-')
333 gnuplot_output_filename=gnuplot_output_filename.replace('--','-')
334 gnuplot_output_filename=gnuplot_output_filename.replace('.log','')
335 # Insure that we don't have any starting or trailing dash to the filename
336 gnuplot_output_filename = gnuplot_output_filename[:-1] if gnuplot_output_filename.endswith('-') else gnuplot_output_filename
337 gnuplot_output_filename = gnuplot_output_filename[1:] if gnuplot_output_filename.startswith('-') else gnuplot_output_filename
339 if parse_global==True:
340 parse_global_files(fio_data_file, global_search)
342 blk_size=compute_temp_file(fio_data_file,disk_perf)
343 title="%s @ Blocksize = %dK" % (title,blk_size/1024)
344 compute_aggregated_file(fio_data_file, gnuplot_output_filename)
345 compute_math(fio_data_file,title,gnuplot_output_filename,mode,disk_perf)
346 generate_gnuplot_script(fio_data_file,title,gnuplot_output_filename,mode,disk_perf)
348 if (run_gnuplot==True):
351 # Cleaning temporary files
353 os.remove('gnuplot_temp_file.*')
358 if __name__ == "__main__":
359 sys.exit(main(sys.argv))