Commit | Line | Data |
---|---|---|
726d2aee AB |
1 | #! /usr/bin/env python |
2 | # | |
3 | # btt_plot.py: Generate matplotlib plots for BTT generate data files | |
4 | # | |
5 | # (C) Copyright 2009 Hewlett-Packard Development Company, L.P. | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU General Public License as published by | |
9 | # the Free Software Foundation; either version 2 of the License, or | |
10 | # (at your option) any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU General Public License for more details. | |
16 | # | |
17 | # You should have received a copy of the GNU General Public License | |
18 | # along with this program; if not, write to the Free Software | |
19 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
20 | # | |
21 | ||
22 | """ | |
23 | btt_plot.py: Generate matplotlib plots for BTT generated data files | |
24 | ||
25 | Files handled: | |
26 | AQD - Average Queue Depth Running average of queue depths | |
27 | ||
28 | BNOS - Block numbers accessed Markers for each block | |
29 | ||
30 | Q2D - Queue to Issue latencies Running averages | |
31 | D2C - Issue to Complete latencies Running averages | |
32 | Q2C - Queue to Complete latencies Running averages | |
33 | ||
34 | Usage: | |
35 | btt_plot_aqd.py equivalent to: btt_plot.py -t aqd <type>=aqd | |
36 | btt_plot_bnos.py equivalent to: btt_plot.py -t bnos <type>=bnos | |
37 | btt_plot_q2d.py equivalent to: btt_plot.py -t q2d <type>=q2d | |
38 | btt_plot_d2c.py equivalent to: btt_plot.py -t d2c <type>=d2c | |
39 | btt_plot_q2c.py equivalent to: btt_plot.py -t q2c <type>=q2c | |
40 | ||
41 | Arguments: | |
42 | [ -A | --generate-all ] Default: False | |
43 | [ -L | --no-legend ] Default: Legend table produced | |
44 | [ -o <file> | --output=<file> ] Default: <type>.png | |
45 | [ -T <string> | --title=<string> ] Default: Based upon <type> | |
46 | [ -v | --verbose ] Default: False | |
47 | <data-files...> | |
48 | ||
49 | The -A (--generate-all) argument is different: when this is specified, | |
50 | an attempt is made to generate default plots for all 5 types (aqd, bnos, | |
51 | q2d, d2c and q2c). It will find files with the appropriate suffix for | |
52 | each type ('aqd.dat' for example). If such files are found, a plot for | |
53 | that type will be made. The output file name will be the default for | |
54 | each type. The -L (--no-legend) option will be obeyed for all plots, | |
55 | but the -o (--output) and -T (--title) options will be ignored. | |
56 | """ | |
57 | ||
58 | __author__ = 'Alan D. Brunelle <alan.brunelle@hp.com>' | |
59 | ||
60 | #------------------------------------------------------------------------------ | |
61 | ||
62 | import matplotlib | |
63 | matplotlib.use('Agg') | |
64 | import getopt, glob, os, sys | |
65 | import matplotlib.pyplot as plt | |
66 | ||
67 | plot_size = [10.9, 8.4] # inches... | |
68 | ||
69 | add_legend = True | |
70 | generate_all = False | |
71 | output_file = None | |
72 | title_str = None | |
73 | type = None | |
74 | verbose = False | |
75 | ||
2e37a10e | 76 | types = [ 'aqd', 'q2d', 'd2c', 'q2c', 'live', 'bnos' ] |
726d2aee AB |
77 | progs = [ 'btt_plot_%s.py' % t for t in types ] |
78 | ||
79 | get_base = lambda file: file[file.find('_')+1:file.rfind('_')] | |
80 | ||
81 | #------------------------------------------------------------------------------ | |
82 | def fatal(msg): | |
83 | """Generate fatal error message and exit""" | |
84 | ||
85 | print >>sys.stderr, 'FATAL: %s' % msg | |
86 | sys.exit(1) | |
87 | ||
2e37a10e AB |
88 | #------------------------------------------------------------------------------ |
89 | def gen_legends(ax, legends): | |
90 | leg = ax.legend(legends, 'best', shadow=True) | |
91 | frame = leg.get_frame() | |
92 | frame.set_facecolor('0.80') | |
93 | for t in leg.get_texts(): | |
94 | t.set_fontsize('xx-small') | |
95 | ||
726d2aee AB |
96 | #---------------------------------------------------------------------- |
97 | def get_data(files): | |
98 | """Retrieve data from files provided. | |
99 | ||
100 | Returns a database containing: | |
101 | 'min_x', 'max_x' - Minimum and maximum X values found | |
102 | 'min_y', 'max_y' - Minimum and maximum Y values found | |
103 | 'x', 'y' - X & Y value arrays | |
104 | 'ax', 'ay' - Running average over X & Y -- | |
105 | if > 10 values provided... | |
106 | """ | |
107 | #-------------------------------------------------------------- | |
108 | def check(mn, mx, v): | |
109 | """Returns new min, max, and float value for those passed in""" | |
110 | ||
111 | v = float(v) | |
112 | if mn == None or v < mn: mn = v | |
113 | if mx == None or v > mx: mx = v | |
114 | return mn, mx, v | |
115 | ||
116 | #-------------------------------------------------------------- | |
117 | def avg(xs, ys): | |
118 | """Computes running average for Xs and Ys""" | |
119 | ||
120 | #------------------------------------------------------ | |
121 | def _avg(vals): | |
122 | """Computes average for array of values passed""" | |
123 | ||
124 | total = 0.0 | |
125 | for val in vals: | |
126 | total += val | |
127 | return total / len(vals) | |
128 | ||
129 | #------------------------------------------------------ | |
130 | if len(xs) < 1000: | |
131 | return xs, ys | |
132 | ||
133 | axs = [xs[0]] | |
134 | ays = [ys[0]] | |
135 | _xs = [xs[0]] | |
136 | _ys = [ys[0]] | |
137 | ||
138 | x_range = (xs[-1] - xs[0]) / 100 | |
139 | for idx in range(1, len(ys)): | |
140 | if (xs[idx] - _xs[0]) > x_range: | |
141 | axs.append(_avg(_xs)) | |
142 | ays.append(_avg(_ys)) | |
143 | del _xs, _ys | |
144 | ||
145 | _xs = [xs[idx]] | |
146 | _ys = [ys[idx]] | |
147 | else: | |
148 | _xs.append(xs[idx]) | |
149 | _ys.append(ys[idx]) | |
150 | ||
151 | if len(_xs) > 1: | |
152 | axs.append(_avg(_xs)) | |
153 | ays.append(_avg(_ys)) | |
154 | ||
155 | return axs, ays | |
156 | ||
157 | #-------------------------------------------------------------- | |
158 | global verbose | |
159 | ||
160 | db = {} | |
161 | min_x = max_x = min_y = max_y = None | |
162 | for file in files: | |
163 | if not os.path.exists(file): | |
164 | fatal('%s not found' % file) | |
165 | elif verbose: | |
166 | print 'Processing %s' % file | |
167 | ||
168 | xs = [] | |
169 | ys = [] | |
170 | for line in open(file, 'r'): | |
171 | f = line.rstrip().split(None) | |
172 | if line.find('#') == 0 or len(f) < 2: | |
173 | continue | |
174 | (min_x, max_x, x) = check(min_x, max_x, f[0]) | |
175 | (min_y, max_y, y) = check(min_y, max_y, f[1]) | |
176 | xs.append(x) | |
177 | ys.append(y) | |
178 | ||
179 | db[file] = {'x':xs, 'y':ys} | |
180 | if len(xs) > 10: | |
181 | db[file]['ax'], db[file]['ay'] = avg(xs, ys) | |
182 | else: | |
183 | db[file]['ax'] = db[file]['ay'] = None | |
184 | ||
185 | db['min_x'] = min_x | |
186 | db['max_x'] = max_x | |
187 | db['min_y'] = min_y | |
188 | db['max_y'] = max_y | |
189 | return db | |
190 | ||
191 | #---------------------------------------------------------------------- | |
192 | def parse_args(args): | |
193 | """Parse command line arguments. | |
194 | ||
195 | Returns list of (data) files that need to be processed -- /unless/ | |
196 | the -A (--generate-all) option is passed, in which case superfluous | |
197 | data files are ignored... | |
198 | """ | |
199 | ||
200 | global add_legend, output_file, title_str, type, verbose | |
201 | global generate_all | |
202 | ||
203 | prog = args[0][args[0].rfind('/')+1:] | |
204 | if prog == 'btt_plot.py': | |
205 | pass | |
206 | elif not prog in progs: | |
207 | fatal('%s not a valid command name' % prog) | |
208 | else: | |
209 | type = prog[prog.rfind('_')+1:prog.rfind('.py')] | |
210 | ||
211 | s_opts = 'ALo:t:T:v' | |
212 | l_opts = [ 'generate-all', 'type', 'no-legend', 'output', 'title', | |
213 | 'verbose' ] | |
214 | ||
215 | try: | |
216 | (opts, args) = getopt.getopt(args[1:], s_opts, l_opts) | |
217 | except getopt.error, msg: | |
218 | print >>sys.stderr, msg | |
219 | fatal(__doc__) | |
220 | ||
221 | for (o, a) in opts: | |
222 | if o in ('-A', '--generate-all'): | |
223 | generate_all = True | |
224 | elif o in ('-L', '--no-legend'): | |
225 | add_legend = False | |
226 | elif o in ('-o', '--output'): | |
227 | output_file = a | |
228 | elif o in ('-t', '--type'): | |
229 | if not a in types: | |
230 | fatal('Type %s not supported' % a) | |
231 | type = a | |
232 | elif o in ('-T', '--title'): | |
233 | title_str = a | |
234 | elif o in ('-v', '--verbose'): | |
235 | verbose = True | |
236 | ||
237 | if type == None and not generate_all: | |
238 | fatal('Need type of data files to process - (-t <type>)') | |
239 | ||
240 | return args | |
241 | ||
242 | #------------------------------------------------------------------------------ | |
243 | def gen_title(fig, type, title_str): | |
244 | """Sets the title for the figure based upon the type /or/ user title""" | |
245 | ||
246 | if title_str != None: | |
247 | pass | |
248 | elif type == 'aqd': | |
249 | title_str = 'Average Queue Depth' | |
250 | elif type == 'bnos': | |
251 | title_str = 'Block Numbers Accessed' | |
252 | elif type == 'q2d': | |
253 | title_str = 'Queue (Q) To Issue (D) Average Latencies' | |
254 | elif type == 'd2c': | |
255 | title_str = 'Issue (D) To Complete (C) Average Latencies' | |
256 | elif type == 'q2c': | |
257 | title_str = 'Queue (Q) To Complete (C) Average Latencies' | |
258 | ||
259 | title = fig.text(.5, .95, title_str, horizontalalignment='center') | |
260 | title.set_fontsize('large') | |
261 | ||
262 | #------------------------------------------------------------------------------ | |
263 | def gen_labels(db, ax, type): | |
264 | """Generate X & Y 'axis'""" | |
265 | ||
266 | #---------------------------------------------------------------------- | |
267 | def gen_ylabel(ax, type): | |
268 | """Set the Y axis label based upon the type""" | |
269 | ||
270 | if type == 'aqd': | |
271 | str = 'Number of Requests Queued' | |
272 | elif type == 'bnos': | |
273 | str = 'Block Number' | |
274 | else: | |
275 | str = 'Seconds' | |
276 | ax.set_ylabel(str) | |
277 | ||
278 | #---------------------------------------------------------------------- | |
279 | xdelta = 0.1 * (db['max_x'] - db['min_x']) | |
280 | ydelta = 0.1 * (db['max_y'] - db['min_y']) | |
281 | ||
282 | ax.set_xlim(db['min_x'] - xdelta, db['max_x'] + xdelta) | |
283 | ax.set_ylim(db['min_y'] - ydelta, db['max_y'] + ydelta) | |
284 | ax.set_xlabel('Runtime (seconds)') | |
285 | ax.grid(True) | |
286 | gen_ylabel(ax, type) | |
287 | ||
288 | #------------------------------------------------------------------------------ | |
289 | def generate_output(type, db): | |
290 | """Generate the output plot based upon the type and database""" | |
291 | ||
292 | #---------------------------------------------------------------------- | |
293 | def color(idx, style): | |
294 | """Returns a color/symbol type based upon the index passed.""" | |
295 | ||
296 | colors = [ 'b', 'g', 'r', 'c', 'm', 'y', 'k' ] | |
297 | l_styles = [ '-', ':', '--', '-.' ] | |
298 | m_styles = [ 'o', '+', '.', ',', 's', 'v', 'x', '<', '>' ] | |
299 | ||
300 | color = colors[idx % len(colors)] | |
301 | if style == 'line': | |
302 | style = l_styles[(idx / len(l_styles)) % len(l_styles)] | |
303 | elif style == 'marker': | |
304 | style = m_styles[(idx / len(m_styles)) % len(m_styles)] | |
305 | ||
306 | return '%s%s' % (color, style) | |
307 | ||
726d2aee AB |
308 | #---------------------------------------------------------------------- |
309 | global add_legend, output_file, title_str, verbose | |
310 | ||
311 | if output_file != None: | |
312 | ofile = output_file | |
313 | else: | |
314 | ofile = '%s.png' % type | |
315 | ||
316 | if verbose: | |
317 | print 'Generating plot into %s' % ofile | |
318 | ||
319 | fig = plt.figure(figsize=plot_size) | |
320 | ax = fig.add_subplot(111) | |
321 | ||
322 | gen_title(fig, type, title_str) | |
323 | gen_labels(db, ax, type) | |
324 | ||
325 | idx = 0 | |
326 | if add_legend: | |
327 | legends = [] | |
328 | else: | |
329 | legends = None | |
330 | ||
331 | keys = [] | |
332 | for file in db.iterkeys(): | |
333 | if not file in ['min_x', 'max_x', 'min_y', 'max_y']: | |
334 | keys.append(file) | |
335 | ||
336 | keys.sort() | |
337 | for file in keys: | |
338 | dat = db[file] | |
339 | if type == 'bnos': | |
340 | ax.plot(dat['x'], dat['y'], color(idx, 'marker'), | |
341 | markersize=1) | |
342 | elif dat['ax'] == None: | |
343 | continue # Don't add legend | |
344 | else: | |
345 | ax.plot(dat['ax'], dat['ay'], color(idx, 'line'), | |
346 | linewidth=1.0) | |
347 | if add_legend: | |
348 | legends.append(get_base(file)) | |
349 | idx += 1 | |
350 | ||
351 | if add_legend and len(legends) > 0: | |
352 | gen_legends(ax, legends) | |
353 | plt.savefig(ofile) | |
354 | ||
355 | #------------------------------------------------------------------------------ | |
356 | def get_files(type): | |
357 | """Returns the list of files for the -A option based upon type""" | |
358 | ||
359 | if type == 'bnos': | |
360 | files = [] | |
361 | for fn in glob.glob('*c.dat'): | |
362 | for t in [ 'q2q', 'd2d', 'q2c', 'd2c' ]: | |
363 | if fn.find(t) >= 0: | |
364 | break | |
365 | else: | |
366 | files.append(fn) | |
367 | else: | |
368 | files = glob.glob('*%s.dat' % type) | |
369 | return files | |
370 | ||
2e37a10e AB |
371 | #------------------------------------------------------------------------------ |
372 | def do_bnos(files): | |
373 | for file in files: | |
374 | base = get_base(file) | |
375 | title_str = 'Block Numbers Accessed: %s' % base | |
376 | output_file = 'bnos_%s.png' % base | |
377 | generate_output(t, get_data([file])) | |
378 | ||
379 | #------------------------------------------------------------------------------ | |
380 | def do_live(files): | |
381 | global plot_size | |
382 | ||
383 | #---------------------------------------------------------------------- | |
384 | def get_live_data(fn): | |
385 | xs = [] | |
386 | ys = [] | |
387 | for line in open(fn, 'r'): | |
388 | f = line.rstrip().split() | |
389 | if f[0] != '#' and len(f) == 2: | |
390 | xs.append(float(f[0])) | |
391 | ys.append(float(f[1])) | |
392 | return xs, ys | |
393 | ||
394 | #---------------------------------------------------------------------- | |
395 | def live_sort(a, b): | |
396 | if a[0] == 'sys' and b[0] == 'sys': | |
397 | return 0 | |
398 | elif a[0] == 'sys' or a[2][0] < b[2][0]: | |
399 | return -1 | |
400 | elif b[0] == 'sys' or a[2][0] > b[2][0]: | |
401 | return 1 | |
402 | else: | |
403 | return 0 | |
404 | ||
405 | #---------------------------------------------------------------------- | |
406 | def turn_off_ticks(ax): | |
407 | for tick in ax.xaxis.get_major_ticks(): | |
408 | tick.tick1On = tick.tick2On = False | |
409 | for tick in ax.yaxis.get_major_ticks(): | |
410 | tick.tick1On = tick.tick2On = False | |
411 | for tick in ax.xaxis.get_minor_ticks(): | |
412 | tick.tick1On = tick.tick2On = False | |
413 | for tick in ax.yaxis.get_minor_ticks(): | |
414 | tick.tick1On = tick.tick2On = False | |
415 | ||
416 | #---------------------------------------------------------------------- | |
417 | fig = plt.figure(figsize=plot_size) | |
418 | ax = fig.add_subplot(111) | |
419 | ||
420 | db = [] | |
421 | for fn in files: | |
422 | if not os.path.exists(fn): | |
423 | continue | |
424 | (xs, ys) = get_live_data(fn) | |
425 | db.append([fn[:fn.find('_live.dat')], xs, ys]) | |
426 | db.sort(live_sort) | |
427 | ||
428 | for rec in db: | |
429 | ax.plot(rec[1], rec[2]) | |
430 | ||
431 | gen_title(fig, 'live', 'Active I/O Per Device') | |
432 | ax.set_xlabel('Runtime (seconds)') | |
433 | ax.set_ylabel('Device') | |
434 | ax.grid(False) | |
435 | ||
436 | ax.set_xlim(-0.1, db[0][1][-1]+1) | |
437 | ax.set_yticks([idx for idx in range(0, len(db))]) | |
438 | ax.yaxis.set_ticklabels([rec[0] for rec in db]) | |
439 | turn_off_ticks(ax) | |
440 | ||
441 | plt.savefig('live.png') | |
442 | plt.savefig('live.eps') | |
443 | ||
726d2aee AB |
444 | #------------------------------------------------------------------------------ |
445 | if __name__ == '__main__': | |
446 | files = parse_args(sys.argv) | |
447 | ||
448 | if generate_all: | |
449 | output_file = title_str = type = None | |
450 | for t in types: | |
451 | files = get_files(t) | |
452 | if len(files) == 0: | |
453 | continue | |
2e37a10e AB |
454 | elif t == 'bnos': |
455 | do_bnos(files) | |
456 | elif t == 'live': | |
457 | do_live(files) | |
458 | else: | |
726d2aee AB |
459 | generate_output(t, get_data(files)) |
460 | continue | |
461 | ||
726d2aee AB |
462 | elif len(files) < 1: |
463 | fatal('Need data files to process') | |
464 | else: | |
465 | generate_output(type, get_data(files)) | |
466 | sys.exit(0) |