Commit | Line | Data |
---|---|---|
726d2aee AB |
1 | #! /usr/bin/env python |
2 | # | |
3 | # btt_plot.py: Generate matplotlib plots for BTT generate data files | |
4 | # | |
5 | # (C) Copyright 2009 Hewlett-Packard Development Company, L.P. | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU General Public License as published by | |
9 | # the Free Software Foundation; either version 2 of the License, or | |
10 | # (at your option) any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU General Public License for more details. | |
16 | # | |
17 | # You should have received a copy of the GNU General Public License | |
18 | # along with this program; if not, write to the Free Software | |
19 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
20 | # | |
21 | ||
22 | """ | |
23 | btt_plot.py: Generate matplotlib plots for BTT generated data files | |
24 | ||
25 | Files handled: | |
26 | AQD - Average Queue Depth Running average of queue depths | |
27 | ||
28 | BNOS - Block numbers accessed Markers for each block | |
29 | ||
30 | Q2D - Queue to Issue latencies Running averages | |
31 | D2C - Issue to Complete latencies Running averages | |
32 | Q2C - Queue to Complete latencies Running averages | |
33 | ||
34 | Usage: | |
35 | btt_plot_aqd.py equivalent to: btt_plot.py -t aqd <type>=aqd | |
36 | btt_plot_bnos.py equivalent to: btt_plot.py -t bnos <type>=bnos | |
37 | btt_plot_q2d.py equivalent to: btt_plot.py -t q2d <type>=q2d | |
38 | btt_plot_d2c.py equivalent to: btt_plot.py -t d2c <type>=d2c | |
39 | btt_plot_q2c.py equivalent to: btt_plot.py -t q2c <type>=q2c | |
40 | ||
41 | Arguments: | |
42 | [ -A | --generate-all ] Default: False | |
43 | [ -L | --no-legend ] Default: Legend table produced | |
44 | [ -o <file> | --output=<file> ] Default: <type>.png | |
45 | [ -T <string> | --title=<string> ] Default: Based upon <type> | |
46 | [ -v | --verbose ] Default: False | |
47 | <data-files...> | |
48 | ||
49 | The -A (--generate-all) argument is different: when this is specified, | |
50 | an attempt is made to generate default plots for all 5 types (aqd, bnos, | |
51 | q2d, d2c and q2c). It will find files with the appropriate suffix for | |
52 | each type ('aqd.dat' for example). If such files are found, a plot for | |
53 | that type will be made. The output file name will be the default for | |
54 | each type. The -L (--no-legend) option will be obeyed for all plots, | |
55 | but the -o (--output) and -T (--title) options will be ignored. | |
56 | """ | |
57 | ||
70d5ca2d ES |
58 | from __future__ import absolute_import |
59 | from __future__ import print_function | |
60 | import six | |
61 | from six.moves import range | |
726d2aee AB |
62 | __author__ = 'Alan D. Brunelle <alan.brunelle@hp.com>' |
63 | ||
64 | #------------------------------------------------------------------------------ | |
65 | ||
66 | import matplotlib | |
67 | matplotlib.use('Agg') | |
68 | import getopt, glob, os, sys | |
69 | import matplotlib.pyplot as plt | |
70 | ||
71 | plot_size = [10.9, 8.4] # inches... | |
72 | ||
73 | add_legend = True | |
74 | generate_all = False | |
75 | output_file = None | |
76 | title_str = None | |
77 | type = None | |
78 | verbose = False | |
79 | ||
2e37a10e | 80 | types = [ 'aqd', 'q2d', 'd2c', 'q2c', 'live', 'bnos' ] |
726d2aee AB |
81 | progs = [ 'btt_plot_%s.py' % t for t in types ] |
82 | ||
83 | get_base = lambda file: file[file.find('_')+1:file.rfind('_')] | |
84 | ||
85 | #------------------------------------------------------------------------------ | |
86 | def fatal(msg): | |
87 | """Generate fatal error message and exit""" | |
88 | ||
70d5ca2d | 89 | print('FATAL: %s' % msg, file=sys.stderr) |
726d2aee AB |
90 | sys.exit(1) |
91 | ||
2e37a10e AB |
92 | #------------------------------------------------------------------------------ |
93 | def gen_legends(ax, legends): | |
94 | leg = ax.legend(legends, 'best', shadow=True) | |
95 | frame = leg.get_frame() | |
96 | frame.set_facecolor('0.80') | |
97 | for t in leg.get_texts(): | |
98 | t.set_fontsize('xx-small') | |
99 | ||
726d2aee AB |
100 | #---------------------------------------------------------------------- |
101 | def get_data(files): | |
102 | """Retrieve data from files provided. | |
103 | ||
104 | Returns a database containing: | |
105 | 'min_x', 'max_x' - Minimum and maximum X values found | |
106 | 'min_y', 'max_y' - Minimum and maximum Y values found | |
107 | 'x', 'y' - X & Y value arrays | |
108 | 'ax', 'ay' - Running average over X & Y -- | |
109 | if > 10 values provided... | |
110 | """ | |
111 | #-------------------------------------------------------------- | |
112 | def check(mn, mx, v): | |
113 | """Returns new min, max, and float value for those passed in""" | |
114 | ||
115 | v = float(v) | |
116 | if mn == None or v < mn: mn = v | |
117 | if mx == None or v > mx: mx = v | |
118 | return mn, mx, v | |
119 | ||
120 | #-------------------------------------------------------------- | |
121 | def avg(xs, ys): | |
122 | """Computes running average for Xs and Ys""" | |
123 | ||
124 | #------------------------------------------------------ | |
125 | def _avg(vals): | |
126 | """Computes average for array of values passed""" | |
127 | ||
128 | total = 0.0 | |
129 | for val in vals: | |
130 | total += val | |
131 | return total / len(vals) | |
132 | ||
133 | #------------------------------------------------------ | |
134 | if len(xs) < 1000: | |
135 | return xs, ys | |
136 | ||
137 | axs = [xs[0]] | |
138 | ays = [ys[0]] | |
139 | _xs = [xs[0]] | |
140 | _ys = [ys[0]] | |
141 | ||
142 | x_range = (xs[-1] - xs[0]) / 100 | |
143 | for idx in range(1, len(ys)): | |
144 | if (xs[idx] - _xs[0]) > x_range: | |
145 | axs.append(_avg(_xs)) | |
146 | ays.append(_avg(_ys)) | |
147 | del _xs, _ys | |
148 | ||
149 | _xs = [xs[idx]] | |
150 | _ys = [ys[idx]] | |
151 | else: | |
152 | _xs.append(xs[idx]) | |
153 | _ys.append(ys[idx]) | |
154 | ||
155 | if len(_xs) > 1: | |
156 | axs.append(_avg(_xs)) | |
157 | ays.append(_avg(_ys)) | |
158 | ||
159 | return axs, ays | |
160 | ||
161 | #-------------------------------------------------------------- | |
162 | global verbose | |
163 | ||
164 | db = {} | |
165 | min_x = max_x = min_y = max_y = None | |
166 | for file in files: | |
167 | if not os.path.exists(file): | |
168 | fatal('%s not found' % file) | |
169 | elif verbose: | |
70d5ca2d | 170 | print('Processing %s' % file) |
726d2aee AB |
171 | |
172 | xs = [] | |
173 | ys = [] | |
174 | for line in open(file, 'r'): | |
175 | f = line.rstrip().split(None) | |
176 | if line.find('#') == 0 or len(f) < 2: | |
177 | continue | |
178 | (min_x, max_x, x) = check(min_x, max_x, f[0]) | |
179 | (min_y, max_y, y) = check(min_y, max_y, f[1]) | |
180 | xs.append(x) | |
181 | ys.append(y) | |
182 | ||
183 | db[file] = {'x':xs, 'y':ys} | |
184 | if len(xs) > 10: | |
185 | db[file]['ax'], db[file]['ay'] = avg(xs, ys) | |
186 | else: | |
187 | db[file]['ax'] = db[file]['ay'] = None | |
188 | ||
189 | db['min_x'] = min_x | |
190 | db['max_x'] = max_x | |
191 | db['min_y'] = min_y | |
192 | db['max_y'] = max_y | |
193 | return db | |
194 | ||
195 | #---------------------------------------------------------------------- | |
196 | def parse_args(args): | |
197 | """Parse command line arguments. | |
198 | ||
199 | Returns list of (data) files that need to be processed -- /unless/ | |
200 | the -A (--generate-all) option is passed, in which case superfluous | |
201 | data files are ignored... | |
202 | """ | |
203 | ||
204 | global add_legend, output_file, title_str, type, verbose | |
205 | global generate_all | |
206 | ||
207 | prog = args[0][args[0].rfind('/')+1:] | |
208 | if prog == 'btt_plot.py': | |
209 | pass | |
210 | elif not prog in progs: | |
211 | fatal('%s not a valid command name' % prog) | |
212 | else: | |
213 | type = prog[prog.rfind('_')+1:prog.rfind('.py')] | |
214 | ||
215 | s_opts = 'ALo:t:T:v' | |
216 | l_opts = [ 'generate-all', 'type', 'no-legend', 'output', 'title', | |
217 | 'verbose' ] | |
218 | ||
219 | try: | |
220 | (opts, args) = getopt.getopt(args[1:], s_opts, l_opts) | |
70d5ca2d ES |
221 | except getopt.error as msg: |
222 | print(msg, file=sys.stderr) | |
726d2aee AB |
223 | fatal(__doc__) |
224 | ||
225 | for (o, a) in opts: | |
226 | if o in ('-A', '--generate-all'): | |
227 | generate_all = True | |
228 | elif o in ('-L', '--no-legend'): | |
229 | add_legend = False | |
230 | elif o in ('-o', '--output'): | |
231 | output_file = a | |
232 | elif o in ('-t', '--type'): | |
233 | if not a in types: | |
234 | fatal('Type %s not supported' % a) | |
235 | type = a | |
236 | elif o in ('-T', '--title'): | |
237 | title_str = a | |
238 | elif o in ('-v', '--verbose'): | |
239 | verbose = True | |
240 | ||
241 | if type == None and not generate_all: | |
242 | fatal('Need type of data files to process - (-t <type>)') | |
243 | ||
244 | return args | |
245 | ||
246 | #------------------------------------------------------------------------------ | |
247 | def gen_title(fig, type, title_str): | |
248 | """Sets the title for the figure based upon the type /or/ user title""" | |
249 | ||
250 | if title_str != None: | |
251 | pass | |
252 | elif type == 'aqd': | |
253 | title_str = 'Average Queue Depth' | |
254 | elif type == 'bnos': | |
255 | title_str = 'Block Numbers Accessed' | |
256 | elif type == 'q2d': | |
257 | title_str = 'Queue (Q) To Issue (D) Average Latencies' | |
258 | elif type == 'd2c': | |
259 | title_str = 'Issue (D) To Complete (C) Average Latencies' | |
260 | elif type == 'q2c': | |
261 | title_str = 'Queue (Q) To Complete (C) Average Latencies' | |
262 | ||
263 | title = fig.text(.5, .95, title_str, horizontalalignment='center') | |
264 | title.set_fontsize('large') | |
265 | ||
266 | #------------------------------------------------------------------------------ | |
267 | def gen_labels(db, ax, type): | |
268 | """Generate X & Y 'axis'""" | |
269 | ||
270 | #---------------------------------------------------------------------- | |
271 | def gen_ylabel(ax, type): | |
272 | """Set the Y axis label based upon the type""" | |
273 | ||
274 | if type == 'aqd': | |
275 | str = 'Number of Requests Queued' | |
276 | elif type == 'bnos': | |
277 | str = 'Block Number' | |
278 | else: | |
279 | str = 'Seconds' | |
280 | ax.set_ylabel(str) | |
281 | ||
282 | #---------------------------------------------------------------------- | |
283 | xdelta = 0.1 * (db['max_x'] - db['min_x']) | |
284 | ydelta = 0.1 * (db['max_y'] - db['min_y']) | |
285 | ||
286 | ax.set_xlim(db['min_x'] - xdelta, db['max_x'] + xdelta) | |
287 | ax.set_ylim(db['min_y'] - ydelta, db['max_y'] + ydelta) | |
288 | ax.set_xlabel('Runtime (seconds)') | |
289 | ax.grid(True) | |
290 | gen_ylabel(ax, type) | |
291 | ||
292 | #------------------------------------------------------------------------------ | |
293 | def generate_output(type, db): | |
294 | """Generate the output plot based upon the type and database""" | |
295 | ||
296 | #---------------------------------------------------------------------- | |
297 | def color(idx, style): | |
298 | """Returns a color/symbol type based upon the index passed.""" | |
299 | ||
70d5ca2d | 300 | colors = [ 'b', 'g', 'r', 'c', 'm', 'y', 'k' ] |
726d2aee AB |
301 | l_styles = [ '-', ':', '--', '-.' ] |
302 | m_styles = [ 'o', '+', '.', ',', 's', 'v', 'x', '<', '>' ] | |
303 | ||
304 | color = colors[idx % len(colors)] | |
305 | if style == 'line': | |
70d5ca2d | 306 | style = l_styles[int((idx / len(l_styles)) % len(l_styles))] |
726d2aee | 307 | elif style == 'marker': |
70d5ca2d | 308 | style = m_styles[int((idx / len(m_styles)) % len(m_styles))] |
726d2aee AB |
309 | |
310 | return '%s%s' % (color, style) | |
311 | ||
726d2aee AB |
312 | #---------------------------------------------------------------------- |
313 | global add_legend, output_file, title_str, verbose | |
314 | ||
315 | if output_file != None: | |
316 | ofile = output_file | |
317 | else: | |
318 | ofile = '%s.png' % type | |
319 | ||
320 | if verbose: | |
70d5ca2d | 321 | print('Generating plot into %s' % ofile) |
726d2aee AB |
322 | |
323 | fig = plt.figure(figsize=plot_size) | |
324 | ax = fig.add_subplot(111) | |
325 | ||
326 | gen_title(fig, type, title_str) | |
327 | gen_labels(db, ax, type) | |
328 | ||
329 | idx = 0 | |
330 | if add_legend: | |
331 | legends = [] | |
332 | else: | |
333 | legends = None | |
334 | ||
335 | keys = [] | |
70d5ca2d | 336 | for file in six.iterkeys(db): |
726d2aee AB |
337 | if not file in ['min_x', 'max_x', 'min_y', 'max_y']: |
338 | keys.append(file) | |
339 | ||
340 | keys.sort() | |
341 | for file in keys: | |
342 | dat = db[file] | |
343 | if type == 'bnos': | |
344 | ax.plot(dat['x'], dat['y'], color(idx, 'marker'), | |
345 | markersize=1) | |
346 | elif dat['ax'] == None: | |
347 | continue # Don't add legend | |
348 | else: | |
349 | ax.plot(dat['ax'], dat['ay'], color(idx, 'line'), | |
350 | linewidth=1.0) | |
351 | if add_legend: | |
352 | legends.append(get_base(file)) | |
353 | idx += 1 | |
354 | ||
355 | if add_legend and len(legends) > 0: | |
356 | gen_legends(ax, legends) | |
357 | plt.savefig(ofile) | |
358 | ||
359 | #------------------------------------------------------------------------------ | |
360 | def get_files(type): | |
361 | """Returns the list of files for the -A option based upon type""" | |
362 | ||
363 | if type == 'bnos': | |
364 | files = [] | |
365 | for fn in glob.glob('*c.dat'): | |
366 | for t in [ 'q2q', 'd2d', 'q2c', 'd2c' ]: | |
367 | if fn.find(t) >= 0: | |
368 | break | |
369 | else: | |
370 | files.append(fn) | |
371 | else: | |
372 | files = glob.glob('*%s.dat' % type) | |
373 | return files | |
374 | ||
2e37a10e AB |
375 | #------------------------------------------------------------------------------ |
376 | def do_bnos(files): | |
377 | for file in files: | |
378 | base = get_base(file) | |
379 | title_str = 'Block Numbers Accessed: %s' % base | |
380 | output_file = 'bnos_%s.png' % base | |
381 | generate_output(t, get_data([file])) | |
382 | ||
383 | #------------------------------------------------------------------------------ | |
384 | def do_live(files): | |
385 | global plot_size | |
386 | ||
387 | #---------------------------------------------------------------------- | |
388 | def get_live_data(fn): | |
389 | xs = [] | |
390 | ys = [] | |
391 | for line in open(fn, 'r'): | |
392 | f = line.rstrip().split() | |
393 | if f[0] != '#' and len(f) == 2: | |
394 | xs.append(float(f[0])) | |
395 | ys.append(float(f[1])) | |
396 | return xs, ys | |
397 | ||
398 | #---------------------------------------------------------------------- | |
399 | def live_sort(a, b): | |
400 | if a[0] == 'sys' and b[0] == 'sys': | |
401 | return 0 | |
402 | elif a[0] == 'sys' or a[2][0] < b[2][0]: | |
403 | return -1 | |
404 | elif b[0] == 'sys' or a[2][0] > b[2][0]: | |
405 | return 1 | |
406 | else: | |
407 | return 0 | |
408 | ||
409 | #---------------------------------------------------------------------- | |
410 | def turn_off_ticks(ax): | |
411 | for tick in ax.xaxis.get_major_ticks(): | |
412 | tick.tick1On = tick.tick2On = False | |
413 | for tick in ax.yaxis.get_major_ticks(): | |
414 | tick.tick1On = tick.tick2On = False | |
415 | for tick in ax.xaxis.get_minor_ticks(): | |
416 | tick.tick1On = tick.tick2On = False | |
417 | for tick in ax.yaxis.get_minor_ticks(): | |
418 | tick.tick1On = tick.tick2On = False | |
419 | ||
420 | #---------------------------------------------------------------------- | |
421 | fig = plt.figure(figsize=plot_size) | |
422 | ax = fig.add_subplot(111) | |
423 | ||
424 | db = [] | |
425 | for fn in files: | |
426 | if not os.path.exists(fn): | |
427 | continue | |
428 | (xs, ys) = get_live_data(fn) | |
429 | db.append([fn[:fn.find('_live.dat')], xs, ys]) | |
430 | db.sort(live_sort) | |
431 | ||
432 | for rec in db: | |
433 | ax.plot(rec[1], rec[2]) | |
434 | ||
435 | gen_title(fig, 'live', 'Active I/O Per Device') | |
436 | ax.set_xlabel('Runtime (seconds)') | |
437 | ax.set_ylabel('Device') | |
438 | ax.grid(False) | |
439 | ||
440 | ax.set_xlim(-0.1, db[0][1][-1]+1) | |
441 | ax.set_yticks([idx for idx in range(0, len(db))]) | |
442 | ax.yaxis.set_ticklabels([rec[0] for rec in db]) | |
443 | turn_off_ticks(ax) | |
444 | ||
445 | plt.savefig('live.png') | |
446 | plt.savefig('live.eps') | |
447 | ||
726d2aee AB |
448 | #------------------------------------------------------------------------------ |
449 | if __name__ == '__main__': | |
450 | files = parse_args(sys.argv) | |
451 | ||
452 | if generate_all: | |
453 | output_file = title_str = type = None | |
454 | for t in types: | |
455 | files = get_files(t) | |
456 | if len(files) == 0: | |
457 | continue | |
2e37a10e AB |
458 | elif t == 'bnos': |
459 | do_bnos(files) | |
460 | elif t == 'live': | |
461 | do_live(files) | |
462 | else: | |
726d2aee AB |
463 | generate_output(t, get_data(files)) |
464 | continue | |
465 | ||
726d2aee AB |
466 | elif len(files) < 1: |
467 | fatal('Need data files to process') | |
468 | else: | |
469 | generate_output(type, get_data(files)) | |
470 | sys.exit(0) |