2 * Copyright (C) 2012 Fusion-io
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 * Parts of this file were imported from Jens Axboe's blktrace sources (also GPL)
19 #include <sys/types.h>
28 #include <asm/types.h>
41 LIST_HEAD(all_traces);
43 static int found_mpstat = 0;
44 static int cpu_color_index = 0;
45 static int color_index = 0;
49 "orange", "darkviolet",
56 char *pick_color(void) {
57 char *ret = colors[color_index];
60 ret = colors[color_index];
66 char *pick_cpu_color(void) {
67 char *ret = colors[cpu_color_index];
70 ret = colors[cpu_color_index];
85 QUEUE_DEPTH_GRAPH_INDEX,
99 static char *graphs_by_name[] = {
112 static int active_graphs[TOTAL_GRAPHS];
113 static int last_active_graph = IOPS_GRAPH_INDEX;
115 static int label_index = 0;
116 static int num_traces = 0;
117 static int longest_label = 0;
120 struct list_head list;
131 struct graph_line_data *tput_gld;
132 struct graph_line_data *iop_gld;
133 struct graph_line_data *latency_gld;
134 struct graph_line_data *queue_depth_gld;
135 struct graph_dot_data *gdd_writes;
136 struct graph_dot_data *gdd_reads;
139 int mpstat_stop_seconds;
140 struct graph_line_data **mpstat_gld;
143 static void alloc_mpstat_gld(struct trace_file *tf)
145 struct graph_line_data **ptr;
147 if (tf->trace->mpstat_num_cpus == 0)
150 ptr = calloc((tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS,
151 sizeof(struct graph_line_data *));
153 perror("Unable to allocate mpstat arrays\n");
156 tf->mpstat_gld = ptr;
159 static void enable_all_graphs(void)
162 for (i = 0; i < TOTAL_GRAPHS; i++)
163 active_graphs[i] = 1;
166 static void disable_all_graphs(void)
169 for (i = 0; i < TOTAL_GRAPHS; i++)
170 active_graphs[i] = 0;
173 static int enable_one_graph(char *name)
176 for (i = 0; i < TOTAL_GRAPHS; i++) {
177 if (strcmp(name, graphs_by_name[i]) == 0) {
178 active_graphs[i] = 1;
185 static int disable_one_graph(char *name)
188 for (i = 0; i < TOTAL_GRAPHS; i++) {
189 if (strcmp(name, graphs_by_name[i]) == 0) {
190 active_graphs[i] = 0;
197 static int last_graph(void)
200 for (i = TOTAL_GRAPHS - 1; i >= 0; i--) {
201 if (active_graphs[i]) {
207 static void add_trace_file(char *filename)
209 struct trace_file *tf;
211 tf = calloc(1, sizeof(*tf));
213 fprintf(stderr, "Unable to allocate memory\n");
217 tf->filename = strdup(filename);
218 list_add_tail(&tf->list, &all_traces);
219 tf->read_color = pick_color();
220 tf->write_color = pick_color();
224 static void setup_trace_file_graphs(void)
226 struct trace_file *tf;
229 list_for_each_entry(tf, &all_traces, list) {
230 tf->tput_gld = alloc_line_data(tf->seconds, tf->stop_seconds);
231 tf->latency_gld = alloc_line_data(tf->seconds, tf->stop_seconds);
232 tf->queue_depth_gld = alloc_line_data(tf->seconds, tf->stop_seconds);
233 tf->iop_gld = alloc_line_data(tf->seconds, tf->stop_seconds);
234 tf->gdd_writes = alloc_dot_data(tf->seconds, tf->max_offset, tf->stop_seconds);
235 tf->gdd_reads = alloc_dot_data(tf->seconds, tf->max_offset, tf->stop_seconds);
237 if (tf->trace->mpstat_num_cpus == 0)
240 alloc_mpstat_gld(tf);
241 for (i = 0; i < (tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i++) {
243 alloc_line_data(tf->mpstat_seconds,
245 tf->mpstat_gld[i]->max = 100;
250 static void read_traces(void)
252 struct trace_file *tf;
258 list_for_each_entry(tf, &all_traces, list) {
259 trace = open_trace(tf->filename);
263 last_time = find_last_time(trace);
265 tf->seconds = SECONDS(last_time);
266 tf->stop_seconds = SECONDS(last_time);
267 tf->max_offset = find_highest_offset(trace);
269 filter_outliers(trace, tf->max_offset, &ymin, &ymax);
270 tf->max_offset = ymax;
272 read_mpstat(trace, tf->filename);
273 tf->mpstat_stop_seconds = trace->mpstat_seconds;
274 tf->mpstat_seconds = trace->mpstat_seconds;
275 if (tf->mpstat_seconds)
280 static void read_trace_events(void)
283 struct trace_file *tf;
288 double user, sys, iowait, irq, soft;
289 double max_user = 0, max_sys = 0, max_iowait = 0,
290 max_irq = 0, max_soft = 0;
292 list_for_each_entry(tf, &all_traces, list) {
297 add_tput(trace, tf->tput_gld);
298 add_iop(trace, tf->iop_gld);
299 add_io(trace, tf->gdd_writes, tf->gdd_reads);
300 add_pending_io(trace, tf->queue_depth_gld);
301 add_completed_io(trace, tf->latency_gld);
302 ret = next_record(trace);
307 list_for_each_entry(tf, &all_traces, list) {
310 if (trace->mpstat_num_cpus == 0)
315 for (time = 0; time < tf->mpstat_stop_seconds; time ++) {
316 for (i = 0; i < (trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i += MPSTAT_GRAPHS) {
317 ret = read_mpstat_event(trace, &user, &sys,
318 &iowait, &irq, &soft);
321 if (next_mpstat_line(trace))
330 if (iowait > max_iowait)
333 add_mpstat_gld(time, sys, tf->mpstat_gld[i + MPSTAT_SYS]);
334 add_mpstat_gld(time, irq, tf->mpstat_gld[i + MPSTAT_IRQ]);
335 add_mpstat_gld(time, soft, tf->mpstat_gld[i + MPSTAT_SOFT]);
336 add_mpstat_gld(time, user, tf->mpstat_gld[i + MPSTAT_USER]);
337 add_mpstat_gld(time, iowait, tf->mpstat_gld[i + MPSTAT_IO]);
339 if (next_mpstat(trace) == NULL)
345 list_for_each_entry(tf, &all_traces, list) {
348 if (trace->mpstat_num_cpus == 0)
351 tf->mpstat_gld[MPSTAT_SYS]->max = max_sys;
352 tf->mpstat_gld[MPSTAT_IRQ]->max = max_irq;
353 tf->mpstat_gld[MPSTAT_SOFT]->max = max_soft;
354 tf->mpstat_gld[MPSTAT_USER]->max = max_user;
355 tf->mpstat_gld[MPSTAT_IO]->max = max_iowait;;
360 static void set_trace_label(char *label)
363 struct trace_file *tf;
364 int len = strlen(label);
366 if (len > longest_label)
369 list_for_each_entry(tf, &all_traces, list) {
370 if (cur == label_index) {
371 tf->label = strdup(label);
379 static char *graph_title = "";
380 static char *output_filename = "trace.svg";
381 static char *blktrace_device = NULL;
382 static char *blktrace_outfile = "trace";
383 static char *blktrace_dest_dir = ".";
384 static char *program_to_run = NULL;
386 static void set_blktrace_outfile(char *arg)
388 char *s = strdup(arg);
389 char *last_dot = strrchr(s, '.');
392 if (strcmp(last_dot, ".dump") == 0)
395 blktrace_outfile = s;
399 char *option_string = "hT:t:o:l:r:O:N:d:p:";
400 static struct option long_options[] = {
401 {"title", required_argument, 0, 'T'},
402 {"trace", required_argument, 0, 't'},
403 {"output", required_argument, 0, 'o'},
404 {"label", required_argument, 0, 'l'},
405 {"rolling", required_argument, 0, 'r'},
406 {"no-graph", required_argument, 0, 'N'},
407 {"only-graph", required_argument, 0, 'O'},
408 {"device", required_argument, 0, 'd'},
409 {"prog", required_argument, 0, 'p'},
410 {"help", required_argument, 0, 'h'},
414 static void print_usage(void)
416 fprintf(stderr, "iowatcher usage:\n"
417 "\t-d (--device): device for blktrace to trace\n"
418 "\t-t (--trace): trace file name (more than one allowed)\n"
419 "\t-l (--label): trace label in the graph\n"
420 "\t-o (--output): output file name (SVG only)\n"
421 "\t-p (--prog): program to run while blktrace is run\n"
422 "\t-r (--rolling): number of seconds in the rolling averge\n"
423 "\t-T (--title): graph title\n"
424 "\t-N (--no-graph): skip a single graph (io, tput, latency, queue_depth, iops)\n"
425 "\t-O (--only-graph): add a single graph (io, tput, latency, queue_depth, iops)\n"
430 static int parse_options(int ac, char **av)
436 // int this_option_optind = optind ? optind : 1;
437 int option_index = 0;
439 c = getopt_long(ac, av, option_string,
440 long_options, &option_index);
450 graph_title = strdup(optarg);
453 add_trace_file(optarg);
454 set_blktrace_outfile(optarg);
457 output_filename = strdup(optarg);
460 set_trace_label(optarg);
463 set_rolling_avg(atoi(optarg));
467 disable_all_graphs();
470 enable_one_graph(optarg);
473 disable_one_graph(optarg);
476 blktrace_device = strdup(optarg);
479 program_to_run = strdup(optarg);
491 static void compare_max_tf(struct trace_file *tf, int *seconds, u64 *max_offset)
493 if (tf->seconds > *seconds)
494 *seconds = tf->seconds;
495 if (tf->max_offset > *max_offset)
496 *max_offset = tf->max_offset;
499 static void set_all_max_tf(int seconds, u64 max_offset)
501 struct trace_file *tf;
503 list_for_each_entry(tf, &all_traces, list) {
504 tf->seconds = seconds;
505 tf->max_offset = max_offset;
509 static void plot_io(struct plot *plot, int seconds, u64 max_offset)
511 struct trace_file *tf;
513 if (active_graphs[IO_GRAPH_INDEX] == 0)
516 plot->add_xlabel = last_active_graph == IO_GRAPH_INDEX;
519 svg_alloc_legend(plot, num_traces * 2);
521 set_plot_label(plot, "Device IO");
522 set_ylabel(plot, "Offset (MB)");
523 set_yticks(plot, 4, 0, max_offset / (1024 * 1024), "");
524 set_xticks(plot, 9, 0, seconds);
526 list_for_each_entry(tf, &all_traces, list) {
527 char *label = tf->label;
531 svg_io_graph(plot, tf->gdd_reads, tf->read_color);
532 if (tf->gdd_reads->total_ios)
533 svg_add_legend(plot, label, " Reads", tf->read_color);
535 svg_io_graph(plot, tf->gdd_writes, tf->write_color);
536 if (tf->gdd_writes->total_ios) {
537 svg_add_legend(plot, label, " Writes", tf->write_color);
540 if (plot->add_xlabel)
541 set_xlabel(plot, "Time (seconds)");
542 svg_write_legend(plot);
546 static void plot_tput(struct plot *plot, int seconds)
548 struct trace_file *tf;
553 if (active_graphs[TPUT_GRAPH_INDEX] == 0)
557 svg_alloc_legend(plot, num_traces);
558 list_for_each_entry(tf, &all_traces, list) {
559 if (tf->tput_gld->max > max)
560 max = tf->tput_gld->max;
562 list_for_each_entry(tf, &all_traces, list)
563 tf->tput_gld->max = max;
565 plot->add_xlabel = last_active_graph == TPUT_GRAPH_INDEX;
567 set_plot_label(plot, "Throughput");
569 tf = list_entry(all_traces.next, struct trace_file, list);
571 scale_line_graph_bytes(&max, &units, 1024);
572 sprintf(line, "%sB/s", units);
573 set_ylabel(plot, line);
574 set_yticks(plot, 4, 0, max, "");
575 set_xticks(plot, 9, 0, seconds);
577 list_for_each_entry(tf, &all_traces, list) {
578 svg_line_graph(plot, tf->tput_gld, tf->read_color);
580 svg_add_legend(plot, tf->label, "", tf->read_color);
583 if (plot->add_xlabel)
584 set_xlabel(plot, "Time (seconds)");
586 svg_write_legend(plot);
590 static void plot_cpu(struct plot *plot, int seconds, char *label,
591 int active_index, int gld_index)
593 struct trace_file *tf;
603 if (active_graphs[active_index] == 0)
606 list_for_each_entry(tf, &all_traces, list) {
607 if (tf->trace->mpstat_num_cpus > max)
608 max = tf->trace->mpstat_num_cpus;
613 tf = list_entry(all_traces.next, struct trace_file, list);
615 ymax = tf->mpstat_gld[gld_index]->max;
619 svg_alloc_legend(plot, num_traces * max);
621 plot->add_xlabel = last_active_graph == active_index;
623 set_plot_label(plot, label);
625 seconds = tf->mpstat_seconds;
627 set_yticks(plot, 4, 0, tf->mpstat_gld[gld_index]->max, "");
628 set_ylabel(plot, "Percent");
629 set_xticks(plot, 9, 0, seconds);
632 list_for_each_entry(tf, &all_traces, list) {
633 for (i = 0; i < tf->mpstat_gld[0]->stop_seconds; i++) {
634 avg += tf->mpstat_gld[gld_index]->data[i].sum;
636 avg /= tf->mpstat_gld[gld_index]->stop_seconds;
637 color = pick_cpu_color();
638 svg_line_graph(plot, tf->mpstat_gld[0], color);
639 svg_add_legend(plot, tf->label, " avg", color);
641 for (i = 1; i < tf->trace->mpstat_num_cpus + 1; i++) {
642 struct graph_line_data *gld = tf->mpstat_gld[i * MPSTAT_GRAPHS + gld_index];
645 for (gld_i = 0; gld_i < gld->stop_seconds; gld_i++)
646 this_avg += gld->data[i].sum;
648 this_avg /= gld->stop_seconds;
650 for (gld_i = 0; gld_i < gld->stop_seconds; gld_i++) {
651 if (this_avg > avg + 30 ||
652 gld->data[gld_i].sum > 95) {
653 color = pick_cpu_color();
654 svg_line_graph(plot, gld, color);
655 snprintf(line, 128, " CPU %d\n", i - 1);
656 svg_add_legend(plot, tf->label, line, color);
665 if (plot->add_xlabel)
666 set_xlabel(plot, "Time (seconds)");
668 if (plot->legend_index <= 8)
669 svg_write_legend(plot);
671 svg_free_legend(plot);
675 static void plot_latency(struct plot *plot, int seconds)
677 struct trace_file *tf;
682 if (active_graphs[LATENCY_GRAPH_INDEX] == 0)
686 svg_alloc_legend(plot, num_traces);
687 list_for_each_entry(tf, &all_traces, list) {
688 if (tf->latency_gld->max > max)
689 max = tf->latency_gld->max;
691 list_for_each_entry(tf, &all_traces, list)
692 tf->latency_gld->max = max;
694 plot->add_xlabel = last_active_graph == TPUT_GRAPH_INDEX;
696 set_plot_label(plot, "IO Latency");
698 tf = list_entry(all_traces.next, struct trace_file, list);
700 scale_line_graph_time(&max, &units);
701 sprintf(line, "latency (%ss)", units);
702 set_ylabel(plot, line);
703 set_yticks(plot, 4, 0, max, "");
704 set_xticks(plot, 9, 0, seconds);
706 list_for_each_entry(tf, &all_traces, list) {
707 svg_line_graph(plot, tf->latency_gld, tf->read_color);
709 svg_add_legend(plot, tf->label, "", tf->read_color);
712 if (plot->add_xlabel)
713 set_xlabel(plot, "Time (seconds)");
715 svg_write_legend(plot);
719 static void plot_queue_depth(struct plot *plot, int seconds)
721 struct trace_file *tf;
723 if (active_graphs[QUEUE_DEPTH_GRAPH_INDEX] == 0)
726 plot->add_xlabel = last_active_graph == QUEUE_DEPTH_GRAPH_INDEX;
729 set_plot_label(plot, "Queue Depth");
731 svg_alloc_legend(plot, num_traces);
733 tf = list_entry(all_traces.next, struct trace_file, list);
734 set_ylabel(plot, "Pending IO");
735 set_yticks(plot, 4, 0, tf->queue_depth_gld->max, "");
736 set_xticks(plot, 9, 0, seconds);
738 list_for_each_entry(tf, &all_traces, list) {
739 svg_line_graph(plot, tf->queue_depth_gld, tf->read_color);
741 svg_add_legend(plot, tf->label, "", tf->read_color);
744 if (plot->add_xlabel)
745 set_xlabel(plot, "Time (seconds)");
747 svg_write_legend(plot);
751 static void plot_iops(struct plot *plot, int seconds)
753 struct trace_file *tf;
757 if (active_graphs[IOPS_GRAPH_INDEX] == 0)
760 list_for_each_entry(tf, &all_traces, list) {
761 if (tf->iop_gld->max > max)
762 max = tf->iop_gld->max;
765 list_for_each_entry(tf, &all_traces, list)
766 tf->iop_gld->max = max;
769 plot->add_xlabel = last_active_graph == IOPS_GRAPH_INDEX;
771 set_plot_label(plot, "IOPs");
773 svg_alloc_legend(plot, num_traces);
775 tf = list_entry(all_traces.next, struct trace_file, list);
777 scale_line_graph_bytes(&max, &units, 1000);
778 set_ylabel(plot, "IO/s");
780 set_yticks(plot, 4, 0, max, units);
781 set_xticks(plot, 9, 0, seconds);
783 list_for_each_entry(tf, &all_traces, list) {
784 svg_line_graph(plot, tf->iop_gld, tf->read_color);
786 svg_add_legend(plot, tf->label, "", tf->read_color);
789 if (plot->add_xlabel)
790 set_xlabel(plot, "Time (seconds)");
792 svg_write_legend(plot);
797 int main(int ac, char **av)
803 struct trace_file *tf;
806 init_io_hash_table();
810 parse_options(ac, av);
812 last_active_graph = last_graph();
814 if (list_empty(&all_traces)) {
815 fprintf(stderr, "No traces found, exiting\n");
819 if (blktrace_device) {
820 ret = start_blktrace(blktrace_device, blktrace_outfile,
823 fprintf(stderr, "exiting due to blktrace failure\n");
826 start_mpstat(blktrace_outfile);
827 if (program_to_run) {
828 ret = run_program(program_to_run);
830 fprintf(stderr, "failed to run %s\n",
835 blktrace_to_dump(blktrace_outfile);
837 /* no program specified, just wait for
844 /* step one, read all the traces */
847 /* step two, find the maxes for time and offset */
848 list_for_each_entry(tf, &all_traces, list)
849 compare_max_tf(tf, &seconds, &max_offset);
851 /* push the max we found into all the tfs */
852 set_all_max_tf(seconds, max_offset);
854 /* alloc graphing structs for all the traces */
855 setup_trace_file_graphs();
857 /* run through all the traces and read their events */
860 fd = open(output_filename, O_CREAT | O_TRUNC | O_WRONLY, 0600);
862 fprintf(stderr, "Unable to open output file %s %s\n",
863 output_filename, strerror(errno));
867 write_svg_header(fd);
868 plot = alloc_plot(fd);
870 if (active_graphs[IO_GRAPH_INDEX] || found_mpstat)
871 set_legend_width(longest_label + strlen("writes"));
872 else if (num_traces > 1)
873 set_legend_width(longest_label);
877 set_plot_title(plot, graph_title);
879 plot_io(plot, seconds, max_offset);
880 plot_tput(plot, seconds);
881 plot_cpu(plot, seconds, "CPU IO Wait Time",
882 CPU_IO_GRAPH_INDEX, MPSTAT_IO);
883 plot_cpu(plot, seconds, "CPU System Time",
884 CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
885 plot_cpu(plot, seconds, "CPU IRQ Time",
886 CPU_IRQ_GRAPH_INDEX, MPSTAT_IRQ);
887 plot_cpu(plot, seconds, "CPU SoftIRQ Time",
888 CPU_SOFT_GRAPH_INDEX, MPSTAT_SOFT);
889 plot_cpu(plot, seconds, "CPU User Time",
890 CPU_USER_GRAPH_INDEX, MPSTAT_USER);
892 plot_latency(plot, seconds);
893 plot_queue_depth(plot, seconds);
894 plot_iops(plot, seconds);