2 * Copyright (C) 2012 Fusion-io
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 * Parts of this file were imported from Jens Axboe's blktrace sources (also GPL)
19 #include <sys/types.h>
28 #include <asm/types.h>
43 LIST_HEAD(all_traces);
45 static char line[1024];
46 static int line_len = 1024;
47 static int found_mpstat = 0;
48 static int make_movie = 0;
49 static int keep_movie_svgs = 0;
50 static int opt_graph_width = 0;
51 static int opt_graph_height = 0;
53 static int columns = 1;
54 static int num_xticks = 7;
55 static int num_yticks = 4;
57 static double min_time = 0;
58 static double max_time = DBL_MAX;
59 static unsigned long long min_mb = 0;
60 static unsigned long long max_mb = ULLONG_MAX >> 20;
62 int plot_io_action = 0;
63 int io_per_process = 0;
64 unsigned int longest_proc_name = 0;
67 * this doesn't include the IO graph,
68 * but it counts the other graphs as they go out
70 static int total_graphs_written = 1;
81 QUEUE_DEPTH_GRAPH_INDEX,
95 static char *graphs_by_name[] = {
114 char *movie_styles[] = {
120 static int movie_style = 0;
122 static int lookup_movie_style(char *str)
126 for (i = 0; i < NUM_MOVIE_STYLES; i++) {
127 if (strcmp(str, movie_styles[i]) == 0)
133 static int active_graphs[TOTAL_GRAPHS];
134 static int last_active_graph = IOPS_GRAPH_INDEX;
136 static int label_index = 0;
137 static int num_traces = 0;
138 static int longest_label = 0;
140 static char *graph_title = "";
141 static char *output_filename = "trace.svg";
142 static char *blktrace_devices[MAX_DEVICES_PER_TRACE];
143 static int num_blktrace_devices = 0;
144 static char *blktrace_outfile = "trace";
145 static char *blktrace_dest_dir = ".";
146 static char *program_to_run = NULL;
147 static char *ffmpeg_codec = "libx264";
149 static void alloc_mpstat_gld(struct trace_file *tf)
151 struct graph_line_data **ptr;
153 if (tf->trace->mpstat_num_cpus == 0)
156 ptr = calloc((tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS,
157 sizeof(struct graph_line_data *));
159 perror("Unable to allocate mpstat arrays\n");
162 tf->mpstat_gld = ptr;
165 static void enable_all_graphs(void)
168 for (i = 0; i < TOTAL_GRAPHS; i++)
169 active_graphs[i] = 1;
172 static void disable_all_graphs(void)
175 for (i = 0; i < TOTAL_GRAPHS; i++)
176 active_graphs[i] = 0;
179 static int enable_one_graph(char *name)
182 for (i = 0; i < TOTAL_GRAPHS; i++) {
183 if (strcmp(name, graphs_by_name[i]) == 0) {
184 active_graphs[i] = 1;
191 static int disable_one_graph(char *name)
194 for (i = 0; i < TOTAL_GRAPHS; i++) {
195 if (strcmp(name, graphs_by_name[i]) == 0) {
196 active_graphs[i] = 0;
203 static int last_graph(void)
206 for (i = TOTAL_GRAPHS - 1; i >= 0; i--) {
207 if (active_graphs[i]) {
214 static int graphs_left(int cur)
218 for (i = cur; i < TOTAL_GRAPHS; i++) {
219 if (active_graphs[i])
225 static char * join_path(char *dest_dir, char *filename)
227 /* alloc 2 extra bytes for '/' and '\0' */
228 char *path = malloc(strlen(dest_dir) + strlen(filename) + 2);
229 sprintf(path, "%s%s%s", dest_dir, "/", filename);
234 static void add_trace_file(char *filename)
236 struct trace_file *tf;
238 tf = calloc(1, sizeof(*tf));
240 fprintf(stderr, "Unable to allocate memory\n");
244 tf->filename = strdup(filename);
245 list_add_tail(&tf->list, &all_traces);
246 tf->line_color = "black";
250 static void setup_trace_file_graphs(void)
252 struct trace_file *tf;
260 list_for_each_entry(tf, &all_traces, list) {
261 tf->tput_reads_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
262 tf->tput_writes_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
263 tf->latency_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
264 tf->queue_depth_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
265 tf->iop_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
266 tf->gdd_writes = calloc(alloc_ptrs, sizeof(struct graph_dot_data));
267 tf->gdd_reads = calloc(alloc_ptrs, sizeof(struct graph_dot_data));
268 tf->io_plots_allocated = alloc_ptrs;
270 if (tf->trace->mpstat_num_cpus == 0)
273 alloc_mpstat_gld(tf);
274 for (i = 0; i < (tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i++) {
276 alloc_line_data(tf->mpstat_min_seconds,
277 tf->mpstat_max_seconds,
278 tf->mpstat_max_seconds);
279 tf->mpstat_gld[i]->max = 100;
284 static void read_traces(void)
286 struct trace_file *tf;
295 list_for_each_entry(tf, &all_traces, list) {
296 path = join_path(blktrace_dest_dir, tf->filename);
297 trace = open_trace(path);
301 last_time = find_last_time(trace);
303 tf->max_seconds = SECONDS(last_time) + 1;
304 tf->stop_seconds = SECONDS(last_time) + 1;
306 find_extreme_offsets(trace, &tf->min_offset, &tf->max_offset,
307 &max_bank, &max_bank_offset);
308 filter_outliers(trace, tf->min_offset, tf->max_offset, &ymin, &ymax);
309 tf->min_offset = ymin;
310 tf->max_offset = ymax;
312 read_mpstat(trace, path);
313 tf->mpstat_stop_seconds = trace->mpstat_seconds;
314 tf->mpstat_max_seconds = trace->mpstat_seconds;
315 if (tf->mpstat_max_seconds)
321 static void pick_line_graph_color(void)
323 struct trace_file *tf;
326 list_for_each_entry(tf, &all_traces, list) {
327 for (i = 0; i < tf->io_plots; i++) {
328 if (tf->gdd_reads[i]) {
329 tf->line_color = tf->gdd_reads[i]->color;
330 tf->reads_color = tf->gdd_reads[i]->color;
332 if (tf->gdd_writes[i]) {
333 tf->line_color = tf->gdd_writes[i]->color;
334 tf->writes_color = tf->gdd_writes[i]->color;
336 if (tf->writes_color && tf->reads_color)
339 if (!tf->reads_color)
340 tf->reads_color = tf->line_color;
341 if (!tf->writes_color)
342 tf->writes_color = tf->line_color;
346 static void read_trace_events(void)
349 struct trace_file *tf;
354 double user, sys, iowait, irq, soft;
355 double max_user = 0, max_sys = 0, max_iowait = 0,
356 max_irq = 0, max_soft = 0;
358 list_for_each_entry(tf, &all_traces, list) {
363 add_tput(trace, tf->tput_writes_gld, tf->tput_reads_gld);
364 add_iop(trace, tf->iop_gld);
366 add_pending_io(trace, tf->queue_depth_gld);
367 add_completed_io(trace, tf->latency_gld);
368 ret = next_record(trace);
373 list_for_each_entry(tf, &all_traces, list) {
376 if (trace->mpstat_num_cpus == 0)
381 for (time = 0; time < tf->mpstat_stop_seconds; time ++) {
382 for (i = 0; i < (trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i += MPSTAT_GRAPHS) {
383 ret = read_mpstat_event(trace, &user, &sys,
384 &iowait, &irq, &soft);
387 if (next_mpstat_line(trace))
396 if (iowait > max_iowait)
399 add_mpstat_gld(time, sys, tf->mpstat_gld[i + MPSTAT_SYS]);
400 add_mpstat_gld(time, irq, tf->mpstat_gld[i + MPSTAT_IRQ]);
401 add_mpstat_gld(time, soft, tf->mpstat_gld[i + MPSTAT_SOFT]);
402 add_mpstat_gld(time, user, tf->mpstat_gld[i + MPSTAT_USER]);
403 add_mpstat_gld(time, iowait, tf->mpstat_gld[i + MPSTAT_IO]);
405 if (next_mpstat(trace) == NULL)
411 list_for_each_entry(tf, &all_traces, list) {
414 if (trace->mpstat_num_cpus == 0)
417 tf->mpstat_gld[MPSTAT_SYS]->max = max_sys;
418 tf->mpstat_gld[MPSTAT_IRQ]->max = max_irq;
419 tf->mpstat_gld[MPSTAT_SOFT]->max = max_soft;
420 tf->mpstat_gld[MPSTAT_USER]->max = max_user;
421 tf->mpstat_gld[MPSTAT_IO]->max = max_iowait;;
426 static void set_trace_label(char *label)
429 struct trace_file *tf;
430 int len = strlen(label);
432 if (len > longest_label)
435 list_for_each_entry(tf, &all_traces, list) {
436 if (cur == label_index) {
437 tf->label = strdup(label);
445 static void set_blktrace_outfile(char *arg)
447 char *s = strdup(arg);
448 char *last_dot = strrchr(s, '.');
451 if (strcmp(last_dot, ".dump") == 0)
454 blktrace_outfile = s;
458 static void compare_minmax_tf(struct trace_file *tf, int *max_seconds, u64 *min_offset, u64 *max_offset)
460 if (tf->max_seconds > *max_seconds)
461 *max_seconds = tf->max_seconds;
462 if (tf->max_offset > *max_offset)
463 *max_offset = tf->max_offset;
464 if (tf->min_offset < *min_offset)
465 *min_offset = tf->min_offset;
468 static void set_all_minmax_tf(int min_seconds, int max_seconds, u64 min_offset, u64 max_offset)
470 struct trace_file *tf;
472 list_for_each_entry(tf, &all_traces, list) {
473 tf->min_seconds = min_seconds;
474 tf->max_seconds = max_seconds;
475 if (tf->stop_seconds > max_seconds)
476 tf->stop_seconds = max_seconds;
477 if (tf->mpstat_max_seconds) {
478 tf->mpstat_min_seconds = min_seconds;
479 tf->mpstat_max_seconds = max_seconds;
480 if (tf->mpstat_stop_seconds > max_seconds)
481 tf->mpstat_stop_seconds = max_seconds;
483 tf->min_offset = min_offset;
484 tf->max_offset = max_offset;
488 static char *create_movie_temp_dir(void)
491 char *pattern = strdup("io-movie-XXXXXX");;
493 ret = mkdtemp(pattern);
495 perror("Unable to create temp directory for movie files");
501 static struct pid_plot_history *alloc_pid_plot_history(char *color)
503 struct pid_plot_history *pph;
505 pph = calloc(1, sizeof(struct pid_plot_history));
507 perror("memory allocation failed");
510 pph->history = malloc(sizeof(double) * 4096);
512 perror("memory allocation failed");
515 pph->history_len = 4096;
521 static struct plot_history *alloc_plot_history(struct trace_file *tf)
523 struct plot_history *ph = calloc(1, sizeof(struct plot_history));
527 perror("memory allocation failed");
530 ph->read_pid_history = calloc(tf->io_plots, sizeof(struct pid_plot_history *));
531 if (!ph->read_pid_history) {
532 perror("memory allocation failed");
535 ph->write_pid_history = calloc(tf->io_plots, sizeof(struct pid_plot_history *));
536 if (!ph->write_pid_history) {
537 perror("memory allocation failed");
540 ph->pid_history_count = tf->io_plots;
541 for (i = 0; i < tf->io_plots; i++) {
542 if (tf->gdd_reads[i])
543 ph->read_pid_history[i] = alloc_pid_plot_history(tf->gdd_reads[i]->color);
544 if (tf->gdd_writes[i])
545 ph->write_pid_history[i] = alloc_pid_plot_history(tf->gdd_writes[i]->color);
550 LIST_HEAD(movie_history);
551 int num_histories = 0;
553 static void free_plot_history(struct plot_history *ph)
557 for (pid = 0; pid < ph->pid_history_count; pid++) {
558 if (ph->read_pid_history[pid])
559 free(ph->read_pid_history[pid]);
560 if (ph->write_pid_history[pid])
561 free(ph->write_pid_history[pid]);
563 free(ph->read_pid_history);
564 free(ph->write_pid_history);
568 static void add_history(struct plot_history *ph, struct list_head *list)
570 struct plot_history *entry;
572 list_add_tail(&ph->list, list);
575 if (num_histories > 12) {
577 entry = list_entry(list->next, struct plot_history, list);
578 list_del(&entry->list);
579 free_plot_history(entry);
583 static void plot_movie_history(struct plot *plot, struct list_head *list)
585 struct plot_history *ph;
588 if (num_histories > 2)
589 rewind_spindle_steps(num_histories - 1);
591 list_for_each_entry(ph, list, list) {
592 for (pid = 0; pid < ph->pid_history_count; pid++) {
593 if (ph->read_pid_history[pid]) {
594 if (movie_style == MOVIE_SPINDLE) {
595 svg_io_graph_movie_array_spindle(plot,
596 ph->read_pid_history[pid]);
598 svg_io_graph_movie_array(plot,
599 ph->read_pid_history[pid]);
602 if (ph->write_pid_history[pid]) {
603 if (movie_style == MOVIE_SPINDLE) {
604 svg_io_graph_movie_array_spindle(plot,
605 ph->write_pid_history[pid]);
607 svg_io_graph_movie_array(plot,
608 ph->write_pid_history[pid]);
615 static void free_all_plot_history(struct list_head *head)
617 struct plot_history *entry;
619 while (!list_empty(head)) {
620 entry = list_entry(head->next, struct plot_history, list);
621 list_del(&entry->list);
622 free_plot_history(entry);
626 static int count_io_plot_types(void)
628 struct trace_file *tf;
630 int total_io_types = 0;
632 list_for_each_entry(tf, &all_traces, list) {
633 for (i = 0; i < tf->io_plots; i++) {
634 if (tf->gdd_reads[i])
636 if (tf->gdd_writes[i])
640 return total_io_types;
643 static void plot_io(struct plot *plot, int min_seconds, int max_seconds, u64 min_offset, u64 max_offset)
645 struct trace_file *tf;
648 if (active_graphs[IO_GRAPH_INDEX] == 0)
653 svg_alloc_legend(plot, count_io_plot_types() * 2);
655 set_plot_label(plot, "Device IO");
656 set_ylabel(plot, "Offset (MB)");
657 set_yticks(plot, num_yticks, min_offset / (1024 * 1024),
658 max_offset / (1024 * 1024), "");
659 set_xticks(plot, num_xticks, min_seconds, max_seconds);
661 list_for_each_entry(tf, &all_traces, list) {
668 strcpy(label, tf->label);
672 pos = label + strlen(label);
674 for (i = 0; i < tf->io_plots; i++) {
675 if (tf->gdd_writes[i]) {
676 svg_io_graph(plot, tf->gdd_writes[i]);
678 strcpy(pos, tf->gdd_writes[i]->label);
679 svg_add_legend(plot, label, " Writes", tf->gdd_writes[i]->color);
681 if (tf->gdd_reads[i]) {
682 svg_io_graph(plot, tf->gdd_reads[i]);
684 strcpy(pos, tf->gdd_reads[i]->label);
685 svg_add_legend(plot, label, " Reads", tf->gdd_reads[i]->color);
690 if (plot->add_xlabel)
691 set_xlabel(plot, "Time (seconds)");
692 svg_write_legend(plot);
696 static void plot_tput(struct plot *plot, int min_seconds, int max_seconds)
698 struct trace_file *tf;
703 if (active_graphs[TPUT_GRAPH_INDEX] == 0)
706 svg_alloc_legend(plot, num_traces * 2);
708 list_for_each_entry(tf, &all_traces, list) {
709 if (tf->tput_writes_gld->max > max)
710 max = tf->tput_writes_gld->max;
711 if (tf->tput_reads_gld->max > max)
712 max = tf->tput_reads_gld->max;
714 list_for_each_entry(tf, &all_traces, list) {
715 if (tf->tput_writes_gld->max > 0)
716 tf->tput_writes_gld->max = max;
717 if (tf->tput_reads_gld->max > 0)
718 tf->tput_reads_gld->max = max;
722 set_plot_label(plot, "Throughput");
724 tf = list_entry(all_traces.next, struct trace_file, list);
726 scale_line_graph_bytes(&max, &units, 1024);
727 sprintf(line, "%sB/s", units);
728 set_ylabel(plot, line);
729 set_yticks(plot, num_yticks, 0, max, "");
730 set_xticks(plot, num_xticks, min_seconds, max_seconds);
732 list_for_each_entry(tf, &all_traces, list) {
733 if (tf->tput_writes_gld->max > 0) {
734 svg_line_graph(plot, tf->tput_writes_gld, tf->writes_color, 0, 0);
735 svg_add_legend(plot, tf->label, "Writes", tf->writes_color);
737 if (tf->tput_reads_gld->max > 0) {
738 svg_line_graph(plot, tf->tput_reads_gld, tf->reads_color, 0, 0);
739 svg_add_legend(plot, tf->label, "Reads", tf->reads_color);
743 if (plot->add_xlabel)
744 set_xlabel(plot, "Time (seconds)");
746 svg_write_legend(plot);
748 total_graphs_written++;
751 static void plot_cpu(struct plot *plot, int max_seconds, char *label,
752 int active_index, int gld_index)
754 struct trace_file *tf;
763 if (active_graphs[active_index] == 0)
766 list_for_each_entry(tf, &all_traces, list) {
767 if (tf->trace->mpstat_num_cpus > max)
768 max = tf->trace->mpstat_num_cpus;
773 tf = list_entry(all_traces.next, struct trace_file, list);
775 ymax = tf->mpstat_gld[gld_index]->max;
779 svg_alloc_legend(plot, num_traces * max);
782 set_plot_label(plot, label);
784 max_seconds = tf->mpstat_max_seconds;
786 set_yticks(plot, num_yticks, 0, tf->mpstat_gld[gld_index]->max, "");
787 set_ylabel(plot, "Percent");
788 set_xticks(plot, num_xticks, tf->mpstat_min_seconds, max_seconds);
791 list_for_each_entry(tf, &all_traces, list) {
792 if (tf->mpstat_gld == 0)
794 for (i = tf->mpstat_gld[0]->min_seconds;
795 i < tf->mpstat_gld[0]->stop_seconds; i++) {
796 if (tf->mpstat_gld[gld_index]->data[i].count) {
797 avg += (tf->mpstat_gld[gld_index]->data[i].sum /
798 tf->mpstat_gld[gld_index]->data[i].count);
801 avg /= tf->mpstat_gld[gld_index]->stop_seconds -
802 tf->mpstat_gld[gld_index]->min_seconds;
803 color = pick_cpu_color();
804 svg_line_graph(plot, tf->mpstat_gld[0], color, 0, 0);
805 svg_add_legend(plot, tf->label, " avg", color);
807 for (i = 1; i < tf->trace->mpstat_num_cpus + 1; i++) {
808 struct graph_line_data *gld = tf->mpstat_gld[i * MPSTAT_GRAPHS + gld_index];
811 for (gld_i = gld->min_seconds;
812 gld_i < gld->stop_seconds; gld_i++) {
813 if (gld->data[i].count) {
814 this_avg += gld->data[i].sum /
819 this_avg /= gld->stop_seconds - gld->min_seconds;
821 for (gld_i = gld->min_seconds;
822 gld_i < gld->stop_seconds; gld_i++) {
825 if (gld->data[gld_i].count == 0)
827 val = (double)gld->data[gld_i].sum /
828 gld->data[gld_i].count;
830 if (this_avg > avg + 30 || val > 95) {
831 color = pick_cpu_color();
832 svg_line_graph(plot, gld, color, avg + 30, 95);
833 snprintf(line, line_len, " CPU %d\n", i - 1);
834 svg_add_legend(plot, tf->label, line, color);
843 if (plot->add_xlabel)
844 set_xlabel(plot, "Time (seconds)");
846 if (!plot->no_legend) {
847 svg_write_legend(plot);
848 svg_free_legend(plot);
851 total_graphs_written++;
854 static void plot_queue_depth(struct plot *plot, int min_seconds, int max_seconds)
856 struct trace_file *tf;
858 if (active_graphs[QUEUE_DEPTH_GRAPH_INDEX] == 0)
862 set_plot_label(plot, "Queue Depth");
864 svg_alloc_legend(plot, num_traces);
866 tf = list_entry(all_traces.next, struct trace_file, list);
867 set_ylabel(plot, "Pending IO");
868 set_yticks(plot, num_yticks, 0, tf->queue_depth_gld->max, "");
869 set_xticks(plot, num_xticks, min_seconds, max_seconds);
871 list_for_each_entry(tf, &all_traces, list) {
872 svg_line_graph(plot, tf->queue_depth_gld, tf->line_color, 0, 0);
874 svg_add_legend(plot, tf->label, "", tf->line_color);
877 if (plot->add_xlabel)
878 set_xlabel(plot, "Time (seconds)");
880 svg_write_legend(plot);
882 total_graphs_written++;
885 static void convert_movie_files(char *movie_dir)
887 fprintf(stderr, "Converting svg files in %s\n", movie_dir);
888 snprintf(line, line_len, "find %s -name \\*.svg | xargs -I{} -n 1 -P 8 rsvg-convert -o {}.png {}",
893 static void mencode_movie(char *movie_dir)
895 fprintf(stderr, "Creating movie %s with ffmpeg\n", movie_dir);
896 snprintf(line, line_len, "ffmpeg -r 20 -y -i %s/%%10d-%s.svg.png -b:v 250k "
897 "-vcodec %s %s", movie_dir, output_filename, ffmpeg_codec,
902 static void tencode_movie(char *movie_dir)
904 fprintf(stderr, "Creating movie %s with png2theora\n", movie_dir);
905 snprintf(line, line_len, "png2theora -o %s %s/%%010d-%s.svg.png",
906 output_filename, movie_dir, output_filename);
910 static void encode_movie(char *movie_dir)
912 char *last_dot = strrchr(output_filename, '.');
915 (!strncmp(last_dot, ".ogg", 4) || !strncmp(last_dot, ".ogv", 4))) {
916 tencode_movie(movie_dir);
918 mencode_movie(movie_dir);
921 static void cleanup_movie(char *movie_dir)
923 if (keep_movie_svgs) {
924 fprintf(stderr, "Keeping movie dir %s\n", movie_dir);
927 fprintf(stderr, "Removing movie dir %s\n", movie_dir);
928 snprintf(line, line_len, "rm %s/*", movie_dir);
931 snprintf(line, line_len, "rmdir %s", movie_dir);
935 static void plot_io_movie(struct plot *plot)
937 struct trace_file *tf;
938 char *movie_dir = create_movie_temp_dir();
940 struct plot_history *history;
943 int movie_frames_per_sec = 20;
944 int total_frames = movie_len * movie_frames_per_sec;
947 int graph_width_factor = 5;
950 get_graph_size(&cols, &rows);
951 batch_count = cols / total_frames;
953 if (batch_count == 0)
956 list_for_each_entry(tf, &all_traces, list) {
963 strcpy(label, tf->label);
967 pos = label + strlen(label);
971 snprintf(line, line_len, "%s/%010d-%s.svg", movie_dir, i, output_filename);
972 set_plot_output(plot, line);
973 set_plot_title(plot, graph_title);
974 orig_y_offset = plot->start_y_offset;
978 set_graph_size(cols / graph_width_factor, rows / 8);
979 plot->timeline = i / graph_width_factor;
981 plot_tput(plot, tf->min_seconds,
984 plot_cpu(plot, tf->max_seconds,
985 "CPU System Time", CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
987 plot->direction = PLOT_ACROSS;
988 plot_queue_depth(plot, tf->min_seconds, tf->max_seconds);
990 /* movie graph starts here */
991 plot->start_y_offset = orig_y_offset;
992 set_graph_size(cols - cols / graph_width_factor, rows);
995 plot->direction = PLOT_DOWN;;
997 if (movie_style == MOVIE_SPINDLE)
998 setup_axis_spindle(plot);
1002 svg_alloc_legend(plot, count_io_plot_types() * 2);
1004 history = alloc_plot_history(tf);
1007 for (pid = 0; pid < tf->io_plots; pid++) {
1008 if (tf->gdd_reads[pid]) {
1010 strcpy(pos, tf->gdd_reads[pid]->label);
1011 svg_add_legend(plot, label, " Reads", tf->gdd_reads[pid]->color);
1013 if (tf->gdd_writes[pid]) {
1015 strcpy(pos, tf->gdd_writes[pid]->label);
1016 svg_add_legend(plot, label, " Writes", tf->gdd_writes[pid]->color);
1021 while (i < cols && batch_i < batch_count) {
1022 for (pid = 0; pid < tf->io_plots; pid++) {
1023 if (tf->gdd_reads[pid]) {
1024 svg_io_graph_movie(tf->gdd_reads[pid],
1025 history->read_pid_history[pid],
1028 if (tf->gdd_writes[pid]) {
1029 svg_io_graph_movie(tf->gdd_writes[pid],
1030 history->write_pid_history[pid],
1038 add_history(history, &movie_history);
1040 plot_movie_history(plot, &movie_history);
1042 svg_write_legend(plot);
1046 close_plot_file(plot);
1048 free_all_plot_history(&movie_history);
1050 convert_movie_files(movie_dir);
1051 encode_movie(movie_dir);
1052 cleanup_movie(movie_dir);
1056 static void plot_latency(struct plot *plot, int min_seconds, int max_seconds)
1058 struct trace_file *tf;
1063 if (active_graphs[LATENCY_GRAPH_INDEX] == 0)
1067 svg_alloc_legend(plot, num_traces);
1069 list_for_each_entry(tf, &all_traces, list) {
1070 if (tf->latency_gld->max > max)
1071 max = tf->latency_gld->max;
1074 list_for_each_entry(tf, &all_traces, list)
1075 tf->latency_gld->max = max;
1078 set_plot_label(plot, "IO Latency");
1080 tf = list_entry(all_traces.next, struct trace_file, list);
1082 scale_line_graph_time(&max, &units);
1083 sprintf(line, "latency (%ss)", units);
1084 set_ylabel(plot, line);
1085 set_yticks(plot, num_yticks, 0, max, "");
1086 set_xticks(plot, num_xticks, min_seconds, max_seconds);
1088 list_for_each_entry(tf, &all_traces, list) {
1089 svg_line_graph(plot, tf->latency_gld, tf->line_color, 0, 0);
1091 svg_add_legend(plot, tf->label, "", tf->line_color);
1094 if (plot->add_xlabel)
1095 set_xlabel(plot, "Time (seconds)");
1097 svg_write_legend(plot);
1099 total_graphs_written++;
1102 static void plot_iops(struct plot *plot, int min_seconds, int max_seconds)
1104 struct trace_file *tf;
1108 if (active_graphs[IOPS_GRAPH_INDEX] == 0)
1111 list_for_each_entry(tf, &all_traces, list) {
1112 if (tf->iop_gld->max > max)
1113 max = tf->iop_gld->max;
1116 list_for_each_entry(tf, &all_traces, list)
1117 tf->iop_gld->max = max;
1120 set_plot_label(plot, "IOPs");
1122 svg_alloc_legend(plot, num_traces);
1124 tf = list_entry(all_traces.next, struct trace_file, list);
1126 scale_line_graph_bytes(&max, &units, 1000);
1127 set_ylabel(plot, "IO/s");
1129 set_yticks(plot, num_yticks, 0, max, units);
1130 set_xticks(plot, num_xticks, min_seconds, max_seconds);
1132 list_for_each_entry(tf, &all_traces, list) {
1133 svg_line_graph(plot, tf->iop_gld, tf->line_color, 0, 0);
1135 svg_add_legend(plot, tf->label, "", tf->line_color);
1138 if (plot->add_xlabel)
1139 set_xlabel(plot, "Time (seconds)");
1141 svg_write_legend(plot);
1144 total_graphs_written++;
1147 static void check_plot_columns(struct plot *plot, int index)
1151 if (columns > 1 && (total_graphs_written == 0 ||
1152 total_graphs_written % columns != 0)) {
1153 count = graphs_left(index);
1154 if (plot->direction == PLOT_DOWN) {
1155 plot->start_x_offset = 0;
1156 if (count <= columns)
1157 plot->add_xlabel = 1;
1159 plot->direction = PLOT_ACROSS;
1162 plot->direction = PLOT_DOWN;
1163 if (index == last_active_graph)
1164 plot->add_xlabel = 1;
1173 char *option_string = "T:t:o:l:r:O:N:d:D:p:m::h:w:c:x:y:a:C:PK";
1174 static struct option long_options[] = {
1175 {"columns", required_argument, 0, 'c'},
1176 {"title", required_argument, 0, 'T'},
1177 {"trace", required_argument, 0, 't'},
1178 {"output", required_argument, 0, 'o'},
1179 {"label", required_argument, 0, 'l'},
1180 {"rolling", required_argument, 0, 'r'},
1181 {"no-graph", required_argument, 0, 'N'},
1182 {"only-graph", required_argument, 0, 'O'},
1183 {"device", required_argument, 0, 'd'},
1184 {"blktrace-destination", required_argument, 0, 'D'},
1185 {"prog", required_argument, 0, 'p'},
1186 {"movie", optional_argument, 0, 'm'},
1187 {"codec", optional_argument, 0, 'C'},
1188 {"keep-movie-svgs", no_argument, 0, 'K'},
1189 {"width", required_argument, 0, 'w'},
1190 {"height", required_argument, 0, 'h'},
1191 {"xzoom", required_argument, 0, 'x'},
1192 {"yzoom", required_argument, 0, 'y'},
1193 {"io-plot-action", required_argument, 0, 'a'},
1194 {"per-process-io", no_argument, 0, 'P'},
1195 {"help", no_argument, 0, HELP_LONG_OPT},
1199 static void print_usage(void)
1201 fprintf(stderr, "iowatcher usage:\n"
1202 "\t-d (--device): device for blktrace to trace\n"
1203 "\t-D (--blktrace-destination): destination for blktrace\n"
1204 "\t-t (--trace): trace file name (more than one allowed)\n"
1205 "\t-l (--label): trace label in the graph\n"
1206 "\t-o (--output): output file name (SVG only)\n"
1207 "\t-p (--prog): program to run while blktrace is run\n"
1208 "\t-K (--keep-movie-svgs keep svgs generated for movie mode\n"
1209 "\t-m (--movie [=spindle|rect]): create IO animations\n"
1210 "\t-C (--codec): ffmpeg codec. Use ffmpeg -codecs to list\n"
1211 "\t-r (--rolling): number of seconds in the rolling averge\n"
1212 "\t-T (--title): graph title\n"
1213 "\t-N (--no-graph): skip a single graph (io, tput, latency, queue_depth, \n"
1214 "\t\t\tiops, cpu-sys, cpu-io, cpu-irq cpu-soft cpu-user)\n"
1215 "\t-O (--only-graph): add a single graph to the output\n"
1216 "\t-h (--height): set the height of each graph\n"
1217 "\t-w (--width): set the width of each graph\n"
1218 "\t-c (--columns): numbers of columns in graph output\n"
1219 "\t-x (--xzoom): limit processed time to min:max\n"
1220 "\t-y (--yzoom): limit processed sectors to min:max\n"
1221 "\t-a (--io-plot-action): plot given action (one of Q,D,C) in IO graph\n"
1222 "\t-P (--per-process-io): distinguish between processes in IO graph\n"
1227 static int parse_double_range(char *str, double *min, double *max)
1231 /* Empty lower bound - leave original value */
1232 if (str[0] != ':') {
1233 *min = strtod(str, &end);
1234 if (*min == HUGE_VAL || *min == -HUGE_VAL)
1240 /* Empty upper bound - leave original value */
1242 *max = strtod(end+1, &end);
1243 if (*max == HUGE_VAL || *max == -HUGE_VAL)
1253 static int parse_ull_range(char *str, unsigned long long *min,
1254 unsigned long long *max)
1258 /* Empty lower bound - leave original value */
1259 if (str[0] != ':') {
1260 *min = strtoull(str, &end, 10);
1261 if (*min == ULLONG_MAX && errno == ERANGE)
1267 /* Empty upper bound - leave original value */
1269 *max = strtoull(end+1, &end, 10);
1270 if (*max == ULLONG_MAX && errno == ERANGE)
1280 static int parse_options(int ac, char **av)
1286 // int this_option_optind = optind ? optind : 1;
1287 int option_index = 0;
1289 c = getopt_long(ac, av, option_string,
1290 long_options, &option_index);
1297 graph_title = strdup(optarg);
1300 add_trace_file(optarg);
1301 set_blktrace_outfile(optarg);
1304 output_filename = strdup(optarg);
1307 set_trace_label(optarg);
1310 set_rolling_avg(atoi(optarg));
1314 disable_all_graphs();
1317 enable_one_graph(optarg);
1320 disable_one_graph(optarg);
1323 if (num_blktrace_devices == MAX_DEVICES_PER_TRACE - 1) {
1324 fprintf(stderr, "Too many blktrace devices provided\n");
1327 blktrace_devices[num_blktrace_devices++] = strdup(optarg);
1330 blktrace_dest_dir = strdup(optarg);
1331 if (!strcmp(blktrace_dest_dir, "")) {
1332 fprintf(stderr, "Need a directory\n");
1337 program_to_run = strdup(optarg);
1340 keep_movie_svgs = 1;
1345 movie_style = lookup_movie_style(optarg);
1346 if (movie_style < 0) {
1347 fprintf(stderr, "Unknown movie style %s\n", optarg);
1351 fprintf(stderr, "Using movie style: %s\n",
1352 movie_styles[movie_style]);
1355 ffmpeg_codec = strdup(optarg);
1358 opt_graph_height = atoi(optarg);
1361 opt_graph_width = atoi(optarg);
1364 columns = atoi(optarg);
1367 if (parse_double_range(optarg, &min_time, &max_time)
1369 fprintf(stderr, "Cannot parse time range %s\n",
1375 if (parse_ull_range(optarg, &min_mb, &max_mb)
1378 "Cannot parse offset range %s\n",
1382 if (max_mb > ULLONG_MAX >> 20) {
1384 "Upper range limit too big."
1385 " Maximum is %llu.\n", ULLONG_MAX >> 20);
1390 if (strlen(optarg) != 1) {
1392 fprintf(stderr, "Action must be one of Q, D, C.");
1395 plot_io_action = action_char_to_num(optarg[0]);
1396 if (plot_io_action < 0)
1413 static void dest_mkdir(char *dir)
1417 ret = mkdir(dir, 0777);
1418 if (ret && errno != EEXIST) {
1419 fprintf(stderr, "failed to mkdir error %s\n", strerror(errno));
1424 int main(int ac, char **av)
1427 int min_seconds = 0;
1428 int max_seconds = 0;
1430 u64 min_offset = ~(u64)0;
1431 struct trace_file *tf;
1435 init_io_hash_table();
1436 init_process_hash_table();
1438 enable_all_graphs();
1440 parse_options(ac, av);
1442 last_active_graph = last_graph();
1444 set_io_graph_scale(256);
1445 if (movie_style == MOVIE_SPINDLE)
1446 set_graph_size(750, 550);
1448 set_graph_size(700, 400);
1451 * the plots in the movie don't have a seconds
1452 * line yet, this makes us skip it
1454 last_active_graph = TOTAL_GRAPHS + 1;
1456 if (opt_graph_height)
1457 set_graph_height(opt_graph_height);
1459 if (opt_graph_width)
1460 set_graph_width(opt_graph_width);
1462 if (list_empty(&all_traces)) {
1463 fprintf(stderr, "No traces found, exiting\n");
1467 if (num_blktrace_devices) {
1470 dest_mkdir(blktrace_dest_dir);
1471 if (num_blktrace_devices > 1) {
1472 snprintf(line, line_len, "%s/%s", blktrace_dest_dir,
1477 path = join_path(blktrace_dest_dir, blktrace_outfile);
1479 snprintf(line, line_len, "%s.dump", path);
1482 ret = start_blktrace(blktrace_devices, num_blktrace_devices,
1483 blktrace_outfile, blktrace_dest_dir);
1485 fprintf(stderr, "exiting due to blktrace failure\n");
1490 if (program_to_run) {
1491 ret = run_program(program_to_run);
1493 fprintf(stderr, "failed to run %s\n",
1499 /* no program specified, just wait for
1507 /* step one, read all the traces */
1510 /* step two, find the maxes for time and offset */
1511 list_for_each_entry(tf, &all_traces, list)
1512 compare_minmax_tf(tf, &max_seconds, &min_offset, &max_offset);
1513 min_seconds = min_time;
1514 if (max_seconds > max_time)
1515 max_seconds = ceil(max_time);
1516 if (min_offset < min_mb << 20)
1517 min_offset = min_mb << 20;
1518 if (max_offset > max_mb << 20)
1519 max_offset = max_mb << 20;
1521 /* push the max we found into all the tfs */
1522 set_all_minmax_tf(min_seconds, max_seconds, min_offset, max_offset);
1524 /* alloc graphing structs for all the traces */
1525 setup_trace_file_graphs();
1527 /* run through all the traces and read their events */
1528 read_trace_events();
1530 pick_line_graph_color();
1532 plot = alloc_plot();
1535 set_legend_width(longest_label + longest_proc_name + 1 + strlen("writes"));
1536 plot_io_movie(plot);
1540 set_plot_output(plot, output_filename);
1542 if (active_graphs[IO_GRAPH_INDEX] || found_mpstat)
1543 set_legend_width(longest_label + longest_proc_name + 1 + strlen("writes"));
1544 else if (num_traces > 1)
1545 set_legend_width(longest_label);
1547 set_legend_width(0);
1549 get_graph_size(&cols, &rows);
1551 plot->add_xlabel = 1;
1552 set_plot_title(plot, graph_title);
1554 check_plot_columns(plot, IO_GRAPH_INDEX);
1555 plot_io(plot, min_seconds, max_seconds, min_offset, max_offset);
1556 plot->add_xlabel = 0;
1559 set_graph_size(cols / columns, rows);
1560 num_xticks /= columns;
1567 check_plot_columns(plot, TPUT_GRAPH_INDEX);
1568 plot_tput(plot, min_seconds, max_seconds);
1570 check_plot_columns(plot, CPU_IO_GRAPH_INDEX);
1571 plot_cpu(plot, max_seconds, "CPU IO Wait Time",
1572 CPU_IO_GRAPH_INDEX, MPSTAT_IO);
1574 check_plot_columns(plot, CPU_SYS_GRAPH_INDEX);
1575 plot_cpu(plot, max_seconds, "CPU System Time",
1576 CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
1578 check_plot_columns(plot, CPU_IRQ_GRAPH_INDEX);
1579 plot_cpu(plot, max_seconds, "CPU IRQ Time",
1580 CPU_IRQ_GRAPH_INDEX, MPSTAT_IRQ);
1582 check_plot_columns(plot, CPU_SOFT_GRAPH_INDEX);
1583 plot_cpu(plot, max_seconds, "CPU SoftIRQ Time",
1584 CPU_SOFT_GRAPH_INDEX, MPSTAT_SOFT);
1586 check_plot_columns(plot, CPU_USER_GRAPH_INDEX);
1587 plot_cpu(plot, max_seconds, "CPU User Time",
1588 CPU_USER_GRAPH_INDEX, MPSTAT_USER);
1590 check_plot_columns(plot, LATENCY_GRAPH_INDEX);
1591 plot_latency(plot, min_seconds, max_seconds);
1593 check_plot_columns(plot, QUEUE_DEPTH_GRAPH_INDEX);
1594 plot_queue_depth(plot, min_seconds, max_seconds);
1596 check_plot_columns(plot, IOPS_GRAPH_INDEX);
1597 plot_iops(plot, min_seconds, max_seconds);
1601 close_plot_file(plot);