iowatcher: Add back missing plot title
[blktrace.git] / iowatcher / main.c
index ded1afec8ba0b7dc9bbdc017a1e03cb79d6e2b06..760fffb5f5ffdc857994bb088e8cd0dad56d7263 100644 (file)
 #include "blkparse.h"
 #include "list.h"
 #include "tracers.h"
+#include "mpstat.h"
 
 LIST_HEAD(all_traces);
 
+static char line[1024];
+static int line_len = 1024;
+static int found_mpstat = 0;
+static int cpu_color_index = 0;
 static int color_index = 0;
+static int make_movie = 0;
+static int opt_graph_width = 0;
+static int opt_graph_height = 0;
+
 char *colors[] = {
        "blue", "darkgreen",
        "red", "aqua",
@@ -61,27 +70,46 @@ char *pick_color(void) {
 }
 
 char *pick_cpu_color(void) {
-       char *ret = colors[color_index];
+       char *ret = colors[cpu_color_index];
        if (!ret) {
                color_index = 0;
-               ret = colors[color_index];
+               ret = colors[cpu_color_index];
        }
-       color_index++;
+       cpu_color_index++;
        return ret;
 }
 
 enum {
        IO_GRAPH_INDEX = 0,
        TPUT_GRAPH_INDEX,
+       CPU_SYS_GRAPH_INDEX,
+       CPU_IO_GRAPH_INDEX,
+       CPU_IRQ_GRAPH_INDEX,
+       CPU_SOFT_GRAPH_INDEX,
+       CPU_USER_GRAPH_INDEX,
        LATENCY_GRAPH_INDEX,
        QUEUE_DEPTH_GRAPH_INDEX,
        IOPS_GRAPH_INDEX,
        TOTAL_GRAPHS
 };
 
+enum {
+       MPSTAT_SYS = 0,
+       MPSTAT_IRQ,
+       MPSTAT_IO,
+       MPSTAT_SOFT,
+       MPSTAT_USER,
+       MPSTAT_GRAPHS
+};
+
 static char *graphs_by_name[] = {
        "io",
        "tput",
+       "cpu-sys",
+       "cpu-io",
+       "cpu-irq",
+       "cpu-soft",
+       "cpu-user",
        "latency",
        "queue-depth",
        "iops",
@@ -112,8 +140,28 @@ struct trace_file {
        struct graph_line_data *queue_depth_gld;
        struct graph_dot_data *gdd_writes;
        struct graph_dot_data *gdd_reads;
+
+       int mpstat_seconds;
+       int mpstat_stop_seconds;
+       struct graph_line_data **mpstat_gld;
 };
 
+static void alloc_mpstat_gld(struct trace_file *tf)
+{
+       struct graph_line_data **ptr;
+
+       if (tf->trace->mpstat_num_cpus == 0)
+               return;
+
+       ptr = calloc((tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS,
+                    sizeof(struct graph_line_data *));
+       if (!ptr) {
+               perror("Unable to allocate mpstat arrays\n");
+               exit(1);
+       }
+       tf->mpstat_gld = ptr;
+}
+
 static void enable_all_graphs(void)
 {
        int i;
@@ -162,7 +210,6 @@ static int last_graph(void)
        }
        return -ENOENT;
 }
-
 static void add_trace_file(char *filename)
 {
        struct trace_file *tf;
@@ -172,6 +219,7 @@ static void add_trace_file(char *filename)
                fprintf(stderr, "Unable to allocate memory\n");
                exit(1);
        }
+       tf->label = "";
        tf->filename = strdup(filename);
        list_add_tail(&tf->list, &all_traces);
        tf->read_color = pick_color();
@@ -182,6 +230,7 @@ static void add_trace_file(char *filename)
 static void setup_trace_file_graphs(void)
 {
        struct trace_file *tf;
+       int i;
 
        list_for_each_entry(tf, &all_traces, list) {
                tf->tput_gld = alloc_line_data(tf->seconds, tf->stop_seconds);
@@ -190,6 +239,17 @@ static void setup_trace_file_graphs(void)
                tf->iop_gld = alloc_line_data(tf->seconds, tf->stop_seconds);
                tf->gdd_writes = alloc_dot_data(tf->seconds, tf->max_offset, tf->stop_seconds);
                tf->gdd_reads = alloc_dot_data(tf->seconds, tf->max_offset, tf->stop_seconds);
+
+               if (tf->trace->mpstat_num_cpus == 0)
+                       continue;
+
+               alloc_mpstat_gld(tf);
+               for (i = 0; i < (tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i++) {
+                       tf->mpstat_gld[i] =
+                               alloc_line_data(tf->mpstat_seconds,
+                                               tf->mpstat_seconds);
+                       tf->mpstat_gld[i]->max = 100;
+               }
        }
 }
 
@@ -214,6 +274,12 @@ static void read_traces(void)
 
                filter_outliers(trace, tf->max_offset, &ymin, &ymax);
                tf->max_offset = ymax;
+
+               read_mpstat(trace, tf->filename);
+               tf->mpstat_stop_seconds = trace->mpstat_seconds;
+               tf->mpstat_seconds = trace->mpstat_seconds;
+               if (tf->mpstat_seconds)
+                       found_mpstat = 1;
        }
 }
 
@@ -223,6 +289,11 @@ static void read_trace_events(void)
        struct trace_file *tf;
        struct trace *trace;
        int ret;
+       int i;
+       int time;
+       double user, sys, iowait, irq, soft;
+       double max_user = 0, max_sys = 0, max_iowait = 0,
+              max_irq = 0, max_soft = 0;
 
        list_for_each_entry(tf, &all_traces, list) {
                trace = tf->trace;
@@ -239,6 +310,57 @@ static void read_trace_events(void)
                                break;
                }
        }
+       list_for_each_entry(tf, &all_traces, list) {
+               trace = tf->trace;
+
+               if (trace->mpstat_num_cpus == 0)
+                       continue;
+
+               first_mpstat(trace);
+
+               for (time = 0; time < tf->mpstat_stop_seconds; time ++) {
+                       for (i = 0; i < (trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i += MPSTAT_GRAPHS) {
+                               ret = read_mpstat_event(trace, &user, &sys,
+                                                       &iowait, &irq, &soft);
+                               if (ret)
+                                       goto mpstat_done;
+                               if (next_mpstat_line(trace))
+                                       goto mpstat_done;
+
+                               if (sys > max_sys)
+                                       max_sys = sys;
+                               if (user > max_user)
+                                       max_user = user;
+                               if (irq > max_irq)
+                                       max_irq = irq;
+                               if (iowait > max_iowait)
+                                       max_iowait = iowait;
+
+                               add_mpstat_gld(time, sys, tf->mpstat_gld[i + MPSTAT_SYS]);
+                               add_mpstat_gld(time, irq, tf->mpstat_gld[i + MPSTAT_IRQ]);
+                               add_mpstat_gld(time, soft, tf->mpstat_gld[i + MPSTAT_SOFT]);
+                               add_mpstat_gld(time, user, tf->mpstat_gld[i + MPSTAT_USER]);
+                               add_mpstat_gld(time, iowait, tf->mpstat_gld[i + MPSTAT_IO]);
+                       }
+                       if (next_mpstat(trace) == NULL)
+                               break;
+               }
+       }
+
+mpstat_done:
+       list_for_each_entry(tf, &all_traces, list) {
+               trace = tf->trace;
+
+               if (trace->mpstat_num_cpus == 0)
+                       continue;
+
+               tf->mpstat_gld[MPSTAT_SYS]->max = max_sys;
+               tf->mpstat_gld[MPSTAT_IRQ]->max = max_irq;
+               tf->mpstat_gld[MPSTAT_SOFT]->max = max_soft;
+               tf->mpstat_gld[MPSTAT_USER]->max = max_user;
+               tf->mpstat_gld[MPSTAT_IO]->max = max_iowait;;
+       }
+       return;
 }
 
 static void set_trace_label(char *label)
@@ -280,124 +402,108 @@ static void set_blktrace_outfile(char *arg)
 }
 
 
-char *option_string = "hT:t:o:l:r:O:N:d:p:";
-static struct option long_options[] = {
-       {"title", required_argument, 0, 'T'},
-       {"trace", required_argument, 0, 't'},
-       {"output", required_argument, 0, 'o'},
-       {"label", required_argument, 0, 'l'},
-       {"rolling", required_argument, 0, 'r'},
-       {"no-graph", required_argument, 0, 'N'},
-       {"only-graph", required_argument, 0, 'O'},
-       {"device", required_argument, 0, 'd'},
-       {"prog", required_argument, 0, 'p'},
-       {"help", required_argument, 0, 'h'},
-       {0, 0, 0, 0}
-};
-
-static void print_usage(void)
+static void compare_max_tf(struct trace_file *tf, int *seconds, u64 *max_offset)
 {
-       fprintf(stderr, "iowatcher usage:\n"
-               "\t-d (--device): device for blktrace to trace\n"
-               "\t-t (--trace): trace file name (more than one allowed)\n"
-               "\t-l (--label): trace label in the graph\n"
-               "\t-o (--output): output file name (SVG only)\n"
-               "\t-p (--prog): program to run while blktrace is run\n"
-               "\t-r (--rolling): number of seconds in the rolling averge\n"
-               "\t-T (--title): graph title\n"
-               "\t-N (--no-graph): skip a single graph (io, tput, latency, queue_depth, iops)\n"
-               "\t-O (--only-graph): add a single graph (io, tput, latency, queue_depth, iops)\n"
-              );
-       exit(1);
+       if (tf->seconds > *seconds)
+               *seconds = tf->seconds;
+       if (tf->max_offset > *max_offset)
+               *max_offset = tf->max_offset;
 }
 
-static int parse_options(int ac, char **av)
+static void set_all_max_tf(int seconds, u64 max_offset)
 {
-       int c;
-       int disabled = 0;
+       struct trace_file *tf;
 
-       while (1) {
-               // int this_option_optind = optind ? optind : 1;
-               int option_index = 0;
+       list_for_each_entry(tf, &all_traces, list) {
+               tf->seconds = seconds;
+               tf->max_offset = max_offset;
+       }
+}
 
-               c = getopt_long(ac, av, option_string,
-                               long_options, &option_index);
+static char *create_movie_temp_dir(void)
+{
+       char *ret;
+       char *pattern = strdup("btrfs-movie-XXXXXX");;
 
-               if (c == -1)
-                       break;
+       ret = mkdtemp(pattern);
+       if (!ret) {
+               perror("Unable to create temp directory for movie files");
+               exit(1);
+       }
+       return ret;
+}
 
-               switch(c) {
-               case 'h':
-                       print_usage();
-                       break;
-               case 'T':
-                       graph_title = strdup(optarg);
-                       break;
-               case 't':
-                       add_trace_file(optarg);
-                       set_blktrace_outfile(optarg);
-                       break;
-               case 'o':
-                       output_filename = strdup(optarg);
-                       break;
-               case 'l':
-                       set_trace_label(optarg);
-                       break;
-               case 'r':
-                       set_rolling_avg(atoi(optarg));
-                       break;
-               case 'O':
-                       if (!disabled) {
-                               disable_all_graphs();
-                               disabled = 1;
-                       }
-                       enable_one_graph(optarg);
-                       break;
-               case 'N':
-                       disable_one_graph(optarg);
-                       break;
-               case 'd':
-                       blktrace_device = strdup(optarg);
-                       break;
-               case 'p':
-                       program_to_run = strdup(optarg);
-                       break;
-               case '?':
-                       print_usage();
-                       break;
-               default:
-                       break;
-               }
+static struct plot_history *alloc_plot_history(char *color)
+{
+       struct plot_history *ph = calloc(1, sizeof(struct plot_history));
+
+       if (!ph) {
+               perror("memory allocation failed");
+               exit(1);
        }
-       return 0;
+       ph->history = calloc(4096, sizeof(double));
+       if (!ph->history) {
+               perror("memory allocation failed");
+               exit(1);
+       }
+       ph->history_len = 4096;
+       ph->color = color;
+       return ph;
 }
 
-static void compare_max_tf(struct trace_file *tf, int *seconds, u64 *max_offset)
+LIST_HEAD(movie_history_writes);
+LIST_HEAD(movie_history_reads);
+int num_histories = 0;
+
+static void add_history(struct plot_history *ph, struct list_head *list)
 {
-       if (tf->seconds > *seconds)
-               *seconds = tf->seconds;
-       if (tf->max_offset > *max_offset)
-               *max_offset = tf->max_offset;
+       struct plot_history *entry;
+
+       list_add_tail(&ph->list, list);
+       num_histories++;
+
+       if (num_histories > 12) {
+               num_histories--;
+               entry = list_entry(list->next, struct plot_history, list);
+               list_del(&entry->list);
+               free(entry->history);
+               free(entry);
+       }
 }
 
-static void set_all_max_tf(int seconds, u64 max_offset)
+static void plot_movie_history(struct plot *plot, struct list_head *list)
 {
-       struct trace_file *tf;
+       float alpha = 0.1;
+       struct plot_history *ph;
+
+       list_for_each_entry(ph, list, list) {
+               if (ph->list.next == list)
+                       alpha = 1;
+               svg_io_graph_movie_array(plot, ph, 1);
+               alpha += 0.2;
+               if (alpha > 1)
+                       alpha = 0.8;
+        }
+}
 
-       list_for_each_entry(tf, &all_traces, list) {
-               tf->seconds = seconds;
-               tf->max_offset = max_offset;
+static void free_all_plot_history(struct list_head *head)
+{
+       struct plot_history *entry;
+       while (!list_empty(head)) {
+               entry = list_entry(head->next, struct plot_history, list);
+               list_del(&entry->list);
+               free(entry->history);
+               free(entry);
        }
 }
 
-static void plot_io(struct plot *plot, int seconds, u64 max_offset)
+static void __plot_io(struct plot *plot, int seconds, u64 max_offset)
 {
        struct trace_file *tf;
 
        if (active_graphs[IO_GRAPH_INDEX] == 0)
                return;
 
-       plot->add_xlabel = last_active_graph == IO_GRAPH_INDEX;
        setup_axis(plot);
 
        svg_alloc_legend(plot, num_traces * 2);
@@ -424,10 +530,17 @@ static void plot_io(struct plot *plot, int seconds, u64 max_offset)
        if (plot->add_xlabel)
                set_xlabel(plot, "Time (seconds)");
        svg_write_legend(plot);
+}
+
+static void plot_io(struct plot *plot, int seconds, u64 max_offset)
+{
+       plot->add_xlabel = last_active_graph == IO_GRAPH_INDEX;
+
+       __plot_io(plot, seconds, max_offset);
        close_plot(plot);
 }
 
-static void plot_tput(struct plot *plot, int seconds)
+static void __plot_tput(struct plot *plot, int seconds)
 {
        struct trace_file *tf;
        char *units;
@@ -446,7 +559,6 @@ static void plot_tput(struct plot *plot, int seconds)
        list_for_each_entry(tf, &all_traces, list)
                tf->tput_gld->max = max;
 
-       plot->add_xlabel = last_active_graph == TPUT_GRAPH_INDEX;
        setup_axis(plot);
        set_plot_label(plot, "Throughput");
 
@@ -468,6 +580,205 @@ static void plot_tput(struct plot *plot, int seconds)
                set_xlabel(plot, "Time (seconds)");
        if (num_traces > 1)
                svg_write_legend(plot);
+}
+
+static void plot_tput(struct plot *plot, int seconds)
+{
+       plot->add_xlabel = last_active_graph == TPUT_GRAPH_INDEX;
+       __plot_tput(plot, seconds);
+       close_plot(plot);
+}
+
+static void convert_movie_files(char *movie_dir)
+{
+       fprintf(stderr, "Converting svg files in %s\n", movie_dir);
+       snprintf(line, line_len, "find %s -name \\*.svg | xargs -I{} -n 1 -P 8 rsvg-convert -o {}.png {}",
+                movie_dir);
+       system(line);
+}
+
+static void mencode_movie(char *movie_dir)
+{
+       fprintf(stderr, "Creating movie %s\n", movie_dir);
+       snprintf(line, line_len, "mencoder mf://%s/*.png -mf type=png:fps=16 -of lavf "
+                "-ovc x264 -oac copy -o %s",
+                movie_dir, output_filename);
+       system(line);
+}
+
+static void cleanup_movie(char *movie_dir)
+{
+       fprintf(stderr, "Removing movie dir %s\n", movie_dir);
+       snprintf(line, line_len, "rm %s/*", movie_dir);
+       system(line);
+
+       snprintf(line, line_len, "rmdir %s", movie_dir);
+       system(line);
+}
+
+static void plot_io_movie(struct plot *plot)
+{
+       struct trace_file *tf;
+       char *movie_dir = create_movie_temp_dir();
+       int i;
+       struct plot_history *read_history;
+       struct plot_history *write_history;
+       int batch_i;
+       int movie_len = 30;
+       int movie_frames_per_sec = 16;
+       int total_frames = movie_len * movie_frames_per_sec;
+       int rows, cols;
+       int batch_count;
+
+       get_graph_size(&cols, &rows);
+       batch_count = cols / total_frames;
+
+       if (batch_count == 0)
+               batch_count = 1;
+
+       list_for_each_entry(tf, &all_traces, list) {
+               char *label = tf->label;
+               if (!label)
+                       label = "";
+
+               i = 0;
+               while (i < cols) {
+                       snprintf(line, line_len, "%s/%010d-%s.svg", movie_dir, i, output_filename);
+                       set_plot_output(plot, line);
+
+                       set_plot_title(plot, graph_title);
+                       setup_axis(plot);
+                       svg_alloc_legend(plot, num_traces * 2);
+
+                       read_history = alloc_plot_history(tf->read_color);
+                       write_history = alloc_plot_history(tf->write_color);
+                       read_history->col = i;
+                       write_history->col = i;
+
+                       if (tf->gdd_reads->total_ios)
+                               svg_add_legend(plot, label, " Reads", tf->read_color);
+                       if (tf->gdd_writes->total_ios)
+                               svg_add_legend(plot, label, " Writes", tf->write_color);
+
+                       batch_i = 0;
+                       while (i < cols && batch_i < batch_count) {
+                               /* print just this column */
+                               svg_io_graph_movie(tf->gdd_reads, read_history, i);
+
+                               svg_io_graph_movie(tf->gdd_writes, write_history, i);
+                               i++;
+                               batch_i++;
+                       }
+
+                       add_history(read_history, &movie_history_reads);
+                       add_history(write_history, &movie_history_writes);
+
+                       plot_movie_history(plot, &movie_history_reads);
+                       plot_movie_history(plot, &movie_history_writes);
+
+                       svg_write_legend(plot);
+                       close_plot(plot);
+
+                       set_graph_size(cols, rows / 3);
+                       plot->add_xlabel = 1;
+                       __plot_tput(plot, tf->gdd_reads->seconds);
+                       svg_write_time_line(plot, i);
+                       close_plot(plot);
+                       set_graph_size(cols, rows);
+
+                       close_plot(plot);
+               }
+               free_all_plot_history(&movie_history_reads);
+               free_all_plot_history(&movie_history_writes);
+       }
+       convert_movie_files(movie_dir);
+       mencode_movie(movie_dir);
+       cleanup_movie(movie_dir);
+       free(movie_dir);
+}
+
+static void plot_cpu(struct plot *plot, int seconds, char *label,
+                    int active_index, int gld_index)
+{
+       struct trace_file *tf;
+       char line[128];
+       int max = 0;
+       int i;
+       int gld_i;
+       char *color;
+       double avg = 0;
+       int ymax;
+       int plotted = 0;
+
+       if (active_graphs[active_index] == 0)
+               return;
+
+       list_for_each_entry(tf, &all_traces, list) {
+               if (tf->trace->mpstat_num_cpus > max)
+                       max = tf->trace->mpstat_num_cpus;
+       }
+       if (max == 0)
+               return;
+
+       tf = list_entry(all_traces.next, struct trace_file, list);
+
+       ymax = tf->mpstat_gld[gld_index]->max;
+       if (ymax == 0)
+               return;
+
+       svg_alloc_legend(plot, num_traces * max);
+
+       plot->add_xlabel = last_active_graph == active_index;
+       setup_axis(plot);
+       set_plot_label(plot, label);
+
+       seconds = tf->mpstat_seconds;
+
+       set_yticks(plot, 4, 0, tf->mpstat_gld[gld_index]->max, "");
+       set_ylabel(plot, "Percent");
+       set_xticks(plot, 9, 0, seconds);
+
+       cpu_color_index = 0;
+       list_for_each_entry(tf, &all_traces, list) {
+               for (i = 0; i < tf->mpstat_gld[0]->stop_seconds; i++) {
+                       avg += tf->mpstat_gld[gld_index]->data[i].sum;
+               }
+               avg /= tf->mpstat_gld[gld_index]->stop_seconds;
+               color = pick_cpu_color();
+               svg_line_graph(plot, tf->mpstat_gld[0], color);
+               svg_add_legend(plot, tf->label, " avg", color);
+
+               for (i = 1; i < tf->trace->mpstat_num_cpus + 1; i++) {
+                       struct graph_line_data *gld = tf->mpstat_gld[i * MPSTAT_GRAPHS + gld_index];
+                       double this_avg = 0;
+
+                       for (gld_i = 0; gld_i < gld->stop_seconds; gld_i++)
+                               this_avg += gld->data[i].sum;
+
+                       this_avg /= gld->stop_seconds;
+
+                       for (gld_i = 0; gld_i < gld->stop_seconds; gld_i++) {
+                               if (this_avg > avg + 30 ||
+                                   gld->data[gld_i].sum > 95) {
+                                       color = pick_cpu_color();
+                                       svg_line_graph(plot, gld, color);
+                                       snprintf(line, 128, " CPU %d\n", i - 1);
+                                       svg_add_legend(plot, tf->label, line, color);
+                                       plotted++;
+                                       break;
+                               }
+
+                       }
+               }
+       }
+
+       if (plot->add_xlabel)
+               set_xlabel(plot, "Time (seconds)");
+
+       if (plot->legend_index <= 8)
+               svg_write_legend(plot);
+       else
+               svg_free_legend(plot);
        close_plot(plot);
 }
 
@@ -593,12 +904,120 @@ static void plot_iops(struct plot *plot, int seconds)
        close_plot(plot);
 }
 
+enum {
+       HELP_LONG_OPT = 1,
+};
+
+char *option_string = "T:t:o:l:r:O:N:d:p:mh:w:";
+static struct option long_options[] = {
+       {"title", required_argument, 0, 'T'},
+       {"trace", required_argument, 0, 't'},
+       {"output", required_argument, 0, 'o'},
+       {"label", required_argument, 0, 'l'},
+       {"rolling", required_argument, 0, 'r'},
+       {"no-graph", required_argument, 0, 'N'},
+       {"only-graph", required_argument, 0, 'O'},
+       {"device", required_argument, 0, 'd'},
+       {"prog", required_argument, 0, 'p'},
+       {"movie", no_argument, 0, 'm'},
+       {"width", required_argument, 0, 'w'},
+       {"height", required_argument, 0, 'h'},
+       {"help", no_argument, 0, HELP_LONG_OPT},
+       {0, 0, 0, 0}
+};
+
+static void print_usage(void)
+{
+       fprintf(stderr, "iowatcher usage:\n"
+               "\t-d (--device): device for blktrace to trace\n"
+               "\t-t (--trace): trace file name (more than one allowed)\n"
+               "\t-l (--label): trace label in the graph\n"
+               "\t-o (--output): output file name (SVG only)\n"
+               "\t-p (--prog): program to run while blktrace is run\n"
+               "\t-p (--movie): create IO animations\n"
+               "\t-r (--rolling): number of seconds in the rolling averge\n"
+               "\t-T (--title): graph title\n"
+               "\t-N (--no-graph): skip a single graph (io, tput, latency, queue_depth, iops)\n"
+               "\t-h (--height): set the height of each graph\n"
+               "\t-w (--width): set the width of each graph\n"
+              );
+       exit(1);
+}
+
+static int parse_options(int ac, char **av)
+{
+       int c;
+       int disabled = 0;
+
+       while (1) {
+               // int this_option_optind = optind ? optind : 1;
+               int option_index = 0;
+
+               c = getopt_long(ac, av, option_string,
+                               long_options, &option_index);
+
+               if (c == -1)
+                       break;
+
+               switch(c) {
+               case 'T':
+                       graph_title = strdup(optarg);
+                       break;
+               case 't':
+                       add_trace_file(optarg);
+                       set_blktrace_outfile(optarg);
+                       break;
+               case 'o':
+                       output_filename = strdup(optarg);
+                       break;
+               case 'l':
+                       set_trace_label(optarg);
+                       break;
+               case 'r':
+                       set_rolling_avg(atoi(optarg));
+                       break;
+               case 'O':
+                       if (!disabled) {
+                               disable_all_graphs();
+                               disabled = 1;
+                       }
+                       enable_one_graph(optarg);
+                       break;
+               case 'N':
+                       disable_one_graph(optarg);
+                       break;
+               case 'd':
+                       blktrace_device = strdup(optarg);
+                       break;
+               case 'p':
+                       program_to_run = strdup(optarg);
+                       break;
+               case 'm':
+                       make_movie = 1;
+                       break;
+               case 'h':
+                       opt_graph_height = atoi(optarg);
+                       break;
+               case 'w':
+                       opt_graph_width = atoi(optarg);
+                       break;
+               case '?':
+               case HELP_LONG_OPT:
+                       print_usage();
+                       break;
+               default:
+                       break;
+               }
+       }
+       return 0;
+}
+
+
 int main(int ac, char **av)
 {
        struct plot *plot;
        int seconds = 0;
        u64 max_offset = 0;
-       int fd;
        struct trace_file *tf;
        int ret;
 
@@ -609,6 +1028,15 @@ int main(int ac, char **av)
        parse_options(ac, av);
 
        last_active_graph = last_graph();
+       if (make_movie) {
+               set_io_graph_scale(256);
+               set_graph_size(700, 250);
+       }
+       if (opt_graph_height)
+               set_graph_height(opt_graph_height);
+
+       if (opt_graph_width)
+               set_graph_width(opt_graph_height);
 
        if (list_empty(&all_traces)) {
                fprintf(stderr, "No traces found, exiting\n");
@@ -622,6 +1050,7 @@ int main(int ac, char **av)
                        fprintf(stderr, "exiting due to blktrace failure\n");
                        exit(1);
                }
+               start_mpstat(blktrace_outfile);
                if (program_to_run) {
                        ret = run_program(program_to_run);
                        if (ret) {
@@ -655,17 +1084,16 @@ int main(int ac, char **av)
        /* run through all the traces and read their events */
        read_trace_events();
 
-       fd = open(output_filename, O_CREAT | O_TRUNC | O_WRONLY, 0600);
-       if (fd < 0) {
-               fprintf(stderr, "Unable to open output file %s %s\n",
-                       output_filename, strerror(errno));
-               exit(1);
+       plot = alloc_plot();
+
+       if (make_movie) {
+               plot_io_movie(plot);
+               exit(0);
        }
 
-       write_svg_header(fd);
-       plot = alloc_plot(fd);
+       set_plot_output(plot, output_filename);
 
-       if (active_graphs[IO_GRAPH_INDEX])
+       if (active_graphs[IO_GRAPH_INDEX] || found_mpstat)
                set_legend_width(longest_label + strlen("writes"));
        else if (num_traces > 1)
                set_legend_width(longest_label);
@@ -673,15 +1101,24 @@ int main(int ac, char **av)
                set_legend_width(0);
 
        set_plot_title(plot, graph_title);
-
        plot_io(plot, seconds, max_offset);
        plot_tput(plot, seconds);
+       plot_cpu(plot, seconds, "CPU IO Wait Time",
+                CPU_IO_GRAPH_INDEX, MPSTAT_IO);
+       plot_cpu(plot, seconds, "CPU System Time",
+                CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
+       plot_cpu(plot, seconds, "CPU IRQ Time",
+                CPU_IRQ_GRAPH_INDEX, MPSTAT_IRQ);
+       plot_cpu(plot, seconds, "CPU SoftIRQ Time",
+                CPU_SOFT_GRAPH_INDEX, MPSTAT_SOFT);
+       plot_cpu(plot, seconds, "CPU User Time",
+                CPU_USER_GRAPH_INDEX, MPSTAT_USER);
+
        plot_latency(plot, seconds);
        plot_queue_depth(plot, seconds);
        plot_iops(plot, seconds);
 
        /* once for all */
        close_plot(plot);
-       close(fd);
        return 0;
 }