iowatcher/main.c

   1 /*
   2  * Copyright (C) 2012 Fusion-io
   3  *
   4  *  This program is free software; you can redistribute it and/or
   5  *  modify it under the terms of the GNU General Public
   6  *  License v2 as published by the Free Software Foundation.
   7  *
   8  *  This program is distributed in the hope that it will be useful,
   9  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  *  GNU General Public License for more details.
  12  *
  13  *  You should have received a copy of the GNU General Public License
  14  *  along with this program; if not, write to the Free Software
  15  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  16  *
  17  *  Parts of this file were imported from Jens Axboe's blktrace sources (also GPL)
  18  */
  19 #include <sys/types.h>
  20 #include <sys/stat.h>
  21 #include <fcntl.h>
  22 #include <unistd.h>
  23 #include <stdlib.h>
  24 #include <stdio.h>
  25 #include <math.h>
  26 #include <inttypes.h>
  27 #include <string.h>
  28 #include <asm/types.h>
  29 #include <errno.h>
  30 #include <sys/mman.h>
  31 #include <time.h>
  32 #include <math.h>
  33 #include <getopt.h>
  34 #include <limits.h>
  35 #include <float.h>
  36
  37 #include "plot.h"
  38 #include "blkparse.h"
  39 #include "list.h"
  40 #include "tracers.h"
  41 #include "mpstat.h"
  42
  43 LIST_HEAD(all_traces);
  44
  45 static char line[1024];
  46 static int line_len = 1024;
  47 static int found_mpstat = 0;
  48 static int cpu_color_index = 0;
  49 static int color_index = 0;
  50 static int make_movie = 0;
  51 static int opt_graph_width = 0;
  52 static int opt_graph_height = 0;
  53
  54 static int columns = 1;
  55 static int num_xticks = 9;
  56
  57 static double min_time = 0;
  58 static double max_time = DBL_MAX;
  59 static unsigned long long min_mb = 0;
  60 static unsigned long long max_mb = ULLONG_MAX >> 20;
  61
  62 /*
  63  * this doesn't include the IO graph,
  64  * but it counts the other graphs as they go out
  65  */
  66 static int total_graphs_written = 1;
  67
  68 char *colors[] = {
  69         "blue", "darkgreen",
  70         "red", "aqua",
  71         "orange", "darkviolet",
  72         "brown", "#00FF00",
  73         "yellow", "coral",
  74         "black", "darkred",
  75         "fuchsia", "crimson",
  76         NULL };
  77
  78 char *pick_color(void) {
  79         char *ret = colors[color_index];
  80         if (!ret) {
  81                 color_index = 0;
  82                 ret = colors[color_index];
  83         }
  84         color_index++;
  85         return ret;
  86 }
  87
  88 char *pick_cpu_color(void) {
  89         char *ret = colors[cpu_color_index];
  90         if (!ret) {
  91                 color_index = 0;
  92                 ret = colors[cpu_color_index];
  93         }
  94         cpu_color_index++;
  95         return ret;
  96 }
  97
  98 enum {
  99         IO_GRAPH_INDEX = 0,
 100         TPUT_GRAPH_INDEX,
 101         CPU_SYS_GRAPH_INDEX,
 102         CPU_IO_GRAPH_INDEX,
 103         CPU_IRQ_GRAPH_INDEX,
 104         CPU_SOFT_GRAPH_INDEX,
 105         CPU_USER_GRAPH_INDEX,
 106         LATENCY_GRAPH_INDEX,
 107         QUEUE_DEPTH_GRAPH_INDEX,
 108         IOPS_GRAPH_INDEX,
 109         TOTAL_GRAPHS
 110 };
 111
 112 enum {
 113         MPSTAT_SYS = 0,
 114         MPSTAT_IRQ,
 115         MPSTAT_IO,
 116         MPSTAT_SOFT,
 117         MPSTAT_USER,
 118         MPSTAT_GRAPHS
 119 };
 120
 121 static char *graphs_by_name[] = {
 122         "io",
 123         "tput",
 124         "cpu-sys",
 125         "cpu-io",
 126         "cpu-irq",
 127         "cpu-soft",
 128         "cpu-user",
 129         "latency",
 130         "queue-depth",
 131         "iops",
 132 };
 133
 134 enum {
 135         MOVIE_SPINDLE,
 136         MOVIE_RECT,
 137         NUM_MOVIE_STYLES,
 138 };
 139
 140 char *movie_styles[] = {
 141         "spindle",
 142         "rect",
 143         NULL
 144 };
 145
 146 static int movie_style = 0;
 147
 148 static int lookup_movie_style(char *str)
 149 {
 150         int i;
 151
 152         for (i = 0; i < NUM_MOVIE_STYLES; i++) {
 153                 if (strcmp(str, movie_styles[i]) == 0)
 154                         return i;
 155         }
 156         return -1;
 157 }
 158
 159 static int active_graphs[TOTAL_GRAPHS];
 160 static int last_active_graph = IOPS_GRAPH_INDEX;
 161
 162 static int label_index = 0;
 163 static int num_traces = 0;
 164 static int longest_label = 0;
 165
 166 struct trace_file {
 167         struct list_head list;
 168         char *filename;
 169         char *label;
 170         struct trace *trace;
 171         int stop_seconds;       /* Time when trace stops */
 172         int min_seconds;        /* Beginning of the interval we should plot */
 173         int max_seconds;        /* End of the interval we should plot */
 174         u64 min_offset;
 175         u64 max_offset;
 176
 177         char *read_color;
 178         char *write_color;
 179
 180         struct graph_line_data *tput_gld;
 181         struct graph_line_data *iop_gld;
 182         struct graph_line_data *latency_gld;
 183         struct graph_line_data *queue_depth_gld;
 184         struct graph_dot_data *gdd_writes;
 185         struct graph_dot_data *gdd_reads;
 186
 187         int mpstat_min_seconds;
 188         int mpstat_max_seconds;
 189         int mpstat_stop_seconds;
 190         struct graph_line_data **mpstat_gld;
 191 };
 192
 193 static void alloc_mpstat_gld(struct trace_file *tf)
 194 {
 195         struct graph_line_data **ptr;
 196
 197         if (tf->trace->mpstat_num_cpus == 0)
 198                 return;
 199
 200         ptr = calloc((tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS,
 201                      sizeof(struct graph_line_data *));
 202         if (!ptr) {
 203                 perror("Unable to allocate mpstat arrays\n");
 204                 exit(1);
 205         }
 206         tf->mpstat_gld = ptr;
 207 }
 208
 209 static void enable_all_graphs(void)
 210 {
 211         int i;
 212         for (i = 0; i < TOTAL_GRAPHS; i++)
 213                 active_graphs[i] = 1;
 214 }
 215
 216 static void disable_all_graphs(void)
 217 {
 218         int i;
 219         for (i = 0; i < TOTAL_GRAPHS; i++)
 220                 active_graphs[i] = 0;
 221 }
 222
 223 static int enable_one_graph(char *name)
 224 {
 225         int i;
 226         for (i = 0; i < TOTAL_GRAPHS; i++) {
 227                 if (strcmp(name, graphs_by_name[i]) == 0) {
 228                         active_graphs[i] = 1;
 229                         return 0;
 230                 }
 231         }
 232         return -ENOENT;
 233 }
 234
 235 static int disable_one_graph(char *name)
 236 {
 237         int i;
 238         for (i = 0; i < TOTAL_GRAPHS; i++) {
 239                 if (strcmp(name, graphs_by_name[i]) == 0) {
 240                         active_graphs[i] = 0;
 241                         return 0;
 242                 }
 243         }
 244         return -ENOENT;
 245 }
 246
 247 static int last_graph(void)
 248 {
 249         int i;
 250         for (i = TOTAL_GRAPHS - 1; i >= 0; i--) {
 251                 if (active_graphs[i]) {
 252                         return i;
 253                 }
 254         }
 255         return -ENOENT;
 256 }
 257
 258 static int graphs_left(int cur)
 259 {
 260         int i;
 261         int left = 0;
 262         for (i = cur; i < TOTAL_GRAPHS; i++) {
 263                 if (active_graphs[i])
 264                         left++;
 265         }
 266         return left;
 267 }
 268
 269 static void add_trace_file(char *filename)
 270 {
 271         struct trace_file *tf;
 272
 273         tf = calloc(1, sizeof(*tf));
 274         if (!tf) {
 275                 fprintf(stderr, "Unable to allocate memory\n");
 276                 exit(1);
 277         }
 278         tf->label = "";
 279         tf->filename = strdup(filename);
 280         list_add_tail(&tf->list, &all_traces);
 281         tf->read_color = pick_color();
 282         tf->write_color = pick_color();
 283         num_traces++;
 284 }
 285
 286 static void setup_trace_file_graphs(void)
 287 {
 288         struct trace_file *tf;
 289         int i;
 290
 291         list_for_each_entry(tf, &all_traces, list) {
 292                 tf->tput_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 293                 tf->latency_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 294                 tf->queue_depth_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 295                 tf->iop_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 296                 tf->gdd_writes = alloc_dot_data(tf->min_seconds, tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds);
 297                 tf->gdd_reads = alloc_dot_data(tf->min_seconds, tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds);
 298
 299                 if (tf->trace->mpstat_num_cpus == 0)
 300                         continue;
 301
 302                 alloc_mpstat_gld(tf);
 303                 for (i = 0; i < (tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i++) {
 304                         tf->mpstat_gld[i] =
 305                                 alloc_line_data(tf->mpstat_min_seconds,
 306                                                 tf->mpstat_max_seconds,
 307                                                 tf->mpstat_max_seconds);
 308                         tf->mpstat_gld[i]->max = 100;
 309                 }
 310         }
 311 }
 312
 313 static void read_traces(void)
 314 {
 315         struct trace_file *tf;
 316         struct trace *trace;
 317         u64 last_time;
 318         u64 ymin;
 319         u64 ymax;
 320         u64 max_bank;
 321         u64 max_bank_offset;
 322
 323         list_for_each_entry(tf, &all_traces, list) {
 324                 trace = open_trace(tf->filename);
 325                 if (!trace)
 326                         exit(1);
 327
 328                 last_time = find_last_time(trace);
 329                 tf->trace = trace;
 330                 tf->max_seconds = SECONDS(last_time);
 331                 tf->stop_seconds = SECONDS(last_time);
 332                 find_extreme_offsets(trace, &tf->min_offset, &tf->max_offset,
 333                                     &max_bank, &max_bank_offset);
 334                 filter_outliers(trace, tf->min_offset, tf->max_offset, &ymin, &ymax);
 335                 tf->min_offset = ymin;
 336                 tf->max_offset = ymax;
 337
 338                 read_mpstat(trace, tf->filename);
 339                 tf->mpstat_stop_seconds = trace->mpstat_seconds;
 340                 tf->mpstat_max_seconds = trace->mpstat_seconds;
 341                 if (tf->mpstat_max_seconds)
 342                         found_mpstat = 1;
 343         }
 344 }
 345
 346 static void read_trace_events(void)
 347 {
 348
 349         struct trace_file *tf;
 350         struct trace *trace;
 351         int ret;
 352         int i;
 353         int time;
 354         double user, sys, iowait, irq, soft;
 355         double max_user = 0, max_sys = 0, max_iowait = 0,
 356                max_irq = 0, max_soft = 0;
 357
 358         list_for_each_entry(tf, &all_traces, list) {
 359                 trace = tf->trace;
 360                 first_record(trace);
 361                 while (1) {
 362                         check_record(trace);
 363                         add_tput(trace, tf->tput_gld);
 364                         add_iop(trace, tf->iop_gld);
 365                         add_io(trace, tf->gdd_writes, tf->gdd_reads);
 366                         add_pending_io(trace, tf->queue_depth_gld);
 367                         add_completed_io(trace, tf->latency_gld);
 368                         ret = next_record(trace);
 369                         if (ret)
 370                                 break;
 371                 }
 372         }
 373         list_for_each_entry(tf, &all_traces, list) {
 374                 trace = tf->trace;
 375
 376                 if (trace->mpstat_num_cpus == 0)
 377                         continue;
 378
 379                 first_mpstat(trace);
 380
 381                 for (time = 0; time < tf->mpstat_stop_seconds; time ++) {
 382                         for (i = 0; i < (trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i += MPSTAT_GRAPHS) {
 383                                 ret = read_mpstat_event(trace, &user, &sys,
 384                                                         &iowait, &irq, &soft);
 385                                 if (ret)
 386                                         goto mpstat_done;
 387                                 if (next_mpstat_line(trace))
 388                                         goto mpstat_done;
 389
 390                                 if (sys > max_sys)
 391                                         max_sys = sys;
 392                                 if (user > max_user)
 393                                         max_user = user;
 394                                 if (irq > max_irq)
 395                                         max_irq = irq;
 396                                 if (iowait > max_iowait)
 397                                         max_iowait = iowait;
 398
 399                                 add_mpstat_gld(time, sys, tf->mpstat_gld[i + MPSTAT_SYS]);
 400                                 add_mpstat_gld(time, irq, tf->mpstat_gld[i + MPSTAT_IRQ]);
 401                                 add_mpstat_gld(time, soft, tf->mpstat_gld[i + MPSTAT_SOFT]);
 402                                 add_mpstat_gld(time, user, tf->mpstat_gld[i + MPSTAT_USER]);
 403                                 add_mpstat_gld(time, iowait, tf->mpstat_gld[i + MPSTAT_IO]);
 404                         }
 405                         if (next_mpstat(trace) == NULL)
 406                                 break;
 407                 }
 408         }
 409
 410 mpstat_done:
 411         list_for_each_entry(tf, &all_traces, list) {
 412                 trace = tf->trace;
 413
 414                 if (trace->mpstat_num_cpus == 0)
 415                         continue;
 416
 417                 tf->mpstat_gld[MPSTAT_SYS]->max = max_sys;
 418                 tf->mpstat_gld[MPSTAT_IRQ]->max = max_irq;
 419                 tf->mpstat_gld[MPSTAT_SOFT]->max = max_soft;
 420                 tf->mpstat_gld[MPSTAT_USER]->max = max_user;
 421                 tf->mpstat_gld[MPSTAT_IO]->max = max_iowait;;
 422         }
 423         return;
 424 }
 425
 426 static void set_trace_label(char *label)
 427 {
 428         int cur = 0;
 429         struct trace_file *tf;
 430         int len = strlen(label);
 431
 432         if (len > longest_label)
 433                 longest_label = len;
 434
 435         list_for_each_entry(tf, &all_traces, list) {
 436                 if (cur == label_index) {
 437                         tf->label = strdup(label);
 438                         label_index++;
 439                         break;
 440                 }
 441                 cur++;
 442         }
 443 }
 444
 445 static char *graph_title = "";
 446 static char *output_filename = "trace.svg";
 447 static char *blktrace_device = NULL;
 448 static char *blktrace_outfile = "trace";
 449 static char *blktrace_dest_dir = ".";
 450 static char *program_to_run = NULL;
 451
 452 static void set_blktrace_outfile(char *arg)
 453 {
 454         char *s = strdup(arg);
 455         char *last_dot = strrchr(s, '.');
 456
 457         if (last_dot) {
 458                 if (strcmp(last_dot, ".dump") == 0)
 459                         *last_dot = '\0';
 460         }
 461         blktrace_outfile = s;
 462 }
 463
 464
 465 static void compare_minmax_tf(struct trace_file *tf, int *max_seconds, u64 *min_offset, u64 *max_offset)
 466 {
 467         if (tf->max_seconds > *max_seconds)
 468                 *max_seconds = tf->max_seconds;
 469         if (tf->max_offset > *max_offset)
 470                 *max_offset = tf->max_offset;
 471         if (tf->min_offset < *min_offset)
 472                 *min_offset = tf->min_offset;
 473 }
 474
 475 static void set_all_minmax_tf(int min_seconds, int max_seconds, u64 min_offset, u64 max_offset)
 476 {
 477         struct trace_file *tf;
 478
 479         list_for_each_entry(tf, &all_traces, list) {
 480                 tf->min_seconds = min_seconds;
 481                 tf->max_seconds = max_seconds;
 482                 if (tf->stop_seconds > max_seconds)
 483                         tf->stop_seconds = max_seconds;
 484                 if (tf->mpstat_max_seconds) {
 485                         tf->mpstat_min_seconds = min_seconds;
 486                         tf->mpstat_max_seconds = max_seconds;
 487                         if (tf->mpstat_stop_seconds > max_seconds)
 488                                 tf->mpstat_stop_seconds = max_seconds;
 489                 }
 490                 tf->min_offset = min_offset;
 491                 tf->max_offset = max_offset;
 492         }
 493 }
 494
 495 static char *create_movie_temp_dir(void)
 496 {
 497         char *ret;
 498         char *pattern = strdup("io-movie-XXXXXX");;
 499
 500         ret = mkdtemp(pattern);
 501         if (!ret) {
 502                 perror("Unable to create temp directory for movie files");
 503                 exit(1);
 504         }
 505         return ret;
 506 }
 507
 508 static struct plot_history *alloc_plot_history(char *color)
 509 {
 510         struct plot_history *ph = calloc(1, sizeof(struct plot_history));
 511
 512         if (!ph) {
 513                 perror("memory allocation failed");
 514                 exit(1);
 515         }
 516         ph->history = calloc(4096, sizeof(double));
 517         if (!ph->history) {
 518                 perror("memory allocation failed");
 519                 exit(1);
 520         }
 521         ph->history_len = 4096;
 522         ph->color = color;
 523         return ph;
 524 }
 525
 526 LIST_HEAD(movie_history_writes);
 527 LIST_HEAD(movie_history_reads);
 528 int num_histories = 0;
 529
 530 static void add_history(struct plot_history *ph, struct list_head *list)
 531 {
 532         struct plot_history *entry;
 533
 534         list_add_tail(&ph->list, list);
 535         num_histories++;
 536
 537         if (num_histories > 12) {
 538                 num_histories--;
 539                 entry = list_entry(list->next, struct plot_history, list);
 540                 list_del(&entry->list);
 541                 free(entry->history);
 542                 free(entry);
 543         }
 544 }
 545
 546 static void plot_movie_history(struct plot *plot, struct list_head *list)
 547 {
 548         struct plot_history *ph;
 549
 550         if (num_histories > 2)
 551                 rewind_spindle_steps(num_histories - 1);
 552
 553         list_for_each_entry(ph, list, list) {
 554                 if (movie_style == MOVIE_SPINDLE)
 555                         svg_io_graph_movie_array_spindle(plot, ph);
 556                 else
 557                         svg_io_graph_movie_array(plot, ph);
 558          }
 559 }
 560
 561 static void free_all_plot_history(struct list_head *head)
 562 {
 563         struct plot_history *entry;
 564         while (!list_empty(head)) {
 565                 entry = list_entry(head->next, struct plot_history, list);
 566                 list_del(&entry->list);
 567                 free(entry->history);
 568                 free(entry);
 569         }
 570 }
 571
 572 static void plot_io(struct plot *plot, int min_seconds, int max_seconds, u64 min_offset, u64 max_offset)
 573 {
 574         struct trace_file *tf;
 575
 576         if (active_graphs[IO_GRAPH_INDEX] == 0)
 577                 return;
 578
 579         setup_axis(plot);
 580
 581         svg_alloc_legend(plot, num_traces * 2);
 582
 583         set_plot_label(plot, "Device IO");
 584         set_ylabel(plot, "Offset (MB)");
 585         set_yticks(plot, 4, min_offset / (1024 * 1024),
 586                    max_offset / (1024 * 1024), "");
 587         set_xticks(plot, num_xticks, min_seconds, max_seconds);
 588
 589         list_for_each_entry(tf, &all_traces, list) {
 590                 char *label = tf->label;
 591
 592                 if (!label)
 593                         label = "";
 594                 svg_io_graph(plot, tf->gdd_reads, tf->read_color);
 595                 if (tf->gdd_reads->total_ios)
 596                         svg_add_legend(plot, label, " Reads", tf->read_color);
 597
 598                 svg_io_graph(plot, tf->gdd_writes, tf->write_color);
 599                 if (tf->gdd_writes->total_ios) {
 600                         svg_add_legend(plot, label, " Writes", tf->write_color);
 601                 }
 602         }
 603         if (plot->add_xlabel)
 604                 set_xlabel(plot, "Time (seconds)");
 605         svg_write_legend(plot);
 606         close_plot(plot);
 607 }
 608
 609 static void plot_tput(struct plot *plot, int min_seconds, int max_seconds)
 610 {
 611         struct trace_file *tf;
 612         char *units;
 613         char line[128];
 614         u64 max = 0;
 615
 616         if (active_graphs[TPUT_GRAPH_INDEX] == 0)
 617                 return;
 618
 619         if (num_traces > 1)
 620                 svg_alloc_legend(plot, num_traces);
 621         list_for_each_entry(tf, &all_traces, list) {
 622                 if (tf->tput_gld->max > max)
 623                         max = tf->tput_gld->max;
 624         }
 625         list_for_each_entry(tf, &all_traces, list)
 626                 tf->tput_gld->max = max;
 627
 628         setup_axis(plot);
 629         set_plot_label(plot, "Throughput");
 630
 631         tf = list_entry(all_traces.next, struct trace_file, list);
 632
 633         scale_line_graph_bytes(&max, &units, 1024);
 634         sprintf(line, "%sB/s", units);
 635         set_ylabel(plot, line);
 636         set_yticks(plot, 4, 0, max, "");
 637         set_xticks(plot, num_xticks, min_seconds, max_seconds);
 638
 639         list_for_each_entry(tf, &all_traces, list) {
 640                 svg_line_graph(plot, tf->tput_gld, tf->read_color, 0, 0);
 641                 if (num_traces > 1)
 642                         svg_add_legend(plot, tf->label, "", tf->read_color);
 643         }
 644
 645         if (plot->add_xlabel)
 646                 set_xlabel(plot, "Time (seconds)");
 647         if (num_traces > 1)
 648                 svg_write_legend(plot);
 649         close_plot(plot);
 650         total_graphs_written++;
 651 }
 652
 653 static void plot_cpu(struct plot *plot, int max_seconds, char *label,
 654                      int active_index, int gld_index)
 655 {
 656         struct trace_file *tf;
 657         int max = 0;
 658         int i;
 659         int gld_i;
 660         char *color;
 661         double avg = 0;
 662         int ymax;
 663         int plotted = 0;
 664
 665         if (active_graphs[active_index] == 0)
 666                 return;
 667
 668         list_for_each_entry(tf, &all_traces, list) {
 669                 if (tf->trace->mpstat_num_cpus > max)
 670                         max = tf->trace->mpstat_num_cpus;
 671         }
 672         if (max == 0)
 673                 return;
 674
 675         tf = list_entry(all_traces.next, struct trace_file, list);
 676
 677         ymax = tf->mpstat_gld[gld_index]->max;
 678         if (ymax == 0)
 679                 return;
 680
 681         svg_alloc_legend(plot, num_traces * max);
 682
 683         setup_axis(plot);
 684         set_plot_label(plot, label);
 685
 686         max_seconds = tf->mpstat_max_seconds;
 687
 688         set_yticks(plot, 4, 0, tf->mpstat_gld[gld_index]->max, "");
 689         set_ylabel(plot, "Percent");
 690         set_xticks(plot, num_xticks, tf->mpstat_min_seconds, max_seconds);
 691
 692         cpu_color_index = 0;
 693         list_for_each_entry(tf, &all_traces, list) {
 694                 for (i = tf->mpstat_gld[0]->min_seconds;
 695                      i < tf->mpstat_gld[0]->stop_seconds; i++) {
 696                         if (tf->mpstat_gld[gld_index]->data[i].count) {
 697                                 avg += (tf->mpstat_gld[gld_index]->data[i].sum /
 698                                         tf->mpstat_gld[gld_index]->data[i].count);
 699                         }
 700                 }
 701                 avg /= tf->mpstat_gld[gld_index]->stop_seconds -
 702                        tf->mpstat_gld[gld_index]->min_seconds;
 703                 color = pick_cpu_color();
 704                 svg_line_graph(plot, tf->mpstat_gld[0], color, 0, 0);
 705                 svg_add_legend(plot, tf->label, " avg", color);
 706
 707                 for (i = 1; i < tf->trace->mpstat_num_cpus + 1; i++) {
 708                         struct graph_line_data *gld = tf->mpstat_gld[i * MPSTAT_GRAPHS + gld_index];
 709                         double this_avg = 0;
 710
 711                         for (gld_i = gld->min_seconds;
 712                              gld_i < gld->stop_seconds; gld_i++) {
 713                                 if (gld->data[i].count) {
 714                                         this_avg += gld->data[i].sum /
 715                                                 gld->data[i].count;
 716                                 }
 717                         }
 718
 719                         this_avg /= gld->stop_seconds - gld->min_seconds;
 720
 721                         for (gld_i = gld->min_seconds;
 722                              gld_i < gld->stop_seconds; gld_i++) {
 723                                 double val;
 724
 725                                 if (gld->data[gld_i].count == 0)
 726                                         continue;
 727                                 val = (double)gld->data[gld_i].sum /
 728                                         gld->data[gld_i].count;
 729
 730                                 if (this_avg > avg + 30 || val > 95) {
 731                                         color = pick_cpu_color();
 732                                         svg_line_graph(plot, gld, color, avg + 30, 95);
 733                                         snprintf(line, line_len, " CPU %d\n", i - 1);
 734                                         svg_add_legend(plot, tf->label, line, color);
 735                                         plotted++;
 736                                         break;
 737                                 }
 738
 739                         }
 740                 }
 741         }
 742
 743         if (plot->add_xlabel)
 744                 set_xlabel(plot, "Time (seconds)");
 745
 746         if (!plot->no_legend) {
 747                 svg_write_legend(plot);
 748                 svg_free_legend(plot);
 749         }
 750         close_plot(plot);
 751         total_graphs_written++;
 752 }
 753
 754 static void plot_queue_depth(struct plot *plot, int min_seconds, int max_seconds)
 755 {
 756         struct trace_file *tf;
 757
 758         if (active_graphs[QUEUE_DEPTH_GRAPH_INDEX] == 0)
 759                 return;
 760
 761         setup_axis(plot);
 762         set_plot_label(plot, "Queue Depth");
 763         if (num_traces > 1)
 764                 svg_alloc_legend(plot, num_traces);
 765
 766         tf = list_entry(all_traces.next, struct trace_file, list);
 767         set_ylabel(plot, "Pending IO");
 768         set_yticks(plot, 4, 0, tf->queue_depth_gld->max, "");
 769         set_xticks(plot, num_xticks, min_seconds, max_seconds);
 770
 771         list_for_each_entry(tf, &all_traces, list) {
 772                 svg_line_graph(plot, tf->queue_depth_gld, tf->read_color, 0, 0);
 773                 if (num_traces > 1)
 774                         svg_add_legend(plot, tf->label, "", tf->read_color);
 775         }
 776
 777         if (plot->add_xlabel)
 778                 set_xlabel(plot, "Time (seconds)");
 779         if (num_traces > 1)
 780                 svg_write_legend(plot);
 781         close_plot(plot);
 782         total_graphs_written++;
 783 }
 784
 785 static void convert_movie_files(char *movie_dir)
 786 {
 787         fprintf(stderr, "Converting svg files in %s\n", movie_dir);
 788         snprintf(line, line_len, "find %s -name \\*.svg | xargs -I{} -n 1 -P 8 rsvg-convert -o {}.png {}",
 789                  movie_dir);
 790         system(line);
 791 }
 792
 793 static void mencode_movie(char *movie_dir)
 794 {
 795         fprintf(stderr, "Creating movie %s\n", movie_dir);
 796         snprintf(line, line_len, "ffmpeg -r 20 -y -i %s/%%10d-%s.svg.png -b:v 250k "
 797                  "-vcodec libx264 %s", movie_dir, output_filename, output_filename);
 798         system(line);
 799 }
 800
 801 static void cleanup_movie(char *movie_dir)
 802 {
 803         fprintf(stderr, "Removing movie dir %s\n", movie_dir);
 804         snprintf(line, line_len, "rm %s/*", movie_dir);
 805         system(line);
 806
 807         snprintf(line, line_len, "rmdir %s", movie_dir);
 808         system(line);
 809 }
 810
 811 static void plot_io_movie(struct plot *plot)
 812 {
 813         struct trace_file *tf;
 814         char *movie_dir = create_movie_temp_dir();
 815         int i;
 816         struct plot_history *read_history;
 817         struct plot_history *write_history;
 818         int batch_i;
 819         int movie_len = 30;
 820         int movie_frames_per_sec = 20;
 821         int total_frames = movie_len * movie_frames_per_sec;
 822         int rows, cols;
 823         int batch_count;
 824         int graph_width_factor = 5;
 825         int orig_y_offset;
 826
 827         get_graph_size(&cols, &rows);
 828         batch_count = cols / total_frames;
 829
 830         if (batch_count == 0)
 831                 batch_count = 1;
 832
 833         list_for_each_entry(tf, &all_traces, list) {
 834                 char *label = tf->label;
 835                 if (!label)
 836                         label = "";
 837
 838                 i = 0;
 839                 while (i < cols) {
 840                         snprintf(line, line_len, "%s/%010d-%s.svg", movie_dir, i, output_filename);
 841                         set_plot_output(plot, line);
 842                         set_plot_title(plot, graph_title);
 843                         orig_y_offset = plot->start_y_offset;
 844
 845                         plot->no_legend = 1;
 846
 847                         set_graph_size(cols / graph_width_factor, rows / 8);
 848                         plot->timeline = i / graph_width_factor;
 849
 850                         plot_tput(plot, tf->gdd_reads->min_seconds,
 851                                   tf->gdd_reads->max_seconds);
 852
 853                         plot_cpu(plot, tf->gdd_reads->max_seconds,
 854                                    "CPU System Time", CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
 855
 856                         plot->direction = PLOT_ACROSS;
 857                         plot_queue_depth(plot, tf->gdd_reads->min_seconds,
 858                                          tf->gdd_reads->max_seconds);
 859
 860                         /* movie graph starts here */
 861                         plot->start_y_offset = orig_y_offset;
 862                         set_graph_size(cols - cols / graph_width_factor, rows);
 863                         plot->no_legend = 0;
 864                         plot->timeline = 0;
 865                         plot->direction = PLOT_DOWN;;
 866
 867                         if (movie_style == MOVIE_SPINDLE)
 868                                 setup_axis_spindle(plot);
 869                         else
 870                                 setup_axis(plot);
 871
 872                         svg_alloc_legend(plot, num_traces * 2);
 873
 874                         read_history = alloc_plot_history(tf->read_color);
 875                         write_history = alloc_plot_history(tf->write_color);
 876                         read_history->col = i;
 877                         write_history->col = i;
 878
 879                         if (tf->gdd_reads->total_ios)
 880                                 svg_add_legend(plot, label, " Reads", tf->read_color);
 881                         if (tf->gdd_writes->total_ios)
 882                                 svg_add_legend(plot, label, " Writes", tf->write_color);
 883
 884                         batch_i = 0;
 885                         while (i < cols && batch_i < batch_count) {
 886                                 svg_io_graph_movie(tf->gdd_reads, read_history, i);
 887                                 svg_io_graph_movie(tf->gdd_writes, write_history, i);
 888                                 i++;
 889                                 batch_i++;
 890                         }
 891
 892                         add_history(read_history, &movie_history_reads);
 893                         add_history(write_history, &movie_history_writes);
 894
 895                         plot_movie_history(plot, &movie_history_reads);
 896                         plot_movie_history(plot, &movie_history_writes);
 897
 898                         svg_write_legend(plot);
 899                         close_plot(plot);
 900                         close_plot(plot);
 901
 902                         close_plot_file(plot);
 903                 }
 904                 free_all_plot_history(&movie_history_reads);
 905                 free_all_plot_history(&movie_history_writes);
 906         }
 907         convert_movie_files(movie_dir);
 908         mencode_movie(movie_dir);
 909         cleanup_movie(movie_dir);
 910         free(movie_dir);
 911 }
 912
 913 static void plot_latency(struct plot *plot, int min_seconds, int max_seconds)
 914 {
 915         struct trace_file *tf;
 916         char *units;
 917         char line[128];
 918         u64 max = 0;
 919
 920         if (active_graphs[LATENCY_GRAPH_INDEX] == 0)
 921                 return;
 922
 923         if (num_traces > 1)
 924                 svg_alloc_legend(plot, num_traces);
 925         list_for_each_entry(tf, &all_traces, list) {
 926                 if (tf->latency_gld->max > max)
 927                         max = tf->latency_gld->max;
 928         }
 929         list_for_each_entry(tf, &all_traces, list)
 930                 tf->latency_gld->max = max;
 931
 932         setup_axis(plot);
 933         set_plot_label(plot, "IO Latency");
 934
 935         tf = list_entry(all_traces.next, struct trace_file, list);
 936
 937         scale_line_graph_time(&max, &units);
 938         sprintf(line, "latency (%ss)", units);
 939         set_ylabel(plot, line);
 940         set_yticks(plot, 4, 0, max, "");
 941         set_xticks(plot, num_xticks, min_seconds, max_seconds);
 942
 943         list_for_each_entry(tf, &all_traces, list) {
 944                 svg_line_graph(plot, tf->latency_gld, tf->read_color, 0, 0);
 945                 if (num_traces > 1)
 946                         svg_add_legend(plot, tf->label, "", tf->read_color);
 947         }
 948
 949         if (plot->add_xlabel)
 950                 set_xlabel(plot, "Time (seconds)");
 951         if (num_traces > 1)
 952                 svg_write_legend(plot);
 953         close_plot(plot);
 954         total_graphs_written++;
 955 }
 956
 957 static void plot_iops(struct plot *plot, int min_seconds, int max_seconds)
 958 {
 959         struct trace_file *tf;
 960         char *units;
 961         u64 max = 0;
 962
 963         if (active_graphs[IOPS_GRAPH_INDEX] == 0)
 964                 return;
 965
 966         list_for_each_entry(tf, &all_traces, list) {
 967                 if (tf->iop_gld->max > max)
 968                         max = tf->iop_gld->max;
 969         }
 970
 971         list_for_each_entry(tf, &all_traces, list)
 972                 tf->iop_gld->max = max;
 973
 974         setup_axis(plot);
 975         set_plot_label(plot, "IOPs");
 976         if (num_traces > 1)
 977                 svg_alloc_legend(plot, num_traces);
 978
 979         tf = list_entry(all_traces.next, struct trace_file, list);
 980
 981         scale_line_graph_bytes(&max, &units, 1000);
 982         set_ylabel(plot, "IO/s");
 983
 984         set_yticks(plot, 4, 0, max, units);
 985         set_xticks(plot, num_xticks, min_seconds, max_seconds);
 986
 987         list_for_each_entry(tf, &all_traces, list) {
 988                 svg_line_graph(plot, tf->iop_gld, tf->read_color, 0, 0);
 989                 if (num_traces > 1)
 990                         svg_add_legend(plot, tf->label, "", tf->read_color);
 991         }
 992
 993         if (plot->add_xlabel)
 994                 set_xlabel(plot, "Time (seconds)");
 995         if (num_traces > 1)
 996                 svg_write_legend(plot);
 997
 998         close_plot(plot);
 999         total_graphs_written++;
1000 }
1001
1002 static void check_plot_columns(struct plot *plot, int index)
1003 {
1004         int count;
1005
1006         if (columns > 1 && (total_graphs_written == 0 ||
1007             total_graphs_written % columns != 0)) {
1008                 count = graphs_left(index);
1009                 if (plot->direction == PLOT_DOWN) {
1010                         plot->start_x_offset = 0;
1011                         if (count <= columns)
1012                                 plot->add_xlabel = 1;
1013                 }
1014                 plot->direction = PLOT_ACROSS;
1015
1016         } else {
1017                 plot->direction = PLOT_DOWN;
1018                 if (index == last_active_graph)
1019                         plot->add_xlabel = 1;
1020         }
1021
1022 }
1023
1024 enum {
1025         HELP_LONG_OPT = 1,
1026 };
1027
1028 char *option_string = "T:t:o:l:r:O:N:d:p:m::h:w:c:x:y:";
1029 static struct option long_options[] = {
1030         {"columns", required_argument, 0, 'c'},
1031         {"title", required_argument, 0, 'T'},
1032         {"trace", required_argument, 0, 't'},
1033         {"output", required_argument, 0, 'o'},
1034         {"label", required_argument, 0, 'l'},
1035         {"rolling", required_argument, 0, 'r'},
1036         {"no-graph", required_argument, 0, 'N'},
1037         {"only-graph", required_argument, 0, 'O'},
1038         {"device", required_argument, 0, 'd'},
1039         {"prog", required_argument, 0, 'p'},
1040         {"movie", optional_argument, 0, 'm'},
1041         {"width", required_argument, 0, 'w'},
1042         {"height", required_argument, 0, 'h'},
1043         {"xzoom", required_argument, 0, 'x'},
1044         {"yzoom", required_argument, 0, 'y'},
1045         {"help", no_argument, 0, HELP_LONG_OPT},
1046         {0, 0, 0, 0}
1047 };
1048
1049 static void print_usage(void)
1050 {
1051         fprintf(stderr, "iowatcher usage:\n"
1052                 "\t-d (--device): device for blktrace to trace\n"
1053                 "\t-t (--trace): trace file name (more than one allowed)\n"
1054                 "\t-l (--label): trace label in the graph\n"
1055                 "\t-o (--output): output file name (SVG only)\n"
1056                 "\t-p (--prog): program to run while blktrace is run\n"
1057                 "\t-m (--movie [=spindle|rect]): create IO animations\n"
1058                 "\t-r (--rolling): number of seconds in the rolling averge\n"
1059                 "\t-T (--title): graph title\n"
1060                 "\t-N (--no-graph): skip a single graph (io, tput, latency, queue_depth, \n"
1061                 "\t\t\tiops, cpu-sys, cpu-io, cpu-irq cpu-soft cpu-user)\n"
1062                 "\t-O (--only-graph): add a single graph to the output\n"
1063                 "\t-h (--height): set the height of each graph\n"
1064                 "\t-w (--width): set the width of each graph\n"
1065                 "\t-c (--columns): numbers of columns in graph output\n"
1066                 "\t-x (--xzoom): limit processed time to min:max\n"
1067                 "\t-y (--yzoom): limit processed sectors to min:max\n"
1068                );
1069         exit(1);
1070 }
1071
1072 static int parse_double_range(char *str, double *min, double *max)
1073 {
1074         char *end;
1075
1076         /* Empty lower bound - leave original value */
1077         if (str[0] != ':') {
1078                 *min = strtod(str, &end);
1079                 if (*min == HUGE_VAL || *min == -HUGE_VAL)
1080                         return -ERANGE;
1081                 if (*end != ':')
1082                         return -EINVAL;
1083         } else
1084                 end = str;
1085         /* Empty upper bound - leave original value */
1086         if (end[1]) {
1087                 *max = strtod(end+1, &end);
1088                 if (*max == HUGE_VAL || *max == -HUGE_VAL)
1089                         return -ERANGE;
1090                 if (*end != 0)
1091                         return -EINVAL;
1092         }
1093         if (*min > *max)
1094                 return -EINVAL;
1095         return 0;
1096 }
1097
1098 static int parse_ull_range(char *str, unsigned long long *min,
1099                            unsigned long long *max)
1100 {
1101         char *end;
1102
1103         /* Empty lower bound - leave original value */
1104         if (str[0] != ':') {
1105                 *min = strtoull(str, &end, 10);
1106                 if (*min == ULLONG_MAX && errno == ERANGE)
1107                         return -ERANGE;
1108                 if (*end != ':')
1109                         return -EINVAL;
1110         } else
1111                 end = str;
1112         /* Empty upper bound - leave original value */
1113         if (end[1]) {
1114                 *max = strtoull(end+1, &end, 10);
1115                 if (*max == ULLONG_MAX && errno == ERANGE)
1116                         return -ERANGE;
1117                 if (*end != 0)
1118                         return -EINVAL;
1119         }
1120         if (*min > *max)
1121                 return -EINVAL;
1122         return 0;
1123 }
1124
1125 static int parse_options(int ac, char **av)
1126 {
1127         int c;
1128         int disabled = 0;
1129
1130         while (1) {
1131                 // int this_option_optind = optind ? optind : 1;
1132                 int option_index = 0;
1133
1134                 c = getopt_long(ac, av, option_string,
1135                                 long_options, &option_index);
1136
1137                 if (c == -1)
1138                         break;
1139
1140                 switch(c) {
1141                 case 'T':
1142                         graph_title = strdup(optarg);
1143                         break;
1144                 case 't':
1145                         add_trace_file(optarg);
1146                         set_blktrace_outfile(optarg);
1147                         break;
1148                 case 'o':
1149                         output_filename = strdup(optarg);
1150                         break;
1151                 case 'l':
1152                         set_trace_label(optarg);
1153                         break;
1154                 case 'r':
1155                         set_rolling_avg(atoi(optarg));
1156                         break;
1157                 case 'O':
1158                         if (!disabled) {
1159                                 disable_all_graphs();
1160                                 disabled = 1;
1161                         }
1162                         enable_one_graph(optarg);
1163                         break;
1164                 case 'N':
1165                         disable_one_graph(optarg);
1166                         break;
1167                 case 'd':
1168                         blktrace_device = strdup(optarg);
1169                         break;
1170                 case 'p':
1171                         program_to_run = strdup(optarg);
1172                         break;
1173                 case 'm':
1174                         make_movie = 1;
1175                         if (optarg) {
1176                                 movie_style = lookup_movie_style(optarg);
1177                                 if (movie_style < 0) {
1178                                         fprintf(stderr, "Unknown movie style %s\n", optarg);
1179                                         print_usage();
1180                                 }
1181                         }
1182                         fprintf(stderr, "Using movie style: %s\n",
1183                                 movie_styles[movie_style]);
1184                         break;
1185                 case 'h':
1186                         opt_graph_height = atoi(optarg);
1187                         break;
1188                 case 'w':
1189                         opt_graph_width = atoi(optarg);
1190                         break;
1191                 case 'c':
1192                         columns = atoi(optarg);
1193                         break;
1194                 case 'x':
1195                         if (parse_double_range(optarg, &min_time, &max_time)
1196                             < 0) {
1197                                 fprintf(stderr, "Cannot parse time range %s\n",
1198                                         optarg);
1199                                 exit(1);
1200                         }
1201                         break;
1202                 case 'y':
1203                         if (parse_ull_range(optarg, &min_mb, &max_mb)
1204                             < 0) {
1205                                 fprintf(stderr,
1206                                         "Cannot parse offset range %s\n",
1207                                         optarg);
1208                                 exit(1);
1209                         }
1210                         if (max_mb > ULLONG_MAX >> 20) {
1211                                 fprintf(stderr,
1212                                         "Upper range limit too big."
1213                                         " Maximum is %llu.\n", ULLONG_MAX >> 20);
1214                                 exit(1);
1215                         }
1216                         break;
1217                 case '?':
1218                 case HELP_LONG_OPT:
1219                         print_usage();
1220                         break;
1221                 default:
1222                         break;
1223                 }
1224         }
1225         return 0;
1226 }
1227
1228
1229 int main(int ac, char **av)
1230 {
1231         struct plot *plot;
1232         int min_seconds = 0;
1233         int max_seconds = 0;
1234         u64 max_offset = 0;
1235         u64 min_offset = ~(u64)0;
1236         struct trace_file *tf;
1237         int ret;
1238         int rows, cols;
1239
1240         init_io_hash_table();
1241
1242         enable_all_graphs();
1243
1244         parse_options(ac, av);
1245
1246         last_active_graph = last_graph();
1247         if (make_movie) {
1248                 set_io_graph_scale(256);
1249                 if (movie_style == MOVIE_SPINDLE)
1250                         set_graph_size(750, 550);
1251                 else
1252                         set_graph_size(700, 400);
1253
1254                 /*
1255                  * the plots in the movie don't have a seconds
1256                  * line yet, this makes us skip it
1257                  */
1258                 last_active_graph = TOTAL_GRAPHS + 1;
1259         }
1260         if (opt_graph_height)
1261                 set_graph_height(opt_graph_height);
1262
1263         if (opt_graph_width)
1264                 set_graph_width(opt_graph_width);
1265
1266         if (list_empty(&all_traces)) {
1267                 fprintf(stderr, "No traces found, exiting\n");
1268                 exit(1);
1269         }
1270
1271         if (blktrace_device) {
1272                 ret = start_blktrace(blktrace_device, blktrace_outfile,
1273                                      blktrace_dest_dir);
1274                 if (ret) {
1275                         fprintf(stderr, "exiting due to blktrace failure\n");
1276                         exit(1);
1277                 }
1278                 start_mpstat(blktrace_outfile);
1279                 if (program_to_run) {
1280                         ret = run_program(program_to_run);
1281                         if (ret) {
1282                                 fprintf(stderr, "failed to run %s\n",
1283                                         program_to_run);
1284                                 exit(1);
1285                         }
1286                         wait_for_tracers();
1287                         blktrace_to_dump(blktrace_outfile);
1288                 } else {
1289                         /* no program specified, just wait for
1290                          * blktrace to exit
1291                          */
1292                         wait_for_tracers();
1293                 }
1294         }
1295
1296         /* step one, read all the traces */
1297         read_traces();
1298
1299         /* step two, find the maxes for time and offset */
1300         list_for_each_entry(tf, &all_traces, list)
1301                 compare_minmax_tf(tf, &max_seconds, &min_offset, &max_offset);
1302         min_seconds = min_time;
1303         if (max_seconds > max_time)
1304                 max_seconds = ceil(max_time);
1305         if (min_offset < min_mb << 20)
1306                 min_offset = min_mb << 20;
1307         if (max_offset > max_mb << 20)
1308                 max_offset = max_mb << 20;
1309
1310         /* push the max we found into all the tfs */
1311         set_all_minmax_tf(min_seconds, max_seconds, min_offset, max_offset);
1312
1313         /* alloc graphing structs for all the traces */
1314         setup_trace_file_graphs();
1315
1316         /* run through all the traces and read their events */
1317         read_trace_events();
1318
1319         plot = alloc_plot();
1320
1321         if (make_movie) {
1322                 plot_io_movie(plot);
1323                 exit(0);
1324         }
1325
1326         set_plot_output(plot, output_filename);
1327
1328         if (active_graphs[IO_GRAPH_INDEX] || found_mpstat)
1329                 set_legend_width(longest_label + strlen("writes"));
1330         else if (num_traces > 1)
1331                 set_legend_width(longest_label);
1332         else
1333                 set_legend_width(0);
1334
1335         get_graph_size(&cols, &rows);
1336         if (columns > 1)
1337                 plot->add_xlabel = 1;
1338         set_plot_title(plot, graph_title);
1339
1340         plot_io(plot, min_seconds, max_seconds, min_offset, max_offset);
1341         plot->add_xlabel = 0;
1342
1343         if (columns > 1) {
1344                 set_graph_size(cols / columns, rows);
1345                 num_xticks /= columns;
1346                 if (num_xticks < 2)
1347                         num_xticks = 2;
1348         }
1349
1350         check_plot_columns(plot, TPUT_GRAPH_INDEX);
1351         plot_tput(plot, min_seconds, max_seconds);
1352
1353         check_plot_columns(plot, CPU_IO_GRAPH_INDEX);
1354         plot_cpu(plot, max_seconds, "CPU IO Wait Time",
1355                  CPU_IO_GRAPH_INDEX, MPSTAT_IO);
1356
1357         check_plot_columns(plot, CPU_SYS_GRAPH_INDEX);
1358         plot_cpu(plot, max_seconds, "CPU System Time",
1359                  CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
1360
1361         check_plot_columns(plot, CPU_IRQ_GRAPH_INDEX);
1362         plot_cpu(plot, max_seconds, "CPU IRQ Time",
1363                  CPU_IRQ_GRAPH_INDEX, MPSTAT_IRQ);
1364
1365         check_plot_columns(plot, CPU_SOFT_GRAPH_INDEX);
1366         plot_cpu(plot, max_seconds, "CPU SoftIRQ Time",
1367                  CPU_SOFT_GRAPH_INDEX, MPSTAT_SOFT);
1368
1369         check_plot_columns(plot, CPU_USER_GRAPH_INDEX);
1370         plot_cpu(plot, max_seconds, "CPU User Time",
1371                  CPU_USER_GRAPH_INDEX, MPSTAT_USER);
1372
1373         check_plot_columns(plot, LATENCY_GRAPH_INDEX);
1374         plot_latency(plot, min_seconds, max_seconds);
1375
1376         check_plot_columns(plot, QUEUE_DEPTH_GRAPH_INDEX);
1377         plot_queue_depth(plot, min_seconds, max_seconds);
1378
1379         check_plot_columns(plot, IOPS_GRAPH_INDEX);
1380         plot_iops(plot, min_seconds, max_seconds);
1381
1382         /* once for all */
1383         close_plot(plot);
1384         close_plot_file(plot);
1385         return 0;
1386 }