iowatcher/main.c

   1 /*
   2  * Copyright (C) 2012 Fusion-io
   3  *
   4  *  This program is free software; you can redistribute it and/or
   5  *  modify it under the terms of the GNU General Public
   6  *  License v2 as published by the Free Software Foundation.
   7  *
   8  *  This program is distributed in the hope that it will be useful,
   9  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  *  GNU General Public License for more details.
  12  *
  13  *  You should have received a copy of the GNU General Public License
  14  *  along with this program; if not, write to the Free Software
  15  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  16  *
  17  *  Parts of this file were imported from Jens Axboe's blktrace sources (also GPL)
  18  */
  19 #include <sys/types.h>
  20 #include <sys/stat.h>
  21 #include <fcntl.h>
  22 #include <unistd.h>
  23 #include <stdlib.h>
  24 #include <stdio.h>
  25 #include <math.h>
  26 #include <inttypes.h>
  27 #include <string.h>
  28 #include <asm/types.h>
  29 #include <errno.h>
  30 #include <sys/mman.h>
  31 #include <time.h>
  32 #include <math.h>
  33 #include <getopt.h>
  34
  35 #include "plot.h"
  36 #include "blkparse.h"
  37 #include "list.h"
  38 #include "tracers.h"
  39 #include "mpstat.h"
  40
  41 LIST_HEAD(all_traces);
  42
  43 static char line[1024];
  44 static int line_len = 1024;
  45 static int found_mpstat = 0;
  46 static int cpu_color_index = 0;
  47 static int color_index = 0;
  48 static int make_movie = 0;
  49 static int opt_graph_width = 0;
  50 static int opt_graph_height = 0;
  51
  52 static int columns = 1;
  53 static int num_xticks = 9;
  54
  55 /*
  56  * this doesn't include the IO graph,
  57  * but it counts the other graphs as they go out
  58  */
  59 static int total_graphs_written = 1;
  60
  61 char *colors[] = {
  62         "blue", "darkgreen",
  63         "red", "aqua",
  64         "orange", "darkviolet",
  65         "brown", "#00FF00",
  66         "yellow", "coral",
  67         "black", "darkred",
  68         "fuchsia", "crimson",
  69         NULL };
  70
  71 char *pick_color(void) {
  72         char *ret = colors[color_index];
  73         if (!ret) {
  74                 color_index = 0;
  75                 ret = colors[color_index];
  76         }
  77         color_index++;
  78         return ret;
  79 }
  80
  81 char *pick_cpu_color(void) {
  82         char *ret = colors[cpu_color_index];
  83         if (!ret) {
  84                 color_index = 0;
  85                 ret = colors[cpu_color_index];
  86         }
  87         cpu_color_index++;
  88         return ret;
  89 }
  90
  91 enum {
  92         IO_GRAPH_INDEX = 0,
  93         TPUT_GRAPH_INDEX,
  94         CPU_SYS_GRAPH_INDEX,
  95         CPU_IO_GRAPH_INDEX,
  96         CPU_IRQ_GRAPH_INDEX,
  97         CPU_SOFT_GRAPH_INDEX,
  98         CPU_USER_GRAPH_INDEX,
  99         LATENCY_GRAPH_INDEX,
 100         QUEUE_DEPTH_GRAPH_INDEX,
 101         IOPS_GRAPH_INDEX,
 102         TOTAL_GRAPHS
 103 };
 104
 105 enum {
 106         MPSTAT_SYS = 0,
 107         MPSTAT_IRQ,
 108         MPSTAT_IO,
 109         MPSTAT_SOFT,
 110         MPSTAT_USER,
 111         MPSTAT_GRAPHS
 112 };
 113
 114 static char *graphs_by_name[] = {
 115         "io",
 116         "tput",
 117         "cpu-sys",
 118         "cpu-io",
 119         "cpu-irq",
 120         "cpu-soft",
 121         "cpu-user",
 122         "latency",
 123         "queue-depth",
 124         "iops",
 125 };
 126
 127 enum {
 128         MOVIE_SPINDLE,
 129         MOVIE_RECT,
 130         NUM_MOVIE_STYLES,
 131 };
 132
 133 char *movie_styles[] = {
 134         "spindle",
 135         "rect",
 136         NULL
 137 };
 138
 139 static int movie_style = 0;
 140
 141 static int lookup_movie_style(char *str)
 142 {
 143         int i;
 144
 145         for (i = 0; i < NUM_MOVIE_STYLES; i++) {
 146                 if (strcmp(str, movie_styles[i]) == 0)
 147                         return i;
 148         }
 149         return -1;
 150 }
 151
 152 static int active_graphs[TOTAL_GRAPHS];
 153 static int last_active_graph = IOPS_GRAPH_INDEX;
 154
 155 static int label_index = 0;
 156 static int num_traces = 0;
 157 static int longest_label = 0;
 158
 159 struct trace_file {
 160         struct list_head list;
 161         char *filename;
 162         char *label;
 163         struct trace *trace;
 164         int max_seconds;
 165         int stop_seconds;
 166         u64 min_offset;
 167         u64 max_offset;
 168
 169         char *read_color;
 170         char *write_color;
 171
 172         struct graph_line_data *tput_gld;
 173         struct graph_line_data *iop_gld;
 174         struct graph_line_data *latency_gld;
 175         struct graph_line_data *queue_depth_gld;
 176         struct graph_dot_data *gdd_writes;
 177         struct graph_dot_data *gdd_reads;
 178
 179         int mpstat_max_seconds;
 180         int mpstat_stop_seconds;
 181         struct graph_line_data **mpstat_gld;
 182 };
 183
 184 static void alloc_mpstat_gld(struct trace_file *tf)
 185 {
 186         struct graph_line_data **ptr;
 187
 188         if (tf->trace->mpstat_num_cpus == 0)
 189                 return;
 190
 191         ptr = calloc((tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS,
 192                      sizeof(struct graph_line_data *));
 193         if (!ptr) {
 194                 perror("Unable to allocate mpstat arrays\n");
 195                 exit(1);
 196         }
 197         tf->mpstat_gld = ptr;
 198 }
 199
 200 static void enable_all_graphs(void)
 201 {
 202         int i;
 203         for (i = 0; i < TOTAL_GRAPHS; i++)
 204                 active_graphs[i] = 1;
 205 }
 206
 207 static void disable_all_graphs(void)
 208 {
 209         int i;
 210         for (i = 0; i < TOTAL_GRAPHS; i++)
 211                 active_graphs[i] = 0;
 212 }
 213
 214 static int enable_one_graph(char *name)
 215 {
 216         int i;
 217         for (i = 0; i < TOTAL_GRAPHS; i++) {
 218                 if (strcmp(name, graphs_by_name[i]) == 0) {
 219                         active_graphs[i] = 1;
 220                         return 0;
 221                 }
 222         }
 223         return -ENOENT;
 224 }
 225
 226 static int disable_one_graph(char *name)
 227 {
 228         int i;
 229         for (i = 0; i < TOTAL_GRAPHS; i++) {
 230                 if (strcmp(name, graphs_by_name[i]) == 0) {
 231                         active_graphs[i] = 0;
 232                         return 0;
 233                 }
 234         }
 235         return -ENOENT;
 236 }
 237
 238 static int last_graph(void)
 239 {
 240         int i;
 241         for (i = TOTAL_GRAPHS - 1; i >= 0; i--) {
 242                 if (active_graphs[i]) {
 243                         return i;
 244                 }
 245         }
 246         return -ENOENT;
 247 }
 248
 249 static int graphs_left(int cur)
 250 {
 251         int i;
 252         int left = 0;
 253         for (i = cur; i < TOTAL_GRAPHS; i++) {
 254                 if (active_graphs[i])
 255                         left++;
 256         }
 257         return left;
 258 }
 259
 260 static void add_trace_file(char *filename)
 261 {
 262         struct trace_file *tf;
 263
 264         tf = calloc(1, sizeof(*tf));
 265         if (!tf) {
 266                 fprintf(stderr, "Unable to allocate memory\n");
 267                 exit(1);
 268         }
 269         tf->label = "";
 270         tf->filename = strdup(filename);
 271         list_add_tail(&tf->list, &all_traces);
 272         tf->read_color = pick_color();
 273         tf->write_color = pick_color();
 274         num_traces++;
 275 }
 276
 277 static void setup_trace_file_graphs(void)
 278 {
 279         struct trace_file *tf;
 280         int i;
 281
 282         list_for_each_entry(tf, &all_traces, list) {
 283                 tf->tput_gld = alloc_line_data(tf->max_seconds, tf->stop_seconds);
 284                 tf->latency_gld = alloc_line_data(tf->max_seconds, tf->stop_seconds);
 285                 tf->queue_depth_gld = alloc_line_data(tf->max_seconds, tf->stop_seconds);
 286                 tf->iop_gld = alloc_line_data(tf->max_seconds, tf->stop_seconds);
 287                 tf->gdd_writes = alloc_dot_data(tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds);
 288                 tf->gdd_reads = alloc_dot_data(tf->max_seconds, tf->min_offset, tf->max_offset, tf->stop_seconds);
 289
 290                 if (tf->trace->mpstat_num_cpus == 0)
 291                         continue;
 292
 293                 alloc_mpstat_gld(tf);
 294                 for (i = 0; i < (tf->trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i++) {
 295                         tf->mpstat_gld[i] =
 296                                 alloc_line_data(tf->mpstat_max_seconds,
 297                                                 tf->mpstat_max_seconds);
 298                         tf->mpstat_gld[i]->max = 100;
 299                 }
 300         }
 301 }
 302
 303 static void read_traces(void)
 304 {
 305         struct trace_file *tf;
 306         struct trace *trace;
 307         u64 last_time;
 308         u64 ymin;
 309         u64 ymax;
 310         u64 max_bank;
 311         u64 max_bank_offset;
 312
 313         list_for_each_entry(tf, &all_traces, list) {
 314                 trace = open_trace(tf->filename);
 315                 if (!trace)
 316                         exit(1);
 317
 318                 last_time = find_last_time(trace);
 319                 tf->trace = trace;
 320                 tf->max_seconds = SECONDS(last_time);
 321                 tf->stop_seconds = SECONDS(last_time);
 322                 find_extreme_offsets(trace, &tf->min_offset, &tf->max_offset,
 323                                     &max_bank, &max_bank_offset);
 324                 filter_outliers(trace, tf->min_offset, tf->max_offset, &ymin, &ymax);
 325                 tf->min_offset = ymin;
 326                 tf->max_offset = ymax;
 327
 328                 read_mpstat(trace, tf->filename);
 329                 tf->mpstat_stop_seconds = trace->mpstat_seconds;
 330                 tf->mpstat_max_seconds = trace->mpstat_seconds;
 331                 if (tf->mpstat_max_seconds)
 332                         found_mpstat = 1;
 333         }
 334 }
 335
 336 static void read_trace_events(void)
 337 {
 338
 339         struct trace_file *tf;
 340         struct trace *trace;
 341         int ret;
 342         int i;
 343         int time;
 344         double user, sys, iowait, irq, soft;
 345         double max_user = 0, max_sys = 0, max_iowait = 0,
 346                max_irq = 0, max_soft = 0;
 347
 348         list_for_each_entry(tf, &all_traces, list) {
 349                 trace = tf->trace;
 350                 first_record(trace);
 351                 while (1) {
 352                         check_record(trace);
 353                         add_tput(trace, tf->tput_gld);
 354                         add_iop(trace, tf->iop_gld);
 355                         add_io(trace, tf->gdd_writes, tf->gdd_reads);
 356                         add_pending_io(trace, tf->queue_depth_gld);
 357                         add_completed_io(trace, tf->latency_gld);
 358                         ret = next_record(trace);
 359                         if (ret)
 360                                 break;
 361                 }
 362         }
 363         list_for_each_entry(tf, &all_traces, list) {
 364                 trace = tf->trace;
 365
 366                 if (trace->mpstat_num_cpus == 0)
 367                         continue;
 368
 369                 first_mpstat(trace);
 370
 371                 for (time = 0; time < tf->mpstat_stop_seconds; time ++) {
 372                         for (i = 0; i < (trace->mpstat_num_cpus + 1) * MPSTAT_GRAPHS; i += MPSTAT_GRAPHS) {
 373                                 ret = read_mpstat_event(trace, &user, &sys,
 374                                                         &iowait, &irq, &soft);
 375                                 if (ret)
 376                                         goto mpstat_done;
 377                                 if (next_mpstat_line(trace))
 378                                         goto mpstat_done;
 379
 380                                 if (sys > max_sys)
 381                                         max_sys = sys;
 382                                 if (user > max_user)
 383                                         max_user = user;
 384                                 if (irq > max_irq)
 385                                         max_irq = irq;
 386                                 if (iowait > max_iowait)
 387                                         max_iowait = iowait;
 388
 389                                 add_mpstat_gld(time, sys, tf->mpstat_gld[i + MPSTAT_SYS]);
 390                                 add_mpstat_gld(time, irq, tf->mpstat_gld[i + MPSTAT_IRQ]);
 391                                 add_mpstat_gld(time, soft, tf->mpstat_gld[i + MPSTAT_SOFT]);
 392                                 add_mpstat_gld(time, user, tf->mpstat_gld[i + MPSTAT_USER]);
 393                                 add_mpstat_gld(time, iowait, tf->mpstat_gld[i + MPSTAT_IO]);
 394                         }
 395                         if (next_mpstat(trace) == NULL)
 396                                 break;
 397                 }
 398         }
 399
 400 mpstat_done:
 401         list_for_each_entry(tf, &all_traces, list) {
 402                 trace = tf->trace;
 403
 404                 if (trace->mpstat_num_cpus == 0)
 405                         continue;
 406
 407                 tf->mpstat_gld[MPSTAT_SYS]->max = max_sys;
 408                 tf->mpstat_gld[MPSTAT_IRQ]->max = max_irq;
 409                 tf->mpstat_gld[MPSTAT_SOFT]->max = max_soft;
 410                 tf->mpstat_gld[MPSTAT_USER]->max = max_user;
 411                 tf->mpstat_gld[MPSTAT_IO]->max = max_iowait;;
 412         }
 413         return;
 414 }
 415
 416 static void set_trace_label(char *label)
 417 {
 418         int cur = 0;
 419         struct trace_file *tf;
 420         int len = strlen(label);
 421
 422         if (len > longest_label)
 423                 longest_label = len;
 424
 425         list_for_each_entry(tf, &all_traces, list) {
 426                 if (cur == label_index) {
 427                         tf->label = strdup(label);
 428                         label_index++;
 429                         break;
 430                 }
 431                 cur++;
 432         }
 433 }
 434
 435 static char *graph_title = "";
 436 static char *output_filename = "trace.svg";
 437 static char *blktrace_device = NULL;
 438 static char *blktrace_outfile = "trace";
 439 static char *blktrace_dest_dir = ".";
 440 static char *program_to_run = NULL;
 441
 442 static void set_blktrace_outfile(char *arg)
 443 {
 444         char *s = strdup(arg);
 445         char *last_dot = strrchr(s, '.');
 446
 447         if (last_dot) {
 448                 if (strcmp(last_dot, ".dump") == 0)
 449                         *last_dot = '\0';
 450         }
 451         blktrace_outfile = s;
 452 }
 453
 454
 455 static void compare_minmax_tf(struct trace_file *tf, int *max_seconds, u64 *min_offset, u64 *max_offset)
 456 {
 457         if (tf->max_seconds > *max_seconds)
 458                 *max_seconds = tf->max_seconds;
 459         if (tf->max_offset > *max_offset)
 460                 *max_offset = tf->max_offset;
 461         if (tf->min_offset < *min_offset)
 462                 *min_offset = tf->min_offset;
 463 }
 464
 465 static void set_all_minmax_tf(int max_seconds, u64 min_offset, u64 max_offset)
 466 {
 467         struct trace_file *tf;
 468
 469         list_for_each_entry(tf, &all_traces, list) {
 470                 tf->max_seconds = max_seconds;
 471                 tf->min_offset = min_offset;
 472                 tf->max_offset = max_offset;
 473         }
 474 }
 475
 476 static char *create_movie_temp_dir(void)
 477 {
 478         char *ret;
 479         char *pattern = strdup("io-movie-XXXXXX");;
 480
 481         ret = mkdtemp(pattern);
 482         if (!ret) {
 483                 perror("Unable to create temp directory for movie files");
 484                 exit(1);
 485         }
 486         return ret;
 487 }
 488
 489 static struct plot_history *alloc_plot_history(char *color)
 490 {
 491         struct plot_history *ph = calloc(1, sizeof(struct plot_history));
 492
 493         if (!ph) {
 494                 perror("memory allocation failed");
 495                 exit(1);
 496         }
 497         ph->history = calloc(4096, sizeof(double));
 498         if (!ph->history) {
 499                 perror("memory allocation failed");
 500                 exit(1);
 501         }
 502         ph->history_len = 4096;
 503         ph->color = color;
 504         return ph;
 505 }
 506
 507 LIST_HEAD(movie_history_writes);
 508 LIST_HEAD(movie_history_reads);
 509 int num_histories = 0;
 510
 511 static void add_history(struct plot_history *ph, struct list_head *list)
 512 {
 513         struct plot_history *entry;
 514
 515         list_add_tail(&ph->list, list);
 516         num_histories++;
 517
 518         if (num_histories > 12) {
 519                 num_histories--;
 520                 entry = list_entry(list->next, struct plot_history, list);
 521                 list_del(&entry->list);
 522                 free(entry->history);
 523                 free(entry);
 524         }
 525 }
 526
 527 static void plot_movie_history(struct plot *plot, struct list_head *list)
 528 {
 529         struct plot_history *ph;
 530
 531         if (num_histories > 2)
 532                 rewind_spindle_steps(num_histories - 1);
 533
 534         list_for_each_entry(ph, list, list) {
 535                 if (movie_style == MOVIE_SPINDLE)
 536                         svg_io_graph_movie_array_spindle(plot, ph);
 537                 else
 538                         svg_io_graph_movie_array(plot, ph);
 539          }
 540 }
 541
 542 static void free_all_plot_history(struct list_head *head)
 543 {
 544         struct plot_history *entry;
 545         while (!list_empty(head)) {
 546                 entry = list_entry(head->next, struct plot_history, list);
 547                 list_del(&entry->list);
 548                 free(entry->history);
 549                 free(entry);
 550         }
 551 }
 552
 553 static void plot_io(struct plot *plot, int max_seconds, u64 min_offset, u64 max_offset)
 554 {
 555         struct trace_file *tf;
 556
 557         if (active_graphs[IO_GRAPH_INDEX] == 0)
 558                 return;
 559
 560         setup_axis(plot);
 561
 562         svg_alloc_legend(plot, num_traces * 2);
 563
 564         set_plot_label(plot, "Device IO");
 565         set_ylabel(plot, "Offset (MB)");
 566         set_yticks(plot, 4, min_offset / (1024 * 1024),
 567                    max_offset / (1024 * 1024), "");
 568         set_xticks(plot, num_xticks, 0, max_seconds);
 569
 570         list_for_each_entry(tf, &all_traces, list) {
 571                 char *label = tf->label;
 572
 573                 if (!label)
 574                         label = "";
 575                 svg_io_graph(plot, tf->gdd_reads, tf->read_color);
 576                 if (tf->gdd_reads->total_ios)
 577                         svg_add_legend(plot, label, " Reads", tf->read_color);
 578
 579                 svg_io_graph(plot, tf->gdd_writes, tf->write_color);
 580                 if (tf->gdd_writes->total_ios) {
 581                         svg_add_legend(plot, label, " Writes", tf->write_color);
 582                 }
 583         }
 584         if (plot->add_xlabel)
 585                 set_xlabel(plot, "Time (seconds)");
 586         svg_write_legend(plot);
 587         close_plot(plot);
 588 }
 589
 590 static void plot_tput(struct plot *plot, int max_seconds)
 591 {
 592         struct trace_file *tf;
 593         char *units;
 594         char line[128];
 595         u64 max = 0;
 596
 597         if (active_graphs[TPUT_GRAPH_INDEX] == 0)
 598                 return;
 599
 600         if (num_traces > 1)
 601                 svg_alloc_legend(plot, num_traces);
 602         list_for_each_entry(tf, &all_traces, list) {
 603                 if (tf->tput_gld->max > max)
 604                         max = tf->tput_gld->max;
 605         }
 606         list_for_each_entry(tf, &all_traces, list)
 607                 tf->tput_gld->max = max;
 608
 609         setup_axis(plot);
 610         set_plot_label(plot, "Throughput");
 611
 612         tf = list_entry(all_traces.next, struct trace_file, list);
 613
 614         scale_line_graph_bytes(&max, &units, 1024);
 615         sprintf(line, "%sB/s", units);
 616         set_ylabel(plot, line);
 617         set_yticks(plot, 4, 0, max, "");
 618         set_xticks(plot, num_xticks, 0, max_seconds);
 619
 620         list_for_each_entry(tf, &all_traces, list) {
 621                 svg_line_graph(plot, tf->tput_gld, tf->read_color, 0, 0);
 622                 if (num_traces > 1)
 623                         svg_add_legend(plot, tf->label, "", tf->read_color);
 624         }
 625
 626         if (plot->add_xlabel)
 627                 set_xlabel(plot, "Time (seconds)");
 628         if (num_traces > 1)
 629                 svg_write_legend(plot);
 630         close_plot(plot);
 631         total_graphs_written++;
 632 }
 633
 634 static void plot_cpu(struct plot *plot, int max_seconds, char *label,
 635                      int active_index, int gld_index)
 636 {
 637         struct trace_file *tf;
 638         int max = 0;
 639         int i;
 640         int gld_i;
 641         char *color;
 642         double avg = 0;
 643         int ymax;
 644         int plotted = 0;
 645
 646         if (active_graphs[active_index] == 0)
 647                 return;
 648
 649         list_for_each_entry(tf, &all_traces, list) {
 650                 if (tf->trace->mpstat_num_cpus > max)
 651                         max = tf->trace->mpstat_num_cpus;
 652         }
 653         if (max == 0)
 654                 return;
 655
 656         tf = list_entry(all_traces.next, struct trace_file, list);
 657
 658         ymax = tf->mpstat_gld[gld_index]->max;
 659         if (ymax == 0)
 660                 return;
 661
 662         svg_alloc_legend(plot, num_traces * max);
 663
 664         setup_axis(plot);
 665         set_plot_label(plot, label);
 666
 667         max_seconds = tf->mpstat_max_seconds;
 668
 669         set_yticks(plot, 4, 0, tf->mpstat_gld[gld_index]->max, "");
 670         set_ylabel(plot, "Percent");
 671         set_xticks(plot, num_xticks, 0, max_seconds);
 672
 673         cpu_color_index = 0;
 674         list_for_each_entry(tf, &all_traces, list) {
 675                 for (i = 0; i < tf->mpstat_gld[0]->stop_seconds; i++) {
 676                         if (tf->mpstat_gld[gld_index]->data[i].count) {
 677                                 avg += (tf->mpstat_gld[gld_index]->data[i].sum /
 678                                         tf->mpstat_gld[gld_index]->data[i].count);
 679                         }
 680                 }
 681                 avg /= tf->mpstat_gld[gld_index]->stop_seconds;
 682                 color = pick_cpu_color();
 683                 svg_line_graph(plot, tf->mpstat_gld[0], color, 0, 0);
 684                 svg_add_legend(plot, tf->label, " avg", color);
 685
 686                 for (i = 1; i < tf->trace->mpstat_num_cpus + 1; i++) {
 687                         struct graph_line_data *gld = tf->mpstat_gld[i * MPSTAT_GRAPHS + gld_index];
 688                         double this_avg = 0;
 689
 690                         for (gld_i = 0; gld_i < gld->stop_seconds; gld_i++) {
 691                                 if (gld->data[i].count) {
 692                                         this_avg += gld->data[i].sum /
 693                                                 gld->data[i].count;
 694                                 }
 695                         }
 696
 697                         this_avg /= gld->stop_seconds;
 698
 699                         for (gld_i = 0; gld_i < gld->stop_seconds; gld_i++) {
 700                                 double val;
 701
 702                                 if (gld->data[gld_i].count == 0)
 703                                         continue;
 704                                 val = (double)gld->data[gld_i].sum /
 705                                         gld->data[gld_i].count;
 706
 707                                 if (this_avg > avg + 30 || val > 95) {
 708                                         color = pick_cpu_color();
 709                                         svg_line_graph(plot, gld, color, avg + 30, 95);
 710                                         snprintf(line, line_len, " CPU %d\n", i - 1);
 711                                         svg_add_legend(plot, tf->label, line, color);
 712                                         plotted++;
 713                                         break;
 714                                 }
 715
 716                         }
 717                 }
 718         }
 719
 720         if (plot->add_xlabel)
 721                 set_xlabel(plot, "Time (seconds)");
 722
 723         if (!plot->no_legend) {
 724                 svg_write_legend(plot);
 725                 svg_free_legend(plot);
 726         }
 727         close_plot(plot);
 728         total_graphs_written++;
 729 }
 730
 731 static void plot_queue_depth(struct plot *plot, int max_seconds)
 732 {
 733         struct trace_file *tf;
 734
 735         if (active_graphs[QUEUE_DEPTH_GRAPH_INDEX] == 0)
 736                 return;
 737
 738         setup_axis(plot);
 739         set_plot_label(plot, "Queue Depth");
 740         if (num_traces > 1)
 741                 svg_alloc_legend(plot, num_traces);
 742
 743         tf = list_entry(all_traces.next, struct trace_file, list);
 744         set_ylabel(plot, "Pending IO");
 745         set_yticks(plot, 4, 0, tf->queue_depth_gld->max, "");
 746         set_xticks(plot, num_xticks, 0, max_seconds);
 747
 748         list_for_each_entry(tf, &all_traces, list) {
 749                 svg_line_graph(plot, tf->queue_depth_gld, tf->read_color, 0, 0);
 750                 if (num_traces > 1)
 751                         svg_add_legend(plot, tf->label, "", tf->read_color);
 752         }
 753
 754         if (plot->add_xlabel)
 755                 set_xlabel(plot, "Time (seconds)");
 756         if (num_traces > 1)
 757                 svg_write_legend(plot);
 758         close_plot(plot);
 759         total_graphs_written++;
 760 }
 761
 762 static void convert_movie_files(char *movie_dir)
 763 {
 764         fprintf(stderr, "Converting svg files in %s\n", movie_dir);
 765         snprintf(line, line_len, "find %s -name \\*.svg | xargs -I{} -n 1 -P 8 rsvg-convert -o {}.png {}",
 766                  movie_dir);
 767         system(line);
 768 }
 769
 770 static void mencode_movie(char *movie_dir)
 771 {
 772         fprintf(stderr, "Creating movie %s\n", movie_dir);
 773         snprintf(line, line_len, "ffmpeg -r 20 -y -i %s/%%10d-%s.svg.png -b:v 250k "
 774                  "-vcodec libx264 %s", movie_dir, output_filename, output_filename);
 775         system(line);
 776 }
 777
 778 static void cleanup_movie(char *movie_dir)
 779 {
 780         fprintf(stderr, "Removing movie dir %s\n", movie_dir);
 781         snprintf(line, line_len, "rm %s/*", movie_dir);
 782         system(line);
 783
 784         snprintf(line, line_len, "rmdir %s", movie_dir);
 785         system(line);
 786 }
 787
 788 static void plot_io_movie(struct plot *plot)
 789 {
 790         struct trace_file *tf;
 791         char *movie_dir = create_movie_temp_dir();
 792         int i;
 793         struct plot_history *read_history;
 794         struct plot_history *write_history;
 795         int batch_i;
 796         int movie_len = 30;
 797         int movie_frames_per_sec = 20;
 798         int total_frames = movie_len * movie_frames_per_sec;
 799         int rows, cols;
 800         int batch_count;
 801         int graph_width_factor = 5;
 802         int orig_y_offset;
 803
 804         get_graph_size(&cols, &rows);
 805         batch_count = cols / total_frames;
 806
 807         if (batch_count == 0)
 808                 batch_count = 1;
 809
 810         list_for_each_entry(tf, &all_traces, list) {
 811                 char *label = tf->label;
 812                 if (!label)
 813                         label = "";
 814
 815                 i = 0;
 816                 while (i < cols) {
 817                         snprintf(line, line_len, "%s/%010d-%s.svg", movie_dir, i, output_filename);
 818                         set_plot_output(plot, line);
 819                         set_plot_title(plot, graph_title);
 820                         orig_y_offset = plot->start_y_offset;
 821
 822                         plot->no_legend = 1;
 823
 824                         set_graph_size(cols / graph_width_factor, rows / 8);
 825                         plot->timeline = i / graph_width_factor;
 826
 827                         plot_tput(plot, tf->gdd_reads->max_seconds);
 828
 829                         plot_cpu(plot, tf->gdd_reads->max_seconds,
 830                                    "CPU System Time", CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
 831
 832                         plot->direction = PLOT_ACROSS;
 833                         plot_queue_depth(plot, tf->gdd_reads->max_seconds);
 834
 835                         /* movie graph starts here */
 836                         plot->start_y_offset = orig_y_offset;
 837                         set_graph_size(cols - cols / graph_width_factor, rows);
 838                         plot->no_legend = 0;
 839                         plot->timeline = 0;
 840                         plot->direction = PLOT_DOWN;;
 841
 842                         if (movie_style == MOVIE_SPINDLE)
 843                                 setup_axis_spindle(plot);
 844                         else
 845                                 setup_axis(plot);
 846
 847                         svg_alloc_legend(plot, num_traces * 2);
 848
 849                         read_history = alloc_plot_history(tf->read_color);
 850                         write_history = alloc_plot_history(tf->write_color);
 851                         read_history->col = i;
 852                         write_history->col = i;
 853
 854                         if (tf->gdd_reads->total_ios)
 855                                 svg_add_legend(plot, label, " Reads", tf->read_color);
 856                         if (tf->gdd_writes->total_ios)
 857                                 svg_add_legend(plot, label, " Writes", tf->write_color);
 858
 859                         batch_i = 0;
 860                         while (i < cols && batch_i < batch_count) {
 861                                 svg_io_graph_movie(tf->gdd_reads, read_history, i);
 862                                 svg_io_graph_movie(tf->gdd_writes, write_history, i);
 863                                 i++;
 864                                 batch_i++;
 865                         }
 866
 867                         add_history(read_history, &movie_history_reads);
 868                         add_history(write_history, &movie_history_writes);
 869
 870                         plot_movie_history(plot, &movie_history_reads);
 871                         plot_movie_history(plot, &movie_history_writes);
 872
 873                         svg_write_legend(plot);
 874                         close_plot(plot);
 875                         close_plot(plot);
 876
 877                         close_plot_file(plot);
 878                 }
 879                 free_all_plot_history(&movie_history_reads);
 880                 free_all_plot_history(&movie_history_writes);
 881         }
 882         convert_movie_files(movie_dir);
 883         mencode_movie(movie_dir);
 884         cleanup_movie(movie_dir);
 885         free(movie_dir);
 886 }
 887
 888 static void plot_latency(struct plot *plot, int max_seconds)
 889 {
 890         struct trace_file *tf;
 891         char *units;
 892         char line[128];
 893         u64 max = 0;
 894
 895         if (active_graphs[LATENCY_GRAPH_INDEX] == 0)
 896                 return;
 897
 898         if (num_traces > 1)
 899                 svg_alloc_legend(plot, num_traces);
 900         list_for_each_entry(tf, &all_traces, list) {
 901                 if (tf->latency_gld->max > max)
 902                         max = tf->latency_gld->max;
 903         }
 904         list_for_each_entry(tf, &all_traces, list)
 905                 tf->latency_gld->max = max;
 906
 907         setup_axis(plot);
 908         set_plot_label(plot, "IO Latency");
 909
 910         tf = list_entry(all_traces.next, struct trace_file, list);
 911
 912         scale_line_graph_time(&max, &units);
 913         sprintf(line, "latency (%ss)", units);
 914         set_ylabel(plot, line);
 915         set_yticks(plot, 4, 0, max, "");
 916         set_xticks(plot, num_xticks, 0, max_seconds);
 917
 918         list_for_each_entry(tf, &all_traces, list) {
 919                 svg_line_graph(plot, tf->latency_gld, tf->read_color, 0, 0);
 920                 if (num_traces > 1)
 921                         svg_add_legend(plot, tf->label, "", tf->read_color);
 922         }
 923
 924         if (plot->add_xlabel)
 925                 set_xlabel(plot, "Time (seconds)");
 926         if (num_traces > 1)
 927                 svg_write_legend(plot);
 928         close_plot(plot);
 929         total_graphs_written++;
 930 }
 931
 932 static void plot_iops(struct plot *plot, int max_seconds)
 933 {
 934         struct trace_file *tf;
 935         char *units;
 936         u64 max = 0;
 937
 938         if (active_graphs[IOPS_GRAPH_INDEX] == 0)
 939                 return;
 940
 941         list_for_each_entry(tf, &all_traces, list) {
 942                 if (tf->iop_gld->max > max)
 943                         max = tf->iop_gld->max;
 944         }
 945
 946         list_for_each_entry(tf, &all_traces, list)
 947                 tf->iop_gld->max = max;
 948
 949         setup_axis(plot);
 950         set_plot_label(plot, "IOPs");
 951         if (num_traces > 1)
 952                 svg_alloc_legend(plot, num_traces);
 953
 954         tf = list_entry(all_traces.next, struct trace_file, list);
 955
 956         scale_line_graph_bytes(&max, &units, 1000);
 957         set_ylabel(plot, "IO/s");
 958
 959         set_yticks(plot, 4, 0, max, units);
 960         set_xticks(plot, num_xticks, 0, max_seconds);
 961
 962         list_for_each_entry(tf, &all_traces, list) {
 963                 svg_line_graph(plot, tf->iop_gld, tf->read_color, 0, 0);
 964                 if (num_traces > 1)
 965                         svg_add_legend(plot, tf->label, "", tf->read_color);
 966         }
 967
 968         if (plot->add_xlabel)
 969                 set_xlabel(plot, "Time (seconds)");
 970         if (num_traces > 1)
 971                 svg_write_legend(plot);
 972
 973         close_plot(plot);
 974         total_graphs_written++;
 975 }
 976
 977 static void check_plot_columns(struct plot *plot, int index)
 978 {
 979         int count;
 980
 981         if (columns > 1 && (total_graphs_written == 0 ||
 982             total_graphs_written % columns != 0)) {
 983                 count = graphs_left(index);
 984                 if (plot->direction == PLOT_DOWN) {
 985                         plot->start_x_offset = 0;
 986                         if (count <= columns)
 987                                 plot->add_xlabel = 1;
 988                 }
 989                 plot->direction = PLOT_ACROSS;
 990
 991         } else {
 992                 plot->direction = PLOT_DOWN;
 993                 if (index == last_active_graph)
 994                         plot->add_xlabel = 1;
 995         }
 996
 997 }
 998
 999 enum {
1000         HELP_LONG_OPT = 1,
1001 };
1002
1003 char *option_string = "T:t:o:l:r:O:N:d:p:m::h:w:c:";
1004 static struct option long_options[] = {
1005         {"columns", required_argument, 0, 'c'},
1006         {"title", required_argument, 0, 'T'},
1007         {"trace", required_argument, 0, 't'},
1008         {"output", required_argument, 0, 'o'},
1009         {"label", required_argument, 0, 'l'},
1010         {"rolling", required_argument, 0, 'r'},
1011         {"no-graph", required_argument, 0, 'N'},
1012         {"only-graph", required_argument, 0, 'O'},
1013         {"device", required_argument, 0, 'd'},
1014         {"prog", required_argument, 0, 'p'},
1015         {"movie", optional_argument, 0, 'm'},
1016         {"width", required_argument, 0, 'w'},
1017         {"height", required_argument, 0, 'h'},
1018         {"help", no_argument, 0, HELP_LONG_OPT},
1019         {0, 0, 0, 0}
1020 };
1021
1022 static void print_usage(void)
1023 {
1024         fprintf(stderr, "iowatcher usage:\n"
1025                 "\t-d (--device): device for blktrace to trace\n"
1026                 "\t-t (--trace): trace file name (more than one allowed)\n"
1027                 "\t-l (--label): trace label in the graph\n"
1028                 "\t-o (--output): output file name (SVG only)\n"
1029                 "\t-p (--prog): program to run while blktrace is run\n"
1030                 "\t-m (--movie [=spindle|rect]): create IO animations\n"
1031                 "\t-r (--rolling): number of seconds in the rolling averge\n"
1032                 "\t-T (--title): graph title\n"
1033                 "\t-N (--no-graph): skip a single graph (io, tput, latency, queue_depth, \n"
1034                 "\t\t\tiops, cpu-sys, cpu-io, cpu-irq cpu-soft cpu-user)\n"
1035                 "\t-O (--only-graph): add a single graph to the output\n"
1036                 "\t-h (--height): set the height of each graph\n"
1037                 "\t-w (--width): set the width of each graph\n"
1038                 "\t-c (--columns): numbers of columns in graph output\n"
1039                );
1040         exit(1);
1041 }
1042
1043 static int parse_options(int ac, char **av)
1044 {
1045         int c;
1046         int disabled = 0;
1047
1048         while (1) {
1049                 // int this_option_optind = optind ? optind : 1;
1050                 int option_index = 0;
1051
1052                 c = getopt_long(ac, av, option_string,
1053                                 long_options, &option_index);
1054
1055                 if (c == -1)
1056                         break;
1057
1058                 switch(c) {
1059                 case 'T':
1060                         graph_title = strdup(optarg);
1061                         break;
1062                 case 't':
1063                         add_trace_file(optarg);
1064                         set_blktrace_outfile(optarg);
1065                         break;
1066                 case 'o':
1067                         output_filename = strdup(optarg);
1068                         break;
1069                 case 'l':
1070                         set_trace_label(optarg);
1071                         break;
1072                 case 'r':
1073                         set_rolling_avg(atoi(optarg));
1074                         break;
1075                 case 'O':
1076                         if (!disabled) {
1077                                 disable_all_graphs();
1078                                 disabled = 1;
1079                         }
1080                         enable_one_graph(optarg);
1081                         break;
1082                 case 'N':
1083                         disable_one_graph(optarg);
1084                         break;
1085                 case 'd':
1086                         blktrace_device = strdup(optarg);
1087                         break;
1088                 case 'p':
1089                         program_to_run = strdup(optarg);
1090                         break;
1091                 case 'm':
1092                         make_movie = 1;
1093                         if (optarg) {
1094                                 movie_style = lookup_movie_style(optarg);
1095                                 if (movie_style < 0) {
1096                                         fprintf(stderr, "Unknown movie style %s\n", optarg);
1097                                         print_usage();
1098                                 }
1099                         }
1100                         fprintf(stderr, "Using movie style: %s\n",
1101                                 movie_styles[movie_style]);
1102                         break;
1103                 case 'h':
1104                         opt_graph_height = atoi(optarg);
1105                         break;
1106                 case 'w':
1107                         opt_graph_width = atoi(optarg);
1108                         break;
1109                 case 'c':
1110                         columns = atoi(optarg);
1111                         break;
1112                 case '?':
1113                 case HELP_LONG_OPT:
1114                         print_usage();
1115                         break;
1116                 default:
1117                         break;
1118                 }
1119         }
1120         return 0;
1121 }
1122
1123
1124 int main(int ac, char **av)
1125 {
1126         struct plot *plot;
1127         int max_seconds = 0;
1128         u64 max_offset = 0;
1129         u64 min_offset = ~(u64)0;
1130         struct trace_file *tf;
1131         int ret;
1132         int rows, cols;
1133
1134         init_io_hash_table();
1135
1136         enable_all_graphs();
1137
1138         parse_options(ac, av);
1139
1140         last_active_graph = last_graph();
1141         if (make_movie) {
1142                 set_io_graph_scale(256);
1143                 if (movie_style == MOVIE_SPINDLE)
1144                         set_graph_size(750, 550);
1145                 else
1146                         set_graph_size(700, 400);
1147
1148                 /*
1149                  * the plots in the movie don't have a seconds
1150                  * line yet, this makes us skip it
1151                  */
1152                 last_active_graph = TOTAL_GRAPHS + 1;
1153         }
1154         if (opt_graph_height)
1155                 set_graph_height(opt_graph_height);
1156
1157         if (opt_graph_width)
1158                 set_graph_width(opt_graph_width);
1159
1160         if (list_empty(&all_traces)) {
1161                 fprintf(stderr, "No traces found, exiting\n");
1162                 exit(1);
1163         }
1164
1165         if (blktrace_device) {
1166                 ret = start_blktrace(blktrace_device, blktrace_outfile,
1167                                      blktrace_dest_dir);
1168                 if (ret) {
1169                         fprintf(stderr, "exiting due to blktrace failure\n");
1170                         exit(1);
1171                 }
1172                 start_mpstat(blktrace_outfile);
1173                 if (program_to_run) {
1174                         ret = run_program(program_to_run);
1175                         if (ret) {
1176                                 fprintf(stderr, "failed to run %s\n",
1177                                         program_to_run);
1178                                 exit(1);
1179                         }
1180                         wait_for_tracers();
1181                         blktrace_to_dump(blktrace_outfile);
1182                 } else {
1183                         /* no program specified, just wait for
1184                          * blktrace to exit
1185                          */
1186                         wait_for_tracers();
1187                 }
1188         }
1189
1190         /* step one, read all the traces */
1191         read_traces();
1192
1193         /* step two, find the maxes for time and offset */
1194         list_for_each_entry(tf, &all_traces, list)
1195                 compare_minmax_tf(tf, &max_seconds, &min_offset, &max_offset);
1196         /* push the max we found into all the tfs */
1197         set_all_minmax_tf(max_seconds, min_offset, max_offset);
1198
1199         /* alloc graphing structs for all the traces */
1200         setup_trace_file_graphs();
1201
1202         /* run through all the traces and read their events */
1203         read_trace_events();
1204
1205         plot = alloc_plot();
1206
1207         if (make_movie) {
1208                 plot_io_movie(plot);
1209                 exit(0);
1210         }
1211
1212         set_plot_output(plot, output_filename);
1213
1214         if (active_graphs[IO_GRAPH_INDEX] || found_mpstat)
1215                 set_legend_width(longest_label + strlen("writes"));
1216         else if (num_traces > 1)
1217                 set_legend_width(longest_label);
1218         else
1219                 set_legend_width(0);
1220
1221         get_graph_size(&cols, &rows);
1222         if (columns > 1)
1223                 plot->add_xlabel = 1;
1224         set_plot_title(plot, graph_title);
1225
1226         plot_io(plot, max_seconds, min_offset, max_offset);
1227         plot->add_xlabel = 0;
1228
1229         if (columns > 1) {
1230                 set_graph_size(cols / columns, rows);
1231                 num_xticks /= columns;
1232                 if (num_xticks < 2)
1233                         num_xticks = 2;
1234         }
1235
1236         check_plot_columns(plot, TPUT_GRAPH_INDEX);
1237         plot_tput(plot, max_seconds);
1238
1239         check_plot_columns(plot, CPU_IO_GRAPH_INDEX);
1240         plot_cpu(plot, max_seconds, "CPU IO Wait Time",
1241                  CPU_IO_GRAPH_INDEX, MPSTAT_IO);
1242
1243         check_plot_columns(plot, CPU_SYS_GRAPH_INDEX);
1244         plot_cpu(plot, max_seconds, "CPU System Time",
1245                  CPU_SYS_GRAPH_INDEX, MPSTAT_SYS);
1246
1247         check_plot_columns(plot, CPU_IRQ_GRAPH_INDEX);
1248         plot_cpu(plot, max_seconds, "CPU IRQ Time",
1249                  CPU_IRQ_GRAPH_INDEX, MPSTAT_IRQ);
1250
1251         check_plot_columns(plot, CPU_SOFT_GRAPH_INDEX);
1252         plot_cpu(plot, max_seconds, "CPU SoftIRQ Time",
1253                  CPU_SOFT_GRAPH_INDEX, MPSTAT_SOFT);
1254
1255         check_plot_columns(plot, CPU_USER_GRAPH_INDEX);
1256         plot_cpu(plot, max_seconds, "CPU User Time",
1257                  CPU_USER_GRAPH_INDEX, MPSTAT_USER);
1258
1259         check_plot_columns(plot, LATENCY_GRAPH_INDEX);
1260         plot_latency(plot, max_seconds);
1261
1262         check_plot_columns(plot, QUEUE_DEPTH_GRAPH_INDEX);
1263         plot_queue_depth(plot, max_seconds);
1264
1265         check_plot_columns(plot, IOPS_GRAPH_INDEX);
1266         plot_iops(plot, max_seconds);
1267
1268         /* once for all */
1269         close_plot(plot);
1270         close_plot_file(plot);
1271         return 0;
1272 }