blktrace.c

   1 /*
   2  * block queue tracing application
   3  *
   4  * Copyright (C) 2005 Jens Axboe <axboe@suse.de>
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 2 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, write to the Free Software
  18  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  *
  20  */
  21 #include <pthread.h>
  22 #include <sys/types.h>
  23 #include <sys/stat.h>
  24 #include <unistd.h>
  25 #include <locale.h>
  26 #include <signal.h>
  27 #include <fcntl.h>
  28 #include <string.h>
  29 #include <sys/ioctl.h>
  30 #include <sys/param.h>
  31 #include <sys/statfs.h>
  32 #include <sys/poll.h>
  33 #include <stdio.h>
  34 #include <stdlib.h>
  35 #include <sched.h>
  36 #include <ctype.h>
  37 #include <getopt.h>
  38 #include <errno.h>
  39 #include <assert.h>
  40
  41 #include "blktrace.h"
  42
  43 static char blktrace_version[] = "0.99";
  44
  45 /*
  46  * You may want to increase this even more, if you are logging at a high
  47  * rate and see skipped/missed events
  48  */
  49 #define BUF_SIZE        (512 * 1024)
  50 #define BUF_NR          (4)
  51
  52 #define OFILE_BUF       (128 * 1024)
  53
  54 #define RELAYFS_TYPE    0xF0B4A981
  55
  56 #define RING_INIT_NR    (2)
  57 #define RING_MAX_NR     (16UL)
  58
  59 #define S_OPTS  "d:a:A:r:o:kw:Vb:n:D:"
  60 static struct option l_opts[] = {
  61         {
  62                 .name = "dev",
  63                 .has_arg = required_argument,
  64                 .flag = NULL,
  65                 .val = 'd'
  66         },
  67         {
  68                 .name = "act-mask",
  69                 .has_arg = required_argument,
  70                 .flag = NULL,
  71                 .val = 'a'
  72         },
  73         {
  74                 .name = "set-mask",
  75                 .has_arg = required_argument,
  76                 .flag = NULL,
  77                 .val = 'A'
  78         },
  79         {
  80                 .name = "relay",
  81                 .has_arg = required_argument,
  82                 .flag = NULL,
  83                 .val = 'r'
  84         },
  85         {
  86                 .name = "output",
  87                 .has_arg = required_argument,
  88                 .flag = NULL,
  89                 .val = 'o'
  90         },
  91         {
  92                 .name = "kill",
  93                 .has_arg = no_argument,
  94                 .flag = NULL,
  95                 .val = 'k'
  96         },
  97         {
  98                 .name = "stopwatch",
  99                 .has_arg = required_argument,
 100                 .flag = NULL,
 101                 .val = 'w'
 102         },
 103         {
 104                 .name = "version",
 105                 .has_arg = no_argument,
 106                 .flag = NULL,
 107                 .val = 'V'
 108         },
 109         {
 110                 .name = "buffer-size",
 111                 .has_arg = required_argument,
 112                 .flag = NULL,
 113                 .val = 'b'
 114         },
 115         {
 116                 .name = "num-sub-buffers",
 117                 .has_arg = required_argument,
 118                 .flag = NULL,
 119                 .val = 'n'
 120         },
 121         {
 122                 .name = "output-dir",
 123                 .has_arg = required_argument,
 124                 .flag = NULL,
 125                 .val = 'D'
 126         },
 127         {
 128                 .name = NULL,
 129         }
 130 };
 131
 132 struct thread_information {
 133         int cpu;
 134         pthread_t thread;
 135
 136         int fd;
 137         void *fd_buf;
 138         unsigned long fd_off;
 139         unsigned long fd_size;
 140         unsigned long fd_max_size;
 141         char fn[MAXPATHLEN + 64];
 142
 143         pthread_mutex_t *fd_lock;
 144         FILE *ofile;
 145         char *ofile_buffer;
 146
 147         unsigned long events_processed;
 148         struct device_information *device;
 149 };
 150
 151 struct device_information {
 152         int fd;
 153         char *path;
 154         char buts_name[32];
 155         volatile int trace_started;
 156         unsigned long drop_count;
 157         struct thread_information *threads;
 158 };
 159
 160 static int ncpus;
 161 static struct thread_information *thread_information;
 162 static int ndevs;
 163 static struct device_information *device_information;
 164
 165 /* command line option globals */
 166 static char *relay_path;
 167 static char *output_name;
 168 static char *output_dir;
 169 static int act_mask = ~0U;
 170 static int kill_running_trace;
 171 static unsigned long buf_size = BUF_SIZE;
 172 static unsigned long buf_nr = BUF_NR;
 173
 174 #define is_done()       (*(volatile int *)(&done))
 175 static volatile int done;
 176
 177 #define is_trace_stopped()      (*(volatile int *)(&trace_stopped))
 178 static volatile int trace_stopped;
 179
 180 #define is_stat_shown() (*(volatile int *)(&stat_shown))
 181 static volatile int stat_shown;
 182
 183 static pthread_mutex_t stdout_mutex = PTHREAD_MUTEX_INITIALIZER;
 184
 185 static void exit_trace(int status);
 186
 187 #define dip_tracing(dip)        (*(volatile int *)(&(dip)->trace_started))
 188 #define dip_set_tracing(dip, v) ((dip)->trace_started = (v))
 189
 190 #define __for_each_dip(__d, __i, __e)   \
 191         for (__i = 0, __d = device_information; __i < __e; __i++, __d++)
 192
 193 #define for_each_dip(__d, __i)  __for_each_dip(__d, __i, ndevs)
 194 #define for_each_tip(__d, __t, __i)     \
 195         for (__i = 0, __t = (__d)->threads; __i < ncpus; __i++, __t++)
 196
 197 static int get_dropped_count(const char *buts_name)
 198 {
 199         int fd;
 200         char tmp[MAXPATHLEN + 64];
 201
 202         snprintf(tmp, sizeof(tmp), "%s/block/%s/dropped",
 203                  relay_path, buts_name);
 204
 205         fd = open(tmp, O_RDONLY);
 206         if (fd < 0) {
 207                 /*
 208                  * this may be ok, if the kernel doesn't support dropped counts
 209                  */
 210                 if (errno == ENOENT)
 211                         return 0;
 212
 213                 fprintf(stderr, "Couldn't open dropped file %s\n", tmp);
 214                 return -1;
 215         }
 216
 217         if (read(fd, tmp, sizeof(tmp)) < 0) {
 218                 perror(tmp);
 219                 close(fd);
 220                 return -1;
 221         }
 222
 223         close(fd);
 224
 225         return atoi(tmp);
 226 }
 227
 228 static int start_trace(struct device_information *dip)
 229 {
 230         struct blk_user_trace_setup buts;
 231
 232         memset(&buts, 0, sizeof(buts));
 233         buts.buf_size = buf_size;
 234         buts.buf_nr = buf_nr;
 235         buts.act_mask = act_mask;
 236
 237         if (ioctl(dip->fd, BLKSTARTTRACE, &buts) < 0) {
 238                 perror("BLKSTARTTRACE");
 239                 return 1;
 240         }
 241
 242         memcpy(dip->buts_name, buts.name, sizeof(dip->buts_name));
 243         dip_set_tracing(dip, 1);
 244         return 0;
 245 }
 246
 247 static void stop_trace(struct device_information *dip)
 248 {
 249         if (dip_tracing(dip) || kill_running_trace) {
 250                 dip_set_tracing(dip, 0);
 251
 252                 if (ioctl(dip->fd, BLKSTOPTRACE) < 0)
 253                         perror("BLKSTOPTRACE");
 254
 255                 close(dip->fd);
 256                 dip->fd = -1;
 257         }
 258 }
 259
 260 static void stop_all_traces(void)
 261 {
 262         struct device_information *dip;
 263         int i;
 264
 265         for_each_dip(dip, i) {
 266                 dip->drop_count = get_dropped_count(dip->buts_name);
 267                 stop_trace(dip);
 268         }
 269 }
 270
 271 static void wait_for_data(struct thread_information *tip)
 272 {
 273         struct pollfd pfd = { .fd = tip->fd, .events = POLLIN };
 274
 275         while (!is_done()) {
 276                 poll(&pfd, 1, 10);
 277                 if (pfd.revents & POLLIN)
 278                         break;
 279         }
 280 }
 281
 282 static int __read_data(struct thread_information *tip, void *buf, int len,
 283                        int block)
 284 {
 285         int ret = 0, waited = 0;
 286
 287         while (!is_done() || waited) {
 288                 ret = read(tip->fd, buf, len);
 289                 waited = 0;
 290                 fprintf(stderr, "got %d, block %d\n", ret, block);
 291                 if (ret > 0)
 292                         break;
 293                 else if (!ret) {
 294                         if (!block)
 295                                 break;
 296                         /*
 297                          * the waited logic is needed, because the relayfs
 298                          * poll works on a sub-buffer granularity
 299                          */
 300                         wait_for_data(tip);
 301                         waited = 1;
 302                 } else {
 303                         if (errno != EAGAIN) {
 304                                 perror(tip->fn);
 305                                 fprintf(stderr,"Thread %d failed read of %s\n",
 306                                         tip->cpu, tip->fn);
 307                                 break;
 308                         }
 309                         if (!block) {
 310                                 ret = 0;
 311                                 break;
 312                         }
 313
 314                         wait_for_data(tip);
 315                         waited = 0;
 316                 }
 317         }
 318
 319         return ret;
 320 }
 321
 322 #define can_grow_ring(tip)      ((tip)->fd_max_size < RING_MAX_NR * buf_size * buf_nr)
 323
 324 static int resize_ringbuffer(struct thread_information *tip)
 325 {
 326         if (!can_grow_ring(tip))
 327                 return 1;
 328
 329         tip->fd_buf = realloc(tip->fd_buf, 2 * tip->fd_max_size);
 330
 331         /*
 332          * if the ring currently wraps, copy range over
 333          */
 334         if (tip->fd_off + tip->fd_size > tip->fd_max_size) {
 335                 unsigned long wrap_size = tip->fd_size - (tip->fd_max_size - tip->fd_off);
 336                 memmove(tip->fd_buf + tip->fd_off, tip->fd_buf, wrap_size);
 337         }
 338
 339         tip->fd_max_size <<= 1;
 340         return 0;
 341 }
 342
 343 static int __refill_ringbuffer(struct thread_information *tip, int len,
 344                                int block)
 345 {
 346         unsigned long off;
 347         int ret;
 348
 349         off = (tip->fd_size + tip->fd_off) & (tip->fd_max_size - 1);
 350         if (off + len > tip->fd_max_size)
 351                 len = tip->fd_max_size - off;
 352
 353         assert(len > 0);
 354
 355         ret = __read_data(tip, tip->fd_buf + off, len, block);
 356         if (ret < 0)
 357                 return -1;
 358
 359         tip->fd_size += ret;
 360         return ret;
 361 }
 362
 363 /*
 364  * keep filling ring until we get a short read
 365  */
 366 static void refill_ringbuffer(struct thread_information *tip, int block)
 367 {
 368         int len = buf_size;
 369         int ret;
 370
 371         do {
 372                 if (len + tip->fd_size > tip->fd_max_size)
 373                         resize_ringbuffer(tip);
 374
 375                 ret = __refill_ringbuffer(tip, len, block);
 376         } while ((ret = len) && !is_done());
 377 }
 378
 379 static int read_data(struct thread_information *tip, void *buf,
 380                      unsigned int len)
 381 {
 382         unsigned int start_size, end_size;
 383
 384         refill_ringbuffer(tip, len > tip->fd_size);
 385
 386         if (len > tip->fd_size)
 387                 return -1;
 388
 389         /*
 390          * see if we wrap the ring
 391          */
 392         start_size = len;
 393         end_size = 0;
 394         if (len > (tip->fd_max_size - tip->fd_off)) {
 395                 start_size = tip->fd_max_size - tip->fd_off;
 396                 end_size = len - start_size;
 397         }
 398
 399         memcpy(buf, tip->fd_buf + tip->fd_off, start_size);
 400         if (end_size)
 401                 memcpy(buf + start_size, tip->fd_buf, end_size);
 402
 403         tip->fd_off = (tip->fd_off + len) & (tip->fd_max_size - 1);
 404         tip->fd_size -= len;
 405         return 0;
 406 }
 407
 408 static int write_data(FILE *file, void *buf, unsigned int buf_len)
 409 {
 410         int ret, bytes_left;
 411         char *p = buf;
 412
 413         bytes_left = buf_len;
 414         while (bytes_left > 0) {
 415                 ret = fwrite(p, bytes_left, 1, file);
 416                 if (ret == 1)
 417                         break;
 418
 419                 if (ret < 0) {
 420                         perror("write");
 421                         return 1;
 422                 }
 423         }
 424
 425         return 0;
 426 }
 427
 428 static void *extract_data(struct thread_information *tip, int nb)
 429 {
 430         unsigned char *buf;
 431
 432         buf = malloc(nb);
 433         if (!read_data(tip, buf, nb))
 434                 return buf;
 435
 436         free(buf);
 437         return NULL;
 438 }
 439
 440 /*
 441  * trace may start inside 'bit' or may need to be gotten further on
 442  */
 443 static int get_event_slow(struct thread_information *tip,
 444                           struct blk_io_trace *bit)
 445 {
 446         const int inc = sizeof(__u32);
 447         struct blk_io_trace foo;
 448         unsigned int offset;
 449         void *p;
 450
 451         /*
 452          * check if trace is inside
 453          */
 454         offset = 0;
 455         p = bit;
 456         while (offset < sizeof(*bit)) {
 457                 p += inc;
 458                 offset += inc;
 459
 460                 memcpy(&foo, p, inc);
 461
 462                 if (CHECK_MAGIC(&foo))
 463                         break;
 464         }
 465
 466         /*
 467          * part trace found inside, read the rest
 468          */
 469         if (offset < sizeof(*bit)) {
 470                 int good_bytes = sizeof(*bit) - offset;
 471
 472                 memmove(bit, p, good_bytes);
 473                 p = (void *) bit + good_bytes;
 474
 475                 return read_data(tip, p, offset);
 476         }
 477
 478         /*
 479          * nothing found, keep looking for start of trace
 480          */
 481         do {
 482                 if (read_data(tip, bit, sizeof(bit->magic)))
 483                         return -1;
 484         } while (!CHECK_MAGIC(bit));
 485
 486         /*
 487          * now get the rest of it
 488          */
 489         p = &bit->sequence;
 490         if (read_data(tip, p, sizeof(*bit) - inc))
 491                 return -1;
 492
 493         return 0;
 494 }
 495
 496 /*
 497  * Sometimes relayfs screws us a little, if an event crosses a sub buffer
 498  * boundary. So keep looking forward in the trace data until an event
 499  * is found
 500  */
 501 static int get_event(struct thread_information *tip, struct blk_io_trace *bit)
 502 {
 503         /*
 504          * optimize for the common fast case, a full trace read that
 505          * succeeds
 506          */
 507         if (read_data(tip, bit, sizeof(*bit)))
 508                 return -1;
 509
 510         if (CHECK_MAGIC(bit))
 511                 return 0;
 512
 513         /*
 514          * ok that didn't work, the event may start somewhere inside the
 515          * trace itself
 516          */
 517         return get_event_slow(tip, bit);
 518 }
 519
 520 static inline void tip_fd_unlock(struct thread_information *tip)
 521 {
 522         if (tip->fd_lock)
 523                 pthread_mutex_unlock(tip->fd_lock);
 524 }
 525
 526 static inline void tip_fd_lock(struct thread_information *tip)
 527 {
 528         if (tip->fd_lock)
 529                 pthread_mutex_lock(tip->fd_lock);
 530 }
 531
 532 static void close_thread(struct thread_information *tip)
 533 {
 534         if (tip->fd != -1)
 535                 close(tip->fd);
 536         if (tip->ofile)
 537                 fclose(tip->ofile);
 538         if (tip->ofile_buffer)
 539                 free(tip->ofile_buffer);
 540         if (tip->fd_buf)
 541                 free(tip->fd_buf);
 542
 543         tip->fd = -1;
 544         tip->ofile = NULL;
 545         tip->ofile_buffer = NULL;
 546         tip->fd_buf = NULL;
 547 }
 548
 549 static void *extract(void *arg)
 550 {
 551         struct thread_information *tip = arg;
 552         int pdu_len;
 553         char *pdu_data;
 554         struct blk_io_trace t;
 555         pid_t pid = getpid();
 556         cpu_set_t cpu_mask;
 557
 558         CPU_ZERO(&cpu_mask);
 559         CPU_SET((tip->cpu), &cpu_mask);
 560
 561         if (sched_setaffinity(pid, sizeof(cpu_mask), &cpu_mask) == -1) {
 562                 perror("sched_setaffinity");
 563                 exit_trace(1);
 564         }
 565
 566         snprintf(tip->fn, sizeof(tip->fn), "%s/block/%s/trace%d",
 567                         relay_path, tip->device->buts_name, tip->cpu);
 568         tip->fd = open(tip->fn, O_RDONLY | O_NONBLOCK);
 569         if (tip->fd < 0) {
 570                 perror(tip->fn);
 571                 fprintf(stderr,"Thread %d failed open of %s\n", tip->cpu,
 572                         tip->fn);
 573                 exit_trace(1);
 574         }
 575
 576         /*
 577          * start with a ringbuffer that is twice the size of the kernel side
 578          */
 579         tip->fd_max_size = buf_size * buf_nr * RING_INIT_NR;
 580         tip->fd_buf = malloc(tip->fd_max_size);
 581         tip->fd_off = 0;
 582         tip->fd_size = 0;
 583
 584         pdu_data = NULL;
 585         while (1) {
 586                 if (get_event(tip, &t))
 587                         break;
 588
 589                 if (verify_trace(&t))
 590                         break;
 591
 592                 pdu_len = t.pdu_len;
 593
 594                 trace_to_be(&t);
 595
 596                 if (pdu_len) {
 597                         pdu_data = extract_data(tip, pdu_len);
 598                         if (!pdu_data)
 599                                 break;
 600                 }
 601
 602                 /*
 603                  * now we have both trace and payload, get a lock on the
 604                  * output descriptor and send it off
 605                  */
 606                 tip_fd_lock(tip);
 607
 608                 if (write_data(tip->ofile, &t, sizeof(t))) {
 609                         tip_fd_unlock(tip);
 610                         break;
 611                 }
 612
 613                 if (pdu_data && write_data(tip->ofile, pdu_data, pdu_len)) {
 614                         tip_fd_unlock(tip);
 615                         break;
 616                 }
 617
 618                 tip_fd_unlock(tip);
 619
 620                 if (pdu_data) {
 621                         free(pdu_data);
 622                         pdu_data = NULL;
 623                 }
 624
 625                 tip->events_processed++;
 626         }
 627
 628         close_thread(tip);
 629         return NULL;
 630 }
 631
 632 static int start_threads(struct device_information *dip)
 633 {
 634         struct thread_information *tip;
 635         char op[64];
 636         int j, pipeline = output_name && !strcmp(output_name, "-");
 637         int len, mode;
 638
 639         for_each_tip(dip, tip, j) {
 640                 tip->cpu = j;
 641                 tip->device = dip;
 642                 tip->fd_lock = NULL;
 643                 tip->events_processed = 0;
 644
 645                 if (pipeline) {
 646                         tip->ofile = fdopen(STDOUT_FILENO, "w");
 647                         tip->fd_lock = &stdout_mutex;
 648                         mode = _IOLBF;
 649                         buf_size = 512;
 650                 } else {
 651                         len = 0;
 652
 653                         if (output_dir)
 654                                 len = sprintf(op, "%s/", output_dir);
 655
 656                         if (output_name) {
 657                                 sprintf(op + len, "%s.blktrace.%d", output_name,
 658                                         tip->cpu);
 659                         } else {
 660                                 sprintf(op + len, "%s.blktrace.%d",
 661                                         dip->buts_name, tip->cpu);
 662                         }
 663                         tip->ofile = fopen(op, "w");
 664                         mode = _IOFBF;
 665                         buf_size = OFILE_BUF;
 666                 }
 667
 668                 if (tip->ofile == NULL) {
 669                         perror(op);
 670                         return 1;
 671                 }
 672
 673                 tip->ofile_buffer = malloc(buf_size);
 674                 if (setvbuf(tip->ofile, tip->ofile_buffer, mode, buf_size)) {
 675                         perror("setvbuf");
 676                         close_thread(tip);
 677                         return 1;
 678                 }
 679
 680                 if (pthread_create(&tip->thread, NULL, extract, tip)) {
 681                         perror("pthread_create");
 682                         close_thread(tip);
 683                         return 1;
 684                 }
 685         }
 686
 687         return 0;
 688 }
 689
 690 static void stop_threads(struct device_information *dip)
 691 {
 692         struct thread_information *tip;
 693         unsigned long ret;
 694         int i;
 695
 696         for_each_tip(dip, tip, i)
 697                 (void) pthread_join(tip->thread, (void *) &ret);
 698 }
 699
 700 static void stop_all_threads(void)
 701 {
 702         struct device_information *dip;
 703         int i;
 704
 705         for_each_dip(dip, i)
 706                 stop_threads(dip);
 707 }
 708
 709 static void stop_all_tracing(void)
 710 {
 711         struct device_information *dip;
 712         int i;
 713
 714         for_each_dip(dip, i)
 715                 stop_trace(dip);
 716 }
 717
 718 static void exit_trace(int status)
 719 {
 720         if (!is_trace_stopped()) {
 721                 trace_stopped = 1;
 722                 stop_all_threads();
 723                 stop_all_tracing();
 724         }
 725
 726         exit(status);
 727 }
 728
 729 static int resize_devices(char *path)
 730 {
 731         int size = (ndevs + 1) * sizeof(struct device_information);
 732
 733         device_information = realloc(device_information, size);
 734         if (!device_information) {
 735                 fprintf(stderr, "Out of memory, device %s (%d)\n", path, size);
 736                 return 1;
 737         }
 738         device_information[ndevs].path = path;
 739         ndevs++;
 740         return 0;
 741 }
 742
 743 static int open_devices(void)
 744 {
 745         struct device_information *dip;
 746         int i;
 747
 748         for_each_dip(dip, i) {
 749                 dip->fd = open(dip->path, O_RDONLY | O_NONBLOCK);
 750                 if (dip->fd < 0) {
 751                         perror(dip->path);
 752                         return 1;
 753                 }
 754         }
 755
 756         return 0;
 757 }
 758
 759 static int start_devices(void)
 760 {
 761         struct device_information *dip;
 762         int i, j, size;
 763
 764         size = ncpus * sizeof(struct thread_information);
 765         thread_information = malloc(size * ndevs);
 766         if (!thread_information) {
 767                 fprintf(stderr, "Out of memory, threads (%d)\n", size * ndevs);
 768                 return 1;
 769         }
 770
 771         for_each_dip(dip, i) {
 772                 if (start_trace(dip)) {
 773                         close(dip->fd);
 774                         fprintf(stderr, "Failed to start trace on %s\n",
 775                                 dip->path);
 776                         break;
 777                 }
 778         }
 779
 780         if (i != ndevs) {
 781                 __for_each_dip(dip, j, i)
 782                         stop_trace(dip);
 783
 784                 return 1;
 785         }
 786
 787         for_each_dip(dip, i) {
 788                 dip->threads = thread_information + (i * ncpus);
 789                 if (start_threads(dip)) {
 790                         fprintf(stderr, "Failed to start worker threads\n");
 791                         break;
 792                 }
 793         }
 794
 795         if (i != ndevs) {
 796                 __for_each_dip(dip, j, i)
 797                         stop_threads(dip);
 798                 for_each_dip(dip, i)
 799                         stop_trace(dip);
 800
 801                 return 1;
 802         }
 803
 804         return 0;
 805 }
 806
 807 static void show_stats(void)
 808 {
 809         int i, j, no_stdout = 0;
 810         struct device_information *dip;
 811         struct thread_information *tip;
 812         unsigned long long events_processed;
 813         unsigned long total_drops;
 814
 815         if (is_stat_shown())
 816                 return;
 817
 818         stat_shown = 1;
 819
 820         if (output_name && !strcmp(output_name, "-"))
 821                 no_stdout = 1;
 822
 823         total_drops = 0;
 824         for_each_dip(dip, i) {
 825                 if (!no_stdout)
 826                         printf("Device: %s\n", dip->path);
 827                 events_processed = 0;
 828                 for_each_tip(dip, tip, j) {
 829                         if (!no_stdout)
 830                                 printf("  CPU%3d: %20ld events\n",
 831                                         tip->cpu, tip->events_processed);
 832                         events_processed += tip->events_processed;
 833                 }
 834                 total_drops += dip->drop_count;
 835                 if (!no_stdout)
 836                         printf("  Total:  %20lld events (dropped %lu)\n",
 837                                         events_processed, dip->drop_count);
 838         }
 839
 840         if (total_drops)
 841                 fprintf(stderr, "You have dropped events, consider using a larger buffer size (-b)\n");
 842 }
 843
 844 static char usage_str[] = \
 845         "-d <dev> [ -r relay path ] [ -o <output> ] [-k ] [ -w time ]\n" \
 846         "[ -a action ] [ -A action mask ] [ -v ]\n\n" \
 847         "\t-d Use specified device. May also be given last after options\n" \
 848         "\t-r Path to mounted relayfs, defaults to /relay\n" \
 849         "\t-o File(s) to send output to\n" \
 850         "\t-D Directory to prepend to output file names\n" \
 851         "\t-k Kill a running trace\n" \
 852         "\t-w Stop after defined time, in seconds\n" \
 853         "\t-a Only trace specified actions. See documentation\n" \
 854         "\t-A Give trace mask as a single value. See documentation\n" \
 855         "\t-b Sub buffer size in KiB\n" \
 856         "\t-n Number of sub buffers\n" \
 857         "\t-v Print program version info\n\n";
 858
 859 static void show_usage(char *program)
 860 {
 861         fprintf(stderr, "Usage: %s %s %s",program, blktrace_version, usage_str);
 862 }
 863 static void handle_sigint(__attribute__((__unused__)) int sig)
 864 {
 865         done = 1;
 866         if (!is_trace_stopped()) {
 867                 trace_stopped = 1;
 868                 stop_all_threads();
 869                 stop_all_traces();
 870         }
 871
 872         show_stats();
 873 }
 874
 875 int main(int argc, char *argv[])
 876 {
 877         static char default_relay_path[] = "/relay";
 878         struct statfs st;
 879         int i, c;
 880         int stop_watch = 0;
 881         int act_mask_tmp = 0;
 882
 883         while ((c = getopt_long(argc, argv, S_OPTS, l_opts, NULL)) >= 0) {
 884                 switch (c) {
 885                 case 'a':
 886                         i = find_mask_map(optarg);
 887                         if (i < 0) {
 888                                 fprintf(stderr,"Invalid action mask %s\n",
 889                                         optarg);
 890                                 return 1;
 891                         }
 892                         act_mask_tmp |= i;
 893                         break;
 894
 895                 case 'A':
 896                         if ((sscanf(optarg, "%x", &i) != 1) ||
 897                                                         !valid_act_opt(i)) {
 898                                 fprintf(stderr,
 899                                         "Invalid set action mask %s/0x%x\n",
 900                                         optarg, i);
 901                                 return 1;
 902                         }
 903                         act_mask_tmp = i;
 904                         break;
 905
 906                 case 'd':
 907                         if (resize_devices(optarg) != 0)
 908                                 return 1;
 909                         break;
 910
 911                 case 'r':
 912                         relay_path = optarg;
 913                         break;
 914
 915                 case 'o':
 916                         output_name = optarg;
 917                         break;
 918                 case 'k':
 919                         kill_running_trace = 1;
 920                         break;
 921                 case 'w':
 922                         stop_watch = atoi(optarg);
 923                         if (stop_watch <= 0) {
 924                                 fprintf(stderr,
 925                                         "Invalid stopwatch value (%d secs)\n",
 926                                         stop_watch);
 927                                 return 1;
 928                         }
 929                         break;
 930                 case 'V':
 931                         printf("%s version %s\n", argv[0], blktrace_version);
 932                         return 0;
 933                 case 'b':
 934                         buf_size = strtoul(optarg, NULL, 10);
 935                         if (buf_size <= 0 || buf_size > 16*1024) {
 936                                 fprintf(stderr,
 937                                         "Invalid buffer size (%lu)\n",buf_size);
 938                                 return 1;
 939                         }
 940                         buf_size <<= 10;
 941                         break;
 942                 case 'n':
 943                         buf_nr = strtoul(optarg, NULL, 10);
 944                         if (buf_nr <= 0) {
 945                                 fprintf(stderr,
 946                                         "Invalid buffer nr (%lu)\n", buf_nr);
 947                                 return 1;
 948                         }
 949                         break;
 950                 case 'D':
 951                         output_dir = optarg;
 952                         break;
 953                 default:
 954                         show_usage(argv[0]);
 955                         return 1;
 956                 }
 957         }
 958
 959         while (optind < argc) {
 960                 if (resize_devices(argv[optind++]) != 0)
 961                         return 1;
 962         }
 963
 964         if (ndevs == 0) {
 965                 show_usage(argv[0]);
 966                 return 1;
 967         }
 968
 969         if (!relay_path)
 970                 relay_path = default_relay_path;
 971
 972         if (act_mask_tmp != 0)
 973                 act_mask = act_mask_tmp;
 974
 975         if (statfs(relay_path, &st) < 0) {
 976                 perror("statfs");
 977                 fprintf(stderr,"%s does not appear to be a valid path\n",
 978                         relay_path);
 979                 return 1;
 980         } else if (st.f_type != (long) RELAYFS_TYPE) {
 981                 fprintf(stderr,"%s does not appear to be a relay filesystem\n",
 982                         relay_path);
 983                 return 1;
 984         }
 985
 986         if (open_devices() != 0)
 987                 return 1;
 988
 989         if (kill_running_trace) {
 990                 stop_all_traces();
 991                 return 0;
 992         }
 993
 994         setlocale(LC_NUMERIC, "en_US");
 995
 996         ncpus = sysconf(_SC_NPROCESSORS_ONLN);
 997         if (ncpus < 0) {
 998                 fprintf(stderr, "sysconf(_SC_NPROCESSORS_ONLN) failed\n");
 999                 return 1;
1000         }
1001
1002         if (start_devices() != 0)
1003                 return 1;
1004
1005         signal(SIGINT, handle_sigint);
1006         signal(SIGHUP, handle_sigint);
1007         signal(SIGTERM, handle_sigint);
1008         signal(SIGALRM, handle_sigint);
1009
1010         atexit(stop_all_tracing);
1011
1012         if (stop_watch)
1013                 alarm(stop_watch);
1014
1015         while (!is_done())
1016                 sleep(1);
1017
1018         if (!is_trace_stopped()) {
1019                 trace_stopped = 1;
1020                 stop_all_threads();
1021                 stop_all_traces();
1022         }
1023
1024         show_stats();
1025
1026         return 0;
1027 }
1028