[PATCH] blkparse: another stab at stopwatch_end fixing
[blktrace.git] / blkparse.c
index 3ebbcdb7f34788c5296ccd530786aaacfc31a73d..c9b37593d161ab28d2bf33c1b418c3c762e162bc 100644 (file)
 #include <signal.h>
 #include <locale.h>
 #include <limits.h>
-#include <ctype.h>
 
 #include "blktrace.h"
 #include "rbtree.h"
+#include "jhash.h"
 
 static char blkparse_version[] = "0.90";
 
-#define SECONDS(x)             ((unsigned long long)(x) / 1000000000)
-#define NANO_SECONDS(x)                ((unsigned long long)(x) % 1000000000)
-#define DOUBLE_TO_NANO_ULL(d)  ((unsigned long long)((d) * 1000000000))
-
-#define MINORBITS      20
-#define MINORMASK      ((1U << MINORBITS) - 1)
-#define MAJOR(dev)     ((unsigned int) ((dev) >> MINORBITS))
-#define MINOR(dev)     ((unsigned int) ((dev) & MINORMASK))
-
-#define min(a, b)      ((a) < (b) ? (a) : (b))
-
-#define HEADER         "%D %2c %8s %5T.%9t %5p %2a %3d "
-
-struct io_stats {
-       unsigned long qreads, qwrites, creads, cwrites, mreads, mwrites;
-       unsigned long ireads, iwrites;
-       unsigned long long qread_kb, qwrite_kb, cread_kb, cwrite_kb;
-       unsigned long long iread_kb, iwrite_kb;
-       unsigned long io_unplugs, timer_unplugs;
-};
-
-struct per_cpu_info {
-       int cpu;
-       int nelems;
-
-       int fd;
-       char fname[128];
-
-       struct io_stats io_stats;
-};
-
 struct per_dev_info {
        dev_t id;
        char *name;
@@ -75,6 +44,7 @@ struct per_dev_info {
        int backwards;
        unsigned long long events;
        unsigned long long last_reported_time;
+       unsigned long long last_read_time;
        struct io_stats io_stats;
        unsigned long last_sequence;
        unsigned long skips;
@@ -89,6 +59,7 @@ struct per_process_info {
        __u32 pid;
        struct io_stats io_stats;
        struct per_process_info *hash_next, *list_next;
+       int more_than_one;
 
        /*
         * individual io stats
@@ -99,11 +70,13 @@ struct per_process_info {
 };
 
 #define PPI_HASH_SHIFT (8)
-static struct per_process_info *ppi_hash[1 << PPI_HASH_SHIFT];
+#define PPI_HASH_SIZE  (1 << PPI_HASH_SHIFT)
+#define PPI_HASH_MASK  (PPI_HASH_SIZE - 1)
+static struct per_process_info *ppi_hash_table[PPI_HASH_SIZE];
 static struct per_process_info *ppi_list;
 static int ppi_list_entries;
 
-#define S_OPTS "i:o:b:stqw:f:F:v"
+#define S_OPTS "i:o:b:stqw:f:F:vn"
 static struct option l_opts[] = {
        {
                .name = "input",
@@ -159,6 +132,12 @@ static struct option l_opts[] = {
                .flag = NULL,
                .val = 'F'
        },
+       {
+               .name = "hash by name",
+               .has_arg = no_argument,
+               .flag = NULL,
+               .val = 'n'
+       },
        {
                .name = "version",
                .has_arg = no_argument,
@@ -178,6 +157,8 @@ struct trace {
 };
 
 static struct rb_root rb_sort_root;
+static unsigned long rb_sort_entries;
+
 static struct rb_root rb_track_root;
 
 static struct trace *trace_list;
@@ -197,6 +178,7 @@ struct io_track {
        dev_t device;
        __u64 sector;
        __u32 pid;
+       char comm[16];
        unsigned long long allocation_time;
        unsigned long long queue_time;
        unsigned long long dispatch_time;
@@ -207,15 +189,17 @@ static int ndevices;
 static struct per_dev_info *devices;
 static char *get_dev_name(struct per_dev_info *, char *, int);
 
-static FILE *ofp;
+FILE *ofp = NULL;
 static char *output_name;
 
 static unsigned long long genesis_time;
+static unsigned long long last_allowed_time;
 static unsigned long long stopwatch_start;     /* start from zero by default */
 static unsigned long long stopwatch_end = ULONG_LONG_MAX;      /* "infinity" */
 
 static int per_process_stats;
 static int track_ios;
+static int ppi_hash_by_pid = 1;
 
 #define RB_BATCH_DEFAULT       (512)
 static int rb_batch = RB_BATCH_DEFAULT;
@@ -225,25 +209,32 @@ static int pipeline;
 #define is_done()      (*(volatile int *)(&done))
 static volatile int done;
 
-static inline unsigned long hash_long(unsigned long val)
+#define JHASH_RANDOM   (0x3af5f2ee)
+
+static inline int ppi_hash_pid(__u32 pid)
 {
-#if __WORDSIZE == 32
-       val *= 0x9e370001UL;
-#elif __WORDSIZE == 64
-       val *= 0x9e37fffffffc0001UL;
-#else
-#error unknown word size
-#endif
+       return jhash_1word(pid, JHASH_RANDOM) & PPI_HASH_MASK;
+}
 
-       return val >> (__WORDSIZE - PPI_HASH_SHIFT);
+static inline int ppi_hash_name(const char *name)
+{
+       return jhash(name, 16, JHASH_RANDOM) & PPI_HASH_MASK;
+}
+
+static inline int ppi_hash(struct per_process_info *ppi)
+{
+       if (ppi_hash_by_pid)
+               return ppi_hash_pid(ppi->pid);
+
+       return ppi_hash_name(ppi->name);
 }
 
 static inline void add_process_to_hash(struct per_process_info *ppi)
 {
-       const int hash_idx = hash_long(ppi->pid);
+       const int hash_idx = ppi_hash(ppi);
 
-       ppi->hash_next = ppi_hash[hash_idx];
-       ppi_hash[hash_idx] = ppi;
+       ppi->hash_next = ppi_hash_table[hash_idx];
+       ppi_hash_table[hash_idx] = ppi;
 }
 
 static inline void add_process_to_list(struct per_process_info *ppi)
@@ -253,12 +244,28 @@ static inline void add_process_to_list(struct per_process_info *ppi)
        ppi_list_entries++;
 }
 
+static struct per_process_info *find_process_by_name(char *name)
+{
+       const int hash_idx = ppi_hash_name(name);
+       struct per_process_info *ppi;
+
+       ppi = ppi_hash_table[hash_idx];
+       while (ppi) {
+               if (!strcmp(ppi->name, name))
+                       return ppi;
+
+               ppi = ppi->hash_next;
+       }
+
+       return NULL;
+}
+
 static struct per_process_info *find_process_by_pid(__u32 pid)
 {
-       const int hash_idx = hash_long(pid);
+       const int hash_idx = ppi_hash_pid(pid);
        struct per_process_info *ppi;
 
-       ppi = ppi_hash[hash_idx];
+       ppi = ppi_hash_table[hash_idx];
        while (ppi) {
                if (ppi->pid == pid)
                        return ppi;
@@ -269,15 +276,26 @@ static struct per_process_info *find_process_by_pid(__u32 pid)
        return NULL;
 }
 
+static struct per_process_info *find_process(__u32 pid, char *name)
+{
+       struct per_process_info *ppi;
+
+       if (ppi_hash_by_pid)
+               return find_process_by_pid(pid);
+
+       ppi = find_process_by_name(name);
+       if (ppi && ppi->pid != pid)
+               ppi->more_than_one = 1;
+
+       return ppi;
+}
+
 static inline int trace_rb_insert(struct trace *t)
 {
        struct rb_node **p = &rb_sort_root.rb_node;
        struct rb_node *parent = NULL;
        struct trace *__t;
 
-       if (genesis_time == 0 || t->bit->time < genesis_time)
-               genesis_time = t->bit->time;
-
        while (*p) {
                parent = *p;
                __t = rb_entry(parent, struct trace, rb_node);
@@ -303,11 +321,37 @@ static inline int trace_rb_insert(struct trace *t)
                }
        }
 
+       rb_sort_entries++;
        rb_link_node(&t->rb_node, parent, p);
        rb_insert_color(&t->rb_node, &rb_sort_root);
        return 0;
 }
 
+static struct trace *trace_rb_find(dev_t device, unsigned long sequence)
+{
+       struct rb_node **p = &rb_sort_root.rb_node;
+       struct rb_node *parent = NULL;
+       struct trace *__t;
+
+       while (*p) {
+               parent = *p;
+               __t = rb_entry(parent, struct trace, rb_node);
+
+               if (device < __t->bit->device)
+                       p = &(*p)->rb_left;
+               else if (device > __t->bit->device)
+                       p = &(*p)->rb_right;
+               else if (sequence < __t->bit->sequence)
+                       p = &(*p)->rb_left;
+               else if (sequence > __t->bit->sequence)
+                       p = &(*p)->rb_right;
+               else
+                       return __t;
+       }
+
+       return NULL;
+}
+
 static inline int track_rb_insert(struct io_track *iot)
 {
        struct rb_node **p = &rb_track_root.rb_node;
@@ -367,7 +411,8 @@ static struct io_track *__find_track(dev_t device, __u64 sector)
        return NULL;
 }
 
-static struct io_track *find_track(__u32 pid, dev_t device, __u64 sector)
+static struct io_track *find_track(__u32 pid, char *comm, dev_t device,
+                                  __u64 sector)
 {
        struct io_track *iot;
 
@@ -375,6 +420,7 @@ static struct io_track *find_track(__u32 pid, dev_t device, __u64 sector)
        if (!iot) {
                iot = malloc(sizeof(*iot));
                iot->pid = pid;
+               memcpy(iot->comm, comm, sizeof(iot->comm));
                iot->device = device;
                iot->sector = sector;
                track_rb_insert(iot);
@@ -408,7 +454,7 @@ static void log_track_getrq(struct blk_io_trace *t)
        if (!track_ios)
                return;
 
-       iot = find_track(t->pid, t->device, t->sector);
+       iot = find_track(t->pid, t->comm, t->device, t->sector);
        iot->allocation_time = t->time;
 }
 
@@ -424,12 +470,12 @@ static unsigned long long log_track_insert(struct blk_io_trace *t)
        if (!track_ios)
                return -1;
 
-       iot = find_track(t->pid, t->device, t->sector);
+       iot = find_track(t->pid, t->comm, t->device, t->sector);
        iot->queue_time = t->time;
        elapsed = iot->queue_time - iot->allocation_time;
 
        if (per_process_stats) {
-               struct per_process_info *ppi = find_process_by_pid(iot->pid);
+               struct per_process_info *ppi = find_process(iot->pid,iot->comm);
                int w = (t->action & BLK_TC_ACT(BLK_TC_WRITE)) != 0;
 
                if (ppi && elapsed > ppi->longest_allocation_wait[w])
@@ -462,7 +508,7 @@ static unsigned long long log_track_issue(struct blk_io_trace *t)
        elapsed = iot->dispatch_time - iot->queue_time;
 
        if (per_process_stats) {
-               struct per_process_info *ppi = find_process_by_pid(iot->pid);
+               struct per_process_info *ppi = find_process(iot->pid,iot->comm);
                int w = (t->action & BLK_TC_ACT(BLK_TC_WRITE)) != 0;
 
                if (ppi && elapsed > ppi->longest_dispatch_wait[w])
@@ -495,7 +541,7 @@ static unsigned long long log_track_complete(struct blk_io_trace *t)
        elapsed = iot->completion_time - iot->dispatch_time;
 
        if (per_process_stats) {
-               struct per_process_info *ppi = find_process_by_pid(iot->pid);
+               struct per_process_info *ppi = find_process(iot->pid,iot->comm);
                int w = (t->action & BLK_TC_ACT(BLK_TC_WRITE)) != 0;
 
                if (ppi && elapsed > ppi->longest_completion_wait[w])
@@ -514,12 +560,12 @@ static unsigned long long log_track_complete(struct blk_io_trace *t)
 
 static struct io_stats *find_process_io_stats(__u32 pid, char *name)
 {
-       struct per_process_info *ppi = find_process_by_pid(pid);
+       struct per_process_info *ppi = find_process(pid, name);
 
        if (!ppi) {
                ppi = malloc(sizeof(*ppi));
                memset(ppi, 0, sizeof(*ppi));
-               strncpy(ppi->name, name, sizeof(ppi->name));
+               memcpy(ppi->name, name, 16);
                ppi->pid = pid;
                add_process_to_hash(ppi);
                add_process_to_list(ppi);
@@ -528,7 +574,6 @@ static struct io_stats *find_process_io_stats(__u32 pid, char *name)
        return &ppi->io_stats;
 }
 
-
 static void resize_cpu_info(struct per_dev_info *pdi, int cpu)
 {
        struct per_cpu_info *cpus = pdi->cpus;
@@ -599,7 +644,8 @@ static struct per_dev_info *get_dev_info(dev_t id)
 
        pdi = &devices[ndevices - 1];
        pdi->id = id;
-       pdi->last_sequence = -1;
+       pdi->last_sequence = 0;
+       pdi->last_read_time = 0;
        return pdi;
 }
 
@@ -736,317 +782,6 @@ static inline void account_unplug(struct blk_io_trace *t,
        }
 }
 
-#define VALID_SPECS    "BCDFGMPQRSTU"
-static char *override_format[256];
-static inline int valid_spec(int spec)
-{
-       return strchr(VALID_SPECS, spec) != NULL;
-}
-
-static void set_all_format_specs(char *optarg)
-{
-       char *p;
-
-       for (p = VALID_SPECS; *p; p++)
-               if (override_format[(int)(*p)] == NULL)
-                       override_format[(int)(*p)] = strdup(optarg);
-}
-
-static int add_format_spec(char *optarg)
-{
-       int spec = optarg[0];
-
-       if (!valid_spec(spec)) {
-               fprintf(stderr,"Bad format specifier %c\n", spec);
-               return 1;
-       }
-       if (optarg[1] != ',') {
-               fprintf(stderr,"Bad format specifier - need ',' %s\n", optarg);
-               return 1;
-       }
-       optarg += 2;
-       if (*optarg == '\0') {
-               fprintf(stderr,"Bad format specifier - need fmt %s\n", optarg);
-               return 1;
-       }
-
-       /*
-        * Set both merges (front and back)
-        */
-       if (spec == 'M') {
-               override_format['B'] = strdup(optarg);
-               override_format['M'] = strdup(optarg);
-       } else
-               override_format[spec] = strdup(optarg);
-
-       return 0;
-}
-
-static void print_field(char *act, struct per_cpu_info *pci,
-                       struct blk_io_trace *t, unsigned long long elapsed,
-                       int pdu_len, unsigned char *pdu_buf, char field,
-                       int minus, int has_w, int width)
-{
-       char format[64];
-
-       if (has_w) {
-               if (minus)
-                       sprintf(format, "%%-%d", width);
-               else
-                       sprintf(format, "%%%d", width);
-       } else
-               sprintf(format, "%%");
-
-       switch (field) {
-       case 'a':
-               fprintf(ofp, strcat(format, "s"), act);
-               break;
-       case 'c':
-               fprintf(ofp, strcat(format, "d"), pci->cpu);
-               break;
-       case 'C':
-               fprintf(ofp, strcat(format, "s"), t->comm);
-               break;
-       case 'd': {
-               char rwbs[4];
-               int i = 0;
-               int w = t->action & BLK_TC_ACT(BLK_TC_WRITE);
-               int b = t->action & BLK_TC_ACT(BLK_TC_BARRIER);
-               int s = t->action & BLK_TC_ACT(BLK_TC_SYNC);
-               if (w)
-                       rwbs[i++] = 'W';
-               else
-                       rwbs[i++] = 'R';
-               if (b)
-                       rwbs[i++] = 'B';
-               if (s)
-                       rwbs[i++] = 'S';
-               rwbs[i] = '\0';
-               fprintf(ofp, strcat(format, "s"), rwbs);
-               break;
-       }
-       case 'D':       /* format width ignored */
-               fprintf(ofp,"%3d,%-3d", MAJOR(t->device), MINOR(t->device));
-               break;
-       case 'e':
-               fprintf(ofp, strcat(format, "d"), t->error);
-               break;
-       case 'M':
-               fprintf(ofp, strcat(format, "d"), MAJOR(t->device));
-               break;
-       case 'm':
-               fprintf(ofp, strcat(format, "d"), MINOR(t->device));
-               break;
-       case 'n':
-               fprintf(ofp, strcat(format, "u"), t->bytes >> 9);
-               break;
-       case 'p':
-               fprintf(ofp, strcat(format, "u"), t->pid);
-               break;
-       case 'P':       /* format width ignored */
-               if ((pdu_len > 0) && (pdu_buf != NULL)) {
-                       int i;
-                       unsigned char *p = pdu_buf;
-                       for (i = 0; i < pdu_len; i++) {
-                               if (i)
-                                       fprintf(ofp, " ");
-
-                               fprintf(ofp, "%02x", *p++);
-                       }
-               }
-               break;
-       case 's':
-               fprintf(ofp, strcat(format, "ld"), t->sequence);
-               break;
-       case 'S':
-               fprintf(ofp, strcat(format, "lu"), t->sector);
-               break;
-       case 't':
-               sprintf(format, "%%0%dlu", has_w ? width : 9);
-               fprintf(ofp, format, NANO_SECONDS(t->time));
-               break;
-       case 'T':
-               fprintf(ofp, strcat(format, "d"), SECONDS(t->time));
-               break;
-       case 'u':
-               if (elapsed == -1ULL) {
-                       fprintf(stderr, "Expecting elapsed value\n");
-                       exit(1);
-               }
-               fprintf(ofp, strcat(format, "llu"), elapsed / 1000);
-               break;
-       case 'U': {
-               __u64 *depth = (__u64 *) ((char *) t + sizeof(*t));
-               fprintf(ofp, strcat(format, "u"),
-                                       (unsigned int) be64_to_cpu(*depth));
-               break;
-       }
-       default:
-               fprintf(ofp,strcat(format, "c"), field);
-               break;
-       }
-}
-
-static char *parse_field(char *act, struct per_cpu_info *pci, 
-                        struct blk_io_trace *t, unsigned long long elapsed, 
-                        int pdu_len, unsigned char *pdu_buf, 
-                        char *master_format)
-{
-       int minus = 0;
-       int has_w = 0;
-       int width = 0;
-       char *p = master_format;
-
-       if (*p == '-') {
-               minus = 1;
-               p++;
-       }
-       if (isdigit(*p)) {
-               has_w = 1;
-               do {
-                       width = (width * 10) + (*p++ - '0');
-               } while ((*p) && (isdigit(*p)));
-       }
-       if (*p) {
-               print_field(act, pci, t, elapsed, pdu_len, pdu_buf, *p++,
-                           minus, has_w, width);
-       }
-       return p;
-}
-
-static char *fmt_select(int fmt_spec, struct blk_io_trace *t,
-                       unsigned long long elapsed)
-{
-       char *fmt;
-       char scratch_format[1024];
-
-       if (override_format[fmt_spec] != NULL)
-               return override_format[fmt_spec];
-
-       switch (fmt_spec) {
-       case 'C':       /* Complete */
-               if (t->action & BLK_TC_ACT(BLK_TC_PC))
-                       strcpy(scratch_format, HEADER "(%P) [%e]\n");
-               else {
-                       if (elapsed != -1ULL) {
-                               strcpy(scratch_format,
-                                       HEADER "%S +%n (%8u) [%e]\n");
-                       } else
-                               strcpy(scratch_format, HEADER "%S + %n [%e]\n");
-               }
-               fmt = scratch_format;
-               break;
-
-       case 'D':       /* Issue */
-               if (t->action & BLK_TC_ACT(BLK_TC_PC))
-                       strcpy(scratch_format, HEADER "%n (%P) [%C]\n");
-               else {
-                       if (elapsed != -1ULL) {
-                               strcpy(scratch_format,
-                                       HEADER "%S + %n (%8u) [%C]\n");
-                       } else
-                               strcpy(scratch_format, HEADER "%S + %n [%C]\n");
-               }
-               fmt = scratch_format;
-               break;
-
-       case 'I':       /* Insert */
-               if (t->action & BLK_TC_ACT(BLK_TC_PC))
-                       strcpy(scratch_format, HEADER "%n (%P) [%C]\n");
-               else {
-                       if (elapsed != -1ULL) {
-                               strcpy(scratch_format,
-                                       HEADER "%S + %n (%8u) [%C]\n");
-                       } else
-                               strcpy(scratch_format, HEADER "%S + %n [%C]\n");
-               }
-               fmt = scratch_format;
-               break;
-
-       case 'Q':       /* Queue */
-       case 'W':       /* Bounce */
-               if (elapsed != -1ULL) {
-                       strcpy(scratch_format, HEADER "%S + %n (%8u) [%C]\n");
-               } else
-                       strcpy(scratch_format, HEADER "%S + %n [%C]\n");
-               fmt = scratch_format;
-               break;
-
-       case 'B':       /* Back merge */
-       case 'F':       /* Front merge */
-       case 'M':       /* Front or back merge */
-               fmt = HEADER "%S + %n [%C]\n";
-               break;
-
-       case 'P':       /* Plug */
-               fmt = HEADER "[%C]\n";
-               break;
-
-       case 'G':       /* Get request */
-       case 'S':       /* Sleep request */
-               fmt = HEADER "%S + %n [%C]\n";
-               break;
-
-       case 'U':       /* Unplug IO */
-       case 'T':       /* Unplug timer */
-               fmt = HEADER "[%C] %U\n";
-               break;
-
-       case 'X':       /* Split */
-               strcpy(scratch_format, HEADER "%S / %U [%C]\n");
-               fmt = scratch_format;
-               break;
-
-       default:
-               fprintf(stderr,"FATAL: Invalid format spec %c\n", fmt_spec);
-               exit(1);
-               /*NOTREACHED*/
-       }
-
-       return fmt;
-}
-
-static void process_fmt(char *act, struct per_cpu_info *pci,
-                          struct blk_io_trace *t, unsigned long long elapsed,
-                          int pdu_len, unsigned char *pdu_buf)
-{
-       char *p = fmt_select(act[0], t, elapsed);
-
-       while (*p) {
-               switch (*p) {
-               case '%':       /* Field specifier */
-                       p++;
-                       if (*p == '%')
-                               fprintf(ofp, "%c", *p++);
-                       else if (!*p)
-                               fprintf(ofp, "%c", '%');
-                       else
-                               p = parse_field(act, pci, t, elapsed,
-                                               pdu_len, pdu_buf, p);
-                       break;
-               case '\\': {    /* escape */
-                       switch (p[1]) {
-                       case 'b': fprintf(ofp, "\b"); break;
-                       case 'n': fprintf(ofp, "\n"); break;
-                       case 'r': fprintf(ofp, "\r"); break;
-                       case 't': fprintf(ofp, "\t"); break;
-                       default:
-                               fprintf(stderr, 
-                                       "Invalid escape char in format %c\n",
-                                       p[1]);
-                               exit(1);
-                               /*NOTREACHED*/
-                       }
-                       p += 2;
-                       break;
-               }
-               default:
-                       fprintf(ofp, "%c", *p++);
-                       break;
-               }
-       }
-}
-
 static void log_complete(struct per_cpu_info *pci, struct blk_io_trace *t,
                         char *act)
 {
@@ -1303,7 +1038,11 @@ static void show_process_stats(void)
        while (ppi) {
                char name[64];
 
-               snprintf(name, sizeof(name)-1, "%s (%u)", ppi->name, ppi->pid);
+               if (ppi->more_than_one)
+                       sprintf(name, "%s (%u, ...)", ppi->name, ppi->pid);
+               else
+                       sprintf(name, "%s (%u)", ppi->name, ppi->pid);
+
                dump_io_stats(&ppi->io_stats, name);
                dump_wait_stats(ppi);
                ppi = ppi->list_next;
@@ -1370,24 +1109,6 @@ static void show_device_and_cpu_stats(void)
        }
 }
 
-static int sort_entries(void)
-{
-       struct trace *t;
-       int nr = 0;
-
-       while ((t = trace_list) != NULL) {
-               trace_list = t->next;
-
-               if (verify_trace(t->bit))
-                       continue;
-               if (trace_rb_insert(t))
-                       break;
-               nr++;
-       }
-
-       return nr;
-}
-
 /*
  * struct trace and blktrace allocation cache, we do potentially
  * millions of mallocs for these structures while only using at most
@@ -1433,15 +1154,68 @@ static inline struct blk_io_trace *bit_alloc(void)
        return malloc(sizeof(*bit));
 }
 
-static void show_entries_rb(void)
+static void find_genesis(void)
+{
+       struct trace *t = trace_list;
+
+       genesis_time = -1ULL;
+       while (t != NULL) {
+               if (t->bit->time < genesis_time)
+                       genesis_time = t->bit->time;
+
+               t = t->next;
+       }
+}
+
+static inline int check_stopwatch(struct blk_io_trace *bit)
+{
+       if (bit->time < stopwatch_end &&
+           bit->time >= stopwatch_start)
+               return 0;
+
+       return 1;
+}
+
+static int sort_entries(void)
+{
+       struct trace *t;
+       int nr = 0;
+
+       if (!genesis_time)
+               find_genesis();
+
+       while ((t = trace_list) != NULL) {
+               struct blk_io_trace *bit = t->bit;
+
+               trace_list = t->next;
+
+               if (verify_trace(bit))
+                       continue;
+
+               bit->time -= genesis_time;
+
+               if (check_stopwatch(bit)) {
+                       bit_free(bit);
+                       t_free(t);
+                       continue;
+               }
+
+               if (trace_rb_insert(t))
+                       break;
+
+               nr++;
+       }
+
+       return nr;
+}
+
+static void show_entries_rb(int force)
 {
        struct per_dev_info *pdi = NULL;
        struct per_cpu_info *pci = NULL;
        struct blk_io_trace *bit;
        struct rb_node *n;
        struct trace *t;
-       __u32 device = 0;
-       int cpu = 0;
 
        while ((n = rb_first(&rb_sort_root)) != NULL) {
 
@@ -1451,10 +1225,8 @@ static void show_entries_rb(void)
                t = rb_entry(n, struct trace, rb_node);
                bit = t->bit;
 
-               if (!pdi || device != bit->device) {
-                       device = bit->device;
-                       pdi = get_dev_info(device);
-               }
+               if (!pdi || pdi->id != bit->device)
+                       pdi = get_dev_info(bit->device);
 
                if (!pdi) {
                        fprintf(stderr, "Unknown device ID? (%d,%d)\n",
@@ -1469,40 +1241,45 @@ static void show_entries_rb(void)
                        break;
                }
 
-               if (!pci || cpu != bit->cpu) {
-                       cpu = bit->cpu;
-                       pci = get_cpu_info(pdi, cpu);
-               }
-
                /*
                 * back off displaying more info if we are out of sync
                 * on SMP systems. to prevent stalling on lost events,
-                * only allow an event to us a few times
+                * only allow an event to skip us a few times
                 */
-               if (bit->sequence != (pdi->last_sequence + 1)
-                   && pdi->last_sequence != -1) {
+               if (bit->sequence > (pdi->last_sequence + 1) && !force) {
+                       struct trace *__t;
+
+                       /*
+                        * the wanted sequence is really there, continue
+                        * because this means that the log time is earlier
+                        * on the trace we have now
+                        */
+                       __t = trace_rb_find(pdi->id, pdi->last_sequence + 1);
+                       if (__t)
+                               goto ok;
+
                        if (t->skipped < 5) {
                                t->skipped++;
                                break;
-                       } else {
-                               fprintf(stderr, "skipping from %lu to %u\n", pdi->last_sequence, bit->sequence);
+                       } else
                                pdi->skips++;
-                       }
                }
 
+ok:
+               if (!force && bit->time > last_allowed_time)
+                       break;
+
                pdi->last_sequence = bit->sequence;
 
-               bit->time -= genesis_time;
-               if (bit->time >= stopwatch_end)
-                       break;
+               check_time(pdi, bit);
 
-               if (bit->time >= stopwatch_start) {
-                       check_time(pdi, bit);
+               if (!pci || pci->cpu != bit->cpu)
+                       pci = get_cpu_info(pdi, bit->cpu);
 
-                       dump_trace(bit, pci, pdi);
-               }
+               dump_trace(bit, pci, pdi);
 
                rb_erase(&t->rb_node, &rb_sort_root);
+               rb_sort_entries--;
                bit_free(bit);
                t_free(t);
        }
@@ -1529,6 +1306,7 @@ static int read_data(int fd, void *buffer, int bytes, int block)
                else if (ret < 0) {
                        if (errno != EAGAIN)
                                perror("read");
+
                        return -1;
                } else {
                        p += ret;
@@ -1539,8 +1317,9 @@ static int read_data(int fd, void *buffer, int bytes, int block)
        return 0;
 }
 
-static int read_sort_events(int fd)
+static int read_events(int fd, int always_block)
 {
+       struct per_dev_info *pdi = NULL;
        int events = 0;
 
        while (!is_done() && events < rb_batch) {
@@ -1551,7 +1330,7 @@ static int read_sort_events(int fd)
 
                bit = bit_alloc();
 
-               if (read_data(fd, bit, sizeof(*bit), !events))
+               if (read_data(fd, bit, sizeof(*bit), !events || always_block))
                        break;
 
                magic = be32_to_cpu(bit->magic);
@@ -1575,9 +1354,16 @@ static int read_sort_events(int fd)
                t->bit = bit;
 
                trace_to_cpu(bit);
+
                t->next = trace_list;
                trace_list = t;
 
+               if (!pdi || pdi->id != bit->device)
+                       pdi = get_dev_info(bit->device);
+
+               if (bit->time > pdi->last_read_time)
+                       pdi->last_read_time = bit->time;
+
                events++;
        }
 
@@ -1596,7 +1382,7 @@ static int do_file(void)
        for (i = 0; i < ndevices; i++) {
                pdi = &devices[i];
                pdi->nfiles = 0;
-               pdi->last_sequence = -1;
+               pdi->last_sequence = 0;
 
                for (j = 0;; j++) {
                        struct stat st;
@@ -1627,6 +1413,7 @@ static int do_file(void)
         */
        do {
                events_added = 0;
+               last_allowed_time = -1ULL;
 
                for (i = 0; i < ndevices; i++) {
                        pdi = &devices[i];
@@ -1638,24 +1425,30 @@ static int do_file(void)
                                if (pci->fd == -1)
                                        continue;
 
-                               events = read_sort_events(pci->fd);
+                               events = read_events(pci->fd, 1);
                                if (!events) {
                                        close(pci->fd);
                                        pci->fd = -1;
                                        continue;
                                }
 
-                               if (sort_entries() == -1)
-                                       continue;
+                               if (pdi->last_read_time < last_allowed_time)
+                                       last_allowed_time = pdi->last_read_time;
 
                                events_added += events;
                        }
                }
 
-               show_entries_rb();
+               if (sort_entries() == -1)
+                       break;
+
+               show_entries_rb(0);
 
        } while (events_added);
 
+       if (rb_sort_entries)
+               show_entries_rb(1);
+
        return 0;
 }
 
@@ -1663,20 +1456,24 @@ static int do_stdin(void)
 {
        int fd;
 
+       last_allowed_time = -1ULL;
        fd = dup(STDIN_FILENO);
        do {
                int events;
 
-               events = read_sort_events(fd);
+               events = read_events(fd, 0);
                if (!events)
                        break;
        
                if (sort_entries() == -1)
                        break;
 
-               show_entries_rb();
+               show_entries_rb(0);
        } while (1);
 
+       if (rb_sort_entries)
+               show_entries_rb(1);
+
        close(fd);
        return 0;
 }
@@ -1737,6 +1534,7 @@ static char usage_str[] = \
        "\t-o Output file. If not given, output is stdout\n" \
        "\t-b stdin read batching\n" \
        "\t-s Show per-program io statistics\n" \
+       "\t-n Hash processes by name, not pid\n" \
        "\t-t Track individual ios. Will tell you the time a request took\n" \
        "\t   to get queued, to get dispatched, and to get completed\n" \
        "\t-q Quiet. Don't display any stats at the end of the trace\n" \
@@ -1794,6 +1592,9 @@ int main(int argc, char *argv[])
                        if (add_format_spec(optarg) != 0)
                                return 1;
                        break;
+               case 'n':
+                       ppi_hash_by_pid = 0;
+                       break;
                case 'v':
                        printf("%s version %s\n", argv[0], blkparse_version);
                        return 0;