Merge branch 'master' of ssh://axboe@router.home.kernel.dk/data/git/blktrace
[blktrace.git] / blktrace.c
index 621e238daea11a322b40bca2d7bd323b363f4b92..26b3afdd671c5ca02697a36a36ae3bf167e675b7 100644 (file)
@@ -71,6 +71,12 @@ enum {
        Net_client,
 };
 
+enum thread_status {
+       Th_running,
+       Th_leaving,
+       Th_error
+};
+
 /*
  * Generic stats collected: nevents can be _roughly_ estimated by data_read
  * (discounting pdu...)
@@ -159,10 +165,8 @@ struct tracer {
        struct io_info *ios;
        struct pollfd *pfds;
        pthread_t thread;
-       pthread_mutex_t mutex;
-       pthread_cond_t cond;
        int cpu, nios;
-       volatile int running, status, is_done;
+       volatile int status, is_done;
 };
 
 /*
@@ -269,28 +273,46 @@ static char blktrace_version[] = "2.0.0";
  */
 int data_is_native = -1;
 
+static int ndevs;
 static int ncpus;
 static int pagesize;
 static int act_mask = ~0U;
+static int kill_running_trace;
+static int stop_watch;
+static int piped_output;
+
 static char *debugfs_path = "/sys/kernel/debug";
 static char *output_name;
 static char *output_dir;
-static int kill_running_trace;
-static int stop_watch;
+
 static unsigned long buf_size = BUF_SIZE;
 static unsigned long buf_nr = BUF_NR;
+
+static FILE *pfp;
+
 static LIST_HEAD(devpaths);
 static LIST_HEAD(tracers);
-static int ndevs;
+
 static volatile int done;
-static FILE *pfp;
-static int piped_output;
-static int ntracers;
 
+/*
+ * tracer threads add entries, the main thread takes them off and processes
+ * them. These protect the dp_entries variable.
+ */
 static pthread_cond_t dp_cond = PTHREAD_COND_INITIALIZER;
 static pthread_mutex_t dp_mutex = PTHREAD_MUTEX_INITIALIZER;
 static volatile int dp_entries;
 
+/*
+ * These synchronize master / thread interactions.
+ */
+static pthread_cond_t mt_cond = PTHREAD_COND_INITIALIZER;
+static pthread_mutex_t mt_mutex = PTHREAD_MUTEX_INITIALIZER;
+static volatile int nthreads_running;
+static volatile int nthreads_leaving;
+static volatile int nthreads_error;
+static volatile int tracers_run;
+
 /*
  * network cmd line params
  */
@@ -474,6 +496,87 @@ static void show_usage(char *prog)
        fprintf(stderr, "Usage: %s %s %s", prog, blktrace_version, usage_str);
 }
 
+/*
+ * Create a timespec 'msec' milliseconds into the future
+ */
+static inline void make_timespec(struct timespec *tsp, long delta_msec)
+{
+       struct timeval now;
+
+       gettimeofday(&now, NULL);
+       tsp->tv_sec = now.tv_sec;
+       tsp->tv_nsec = 1000L * now.tv_usec;
+
+       tsp->tv_nsec += (delta_msec * 1000000L);
+       if (tsp->tv_nsec > 1000000000L) {
+               long secs = tsp->tv_nsec / 1000000000L;
+
+               tsp->tv_sec += secs;
+               tsp->tv_nsec -= (secs * 1000000000L);
+       }
+}
+
+/*
+ * Add a timer to ensure wait ends
+ */
+static void t_pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
+{
+       struct timespec ts;
+
+       make_timespec(&ts, 50);
+       pthread_cond_timedwait(cond, mutex, &ts);
+}
+
+static void unblock_tracers(void)
+{
+       pthread_mutex_lock(&mt_mutex);
+       tracers_run = 1;
+       pthread_cond_broadcast(&mt_cond);
+       pthread_mutex_unlock(&mt_mutex);
+}
+
+static void tracer_wait_unblock(struct tracer *tp)
+{
+       pthread_mutex_lock(&mt_mutex);
+       while (!tp->is_done && !tracers_run)
+               pthread_cond_wait(&mt_cond, &mt_mutex);
+       pthread_mutex_unlock(&mt_mutex);
+}
+
+static void tracer_signal_ready(struct tracer *tp,
+                               enum thread_status th_status,
+                               int status)
+{
+       pthread_mutex_lock(&mt_mutex);
+       tp->status = status;
+
+       if (th_status == Th_running)
+               nthreads_running++;
+       else if (th_status == Th_error)
+               nthreads_error++;
+       else
+               nthreads_leaving++;
+
+       pthread_cond_signal(&mt_cond);
+       pthread_mutex_unlock(&mt_mutex);
+}
+
+static void wait_tracers_ready(int ncpus_started)
+{
+       pthread_mutex_lock(&mt_mutex);
+       while ((nthreads_running + nthreads_error) < ncpus_started)
+               t_pthread_cond_wait(&mt_cond, &mt_mutex);
+       pthread_mutex_unlock(&mt_mutex);
+}
+
+static void wait_tracers_leaving(void)
+{
+       pthread_mutex_lock(&mt_mutex);
+       while (nthreads_leaving < nthreads_running)
+               t_pthread_cond_wait(&mt_cond, &mt_mutex);
+       pthread_mutex_unlock(&mt_mutex);
+}
+
 static void init_mmap_info(struct mmap_info *mip)
 {
        mip->buf_size = buf_size;
@@ -513,26 +616,6 @@ static int lock_on_cpu(int cpu)
        return 0;
 }
 
-/*
- * Create a timespec 'msec' milliseconds into the future
- */
-static inline void make_timespec(struct timespec *tsp, long delta_msec)
-{
-       struct timeval now;
-
-       gettimeofday(&now, NULL);
-       tsp->tv_sec = now.tv_sec;
-       tsp->tv_nsec = 1000L * now.tv_usec;
-
-       tsp->tv_nsec += (delta_msec * 1000000L);
-       if (tsp->tv_nsec > 1000000000L) {
-               long secs = tsp->tv_nsec / 1000000000L;
-
-               tsp->tv_sec += secs;
-               tsp->tv_nsec -= (secs * 1000000000L);
-       }
-}
-
 static int increase_limit(int resource, rlim_t increase)
 {
        struct rlimit rlim;
@@ -634,6 +717,39 @@ static int my_mlock(const void *addr, size_t len)
        return ret;
 }
 
+static int setup_mmap(int fd, unsigned int maxlen, struct mmap_info *mip)
+{
+       if (mip->fs_off + maxlen > mip->fs_buf_len) {
+               unsigned long nr = max(16, mip->buf_nr);
+
+               if (mip->fs_buf) {
+                       munlock(mip->fs_buf, mip->fs_buf_len);
+                       munmap(mip->fs_buf, mip->fs_buf_len);
+                       mip->fs_buf = NULL;
+               }
+
+               mip->fs_off = mip->fs_size & (mip->pagesize - 1);
+               mip->fs_buf_len = (nr * mip->buf_size) - mip->fs_off;
+               mip->fs_max_size += mip->fs_buf_len;
+
+               if (ftruncate(fd, mip->fs_max_size) < 0) {
+                       perror("setup_mmap: ftruncate");
+                       return 1;
+               }
+
+               mip->fs_buf = my_mmap(NULL, mip->fs_buf_len, PROT_WRITE,
+                                     MAP_SHARED, fd,
+                                     mip->fs_size - mip->fs_off);
+               if (mip->fs_buf == MAP_FAILED) {
+                       perror("setup_mmap: mmap");
+                       return 1;
+               }
+               my_mlock(mip->fs_buf, mip->fs_buf_len);
+       }
+
+       return 0;
+}
+
 static int __stop_trace(int fd)
 {
        /*
@@ -683,11 +799,12 @@ static int __net_recv_data(int fd, void *buf, unsigned int len)
                if (ret == 0)
                        break;
                else if (ret < 0) {
-                       if (errno != EAGAIN) {
-                               perror("server: net_recv_data: recv failed");
-                               break;
-                       } else
-                               break;
+                       if (errno == EAGAIN) {
+                               usleep(50);
+                               continue;
+                       }
+                       perror("server: net_recv_data: recv failed");
+                       break;
                } else {
                        buf += ret;
                        bytes_left -= ret;
@@ -788,9 +905,9 @@ static void net_send_drops(int fd)
 
 /*
  * Returns:
- *      0: "EOF"
- *      1: OK
- *     -1: Error
+ *      0: "EOF"
+ *      1: OK
+ *     -1: Error
  */
 static int net_get_header(struct cl_conn *nc, struct blktrace_net_hdr *bnh)
 {
@@ -805,7 +922,8 @@ static int net_get_header(struct cl_conn *nc, struct blktrace_net_hdr *bnh)
                return 1;
        else if (bytes_read == 0)
                return 0;
-       return -1;
+       else
+               return -1;
 }
 
 static int net_setup_addr(void)
@@ -864,6 +982,7 @@ static int net_setup_client(void)
                                hostname);
                else
                        perror("client: connect");
+
                close(fd);
                return -1;
        }
@@ -918,22 +1037,30 @@ static void setup_buts(void)
                buts.buf_nr = buf_nr;
                buts.act_mask = act_mask;
 
-               if (ioctl(dpp->fd, BLKTRACESETUP, &buts) < 0) {
+               if (ioctl(dpp->fd, BLKTRACESETUP, &buts) >= 0) {
+                       dpp->ncpus = ncpus;
+                       dpp->buts_name = strdup(buts.name);
+                       if (dpp->stats)
+                               free(dpp->stats);
+                       dpp->stats = calloc(dpp->ncpus, sizeof(*dpp->stats));
+                       memset(dpp->stats, 0, dpp->ncpus * sizeof(*dpp->stats));
+               } else
                        fprintf(stderr, "BLKTRACESETUP(2) %s failed: %d/%s\n",
                                dpp->path, errno, strerror(errno));
-                       continue;
-               } else if (ioctl(dpp->fd, BLKTRACESTART) < 0) {
+       }
+}
+
+static void start_buts(void)
+{
+       struct list_head *p;
+
+       __list_for_each(p, &devpaths) {
+               struct devpath *dpp = list_entry(p, struct devpath, head);
+
+               if (ioctl(dpp->fd, BLKTRACESTART) < 0) {
                        fprintf(stderr, "BLKTRACESTART %s failed: %d/%s\n",
                                dpp->path, errno, strerror(errno));
-                       continue;
                }
-
-               dpp->ncpus = ncpus;
-               dpp->buts_name = strdup(buts.name);
-               if (dpp->stats)
-                       free(dpp->stats);
-               dpp->stats = calloc(dpp->ncpus, sizeof(*dpp->stats));
-               memset(dpp->stats, 0, dpp->ncpus * sizeof(*dpp->stats));
        }
 }
 
@@ -971,6 +1098,7 @@ static void get_all_drops(void)
 
        __list_for_each(p, &devpaths) {
                struct devpath *dpp = list_entry(p, struct devpath, head);
+
                dpp->drops = get_drops(dpp);
        }
 }
@@ -997,6 +1125,7 @@ static void free_tracer_heads(struct devpath *dpp)
        for (cpu = 0, hd = dpp->heads; cpu < ncpus; cpu++, hd++) {
                if (hd->prev)
                        free(hd->prev);
+
                pthread_mutex_destroy(&hd->mutex);
        }
        free(dpp->heads);
@@ -1050,6 +1179,23 @@ static inline void incr_entries(int entries_handled)
        pthread_mutex_unlock(&dp_mutex);
 }
 
+static void decr_entries(int handled)
+{
+       pthread_mutex_lock(&dp_mutex);
+       dp_entries -= handled;
+       pthread_mutex_unlock(&dp_mutex);
+}
+
+static int wait_empty_entries(void)
+{
+       pthread_mutex_lock(&dp_mutex);
+       while (!done && dp_entries == 0)
+               t_pthread_cond_wait(&dp_cond, &dp_mutex);
+       pthread_mutex_unlock(&dp_mutex);
+
+       return !done;
+}
+
 static int add_devpath(char *path)
 {
        int fd;
@@ -1101,8 +1247,7 @@ static int flush_subbuf_net(struct trace_buf *tbp)
 
        if (net_send_header(fd, tbp->cpu, dpp->buts_name, tbp->len))
                return 1;
-
-       if (net_send_data(fd, tbp->buf, tbp->len) != tbp->len)
+       else if (net_send_data(fd, tbp->buf, tbp->len) != tbp->len)
                return 1;
 
        return 0;
@@ -1135,6 +1280,34 @@ handle_list_net(__attribute__((__unused__))struct tracer_devpath_head *hd,
        return entries_handled;
 }
 
+/*
+ * Tack 'tbp's buf onto the tail of 'prev's buf
+ */
+static struct trace_buf *tb_combine(struct trace_buf *prev,
+                                   struct trace_buf *tbp)
+{
+       unsigned long tot_len;
+
+       tot_len = prev->len + tbp->len;
+       if (tot_len > buf_size) {
+               /*
+                * tbp->head isn't connected (it was 'prev'
+                * so it had been taken off of the list
+                * before). Therefore, we can realloc
+                * the whole structures, as the other fields
+                * are "static".
+                */
+               prev = realloc(prev->buf, sizeof(*prev) + tot_len);
+               prev->buf = (void *)(prev + 1);
+       }
+
+       memcpy(prev->buf + prev->len, tbp->buf, tbp->len);
+       prev->len = tot_len;
+
+       free(tbp);
+       return prev;
+}
+
 static int handle_list_file(struct tracer_devpath_head *hd,
                            struct list_head *list)
 {
@@ -1154,31 +1327,8 @@ static int handle_list_file(struct tracer_devpath_head *hd,
                 * If there was some leftover before, tack this new
                 * entry onto the tail of the previous one.
                 */
-               if (prev) {
-                       unsigned long tot_len;
-                       struct trace_buf *tmp = tbp;
-
-                       tbp = prev;
-                       prev = NULL;
-
-                       tot_len = tbp->len + tmp->len;
-                       if (tot_len > buf_size) {
-                               /*
-                                * tbp->head isn't connected (it was 'prev'
-                                * so it had been taken off of the list
-                                * before). Therefore, we can realloc
-                                * the whole structures, as the other fields
-                                * are "static".
-                                */
-                               tbp = realloc(tbp->buf, sizeof(*tbp) + tot_len);
-                               tbp->buf = (void *)(tbp + 1);
-                       }
-
-                       memcpy(tbp->buf + tbp->len, tmp->buf, tmp->len);
-                       tbp->len = tot_len;
-
-                       free(tmp);
-               }
+               if (prev)
+                       tbp = tb_combine(prev, tbp);
 
                /*
                 * See how many whole traces there are - send them
@@ -1203,8 +1353,10 @@ static int handle_list_file(struct tracer_devpath_head *hd,
                 * for the next pass.
                 */
                if (off) {
-                       if (write_data(tbp->buf, off) || off == tbp->len)
+                       if (write_data(tbp->buf, off) || off == tbp->len) {
                                free(tbp);
+                               prev = NULL;
+                       }
                        else {
                                /*
                                 * Move valid data to beginning of buffer
@@ -1246,27 +1398,14 @@ static void __process_trace_bufs(void)
                }
        }
 
-       if (handled) {
-               pthread_mutex_lock(&dp_mutex);
-               dp_entries -= handled;
-               pthread_mutex_unlock(&dp_mutex);
-       }
+       if (handled)
+               decr_entries(handled);
 }
 
 static void process_trace_bufs(void)
 {
-       while (!done) {
-               pthread_mutex_lock(&dp_mutex);
-               while (!done && dp_entries == 0) {
-                       struct timespec ts;
-
-                       make_timespec(&ts, 50);
-                       pthread_cond_timedwait(&dp_cond, &dp_mutex, &ts);
-               }
-               pthread_mutex_unlock(&dp_mutex);
-
+       while (wait_empty_entries())
                __process_trace_bufs();
-       }
 }
 
 static void clean_trace_bufs(void)
@@ -1312,78 +1451,6 @@ static inline int net_sendfile_data(struct tracer *tp, struct io_info *iop)
        return net_sendfile(iop);
 }
 
-static int handle_pfds_netclient(struct tracer *tp, int nevs, int force_read)
-{
-       struct stat sb;
-       int i, nentries = 0;
-       struct pdc_stats *sp;
-       struct pollfd *pfd = tp->pfds;
-       struct io_info *iop = tp->ios;
-
-       for (i = 0; nevs > 0 && i < ndevs; i++, pfd++, iop++, sp++) {
-               if (pfd->revents & POLLIN || force_read) {
-                       if (fstat(iop->ifd, &sb) < 0) {
-                               perror(iop->ifn);
-                               pfd->events = 0;
-                       } else if (sb.st_size > (off_t)iop->data_queued) {
-                               iop->ready = sb.st_size - iop->data_queued;
-                               iop->data_queued = sb.st_size;
-                               if (!net_sendfile_data(tp, iop)) {
-                                       pdc_dr_update(iop->dpp, tp->cpu,
-                                                     iop->ready);
-                                       nentries++;
-                               } else
-                                       clear_events(pfd);
-                       }
-                       nevs--;
-               }
-       }
-
-       if (nentries)
-               incr_entries(nentries);
-
-       return nentries;
-}
-
-static int handle_pfds_entries(struct tracer *tp, int nevs, int force_read)
-{
-       int i, nentries = 0;
-       struct trace_buf *tbp;
-       struct pollfd *pfd = tp->pfds;
-       struct io_info *iop = tp->ios;
-
-       tbp = alloc_trace_buf(tp->cpu, buf_size);
-       for (i = 0; i < ndevs; i++, pfd++, iop++) {
-               if (pfd->revents & POLLIN || force_read) {
-                       tbp->len = read(iop->ifd, tbp->buf, buf_size);
-                       if (tbp->len > 0) {
-                               pdc_dr_update(iop->dpp, tp->cpu, tbp->len);
-                               add_trace_buf(iop->dpp, tp->cpu, &tbp);
-                               nentries++;
-                       } else if (tbp->len == 0) {
-                               /*
-                                * Short reads after we're done stop us
-                                * from trying reads.
-                                */
-                               if (tp->is_done)
-                                       clear_events(pfd);
-                       } else {
-                               read_err(tp->cpu, iop->ifn);
-                               if (errno != EAGAIN || tp->is_done)
-                                       clear_events(pfd);
-                       }
-                       if (!piped_output && --nevs == 0)
-                               break;
-               }
-       }
-       free(tbp);
-
-       if (nentries)
-               incr_entries(nentries);
-
-       return nentries;
-}
-
 static int fill_ofname(struct io_info *iop, int cpu)
 {
        int len;
@@ -1454,6 +1521,7 @@ static int iop_open(struct io_info *iop, int cpu)
                        iop->ofn, errno, strerror(errno));
                return 1;
        }
+
        if (set_vbuf(iop, _IOLBF, FILE_VBUF_SIZE)) {
                fprintf(stderr, "set_vbuf for file %s failed: %d/%s\n",
                        iop->ofn, errno, strerror(errno));
@@ -1465,6 +1533,50 @@ static int iop_open(struct io_info *iop, int cpu)
        return 0;
 }
 
+static void close_iop(struct io_info *iop)
+{
+       struct mmap_info *mip = &iop->mmap_info;
+
+       if (mip->fs_buf)
+               munmap(mip->fs_buf, mip->fs_buf_len);
+
+       if (!piped_output) {
+               if (ftruncate(fileno(iop->ofp), mip->fs_size) < 0) {
+                       fprintf(stderr,
+                               "Ignoring err: ftruncate(%s): %d/%s\n",
+                               iop->ofn, errno, strerror(errno));
+               }
+       }
+
+       if (iop->ofp)
+               fclose(iop->ofp);
+       if (iop->obuf)
+               free(iop->obuf);
+}
+
+static void close_ios(struct tracer *tp)
+{
+       while (tp->nios > 0) {
+               struct io_info *iop = &tp->ios[--tp->nios];
+
+               iop->dpp->drops = get_drops(iop->dpp);
+               if (iop->ifd >= 0)
+                       close(iop->ifd);
+
+               if (iop->ofp)
+                       close_iop(iop);
+               else if (iop->ofd >= 0) {
+                       struct devpath *dpp = iop->dpp;
+
+                       net_send_close(iop->ofd, dpp->buts_name, dpp->drops);
+                       net_close_connection(&iop->ofd);
+               }
+       }
+
+       free(tp->ios);
+       free(tp->pfds);
+}
+
 static int open_ios(struct tracer *tp)
 {
        struct pollfd *pfd;
@@ -1472,9 +1584,9 @@ static int open_ios(struct tracer *tp)
        struct list_head *p;
 
        tp->ios = calloc(ndevs, sizeof(struct io_info));
-       tp->pfds = calloc(ndevs, sizeof(struct pollfd));
-
        memset(tp->ios, 0, ndevs * sizeof(struct io_info));
+
+       tp->pfds = calloc(ndevs, sizeof(struct pollfd));
        memset(tp->pfds, 0, ndevs * sizeof(struct pollfd));
 
        tp->nios = 0;
@@ -1527,86 +1639,10 @@ static int open_ios(struct tracer *tp)
 
 err:
        close(iop->ifd);        /* tp->nios _not_ bumped */
+       close_ios(tp);
        return 1;
 }
 
-static void close_iop(struct io_info *iop)
-{
-       struct mmap_info *mip = &iop->mmap_info;
-
-       if (mip->fs_buf)
-               munmap(mip->fs_buf, mip->fs_buf_len);
-
-       if (!piped_output) {
-               if (ftruncate(fileno(iop->ofp), mip->fs_size) < 0) {
-                       fprintf(stderr,
-                               "Ignoring err: ftruncate(%s): %d/%s\n",
-                               iop->ofn, errno, strerror(errno));
-               }
-       }
-
-       if (iop->ofp)
-               fclose(iop->ofp);
-       if (iop->obuf)
-               free(iop->obuf);
-}
-
-static void close_ios(struct tracer *tp)
-{
-       while (tp->nios > 0) {
-               struct io_info *iop = &tp->ios[--tp->nios];
-
-               iop->dpp->drops = get_drops(iop->dpp);
-               if (iop->ifd >= 0)
-                       close(iop->ifd);
-
-               if (iop->ofp)
-                       close_iop(iop);
-               else if (iop->ofd >= 0) {
-                       struct devpath *dpp = iop->dpp;
-
-                       net_send_close(iop->ofd, dpp->buts_name, dpp->drops);
-                       net_close_connection(&iop->ofd);
-               }
-       }
-
-       free(tp->ios);
-       free(tp->pfds);
-}
-
-static int setup_mmap(int fd, unsigned int maxlen, struct mmap_info *mip)
-{
-       if (mip->fs_off + maxlen > mip->fs_buf_len) {
-               unsigned long nr = max(16, mip->buf_nr);
-
-               if (mip->fs_buf) {
-                       munlock(mip->fs_buf, mip->fs_buf_len);
-                       munmap(mip->fs_buf, mip->fs_buf_len);
-                       mip->fs_buf = NULL;
-               }
-
-               mip->fs_off = mip->fs_size & (mip->pagesize - 1);
-               mip->fs_buf_len = (nr * mip->buf_size) - mip->fs_off;
-               mip->fs_max_size += mip->fs_buf_len;
-
-               if (ftruncate(fd, mip->fs_max_size) < 0) {
-                       perror("__setup_mmap: ftruncate");
-                       return 1;
-               }
-
-               mip->fs_buf = my_mmap(NULL, mip->fs_buf_len, PROT_WRITE,
-                                     MAP_SHARED, fd,
-                                     mip->fs_size - mip->fs_off);
-               if (mip->fs_buf == MAP_FAILED) {
-                       perror("__setup_mmap: mmap");
-                       return 1;
-               }
-               my_mlock(mip->fs_buf, mip->fs_buf_len);
-       }
-
-       return 0;
-}
-
 static int handle_pfds_file(struct tracer *tp, int nevs, int force_read)
 {
        struct mmap_info *mip;
@@ -1650,11 +1686,83 @@ static int handle_pfds_file(struct tracer *tp, int nevs, int force_read)
        return nentries;
 }
 
-static void *thread_main(void *arg)
+static int handle_pfds_netclient(struct tracer *tp, int nevs, int force_read)
 {
-       int ret, ndone;
-       int to_val;
+       struct stat sb;
+       int i, nentries = 0;
+       struct pdc_stats *sp;
+       struct pollfd *pfd = tp->pfds;
+       struct io_info *iop = tp->ios;
 
+       for (i = 0; i < ndevs; i++, pfd++, iop++, sp++) {
+               if (pfd->revents & POLLIN || force_read) {
+                       if (fstat(iop->ifd, &sb) < 0) {
+                               perror(iop->ifn);
+                               pfd->events = 0;
+                       } else if (sb.st_size > (off_t)iop->data_queued) {
+                               iop->ready = sb.st_size - iop->data_queued;
+                               iop->data_queued = sb.st_size;
+
+                               if (!net_sendfile_data(tp, iop)) {
+                                       pdc_dr_update(iop->dpp, tp->cpu,
+                                                     iop->ready);
+                                       nentries++;
+                               } else
+                                       clear_events(pfd);
+                       }
+                       if (--nevs == 0)
+                               break;
+               }
+       }
+
+       if (nentries)
+               incr_entries(nentries);
+
+       return nentries;
+}
+
+static int handle_pfds_entries(struct tracer *tp, int nevs, int force_read)
+{
+       int i, nentries = 0;
+       struct trace_buf *tbp;
+       struct pollfd *pfd = tp->pfds;
+       struct io_info *iop = tp->ios;
+
+       tbp = alloc_trace_buf(tp->cpu, buf_size);
+       for (i = 0; i < ndevs; i++, pfd++, iop++) {
+               if (pfd->revents & POLLIN || force_read) {
+                       tbp->len = read(iop->ifd, tbp->buf, buf_size);
+                       if (tbp->len > 0) {
+                               pdc_dr_update(iop->dpp, tp->cpu, tbp->len);
+                               add_trace_buf(iop->dpp, tp->cpu, &tbp);
+                               nentries++;
+                       } else if (tbp->len == 0) {
+                               /*
+                                * Short reads after we're done stop us
+                                * from trying reads.
+                                */
+                               if (tp->is_done)
+                                       clear_events(pfd);
+                       } else {
+                               read_err(tp->cpu, iop->ifn);
+                               if (errno != EAGAIN || tp->is_done)
+                                       clear_events(pfd);
+                       }
+                       if (!piped_output && --nevs == 0)
+                               break;
+               }
+       }
+       free(tbp);
+
+       if (nentries)
+               incr_entries(nentries);
+
+       return nentries;
+}
+
+static void *thread_main(void *arg)
+{
+       int ret, ndone, to_val;
        struct tracer *tp = arg;
 
        ret = lock_on_cpu(tp->cpu);
@@ -1662,21 +1770,18 @@ static void *thread_main(void *arg)
                goto err;
 
        ret = open_ios(tp);
-       if (ret) {
-               close_ios(tp);
+       if (ret)
                goto err;
-       }
-
-       pthread_mutex_lock(&tp->mutex);
-       tp->running = 1;
-       pthread_cond_signal(&tp->cond);
-       pthread_mutex_unlock(&tp->mutex);
 
        if (piped_output)
                to_val = 50;            /* Frequent partial handles */
        else
                to_val = 500;           /* 1/2 second intervals */
 
+
+       tracer_signal_ready(tp, Th_running, 0);
+       tracer_wait_unblock(tp);
+
        while (!tp->is_done) {
                ndone = poll(tp->pfds, ndevs, to_val);
                if (ndone || piped_output)
@@ -1693,13 +1798,11 @@ static void *thread_main(void *arg)
                ;
 
        close_ios(tp);
+       tracer_signal_ready(tp, Th_leaving, 0);
+       return NULL;
 
 err:
-       pthread_mutex_lock(&tp->mutex);
-       tp->running = 0;
-       tp->status = ret;
-       pthread_cond_signal(&tp->cond);
-       pthread_mutex_unlock(&tp->mutex);
+       tracer_signal_ready(tp, Th_error, ret);
        return NULL;
 }
 
@@ -1711,46 +1814,38 @@ static int start_tracer(int cpu)
        memset(tp, 0, sizeof(*tp));
 
        INIT_LIST_HEAD(&tp->head);
-       pthread_mutex_init(&tp->mutex, NULL);
-       pthread_cond_init(&tp->cond, NULL);
-       tp->running = 0;
        tp->status = 0;
        tp->cpu = cpu;
 
        if (pthread_create(&tp->thread, NULL, thread_main, tp)) {
                fprintf(stderr, "FAILED to start thread on CPU %d: %d/%s\n",
                        cpu, errno, strerror(errno));
-               goto err;
-       }
-
-       pthread_mutex_lock(&tp->mutex);
-       while (!tp->running && (tp->status == 0))
-               pthread_cond_wait(&tp->cond, &tp->mutex);
-       pthread_mutex_unlock(&tp->mutex);
-
-       if (tp->status == 0) {
-               list_add_tail(&tp->head, &tracers);
-               return 0;
+               free(tp);
+               return 1;
        }
 
-       fprintf(stderr, "FAILED to start thread on CPU %d\n", cpu);
-
-err:
-       pthread_mutex_destroy(&tp->mutex);
-       pthread_cond_destroy(&tp->cond);
-       free(tp);
-       return 1;
+       list_add_tail(&tp->head, &tracers);
+       return 0;
 }
 
-static int start_tracers(void)
+static void start_tracers(void)
 {
        int cpu;
+       struct list_head *p;
 
        for (cpu = 0; cpu < ncpus; cpu++)
                if (start_tracer(cpu))
                        break;
 
-       return cpu;
+       wait_tracers_ready(cpu);
+
+       __list_for_each(p, &tracers) {
+               struct tracer *tp = list_entry(p, struct tracer, head);
+               if (tp->status)
+                       fprintf(stderr,
+                               "FAILED to start thread on CPU %d: %d/%s\n",
+                               tp->cpu, tp->status, strerror(tp->status));
+       }
 }
 
 static void stop_tracers(void)
@@ -1784,7 +1879,6 @@ static void del_tracers(void)
                list_del(&tp->head);
                free(tp);
        }
-       ntracers = 0;
 }
 
 static void wait_tracers(void)
@@ -1794,15 +1888,12 @@ static void wait_tracers(void)
        if (use_tracer_devpaths())
                process_trace_bufs();
 
+       wait_tracers_leaving();
+
        __list_for_each(p, &tracers) {
                int ret;
                struct tracer *tp = list_entry(p, struct tracer, head);
 
-               pthread_mutex_lock(&tp->mutex);
-               while (tp->running)
-                       pthread_cond_wait(&tp->cond, &tp->mutex);
-               pthread_mutex_unlock(&tp->mutex);
-
                ret = pthread_join(tp->thread, NULL);
                if (ret)
                        fprintf(stderr, "Thread join %d failed %d\n",
@@ -2466,6 +2557,45 @@ out:
        return ret;
 }
 
+static int run_tracers(void)
+{
+       atexit(exit_tracing);
+       if (net_mode == Net_client)
+               printf("blktrace: connecting to %s\n", hostname);
+
+       setup_buts();
+
+       if (use_tracer_devpaths()) {
+               if (setup_tracer_devpaths())
+                       return 1;
+
+               if (piped_output)
+                       handle_list = handle_list_file;
+               else
+                       handle_list = handle_list_net;
+       }
+
+       start_tracers();
+       if (nthreads_running == ncpus) {
+               unblock_tracers();
+               start_buts();
+               if (net_mode == Net_client)
+                       printf("blktrace: connected!\n");
+               if (stop_watch)
+                       alarm(stop_watch);
+       } else
+               stop_tracers();
+
+       wait_tracers();
+       if (nthreads_running == ncpus)
+               show_stats(&devpaths);
+       if (net_client_use_send())
+               close_client_connections();
+       del_tracers();
+
+       return 0;
+}
+
 int main(int argc, char *argv[])
 {
        int ret = 0;
@@ -2478,9 +2608,7 @@ int main(int argc, char *argv[])
                        errno, strerror(errno));
                ret = 1;
                goto out;
-       }
-
-       if (handle_args(argc, argv)) {
+       } else if (handle_args(argc, argv)) {
                ret = 1;
                goto out;
        }
@@ -2508,44 +2636,9 @@ int main(int argc, char *argv[])
                        fprintf(stderr, "-o ignored in server mode\n");
                        output_name = NULL;
                }
-
                ret = net_server();
-       } else {
-               atexit(exit_tracing);
-
-               if (net_mode == Net_client)
-                       printf("blktrace: connecting to %s\n", hostname);
-
-               setup_buts();
-
-               if (use_tracer_devpaths()) {
-                       if (setup_tracer_devpaths())
-                               goto out;
-
-                       if (piped_output)
-                               handle_list = handle_list_file;
-                       else
-                               handle_list = handle_list_net;
-               }
-
-               ntracers = start_tracers();
-               if (ntracers != ncpus)
-                       stop_tracers();
-               else {
-                       if (net_mode == Net_client)
-                               printf("blktrace: connected!\n");
-                       if (stop_watch)
-                               alarm(stop_watch);
-               }
-
-               wait_tracers();
-               if (ntracers == ncpus)
-                       show_stats(&devpaths);
-
-               if (net_client_use_send())
-                       close_client_connections();
-               del_tracers();
-       }
+       } else
+               ret = run_tracers();
 
 out:
        if (pfp)