[PATCH] blktrace: fix get_subbuf() leak
[blktrace.git] / blktrace.c
index beadcf73ddd61a7742aa455bdfa2d09a6edbce16..9acb0dcc562c213ca5ad5f8572812b1d34466b85 100644 (file)
@@ -46,7 +46,7 @@
 #include "blktrace.h"
 #include "barrier.h"
 
-static char blktrace_version[] = "0.99";
+static char blktrace_version[] = "0.99.1";
 
 /*
  * You may want to increase this even more, if you are logging at a high
@@ -57,7 +57,7 @@ static char blktrace_version[] = "0.99";
 
 #define OFILE_BUF      (128 * 1024)
 
-#define RELAYFS_TYPE   0xF0B4A981
+#define DEBUGFS_TYPE   0x64626720
 
 #define S_OPTS "d:a:A:r:o:kw:Vb:n:D:lh:p:s"
 static struct option l_opts[] = {
@@ -146,7 +146,7 @@ static struct option l_opts[] = {
                .val = 'p'
        },
        {
-               .name = "sendfile",
+               .name = "no-sendfile",
                .has_arg = no_argument,
                .flag = NULL,
                .val = 's'
@@ -210,6 +210,8 @@ struct thread_information {
        unsigned long fs_off;
        void *fs_buf;
        unsigned long fs_buf_len;
+
+       struct net_connection *nc;
 };
 
 struct device_information {
@@ -218,18 +220,20 @@ struct device_information {
        char buts_name[32];
        volatile int trace_started;
        unsigned long drop_count;
-       struct in_addr cl_in_addr;
        struct thread_information *threads;
+
+       struct cl_host *ch;
+       u32 cl_id;
+       time_t cl_connect_time;
 };
 
 static int ncpus;
 static struct thread_information *thread_information;
 static int ndevs;
 static struct device_information *device_information;
-static int ndevs_running;
 
 /* command line option globals */
-static char *relay_path;
+static char *debugfs_path;
 static char *output_name;
 static char *output_dir;
 static int act_mask = ~0U;
@@ -254,12 +258,20 @@ static void exit_trace(int status);
 #define dip_tracing(dip)       (*(volatile int *)(&(dip)->trace_started))
 #define dip_set_tracing(dip, v)        ((dip)->trace_started = (v))
 
-#define __for_each_dip(__d, __i, __e)  \
-       for (__i = 0, __d = device_information; __i < __e; __i++, __d++)
+#define __for_each_dip(__d, __di, __e, __i)    \
+       for (__i = 0, __d = __di; __i < __e; __i++, __d++)
+
+#define for_each_dip(__d, __i)         \
+       __for_each_dip(__d, device_information, ndevs, __i)
+#define for_each_nc_dip(__nc, __d, __i)                \
+       __for_each_dip(__d, (__nc)->ch->device_information, (__nc)->ch->ndevs, __i)
 
-#define for_each_dip(__d, __i) __for_each_dip(__d, __i, ndevs)
+#define __for_each_tip(__d, __t, __ncpus, __j) \
+       for (__j = 0, __t = (__d)->threads; __j < __ncpus; __j++, __t++)
 #define for_each_tip(__d, __t, __j)    \
-       for (__j = 0, __t = (__d)->threads; __j < ncpus; __j++, __t++)
+       __for_each_tip(__d, __t, ncpus, __j)
+#define for_each_cl_host(__c)  \
+       for (__c = cl_host_list; __c; __c = __c->list_next)
 
 /*
  * networking stuff follows. we include a magic number so we know whether
@@ -271,6 +283,7 @@ struct blktrace_net_hdr {
        u32 cpu;                /* for which cpu */
        u32 max_cpus;
        u32 len;                /* length of following trace data */
+       u32 cl_id;              /* id for set of client per-cpu connections */
 };
 
 #define TRACE_NET_PORT         (8462)
@@ -287,11 +300,33 @@ enum {
 static char hostname[MAXHOSTNAMELEN];
 static int net_port = TRACE_NET_PORT;
 static int net_mode = 0;
-static int net_use_sendfile;
+static int net_use_sendfile = 1;
+
+struct cl_host {
+       struct cl_host *list_next;
+       struct in_addr cl_in_addr;
+       struct net_connection *net_connections;
+       int nconn;
+       struct device_information *device_information;
+       int ndevs;
+       int ncpus;
+       int ndevs_done;
+};
 
-static int net_in_fd = -1;
-static time_t net_connect_time;
-static int net_out_fd = -1;
+struct net_connection {
+       int in_fd;
+       struct pollfd pfd;
+       time_t connect_time;
+       struct cl_host *ch;
+       int ncpus;
+};
+
+#define NET_MAX_CL_HOSTS       (1024)
+static struct cl_host *cl_host_list;
+static int cl_hosts;
+static int net_connects;
+
+static int *net_out_fd;
 
 static void handle_sigint(__attribute__((__unused__)) int sig)
 {
@@ -317,7 +352,7 @@ static int get_dropped_count(const char *buts_name)
        char tmp[MAXPATHLEN + 64];
 
        snprintf(tmp, sizeof(tmp), "%s/block/%s/dropped",
-                relay_path, buts_name);
+                debugfs_path, buts_name);
 
        fd = open(tmp, O_RDONLY);
        if (fd < 0) {
@@ -395,20 +430,20 @@ static void stop_all_traces(void)
        }
 }
 
-static void wait_for_data(struct thread_information *tip, int events)
+static void wait_for_data(struct thread_information *tip, int timeout)
 {
-       struct pollfd pfd = { .fd = tip->fd, .events = events };
+       struct pollfd pfd = { .fd = tip->fd, .events = POLLIN };
 
-       do {
-               if (poll(&pfd, 1, 100) < 0) {
+       while (!is_done()) {
+               if (poll(&pfd, 1, timeout) < 0) {
                        perror("poll");
                        break;
                }
-               if (pfd.revents & events)
+               if (pfd.revents & POLLIN)
                        break;
                if (tip->ofile_stdout)
                        break;
-       } while (!is_done());
+       }
 }
 
 static int read_data_file(struct thread_information *tip, void *buf,
@@ -417,7 +452,7 @@ static int read_data_file(struct thread_information *tip, void *buf,
        int ret = 0;
 
        do {
-               wait_for_data(tip, POLLIN);
+               wait_for_data(tip, 100);
 
                ret = read(tip->fd, buf, len);
                if (!ret)
@@ -442,11 +477,12 @@ static int read_data_file(struct thread_information *tip, void *buf,
 static int read_data_net(struct thread_information *tip, void *buf,
                         unsigned int len)
 {
+       struct net_connection *nc = tip->nc;
        unsigned int bytes_left = len;
        int ret = 0;
 
        do {
-               ret = recv(net_in_fd, buf, bytes_left, MSG_WAITALL);
+               ret = recv(nc->in_fd, buf, bytes_left, MSG_WAITALL);
 
                if (!ret)
                        continue;
@@ -563,47 +599,15 @@ static int get_subbuf(struct thread_information *tip, unsigned int maxlen)
                ts->len = ret;
                tip->data_read += ret;
                if (subbuf_fifo_queue(tip, ts))
-                       return -1;
+                       ret = -1;
        }
 
-       return ret;
-}
-
-static int get_subbuf_sendfile(struct thread_information *tip,
-                              unsigned int maxlen)
-{
-       struct tip_subbuf *ts;
-       struct stat sb;
-       unsigned int ready;
-
-       wait_for_data(tip, POLLMSG);
-
-       /*
-        * hack to get last data out, we can't use sendfile for that
-        */
-       if (is_done())
-               return get_subbuf(tip, maxlen);
-
-       if (fstat(tip->fd, &sb) < 0) {
-               perror("trace stat");
-               return -1;
-       }
-       ready = sb.st_size - tip->data_queued;
-       if (!ready) {
-               usleep(1000);
-               return 0;
+       if (ret <= 0) {
+               free(ts->buf);
+               free(ts);
        }
 
-       ts = malloc(sizeof(*ts));
-       ts->buf = NULL;
-       ts->max_len = 0;
-       ts->len = ready;
-       tip->data_queued += ready;
-
-       if (subbuf_fifo_queue(tip, ts))
-               return -1;
-
-       return ready;
+       return ret;
 }
 
 static void close_thread(struct thread_information *tip)
@@ -628,7 +632,7 @@ static void tip_ftrunc_final(struct thread_information *tip)
        /*
         * truncate to right size and cleanup mmap
         */
-       if (tip->ofile_mmap) {
+       if (tip->ofile_mmap && tip->ofile) {
                int ofd = fileno(tip->ofile);
 
                if (tip->fs_buf)
@@ -653,7 +657,7 @@ static void *thread_main(void *arg)
        }
 
        snprintf(tip->fn, sizeof(tip->fn), "%s/block/%s/trace%d",
-                       relay_path, tip->device->buts_name, tip->cpu);
+                       debugfs_path, tip->device->buts_name, tip->cpu);
        tip->fd = open(tip->fn, O_RDONLY);
        if (tip->fd < 0) {
                perror(tip->fn);
@@ -706,8 +710,9 @@ static int net_send_header(struct thread_information *tip, unsigned int len)
        hdr.cpu = tip->cpu;
        hdr.max_cpus = ncpus;
        hdr.len = len;
+       hdr.cl_id = getpid();
 
-       return write_data_net(net_out_fd, &hdr, sizeof(hdr));
+       return write_data_net(net_out_fd[tip->cpu], &hdr, sizeof(hdr));
 }
 
 /*
@@ -725,8 +730,9 @@ static void net_client_send_close(void)
                hdr.len = 0;
                strcpy(hdr.buts_name, dip->buts_name);
                hdr.cpu = get_dropped_count(dip->buts_name);
+               hdr.cl_id = getpid();
 
-               write_data_net(net_out_fd, &hdr, sizeof(hdr));
+               write_data_net(net_out_fd[0], &hdr, sizeof(hdr));
        }
 
 }
@@ -736,7 +742,7 @@ static int flush_subbuf_net(struct thread_information *tip,
 {
        if (net_send_header(tip, ts->len))
                return -1;
-       if (write_data_net(net_out_fd, ts->buf, ts->len))
+       if (write_data_net(net_out_fd[tip->cpu], ts->buf, ts->len))
                return -1;
 
        free(ts->buf);
@@ -746,7 +752,7 @@ static int flush_subbuf_net(struct thread_information *tip,
 
 static int net_sendfile(struct thread_information *tip, struct tip_subbuf *ts)
 {
-       int ret = sendfile(net_out_fd, tip->fd, NULL, ts->len);
+       int ret = sendfile(net_out_fd[tip->cpu], tip->fd, NULL, ts->len);
 
        if (ret < 0) {
                perror("sendfile");
@@ -764,14 +770,6 @@ static int flush_subbuf_sendfile(struct thread_information *tip,
 {
        int ret = -1;
 
-       /*
-        * currently we cannot use sendfile() on the last bytes read, as they
-        * may not be a full subbuffer. get_subbuf_sendfile() falls back to
-        * the read approach for those, so use send() to ship them out
-        */
-       if (ts->buf)
-               return flush_subbuf_net(tip, ts);
-       
        if (net_send_header(tip, ts->len))
                goto err;
        if (net_sendfile(tip, ts))
@@ -785,6 +783,38 @@ err:
        return ret;
 }
 
+static int get_subbuf_sendfile(struct thread_information *tip,
+                              __attribute__((__unused__)) unsigned int maxlen)
+{
+       struct tip_subbuf *ts;
+       struct stat sb;
+       unsigned int ready;
+
+       wait_for_data(tip, -1);
+
+       if (fstat(tip->fd, &sb) < 0) {
+               perror("trace stat");
+               return -1;
+       }
+
+       ready = sb.st_size - tip->data_queued;
+       if (!ready) {
+               usleep(1000);
+               return 0;
+       }
+
+       ts = malloc(sizeof(*ts));
+       ts->buf = NULL;
+       ts->max_len = 0;
+       ts->len = ready;
+       tip->data_queued += ready;
+
+       if (flush_subbuf_sendfile(tip, ts) < 0)
+               return -1;
+
+       return ready;
+}
+
 static int write_data(struct thread_information *tip, void *buf,
                      unsigned int buf_len)
 {
@@ -934,7 +964,7 @@ static void wait_for_threads(void)
         * for files, we just wait around for trace threads to exit
         */
        if ((output_name && !strcmp(output_name, "-")) ||
-           net_mode == Net_client)
+           ((net_mode == Net_client) && !net_use_sendfile))
                get_and_write_events();
        else {
                struct device_information *dip;
@@ -964,10 +994,14 @@ static int fill_ofname(struct device_information *dip,
 
        if (output_dir)
                len = sprintf(dst, "%s/", output_dir);
+       else
+               len = sprintf(dst, "./");
 
        if (net_mode == Net_server) {
-               len += sprintf(dst + len, "%s-", inet_ntoa(dip->cl_in_addr));
-               len += strftime(dst + len, 64, "%F-%T/", gmtime(&net_connect_time));
+               struct net_connection *nc = tip->nc;
+
+               len += sprintf(dst + len, "%s-", inet_ntoa(nc->ch->cl_in_addr));
+               len += strftime(dst + len, 64, "%F-%T/", gmtime(&dip->cl_connect_time));
        }
 
        if (stat(dst, &sb) < 0) {
@@ -998,7 +1032,7 @@ static void fill_ops(struct thread_information *tip)
        if (net_mode == Net_client) {
                if (net_use_sendfile) {
                        tip->get_subbuf = get_subbuf_sendfile;
-                       tip->flush_subbuf = flush_subbuf_sendfile;
+                       tip->flush_subbuf = NULL;
                } else {
                        tip->get_subbuf = get_subbuf;
                        tip->flush_subbuf = flush_subbuf_net;
@@ -1182,7 +1216,7 @@ static int start_devices(void)
        }
 
        if (i != ndevs) {
-               __for_each_dip(dip, j, i)
+               __for_each_dip(dip, device_information, i, j)
                        stop_trace(dip);
 
                return 1;
@@ -1197,7 +1231,7 @@ static int start_devices(void)
        }
 
        if (i != ndevs) {
-               __for_each_dip(dip, j, i)
+               __for_each_dip(dip, device_information, i, j)
                        stop_threads(dip);
                for_each_dip(dip, i)
                        stop_trace(dip);
@@ -1208,7 +1242,7 @@ static int start_devices(void)
        return 0;
 }
 
-static void show_stats(void)
+static void show_stats(struct device_information *dips, int ndips, int cpus)
 {
        struct device_information *dip;
        struct thread_information *tip;
@@ -1225,12 +1259,12 @@ static void show_stats(void)
        stat_shown = 1;
 
        total_drops = 0;
-       for_each_dip(dip, i) {
+       __for_each_dip(dip, dips, ndips, i) {
                if (!no_stdout)
                        printf("Device: %s\n", dip->path);
                events_processed = 0;
                data_read = 0;
-               for_each_tip(dip, tip, j) {
+               __for_each_tip(dip, tip, cpus, j) {
                        if (!no_stdout)
                                printf("  CPU%3d: %20lu events, %8llu KiB data\n",
                                        tip->cpu, tip->events_processed,
@@ -1249,74 +1283,90 @@ static void show_stats(void)
                fprintf(stderr, "You have dropped events, consider using a larger buffer size (-b)\n");
 }
 
-static struct device_information *net_get_dip(char *buts_name,
-                                             struct in_addr *cl_in_addr)
+static struct device_information *net_get_dip(struct net_connection *nc,
+                                             char *buts_name, u32 cl_id)
 {
-       struct device_information *dip;
+       struct device_information *dip, *cl_dip = NULL;
+       struct cl_host *ch = nc->ch;
        int i;
 
-       for (i = 0; i < ndevs; i++) {
-               dip = &device_information[i];
+       for (i = 0; i < ch->ndevs; i++) {
+               dip = &ch->device_information[i];
 
                if (!strcmp(dip->buts_name, buts_name))
                        return dip;
+
+               if (dip->cl_id == cl_id)
+                       cl_dip = dip;
        }
 
-       device_information = realloc(device_information, (ndevs + 1) * sizeof(*dip));
-       dip = &device_information[ndevs];
+       ch->device_information = realloc(ch->device_information, (ch->ndevs + 1) * sizeof(*dip));
+       dip = &ch->device_information[ch->ndevs];
        memset(dip, 0, sizeof(*dip));
        dip->fd = -1;
-       dip->cl_in_addr = *cl_in_addr;
+       dip->ch = ch;
+       dip->cl_id = cl_id;
+       if (cl_dip)
+               dip->cl_connect_time = cl_dip->cl_connect_time;
+       else
+               dip->cl_connect_time = nc->connect_time;
        strcpy(dip->buts_name, buts_name);
        dip->path = strdup(buts_name);
        dip->trace_started = 1;
-       ndevs++;
-       ndevs_running++;
-       dip->threads = malloc(ncpus * sizeof(struct thread_information));
-       memset(dip->threads, 0, ncpus * sizeof(struct thread_information));
+       ch->ndevs++;
+       dip->threads = malloc(nc->ncpus * sizeof(struct thread_information));
+       memset(dip->threads, 0, nc->ncpus * sizeof(struct thread_information));
 
        /*
         * open all files
         */
-       for (i = 0; i < ncpus; i++) {
+       for (i = 0; i < nc->ncpus; i++) {
                struct thread_information *tip = &dip->threads[i];
 
                tip->cpu = i;
                tip->device = dip;
                tip->fd = -1;
-
+               tip->nc = nc;
+               
                if (tip_open_output(dip, tip))
                        return NULL;
+
+               tip->nc = NULL;
        }
 
        return dip;
 }
 
-static struct thread_information *net_get_tip(struct blktrace_net_hdr *bnh,
-                                             struct in_addr *cl_in_addr)
+static struct thread_information *net_get_tip(struct net_connection *nc,
+                                             struct blktrace_net_hdr *bnh)
 {
        struct device_information *dip;
+       struct thread_information *tip;
 
-       ncpus = bnh->max_cpus;
-       dip = net_get_dip(bnh->buts_name, cl_in_addr);
+       dip = net_get_dip(nc, bnh->buts_name, bnh->cl_id);
        if (!dip->trace_started) {
                fprintf(stderr, "Events for closed devices %s\n", dip->buts_name);
                return NULL;
        }
 
-       return &dip->threads[bnh->cpu];
+       tip = &dip->threads[bnh->cpu];
+       if (!tip->nc)
+               tip->nc = nc;
+       
+       return tip;
 }
 
-static int net_get_header(struct blktrace_net_hdr *bnh)
+static int net_get_header(struct net_connection *nc,
+                         struct blktrace_net_hdr *bnh)
 {
-       int fl = fcntl(net_in_fd, F_GETFL);
+       int fl = fcntl(nc->in_fd, F_GETFL);
        int bytes_left, ret;
        void *p = bnh;
 
-       fcntl(net_in_fd, F_SETFL, fl | O_NONBLOCK);
+       fcntl(nc->in_fd, F_SETFL, fl | O_NONBLOCK);
        bytes_left = sizeof(*bnh);
        while (bytes_left && !is_done()) {
-               ret = recv(net_in_fd, p, bytes_left, MSG_WAITALL);
+               ret = recv(nc->in_fd, p, bytes_left, MSG_WAITALL);
                if (ret < 0) {
                        if (errno != EAGAIN) {
                                perror("recv header");
@@ -1332,16 +1382,100 @@ static int net_get_header(struct blktrace_net_hdr *bnh)
                        bytes_left -= ret;
                }
        }
-       fcntl(net_in_fd, F_SETFL, fl & ~O_NONBLOCK);
+       fcntl(nc->in_fd, F_SETFL, fl & ~O_NONBLOCK);
        return bytes_left;
 }
 
-static int net_server_loop(struct in_addr *cl_in_addr)
+/*
+ * finalize a net client: truncate files, show stats, cleanup, etc
+ */
+static void device_done(struct net_connection *nc, struct device_information *dip)
+{
+       struct thread_information *tip;
+       int i;
+
+       __for_each_tip(dip, tip, nc->ncpus, i)
+               tip_ftrunc_final(tip);
+
+       show_stats(dip, 1, nc->ncpus);
+
+       /*
+        * cleanup for next run
+        */
+       __for_each_tip(dip, tip, nc->ncpus, i) {
+               if (tip->ofile)
+                       fclose(tip->ofile);
+       }
+
+       free(dip->threads);
+       free(dip->path);
+
+       close(nc->in_fd);
+       nc->in_fd = -1;
+
+       stat_shown = 0;
+}
+
+static inline int in_addr_eq(struct in_addr a, struct in_addr b)
+{
+       return a.s_addr == b.s_addr;
+}
+
+static void net_add_client_host(struct cl_host *ch)
+{
+       ch->list_next = cl_host_list;
+       cl_host_list = ch;
+       cl_hosts++;
+}
+
+static void net_remove_client_host(struct cl_host *ch)
+{
+       struct cl_host *p, *c;
+       
+       for (p = c = cl_host_list; c; c = c->list_next) {
+               if (c == ch) {
+                       if (p == c)
+                               cl_host_list = c->list_next;
+                       else
+                               p->list_next = c->list_next;
+                       cl_hosts--;
+                       return;
+               }
+               p = c;
+       }
+}
+
+static struct cl_host *net_find_client_host(struct in_addr cl_in_addr)
+{
+       struct cl_host *ch = cl_host_list;
+
+       while (ch) {
+               if (in_addr_eq(ch->cl_in_addr, cl_in_addr))
+                       return ch;
+               ch = ch->list_next;
+       }
+
+       return NULL;
+}
+
+static void net_client_host_done(struct cl_host *ch)
+{
+       free(ch->device_information);
+       free(ch->net_connections);
+       net_connects -= ch->nconn;
+       net_remove_client_host(ch);
+       free(ch);
+}
+
+/*
+ * handle incoming events from a net client
+ */
+static int net_client_data(struct net_connection *nc)
 {
        struct thread_information *tip;
        struct blktrace_net_hdr bnh;
 
-       if (net_get_header(&bnh))
+       if (net_get_header(nc, &bnh))
                return 1;
 
        if (data_is_native == -1 && check_data_endianness(bnh.magic)) {
@@ -1352,7 +1486,9 @@ static int net_server_loop(struct in_addr *cl_in_addr)
        if (!data_is_native) {
                bnh.magic = be32_to_cpu(bnh.magic);
                bnh.cpu = be32_to_cpu(bnh.cpu);
+               bnh.max_cpus = be32_to_cpu(bnh.max_cpus);
                bnh.len = be32_to_cpu(bnh.len);
+               bnh.cl_id = be32_to_cpu(bnh.cl_id);
        }
 
        if ((bnh.magic & 0xffffff00) != BLK_IO_TRACE_MAGIC) {
@@ -1360,6 +1496,9 @@ static int net_server_loop(struct in_addr *cl_in_addr)
                return 1;
        }
 
+       if (nc->ncpus == -1)
+               nc->ncpus = bnh.max_cpus;
+
        /*
         * len == 0 means that the other end signalled end-of-run
         */
@@ -1369,16 +1508,21 @@ static int net_server_loop(struct in_addr *cl_in_addr)
                 */
                struct device_information *dip;
 
-               dip = net_get_dip(bnh.buts_name, cl_in_addr);
+               dip = net_get_dip(nc, bnh.buts_name, bnh.cl_id);
                dip->drop_count = bnh.cpu;
                dip->trace_started = 0;
-               ndevs_running--;
 
-               fprintf(stderr, "server: end of run for %s\n", dip->buts_name);
-               return !ndevs_running;
+               printf("server: end of run for %s\n", dip->buts_name);
+
+               device_done(nc, dip);
+
+               if (++nc->ch->ndevs_done == nc->ch->ndevs)
+                       net_client_host_done(nc->ch);
+
+               return 0;
        }
 
-       tip = net_get_tip(&bnh, cl_in_addr);
+       tip = net_get_tip(nc, &bnh);
        if (!tip)
                return 1;
 
@@ -1388,30 +1532,104 @@ static int net_server_loop(struct in_addr *cl_in_addr)
        return 0;
 }
 
-static int get_connection(int fd, struct sockaddr_in *addr)
+static void net_add_connection(int listen_fd, struct sockaddr_in *addr)
 {
-       struct pollfd pfd = { .fd = fd, .events = POLLIN };
-       socklen_t socklen;
+       socklen_t socklen = sizeof(*addr);
+       struct net_connection *nc;
+       struct cl_host *ch;
+       int in_fd;
 
-       printf("blktrace: waiting for incoming connection...\n");
-
-       if (poll(&pfd, 1, -1) < 0) {
-               perror("poll for connection");
-               return 1;
+       in_fd = accept(listen_fd, (struct sockaddr *) addr, &socklen);
+       if (in_fd < 0) {
+               perror("accept");
+               return;
        }
-       if ((pfd.revents & POLLIN) == 0)
-               return 1;
 
-       socklen = sizeof(*addr);
-       net_in_fd = accept(fd, (struct sockaddr *) addr, &socklen);
-       if (net_in_fd < 0) {
-               perror("accept");
-               return 1;
+       ch = net_find_client_host(addr->sin_addr);
+       if (!ch) {
+               if (cl_hosts == NET_MAX_CL_HOSTS) {
+                       fprintf(stderr, "server: no more clients allowed\n");
+                       return;
+               }
+               ch = malloc(sizeof(struct cl_host));
+               memset(ch, 0, sizeof(*ch));
+               ch->cl_in_addr = addr->sin_addr;
+               net_add_client_host(ch);
+
+               printf("server: connection from %s\n", inet_ntoa(addr->sin_addr));
        }
 
-       time(&net_connect_time);
-       printf("blktrace: connection from %s\n", inet_ntoa(addr->sin_addr));
-       return 0;
+       ch->net_connections = realloc(ch->net_connections, (ch->nconn + 1) * sizeof(*nc));
+       nc = &ch->net_connections[ch->nconn++];
+       memset(nc, 0, sizeof(*nc));
+
+       time(&nc->connect_time);
+       nc->ch = ch;
+       nc->in_fd = in_fd;
+       nc->ncpus = -1;
+       net_connects++;
+}
+
+/*
+ * event driven loop, handle new incoming connections and data from
+ * existing connections
+ */
+static void net_server_handle_connections(int listen_fd,
+                                         struct sockaddr_in *addr)
+{
+       struct pollfd *pfds = NULL;
+       struct net_connection **ncs = NULL;
+       int max_connects = 0;
+       int i, nconns, events;
+       struct cl_host *ch;
+       struct net_connection *nc;
+       
+       printf("server: waiting for connections...\n");
+
+       while (!is_done()) {
+               if (net_connects >= max_connects) {
+                       pfds = realloc(pfds, (net_connects + 1) * sizeof(*pfds));
+                       ncs = realloc(ncs, (net_connects + 1) * sizeof(*ncs));
+                       max_connects = net_connects + 1;
+               }
+               /*
+                * the zero entry is for incoming connections, remaining
+                * entries for clients
+                */
+               pfds[0].fd = listen_fd;
+               pfds[0].events = POLLIN;
+               nconns = 0;
+               for_each_cl_host(ch) {
+                       for (i = 0; i < ch->nconn; i++) {
+                               nc = &ch->net_connections[i];
+                               pfds[nconns + 1].fd = nc->in_fd;
+                               pfds[nconns + 1].events = POLLIN;
+                               ncs[nconns++] = nc;
+                       }
+               }
+
+               events = poll(pfds, 1 + nconns, -1);
+               if (events < 0) {
+                       if (errno == EINTR)
+                               continue;
+
+                       perror("poll");
+                       break;
+               } else if (!events)
+                       continue;
+
+               if (pfds[0].revents & POLLIN) {
+                       net_add_connection(listen_fd, addr);
+                       events--;
+               }
+
+               for (i = 0; events && i < nconns; i++) {
+                       if (pfds[i + 1].revents & POLLIN) {
+                               net_client_data(ncs[i]);
+                               events--;
+                       }
+               }
+       }
 }
 
 /*
@@ -1420,10 +1638,8 @@ static int get_connection(int fd, struct sockaddr_in *addr)
  */
 static int net_server(void)
 {
-       struct device_information *dip;
-       struct thread_information *tip;
        struct sockaddr_in addr;
-       int fd, opt, i, j;
+       int fd, opt;
 
        fd = socket(AF_INET, SOCK_STREAM, 0);
        if (fd < 0) {
@@ -1452,51 +1668,15 @@ static int net_server(void)
                return 1;
        }
 
-repeat:
-       if (get_connection(fd, &addr))
-               return 0;
-
-       while (!is_done()) {
-               if (net_server_loop(&addr.sin_addr))
-                       break;
-       }
-
-       for_each_dip(dip, i)
-               for_each_tip(dip, tip, j)
-                       tip_ftrunc_final(tip);
-
-       show_stats();
-
-       if (is_done())
-               return 0;
-
-       /*
-        * cleanup for next run
-        */
-       for_each_dip(dip, i) {
-               for_each_tip(dip, tip, j)
-                       fclose(tip->ofile);
-
-               free(dip->threads);
-               free(dip->path);
-       }
-
-       free(device_information);
-       device_information = NULL;
-       ncpus = ndevs = 0;
-
-       close(net_in_fd);
-       net_in_fd = -1;
-       stat_shown = 0;
-       goto repeat;
+       net_server_handle_connections(fd, &addr);
+       return 0;
 }
 
 /*
  * Setup outgoing network connection where we will transmit data
  */
-static int net_setup_client(void)
+static int net_setup_client_cpu(int i, struct sockaddr_in *addr)
 {
-       struct sockaddr_in addr;
        int fd;
 
        fd = socket(AF_INET, SOCK_STREAM, 0);
@@ -1505,6 +1685,20 @@ static int net_setup_client(void)
                return 1;
        }
 
+       if (connect(fd, (struct sockaddr *) addr, sizeof(*addr)) < 0) {
+               perror("client: connect");
+               return 1;
+       }
+
+       net_out_fd[i] = fd;
+       return 0;
+}
+
+static int net_setup_client(void)
+{
+       struct sockaddr_in addr;
+       int i;
+
        memset(&addr, 0, sizeof(addr));
        addr.sin_family = AF_INET;
        addr.sin_port = htons(net_port);
@@ -1522,21 +1716,22 @@ static int net_setup_client(void)
 
        printf("blktrace: connecting to %s\n", hostname);
 
-       if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) < 0) {
-               perror("client: connect");
-               return 1;
+       net_out_fd = malloc(ncpus * sizeof(*net_out_fd));
+       for (i = 0; i < ncpus; i++) {
+               if (net_setup_client_cpu(i, &addr))
+                       return 1;
        }
 
        printf("blktrace: connected!\n");
-       net_out_fd = fd;
+       
        return 0;
 }
 
 static char usage_str[] = \
-       "-d <dev> [ -r relay path ] [ -o <output> ] [-k ] [ -w time ]\n" \
+       "-d <dev> [ -r debugfs path ] [ -o <output> ] [-k ] [ -w time ]\n" \
        "[ -a action ] [ -A action mask ] [ -v ]\n\n" \
        "\t-d Use specified device. May also be given last after options\n" \
-       "\t-r Path to mounted relayfs, defaults to /relay\n" \
+       "\t-r Path to mounted debugfs, defaults to /debug\n" \
        "\t-o File(s) to send output to\n" \
        "\t-D Directory to prepend to output file names\n" \
        "\t-k Kill a running trace\n" \
@@ -1548,7 +1743,7 @@ static char usage_str[] = \
        "\t-l Run in network listen mode (blktrace server)\n" \
        "\t-h Run in network client mode, connecting to the given host\n" \
        "\t-p Network port to use (default 8462)\n" \
-       "\t-s Make the network client use sendfile() to transfer data\n" \
+       "\t-s Make the network client NOT use sendfile() to transfer data\n" \
        "\t-V Print program version info\n\n";
 
 static void show_usage(char *program)
@@ -1558,7 +1753,7 @@ static void show_usage(char *program)
 
 int main(int argc, char *argv[])
 {
-       static char default_relay_path[] = "/relay";
+       static char default_debugfs_path[] = "/debug";
        struct statfs st;
        int i, c;
        int stop_watch = 0;
@@ -1593,7 +1788,7 @@ int main(int argc, char *argv[])
                        break;
 
                case 'r':
-                       relay_path = optarg;
+                       debugfs_path = optarg;
                        break;
 
                case 'o':
@@ -1645,7 +1840,7 @@ int main(int argc, char *argv[])
                        net_port = atoi(optarg);
                        break;
                case 's':
-                       net_use_sendfile = 1;
+                       net_use_sendfile = 0;
                        break;
                default:
                        show_usage(argv[0]);
@@ -1670,20 +1865,20 @@ int main(int argc, char *argv[])
                return 1;
        }
 
-       if (!relay_path)
-               relay_path = default_relay_path;
-
        if (act_mask_tmp != 0)
                act_mask = act_mask_tmp;
 
-       if (statfs(relay_path, &st) < 0) {
+       if (!debugfs_path)
+               debugfs_path = default_debugfs_path;
+
+       if (statfs(debugfs_path, &st) < 0) {
                perror("statfs");
                fprintf(stderr,"%s does not appear to be a valid path\n",
-                       relay_path);
+                       debugfs_path);
                return 1;
-       } else if (st.f_type != (long) RELAYFS_TYPE) {
-               fprintf(stderr,"%s does not appear to be a relay filesystem\n",
-                       relay_path);
+       } else if (st.f_type != (long) DEBUGFS_TYPE) {
+               fprintf(stderr,"%s does not appear to be a debug filesystem\n",
+                       debugfs_path);
                return 1;
        }
 
@@ -1725,7 +1920,7 @@ int main(int argc, char *argv[])
                stop_all_traces();
        }
 
-       show_stats();
+       show_stats(device_information, ndevs, ncpus);
 
        return 0;
 }