Make use of zlib optional
authorJens Axboe <axboe@kernel.dk>
Fri, 12 Apr 2013 11:13:24 +0000 (13:13 +0200)
committerJens Axboe <axboe@kernel.dk>
Fri, 12 Apr 2013 11:13:24 +0000 (13:13 +0200)
Fio uses it to compress io logs between the client and server.
But let them handshake on this requirement when they connect
and probe, and fallback to just sending the logs in plain
format if both of them do not support zlib compression.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
client.c
configure
server.c
server.h

index eb695ff..13e84f6 100644 (file)
--- a/client.c
+++ b/client.c
@@ -14,7 +14,9 @@
 #include <arpa/inet.h>
 #include <netdb.h>
 #include <signal.h>
+#ifdef CONFIG_ZLIB
 #include <zlib.h>
+#endif
 
 #include "fio.h"
 #include "client.h"
@@ -296,9 +298,18 @@ int fio_client_add(struct client_ops *ops, const char *hostname, void **cookie)
 
 static void probe_client(struct fio_client *client)
 {
+       struct cmd_client_probe_pdu pdu;
+       uint64_t tag;
+
        dprint(FD_NET, "client: send probe\n");
 
-       fio_net_send_simple_cmd(client->fd, FIO_NET_CMD_PROBE, 0, &client->cmd_list);
+#ifdef CONFIG_ZLIB
+       pdu.flags = __le64_to_cpu(FIO_PROBE_FLAG_ZLIB);
+#else
+       pdu.flags = 0;
+#endif
+
+       fio_net_send_cmd(client->fd, FIO_NET_CMD_PROBE, &pdu, sizeof(pdu), &tag, &client->cmd_list);
 }
 
 static int fio_client_connect_ip(struct fio_client *client)
@@ -998,7 +1009,7 @@ static void handle_eta(struct fio_client *client, struct fio_net_cmd *cmd)
 
 static void handle_probe(struct fio_client *client, struct fio_net_cmd *cmd)
 {
-       struct cmd_probe_pdu *probe = (struct cmd_probe_pdu *) cmd->payload;
+       struct cmd_probe_reply_pdu *probe = (struct cmd_probe_reply_pdu *) cmd->payload;
        const char *os, *arch;
        char bit[16];
 
@@ -1011,10 +1022,11 @@ static void handle_probe(struct fio_client *client, struct fio_net_cmd *cmd)
                os = "unknown";
 
        sprintf(bit, "%d-bit", probe->bpp * 8);
+       probe->flags = le64_to_cpu(probe->flags);
 
-       log_info("hostname=%s, be=%u, %s, os=%s, arch=%s, fio=%s\n",
+       log_info("hostname=%s, be=%u, %s, os=%s, arch=%s, fio=%s, flags=%lx\n",
                probe->hostname, probe->bigendian, bit, os, arch,
-               probe->fio_version);
+               probe->fio_version, (unsigned long) probe->flags);
 
        if (!client->name)
                client->name = strdup((char *) probe->hostname);
@@ -1057,19 +1069,15 @@ static void convert_text(struct fio_net_cmd *cmd)
        pdu->log_usec   = le64_to_cpu(pdu->log_usec);
 }
 
-/*
- * This has been compressed on the server side, since it can be big.
- * Uncompress here.
- */
-static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
+static struct cmd_iolog_pdu *convert_iolog_gz(struct fio_net_cmd *cmd,
+                                             struct cmd_iolog_pdu *pdu)
 {
-       struct cmd_iolog_pdu *pdu = (struct cmd_iolog_pdu *) cmd->payload;
+#ifdef CONFIG_ZLIB
        struct cmd_iolog_pdu *ret;
-       uint32_t nr_samples;
-       unsigned long total;
        z_stream stream;
+       uint32_t nr_samples;
+       size_t total;
        void *p;
-       int i;
 
        stream.zalloc = Z_NULL;
        stream.zfree = Z_NULL;
@@ -1087,10 +1095,9 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
 
        total = nr_samples * sizeof(struct io_sample);
        ret = malloc(total + sizeof(*pdu));
-       ret->thread_number = le32_to_cpu(pdu->thread_number);
        ret->nr_samples = nr_samples;
-       ret->log_type = le32_to_cpu(pdu->log_type);
-       strcpy((char *) ret->name, (char *) pdu->name);
+
+       memcpy(ret, pdu, sizeof(*pdu));
 
        p = (void *) ret + sizeof(*pdu);
 
@@ -1112,7 +1119,7 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
                        log_err("fio: inflate error %d\n", err);
                        free(ret);
                        ret = NULL;
-                       goto out;
+                       goto err;
                }
 
                this_len = this_chunk - stream.avail_out;
@@ -1120,6 +1127,46 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
                total -= this_len;
        }
 
+err:
+       inflateEnd(&stream);
+       return ret;
+#else
+       return NULL;
+#endif
+}
+
+/*
+ * This has been compressed on the server side, since it can be big.
+ * Uncompress here.
+ */
+static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
+{
+       struct cmd_iolog_pdu *pdu = (struct cmd_iolog_pdu *) cmd->payload;
+       struct cmd_iolog_pdu *ret;
+       int i;
+
+       /*
+        * Convert if compressed and we support it. If it's not
+        * compressed, we need not do anything.
+        */
+       if (le32_to_cpu(pdu->compressed)) {
+#ifndef CONFIG_ZLIB
+               log_err("fio: server sent compressed data by mistake\n");
+               return NULL;
+#endif
+               ret = convert_iolog_gz(cmd, pdu);
+               if (!ret) {
+                       log_err("fio: failed decompressing log\n");
+                       return NULL;
+               }
+       } else
+               ret = pdu;
+
+       ret->thread_number      = le32_to_cpu(ret->thread_number);
+       ret->nr_samples         = le32_to_cpu(ret->nr_samples);
+       ret->log_type           = le32_to_cpu(ret->log_type);
+       ret->compressed         = le32_to_cpu(ret->compressed);
+
        for (i = 0; i < ret->nr_samples; i++) {
                struct io_sample *s = &ret->samples[i];
 
@@ -1129,8 +1176,6 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
                s->bs   = le32_to_cpu(s->bs);
        }
 
-out:
-       inflateEnd(&stream);
        return ret;
 }
 
index 87d659c..7be8789 100755 (executable)
--- a/configure
+++ b/configure
@@ -415,9 +415,6 @@ EOF
 if compile_prog "" "-lz" "zlib" ; then
   zlib=yes
   LIBS="-lz $LIBS"
-else
-  feature_not_found "zlib" "zlib development files"
-  zlib=no
 fi
 echo "zlib                          $zlib"
 
@@ -1083,6 +1080,9 @@ if test "$bigendian" = "yes" ; then
 else
   output_sym "CONFIG_LITTLE_ENDIAN"
 fi
+if test "$zlib" = "yes" ; then
+  output_sym "CONFIG_ZLIB"
+fi
 if test "$libaio" = "yes" ; then
   output_sym "CONFIG_LIBAIO"
 fi
index 8a6275d..020d1d7 100644 (file)
--- a/server.c
+++ b/server.c
@@ -16,7 +16,9 @@
 #include <netdb.h>
 #include <syslog.h>
 #include <signal.h>
+#ifdef CONFIG_ZLIB
 #include <zlib.h>
+#endif
 
 #include "fio.h"
 #include "server.h"
@@ -33,6 +35,12 @@ static char *bind_sock;
 static struct sockaddr_in saddr_in;
 static struct sockaddr_in6 saddr_in6;
 static int use_ipv6;
+#ifdef CONFIG_ZLIB
+static unsigned int has_zlib = 1;
+#else
+static unsigned int has_zlib = 0;
+#endif
+static unsigned int use_zlib;
 
 struct fio_fork_item {
        struct flist_head list;
@@ -616,7 +624,8 @@ static int handle_jobline_cmd(struct fio_net_cmd *cmd)
 
 static int handle_probe_cmd(struct fio_net_cmd *cmd)
 {
-       struct cmd_probe_pdu probe;
+       struct cmd_client_probe_pdu *pdu = (struct cmd_client_probe_pdu *) cmd->payload;
+       struct cmd_probe_reply_pdu probe;
        uint64_t tag = cmd->tag;
 
        dprint(FD_NET, "server: sending probe reply\n");
@@ -632,7 +641,17 @@ static int handle_probe_cmd(struct fio_net_cmd *cmd)
        probe.arch      = FIO_ARCH;
        probe.bpp       = sizeof(void *);
        probe.cpus      = __cpu_to_le32(cpus_online());
-       probe.flags     = 0;
+
+       /*
+        * If the client supports compression and we do too, then enable it
+        */
+       if (has_zlib && le64_to_cpu(pdu->flags) & FIO_PROBE_FLAG_ZLIB) {
+               probe.flags = __cpu_to_le64(FIO_PROBE_FLAG_ZLIB);
+               use_zlib = 1;
+       } else {
+               probe.flags = 0;
+               use_zlib = 0;
+       }
 
        return fio_net_send_cmd(server_fd, FIO_NET_CMD_PROBE, &probe, sizeof(probe), &tag, NULL);
 }
@@ -1117,26 +1136,12 @@ static int fio_send_cmd_ext_pdu(int sk, uint16_t opcode, const void *buf,
        return fio_sendv_data(sk, iov, 2);
 }
 
-int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
+static int fio_send_iolog_gz(struct cmd_iolog_pdu *pdu, struct io_log *log)
 {
-       struct cmd_iolog_pdu pdu;
+       int ret = 0;
+#ifdef CONFIG_ZLIB
        z_stream stream;
        void *out_pdu;
-       int i, ret = 0;
-
-       pdu.thread_number = cpu_to_le32(td->thread_number);
-       pdu.nr_samples = __cpu_to_le32(log->nr_samples);
-       pdu.log_type = cpu_to_le32(log->log_type);
-       strcpy((char *) pdu.name, name);
-
-       for (i = 0; i < log->nr_samples; i++) {
-               struct io_sample *s = &log->log[i];
-
-               s->time = cpu_to_le64(s->time);
-               s->val  = cpu_to_le64(s->val);
-               s->ddir = cpu_to_le32(s->ddir);
-               s->bs   = cpu_to_le32(s->bs);
-       }
 
        /*
         * Dirty - since the log is potentially huge, compress it into
@@ -1154,14 +1159,6 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
                goto err;
        }
 
-       /*
-        * Send header first, it's not compressed.
-        */
-       ret = fio_send_cmd_ext_pdu(server_fd, FIO_NET_CMD_IOLOG, &pdu,
-                                       sizeof(pdu), 0, FIO_NET_CMD_F_MORE);
-       if (ret)
-               goto err_zlib;
-
        stream.next_in = (void *) log->log;
        stream.avail_in = log->nr_samples * sizeof(struct io_sample);
 
@@ -1191,9 +1188,48 @@ err_zlib:
        deflateEnd(&stream);
 err:
        free(out_pdu);
+#endif
        return ret;
 }
 
+int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
+{
+       struct cmd_iolog_pdu pdu;
+       int i, ret = 0;
+
+       pdu.thread_number = cpu_to_le32(td->thread_number);
+       pdu.nr_samples = __cpu_to_le32(log->nr_samples);
+       pdu.log_type = cpu_to_le32(log->log_type);
+       pdu.compressed = cpu_to_le32(use_zlib);
+       strcpy((char *) pdu.name, name);
+
+       for (i = 0; i < log->nr_samples; i++) {
+               struct io_sample *s = &log->log[i];
+
+               s->time = cpu_to_le64(s->time);
+               s->val  = cpu_to_le64(s->val);
+               s->ddir = cpu_to_le32(s->ddir);
+               s->bs   = cpu_to_le32(s->bs);
+       }
+
+       /*
+        * Send header first, it's not compressed.
+        */
+       ret = fio_send_cmd_ext_pdu(server_fd, FIO_NET_CMD_IOLOG, &pdu,
+                                       sizeof(pdu), 0, FIO_NET_CMD_F_MORE);
+       if (ret)
+               return ret;
+
+       /*
+        * Now send actual log, compress if we can, otherwise just plain
+        */
+       if (use_zlib)
+               return fio_send_iolog_gz(&pdu, log);
+
+       return fio_send_cmd_ext_pdu(server_fd, FIO_NET_CMD_IOLOG, log->log,
+                       log->nr_samples * sizeof(struct io_sample), 0, 0);
+}
+
 void fio_server_send_add_job(struct thread_data *td)
 {
        struct cmd_add_job_pdu pdu;
index 067cdc0..478c283 100644 (file)
--- a/server.h
+++ b/server.h
@@ -38,7 +38,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-       FIO_SERVER_VER                  = 21,
+       FIO_SERVER_VER                  = 22,
 
        FIO_SERVER_MAX_FRAGMENT_PDU     = 1024,
 
@@ -71,6 +71,8 @@ enum {
        FIO_NET_NAME_MAX                = 256,
 
        FIO_NET_CLIENT_TIMEOUT          = 5000,
+
+       FIO_PROBE_FLAG_ZLIB             = 1UL << 0,
 };
 
 struct cmd_ts_pdu {
@@ -83,7 +85,11 @@ struct cmd_du_pdu {
        struct disk_util_agg agg;
 };
 
-struct cmd_probe_pdu {
+struct cmd_client_probe_pdu {
+       uint64_t flags;
+};
+
+struct cmd_probe_reply_pdu {
        uint8_t hostname[64];
        uint8_t bigendian;
        uint8_t fio_version[32];
@@ -139,6 +145,7 @@ struct cmd_iolog_pdu {
        uint32_t thread_number;
        uint32_t nr_samples;
        uint32_t log_type;
+       uint32_t compressed;
        uint8_t name[FIO_NET_NAME_MAX];
        struct io_sample samples[0];
 };