t/nvmept_trim: increase transfer size for some tests
[fio.git] / server.c
index 90c7057719d200871f1a325a6ad042f8a610b775..afaeb3482b0fd0072c123bf78b925b5a35176bf1 100644 (file)
--- a/server.c
+++ b/server.c
@@ -1,5 +1,6 @@
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
 #include <errno.h>
 #include <poll.h>
@@ -28,7 +29,7 @@
 
 int fio_net_port = FIO_NET_PORT;
 
-int exit_backend = 0;
+bool exit_backend = false;
 
 enum {
        SK_F_FREE       = 1,
@@ -63,12 +64,28 @@ static char me[128];
 
 static pthread_key_t sk_out_key;
 
+#ifdef WIN32
+static char *fio_server_pipe_name  = NULL;
+static HANDLE hjob = INVALID_HANDLE_VALUE;
+struct ffi_element {
+       union {
+               pthread_t thread;
+               HANDLE hProcess;
+       };
+       bool is_thread;
+};
+#endif
+
 struct fio_fork_item {
        struct flist_head list;
        int exitval;
        int signal;
        int exited;
+#ifdef WIN32
+       struct ffi_element element;
+#else
        pid_t pid;
+#endif
 };
 
 struct cmd_reply {
@@ -250,6 +267,28 @@ static int fio_send_data(int sk, const void *p, unsigned int len)
        return fio_sendv_data(sk, &iov, 1);
 }
 
+bool fio_server_poll_fd(int fd, short events, int timeout)
+{
+       struct pollfd pfd = {
+               .fd     = fd,
+               .events = events,
+       };
+       int ret;
+
+       ret = poll(&pfd, 1, timeout);
+       if (ret < 0) {
+               if (errno == EINTR)
+                       return false;
+               log_err("fio: poll: %s\n", strerror(errno));
+               return false;
+       } else if (!ret) {
+               return false;
+       }
+       if (pfd.revents & events)
+               return true;
+       return false;
+}
+
 static int fio_recv_data(int sk, void *buf, unsigned int len, bool wait)
 {
        int flags;
@@ -296,6 +335,8 @@ static int verify_convert_cmd(struct fio_net_cmd *cmd)
        if (crc != cmd->cmd_crc16) {
                log_err("fio: server bad crc on command (got %x, wanted %x)\n",
                                cmd->cmd_crc16, crc);
+               fprintf(f_err, "fio: server bad crc on command (got %x, wanted %x)\n",
+                               cmd->cmd_crc16, crc);
                return 1;
        }
 
@@ -310,6 +351,8 @@ static int verify_convert_cmd(struct fio_net_cmd *cmd)
                break;
        default:
                log_err("fio: bad server cmd version %d\n", cmd->version);
+               fprintf(f_err, "fio: client/server version mismatch (%d != %d)\n",
+                               cmd->version, FIO_SERVER_VER);
                return 1;
        }
 
@@ -405,8 +448,9 @@ struct fio_net_cmd *fio_net_recv_cmd(int sk, bool wait)
                        if (cmdret->opcode == FIO_NET_CMD_TEXT) {
                                struct cmd_text_pdu *__pdu = (struct cmd_text_pdu *) cmdret->payload;
                                char *buf = (char *) __pdu->buf;
+                               int len = le32_to_cpu(__pdu->buf_len);
 
-                               buf[__pdu->buf_len] = '\0';
+                               buf[len] = '\0';
                        } else if (cmdret->opcode == FIO_NET_CMD_JOB) {
                                struct cmd_job_pdu *__pdu = (struct cmd_job_pdu *) cmdret->payload;
                                char *buf = (char *) __pdu->buf;
@@ -646,6 +690,63 @@ static int fio_net_queue_stop(int error, int signal)
        return fio_net_send_ack(NULL, error, signal);
 }
 
+#ifdef WIN32
+static void fio_server_add_fork_item(struct ffi_element *element, struct flist_head *list)
+{
+       struct fio_fork_item *ffi;
+
+       ffi = malloc(sizeof(*ffi));
+       ffi->exitval = 0;
+       ffi->signal = 0;
+       ffi->exited = 0;
+       ffi->element = *element;
+       flist_add_tail(&ffi->list, list);
+}
+
+static void fio_server_add_conn_pid(struct flist_head *conn_list, HANDLE hProcess)
+{
+       struct ffi_element element = {.hProcess = hProcess, .is_thread=FALSE};
+       dprint(FD_NET, "server: forked off connection job (tid=%u)\n", (int) element.thread);
+
+       fio_server_add_fork_item(&element, conn_list);
+}
+
+static void fio_server_add_job_pid(struct flist_head *job_list, pthread_t thread)
+{
+       struct ffi_element element = {.thread = thread, .is_thread=TRUE};
+       dprint(FD_NET, "server: forked off job job (tid=%u)\n", (int) element.thread);
+       fio_server_add_fork_item(&element, job_list);
+}
+
+static void fio_server_check_fork_item(struct fio_fork_item *ffi)
+{
+       int ret;
+
+       if (ffi->element.is_thread) {
+
+               ret = pthread_kill(ffi->element.thread, 0);
+               if (ret) {
+                       int rev_val;
+                       pthread_join(ffi->element.thread, (void**) &rev_val); /*if the thread is dead, then join it to get status*/
+
+                       ffi->exitval = rev_val;
+                       if (ffi->exitval)
+                               log_err("thread (tid=%u) exited with %x\n", (int) ffi->element.thread, (int) ffi->exitval);
+                       dprint(FD_PROCESS, "thread (tid=%u) exited with %x\n", (int) ffi->element.thread, (int) ffi->exitval);
+                       ffi->exited = 1;
+               }
+       } else {
+               DWORD exit_val;
+               GetExitCodeProcess(ffi->element.hProcess, &exit_val);
+
+               if (exit_val != STILL_ACTIVE) {
+                       dprint(FD_PROCESS, "process %u exited with %d\n", GetProcessId(ffi->element.hProcess), exit_val);
+                       ffi->exited = 1;
+                       ffi->exitval = exit_val;
+               }
+       }
+}
+#else
 static void fio_server_add_fork_item(pid_t pid, struct flist_head *list)
 {
        struct fio_fork_item *ffi;
@@ -693,10 +794,21 @@ static void fio_server_check_fork_item(struct fio_fork_item *ffi)
                }
        }
 }
+#endif
 
 static void fio_server_fork_item_done(struct fio_fork_item *ffi, bool stop)
 {
+#ifdef WIN32
+       if (ffi->element.is_thread)
+               dprint(FD_NET, "tid %u exited, sig=%u, exitval=%d\n", (int) ffi->element.thread, ffi->signal, ffi->exitval);
+       else {
+               dprint(FD_NET, "pid %u exited, sig=%u, exitval=%d\n", (int)  GetProcessId(ffi->element.hProcess), ffi->signal, ffi->exitval);
+               CloseHandle(ffi->element.hProcess);
+               ffi->element.hProcess = INVALID_HANDLE_VALUE;
+       }
+#else
        dprint(FD_NET, "pid %u exited, sig=%u, exitval=%d\n", (int) ffi->pid, ffi->signal, ffi->exitval);
+#endif
 
        /*
         * Fold STOP and QUIT...
@@ -757,27 +869,62 @@ static int handle_load_file_cmd(struct fio_net_cmd *cmd)
        return 0;
 }
 
-static int handle_run_cmd(struct sk_out *sk_out, struct flist_head *job_list,
-                         struct fio_net_cmd *cmd)
+#ifdef WIN32
+static void *fio_backend_thread(void *data)
 {
-       pid_t pid;
        int ret;
+       struct sk_out *sk_out = (struct sk_out *) data;
 
        sk_out_assign(sk_out);
 
+       ret = fio_backend(sk_out);
+       sk_out_drop();
+
+       pthread_exit((void*) (intptr_t) ret);
+       return NULL;
+}
+#endif
+
+static int handle_run_cmd(struct sk_out *sk_out, struct flist_head *job_list,
+                         struct fio_net_cmd *cmd)
+{
+       int ret;
+
        fio_time_init();
        set_genesis_time();
 
-       pid = fork();
-       if (pid) {
-               fio_server_add_job_pid(job_list, pid);
-               return 0;
+#ifdef WIN32
+       {
+               pthread_t thread;
+               /* both this thread and backend_thread call sk_out_assign() to double increment
+                * the ref count.  This ensures struct is valid until both threads are done with it
+                */
+               sk_out_assign(sk_out);
+               ret = pthread_create(&thread, NULL,     fio_backend_thread, sk_out);
+               if (ret) {
+                       log_err("pthread_create: %s\n", strerror(ret));
+                       return ret;
+               }
+
+               fio_server_add_job_pid(job_list, thread);
+               return ret;
        }
+#else
+    {
+               pid_t pid;
+               sk_out_assign(sk_out);
+               pid = fork();
+               if (pid) {
+                       fio_server_add_job_pid(job_list, pid);
+                       return 0;
+               }
 
-       ret = fio_backend(sk_out);
-       free_threads_shm();
-       sk_out_drop();
-       _exit(ret);
+               ret = fio_backend(sk_out);
+               free_threads_shm();
+               sk_out_drop();
+               _exit(ret);
+       }
+#endif
 }
 
 static int handle_job_cmd(struct fio_net_cmd *cmd)
@@ -853,7 +1000,7 @@ static int handle_probe_cmd(struct fio_net_cmd *cmd)
                .os             = FIO_OS,
                .arch           = FIO_ARCH,
                .bpp            = sizeof(void *),
-               .cpus           = __cpu_to_le32(cpus_online()),
+               .cpus           = __cpu_to_le32(cpus_configured()),
        };
 
        dprint(FD_NET, "server: sending probe reply\n");
@@ -861,7 +1008,8 @@ static int handle_probe_cmd(struct fio_net_cmd *cmd)
        strcpy(me, (char *) pdu->server);
 
        gethostname((char *) probe.hostname, sizeof(probe.hostname));
-       strncpy((char *) probe.fio_version, fio_version_string, sizeof(probe.fio_version) - 1);
+       snprintf((char *) probe.fio_version, sizeof(probe.fio_version), "%s",
+                fio_version_string);
 
        /*
         * If the client supports compression and we do too, then enable it
@@ -935,6 +1083,7 @@ static int handle_update_job_cmd(struct fio_net_cmd *cmd)
        struct cmd_add_job_pdu *pdu = (struct cmd_add_job_pdu *) cmd->payload;
        struct thread_data *td;
        uint32_t tnumber;
+       int ret;
 
        tnumber = le32_to_cpu(pdu->thread_number);
 
@@ -945,9 +1094,10 @@ static int handle_update_job_cmd(struct fio_net_cmd *cmd)
                return 0;
        }
 
-       td = &threads[tnumber - 1];
-       convert_thread_options_to_cpu(&td->o, &pdu->top);
-       send_update_job_reply(cmd->tag, 0);
+       td = tnumber_to_td(tnumber);
+       ret = convert_thread_options_to_cpu(&td->o, &pdu->top,
+                       cmd->pdu_len - offsetof(struct cmd_add_job_pdu, top));
+       send_update_job_reply(cmd->tag, ret);
        return 0;
 }
 
@@ -970,7 +1120,7 @@ static int handle_trigger_cmd(struct fio_net_cmd *cmd, struct flist_head *job_li
        } else
                fio_net_queue_cmd(FIO_NET_CMD_VTRIGGER, rep, sz, NULL, SK_F_FREE | SK_F_INLINE);
 
-       fio_terminate_threads(TERMINATE_ALL);
+       fio_terminate_threads(TERMINATE_ALL, TERMINATE_ALL);
        fio_server_check_jobs(job_list);
        exec_trigger(buf);
        return 0;
@@ -987,11 +1137,11 @@ static int handle_command(struct sk_out *sk_out, struct flist_head *job_list,
 
        switch (cmd->opcode) {
        case FIO_NET_CMD_QUIT:
-               fio_terminate_threads(TERMINATE_ALL);
+               fio_terminate_threads(TERMINATE_ALL, TERMINATE_ALL);
                ret = 0;
                break;
        case FIO_NET_CMD_EXIT:
-               exit_backend = 1;
+               exit_backend = true;
                return -1;
        case FIO_NET_CMD_LOAD_FILE:
                ret = handle_load_file_cmd(cmd);
@@ -1176,7 +1326,7 @@ static int handle_xmits(struct sk_out *sk_out)
        sk_unlock(sk_out);
 
        while (!flist_empty(&list)) {
-               entry = flist_entry(list.next, struct sk_entry, list);
+               entry = flist_first_entry(&list, struct sk_entry, list);
                flist_del(&entry->list);
                ret += handle_sk_entry(sk_out, entry);
        }
@@ -1199,7 +1349,6 @@ static int handle_connection(struct sk_out *sk_out)
                        .events = POLLIN,
                };
 
-               ret = 0;
                do {
                        int timeout = 1000;
 
@@ -1233,7 +1382,8 @@ static int handle_connection(struct sk_out *sk_out)
                if (ret < 0)
                        break;
 
-               cmd = fio_net_recv_cmd(sk_out->sk, true);
+               if (pfd.revents & POLLIN)
+                       cmd = fio_net_recv_cmd(sk_out->sk, true);
                if (!cmd) {
                        ret = -1;
                        break;
@@ -1258,7 +1408,7 @@ static int handle_connection(struct sk_out *sk_out)
        _exit(ret);
 }
 
-/* get the address on this host bound by the input socket, 
+/* get the address on this host bound by the input socket,
  * whether it is ipv6 or ipv4 */
 
 static int get_my_addr_str(int sk)
@@ -1295,6 +1445,73 @@ static int get_my_addr_str(int sk)
        return 0;
 }
 
+#ifdef WIN32
+static int handle_connection_process(void)
+{
+       WSAPROTOCOL_INFO protocol_info;
+       DWORD bytes_read;
+       HANDLE hpipe;
+       int sk;
+       struct sk_out *sk_out;
+       int ret;
+       char *msg = (char *) "connected";
+
+       log_info("server enter accept loop.  ProcessID %d\n", GetCurrentProcessId());
+
+       hpipe = CreateFile(
+                                       fio_server_pipe_name,
+                                       GENERIC_READ | GENERIC_WRITE,
+                                       0, NULL,
+                                       OPEN_EXISTING,
+                                       0, NULL);
+
+       if (hpipe == INVALID_HANDLE_VALUE) {
+               log_err("couldnt open pipe %s error %lu\n",
+                               fio_server_pipe_name, GetLastError());
+               return -1;
+       }
+
+       if (!ReadFile(hpipe, &protocol_info, sizeof(protocol_info), &bytes_read, NULL)) {
+               log_err("couldnt read pi from pipe %s error %lu\n", fio_server_pipe_name,
+                               GetLastError());
+       }
+
+       if (use_ipv6) /* use protocol_info to create a duplicate of parents socket */
+               sk = WSASocket(AF_INET6, SOCK_STREAM, 0, &protocol_info, 0, 0);
+       else
+               sk = WSASocket(AF_INET,  SOCK_STREAM, 0, &protocol_info, 0, 0);
+
+       sk_out = scalloc(1, sizeof(*sk_out));
+       if (!sk_out) {
+               CloseHandle(hpipe);
+               close(sk);
+               return -1;
+       }
+
+       sk_out->sk = sk;
+       sk_out->hProcess = INVALID_HANDLE_VALUE;
+       INIT_FLIST_HEAD(&sk_out->list);
+       __fio_sem_init(&sk_out->lock, FIO_SEM_UNLOCKED);
+       __fio_sem_init(&sk_out->wait, FIO_SEM_LOCKED);
+       __fio_sem_init(&sk_out->xmit, FIO_SEM_UNLOCKED);
+
+       get_my_addr_str(sk);
+
+       if (!WriteFile(hpipe, msg, strlen(msg), NULL, NULL)) {
+               log_err("couldnt write pipe\n");
+               close(sk);
+               return -1;
+       }
+       CloseHandle(hpipe);
+
+       sk_out_assign(sk_out);
+
+       ret = handle_connection(sk_out);
+       __sk_out_drop(sk_out);
+       return ret;
+}
+#endif
+
 static int accept_loop(int listen_sk)
 {
        struct sockaddr_in addr;
@@ -1312,8 +1529,11 @@ static int accept_loop(int listen_sk)
                struct sk_out *sk_out;
                const char *from;
                char buf[64];
+#ifdef WIN32
+               HANDLE hProcess;
+#else
                pid_t pid;
-
+#endif
                pfd.fd = listen_sk;
                pfd.events = POLLIN;
                do {
@@ -1371,6 +1591,13 @@ static int accept_loop(int listen_sk)
                __fio_sem_init(&sk_out->wait, FIO_SEM_LOCKED);
                __fio_sem_init(&sk_out->xmit, FIO_SEM_UNLOCKED);
 
+#ifdef WIN32
+               hProcess = windows_handle_connection(hjob, sk);
+               if (hProcess == INVALID_HANDLE_VALUE)
+                       return -1;
+               sk_out->hProcess = hProcess;
+               fio_server_add_conn_pid(&conn_list, hProcess);
+#else
                pid = fork();
                if (pid) {
                        close(sk);
@@ -1387,6 +1614,7 @@ static int accept_loop(int listen_sk)
                 */
                sk_out_assign(sk_out);
                handle_connection(sk_out);
+#endif
        }
 
        return exitval;
@@ -1459,24 +1687,31 @@ static void convert_gs(struct group_run_stats *dst, struct group_run_stats *src)
 void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs)
 {
        struct cmd_ts_pdu p;
-       int i, j;
-       void *ss_buf;
-       uint64_t *ss_iops, *ss_bw;
+       int i, j, k;
+       size_t clat_prio_stats_extra_size = 0;
+       size_t ss_extra_size = 0;
+       size_t extended_buf_size = 0;
+       void *extended_buf;
+       void *extended_buf_wp;
 
        dprint(FD_NET, "server sending end stats\n");
 
        memset(&p, 0, sizeof(p));
 
-       strncpy(p.ts.name, ts->name, FIO_JOBNAME_SIZE - 1);
-       strncpy(p.ts.verror, ts->verror, FIO_VERROR_SIZE - 1);
-       strncpy(p.ts.description, ts->description, FIO_JOBDESC_SIZE - 1);
+       snprintf(p.ts.name, sizeof(p.ts.name), "%s", ts->name);
+       snprintf(p.ts.verror, sizeof(p.ts.verror), "%s", ts->verror);
+       snprintf(p.ts.description, sizeof(p.ts.description), "%s",
+                ts->description);
 
        p.ts.error              = cpu_to_le32(ts->error);
        p.ts.thread_number      = cpu_to_le32(ts->thread_number);
        p.ts.groupid            = cpu_to_le32(ts->groupid);
+       p.ts.job_start          = cpu_to_le64(ts->job_start);
        p.ts.pid                = cpu_to_le32(ts->pid);
        p.ts.members            = cpu_to_le32(ts->members);
        p.ts.unified_rw_rep     = cpu_to_le32(ts->unified_rw_rep);
+       p.ts.ioprio             = cpu_to_le32(ts->ioprio);
+       p.ts.disable_prio_stat  = cpu_to_le32(ts->disable_prio_stat);
 
        for (i = 0; i < DDIR_RWDIR_CNT; i++) {
                convert_io_stat(&p.ts.clat_stat[i], &ts->clat_stat[i]);
@@ -1485,6 +1720,7 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs)
                convert_io_stat(&p.ts.bw_stat[i], &ts->bw_stat[i]);
                convert_io_stat(&p.ts.iops_stat[i], &ts->iops_stat[i]);
        }
+       convert_io_stat(&p.ts.sync_stat, &ts->sync_stat);
 
        p.ts.usr_time           = cpu_to_le64(ts->usr_time);
        p.ts.sys_time           = cpu_to_le64(ts->sys_time);
@@ -1493,6 +1729,7 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs)
        p.ts.majf               = cpu_to_le64(ts->majf);
        p.ts.clat_percentiles   = cpu_to_le32(ts->clat_percentiles);
        p.ts.lat_percentiles    = cpu_to_le32(ts->lat_percentiles);
+       p.ts.slat_percentiles   = cpu_to_le32(ts->slat_percentiles);
        p.ts.percentile_precision = cpu_to_le64(ts->percentile_precision);
 
        for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) {
@@ -1515,18 +1752,25 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs)
        for (i = 0; i < FIO_IO_U_LAT_M_NR; i++)
                p.ts.io_u_lat_m[i]      = cpu_to_le64(ts->io_u_lat_m[i]);
 
-       for (i = 0; i < DDIR_RWDIR_CNT; i++)
-               for (j = 0; j < FIO_IO_U_PLAT_NR; j++)
-                       p.ts.io_u_plat[i][j] = cpu_to_le64(ts->io_u_plat[i][j]);
+       for (i = 0; i < FIO_LAT_CNT; i++)
+               for (j = 0; j < DDIR_RWDIR_CNT; j++)
+                       for (k = 0; k < FIO_IO_U_PLAT_NR; k++)
+                               p.ts.io_u_plat[i][j][k] = cpu_to_le64(ts->io_u_plat[i][j][k]);
 
-       for (i = 0; i < DDIR_RWDIR_CNT; i++) {
+       for (j = 0; j < FIO_IO_U_PLAT_NR; j++)
+               p.ts.io_u_sync_plat[j] = cpu_to_le64(ts->io_u_sync_plat[j]);
+
+       for (i = 0; i < DDIR_RWDIR_SYNC_CNT; i++)
                p.ts.total_io_u[i]      = cpu_to_le64(ts->total_io_u[i]);
+
+       for (i = 0; i < DDIR_RWDIR_CNT; i++) {
                p.ts.short_io_u[i]      = cpu_to_le64(ts->short_io_u[i]);
                p.ts.drop_io_u[i]       = cpu_to_le64(ts->drop_io_u[i]);
        }
 
        p.ts.total_submit       = cpu_to_le64(ts->total_submit);
        p.ts.total_complete     = cpu_to_le64(ts->total_complete);
+       p.ts.nr_zone_resets     = cpu_to_le64(ts->nr_zone_resets);
 
        for (i = 0; i < DDIR_RWDIR_CNT; i++) {
                p.ts.io_bytes[i]        = cpu_to_le64(ts->io_bytes[i]);
@@ -1559,29 +1803,90 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs)
        p.ts.ss_deviation.u.i   = cpu_to_le64(fio_double_to_uint64(ts->ss_deviation.u.f));
        p.ts.ss_criterion.u.i   = cpu_to_le64(fio_double_to_uint64(ts->ss_criterion.u.f));
 
+       p.ts.cachehit           = cpu_to_le64(ts->cachehit);
+       p.ts.cachemiss          = cpu_to_le64(ts->cachemiss);
+
        convert_gs(&p.rs, rs);
 
+       for (i = 0; i < DDIR_RWDIR_CNT; i++) {
+               if (ts->nr_clat_prio[i])
+                       clat_prio_stats_extra_size += ts->nr_clat_prio[i] * sizeof(*ts->clat_prio[i]);
+       }
+       extended_buf_size += clat_prio_stats_extra_size;
+
        dprint(FD_NET, "ts->ss_state = %d\n", ts->ss_state);
-       if (ts->ss_state & FIO_SS_DATA) {
-               dprint(FD_NET, "server sending steadystate ring buffers\n");
+       if (ts->ss_state & FIO_SS_DATA)
+               ss_extra_size = 2 * ts->ss_dur * sizeof(uint64_t);
 
-               ss_buf = malloc(sizeof(p) + 2*ts->ss_dur*sizeof(uint64_t));
+       extended_buf_size += ss_extra_size;
+       if (!extended_buf_size) {
+               fio_net_queue_cmd(FIO_NET_CMD_TS, &p, sizeof(p), NULL, SK_F_COPY);
+               return;
+       }
 
-               memcpy(ss_buf, &p, sizeof(p));
+       extended_buf_size += sizeof(p);
+       extended_buf = calloc(1, extended_buf_size);
+       if (!extended_buf) {
+               log_err("fio: failed to allocate FIO_NET_CMD_TS buffer\n");
+               return;
+       }
 
-               ss_iops = (uint64_t *) ((struct cmd_ts_pdu *)ss_buf + 1);
-               ss_bw = ss_iops + (int) ts->ss_dur;
-               for (i = 0; i < ts->ss_dur; i++) {
-                       ss_iops[i] = cpu_to_le64(ts->ss_iops_data[i]);
-                       ss_bw[i] = cpu_to_le64(ts->ss_bw_data[i]);
+       memcpy(extended_buf, &p, sizeof(p));
+       extended_buf_wp = (struct cmd_ts_pdu *)extended_buf + 1;
+
+       if (clat_prio_stats_extra_size) {
+               for (i = 0; i < DDIR_RWDIR_CNT; i++) {
+                       struct clat_prio_stat *prio = (struct clat_prio_stat *) extended_buf_wp;
+
+                       for (j = 0; j < ts->nr_clat_prio[i]; j++) {
+                               for (k = 0; k < FIO_IO_U_PLAT_NR; k++)
+                                       prio->io_u_plat[k] =
+                                               cpu_to_le64(ts->clat_prio[i][j].io_u_plat[k]);
+                               convert_io_stat(&prio->clat_stat,
+                                               &ts->clat_prio[i][j].clat_stat);
+                               prio->ioprio = cpu_to_le32(ts->clat_prio[i][j].ioprio);
+                               prio++;
+                       }
+
+                       if (ts->nr_clat_prio[i]) {
+                               uint64_t offset = (char *)extended_buf_wp - (char *)extended_buf;
+                               struct cmd_ts_pdu *ptr = extended_buf;
+
+                               ptr->ts.clat_prio_offset[i] = cpu_to_le64(offset);
+                               ptr->ts.nr_clat_prio[i] = cpu_to_le32(ts->nr_clat_prio[i]);
+                       }
+
+                       extended_buf_wp = prio;
                }
+       }
+
+       if (ss_extra_size) {
+               uint64_t *ss_iops, *ss_bw;
+               uint64_t offset;
+               struct cmd_ts_pdu *ptr = extended_buf;
+
+               dprint(FD_NET, "server sending steadystate ring buffers\n");
 
-               fio_net_queue_cmd(FIO_NET_CMD_TS, ss_buf, sizeof(p) + 2*ts->ss_dur*sizeof(uint64_t), NULL, SK_F_COPY);
+               /* ss iops */
+               ss_iops = (uint64_t *) extended_buf_wp;
+               for (i = 0; i < ts->ss_dur; i++)
+                       ss_iops[i] = cpu_to_le64(ts->ss_iops_data[i]);
+
+               offset = (char *)extended_buf_wp - (char *)extended_buf;
+               ptr->ts.ss_iops_data_offset = cpu_to_le64(offset);
+               extended_buf_wp = ss_iops + (int) ts->ss_dur;
+
+               /* ss bw */
+               ss_bw = extended_buf_wp;
+               for (i = 0; i < ts->ss_dur; i++)
+                       ss_bw[i] = cpu_to_le64(ts->ss_bw_data[i]);
 
-               free(ss_buf);
+               offset = (char *)extended_buf_wp - (char *)extended_buf;
+               ptr->ts.ss_bw_data_offset = cpu_to_le64(offset);
        }
-       else
-               fio_net_queue_cmd(FIO_NET_CMD_TS, &p, sizeof(p), NULL, SK_F_COPY);
+
+       fio_net_queue_cmd(FIO_NET_CMD_TS, extended_buf, extended_buf_size, NULL, SK_F_COPY);
+       free(extended_buf);
 }
 
 void fio_server_send_gs(struct group_run_stats *rs)
@@ -1595,7 +1900,7 @@ void fio_server_send_gs(struct group_run_stats *rs)
 }
 
 void fio_server_send_job_options(struct flist_head *opt_list,
-                                unsigned int groupid)
+                                unsigned int gid)
 {
        struct cmd_job_option pdu;
        struct flist_head *entry;
@@ -1610,12 +1915,12 @@ void fio_server_send_job_options(struct flist_head *opt_list,
                p = flist_entry(entry, struct print_option, list);
                memset(&pdu, 0, sizeof(pdu));
 
-               if (groupid == -1U) {
+               if (gid == -1U) {
                        pdu.global = __cpu_to_le16(1);
                        pdu.groupid = 0;
                } else {
                        pdu.global = 0;
-                       pdu.groupid = cpu_to_le32(groupid);
+                       pdu.groupid = cpu_to_le32(gid);
                }
                len = strlen(p->name);
                if (len >= sizeof(pdu.name)) {
@@ -1656,8 +1961,7 @@ static void convert_dus(struct disk_util_stat *dst, struct disk_util_stat *src)
 {
        int i;
 
-       dst->name[FIO_DU_NAME_SZ - 1] = '\0';
-       strncpy((char *) dst->name, (char *) src->name, FIO_DU_NAME_SZ - 1);
+       snprintf((char *) dst->name, sizeof(dst->name), "%s", src->name);
 
        for (i = 0; i < 2; i++) {
                dst->s.ios[i]           = cpu_to_le64(src->s.ios[i]);
@@ -1884,7 +2188,7 @@ static int fio_append_iolog_gz(struct sk_entry *first, struct io_log *log)
                        break;
                }
                flist_add_tail(&entry->list, &first->next);
-       } while (ret != Z_STREAM_END);
+       }
 
        ret = deflateEnd(&stream);
        if (ret == Z_OK)
@@ -1955,6 +2259,7 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
                .thread_number          = cpu_to_le32(td->thread_number),
                .log_type               = cpu_to_le32(log->log_type),
                .log_hist_coarseness    = cpu_to_le32(log->hist_coarseness),
+               .per_job_logs           = cpu_to_le32(td->o.per_job_logs),
        };
        struct sk_entry *first;
        struct flist_head *entry;
@@ -1967,8 +2272,7 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
        else
                pdu.compressed = 0;
 
-       strncpy((char *) pdu.name, name, FIO_NET_NAME_MAX);
-       pdu.name[FIO_NET_NAME_MAX - 1] = '\0';
+       snprintf((char *) pdu.name, sizeof(pdu.name), "%s", name);
 
        /*
         * We can't do this for a pre-compressed log, but for that case,
@@ -1984,9 +2288,12 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
                        struct io_sample *s = get_sample(log, cur_log, i);
 
                        s->time         = cpu_to_le64(s->time);
-                       s->data.val     = cpu_to_le64(s->data.val);
-                       s->__ddir       = cpu_to_le32(s->__ddir);
-                       s->bs           = cpu_to_le32(s->bs);
+                       if (log->log_type != IO_LOG_TYPE_HIST) {
+                               s->data.val.val0        = cpu_to_le64(s->data.val.val0);
+                               s->data.val.val1        = cpu_to_le64(s->data.val.val1);
+                       }
+                       s->__ddir       = __cpu_to_le32(s->__ddir);
+                       s->bs           = cpu_to_le64(s->bs);
 
                        if (log->log_offset) {
                                struct io_sample_offset *so = (void *) s;
@@ -2022,22 +2329,29 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
 
 void fio_server_send_add_job(struct thread_data *td)
 {
-       struct cmd_add_job_pdu pdu = {
-               .thread_number = cpu_to_le32(td->thread_number),
-               .groupid = cpu_to_le32(td->groupid),
-       };
+       struct cmd_add_job_pdu *pdu;
+       size_t cmd_sz = offsetof(struct cmd_add_job_pdu, top) +
+               thread_options_pack_size(&td->o);
 
-       convert_thread_options_to_net(&pdu.top, &td->o);
+       pdu = malloc(cmd_sz);
+       pdu->thread_number = cpu_to_le32(td->thread_number);
+       pdu->groupid = cpu_to_le32(td->groupid);
 
-       fio_net_queue_cmd(FIO_NET_CMD_ADD_JOB, &pdu, sizeof(pdu), NULL,
-                               SK_F_COPY);
+       convert_thread_options_to_net(&pdu->top, &td->o);
+
+       fio_net_queue_cmd(FIO_NET_CMD_ADD_JOB, pdu, cmd_sz, NULL, SK_F_COPY);
+       free(pdu);
 }
 
 void fio_server_send_start(struct thread_data *td)
 {
        struct sk_out *sk_out = pthread_getspecific(sk_out_key);
 
-       assert(sk_out->sk != -1);
+       if (sk_out->sk == -1) {
+               log_err("pthread getting specific for key failed, sk_out %p, sk %i, err: %i:%s",
+                       sk_out, sk_out->sk, errno, strerror(errno));
+               abort();
+       }
 
        fio_net_queue_cmd(FIO_NET_CMD_SERVER_START, NULL, 0, NULL, SK_F_SIMPLE);
 }
@@ -2141,7 +2455,8 @@ static int fio_init_server_ip(void)
        /*
         * Not fatal if fails, so just ignore it if that happens
         */
-       setsockopt(sk, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));
+       if (setsockopt(sk, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt))) {
+       }
 #endif
 
        if (use_ipv6) {
@@ -2185,9 +2500,8 @@ static int fio_init_server_sock(void)
 
        mode = umask(000);
 
-       memset(&addr, 0, sizeof(addr));
        addr.sun_family = AF_UNIX;
-       strncpy(addr.sun_path, bind_sock, sizeof(addr.sun_path) - 1);
+       snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", bind_sock);
 
        len = sizeof(addr.sun_family) + strlen(bind_sock) + 1;
 
@@ -2237,9 +2551,9 @@ static int fio_init_server_connection(void)
                if (p)
                        strcat(p, port);
                else
-                       strncpy(bind_str, port, sizeof(bind_str) - 1);
+                       snprintf(bind_str, sizeof(bind_str), "%s", port);
        } else
-               strncpy(bind_str, bind_sock, sizeof(bind_str) - 1);
+               snprintf(bind_str, sizeof(bind_str), "%s", bind_sock);
 
        log_info("fio: server listening on %s\n", bind_str);
 
@@ -2432,6 +2746,11 @@ static void set_sig_handlers(void)
        };
 
        sigaction(SIGINT, &act, NULL);
+
+       /* Windows uses SIGBREAK as a quit signal from other applications */
+#ifdef WIN32
+       sigaction(SIGBREAK, &act, NULL);
+#endif
 }
 
 void fio_server_destroy_sk_key(void)
@@ -2459,12 +2778,25 @@ static int fio_server(void)
        if (fio_handle_server_arg())
                return -1;
 
+       set_sig_handlers();
+
+#ifdef WIN32
+       /* if this is a child process, go handle the connection */
+       if (fio_server_pipe_name != NULL) {
+               ret = handle_connection_process();
+               return ret;
+       }
+
+       /* job to link child processes so they terminate together */
+       hjob = windows_create_job();
+       if (hjob == INVALID_HANDLE_VALUE)
+               return -1;
+#endif
+
        sk = fio_init_server_connection();
        if (sk < 0)
                return -1;
 
-       set_sig_handlers();
-
        ret = accept_loop(sk);
 
        close(sk);
@@ -2489,7 +2821,7 @@ void fio_server_got_signal(int signal)
                sk_out->sk = -1;
        else {
                log_info("\nfio: terminating on signal %d\n", signal);
-               exit_backend = 1;
+               exit_backend = true;
        }
 }
 
@@ -2540,6 +2872,7 @@ static int write_pid(pid_t pid, const char *pidfile)
  */
 int fio_start_server(char *pidfile)
 {
+       FILE *file;
        pid_t pid;
        int ret;
 
@@ -2571,15 +2904,29 @@ int fio_start_server(char *pidfile)
 
        setsid();
        openlog("fio", LOG_NDELAY|LOG_NOWAIT|LOG_PID, LOG_USER);
-       log_syslog = 1;
-       close(STDIN_FILENO);
-       close(STDOUT_FILENO);
-       close(STDERR_FILENO);
+       log_syslog = true;
+
+       file = freopen("/dev/null", "r", stdin);
+       if (!file)
+               perror("freopen");
+
+       file = freopen("/dev/null", "w", stdout);
+       if (!file)
+               perror("freopen");
+
+       file = freopen("/dev/null", "w", stderr);
+       if (!file)
+               perror("freopen");
+
        f_out = NULL;
        f_err = NULL;
 
        ret = fio_server();
 
+       fclose(stdin);
+       fclose(stdout);
+       fclose(stderr);
+
        closelog();
        unlink(pidfile);
        free(pidfile);
@@ -2590,3 +2937,10 @@ void fio_server_set_arg(const char *arg)
 {
        fio_server_arg = strdup(arg);
 }
+
+#ifdef WIN32
+void fio_server_internal_set(const char *arg)
+{
+       fio_server_pipe_name = strdup(arg);
+}
+#endif