Merge branch 'doc-patches' of https://github.com/vincentkfu/fio
authorJens Axboe <axboe@kernel.dk>
Thu, 14 Sep 2017 17:37:34 +0000 (11:37 -0600)
committerJens Axboe <axboe@kernel.dk>
Thu, 14 Sep 2017 17:37:34 +0000 (11:37 -0600)
12 files changed:
HOWTO
cconv.c
client.c
fio.1
gclient.c
init.c
options.c
server.c
server.h
stat.c
stat.h
thread_options.h

diff --git a/HOWTO b/HOWTO
index 342bad8..8fad2ce 100644 (file)
--- a/HOWTO
+++ b/HOWTO
@@ -2862,7 +2862,15 @@ Measurements and reporting
 
 .. option:: clat_percentiles=bool
 
-       Enable the reporting of percentiles of completion latencies.
+       Enable the reporting of percentiles of completion latencies.  This
+       option is mutually exclusive with :option:`lat_percentiles`.
+
+.. option:: lat_percentiles=bool
+
+       Enable the reporting of percentiles of IO latencies. This is similar
+       to :option:`clat_percentiles`, except that this includes the
+       submission latency. This option is mutually exclusive with
+       :option:`clat_percentiles`.
 
 .. option:: percentile_list=float_list
 
diff --git a/cconv.c b/cconv.c
index ac58705..f809fd5 100644 (file)
--- a/cconv.c
+++ b/cconv.c
@@ -267,6 +267,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
        o->trim_batch = le32_to_cpu(top->trim_batch);
        o->trim_zero = le32_to_cpu(top->trim_zero);
        o->clat_percentiles = le32_to_cpu(top->clat_percentiles);
+       o->lat_percentiles = le32_to_cpu(top->lat_percentiles);
        o->percentile_precision = le32_to_cpu(top->percentile_precision);
        o->continue_on_error = le32_to_cpu(top->continue_on_error);
        o->cgroup_weight = le32_to_cpu(top->cgroup_weight);
@@ -454,6 +455,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
        top->trim_batch = cpu_to_le32(o->trim_batch);
        top->trim_zero = cpu_to_le32(o->trim_zero);
        top->clat_percentiles = cpu_to_le32(o->clat_percentiles);
+       top->lat_percentiles = cpu_to_le32(o->lat_percentiles);
        top->percentile_precision = cpu_to_le32(o->percentile_precision);
        top->continue_on_error = cpu_to_le32(o->continue_on_error);
        top->cgroup_weight = cpu_to_le32(o->cgroup_weight);
index 281d853..09e810a 100644 (file)
--- a/client.c
+++ b/client.c
@@ -893,7 +893,8 @@ static void convert_ts(struct thread_stat *dst, struct thread_stat *src)
        dst->ctx                = le64_to_cpu(src->ctx);
        dst->minf               = le64_to_cpu(src->minf);
        dst->majf               = le64_to_cpu(src->majf);
-       dst->clat_percentiles   = le64_to_cpu(src->clat_percentiles);
+       dst->clat_percentiles   = le32_to_cpu(src->clat_percentiles);
+       dst->lat_percentiles    = le32_to_cpu(src->lat_percentiles);
        dst->percentile_precision = le64_to_cpu(src->percentile_precision);
 
        for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) {
diff --git a/fio.1 b/fio.1
index 4837c50..b943db2 100644 (file)
--- a/fio.1
+++ b/fio.1
@@ -2545,7 +2545,13 @@ Disable measurements of throughput/bandwidth numbers. See
 \fBdisable_lat\fR.
 .TP
 .BI clat_percentiles \fR=\fPbool
-Enable the reporting of percentiles of completion latencies.
+Enable the reporting of percentiles of completion latencies. This option is
+mutually exclusive with \fBlat_percentiles\fR.
+.TP
+.BI lat_percentiles \fR=\fPbool
+Enable the reporting of percentiles of IO latencies. This is similar to
+\fBclat_percentiles\fR, except that this includes the submission latency.
+This option is mutually exclusive with \fBclat_percentiles\fR.
 .TP
 .BI percentile_list \fR=\fPfloat_list
 Overwrite the default list of percentiles for completion latencies and the
index 4eb99a0..43c8a08 100644 (file)
--- a/gclient.c
+++ b/gclient.c
@@ -1127,7 +1127,11 @@ static void gfio_show_clat_percentiles(struct gfio_client *gc,
                base = "nsec";
         }
 
-       sprintf(tmp, "Completion percentiles (%s)", base);
+       if (ts->clat_percentiles)
+               sprintf(tmp, "Completion percentiles (%s)", base);
+       else
+               sprintf(tmp, "Latency percentiles (%s)", base);
+
        tree_view = gfio_output_clat_percentiles(ovals, plist, len, base, scale_down);
        ge->clat_graph = setup_clat_graph(tmp, ovals, plist, len, 700.0, 300.0);
 
diff --git a/init.c b/init.c
index cf5c646..6ac5212 100644 (file)
--- a/init.c
+++ b/init.c
@@ -837,7 +837,7 @@ static int fixup_options(struct thread_data *td)
         * Windows doesn't support O_DIRECT or O_SYNC with the _open interface,
         * so fail if we're passed those flags
         */
-       if (td_ioengine_flagged(td, FIO_SYNCIO) && (td->o.odirect || td->o.sync_io)) {
+       if (td_ioengine_flagged(td, FIO_SYNCIO) && (o->odirect || o->sync_io)) {
                log_err("fio: Windows does not support direct or non-buffered io with"
                                " the synchronous ioengines. Use the 'windowsaio' ioengine"
                                " with 'direct=1' and 'iodepth=1' instead.\n");
@@ -863,8 +863,8 @@ static int fixup_options(struct thread_data *td)
         * Using a non-uniform random distribution excludes usage of
         * a random map
         */
-       if (td->o.random_distribution != FIO_RAND_DIST_RANDOM)
-               td->o.norandommap = 1;
+       if (o->random_distribution != FIO_RAND_DIST_RANDOM)
+               o->norandommap = 1;
 
        /*
         * If size is set but less than the min block size, complain
@@ -878,16 +878,16 @@ static int fixup_options(struct thread_data *td)
        /*
         * O_ATOMIC implies O_DIRECT
         */
-       if (td->o.oatomic)
-               td->o.odirect = 1;
+       if (o->oatomic)
+               o->odirect = 1;
 
        /*
         * If randseed is set, that overrides randrepeat
         */
-       if (fio_option_is_set(&td->o, rand_seed))
-               td->o.rand_repeatable = 0;
+       if (fio_option_is_set(o, rand_seed))
+               o->rand_repeatable = 0;
 
-       if (td_ioengine_flagged(td, FIO_NOEXTEND) && td->o.file_append) {
+       if (td_ioengine_flagged(td, FIO_NOEXTEND) && o->file_append) {
                log_err("fio: can't append/extent with IO engine %s\n", td->io_ops->name);
                ret = 1;
        }
@@ -902,10 +902,24 @@ static int fixup_options(struct thread_data *td)
        if (!td->loops)
                td->loops = 1;
 
-       if (td->o.block_error_hist && td->o.nr_files != 1) {
+       if (o->block_error_hist && o->nr_files != 1) {
                log_err("fio: block error histogram only available "
                        "with a single file per job, but %d files "
-                       "provided\n", td->o.nr_files);
+                       "provided\n", o->nr_files);
+               ret = 1;
+       }
+
+       if (fio_option_is_set(o, clat_percentiles) &&
+           !fio_option_is_set(o, lat_percentiles)) {
+               o->lat_percentiles = !o->clat_percentiles;
+       } else if (fio_option_is_set(o, lat_percentiles) &&
+                  !fio_option_is_set(o, clat_percentiles)) {
+               o->clat_percentiles = !o->lat_percentiles;
+       } else if (fio_option_is_set(o, lat_percentiles) &&
+                  fio_option_is_set(o, clat_percentiles) &&
+                  o->lat_percentiles && o->clat_percentiles) {
+               log_err("fio: lat_percentiles and clat_percentiles are "
+                       "mutually exclusive\n");
                ret = 1;
        }
 
@@ -1401,6 +1415,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
        td->mutex = fio_mutex_init(FIO_MUTEX_LOCKED);
 
        td->ts.clat_percentiles = o->clat_percentiles;
+       td->ts.lat_percentiles = o->lat_percentiles;
        td->ts.percentile_precision = o->percentile_precision;
        memcpy(td->ts.percentile_list, o->percentile_list, sizeof(o->percentile_list));
 
index 54fa4ee..5c1abe9 100644 (file)
--- a/options.c
+++ b/options.c
@@ -4076,6 +4076,18 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                .off1   = offsetof(struct thread_options, clat_percentiles),
                .help   = "Enable the reporting of completion latency percentiles",
                .def    = "1",
+               .inverse = "lat_percentiles",
+               .category = FIO_OPT_C_STAT,
+               .group  = FIO_OPT_G_INVALID,
+       },
+       {
+               .name   = "lat_percentiles",
+               .lname  = "IO latency percentiles",
+               .type   = FIO_OPT_BOOL,
+               .off1   = offsetof(struct thread_options, lat_percentiles),
+               .help   = "Enable the reporting of IO latency percentiles",
+               .def    = "0",
+               .inverse = "clat_percentiles",
                .category = FIO_OPT_C_STAT,
                .group  = FIO_OPT_G_INVALID,
        },
index 2c08c3e..0469cea 100644 (file)
--- a/server.c
+++ b/server.c
@@ -1484,7 +1484,8 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs)
        p.ts.ctx                = cpu_to_le64(ts->ctx);
        p.ts.minf               = cpu_to_le64(ts->minf);
        p.ts.majf               = cpu_to_le64(ts->majf);
-       p.ts.clat_percentiles   = cpu_to_le64(ts->clat_percentiles);
+       p.ts.clat_percentiles   = cpu_to_le32(ts->clat_percentiles);
+       p.ts.lat_percentiles    = cpu_to_le32(ts->lat_percentiles);
        p.ts.percentile_precision = cpu_to_le64(ts->percentile_precision);
 
        for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) {
index f63a518..ba3abfe 100644 (file)
--- a/server.h
+++ b/server.h
@@ -49,7 +49,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-       FIO_SERVER_VER                  = 65,
+       FIO_SERVER_VER                  = 66,
 
        FIO_SERVER_MAX_FRAGMENT_PDU     = 1024,
        FIO_SERVER_MAX_CMD_MB           = 2048,
diff --git a/stat.c b/stat.c
index 63353cc..09afa5b 100644 (file)
--- a/stat.c
+++ b/stat.c
@@ -143,7 +143,7 @@ unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long nr,
        unsigned int len, i, j = 0;
        unsigned int oval_len = 0;
        unsigned long long *ovals = NULL;
-       int is_last;
+       bool is_last;
 
        *minv = -1ULL;
        *maxv = 0;
@@ -166,7 +166,7 @@ unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long nr,
        /*
         * Calculate bucket values, note down max and min values
         */
-       is_last = 0;
+       is_last = false;
        for (i = 0; i < FIO_IO_U_PLAT_NR && !is_last; i++) {
                sum += io_u_plat[i];
                while (sum >= (plist[j].u.f / 100.0 * nr)) {
@@ -183,7 +183,7 @@ unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long nr,
                        if (ovals[j] > *maxv)
                                *maxv = ovals[j];
 
-                       is_last = (j == len - 1);
+                       is_last = (j == len - 1) != 0;
                        if (is_last)
                                break;
 
@@ -200,12 +200,14 @@ unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long nr,
  */
 static void show_clat_percentiles(unsigned int *io_u_plat, unsigned long nr,
                                  fio_fp64_t *plist, unsigned int precision,
-                                 struct buf_output *out)
+                                 bool is_clat, struct buf_output *out)
 {
        unsigned int divisor, len, i, j = 0;
        unsigned long long minv, maxv;
        unsigned long long *ovals;
-       int is_last, per_line, scale_down, time_width;
+       int per_line, scale_down, time_width;
+       const char *pre = is_clat ? "clat" : " lat";
+       bool is_last;
        char fmt[32];
 
        len = calc_clat_percentiles(io_u_plat, nr, plist, &ovals, &maxv, &minv);
@@ -219,15 +221,15 @@ static void show_clat_percentiles(unsigned int *io_u_plat, unsigned long nr,
        if (minv > 2000000 && maxv > 99999999ULL) {
                scale_down = 2;
                divisor = 1000000;
-               log_buf(out, "    clat percentiles (msec):\n     |");
+               log_buf(out, "    %s percentiles (msec):\n     |", pre);
        } else if (minv > 2000 && maxv > 99999) {
                scale_down = 1;
                divisor = 1000;
-               log_buf(out, "    clat percentiles (usec):\n     |");
+               log_buf(out, "    %s percentiles (usec):\n     |", pre);
        } else {
                scale_down = 0;
                divisor = 1;
-               log_buf(out, "    clat percentiles (nsec):\n     |");
+               log_buf(out, "    %s percentiles (nsec):\n     |", pre);
        }
 
 
@@ -243,7 +245,7 @@ static void show_clat_percentiles(unsigned int *io_u_plat, unsigned long nr,
                        log_buf(out, "     |");
 
                /* end of the list */
-               is_last = (j == len - 1);
+               is_last = (j == len - 1) != 0;
 
                for (i = 0; i < scale_down; i++)
                        ovals[j] = (ovals[j] + 999) / 1000;
@@ -457,11 +459,12 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts,
        if (calc_lat(&ts->lat_stat[ddir], &min, &max, &mean, &dev))
                display_lat(" lat", min, max, mean, dev, out);
 
-       if (ts->clat_percentiles) {
+       if (ts->clat_percentiles || ts->lat_percentiles) {
                show_clat_percentiles(ts->io_u_plat[ddir],
                                        ts->clat_stat[ddir].samples,
                                        ts->percentile_list,
-                                       ts->percentile_precision, out);
+                                       ts->percentile_precision,
+                                       ts->clat_percentiles, out);
        }
        if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) {
                double p_of_agg = 100.0, fkb_base = (double)rs->kb_base;
@@ -509,20 +512,21 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts,
        }
 }
 
-static int show_lat(double *io_u_lat, int nr, const char **ranges,
-                   const char *msg, struct buf_output *out)
+static bool show_lat(double *io_u_lat, int nr, const char **ranges,
+                    const char *msg, struct buf_output *out)
 {
-       int new_line = 1, i, line = 0, shown = 0;
+       bool new_line = true, shown = false;
+       int i, line = 0;
 
        for (i = 0; i < nr; i++) {
                if (io_u_lat[i] <= 0.0)
                        continue;
-               shown = 1;
+               shown = true;
                if (new_line) {
                        if (line)
                                log_buf(out, "\n");
                        log_buf(out, "  lat (%s)   : ", msg);
-                       new_line = 0;
+                       new_line = false;
                        line = 0;
                }
                if (line)
@@ -530,13 +534,13 @@ static int show_lat(double *io_u_lat, int nr, const char **ranges,
                log_buf(out, "%s%3.2f%%", ranges[i], io_u_lat[i]);
                line++;
                if (line == 5)
-                       new_line = 1;
+                       new_line = true;
        }
 
        if (shown)
                log_buf(out, "\n");
 
-       return shown;
+       return true;
 }
 
 static void show_lat_n(double *io_u_lat_n, struct buf_output *out)
@@ -896,7 +900,7 @@ static void show_ddir_status_terse(struct thread_stat *ts,
        else
                log_buf(out, ";%llu;%llu;%f;%f", 0ULL, 0ULL, 0.0, 0.0);
 
-       if (ts->clat_percentiles) {
+       if (ts->clat_percentiles || ts->lat_percentiles) {
                len = calc_clat_percentiles(ts->io_u_plat[ddir],
                                        ts->clat_stat[ddir].samples,
                                        ts->percentile_list, &ovals, &maxv,
@@ -1011,7 +1015,7 @@ static void add_ddir_status_json(struct thread_stat *ts,
        json_object_add_value_float(tmp_object, "mean", mean);
        json_object_add_value_float(tmp_object, "stddev", dev);
 
-       if (ts->clat_percentiles) {
+       if (ts->clat_percentiles || ts->lat_percentiles) {
                len = calc_clat_percentiles(ts->io_u_plat[ddir],
                                        ts->clat_stat[ddir].samples,
                                        ts->percentile_list, &ovals, &maxv,
@@ -1588,8 +1592,8 @@ void __show_run_stats(void)
        struct thread_data *td;
        struct thread_stat *threadstats, *ts;
        int i, j, k, nr_ts, last_ts, idx;
-       int kb_base_warned = 0;
-       int unit_base_warned = 0;
+       bool kb_base_warned = false;
+       bool unit_base_warned = false;
        struct json_object *root = NULL;
        struct json_array *array = NULL;
        struct buf_output output[FIO_OUTPUT_NR];
@@ -1645,6 +1649,7 @@ void __show_run_stats(void)
                ts = &threadstats[j];
 
                ts->clat_percentiles = td->o.clat_percentiles;
+               ts->lat_percentiles = td->o.lat_percentiles;
                ts->percentile_precision = td->o.percentile_precision;
                memcpy(ts->percentile_list, td->o.percentile_list, sizeof(td->o.percentile_list));
                opt_lists[j] = &td->opt_list;
@@ -1681,11 +1686,11 @@ void __show_run_stats(void)
                } else if (ts->kb_base != td->o.kb_base && !kb_base_warned) {
                        log_info("fio: kb_base differs for jobs in group, using"
                                 " %u as the base\n", ts->kb_base);
-                       kb_base_warned = 1;
+                       kb_base_warned = true;
                } else if (ts->unit_base != td->o.unit_base && !unit_base_warned) {
                        log_info("fio: unit_base differs for jobs in group, using"
                                 " %u as the base\n", ts->unit_base);
-                       unit_base_warned = 1;
+                       unit_base_warned = true;
                }
 
                ts->continue_on_error = td->o.continue_on_error;
@@ -1929,9 +1934,9 @@ void __show_running_run_stats(void)
        fio_mutex_up(stat_mutex);
 }
 
-static int status_interval_init;
+static bool status_interval_init;
 static struct timespec status_time;
-static int status_file_disabled;
+static bool status_file_disabled;
 
 #define FIO_STATUS_FILE                "fio-dump-status"
 
@@ -1962,7 +1967,7 @@ static int check_status_file(void)
                log_err("fio: failed to unlink %s: %s\n", fio_status_file_path,
                                                        strerror(errno));
                log_err("fio: disabling status file updates\n");
-               status_file_disabled = 1;
+               status_file_disabled = true;
        }
 
        return 1;
@@ -1973,7 +1978,7 @@ void check_for_running_stats(void)
        if (status_interval) {
                if (!status_interval_init) {
                        fio_gettime(&status_time, NULL);
-                       status_interval_init = 1;
+                       status_interval_init = true;
                } else if (mtime_since_now(&status_time) >= status_interval) {
                        show_running_run_stats();
                        fio_gettime(&status_time, NULL);
@@ -2437,6 +2442,9 @@ void add_lat_sample(struct thread_data *td, enum fio_ddir ddir,
                add_log_sample(td, td->lat_log, sample_val(nsec), ddir, bs,
                               offset);
 
+       if (ts->lat_percentiles)
+               add_clat_percentile_sample(ts, nsec, ddir);
+
        td_io_u_unlock(td);
 }
 
diff --git a/stat.h b/stat.h
index 132dee3..848331b 100644 (file)
--- a/stat.h
+++ b/stat.h
@@ -172,7 +172,8 @@ struct thread_stat {
        /*
         * IO depth and latency stats
         */
-       uint64_t clat_percentiles;
+       uint32_t clat_percentiles;
+       uint32_t lat_percentiles;
        uint64_t percentile_precision;
        fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN];
 
index fd6576e..1813cdc 100644 (file)
@@ -240,6 +240,7 @@ struct thread_options {
        unsigned int trim_zero;
        unsigned long long trim_backlog;
        unsigned int clat_percentiles;
+       unsigned int lat_percentiles;
        unsigned int percentile_precision;      /* digits after decimal for percentiles */
        fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN];
 
@@ -343,7 +344,7 @@ struct thread_options_pack {
        uint32_t iodepth_batch_complete_min;
        uint32_t iodepth_batch_complete_max;
        uint32_t serialize_overlap;
-       uint32_t pad3;
+       uint32_t lat_percentiles;
 
        uint64_t size;
        uint64_t io_size;