fio: report percentiles for slat, clat, lat
authorVincent Fu <vincent.fu@wdc.com>
Wed, 29 Jan 2020 19:08:26 +0000 (14:08 -0500)
committerVincent Fu <vincent.fu@wdc.com>
Tue, 4 Feb 2020 15:14:04 +0000 (10:14 -0500)
Currently, fio can report percentiles for either completion latencies or
total latencies. This patch adds the ability to simultaneously report
percentiles for submission, completion, and total latencies.

- simplify the JSON lat percentile reporting code
- parsing the JSON output will be unaffected, although now setting
lat_percentiles=1 no longer suppresses clat percentiles
- this may break parsing of the normal output since more latency
percentile blocks may be displayed
- to maintain backward compatibility in the terse output, total
latencies are reported if lat_percentiles=1 even when clat percentiles
have also been computed
- change the gtod_reduce callback to only change option values when
gtod_reduce=1
- for the new cmdprio_percentage latencies, if lat_percentiles=1,
*total* latency percentiles will be tracked. Otherwise, *completion*
latency percentiles will be tracked.
- also make sure we skip the high/low priority latency code if we are
producing output for sync latencies.
- make add_slat_sample use an unsigned long long for the time to be
consistent with other functions

Signed-off-by: Vincent Fu <vincent.fu@wdc.com>
cconv.c
client.c
init.c
options.c
server.c
server.h
stat.c
stat.h
thread_options.h

diff --git a/cconv.c b/cconv.c
index 04854b0..48218dc 100644 (file)
--- a/cconv.c
+++ b/cconv.c
@@ -271,6 +271,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
        o->trim_zero = le32_to_cpu(top->trim_zero);
        o->clat_percentiles = le32_to_cpu(top->clat_percentiles);
        o->lat_percentiles = le32_to_cpu(top->lat_percentiles);
+       o->slat_percentiles = le32_to_cpu(top->slat_percentiles);
        o->percentile_precision = le32_to_cpu(top->percentile_precision);
        o->sig_figs = le32_to_cpu(top->sig_figs);
        o->continue_on_error = le32_to_cpu(top->continue_on_error);
@@ -469,6 +470,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
        top->trim_zero = cpu_to_le32(o->trim_zero);
        top->clat_percentiles = cpu_to_le32(o->clat_percentiles);
        top->lat_percentiles = cpu_to_le32(o->lat_percentiles);
+       top->slat_percentiles = cpu_to_le32(o->slat_percentiles);
        top->percentile_precision = cpu_to_le32(o->percentile_precision);
        top->sig_figs = cpu_to_le32(o->sig_figs);
        top->continue_on_error = cpu_to_le32(o->continue_on_error);
index ac24651..e8f6fd4 100644 (file)
--- a/client.c
+++ b/client.c
@@ -969,6 +969,7 @@ static void convert_ts(struct thread_stat *dst, struct thread_stat *src)
        dst->majf               = le64_to_cpu(src->majf);
        dst->clat_percentiles   = le32_to_cpu(src->clat_percentiles);
        dst->lat_percentiles    = le32_to_cpu(src->lat_percentiles);
+       dst->slat_percentiles   = le32_to_cpu(src->slat_percentiles);
        dst->percentile_precision = le64_to_cpu(src->percentile_precision);
 
        for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) {
diff --git a/init.c b/init.c
index dca44bc..e231336 100644 (file)
--- a/init.c
+++ b/init.c
@@ -944,24 +944,12 @@ static int fixup_options(struct thread_data *td)
                ret |= 1;
        }
 
-       if (fio_option_is_set(o, clat_percentiles) &&
-           !fio_option_is_set(o, lat_percentiles)) {
-               o->lat_percentiles = !o->clat_percentiles;
-       } else if (fio_option_is_set(o, lat_percentiles) &&
-                  !fio_option_is_set(o, clat_percentiles)) {
-               o->clat_percentiles = !o->lat_percentiles;
-       } else if (fio_option_is_set(o, lat_percentiles) &&
-                  fio_option_is_set(o, clat_percentiles) &&
-                  o->lat_percentiles && o->clat_percentiles) {
-               log_err("fio: lat_percentiles and clat_percentiles are "
-                       "mutually exclusive\n");
-               ret |= 1;
-       }
-
        if (o->disable_lat)
                o->lat_percentiles = 0;
        if (o->disable_clat)
                o->clat_percentiles = 0;
+       if (o->disable_slat)
+               o->slat_percentiles = 0;
 
        /*
         * Fix these up to be nsec internally
@@ -1509,6 +1497,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
 
        td->ts.clat_percentiles = o->clat_percentiles;
        td->ts.lat_percentiles = o->lat_percentiles;
+       td->ts.slat_percentiles = o->slat_percentiles;
        td->ts.percentile_precision = o->percentile_precision;
        memcpy(td->ts.percentile_list, o->percentile_list, sizeof(o->percentile_list));
        td->ts.sig_figs = o->sig_figs;
index 287f043..4714a3a 100644 (file)
--- a/options.c
+++ b/options.c
@@ -1408,13 +1408,20 @@ static int str_gtod_reduce_cb(void *data, int *il)
        struct thread_data *td = cb_data_to_td(data);
        int val = *il;
 
-       td->o.disable_lat = !!val;
-       td->o.disable_clat = !!val;
-       td->o.disable_slat = !!val;
-       td->o.disable_bw = !!val;
-       td->o.clat_percentiles = !val;
-       if (val)
+       /*
+        * Only modfiy options if gtod_reduce==1
+        * Otherwise leave settings alone.
+        */
+       if (val) {
+               td->o.disable_lat = 1;
+               td->o.disable_clat = 1;
+               td->o.disable_slat = 1;
+               td->o.disable_bw = 1;
+               td->o.clat_percentiles = 0;
+               td->o.lat_percentiles = 0;
+               td->o.slat_percentiles = 0;
                td->ts_cache_mask = 63;
+       }
 
        return 0;
 }
@@ -4312,7 +4319,6 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                .off1   = offsetof(struct thread_options, clat_percentiles),
                .help   = "Enable the reporting of completion latency percentiles",
                .def    = "1",
-               .inverse = "lat_percentiles",
                .category = FIO_OPT_C_STAT,
                .group  = FIO_OPT_G_INVALID,
        },
@@ -4323,7 +4329,16 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                .off1   = offsetof(struct thread_options, lat_percentiles),
                .help   = "Enable the reporting of IO latency percentiles",
                .def    = "0",
-               .inverse = "clat_percentiles",
+               .category = FIO_OPT_C_STAT,
+               .group  = FIO_OPT_G_INVALID,
+       },
+       {
+               .name   = "slat_percentiles",
+               .lname  = "Submission latency percentiles",
+               .type   = FIO_OPT_BOOL,
+               .off1   = offsetof(struct thread_options, slat_percentiles),
+               .help   = "Enable the reporting of submission latency percentiles",
+               .def    = "0",
                .category = FIO_OPT_C_STAT,
                .group  = FIO_OPT_G_INVALID,
        },
index 2fa991a..558c599 100644 (file)
--- a/server.c
+++ b/server.c
@@ -1499,6 +1499,7 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs)
        p.ts.majf               = cpu_to_le64(ts->majf);
        p.ts.clat_percentiles   = cpu_to_le32(ts->clat_percentiles);
        p.ts.lat_percentiles    = cpu_to_le32(ts->lat_percentiles);
+       p.ts.slat_percentiles   = cpu_to_le32(ts->slat_percentiles);
        p.ts.percentile_precision = cpu_to_le64(ts->percentile_precision);
 
        for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) {
index 6ac7536..279b691 100644 (file)
--- a/server.h
+++ b/server.h
@@ -48,7 +48,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-       FIO_SERVER_VER                  = 81,
+       FIO_SERVER_VER                  = 82,
 
        FIO_SERVER_MAX_FRAGMENT_PDU     = 1024,
        FIO_SERVER_MAX_CMD_MB           = 2048,
diff --git a/stat.c b/stat.c
index 601652a..5082486 100644 (file)
--- a/stat.c
+++ b/stat.c
@@ -485,20 +485,31 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts,
        if (calc_lat(&ts->clat_high_prio_stat[ddir], &min, &max, &mean, &dev))
                display_lat("prio_clat", min, max, mean, dev, out);
 
+       if (ts->slat_percentiles && ts->slat_stat[ddir].samples > 0)
+               show_clat_percentiles(ts->io_u_plat[FIO_SLAT][ddir],
+                                       ts->slat_stat[ddir].samples,
+                                       ts->percentile_list,
+                                       ts->percentile_precision, "slat", out);
+       if (ts->clat_percentiles && ts->clat_stat[ddir].samples > 0)
+               show_clat_percentiles(ts->io_u_plat[FIO_CLAT][ddir],
+                                       ts->clat_stat[ddir].samples,
+                                       ts->percentile_list,
+                                       ts->percentile_precision, "clat", out);
+       if (ts->lat_percentiles && ts->lat_stat[ddir].samples > 0)
+               show_clat_percentiles(ts->io_u_plat[FIO_LAT][ddir],
+                                       ts->lat_stat[ddir].samples,
+                                       ts->percentile_list,
+                                       ts->percentile_precision, "lat", out);
+
        if (ts->clat_percentiles || ts->lat_percentiles) {
-               const char *name = ts->clat_percentiles ? "clat" : " lat";
+               const char *name = ts->lat_percentiles ? " lat" : "clat";
                char prio_name[32];
                uint64_t samples;
 
-               if (ts->clat_percentiles)
-                       samples = ts->clat_stat[ddir].samples;
-               else
+               if (ts->lat_percentiles)
                        samples = ts->lat_stat[ddir].samples;
-
-               show_clat_percentiles(ts->io_u_plat[FIO_CLAT][ddir],
-                                       samples,
-                                       ts->percentile_list,
-                                       ts->percentile_precision, name, out);
+               else
+                       samples = ts->clat_stat[ddir].samples;
 
                /* Only print this if some high and low priority stats were collected */
                if (ts->clat_high_prio_stat[ddir].samples > 0 &&
@@ -521,6 +532,7 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts,
                                                ts->percentile_precision, prio_name, out);
                }
        }
+
        if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) {
                double p_of_agg = 100.0, fkb_base = (double)rs->kb_base;
                const char *bw_str;
@@ -1170,12 +1182,17 @@ static void show_ddir_status_terse(struct thread_stat *ts,
        else
                log_buf(out, ";%llu;%llu;%f;%f", 0ULL, 0ULL, 0.0, 0.0);
 
-       if (ts->clat_percentiles || ts->lat_percentiles) {
+       if (ts->lat_percentiles)
+               len = calc_clat_percentiles(ts->io_u_plat[FIO_LAT][ddir],
+                                       ts->lat_stat[ddir].samples,
+                                       ts->percentile_list, &ovals, &maxv,
+                                       &minv);
+       else if (ts->clat_percentiles)
                len = calc_clat_percentiles(ts->io_u_plat[FIO_CLAT][ddir],
                                        ts->clat_stat[ddir].samples,
                                        ts->percentile_list, &ovals, &maxv,
                                        &minv);
-       else
+       else
                len = 0;
 
        for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) {
@@ -1221,18 +1238,63 @@ static void show_ddir_status_terse(struct thread_stat *ts,
        }
 }
 
+static struct json_object *add_ddir_lat_json(struct thread_stat *ts, uint32_t percentiles,
+               struct io_stat *lat_stat, uint64_t *io_u_plat)
+{
+       char buf[120];
+       double mean, dev;
+       unsigned int i, len;
+       struct json_object *lat_object, *percentile_object, *clat_bins_object;
+       unsigned long long min, max, maxv, minv, *ovals = NULL;
+
+       if (!calc_lat(lat_stat, &min, &max, &mean, &dev)) {
+               min = max = 0;
+               mean = dev = 0.0;
+       }
+       lat_object = json_create_object();
+       json_object_add_value_int(lat_object, "min", min);
+       json_object_add_value_int(lat_object, "max", max);
+       json_object_add_value_float(lat_object, "mean", mean);
+       json_object_add_value_float(lat_object, "stddev", dev);
+       json_object_add_value_int(lat_object, "N", lat_stat->samples);
+
+       if (percentiles && lat_stat->samples) {
+               len = calc_clat_percentiles(io_u_plat, lat_stat->samples,
+                               ts->percentile_list, &ovals, &maxv, &minv);
+
+               if (len > FIO_IO_U_LIST_MAX_LEN)
+                       len = FIO_IO_U_LIST_MAX_LEN;
+
+               percentile_object = json_create_object();
+               json_object_add_value_object(lat_object, "percentile", percentile_object);
+               for (i = 0; i < len; i++) {
+                       snprintf(buf, sizeof(buf), "%f", ts->percentile_list[i].u.f);
+                       json_object_add_value_int(percentile_object, buf, ovals[i]);
+               }
+               free(ovals);
+
+               if (output_format & FIO_OUTPUT_JSON_PLUS) {
+                       clat_bins_object = json_create_object();
+                       json_object_add_value_object(lat_object, "bins", clat_bins_object);
+
+                       for(i = 0; i < FIO_IO_U_PLAT_NR; i++)
+                               if (io_u_plat[i]) {
+                                       snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i));
+                                       json_object_add_value_int(clat_bins_object, buf, io_u_plat[i]);
+                               }
+               }
+       }
+
+       return lat_object;
+}
+
 static void add_ddir_status_json(struct thread_stat *ts,
                struct group_run_stats *rs, int ddir, struct json_object *parent)
 {
-       unsigned long long min, max, minv, maxv;
+       unsigned long long min, max;
        unsigned long long bw_bytes, bw;
-       unsigned long long *ovals = NULL;
        double mean, dev, iops;
-       unsigned int len;
-       int i;
-       struct json_object *dir_object, *tmp_object, *percentile_object = NULL,
-               *clat_bins_object = NULL;
-       char buf[120];
+       struct json_object *dir_object, *tmp_object;
        double p_of_agg = 100.0;
 
        assert(ddir_rw(ddir) || ddir_sync(ddir));
@@ -1266,222 +1328,47 @@ static void add_ddir_status_json(struct thread_stat *ts,
                json_object_add_value_int(dir_object, "short_ios", ts->short_io_u[ddir]);
                json_object_add_value_int(dir_object, "drop_ios", ts->drop_io_u[ddir]);
 
-               if (!calc_lat(&ts->slat_stat[ddir], &min, &max, &mean, &dev)) {
-                       min = max = 0;
-                       mean = dev = 0.0;
-               }
-               tmp_object = json_create_object();
+               tmp_object = add_ddir_lat_json(ts, ts->slat_percentiles,
+                               &ts->slat_stat[ddir], ts->io_u_plat[FIO_SLAT][ddir]);
                json_object_add_value_object(dir_object, "slat_ns", tmp_object);
-               json_object_add_value_int(tmp_object, "min", min);
-               json_object_add_value_int(tmp_object, "max", max);
-               json_object_add_value_float(tmp_object, "mean", mean);
-               json_object_add_value_float(tmp_object, "stddev", dev);
-
-               if (!calc_lat(&ts->clat_stat[ddir], &min, &max, &mean, &dev)) {
-                       min = max = 0;
-                       mean = dev = 0.0;
-               }
-               tmp_object = json_create_object();
+
+               tmp_object = add_ddir_lat_json(ts, ts->clat_percentiles,
+                               &ts->clat_stat[ddir], ts->io_u_plat[FIO_CLAT][ddir]);
                json_object_add_value_object(dir_object, "clat_ns", tmp_object);
-               json_object_add_value_int(tmp_object, "min", min);
-               json_object_add_value_int(tmp_object, "max", max);
-               json_object_add_value_float(tmp_object, "mean", mean);
-               json_object_add_value_float(tmp_object, "stddev", dev);
-       } else {
-               if (!calc_lat(&ts->sync_stat, &min, &max, &mean, &dev)) {
-                       min = max = 0;
-                       mean = dev = 0.0;
-               }
 
-               tmp_object = json_create_object();
+               tmp_object = add_ddir_lat_json(ts, ts->lat_percentiles,
+                               &ts->lat_stat[ddir], ts->io_u_plat[FIO_LAT][ddir]);
                json_object_add_value_object(dir_object, "lat_ns", tmp_object);
+       } else {
                json_object_add_value_int(dir_object, "total_ios", ts->total_io_u[DDIR_SYNC]);
-               json_object_add_value_int(tmp_object, "min", min);
-               json_object_add_value_int(tmp_object, "max", max);
-               json_object_add_value_float(tmp_object, "mean", mean);
-               json_object_add_value_float(tmp_object, "stddev", dev);
-       }
-
-       if (ts->clat_percentiles || ts->lat_percentiles) {
-               if (ddir_rw(ddir)) {
-                       uint64_t samples;
-
-                       if (ts->clat_percentiles)
-                               samples = ts->clat_stat[ddir].samples;
-                       else
-                               samples = ts->lat_stat[ddir].samples;
-
-                       len = calc_clat_percentiles(ts->io_u_plat[FIO_CLAT][ddir],
-                                       samples, ts->percentile_list, &ovals,
-                                       &maxv, &minv);
-               } else {
-                       len = calc_clat_percentiles(ts->io_u_sync_plat,
-                                       ts->sync_stat.samples,
-                                       ts->percentile_list, &ovals, &maxv,
-                                       &minv);
-               }
-
-               if (len > FIO_IO_U_LIST_MAX_LEN)
-                       len = FIO_IO_U_LIST_MAX_LEN;
-       } else
-               len = 0;
-
-       if (ts->clat_percentiles) {
-               percentile_object = json_create_object();
-               json_object_add_value_object(tmp_object, "percentile", percentile_object);
-               for (i = 0; i < len; i++) {
-                       snprintf(buf, sizeof(buf), "%f",
-                                ts->percentile_list[i].u.f);
-                       json_object_add_value_int(percentile_object, buf,
-                                                 ovals[i]);
-               }
-       }
-
-       free(ovals);
-
-       if (output_format & FIO_OUTPUT_JSON_PLUS && ts->clat_percentiles) {
-               clat_bins_object = json_create_object();
-               json_object_add_value_object(tmp_object, "bins",
-                                            clat_bins_object);
-
-               for(i = 0; i < FIO_IO_U_PLAT_NR; i++) {
-                       if (ddir_rw(ddir)) {
-                               if (ts->io_u_plat[FIO_CLAT][ddir][i]) {
-                                       snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i));
-                                       json_object_add_value_int(clat_bins_object, buf, ts->io_u_plat[FIO_CLAT][ddir][i]);
-                               }
-                       } else {
-                               if (ts->io_u_sync_plat[i]) {
-                                       snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i));
-                                       json_object_add_value_int(clat_bins_object, buf, ts->io_u_sync_plat[i]);
-                               }
-                       }
-               }
+               tmp_object = add_ddir_lat_json(ts, ts->lat_percentiles | ts->clat_percentiles,
+                               &ts->sync_stat, ts->io_u_sync_plat);
+               json_object_add_value_object(dir_object, "lat_ns", tmp_object);
        }
 
+       if (!ddir_rw(ddir))
+               return;
 
        /* Only print PRIO latencies if some high priority samples were gathered */
        if (ts->clat_high_prio_stat[ddir].samples > 0) {
-               /* START OF HIGH PRIO CLAT */
-           if (!calc_lat(&ts->clat_high_prio_stat[ddir], &min, &max, &mean, &dev)) {
-                       min = max = 0;
-                       mean = dev = 0.0;
-               }
-               tmp_object = json_create_object();
-               json_object_add_value_object(dir_object, "clat_prio",
-                               tmp_object);
-               json_object_add_value_int(tmp_object, "samples",
-                               ts->clat_high_prio_stat[ddir].samples);
-               json_object_add_value_int(tmp_object, "min", min);
-               json_object_add_value_int(tmp_object, "max", max);
-               json_object_add_value_float(tmp_object, "mean", mean);
-               json_object_add_value_float(tmp_object, "stddev", dev);
-
-               if (ts->clat_percentiles) {
-                       len = calc_clat_percentiles(ts->io_u_plat_high_prio[ddir],
-                                               ts->clat_high_prio_stat[ddir].samples,
-                                               ts->percentile_list, &ovals, &maxv,
-                                               &minv);
-               } else
-                       len = 0;
-
-               percentile_object = json_create_object();
-               json_object_add_value_object(tmp_object, "percentile", percentile_object);
-               for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) {
-                       if (i >= len) {
-                               json_object_add_value_int(percentile_object, "0.00", 0);
-                               continue;
-                       }
-                       snprintf(buf, sizeof(buf), "%f", ts->percentile_list[i].u.f);
-                       json_object_add_value_int(percentile_object, (const char *)buf, ovals[i]);
-               }
-
-               if (output_format & FIO_OUTPUT_JSON_PLUS) {
-                       clat_bins_object = json_create_object();
-                       json_object_add_value_object(tmp_object, "bins", clat_bins_object);
-                       for(i = 0; i < FIO_IO_U_PLAT_NR; i++) {
-                               snprintf(buf, sizeof(buf), "%d", i);
-                               json_object_add_value_int(clat_bins_object, (const char *)buf,
-                                               ts->io_u_plat_high_prio[ddir][i]);
-                       }
-                       json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_BITS",
-                                       FIO_IO_U_PLAT_BITS);
-                       json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_VAL",
-                                       FIO_IO_U_PLAT_VAL);
-                       json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_NR",
-                                       FIO_IO_U_PLAT_NR);
-               }
-               /* END OF HIGH PRIO CLAT */
+               const char *high, *low;
 
-               /* START OF PRIO CLAT */
-           if (!calc_lat(&ts->clat_prio_stat[ddir], &min, &max, &mean, &dev)) {
-                       min = max = 0;
-                       mean = dev = 0.0;
-               }
-               tmp_object = json_create_object();
-               json_object_add_value_object(dir_object, "clat_low_prio",
-                               tmp_object);
-               json_object_add_value_int(tmp_object, "samples",
-                               ts->clat_prio_stat[ddir].samples);
-               json_object_add_value_int(tmp_object, "min", min);
-               json_object_add_value_int(tmp_object, "max", max);
-               json_object_add_value_float(tmp_object, "mean", mean);
-               json_object_add_value_float(tmp_object, "stddev", dev);
-
-               if (ts->clat_percentiles) {
-                       len = calc_clat_percentiles(ts->io_u_plat_prio[ddir],
-                                               ts->clat_prio_stat[ddir].samples,
-                                               ts->percentile_list, &ovals, &maxv,
-                                               &minv);
-               } else
-                       len = 0;
-
-               percentile_object = json_create_object();
-               json_object_add_value_object(tmp_object, "percentile", percentile_object);
-               for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) {
-                       if (i >= len) {
-                               json_object_add_value_int(percentile_object, "0.00", 0);
-                               continue;
-                       }
-                       snprintf(buf, sizeof(buf), "%f", ts->percentile_list[i].u.f);
-                       json_object_add_value_int(percentile_object, (const char *)buf, ovals[i]);
-               }
-
-               if (output_format & FIO_OUTPUT_JSON_PLUS) {
-                       clat_bins_object = json_create_object();
-                       json_object_add_value_object(tmp_object, "bins", clat_bins_object);
-                       for(i = 0; i < FIO_IO_U_PLAT_NR; i++) {
-                               snprintf(buf, sizeof(buf), "%d", i);
-                               json_object_add_value_int(clat_bins_object, (const char *)buf,
-                                               ts->io_u_plat_prio[ddir][i]);
-                       }
-                       json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_BITS",
-                                       FIO_IO_U_PLAT_BITS);
-                       json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_VAL",
-                                       FIO_IO_U_PLAT_VAL);
-                       json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_NR",
-                                       FIO_IO_U_PLAT_NR);
+               if (ts->lat_percentiles) {
+                       high = "lat_prio";
+                       low = "lat_low_prio";
+               } else {
+                       high = "clat_prio";
+                       low = "clat_low_prio";
                }
-               /* END OF PRIO CLAT */
-       }
 
-       if (!ddir_rw(ddir))
-               return;
+               tmp_object = add_ddir_lat_json(ts, ts->clat_percentiles | ts->lat_percentiles,
+                               &ts->clat_high_prio_stat[ddir], ts->io_u_plat_high_prio[ddir]);
+               json_object_add_value_object(dir_object, high, tmp_object);
 
-       if (!calc_lat(&ts->lat_stat[ddir], &min, &max, &mean, &dev)) {
-               min = max = 0;
-               mean = dev = 0.0;
+               tmp_object = add_ddir_lat_json(ts, ts->clat_percentiles | ts->lat_percentiles,
+                               &ts->clat_prio_stat[ddir], ts->io_u_plat_prio[ddir]);
+               json_object_add_value_object(dir_object, low, tmp_object);
        }
-       tmp_object = json_create_object();
-       json_object_add_value_object(dir_object, "lat_ns", tmp_object);
-       json_object_add_value_int(tmp_object, "min", min);
-       json_object_add_value_int(tmp_object, "max", max);
-       json_object_add_value_float(tmp_object, "mean", mean);
-       json_object_add_value_float(tmp_object, "stddev", dev);
-       if (ts->lat_percentiles)
-               json_object_add_value_object(tmp_object, "percentile", percentile_object);
-       if (output_format & FIO_OUTPUT_JSON_PLUS && ts->lat_percentiles)
-               json_object_add_value_object(tmp_object, "bins", clat_bins_object);
 
        if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) {
                if (rs->agg[ddir]) {
@@ -1493,6 +1380,7 @@ static void add_ddir_status_json(struct thread_stat *ts,
                min = max = 0;
                p_of_agg = mean = dev = 0.0;
        }
+
        json_object_add_value_int(dir_object, "bw_min", min);
        json_object_add_value_int(dir_object, "bw_max", max);
        json_object_add_value_float(dir_object, "bw_agg", p_of_agg);
@@ -2177,6 +2065,7 @@ void __show_run_stats(void)
 
                ts->clat_percentiles = td->o.clat_percentiles;
                ts->lat_percentiles = td->o.lat_percentiles;
+               ts->slat_percentiles = td->o.slat_percentiles;
                ts->percentile_precision = td->o.percentile_precision;
                memcpy(ts->percentile_list, td->o.percentile_list, sizeof(td->o.percentile_list));
                opt_lists[j] = &td->opt_list;
@@ -2900,19 +2789,27 @@ void add_sync_clat_sample(struct thread_stat *ts, unsigned long long nsec)
        add_stat_sample(&ts->sync_stat, nsec);
 }
 
-static void add_clat_percentile_sample(struct thread_stat *ts,
-                               unsigned long long nsec, enum fio_ddir ddir, uint8_t priority_bit)
+static void add_lat_percentile_sample_noprio(struct thread_stat *ts,
+                               unsigned long long nsec, enum fio_ddir ddir, enum fio_lat lat)
 {
        unsigned int idx = plat_val_to_idx(nsec);
        assert(idx < FIO_IO_U_PLAT_NR);
 
-       ts->io_u_plat[FIO_CLAT][ddir][idx]++;
+       ts->io_u_plat[lat][ddir][idx]++;
+}
 
-       if (!priority_bit) {
+static void add_lat_percentile_sample(struct thread_stat *ts,
+                               unsigned long long nsec, enum fio_ddir ddir, uint8_t priority_bit,
+                               enum fio_lat lat)
+{
+       unsigned int idx = plat_val_to_idx(nsec);
+
+       add_lat_percentile_sample_noprio(ts, nsec, ddir, lat);
+
+       if (!priority_bit)
                ts->io_u_plat_prio[ddir][idx]++;
-       } else {
+       else
                ts->io_u_plat_high_prio[ddir][idx]++;
-       }
 }
 
 void add_clat_sample(struct thread_data *td, enum fio_ddir ddir,
@@ -2940,7 +2837,10 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir,
                               offset, priority_bit);
 
        if (ts->clat_percentiles) {
-               add_clat_percentile_sample(ts, nsec, ddir, priority_bit);
+               if (ts->lat_percentiles)
+                       add_lat_percentile_sample_noprio(ts, nsec, ddir, FIO_CLAT);
+               else
+                       add_lat_percentile_sample(ts, nsec, ddir, priority_bit, FIO_CLAT);
        }
 
        if (iolog && iolog->hist_msec) {
@@ -2986,7 +2886,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir,
 }
 
 void add_slat_sample(struct thread_data *td, enum fio_ddir ddir,
-                       unsigned long usec, unsigned long long bs, uint64_t offset,
+                       unsigned long long nsec, unsigned long long bs, uint64_t offset,
                        uint8_t priority_bit)
 {
        const bool needs_lock = td_async_processing(td);
@@ -2998,12 +2898,15 @@ void add_slat_sample(struct thread_data *td, enum fio_ddir ddir,
        if (needs_lock)
                __td_io_u_lock(td);
 
-       add_stat_sample(&ts->slat_stat[ddir], usec);
+       add_stat_sample(&ts->slat_stat[ddir], nsec);
 
        if (td->slat_log)
-               add_log_sample(td, td->slat_log, sample_val(usec), ddir, bs, offset,
+               add_log_sample(td, td->slat_log, sample_val(nsec), ddir, bs, offset,
                        priority_bit);
 
+       if (ts->slat_percentiles)
+               add_lat_percentile_sample_noprio(ts, nsec, ddir, FIO_SLAT);
+
        if (needs_lock)
                __td_io_u_unlock(td);
 }
@@ -3028,7 +2931,7 @@ void add_lat_sample(struct thread_data *td, enum fio_ddir ddir,
                               offset, priority_bit);
 
        if (ts->lat_percentiles)
-               add_clat_percentile_sample(ts, nsec, ddir, priority_bit);
+               add_lat_percentile_sample(ts, nsec, ddir, priority_bit, FIO_LAT);
 
        if (needs_lock)
                __td_io_u_unlock(td);
diff --git a/stat.h b/stat.h
index 0dc7c4d..3c1653f 100644 (file)
--- a/stat.h
+++ b/stat.h
@@ -37,7 +37,7 @@ struct group_run_stats {
                                        list of percentiles */
 
 /*
- * Aggregate clat samples to report percentile(s) of them.
+ * Aggregate latency samples for reporting percentile(s).
  *
  * EXECUTIVE SUMMARY
  *
@@ -58,7 +58,7 @@ struct group_run_stats {
  *
  * DETAILS
  *
- * Suppose the clat varies from 0 to 999 (usec), the straightforward
+ * Suppose the lat varies from 0 to 999 (usec), the straightforward
  * method is to keep an array of (999 + 1) buckets, in which a counter
  * keeps the count of samples which fall in the bucket, e.g.,
  * {[0],[1],...,[999]}. However this consumes a huge amount of space,
@@ -147,7 +147,7 @@ enum block_info_state {
 #define FIO_JOBDESC_SIZE       256
 #define FIO_VERROR_SIZE                128
 
-enum {
+enum fio_lat {
        FIO_SLAT = 0,
        FIO_CLAT,
        FIO_LAT,
@@ -189,6 +189,8 @@ struct thread_stat {
         */
        uint32_t clat_percentiles;
        uint32_t lat_percentiles;
+       uint32_t slat_percentiles;
+       uint32_t pad;
        uint64_t percentile_precision;
        fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN];
 
@@ -339,7 +341,7 @@ extern void add_lat_sample(struct thread_data *, enum fio_ddir, unsigned long lo
                                unsigned long long, uint64_t, uint8_t);
 extern void add_clat_sample(struct thread_data *, enum fio_ddir, unsigned long long,
                                unsigned long long, uint64_t, uint8_t);
-extern void add_slat_sample(struct thread_data *, enum fio_ddir, unsigned long,
+extern void add_slat_sample(struct thread_data *, enum fio_ddir, unsigned long long,
                                unsigned long long, uint64_t, uint8_t);
 extern void add_agg_sample(union io_sample_data, enum fio_ddir, unsigned long long bs,
                                uint8_t priority_bit);
index 4b131bd..c78ed43 100644 (file)
@@ -250,6 +250,7 @@ struct thread_options {
        unsigned int trim_zero;
        unsigned long long trim_backlog;
        unsigned int clat_percentiles;
+       unsigned int slat_percentiles;
        unsigned int lat_percentiles;
        unsigned int percentile_precision;      /* digits after decimal for percentiles */
        fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN];
@@ -370,7 +371,7 @@ struct thread_options_pack {
        uint32_t iodepth_batch_complete_min;
        uint32_t iodepth_batch_complete_max;
        uint32_t serialize_overlap;
-       uint32_t lat_percentiles;
+       uint32_t pad;
 
        uint64_t size;
        uint64_t io_size;
@@ -430,7 +431,7 @@ struct thread_options_pack {
        uint32_t override_sync;
        uint32_t rand_repeatable;
        uint32_t allrand_repeatable;
-       uint32_t pad;
+       uint32_t pad2;
        uint64_t rand_seed;
        uint32_t log_avg_msec;
        uint32_t log_hist_msec;
@@ -464,7 +465,6 @@ struct thread_options_pack {
 
        uint32_t hugepage_size;
        uint64_t rw_min_bs;
-       uint32_t pad2;
        uint32_t thinktime;
        uint32_t thinktime_spin;
        uint32_t thinktime_blocks;
@@ -539,7 +539,10 @@ struct thread_options_pack {
        uint32_t trim_zero;
        uint64_t trim_backlog;
        uint32_t clat_percentiles;
+       uint32_t lat_percentiles;
+       uint32_t slat_percentiles;
        uint32_t percentile_precision;
+       uint32_t pad3;
        fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN];
 
        uint8_t read_iolog_file[FIO_TOP_STR_MAX];
@@ -573,7 +576,6 @@ struct thread_options_pack {
        uint32_t rate_iops_min[DDIR_RWDIR_CNT];
        uint32_t rate_process;
        uint32_t rate_ign_think;
-       uint32_t pad3;
 
        uint8_t ioscheduler[FIO_TOP_STR_MAX];
 
@@ -603,7 +605,6 @@ struct thread_options_pack {
        uint32_t flow_sleep;
 
        uint32_t offset_increment_percent;
-       uint32_t pad4;
        uint64_t offset_increment;
        uint64_t number_ios;