X-Git-Url: https://git.kernel.dk/?a=blobdiff_plain;f=stat.c;h=d8c01d14d594769df3c4e95a1451d9e767bb39af;hb=409a4f291e7f4011f6ccfe748c6ed9786431ddb3;hp=4d3b728c3815dac1614eef6bee10475e0a617bd1;hpb=d9b7596a1fad5adf7f6731d067e1513c56eabe96;p=fio.git diff --git a/stat.c b/stat.c index 4d3b728c..d8c01d14 100644 --- a/stat.c +++ b/stat.c @@ -482,21 +482,62 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, display_lat("clat", min, max, mean, dev, out); if (calc_lat(&ts->lat_stat[ddir], &min, &max, &mean, &dev)) display_lat(" lat", min, max, mean, dev, out); + if (calc_lat(&ts->clat_high_prio_stat[ddir], &min, &max, &mean, &dev)) { + display_lat(ts->lat_percentiles ? "high prio_lat" : "high prio_clat", + min, max, mean, dev, out); + if (calc_lat(&ts->clat_low_prio_stat[ddir], &min, &max, &mean, &dev)) + display_lat(ts->lat_percentiles ? "low prio_lat" : "low prio_clat", + min, max, mean, dev, out); + } + + if (ts->slat_percentiles && ts->slat_stat[ddir].samples > 0) + show_clat_percentiles(ts->io_u_plat[FIO_SLAT][ddir], + ts->slat_stat[ddir].samples, + ts->percentile_list, + ts->percentile_precision, "slat", out); + if (ts->clat_percentiles && ts->clat_stat[ddir].samples > 0) + show_clat_percentiles(ts->io_u_plat[FIO_CLAT][ddir], + ts->clat_stat[ddir].samples, + ts->percentile_list, + ts->percentile_precision, "clat", out); + if (ts->lat_percentiles && ts->lat_stat[ddir].samples > 0) + show_clat_percentiles(ts->io_u_plat[FIO_LAT][ddir], + ts->lat_stat[ddir].samples, + ts->percentile_list, + ts->percentile_precision, "lat", out); if (ts->clat_percentiles || ts->lat_percentiles) { - const char *name = ts->clat_percentiles ? "clat" : " lat"; + const char *name = ts->lat_percentiles ? "lat" : "clat"; + char prio_name[32]; uint64_t samples; - if (ts->clat_percentiles) - samples = ts->clat_stat[ddir].samples; - else + if (ts->lat_percentiles) samples = ts->lat_stat[ddir].samples; + else + samples = ts->clat_stat[ddir].samples; - show_clat_percentiles(ts->io_u_plat[ddir], - samples, - ts->percentile_list, - ts->percentile_precision, name, out); + /* Only print this if some high and low priority stats were collected */ + if (ts->clat_high_prio_stat[ddir].samples > 0 && + ts->clat_low_prio_stat[ddir].samples > 0) + { + sprintf(prio_name, "high prio (%.2f%%) %s", + 100. * (double) ts->clat_high_prio_stat[ddir].samples / (double) samples, + name); + show_clat_percentiles(ts->io_u_plat_high_prio[ddir], + ts->clat_high_prio_stat[ddir].samples, + ts->percentile_list, + ts->percentile_precision, prio_name, out); + + sprintf(prio_name, "low prio (%.2f%%) %s", + 100. * (double) ts->clat_low_prio_stat[ddir].samples / (double) samples, + name); + show_clat_percentiles(ts->io_u_plat_low_prio[ddir], + ts->clat_low_prio_stat[ddir].samples, + ts->percentile_list, + ts->percentile_precision, prio_name, out); + } } + if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) { double p_of_agg = 100.0, fkb_base = (double)rs->kb_base; const char *bw_str; @@ -913,7 +954,7 @@ void json_array_add_disk_util(struct disk_util_stat *dus, obj = json_create_object(); json_array_add_value_object(array, obj); - json_object_add_value_string(obj, "name", dus->name); + json_object_add_value_string(obj, "name", (const char *)dus->name); json_object_add_value_int(obj, "read_ios", dus->s.ios[0]); json_object_add_value_int(obj, "write_ios", dus->s.ios[1]); json_object_add_value_int(obj, "read_merges", dus->s.merges[0]); @@ -1009,7 +1050,7 @@ static void show_thread_status_normal(struct thread_stat *ts, if (!ddir_rw_sum(ts->io_bytes) && !ddir_rw_sum(ts->total_io_u)) return; - + memset(time_buf, 0, sizeof(time_buf)); time(&time_p); @@ -1146,12 +1187,17 @@ static void show_ddir_status_terse(struct thread_stat *ts, else log_buf(out, ";%llu;%llu;%f;%f", 0ULL, 0ULL, 0.0, 0.0); - if (ts->clat_percentiles || ts->lat_percentiles) { - len = calc_clat_percentiles(ts->io_u_plat[ddir], + if (ts->lat_percentiles) + len = calc_clat_percentiles(ts->io_u_plat[FIO_LAT][ddir], + ts->lat_stat[ddir].samples, + ts->percentile_list, &ovals, &maxv, + &minv); + else if (ts->clat_percentiles) + len = calc_clat_percentiles(ts->io_u_plat[FIO_CLAT][ddir], ts->clat_stat[ddir].samples, ts->percentile_list, &ovals, &maxv, &minv); - } else + else len = 0; for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) { @@ -1197,18 +1243,63 @@ static void show_ddir_status_terse(struct thread_stat *ts, } } +static struct json_object *add_ddir_lat_json(struct thread_stat *ts, uint32_t percentiles, + struct io_stat *lat_stat, uint64_t *io_u_plat) +{ + char buf[120]; + double mean, dev; + unsigned int i, len; + struct json_object *lat_object, *percentile_object, *clat_bins_object; + unsigned long long min, max, maxv, minv, *ovals = NULL; + + if (!calc_lat(lat_stat, &min, &max, &mean, &dev)) { + min = max = 0; + mean = dev = 0.0; + } + lat_object = json_create_object(); + json_object_add_value_int(lat_object, "min", min); + json_object_add_value_int(lat_object, "max", max); + json_object_add_value_float(lat_object, "mean", mean); + json_object_add_value_float(lat_object, "stddev", dev); + json_object_add_value_int(lat_object, "N", lat_stat->samples); + + if (percentiles && lat_stat->samples) { + len = calc_clat_percentiles(io_u_plat, lat_stat->samples, + ts->percentile_list, &ovals, &maxv, &minv); + + if (len > FIO_IO_U_LIST_MAX_LEN) + len = FIO_IO_U_LIST_MAX_LEN; + + percentile_object = json_create_object(); + json_object_add_value_object(lat_object, "percentile", percentile_object); + for (i = 0; i < len; i++) { + snprintf(buf, sizeof(buf), "%f", ts->percentile_list[i].u.f); + json_object_add_value_int(percentile_object, buf, ovals[i]); + } + free(ovals); + + if (output_format & FIO_OUTPUT_JSON_PLUS) { + clat_bins_object = json_create_object(); + json_object_add_value_object(lat_object, "bins", clat_bins_object); + + for(i = 0; i < FIO_IO_U_PLAT_NR; i++) + if (io_u_plat[i]) { + snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i)); + json_object_add_value_int(clat_bins_object, buf, io_u_plat[i]); + } + } + } + + return lat_object; +} + static void add_ddir_status_json(struct thread_stat *ts, struct group_run_stats *rs, int ddir, struct json_object *parent) { - unsigned long long min, max, minv, maxv; + unsigned long long min, max; unsigned long long bw_bytes, bw; - unsigned long long *ovals = NULL; double mean, dev, iops; - unsigned int len; - int i; - struct json_object *dir_object, *tmp_object, *percentile_object = NULL, - *clat_bins_object = NULL; - char buf[120]; + struct json_object *dir_object, *tmp_object; double p_of_agg = 100.0; assert(ddir_rw(ddir) || ddir_sync(ddir)); @@ -1242,117 +1333,48 @@ static void add_ddir_status_json(struct thread_stat *ts, json_object_add_value_int(dir_object, "short_ios", ts->short_io_u[ddir]); json_object_add_value_int(dir_object, "drop_ios", ts->drop_io_u[ddir]); - if (!calc_lat(&ts->slat_stat[ddir], &min, &max, &mean, &dev)) { - min = max = 0; - mean = dev = 0.0; - } - tmp_object = json_create_object(); + tmp_object = add_ddir_lat_json(ts, ts->slat_percentiles, + &ts->slat_stat[ddir], ts->io_u_plat[FIO_SLAT][ddir]); json_object_add_value_object(dir_object, "slat_ns", tmp_object); - json_object_add_value_int(tmp_object, "min", min); - json_object_add_value_int(tmp_object, "max", max); - json_object_add_value_float(tmp_object, "mean", mean); - json_object_add_value_float(tmp_object, "stddev", dev); - - if (!calc_lat(&ts->clat_stat[ddir], &min, &max, &mean, &dev)) { - min = max = 0; - mean = dev = 0.0; - } - tmp_object = json_create_object(); + + tmp_object = add_ddir_lat_json(ts, ts->clat_percentiles, + &ts->clat_stat[ddir], ts->io_u_plat[FIO_CLAT][ddir]); json_object_add_value_object(dir_object, "clat_ns", tmp_object); - json_object_add_value_int(tmp_object, "min", min); - json_object_add_value_int(tmp_object, "max", max); - json_object_add_value_float(tmp_object, "mean", mean); - json_object_add_value_float(tmp_object, "stddev", dev); - } else { - if (!calc_lat(&ts->sync_stat, &min, &max, &mean, &dev)) { - min = max = 0; - mean = dev = 0.0; - } - tmp_object = json_create_object(); + tmp_object = add_ddir_lat_json(ts, ts->lat_percentiles, + &ts->lat_stat[ddir], ts->io_u_plat[FIO_LAT][ddir]); json_object_add_value_object(dir_object, "lat_ns", tmp_object); + } else { json_object_add_value_int(dir_object, "total_ios", ts->total_io_u[DDIR_SYNC]); - json_object_add_value_int(tmp_object, "min", min); - json_object_add_value_int(tmp_object, "max", max); - json_object_add_value_float(tmp_object, "mean", mean); - json_object_add_value_float(tmp_object, "stddev", dev); + tmp_object = add_ddir_lat_json(ts, ts->lat_percentiles | ts->clat_percentiles, + &ts->sync_stat, ts->io_u_sync_plat); + json_object_add_value_object(dir_object, "lat_ns", tmp_object); } - if (ts->clat_percentiles || ts->lat_percentiles) { - if (ddir_rw(ddir)) { - uint64_t samples; + if (!ddir_rw(ddir)) + return; - if (ts->clat_percentiles) - samples = ts->clat_stat[ddir].samples; - else - samples = ts->lat_stat[ddir].samples; + /* Only print PRIO latencies if some high priority samples were gathered */ + if (ts->clat_high_prio_stat[ddir].samples > 0) { + const char *high, *low; - len = calc_clat_percentiles(ts->io_u_plat[ddir], - samples, ts->percentile_list, &ovals, - &maxv, &minv); + if (ts->lat_percentiles) { + high = "lat_high_prio"; + low = "lat_low_prio"; } else { - len = calc_clat_percentiles(ts->io_u_sync_plat, - ts->sync_stat.samples, - ts->percentile_list, &ovals, &maxv, - &minv); + high = "clat_high_prio"; + low = "clat_low_prio"; } - if (len > FIO_IO_U_LIST_MAX_LEN) - len = FIO_IO_U_LIST_MAX_LEN; - } else - len = 0; + tmp_object = add_ddir_lat_json(ts, ts->clat_percentiles | ts->lat_percentiles, + &ts->clat_high_prio_stat[ddir], ts->io_u_plat_high_prio[ddir]); + json_object_add_value_object(dir_object, high, tmp_object); - if (ts->clat_percentiles) { - percentile_object = json_create_object(); - json_object_add_value_object(tmp_object, "percentile", percentile_object); - for (i = 0; i < len; i++) { - snprintf(buf, sizeof(buf), "%f", - ts->percentile_list[i].u.f); - json_object_add_value_int(percentile_object, buf, - ovals[i]); - } + tmp_object = add_ddir_lat_json(ts, ts->clat_percentiles | ts->lat_percentiles, + &ts->clat_low_prio_stat[ddir], ts->io_u_plat_low_prio[ddir]); + json_object_add_value_object(dir_object, low, tmp_object); } - free(ovals); - - if (output_format & FIO_OUTPUT_JSON_PLUS && ts->clat_percentiles) { - clat_bins_object = json_create_object(); - json_object_add_value_object(tmp_object, "bins", - clat_bins_object); - - for(i = 0; i < FIO_IO_U_PLAT_NR; i++) { - if (ddir_rw(ddir)) { - if (ts->io_u_plat[ddir][i]) { - snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i)); - json_object_add_value_int(clat_bins_object, buf, ts->io_u_plat[ddir][i]); - } - } else { - if (ts->io_u_sync_plat[i]) { - snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i)); - json_object_add_value_int(clat_bins_object, buf, ts->io_u_sync_plat[i]); - } - } - } - } - - if (!ddir_rw(ddir)) - return; - - if (!calc_lat(&ts->lat_stat[ddir], &min, &max, &mean, &dev)) { - min = max = 0; - mean = dev = 0.0; - } - tmp_object = json_create_object(); - json_object_add_value_object(dir_object, "lat_ns", tmp_object); - json_object_add_value_int(tmp_object, "min", min); - json_object_add_value_int(tmp_object, "max", max); - json_object_add_value_float(tmp_object, "mean", mean); - json_object_add_value_float(tmp_object, "stddev", dev); - if (ts->lat_percentiles) - json_object_add_value_object(tmp_object, "percentile", percentile_object); - if (output_format & FIO_OUTPUT_JSON_PLUS && ts->lat_percentiles) - json_object_add_value_object(tmp_object, "bins", clat_bins_object); - if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) { if (rs->agg[ddir]) { p_of_agg = mean * 100 / (double) (rs->agg[ddir] / 1024); @@ -1363,6 +1385,7 @@ static void add_ddir_status_json(struct thread_stat *ts, min = max = 0; p_of_agg = mean = dev = 0.0; } + json_object_add_value_int(dir_object, "bw_min", min); json_object_add_value_int(dir_object, "bw_max", max); json_object_add_value_float(dir_object, "bw_agg", p_of_agg); @@ -1851,11 +1874,13 @@ void sum_group_stats(struct group_run_stats *dst, struct group_run_stats *src) void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src, bool first) { - int l, k; + int k, l, m; for (l = 0; l < DDIR_RWDIR_CNT; l++) { if (!dst->unified_rw_rep) { sum_stat(&dst->clat_stat[l], &src->clat_stat[l], first, false); + sum_stat(&dst->clat_high_prio_stat[l], &src->clat_high_prio_stat[l], first, false); + sum_stat(&dst->clat_low_prio_stat[l], &src->clat_low_prio_stat[l], first, false); sum_stat(&dst->slat_stat[l], &src->slat_stat[l], first, false); sum_stat(&dst->lat_stat[l], &src->lat_stat[l], first, false); sum_stat(&dst->bw_stat[l], &src->bw_stat[l], first, true); @@ -1867,6 +1892,8 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src, dst->runtime[l] = src->runtime[l]; } else { sum_stat(&dst->clat_stat[0], &src->clat_stat[l], first, false); + sum_stat(&dst->clat_high_prio_stat[0], &src->clat_high_prio_stat[l], first, false); + sum_stat(&dst->clat_low_prio_stat[0], &src->clat_low_prio_stat[l], first, false); sum_stat(&dst->slat_stat[0], &src->slat_stat[l], first, false); sum_stat(&dst->lat_stat[0], &src->lat_stat[l], first, false); sum_stat(&dst->bw_stat[0], &src->bw_stat[l], first, true); @@ -1905,9 +1932,6 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src, for (k = 0; k < FIO_IO_U_LAT_M_NR; k++) dst->io_u_lat_m[k] += src->io_u_lat_m[k]; - for (k = 0; k < FIO_IO_U_PLAT_NR; k++) - dst->io_u_sync_plat[k] += src->io_u_sync_plat[k]; - for (k = 0; k < DDIR_RWDIR_CNT; k++) { if (!dst->unified_rw_rep) { dst->total_io_u[k] += src->total_io_u[k]; @@ -1922,14 +1946,27 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src, dst->total_io_u[DDIR_SYNC] += src->total_io_u[DDIR_SYNC]; - for (k = 0; k < DDIR_RWDIR_CNT; k++) { - int m; + for (k = 0; k < FIO_LAT_CNT; k++) + for (l = 0; l < DDIR_RWDIR_CNT; l++) + for (m = 0; m < FIO_IO_U_PLAT_NR; m++) + if (!dst->unified_rw_rep) + dst->io_u_plat[k][l][m] += src->io_u_plat[k][l][m]; + else + dst->io_u_plat[k][0][m] += src->io_u_plat[k][l][m]; + for (k = 0; k < FIO_IO_U_PLAT_NR; k++) + dst->io_u_sync_plat[k] += src->io_u_sync_plat[k]; + + for (k = 0; k < DDIR_RWDIR_CNT; k++) { for (m = 0; m < FIO_IO_U_PLAT_NR; m++) { - if (!dst->unified_rw_rep) - dst->io_u_plat[k][m] += src->io_u_plat[k][m]; - else - dst->io_u_plat[0][m] += src->io_u_plat[k][m]; + if (!dst->unified_rw_rep) { + dst->io_u_plat_high_prio[k][m] += src->io_u_plat_high_prio[k][m]; + dst->io_u_plat_low_prio[k][m] += src->io_u_plat_low_prio[k][m]; + } else { + dst->io_u_plat_high_prio[0][m] += src->io_u_plat_high_prio[k][m]; + dst->io_u_plat_low_prio[0][m] += src->io_u_plat_low_prio[k][m]; + } + } } @@ -1962,6 +1999,8 @@ void init_thread_stat(struct thread_stat *ts) ts->slat_stat[j].min_val = -1UL; ts->bw_stat[j].min_val = -1UL; ts->iops_stat[j].min_val = -1UL; + ts->clat_high_prio_stat[j].min_val = -1UL; + ts->clat_low_prio_stat[j].min_val = -1UL; } ts->sync_stat.min_val = -1UL; ts->groupid = -1; @@ -2031,6 +2070,7 @@ void __show_run_stats(void) ts->clat_percentiles = td->o.clat_percentiles; ts->lat_percentiles = td->o.lat_percentiles; + ts->slat_percentiles = td->o.slat_percentiles; ts->percentile_precision = td->o.percentile_precision; memcpy(ts->percentile_list, td->o.percentile_list, sizeof(td->o.percentile_list)); opt_lists[j] = &td->opt_list; @@ -2542,7 +2582,7 @@ static struct io_logs *get_cur_log(struct io_log *iolog) static void __add_log_sample(struct io_log *iolog, union io_sample_data data, enum fio_ddir ddir, unsigned long long bs, - unsigned long t, uint64_t offset) + unsigned long t, uint64_t offset, uint8_t priority_bit) { struct io_logs *cur_log; @@ -2561,6 +2601,7 @@ static void __add_log_sample(struct io_log *iolog, union io_sample_data data, s->time = t + (iolog->td ? iolog->td->unix_epoch : 0); io_sample_set_ddir(iolog, s, ddir); s->bs = bs; + s->priority_bit = priority_bit; if (iolog->log_offset) { struct io_sample_offset *so = (void *) s; @@ -2585,9 +2626,11 @@ static inline void reset_io_stat(struct io_stat *ios) void reset_io_stats(struct thread_data *td) { struct thread_stat *ts = &td->ts; - int i, j; + int i, j, k; for (i = 0; i < DDIR_RWDIR_CNT; i++) { + reset_io_stat(&ts->clat_high_prio_stat[i]); + reset_io_stat(&ts->clat_low_prio_stat[i]); reset_io_stat(&ts->clat_stat[i]); reset_io_stat(&ts->slat_stat[i]); reset_io_stat(&ts->lat_stat[i]); @@ -2601,12 +2644,18 @@ void reset_io_stats(struct thread_data *td) ts->drop_io_u[i] = 0; for (j = 0; j < FIO_IO_U_PLAT_NR; j++) { - ts->io_u_plat[i][j] = 0; + ts->io_u_plat_high_prio[i][j] = 0; + ts->io_u_plat_low_prio[i][j] = 0; if (!i) ts->io_u_sync_plat[j] = 0; } } + for (i = 0; i < FIO_LAT_CNT; i++) + for (j = 0; j < DDIR_RWDIR_CNT; j++) + for (k = 0; k < FIO_IO_U_PLAT_NR; k++) + ts->io_u_plat[i][j][k] = 0; + ts->total_io_u[DDIR_SYNC] = 0; for (i = 0; i < FIO_IO_U_MAP_NR; i++) { @@ -2629,7 +2678,7 @@ void reset_io_stats(struct thread_data *td) } static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir, - unsigned long elapsed, bool log_max) + unsigned long elapsed, bool log_max, uint8_t priority_bit) { /* * Note an entry in the log. Use the mean from the logged samples, @@ -2644,26 +2693,26 @@ static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir, else data.val = iolog->avg_window[ddir].mean.u.f + 0.50; - __add_log_sample(iolog, data, ddir, 0, elapsed, 0); + __add_log_sample(iolog, data, ddir, 0, elapsed, 0, priority_bit); } reset_io_stat(&iolog->avg_window[ddir]); } static void _add_stat_to_log(struct io_log *iolog, unsigned long elapsed, - bool log_max) + bool log_max, uint8_t priority_bit) { int ddir; for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) - __add_stat_to_log(iolog, ddir, elapsed, log_max); + __add_stat_to_log(iolog, ddir, elapsed, log_max, priority_bit); } static unsigned long add_log_sample(struct thread_data *td, struct io_log *iolog, union io_sample_data data, enum fio_ddir ddir, unsigned long long bs, - uint64_t offset) + uint64_t offset, uint8_t priority_bit) { unsigned long elapsed, this_window; @@ -2676,7 +2725,7 @@ static unsigned long add_log_sample(struct thread_data *td, * If no time averaging, just add the log sample. */ if (!iolog->avg_msec) { - __add_log_sample(iolog, data, ddir, bs, elapsed, offset); + __add_log_sample(iolog, data, ddir, bs, elapsed, offset, priority_bit); return 0; } @@ -2700,7 +2749,7 @@ static unsigned long add_log_sample(struct thread_data *td, return diff; } - __add_stat_to_log(iolog, ddir, elapsed, td->o.log_max != 0); + _add_stat_to_log(iolog, elapsed, td->o.log_max != 0, priority_bit); iolog->avg_last[ddir] = elapsed - (this_window - iolog->avg_msec); return iolog->avg_msec; @@ -2713,18 +2762,19 @@ void finalize_logs(struct thread_data *td, bool unit_logs) elapsed = mtime_since_now(&td->epoch); if (td->clat_log && unit_logs) - _add_stat_to_log(td->clat_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->clat_log, elapsed, td->o.log_max != 0, 0); if (td->slat_log && unit_logs) - _add_stat_to_log(td->slat_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->slat_log, elapsed, td->o.log_max != 0, 0); if (td->lat_log && unit_logs) - _add_stat_to_log(td->lat_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->lat_log, elapsed, td->o.log_max != 0, 0); if (td->bw_log && (unit_logs == per_unit_log(td->bw_log))) - _add_stat_to_log(td->bw_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->bw_log, elapsed, td->o.log_max != 0, 0); if (td->iops_log && (unit_logs == per_unit_log(td->iops_log))) - _add_stat_to_log(td->iops_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->iops_log, elapsed, td->o.log_max != 0, 0); } -void add_agg_sample(union io_sample_data data, enum fio_ddir ddir, unsigned long long bs) +void add_agg_sample(union io_sample_data data, enum fio_ddir ddir, unsigned long long bs, + uint8_t priority_bit) { struct io_log *iolog; @@ -2732,7 +2782,7 @@ void add_agg_sample(union io_sample_data data, enum fio_ddir ddir, unsigned long return; iolog = agg_io_log[ddir]; - __add_log_sample(iolog, data, ddir, bs, mtime_since_genesis(), 0); + __add_log_sample(iolog, data, ddir, bs, mtime_since_genesis(), 0, priority_bit); } void add_sync_clat_sample(struct thread_stat *ts, unsigned long long nsec) @@ -2744,18 +2794,32 @@ void add_sync_clat_sample(struct thread_stat *ts, unsigned long long nsec) add_stat_sample(&ts->sync_stat, nsec); } -static void add_clat_percentile_sample(struct thread_stat *ts, - unsigned long long nsec, enum fio_ddir ddir) +static void add_lat_percentile_sample_noprio(struct thread_stat *ts, + unsigned long long nsec, enum fio_ddir ddir, enum fio_lat lat) { unsigned int idx = plat_val_to_idx(nsec); assert(idx < FIO_IO_U_PLAT_NR); - ts->io_u_plat[ddir][idx]++; + ts->io_u_plat[lat][ddir][idx]++; +} + +static void add_lat_percentile_sample(struct thread_stat *ts, + unsigned long long nsec, enum fio_ddir ddir, uint8_t priority_bit, + enum fio_lat lat) +{ + unsigned int idx = plat_val_to_idx(nsec); + + add_lat_percentile_sample_noprio(ts, nsec, ddir, lat); + + if (!priority_bit) + ts->io_u_plat_low_prio[ddir][idx]++; + else + ts->io_u_plat_high_prio[ddir][idx]++; } void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, unsigned long long nsec, unsigned long long bs, - uint64_t offset) + uint64_t offset, uint8_t priority_bit) { const bool needs_lock = td_async_processing(td); unsigned long elapsed, this_window; @@ -2767,12 +2831,23 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, add_stat_sample(&ts->clat_stat[ddir], nsec); + if (!ts->lat_percentiles) { + if (priority_bit) + add_stat_sample(&ts->clat_high_prio_stat[ddir], nsec); + else + add_stat_sample(&ts->clat_low_prio_stat[ddir], nsec); + } + if (td->clat_log) add_log_sample(td, td->clat_log, sample_val(nsec), ddir, bs, - offset); + offset, priority_bit); - if (ts->clat_percentiles) - add_clat_percentile_sample(ts, nsec, ddir); + if (ts->clat_percentiles) { + if (ts->lat_percentiles) + add_lat_percentile_sample_noprio(ts, nsec, ddir, FIO_CLAT); + else + add_lat_percentile_sample(ts, nsec, ddir, priority_bit, FIO_CLAT); + } if (iolog && iolog->hist_msec) { struct io_hist *hw = &iolog->hist_window[ddir]; @@ -2782,7 +2857,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, if (!hw->hist_last) hw->hist_last = elapsed; this_window = elapsed - hw->hist_last; - + if (this_window >= iolog->hist_msec) { uint64_t *io_u_plat; struct io_u_plat_entry *dst; @@ -2794,13 +2869,13 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, * located in iolog.c after printing this sample to the * log file. */ - io_u_plat = (uint64_t *) td->ts.io_u_plat[ddir]; + io_u_plat = (uint64_t *) td->ts.io_u_plat[FIO_CLAT][ddir]; dst = malloc(sizeof(struct io_u_plat_entry)); memcpy(&(dst->io_u_plat), io_u_plat, FIO_IO_U_PLAT_NR * sizeof(uint64_t)); flist_add(&dst->list, &hw->list); __add_log_sample(iolog, sample_plat(dst), ddir, bs, - elapsed, offset); + elapsed, offset, priority_bit); /* * Update the last time we recorded as being now, minus @@ -2817,7 +2892,8 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, } void add_slat_sample(struct thread_data *td, enum fio_ddir ddir, - unsigned long usec, unsigned long long bs, uint64_t offset) + unsigned long long nsec, unsigned long long bs, uint64_t offset, + uint8_t priority_bit) { const bool needs_lock = td_async_processing(td); struct thread_stat *ts = &td->ts; @@ -2828,10 +2904,14 @@ void add_slat_sample(struct thread_data *td, enum fio_ddir ddir, if (needs_lock) __td_io_u_lock(td); - add_stat_sample(&ts->slat_stat[ddir], usec); + add_stat_sample(&ts->slat_stat[ddir], nsec); if (td->slat_log) - add_log_sample(td, td->slat_log, sample_val(usec), ddir, bs, offset); + add_log_sample(td, td->slat_log, sample_val(nsec), ddir, bs, offset, + priority_bit); + + if (ts->slat_percentiles) + add_lat_percentile_sample_noprio(ts, nsec, ddir, FIO_SLAT); if (needs_lock) __td_io_u_unlock(td); @@ -2839,7 +2919,7 @@ void add_slat_sample(struct thread_data *td, enum fio_ddir ddir, void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, unsigned long long nsec, unsigned long long bs, - uint64_t offset) + uint64_t offset, uint8_t priority_bit) { const bool needs_lock = td_async_processing(td); struct thread_stat *ts = &td->ts; @@ -2854,11 +2934,16 @@ void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, if (td->lat_log) add_log_sample(td, td->lat_log, sample_val(nsec), ddir, bs, - offset); + offset, priority_bit); - if (ts->lat_percentiles) - add_clat_percentile_sample(ts, nsec, ddir); + if (ts->lat_percentiles) { + add_lat_percentile_sample(ts, nsec, ddir, priority_bit, FIO_LAT); + if (priority_bit) + add_stat_sample(&ts->clat_high_prio_stat[ddir], nsec); + else + add_stat_sample(&ts->clat_low_prio_stat[ddir], nsec); + } if (needs_lock) __td_io_u_unlock(td); } @@ -2882,7 +2967,7 @@ void add_bw_sample(struct thread_data *td, struct io_u *io_u, if (td->bw_log) add_log_sample(td, td->bw_log, sample_val(rate), io_u->ddir, - bytes, io_u->offset); + bytes, io_u->offset, io_u_is_prio(io_u)); td->stat_io_bytes[io_u->ddir] = td->this_io_bytes[io_u->ddir]; @@ -2936,14 +3021,14 @@ static int __add_samples(struct thread_data *td, struct timespec *parent_tv, if (td->o.min_bs[ddir] == td->o.max_bs[ddir]) bs = td->o.min_bs[ddir]; - next = add_log_sample(td, log, sample_val(rate), ddir, bs, 0); + next = add_log_sample(td, log, sample_val(rate), ddir, bs, 0, 0); next_log = min(next_log, next); } stat_io_bytes[ddir] = this_io_bytes[ddir]; } - timespec_add_msec(parent_tv, avg_time); + *parent_tv = *t; if (needs_lock) __td_io_u_unlock(td); @@ -2976,7 +3061,7 @@ void add_iops_sample(struct thread_data *td, struct io_u *io_u, if (td->iops_log) add_log_sample(td, td->iops_log, sample_val(1), io_u->ddir, - bytes, io_u->offset); + bytes, io_u->offset, io_u_is_prio(io_u)); td->stat_io_blocks[io_u->ddir] = td->this_io_blocks[io_u->ddir];