From 692dec0cfb4bcf2ddcb6438cfbe73d585c7a3bbc Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Thu, 3 Feb 2022 19:28:30 +0000 Subject: [PATCH] stat: report clat stats on a per priority granularity Convert the stat code to report clat stats on a per priority granularity, rather than simply supporting high/low priority. This is made possible by using the new clat_prio_stat array (per ddir), together with the clat_prio_stat index which is saved in each io_u. The per priority samples are only printed when there are samples for more than one priority in the clat_prio_stat array. If there are only samples for one priority, that means that all I/Os where submitted using the same priority, so no need to print. For example, running the following fio command: fio --name=test --filename=/dev/sdc --direct=1 --runtime=60 --rw=randread \ --ioengine=io_uring --ioscheduler=mq-deadline --iodepth=32 --bs=32k \ --prioclass=2 --prio=7 --cmdprio_bssplit=32k/20/3/0:32k/10/1/4 Now results in the following output: test: (groupid=0, jobs=1): err= 0: pid=465655: Tue Feb 1 02:24:47 2022 read: IOPS=146, BW=4695KiB/s (4808kB/s)(276MiB/60239msec) slat (usec): min=18, max=335, avg=62.87, stdev=22.59 clat (msec): min=2, max=2135, avg=217.97, stdev=287.26 lat (msec): min=2, max=2135, avg=218.03, stdev=287.26 clat prio 2/7 (msec): min=3, max=606, avg=106.57, stdev=86.64 clat prio 3/0 (msec): min=10, max=2135, avg=664.94, stdev=339.42 clat prio 1/4 (msec): min=2, max=300, avg=52.29, stdev=42.52 clat percentiles (msec): | 1.00th=[ 8], 5.00th=[ 14], 10.00th=[ 19], 20.00th=[ 33], | 30.00th=[ 52], 40.00th=[ 77], 50.00th=[ 108], 60.00th=[ 144], | 70.00th=[ 192], 80.00th=[ 300], 90.00th=[ 684], 95.00th=[ 911], | 99.00th=[ 1234], 99.50th=[ 1318], 99.90th=[ 1687], 99.95th=[ 1770], | 99.99th=[ 2140] clat prio 2/7 (69.25% of IOs) percentiles (msec): | 1.00th=[ 7], 5.00th=[ 13], 10.00th=[ 17], 20.00th=[ 28], | 30.00th=[ 44], 40.00th=[ 64], 50.00th=[ 85], 60.00th=[ 111], | 70.00th=[ 140], 80.00th=[ 174], 90.00th=[ 226], 95.00th=[ 279], | 99.00th=[ 368], 99.50th=[ 418], 99.90th=[ 502], 99.95th=[ 567], | 99.99th=[ 609] clat prio 3/0 (20.91% of IOs) percentiles (msec): | 1.00th=[ 44], 5.00th=[ 138], 10.00th=[ 205], 20.00th=[ 347], | 30.00th=[ 464], 40.00th=[ 558], 50.00th=[ 659], 60.00th=[ 760], | 70.00th=[ 860], 80.00th=[ 961], 90.00th=[ 1099], 95.00th=[ 1217], | 99.00th=[ 1485], 99.50th=[ 1687], 99.90th=[ 1871], 99.95th=[ 2140], | 99.99th=[ 2140] clat prio 1/4 (9.84% of IOs) percentiles (msec): | 1.00th=[ 7], 5.00th=[ 10], 10.00th=[ 13], 20.00th=[ 18], | 30.00th=[ 24], 40.00th=[ 30], 50.00th=[ 39], 60.00th=[ 51], | 70.00th=[ 63], 80.00th=[ 84], 90.00th=[ 114], 95.00th=[ 136], | 99.00th=[ 188], 99.50th=[ 197], 99.90th=[ 300], 99.95th=[ 300], | 99.99th=[ 300] bw ( KiB/s): min= 3456, max= 5888, per=100.00%, avg=4697.60, stdev=472.38, samples=120 iops : min= 108, max= 184, avg=146.80, stdev=14.76, samples=120 lat (msec) : 4=0.11%, 10=2.57%, 20=8.67%, 50=18.21%, 100=18.34% lat (msec) : 250=28.87%, 500=9.41%, 750=5.22%, 1000=5.09%, 2000=3.50% lat (msec) : >=2000=0.01% cpu : usr=0.16%, sys=0.97%, ctx=17715, majf=0, minf=262 IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.2%, 32=99.6%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0% issued rwts: total=8839,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=32 Signed-off-by: Niklas Cassel Link: https://lore.kernel.org/r/20220203192814.18552-15-Niklas.Cassel@wdc.com Signed-off-by: Jens Axboe --- client.c | 31 +++- engines/filecreate.c | 2 +- engines/filedelete.c | 2 +- engines/filestat.c | 2 +- io_u.c | 6 +- io_u.h | 2 - server.c | 40 ++++- server.h | 2 +- stat.c | 351 +++++++++++++++++++++++++++++++++---------- stat.h | 4 +- 10 files changed, 338 insertions(+), 104 deletions(-) diff --git a/client.c b/client.c index d7a80f02..605a3ce5 100644 --- a/client.c +++ b/client.c @@ -1038,14 +1038,6 @@ static void convert_ts(struct thread_stat *dst, struct thread_stat *src) dst->nr_block_infos = le64_to_cpu(src->nr_block_infos); for (i = 0; i < dst->nr_block_infos; i++) dst->block_infos[i] = le32_to_cpu(src->block_infos[i]); - for (i = 0; i < DDIR_RWDIR_CNT; i++) { - for (j = 0; j < FIO_IO_U_PLAT_NR; j++) { - dst->io_u_plat_high_prio[i][j] = le64_to_cpu(src->io_u_plat_high_prio[i][j]); - dst->io_u_plat_low_prio[i][j] = le64_to_cpu(src->io_u_plat_low_prio[i][j]); - } - convert_io_stat(&dst->clat_high_prio_stat[i], &src->clat_high_prio_stat[i]); - convert_io_stat(&dst->clat_low_prio_stat[i], &src->clat_low_prio_stat[i]); - } dst->ss_dur = le64_to_cpu(src->ss_dur); dst->ss_state = le32_to_cpu(src->ss_state); @@ -1055,6 +1047,19 @@ static void convert_ts(struct thread_stat *dst, struct thread_stat *src) dst->ss_deviation.u.f = fio_uint64_to_double(le64_to_cpu(src->ss_deviation.u.i)); dst->ss_criterion.u.f = fio_uint64_to_double(le64_to_cpu(src->ss_criterion.u.i)); + for (i = 0; i < DDIR_RWDIR_CNT; i++) { + dst->nr_clat_prio[i] = le32_to_cpu(src->nr_clat_prio[i]); + for (j = 0; j < dst->nr_clat_prio[i]; j++) { + for (k = 0; k < FIO_IO_U_PLAT_NR; k++) + dst->clat_prio[i][j].io_u_plat[k] = + le64_to_cpu(src->clat_prio[i][j].io_u_plat[k]); + convert_io_stat(&dst->clat_prio[i][j].clat_stat, + &src->clat_prio[i][j].clat_stat); + dst->clat_prio[i][j].ioprio = + le32_to_cpu(dst->clat_prio[i][j].ioprio); + } + } + if (dst->ss_state & FIO_SS_DATA) { for (i = 0; i < dst->ss_dur; i++ ) { dst->ss_iops_data[i] = le64_to_cpu(src->ss_iops_data[i]); @@ -1797,6 +1802,15 @@ int fio_handle_client(struct fio_client *client) case FIO_NET_CMD_TS: { struct cmd_ts_pdu *p = (struct cmd_ts_pdu *) cmd->payload; uint64_t offset; + int i; + + for (i = 0; i < DDIR_RWDIR_CNT; i++) { + if (le32_to_cpu(p->ts.nr_clat_prio[i])) { + offset = le64_to_cpu(p->ts.clat_prio_offset[i]); + p->ts.clat_prio[i] = + (struct clat_prio_stat *)((char *)p + offset); + } + } dprint(FD_NET, "client: ts->ss_state = %u\n", (unsigned int) le32_to_cpu(p->ts.ss_state)); if (le32_to_cpu(p->ts.ss_state) & FIO_SS_DATA) { @@ -2157,6 +2171,7 @@ int fio_handle_clients(struct client_ops *ops) fio_client_json_fini(); + free_clat_prio_stats(&client_ts); free(pfds); return retval || error_clients; } diff --git a/engines/filecreate.c b/engines/filecreate.c index 4bb13c34..7884752d 100644 --- a/engines/filecreate.c +++ b/engines/filecreate.c @@ -49,7 +49,7 @@ static int open_file(struct thread_data *td, struct fio_file *f) uint64_t nsec; nsec = ntime_since_now(&start); - add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, false); + add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, 0); } return 0; diff --git a/engines/filedelete.c b/engines/filedelete.c index e882ccf0..df388ac9 100644 --- a/engines/filedelete.c +++ b/engines/filedelete.c @@ -51,7 +51,7 @@ static int delete_file(struct thread_data *td, struct fio_file *f) uint64_t nsec; nsec = ntime_since_now(&start); - add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, false); + add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, 0); } return 0; diff --git a/engines/filestat.c b/engines/filestat.c index 00311247..e587eb54 100644 --- a/engines/filestat.c +++ b/engines/filestat.c @@ -125,7 +125,7 @@ static int stat_file(struct thread_data *td, struct fio_file *f) uint64_t nsec; nsec = ntime_since_now(&start); - add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, false); + add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0, 0); } return 0; diff --git a/io_u.c b/io_u.c index 656b4610..059637e5 100644 --- a/io_u.c +++ b/io_u.c @@ -1595,7 +1595,7 @@ again: assert(io_u->flags & IO_U_F_FREE); io_u_clear(td, io_u, IO_U_F_FREE | IO_U_F_NO_FILE_PUT | IO_U_F_TRIMMED | IO_U_F_BARRIER | - IO_U_F_VER_LIST | IO_U_F_HIGH_PRIO); + IO_U_F_VER_LIST); io_u->error = 0; io_u->acct_ddir = -1; @@ -1890,7 +1890,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, tnsec = ntime_since(&io_u->start_time, &icd->time); add_lat_sample(td, idx, tnsec, bytes, io_u->offset, - io_u->ioprio, io_u_is_high_prio(io_u)); + io_u->ioprio, io_u->clat_prio_index); if (td->flags & TD_F_PROFILE_OPS) { struct prof_io_ops *ops = &td->prof_io_ops; @@ -1912,7 +1912,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, if (ddir_rw(idx)) { if (!td->o.disable_clat) { add_clat_sample(td, idx, llnsec, bytes, io_u->offset, - io_u->ioprio, io_u_is_high_prio(io_u)); + io_u->ioprio, io_u->clat_prio_index); io_u_mark_latency(td, llnsec); } diff --git a/io_u.h b/io_u.h index d88d5f2c..206e24fe 100644 --- a/io_u.h +++ b/io_u.h @@ -21,7 +21,6 @@ enum { IO_U_F_TRIMMED = 1 << 5, IO_U_F_BARRIER = 1 << 6, IO_U_F_VER_LIST = 1 << 7, - IO_U_F_HIGH_PRIO = 1 << 8, }; /* @@ -194,6 +193,5 @@ static inline enum fio_ddir acct_ddir(struct io_u *io_u) td_flags_clear((td), &(io_u->flags), (val)) #define io_u_set(td, io_u, val) \ td_flags_set((td), &(io_u)->flags, (val)) -#define io_u_is_high_prio(io_u) (io_u->flags & IO_U_F_HIGH_PRIO) #endif diff --git a/server.c b/server.c index 6b45ff2e..914a8c74 100644 --- a/server.c +++ b/server.c @@ -1685,6 +1685,7 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs) { struct cmd_ts_pdu p; int i, j, k; + size_t clat_prio_stats_extra_size = 0; size_t ss_extra_size = 0; size_t extended_buf_size = 0; void *extended_buf; @@ -1801,16 +1802,13 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs) p.ts.cachehit = cpu_to_le64(ts->cachehit); p.ts.cachemiss = cpu_to_le64(ts->cachemiss); + convert_gs(&p.rs, rs); + for (i = 0; i < DDIR_RWDIR_CNT; i++) { - for (j = 0; j < FIO_IO_U_PLAT_NR; j++) { - p.ts.io_u_plat_high_prio[i][j] = cpu_to_le64(ts->io_u_plat_high_prio[i][j]); - p.ts.io_u_plat_low_prio[i][j] = cpu_to_le64(ts->io_u_plat_low_prio[i][j]); - } - convert_io_stat(&p.ts.clat_high_prio_stat[i], &ts->clat_high_prio_stat[i]); - convert_io_stat(&p.ts.clat_low_prio_stat[i], &ts->clat_low_prio_stat[i]); + if (ts->nr_clat_prio[i]) + clat_prio_stats_extra_size += ts->nr_clat_prio[i] * sizeof(*ts->clat_prio[i]); } - - convert_gs(&p.rs, rs); + extended_buf_size += clat_prio_stats_extra_size; dprint(FD_NET, "ts->ss_state = %d\n", ts->ss_state); if (ts->ss_state & FIO_SS_DATA) @@ -1832,6 +1830,32 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs) memcpy(extended_buf, &p, sizeof(p)); extended_buf_wp = (struct cmd_ts_pdu *)extended_buf + 1; + if (clat_prio_stats_extra_size) { + for (i = 0; i < DDIR_RWDIR_CNT; i++) { + struct clat_prio_stat *prio = (struct clat_prio_stat *) extended_buf_wp; + + for (j = 0; j < ts->nr_clat_prio[i]; j++) { + for (k = 0; k < FIO_IO_U_PLAT_NR; k++) + prio->io_u_plat[k] = + cpu_to_le64(ts->clat_prio[i][j].io_u_plat[k]); + convert_io_stat(&prio->clat_stat, + &ts->clat_prio[i][j].clat_stat); + prio->ioprio = cpu_to_le32(ts->clat_prio[i][j].ioprio); + prio++; + } + + if (ts->nr_clat_prio[i]) { + uint64_t offset = (char *)extended_buf_wp - (char *)extended_buf; + struct cmd_ts_pdu *ptr = extended_buf; + + ptr->ts.clat_prio_offset[i] = cpu_to_le64(offset); + ptr->ts.nr_clat_prio[i] = cpu_to_le32(ts->nr_clat_prio[i]); + } + + extended_buf_wp = prio; + } + } + if (ss_extra_size) { uint64_t *ss_iops, *ss_bw; uint64_t offset; diff --git a/server.h b/server.h index 35a3dc1d..0e62b6df 100644 --- a/server.h +++ b/server.h @@ -51,7 +51,7 @@ struct fio_net_cmd_reply { }; enum { - FIO_SERVER_VER = 95, + FIO_SERVER_VER = 96, FIO_SERVER_MAX_FRAGMENT_PDU = 1024, FIO_SERVER_MAX_CMD_MB = 2048, diff --git a/stat.c b/stat.c index f783aed8..a6810d9b 100644 --- a/stat.c +++ b/stat.c @@ -265,6 +265,18 @@ static void show_clat_percentiles(uint64_t *io_u_plat, unsigned long long nr, free(ovals); } +static int get_nr_prios_with_samples(struct thread_stat *ts, enum fio_ddir ddir) +{ + int i, nr_prios_with_samples = 0; + + for (i = 0; i < ts->nr_clat_prio[ddir]; i++) { + if (ts->clat_prio[ddir][i].clat_stat.samples) + nr_prios_with_samples++; + } + + return nr_prios_with_samples; +} + bool calc_lat(struct io_stat *is, unsigned long long *min, unsigned long long *max, double *mean, double *dev) { @@ -511,7 +523,8 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, unsigned long long min, max, bw, iops; double mean, dev; char *io_p, *bw_p, *bw_p_alt, *iops_p, *post_st = NULL; - int i2p; + int i2p, i; + const char *clat_type = ts->lat_percentiles ? "lat" : "clat"; if (ddir_sync(ddir)) { if (calc_lat(&ts->sync_stat, &min, &max, &mean, &dev)) { @@ -572,12 +585,22 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, display_lat("clat", min, max, mean, dev, out); if (calc_lat(&ts->lat_stat[ddir], &min, &max, &mean, &dev)) display_lat(" lat", min, max, mean, dev, out); - if (calc_lat(&ts->clat_high_prio_stat[ddir], &min, &max, &mean, &dev)) { - display_lat(ts->lat_percentiles ? "high prio_lat" : "high prio_clat", - min, max, mean, dev, out); - if (calc_lat(&ts->clat_low_prio_stat[ddir], &min, &max, &mean, &dev)) - display_lat(ts->lat_percentiles ? "low prio_lat" : "low prio_clat", - min, max, mean, dev, out); + + /* Only print per prio stats if there are >= 2 prios with samples */ + if (get_nr_prios_with_samples(ts, ddir) >= 2) { + for (i = 0; i < ts->nr_clat_prio[ddir]; i++) { + if (calc_lat(&ts->clat_prio[ddir][i].clat_stat, &min, + &max, &mean, &dev)) { + char buf[64]; + + snprintf(buf, sizeof(buf), + "%s prio %u/%u", + clat_type, + ts->clat_prio[ddir][i].ioprio >> 13, + ts->clat_prio[ddir][i].ioprio & 7); + display_lat(buf, min, max, mean, dev, out); + } + } } if (ts->slat_percentiles && ts->slat_stat[ddir].samples > 0) @@ -597,8 +620,7 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, ts->percentile_precision, "lat", out); if (ts->clat_percentiles || ts->lat_percentiles) { - const char *name = ts->lat_percentiles ? "lat" : "clat"; - char prio_name[32]; + char prio_name[64]; uint64_t samples; if (ts->lat_percentiles) @@ -606,25 +628,24 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, else samples = ts->clat_stat[ddir].samples; - /* Only print this if some high and low priority stats were collected */ - if (ts->clat_high_prio_stat[ddir].samples > 0 && - ts->clat_low_prio_stat[ddir].samples > 0) - { - sprintf(prio_name, "high prio (%.2f%%) %s", - 100. * (double) ts->clat_high_prio_stat[ddir].samples / (double) samples, - name); - show_clat_percentiles(ts->io_u_plat_high_prio[ddir], - ts->clat_high_prio_stat[ddir].samples, - ts->percentile_list, - ts->percentile_precision, prio_name, out); - - sprintf(prio_name, "low prio (%.2f%%) %s", - 100. * (double) ts->clat_low_prio_stat[ddir].samples / (double) samples, - name); - show_clat_percentiles(ts->io_u_plat_low_prio[ddir], - ts->clat_low_prio_stat[ddir].samples, - ts->percentile_list, - ts->percentile_precision, prio_name, out); + /* Only print per prio stats if there are >= 2 prios with samples */ + if (get_nr_prios_with_samples(ts, ddir) >= 2) { + for (i = 0; i < ts->nr_clat_prio[ddir]; i++) { + uint64_t prio_samples = ts->clat_prio[ddir][i].clat_stat.samples; + + if (prio_samples > 0) { + snprintf(prio_name, sizeof(prio_name), + "%s prio %u/%u (%.2f%% of IOs)", + clat_type, + ts->clat_prio[ddir][i].ioprio >> 13, + ts->clat_prio[ddir][i].ioprio & 7, + 100. * (double) prio_samples / (double) samples); + show_clat_percentiles(ts->clat_prio[ddir][i].io_u_plat, + prio_samples, ts->percentile_list, + ts->percentile_precision, + prio_name, out); + } + } } } @@ -679,6 +700,7 @@ static void show_mixed_ddir_status(struct group_run_stats *rs, if (ts_lcl) show_ddir_status(rs, ts_lcl, DDIR_READ, out); + free_clat_prio_stats(ts_lcl); free(ts_lcl); } @@ -1353,6 +1375,7 @@ static void show_mixed_ddir_status_terse(struct thread_stat *ts, if (ts_lcl) show_ddir_status_terse(ts_lcl, rs, DDIR_READ, ver, out); + free_clat_prio_stats(ts_lcl); free(ts_lcl); } @@ -1537,6 +1560,7 @@ static void add_mixed_ddir_status_json(struct thread_stat *ts, if (ts_lcl) add_ddir_status_json(ts_lcl, rs, DDIR_READ, parent); + free_clat_prio_stats(ts_lcl); free(ts_lcl); } @@ -2038,6 +2062,175 @@ int alloc_clat_prio_stat_ddir(struct thread_stat *ts, enum fio_ddir ddir, return 0; } +static int grow_clat_prio_stat(struct thread_stat *dst, enum fio_ddir ddir) +{ + int curr_len = dst->nr_clat_prio[ddir]; + void *new_arr; + + new_arr = scalloc(curr_len + 1, sizeof(*dst->clat_prio[ddir])); + if (!new_arr) { + log_err("fio: failed to grow clat prio array\n"); + return 1; + } + + memcpy(new_arr, dst->clat_prio[ddir], + curr_len * sizeof(*dst->clat_prio[ddir])); + sfree(dst->clat_prio[ddir]); + + dst->clat_prio[ddir] = new_arr; + dst->clat_prio[ddir][curr_len].clat_stat.min_val = ULONG_MAX; + dst->nr_clat_prio[ddir]++; + + return 0; +} + +static int find_clat_prio_index(struct thread_stat *dst, enum fio_ddir ddir, + uint32_t ioprio) +{ + int i, nr_prios = dst->nr_clat_prio[ddir]; + + for (i = 0; i < nr_prios; i++) { + if (dst->clat_prio[ddir][i].ioprio == ioprio) + return i; + } + + return -1; +} + +static int alloc_or_get_clat_prio_index(struct thread_stat *dst, + enum fio_ddir ddir, uint32_t ioprio, + int *idx) +{ + int index = find_clat_prio_index(dst, ddir, ioprio); + + if (index == -1) { + index = dst->nr_clat_prio[ddir]; + + if (grow_clat_prio_stat(dst, ddir)) + return 1; + + dst->clat_prio[ddir][index].ioprio = ioprio; + } + + *idx = index; + + return 0; +} + +static int clat_prio_stats_copy(struct thread_stat *dst, struct thread_stat *src, + enum fio_ddir dst_ddir, enum fio_ddir src_ddir) +{ + size_t sz = sizeof(*src->clat_prio[src_ddir]) * + src->nr_clat_prio[src_ddir]; + + dst->clat_prio[dst_ddir] = smalloc(sz); + if (!dst->clat_prio[dst_ddir]) { + log_err("fio: failed to alloc clat prio array\n"); + return 1; + } + + memcpy(dst->clat_prio[dst_ddir], src->clat_prio[src_ddir], sz); + dst->nr_clat_prio[dst_ddir] = src->nr_clat_prio[src_ddir]; + + return 0; +} + +static int clat_prio_stat_add_samples(struct thread_stat *dst, + enum fio_ddir dst_ddir, uint32_t ioprio, + struct io_stat *io_stat, + uint64_t *io_u_plat) +{ + int i, dst_index; + + if (!io_stat->samples) + return 0; + + if (alloc_or_get_clat_prio_index(dst, dst_ddir, ioprio, &dst_index)) + return 1; + + sum_stat(&dst->clat_prio[dst_ddir][dst_index].clat_stat, io_stat, + false); + + for (i = 0; i < FIO_IO_U_PLAT_NR; i++) + dst->clat_prio[dst_ddir][dst_index].io_u_plat[i] += io_u_plat[i]; + + return 0; +} + +static int sum_clat_prio_stats_src_single_prio(struct thread_stat *dst, + struct thread_stat *src, + enum fio_ddir dst_ddir, + enum fio_ddir src_ddir) +{ + struct io_stat *io_stat; + uint64_t *io_u_plat; + + /* + * If src ts has no clat_prio_stat array, then all I/Os were submitted + * using src->ioprio. Thus, the global samples in src->clat_stat (or + * src->lat_stat) can be used as the 'per prio' samples for src->ioprio. + */ + assert(!src->clat_prio[src_ddir]); + assert(src->nr_clat_prio[src_ddir] == 0); + + if (src->lat_percentiles) { + io_u_plat = src->io_u_plat[FIO_LAT][src_ddir]; + io_stat = &src->lat_stat[src_ddir]; + } else { + io_u_plat = src->io_u_plat[FIO_CLAT][src_ddir]; + io_stat = &src->clat_stat[src_ddir]; + } + + return clat_prio_stat_add_samples(dst, dst_ddir, src->ioprio, io_stat, + io_u_plat); +} + +static int sum_clat_prio_stats_src_multi_prio(struct thread_stat *dst, + struct thread_stat *src, + enum fio_ddir dst_ddir, + enum fio_ddir src_ddir) +{ + int i; + + /* + * If src ts has a clat_prio_stat array, then there are multiple prios + * in use (i.e. src ts had cmdprio_percentage or cmdprio_bssplit set). + * The samples for the default prio will exist in the src->clat_prio + * array, just like the samples for any other prio. + */ + assert(src->clat_prio[src_ddir]); + assert(src->nr_clat_prio[src_ddir]); + + /* If the dst ts doesn't yet have a clat_prio array, simply memcpy. */ + if (!dst->clat_prio[dst_ddir]) + return clat_prio_stats_copy(dst, src, dst_ddir, src_ddir); + + /* The dst ts already has a clat_prio_array, add src stats into it. */ + for (i = 0; i < src->nr_clat_prio[src_ddir]; i++) { + struct io_stat *io_stat = &src->clat_prio[src_ddir][i].clat_stat; + uint64_t *io_u_plat = src->clat_prio[src_ddir][i].io_u_plat; + uint32_t ioprio = src->clat_prio[src_ddir][i].ioprio; + + if (clat_prio_stat_add_samples(dst, dst_ddir, ioprio, io_stat, io_u_plat)) + return 1; + } + + return 0; +} + +static int sum_clat_prio_stats(struct thread_stat *dst, struct thread_stat *src, + enum fio_ddir dst_ddir, enum fio_ddir src_ddir) +{ + if (dst->disable_prio_stat) + return 0; + + if (!src->clat_prio[src_ddir]) + return sum_clat_prio_stats_src_single_prio(dst, src, dst_ddir, + src_ddir); + + return sum_clat_prio_stats_src_multi_prio(dst, src, dst_ddir, src_ddir); +} + void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src) { int k, l, m; @@ -2045,12 +2238,11 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src) for (l = 0; l < DDIR_RWDIR_CNT; l++) { if (dst->unified_rw_rep != UNIFIED_MIXED) { sum_stat(&dst->clat_stat[l], &src->clat_stat[l], false); - sum_stat(&dst->clat_high_prio_stat[l], &src->clat_high_prio_stat[l], false); - sum_stat(&dst->clat_low_prio_stat[l], &src->clat_low_prio_stat[l], false); sum_stat(&dst->slat_stat[l], &src->slat_stat[l], false); sum_stat(&dst->lat_stat[l], &src->lat_stat[l], false); sum_stat(&dst->bw_stat[l], &src->bw_stat[l], true); sum_stat(&dst->iops_stat[l], &src->iops_stat[l], true); + sum_clat_prio_stats(dst, src, l, l); dst->io_bytes[l] += src->io_bytes[l]; @@ -2058,12 +2250,11 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src) dst->runtime[l] = src->runtime[l]; } else { sum_stat(&dst->clat_stat[0], &src->clat_stat[l], false); - sum_stat(&dst->clat_high_prio_stat[0], &src->clat_high_prio_stat[l], false); - sum_stat(&dst->clat_low_prio_stat[0], &src->clat_low_prio_stat[l], false); sum_stat(&dst->slat_stat[0], &src->slat_stat[l], false); sum_stat(&dst->lat_stat[0], &src->lat_stat[l], false); sum_stat(&dst->bw_stat[0], &src->bw_stat[l], true); sum_stat(&dst->iops_stat[0], &src->iops_stat[l], true); + sum_clat_prio_stats(dst, src, 0, l); dst->io_bytes[0] += src->io_bytes[l]; @@ -2117,19 +2308,6 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src) for (k = 0; k < FIO_IO_U_PLAT_NR; k++) dst->io_u_sync_plat[k] += src->io_u_sync_plat[k]; - for (k = 0; k < DDIR_RWDIR_CNT; k++) { - for (m = 0; m < FIO_IO_U_PLAT_NR; m++) { - if (dst->unified_rw_rep != UNIFIED_MIXED) { - dst->io_u_plat_high_prio[k][m] += src->io_u_plat_high_prio[k][m]; - dst->io_u_plat_low_prio[k][m] += src->io_u_plat_low_prio[k][m]; - } else { - dst->io_u_plat_high_prio[0][m] += src->io_u_plat_high_prio[k][m]; - dst->io_u_plat_low_prio[0][m] += src->io_u_plat_low_prio[k][m]; - } - - } - } - dst->total_run_time += src->total_run_time; dst->total_submit += src->total_submit; dst->total_complete += src->total_complete; @@ -2157,8 +2335,6 @@ void init_thread_stat_min_vals(struct thread_stat *ts) ts->lat_stat[i].min_val = ULONG_MAX; ts->bw_stat[i].min_val = ULONG_MAX; ts->iops_stat[i].min_val = ULONG_MAX; - ts->clat_high_prio_stat[i].min_val = ULONG_MAX; - ts->clat_low_prio_stat[i].min_val = ULONG_MAX; } ts->sync_stat.min_val = ULONG_MAX; } @@ -2517,6 +2693,12 @@ void __show_run_stats(void) log_info_flush(); free(runstats); + + /* free arrays allocated by sum_thread_stats(), if any */ + for (i = 0; i < nr_ts; i++) { + ts = &threadstats[i]; + free_clat_prio_stats(ts); + } free(threadstats); free(opt_lists); } @@ -2643,6 +2825,14 @@ static inline void add_stat_sample(struct io_stat *is, unsigned long long data) is->samples++; } +static inline void add_stat_prio_sample(struct clat_prio_stat *clat_prio, + unsigned short clat_prio_index, + unsigned long long nsec) +{ + if (clat_prio) + add_stat_sample(&clat_prio[clat_prio_index].clat_stat, nsec); +} + /* * Return a struct io_logs, which is added to the tail of the log * list for 'iolog'. @@ -2848,14 +3038,28 @@ static inline void reset_io_u_plat(uint64_t *io_u_plat) io_u_plat[i] = 0; } +static inline void reset_clat_prio_stats(struct thread_stat *ts) +{ + enum fio_ddir ddir; + int i; + + for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) { + if (!ts->clat_prio[ddir]) + continue; + + for (i = 0; i < ts->nr_clat_prio[ddir]; i++) { + reset_io_stat(&ts->clat_prio[ddir][i].clat_stat); + reset_io_u_plat(ts->clat_prio[ddir][i].io_u_plat); + } + } +} + void reset_io_stats(struct thread_data *td) { struct thread_stat *ts = &td->ts; int i, j; for (i = 0; i < DDIR_RWDIR_CNT; i++) { - reset_io_stat(&ts->clat_high_prio_stat[i]); - reset_io_stat(&ts->clat_low_prio_stat[i]); reset_io_stat(&ts->clat_stat[i]); reset_io_stat(&ts->slat_stat[i]); reset_io_stat(&ts->lat_stat[i]); @@ -2867,15 +3071,14 @@ void reset_io_stats(struct thread_data *td) ts->total_io_u[i] = 0; ts->short_io_u[i] = 0; ts->drop_io_u[i] = 0; - - reset_io_u_plat(ts->io_u_plat_high_prio[i]); - reset_io_u_plat(ts->io_u_plat_low_prio[i]); } for (i = 0; i < FIO_LAT_CNT; i++) for (j = 0; j < DDIR_RWDIR_CNT; j++) reset_io_u_plat(ts->io_u_plat[i][j]); + reset_clat_prio_stats(ts); + ts->total_io_u[DDIR_SYNC] = 0; reset_io_u_plat(ts->io_u_sync_plat); @@ -3028,22 +3231,21 @@ static inline void add_lat_percentile_sample(struct thread_stat *ts, ts->io_u_plat[lat][ddir][idx]++; } -static inline void add_lat_percentile_prio_sample(struct thread_stat *ts, - unsigned long long nsec, - enum fio_ddir ddir, - bool high_prio) +static inline void +add_lat_percentile_prio_sample(struct thread_stat *ts, unsigned long long nsec, + enum fio_ddir ddir, + unsigned short clat_prio_index) { unsigned int idx = plat_val_to_idx(nsec); - if (!high_prio) - ts->io_u_plat_low_prio[ddir][idx]++; - else - ts->io_u_plat_high_prio[ddir][idx]++; + if (ts->clat_prio[ddir]) + ts->clat_prio[ddir][clat_prio_index].io_u_plat[idx]++; } void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, unsigned long long nsec, unsigned long long bs, - uint64_t offset, unsigned int ioprio, bool high_prio) + uint64_t offset, unsigned int ioprio, + unsigned short clat_prio_index) { const bool needs_lock = td_async_processing(td); unsigned long elapsed, this_window; @@ -3056,7 +3258,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, add_stat_sample(&ts->clat_stat[ddir], nsec); /* - * When lat_percentiles=1 (default 0), the reported high/low priority + * When lat_percentiles=1 (default 0), the reported per priority * percentiles and stats are used for describing total latency values, * even though the variable names themselves start with clat_. * @@ -3064,12 +3266,9 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, * lat_percentiles=0. add_lat_sample() will add the prio stat sample * when lat_percentiles=1. */ - if (!ts->lat_percentiles) { - if (high_prio) - add_stat_sample(&ts->clat_high_prio_stat[ddir], nsec); - else - add_stat_sample(&ts->clat_low_prio_stat[ddir], nsec); - } + if (!ts->lat_percentiles) + add_stat_prio_sample(ts->clat_prio[ddir], clat_prio_index, + nsec); if (td->clat_log) add_log_sample(td, td->clat_log, sample_val(nsec), ddir, bs, @@ -3084,7 +3283,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, add_lat_percentile_sample(ts, nsec, ddir, FIO_CLAT); if (!ts->lat_percentiles) add_lat_percentile_prio_sample(ts, nsec, ddir, - high_prio); + clat_prio_index); } if (iolog && iolog->hist_msec) { @@ -3157,7 +3356,8 @@ void add_slat_sample(struct thread_data *td, enum fio_ddir ddir, void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, unsigned long long nsec, unsigned long long bs, - uint64_t offset, unsigned int ioprio, bool high_prio) + uint64_t offset, unsigned int ioprio, + unsigned short clat_prio_index) { const bool needs_lock = td_async_processing(td); struct thread_stat *ts = &td->ts; @@ -3175,7 +3375,7 @@ void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, offset, ioprio); /* - * When lat_percentiles=1 (default 0), the reported high/low priority + * When lat_percentiles=1 (default 0), the reported per priority * percentiles and stats are used for describing total latency values, * even though the variable names themselves start with clat_. * @@ -3186,12 +3386,9 @@ void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, */ if (ts->lat_percentiles) { add_lat_percentile_sample(ts, nsec, ddir, FIO_LAT); - add_lat_percentile_prio_sample(ts, nsec, ddir, high_prio); - if (high_prio) - add_stat_sample(&ts->clat_high_prio_stat[ddir], nsec); - else - add_stat_sample(&ts->clat_low_prio_stat[ddir], nsec); - + add_lat_percentile_prio_sample(ts, nsec, ddir, clat_prio_index); + add_stat_prio_sample(ts->clat_prio[ddir], clat_prio_index, + nsec); } if (needs_lock) __td_io_u_unlock(td); diff --git a/stat.h b/stat.h index 6c86fa22..05228fd2 100644 --- a/stat.h +++ b/stat.h @@ -373,9 +373,9 @@ extern void update_rusage_stat(struct thread_data *); extern void clear_rusage_stat(struct thread_data *); extern void add_lat_sample(struct thread_data *, enum fio_ddir, unsigned long long, - unsigned long long, uint64_t, unsigned int, bool); + unsigned long long, uint64_t, unsigned int, unsigned short); extern void add_clat_sample(struct thread_data *, enum fio_ddir, unsigned long long, - unsigned long long, uint64_t, unsigned int, bool); + unsigned long long, uint64_t, unsigned int, unsigned short); extern void add_slat_sample(struct thread_data *, enum fio_ddir, unsigned long long, unsigned long long, uint64_t, unsigned int); extern void add_agg_sample(union io_sample_data, enum fio_ddir, unsigned long long); -- 2.25.1