X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=stat.c;h=949af5edd49ffa05dfa16bbef70c7d1037de8261;hp=d7f7362b5de19896d22cc06f7bbaf90496a6eb8a;hb=HEAD;hpb=2e5455223ed239eaa516491d33aed8e49325ec04 diff --git a/stat.c b/stat.c index d7f7362b..b98e8b27 100644 --- a/stat.c +++ b/stat.c @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -265,6 +266,18 @@ static void show_clat_percentiles(uint64_t *io_u_plat, unsigned long long nr, free(ovals); } +static int get_nr_prios_with_samples(struct thread_stat *ts, enum fio_ddir ddir) +{ + int i, nr_prios_with_samples = 0; + + for (i = 0; i < ts->nr_clat_prio[ddir]; i++) { + if (ts->clat_prio[ddir][i].clat_stat.samples) + nr_prios_with_samples++; + } + + return nr_prios_with_samples; +} + bool calc_lat(struct io_stat *is, unsigned long long *min, unsigned long long *max, double *mean, double *dev) { @@ -365,7 +378,7 @@ void show_group_stats(struct group_run_stats *rs, struct buf_output *out) free(maxalt); } - /* Need to aggregate statisitics to show mixed values */ + /* Need to aggregate statistics to show mixed values */ if (rs->unified_rw_rep == UNIFIED_BOTH) show_mixed_group_stats(rs, out); } @@ -511,7 +524,8 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, unsigned long long min, max, bw, iops; double mean, dev; char *io_p, *bw_p, *bw_p_alt, *iops_p, *post_st = NULL; - int i2p; + int i2p, i; + const char *clat_type = ts->lat_percentiles ? "lat" : "clat"; if (ddir_sync(ddir)) { if (calc_lat(&ts->sync_stat, &min, &max, &mean, &dev)) { @@ -541,7 +555,7 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, iops = (1000 * (uint64_t)ts->total_io_u[ddir]) / runt; iops_p = num2str(iops, ts->sig_figs, 1, 0, N2S_NONE); - if (ddir == DDIR_WRITE) + if (ddir == DDIR_WRITE || ddir == DDIR_TRIM) post_st = zbd_write_status(ts); else if (ddir == DDIR_READ && ts->cachehit && ts->cachemiss) { uint64_t total; @@ -572,12 +586,24 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, display_lat("clat", min, max, mean, dev, out); if (calc_lat(&ts->lat_stat[ddir], &min, &max, &mean, &dev)) display_lat(" lat", min, max, mean, dev, out); - if (calc_lat(&ts->clat_high_prio_stat[ddir], &min, &max, &mean, &dev)) { - display_lat(ts->lat_percentiles ? "high prio_lat" : "high prio_clat", - min, max, mean, dev, out); - if (calc_lat(&ts->clat_low_prio_stat[ddir], &min, &max, &mean, &dev)) - display_lat(ts->lat_percentiles ? "low prio_lat" : "low prio_clat", - min, max, mean, dev, out); + + /* Only print per prio stats if there are >= 2 prios with samples */ + if (get_nr_prios_with_samples(ts, ddir) >= 2) { + for (i = 0; i < ts->nr_clat_prio[ddir]; i++) { + char buf[64]; + + if (!calc_lat(&ts->clat_prio[ddir][i].clat_stat, &min, + &max, &mean, &dev)) + continue; + + snprintf(buf, sizeof(buf), + "%s prio %u/%u/%u", + clat_type, + ioprio_class(ts->clat_prio[ddir][i].ioprio), + ioprio(ts->clat_prio[ddir][i].ioprio), + ioprio_hint(ts->clat_prio[ddir][i].ioprio)); + display_lat(buf, min, max, mean, dev, out); + } } if (ts->slat_percentiles && ts->slat_stat[ddir].samples > 0) @@ -597,8 +623,7 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, ts->percentile_precision, "lat", out); if (ts->clat_percentiles || ts->lat_percentiles) { - const char *name = ts->lat_percentiles ? "lat" : "clat"; - char prio_name[32]; + char prio_name[64]; uint64_t samples; if (ts->lat_percentiles) @@ -606,25 +631,27 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, else samples = ts->clat_stat[ddir].samples; - /* Only print this if some high and low priority stats were collected */ - if (ts->clat_high_prio_stat[ddir].samples > 0 && - ts->clat_low_prio_stat[ddir].samples > 0) - { - sprintf(prio_name, "high prio (%.2f%%) %s", - 100. * (double) ts->clat_high_prio_stat[ddir].samples / (double) samples, - name); - show_clat_percentiles(ts->io_u_plat_high_prio[ddir], - ts->clat_high_prio_stat[ddir].samples, - ts->percentile_list, - ts->percentile_precision, prio_name, out); - - sprintf(prio_name, "low prio (%.2f%%) %s", - 100. * (double) ts->clat_low_prio_stat[ddir].samples / (double) samples, - name); - show_clat_percentiles(ts->io_u_plat_low_prio[ddir], - ts->clat_low_prio_stat[ddir].samples, - ts->percentile_list, - ts->percentile_precision, prio_name, out); + /* Only print per prio stats if there are >= 2 prios with samples */ + if (get_nr_prios_with_samples(ts, ddir) >= 2) { + for (i = 0; i < ts->nr_clat_prio[ddir]; i++) { + uint64_t prio_samples = + ts->clat_prio[ddir][i].clat_stat.samples; + + if (!prio_samples) + continue; + + snprintf(prio_name, sizeof(prio_name), + "%s prio %u/%u/%u (%.2f%% of IOs)", + clat_type, + ioprio_class(ts->clat_prio[ddir][i].ioprio), + ioprio(ts->clat_prio[ddir][i].ioprio), + ioprio_hint(ts->clat_prio[ddir][i].ioprio), + 100. * (double) prio_samples / (double) samples); + show_clat_percentiles(ts->clat_prio[ddir][i].io_u_plat, + prio_samples, ts->percentile_list, + ts->percentile_precision, + prio_name, out); + } } } @@ -679,6 +706,7 @@ static void show_mixed_ddir_status(struct group_run_stats *rs, if (ts_lcl) show_ddir_status(rs, ts_lcl, DDIR_READ, out); + free_clat_prio_stats(ts_lcl); free(ts_lcl); } @@ -931,11 +959,13 @@ static void show_agg_stats(struct disk_util_agg *agg, int terse, return; if (!terse) { - log_buf(out, ", aggrios=%llu/%llu, aggrmerge=%llu/%llu, " - "aggrticks=%llu/%llu, aggrin_queue=%llu, " - "aggrutil=%3.2f%%", + log_buf(out, ", aggrios=%llu/%llu, aggsectors=%llu/%llu, " + "aggrmerge=%llu/%llu, aggrticks=%llu/%llu, " + "aggrin_queue=%llu, aggrutil=%3.2f%%", (unsigned long long) agg->ios[0] / agg->slavecount, (unsigned long long) agg->ios[1] / agg->slavecount, + (unsigned long long) agg->sectors[0] / agg->slavecount, + (unsigned long long) agg->sectors[1] / agg->slavecount, (unsigned long long) agg->merges[0] / agg->slavecount, (unsigned long long) agg->merges[1] / agg->slavecount, (unsigned long long) agg->ticks[0] / agg->slavecount, @@ -1004,11 +1034,14 @@ void print_disk_util(struct disk_util_stat *dus, struct disk_util_agg *agg, if (agg->slavecount) log_buf(out, " "); - log_buf(out, " %s: ios=%llu/%llu, merge=%llu/%llu, " - "ticks=%llu/%llu, in_queue=%llu, util=%3.2f%%", + log_buf(out, " %s: ios=%llu/%llu, sectors=%llu/%llu, " + "merge=%llu/%llu, ticks=%llu/%llu, in_queue=%llu, " + "util=%3.2f%%", dus->name, (unsigned long long) dus->s.ios[0], (unsigned long long) dus->s.ios[1], + (unsigned long long) dus->s.sectors[0], + (unsigned long long) dus->s.sectors[1], (unsigned long long) dus->s.merges[0], (unsigned long long) dus->s.merges[1], (unsigned long long) dus->s.ticks[0], @@ -1055,6 +1088,8 @@ void json_array_add_disk_util(struct disk_util_stat *dus, json_object_add_value_string(obj, "name", (const char *)dus->name); json_object_add_value_int(obj, "read_ios", dus->s.ios[0]); json_object_add_value_int(obj, "write_ios", dus->s.ios[1]); + json_object_add_value_int(obj, "read_sectors", dus->s.sectors[0]); + json_object_add_value_int(obj, "write_sectors", dus->s.sectors[1]); json_object_add_value_int(obj, "read_merges", dus->s.merges[0]); json_object_add_value_int(obj, "write_merges", dus->s.merges[1]); json_object_add_value_int(obj, "read_ticks", dus->s.ticks[0]); @@ -1072,6 +1107,10 @@ void json_array_add_disk_util(struct disk_util_stat *dus, agg->ios[0] / agg->slavecount); json_object_add_value_int(obj, "aggr_write_ios", agg->ios[1] / agg->slavecount); + json_object_add_value_int(obj, "aggr_read_sectors", + agg->sectors[0] / agg->slavecount); + json_object_add_value_int(obj, "aggr_write_sectors", + agg->sectors[1] / agg->slavecount); json_object_add_value_int(obj, "aggr_read_merges", agg->merges[0] / agg->slavecount); json_object_add_value_int(obj, "aggr_write_merge", @@ -1353,6 +1392,7 @@ static void show_mixed_ddir_status_terse(struct thread_stat *ts, if (ts_lcl) show_ddir_status_terse(ts_lcl, rs, DDIR_READ, ver, out); + free_clat_prio_stats(ts_lcl); free(ts_lcl); } @@ -1470,25 +1510,41 @@ static void add_ddir_status_json(struct thread_stat *ts, if (!ddir_rw(ddir)) return; - /* Only print PRIO latencies if some high priority samples were gathered */ - if (ts->clat_high_prio_stat[ddir].samples > 0) { - const char *high, *low; + /* Only include per prio stats if there are >= 2 prios with samples */ + if (get_nr_prios_with_samples(ts, ddir) >= 2) { + struct json_array *array = json_create_array(); + const char *obj_name; + int i; - if (ts->lat_percentiles) { - high = "lat_high_prio"; - low = "lat_low_prio"; - } else { - high = "clat_high_prio"; - low = "clat_low_prio"; - } + if (ts->lat_percentiles) + obj_name = "lat_ns"; + else + obj_name = "clat_ns"; + + json_object_add_value_array(dir_object, "prios", array); + + for (i = 0; i < ts->nr_clat_prio[ddir]; i++) { + struct json_object *obj; - tmp_object = add_ddir_lat_json(ts, ts->clat_percentiles | ts->lat_percentiles, - &ts->clat_high_prio_stat[ddir], ts->io_u_plat_high_prio[ddir]); - json_object_add_value_object(dir_object, high, tmp_object); + if (!ts->clat_prio[ddir][i].clat_stat.samples) + continue; - tmp_object = add_ddir_lat_json(ts, ts->clat_percentiles | ts->lat_percentiles, - &ts->clat_low_prio_stat[ddir], ts->io_u_plat_low_prio[ddir]); - json_object_add_value_object(dir_object, low, tmp_object); + obj = json_create_object(); + + json_object_add_value_int(obj, "prioclass", + ioprio_class(ts->clat_prio[ddir][i].ioprio)); + json_object_add_value_int(obj, "prio", + ioprio(ts->clat_prio[ddir][i].ioprio)); + json_object_add_value_int(obj, "priohint", + ioprio_hint(ts->clat_prio[ddir][i].ioprio)); + + tmp_object = add_ddir_lat_json(ts, + ts->clat_percentiles | ts->lat_percentiles, + &ts->clat_prio[ddir][i].clat_stat, + ts->clat_prio[ddir][i].io_u_plat); + json_object_add_value_object(obj, obj_name, tmp_object); + json_array_add_value_object(array, obj); + } } if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) { @@ -1537,6 +1593,7 @@ static void add_mixed_ddir_status_json(struct thread_stat *ts, if (ts_lcl) add_ddir_status_json(ts_lcl, rs, DDIR_READ, parent); + free_clat_prio_stats(ts_lcl); free(ts_lcl); } @@ -1655,6 +1712,7 @@ static struct json_object *show_thread_status_json(struct thread_stat *ts, root = json_create_object(); json_object_add_value_string(root, "jobname", ts->name); json_object_add_value_int(root, "groupid", ts->groupid); + json_object_add_value_int(root, "job_start", ts->job_start); json_object_add_value_int(root, "error", ts->error); /* ETA Info */ @@ -1662,6 +1720,7 @@ static struct json_object *show_thread_status_json(struct thread_stat *ts, if (je) { json_object_add_value_int(root, "eta", je->eta_sec); json_object_add_value_int(root, "elapsed", je->elapsed_sec); + free(je); } if (opt_list) @@ -1831,6 +1890,7 @@ static struct json_object *show_thread_status_json(struct thread_stat *ts, struct json_array *iops, *bw; int j, k, l; char ss_buf[64]; + int intervals = ts->ss_dur / (ss_check_interval / 1000L); snprintf(ss_buf, sizeof(ss_buf), "%s%s:%f%s", ts->ss_state & FIO_SS_IOPS ? "iops" : "bw", @@ -1864,9 +1924,9 @@ static struct json_object *show_thread_status_json(struct thread_stat *ts, if ((ts->ss_state & FIO_SS_ATTAINED) || !(ts->ss_state & FIO_SS_BUFFER_FULL)) j = ts->ss_head; else - j = ts->ss_head == 0 ? ts->ss_dur - 1 : ts->ss_head - 1; - for (l = 0; l < ts->ss_dur; l++) { - k = (j + l) % ts->ss_dur; + j = ts->ss_head == 0 ? intervals - 1 : ts->ss_head - 1; + for (l = 0; l < intervals; l++) { + k = (j + l) % intervals; json_array_add_value_int(bw, ts->ss_bw_data[k]); json_array_add_value_int(iops, ts->ss_iops_data[k]); } @@ -2005,6 +2065,9 @@ void free_clat_prio_stats(struct thread_stat *ts) { enum fio_ddir ddir; + if (!ts) + return; + for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) { sfree(ts->clat_prio[ddir]); ts->clat_prio[ddir] = NULL; @@ -2038,6 +2101,175 @@ int alloc_clat_prio_stat_ddir(struct thread_stat *ts, enum fio_ddir ddir, return 0; } +static int grow_clat_prio_stat(struct thread_stat *dst, enum fio_ddir ddir) +{ + int curr_len = dst->nr_clat_prio[ddir]; + void *new_arr; + + new_arr = scalloc(curr_len + 1, sizeof(*dst->clat_prio[ddir])); + if (!new_arr) { + log_err("fio: failed to grow clat prio array\n"); + return 1; + } + + memcpy(new_arr, dst->clat_prio[ddir], + curr_len * sizeof(*dst->clat_prio[ddir])); + sfree(dst->clat_prio[ddir]); + + dst->clat_prio[ddir] = new_arr; + dst->clat_prio[ddir][curr_len].clat_stat.min_val = ULONG_MAX; + dst->nr_clat_prio[ddir]++; + + return 0; +} + +static int find_clat_prio_index(struct thread_stat *dst, enum fio_ddir ddir, + uint32_t ioprio) +{ + int i, nr_prios = dst->nr_clat_prio[ddir]; + + for (i = 0; i < nr_prios; i++) { + if (dst->clat_prio[ddir][i].ioprio == ioprio) + return i; + } + + return -1; +} + +static int alloc_or_get_clat_prio_index(struct thread_stat *dst, + enum fio_ddir ddir, uint32_t ioprio, + int *idx) +{ + int index = find_clat_prio_index(dst, ddir, ioprio); + + if (index == -1) { + index = dst->nr_clat_prio[ddir]; + + if (grow_clat_prio_stat(dst, ddir)) + return 1; + + dst->clat_prio[ddir][index].ioprio = ioprio; + } + + *idx = index; + + return 0; +} + +static int clat_prio_stats_copy(struct thread_stat *dst, struct thread_stat *src, + enum fio_ddir dst_ddir, enum fio_ddir src_ddir) +{ + size_t sz = sizeof(*src->clat_prio[src_ddir]) * + src->nr_clat_prio[src_ddir]; + + dst->clat_prio[dst_ddir] = smalloc(sz); + if (!dst->clat_prio[dst_ddir]) { + log_err("fio: failed to alloc clat prio array\n"); + return 1; + } + + memcpy(dst->clat_prio[dst_ddir], src->clat_prio[src_ddir], sz); + dst->nr_clat_prio[dst_ddir] = src->nr_clat_prio[src_ddir]; + + return 0; +} + +static int clat_prio_stat_add_samples(struct thread_stat *dst, + enum fio_ddir dst_ddir, uint32_t ioprio, + struct io_stat *io_stat, + uint64_t *io_u_plat) +{ + int i, dst_index; + + if (!io_stat->samples) + return 0; + + if (alloc_or_get_clat_prio_index(dst, dst_ddir, ioprio, &dst_index)) + return 1; + + sum_stat(&dst->clat_prio[dst_ddir][dst_index].clat_stat, io_stat, + false); + + for (i = 0; i < FIO_IO_U_PLAT_NR; i++) + dst->clat_prio[dst_ddir][dst_index].io_u_plat[i] += io_u_plat[i]; + + return 0; +} + +static int sum_clat_prio_stats_src_single_prio(struct thread_stat *dst, + struct thread_stat *src, + enum fio_ddir dst_ddir, + enum fio_ddir src_ddir) +{ + struct io_stat *io_stat; + uint64_t *io_u_plat; + + /* + * If src ts has no clat_prio_stat array, then all I/Os were submitted + * using src->ioprio. Thus, the global samples in src->clat_stat (or + * src->lat_stat) can be used as the 'per prio' samples for src->ioprio. + */ + assert(!src->clat_prio[src_ddir]); + assert(src->nr_clat_prio[src_ddir] == 0); + + if (src->lat_percentiles) { + io_u_plat = src->io_u_plat[FIO_LAT][src_ddir]; + io_stat = &src->lat_stat[src_ddir]; + } else { + io_u_plat = src->io_u_plat[FIO_CLAT][src_ddir]; + io_stat = &src->clat_stat[src_ddir]; + } + + return clat_prio_stat_add_samples(dst, dst_ddir, src->ioprio, io_stat, + io_u_plat); +} + +static int sum_clat_prio_stats_src_multi_prio(struct thread_stat *dst, + struct thread_stat *src, + enum fio_ddir dst_ddir, + enum fio_ddir src_ddir) +{ + int i; + + /* + * If src ts has a clat_prio_stat array, then there are multiple prios + * in use (i.e. src ts had cmdprio_percentage or cmdprio_bssplit set). + * The samples for the default prio will exist in the src->clat_prio + * array, just like the samples for any other prio. + */ + assert(src->clat_prio[src_ddir]); + assert(src->nr_clat_prio[src_ddir]); + + /* If the dst ts doesn't yet have a clat_prio array, simply memcpy. */ + if (!dst->clat_prio[dst_ddir]) + return clat_prio_stats_copy(dst, src, dst_ddir, src_ddir); + + /* The dst ts already has a clat_prio_array, add src stats into it. */ + for (i = 0; i < src->nr_clat_prio[src_ddir]; i++) { + struct io_stat *io_stat = &src->clat_prio[src_ddir][i].clat_stat; + uint64_t *io_u_plat = src->clat_prio[src_ddir][i].io_u_plat; + uint32_t ioprio = src->clat_prio[src_ddir][i].ioprio; + + if (clat_prio_stat_add_samples(dst, dst_ddir, ioprio, io_stat, io_u_plat)) + return 1; + } + + return 0; +} + +static int sum_clat_prio_stats(struct thread_stat *dst, struct thread_stat *src, + enum fio_ddir dst_ddir, enum fio_ddir src_ddir) +{ + if (dst->disable_prio_stat) + return 0; + + if (!src->clat_prio[src_ddir]) + return sum_clat_prio_stats_src_single_prio(dst, src, dst_ddir, + src_ddir); + + return sum_clat_prio_stats_src_multi_prio(dst, src, dst_ddir, src_ddir); +} + void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src) { int k, l, m; @@ -2045,12 +2277,11 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src) for (l = 0; l < DDIR_RWDIR_CNT; l++) { if (dst->unified_rw_rep != UNIFIED_MIXED) { sum_stat(&dst->clat_stat[l], &src->clat_stat[l], false); - sum_stat(&dst->clat_high_prio_stat[l], &src->clat_high_prio_stat[l], false); - sum_stat(&dst->clat_low_prio_stat[l], &src->clat_low_prio_stat[l], false); sum_stat(&dst->slat_stat[l], &src->slat_stat[l], false); sum_stat(&dst->lat_stat[l], &src->lat_stat[l], false); sum_stat(&dst->bw_stat[l], &src->bw_stat[l], true); sum_stat(&dst->iops_stat[l], &src->iops_stat[l], true); + sum_clat_prio_stats(dst, src, l, l); dst->io_bytes[l] += src->io_bytes[l]; @@ -2058,12 +2289,11 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src) dst->runtime[l] = src->runtime[l]; } else { sum_stat(&dst->clat_stat[0], &src->clat_stat[l], false); - sum_stat(&dst->clat_high_prio_stat[0], &src->clat_high_prio_stat[l], false); - sum_stat(&dst->clat_low_prio_stat[0], &src->clat_low_prio_stat[l], false); sum_stat(&dst->slat_stat[0], &src->slat_stat[l], false); sum_stat(&dst->lat_stat[0], &src->lat_stat[l], false); sum_stat(&dst->bw_stat[0], &src->bw_stat[l], true); sum_stat(&dst->iops_stat[0], &src->iops_stat[l], true); + sum_clat_prio_stats(dst, src, 0, l); dst->io_bytes[0] += src->io_bytes[l]; @@ -2117,19 +2347,6 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src) for (k = 0; k < FIO_IO_U_PLAT_NR; k++) dst->io_u_sync_plat[k] += src->io_u_sync_plat[k]; - for (k = 0; k < DDIR_RWDIR_CNT; k++) { - for (m = 0; m < FIO_IO_U_PLAT_NR; m++) { - if (dst->unified_rw_rep != UNIFIED_MIXED) { - dst->io_u_plat_high_prio[k][m] += src->io_u_plat_high_prio[k][m]; - dst->io_u_plat_low_prio[k][m] += src->io_u_plat_low_prio[k][m]; - } else { - dst->io_u_plat_high_prio[0][m] += src->io_u_plat_high_prio[k][m]; - dst->io_u_plat_low_prio[0][m] += src->io_u_plat_low_prio[k][m]; - } - - } - } - dst->total_run_time += src->total_run_time; dst->total_submit += src->total_submit; dst->total_complete += src->total_complete; @@ -2157,8 +2374,6 @@ void init_thread_stat_min_vals(struct thread_stat *ts) ts->lat_stat[i].min_val = ULONG_MAX; ts->bw_stat[i].min_val = ULONG_MAX; ts->iops_stat[i].min_val = ULONG_MAX; - ts->clat_high_prio_stat[i].min_val = ULONG_MAX; - ts->clat_low_prio_stat[i].min_val = ULONG_MAX; } ts->sync_stat.min_val = ULONG_MAX; } @@ -2171,10 +2386,60 @@ void init_thread_stat(struct thread_stat *ts) ts->groupid = -1; } +static void init_per_prio_stats(struct thread_stat *threadstats, int nr_ts) +{ + struct thread_stat *ts; + int i, j, last_ts, idx; + enum fio_ddir ddir; + + j = 0; + last_ts = -1; + idx = 0; + + /* + * Loop through all tds, if a td requires per prio stats, temporarily + * store a 1 in ts->disable_prio_stat, and then do an additional + * loop at the end where we invert the ts->disable_prio_stat values. + */ + for_each_td(td) { + if (!td->o.stats) + continue; + if (idx && + (!td->o.group_reporting || + (td->o.group_reporting && last_ts != td->groupid))) { + idx = 0; + j++; + } + + last_ts = td->groupid; + ts = &threadstats[j]; + + /* idx == 0 means first td in group, or td is not in a group. */ + if (idx == 0) + ts->ioprio = td->ioprio; + else if (td->ioprio != ts->ioprio) + ts->disable_prio_stat = 1; + + for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) { + if (td->ts.clat_prio[ddir]) { + ts->disable_prio_stat = 1; + break; + } + } + + idx++; + } end_for_each(); + + /* Loop through all dst threadstats and fixup the values. */ + for (i = 0; i < nr_ts; i++) { + ts = &threadstats[i]; + ts->disable_prio_stat = !ts->disable_prio_stat; + } +} + void __show_run_stats(void) { struct group_run_stats *runstats, *rs; - struct thread_data *td; struct thread_stat *threadstats, *ts; int i, j, k, nr_ts, last_ts, idx; bool kb_base_warned = false; @@ -2195,7 +2460,7 @@ void __show_run_stats(void) */ nr_ts = 0; last_ts = -1; - for_each_td(td, i) { + for_each_td(td) { if (!td->o.group_reporting) { nr_ts++; continue; @@ -2207,7 +2472,7 @@ void __show_run_stats(void) last_ts = td->groupid; nr_ts++; - } + } end_for_each(); threadstats = malloc(nr_ts * sizeof(struct thread_stat)); opt_lists = malloc(nr_ts * sizeof(struct flist_head *)); @@ -2217,10 +2482,12 @@ void __show_run_stats(void) opt_lists[i] = NULL; } + init_per_prio_stats(threadstats, nr_ts); + j = 0; last_ts = -1; idx = 0; - for_each_td(td, i) { + for_each_td(td) { if (!td->o.stats) continue; if (idx && (!td->o.group_reporting || @@ -2241,7 +2508,6 @@ void __show_run_stats(void) opt_lists[j] = &td->opt_list; idx++; - ts->members++; if (ts->groupid == -1) { /* @@ -2261,6 +2527,7 @@ void __show_run_stats(void) */ ts->thread_number = td->thread_number; ts->groupid = td->groupid; + ts->job_start = td->job_start; /* * first pid in group, not very useful... @@ -2308,6 +2575,8 @@ void __show_run_stats(void) sum_thread_stats(ts, &td->ts); + ts->members++; + if (td->o.ss_dur) { ts->ss_state = td->ss.state; ts->ss_dur = td->ss.dur; @@ -2321,7 +2590,7 @@ void __show_run_stats(void) } else ts->ss_dur = ts->ss_state = 0; - } + } end_for_each(); for (i = 0; i < nr_ts; i++) { unsigned long long bw; @@ -2462,39 +2731,46 @@ void __show_run_stats(void) log_info_flush(); free(runstats); + + /* free arrays allocated by sum_thread_stats(), if any */ + for (i = 0; i < nr_ts; i++) { + ts = &threadstats[i]; + free_clat_prio_stats(ts); + } free(threadstats); free(opt_lists); } int __show_running_run_stats(void) { - struct thread_data *td; unsigned long long *rt; struct timespec ts; - int i; fio_sem_down(stat_sem); rt = malloc(thread_number * sizeof(unsigned long long)); fio_gettime(&ts, NULL); - for_each_td(td, i) { + for_each_td(td) { + if (td->runstate >= TD_EXITED) + continue; + td->update_rusage = 1; for_each_rw_ddir(ddir) { td->ts.io_bytes[ddir] = td->io_bytes[ddir]; } td->ts.total_run_time = mtime_since(&td->epoch, &ts); - rt[i] = mtime_since(&td->start, &ts); + rt[__td_index] = mtime_since(&td->start, &ts); if (td_read(td) && td->ts.io_bytes[DDIR_READ]) - td->ts.runtime[DDIR_READ] += rt[i]; + td->ts.runtime[DDIR_READ] += rt[__td_index]; if (td_write(td) && td->ts.io_bytes[DDIR_WRITE]) - td->ts.runtime[DDIR_WRITE] += rt[i]; + td->ts.runtime[DDIR_WRITE] += rt[__td_index]; if (td_trim(td) && td->ts.io_bytes[DDIR_TRIM]) - td->ts.runtime[DDIR_TRIM] += rt[i]; - } + td->ts.runtime[DDIR_TRIM] += rt[__td_index]; + } end_for_each(); - for_each_td(td, i) { + for_each_td(td) { if (td->runstate >= TD_EXITED) continue; if (td->rusage_sem) { @@ -2502,18 +2778,21 @@ int __show_running_run_stats(void) fio_sem_down(td->rusage_sem); } td->update_rusage = 0; - } + } end_for_each(); __show_run_stats(); - for_each_td(td, i) { + for_each_td(td) { + if (td->runstate >= TD_EXITED) + continue; + if (td_read(td) && td->ts.io_bytes[DDIR_READ]) - td->ts.runtime[DDIR_READ] -= rt[i]; + td->ts.runtime[DDIR_READ] -= rt[__td_index]; if (td_write(td) && td->ts.io_bytes[DDIR_WRITE]) - td->ts.runtime[DDIR_WRITE] -= rt[i]; + td->ts.runtime[DDIR_WRITE] -= rt[__td_index]; if (td_trim(td) && td->ts.io_bytes[DDIR_TRIM]) - td->ts.runtime[DDIR_TRIM] -= rt[i]; - } + td->ts.runtime[DDIR_TRIM] -= rt[__td_index]; + } end_for_each(); free(rt); fio_sem_up(stat_sem); @@ -2588,6 +2867,14 @@ static inline void add_stat_sample(struct io_stat *is, unsigned long long data) is->samples++; } +static inline void add_stat_prio_sample(struct clat_prio_stat *clat_prio, + unsigned short clat_prio_index, + unsigned long long nsec) +{ + if (clat_prio) + add_stat_sample(&clat_prio[clat_prio_index].clat_stat, nsec); +} + /* * Return a struct io_logs, which is added to the tail of the log * list for 'iolog'. @@ -2602,7 +2889,10 @@ static struct io_logs *get_new_log(struct io_log *iolog) * forever */ if (!iolog->cur_log_max) { - new_samples = iolog->td->o.log_entries; + if (iolog->td) + new_samples = iolog->td->o.log_entries; + else + new_samples = DEF_LOG_ENTRIES; } else { new_samples = iolog->cur_log_max * 2; if (new_samples > MAX_LOG_ENTRIES) @@ -2760,7 +3050,9 @@ static void __add_log_sample(struct io_log *iolog, union io_sample_data data, s = get_sample(iolog, cur_log, cur_log->nr_samples); s->data = data; - s->time = t + (iolog->td ? iolog->td->unix_epoch : 0); + s->time = t; + if (iolog->td && iolog->td->o.log_alternate_epoch) + s->time += iolog->td->alternate_epoch; io_sample_set_ddir(iolog, s, ddir); s->bs = bs; s->priority = priority; @@ -2785,14 +3077,36 @@ static inline void reset_io_stat(struct io_stat *ios) ios->mean.u.f = ios->S.u.f = 0; } +static inline void reset_io_u_plat(uint64_t *io_u_plat) +{ + int i; + + for (i = 0; i < FIO_IO_U_PLAT_NR; i++) + io_u_plat[i] = 0; +} + +static inline void reset_clat_prio_stats(struct thread_stat *ts) +{ + enum fio_ddir ddir; + int i; + + for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) { + if (!ts->clat_prio[ddir]) + continue; + + for (i = 0; i < ts->nr_clat_prio[ddir]; i++) { + reset_io_stat(&ts->clat_prio[ddir][i].clat_stat); + reset_io_u_plat(ts->clat_prio[ddir][i].io_u_plat); + } + } +} + void reset_io_stats(struct thread_data *td) { struct thread_stat *ts = &td->ts; - int i, j, k; + int i, j; for (i = 0; i < DDIR_RWDIR_CNT; i++) { - reset_io_stat(&ts->clat_high_prio_stat[i]); - reset_io_stat(&ts->clat_low_prio_stat[i]); reset_io_stat(&ts->clat_stat[i]); reset_io_stat(&ts->slat_stat[i]); reset_io_stat(&ts->lat_stat[i]); @@ -2804,21 +3118,16 @@ void reset_io_stats(struct thread_data *td) ts->total_io_u[i] = 0; ts->short_io_u[i] = 0; ts->drop_io_u[i] = 0; - - for (j = 0; j < FIO_IO_U_PLAT_NR; j++) { - ts->io_u_plat_high_prio[i][j] = 0; - ts->io_u_plat_low_prio[i][j] = 0; - if (!i) - ts->io_u_sync_plat[j] = 0; - } } for (i = 0; i < FIO_LAT_CNT; i++) for (j = 0; j < DDIR_RWDIR_CNT; j++) - for (k = 0; k < FIO_IO_U_PLAT_NR; k++) - ts->io_u_plat[i][j][k] = 0; + reset_io_u_plat(ts->io_u_plat[i][j]); + + reset_clat_prio_stats(ts); ts->total_io_u[DDIR_SYNC] = 0; + reset_io_u_plat(ts->io_u_sync_plat); for (i = 0; i < FIO_IO_U_MAP_NR; i++) { ts->io_u_map[i] = 0; @@ -2840,7 +3149,7 @@ void reset_io_stats(struct thread_data *td) } static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir, - unsigned long elapsed, bool log_max) + unsigned long elapsed, int log_max) { /* * Note an entry in the log. Use the mean from the logged samples, @@ -2850,10 +3159,16 @@ static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir, if (iolog->avg_window[ddir].samples) { union io_sample_data data; - if (log_max) - data.val = iolog->avg_window[ddir].max_val; - else - data.val = iolog->avg_window[ddir].mean.u.f + 0.50; + if (log_max == IO_LOG_SAMPLE_AVG) { + data.val.val0 = iolog->avg_window[ddir].mean.u.f + 0.50; + data.val.val1 = 0; + } else if (log_max == IO_LOG_SAMPLE_MAX) { + data.val.val0 = iolog->avg_window[ddir].max_val; + data.val.val1 = 0; + } else { + data.val.val0 = iolog->avg_window[ddir].mean.u.f + 0.50; + data.val.val1 = iolog->avg_window[ddir].max_val; + } __add_log_sample(iolog, data, ddir, 0, elapsed, 0, 0); } @@ -2862,7 +3177,7 @@ static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir, } static void _add_stat_to_log(struct io_log *iolog, unsigned long elapsed, - bool log_max) + int log_max) { enum fio_ddir ddir; @@ -2896,7 +3211,7 @@ static unsigned long add_log_sample(struct thread_data *td, * Add the sample. If the time period has passed, then * add that entry to the log and clear. */ - add_stat_sample(&iolog->avg_window[ddir], data.val); + add_stat_sample(&iolog->avg_window[ddir], data.val.val0); /* * If period hasn't passed, adding the above sample is all we @@ -2912,7 +3227,7 @@ static unsigned long add_log_sample(struct thread_data *td, return diff; } - __add_stat_to_log(iolog, ddir, elapsed, td->o.log_max != 0); + __add_stat_to_log(iolog, ddir, elapsed, td->o.log_max); iolog->avg_last[ddir] = elapsed - (elapsed % iolog->avg_msec); @@ -2926,15 +3241,15 @@ void finalize_logs(struct thread_data *td, bool unit_logs) elapsed = mtime_since_now(&td->epoch); if (td->clat_log && unit_logs) - _add_stat_to_log(td->clat_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->clat_log, elapsed, td->o.log_max); if (td->slat_log && unit_logs) - _add_stat_to_log(td->slat_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->slat_log, elapsed, td->o.log_max); if (td->lat_log && unit_logs) - _add_stat_to_log(td->lat_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->lat_log, elapsed, td->o.log_max); if (td->bw_log && (unit_logs == per_unit_log(td->bw_log))) - _add_stat_to_log(td->bw_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->bw_log, elapsed, td->o.log_max); if (td->iops_log && (unit_logs == per_unit_log(td->iops_log))) - _add_stat_to_log(td->iops_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->iops_log, elapsed, td->o.log_max); } void add_agg_sample(union io_sample_data data, enum fio_ddir ddir, @@ -2969,22 +3284,21 @@ static inline void add_lat_percentile_sample(struct thread_stat *ts, ts->io_u_plat[lat][ddir][idx]++; } -static inline void add_lat_percentile_prio_sample(struct thread_stat *ts, - unsigned long long nsec, - enum fio_ddir ddir, - bool high_prio) +static inline void +add_lat_percentile_prio_sample(struct thread_stat *ts, unsigned long long nsec, + enum fio_ddir ddir, + unsigned short clat_prio_index) { unsigned int idx = plat_val_to_idx(nsec); - if (!high_prio) - ts->io_u_plat_low_prio[ddir][idx]++; - else - ts->io_u_plat_high_prio[ddir][idx]++; + if (ts->clat_prio[ddir]) + ts->clat_prio[ddir][clat_prio_index].io_u_plat[idx]++; } void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, unsigned long long nsec, unsigned long long bs, - uint64_t offset, unsigned int ioprio, bool high_prio) + uint64_t offset, unsigned int ioprio, + unsigned short clat_prio_index) { const bool needs_lock = td_async_processing(td); unsigned long elapsed, this_window; @@ -2997,7 +3311,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, add_stat_sample(&ts->clat_stat[ddir], nsec); /* - * When lat_percentiles=1 (default 0), the reported high/low priority + * When lat_percentiles=1 (default 0), the reported per priority * percentiles and stats are used for describing total latency values, * even though the variable names themselves start with clat_. * @@ -3005,12 +3319,9 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, * lat_percentiles=0. add_lat_sample() will add the prio stat sample * when lat_percentiles=1. */ - if (!ts->lat_percentiles) { - if (high_prio) - add_stat_sample(&ts->clat_high_prio_stat[ddir], nsec); - else - add_stat_sample(&ts->clat_low_prio_stat[ddir], nsec); - } + if (!ts->lat_percentiles) + add_stat_prio_sample(ts->clat_prio[ddir], clat_prio_index, + nsec); if (td->clat_log) add_log_sample(td, td->clat_log, sample_val(nsec), ddir, bs, @@ -3025,7 +3336,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, add_lat_percentile_sample(ts, nsec, ddir, FIO_CLAT); if (!ts->lat_percentiles) add_lat_percentile_prio_sample(ts, nsec, ddir, - high_prio); + clat_prio_index); } if (iolog && iolog->hist_msec) { @@ -3098,7 +3409,8 @@ void add_slat_sample(struct thread_data *td, enum fio_ddir ddir, void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, unsigned long long nsec, unsigned long long bs, - uint64_t offset, unsigned int ioprio, bool high_prio) + uint64_t offset, unsigned int ioprio, + unsigned short clat_prio_index) { const bool needs_lock = td_async_processing(td); struct thread_stat *ts = &td->ts; @@ -3116,7 +3428,7 @@ void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, offset, ioprio); /* - * When lat_percentiles=1 (default 0), the reported high/low priority + * When lat_percentiles=1 (default 0), the reported per priority * percentiles and stats are used for describing total latency values, * even though the variable names themselves start with clat_. * @@ -3127,12 +3439,9 @@ void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, */ if (ts->lat_percentiles) { add_lat_percentile_sample(ts, nsec, ddir, FIO_LAT); - add_lat_percentile_prio_sample(ts, nsec, ddir, high_prio); - if (high_prio) - add_stat_sample(&ts->clat_high_prio_stat[ddir], nsec); - else - add_stat_sample(&ts->clat_low_prio_stat[ddir], nsec); - + add_lat_percentile_prio_sample(ts, nsec, ddir, clat_prio_index); + add_stat_prio_sample(ts->clat_prio[ddir], clat_prio_index, + nsec); } if (needs_lock) __td_io_u_unlock(td); @@ -3267,26 +3576,38 @@ static int add_iops_samples(struct thread_data *td, struct timespec *t) td->ts.iops_stat, td->iops_log, false); } +static bool td_in_logging_state(struct thread_data *td) +{ + if (in_ramp_time(td)) + return false; + + switch(td->runstate) { + case TD_RUNNING: + case TD_VERIFYING: + case TD_FINISHING: + case TD_EXITED: + return true; + default: + return false; + } +} + /* * Returns msecs to next event */ int calc_log_samples(void) { - struct thread_data *td; unsigned int next = ~0U, tmp = 0, next_mod = 0, log_avg_msec_min = -1U; struct timespec now; - int i; long elapsed_time = 0; - fio_gettime(&now, NULL); - - for_each_td(td, i) { - elapsed_time = mtime_since_now(&td->epoch); + for_each_td(td) { + fio_gettime(&now, NULL); + elapsed_time = mtime_since(&td->epoch, &now); if (!td->o.stats) continue; - if (in_ramp_time(td) || - !(td->runstate == TD_RUNNING || td->runstate == TD_VERIFYING)) { + if (!td_in_logging_state(td)) { next = min(td->o.iops_avg_time, td->o.bw_avg_time); continue; } @@ -3307,7 +3628,7 @@ int calc_log_samples(void) if (tmp < next) next = tmp; - } + } end_for_each(); /* if log_avg_msec_min has not been changed, set it to 0 */ if (log_avg_msec_min == -1U)