+ int curr_len = dst->nr_clat_prio[ddir];
+ void *new_arr;
+
+ new_arr = scalloc(curr_len + 1, sizeof(*dst->clat_prio[ddir]));
+ if (!new_arr) {
+ log_err("fio: failed to grow clat prio array\n");
+ return 1;
+ }
+
+ memcpy(new_arr, dst->clat_prio[ddir],
+ curr_len * sizeof(*dst->clat_prio[ddir]));
+ sfree(dst->clat_prio[ddir]);
+
+ dst->clat_prio[ddir] = new_arr;
+ dst->clat_prio[ddir][curr_len].clat_stat.min_val = ULONG_MAX;
+ dst->nr_clat_prio[ddir]++;
+
+ return 0;
+}
+
+static int find_clat_prio_index(struct thread_stat *dst, enum fio_ddir ddir,
+ uint32_t ioprio)
+{
+ int i, nr_prios = dst->nr_clat_prio[ddir];
+
+ for (i = 0; i < nr_prios; i++) {
+ if (dst->clat_prio[ddir][i].ioprio == ioprio)
+ return i;
+ }
+
+ return -1;
+}
+
+static int alloc_or_get_clat_prio_index(struct thread_stat *dst,
+ enum fio_ddir ddir, uint32_t ioprio,
+ int *idx)
+{
+ int index = find_clat_prio_index(dst, ddir, ioprio);
+
+ if (index == -1) {
+ index = dst->nr_clat_prio[ddir];
+
+ if (grow_clat_prio_stat(dst, ddir))
+ return 1;
+
+ dst->clat_prio[ddir][index].ioprio = ioprio;
+ }
+
+ *idx = index;
+
+ return 0;
+}
+
+static int clat_prio_stats_copy(struct thread_stat *dst, struct thread_stat *src,
+ enum fio_ddir dst_ddir, enum fio_ddir src_ddir)
+{
+ size_t sz = sizeof(*src->clat_prio[src_ddir]) *
+ src->nr_clat_prio[src_ddir];
+
+ dst->clat_prio[dst_ddir] = smalloc(sz);
+ if (!dst->clat_prio[dst_ddir]) {
+ log_err("fio: failed to alloc clat prio array\n");
+ return 1;
+ }
+
+ memcpy(dst->clat_prio[dst_ddir], src->clat_prio[src_ddir], sz);
+ dst->nr_clat_prio[dst_ddir] = src->nr_clat_prio[src_ddir];
+
+ return 0;
+}
+
+static int clat_prio_stat_add_samples(struct thread_stat *dst,
+ enum fio_ddir dst_ddir, uint32_t ioprio,
+ struct io_stat *io_stat,
+ uint64_t *io_u_plat)
+{
+ int i, dst_index;
+
+ if (!io_stat->samples)
+ return 0;
+
+ if (alloc_or_get_clat_prio_index(dst, dst_ddir, ioprio, &dst_index))
+ return 1;
+
+ sum_stat(&dst->clat_prio[dst_ddir][dst_index].clat_stat, io_stat,
+ false);
+
+ for (i = 0; i < FIO_IO_U_PLAT_NR; i++)
+ dst->clat_prio[dst_ddir][dst_index].io_u_plat[i] += io_u_plat[i];
+
+ return 0;
+}
+
+static int sum_clat_prio_stats_src_single_prio(struct thread_stat *dst,
+ struct thread_stat *src,
+ enum fio_ddir dst_ddir,
+ enum fio_ddir src_ddir)
+{
+ struct io_stat *io_stat;
+ uint64_t *io_u_plat;
+
+ /*
+ * If src ts has no clat_prio_stat array, then all I/Os were submitted
+ * using src->ioprio. Thus, the global samples in src->clat_stat (or
+ * src->lat_stat) can be used as the 'per prio' samples for src->ioprio.
+ */
+ assert(!src->clat_prio[src_ddir]);
+ assert(src->nr_clat_prio[src_ddir] == 0);
+
+ if (src->lat_percentiles) {
+ io_u_plat = src->io_u_plat[FIO_LAT][src_ddir];
+ io_stat = &src->lat_stat[src_ddir];
+ } else {
+ io_u_plat = src->io_u_plat[FIO_CLAT][src_ddir];
+ io_stat = &src->clat_stat[src_ddir];
+ }
+
+ return clat_prio_stat_add_samples(dst, dst_ddir, src->ioprio, io_stat,
+ io_u_plat);
+}
+
+static int sum_clat_prio_stats_src_multi_prio(struct thread_stat *dst,
+ struct thread_stat *src,
+ enum fio_ddir dst_ddir,
+ enum fio_ddir src_ddir)
+{
+ int i;
+
+ /*
+ * If src ts has a clat_prio_stat array, then there are multiple prios
+ * in use (i.e. src ts had cmdprio_percentage or cmdprio_bssplit set).
+ * The samples for the default prio will exist in the src->clat_prio
+ * array, just like the samples for any other prio.
+ */
+ assert(src->clat_prio[src_ddir]);
+ assert(src->nr_clat_prio[src_ddir]);
+
+ /* If the dst ts doesn't yet have a clat_prio array, simply memcpy. */
+ if (!dst->clat_prio[dst_ddir])
+ return clat_prio_stats_copy(dst, src, dst_ddir, src_ddir);
+
+ /* The dst ts already has a clat_prio_array, add src stats into it. */
+ for (i = 0; i < src->nr_clat_prio[src_ddir]; i++) {
+ struct io_stat *io_stat = &src->clat_prio[src_ddir][i].clat_stat;
+ uint64_t *io_u_plat = src->clat_prio[src_ddir][i].io_u_plat;
+ uint32_t ioprio = src->clat_prio[src_ddir][i].ioprio;
+
+ if (clat_prio_stat_add_samples(dst, dst_ddir, ioprio, io_stat, io_u_plat))
+ return 1;
+ }
+
+ return 0;
+}
+
+static int sum_clat_prio_stats(struct thread_stat *dst, struct thread_stat *src,
+ enum fio_ddir dst_ddir, enum fio_ddir src_ddir)
+{
+ if (dst->disable_prio_stat)
+ return 0;
+
+ if (!src->clat_prio[src_ddir])
+ return sum_clat_prio_stats_src_single_prio(dst, src, dst_ddir,
+ src_ddir);
+
+ return sum_clat_prio_stats_src_multi_prio(dst, src, dst_ddir, src_ddir);
+}
+
+void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src)
+{
+ int k, l, m;