const char *filename)
{
struct io_log *l;
+ int i;
+ struct io_u_plat_entry *entry;
+ struct flist_head *list;
l = scalloc(1, sizeof(*l));
INIT_FLIST_HEAD(&l->io_logs);
l->filename = strdup(filename);
l->td = p->td;
+ /* Initialize histogram lists for each r/w direction,
+ * with initial io_u_plat of all zeros:
+ */
+ for (i = 0; i < DDIR_RWDIR_CNT; i++) {
+ list = &l->hist_window[i].list.list;
+ INIT_FLIST_HEAD(list);
+ entry = calloc(1, sizeof(struct io_u_plat_entry));
+ flist_add(&entry->list, list);
+ }
+
if (l->td && l->td->o.io_submit_mode != IO_MODE_OFFLOAD) {
struct io_logs *p;
sfree(log);
}
-static inline unsigned long hist_sum(int j, int stride, unsigned int *io_u_plat)
+static inline unsigned long hist_sum(int j, int stride, unsigned int *io_u_plat,
+ unsigned int *io_u_plat_last)
{
unsigned long sum;
int k;
for (k = sum = 0; k < stride; k++)
- sum += io_u_plat[j + k];
+ sum += io_u_plat[j + k] - io_u_plat_last[j + k];
return sum;
}
struct io_sample *s;
int log_offset;
uint64_t i, j, nr_samples;
+ struct io_u_plat_entry *entry, *entry_before;
unsigned int *io_u_plat;
+ unsigned int *io_u_plat_before;
int stride = 1 << hist_coarseness;
for (i = 0; i < nr_samples; i++) {
s = __get_sample(samples, log_offset, i);
- io_u_plat = (unsigned int *) (uintptr_t) s->val;
+
+ entry = (struct io_u_plat_entry *) s->val;
+ io_u_plat = entry->io_u_plat;
+
+ entry_before = flist_first_entry(&entry->list, struct io_u_plat_entry, list);
+ io_u_plat_before = entry_before->io_u_plat;
+
fprintf(f, "%lu, %u, %u, ", (unsigned long)s->time,
io_sample_ddir(s), s->bs);
for (j = 0; j < FIO_IO_U_PLAT_NR - stride; j += stride) {
- fprintf(f, "%lu, ", hist_sum(j, stride, io_u_plat));
+ fprintf(f, "%lu, ", hist_sum(j, stride, io_u_plat, io_u_plat_before));
}
fprintf(f, "%lu\n", (unsigned long)
- hist_sum(FIO_IO_U_PLAT_NR - stride, stride, io_u_plat));
- free(io_u_plat);
+ hist_sum(FIO_IO_U_PLAT_NR - stride, stride, io_u_plat,
+ io_u_plat_before));
+
+ flist_del(&entry_before->list);
+ free(entry_before);
}
}
to get weighted histograms.
* We convert files given on the command line, assumed to be fio histogram files,
- on-the-fly into their corresponding differenced files i.e. non-cumulative histograms
- because fio outputs cumulative histograms, but we want histograms corresponding
- to individual time intervals. An individual histogram file can contain the cumulative
+ An individual histogram file can contain the
histograms for multiple different r/w directions (notably when --rw=randrw). This
is accounted for by tracking each r/w direction separately. In the statistics
reported we ultimately merge *all* histograms (regardless of r/w direction).
__NON_HIST_COLUMNS = 3
__TOTAL_COLUMNS = __HIST_COLUMNS + __NON_HIST_COLUMNS
-def sequential_diffs(head_row, times, rws, hists):
- """ Take the difference of sequential (in time) histograms with the same
- r/w direction, returning a new array of differenced histograms. """
- result = np.empty(shape=(0, __HIST_COLUMNS))
- result_times = np.empty(shape=(1, 0))
- for i in range(8):
- idx = np.where(rws == i)
- diff = np.diff(np.append(head_row[i], hists[idx], axis=0), axis=0).astype(int)
- result = np.append(diff, result, axis=0)
- result_times = np.append(times[idx], result_times)
- idx = np.argsort(result_times)
- return result[idx]
-
-def read_chunk(head_row, rdr, sz):
- """ Read the next chunk of size sz from the given reader, computing the
- differences across neighboring histogram samples.
- """
+def read_chunk(rdr, sz):
+ """ Read the next chunk of size sz from the given reader. """
try:
""" StopIteration occurs when the pandas reader is empty, and AttributeError
occurs if rdr is None due to the file being empty. """
except (StopIteration, AttributeError):
return None
- """ Extract array of just the times, and histograms matrix without times column.
- Then, take the sequential difference of each of the rows in the histogram
- matrix. This is necessary because fio outputs *cumulative* histograms as
- opposed to histograms with counts just for a particular interval. """
+ """ Extract array of just the times, and histograms matrix without times column. """
times, rws, szs = new_arr[:,0], new_arr[:,1], new_arr[:,2]
hists = new_arr[:,__NON_HIST_COLUMNS:]
- hists_diff = sequential_diffs(head_row, times, rws, hists)
times = times.reshape((len(times),1))
- arr = np.append(times, hists_diff, axis=1)
+ arr = np.append(times, hists, axis=1)
- """ hists[-1] will be the row we need to start our differencing with the
- next time we call read_chunk() on the same rdr """
- return arr, hists[-1]
+ return arr
def get_min(fps, arrs):
""" Find the file with the current first row with the smallest start time """
- return min([fp for fp in fps if not arrs[fp] is None], key=lambda fp: arrs.get(fp)[0][0][0])
+ return min([fp for fp in fps if not arrs[fp] is None], key=lambda fp: arrs.get(fp)[0][0])
def histogram_generator(ctx, fps, sz):
- """ head_row for a particular file keeps track of the last (cumulative)
- histogram we read so that we have a reference point to subtract off
- when computing sequential differences. """
- head_row = np.zeros(shape=(1, __HIST_COLUMNS))
- head_rows = {fp: {i: head_row for i in range(8)} for fp in fps}
-
# Create a chunked pandas reader for each of the files:
rdrs = {}
for fp in fps:
else:
raise(e)
- # Initial histograms and corresponding head_rows:
- arrs = {fp: read_chunk(head_rows[fp], rdr, sz) for fp,rdr in rdrs.items()}
+ # Initial histograms from disk:
+ arrs = {fp: read_chunk(rdr, sz) for fp,rdr in rdrs.items()}
while True:
try:
fp = get_min(fps, arrs)
except ValueError:
return
- arr, head_row = arrs[fp]
+ arr = arrs[fp]
yield np.insert(arr[0], 1, fps.index(fp))
- arrs[fp] = arr[1:], head_row
- head_rows[fp] = head_row
+ arrs[fp] = arr[1:]
- if arrs[fp][0].shape[0] == 0:
- arrs[fp] = read_chunk(head_rows[fp], rdrs[fp], sz)
+ if arrs[fp].shape[0] == 0:
+ arrs[fp] = read_chunk(rdrs[fp], sz)
def _plat_idx_to_val(idx, edge=0.5, FIO_IO_U_PLAT_BITS=6, FIO_IO_U_PLAT_VAL=64):
""" Taken from fio's stat.c for calculating the latency value of a bin
projects / fio.git / commitdiff