X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=t%2Fbtrace2fio.c;h=4cdb38d10d3ec8c53ee265787124b70a76893556;hp=5666a560c0beca15941ad596a3469a4422264d36;hb=c6b80f7e095985237136cfb41eb8521d0a51279f;hpb=761c27292d4add2be17056d8065cad4ff4e87036 diff --git a/t/btrace2fio.c b/t/btrace2fio.c index 5666a560..4cdb38d1 100644 --- a/t/btrace2fio.c +++ b/t/btrace2fio.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "../io_ddir.h" @@ -11,7 +12,7 @@ #include "../blktrace_api.h" #include "../os/os.h" #include "../log.h" -#include "../lib/linux-dev-lookup.h" +#include "../oslib/linux-dev-lookup.h" #define TRACE_FIFO_SIZE 8192 @@ -19,9 +20,20 @@ static unsigned int rt_threshold = 1000000; static unsigned int ios_threshold = 10; static unsigned int rate_threshold; static unsigned int set_rate; +static unsigned int max_depth = 256; static int output_ascii = 1; static char *filename; +static char **add_opts; +static int n_add_opts; + +/* + * Collapse defaults + */ +static unsigned int collapse_entries = 0; +static unsigned int depth_diff = 1; +static unsigned int random_diff = 5; + struct bs { unsigned int bs; unsigned int nr; @@ -45,9 +57,12 @@ struct btrace_out { int inflight; unsigned int depth; + int depth_disabled; + int complete_seen; + uint64_t first_ttime[DDIR_RWDIR_CNT]; uint64_t last_ttime[DDIR_RWDIR_CNT]; - uint64_t kb[DDIR_RWDIR_CNT]; + uint64_t kib[DDIR_RWDIR_CNT]; uint64_t start_delay; }; @@ -57,9 +72,14 @@ struct btrace_pid { struct flist_head pid_list; pid_t pid; + pid_t *merge_pids; + unsigned int nr_merge_pids; + struct trace_file *files; int nr_files; unsigned int last_major, last_minor; + int numjobs; + int ignore; struct btrace_out o; }; @@ -125,7 +145,13 @@ static void inflight_add(struct btrace_pid *p, uint64_t sector, uint32_t len) i = calloc(1, sizeof(*i)); i->p = p; o->inflight++; - o->depth = max((int) o->depth, o->inflight); + if (!o->depth_disabled) { + o->depth = max((int) o->depth, o->inflight); + if (o->depth >= max_depth && !o->complete_seen) { + o->depth_disabled = 1; + o->depth = max_depth; + } + } i->end_sector = sector + (len >> 9); __inflight_add(i); } @@ -355,8 +381,6 @@ static int handle_trace(struct blk_io_trace *t, struct btrace_pid *p) if (act == __BLK_TA_QUEUE) { inflight_add(p, t->sector, t->bytes); ret = handle_queue_trace(t, p); - } else if (act == __BLK_TA_REQUEUE) { - p->o.inflight--; } else if (act == __BLK_TA_BACKMERGE) { struct inflight *i; @@ -382,7 +406,8 @@ static int handle_trace(struct blk_io_trace *t, struct btrace_pid *p) i = inflight_find(t->sector + (t->bytes >> 9)); if (i) { - i->p->o.kb[t_to_rwdir(t)] += (t->bytes >> 10); + i->p->o.kib[t_to_rwdir(t)] += (t->bytes >> 10); + i->p->o.complete_seen = 1; inflight_remove(i); } } @@ -439,6 +464,7 @@ static struct btrace_pid *pid_hash_get(pid_t pid) } p->pid = pid; + p->numjobs = 1; flist_add_tail(&p->hash_list, hash_list); flist_add_tail(&p->pid_list, &pid_list); } @@ -450,7 +476,7 @@ static struct btrace_pid *pid_hash_get(pid_t pid) * Load a blktrace file by reading all the blk_io_trace entries, and storing * them as io_pieces like the fio text version would do. */ -static int load_blktrace(const char *filename, int need_swap) +static int load_blktrace(const char *fname, int need_swap) { struct btrace_pid *p; unsigned long traces; @@ -458,7 +484,7 @@ static int load_blktrace(const char *filename, int need_swap) struct fifo *fifo; int fd, ret = 0; - fd = open(filename, O_RDONLY); + fd = open(fname, O_RDONLY); if (fd < 0) { perror("open trace file\n"); return 1; @@ -468,8 +494,7 @@ static int load_blktrace(const char *filename, int need_swap) traces = 0; do { - int ret = trace_fifo_get(fifo, fd, &t, sizeof(t)); - + ret = trace_fifo_get(fifo, fd, &t, sizeof(t)); if (ret < 0) goto err; else if (!ret) @@ -531,7 +556,7 @@ static int bs_cmp(const void *ba, const void *bb) return bsb->nr - bsa->nr; } -static unsigned long o_to_kb_rate(struct btrace_out *o, int rw) +static unsigned long o_to_kib_rate(struct btrace_out *o, int rw) { uint64_t usec = (o->last_ttime[rw] - o->first_ttime[rw]) / 1000ULL; uint64_t val; @@ -539,8 +564,12 @@ static unsigned long o_to_kb_rate(struct btrace_out *o, int rw) if (!usec) return 0; - val = o->kb[rw] * 1000ULL; - return val / (usec / 1000ULL); + usec /= 1000; + if (!usec) + return 0; + + val = o->kib[rw] * 1000ULL; + return val / usec; } static uint64_t o_first_ttime(struct btrace_out *o) @@ -573,7 +602,11 @@ static void __output_p_ascii(struct btrace_pid *p, unsigned long *ios) unsigned long total, usec; int i, j; - printf("[pid:\t%u]\n", p->pid); + printf("[pid:\t%u", p->pid); + if (p->nr_merge_pids) + for (i = 0; i < p->nr_merge_pids; i++) + printf(", %u", p->merge_pids[i]); + printf("]\n"); total = ddir_rw_sum(o->ios); for (i = 0; i < DDIR_RWDIR_CNT; i++) { @@ -590,7 +623,7 @@ static void __output_p_ascii(struct btrace_pid *p, unsigned long *ios) printf("\tmerges: %lu (perc=%3.2f%%)\n", o->merges[i], perc); perc = ((float) o->seq[i] * 100.0) / (float) o->ios[i]; printf("\tseq: %lu (perc=%3.2f%%)\n", (unsigned long) o->seq[i], perc); - printf("\trate: %lu KB/sec\n", o_to_kb_rate(o, i)); + printf("\trate: %lu KiB/sec\n", o_to_kib_rate(o, i)); for (j = 0; j < o->nr_bs[i]; j++) { struct bs *bs = &o->bs[i][j]; @@ -629,7 +662,13 @@ static int __output_p_fio(struct btrace_pid *p, unsigned long *ios) return 1; } - printf("[pid%u]\n", p->pid); + printf("[pid%u", p->pid); + if (p->nr_merge_pids) + for (i = 0; i < p->nr_merge_pids; i++) + printf(",pid%u", p->merge_pids[i]); + printf("]\n"); + + printf("numjobs=%u\n", p->numjobs); printf("direct=1\n"); if (o->depth == 1) printf("ioengine=sync\n"); @@ -646,7 +685,7 @@ static int __output_p_fio(struct btrace_pid *p, unsigned long *ios) printf("rw=randrw\n"); total = ddir_rw_sum(o->ios); perc = ((float) o->ios[0] * 100.0) / (float) total; - printf("rwmixread=%u\n", (int) (perc + 0.99)); + printf("rwmixread=%u\n", (int) floor(perc + 0.50)); } printf("percentage_random="); @@ -661,7 +700,7 @@ static int __output_p_fio(struct btrace_pid *p, unsigned long *ios) if (i) printf(","); perc = 100.0 - perc; - printf("%u", (int) perc); + printf("%u", (int) floor(perc + 0.5)); } printf("\n"); @@ -673,7 +712,8 @@ static int __output_p_fio(struct btrace_pid *p, unsigned long *ios) } printf("\n"); - printf("startdelay=%llus\n", o->start_delay / 1000000ULL); + if (o->start_delay / 1000000ULL) + printf("startdelay=%llus\n", o->start_delay / 1000000ULL); time = o_longest_ttime(o); time = (time + 1000000000ULL - 1) / 1000000000ULL; @@ -696,7 +736,7 @@ static int __output_p_fio(struct btrace_pid *p, unsigned long *ios) if (j + 1 == o->nr_bs[i]) printf("%u/", bs->bs); else - printf("%u/%u", bs->bs, (int) perc); + printf("%u/%u", bs->bs, (int) floor(perc + 0.5)); } } printf("\n"); @@ -706,7 +746,7 @@ static int __output_p_fio(struct btrace_pid *p, unsigned long *ios) for (i = 0; i < DDIR_RWDIR_CNT; i++) { unsigned long rate; - rate = o_to_kb_rate(o, i); + rate = o_to_kib_rate(o, i); if (i) printf(","); if (rate) @@ -715,6 +755,10 @@ static int __output_p_fio(struct btrace_pid *p, unsigned long *ios) printf("\n"); } + if (n_add_opts) + for (i = 0; i < n_add_opts; i++) + printf("%s\n", add_opts[i]); + printf("\n"); return 0; } @@ -766,7 +810,7 @@ static int prune_entry(struct btrace_out *o) for (i = 0; i < DDIR_RWDIR_CNT; i++) { unsigned long this_rate; - this_rate = o_to_kb_rate(o, i); + this_rate = o_to_kib_rate(o, i); if (this_rate < rate_threshold) { remove_ddir(o, i); this_rate = 0; @@ -807,10 +851,118 @@ static void free_p(struct btrace_pid *p) free(p); } +static int entries_close(struct btrace_pid *pida, struct btrace_pid *pidb) +{ + float perca, percb, fdiff; + int i, idiff; + + for (i = 0; i < DDIR_RWDIR_CNT; i++) { + if ((pida->o.ios[i] && !pidb->o.ios[i]) || + (pidb->o.ios[i] && !pida->o.ios[i])) + return 0; + if (pida->o.ios[i] && pidb->o.ios[i]) { + perca = ((float) pida->o.seq[i] * 100.0) / (float) pida->o.ios[i]; + percb = ((float) pidb->o.seq[i] * 100.0) / (float) pidb->o.ios[i]; + fdiff = perca - percb; + if (fabs(fdiff) > random_diff) + return 0; + } + + idiff = pida->o.depth - pidb->o.depth; + if (abs(idiff) > depth_diff) + return 0; + } + + return 1; +} + +static void merge_bs(struct bs **bsap, unsigned int *nr_bsap, + struct bs *bsb, unsigned int nr_bsb) +{ + struct bs *bsa = *bsap; + unsigned int nr_bsa = *nr_bsap; + int a, b; + + for (b = 0; b < nr_bsb; b++) { + int next, found = 0; + + for (a = 0; a < nr_bsa; a++) { + if (bsb[b].bs != bsa[a].bs) + continue; + + bsa[a].nr += bsb[b].nr; + bsa[a].merges += bsb[b].merges; + found = 1; + break; + } + + if (found) + continue; + + next = *nr_bsap; + bsa = realloc(bsa, (next + 1) * sizeof(struct bs)); + bsa[next].bs = bsb[b].bs; + bsa[next].nr = bsb[b].nr; + (*nr_bsap)++; + *bsap = bsa; + } +} + +static int merge_entries(struct btrace_pid *pida, struct btrace_pid *pidb) +{ + int i; + + if (!entries_close(pida, pidb)) + return 0; + + pida->nr_merge_pids++; + pida->merge_pids = realloc(pida->merge_pids, pida->nr_merge_pids * sizeof(pid_t)); + pida->merge_pids[pida->nr_merge_pids - 1] = pidb->pid; + + for (i = 0; i < DDIR_RWDIR_CNT; i++) { + struct btrace_out *oa = &pida->o; + struct btrace_out *ob = &pidb->o; + + oa->ios[i] += ob->ios[i]; + oa->merges[i] += ob->merges[i]; + oa->seq[i] += ob->seq[i]; + oa->kib[i] += ob->kib[i]; + oa->first_ttime[i] = min(oa->first_ttime[i], ob->first_ttime[i]); + oa->last_ttime[i] = max(oa->last_ttime[i], ob->last_ttime[i]); + merge_bs(&oa->bs[i], &oa->nr_bs[i], ob->bs[i], ob->nr_bs[i]); + } + + pida->o.start_delay = min(pida->o.start_delay, pidb->o.start_delay); + pida->o.depth = (pida->o.depth + pidb->o.depth) / 2; + return 1; +} + +static void check_merges(struct btrace_pid *p, struct flist_head *pidlist) +{ + struct flist_head *e, *tmp; + + if (p->ignore) + return; + + flist_for_each_safe(e, tmp, pidlist) { + struct btrace_pid *pidb; + + pidb = flist_entry(e, struct btrace_pid, pid_list); + if (pidb == p) + continue; + + if (merge_entries(p, pidb)) { + pidb->ignore = 1; + p->numjobs++; + } + } +} + static int output_p(void) { unsigned long ios[DDIR_RWDIR_CNT]; struct flist_head *e, *tmp; + int depth_disabled = 0; int ret = 0; flist_for_each_safe(e, tmp, &pid_list) { @@ -822,8 +974,27 @@ static int output_p(void) continue; } p->o.start_delay = (o_first_ttime(&p->o) / 1000ULL) - first_ttime; + depth_disabled += p->o.depth_disabled; } + if (collapse_entries) { + struct btrace_pid *p; + + flist_for_each_safe(e, tmp, &pid_list) { + p = flist_entry(e, struct btrace_pid, pid_list); + check_merges(p, &pid_list); + } + + flist_for_each_safe(e, tmp, &pid_list) { + p = flist_entry(e, struct btrace_pid, pid_list); + if (p->ignore) + free_p(p); + } + } + + if (depth_disabled) + log_err("fio: missing completion traces, depths capped at %u\n", max_depth); + memset(ios, 0, sizeof(ios)); flist_sort(NULL, &pid_list, entry_cmp); @@ -845,13 +1016,18 @@ static int output_p(void) static int usage(char *argv[]) { - log_err("%s: \n", argv[0]); + log_err("%s: [options] \n", argv[0]); log_err("\t-t\tUsec threshold to ignore task\n"); log_err("\t-n\tNumber IOS threshold to ignore task\n"); log_err("\t-f\tFio job file output\n"); log_err("\t-d\tUse this file/device for replay\n"); - log_err("\t-r\tIgnore jobs with less than this KB/sec rate\n"); - log_err("\t-R\tSet rate in fio job\n"); + log_err("\t-r\tIgnore jobs with less than this KiB/sec rate\n"); + log_err("\t-R\tSet rate in fio job (def=%u)\n", set_rate); + log_err("\t-D\tCap queue depth at this value (def=%u)\n", max_depth); + log_err("\t-c\tCollapse \"identical\" jobs (def=%u)\n", collapse_entries); + log_err("\t-u\tDepth difference for collapse (def=%u)\n", depth_diff); + log_err("\t-x\tRandom difference for collapse (def=%u)\n", random_diff); + log_err("\t-a\tAdditional fio option to add to job file\n"); return 1; } @@ -870,9 +1046,11 @@ static int trace_needs_swap(const char *trace_file, int *swap) ret = read(fd, &t, sizeof(t)); if (ret < 0) { + close(fd); perror("read"); return 1; } else if (ret != sizeof(t)) { + close(fd); log_err("fio: short read on trace file\n"); return 1; } @@ -905,7 +1083,7 @@ int main(int argc, char *argv[]) if (argc < 2) return usage(argv); - while ((c = getopt(argc, argv, "t:n:fd:r:R")) != -1) { + while ((c = getopt(argc, argv, "t:n:fd:r:RD:c:u:x:a:")) != -1) { switch (c) { case 'R': set_rate = 1; @@ -925,6 +1103,23 @@ int main(int argc, char *argv[]) case 'd': filename = strdup(optarg); break; + case 'D': + max_depth = atoi(optarg); + break; + case 'c': + collapse_entries = atoi(optarg); + break; + case 'u': + depth_diff = atoi(optarg); + break; + case 'x': + random_diff = atoi(optarg); + break; + case 'a': + add_opts = realloc(add_opts, (n_add_opts + 1) * sizeof(char *)); + add_opts[n_add_opts] = strdup(optarg); + n_add_opts++; + break; case '?': default: return usage(argv);