X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=iolog.c;h=05cabd0c05abb7ff69a75dbb39b1dd4a0e01f036;hp=71afe86c29a1e50847128591be9bb10adbcea70b;hb=dab41b1c4322d968da5d69ceddac85a227675614;hpb=a47591e4923fb8faef18e1cd5125a50429282089 diff --git a/iolog.c b/iolog.c index 71afe86c..05cabd0c 100644 --- a/iolog.c +++ b/iolog.c @@ -19,6 +19,9 @@ #include "trim.h" #include "filelock.h" #include "smalloc.h" +#include "blktrace.h" + +static int iolog_flush(struct io_log *log); static const char iolog_ver2[] = "fio version 2 iolog"; @@ -62,7 +65,7 @@ static void iolog_delay(struct thread_data *td, unsigned long delay) { uint64_t usec = utime_since_now(&td->last_issue); uint64_t this_delay; - struct timeval tv; + struct timespec ts; if (delay < td->time_offset) { td->time_offset = 0; @@ -75,7 +78,7 @@ static void iolog_delay(struct thread_data *td, unsigned long delay) delay -= usec; - fio_gettime(&tv, NULL); + fio_gettime(&ts, NULL); while (delay && !td->terminate) { this_delay = delay; if (this_delay > 500000) @@ -85,7 +88,7 @@ static void iolog_delay(struct thread_data *td, unsigned long delay) delay -= this_delay; } - usec = utime_since_now(&tv); + usec = utime_since_now(&ts); if (usec > delay) td->time_offset = usec - delay; else @@ -107,6 +110,11 @@ static int ipo_special(struct thread_data *td, struct io_piece *ipo) switch (ipo->file_action) { case FIO_LOG_OPEN_FILE: + if (td->o.replay_redirect && fio_file_open(f)) { + dprint(FD_FILE, "iolog: ignoring re-open of file %s\n", + f->file_name); + break; + } ret = td_io_open_file(td, f); if (!ret) break; @@ -270,7 +278,7 @@ restart: overlap = 1; if (overlap) { - dprint(FD_IO, "iolog: overlap %llu/%lu, %llu/%lu", + dprint(FD_IO, "iolog: overlap %llu/%lu, %llu/%lu\n", __ipo->offset, __ipo->len, ipo->offset, ipo->len); td->io_hist_len--; @@ -344,7 +352,7 @@ static int read_iolog2(struct thread_data *td, FILE *f) unsigned long long offset; unsigned int bytes; int reads, writes, waits, fileno = 0, file_action = 0; /* stupid gcc */ - char *fname, *act; + char *rfname, *fname, *act; char *str, *p; enum fio_ddir rw; @@ -355,7 +363,7 @@ static int read_iolog2(struct thread_data *td, FILE *f) * for doing verifications. */ str = malloc(4096); - fname = malloc(256+16); + rfname = fname = malloc(256+16); act = malloc(256+16); reads = writes = waits = 0; @@ -363,8 +371,12 @@ static int read_iolog2(struct thread_data *td, FILE *f) struct io_piece *ipo; int r; - r = sscanf(p, "%256s %256s %llu %u", fname, act, &offset, + r = sscanf(p, "%256s %256s %llu %u", rfname, act, &offset, &bytes); + + if (td->o.replay_redirect) + fname = td->o.replay_redirect; + if (r == 4) { /* * Check action first @@ -390,8 +402,14 @@ static int read_iolog2(struct thread_data *td, FILE *f) } else if (r == 2) { rw = DDIR_INVAL; if (!strcmp(act, "add")) { - fileno = add_file(td, fname, 0, 1); - file_action = FIO_LOG_ADD_FILE; + if (td->o.replay_redirect && + get_fileno(td, fname) != -1) { + dprint(FD_FILE, "iolog: ignoring" + " re-add of file %s\n", fname); + } else { + fileno = add_file(td, fname, 0, 1); + file_action = FIO_LOG_ADD_FILE; + } continue; } else if (!strcmp(act, "open")) { fileno = get_fileno(td, fname); @@ -405,7 +423,7 @@ static int read_iolog2(struct thread_data *td, FILE *f) continue; } } else { - log_err("bad iolog2: %s", p); + log_err("bad iolog2: %s\n", p); continue; } @@ -419,6 +437,8 @@ static int read_iolog2(struct thread_data *td, FILE *f) continue; writes++; } else if (rw == DDIR_WAIT) { + if (td->o.no_stall) + continue; waits++; } else if (rw == DDIR_INVAL) { } else if (!ddir_sync(rw)) { @@ -435,7 +455,12 @@ static int read_iolog2(struct thread_data *td, FILE *f) if (rw == DDIR_WAIT) { ipo->delay = offset; } else { - ipo->offset = offset; + if (td->o.replay_scale) + ipo->offset = offset / td->o.replay_scale; + else + ipo->offset = offset; + ipo_bytes_align(td->o.replay_align, ipo); + ipo->len = bytes; if (rw != DDIR_INVAL && bytes > td->o.max_bs[rw]) td->o.max_bs[rw] = bytes; @@ -449,7 +474,7 @@ static int read_iolog2(struct thread_data *td, FILE *f) free(str); free(act); - free(fname); + free(rfname); if (writes && read_only) { log_err("fio: <%s> skips replay of %d writes due to" @@ -574,17 +599,41 @@ void setup_log(struct io_log **log, struct log_params *p, const char *filename) { struct io_log *l; + int i; + struct io_u_plat_entry *entry; + struct flist_head *list; - l = smalloc(sizeof(*l)); - l->nr_samples = 0; + l = scalloc(1, sizeof(*l)); + INIT_FLIST_HEAD(&l->io_logs); l->log_type = p->log_type; l->log_offset = p->log_offset; l->log_gz = p->log_gz; l->log_gz_store = p->log_gz_store; l->avg_msec = p->avg_msec; + l->hist_msec = p->hist_msec; + l->hist_coarseness = p->hist_coarseness; l->filename = strdup(filename); l->td = p->td; + /* Initialize histogram lists for each r/w direction, + * with initial io_u_plat of all zeros: + */ + for (i = 0; i < DDIR_RWDIR_CNT; i++) { + list = &l->hist_window[i].list; + INIT_FLIST_HEAD(list); + entry = calloc(1, sizeof(struct io_u_plat_entry)); + flist_add(&entry->list, list); + } + + if (l->td && l->td->o.io_submit_mode != IO_MODE_OFFLOAD) { + struct io_logs *p; + + p = calloc(1, sizeof(*l->pending)); + p->max_samples = DEF_LOG_ENTRIES; + p->log = calloc(p->max_samples, log_entry_sz(l)); + l->pending = p; + } + if (l->log_offset) l->log_ddir_mask = LOG_OFFSET_SAMPLE_BIT; @@ -593,7 +642,7 @@ void setup_log(struct io_log **log, struct log_params *p, if (l->log_gz && !p->td) l->log_gz = 0; else if (l->log_gz || l->log_gz_store) { - pthread_mutex_init(&l->chunk_lock, NULL); + mutex_init_pshared(&l->chunk_lock); p->td->flags |= TD_F_COMPRESS_LOG; } @@ -628,11 +677,87 @@ static void clear_file_buffer(void *buf) void free_log(struct io_log *log) { - free(log->log); + while (!flist_empty(&log->io_logs)) { + struct io_logs *cur_log; + + cur_log = flist_first_entry(&log->io_logs, struct io_logs, list); + flist_del_init(&cur_log->list); + free(cur_log->log); + sfree(cur_log); + } + + if (log->pending) { + free(log->pending->log); + free(log->pending); + log->pending = NULL; + } + + free(log->pending); free(log->filename); sfree(log); } +unsigned long hist_sum(int j, int stride, unsigned int *io_u_plat, + unsigned int *io_u_plat_last) +{ + unsigned long sum; + int k; + + if (io_u_plat_last) { + for (k = sum = 0; k < stride; k++) + sum += io_u_plat[j + k] - io_u_plat_last[j + k]; + } else { + for (k = sum = 0; k < stride; k++) + sum += io_u_plat[j + k]; + } + + return sum; +} + +static void flush_hist_samples(FILE *f, int hist_coarseness, void *samples, + uint64_t sample_size) +{ + struct io_sample *s; + int log_offset; + uint64_t i, j, nr_samples; + struct io_u_plat_entry *entry, *entry_before; + unsigned int *io_u_plat; + unsigned int *io_u_plat_before; + + int stride = 1 << hist_coarseness; + + if (!sample_size) + return; + + s = __get_sample(samples, 0, 0); + log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0; + + nr_samples = sample_size / __log_entry_sz(log_offset); + + for (i = 0; i < nr_samples; i++) { + s = __get_sample(samples, log_offset, i); + + entry = s->data.plat_entry; + io_u_plat = entry->io_u_plat; + + entry_before = flist_first_entry(&entry->list, struct io_u_plat_entry, list); + io_u_plat_before = entry_before->io_u_plat; + + fprintf(f, "%lu, %u, %u, ", (unsigned long) s->time, + io_sample_ddir(s), s->bs); + for (j = 0; j < FIO_IO_U_PLAT_NR - stride; j += stride) { + fprintf(f, "%lu, ", hist_sum(j, stride, io_u_plat, + io_u_plat_before)); + } + fprintf(f, "%lu\n", (unsigned long) + hist_sum(FIO_IO_U_PLAT_NR - stride, stride, io_u_plat, + io_u_plat_before)); + + flist_del(&entry_before->list); + free(entry_before); + } +} + void flush_samples(FILE *f, void *samples, uint64_t sample_size) { struct io_sample *s; @@ -651,16 +776,16 @@ void flush_samples(FILE *f, void *samples, uint64_t sample_size) s = __get_sample(samples, log_offset, i); if (!log_offset) { - fprintf(f, "%lu, %lu, %u, %u\n", + fprintf(f, "%lu, %" PRId64 ", %u, %u\n", (unsigned long) s->time, - (unsigned long) s->val, + s->data.val, io_sample_ddir(s), s->bs); } else { struct io_sample_offset *so = (void *) s; - fprintf(f, "%lu, %lu, %u, %u, %llu\n", + fprintf(f, "%lu, %" PRId64 ", %u, %u, %llu\n", (unsigned long) s->time, - (unsigned long) s->val, + s->data.val, io_sample_ddir(s), s->bs, (unsigned long long) so->offset); } @@ -673,7 +798,8 @@ struct iolog_flush_data { struct workqueue_work work; struct io_log *log; void *samples; - uint64_t nr_samples; + uint32_t nr_samples; + bool free; }; #define GZ_CHUNK 131072 @@ -700,6 +826,7 @@ static int z_stream_init(z_stream *stream, int gz_hdr) { int wbits = 15; + memset(stream, 0, sizeof(*stream)); stream->zalloc = Z_NULL; stream->zfree = Z_NULL; stream->opaque = Z_NULL; @@ -734,7 +861,8 @@ static void finish_chunk(z_stream *stream, FILE *f, ret = inflateEnd(stream); if (ret != Z_OK) - log_err("fio: failed to end log inflation (%d)\n", ret); + log_err("fio: failed to end log inflation seq %d (%d)\n", + iter->seq, ret); flush_samples(f, iter->buf, iter->buf_used); free(iter->buf); @@ -751,7 +879,7 @@ static size_t inflate_chunk(struct iolog_compress *ic, int gz_hdr, FILE *f, { size_t ret; - dprint(FD_COMPRESS, "inflate chunk size=%lu, seq=%u", + dprint(FD_COMPRESS, "inflate chunk size=%lu, seq=%u\n", (unsigned long) ic->len, ic->seq); if (ic->seq != iter->seq) { @@ -798,7 +926,7 @@ static size_t inflate_chunk(struct iolog_compress *ic, int gz_hdr, FILE *f, ret = (void *) stream->next_in - ic->buf; - dprint(FD_COMPRESS, "inflated to size=%lu\n", (unsigned long) ret); + dprint(FD_COMPRESS, "inflated to size=%lu\n", (unsigned long) iter->buf_size); return ret; } @@ -936,7 +1064,7 @@ int iolog_file_inflate(const char *file) #endif -void flush_log(struct io_log *log, int do_append) +void flush_log(struct io_log *log, bool do_append) { void *buf; FILE *f; @@ -954,7 +1082,20 @@ void flush_log(struct io_log *log, int do_append) inflate_gz_chunks(log, f); - flush_samples(f, log->log, log->nr_samples * log_entry_sz(log)); + while (!flist_empty(&log->io_logs)) { + struct io_logs *cur_log; + + cur_log = flist_first_entry(&log->io_logs, struct io_logs, list); + flist_del_init(&cur_log->list); + + if (log->td && log == log->td->clat_hist_log) + flush_hist_samples(f, log->hist_coarseness, cur_log->log, + log_sample_sz(log, cur_log)); + else + flush_samples(f, cur_log->log, log_sample_sz(log, cur_log)); + + sfree(cur_log); + } fclose(f); clear_file_buffer(buf); @@ -963,7 +1104,7 @@ void flush_log(struct io_log *log, int do_append) static int finish_log(struct thread_data *td, struct io_log *log, int trylock) { if (td->flags & TD_F_COMPRESS_LOG) - iolog_flush(log, 1); + iolog_flush(log); if (trylock) { if (fio_trylock_file(log->filename)) @@ -1005,7 +1146,7 @@ size_t log_chunk_sizes(struct io_log *log) static int gz_work(struct iolog_flush_data *data) { - struct iolog_compress *c; + struct iolog_compress *c = NULL; struct flist_head list; unsigned int seq; z_stream stream; @@ -1014,6 +1155,7 @@ static int gz_work(struct iolog_flush_data *data) INIT_FLIST_HEAD(&list); + memset(&stream, 0, sizeof(stream)); stream.zalloc = Z_NULL; stream.zfree = Z_NULL; stream.opaque = Z_NULL; @@ -1029,9 +1171,13 @@ static int gz_work(struct iolog_flush_data *data) stream.next_in = (void *) data->samples; stream.avail_in = data->nr_samples * log_entry_sz(data->log); - dprint(FD_COMPRESS, "deflate input size=%lu, seq=%u\n", - (unsigned long) stream.avail_in, seq); + dprint(FD_COMPRESS, "deflate input size=%lu, seq=%u, log=%s\n", + (unsigned long) stream.avail_in, seq, + data->log->filename); do { + if (c) + dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq, + (unsigned long) c->len); c = get_new_chunk(seq); stream.avail_out = GZ_CHUNK; stream.next_out = c->buf; @@ -1051,9 +1197,26 @@ static int gz_work(struct iolog_flush_data *data) stream.avail_out = GZ_CHUNK - c->len; ret = deflate(&stream, Z_FINISH); - if (ret == Z_STREAM_END) - c->len = GZ_CHUNK - stream.avail_out; - else { + if (ret < 0) { + /* + * Z_BUF_ERROR is special, it just means we need more + * output space. We'll handle that below. Treat any other + * error as fatal. + */ + if (ret != Z_BUF_ERROR) { + log_err("fio: deflate log (%d)\n", ret); + flist_del(&c->list); + free_chunk(c); + goto err; + } + } + + total -= c->len; + c->len = GZ_CHUNK - stream.avail_out; + total += c->len; + dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq, (unsigned long) c->len); + + if (ret != Z_STREAM_END) { do { c = get_new_chunk(seq); stream.avail_out = GZ_CHUNK; @@ -1062,6 +1225,8 @@ static int gz_work(struct iolog_flush_data *data) c->len = GZ_CHUNK - stream.avail_out; total += c->len; flist_add_tail(&c->list, &list); + dprint(FD_COMPRESS, "seq=%d, chunk=%lu\n", seq, + (unsigned long) c->len); } while (ret != Z_STREAM_END); } @@ -1081,7 +1246,8 @@ static int gz_work(struct iolog_flush_data *data) ret = 0; done: - free(data); + if (data->free) + sfree(data); return ret; err: while (!flist_empty(&list)) { @@ -1145,39 +1311,67 @@ void iolog_compress_exit(struct thread_data *td) * Queue work item to compress the existing log entries. We reset the * current log to a small size, and reference the existing log in the * data that we queue for compression. Once compression has been done, - * this old log is freed. If called with wait == 1, will not return until - * the log compression has completed. + * this old log is freed. If called with finish == true, will not return + * until the log compression has completed, and will flush all previous + * logs too */ -int iolog_flush(struct io_log *log, int wait) +static int iolog_flush(struct io_log *log) { struct iolog_flush_data *data; - io_u_quiesce(log->td); - data = malloc(sizeof(*data)); if (!data) return 1; data->log = log; + data->free = false; - data->samples = log->log; - data->nr_samples = log->nr_samples; + while (!flist_empty(&log->io_logs)) { + struct io_logs *cur_log; - log->nr_samples = 0; - log->max_samples = DEF_LOG_ENTRIES; - log->log = malloc(log->max_samples * log_entry_sz(log)); + cur_log = flist_first_entry(&log->io_logs, struct io_logs, list); + flist_del_init(&cur_log->list); + + data->samples = cur_log->log; + data->nr_samples = cur_log->nr_samples; + + sfree(cur_log); - if (!wait) - workqueue_enqueue(&log->td->log_compress_wq, &data->work); - else gz_work(data); + } + free(data); return 0; } +int iolog_cur_flush(struct io_log *log, struct io_logs *cur_log) +{ + struct iolog_flush_data *data; + + data = smalloc(sizeof(*data)); + if (!data) + return 1; + + data->log = log; + + data->samples = cur_log->log; + data->nr_samples = cur_log->nr_samples; + data->free = true; + + cur_log->nr_samples = cur_log->max_samples = 0; + cur_log->log = NULL; + + workqueue_enqueue(&log->td->log_compress_wq, &data->work); + return 0; +} #else -int iolog_flush(struct io_log *log, int wait) +static int iolog_flush(struct io_log *log) +{ + return 1; +} + +int iolog_cur_flush(struct io_log *log, struct io_logs *cur_log) { return 1; } @@ -1193,6 +1387,29 @@ void iolog_compress_exit(struct thread_data *td) #endif +struct io_logs *iolog_cur_log(struct io_log *log) +{ + if (flist_empty(&log->io_logs)) + return NULL; + + return flist_last_entry(&log->io_logs, struct io_logs, list); +} + +uint64_t iolog_nr_samples(struct io_log *iolog) +{ + struct flist_head *entry; + uint64_t ret = 0; + + flist_for_each(entry, &iolog->io_logs) { + struct io_logs *cur_log; + + cur_log = flist_entry(entry, struct io_logs, list); + ret += cur_log->nr_samples; + } + + return ret; +} + static int __write_log(struct thread_data *td, struct io_log *log, int try) { if (log) @@ -1243,6 +1460,20 @@ static int write_clat_log(struct thread_data *td, int try, bool unit_log) return ret; } +static int write_clat_hist_log(struct thread_data *td, int try, bool unit_log) +{ + int ret; + + if (!unit_log) + return 0; + + ret = __write_log(td, td->clat_hist_log, try); + if (!ret) + td->clat_hist_log = NULL; + + return ret; +} + static int write_lat_log(struct thread_data *td, int try, bool unit_log) { int ret; @@ -1277,8 +1508,9 @@ enum { SLAT_LOG_MASK = 4, CLAT_LOG_MASK = 8, IOPS_LOG_MASK = 16, + CLAT_HIST_LOG_MASK = 32, - ALL_LOG_NR = 5, + ALL_LOG_NR = 6, }; struct log_type { @@ -1307,6 +1539,10 @@ static struct log_type log_types[] = { .mask = IOPS_LOG_MASK, .fn = write_iops_log, }, + { + .mask = CLAT_HIST_LOG_MASK, + .fn = write_clat_hist_log, + } }; void td_writeout_logs(struct thread_data *td, bool unit_logs)