iolog: regrow log out-of-line

From the completion side, when we run out of entries, store in a
temporary location that we size at log init time. Mark 'td' as
needing log regrow, and handle that on the submission side. We
can't easily quiesce and grow logs at completion time, since
it'll potentially recurse. On top of that, the IO engines rely
on the fact that we serialize getevents/completions, we can't
nest them.

Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/backend.c b/backend.c
index f8300405f341785fe090c208f1fbce872eb5e832..6d503606b10ebe7145c449b1d8c14390a4f99549 100644 (file)
--- a/backend.c
+++ b/backend.c
@@ -441,6 +441,12 @@ static int wait_for_completions(struct thread_data *td, struct timeval *time)
        int min_evts = 0;
        int ret;
 
+       if (td->flags & TD_F_REGROW_LOGS) {
+               ret = io_u_quiesce(td);
+               regrow_logs(td);
+               return ret;
+       }
+
        /*
         * if the queue is full, we MUST reap at least 1 event
         */

diff --git a/fio.h b/fio.h
index 8b6a27220db7e3e1ea9323a6f5dd872ef3d946af..7e6311c8899e0403f8f7d1e93f0029a4540e7aeb 100644 (file)
--- a/fio.h
+++ b/fio.h
@@ -79,6 +79,7 @@ enum {
        TD_F_NEED_LOCK          = 1U << 11,
        TD_F_CHILD              = 1U << 12,
        TD_F_NO_PROGRESS        = 1U << 13,
+       TD_F_REGROW_LOGS        = 1U << 14,
 };
 
 enum {

diff --git a/iolog.c b/iolog.c
index aec0881ed51784f69d16d32ce01132f070674f66..3723e0a86025afe460c11a7585e14a365f37022b 100644 (file)
--- a/iolog.c
+++ b/iolog.c
@@ -587,6 +587,15 @@ void setup_log(struct io_log **log, struct log_params *p,
        l->filename = strdup(filename);
        l->td = p->td;
 
+       if (l->td && l->td->o.io_submit_mode != IO_MODE_OFFLOAD) {
+               struct io_logs *p;
+
+               p = calloc(1, sizeof(*l->pending));
+               p->max_samples = l->td->o.iodepth;
+               p->log = calloc(p->max_samples, log_entry_sz(l));
+               l->pending = p;
+       }
+
        if (l->log_offset)
                l->log_ddir_mask = LOG_OFFSET_SAMPLE_BIT;
 
@@ -638,6 +647,13 @@ void free_log(struct io_log *log)
                free(cur_log->log);
        }
 
+       if (log->pending) {
+               free(log->pending->log);
+               free(log->pending);
+               log->pending = NULL;
+       }
+
+       free(log->pending);
        free(log->filename);
        sfree(log);
 }

diff --git a/iolog.h b/iolog.h
index 2b7813b91f22095d135d96be444fc12809c97cd6..f7ef6229ed08a5c92ace84e2e8e45f4633987e2e 100644 (file)
--- a/iolog.h
+++ b/iolog.h
@@ -63,6 +63,12 @@ struct io_log {
        struct flist_head io_logs;
        uint32_t cur_log_max;
 
+       /*
+        * When the current log runs out of space, store events here until
+        * we have a chance to regrow
+        */
+       struct io_logs *pending;
+
        unsigned int log_ddir_mask;
 
        char *filename;
@@ -139,6 +145,7 @@ static inline struct io_sample *__get_sample(void *samples, int log_offset,
 
 struct io_logs *iolog_cur_log(struct io_log *);
 uint64_t iolog_nr_samples(struct io_log *);
+void regrow_logs(struct thread_data *);
 
 static inline struct io_sample *get_sample(struct io_log *iolog,
                                           struct io_logs *cur_log,

diff --git a/stat.c b/stat.c
index 5eb1aab1de246f657e398e6adb4443b262bc6cfa..f55cb2f5cc8dfdd4fcc3087d6435134e7405c07b 100644 (file)
--- a/stat.c
+++ b/stat.c
@@ -1889,17 +1889,18 @@ static struct io_logs *get_new_log(struct io_log *iolog)
        return NULL;
 }
 
-static struct io_logs *get_cur_log(struct io_log *iolog)
+/*
+ * Add and return a new log chunk, or return current log if big enough
+ */
+static struct io_logs *regrow_log(struct io_log *iolog)
 {
        struct io_logs *cur_log;
+       int i;
 
-       cur_log = iolog_cur_log(iolog);
-       if (!cur_log) {
-               cur_log = get_new_log(iolog);
-               if (!cur_log)
-                       return NULL;
-       }
+       if (!iolog || iolog->disabled)
+               return NULL;
 
+       cur_log = iolog_cur_log(iolog);
        if (cur_log->nr_samples < cur_log->max_samples)
                return cur_log;
 
@@ -1918,11 +1919,70 @@ static struct io_logs *get_cur_log(struct io_log *iolog)
         * Get a new log array, and add to our list
         */
        cur_log = get_new_log(iolog);
-       if (cur_log)
+       if (!cur_log) {
+               log_err("fio: failed extending iolog! Will stop logging.\n");
+               return NULL;
+       }
+
+       if (!iolog->pending || !iolog->pending->nr_samples)
                return cur_log;
 
-       log_err("fio: failed extending iolog! Will stop logging.\n");
-       return NULL;
+       /*
+        * Flush pending items to new log
+        */
+       for (i = 0; i < iolog->pending->nr_samples; i++) {
+               struct io_sample *src, *dst;
+
+               src = get_sample(iolog, iolog->pending, i);
+               dst = get_sample(iolog, cur_log, i);
+               memcpy(dst, src, log_entry_sz(iolog));
+       }
+
+       iolog->pending->nr_samples = 0;
+       return cur_log;
+}
+
+void regrow_logs(struct thread_data *td)
+{
+       if (!regrow_log(td->slat_log))
+               td->slat_log->disabled = true;
+       if (!regrow_log(td->clat_log))
+               td->clat_log->disabled = true;
+       if (!regrow_log(td->lat_log))
+               td->lat_log->disabled = true;
+       if (!regrow_log(td->bw_log))
+               td->bw_log->disabled = true;
+       if (!regrow_log(td->iops_log))
+               td->iops_log->disabled = true;
+
+       td->flags &= ~TD_F_REGROW_LOGS;
+}
+
+static struct io_logs *get_cur_log(struct io_log *iolog)
+{
+       struct io_logs *cur_log;
+
+       cur_log = iolog_cur_log(iolog);
+       if (!cur_log) {
+               cur_log = get_new_log(iolog);
+               if (!cur_log)
+                       return NULL;
+       }
+
+       if (cur_log->nr_samples < cur_log->max_samples)
+               return cur_log;
+
+       /*
+        * Out of space. If we're in IO offload mode, add a new log chunk
+        * inline. If we're doing inline submissions, flag 'td' as needing
+        * a log regrow and we'll take care of it on the submission side.
+        */
+       if (iolog->td->o.io_submit_mode == IO_MODE_OFFLOAD)
+               return regrow_log(iolog);
+
+       iolog->td->flags |= TD_F_REGROW_LOGS;
+       assert(iolog->pending->nr_samples < iolog->pending->max_samples);
+       return iolog->pending;
 }
 
 static void __add_log_sample(struct io_log *iolog, unsigned long val,