rate: ensure IO issue restarts right after sleep

[fio.git] / backend.c

diff --git a/backend.c b/backend.c
index d3e9bf8160615b64ae714b47f4524403df272f44..5304ddc0ae1df0245405d2f6136c1b45755473d2 100644 (file)
--- a/backend.c
+++ b/backend.c
@@ -844,14 +844,13 @@ static bool io_complete_bytes_exceeded(struct thread_data *td)
  */
 static long long usec_for_io(struct thread_data *td, enum fio_ddir ddir)
 {
-       uint64_t secs, remainder, bps, bytes, iops;
+       uint64_t bps = td->rate_bps[ddir];
 
        assert(!(td->flags & TD_F_CHILD));
-       bytes = td->rate_io_issue_bytes[ddir];
-       bps = td->rate_bps[ddir];
 
        if (td->o.rate_process == RATE_PROCESS_POISSON) {
-               uint64_t val;
+               uint64_t val, iops;
+
                iops = bps / td->o.bs[ddir];
                val = (int64_t) (1000000 / iops) *
                                -logf(__rand_0_1(&td->poisson_state[ddir]));
@@ -863,14 +862,47 @@ static long long usec_for_io(struct thread_data *td, enum fio_ddir ddir)
                td->last_usec[ddir] += val;
                return td->last_usec[ddir];
        } else if (bps) {
-               secs = bytes / bps;
-               remainder = bytes % bps;
+               uint64_t bytes = td->rate_io_issue_bytes[ddir];
+               uint64_t secs = bytes / bps;
+               uint64_t remainder = bytes % bps;
+
                return remainder * 1000000 / bps + secs * 1000000;
        }
 
        return 0;
 }
 
+static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir)
+{
+       unsigned long long b;
+       uint64_t total;
+       int left;
+
+       b = ddir_rw_sum(td->io_blocks);
+       if (b % td->o.thinktime_blocks)
+               return;
+
+       io_u_quiesce(td);
+
+       total = 0;
+       if (td->o.thinktime_spin)
+               total = usec_spin(td->o.thinktime_spin);
+
+       left = td->o.thinktime - total;
+       if (left)
+               total += usec_sleep(td, left);
+
+       /*
+        * If we're ignoring thinktime for the rate, add the number of bytes
+        * we would have done while sleeping, minus one block to ensure we
+        * start issuing immediately after the sleep.
+        */
+       if (total && td->rate_bps[ddir] && td->o.rate_ign_think) {
+               td->rate_io_issue_bytes[ddir] += (td->rate_bps[ddir] * 1000000) / total;
+               td->rate_io_issue_bytes[ddir] -= td->o.min_bs[ddir];
+       }
+}
+
 /*
  * Main IO worker function. It retrieves io_u's to process and queues
  * and reaps them, checking for rate and errors along the way.
@@ -955,6 +987,7 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)
                        int err = PTR_ERR(io_u);
 
                        io_u = NULL;
+                       ddir = DDIR_INVAL;
                        if (err == -EBUSY) {
                                ret = FIO_Q_BUSY;
                                goto reap;
@@ -1062,23 +1095,8 @@ reap:
                if (!in_ramp_time(td) && td->o.latency_target)
                        lat_target_check(td);
 
-               if (td->o.thinktime) {
-                       unsigned long long b;
-
-                       b = ddir_rw_sum(td->io_blocks);
-                       if (!(b % td->o.thinktime_blocks)) {
-                               int left;
-
-                               io_u_quiesce(td);
-
-                               if (td->o.thinktime_spin)
-                                       usec_spin(td->o.thinktime_spin);
-
-                               left = td->o.thinktime - td->o.thinktime_spin;
-                               if (left)
-                                       usec_sleep(td, left);
-                       }
-               }
+               if (ddir_rw(ddir) && td->o.thinktime)
+                       handle_thinktime(td, ddir);
        }
 
        check_update_rusage(td);
@@ -1203,9 +1221,9 @@ static int init_io_u(struct thread_data *td)
                data_xfer = 0;
 
        err = 0;
-       err += io_u_rinit(&td->io_u_requeues, td->o.iodepth);
-       err += io_u_qinit(&td->io_u_freelist, td->o.iodepth);
-       err += io_u_qinit(&td->io_u_all, td->o.iodepth);
+       err += !io_u_rinit(&td->io_u_requeues, td->o.iodepth);
+       err += !io_u_qinit(&td->io_u_freelist, td->o.iodepth);
+       err += !io_u_qinit(&td->io_u_all, td->o.iodepth);
 
        if (err) {
                log_err("fio: failed setting up IO queues\n");
@@ -2318,6 +2336,7 @@ reap:
                                        nr_started--;
                                        break;
                                }
+                               fd = NULL;
                                ret = pthread_detach(td->thread);
                                if (ret)
                                        log_err("pthread_detach: %s",
@@ -2479,12 +2498,7 @@ int fio_backend(struct sk_out *sk_out)
        }
 
        for_each_td(td, i) {
-               if (td->ss.dur) {
-                       if (td->ss.iops_data != NULL) {
-                               free(td->ss.iops_data);
-                               free(td->ss.bw_data);
-                       }
-               }
+               steadystate_free(td);
                fio_options_free(td);
                if (td->rusage_sem) {
                        fio_mutex_remove(td->rusage_sem);