Merge branch 'master' of https://github.com/bvanassche/fio

[fio.git] / io_u.c

diff --git a/io_u.c b/io_u.c
index f30fc03701300e0b41f8a1be7c906a9eaa7996a1..586a4befdce0969e596f9243abc73044b4b75fda 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -1326,8 +1326,10 @@ static struct fio_file *__get_next_file(struct thread_data *td)
        if (f && fio_file_open(f) && !fio_file_closing(f)) {
                if (td->o.file_service_type == FIO_FSERVICE_SEQ)
                        goto out;
-               if (td->file_service_left--)
-                       goto out;
+               if (td->file_service_left) {
+                 td->file_service_left--;
+                 goto out;
+               }
        }
 
        if (td->o.file_service_type == FIO_FSERVICE_RR ||
@@ -1387,11 +1389,16 @@ static long set_io_u_file(struct thread_data *td, struct io_u *io_u)
        return 0;
 }
 
-static void lat_fatal(struct thread_data *td, struct io_completion_data *icd,
+static void lat_fatal(struct thread_data *td, struct io_u *io_u, struct io_completion_data *icd,
                      unsigned long long tnsec, unsigned long long max_nsec)
 {
-       if (!td->error)
-               log_err("fio: latency of %llu nsec exceeds specified max (%llu nsec)\n", tnsec, max_nsec);
+       if (!td->error) {
+               log_err("fio: latency of %llu nsec exceeds specified max (%llu nsec): %s %s %llu %llu\n",
+                                       tnsec, max_nsec,
+                                       io_u->file->file_name,
+                                       io_ddir_name(io_u->ddir),
+                                       io_u->offset, io_u->buflen);
+       }
        td_verror(td, ETIMEDOUT, "max latency exceeded");
        icd->error = ETIMEDOUT;
 }
@@ -1588,7 +1595,7 @@ again:
                assert(io_u->flags & IO_U_F_FREE);
                io_u_clear(td, io_u, IO_U_F_FREE | IO_U_F_NO_FILE_PUT |
                                 IO_U_F_TRIMMED | IO_U_F_BARRIER |
-                                IO_U_F_VER_LIST | IO_U_F_PRIORITY);
+                                IO_U_F_VER_LIST | IO_U_F_HIGH_PRIO);
 
                io_u->error = 0;
                io_u->acct_ddir = -1;
@@ -1792,6 +1799,10 @@ struct io_u *get_io_u(struct thread_data *td)
        io_u->xfer_buf = io_u->buf;
        io_u->xfer_buflen = io_u->buflen;
 
+       /*
+        * Remember the issuing context priority. The IO engine may change this.
+        */
+       io_u->ioprio = td->ioprio;
 out:
        assert(io_u->file);
        if (!td_io_prep(td, io_u)) {
@@ -1877,7 +1888,8 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u,
                unsigned long long tnsec;
 
                tnsec = ntime_since(&io_u->start_time, &icd->time);
-               add_lat_sample(td, idx, tnsec, bytes, io_u->offset, io_u_is_prio(io_u));
+               add_lat_sample(td, idx, tnsec, bytes, io_u->offset,
+                              io_u->ioprio, io_u_is_high_prio(io_u));
 
                if (td->flags & TD_F_PROFILE_OPS) {
                        struct prof_io_ops *ops = &td->prof_io_ops;
@@ -1886,17 +1898,20 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u,
                                icd->error = ops->io_u_lat(td, tnsec);
                }
 
-               if (td->o.max_latency && tnsec > td->o.max_latency)
-                       lat_fatal(td, icd, tnsec, td->o.max_latency);
-               if (td->o.latency_target && tnsec > td->o.latency_target) {
-                       if (lat_target_failed(td))
-                               lat_fatal(td, icd, tnsec, td->o.latency_target);
+               if (ddir_rw(idx)) {
+                       if (td->o.max_latency[idx] && tnsec > td->o.max_latency[idx])
+                               lat_fatal(td, io_u, icd, tnsec, td->o.max_latency[idx]);
+                       if (td->o.latency_target && tnsec > td->o.latency_target) {
+                               if (lat_target_failed(td))
+                                       lat_fatal(td, io_u, icd, tnsec, td->o.latency_target);
+                       }
                }
        }
 
        if (ddir_rw(idx)) {
                if (!td->o.disable_clat) {
-                       add_clat_sample(td, idx, llnsec, bytes, io_u->offset, io_u_is_prio(io_u));
+                       add_clat_sample(td, idx, llnsec, bytes, io_u->offset,
+                                       io_u->ioprio, io_u_is_high_prio(io_u));
                        io_u_mark_latency(td, llnsec);
                }
 
@@ -1989,7 +2004,7 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
                 * Make sure we notice short IO from here, and requeue them
                 * appropriately!
                 */
-               if (io_u->resid) {
+               if (bytes && io_u->resid) {
                        io_u->xfer_buflen = io_u->resid;
                        io_u->xfer_buf += bytes;
                        io_u->offset += bytes;
@@ -2153,7 +2168,7 @@ void io_u_queued(struct thread_data *td, struct io_u *io_u)
                        td = td->parent;
 
                add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen,
-                               io_u->offset, io_u_is_prio(io_u));
+                               io_u->offset, io_u->ioprio);
        }
 }
 
@@ -2163,6 +2178,7 @@ void io_u_queued(struct thread_data *td, struct io_u *io_u)
 static struct frand_state *get_buf_state(struct thread_data *td)
 {
        unsigned int v;
+       unsigned long long i;
 
        if (!td->o.dedupe_percentage)
                return &td->buf_state;
@@ -2174,7 +2190,24 @@ static struct frand_state *get_buf_state(struct thread_data *td)
        v = rand_between(&td->dedupe_state, 1, 100);
 
        if (v <= td->o.dedupe_percentage)
-               return &td->buf_state_prev;
+               switch (td->o.dedupe_mode) {
+               case DEDUPE_MODE_REPEAT:
+                       /*
+                       * The caller advances the returned frand_state.
+                       * A copy of prev should be returned instead since
+                       * a subsequent intention to generate a deduped buffer
+                       * might result in generating a unique one
+                       */
+                       frand_copy(&td->buf_state_ret, &td->buf_state_prev);
+                       return &td->buf_state_ret;
+               case DEDUPE_MODE_WORKING_SET:
+                       i = rand_between(&td->dedupe_working_set_index_state, 0, td->num_unique_pages - 1);
+                       frand_copy(&td->buf_state_ret, &td->dedupe_working_set_states[i]);
+                       return &td->buf_state_ret;
+               default:
+                       log_err("unexpected dedupe mode %u\n", td->o.dedupe_mode);
+                       assert(0);
+               }
 
        return &td->buf_state;
 }
@@ -2290,10 +2323,19 @@ int do_io_u_trim(const struct thread_data *td, struct io_u *io_u)
        struct fio_file *f = io_u->file;
        int ret;
 
+       if (td->o.zone_mode == ZONE_MODE_ZBD) {
+               ret = zbd_do_io_u_trim(td, io_u);
+               if (ret == io_u_completed)
+                       return io_u->xfer_buflen;
+               if (ret)
+                       goto err;
+       }
+
        ret = os_trim(f, io_u->offset, io_u->xfer_buflen);
        if (!ret)
                return io_u->xfer_buflen;
 
+err:
        io_u->error = ret;
        return 0;
 #endif