Merge branch 'atomics' of https://github.com/bvanassche/fio

[fio.git] / io_u.c

diff --git a/io_u.c b/io_u.c
index 5d62a76cb5a72c81ffd0cfee35ace6bf3ee162e6..7f50906ba3f9792eb34755274bf5b2efab9464f1 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -746,6 +746,9 @@ static void set_rw_ddir(struct thread_data *td, struct io_u *io_u)
 {
        enum fio_ddir ddir = get_rw_ddir(td);
 
+       if (td->o.zone_mode == ZONE_MODE_ZBD)
+               ddir = zbd_adjust_ddir(td, io_u, ddir);
+
        if (td_trimwrite(td)) {
                struct fio_file *f = io_u->file;
                if (f->last_pos[DDIR_WRITE] == f->last_pos[DDIR_TRIM])
@@ -1388,6 +1391,7 @@ static bool __lat_target_failed(struct thread_data *td)
                td->latency_qd_low--;
 
        td->latency_qd = (td->latency_qd + td->latency_qd_low) / 2;
+       td->latency_stable_count = 0;
 
        dprint(FD_RATE, "Ramped down: %d %d %d\n", td->latency_qd_low, td->latency_qd, td->latency_qd_high);
 
@@ -1437,6 +1441,21 @@ static void lat_target_success(struct thread_data *td)
 
        td->latency_qd_low = td->latency_qd;
 
+       if (td->latency_qd + 1 == td->latency_qd_high) {
+               /*
+                * latency_qd will not incease on lat_target_success(), so
+                * called stable. If we stick with this queue depth, the
+                * final latency is likely lower than latency_target. Fix
+                * this by increasing latency_qd_high slowly. Use a naive
+                * heuristic here. If we get lat_target_success() 3 times
+                * in a row, increase latency_qd_high by 1.
+                */
+               if (++td->latency_stable_count >= 3) {
+                       td->latency_qd_high++;
+                       td->latency_stable_count = 0;
+               }
+       }
+
        /*
         * If we haven't failed yet, we double up to a failing value instead
         * of bisecting from highest possible queue depth. If we have set
@@ -1456,7 +1475,7 @@ static void lat_target_success(struct thread_data *td)
         * Same as last one, we are done. Let it run a latency cycle, so
         * we get only the results from the targeted depth.
         */
-       if (td->latency_qd == qd) {
+       if (!o->latency_run && td->latency_qd == qd) {
                if (td->latency_end_run) {
                        dprint(FD_RATE, "We are done\n");
                        td->done = 1;
@@ -1915,8 +1934,8 @@ static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
                if (io_u->error)
                        unlog_io_piece(td, io_u);
                else {
-                       io_u->ipo->flags &= ~IP_F_IN_FLIGHT;
-                       write_barrier();
+                       atomic_store_release(&io_u->ipo->flags,
+                                       io_u->ipo->flags & ~IP_F_IN_FLIGHT);
                }
        }