verify: always log IO in the order they are issued

We currently log verify_backlog verifies when they complete,
which means the sequence of verify and issue might be different.

Change this to log in one spot, prior to issue, and track the
completion state of the logged unit instead. This unifies the
handling of verifies.

Signed-off-by: Jens Axboe <axboe@fb.com>

diff --git a/backend.c b/backend.c
index 62fa17c3a209665252a25e322aa084b15ce86c25..501c59a322a5551e174a108c7989c70c0db9f09c 100644 (file)
--- a/backend.c
+++ b/backend.c
@@ -725,14 +725,14 @@ static uint64_t do_io(struct thread_data *td)
                        td_set_runstate(td, TD_RUNNING);
 
                /*
                        td_set_runstate(td, TD_RUNNING);
 
                /*

-                * Verify_backlog disabled: We need to log rand seed before the
-                * actual IO to be able to replay it correctly in the verify phase.
+                * Always log IO before it's issued, so we know the specific
+                * order of it. The logged unit will track when the IO has
+                * completed.

                 */
                if (td_write(td) && io_u->ddir == DDIR_WRITE &&
                    td->o.do_verify &&
                    td->o.verify != VERIFY_NONE &&
                 */
                if (td_write(td) && io_u->ddir == DDIR_WRITE &&
                    td->o.do_verify &&
                    td->o.verify != VERIFY_NONE &&

-                   !td->o.experimental_verify &&
-                   !(td->flags & TD_F_VER_BACKLOG))
+                   !td->o.experimental_verify)

                        log_io_piece(td, io_u);
 
                ret = td_io_queue(td, io_u);
                        log_io_piece(td, io_u);
 
                ret = td_io_queue(td, io_u);

diff --git a/io_u.c b/io_u.c
index 4264cd54115ca39e2e70d97857179f009d24b742..64ff73cd5555b8d4efa0303085f7944cae07544c 100644 (file)
--- a/io_u.c
+++ b/io_u.c
@@ -1285,6 +1285,7 @@ again:
                io_u->acct_ddir = -1;
                td->cur_depth++;
                io_u->flags |= IO_U_F_IN_CUR_DEPTH;
                io_u->acct_ddir = -1;
                td->cur_depth++;
                io_u->flags |= IO_U_F_IN_CUR_DEPTH;

+               io_u->ipo = NULL;

        } else if (td->o.verify_async) {
                /*
                 * We ran out, wait for async verify threads to finish and
        } else if (td->o.verify_async) {
                /*
                 * We ran out, wait for async verify threads to finish and


@@ -1568,6 +1569,15 @@ static void io_completed(struct thread_data *td, struct io_u *io_u,
        td_io_u_lock(td);
        assert(io_u->flags & IO_U_F_FLIGHT);
        io_u->flags &= ~(IO_U_F_FLIGHT | IO_U_F_BUSY_OK);
        td_io_u_lock(td);
        assert(io_u->flags & IO_U_F_FLIGHT);
        io_u->flags &= ~(IO_U_F_FLIGHT | IO_U_F_BUSY_OK);

+
+       /*
+        * Mark IO ok to verify
+        */
+       if (io_u->ipo) {
+               io_u->ipo->flags &= ~IP_F_IN_FLIGHT;
+               write_barrier();
+       }
+

        td_io_u_unlock(td);
 
        if (ddir_sync(io_u->ddir)) {
        td_io_u_unlock(td);
 
        if (ddir_sync(io_u->ddir)) {


@@ -1623,17 +1633,6 @@ static void io_completed(struct thread_data *td, struct io_u *io_u,
                                         utime_since_now(&td->start));
                }
 
                                         utime_since_now(&td->start));
                }
 

-               /*
-                * Verify_backlog enable: We need to log the write job after
-                * finishing it to prevent verifying before finish writing.
-                */
-               if (td_write(td) && idx == DDIR_WRITE &&
-                   td->o.do_verify &&
-                   td->o.verify != VERIFY_NONE &&
-                   !td->o.experimental_verify &&
-                   (td->flags & TD_F_VER_BACKLOG))
-                       log_io_piece(td, io_u);
-

                icd->bytes_done[idx] += bytes;
 
                if (io_u->end_io) {
                icd->bytes_done[idx] += bytes;
 
                if (io_u->end_io) {

diff --git a/ioengine.h b/ioengine.h
index 0756bc7e6c1335dc729bbe6c08b9f78630121d66..19ed10bf7a5401dd13fafe5cef97cb2d32748bd0 100644 (file)
--- a/ioengine.h
+++ b/ioengine.h
@@ -15,7 +15,7 @@
 #include <guasi.h>
 #endif
 
 #include <guasi.h>
 #endif
 

-#define FIO_IOOPS_VERSION      16
+#define FIO_IOOPS_VERSION      17

 
 enum {
        IO_U_F_FREE             = 1 << 0,
 
 enum {
        IO_U_F_FREE             = 1 << 0,


@@ -71,6 +71,8 @@ struct io_u {
         */
        unsigned long buf_filled_len;
 
         */
        unsigned long buf_filled_len;
 

+       struct io_piece *ipo;
+

        union {
 #ifdef CONFIG_LIBAIO
                struct iocb iocb;
        union {
 #ifdef CONFIG_LIBAIO
                struct iocb iocb;

diff --git a/iolog.c b/iolog.c
index 017b235c217a4e0ce52b298fd958386213001ef0..5fd9416c036ec2454030b4ddf6772eab487efa30 100644 (file)
--- a/iolog.c
+++ b/iolog.c
@@ -189,6 +189,9 @@ void log_io_piece(struct thread_data *td, struct io_u *io_u)
        ipo->offset = io_u->offset;
        ipo->len = io_u->buflen;
        ipo->numberio = io_u->numberio;
        ipo->offset = io_u->offset;
        ipo->len = io_u->buflen;
        ipo->numberio = io_u->numberio;

+       ipo->flags = IP_F_IN_FLIGHT;
+
+       io_u->ipo = ipo;

 
        if (io_u_should_trim(td, io_u)) {
                flist_add_tail(&ipo->trim_list, &td->trim_list);
 
        if (io_u_should_trim(td, io_u)) {
                flist_add_tail(&ipo->trim_list, &td->trim_list);

diff --git a/iolog.h b/iolog.h
index 321576dbe6117946e1458ee3233ed14f0d7612cf..3ec48f2100fe04f1cc8981a81d61e0d60fb3f9ea 100644 (file)
--- a/iolog.h
+++ b/iolog.h
@@ -67,6 +67,7 @@ enum {
        IP_F_ONRB       = 1,
        IP_F_ONLIST     = 2,
        IP_F_TRIMMED    = 4,
        IP_F_ONRB       = 1,
        IP_F_ONLIST     = 2,
        IP_F_TRIMMED    = 4,

+       IP_F_IN_FLIGHT  = 8,

 };
 
 /*
 };
 
 /*

diff --git a/verify.c b/verify.c
index 90cd093add1fbbb13ba2f9d713ce60dc9e4f49e9..93731228f1b68661e0edd3293bc491ccb650774d 100644 (file)
--- a/verify.c
+++ b/verify.c
@@ -1022,11 +1022,27 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
                struct rb_node *n = rb_first(&td->io_hist_tree);
 
                ipo = rb_entry(n, struct io_piece, rb_node);
                struct rb_node *n = rb_first(&td->io_hist_tree);
 
                ipo = rb_entry(n, struct io_piece, rb_node);

+
+               /*
+                * Ensure that the associated IO has completed
+                */
+               read_barrier();
+               if (ipo->flags & IP_F_IN_FLIGHT)
+                       goto nothing;
+

                rb_erase(n, &td->io_hist_tree);
                assert(ipo->flags & IP_F_ONRB);
                ipo->flags &= ~IP_F_ONRB;
        } else if (!flist_empty(&td->io_hist_list)) {
                ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
                rb_erase(n, &td->io_hist_tree);
                assert(ipo->flags & IP_F_ONRB);
                ipo->flags &= ~IP_F_ONRB;
        } else if (!flist_empty(&td->io_hist_list)) {
                ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);

+
+               /*
+                * Ensure that the associated IO has completed
+                */
+               read_barrier();
+               if (ipo->flags & IP_F_IN_FLIGHT)
+                       goto nothing;
+

                flist_del(&ipo->list);
                assert(ipo->flags & IP_F_ONLIST);
                ipo->flags &= ~IP_F_ONLIST;
                flist_del(&ipo->list);
                assert(ipo->flags & IP_F_ONLIST);
                ipo->flags &= ~IP_F_ONLIST;


@@ -1072,6 +1088,7 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
                return 0;
        }
 
                return 0;
        }
 

+nothing:

        dprint(FD_VERIFY, "get_next_verify: empty\n");
        return 1;
 }
        dprint(FD_VERIFY, "get_next_verify: empty\n");
        return 1;
 }