td_set_runstate(td, TD_RUNNING);
/*
- * Verify_backlog disabled: We need to log rand seed before the
- * actual IO to be able to replay it correctly in the verify phase.
+ * Always log IO before it's issued, so we know the specific
+ * order of it. The logged unit will track when the IO has
+ * completed.
*/
if (td_write(td) && io_u->ddir == DDIR_WRITE &&
td->o.do_verify &&
td->o.verify != VERIFY_NONE &&
- !td->o.experimental_verify &&
- !(td->flags & TD_F_VER_BACKLOG))
+ !td->o.experimental_verify)
log_io_piece(td, io_u);
ret = td_io_queue(td, io_u);
io_u->acct_ddir = -1;
td->cur_depth++;
io_u->flags |= IO_U_F_IN_CUR_DEPTH;
+ io_u->ipo = NULL;
} else if (td->o.verify_async) {
/*
* We ran out, wait for async verify threads to finish and
td_io_u_lock(td);
assert(io_u->flags & IO_U_F_FLIGHT);
io_u->flags &= ~(IO_U_F_FLIGHT | IO_U_F_BUSY_OK);
+
+ /*
+ * Mark IO ok to verify
+ */
+ if (io_u->ipo) {
+ io_u->ipo->flags &= ~IP_F_IN_FLIGHT;
+ write_barrier();
+ }
+
td_io_u_unlock(td);
if (ddir_sync(io_u->ddir)) {
utime_since_now(&td->start));
}
- /*
- * Verify_backlog enable: We need to log the write job after
- * finishing it to prevent verifying before finish writing.
- */
- if (td_write(td) && idx == DDIR_WRITE &&
- td->o.do_verify &&
- td->o.verify != VERIFY_NONE &&
- !td->o.experimental_verify &&
- (td->flags & TD_F_VER_BACKLOG))
- log_io_piece(td, io_u);
-
icd->bytes_done[idx] += bytes;
if (io_u->end_io) {
#include <guasi.h>
#endif
-#define FIO_IOOPS_VERSION 16
+#define FIO_IOOPS_VERSION 17
enum {
IO_U_F_FREE = 1 << 0,
*/
unsigned long buf_filled_len;
+ struct io_piece *ipo;
+
union {
#ifdef CONFIG_LIBAIO
struct iocb iocb;
ipo->offset = io_u->offset;
ipo->len = io_u->buflen;
ipo->numberio = io_u->numberio;
+ ipo->flags = IP_F_IN_FLIGHT;
+
+ io_u->ipo = ipo;
if (io_u_should_trim(td, io_u)) {
flist_add_tail(&ipo->trim_list, &td->trim_list);
IP_F_ONRB = 1,
IP_F_ONLIST = 2,
IP_F_TRIMMED = 4,
+ IP_F_IN_FLIGHT = 8,
};
/*
struct rb_node *n = rb_first(&td->io_hist_tree);
ipo = rb_entry(n, struct io_piece, rb_node);
+
+ /*
+ * Ensure that the associated IO has completed
+ */
+ read_barrier();
+ if (ipo->flags & IP_F_IN_FLIGHT)
+ goto nothing;
+
rb_erase(n, &td->io_hist_tree);
assert(ipo->flags & IP_F_ONRB);
ipo->flags &= ~IP_F_ONRB;
} else if (!flist_empty(&td->io_hist_list)) {
ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
+
+ /*
+ * Ensure that the associated IO has completed
+ */
+ read_barrier();
+ if (ipo->flags & IP_F_IN_FLIGHT)
+ goto nothing;
+
flist_del(&ipo->list);
assert(ipo->flags & IP_F_ONLIST);
ipo->flags &= ~IP_F_ONLIST;
return 0;
}
+nothing:
dprint(FD_VERIFY, "get_next_verify: empty\n");
return 1;
}