#include "idletime.h"
#include "err.h"
#include "lib/tp.h"
+#include "workqueue.h"
static pthread_t helper_thread;
static pthread_mutex_t helper_lock;
return 0;
}
+/*
+ * We need to update the runtime consistently in ms, but keep a running
+ * tally of the current elapsed time in microseconds for sub millisecond
+ * updates.
+ */
+static inline void update_runtime(struct thread_data *td,
+ unsigned long long *elapsed_us,
+ const enum fio_ddir ddir)
+{
+ td->ts.runtime[ddir] -= (elapsed_us[ddir] + 999) / 1000;
+ elapsed_us[ddir] += utime_since_now(&td->start);
+ td->ts.runtime[ddir] += (elapsed_us[ddir] + 999) / 1000;
+}
+
static int break_on_this_error(struct thread_data *td, enum fio_ddir ddir,
int *retptr)
{
* if the queue is full, we MUST reap at least 1 event
*/
min_evts = min(td->o.iodepth_batch_complete, td->cur_depth);
- if (full && !min_evts)
+ if ((full && !min_evts) || !td->o.iodepth_batch_complete)
min_evts = 1;
if (time && (__should_check_rate(td, DDIR_READ) ||
return ret;
}
+int io_queue_event(struct thread_data *td, struct io_u *io_u, int *ret,
+ enum fio_ddir ddir, uint64_t *bytes_issued, int from_verify,
+ struct timeval *comp_time)
+{
+ int ret2;
+
+ switch (*ret) {
+ case FIO_Q_COMPLETED:
+ if (io_u->error) {
+ *ret = -io_u->error;
+ clear_io_u(td, io_u);
+ } else if (io_u->resid) {
+ int bytes = io_u->xfer_buflen - io_u->resid;
+ struct fio_file *f = io_u->file;
+
+ if (bytes_issued)
+ *bytes_issued += bytes;
+
+ if (!from_verify)
+ trim_io_piece(td, io_u);
+
+ /*
+ * zero read, fail
+ */
+ if (!bytes) {
+ if (!from_verify)
+ unlog_io_piece(td, io_u);
+ td_verror(td, EIO, "full resid");
+ put_io_u(td, io_u);
+ break;
+ }
+
+ io_u->xfer_buflen = io_u->resid;
+ io_u->xfer_buf += bytes;
+ io_u->offset += bytes;
+
+ if (ddir_rw(io_u->ddir))
+ td->ts.short_io_u[io_u->ddir]++;
+
+ f = io_u->file;
+ if (io_u->offset == f->real_file_size)
+ goto sync_done;
+
+ requeue_io_u(td, &io_u);
+ } else {
+sync_done:
+ if (comp_time && (__should_check_rate(td, DDIR_READ) ||
+ __should_check_rate(td, DDIR_WRITE) ||
+ __should_check_rate(td, DDIR_TRIM)))
+ fio_gettime(comp_time, NULL);
+
+ *ret = io_u_sync_complete(td, io_u);
+ if (*ret < 0)
+ break;
+ }
+ return 0;
+ case FIO_Q_QUEUED:
+ /*
+ * if the engine doesn't have a commit hook,
+ * the io_u is really queued. if it does have such
+ * a hook, it has to call io_u_queued() itself.
+ */
+ if (td->io_ops->commit == NULL)
+ io_u_queued(td, io_u);
+ if (bytes_issued)
+ *bytes_issued += io_u->xfer_buflen;
+ break;
+ case FIO_Q_BUSY:
+ if (!from_verify)
+ unlog_io_piece(td, io_u);
+ requeue_io_u(td, &io_u);
+ ret2 = td_io_commit(td);
+ if (ret2 < 0)
+ *ret = ret2;
+ break;
+ default:
+ assert(ret < 0);
+ td_verror(td, -(*ret), "td_io_queue");
+ break;
+ }
+
+ if (break_on_this_error(td, ddir, ret))
+ return 1;
+
+ return 0;
+}
+
/*
* The main verify engine. Runs over the writes we previously submitted,
* reads the blocks back in, and checks the crc/md5 of the data.
io_u = NULL;
while (!td->terminate) {
enum fio_ddir ddir;
- int ret2, full;
+ int full;
update_tv_cache(td);
check_update_rusage(td);
continue;
} else if (io_u->ddir == DDIR_TRIM) {
io_u->ddir = DDIR_READ;
- io_u->flags |= IO_U_F_TRIMMED;
+ io_u_set(io_u, IO_U_F_TRIMMED);
break;
} else if (io_u->ddir == DDIR_WRITE) {
io_u->ddir = DDIR_READ;
fio_gettime(&io_u->start_time, NULL);
ret = td_io_queue(td, io_u);
- switch (ret) {
- case FIO_Q_COMPLETED:
- if (io_u->error) {
- ret = -io_u->error;
- clear_io_u(td, io_u);
- } else if (io_u->resid) {
- int bytes = io_u->xfer_buflen - io_u->resid;
-
- /*
- * zero read, fail
- */
- if (!bytes) {
- td_verror(td, EIO, "full resid");
- put_io_u(td, io_u);
- break;
- }
-
- io_u->xfer_buflen = io_u->resid;
- io_u->xfer_buf += bytes;
- io_u->offset += bytes;
-
- if (ddir_rw(io_u->ddir))
- td->ts.short_io_u[io_u->ddir]++;
-
- f = io_u->file;
- if (io_u->offset == f->real_file_size)
- goto sync_done;
-
- requeue_io_u(td, &io_u);
- } else {
-sync_done:
- ret = io_u_sync_complete(td, io_u);
- if (ret < 0)
- break;
- }
- continue;
- case FIO_Q_QUEUED:
- break;
- case FIO_Q_BUSY:
- requeue_io_u(td, &io_u);
- ret2 = td_io_commit(td);
- if (ret2 < 0)
- ret = ret2;
- break;
- default:
- assert(ret < 0);
- td_verror(td, -ret, "td_io_queue");
- break;
- }
- if (break_on_this_error(td, ddir, &ret))
+ if (io_queue_event(td, io_u, &ret, ddir, NULL, 1, NULL))
break;
/*
td->o.time_based) {
struct timeval comp_time;
struct io_u *io_u;
- int ret2, full;
+ int full;
enum fio_ddir ddir;
check_update_rusage(td);
!td->o.experimental_verify)
log_io_piece(td, io_u);
- ret = td_io_queue(td, io_u);
- switch (ret) {
- case FIO_Q_COMPLETED:
- if (io_u->error) {
- ret = -io_u->error;
- unlog_io_piece(td, io_u);
- clear_io_u(td, io_u);
- } else if (io_u->resid) {
- int bytes = io_u->xfer_buflen - io_u->resid;
- struct fio_file *f = io_u->file;
-
- bytes_issued += bytes;
-
- trim_io_piece(td, io_u);
-
- /*
- * zero read, fail
- */
- if (!bytes) {
- unlog_io_piece(td, io_u);
- td_verror(td, EIO, "full resid");
- put_io_u(td, io_u);
- break;
- }
-
- io_u->xfer_buflen = io_u->resid;
- io_u->xfer_buf += bytes;
- io_u->offset += bytes;
-
- if (ddir_rw(io_u->ddir))
- td->ts.short_io_u[io_u->ddir]++;
-
- if (io_u->offset == f->real_file_size)
- goto sync_done;
+ if (td->o.io_submit_mode == IO_MODE_OFFLOAD) {
+ if (td->error)
+ break;
+ ret = workqueue_enqueue(&td->io_wq, io_u);
+ } else {
+ ret = td_io_queue(td, io_u);
- requeue_io_u(td, &io_u);
- } else {
-sync_done:
- if (__should_check_rate(td, DDIR_READ) ||
- __should_check_rate(td, DDIR_WRITE) ||
- __should_check_rate(td, DDIR_TRIM))
- fio_gettime(&comp_time, NULL);
+ if (io_queue_event(td, io_u, &ret, ddir, &bytes_issued, 1, &comp_time))
+ break;
- ret = io_u_sync_complete(td, io_u);
- if (ret < 0)
- break;
- bytes_issued += io_u->xfer_buflen;
- }
- break;
- case FIO_Q_QUEUED:
/*
- * if the engine doesn't have a commit hook,
- * the io_u is really queued. if it does have such
- * a hook, it has to call io_u_queued() itself.
+ * See if we need to complete some commands. Note that
+ * we can get BUSY even without IO queued, if the
+ * system is resource starved.
*/
- if (td->io_ops->commit == NULL)
- io_u_queued(td, io_u);
- bytes_issued += io_u->xfer_buflen;
- break;
- case FIO_Q_BUSY:
- unlog_io_piece(td, io_u);
- requeue_io_u(td, &io_u);
- ret2 = td_io_commit(td);
- if (ret2 < 0)
- ret = ret2;
- break;
- default:
- assert(ret < 0);
- put_io_u(td, io_u);
- break;
- }
-
- if (break_on_this_error(td, ddir, &ret))
- break;
-
- /*
- * See if we need to complete some commands. Note that we
- * can get BUSY even without IO queued, if the system is
- * resource starved.
- */
reap:
- full = queue_full(td) || (ret == FIO_Q_BUSY && td->cur_depth);
- if (full || !td->o.iodepth_batch_complete)
- ret = wait_for_completions(td, &comp_time);
+ full = queue_full(td) ||
+ (ret == FIO_Q_BUSY && td->cur_depth);
+ if (full || !td->o.iodepth_batch_complete)
+ ret = wait_for_completions(td, &comp_time);
+ }
if (ret < 0)
break;
if (!ddir_rw_sum(td->bytes_done) &&
if (!td->error) {
struct fio_file *f;
- i = td->cur_depth;
+ if (td->o.io_submit_mode == IO_MODE_OFFLOAD) {
+ workqueue_flush(&td->io_wq);
+ i = 0;
+ } else
+ i = td->cur_depth;
+
if (i) {
ret = io_u_queued_complete(td, i);
if (td->o.fill_device && td->error == ENOSPC)
if (!io_u)
break;
- io_u->flags |= IO_U_F_FLIGHT;
+ io_u_set(io_u, IO_U_F_FLIGHT);
io_u->error = 0;
io_u->resid = 0;
if (ddir_rw(acct_ddir(io_u)))
return td->bytes_done[DDIR_WRITE] + td->bytes_done[DDIR_TRIM];
}
+static void io_workqueue_fn(struct thread_data *td, struct io_u *io_u)
+{
+ const enum fio_ddir ddir = io_u->ddir;
+ int ret;
+
+ dprint(FD_RATE, "io_u %p queued by %u\n", io_u, gettid());
+
+ io_u_set(io_u, IO_U_F_NO_FILE_PUT);
+
+ td->cur_depth++;
+
+ ret = td_io_queue(td, io_u);
+
+ dprint(FD_RATE, "io_u %p ret %d by %u\n", io_u, ret, gettid());
+
+ io_queue_event(td, io_u, &ret, ddir, NULL, 0, NULL);
+
+ if (ret == FIO_Q_QUEUED)
+ ret = io_u_queued_complete(td, 1);
+
+ td->cur_depth--;
+}
+
/*
* Entry point for the thread based jobs. The process based jobs end up
* here as well, after a little setup.
*/
static void *thread_main(void *data)
{
- unsigned long long elapsed;
+ unsigned long long elapsed_us[DDIR_RWDIR_CNT] = { 0, };
struct thread_data *td = data;
struct thread_options *o = &td->o;
pthread_condattr_t attr;
fio_verify_init(td);
+ if ((o->io_submit_mode == IO_MODE_OFFLOAD) &&
+ workqueue_init(td, &td->io_wq, io_workqueue_fn, td->o.iodepth))
+ goto err;
+
fio_gettime(&td->epoch, NULL);
fio_getrusage(&td->ru_start);
clear_state = 0;
check_update_rusage(td);
fio_mutex_down(stat_mutex);
- if (td_read(td) && td->io_bytes[DDIR_READ]) {
- elapsed = mtime_since_now(&td->start);
- td->ts.runtime[DDIR_READ] += elapsed;
- }
- if (td_write(td) && td->io_bytes[DDIR_WRITE]) {
- elapsed = mtime_since_now(&td->start);
- td->ts.runtime[DDIR_WRITE] += elapsed;
- }
- if (td_trim(td) && td->io_bytes[DDIR_TRIM]) {
- elapsed = mtime_since_now(&td->start);
- td->ts.runtime[DDIR_TRIM] += elapsed;
- }
+ if (td_read(td) && td->io_bytes[DDIR_READ])
+ update_runtime(td, elapsed_us, DDIR_READ);
+ if (td_write(td) && td->io_bytes[DDIR_WRITE])
+ update_runtime(td, elapsed_us, DDIR_WRITE);
+ if (td_trim(td) && td->io_bytes[DDIR_TRIM])
+ update_runtime(td, elapsed_us, DDIR_TRIM);
fio_gettime(&td->start, NULL);
fio_mutex_up(stat_mutex);
check_update_rusage(td);
fio_mutex_down(stat_mutex);
- td->ts.runtime[DDIR_READ] += mtime_since_now(&td->start);
+ update_runtime(td, elapsed_us, DDIR_READ);
fio_gettime(&td->start, NULL);
fio_mutex_up(stat_mutex);
fio_writeout_logs(td);
+ if (o->io_submit_mode == IO_MODE_OFFLOAD)
+ workqueue_exit(&td->io_wq);
+
if (td->flags & TD_F_COMPRESS_LOG)
tp_exit(&td->tp_data);
for (i = 0; i < DDIR_RWDIR_CNT; i++) {
struct io_log *log = agg_io_log[i];
- flush_log(log);
+ flush_log(log, 0);
free_log(log);
}
}
projects / fio.git / blobdiff