#include "lib/getrusage.h"
#include "idletime.h"
#include "err.h"
-#include "lib/tp.h"
#include "workqueue.h"
#include "lib/mountcheck.h"
+#include "rate-submit.h"
static pthread_t helper_thread;
static pthread_mutex_t helper_lock;
if (td->error)
break;
- ret = workqueue_enqueue(&td->io_wq, io_u);
+ workqueue_enqueue(&td->io_wq, &io_u->work);
+ ret = FIO_Q_QUEUED;
- if (ret == FIO_Q_QUEUED && ddir_rw(ddir)) {
+ if (ddir_rw(ddir)) {
td->io_issues[ddir]++;
td->io_issue_bytes[ddir] += blen;
td->rate_io_issue_bytes[ddir] += blen;
if (!in_ramp_time(td) && should_check_rate(td)) {
if (check_min_rate(td, &comp_time)) {
- if (exitall_on_terminate)
+ if (exitall_on_terminate || td->o.exitall_error)
fio_terminate_threads(td->groupid);
td_verror(td, EIO, "check_min_rate");
break;
return td->bytes_done[DDIR_WRITE] + td->bytes_done[DDIR_TRIM];
}
-static void io_workqueue_fn(struct thread_data *td, struct io_u *io_u)
-{
- const enum fio_ddir ddir = io_u->ddir;
- int ret;
-
- dprint(FD_RATE, "io_u %p queued by %u\n", io_u, gettid());
-
- io_u_set(io_u, IO_U_F_NO_FILE_PUT);
-
- td->cur_depth++;
-
- do {
- ret = td_io_queue(td, io_u);
- if (ret != FIO_Q_BUSY)
- break;
- ret = io_u_queued_complete(td, 1);
- if (ret > 0)
- td->cur_depth -= ret;
- io_u_clear(io_u, IO_U_F_FLIGHT);
- } while (1);
-
- dprint(FD_RATE, "io_u %p ret %d by %u\n", io_u, ret, gettid());
-
- io_queue_event(td, io_u, &ret, ddir, NULL, 0, NULL);
-
- if (ret == FIO_Q_COMPLETED)
- td->cur_depth--;
- else if (ret == FIO_Q_QUEUED) {
- unsigned int min_evts;
-
- if (td->o.iodepth == 1)
- min_evts = 1;
- else
- min_evts = 0;
-
- ret = io_u_queued_complete(td, min_evts);
- if (ret > 0)
- td->cur_depth -= ret;
- } else if (ret == FIO_Q_BUSY) {
- ret = io_u_queued_complete(td, td->cur_depth);
- if (ret > 0)
- td->cur_depth -= ret;
- }
-}
-
/*
* Entry point for the thread based jobs. The process based jobs end up
* here as well, after a little setup.
goto err;
}
- if (td->flags & TD_F_COMPRESS_LOG)
- tp_init(&td->tp_data);
+ if (iolog_compress_init(td))
+ goto err;
fio_verify_init(td);
- if ((o->io_submit_mode == IO_MODE_OFFLOAD) &&
- workqueue_init(td, &td->io_wq, io_workqueue_fn, td->o.iodepth))
+ if (rate_submit_init(td))
goto err;
fio_gettime(&td->epoch, NULL);
if (td->o.verify_only && (td_write(td) || td_rw(td)))
verify_bytes = do_dry_run(td);
- else
+ else {
verify_bytes = do_io(td);
+ if (!verify_bytes)
+ fio_mark_td_terminate(td);
+ }
clear_state = 1;
fio_writeout_logs(td);
- if (o->io_submit_mode == IO_MODE_OFFLOAD)
- workqueue_exit(&td->io_wq);
-
- if (td->flags & TD_F_COMPRESS_LOG)
- tp_exit(&td->tp_data);
+ iolog_compress_exit(td);
+ rate_submit_exit(td);
if (o->exec_postrun)
exec_string(o, o->exec_postrun, (const char *)"postrun");
- if (exitall_on_terminate)
+ if (exitall_on_terminate || (o->exitall_error && td->error))
fio_terminate_threads(td->groupid);
err: