#include "helper_thread.h"
#include "pshared.h"
#include "zone-dist.h"
+#include "fio_time.h"
static struct fio_sem *startup_sem;
static struct flist_head *cgroup_list;
#ifdef WIN32
static void sig_break(int sig)
{
- struct thread_data *td;
- int i;
-
sig_int(sig);
/**
* Windows terminates all job processes on SIGBREAK after the handler
* returns, so give them time to wrap-up and give stats
*/
- for_each_td(td, i) {
+ for_each_td(td) {
while (td->runstate < TD_EXITED)
sleep(1);
- }
+ } end_for_each();
}
#endif
if (!from_verify)
unlog_io_piece(td, io_u);
td_verror(td, EIO, "full resid");
- put_io_u(td, io_u);
+ clear_io_u(td, io_u);
break;
}
if (td->error)
return;
- /*
- * verify_state needs to be reset before verification
- * proceeds so that expected random seeds match actual
- * random seeds in headers. The main loop will reset
- * all random number generators if randrepeat is set.
- */
- if (!td->o.rand_repeatable)
- td_fill_verify_state_seed(td);
-
td_set_runstate(td, TD_VERIFYING);
io_u = NULL;
break;
}
} else {
- if (ddir_rw_sum(td->bytes_done) + td->o.rw_min_bs > verify_bytes)
+ if (td->bytes_verified + td->o.rw_min_bs > verify_bytes)
break;
while ((io_u = get_io_u(td)) != NULL) {
break;
} else if (io_u->ddir == DDIR_WRITE) {
io_u->ddir = DDIR_READ;
+ io_u->numberio = td->verify_read_issues;
+ td->verify_read_issues++;
populate_verify_io_u(td, io_u);
break;
} else {
struct timespec *time)
{
unsigned long long b;
+ unsigned long long runtime_left;
uint64_t total;
int left;
struct timespec now;
if (td->o.thinktime_iotime) {
fio_gettime(&now, NULL);
if (utime_since(&td->last_thinktime, &now)
- >= td->o.thinktime_iotime + td->o.thinktime) {
+ >= td->o.thinktime_iotime) {
stall = true;
} else if (!fio_option_is_set(&td->o, thinktime_blocks)) {
/*
io_u_quiesce(td);
+ left = td->o.thinktime_spin;
+ if (td->o.timeout) {
+ runtime_left = td->o.timeout - utime_since_now(&td->epoch);
+ if (runtime_left < (unsigned long long)left)
+ left = runtime_left;
+ }
+
total = 0;
- if (td->o.thinktime_spin)
- total = usec_spin(td->o.thinktime_spin);
+ if (left)
+ total = usec_spin(left);
+
+ /*
+ * usec_spin() might run for slightly longer than intended in a VM
+ * where the vCPU could get descheduled or the hypervisor could steal
+ * CPU time. Ensure "left" doesn't become negative.
+ */
+ if (total < td->o.thinktime)
+ left = td->o.thinktime - total;
+ else
+ left = 0;
+
+ if (td->o.timeout) {
+ runtime_left = td->o.timeout - utime_since_now(&td->epoch);
+ if (runtime_left < (unsigned long long)left)
+ left = runtime_left;
+ }
- left = td->o.thinktime - total;
if (left)
total += usec_sleep(td, left);
fio_gettime(time, NULL);
td->last_thinktime_blocks = b;
- if (td->o.thinktime_iotime)
+ if (td->o.thinktime_iotime) {
+ fio_gettime(&now, NULL);
td->last_thinktime = now;
+ }
}
/*
*/
if (td_write(td) && td_random(td) && td->o.norandommap)
total_bytes = max(total_bytes, (uint64_t) td->o.io_size);
+
+ /* Don't break too early if io_size > size */
+ if (td_rw(td) && !td_random(td))
+ total_bytes = max(total_bytes, (uint64_t)td->o.io_size);
+
/*
* If verify_backlog is enabled, we'll run the verify in this
* handler as well. For that case, we may need up to twice the
break;
}
- if (io_u->ddir == DDIR_WRITE && td->flags & TD_F_DO_VERIFY)
- populate_verify_io_u(td, io_u);
+ if (io_u->ddir == DDIR_WRITE && td->flags & TD_F_DO_VERIFY) {
+ if (!(io_u->flags & IO_U_F_PATTERN_DONE)) {
+ io_u_set(td, io_u, IO_U_F_PATTERN_DONE);
+ io_u->numberio = td->io_issues[io_u->ddir];
+ populate_verify_io_u(td, io_u);
+ }
+ }
ddir = io_u->ddir;
if (ret < 0)
break;
+ if (ddir_rw(ddir) && td->o.thinkcycles)
+ cycles_spin(td->o.thinkcycles);
+
if (ddir_rw(ddir) && td->o.thinktime)
handle_thinktime(td, ddir, &comp_time);
}
}
- init_io_u_buffers(td);
+ if (init_io_u_buffers(td))
+ return 1;
if (init_file_completion_logging(td, max_units))
return 1;
int init_io_u_buffers(struct thread_data *td)
{
struct io_u *io_u;
- unsigned long long max_bs, min_write;
+ unsigned long long max_bs, min_write, trim_bs = 0;
int i, max_units;
int data_xfer = 1;
char *p;
td->orig_buffer_size = (unsigned long long) max_bs
* (unsigned long long) max_units;
- if (td_ioengine_flagged(td, FIO_NOIO) || !(td_read(td) || td_write(td)))
+ if (td_trim(td) && td->o.num_range > 1) {
+ trim_bs = td->o.num_range * sizeof(struct trim_range);
+ td->orig_buffer_size = trim_bs
+ * (unsigned long long) max_units;
+ }
+
+ /*
+ * For reads, writes, and multi-range trim operations we need a
+ * data buffer
+ */
+ if (td_ioengine_flagged(td, FIO_NOIO) ||
+ !(td_read(td) || td_write(td) || (td_trim(td) && td->o.num_range > 1)))
data_xfer = 0;
/*
* overflow later. this adjustment may be too much if we get
* lucky and the allocator gives us an aligned address.
*/
- if (td->o.odirect || td->o.mem_align || td->o.oatomic ||
+ if (td->o.odirect || td->o.mem_align ||
td_ioengine_flagged(td, FIO_RAWIO))
td->orig_buffer_size += page_mask + td->o.mem_align;
if (data_xfer && allocate_io_mem(td))
return 1;
- if (td->o.odirect || td->o.mem_align || td->o.oatomic ||
+ if (td->o.odirect || td->o.mem_align ||
td_ioengine_flagged(td, FIO_RAWIO))
p = PTR_ALIGN(td->orig_buffer, page_mask) + td->o.mem_align;
else
fill_verify_pattern(td, io_u->buf, max_bs, io_u, 0, 0);
}
}
- p += max_bs;
+ if (td_trim(td) && td->o.num_range > 1)
+ p += trim_bs;
+ else
+ p += max_bs;
}
return 0;
uint64_t bytes_done[DDIR_RWDIR_CNT];
int deadlock_loop_cnt;
bool clear_state;
- int res, ret;
+ int ret;
sk_out_assign(sk_out);
free(fd);
/* ioprio_set() has to be done before td_io_init() */
if (fio_option_is_set(o, ioprio) ||
- fio_option_is_set(o, ioprio_class)) {
- ret = ioprio_set(IOPRIO_WHO_PROCESS, 0, o->ioprio_class, o->ioprio);
+ fio_option_is_set(o, ioprio_class) ||
+ fio_option_is_set(o, ioprio_hint)) {
+ ret = ioprio_set(IOPRIO_WHO_PROCESS, 0, o->ioprio_class,
+ o->ioprio, o->ioprio_hint);
if (ret == -1) {
td_verror(td, errno, "ioprio_set");
goto err;
}
- td->ioprio = ioprio_value(o->ioprio_class, o->ioprio);
+ td->ioprio = ioprio_value(o->ioprio_class, o->ioprio,
+ o->ioprio_hint);
td->ts.ioprio = td->ioprio;
}
if (td_io_init(td))
goto err;
- if (td_ioengine_flagged(td, FIO_SYNCIO) && td->o.iodepth > 1) {
+ if (td_ioengine_flagged(td, FIO_SYNCIO) && td->o.iodepth > 1 && td->o.io_submit_mode != IO_MODE_OFFLOAD) {
log_info("note: both iodepth >= 1 and synchronous I/O engine "
"are selected, queue depth will be capped at 1\n");
}
if (rate_submit_init(td, sk_out))
goto err;
- set_epoch_time(td, o->log_unix_epoch | o->log_alternate_epoch, o->log_alternate_epoch_clock_id);
+ set_epoch_time(td, o->log_alternate_epoch_clock_id, o->job_start_clock_id);
fio_getrusage(&td->ru_start);
memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch));
memcpy(&td->iops_sample_time, &td->epoch, sizeof(td->epoch));
if (td->o.verify_only && td_write(td))
verify_bytes = do_dry_run(td);
else {
+ if (!td->o.rand_repeatable)
+ /* save verify rand state to replay hdr seeds later at verify */
+ frand_copy(&td->verify_state_last_do_io, &td->verify_state);
do_io(td, bytes_done);
-
+ if (!td->o.rand_repeatable)
+ frand_copy(&td->verify_state, &td->verify_state_last_do_io);
if (!ddir_rw_sum(bytes_done)) {
fio_mark_td_terminate(td);
verify_bytes = 0;
}
} while (1);
- if (td_read(td) && td->io_bytes[DDIR_READ])
+ if (td->io_bytes[DDIR_READ] && (td_read(td) ||
+ ((td->flags & TD_F_VER_BACKLOG) && td_write(td))))
update_runtime(td, elapsed_us, DDIR_READ);
if (td_write(td) && td->io_bytes[DDIR_WRITE])
update_runtime(td, elapsed_us, DDIR_WRITE);
* another thread is checking its io_u's for overlap
*/
if (td_offload_overlap(td)) {
- int res = pthread_mutex_lock(&overlap_check);
- assert(res == 0);
+ int res;
+
+ res = pthread_mutex_lock(&overlap_check);
+ if (res) {
+ td->error = errno;
+ goto err;
+ }
}
td_set_runstate(td, TD_FINISHING);
if (td_offload_overlap(td)) {
+ int res;
+
res = pthread_mutex_unlock(&overlap_check);
- assert(res == 0);
+ if (res) {
+ td->error = errno;
+ goto err;
+ }
}
update_rusage_stat(td);
static void reap_threads(unsigned int *nr_running, uint64_t *t_rate,
uint64_t *m_rate)
{
- struct thread_data *td;
unsigned int cputhreads, realthreads, pending;
- int i, status, ret;
+ int ret;
/*
* reap exited threads (TD_EXITED -> TD_REAPED)
*/
realthreads = pending = cputhreads = 0;
- for_each_td(td, i) {
- int flags = 0;
+ for_each_td(td) {
+ int flags = 0, status;
if (!strcmp(td->o.ioengine, "cpuio"))
cputhreads++;
done_secs += mtime_since_now(&td->epoch) / 1000;
profile_td_exit(td);
flow_exit_job(td);
- }
+ } end_for_each();
if (*nr_running == cputhreads && !pending && realthreads)
fio_terminate_threads(TERMINATE_ALL, TERMINATE_ALL);
{
const char *waitee = me->o.wait_for;
const char *self = me->o.name;
- struct thread_data *td;
- int i;
if (!waitee)
return false;
- for_each_td(td, i) {
+ for_each_td(td) {
if (!strcmp(td->o.name, self) || strcmp(td->o.name, waitee))
continue;
runstate_to_name(td->runstate));
return true;
}
- }
+ } end_for_each();
dprint(FD_PROCESS, "%s: %s completed, can run\n", self, waitee);
return false;
set_sig_handlers();
nr_thread = nr_process = 0;
- for_each_td(td, i) {
+ for_each_td(td) {
if (check_mount_writes(td))
return;
if (td->o.use_thread)
nr_thread++;
else
nr_process++;
- }
+ } end_for_each();
if (output_format & FIO_OUTPUT_NORMAL) {
struct buf_output out;
nr_started = 0;
m_rate = t_rate = 0;
- for_each_td(td, i) {
+ for_each_td(td) {
print_status_init(td->thread_number - 1);
if (!td->o.create_serialize)
td_io_close_file(td, f);
}
}
- }
+ } end_for_each();
/* start idle threads before io threads start to run */
fio_idle_prof_start();
/*
* create threads (TD_NOT_CREATED -> TD_CREATED)
*/
- for_each_td(td, i) {
+ for_each_td(td) {
if (td->runstate != TD_NOT_CREATED)
continue;
ret = (int)(uintptr_t)thread_main(fd);
_exit(ret);
- } else if (i == fio_debug_jobno)
+ } else if (__td_index == fio_debug_jobno)
*fio_debug_jobp = pid;
free(eo);
free(fd);
break;
}
dprint(FD_MUTEX, "done waiting on startup_sem\n");
- }
+ } end_for_each();
/*
* Wait for the started threads to transition to
/*
* start created threads (TD_INITIALIZED -> TD_RUNNING).
*/
- for_each_td(td, i) {
+ for_each_td(td) {
if (td->runstate != TD_INITIALIZED)
continue;
t_rate += ddir_rw_sum(td->o.rate);
todo--;
fio_sem_up(td->sem);
- }
+ } end_for_each();
reap_threads(&nr_running, &t_rate, &m_rate);
int fio_backend(struct sk_out *sk_out)
{
- struct thread_data *td;
int i;
-
if (exec_profile) {
if (load_profile(exec_profile))
return 1;
}
}
- for_each_td(td, i) {
+ for_each_td(td) {
struct thread_stat *ts = &td->ts;
free_clat_prio_stats(ts);
}
fio_sem_remove(td->sem);
td->sem = NULL;
- }
+ } end_for_each();
free_disk_util();
if (cgroup_list) {
projects / fio.git / blobdiff