#include "workqueue.h"
#include "lib/mountcheck.h"
#include "rate-submit.h"
+#include "helper_thread.h"
static struct fio_mutex *startup_mutex;
static struct flist_head *cgroup_list;
int temp_stall_ts;
unsigned long done_secs = 0;
-static struct helper_data {
- volatile int exit;
- volatile int reset;
- volatile int do_stat;
- struct sk_out *sk_out;
- pthread_t thread;
- pthread_mutex_t lock;
- pthread_cond_t cond;
-} *helper_data;
-
#define PAGE_ALIGN(buf) \
(char *) (((uintptr_t) (buf) + page_mask) & ~page_mask)
int min_evts = 0;
int ret;
+ if (td->flags & TD_F_REGROW_LOGS) {
+ ret = io_u_quiesce(td);
+ regrow_logs(td);
+ return ret;
+ }
+
/*
* if the queue is full, we MUST reap at least 1 event
*/
break;
}
+ if (td->flags & TD_F_REGROW_LOGS)
+ regrow_logs(td);
+
/*
* when doing I/O (not when verifying),
* check for any errors that are to be ignored
return !td->o.iodepth_batch_complete_min &&
!td->o.iodepth_batch_complete_max;
}
+/*
+ * Unlinks files from thread data fio_file structure
+ */
+static int unlink_all_files(struct thread_data *td)
+{
+ struct fio_file *f;
+ unsigned int i;
+ int ret = 0;
+
+ for_each_file(td, f, i) {
+ if (f->filetype != FIO_TYPE_FILE)
+ continue;
+ ret = td_io_unlink_file(td, f);
+ if (ret)
+ break;
+ }
+
+ if (ret)
+ td_verror(td, ret, "unlink_all_files");
+
+ return ret;
+}
/*
* The main verify engine. Runs over the writes we previously submitted,
if (td->error)
return;
+ /*
+ * verify_state needs to be reset before verification
+ * proceeds so that expected random seeds match actual
+ * random seeds in headers. The main loop will reset
+ * all random number generators if randrepeat is set.
+ */
+ if (!td->o.rand_repeatable)
+ td_fill_verify_state_seed(td);
+
td_set_runstate(td, TD_VERIFYING);
io_u = NULL;
break;
while ((io_u = get_io_u(td)) != NULL) {
- if (IS_ERR(io_u)) {
+ if (IS_ERR_OR_NULL(io_u)) {
io_u = NULL;
ret = FIO_Q_BUSY;
goto reap;
continue;
} else if (io_u->ddir == DDIR_TRIM) {
io_u->ddir = DDIR_READ;
- io_u_set(io_u, IO_U_F_TRIMMED);
+ io_u_set(td, io_u, IO_U_F_TRIMMED);
break;
} else if (io_u->ddir == DDIR_WRITE) {
io_u->ddir = DDIR_READ;
if (ret < 0)
break;
if (!ddir_rw_sum(td->bytes_done) &&
- !(td->io_ops->flags & FIO_NOIO))
+ !td_ioengine_flagged(td, FIO_NOIO))
continue;
if (!in_ramp_time(td) && should_check_rate(td)) {
td->orig_buffer_size = (unsigned long long) max_bs
* (unsigned long long) max_units;
- if ((td->io_ops->flags & FIO_NOIO) || !(td_read(td) || td_write(td)))
+ if (td_ioengine_flagged(td, FIO_NOIO) || !(td_read(td) || td_write(td)))
data_xfer = 0;
err = 0;
* lucky and the allocator gives us an aligned address.
*/
if (td->o.odirect || td->o.mem_align || td->o.oatomic ||
- (td->io_ops->flags & FIO_RAWIO))
+ td_ioengine_flagged(td, FIO_RAWIO))
td->orig_buffer_size += page_mask + td->o.mem_align;
if (td->o.mem_type == MEM_SHMHUGE || td->o.mem_type == MEM_MMAPHUGE) {
return 1;
if (td->o.odirect || td->o.mem_align || td->o.oatomic ||
- (td->io_ops->flags & FIO_RAWIO))
+ td_ioengine_flagged(td, FIO_RAWIO))
p = PAGE_ALIGN(td->orig_buffer) + td->o.mem_align;
else
p = td->orig_buffer;
static int switch_ioscheduler(struct thread_data *td)
{
+#ifdef FIO_HAVE_IOSCHED_SWITCH
char tmp[256], tmp2[128];
FILE *f;
int ret;
- if (td->io_ops->flags & FIO_DISKLESSIO)
+ if (td_ioengine_flagged(td, FIO_DISKLESSIO))
return 0;
sprintf(tmp, "%s/queue/scheduler", td->sysfs_root);
*/
tmp[strlen(tmp) - 1] = '\0';
+ /*
+ * Write to "none" entry doesn't fail, so check the result here.
+ */
+ if (!strcmp(tmp, "none")) {
+ log_err("fio: io scheduler is not tunable\n");
+ fclose(f);
+ return 0;
+ }
sprintf(tmp2, "[%s]", td->o.ioscheduler);
if (!strstr(tmp, tmp2)) {
fclose(f);
return 0;
+#else
+ return 0;
+#endif
}
static bool keep_running(struct thread_data *td)
break;
io_u = get_io_u(td);
- if (!io_u)
+ if (IS_ERR_OR_NULL(io_u))
break;
- io_u_set(io_u, IO_U_F_FLIGHT);
+ io_u_set(td, io_u, IO_U_F_FLIGHT);
io_u->error = 0;
io_u->resid = 0;
if (ddir_rw(acct_ddir(io_u)))
struct thread_data *td = fd->td;
struct thread_options *o = &td->o;
struct sk_out *sk_out = fd->sk_out;
- pthread_condattr_t attr;
+ int deadlock_loop_cnt;
int clear_state;
int ret;
INIT_FLIST_HEAD(&td->verify_list);
INIT_FLIST_HEAD(&td->trim_list);
INIT_FLIST_HEAD(&td->next_rand_list);
- pthread_mutex_init(&td->io_u_lock, NULL);
td->io_hist_tree = RB_ROOT;
- pthread_condattr_init(&attr);
- pthread_cond_init(&td->verify_cond, &attr);
- pthread_cond_init(&td->free_cond, &attr);
+ ret = mutex_cond_init_pshared(&td->io_u_lock, &td->free_cond);
+ if (ret) {
+ td_verror(td, ret, "mutex_cond_init_pshared");
+ goto err;
+ }
+ ret = cond_init_pshared(&td->verify_cond);
+ if (ret) {
+ td_verror(td, ret, "mutex_cond_pshared");
+ goto err;
+ }
td_set_runstate(td, TD_INITIALIZED);
dprint(FD_MUTEX, "up startup_mutex\n");
goto err;
}
+ /*
+ * Do this early, we don't want the compress threads to be limited
+ * to the same CPUs as the IO workers. So do this before we set
+ * any potential CPU affinity
+ */
+ if (iolog_compress_init(td, sk_out))
+ goto err;
+
/*
* If we have a gettimeofday() thread, make sure we exclude that
* thread from this job
goto err;
}
- if (iolog_compress_init(td, sk_out))
- goto err;
-
fio_verify_init(td);
if (rate_submit_init(td, sk_out))
goto err;
- fio_gettime(&td->epoch, NULL);
+ set_epoch_time(td, o->log_unix_epoch);
fio_getrusage(&td->ru_start);
memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch));
memcpy(&td->iops_sample_time, &td->epoch, sizeof(td->epoch));
fio_gettime(&td->start, NULL);
memcpy(&td->tv_cache, &td->start, sizeof(td->start));
- if (clear_state)
+ if (clear_state) {
clear_io_state(td, 0);
+ if (o->unlink_each_loop && unlink_all_files(td))
+ break;
+ }
+
prune_io_piece_log(td);
if (td->o.verify_only && (td_write(td) || td_rw(td)))
}
}
+ /*
+ * If we took too long to shut down, the main thread could
+ * already consider us reaped/exited. If that happens, break
+ * out and clean up.
+ */
+ if (td->runstate >= TD_EXITED)
+ break;
+
clear_state = 1;
/*
* the rusage_sem, which would never get upped because
* this thread is waiting for the stat mutex.
*/
- check_update_rusage(td);
+ deadlock_loop_cnt = 0;
+ do {
+ check_update_rusage(td);
+ if (!fio_mutex_down_trylock(stat_mutex))
+ break;
+ usleep(1000);
+ if (deadlock_loop_cnt++ > 5000) {
+ log_err("fio seems to be stuck grabbing stat_mutex, forcibly exiting\n");
+ td->error = EDEADLK;
+ goto err;
+ }
+ } while (1);
- fio_mutex_down(stat_mutex);
if (td_read(td) && td->io_bytes[DDIR_READ])
update_runtime(td, elapsed_us, DDIR_READ);
if (td_write(td) && td->io_bytes[DDIR_WRITE])
if (!o->do_verify ||
o->verify == VERIFY_NONE ||
- (td->io_ops->flags & FIO_UNIDIR))
+ td_ioengine_flagged(td, FIO_UNIDIR))
continue;
clear_io_state(td, 0);
break;
}
+ td_set_runstate(td, TD_FINISHING);
+
update_rusage_stat(td);
td->ts.total_run_time = mtime_since_now(&td->epoch);
td->ts.io_bytes[DDIR_READ] = td->io_bytes[DDIR_READ];
return (void *) (uintptr_t) td->error;
}
-
-/*
- * We cannot pass the td data into a forked process, so attach the td and
- * pass it to the thread worker.
- */
-static int fork_main(struct sk_out *sk_out, int shmid, int offset)
-{
- struct fork_data *fd;
- void *data, *ret;
-
-#if !defined(__hpux) && !defined(CONFIG_NO_SHM)
- data = shmat(shmid, NULL, 0);
- if (data == (void *) -1) {
- int __err = errno;
-
- perror("shmat");
- return __err;
- }
-#else
- /*
- * HP-UX inherits shm mappings?
- */
- data = threads;
-#endif
-
- fd = calloc(1, sizeof(*fd));
- fd->td = data + offset * sizeof(struct thread_data);
- fd->sk_out = sk_out;
- ret = thread_main(fd);
- shmdt(data);
- return (int) (uintptr_t) ret;
-}
-
static void dump_td_info(struct thread_data *td)
{
- log_err("fio: job '%s' hasn't exited in %lu seconds, it appears to "
- "be stuck. Doing forceful exit of this job.\n", td->o.name,
+ log_err("fio: job '%s' (state=%d) hasn't exited in %lu seconds, it "
+ "appears to be stuck. Doing forceful exit of this job.\n",
+ td->o.name, td->runstate,
(unsigned long) time_since_now(&td->terminate_time));
}
* move on.
*/
if (td->terminate &&
+ td->runstate < TD_FSYNCING &&
time_since_now(&td->terminate_time) >= FIO_REAP_TIMEOUT) {
dump_td_info(td);
td_set_runstate(td, TD_REAPED);
struct thread_data *map[REAL_MAX_JOBS];
struct timeval this_start;
int this_jobs = 0, left;
+ struct fork_data *fd;
/*
* create threads (TD_NOT_CREATED -> TD_CREATED)
map[this_jobs++] = td;
nr_started++;
+ fd = calloc(1, sizeof(*fd));
+ fd->td = td;
+ fd->sk_out = sk_out;
+
if (td->o.use_thread) {
- struct fork_data *fd;
int ret;
- fd = calloc(1, sizeof(*fd));
- fd->td = td;
- fd->sk_out = sk_out;
-
dprint(FD_PROCESS, "will pthread_create\n");
ret = pthread_create(&td->thread, NULL,
thread_main, fd);
dprint(FD_PROCESS, "will fork\n");
pid = fork();
if (!pid) {
- int ret = fork_main(sk_out, shm_id, i);
+ int ret;
+ ret = (int)(uintptr_t)thread_main(fd);
_exit(ret);
} else if (i == fio_debug_jobno)
*fio_debug_jobp = pid;
update_io_ticks();
}
-void helper_reset(void)
-{
- if (!helper_data)
- return;
-
- pthread_mutex_lock(&helper_data->lock);
-
- if (!helper_data->reset) {
- helper_data->reset = 1;
- pthread_cond_signal(&helper_data->cond);
- }
-
- pthread_mutex_unlock(&helper_data->lock);
-}
-
-void helper_do_stat(void)
-{
- if (!helper_data)
- return;
-
- pthread_mutex_lock(&helper_data->lock);
- helper_data->do_stat = 1;
- pthread_cond_signal(&helper_data->cond);
- pthread_mutex_unlock(&helper_data->lock);
-}
-
-bool helper_should_exit(void)
-{
- if (!helper_data)
- return true;
-
- return helper_data->exit;
-}
-
-static void wait_for_helper_thread_exit(void)
-{
- void *ret;
-
- pthread_mutex_lock(&helper_data->lock);
- helper_data->exit = 1;
- pthread_cond_signal(&helper_data->cond);
- pthread_mutex_unlock(&helper_data->lock);
-
- pthread_join(helper_data->thread, &ret);
-}
-
static void free_disk_util(void)
{
disk_util_prune_entries();
-
- pthread_cond_destroy(&helper_data->cond);
- pthread_mutex_destroy(&helper_data->lock);
- sfree(helper_data);
-}
-
-static void *helper_thread_main(void *data)
-{
- struct helper_data *hd = data;
- unsigned int msec_to_next_event, next_log;
- struct timeval tv, last_du;
- int ret = 0;
-
- sk_out_assign(hd->sk_out);
-
- gettimeofday(&tv, NULL);
- memcpy(&last_du, &tv, sizeof(tv));
-
- fio_mutex_up(startup_mutex);
-
- msec_to_next_event = DISK_UTIL_MSEC;
- while (!ret && !hd->exit) {
- struct timespec ts;
- struct timeval now;
- uint64_t since_du;
-
- timeval_add_msec(&tv, msec_to_next_event);
- ts.tv_sec = tv.tv_sec;
- ts.tv_nsec = tv.tv_usec * 1000;
-
- pthread_mutex_lock(&hd->lock);
- pthread_cond_timedwait(&hd->cond, &hd->lock, &ts);
-
- gettimeofday(&now, NULL);
-
- if (hd->reset) {
- memcpy(&tv, &now, sizeof(tv));
- memcpy(&last_du, &now, sizeof(last_du));
- hd->reset = 0;
- }
-
- pthread_mutex_unlock(&hd->lock);
-
- since_du = mtime_since(&last_du, &now);
- if (since_du >= DISK_UTIL_MSEC || DISK_UTIL_MSEC - since_du < 10) {
- ret = update_io_ticks();
- timeval_add_msec(&last_du, DISK_UTIL_MSEC);
- msec_to_next_event = DISK_UTIL_MSEC;
- if (since_du >= DISK_UTIL_MSEC)
- msec_to_next_event -= (since_du - DISK_UTIL_MSEC);
- } else {
- if (since_du >= DISK_UTIL_MSEC)
- msec_to_next_event = DISK_UTIL_MSEC - (DISK_UTIL_MSEC - since_du);
- else
- msec_to_next_event = DISK_UTIL_MSEC;
- }
-
- if (hd->do_stat) {
- hd->do_stat = 0;
- __show_running_run_stats();
- }
-
- next_log = calc_log_samples();
- if (!next_log)
- next_log = DISK_UTIL_MSEC;
-
- msec_to_next_event = min(next_log, msec_to_next_event);
-
- if (!is_backend)
- print_thread_status();
- }
-
- fio_writeout_logs(false);
-
- sk_out_drop();
- return NULL;
-}
-
-static int create_helper_thread(struct sk_out *sk_out)
-{
- struct helper_data *hd;
- int ret;
-
- hd = smalloc(sizeof(*hd));
-
- setup_disk_util();
-
- hd->sk_out = sk_out;
- pthread_cond_init(&hd->cond, NULL);
- pthread_mutex_init(&hd->lock, NULL);
-
- ret = pthread_create(&hd->thread, NULL, helper_thread_main, hd);
- if (ret) {
- log_err("Can't create helper thread: %s\n", strerror(ret));
- return 1;
- }
-
- helper_data = hd;
-
- dprint(FD_MUTEX, "wait on startup_mutex\n");
- fio_mutex_down(startup_mutex);
- dprint(FD_MUTEX, "done waiting on startup_mutex\n");
- return 0;
+ helper_thread_destroy();
}
int fio_backend(struct sk_out *sk_out)
set_genesis_time();
stat_init();
- create_helper_thread(sk_out);
+ helper_thread_create(startup_mutex, sk_out);
cgroup_list = smalloc(sizeof(*cgroup_list));
INIT_FLIST_HEAD(cgroup_list);
run_threads(sk_out);
- wait_for_helper_thread_exit();
+ helper_thread_exit();
if (!fio_abort) {
__show_run_stats();
for (i = 0; i < DDIR_RWDIR_CNT; i++) {
struct io_log *log = agg_io_log[i];
- flush_log(log, 0);
+ flush_log(log, false);
free_log(log);
}
}
projects / fio.git / blobdiff