X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=backend.c;h=2f46329376451f18a49c94696e84687dda4390d8;hp=180348f841267df45bb2c9d7852d3e70097545ba;hb=da92f848c5e0d29db6dda12c4c97bf1338a01128;hpb=182cab24328ab22a5b76ba1edac2c801b83aeffb diff --git a/backend.c b/backend.c index 180348f8..2f463293 100644 --- a/backend.c +++ b/backend.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "fio.h" #include "smalloc.h" @@ -47,12 +48,13 @@ #include "rate-submit.h" #include "helper_thread.h" #include "pshared.h" +#include "zone-dist.h" static struct fio_sem *startup_sem; static struct flist_head *cgroup_list; -static char *cgroup_mnt; +static struct cgroup_mnt *cgroup_mnt; static int exit_value; -static volatile int fio_abort; +static volatile bool fio_abort; static unsigned int nr_process = 0; static unsigned int nr_thread = 0; @@ -64,6 +66,7 @@ unsigned int stat_number = 0; int shm_id = 0; int temp_stall_ts; unsigned long done_secs = 0; +pthread_mutex_t overlap_check = PTHREAD_MUTEX_INITIALIZER; #define JOB_START_TIMEOUT (5 * 1000) @@ -234,6 +237,9 @@ static void cleanup_pending_aio(struct thread_data *td) { int r; + if (td->error) + return; + /* * get immediately available events, if any */ @@ -432,9 +438,7 @@ static int wait_for_completions(struct thread_data *td, struct timespec *time) if ((full && !min_evts) || !td->o.iodepth_batch_complete_min) min_evts = 1; - if (time && (__should_check_rate(td, DDIR_READ) || - __should_check_rate(td, DDIR_WRITE) || - __should_check_rate(td, DDIR_TRIM))) + if (time && __should_check_rate(td)) fio_gettime(time, NULL); do { @@ -456,14 +460,14 @@ int io_queue_event(struct thread_data *td, struct io_u *io_u, int *ret, *ret = -io_u->error; clear_io_u(td, io_u); } else if (io_u->resid) { - int bytes = io_u->xfer_buflen - io_u->resid; + long long bytes = io_u->xfer_buflen - io_u->resid; struct fio_file *f = io_u->file; if (bytes_issued) *bytes_issued += bytes; if (!from_verify) - trim_io_piece(td, io_u); + trim_io_piece(io_u); /* * zero read, fail @@ -489,9 +493,7 @@ int io_queue_event(struct thread_data *td, struct io_u *io_u, int *ret, requeue_io_u(td, &io_u); } else { sync_done: - if (comp_time && (__should_check_rate(td, DDIR_READ) || - __should_check_rate(td, DDIR_WRITE) || - __should_check_rate(td, DDIR_TRIM))) + if (comp_time && __should_check_rate(td)) fio_gettime(comp_time, NULL); *ret = io_u_sync_complete(td, io_u); @@ -570,7 +572,7 @@ static int unlink_all_files(struct thread_data *td) /* * Check if io_u will overlap an in-flight IO in the queue */ -static bool in_flight_overlap(struct io_u_queue *q, struct io_u *io_u) +bool in_flight_overlap(struct io_u_queue *q, struct io_u *io_u) { bool overlap; struct io_u *check_io_u; @@ -587,7 +589,7 @@ static bool in_flight_overlap(struct io_u_queue *q, struct io_u *io_u) if (x1 < y2 && y1 < x2) { overlap = true; - dprint(FD_IO, "in-flight overlap: %llu/%lu, %llu/%lu\n", + dprint(FD_IO, "in-flight overlap: %llu/%llu, %llu/%llu\n", x1, io_u->buflen, y1, check_io_u->buflen); break; @@ -970,8 +972,10 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done) * Break if we exceeded the bytes. The exception is time * based runs, but we still need to break out of the loop * for those to run verification, if enabled. + * Jobs read from iolog do not use this stop condition. */ if (bytes_issued >= total_bytes && + !td->o.read_iolog_file && (!td->o.time_based || (td->o.time_based && td->o.verify != VERIFY_NONE))) break; @@ -1037,8 +1041,8 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done) log_io_piece(td, io_u); if (td->o.io_submit_mode == IO_MODE_OFFLOAD) { - const unsigned long blen = io_u->xfer_buflen; - const enum fio_ddir ddir = acct_ddir(io_u); + const unsigned long long blen = io_u->xfer_buflen; + const enum fio_ddir __ddir = acct_ddir(io_u); if (td->error) break; @@ -1046,14 +1050,14 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done) workqueue_enqueue(&td->io_wq, &io_u->work); ret = FIO_Q_QUEUED; - if (ddir_rw(ddir)) { - td->io_issues[ddir]++; - td->io_issue_bytes[ddir] += blen; - td->rate_io_issue_bytes[ddir] += blen; + if (ddir_rw(__ddir)) { + td->io_issues[__ddir]++; + td->io_issue_bytes[__ddir] += blen; + td->rate_io_issue_bytes[__ddir] += blen; } if (should_check_rate(td)) - td->rate_next_io_time[ddir] = usec_for_io(td, ddir); + td->rate_next_io_time[__ddir] = usec_for_io(td, __ddir); } else { ret = io_u_submit(td, io_u); @@ -1188,14 +1192,14 @@ static void cleanup_io_u(struct thread_data *td) if (td->io_ops->io_u_free) td->io_ops->io_u_free(td, io_u); - fio_memfree(io_u, sizeof(*io_u)); + fio_memfree(io_u, sizeof(*io_u), td_offload_overlap(td)); } free_io_mem(td); io_u_rexit(&td->io_u_requeues); - io_u_qexit(&td->io_u_freelist); - io_u_qexit(&td->io_u_all); + io_u_qexit(&td->io_u_freelist, false); + io_u_qexit(&td->io_u_all, td_offload_overlap(td)); free_file_completion_logging(td); } @@ -1203,30 +1207,85 @@ static void cleanup_io_u(struct thread_data *td) static int init_io_u(struct thread_data *td) { struct io_u *io_u; - unsigned int max_bs, min_write; int cl_align, i, max_units; - int data_xfer = 1, err; - char *p; + int err; max_units = td->o.iodepth; - max_bs = td_max_bs(td); - min_write = td->o.min_bs[DDIR_WRITE]; - td->orig_buffer_size = (unsigned long long) max_bs - * (unsigned long long) max_units; - - if (td_ioengine_flagged(td, FIO_NOIO) || !(td_read(td) || td_write(td))) - data_xfer = 0; err = 0; err += !io_u_rinit(&td->io_u_requeues, td->o.iodepth); - err += !io_u_qinit(&td->io_u_freelist, td->o.iodepth); - err += !io_u_qinit(&td->io_u_all, td->o.iodepth); + err += !io_u_qinit(&td->io_u_freelist, td->o.iodepth, false); + err += !io_u_qinit(&td->io_u_all, td->o.iodepth, td_offload_overlap(td)); if (err) { log_err("fio: failed setting up IO queues\n"); return 1; } + cl_align = os_cache_line_size(); + + for (i = 0; i < max_units; i++) { + void *ptr; + + if (td->terminate) + return 1; + + ptr = fio_memalign(cl_align, sizeof(*io_u), td_offload_overlap(td)); + if (!ptr) { + log_err("fio: unable to allocate aligned memory\n"); + break; + } + + io_u = ptr; + memset(io_u, 0, sizeof(*io_u)); + INIT_FLIST_HEAD(&io_u->verify_list); + dprint(FD_MEM, "io_u alloc %p, index %u\n", io_u, i); + + io_u->index = i; + io_u->flags = IO_U_F_FREE; + io_u_qpush(&td->io_u_freelist, io_u); + + /* + * io_u never leaves this stack, used for iteration of all + * io_u buffers. + */ + io_u_qpush(&td->io_u_all, io_u); + + if (td->io_ops->io_u_init) { + int ret = td->io_ops->io_u_init(td, io_u); + + if (ret) { + log_err("fio: failed to init engine data: %d\n", ret); + return 1; + } + } + } + + init_io_u_buffers(td); + + if (init_file_completion_logging(td, max_units)) + return 1; + + return 0; +} + +int init_io_u_buffers(struct thread_data *td) +{ + struct io_u *io_u; + unsigned long long max_bs, min_write; + int i, max_units; + int data_xfer = 1; + char *p; + + max_units = td->o.iodepth; + max_bs = td_max_bs(td); + min_write = td->o.min_bs[DDIR_WRITE]; + td->orig_buffer_size = (unsigned long long) max_bs + * (unsigned long long) max_units; + + if (td_ioengine_flagged(td, FIO_NOIO) || !(td_read(td) || td_write(td))) + data_xfer = 0; + /* * if we may later need to do address alignment, then add any * possible adjustment here so that we don't cause a buffer @@ -1238,7 +1297,7 @@ static int init_io_u(struct thread_data *td) td->orig_buffer_size += page_mask + td->o.mem_align; if (td->o.mem_type == MEM_SHMHUGE || td->o.mem_type == MEM_MMAPHUGE) { - unsigned long bs; + unsigned long long bs; bs = td->orig_buffer_size + td->o.hugepage_size - 1; td->orig_buffer_size = bs & ~(td->o.hugepage_size - 1); @@ -1258,23 +1317,8 @@ static int init_io_u(struct thread_data *td) else p = td->orig_buffer; - cl_align = os_cache_line_size(); - for (i = 0; i < max_units; i++) { - void *ptr; - - if (td->terminate) - return 1; - - ptr = fio_memalign(cl_align, sizeof(*io_u)); - if (!ptr) { - log_err("fio: unable to allocate aligned memory\n"); - break; - } - - io_u = ptr; - memset(io_u, 0, sizeof(*io_u)); - INIT_FLIST_HEAD(&io_u->verify_list); + io_u = td->io_u_all.io_us[i]; dprint(FD_MEM, "io_u alloc %p, index %u\n", io_u, i); if (data_xfer) { @@ -1291,32 +1335,9 @@ static int init_io_u(struct thread_data *td) fill_verify_pattern(td, io_u->buf, max_bs, io_u, 0, 0); } } - - io_u->index = i; - io_u->flags = IO_U_F_FREE; - io_u_qpush(&td->io_u_freelist, io_u); - - /* - * io_u never leaves this stack, used for iteration of all - * io_u buffers. - */ - io_u_qpush(&td->io_u_all, io_u); - - if (td->io_ops->io_u_init) { - int ret = td->io_ops->io_u_init(td, io_u); - - if (ret) { - log_err("fio: failed to init engine data: %d\n", ret); - return 1; - } - } - p += max_bs; } - if (init_file_completion_logging(td, max_units)) - return 1; - return 0; } @@ -1521,7 +1542,7 @@ static void *thread_main(void *data) struct sk_out *sk_out = fd->sk_out; uint64_t bytes_done[DDIR_RWDIR_CNT]; int deadlock_loop_cnt; - bool clear_state, did_some_io; + bool clear_state; int ret; sk_out_assign(sk_out); @@ -1533,7 +1554,7 @@ static void *thread_main(void *data) } else td->pid = gettid(); - fio_local_clock_init(o->use_thread); + fio_local_clock_init(); dprint(FD_PROCESS, "jobs pid=%d started\n", (int) td->pid); @@ -1577,6 +1598,8 @@ static void *thread_main(void *data) goto err; } + td_zone_gen_index(td); + /* * Do this early, we don't want the compress threads to be limited * to the same CPUs as the IO workers. So do this before we set @@ -1675,9 +1698,15 @@ static void *thread_main(void *data) if (!init_iolog(td)) goto err; + if (td_io_init(td)) + goto err; + if (init_io_u(td)) goto err; + if (td->io_ops->post_init && td->io_ops->post_init(td)) + goto err; + if (o->verify_async && verify_async_init(td)) goto err; @@ -1705,9 +1734,6 @@ static void *thread_main(void *data) if (!o->create_serialize && setup_files(td)) goto err; - if (td_io_init(td)) - goto err; - if (!init_random_map(td)) goto err; @@ -1740,7 +1766,6 @@ static void *thread_main(void *data) memset(bytes_done, 0, sizeof(bytes_done)); clear_state = false; - did_some_io = false; while (keep_running(td)) { uint64_t verify_bytes; @@ -1818,9 +1843,6 @@ static void *thread_main(void *data) td_ioengine_flagged(td, FIO_UNIDIR)) continue; - if (ddir_rw_sum(bytes_done)) - did_some_io = true; - clear_io_state(td, 0); fio_gettime(&td->start, NULL); @@ -1842,19 +1864,15 @@ static void *thread_main(void *data) } /* - * If td ended up with no I/O when it should have had, - * then something went wrong unless FIO_NOIO or FIO_DISKLESSIO. - * (Are we not missing other flags that can be ignored ?) + * Acquire this lock if we were doing overlap checking in + * offload mode so that we don't clean up this job while + * another thread is checking its io_u's for overlap */ - if ((td->o.size || td->o.io_size) && !ddir_rw_sum(bytes_done) && - !did_some_io && !td->o.create_only && - !(td_ioengine_flagged(td, FIO_NOIO) || - td_ioengine_flagged(td, FIO_DISKLESSIO))) - log_err("%s: No I/O performed by %s, " - "perhaps try --debug=io option for details?\n", - td->o.name, td->io_ops->name); - + if (td_offload_overlap(td)) + pthread_mutex_lock(&overlap_check); td_set_runstate(td, TD_FINISHING); + if (td_offload_overlap(td)) + pthread_mutex_unlock(&overlap_check); update_rusage_stat(td); td->ts.total_run_time = mtime_since_now(&td->epoch); @@ -1890,17 +1908,9 @@ err: close_and_free_files(td); cleanup_io_u(td); close_ioengine(td); - cgroup_shutdown(td, &cgroup_mnt); + cgroup_shutdown(td, cgroup_mnt); verify_free_state(td); - - if (td->zone_state_index) { - int i; - - for (i = 0; i < DDIR_RWDIR_CNT; i++) - free(td->zone_state_index[i]); - free(td->zone_state_index); - td->zone_state_index = NULL; - } + td_zone_free_index(td); if (fio_option_is_set(o, cpumask)) { ret = fio_cpuset_exit(&o->cpumask); @@ -1913,6 +1923,8 @@ err: */ if (o->write_iolog_file) write_iolog_close(td); + if (td->io_log_rfile) + fclose(td->io_log_rfile); td_set_runstate(td, TD_EXITED); @@ -2201,18 +2213,22 @@ static void run_threads(struct sk_out *sk_out) } if (output_format & FIO_OUTPUT_NORMAL) { - log_info("Starting "); + struct buf_output out; + + buf_output_init(&out); + __log_buf(&out, "Starting "); if (nr_thread) - log_info("%d thread%s", nr_thread, + __log_buf(&out, "%d thread%s", nr_thread, nr_thread > 1 ? "s" : ""); if (nr_process) { if (nr_thread) - log_info(" and "); - log_info("%d process%s", nr_process, + __log_buf(&out, " and "); + __log_buf(&out, "%d process%s", nr_process, nr_process > 1 ? "es" : ""); } - log_info("\n"); - log_info_flush(); + __log_buf(&out, "\n"); + log_info_buf(out.buf, out.buflen); + buf_output_free(&out); } todo = thread_number; @@ -2355,7 +2371,7 @@ reap: if (fio_sem_down_timeout(startup_sem, 10000)) { log_err("fio: job startup hung? exiting.\n"); fio_terminate_threads(TERMINATE_ALL); - fio_abort = 1; + fio_abort = true; nr_started--; free(fd); break; @@ -2469,6 +2485,8 @@ int fio_backend(struct sk_out *sk_out) } startup_sem = fio_sem_init(FIO_SEM_LOCKED); + if (!sk_out) + is_local_backend = true; if (startup_sem == NULL) return 1; @@ -2512,7 +2530,6 @@ int fio_backend(struct sk_out *sk_out) cgroup_kill(cgroup_list); sfree(cgroup_list); } - sfree(cgroup_mnt); fio_sem_remove(startup_sem); stat_exit();