X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=fio.c;h=9a3cc94897d3c49928ef27f2dade7504c955f81a;hp=8d2a8e5f5ccc023ed32e8c16cccc72e683049890;hb=0be06ea2c87ecf1751a01ed528a2d3170bdca543;hpb=2f9ade3cb72218eb260f4f5e6914218bdcaa2bcc diff --git a/fio.c b/fio.c index 8d2a8e5f..9a3cc948 100644 --- a/fio.c +++ b/fio.c @@ -4,10 +4,12 @@ * Copyright (C) 2005 Jens Axboe * Copyright (C) 2006 Jens Axboe * + * The license below covers all files distributed with fio unless otherwise + * noted in the file itself. + * * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -24,35 +26,41 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include "fio.h" #include "os.h" -#define MASK (4095) - -#define ALIGN(buf) (char *) (((unsigned long) (buf) + MASK) & ~(MASK)) +static unsigned long page_mask; +#define ALIGN(buf) \ + (char *) (((unsigned long) (buf) + page_mask) & ~page_mask) int groupid = 0; int thread_number = 0; int shm_id = 0; int temp_stall_ts; -char *fio_inst_prefix = _INST_PREFIX; - -#define should_fsync(td) ((td_write(td) || td_rw(td)) && (!(td)->odirect || (td)->override_sync)) static volatile int startup_sem; +static volatile int fio_abort; +static int exit_value; + +struct io_log *agg_io_log[2]; #define TERMINATE_ALL (-1) #define JOB_START_TIMEOUT (5 * 1000) -static void terminate_threads(int group_id) +static inline void td_set_runstate(struct thread_data *td, int runstate) +{ + td->runstate = runstate; +} + +static void terminate_threads(int group_id, int forced_kill) { struct thread_data *td; int i; @@ -61,6 +69,8 @@ static void terminate_threads(int group_id) if (group_id == TERMINATE_ALL || groupid == td->groupid) { td->terminate = 1; td->start_delay = 0; + if (forced_kill) + td_set_runstate(td, TD_EXITED); } } } @@ -74,149 +84,27 @@ static void sig_handler(int sig) print_thread_status(); break; default: - printf("\nfio: terminating on signal\n"); + printf("\nfio: terminating on signal %d\n", sig); fflush(stdout); - terminate_threads(TERMINATE_ALL); + terminate_threads(TERMINATE_ALL, 0); break; } } -/* - * The ->file_map[] contains a map of blocks we have or have not done io - * to yet. Used to make sure we cover the entire range in a fair fashion. - */ -static int random_map_free(struct thread_data *td, struct fio_file *f, - unsigned long long block) -{ - unsigned int idx = RAND_MAP_IDX(td, f, block); - unsigned int bit = RAND_MAP_BIT(td, f, block); - - return (f->file_map[idx] & (1UL << bit)) == 0; -} - -/* - * Return the next free block in the map. - */ -static int get_next_free_block(struct thread_data *td, struct fio_file *f, - unsigned long long *b) -{ - int i; - - *b = 0; - i = 0; - while ((*b) * td->min_bs < f->file_size) { - if (f->file_map[i] != -1UL) { - *b += ffz(f->file_map[i]); - return 0; - } - - *b += BLOCKS_PER_MAP; - i++; - } - - return 1; -} - -/* - * Mark a given offset as used in the map. - */ -static void mark_random_map(struct thread_data *td, struct fio_file *f, - struct io_u *io_u) -{ - unsigned long long block = io_u->offset / (unsigned long long) td->min_bs; - unsigned int blocks = 0; - - while (blocks < (io_u->buflen / td->min_bs)) { - unsigned int idx, bit; - - if (!random_map_free(td, f, block)) - break; - - idx = RAND_MAP_IDX(td, f, block); - bit = RAND_MAP_BIT(td, f, block); - - assert(idx < f->num_maps); - - f->file_map[idx] |= (1UL << bit); - block++; - blocks++; - } - - if ((blocks * td->min_bs) < io_u->buflen) - io_u->buflen = blocks * td->min_bs; -} - -/* - * For random io, generate a random new block and see if it's used. Repeat - * until we find a free one. For sequential io, just return the end of - * the last io issued. - */ -static int get_next_offset(struct thread_data *td, struct fio_file *f, - unsigned long long *offset) -{ - unsigned long long b, rb; - long r; - - if (!td->sequential) { - unsigned long long max_blocks = td->io_size / td->min_bs; - int loops = 50; - - do { - r = os_random_long(&td->random_state); - b = ((max_blocks - 1) * r / (unsigned long long) (RAND_MAX+1.0)); - rb = b + (f->file_offset / td->min_bs); - loops--; - } while (!random_map_free(td, f, rb) && loops); - - if (!loops) { - if (get_next_free_block(td, f, &b)) - return 1; - } - } else - b = f->last_pos / td->min_bs; - - *offset = (b * td->min_bs) + f->file_offset; - if (*offset > f->file_size) - return 1; - - return 0; -} - -static unsigned int get_next_buflen(struct thread_data *td) -{ - unsigned int buflen; - long r; - - if (td->min_bs == td->max_bs) - buflen = td->min_bs; - else { - r = os_random_long(&td->bsrange_state); - buflen = (1 + (double) (td->max_bs - 1) * r / (RAND_MAX + 1.0)); - buflen = (buflen + td->min_bs - 1) & ~(td->min_bs - 1); - } - - if (buflen > td->io_size - td->this_io_bytes[td->ddir]) { - /* - * if using direct/raw io, we may not be able to - * shrink the size. so just fail it. - */ - if (td->io_ops->flags & FIO_RAWIO) - return 0; - - buflen = td->io_size - td->this_io_bytes[td->ddir]; - } - - return buflen; -} - /* * Check if we are above the minimum rate given. */ static int check_min_rate(struct thread_data *td, struct timeval *now) { + unsigned long long bytes = 0; unsigned long spent; unsigned long rate; - int ddir = td->ddir; + + /* + * No minimum rate set, always ok + */ + if (!td->ratemin) + return 0; /* * allow a 2 second settle period in the beginning @@ -224,6 +112,11 @@ static int check_min_rate(struct thread_data *td, struct timeval *now) if (mtime_since(&td->start, now) < 2000) return 0; + if (td_read(td)) + bytes += td->this_io_bytes[DDIR_READ]; + if (td_write(td)) + bytes += td->this_io_bytes[DDIR_WRITE]; + /* * if rate blocks is set, sample is running */ @@ -232,14 +125,19 @@ static int check_min_rate(struct thread_data *td, struct timeval *now) if (spent < td->ratecycle) return 0; - rate = (td->this_io_bytes[ddir] - td->rate_bytes) / spent; - if (rate < td->ratemin) { - fprintf(f_out, "%s: min rate %d not met, got %ldKiB/sec\n", td->name, td->ratemin, rate); + if (bytes < td->rate_bytes) { + fprintf(f_out, "%s: min rate %u not met\n", td->name, td->ratemin); return 1; + } else { + rate = (bytes - td->rate_bytes) / spent; + if (rate < td->ratemin || bytes < td->rate_bytes) { + fprintf(f_out, "%s: min rate %u not met, got %luKiB/sec\n", td->name, td->ratemin, rate); + return 1; + } } } - td->rate_bytes = td->this_io_bytes[ddir]; + td->rate_bytes = bytes; memcpy(&td->lastrate, now, sizeof(*now)); return 0; } @@ -255,394 +153,202 @@ static inline int runtime_exceeded(struct thread_data *td, struct timeval *t) } /* - * Return the data direction for the next io_u. If the job is a - * mixed read/write workload, check the rwmix cycle and switch if - * necessary. + * When job exits, we can cancel the in-flight IO if we are using async + * io. Attempt to do so. */ -static int get_rw_ddir(struct thread_data *td) -{ - if (td_rw(td)) { - struct timeval now; - unsigned long elapsed; - - gettimeofday(&now, NULL); - elapsed = mtime_since_now(&td->rwmix_switch); - - /* - * Check if it's time to seed a new data direction. - */ - if (elapsed >= td->rwmixcycle) { - unsigned int v; - long r; - - r = os_random_long(&td->rwmix_state); - v = 1 + (int) (100.0 * (r / (RAND_MAX + 1.0))); - if (v < td->rwmixread) - td->rwmix_ddir = DDIR_READ; - else - td->rwmix_ddir = DDIR_WRITE; - memcpy(&td->rwmix_switch, &now, sizeof(now)); - } - return td->rwmix_ddir; - } else if (td_read(td)) - return DDIR_READ; - else - return DDIR_WRITE; -} - -static int td_io_prep(struct thread_data *td, struct io_u *io_u) -{ - if (td->io_ops->prep && td->io_ops->prep(td, io_u)) - return 1; - - return 0; -} - -void put_io_u(struct thread_data *td, struct io_u *io_u) +static void cleanup_pending_aio(struct thread_data *td) { - io_u->file = NULL; - list_del(&io_u->list); - list_add(&io_u->list, &td->io_u_freelist); - td->cur_depth--; -} + struct list_head *entry, *n; + struct io_u *io_u; + int r; -static int fill_io_u(struct thread_data *td, struct fio_file *f, - struct io_u *io_u) -{ /* - * If using an iolog, grab next piece if any available. + * get immediately available events, if any */ - if (td->read_iolog) - return read_iolog_get(td, io_u); + r = io_u_queued_complete(td, 0); + if (r < 0) + return; /* - * No log, let the seq/rand engine retrieve the next position. + * now cancel remaining active events */ - if (!get_next_offset(td, f, &io_u->offset)) { - io_u->buflen = get_next_buflen(td); - - if (io_u->buflen) { - io_u->ddir = get_rw_ddir(td); + if (td->io_ops->cancel) { + list_for_each_safe(entry, n, &td->io_u_busylist) { + io_u = list_entry(entry, struct io_u, list); /* - * If using a write iolog, store this entry. + * if the io_u isn't in flight, then that generally + * means someone leaked an io_u. complain but fix + * it up, so we don't stall here. */ - if (td->write_iolog) - write_iolog_put(td, io_u); - - io_u->file = f; - return 0; + if ((io_u->flags & IO_U_F_FLIGHT) == 0) { + log_err("fio: non-busy IO on busy list\n"); + put_io_u(td, io_u); + } else { + r = td->io_ops->cancel(td, io_u); + if (!r) + put_io_u(td, io_u); + } } } - return 1; -} - -#define queue_full(td) list_empty(&(td)->io_u_freelist) - -struct io_u *__get_io_u(struct thread_data *td) -{ - struct io_u *io_u = NULL; - - if (!queue_full(td)) { - io_u = list_entry(td->io_u_freelist.next, struct io_u, list); - - io_u->error = 0; - io_u->resid = 0; - list_del(&io_u->list); - list_add(&io_u->list, &td->io_u_busylist); - td->cur_depth++; - } - - return io_u; + if (td->cur_depth) + r = io_u_queued_complete(td, td->cur_depth); } /* - * Return an io_u to be processed. Gets a buflen and offset, sets direction, - * etc. The returned io_u is fully ready to be prepped and submitted. + * Helper to handle the final sync of a file. Works just like the normal + * io path, just does everything sync. */ -static struct io_u *get_io_u(struct thread_data *td, struct fio_file *f) +static int fio_io_sync(struct thread_data *td, struct fio_file *f) { - struct io_u *io_u; + struct io_u *io_u = __get_io_u(td); + int ret; - io_u = __get_io_u(td); if (!io_u) - return NULL; - - if (td->zone_bytes >= td->zone_size) { - td->zone_bytes = 0; - f->last_pos += td->zone_skip; - } - - if (fill_io_u(td, f, io_u)) { - put_io_u(td, io_u); - return NULL; - } - - if (io_u->buflen + io_u->offset > f->file_size) { - if (td->io_ops->flags & FIO_RAWIO) { - put_io_u(td, io_u); - return NULL; - } - - io_u->buflen = f->file_size - io_u->offset; - } - - if (!io_u->buflen) { - put_io_u(td, io_u); - return NULL; - } - - if (!td->read_iolog && !td->sequential) - mark_random_map(td, f, io_u); - - f->last_pos += io_u->buflen; + return 1; - if (td->verify != VERIFY_NONE) - populate_verify_io_u(td, io_u); + io_u->ddir = DDIR_SYNC; + io_u->file = f; if (td_io_prep(td, io_u)) { put_io_u(td, io_u); - return NULL; + return 1; } - gettimeofday(&io_u->start_time, NULL); - return io_u; -} - -static inline void td_set_runstate(struct thread_data *td, int runstate) -{ - td->runstate = runstate; -} - -static struct fio_file *get_next_file(struct thread_data *td) -{ - unsigned int old_next_file = td->next_file; - struct fio_file *f; - - do { - f = &td->files[td->next_file]; - - td->next_file++; - if (td->next_file >= td->nr_files) - td->next_file = 0; - - if (f->fd != -1) - break; - - f = NULL; - } while (td->next_file != old_next_file); - - return f; -} - -static int td_io_sync(struct thread_data *td, struct fio_file *f) -{ - if (td->io_ops->sync) - return td->io_ops->sync(td, f); - - return 0; -} - -static int td_io_getevents(struct thread_data *td, int min, int max, - struct timespec *t) -{ - return td->io_ops->getevents(td, min, max, t); -} - -static int td_io_queue(struct thread_data *td, struct io_u *io_u) -{ - gettimeofday(&io_u->issue_time, NULL); - - return td->io_ops->queue(td, io_u); -} - -#define iocb_time(iocb) ((unsigned long) (iocb)->data) - -static void io_completed(struct thread_data *td, struct io_u *io_u, - struct io_completion_data *icd) -{ - struct timeval e; - unsigned long msec; - - gettimeofday(&e, NULL); - - if (!io_u->error) { - unsigned int bytes = io_u->buflen - io_u->resid; - const int idx = io_u->ddir; - - td->io_blocks[idx]++; - td->io_bytes[idx] += bytes; - td->zone_bytes += bytes; - td->this_io_bytes[idx] += bytes; - - msec = mtime_since(&io_u->issue_time, &e); - - add_clat_sample(td, idx, msec); - add_bw_sample(td, idx); - - if ((td_rw(td) || td_write(td)) && idx == DDIR_WRITE) - log_io_piece(td, io_u); - - icd->bytes_done[idx] += bytes; - } else - icd->error = io_u->error; -} - -static void ios_completed(struct thread_data *td,struct io_completion_data *icd) -{ - struct io_u *io_u; - int i; - - icd->error = 0; - icd->bytes_done[0] = icd->bytes_done[1] = 0; - - for (i = 0; i < icd->nr; i++) { - io_u = td->io_ops->event(td, i); - - io_completed(td, io_u, icd); +requeue: + ret = td_io_queue(td, io_u); + if (ret < 0) { + td_verror(td, io_u->error, "td_io_queue"); put_io_u(td, io_u); - } -} - -/* - * When job exits, we can cancel the in-flight IO if we are using async - * io. Attempt to do so. - */ -static void cleanup_pending_aio(struct thread_data *td) -{ - struct timespec ts = { .tv_sec = 0, .tv_nsec = 0}; - struct list_head *entry, *n; - struct io_completion_data icd; - struct io_u *io_u; - int r; - - /* - * get immediately available events, if any - */ - r = td_io_getevents(td, 0, td->cur_depth, &ts); - if (r > 0) { - icd.nr = r; - ios_completed(td, &icd); - } - - /* - * now cancel remaining active events - */ - if (td->io_ops->cancel) { - list_for_each_safe(entry, n, &td->io_u_busylist) { - io_u = list_entry(entry, struct io_u, list); - - r = td->io_ops->cancel(td, io_u); - if (!r) - put_io_u(td, io_u); + return 1; + } else if (ret == FIO_Q_QUEUED) { + if (io_u_queued_complete(td, 1) < 0) + return 1; + } else if (ret == FIO_Q_COMPLETED) { + if (io_u->error) { + td_verror(td, io_u->error, "td_io_queue"); + return 1; } - } - if (td->cur_depth) { - r = td_io_getevents(td, td->cur_depth, td->cur_depth, NULL); - if (r > 0) { - icd.nr = r; - ios_completed(td, &icd); - } + if (io_u_sync_complete(td, io_u) < 0) + return 1; + } else if (ret == FIO_Q_BUSY) { + if (td_io_commit(td)) + return 1; + goto requeue; } + + return 0; } /* * The main verify engine. Runs over the writes we previusly submitted, * reads the blocks back in, and checks the crc/md5 of the data. */ -void do_verify(struct thread_data *td) +static void do_verify(struct thread_data *td) { - struct timeval t; - struct io_u *io_u, *v_io_u = NULL; - struct io_completion_data icd; struct fio_file *f; - int ret, i; + struct io_u *io_u; + int ret, i, min_events; /* * sync io first and invalidate cache, to make sure we really * read from disk. */ for_each_file(td, f, i) { - td_io_sync(td, f); - file_invalidate_cache(td, f); + if (fio_io_sync(td, f)) + break; + if (file_invalidate_cache(td, f)) + break; } - td_set_runstate(td, TD_VERIFYING); - - do { - if (td->terminate) - break; + if (td->error) + return; - gettimeofday(&t, NULL); - if (runtime_exceeded(td, &t)) - break; + td_set_runstate(td, TD_VERIFYING); + io_u = NULL; + while (!td->terminate) { io_u = __get_io_u(td); if (!io_u) break; - if (get_next_verify(td, io_u)) { + if (runtime_exceeded(td, &io_u->start_time)) { put_io_u(td, io_u); break; } - f = get_next_file(td); - if (!f) + if (get_next_verify(td, io_u)) { + put_io_u(td, io_u); break; - - io_u->file = f; + } if (td_io_prep(td, io_u)) { put_io_u(td, io_u); break; } + io_u->end_io = verify_io_u; +requeue: ret = td_io_queue(td, io_u); - if (ret) { - put_io_u(td, io_u); - td_verror(td, ret); - break; - } - /* - * we have one pending to verify, do that while - * we are doing io on the next one - */ - if (do_io_u_verify(td, &v_io_u)) - break; + switch (ret) { + case FIO_Q_COMPLETED: + if (io_u->error) + ret = -io_u->error; + if (io_u->xfer_buflen != io_u->resid && io_u->resid) { + int bytes = io_u->xfer_buflen - io_u->resid; - ret = td_io_getevents(td, 1, 1, NULL); - if (ret != 1) { + io_u->xfer_buflen = io_u->resid; + io_u->xfer_buf += bytes; + goto requeue; + } + ret = io_u_sync_complete(td, io_u); if (ret < 0) - td_verror(td, ret); + break; + continue; + case FIO_Q_QUEUED: + break; + case FIO_Q_BUSY: + requeue_io_u(td, &io_u); + ret = td_io_commit(td); + break; + default: + assert(ret < 0); + td_verror(td, -ret, "td_io_queue"); break; } - v_io_u = td->io_ops->event(td, 0); - icd.nr = 1; - icd.error = 0; - io_completed(td, v_io_u, &icd); - - if (icd.error) { - td_verror(td, icd.error); - put_io_u(td, v_io_u); - v_io_u = NULL; + if (ret < 0 || td->error) break; + + /* + * if we can queue more, do so. but check if there are + * completed io_u's first. + */ + min_events = 0; + if (queue_full(td) || ret == FIO_Q_BUSY) { + min_events = 1; + + if (td->cur_depth > td->iodepth_low) + min_events = td->cur_depth - td->iodepth_low; } /* - * if we can't submit more io, we need to verify now + * Reap required number of io units, if any, and do the + * verification on them through the callback handler */ - if (queue_full(td) && do_io_u_verify(td, &v_io_u)) + if (io_u_queued_complete(td, min_events) < 0) break; + } - } while (1); - - do_io_u_verify(td, &v_io_u); + if (!td->error) { + min_events = td->cur_depth; - if (td->cur_depth) + if (min_events) + ret = io_u_queued_complete(td, min_events); + } else cleanup_pending_aio(td); td_set_runstate(td, TD_RUNNING); @@ -659,7 +365,7 @@ static void do_cpuio(struct thread_data *td) int i = 0; while (!td->terminate) { - gettimeofday(&e, NULL); + fio_gettime(&e, NULL); if (runtime_exceeded(td, &e)) break; @@ -679,111 +385,142 @@ static void do_cpuio(struct thread_data *td) */ static void do_io(struct thread_data *td) { - struct io_completion_data icd; - struct timeval s, e; + struct timeval s; unsigned long usec; - struct fio_file *f; int i, ret = 0; td_set_runstate(td, TD_RUNNING); - while (td->this_io_bytes[td->ddir] < td->io_size) { - struct timespec ts = { .tv_sec = 0, .tv_nsec = 0}; - struct timespec *timeout; + while ((td->this_io_bytes[0] + td->this_io_bytes[1]) < td->io_size) { + struct timeval comp_time; + long bytes_done = 0; int min_evts = 0; struct io_u *io_u; if (td->terminate) break; - f = get_next_file(td); - if (!f) - break; - - io_u = get_io_u(td, f); + io_u = get_io_u(td); if (!io_u) break; memcpy(&s, &io_u->start_time, sizeof(s)); - ret = td_io_queue(td, io_u); - if (ret) { + if (runtime_exceeded(td, &s)) { put_io_u(td, io_u); - td_verror(td, ret); break; } +requeue: + ret = td_io_queue(td, io_u); - add_slat_sample(td, io_u->ddir, mtime_since(&io_u->start_time, &io_u->issue_time)); + switch (ret) { + case FIO_Q_COMPLETED: + if (io_u->error) { + ret = io_u->error; + break; + } + if (io_u->xfer_buflen != io_u->resid && io_u->resid) { + int bytes = io_u->xfer_buflen - io_u->resid; - if (td->cur_depth < td->iodepth) { - timeout = &ts; - min_evts = 0; - } else { - timeout = NULL; - min_evts = 1; + io_u->xfer_buflen = io_u->resid; + io_u->xfer_buf += bytes; + goto requeue; + } + fio_gettime(&comp_time, NULL); + bytes_done = io_u_sync_complete(td, io_u); + if (bytes_done < 0) + ret = bytes_done; + break; + case FIO_Q_QUEUED: + /* + * if the engine doesn't have a commit hook, + * the io_u is really queued. if it does have such + * a hook, it has to call io_u_queued() itself. + */ + if (td->io_ops->commit == NULL) + io_u_queued(td, io_u); + break; + case FIO_Q_BUSY: + requeue_io_u(td, &io_u); + ret = td_io_commit(td); + break; + default: + assert(ret < 0); + put_io_u(td, io_u); + break; } - - ret = td_io_getevents(td, min_evts, td->cur_depth, timeout); - if (ret < 0) { - td_verror(td, -ret); + if (ret < 0 || td->error) break; - } else if (!ret) - continue; - icd.nr = ret; - ios_completed(td, &icd); - if (icd.error) { - td_verror(td, icd.error); - break; + /* + * See if we need to complete some commands + */ + if (ret == FIO_Q_QUEUED || ret == FIO_Q_BUSY) { + min_evts = 0; + if (queue_full(td) || ret == FIO_Q_BUSY) { + min_evts = 1; + + if (td->cur_depth > td->iodepth_low) + min_evts = td->cur_depth - td->iodepth_low; + } + + fio_gettime(&comp_time, NULL); + bytes_done = io_u_queued_complete(td, min_evts); + if (bytes_done < 0) + break; } + if (!bytes_done) + continue; + /* * the rate is batched for now, it should work for batches * of completions except the very first one which may look * a little bursty */ - gettimeofday(&e, NULL); - usec = utime_since(&s, &e); + usec = utime_since(&s, &comp_time); - rate_throttle(td, usec, icd.bytes_done[td->ddir]); + rate_throttle(td, usec, bytes_done); - if (check_min_rate(td, &e)) { - if (rate_quit) - terminate_threads(td->groupid); - td_verror(td, ENOMEM); + if (check_min_rate(td, &comp_time)) { + if (exitall_on_terminate) + terminate_threads(td->groupid, 0); + td_verror(td, ENODATA, "check_min_rate"); break; } - if (runtime_exceeded(td, &e)) - break; + if (td->thinktime) { + unsigned long long b; + + b = td->io_blocks[0] + td->io_blocks[1]; + if (!(b % td->thinktime_blocks)) { + int left; - if (td->thinktime) - usec_sleep(td, td->thinktime); + if (td->thinktime_spin) + __usec_sleep(td->thinktime_spin); - if (should_fsync(td) && td->fsync_blocks && - (td->io_blocks[DDIR_WRITE] % td->fsync_blocks) == 0) - td_io_sync(td, f); + left = td->thinktime - td->thinktime_spin; + if (left) + usec_sleep(td, left); + } + } } - if (!ret) { - if (td->cur_depth) - cleanup_pending_aio(td); + if (!td->error) { + struct fio_file *f; + + i = td->cur_depth; + if (i) + ret = io_u_queued_complete(td, i); if (should_fsync(td) && td->end_fsync) { td_set_runstate(td, TD_FSYNCING); for_each_file(td, f, i) - td_io_sync(td, f); + fio_io_sync(td, f); } - } -} - -static int td_io_init(struct thread_data *td) -{ - if (td->io_ops->init) - return td->io_ops->init(td); - - return 0; + } else + cleanup_pending_aio(td); } static void cleanup_io_u(struct thread_data *td) @@ -801,9 +538,23 @@ static void cleanup_io_u(struct thread_data *td) free_io_mem(td); } +/* + * "randomly" fill the buffer contents + */ +static void fill_rand_buf(struct io_u *io_u, int max_bs) +{ + int *ptr = io_u->buf; + + while ((void *) ptr - io_u->buf < max_bs) { + *ptr = rand() * 0x9e370001; + ptr++; + } +} + static int init_io_u(struct thread_data *td) { struct io_u *io_u; + unsigned int max_bs; int i, max_units; char *p; @@ -815,7 +566,13 @@ static int init_io_u(struct thread_data *td) else max_units = td->iodepth; - td->orig_buffer_size = td->max_bs * max_units + MASK; + max_bs = max(td->max_bs[DDIR_READ], td->max_bs[DDIR_WRITE]); + td->orig_buffer_size = max_bs * max_units; + + if (td->mem_type == MEM_SHMHUGE || td->mem_type == MEM_MMAPHUGE) + td->orig_buffer_size = (td->orig_buffer_size + td->hugepage_size - 1) & ~(td->hugepage_size - 1); + else + td->orig_buffer_size += page_mask; if (allocate_io_mem(td)) return 1; @@ -826,11 +583,17 @@ static int init_io_u(struct thread_data *td) memset(io_u, 0, sizeof(*io_u)); INIT_LIST_HEAD(&io_u->list); - io_u->buf = p + td->max_bs * i; + io_u->buf = p + max_bs * i; + if (td_write(td) || td_rw(td)) + fill_rand_buf(io_u, max_bs); + io_u->index = i; + io_u->flags = IO_U_F_FREE; list_add(&io_u->list, &td->io_u_freelist); } + io_u_init_timeout(); + return 0; } @@ -840,11 +603,14 @@ static int switch_ioscheduler(struct thread_data *td) FILE *f; int ret; + if (td->io_ops->flags & FIO_CPUIO) + return 0; + sprintf(tmp, "%s/queue/scheduler", td->sysfs_root); f = fopen(tmp, "r+"); if (!f) { - td_verror(td, errno); + td_verror(td, errno, "fopen"); return 1; } @@ -853,7 +619,7 @@ static int switch_ioscheduler(struct thread_data *td) */ ret = fwrite(td->ioscheduler, strlen(td->ioscheduler), 1, f); if (ferror(f) || ret != 1) { - td_verror(td, errno); + td_verror(td, errno, "fwrite"); fclose(f); return 1; } @@ -865,7 +631,7 @@ static int switch_ioscheduler(struct thread_data *td) */ ret = fread(tmp, 1, sizeof(tmp), f); if (ferror(f) || ret < 0) { - td_verror(td, errno); + td_verror(td, errno, "fread"); fclose(f); return 1; } @@ -873,7 +639,7 @@ static int switch_ioscheduler(struct thread_data *td) sprintf(tmp2, "[%s]", td->ioscheduler); if (!strstr(tmp, tmp2)) { log_err("fio: io scheduler %s not found\n", td->ioscheduler); - td_verror(td, EINVAL); + td_verror(td, EINVAL, "iosched_switch"); fclose(f); return 1; } @@ -887,11 +653,15 @@ static void clear_io_state(struct thread_data *td) struct fio_file *f; int i; - td->stat_io_bytes[0] = td->stat_io_bytes[1] = 0; + td->ts.stat_io_bytes[0] = td->ts.stat_io_bytes[1] = 0; td->this_io_bytes[0] = td->this_io_bytes[1] = 0; td->zone_bytes = 0; + td->last_was_sync = 0; + for_each_file(td, f, i) { + f->last_completed_pos = 0; + f->last_pos = 0; if (td->io_ops->flags & FIO_SYNCIO) lseek(f->fd, SEEK_SET, 0); @@ -907,6 +677,7 @@ static void clear_io_state(struct thread_data *td) */ static void *thread_main(void *data) { + unsigned long long runtime[2]; struct thread_data *td = data; if (!td->use_thread) @@ -916,6 +687,7 @@ static void *thread_main(void *data) INIT_LIST_HEAD(&td->io_u_freelist); INIT_LIST_HEAD(&td->io_u_busylist); + INIT_LIST_HEAD(&td->io_u_requeues); INIT_LIST_HEAD(&td->io_hist_list); INIT_LIST_HEAD(&td->io_log_list); @@ -923,25 +695,22 @@ static void *thread_main(void *data) goto err; if (fio_setaffinity(td) == -1) { - td_verror(td, errno); + td_verror(td, errno, "cpu_set_affinity"); goto err; } - if (td_io_init(td)) - goto err; - if (init_iolog(td)) goto err; if (td->ioprio) { if (ioprio_set(IOPRIO_WHO_PROCESS, 0, td->ioprio) == -1) { - td_verror(td, errno); + td_verror(td, errno, "ioprio_set"); goto err; } } if (nice(td->nice) == -1) { - td_verror(td, errno); + td_verror(td, errno, "nice"); goto err; } @@ -957,19 +726,32 @@ static void *thread_main(void *data) if (!td->create_serialize && setup_files(td)) goto err; + if (open_files(td)) + goto err; - gettimeofday(&td->epoch, NULL); + /* + * Do this late, as some IO engines would like to have the + * files setup prior to initializing structures. + */ + if (td_io_init(td)) + goto err; - if (td->exec_prerun) - system(td->exec_prerun); + if (td->exec_prerun) { + if (system(td->exec_prerun) < 0) + goto err; + } + + fio_gettime(&td->epoch, NULL); + memcpy(&td->timeout_end, &td->epoch, sizeof(td->epoch)); + getrusage(RUSAGE_SELF, &td->ts.ru_start); + runtime[0] = runtime[1] = 0; while (td->loops--) { - getrusage(RUSAGE_SELF, &td->ru_start); - gettimeofday(&td->start, NULL); - memcpy(&td->stat_sample_time, &td->start, sizeof(td->start)); + fio_gettime(&td->start, NULL); + memcpy(&td->ts.stat_sample_time, &td->start, sizeof(td->start)); if (td->ratemin) - memcpy(&td->lastrate, &td->stat_sample_time, sizeof(td->lastrate)); + memcpy(&td->lastrate, &td->ts.stat_sample_time, sizeof(td->lastrate)); clear_io_state(td); prune_io_piece_log(td); @@ -979,12 +761,11 @@ static void *thread_main(void *data) else do_io(td); - td->runtime[td->ddir] += mtime_since_now(&td->start); - if (td_rw(td) && td->io_bytes[td->ddir ^ 1]) - td->runtime[td->ddir ^ 1] = td->runtime[td->ddir]; - - update_rusage_stat(td); - + if (td_read(td) && td->io_bytes[DDIR_READ]) + runtime[DDIR_READ] += utime_since_now(&td->start); + if (td_write(td) && td->io_bytes[DDIR_WRITE]) + runtime[DDIR_WRITE] += utime_since_now(&td->start); + if (td->error || td->terminate) break; @@ -992,58 +773,68 @@ static void *thread_main(void *data) continue; clear_io_state(td); - gettimeofday(&td->start, NULL); + fio_gettime(&td->start, NULL); do_verify(td); - td->runtime[DDIR_READ] += mtime_since_now(&td->start); + runtime[DDIR_READ] += utime_since_now(&td->start); if (td->error || td->terminate) break; } - if (td->bw_log) - finish_log(td, td->bw_log, "bw"); - if (td->slat_log) - finish_log(td, td->slat_log, "slat"); - if (td->clat_log) - finish_log(td, td->clat_log, "clat"); - if (td->write_iolog) + update_rusage_stat(td); + fio_gettime(&td->end_time, NULL); + td->runtime[0] = runtime[0] / 1000; + td->runtime[1] = runtime[1] / 1000; + + if (td->ts.bw_log) + finish_log(td, td->ts.bw_log, "bw"); + if (td->ts.slat_log) + finish_log(td, td->ts.slat_log, "slat"); + if (td->ts.clat_log) + finish_log(td, td->ts.clat_log, "clat"); + if (td->write_iolog_file) write_iolog_close(td); - if (td->exec_postrun) - system(td->exec_postrun); + if (td->exec_postrun) { + if (system(td->exec_postrun) < 0) + log_err("fio: postrun %s failed\n", td->exec_postrun); + } if (exitall_on_terminate) - terminate_threads(td->groupid); + terminate_threads(td->groupid, 0); err: + if (td->error) + printf("fio: pid=%d, err=%d/%s\n", td->pid, td->error, td->verror); close_files(td); close_ioengine(td); cleanup_io_u(td); td_set_runstate(td, TD_EXITED); - return NULL; - + return (void *) (unsigned long) td->error; } /* * We cannot pass the td data into a forked process, so attach the td and * pass it to the thread worker. */ -static void *fork_main(int shmid, int offset) +static int fork_main(int shmid, int offset) { struct thread_data *td; - void *data; + void *data, *ret; data = shmat(shmid, NULL, 0); if (data == (void *) -1) { + int __err = errno; + perror("shmat"); - return NULL; + return __err; } td = data + offset * sizeof(struct thread_data); - thread_main(td); + ret = thread_main(td); shmdt(data); - return NULL; + return (int) (unsigned long) ret; } /* @@ -1052,13 +843,15 @@ static void *fork_main(int shmid, int offset) static void reap_threads(int *nr_running, int *t_rate, int *m_rate) { struct thread_data *td; - int i, cputhreads; + int i, cputhreads, pending, status, ret; /* * reap exited threads (TD_EXITED -> TD_REAPED) */ - cputhreads = 0; + pending = cputhreads = 0; for_each_td(td, i) { + int flags = 0; + /* * ->io_ops is NULL for a thread that has closed its * io engine @@ -1066,26 +859,70 @@ static void reap_threads(int *nr_running, int *t_rate, int *m_rate) if (td->io_ops && td->io_ops->flags & FIO_CPUIO) cputhreads++; - if (td->runstate != TD_EXITED) + if (!td->pid || td->runstate == TD_REAPED) + continue; + if (td->use_thread) { + if (td->runstate == TD_EXITED) { + td_set_runstate(td, TD_REAPED); + goto reaped; + } continue; + } - td_set_runstate(td, TD_REAPED); + flags = WNOHANG; + if (td->runstate == TD_EXITED) + flags = 0; + /* + * check if someone quit or got killed in an unusual way + */ + ret = waitpid(td->pid, &status, flags); + if (ret < 0) { + if (errno == ECHILD) { + log_err("fio: pid=%d disappeared %d\n", td->pid, td->runstate); + td_set_runstate(td, TD_REAPED); + goto reaped; + } + perror("waitpid"); + } else if (ret == td->pid) { + if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); + + log_err("fio: pid=%d, got signal=%d\n", td->pid, sig); + td_set_runstate(td, TD_REAPED); + goto reaped; + } + if (WIFEXITED(status)) { + if (WEXITSTATUS(status) && !td->error) + td->error = WEXITSTATUS(status); + + td_set_runstate(td, TD_REAPED); + goto reaped; + } + } + + /* + * thread is not dead, continue + */ + continue; +reaped: if (td->use_thread) { long ret; if (pthread_join(td->thread, (void *) &ret)) - perror("thread_join"); - } else - waitpid(td->pid, NULL, 0); + perror("pthread_join"); + } (*nr_running)--; (*m_rate) -= td->ratemin; (*t_rate) -= td->rate; + + if (td->error) + exit_value++; } - if (*nr_running == cputhreads) - terminate_threads(TERMINATE_ALL); + if (*nr_running == cputhreads && !pending) + terminate_threads(TERMINATE_ALL, 0); } /* @@ -1116,10 +953,10 @@ static void run_threads(void) for_each_td(td, i) { print_status_init(td->thread_number - 1); - init_disk_util(td); - - if (!td->create_serialize) + if (!td->create_serialize) { + init_disk_util(td); continue; + } /* * do file setup here so it happens sequentially, @@ -1127,12 +964,17 @@ static void run_threads(void) * client data interspersed on disk */ if (setup_files(td)) { + exit_value++; + if (td->error) + log_err("fio: pid=%d, err=%d/%s\n", td->pid, td->error, td->verror); td_set_runstate(td, TD_REAPED); todo--; } + + init_disk_util(td); } - time_init(); + set_genesis_time(); while (todo) { struct thread_data *map[MAX_JOBS]; @@ -1183,8 +1025,9 @@ static void run_threads(void) if (fork()) fio_sem_down(&startup_sem); else { - fork_main(shm_id, i); - exit(0); + int ret = fork_main(shm_id, i); + + exit(ret); } } } @@ -1193,9 +1036,9 @@ static void run_threads(void) * Wait for the started threads to transition to * TD_INITIALIZED. */ - gettimeofday(&this_start, NULL); + fio_gettime(&this_start, NULL); left = this_jobs; - while (left) { + while (left && !fio_abort) { if (mtime_since_now(&this_start) > JOB_START_TIMEOUT) break; @@ -1261,6 +1104,15 @@ static void run_threads(void) int main(int argc, char *argv[]) { + long ps; + + /* + * We need locale for number printing, if it isn't set then just + * go with the US format. + */ + if (!getenv("LC_NUMERIC")) + setlocale(LC_NUMERIC, "en_US"); + if (parse_options(argc, argv)) return 1; @@ -1269,10 +1121,32 @@ int main(int argc, char *argv[]) return 1; } + ps = sysconf(_SC_PAGESIZE); + if (ps < 0) { + log_err("Failed to get page size\n"); + return 1; + } + + page_mask = ps - 1; + + if (write_bw_log) { + setup_log(&agg_io_log[DDIR_READ]); + setup_log(&agg_io_log[DDIR_WRITE]); + } + + set_genesis_time(); + disk_util_timer_arm(); run_threads(); - show_run_stats(); - return 0; + if (!fio_abort) { + show_run_stats(); + if (write_bw_log) { + __finish_log(agg_io_log[DDIR_READ],"agg-read_bw.log"); + __finish_log(agg_io_log[DDIR_WRITE],"agg-write_bw.log"); + } + } + + return exit_value; }