#include <unistd.h>
-#include <fcntl.h>
#include <string.h>
-#include <signal.h>
-#include <time.h>
#include <assert.h>
#include "fio.h"
-#include "hash.h"
#include "verify.h"
#include "trim.h"
#include "lib/rand.h"
return max_blocks;
}
-struct rand_off {
- struct flist_head list;
- uint64_t off;
-};
-
static int __get_next_rand_offset(struct thread_data *td, struct fio_file *f,
enum fio_ddir ddir, uint64_t *b,
uint64_t lastb)
{
struct zone_split_index *zsi;
uint64_t lastb, send, stotal;
- static int warned;
unsigned int v;
lastb = last_block(td, f, ddir);
* Should never happen
*/
if (send == -1U) {
- if (!warned) {
+ if (!fio_did_warn(FIO_WARN_ZONED_BUG))
log_err("fio: bug in zoned generation\n");
- warned = 1;
- }
goto bail;
} else if (send > lastb) {
/*
{
unsigned int v, send, stotal;
uint64_t offset, lastb;
- static int warned;
struct zone_split_index *zsi;
lastb = last_block(td, f, ddir);
* Should never happen
*/
if (send == -1U) {
- if (!warned) {
+ if (!fio_did_warn(FIO_WARN_ZONED_BUG))
log_err("fio: bug in zoned generation\n");
- warned = 1;
- }
goto bail;
}
return 0;
}
-static int flist_cmp(void *data, struct flist_head *a, struct flist_head *b)
-{
- struct rand_off *r1 = flist_entry(a, struct rand_off, list);
- struct rand_off *r2 = flist_entry(b, struct rand_off, list);
-
- return r1->off - r2->off;
-}
-
-static int get_off_from_method(struct thread_data *td, struct fio_file *f,
- enum fio_ddir ddir, uint64_t *b)
+static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
+ enum fio_ddir ddir, uint64_t *b)
{
if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) {
uint64_t lastb;
return 1;
}
-/*
- * Sort the reads for a verify phase in batches of verifysort_nr, if
- * specified.
- */
-static inline bool should_sort_io(struct thread_data *td)
-{
- if (!td->o.verifysort_nr || !td->o.do_verify)
- return false;
- if (!td_random(td))
- return false;
- if (td->runstate != TD_VERIFYING)
- return false;
- if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE ||
- td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE64)
- return false;
-
- return true;
-}
-
static bool should_do_random(struct thread_data *td, enum fio_ddir ddir)
{
unsigned int v;
return v <= td->o.perc_rand[ddir];
}
-static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
- enum fio_ddir ddir, uint64_t *b)
-{
- struct rand_off *r;
- int i, ret = 1;
-
- if (!should_sort_io(td))
- return get_off_from_method(td, f, ddir, b);
-
- if (!flist_empty(&td->next_rand_list)) {
-fetch:
- r = flist_first_entry(&td->next_rand_list, struct rand_off, list);
- flist_del(&r->list);
- *b = r->off;
- free(r);
- return 0;
- }
-
- for (i = 0; i < td->o.verifysort_nr; i++) {
- r = malloc(sizeof(*r));
-
- ret = get_off_from_method(td, f, ddir, &r->off);
- if (ret) {
- free(r);
- break;
- }
-
- flist_add(&r->list, &td->next_rand_list);
- }
-
- if (ret && !i)
- return ret;
-
- assert(!flist_empty(&td->next_rand_list));
- flist_sort(NULL, &td->next_rand_list, flist_cmp);
- goto fetch;
-}
-
static void loop_cache_invalidate(struct thread_data *td, struct fio_file *f)
{
struct thread_options *o = &td->o;
if (td->o.time_based ||
(td->o.file_service_type & __FIO_FSERVICE_NONUNIFORM)) {
fio_file_reset(td, f);
+ loop_cache_invalidate(td, f);
if (!get_next_rand_offset(td, f, ddir, b))
return 0;
- loop_cache_invalidate(td, f);
}
dprint(FD_IO, "%s: rand offset failed, last=%llu, size=%llu\n",
if (f->last_pos[ddir] < f->real_file_size) {
uint64_t pos;
- if (f->last_pos[ddir] == f->file_offset && o->ddir_seq_add < 0) {
+ /*
+ * Only rewind if we already hit the end
+ */
+ if (f->last_pos[ddir] == f->file_offset &&
+ f->file_offset && o->ddir_seq_add < 0) {
if (f->real_file_size > f->io_size)
f->last_pos[ddir] = f->io_size;
else
static int get_next_block(struct thread_data *td, struct io_u *io_u,
enum fio_ddir ddir, int rw_seq,
- unsigned int *is_random)
+ bool *is_random)
{
struct fio_file *f = io_u->file;
uint64_t b, offset;
if (td_random(td)) {
if (should_do_random(td, ddir)) {
ret = get_next_rand_block(td, f, ddir, &b);
- *is_random = 1;
+ *is_random = true;
} else {
- *is_random = 0;
+ *is_random = false;
io_u_set(td, io_u, IO_U_F_BUSY_OK);
ret = get_next_seq_offset(td, f, ddir, &offset);
if (ret)
ret = get_next_rand_block(td, f, ddir, &b);
}
} else {
- *is_random = 0;
+ *is_random = false;
ret = get_next_seq_offset(td, f, ddir, &offset);
}
} else {
io_u_set(td, io_u, IO_U_F_BUSY_OK);
- *is_random = 0;
+ *is_random = false;
if (td->o.rw_seq == RW_SEQ_SEQ) {
ret = get_next_seq_offset(td, f, ddir, &offset);
if (ret) {
ret = get_next_rand_block(td, f, ddir, &b);
- *is_random = 0;
+ *is_random = false;
}
} else if (td->o.rw_seq == RW_SEQ_IDENT) {
if (f->last_start[ddir] != -1ULL)
* until we find a free one. For sequential io, just return the end of
* the last io issued.
*/
-static int __get_next_offset(struct thread_data *td, struct io_u *io_u,
- unsigned int *is_random)
+static int get_next_offset(struct thread_data *td, struct io_u *io_u,
+ bool *is_random)
{
struct fio_file *f = io_u->file;
enum fio_ddir ddir = io_u->ddir;
return 0;
}
-static int get_next_offset(struct thread_data *td, struct io_u *io_u,
- unsigned int *is_random)
-{
- if (td->flags & TD_F_PROFILE_OPS) {
- struct prof_io_ops *ops = &td->prof_io_ops;
-
- if (ops->fill_io_u_off)
- return ops->fill_io_u_off(td, io_u, is_random);
- }
-
- return __get_next_offset(td, io_u, is_random);
-}
-
static inline bool io_u_fits(struct thread_data *td, struct io_u *io_u,
unsigned int buflen)
{
return io_u->offset + buflen <= f->io_size + get_start_offset(td, f);
}
-static unsigned int __get_next_buflen(struct thread_data *td, struct io_u *io_u,
- unsigned int is_random)
+static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u,
+ bool is_random)
{
int ddir = io_u->ddir;
unsigned int buflen = 0;
assert(ddir_rw(ddir));
if (td->o.bs_is_seq_rand)
- ddir = is_random ? DDIR_WRITE: DDIR_READ;
+ ddir = is_random ? DDIR_WRITE : DDIR_READ;
minbs = td->o.min_bs[ddir];
maxbs = td->o.max_bs[ddir];
return buflen;
}
-static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u,
- unsigned int is_random)
-{
- if (td->flags & TD_F_PROFILE_OPS) {
- struct prof_io_ops *ops = &td->prof_io_ops;
-
- if (ops->fill_io_u_size)
- return ops->fill_io_u_size(td, io_u, is_random);
- }
-
- return __get_next_buflen(td, io_u, is_random);
-}
-
static void set_rwmix_bytes(struct thread_data *td)
{
unsigned int diff;
* io's that have been actually submitted to an async engine,
* and cur_depth is meaningless for sync engines.
*/
- if (td->io_u_queued || td->cur_depth) {
- int fio_unused ret;
-
- ret = td_io_commit(td);
- }
+ if (td->io_u_queued || td->cur_depth)
+ td_io_commit(td);
while (td->io_u_in_flight) {
int ret;
assert(!(td->flags & TD_F_CHILD));
}
io_u_qpush(&td->io_u_freelist, io_u);
- td_io_u_unlock(td);
td_io_u_free_notify(td);
+ td_io_u_unlock(td);
}
void clear_io_u(struct thread_data *td, struct io_u *io_u)
}
io_u_rpush(&td->io_u_requeues, __io_u);
- td_io_u_unlock(td);
td_io_u_free_notify(td);
+ td_io_u_unlock(td);
*io_u = NULL;
}
static int fill_io_u(struct thread_data *td, struct io_u *io_u)
{
- unsigned int is_random;
+ bool is_random;
if (td_ioengine_flagged(td, FIO_NOIO))
goto out;
return 0;
}
-static void __io_u_mark_map(unsigned int *map, unsigned int nr)
+static void __io_u_mark_map(uint64_t *map, unsigned int nr)
{
int idx = 0;
static struct fio_file *get_next_file(struct thread_data *td)
{
- if (td->flags & TD_F_PROFILE_OPS) {
- struct prof_io_ops *ops = &td->prof_io_ops;
-
- if (ops->get_next_file)
- return ops->get_next_file(td);
- }
-
return __get_next_file(td);
}
struct io_u *__get_io_u(struct thread_data *td)
{
struct io_u *io_u = NULL;
+ int ret;
if (td->stop_io)
return NULL;
* return one
*/
assert(!(td->flags & TD_F_CHILD));
- assert(!pthread_cond_wait(&td->free_cond, &td->io_u_lock));
+ ret = pthread_cond_wait(&td->free_cond, &td->io_u_lock);
+ assert(ret == 0);
goto again;
}
/*
* Return an io_u to be processed. Gets a buflen and offset, sets direction,
- * etc. The returned io_u is fully ready to be prepped and submitted.
+ * etc. The returned io_u is fully ready to be prepped, populated and submitted.
*/
struct io_u *get_io_u(struct thread_data *td)
{
td->o.min_bs[DDIR_WRITE],
io_u->buflen);
} else if ((td->flags & TD_F_SCRAMBLE_BUFFERS) &&
- !(td->flags & TD_F_COMPRESS))
+ !(td->flags & TD_F_COMPRESS) &&
+ !(td->flags & TD_F_DO_VERIFY))
do_scramble = 1;
- if (td->flags & TD_F_VER_NONE) {
- populate_verify_io_u(td, io_u);
- do_scramble = 0;
- }
} else if (io_u->ddir == DDIR_READ) {
/*
* Reset the buf_filled parameters so next time if the
if (no_reduce && per_unit_log(td->iops_log))
add_iops_sample(td, io_u, bytes);
- }
+ } else if (ddir_sync(idx) && !td->o.disable_clat)
+ add_sync_clat_sample(&td->ts, llnsec);
if (td->ts.nr_block_infos && io_u->ddir == DDIR_TRIM) {
uint32_t *info = io_u_block_info(td, io_u);
f->last_write_idx = 0;
}
+static bool should_account(struct thread_data *td)
+{
+ return ramp_time_over(td) && (td->runstate == TD_RUNNING ||
+ td->runstate == TD_VERIFYING);
+}
+
static void io_completed(struct thread_data *td, struct io_u **io_u_ptr,
struct io_completion_data *icd)
{
}
if (ddir_sync(ddir)) {
- td->last_was_sync = 1;
+ td->last_was_sync = true;
if (f) {
f->first_write = -1ULL;
f->last_write = -1ULL;
}
+ if (should_account(td))
+ account_io_completion(td, io_u, icd, ddir, io_u->buflen);
return;
}
- td->last_was_sync = 0;
+ td->last_was_sync = false;
td->last_ddir = ddir;
if (!io_u->error && ddir_rw(ddir)) {
if (ddir == DDIR_WRITE)
file_log_write_comp(td, f, io_u->offset, bytes);
- if (ramp_time_over(td) && (td->runstate == TD_RUNNING ||
- td->runstate == TD_VERIFYING))
+ if (should_account(td))
account_io_completion(td, io_u, icd, ddir, bytes);
icd->bytes_done[ddir] += bytes;
projects / fio.git / blobdiff