[fio.git] / io_u.c

#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <signal.h>
#include <time.h>
#include <assert.h>

#include "fio.h"
#include "os.h"

/*
 * Change this define to play with the timeout handling
 */
#undef FIO_USE_TIMEOUT

struct io_completion_data {
	int nr;				/* input */

	int error;			/* output */
	unsigned long bytes_done[2];	/* output */
	struct timeval time;		/* output */
};

/*
 * The ->file_map[] contains a map of blocks we have or have not done io
 * to yet. Used to make sure we cover the entire range in a fair fashion.
 */
static int random_map_free(struct thread_data *td, struct fio_file *f,
			   unsigned long long block)
{
	unsigned int idx = RAND_MAP_IDX(td, f, block);
	unsigned int bit = RAND_MAP_BIT(td, f, block);

	return (f->file_map[idx] & (1UL << bit)) == 0;
}

/*
 * Mark a given offset as used in the map.
 */
static void mark_random_map(struct thread_data *td, struct io_u *io_u)
{
	unsigned int min_bs = td->rw_min_bs;
	struct fio_file *f = io_u->file;
	unsigned long long block;
	unsigned int blocks;
	unsigned int nr_blocks;

	block = io_u->offset / (unsigned long long) min_bs;
	blocks = 0;
	nr_blocks = (io_u->buflen + min_bs - 1) / min_bs;

	while (blocks < nr_blocks) {
		unsigned int idx, bit;

		if (!random_map_free(td, f, block))
			break;

		idx = RAND_MAP_IDX(td, f, block);
		bit = RAND_MAP_BIT(td, f, block);

		fio_assert(td, idx < f->num_maps);

		f->file_map[idx] |= (1UL << bit);
		block++;
		blocks++;
	}

	if ((blocks * min_bs) < io_u->buflen)
		io_u->buflen = blocks * min_bs;
}

/*
 * Return the next free block in the map.
 */
static int get_next_free_block(struct thread_data *td, struct fio_file *f,
			       unsigned long long *b)
{
	int i;

	i = f->last_free_lookup;
	*b = (i * BLOCKS_PER_MAP);
	while ((*b) * td->rw_min_bs < f->real_file_size) {
		if (f->file_map[i] != -1UL) {
			*b += ffz(f->file_map[i]);
			f->last_free_lookup = i;
			return 0;
		}

		*b += BLOCKS_PER_MAP;
		i++;
	}

	return 1;
}

/*
 * For random io, generate a random new block and see if it's used. Repeat
 * until we find a free one. For sequential io, just return the end of
 * the last io issued.
 */
static int get_next_offset(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	const int ddir = io_u->ddir;
	unsigned long long b, rb;
	long r;

	if (td_random(td)) {
		unsigned long long max_blocks = f->file_size / td->min_bs[ddir];
		int loops = 5;

		do {
			r = os_random_long(&td->random_state);
			b = ((max_blocks - 1) * r / (unsigned long long) (RAND_MAX+1.0));
			if (td->norandommap)
				break;
			rb = b + (f->file_offset / td->min_bs[ddir]);
			loops--;
		} while (!random_map_free(td, f, rb) && loops);

		/*
		 * if we failed to retrieve a truly random offset within
		 * the loops assigned, see if there are free ones left at all
		 */
		if (!loops && get_next_free_block(td, f, &b))
			return 1;
	} else
		b = f->last_pos / td->min_bs[ddir];

	io_u->offset = (b * td->min_bs[ddir]) + f->file_offset;
	if (io_u->offset >= f->real_file_size)
		return 1;

	return 0;
}

static unsigned int get_next_buflen(struct thread_data *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;
	const int ddir = io_u->ddir;
	unsigned int buflen;
	long r;

	if (td->min_bs[ddir] == td->max_bs[ddir])
		buflen = td->min_bs[ddir];
	else {
		r = os_random_long(&td->bsrange_state);
		buflen = (unsigned int) (1 + (double) (td->max_bs[ddir] - 1) * r / (RAND_MAX + 1.0));
		if (!td->bs_unaligned)
			buflen = (buflen + td->min_bs[ddir] - 1) & ~(td->min_bs[ddir] - 1);
	}

	while (buflen + io_u->offset > f->real_file_size) {
		if (buflen == td->min_bs[ddir])
			return 0;

		buflen = td->min_bs[ddir];
	}

	return buflen;
}

/*
 * Return the data direction for the next io_u. If the job is a
 * mixed read/write workload, check the rwmix cycle and switch if
 * necessary.
 */
static enum fio_ddir get_rw_ddir(struct thread_data *td)
{
	if (td_rw(td)) {
		struct timeval now;
		unsigned long elapsed;

		fio_gettime(&now, NULL);
	 	elapsed = mtime_since_now(&td->rwmix_switch);

		/*
		 * Check if it's time to seed a new data direction.
		 */
		if (elapsed >= td->rwmixcycle) {
			unsigned int v;
			long r;

			r = os_random_long(&td->rwmix_state);
			v = 1 + (int) (100.0 * (r / (RAND_MAX + 1.0)));
			if (v < td->rwmixread)
				td->rwmix_ddir = DDIR_READ;
			else
				td->rwmix_ddir = DDIR_WRITE;
			memcpy(&td->rwmix_switch, &now, sizeof(now));
		}
		return td->rwmix_ddir;
	} else if (td_read(td))
		return DDIR_READ;
	else
		return DDIR_WRITE;
}

void put_io_u(struct thread_data *td, struct io_u *io_u)
{
	assert((io_u->flags & IO_U_F_FREE) == 0);
	io_u->flags |= IO_U_F_FREE;

	io_u->file = NULL;
	list_del(&io_u->list);
	list_add(&io_u->list, &td->io_u_freelist);
	td->cur_depth--;
}

void requeue_io_u(struct thread_data *td, struct io_u **io_u)
{
	struct io_u *__io_u = *io_u;

	__io_u->flags |= IO_U_F_FREE;
	__io_u->flags &= ~IO_U_F_FLIGHT;

	list_del(&__io_u->list);
	list_add_tail(&__io_u->list, &td->io_u_requeues);
	td->cur_depth--;
	*io_u = NULL;
}

static int fill_io_u(struct thread_data *td, struct io_u *io_u)
{
	/*
	 * If using an iolog, grab next piece if any available.
	 */
	if (td->read_iolog)
		return read_iolog_get(td, io_u);

	/*
	 * see if it's time to sync
	 */
	if (td->fsync_blocks && !(td->io_issues[DDIR_WRITE] % td->fsync_blocks)
	    && td->io_issues[DDIR_WRITE] && should_fsync(td)) {
		io_u->ddir = DDIR_SYNC;
		return 0;
	}

	io_u->ddir = get_rw_ddir(td);

	/*
	 * No log, let the seq/rand engine retrieve the next buflen and
	 * position.
	 */
	if (get_next_offset(td, io_u))
		return 1;

	io_u->buflen = get_next_buflen(td, io_u);
	if (!io_u->buflen)
		return 1;

	/*
	 * mark entry before potentially trimming io_u
	 */
	if (!td->read_iolog && td_random(td) && !td->norandommap)
		mark_random_map(td, io_u);

	/*
	 * If using a write iolog, store this entry.
	 */
	if (td->write_iolog_file)
		write_iolog_put(td, io_u);

	return 0;
}

void io_u_mark_depth(struct thread_data *td, struct io_u *io_u)
{
	int index = 0;

	if (io_u->ddir == DDIR_SYNC)
		return;

	switch (td->cur_depth) {
	default:
		index++;
	case 32 ... 63:
		index++;
	case 16 ... 31:
		index++;
	case 8 ... 15:
		index++;
	case 4 ... 7:
		index++;
	case 2 ... 3:
		index++;
	case 1:
		break;
	}

	td->ts.io_u_map[index]++;
	td->ts.total_io_u[io_u->ddir]++;
}

static void io_u_mark_latency(struct thread_data *td, unsigned long msec)
{
	int index = 0;

	switch (msec) {
	default:
		index++;
	case 1000 ... 1999:
		index++;
	case 750 ... 999:
		index++;
	case 500 ... 749:
		index++;
	case 250 ... 499:
		index++;
	case 100 ... 249:
		index++;
	case 50 ... 99:
		index++;
	case 20 ... 49:
		index++;
	case 10 ... 19:
		index++;
	case 4 ... 9:
		index++;
	case 2 ... 3:
		index++;
	case 0 ... 1:
		break;
	}

	td->ts.io_u_lat[index]++;
}

/*
 * Get next file to service by choosing one at random
 */
static struct fio_file *get_next_file_rand(struct thread_data *td, int goodf,
					   int badf)
{
	struct fio_file *f;
	int fno;

	do {
		long r = os_random_long(&td->next_file_state);

		fno = (unsigned int) ((double) td->nr_files * (r / (RAND_MAX + 1.0)));
		f = &td->files[fno];

		if ((!goodf || (f->flags & goodf)) && !(f->flags & badf))
			return f;
	} while (1);
}

/*
 * Get next file to service by doing round robin between all available ones
 */
static struct fio_file *get_next_file_rr(struct thread_data *td, int goodf,
					 int badf)
{
	unsigned int old_next_file = td->next_file;
	struct fio_file *f;

	do {
		f = &td->files[td->next_file];

		td->next_file++;
		if (td->next_file >= td->nr_files)
			td->next_file = 0;

		if ((!goodf || (f->flags & goodf)) && !(f->flags & badf))
			break;

		f = NULL;
	} while (td->next_file != old_next_file);

	return f;
}

static struct fio_file *get_next_file(struct thread_data *td)
{
	struct fio_file *f;

	assert(td->nr_files <= td->files_index);

	if (!td->nr_open_files)
		return NULL;

	f = td->file_service_file;
	if (f && (f->flags & FIO_FILE_OPEN) && td->file_service_left--)
		return f;

	if (td->file_service_type == FIO_FSERVICE_RR)
		f = get_next_file_rr(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);
	else
		f = get_next_file_rand(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);

	td->file_service_file = f;
	td->file_service_left = td->file_service_nr - 1;
	return f;
}

static struct fio_file *find_next_new_file(struct thread_data *td)
{
	struct fio_file *f;

	if (td->file_service_type == FIO_FSERVICE_RR)
		f = get_next_file_rr(td, 0, FIO_FILE_OPEN);
	else
		f = get_next_file_rand(td, 0, FIO_FILE_OPEN);

	return f;
}

struct io_u *__get_io_u(struct thread_data *td)
{
	struct io_u *io_u = NULL;

	if (!list_empty(&td->io_u_requeues))
		io_u = list_entry(td->io_u_requeues.next, struct io_u, list);
	else if (!queue_full(td)) {
		io_u = list_entry(td->io_u_freelist.next, struct io_u, list);

		io_u->buflen = 0;
		io_u->resid = 0;
		io_u->file = NULL;
		io_u->end_io = NULL;
	}

	if (io_u) {
		assert(io_u->flags & IO_U_F_FREE);
		io_u->flags &= ~IO_U_F_FREE;

		io_u->error = 0;
		list_del(&io_u->list);
		list_add(&io_u->list, &td->io_u_busylist);
		td->cur_depth++;
	}

	return io_u;
}

/*
 * Return an io_u to be processed. Gets a buflen and offset, sets direction,
 * etc. The returned io_u is fully ready to be prepped and submitted.
 */
struct io_u *get_io_u(struct thread_data *td)
{
	struct fio_file *f;
	struct io_u *io_u;
	int ret;

	io_u = __get_io_u(td);
	if (!io_u)
		return NULL;

	/*
	 * from a requeue, io_u already setup
	 */
	if (io_u->file)
		goto out;

	do {
		f = get_next_file(td);
		if (!f) {
			put_io_u(td, io_u);
			return NULL;
		}

set_file:
		io_u->file = f;

		if (!fill_io_u(td, io_u))
			break;

		/*
		 * No more to do for this file, close it
		 */
		io_u->file = NULL;
		td_io_close_file(td, f);

		/*
		 * probably not the right place to do this, but see
		 * if we need to open a new file
		 */
		if (td->nr_open_files < td->nr_files &&
		    td->open_files != td->nr_files) {
			f = find_next_new_file(td);

			if (!f || (ret = td_io_open_file(td, f))) {
				put_io_u(td, io_u);
				break;
			}
			goto set_file;
		}
	} while (1);

	if (td->zone_bytes >= td->zone_size) {
		td->zone_bytes = 0;
		f->last_pos += td->zone_skip;
	}

	if (io_u->buflen + io_u->offset > f->real_file_size) {
		if (td->io_ops->flags & FIO_RAWIO) {
			put_io_u(td, io_u);
			return NULL;
		}

		io_u->buflen = f->real_file_size - io_u->offset;
	}

	if (io_u->ddir != DDIR_SYNC) {
		if (!io_u->buflen) {
			put_io_u(td, io_u);
			return NULL;
		}

		f->last_pos = io_u->offset + io_u->buflen;

		if (td->verify != VERIFY_NONE)
			populate_verify_io_u(td, io_u);
	}

	/*
	 * Set io data pointers.
	 */
out:
	io_u->xfer_buf = io_u->buf;
	io_u->xfer_buflen = io_u->buflen;

	if (td_io_prep(td, io_u)) {
		put_io_u(td, io_u);
		return NULL;
	}

	fio_gettime(&io_u->start_time, NULL);
	return io_u;
}

void io_u_log_error(struct thread_data *td, struct io_u *io_u)
{
	const char *msg[] = { "read", "write", "sync" };

	log_err("fio: io_u error");

	if (io_u->file)
		log_err(" on file %s", io_u->file->file_name);

	log_err(": %s\n", strerror(io_u->error));

	log_err("     %s offset=%llu, buflen=%lu\n", msg[io_u->ddir], io_u->offset, io_u->xfer_buflen);

	if (!td->error)
		td_verror(td, io_u->error, "io_u error");
}

static void io_completed(struct thread_data *td, struct io_u *io_u,
			 struct io_completion_data *icd)
{
	unsigned long msec;

	assert(io_u->flags & IO_U_F_FLIGHT);
	io_u->flags &= ~IO_U_F_FLIGHT;

	put_file(td, io_u->file);

	if (io_u->ddir == DDIR_SYNC) {
		td->last_was_sync = 1;
		return;
	}

	td->last_was_sync = 0;

	if (!io_u->error) {
		unsigned int bytes = io_u->buflen - io_u->resid;
		const enum fio_ddir idx = io_u->ddir;
		int ret;

		td->io_blocks[idx]++;
		td->io_bytes[idx] += bytes;
		td->zone_bytes += bytes;
		td->this_io_bytes[idx] += bytes;

		io_u->file->last_completed_pos = io_u->offset + io_u->buflen;

		msec = mtime_since(&io_u->issue_time, &icd->time);

		add_clat_sample(td, idx, msec);
		add_bw_sample(td, idx, &icd->time);
		io_u_mark_latency(td, msec);

		if ((td_rw(td) || td_write(td)) && idx == DDIR_WRITE)
			log_io_piece(td, io_u);

		icd->bytes_done[idx] += bytes;

		if (io_u->end_io) {
			ret = io_u->end_io(io_u);
			if (ret && !icd->error)
				icd->error = ret;
		}
	} else {
		icd->error = io_u->error;
		io_u_log_error(td, io_u);
	}
}

static void init_icd(struct io_completion_data *icd, int nr)
{
	fio_gettime(&icd->time, NULL);

	icd->nr = nr;

	icd->error = 0;
	icd->bytes_done[0] = icd->bytes_done[1] = 0;
}

static void ios_completed(struct thread_data *td,
			  struct io_completion_data *icd)
{
	struct io_u *io_u;
	int i;

	for (i = 0; i < icd->nr; i++) {
		io_u = td->io_ops->event(td, i);

		io_completed(td, io_u, icd);
		put_io_u(td, io_u);
	}
}

/*
 * Complete a single io_u for the sync engines.
 */
long io_u_sync_complete(struct thread_data *td, struct io_u *io_u)
{
	struct io_completion_data icd;

	init_icd(&icd, 1);
	io_completed(td, io_u, &icd);
	put_io_u(td, io_u);

	if (!icd.error)
		return icd.bytes_done[0] + icd.bytes_done[1];

	td_verror(td, icd.error, "io_u_sync_complete");
	return -1;
}

/*
 * Called to complete min_events number of io for the async engines.
 */
long io_u_queued_complete(struct thread_data *td, int min_events)
{
	struct io_completion_data icd;
	struct timespec *tvp = NULL;
	int ret;

	if (!min_events) {
		struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, };

		tvp = &ts;
	}

	ret = td_io_getevents(td, min_events, td->cur_depth, tvp);
	if (ret < 0) {
		td_verror(td, -ret, "td_io_getevents");
		return ret;
	} else if (!ret)
		return ret;

	init_icd(&icd, ret);
	ios_completed(td, &icd);
	if (!icd.error)
		return icd.bytes_done[0] + icd.bytes_done[1];

	td_verror(td, icd.error, "io_u_queued_complete");
	return -1;
}

/*
 * Call when io_u is really queued, to update the submission latency.
 */
void io_u_queued(struct thread_data *td, struct io_u *io_u)
{
	unsigned long slat_time;

	slat_time = mtime_since(&io_u->start_time, &io_u->issue_time);
	add_slat_sample(td, io_u->ddir, slat_time);
}

#ifdef FIO_USE_TIMEOUT
void io_u_set_timeout(struct thread_data *td)
{
	assert(td->cur_depth);

	td->timer.it_interval.tv_sec = 0;
	td->timer.it_interval.tv_usec = 0;
	td->timer.it_value.tv_sec = IO_U_TIMEOUT + IO_U_TIMEOUT_INC;
	td->timer.it_value.tv_usec = 0;
	setitimer(ITIMER_REAL, &td->timer, NULL);
	fio_gettime(&td->timeout_end, NULL);
}

static void io_u_dump(struct io_u *io_u)
{
	unsigned long t_start = mtime_since_now(&io_u->start_time);
	unsigned long t_issue = mtime_since_now(&io_u->issue_time);

	log_err("io_u=%p, t_start=%lu, t_issue=%lu\n", io_u, t_start, t_issue);
	log_err("  buf=%p/%p, len=%lu/%lu, offset=%llu\n", io_u->buf, io_u->xfer_buf, io_u->buflen, io_u->xfer_buflen, io_u->offset);
	log_err("  ddir=%d, fname=%s\n", io_u->ddir, io_u->file->file_name);
}
#else
void io_u_set_timeout(struct thread_data fio_unused *td)
{
}
#endif

#ifdef FIO_USE_TIMEOUT
static void io_u_timeout_handler(int fio_unused sig)
{
	struct thread_data *td, *__td;
	pid_t pid = getpid();
	struct list_head *entry;
	struct io_u *io_u;
	int i;

	log_err("fio: io_u timeout\n");

	/*
	 * TLS would be nice...
	 */
	td = NULL;
	for_each_td(__td, i) {
		if (__td->pid == pid) {
			td = __td;
			break;
		}
	}

	if (!td) {
		log_err("fio: io_u timeout, can't find job\n");
		exit(1);
	}

	if (!td->cur_depth) {
		log_err("fio: timeout without pending work?\n");
		return;
	}

	log_err("fio: io_u timeout: job=%s, pid=%d\n", td->name, td->pid);

	list_for_each(entry, &td->io_u_busylist) {
		io_u = list_entry(entry, struct io_u, list);

		io_u_dump(io_u);
	}

	td_verror(td, ETIMEDOUT, "io_u timeout");
	exit(1);
}
#endif

void io_u_init_timeout(void)
{
#ifdef FIO_USE_TIMEOUT
	signal(SIGALRM, io_u_timeout_handler);
#endif
}
Commit	Line	Data
10ba535a JA	1	#include <unistd.h>
	2	#include <fcntl.h>
	3	#include <string.h>
	4	#include <signal.h>
	5	#include <time.h>
0c6e7517	6	#include <assert.h>
10ba535a JA	7
	8	#include "fio.h"
	9	#include "os.h"
	10
5945b9b4 JA	11	/*
	12	* Change this define to play with the timeout handling
	13	*/
	14	#undef FIO_USE_TIMEOUT
	15
97601024 JA	16	struct io_completion_data {
97601024 JA	17	int nr; /* input */
97601024 JA	18
	19	int error; /* output */
	20	unsigned long bytes_done[2]; /* output */
	21	struct timeval time; /* output */
	22	};
	23
10ba535a JA	24	/*
	25	* The ->file_map[] contains a map of blocks we have or have not done io
	26	* to yet. Used to make sure we cover the entire range in a fair fashion.
	27	*/
	28	static int random_map_free(struct thread_data td, struct fio_file f,
	29	unsigned long long block)
	30	{
	31	unsigned int idx = RAND_MAP_IDX(td, f, block);
	32	unsigned int bit = RAND_MAP_BIT(td, f, block);
	33
	34	return (f->file_map[idx] & (1UL << bit)) == 0;
	35	}
	36
df415585 JA	37	/*
	38	* Mark a given offset as used in the map.
	39	*/
9bf2061e	40	static void mark_random_map(struct thread_data td, struct io_u io_u)
df415585	41	{
afdbe580	42	unsigned int min_bs = td->rw_min_bs;
9bf2061e	43	struct fio_file *f = io_u->file;
a00735e6 JA	44	unsigned long long block;
a00735e6 JA	45	unsigned int blocks;
c685b5b2	46	unsigned int nr_blocks;
df415585	47
a00735e6 JA	48	block = io_u->offset / (unsigned long long) min_bs;
a00735e6 JA	49	blocks = 0;
c685b5b2 JA	50	nr_blocks = (io_u->buflen + min_bs - 1) / min_bs;
	51
	52	while (blocks < nr_blocks) {
df415585 JA	53	unsigned int idx, bit;
	54
	55	if (!random_map_free(td, f, block))
	56	break;
	57
	58	idx = RAND_MAP_IDX(td, f, block);
	59	bit = RAND_MAP_BIT(td, f, block);
	60
0032bf9f	61	fio_assert(td, idx < f->num_maps);
df415585 JA	62
	63	f->file_map[idx] \|= (1UL << bit);
	64	block++;
	65	blocks++;
	66	}
	67
a00735e6 JA	68	if ((blocks * min_bs) < io_u->buflen)
a00735e6 JA	69	io_u->buflen = blocks * min_bs;
df415585 JA	70	}
df415585 JA	71
10ba535a JA	72	/*
	73	* Return the next free block in the map.
	74	*/
	75	static int get_next_free_block(struct thread_data td, struct fio_file f,
	76	unsigned long long *b)
	77	{
	78	int i;
	79
c685b5b2 JA	80	i = f->last_free_lookup;
c685b5b2 JA	81	b = (i BLOCKS_PER_MAP);
ee88470c	82	while ((b) td->rw_min_bs < f->real_file_size) {
10ba535a JA	83	if (f->file_map[i] != -1UL) {
10ba535a JA	84	*b += ffz(f->file_map[i]);
c685b5b2	85	f->last_free_lookup = i;
10ba535a JA	86	return 0;
	87	}
	88
	89	*b += BLOCKS_PER_MAP;
	90	i++;
	91	}
	92
	93	return 1;
	94	}
	95
	96	/*
	97	* For random io, generate a random new block and see if it's used. Repeat
	98	* until we find a free one. For sequential io, just return the end of
	99	* the last io issued.
	100	*/
9bf2061e	101	static int get_next_offset(struct thread_data td, struct io_u io_u)
10ba535a	102	{
9bf2061e	103	struct fio_file *f = io_u->file;
c685b5b2	104	const int ddir = io_u->ddir;
10ba535a JA	105	unsigned long long b, rb;
	106	long r;
	107
413dd459	108	if (td_random(td)) {
f697125a	109	unsigned long long max_blocks = f->file_size / td->min_bs[ddir];
e6433576	110	int loops = 5;
10ba535a JA	111
	112	do {
	113	r = os_random_long(&td->random_state);
	114	b = ((max_blocks - 1) * r / (unsigned long long) (RAND_MAX+1.0));
bb8895e0 JA	115	if (td->norandommap)
bb8895e0 JA	116	break;
a00735e6	117	rb = b + (f->file_offset / td->min_bs[ddir]);
10ba535a JA	118	loops--;
	119	} while (!random_map_free(td, f, rb) && loops);
	120
bca4ed4d JA	121	/*
	122	* if we failed to retrieve a truly random offset within
	123	* the loops assigned, see if there are free ones left at all
	124	*/
	125	if (!loops && get_next_free_block(td, f, &b))
	126	return 1;
10ba535a	127	} else
a00735e6	128	b = f->last_pos / td->min_bs[ddir];
10ba535a	129
c685b5b2	130	io_u->offset = (b * td->min_bs[ddir]) + f->file_offset;
bca4ed4d	131	if (io_u->offset >= f->real_file_size)
10ba535a JA	132	return 1;
	133
	134	return 0;
	135	}
	136
9bf2061e	137	static unsigned int get_next_buflen(struct thread_data td, struct io_u io_u)
10ba535a	138	{
9bf2061e	139	struct fio_file *f = io_u->file;
bca4ed4d	140	const int ddir = io_u->ddir;
10ba535a JA	141	unsigned int buflen;
	142	long r;
	143
a00735e6 JA	144	if (td->min_bs[ddir] == td->max_bs[ddir])
a00735e6 JA	145	buflen = td->min_bs[ddir];
10ba535a JA	146	else {
10ba535a JA	147	r = os_random_long(&td->bsrange_state);
1e97cce9	148	buflen = (unsigned int) (1 + (double) (td->max_bs[ddir] - 1) * r / (RAND_MAX + 1.0));
690adba3	149	if (!td->bs_unaligned)
a00735e6	150	buflen = (buflen + td->min_bs[ddir] - 1) & ~(td->min_bs[ddir] - 1);
10ba535a JA	151	}
10ba535a JA	152
bca4ed4d	153	while (buflen + io_u->offset > f->real_file_size) {
bca4ed4d JA	154	if (buflen == td->min_bs[ddir])
	155	return 0;
	156
	157	buflen = td->min_bs[ddir];
10ba535a JA	158	}
	159
	160	return buflen;
	161	}
	162
	163	/*
	164	* Return the data direction for the next io_u. If the job is a
	165	* mixed read/write workload, check the rwmix cycle and switch if
	166	* necessary.
	167	*/
1e97cce9	168	static enum fio_ddir get_rw_ddir(struct thread_data *td)
10ba535a JA	169	{
	170	if (td_rw(td)) {
	171	struct timeval now;
	172	unsigned long elapsed;
	173
02bcaa8c	174	fio_gettime(&now, NULL);
10ba535a JA	175	elapsed = mtime_since_now(&td->rwmix_switch);
	176
	177	/*
	178	* Check if it's time to seed a new data direction.
	179	*/
	180	if (elapsed >= td->rwmixcycle) {
	181	unsigned int v;
	182	long r;
	183
	184	r = os_random_long(&td->rwmix_state);
	185	v = 1 + (int) (100.0 * (r / (RAND_MAX + 1.0)));
	186	if (v < td->rwmixread)
	187	td->rwmix_ddir = DDIR_READ;
	188	else
	189	td->rwmix_ddir = DDIR_WRITE;
	190	memcpy(&td->rwmix_switch, &now, sizeof(now));
	191	}
	192	return td->rwmix_ddir;
	193	} else if (td_read(td))
	194	return DDIR_READ;
	195	else
	196	return DDIR_WRITE;
	197	}
	198
10ba535a JA	199	void put_io_u(struct thread_data td, struct io_u io_u)
10ba535a JA	200	{
0c6e7517 JA	201	assert((io_u->flags & IO_U_F_FREE) == 0);
	202	io_u->flags \|= IO_U_F_FREE;
	203
10ba535a JA	204	io_u->file = NULL;
	205	list_del(&io_u->list);
	206	list_add(&io_u->list, &td->io_u_freelist);
	207	td->cur_depth--;
	208	}
	209
755200a3 JA	210	void requeue_io_u(struct thread_data td, struct io_u *io_u)
	211	{
	212	struct io_u __io_u = io_u;
	213
4d2e0f49 JA	214	__io_u->flags \|= IO_U_F_FREE;
	215	__io_u->flags &= ~IO_U_F_FLIGHT;
	216
755200a3 JA	217	list_del(&__io_u->list);
	218	list_add_tail(&__io_u->list, &td->io_u_requeues);
	219	td->cur_depth--;
	220	*io_u = NULL;
	221	}
	222
9bf2061e	223	static int fill_io_u(struct thread_data td, struct io_u io_u)
10ba535a JA	224	{
	225	/*
	226	* If using an iolog, grab next piece if any available.
	227	*/
	228	if (td->read_iolog)
	229	return read_iolog_get(td, io_u);
	230
87dc1ab1 JA	231	/*
	232	* see if it's time to sync
	233	*/
755200a3 JA	234	if (td->fsync_blocks && !(td->io_issues[DDIR_WRITE] % td->fsync_blocks)
755200a3 JA	235	&& td->io_issues[DDIR_WRITE] && should_fsync(td)) {
87dc1ab1	236	io_u->ddir = DDIR_SYNC;
87dc1ab1 JA	237	return 0;
	238	}
	239
a00735e6 JA	240	io_u->ddir = get_rw_ddir(td);
a00735e6 JA	241
10ba535a	242	/*
c685b5b2 JA	243	* No log, let the seq/rand engine retrieve the next buflen and
c685b5b2 JA	244	* position.
10ba535a	245	*/
9bf2061e	246	if (get_next_offset(td, io_u))
bca4ed4d	247	return 1;
10ba535a	248
9bf2061e	249	io_u->buflen = get_next_buflen(td, io_u);
bca4ed4d JA	250	if (!io_u->buflen)
	251	return 1;
	252
	253	/*
	254	* mark entry before potentially trimming io_u
	255	*/
413dd459	256	if (!td->read_iolog && td_random(td) && !td->norandommap)
9bf2061e	257	mark_random_map(td, io_u);
bca4ed4d JA	258
	259	/*
	260	* If using a write iolog, store this entry.
	261	*/
	262	if (td->write_iolog_file)
	263	write_iolog_put(td, io_u);
	264
bca4ed4d	265	return 0;
10ba535a JA	266	}
10ba535a JA	267
b3605062	268	void io_u_mark_depth(struct thread_data td, struct io_u io_u)
71619dc2 JA	269	{
	270	int index = 0;
	271
b3605062 JA	272	if (io_u->ddir == DDIR_SYNC)
	273	return;
	274
71619dc2 JA	275	switch (td->cur_depth) {
	276	default:
	277	index++;
	278	case 32 ... 63:
	279	index++;
	280	case 16 ... 31:
	281	index++;
	282	case 8 ... 15:
	283	index++;
	284	case 4 ... 7:
	285	index++;
	286	case 2 ... 3:
	287	index++;
	288	case 1:
	289	break;
	290	}
	291
756867bd	292	td->ts.io_u_map[index]++;
b3605062	293	td->ts.total_io_u[io_u->ddir]++;
71619dc2 JA	294	}
71619dc2 JA	295
ec118304 JA	296	static void io_u_mark_latency(struct thread_data *td, unsigned long msec)
	297	{
	298	int index = 0;
	299
	300	switch (msec) {
	301	default:
	302	index++;
8abdce66	303	case 1000 ... 1999:
ec118304	304	index++;
8abdce66	305	case 750 ... 999:
ec118304	306	index++;
8abdce66	307	case 500 ... 749:
ec118304	308	index++;
8abdce66	309	case 250 ... 499:
ec118304	310	index++;
8abdce66	311	case 100 ... 249:
ec118304	312	index++;
8abdce66	313	case 50 ... 99:
ec118304	314	index++;
8abdce66	315	case 20 ... 49:
ec118304	316	index++;
8abdce66	317	case 10 ... 19:
ec118304	318	index++;
8abdce66	319	case 4 ... 9:
ec118304 JA	320	index++;
	321	case 2 ... 3:
	322	index++;
	323	case 0 ... 1:
	324	break;
	325	}
	326
756867bd	327	td->ts.io_u_lat[index]++;
ec118304 JA	328	}
ec118304 JA	329
0aabe160 JA	330	/*
	331	* Get next file to service by choosing one at random
	332	*/
1c178180 JA	333	static struct fio_file get_next_file_rand(struct thread_data td, int goodf,
1c178180 JA	334	int badf)
0aabe160	335	{
0aabe160	336	struct fio_file *f;
1c178180	337	int fno;
0aabe160 JA	338
0aabe160 JA	339	do {
7c83c089 JA	340	long r = os_random_long(&td->next_file_state);
7c83c089 JA	341
1c178180 JA	342	fno = (unsigned int) ((double) td->nr_files * (r / (RAND_MAX + 1.0)));
	343	f = &td->files[fno];
	344
	345	if ((!goodf \|\| (f->flags & goodf)) && !(f->flags & badf))
0aabe160 JA	346	return f;
	347	} while (1);
	348	}
	349
	350	/*
	351	* Get next file to service by doing round robin between all available ones
	352	*/
1c178180 JA	353	static struct fio_file get_next_file_rr(struct thread_data td, int goodf,
1c178180 JA	354	int badf)
3d7c391d JA	355	{
	356	unsigned int old_next_file = td->next_file;
	357	struct fio_file *f;
	358
	359	do {
	360	f = &td->files[td->next_file];
	361
	362	td->next_file++;
1c178180	363	if (td->next_file >= td->nr_files)
3d7c391d JA	364	td->next_file = 0;
3d7c391d JA	365
1c178180	366	if ((!goodf \|\| (f->flags & goodf)) && !(f->flags & badf))
3d7c391d JA	367	break;
	368
	369	f = NULL;
	370	} while (td->next_file != old_next_file);
	371
	372	return f;
	373	}
	374
bdb4e2e9 JA	375	static struct fio_file get_next_file(struct thread_data td)
bdb4e2e9 JA	376	{
1907dbc6 JA	377	struct fio_file *f;
1907dbc6 JA	378
1c178180 JA	379	assert(td->nr_files <= td->files_index);
1c178180 JA	380
bdb4e2e9 JA	381	if (!td->nr_open_files)
	382	return NULL;
	383
1907dbc6	384	f = td->file_service_file;
f11bd94d	385	if (f && (f->flags & FIO_FILE_OPEN) && td->file_service_left--)
1907dbc6 JA	386	return f;
1907dbc6 JA	387
bdb4e2e9	388	if (td->file_service_type == FIO_FSERVICE_RR)
1c178180	389	f = get_next_file_rr(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);
bdb4e2e9	390	else
1c178180	391	f = get_next_file_rand(td, FIO_FILE_OPEN, FIO_FILE_CLOSING);
1907dbc6 JA	392
	393	td->file_service_file = f;
	394	td->file_service_left = td->file_service_nr - 1;
	395	return f;
bdb4e2e9 JA	396	}
bdb4e2e9 JA	397
1c178180 JA	398	static struct fio_file find_next_new_file(struct thread_data td)
	399	{
	400	struct fio_file *f;
	401
	402	if (td->file_service_type == FIO_FSERVICE_RR)
	403	f = get_next_file_rr(td, 0, FIO_FILE_OPEN);
	404	else
	405	f = get_next_file_rand(td, 0, FIO_FILE_OPEN);
	406
	407	return f;
	408	}
	409
10ba535a JA	410	struct io_u __get_io_u(struct thread_data td)
	411	{
	412	struct io_u *io_u = NULL;
	413
755200a3 JA	414	if (!list_empty(&td->io_u_requeues))
	415	io_u = list_entry(td->io_u_requeues.next, struct io_u, list);
	416	else if (!queue_full(td)) {
10ba535a JA	417	io_u = list_entry(td->io_u_freelist.next, struct io_u, list);
10ba535a JA	418
6040dabc	419	io_u->buflen = 0;
10ba535a	420	io_u->resid = 0;
755200a3	421	io_u->file = NULL;
d7762cf8	422	io_u->end_io = NULL;
755200a3 JA	423	}
	424
	425	if (io_u) {
0c6e7517 JA	426	assert(io_u->flags & IO_U_F_FREE);
	427	io_u->flags &= ~IO_U_F_FREE;
	428
755200a3	429	io_u->error = 0;
10ba535a JA	430	list_del(&io_u->list);
	431	list_add(&io_u->list, &td->io_u_busylist);
	432	td->cur_depth++;
	433	}
	434
	435	return io_u;
	436	}
	437
	438	/*
	439	* Return an io_u to be processed. Gets a buflen and offset, sets direction,
	440	* etc. The returned io_u is fully ready to be prepped and submitted.
	441	*/
3d7c391d	442	struct io_u get_io_u(struct thread_data td)
10ba535a	443	{
3d7c391d	444	struct fio_file *f;
10ba535a	445	struct io_u *io_u;
1c178180	446	int ret;
10ba535a JA	447
	448	io_u = __get_io_u(td);
	449	if (!io_u)
	450	return NULL;
	451
755200a3 JA	452	/*
	453	* from a requeue, io_u already setup
	454	*/
	455	if (io_u->file)
77f392bf	456	goto out;
755200a3	457
bdb4e2e9 JA	458	do {
	459	f = get_next_file(td);
	460	if (!f) {
	461	put_io_u(td, io_u);
	462	return NULL;
	463	}
3d7c391d	464
1c178180	465	set_file:
bdb4e2e9	466	io_u->file = f;
3d7c391d	467
bdb4e2e9 JA	468	if (!fill_io_u(td, io_u))
bdb4e2e9 JA	469	break;
10ba535a	470
bdb4e2e9 JA	471	/*
	472	* No more to do for this file, close it
	473	*/
	474	io_u->file = NULL;
b5af8293 JA	475	td_io_close_file(td, f);
	476
	477	/*
	478	* probably not the right place to do this, but see
	479	* if we need to open a new file
	480	*/
	481	if (td->nr_open_files < td->nr_files &&
a978ba68	482	td->open_files != td->nr_files) {
1c178180	483	f = find_next_new_file(td);
a978ba68	484
1c178180	485	if (!f \|\| (ret = td_io_open_file(td, f))) {
a978ba68	486	put_io_u(td, io_u);
1c178180	487	break;
a978ba68	488	}
1c178180	489	goto set_file;
a978ba68	490	}
bdb4e2e9	491	} while (1);
10ba535a	492
9bf2061e JA	493	if (td->zone_bytes >= td->zone_size) {
	494	td->zone_bytes = 0;
	495	f->last_pos += td->zone_skip;
	496	}
	497
ee88470c	498	if (io_u->buflen + io_u->offset > f->real_file_size) {
10ba535a JA	499	if (td->io_ops->flags & FIO_RAWIO) {
	500	put_io_u(td, io_u);
	501	return NULL;
	502	}
	503
ee88470c	504	io_u->buflen = f->real_file_size - io_u->offset;
10ba535a JA	505	}
10ba535a JA	506
87dc1ab1 JA	507	if (io_u->ddir != DDIR_SYNC) {
	508	if (!io_u->buflen) {
	509	put_io_u(td, io_u);
	510	return NULL;
	511	}
10ba535a	512
36167d82	513	f->last_pos = io_u->offset + io_u->buflen;
10ba535a	514
87dc1ab1 JA	515	if (td->verify != VERIFY_NONE)
	516	populate_verify_io_u(td, io_u);
	517	}
10ba535a	518
165faf16 JA	519	/*
	520	* Set io data pointers.
	521	*/
77f392bf	522	out:
cec6b55d JA	523	io_u->xfer_buf = io_u->buf;
cec6b55d JA	524	io_u->xfer_buflen = io_u->buflen;
165faf16	525
36167d82 JA	526	if (td_io_prep(td, io_u)) {
	527	put_io_u(td, io_u);
	528	return NULL;
	529	}
	530
02bcaa8c	531	fio_gettime(&io_u->start_time, NULL);
10ba535a JA	532	return io_u;
	533	}
	534
5451792e JA	535	void io_u_log_error(struct thread_data td, struct io_u io_u)
	536	{
	537	const char *msg[] = { "read", "write", "sync" };
	538
	539	log_err("fio: io_u error");
	540
	541	if (io_u->file)
	542	log_err(" on file %s", io_u->file->file_name);
	543
	544	log_err(": %s\n", strerror(io_u->error));
	545
	546	log_err(" %s offset=%llu, buflen=%lu\n", msg[io_u->ddir], io_u->offset, io_u->xfer_buflen);
	547
	548	if (!td->error)
	549	td_verror(td, io_u->error, "io_u error");
	550	}
	551
97601024 JA	552	static void io_completed(struct thread_data td, struct io_u io_u,
97601024 JA	553	struct io_completion_data *icd)
10ba535a	554	{
10ba535a JA	555	unsigned long msec;
10ba535a JA	556
0c6e7517 JA	557	assert(io_u->flags & IO_U_F_FLIGHT);
	558	io_u->flags &= ~IO_U_F_FLIGHT;
	559
0ad920e7 JA	560	put_file(td, io_u->file);
0ad920e7 JA	561
87dc1ab1 JA	562	if (io_u->ddir == DDIR_SYNC) {
	563	td->last_was_sync = 1;
	564	return;
	565	}
	566
	567	td->last_was_sync = 0;
	568
10ba535a JA	569	if (!io_u->error) {
10ba535a JA	570	unsigned int bytes = io_u->buflen - io_u->resid;
1e97cce9	571	const enum fio_ddir idx = io_u->ddir;
3af6ef39	572	int ret;
10ba535a JA	573
	574	td->io_blocks[idx]++;
	575	td->io_bytes[idx] += bytes;
	576	td->zone_bytes += bytes;
	577	td->this_io_bytes[idx] += bytes;
	578
02bcaa8c JA	579	io_u->file->last_completed_pos = io_u->offset + io_u->buflen;
	580
	581	msec = mtime_since(&io_u->issue_time, &icd->time);
10ba535a JA	582
10ba535a JA	583	add_clat_sample(td, idx, msec);
02bcaa8c	584	add_bw_sample(td, idx, &icd->time);
ec118304	585	io_u_mark_latency(td, msec);
10ba535a JA	586
	587	if ((td_rw(td) \|\| td_write(td)) && idx == DDIR_WRITE)
	588	log_io_piece(td, io_u);
	589
	590	icd->bytes_done[idx] += bytes;
3af6ef39	591
d7762cf8 JA	592	if (io_u->end_io) {
d7762cf8 JA	593	ret = io_u->end_io(io_u);
3af6ef39 JA	594	if (ret && !icd->error)
	595	icd->error = ret;
	596	}
5451792e	597	} else {
10ba535a	598	icd->error = io_u->error;
5451792e JA	599	io_u_log_error(td, io_u);
5451792e JA	600	}
10ba535a JA	601	}
10ba535a JA	602
d7762cf8	603	static void init_icd(struct io_completion_data *icd, int nr)
10ba535a	604	{
02bcaa8c JA	605	fio_gettime(&icd->time, NULL);
02bcaa8c JA	606
3af6ef39 JA	607	icd->nr = nr;
3af6ef39 JA	608
10ba535a JA	609	icd->error = 0;
10ba535a JA	610	icd->bytes_done[0] = icd->bytes_done[1] = 0;
36167d82 JA	611	}
36167d82 JA	612
97601024 JA	613	static void ios_completed(struct thread_data *td,
97601024 JA	614	struct io_completion_data *icd)
36167d82 JA	615	{
	616	struct io_u *io_u;
	617	int i;
	618
10ba535a JA	619	for (i = 0; i < icd->nr; i++) {
	620	io_u = td->io_ops->event(td, i);
	621
	622	io_completed(td, io_u, icd);
	623	put_io_u(td, io_u);
	624	}
	625	}
97601024	626
e7e6cfb4 JA	627	/*
	628	* Complete a single io_u for the sync engines.
	629	*/
d7762cf8	630	long io_u_sync_complete(struct thread_data td, struct io_u io_u)
97601024 JA	631	{
	632	struct io_completion_data icd;
	633
d7762cf8	634	init_icd(&icd, 1);
97601024 JA	635	io_completed(td, io_u, &icd);
	636	put_io_u(td, io_u);
	637
	638	if (!icd.error)
	639	return icd.bytes_done[0] + icd.bytes_done[1];
	640
37e974a5	641	td_verror(td, icd.error, "io_u_sync_complete");
97601024 JA	642	return -1;
	643	}
	644
e7e6cfb4 JA	645	/*
	646	* Called to complete min_events number of io for the async engines.
	647	*/
d7762cf8	648	long io_u_queued_complete(struct thread_data *td, int min_events)
97601024	649	{
97601024	650	struct io_completion_data icd;
00de55ef	651	struct timespec *tvp = NULL;
97601024 JA	652	int ret;
97601024 JA	653
face81b2	654	if (!min_events) {
00de55ef JA	655	struct timespec ts = { .tv_sec = 0, .tv_nsec = 0, };
	656
	657	tvp = &ts;
755200a3	658	}
97601024	659
00de55ef	660	ret = td_io_getevents(td, min_events, td->cur_depth, tvp);
97601024	661	if (ret < 0) {
e1161c32	662	td_verror(td, -ret, "td_io_getevents");
97601024 JA	663	return ret;
	664	} else if (!ret)
	665	return ret;
	666
d7762cf8	667	init_icd(&icd, ret);
97601024 JA	668	ios_completed(td, &icd);
	669	if (!icd.error)
	670	return icd.bytes_done[0] + icd.bytes_done[1];
	671
37e974a5	672	td_verror(td, icd.error, "io_u_queued_complete");
97601024 JA	673	return -1;
97601024 JA	674	}
7e77dd02 JA	675
	676	/*
	677	* Call when io_u is really queued, to update the submission latency.
	678	*/
	679	void io_u_queued(struct thread_data td, struct io_u io_u)
	680	{
	681	unsigned long slat_time;
	682
	683	slat_time = mtime_since(&io_u->start_time, &io_u->issue_time);
	684	add_slat_sample(td, io_u->ddir, slat_time);
	685	}
433afcb4	686
55bc9728	687	#ifdef FIO_USE_TIMEOUT
433afcb4 JA	688	void io_u_set_timeout(struct thread_data *td)
	689	{
	690	assert(td->cur_depth);
	691
	692	td->timer.it_interval.tv_sec = 0;
	693	td->timer.it_interval.tv_usec = 0;
	694	td->timer.it_value.tv_sec = IO_U_TIMEOUT + IO_U_TIMEOUT_INC;
	695	td->timer.it_value.tv_usec = 0;
	696	setitimer(ITIMER_REAL, &td->timer, NULL);
	697	fio_gettime(&td->timeout_end, NULL);
	698	}
5945b9b4 JA	699
	700	static void io_u_dump(struct io_u *io_u)
	701	{
	702	unsigned long t_start = mtime_since_now(&io_u->start_time);
	703	unsigned long t_issue = mtime_since_now(&io_u->issue_time);
	704
	705	log_err("io_u=%p, t_start=%lu, t_issue=%lu\n", io_u, t_start, t_issue);
	706	log_err(" buf=%p/%p, len=%lu/%lu, offset=%llu\n", io_u->buf, io_u->xfer_buf, io_u->buflen, io_u->xfer_buflen, io_u->offset);
	707	log_err(" ddir=%d, fname=%s\n", io_u->ddir, io_u->file->file_name);
	708	}
55bc9728 JA	709	#else
	710	void io_u_set_timeout(struct thread_data fio_unused *td)
	711	{
	712	}
	713	#endif
433afcb4	714
55bc9728	715	#ifdef FIO_USE_TIMEOUT
433afcb4 JA	716	static void io_u_timeout_handler(int fio_unused sig)
	717	{
	718	struct thread_data td, __td;
	719	pid_t pid = getpid();
5945b9b4 JA	720	struct list_head *entry;
5945b9b4 JA	721	struct io_u *io_u;
433afcb4 JA	722	int i;
	723
	724	log_err("fio: io_u timeout\n");
	725
	726	/*
	727	* TLS would be nice...
	728	*/
	729	td = NULL;
	730	for_each_td(__td, i) {
	731	if (__td->pid == pid) {
	732	td = __td;
	733	break;
	734	}
	735	}
	736
	737	if (!td) {
	738	log_err("fio: io_u timeout, can't find job\n");
	739	exit(1);
	740	}
	741
	742	if (!td->cur_depth) {
	743	log_err("fio: timeout without pending work?\n");
	744	return;
	745	}
	746
	747	log_err("fio: io_u timeout: job=%s, pid=%d\n", td->name, td->pid);
5945b9b4 JA	748
	749	list_for_each(entry, &td->io_u_busylist) {
	750	io_u = list_entry(entry, struct io_u, list);
	751
	752	io_u_dump(io_u);
	753	}
	754
	755	td_verror(td, ETIMEDOUT, "io_u timeout");
433afcb4 JA	756	exit(1);
433afcb4 JA	757	}
55bc9728	758	#endif
433afcb4 JA	759
	760	void io_u_init_timeout(void)
	761	{
55bc9728	762	#ifdef FIO_USE_TIMEOUT
433afcb4	763	signal(SIGALRM, io_u_timeout_handler);
55bc9728	764	#endif
433afcb4	765	}