[fio.git] / ioengines.c

/*
 * The io parts of the fio tool, includes workers for sync and mmap'ed
 * io, as well as both posix and linux libaio support.
 *
 * sync io is implemented on top of aio.
 *
 * This is not really specific to fio, if the get_io_u/put_io_u and
 * structures was pulled into this as well it would be a perfectly
 * generic io engine that could be used for other projects.
 *
 */
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <dlfcn.h>
#include <fcntl.h>
#include <assert.h>

#include "fio.h"
#include "diskutil.h"
#include "zbd.h"

static FLIST_HEAD(engine_list);

static bool check_engine_ops(struct ioengine_ops *ops)
{
	if (ops->version != FIO_IOOPS_VERSION) {
		log_err("bad ioops version %d (want %d)\n", ops->version,
							FIO_IOOPS_VERSION);
		return true;
	}

	if (!ops->queue) {
		log_err("%s: no queue handler\n", ops->name);
		return true;
	}

	/*
	 * sync engines only need a ->queue()
	 */
	if (ops->flags & FIO_SYNCIO)
		return false;

	if (!ops->event || !ops->getevents) {
		log_err("%s: no event/getevents handler\n", ops->name);
		return true;
	}

	return false;
}

void unregister_ioengine(struct ioengine_ops *ops)
{
	dprint(FD_IO, "ioengine %s unregistered\n", ops->name);
	flist_del_init(&ops->list);
}

void register_ioengine(struct ioengine_ops *ops)
{
	dprint(FD_IO, "ioengine %s registered\n", ops->name);
	flist_add_tail(&ops->list, &engine_list);
}

static struct ioengine_ops *find_ioengine(const char *name)
{
	struct ioengine_ops *ops;
	struct flist_head *entry;

	flist_for_each(entry, &engine_list) {
		ops = flist_entry(entry, struct ioengine_ops, list);
		if (!strcmp(name, ops->name))
			return ops;
	}

	return NULL;
}

#ifdef CONFIG_DYNAMIC_ENGINES
static void *dlopen_external(struct thread_data *td, const char *engine)
{
	char engine_path[PATH_MAX];

	sprintf(engine_path, "%s/lib%s.so", FIO_EXT_ENG_DIR, engine);

	return dlopen(engine_path, RTLD_LAZY);
}
#else
#define dlopen_external(td, engine) (NULL)
#endif

static struct ioengine_ops *dlopen_ioengine(struct thread_data *td,
					    const char *engine_lib)
{
	struct ioengine_ops *ops;
	void *dlhandle;

	dprint(FD_IO, "dload engine %s\n", engine_lib);

	dlerror();
	dlhandle = dlopen(engine_lib, RTLD_LAZY);
	if (!dlhandle) {
		dlhandle = dlopen_external(td, engine_lib);
		if (!dlhandle) {
			td_vmsg(td, -1, dlerror(), "dlopen");
			return NULL;
		}
	}

	/*
	 * Unlike the included modules, external engines should have a
	 * non-static ioengine structure that we can reference.
	 */
	ops = dlsym(dlhandle, engine_lib);
	if (!ops)
		ops = dlsym(dlhandle, "ioengine");

	/*
	 * For some external engines (like C++ ones) it is not that trivial
	 * to provide a non-static ionengine structure that we can reference.
	 * Instead we call a method which allocates the required ioengine
	 * structure.
	 */
	if (!ops) {
		get_ioengine_t get_ioengine = dlsym(dlhandle, "get_ioengine");

		if (get_ioengine)
			get_ioengine(&ops);
	}

	if (!ops) {
		td_vmsg(td, -1, dlerror(), "dlsym");
		dlclose(dlhandle);
		return NULL;
	}

	td->io_ops_dlhandle = dlhandle;
	return ops;
}

static struct ioengine_ops *__load_ioengine(const char *engine)
{
	/*
	 * linux libaio has alias names, so convert to what we want
	 */
	if (!strncmp(engine, "linuxaio", 8)) {
		dprint(FD_IO, "converting ioengine name: %s -> libaio\n",
		       engine);
		engine = "libaio";
	}

	dprint(FD_IO, "load ioengine %s\n", engine);
	return find_ioengine(engine);
}

struct ioengine_ops *load_ioengine(struct thread_data *td)
{
	struct ioengine_ops *ops = NULL;
	const char *name;

	/*
	 * Use ->ioengine_so_path if an external ioengine path is specified.
	 * In this case, ->ioengine is "external" which also means the prefix
	 * for external ioengines "external:" is properly used.
	 */
	name = td->o.ioengine_so_path ?: td->o.ioengine;

	/*
	 * Try to load ->ioengine first, and if failed try to dlopen(3) either
	 * ->ioengine or ->ioengine_so_path.  This is redundant for an external
	 * ioengine with prefix, and also leaves the possibility of unexpected
	 * behavior (e.g. if the "external" ioengine exists), but we do this
	 * so as not to break job files not using the prefix.
	 */
	ops = __load_ioengine(td->o.ioengine);
	if (!ops)
		ops = dlopen_ioengine(td, name);

	/*
	 * If ops is NULL, we failed to load ->ioengine, and also failed to
	 * dlopen(3) either ->ioengine or ->ioengine_so_path as a path.
	 */
	if (!ops) {
		log_err("fio: engine %s not loadable\n", name);
		return NULL;
	}

	/*
	 * Check that the required methods are there.
	 */
	if (check_engine_ops(ops))
		return NULL;

	return ops;
}

/*
 * For cleaning up an ioengine which never made it to init().
 */
void free_ioengine(struct thread_data *td)
{
	dprint(FD_IO, "free ioengine %s\n", td->io_ops->name);

	if (td->eo && td->io_ops->options) {
		options_free(td->io_ops->options, td->eo);
		free(td->eo);
		td->eo = NULL;
	}

	if (td->io_ops_dlhandle) {
		dlclose(td->io_ops_dlhandle);
		td->io_ops_dlhandle = NULL;
	}

	td->io_ops = NULL;
}

void close_ioengine(struct thread_data *td)
{
	dprint(FD_IO, "close ioengine %s\n", td->io_ops->name);

	if (td->io_ops->cleanup) {
		td->io_ops->cleanup(td);
		td->io_ops_data = NULL;
	}

	free_ioengine(td);
}

int td_io_prep(struct thread_data *td, struct io_u *io_u)
{
	dprint_io_u(io_u, "prep");
	fio_ro_check(td, io_u);

	lock_file(td, io_u->file, io_u->ddir);

	if (td->io_ops->prep) {
		int ret = td->io_ops->prep(td, io_u);

		dprint(FD_IO, "prep: io_u %p: ret=%d\n", io_u, ret);

		if (ret)
			unlock_file(td, io_u->file);
		return ret;
	}

	return 0;
}

int td_io_getevents(struct thread_data *td, unsigned int min, unsigned int max,
		    const struct timespec *t)
{
	int r = 0;

	/*
	 * For ioengine=rdma one side operation RDMA_WRITE or RDMA_READ,
	 * server side gets a message from the client
	 * side that the task is finished, and
	 * td->done is set to 1 after td_io_commit(). In this case,
	 * there is no need to reap complete event in server side.
	 */
	if (td->done)
		return 0;

	if (min > 0 && td->io_ops->commit) {
		r = td->io_ops->commit(td);
		if (r < 0)
			goto out;
	}
	if (max > td->cur_depth)
		max = td->cur_depth;
	if (min > max)
		max = min;

	r = 0;
	if (max && td->io_ops->getevents)
		r = td->io_ops->getevents(td, min, max, t);
out:
	if (r >= 0) {
		/*
		 * Reflect that our submitted requests were retrieved with
		 * whatever OS async calls are in the underlying engine.
		 */
		td->io_u_in_flight -= r;
		io_u_mark_complete(td, r);
	} else
		td_verror(td, r, "get_events");

	dprint(FD_IO, "getevents: %d\n", r);
	return r;
}

enum fio_q_status td_io_queue(struct thread_data *td, struct io_u *io_u)
{
	const enum fio_ddir ddir = acct_ddir(io_u);
	unsigned long long buflen = io_u->xfer_buflen;
	enum fio_q_status ret;

	dprint_io_u(io_u, "queue");
	fio_ro_check(td, io_u);

	assert((io_u->flags & IO_U_F_FLIGHT) == 0);
	io_u_set(td, io_u, IO_U_F_FLIGHT);

	/*
	 * If overlap checking was enabled in offload mode we
	 * can release this lock that was acquired when we
	 * started the overlap check because the IO_U_F_FLIGHT
	 * flag is now set
	 */
	if (td_offload_overlap(td))
		pthread_mutex_unlock(&overlap_check);

	assert(fio_file_open(io_u->file));

	/*
	 * If using a write iolog, store this entry.
	 */
	log_io_u(td, io_u);

	io_u->error = 0;
	io_u->resid = 0;

	if (td_ioengine_flagged(td, FIO_SYNCIO) ||
		(td_ioengine_flagged(td, FIO_ASYNCIO_SYNC_TRIM) && 
		io_u->ddir == DDIR_TRIM)) {
		if (fio_fill_issue_time(td))
			fio_gettime(&io_u->issue_time, NULL);

		/*
		 * only used for iolog
		 */
		if (td->o.read_iolog_file)
			memcpy(&td->last_issue, &io_u->issue_time,
					sizeof(io_u->issue_time));
	}


	if (ddir_rw(ddir)) {
		if (!(io_u->flags & IO_U_F_VER_LIST)) {
			td->io_issues[ddir]++;
			td->io_issue_bytes[ddir] += buflen;
		}
		td->rate_io_issue_bytes[ddir] += buflen;
	}

	ret = td->io_ops->queue(td, io_u);
	zbd_queue_io_u(io_u, ret);

	unlock_file(td, io_u->file);

	if (ret == FIO_Q_BUSY && ddir_rw(ddir)) {
		td->io_issues[ddir]--;
		td->io_issue_bytes[ddir] -= buflen;
		td->rate_io_issue_bytes[ddir] -= buflen;
		io_u_clear(td, io_u, IO_U_F_FLIGHT);
	}

	/*
	 * If an error was seen and the io engine didn't propagate it
	 * back to 'td', do so.
	 */
	if (io_u->error && !td->error)
		td_verror(td, io_u->error, "td_io_queue");

	/*
	 * Add warning for O_DIRECT so that users have an easier time
	 * spotting potentially bad alignment. If this triggers for the first
	 * IO, then it's likely an alignment problem or because the host fs
	 * does not support O_DIRECT
	 */
	if (io_u->error == EINVAL && td->io_issues[io_u->ddir & 1] == 1 &&
	    td->o.odirect) {

		log_info("fio: first direct IO errored. File system may not "
			 "support direct IO, or iomem_align= is bad, or "
			 "invalid block size. Try setting direct=0.\n");
	}

	if (zbd_unaligned_write(io_u->error) &&
	    td->io_issues[io_u->ddir & 1] == 1 &&
	    td->o.zone_mode != ZONE_MODE_ZBD) {
		log_info("fio: first I/O failed. If %s is a zoned block device, consider --zonemode=zbd\n",
			 io_u->file->file_name);
	}

	if (!td->io_ops->commit) {
		io_u_mark_submit(td, 1);
		io_u_mark_complete(td, 1);
		zbd_put_io_u(io_u);
	}

	if (ret == FIO_Q_COMPLETED) {
		if (ddir_rw(io_u->ddir) ||
		    (ddir_sync(io_u->ddir) && td->runstate != TD_FSYNCING)) {
			io_u_mark_depth(td, 1);
			td->ts.total_io_u[io_u->ddir]++;
		}
	} else if (ret == FIO_Q_QUEUED) {
		td->io_u_queued++;

		if (ddir_rw(io_u->ddir) ||
		    (ddir_sync(io_u->ddir) && td->runstate != TD_FSYNCING))
			td->ts.total_io_u[io_u->ddir]++;

		if (td->io_u_queued >= td->o.iodepth_batch)
			td_io_commit(td);
	}

	if (!td_ioengine_flagged(td, FIO_SYNCIO) &&
		(!td_ioengine_flagged(td, FIO_ASYNCIO_SYNC_TRIM) ||
		 io_u->ddir != DDIR_TRIM)) {
		if (fio_fill_issue_time(td))
			fio_gettime(&io_u->issue_time, NULL);

		/*
		 * only used for iolog
		 */
		if (td->o.read_iolog_file)
			memcpy(&td->last_issue, &io_u->issue_time,
					sizeof(io_u->issue_time));
	}

	return ret;
}

int td_io_init(struct thread_data *td)
{
	int ret = 0;

	if (td->io_ops->init) {
		ret = td->io_ops->init(td);
		if (ret)
			log_err("fio: io engine %s init failed.%s\n",
				td->io_ops->name,
				td->o.iodepth > 1 ?
				" Perhaps try reducing io depth?" : "");
		else
			td->io_ops_init = 1;
		if (!td->error)
			td->error = ret;
	}

	return ret;
}

void td_io_commit(struct thread_data *td)
{
	int ret;

	dprint(FD_IO, "calling ->commit(), depth %d\n", td->cur_depth);

	if (!td->cur_depth || !td->io_u_queued)
		return;

	io_u_mark_depth(td, td->io_u_queued);

	if (td->io_ops->commit) {
		ret = td->io_ops->commit(td);
		if (ret)
			td_verror(td, -ret, "io commit");
	}

	/*
	 * Reflect that events were submitted as async IO requests.
	 */
	td->io_u_in_flight += td->io_u_queued;
	td->io_u_queued = 0;
}

int td_io_open_file(struct thread_data *td, struct fio_file *f)
{
	if (fio_file_closing(f)) {
		/*
		 * Open translates to undo closing.
		 */
		fio_file_clear_closing(f);
		get_file(f);
		return 0;
	}
	assert(!fio_file_open(f));
	assert(f->fd == -1);
	assert(td->io_ops->open_file);

	if (td->io_ops->open_file(td, f)) {
		if (td->error == EINVAL && td->o.odirect)
			log_err("fio: destination does not support O_DIRECT\n");
		if (td->error == EMFILE) {
			log_err("fio: try reducing/setting openfiles (failed"
				" at %u of %u)\n", td->nr_open_files,
							td->o.nr_files);
		}

		assert(f->fd == -1);
		assert(!fio_file_open(f));
		return 1;
	}

	fio_file_reset(td, f);
	fio_file_set_open(f);
	fio_file_clear_closing(f);
	disk_util_inc(f->du);

	td->nr_open_files++;
	get_file(f);

	if (f->filetype == FIO_TYPE_PIPE) {
		if (td_random(td)) {
			log_err("fio: can't seek on pipes (no random io)\n");
			goto err;
		}
	}

	if (td_ioengine_flagged(td, FIO_DISKLESSIO))
		goto done;

	if (td->o.invalidate_cache && file_invalidate_cache(td, f))
		goto err;

	if (td->o.fadvise_hint != F_ADV_NONE &&
	    (f->filetype == FIO_TYPE_BLOCK || f->filetype == FIO_TYPE_FILE)) {
		int flags;

		if (td->o.fadvise_hint == F_ADV_TYPE) {
			if (td_random(td))
				flags = POSIX_FADV_RANDOM;
			else
				flags = POSIX_FADV_SEQUENTIAL;
		} else if (td->o.fadvise_hint == F_ADV_RANDOM)
			flags = POSIX_FADV_RANDOM;
		else if (td->o.fadvise_hint == F_ADV_SEQUENTIAL)
			flags = POSIX_FADV_SEQUENTIAL;
		else {
			log_err("fio: unknown fadvise type %d\n",
							td->o.fadvise_hint);
			flags = POSIX_FADV_NORMAL;
		}

		if (posix_fadvise(f->fd, f->file_offset, f->io_size, flags) < 0) {
			if (!fio_did_warn(FIO_WARN_FADVISE))
				log_err("fio: fadvise hint failed\n");
		}
	}
#ifdef FIO_HAVE_WRITE_HINT
	if (fio_option_is_set(&td->o, write_hint) &&
	    (f->filetype == FIO_TYPE_BLOCK || f->filetype == FIO_TYPE_FILE)) {
		uint64_t hint = td->o.write_hint;
		int cmd;

		/*
		 * For direct IO, we just need/want to set the hint on
		 * the file descriptor. For buffered IO, we need to set
		 * it on the inode.
		 */
		if (td->o.odirect)
			cmd = F_SET_FILE_RW_HINT;
		else
			cmd = F_SET_RW_HINT;

		if (fcntl(f->fd, cmd, &hint) < 0) {
			td_verror(td, errno, "fcntl write hint");
			goto err;
		}
	}
#endif

	if (td->o.odirect && !OS_O_DIRECT && fio_set_directio(td, f))
		goto err;

done:
	log_file(td, f, FIO_LOG_OPEN_FILE);
	return 0;
err:
	disk_util_dec(f->du);
	if (td->io_ops->close_file)
		td->io_ops->close_file(td, f);
	return 1;
}

int td_io_close_file(struct thread_data *td, struct fio_file *f)
{
	if (!fio_file_closing(f))
		log_file(td, f, FIO_LOG_CLOSE_FILE);

	/*
	 * mark as closing, do real close when last io on it has completed
	 */
	fio_file_set_closing(f);

	return put_file(td, f);
}

int td_io_unlink_file(struct thread_data *td, struct fio_file *f)
{
	if (td->io_ops->unlink_file)
		return td->io_ops->unlink_file(td, f);
	else {
		int ret;

		ret = unlink(f->file_name);
		if (ret < 0)
			return errno;

		return 0;
	}
}

int td_io_get_file_size(struct thread_data *td, struct fio_file *f)
{
	if (!td->io_ops->get_file_size)
		return 0;

	return td->io_ops->get_file_size(td, f);
}

int fio_show_ioengine_help(const char *engine)
{
	struct flist_head *entry;
	struct thread_data td;
	struct ioengine_ops *io_ops;
	char *sep;
	int ret = 1;

	if (!engine || !*engine) {
		log_info("Available IO engines:\n");
		flist_for_each(entry, &engine_list) {
			io_ops = flist_entry(entry, struct ioengine_ops, list);
			log_info("\t%s\n", io_ops->name);
		}
		return 0;
	}
	sep = strchr(engine, ',');
	if (sep) {
		*sep = 0;
		sep++;
	}

	memset(&td, 0, sizeof(struct thread_data));
	td.o.ioengine = (char *)engine;
	io_ops = load_ioengine(&td);

	if (!io_ops) {
		log_info("IO engine %s not found\n", engine);
		return 1;
	}

	if (io_ops->options)
		ret = show_cmd_help(io_ops->options, sep);
	else
		log_info("IO engine %s has no options\n", io_ops->name);

	free_ioengine(&td);
	return ret;
}
Commit	Line	Data
ebac4655 JA	1	/*
	2	* The io parts of the fio tool, includes workers for sync and mmap'ed
	3	* io, as well as both posix and linux libaio support.
	4	*
	5	* sync io is implemented on top of aio.
	6	*
	7	* This is not really specific to fio, if the get_io_u/put_io_u and
	8	* structures was pulled into this as well it would be a perfectly
	9	* generic io engine that could be used for other projects.
	10	*
	11	*/
ebac4655 JA	12	#include <stdlib.h>
ebac4655 JA	13	#include <unistd.h>
5c4e1dbc	14	#include <string.h>
2866c82d	15	#include <dlfcn.h>
ecc314ba	16	#include <fcntl.h>
0c6e7517	17	#include <assert.h>
8c16d840	18
ebac4655	19	#include "fio.h"
7c9b1bce	20	#include "diskutil.h"
bfbdd35b	21	#include "zbd.h"
ebac4655	22
01743ee1	23	static FLIST_HEAD(engine_list);
5f350952	24
1cc954ba	25	static bool check_engine_ops(struct ioengine_ops *ops)
8c16d840	26	{
5f350952	27	if (ops->version != FIO_IOOPS_VERSION) {
5ec10eaa JA	28	log_err("bad ioops version %d (want %d)\n", ops->version,
5ec10eaa JA	29	FIO_IOOPS_VERSION);
1cc954ba	30	return true;
5f350952 JA	31	}
5f350952 JA	32
36167d82 JA	33	if (!ops->queue) {
36167d82 JA	34	log_err("%s: no queue handler\n", ops->name);
1cc954ba	35	return true;
36167d82 JA	36	}
	37
	38	/*
	39	* sync engines only need a ->queue()
	40	*/
	41	if (ops->flags & FIO_SYNCIO)
1cc954ba	42	return false;
5ec10eaa	43
1cc954ba JA	44	if (!ops->event \|\| !ops->getevents) {
	45	log_err("%s: no event/getevents handler\n", ops->name);
	46	return true;
8c16d840	47	}
5ec10eaa	48
1cc954ba	49	return false;
8c16d840 JA	50	}
8c16d840 JA	51
5f350952	52	void unregister_ioengine(struct ioengine_ops *ops)
ebac4655	53	{
ee56ad50	54	dprint(FD_IO, "ioengine %s unregistered\n", ops->name);
adf075fa	55	flist_del_init(&ops->list);
5f350952 JA	56	}
5f350952 JA	57
b2fdda43	58	void register_ioengine(struct ioengine_ops *ops)
5f350952	59	{
ee56ad50	60	dprint(FD_IO, "ioengine %s registered\n", ops->name);
01743ee1	61	flist_add_tail(&ops->list, &engine_list);
5f350952 JA	62	}
	63
	64	static struct ioengine_ops find_ioengine(const char name)
	65	{
	66	struct ioengine_ops *ops;
01743ee1	67	struct flist_head *entry;
ebac4655	68
01743ee1 JA	69	flist_for_each(entry, &engine_list) {
01743ee1 JA	70	ops = flist_entry(entry, struct ioengine_ops, list);
bc5b77a8	71	if (!strcmp(name, ops->name))
5f350952 JA	72	return ops;
	73	}
	74
	75	return NULL;
	76	}
	77
5a8a6a03 YK	78	#ifdef CONFIG_DYNAMIC_ENGINES
	79	static void dlopen_external(struct thread_data td, const char *engine)
	80	{
	81	char engine_path[PATH_MAX];
	82
	83	sprintf(engine_path, "%s/lib%s.so", FIO_EXT_ENG_DIR, engine);
	84
	85	return dlopen(engine_path, RTLD_LAZY);
	86	}
	87	#else
	88	#define dlopen_external(td, engine) (NULL)
	89	#endif
	90
5f350952 JA	91	static struct ioengine_ops dlopen_ioengine(struct thread_data td,
	92	const char *engine_lib)
	93	{
	94	struct ioengine_ops *ops;
	95	void *dlhandle;
	96
ee56ad50 JA	97	dprint(FD_IO, "dload engine %s\n", engine_lib);
ee56ad50 JA	98
2866c82d JA	99	dlerror();
2866c82d JA	100	dlhandle = dlopen(engine_lib, RTLD_LAZY);
d4dbaaa8	101	if (!dlhandle) {
5a8a6a03 YK	102	dlhandle = dlopen_external(td, engine_lib);
	103	if (!dlhandle) {
	104	td_vmsg(td, -1, dlerror(), "dlopen");
	105	return NULL;
	106	}
d4dbaaa8	107	}
8756e4d4	108
da51c050 JA	109	/*
	110	* Unlike the included modules, external engines should have a
	111	* non-static ioengine structure that we can reference.
	112	*/
0abea0b7 DM	113	ops = dlsym(dlhandle, engine_lib);
	114	if (!ops)
	115	ops = dlsym(dlhandle, "ioengine");
a8075704 DG	116
	117	/*
	118	* For some external engines (like C++ ones) it is not that trivial
	119	* to provide a non-static ionengine structure that we can reference.
	120	* Instead we call a method which allocates the required ioengine
	121	* structure.
	122	*/
	123	if (!ops) {
	124	get_ioengine_t get_ioengine = dlsym(dlhandle, "get_ioengine");
	125
	126	if (get_ioengine)
	127	get_ioengine(&ops);
	128	}
	129
d4dbaaa8	130	if (!ops) {
e1161c32	131	td_vmsg(td, -1, dlerror(), "dlsym");
d4dbaaa8 JA	132	dlclose(dlhandle);
	133	return NULL;
	134	}
8756e4d4	135
565e784d	136	td->io_ops_dlhandle = dlhandle;
5f350952 JA	137	return ops;
	138	}
	139
044be36e	140	static struct ioengine_ops __load_ioengine(const char engine)
97bb54c9	141	{
97bb54c9 TK	142	/*
	143	* linux libaio has alias names, so convert to what we want
	144	*/
59cd9313	145	if (!strncmp(engine, "linuxaio", 8)) {
044be36e BVA	146	dprint(FD_IO, "converting ioengine name: %s -> libaio\n",
	147	engine);
	148	engine = "libaio";
966fcbd4	149	}
97bb54c9 TK	150
	151	dprint(FD_IO, "load ioengine %s\n", engine);
	152	return find_ioengine(engine);
	153	}
	154
4fedf59a	155	struct ioengine_ops load_ioengine(struct thread_data td)
5f350952	156	{
81647a9a	157	struct ioengine_ops *ops = NULL;
ba872a0b	158	const char *name;
81647a9a	159
ba872a0b TK	160	/*
	161	* Use ->ioengine_so_path if an external ioengine path is specified.
	162	* In this case, ->ioengine is "external" which also means the prefix
	163	* for external ioengines "external:" is properly used.
	164	*/
	165	name = td->o.ioengine_so_path ?: td->o.ioengine;
	166
	167	/*
	168	* Try to load ->ioengine first, and if failed try to dlopen(3) either
	169	* ->ioengine or ->ioengine_so_path. This is redundant for an external
	170	* ioengine with prefix, and also leaves the possibility of unexpected
	171	* behavior (e.g. if the "external" ioengine exists), but we do this
	172	* so as not to break job files not using the prefix.
	173	*/
	174	ops = __load_ioengine(td->o.ioengine);
	175	if (!ops)
5f350952 JA	176	ops = dlopen_ioengine(td, name);
5f350952 JA	177
ba872a0b TK	178	/*
	179	* If ops is NULL, we failed to load ->ioengine, and also failed to
	180	* dlopen(3) either ->ioengine or ->ioengine_so_path as a path.
	181	*/
5f350952 JA	182	if (!ops) {
5f350952 JA	183	log_err("fio: engine %s not loadable\n", name);
b902ceb5 JA	184	return NULL;
	185	}
	186
8c16d840 JA	187	/*
	188	* Check that the required methods are there.
	189	*/
5f350952	190	if (check_engine_ops(ops))
8c16d840	191	return NULL;
8c16d840	192
565e784d	193	return ops;
8756e4d4 JA	194	}
8756e4d4 JA	195
de890a1e SL	196	/*
	197	* For cleaning up an ioengine which never made it to init().
	198	*/
	199	void free_ioengine(struct thread_data *td)
8756e4d4	200	{
de890a1e	201	dprint(FD_IO, "free ioengine %s\n", td->io_ops->name);
ee56ad50	202
de890a1e SL	203	if (td->eo && td->io_ops->options) {
	204	options_free(td->io_ops->options, td->eo);
	205	free(td->eo);
	206	td->eo = NULL;
2992b059	207	}
b990b5c0	208
9b50942e	209	if (td->io_ops_dlhandle) {
565e784d	210	dlclose(td->io_ops_dlhandle);
9b50942e JA	211	td->io_ops_dlhandle = NULL;
9b50942e JA	212	}
5f350952	213
84585003	214	td->io_ops = NULL;
b990b5c0	215	}
10ba535a	216
de890a1e SL	217	void close_ioengine(struct thread_data *td)
	218	{
	219	dprint(FD_IO, "close ioengine %s\n", td->io_ops->name);
	220
	221	if (td->io_ops->cleanup) {
	222	td->io_ops->cleanup(td);
565e784d	223	td->io_ops_data = NULL;
de890a1e SL	224	}
	225
	226	free_ioengine(td);
	227	}
	228
10ba535a JA	229	int td_io_prep(struct thread_data td, struct io_u io_u)
10ba535a JA	230	{
ee56ad50	231	dprint_io_u(io_u, "prep");
7101d9c2 JA	232	fio_ro_check(td, io_u);
7101d9c2 JA	233
4d4e80f2	234	lock_file(td, io_u->file, io_u->ddir);
b2bd2bd9	235
2ba1c290 JA	236	if (td->io_ops->prep) {
	237	int ret = td->io_ops->prep(td, io_u);
	238
e5f9a813 RE	239	dprint(FD_IO, "prep: io_u %p: ret=%d\n", io_u, ret);
e5f9a813 RE	240
b2bd2bd9	241	if (ret)
4d4e80f2	242	unlock_file(td, io_u->file);
2ba1c290 JA	243	return ret;
2ba1c290 JA	244	}
10ba535a JA	245
	246	return 0;
	247	}
	248
e7d2e616	249	int td_io_getevents(struct thread_data *td, unsigned int min, unsigned int max,
1f440ece	250	const struct timespec *t)
10ba535a	251	{
ee56ad50	252	int r = 0;
face81b2	253
a05d62b2 YR	254	/*
	255	* For ioengine=rdma one side operation RDMA_WRITE or RDMA_READ,
	256	* server side gets a message from the client
	257	* side that the task is finished, and
	258	* td->done is set to 1 after td_io_commit(). In this case,
	259	* there is no need to reap complete event in server side.
	260	*/
	261	if (td->done)
	262	return 0;
	263
ee56ad50 JA	264	if (min > 0 && td->io_ops->commit) {
ee56ad50 JA	265	r = td->io_ops->commit(td);
face81b2	266	if (r < 0)
ee56ad50	267	goto out;
face81b2	268	}
4950421a JA	269	if (max > td->cur_depth)
	270	max = td->cur_depth;
	271	if (min > max)
	272	max = min;
36167d82	273
ee56ad50	274	r = 0;
4950421a	275	if (max && td->io_ops->getevents)
ee56ad50 JA	276	r = td->io_ops->getevents(td, min, max, t);
ee56ad50 JA	277	out:
422f9e4b RM	278	if (r >= 0) {
422f9e4b RM	279	/*
3fd9efbc	280	* Reflect that our submitted requests were retrieved with
422f9e4b RM	281	* whatever OS async calls are in the underlying engine.
	282	*/
	283	td->io_u_in_flight -= r;
838bc709	284	io_u_mark_complete(td, r);
422f9e4b	285	} else
7c639b14	286	td_verror(td, r, "get_events");
f3e11d05	287
ee56ad50 JA	288	dprint(FD_IO, "getevents: %d\n", r);
ee56ad50 JA	289	return r;
10ba535a JA	290	}
10ba535a JA	291
d3b07186	292	enum fio_q_status td_io_queue(struct thread_data td, struct io_u io_u)
10ba535a	293	{
a9da8ab2	294	const enum fio_ddir ddir = acct_ddir(io_u);
5fff9543	295	unsigned long long buflen = io_u->xfer_buflen;
d3b07186	296	enum fio_q_status ret;
7e77dd02	297
ee56ad50	298	dprint_io_u(io_u, "queue");
7101d9c2 JA	299	fio_ro_check(td, io_u);
7101d9c2 JA	300
0c6e7517	301	assert((io_u->flags & IO_U_F_FLIGHT) == 0);
1651e431	302	io_u_set(td, io_u, IO_U_F_FLIGHT);
817ae977 VF	303
	304	/*
	305	* If overlap checking was enabled in offload mode we
	306	* can release this lock that was acquired when we
	307	* started the overlap check because the IO_U_F_FLIGHT
	308	* flag is now set
	309	*/
1368d95b	310	if (td_offload_overlap(td))
c06379a6	311	pthread_mutex_unlock(&overlap_check);
0c6e7517	312
d6aed795	313	assert(fio_file_open(io_u->file));
3d7b485f	314
bcd5abfa JA	315	/*
	316	* If using a write iolog, store this entry.
	317	*/
	318	log_io_u(td, io_u);
	319
11786802 JA	320	io_u->error = 0;
	321	io_u->resid = 0;
	322
04ba61df VF	323	if (td_ioengine_flagged(td, FIO_SYNCIO) \|\|
	324	(td_ioengine_flagged(td, FIO_ASYNCIO_SYNC_TRIM) &&
	325	io_u->ddir == DDIR_TRIM)) {
12d9d841	326	if (fio_fill_issue_time(td))
9520ebb9	327	fio_gettime(&io_u->issue_time, NULL);
d0c15328 JA	328
	329	/*
	330	* only used for iolog
	331	*/
	332	if (td->o.read_iolog_file)
	333	memcpy(&td->last_issue, &io_u->issue_time,
8b6a404c	334	sizeof(io_u->issue_time));
433afcb4 JA	335	}
433afcb4 JA	336
b2a432bf	337
a9da8ab2	338	if (ddir_rw(ddir)) {
a7d01f02 JA	339	if (!(io_u->flags & IO_U_F_VER_LIST)) {
	340	td->io_issues[ddir]++;
	341	td->io_issue_bytes[ddir] += buflen;
	342	}
50a8ce86	343	td->rate_io_issue_bytes[ddir] += buflen;
74d6277f	344	}
755200a3	345
7e77dd02	346	ret = td->io_ops->queue(td, io_u);
d9ed3e63	347	zbd_queue_io_u(io_u, ret);
5aeb77df	348
4d4e80f2	349	unlock_file(td, io_u->file);
b2bd2bd9	350
a9da8ab2 JA	351	if (ret == FIO_Q_BUSY && ddir_rw(ddir)) {
	352	td->io_issues[ddir]--;
	353	td->io_issue_bytes[ddir] -= buflen;
50a8ce86	354	td->rate_io_issue_bytes[ddir] -= buflen;
871467d9	355	io_u_clear(td, io_u, IO_U_F_FLIGHT);
cd8a19e6 JA	356	}
cd8a19e6 JA	357
39a43d34 JA	358	/*
	359	* If an error was seen and the io engine didn't propagate it
	360	* back to 'td', do so.
	361	*/
	362	if (io_u->error && !td->error)
	363	td_verror(td, io_u->error, "td_io_queue");
	364
cb211682 JA	365	/*
	366	* Add warning for O_DIRECT so that users have an easier time
	367	* spotting potentially bad alignment. If this triggers for the first
	368	* IO, then it's likely an alignment problem or because the host fs
	369	* does not support O_DIRECT
	370	*/
ff58fced	371	if (io_u->error == EINVAL && td->io_issues[io_u->ddir & 1] == 1 &&
cb211682	372	td->o.odirect) {
214ac7e0	373
cb211682	374	log_info("fio: first direct IO errored. File system may not "
78d4a262 TK	375	"support direct IO, or iomem_align= is bad, or "
78d4a262 TK	376	"invalid block size. Try setting direct=0.\n");
cb211682 JA	377	}
cb211682 JA	378
bfbdd35b BVA	379	if (zbd_unaligned_write(io_u->error) &&
	380	td->io_issues[io_u->ddir & 1] == 1 &&
	381	td->o.zone_mode != ZONE_MODE_ZBD) {
	382	log_info("fio: first I/O failed. If %s is a zoned block device, consider --zonemode=zbd\n",
	383	io_u->file->file_name);
	384	}
	385
c0681c9d	386	if (!td->io_ops->commit) {
838bc709 JA	387	io_u_mark_submit(td, 1);
838bc709 JA	388	io_u_mark_complete(td, 1);
d9ed3e63	389	zbd_put_io_u(io_u);
838bc709 JA	390	}
838bc709 JA	391
d8005759	392	if (ret == FIO_Q_COMPLETED) {
00615bfb SW	393	if (ddir_rw(io_u->ddir) \|\|
00615bfb SW	394	(ddir_sync(io_u->ddir) && td->runstate != TD_FSYNCING)) {
d8005759 JA	395	io_u_mark_depth(td, 1);
d8005759 JA	396	td->ts.total_io_u[io_u->ddir]++;
6eaf09d6	397	}
d8005759	398	} else if (ret == FIO_Q_QUEUED) {
e6727cbd JA	399	td->io_u_queued++;
e6727cbd JA	400
00615bfb SW	401	if (ddir_rw(io_u->ddir) \|\|
00615bfb SW	402	(ddir_sync(io_u->ddir) && td->runstate != TD_FSYNCING))
d8005759	403	td->ts.total_io_u[io_u->ddir]++;
d8005759	404
a80cb54b BVA	405	if (td->io_u_queued >= td->o.iodepth_batch)
a80cb54b BVA	406	td_io_commit(td);
eb7c8ae2	407	}
cb5ab512	408
04ba61df VF	409	if (!td_ioengine_flagged(td, FIO_SYNCIO) &&
	410	(!td_ioengine_flagged(td, FIO_ASYNCIO_SYNC_TRIM) \|\|
	411	io_u->ddir != DDIR_TRIM)) {
12d9d841	412	if (fio_fill_issue_time(td))
9520ebb9	413	fio_gettime(&io_u->issue_time, NULL);
d0c15328 JA	414
	415	/*
	416	* only used for iolog
	417	*/
	418	if (td->o.read_iolog_file)
	419	memcpy(&td->last_issue, &io_u->issue_time,
8b6a404c	420	sizeof(io_u->issue_time));
433afcb4 JA	421	}
433afcb4 JA	422
7e77dd02	423	return ret;
10ba535a	424	}
8c16d840 JA	425
	426	int td_io_init(struct thread_data *td)
	427	{
eeb12160	428	int ret = 0;
8c16d840	429
eeb12160 JA	430	if (td->io_ops->init) {
eeb12160 JA	431	ret = td->io_ops->init(td);
356ef1a1 TK	432	if (ret)
	433	log_err("fio: io engine %s init failed.%s\n",
	434	td->io_ops->name,
	435	td->o.iodepth > 1 ?
	436	" Perhaps try reducing io depth?" : "");
	437	else
	438	td->io_ops_init = 1;
7c973896 JA	439	if (!td->error)
7c973896 JA	440	td->error = ret;
eeb12160 JA	441	}
	442
	443	return ret;
8c16d840	444	}
755200a3	445
a80cb54b	446	void td_io_commit(struct thread_data *td)
755200a3	447	{
f3e11d05 JA	448	int ret;
f3e11d05 JA	449
ee56ad50 JA	450	dprint(FD_IO, "calling ->commit(), depth %d\n", td->cur_depth);
ee56ad50 JA	451
d8005759	452	if (!td->cur_depth \|\| !td->io_u_queued)
a80cb54b	453	return;
cb5ab512	454
3fd9efbc	455	io_u_mark_depth(td, td->io_u_queued);
d8005759	456
f3e11d05 JA	457	if (td->io_ops->commit) {
	458	ret = td->io_ops->commit(td);
	459	if (ret)
	460	td_verror(td, -ret, "io commit");
	461	}
3fd9efbc	462
422f9e4b RM	463	/*
	464	* Reflect that events were submitted as async IO requests.
	465	*/
	466	td->io_u_in_flight += td->io_u_queued;
	467	td->io_u_queued = 0;
755200a3	468	}
b5af8293 JA	469
	470	int td_io_open_file(struct thread_data td, struct fio_file f)
	471	{
230f33fb AK	472	if (fio_file_closing(f)) {
	473	/*
	474	* Open translates to undo closing.
	475	*/
	476	fio_file_clear_closing(f);
	477	get_file(f);
	478	return 0;
	479	}
22a57ba8 JA	480	assert(!fio_file_open(f));
22a57ba8 JA	481	assert(f->fd == -1);
966fcbd4	482	assert(td->io_ops->open_file);
22a57ba8	483
413d6693 JA	484	if (td->io_ops->open_file(td, f)) {
	485	if (td->error == EINVAL && td->o.odirect)
	486	log_err("fio: destination does not support O_DIRECT\n");
5ec10eaa JA	487	if (td->error == EMFILE) {
	488	log_err("fio: try reducing/setting openfiles (failed"
	489	" at %u of %u)\n", td->nr_open_files,
	490	td->o.nr_files);
	491	}
413d6693	492
22a57ba8 JA	493	assert(f->fd == -1);
22a57ba8 JA	494	assert(!fio_file_open(f));
413d6693 JA	495	return 1;
	496	}
	497
33c48814	498	fio_file_reset(td, f);
d6aed795 JA	499	fio_file_set_open(f);
d6aed795 JA	500	fio_file_clear_closing(f);
c97bd0fa	501	disk_util_inc(f->du);
d5707a35 JA	502
	503	td->nr_open_files++;
	504	get_file(f);
	505
66159828 JA	506	if (f->filetype == FIO_TYPE_PIPE) {
	507	if (td_random(td)) {
	508	log_err("fio: can't seek on pipes (no random io)\n");
	509	goto err;
	510	}
	511	}
	512
9b87f09b	513	if (td_ioengine_flagged(td, FIO_DISKLESSIO))
413d6693 JA	514	goto done;
	515
	516	if (td->o.invalidate_cache && file_invalidate_cache(td, f))
	517	goto err;
	518
ecb2083d	519	if (td->o.fadvise_hint != F_ADV_NONE &&
686fbd31	520	(f->filetype == FIO_TYPE_BLOCK \|\| f->filetype == FIO_TYPE_FILE)) {
413d6693 JA	521	int flags;
413d6693 JA	522
ecb2083d JA	523	if (td->o.fadvise_hint == F_ADV_TYPE) {
	524	if (td_random(td))
	525	flags = POSIX_FADV_RANDOM;
	526	else
	527	flags = POSIX_FADV_SEQUENTIAL;
	528	} else if (td->o.fadvise_hint == F_ADV_RANDOM)
413d6693	529	flags = POSIX_FADV_RANDOM;
ecb2083d	530	else if (td->o.fadvise_hint == F_ADV_SEQUENTIAL)
413d6693	531	flags = POSIX_FADV_SEQUENTIAL;
ecb2083d JA	532	else {
	533	log_err("fio: unknown fadvise type %d\n",
	534	td->o.fadvise_hint);
	535	flags = POSIX_FADV_NORMAL;
	536	}
413d6693	537
ecc314ba	538	if (posix_fadvise(f->fd, f->file_offset, f->io_size, flags) < 0) {
825b0a75 JA	539	if (!fio_did_warn(FIO_WARN_FADVISE))
825b0a75 JA	540	log_err("fio: fadvise hint failed\n");
413d6693	541	}
7bb48f84	542	}
ae8e559e JA	543	#ifdef FIO_HAVE_WRITE_HINT
ae8e559e JA	544	if (fio_option_is_set(&td->o, write_hint) &&
686fbd31	545	(f->filetype == FIO_TYPE_BLOCK \|\| f->filetype == FIO_TYPE_FILE)) {
ae8e559e	546	uint64_t hint = td->o.write_hint;
bd553af6	547	int cmd;
37659335	548
bd553af6 JA	549	/*
	550	* For direct IO, we just need/want to set the hint on
	551	* the file descriptor. For buffered IO, we need to set
	552	* it on the inode.
	553	*/
	554	if (td->o.odirect)
	555	cmd = F_SET_FILE_RW_HINT;
	556	else
	557	cmd = F_SET_RW_HINT;
	558
	559	if (fcntl(f->fd, cmd, &hint) < 0) {
ae8e559e	560	td_verror(td, errno, "fcntl write hint");
37659335 JA	561	goto err;
	562	}
	563	}
	564	#endif
a978ba68	565
47534cda TK	566	if (td->o.odirect && !OS_O_DIRECT && fio_set_directio(td, f))
47534cda TK	567	goto err;
e116f2b9	568
413d6693	569	done:
f29b25a3	570	log_file(td, f, FIO_LOG_OPEN_FILE);
413d6693 JA	571	return 0;
413d6693 JA	572	err:
c97bd0fa	573	disk_util_dec(f->du);
b284075a JA	574	if (td->io_ops->close_file)
b284075a JA	575	td->io_ops->close_file(td, f);
7bb48f84	576	return 1;
b5af8293 JA	577	}
b5af8293 JA	578
6977bcd0	579	int td_io_close_file(struct thread_data td, struct fio_file f)
b5af8293	580	{
d6aed795	581	if (!fio_file_closing(f))
f29b25a3 JA	582	log_file(td, f, FIO_LOG_CLOSE_FILE);
f29b25a3 JA	583
0ad920e7 JA	584	/*
	585	* mark as closing, do real close when last io on it has completed
	586	*/
d6aed795	587	fio_file_set_closing(f);
0ad920e7	588
6977bcd0	589	return put_file(td, f);
b5af8293	590	}
df9c26b1	591
38ef9c90 CF	592	int td_io_unlink_file(struct thread_data td, struct fio_file f)
	593	{
	594	if (td->io_ops->unlink_file)
	595	return td->io_ops->unlink_file(td, f);
2442c935 JA	596	else {
	597	int ret;
	598
	599	ret = unlink(f->file_name);
	600	if (ret < 0)
	601	return errno;
	602
	603	return 0;
	604	}
38ef9c90 CF	605	}
38ef9c90 CF	606
df9c26b1 JA	607	int td_io_get_file_size(struct thread_data td, struct fio_file f)
	608	{
	609	if (!td->io_ops->get_file_size)
	610	return 0;
	611
	612	return td->io_ops->get_file_size(td, f);
	613	}
44f29692	614
de890a1e SL	615	int fio_show_ioengine_help(const char *engine)
	616	{
	617	struct flist_head *entry;
d04f1d5b	618	struct thread_data td;
755dcbbd	619	struct ioengine_ops *io_ops;
de890a1e SL	620	char *sep;
	621	int ret = 1;
	622
	623	if (!engine \|\| !*engine) {
	624	log_info("Available IO engines:\n");
	625	flist_for_each(entry, &engine_list) {
755dcbbd TK	626	io_ops = flist_entry(entry, struct ioengine_ops, list);
755dcbbd TK	627	log_info("\t%s\n", io_ops->name);
de890a1e SL	628	}
	629	return 0;
	630	}
	631	sep = strchr(engine, ',');
	632	if (sep) {
	633	*sep = 0;
	634	sep++;
	635	}
	636
d04f1d5b JM	637	memset(&td, 0, sizeof(struct thread_data));
	638	td.o.ioengine = (char *)engine;
	639	io_ops = load_ioengine(&td);
	640
755dcbbd	641	if (!io_ops) {
de890a1e SL	642	log_info("IO engine %s not found\n", engine);
	643	return 1;
	644	}
	645
755dcbbd TK	646	if (io_ops->options)
755dcbbd TK	647	ret = show_cmd_help(io_ops->options, sep);
de890a1e	648	else
755dcbbd	649	log_info("IO engine %s has no options\n", io_ops->name);
de890a1e	650
d04f1d5b	651	free_ioengine(&td);
de890a1e SL	652	return ret;
de890a1e SL	653	}