[fio.git] / ioengine.h

#ifndef FIO_IOENGINE_H
#define FIO_IOENGINE_H

#include "compiler/compiler.h"
#include "os/os.h"
#include "log.h"
#include "io_ddir.h"
#include "debug.h"
#include "file.h"
#include "workqueue.h"

#ifdef CONFIG_LIBAIO
#include <libaio.h>
#endif
#ifdef CONFIG_GUASI
#include <guasi.h>
#endif

#define FIO_IOOPS_VERSION	23

enum {
	IO_U_F_FREE		= 1 << 0,
	IO_U_F_FLIGHT		= 1 << 1,
	IO_U_F_NO_FILE_PUT	= 1 << 2,
	IO_U_F_IN_CUR_DEPTH	= 1 << 3,
	IO_U_F_BUSY_OK		= 1 << 4,
	IO_U_F_TRIMMED		= 1 << 5,
	IO_U_F_BARRIER		= 1 << 6,
	IO_U_F_VER_LIST		= 1 << 7,
};

/*
 * The io unit
 */
struct io_u {
	struct timeval start_time;
	struct timeval issue_time;

	struct fio_file *file;
	unsigned int flags;
	enum fio_ddir ddir;

	/*
	 * For replay workloads, we may want to account as a different
	 * IO type than what is being submitted.
	 */
	enum fio_ddir acct_ddir;

	/*
	 * Write generation
	 */
	unsigned short numberio;

	/*
	 * Allocated/set buffer and length
	 */
	unsigned long buflen;
	unsigned long long offset;
	void *buf;

	/*
	 * Initial seed for generating the buffer contents
	 */
	uint64_t rand_seed;

	/*
	 * IO engine state, may be different from above when we get
	 * partial transfers / residual data counts
	 */
	void *xfer_buf;
	unsigned long xfer_buflen;

	/*
	 * Parameter related to pre-filled buffers and
	 * their size to handle variable block sizes.
	 */
	unsigned long buf_filled_len;

	struct io_piece *ipo;

	unsigned int resid;
	unsigned int error;

	/*
	 * io engine private data
	 */
	union {
		unsigned int index;
		unsigned int seen;
		void *engine_data;
	};

	union {
		struct flist_head verify_list;
		struct workqueue_work work;
	};

	/*
	 * Callback for io completion
	 */
	int (*end_io)(struct thread_data *, struct io_u **);

	union {
#ifdef CONFIG_LIBAIO
		struct iocb iocb;
#endif
#ifdef CONFIG_POSIXAIO
		os_aiocb_t aiocb;
#endif
#ifdef FIO_HAVE_SGIO
		struct sg_io_hdr hdr;
#endif
#ifdef CONFIG_GUASI
		guasi_req_t greq;
#endif
#ifdef CONFIG_SOLARISAIO
		aio_result_t resultp;
#endif
#ifdef FIO_HAVE_BINJECT
		struct b_user_cmd buc;
#endif
#ifdef CONFIG_RDMA
		struct ibv_mr *mr;
#endif
		void *mmap_data;
		uint64_t null;
	};
};

/*
 * io_ops->queue() return values
 */
enum {
	FIO_Q_COMPLETED	= 0,		/* completed sync */
	FIO_Q_QUEUED	= 1,		/* queued, will complete async */
	FIO_Q_BUSY	= 2,		/* no more room, call ->commit() */
};

struct ioengine_ops {
	struct flist_head list;
	char name[16];
	int version;
	int flags;
	int (*setup)(struct thread_data *);
	int (*init)(struct thread_data *);
	int (*prep)(struct thread_data *, struct io_u *);
	int (*queue)(struct thread_data *, struct io_u *);
	int (*commit)(struct thread_data *);
	int (*getevents)(struct thread_data *, unsigned int, unsigned int, const struct timespec *);
	struct io_u *(*event)(struct thread_data *, int);
	char *(*errdetails)(struct io_u *);
	int (*cancel)(struct thread_data *, struct io_u *);
	void (*cleanup)(struct thread_data *);
	int (*open_file)(struct thread_data *, struct fio_file *);
	int (*close_file)(struct thread_data *, struct fio_file *);
	int (*invalidate)(struct thread_data *, struct fio_file *);
	int (*unlink_file)(struct thread_data *, struct fio_file *);
	int (*get_file_size)(struct thread_data *, struct fio_file *);
	void (*terminate)(struct thread_data *);
	int (*iomem_alloc)(struct thread_data *, size_t);
	void (*iomem_free)(struct thread_data *);
	int (*io_u_init)(struct thread_data *, struct io_u *);
	void (*io_u_free)(struct thread_data *, struct io_u *);
	int option_struct_size;
	struct fio_option *options;
	void *data;
	void *dlhandle;
};

enum fio_ioengine_flags {
	FIO_SYNCIO	= 1 << 0,	/* io engine has synchronous ->queue */
	FIO_RAWIO	= 1 << 1,	/* some sort of direct/raw io */
	FIO_DISKLESSIO	= 1 << 2,	/* no disk involved */
	FIO_NOEXTEND	= 1 << 3,	/* engine can't extend file */
	FIO_NODISKUTIL  = 1 << 4,	/* diskutil can't handle filename */
	FIO_UNIDIR	= 1 << 5,	/* engine is uni-directional */
	FIO_NOIO	= 1 << 6,	/* thread does only pseudo IO */
	FIO_PIPEIO	= 1 << 7,	/* input/output no seekable */
	FIO_BARRIER	= 1 << 8,	/* engine supports barriers */
	FIO_MEMALIGN	= 1 << 9,	/* engine wants aligned memory */
	FIO_BIT_BASED	= 1 << 10,	/* engine uses a bit base (e.g. uses Kbit as opposed to KB) */
	FIO_FAKEIO	= 1 << 11,	/* engine pretends to do IO */
};

/*
 * External engine defined symbol to fill in the engine ops structure
 */
typedef void (*get_ioengine_t)(struct ioengine_ops **);

/*
 * io engine entry points
 */
extern int __must_check td_io_init(struct thread_data *);
extern int __must_check td_io_prep(struct thread_data *, struct io_u *);
extern int __must_check td_io_queue(struct thread_data *, struct io_u *);
extern int __must_check td_io_sync(struct thread_data *, struct fio_file *);
extern int __must_check td_io_getevents(struct thread_data *, unsigned int, unsigned int, const struct timespec *);
extern int __must_check td_io_commit(struct thread_data *);
extern int __must_check td_io_open_file(struct thread_data *, struct fio_file *);
extern int td_io_close_file(struct thread_data *, struct fio_file *);
extern int td_io_unlink_file(struct thread_data *, struct fio_file *);
extern int __must_check td_io_get_file_size(struct thread_data *, struct fio_file *);

extern struct ioengine_ops *load_ioengine(struct thread_data *, const char *);
extern void register_ioengine(struct ioengine_ops *);
extern void unregister_ioengine(struct ioengine_ops *);
extern void free_ioengine(struct thread_data *);
extern void close_ioengine(struct thread_data *);

extern int fio_show_ioengine_help(const char *engine);

/*
 * io unit handling
 */
extern struct io_u *__get_io_u(struct thread_data *);
extern struct io_u *get_io_u(struct thread_data *);
extern void put_io_u(struct thread_data *, struct io_u *);
extern void clear_io_u(struct thread_data *, struct io_u *);
extern void requeue_io_u(struct thread_data *, struct io_u **);
extern int __must_check io_u_sync_complete(struct thread_data *, struct io_u *);
extern int __must_check io_u_queued_complete(struct thread_data *, int);
extern void io_u_queued(struct thread_data *, struct io_u *);
extern int io_u_quiesce(struct thread_data *);
extern void io_u_log_error(struct thread_data *, struct io_u *);
extern void io_u_mark_depth(struct thread_data *, unsigned int);
extern void fill_io_buffer(struct thread_data *, void *, unsigned int, unsigned int);
extern void io_u_fill_buffer(struct thread_data *td, struct io_u *, unsigned int, unsigned int);
void io_u_mark_complete(struct thread_data *, unsigned int);
void io_u_mark_submit(struct thread_data *, unsigned int);
bool queue_full(const struct thread_data *);

int do_io_u_sync(const struct thread_data *, struct io_u *);
int do_io_u_trim(const struct thread_data *, struct io_u *);

#ifdef FIO_INC_DEBUG
static inline void dprint_io_u(struct io_u *io_u, const char *p)
{
	struct fio_file *f = io_u->file;

	dprint(FD_IO, "%s: io_u %p: off=%llu/len=%lu/ddir=%d", p, io_u,
					(unsigned long long) io_u->offset,
					io_u->buflen, io_u->ddir);
	if (fio_debug & (1 << FD_IO)) {
		if (f)
			log_info("/%s", f->file_name);

		log_info("\n");
	}
}
#else
#define dprint_io_u(io_u, p)
#endif

static inline enum fio_ddir acct_ddir(struct io_u *io_u)
{
	if (io_u->acct_ddir != -1)
		return io_u->acct_ddir;

	return io_u->ddir;
}

static inline void io_u_clear(struct io_u *io_u, unsigned int flags)
{
	__sync_fetch_and_and(&io_u->flags, ~flags);
}

static inline void io_u_set(struct io_u *io_u, unsigned int flags)
{
	__sync_fetch_and_or(&io_u->flags, flags);
}

#endif
Commit	Line	Data
dcefb588 JA	1	#ifndef FIO_IOENGINE_H
	2	#define FIO_IOENGINE_H
	3
41666588	4	#include "compiler/compiler.h"
836fcc0f JA	5	#include "os/os.h"
836fcc0f JA	6	#include "log.h"
41666588	7	#include "io_ddir.h"
ec41265e	8	#include "debug.h"
41666588	9	#include "file.h"
88271841	10	#include "workqueue.h"
ec41265e	11
67bf9823 JA	12	#ifdef CONFIG_LIBAIO
	13	#include <libaio.h>
	14	#endif
	15	#ifdef CONFIG_GUASI
	16	#include <guasi.h>
	17	#endif
	18
a6cb85e2	19	#define FIO_IOOPS_VERSION 23
dcefb588 JA	20
dcefb588 JA	21	enum {
0c41214f RR	22	IO_U_F_FREE = 1 << 0,
0c41214f RR	23	IO_U_F_FLIGHT = 1 << 1,
f8b0bd10	24	IO_U_F_NO_FILE_PUT = 1 << 2,
0c41214f	25	IO_U_F_IN_CUR_DEPTH = 1 << 3,
38dad62d	26	IO_U_F_BUSY_OK = 1 << 4,
0d29de83	27	IO_U_F_TRIMMED = 1 << 5,
1ef2b6be	28	IO_U_F_BARRIER = 1 << 6,
82af2a7c	29	IO_U_F_VER_LIST = 1 << 7,
dcefb588 JA	30	};
	31
	32	/*
	33	* The io unit
	34	*/
	35	struct io_u {
dcefb588 JA	36	struct timeval start_time;
	37	struct timeval issue_time;
	38
d72be545 JA	39	struct fio_file *file;
	40	unsigned int flags;
	41	enum fio_ddir ddir;
	42
bcd5abfa JA	43	/*
	44	* For replay workloads, we may want to account as a different
	45	* IO type than what is being submitted.
	46	*/
	47	enum fio_ddir acct_ddir;
	48
d8f1f7d4 JA	49	/*
	50	* Write generation
	51	*/
	52	unsigned short numberio;
	53
dcefb588 JA	54	/*
	55	* Allocated/set buffer and length
	56	*/
dcefb588 JA	57	unsigned long buflen;
dcefb588 JA	58	unsigned long long offset;
d72be545	59	void *buf;
dcefb588	60
7d9fb455 JA	61	/*
	62	* Initial seed for generating the buffer contents
	63	*/
363cffa7	64	uint64_t rand_seed;
7d9fb455	65
dcefb588 JA	66	/*
	67	* IO engine state, may be different from above when we get
	68	* partial transfers / residual data counts
	69	*/
	70	void *xfer_buf;
	71	unsigned long xfer_buflen;
	72
95228507 JA	73	/*
	74	* Parameter related to pre-filled buffers and
	75	* their size to handle variable block sizes.
	76	*/
	77	unsigned long buf_filled_len;
	78
f9401285 JA	79	struct io_piece *ipo;
f9401285 JA	80
225ba9e3 JA	81	unsigned int resid;
	82	unsigned int error;
	83
	84	/*
	85	* io engine private data
	86	*/
	87	union {
	88	unsigned int index;
	89	unsigned int seen;
	90	void *engine_data;
	91	};
	92
88271841 JA	93	union {
	94	struct flist_head verify_list;
	95	struct workqueue_work work;
	96	};
225ba9e3 JA	97
	98	/*
	99	* Callback for io completion
	100	*/
f8b0bd10	101	int (end_io)(struct thread_data , struct io_u **);
225ba9e3	102
2ae0b204 JA	103	union {
	104	#ifdef CONFIG_LIBAIO
	105	struct iocb iocb;
	106	#endif
	107	#ifdef CONFIG_POSIXAIO
	108	os_aiocb_t aiocb;
	109	#endif
	110	#ifdef FIO_HAVE_SGIO
	111	struct sg_io_hdr hdr;
	112	#endif
	113	#ifdef CONFIG_GUASI
	114	guasi_req_t greq;
	115	#endif
	116	#ifdef CONFIG_SOLARISAIO
	117	aio_result_t resultp;
	118	#endif
	119	#ifdef FIO_HAVE_BINJECT
	120	struct b_user_cmd buc;
	121	#endif
	122	#ifdef CONFIG_RDMA
	123	struct ibv_mr *mr;
	124	#endif
	125	void *mmap_data;
a9da8ab2	126	uint64_t null;
2ae0b204	127	};
dcefb588 JA	128	};
	129
	130	/*
	131	* io_ops->queue() return values
	132	*/
	133	enum {
	134	FIO_Q_COMPLETED = 0, /* completed sync */
	135	FIO_Q_QUEUED = 1, /* queued, will complete async */
	136	FIO_Q_BUSY = 2, /* no more room, call ->commit() */
	137	};
	138
	139	struct ioengine_ops {
	140	struct flist_head list;
	141	char name[16];
	142	int version;
	143	int flags;
	144	int (setup)(struct thread_data );
	145	int (init)(struct thread_data );
	146	int (prep)(struct thread_data , struct io_u *);
	147	int (queue)(struct thread_data , struct io_u *);
	148	int (commit)(struct thread_data );
1f440ece	149	int (getevents)(struct thread_data , unsigned int, unsigned int, const struct timespec *);
dcefb588	150	struct io_u (event)(struct thread_data *, int);
5ad7be56	151	char (errdetails)(struct io_u *);
dcefb588 JA	152	int (cancel)(struct thread_data , struct io_u *);
	153	void (cleanup)(struct thread_data );
	154	int (open_file)(struct thread_data , struct fio_file *);
	155	int (close_file)(struct thread_data , struct fio_file *);
d9b100fc	156	int (invalidate)(struct thread_data , struct fio_file *);
38ef9c90	157	int (unlink_file)(struct thread_data , struct fio_file *);
dcefb588	158	int (get_file_size)(struct thread_data , struct fio_file *);
36d80bc7	159	void (terminate)(struct thread_data );
a6cb85e2 JA	160	int (iomem_alloc)(struct thread_data , size_t);
a6cb85e2 JA	161	void (iomem_free)(struct thread_data );
c73ed246 JA	162	int (io_u_init)(struct thread_data , struct io_u *);
c73ed246 JA	163	void (io_u_free)(struct thread_data , struct io_u *);
de890a1e SL	164	int option_struct_size;
de890a1e SL	165	struct fio_option *options;
dcefb588 JA	166	void *data;
	167	void *dlhandle;
	168	};
	169
2b4f4abe JA	170	enum fio_ioengine_flags {
	171	FIO_SYNCIO = 1 << 0, /* io engine has synchronous ->queue */
	172	FIO_RAWIO = 1 << 1, /* some sort of direct/raw io */
	173	FIO_DISKLESSIO = 1 << 2, /* no disk involved */
	174	FIO_NOEXTEND = 1 << 3, /* engine can't extend file */
93bcfd20	175	FIO_NODISKUTIL = 1 << 4, /* diskutil can't handle filename */
2b4f4abe JA	176	FIO_UNIDIR = 1 << 5, /* engine is uni-directional */
2b4f4abe JA	177	FIO_NOIO = 1 << 6, /* thread does only pseudo IO */
36d80bc7 JA	178	FIO_PIPEIO = 1 << 7, /* input/output no seekable */
	179	FIO_BARRIER = 1 << 8, /* engine supports barriers */
	180	FIO_MEMALIGN = 1 << 9, /* engine wants aligned memory */
ad705bcb	181	FIO_BIT_BASED = 1 << 10, /* engine uses a bit base (e.g. uses Kbit as opposed to KB) */
5c57c084	182	FIO_FAKEIO = 1 << 11, /* engine pretends to do IO */
2b4f4abe	183	};
dcefb588	184
a8075704 DG	185	/*
	186	* External engine defined symbol to fill in the engine ops structure
	187	*/
	188	typedef void (get_ioengine_t)(struct ioengine_ops *);
	189
dcefb588 JA	190	/*
	191	* io engine entry points
	192	*/
	193	extern int __must_check td_io_init(struct thread_data *);
	194	extern int __must_check td_io_prep(struct thread_data , struct io_u );
	195	extern int __must_check td_io_queue(struct thread_data , struct io_u );
	196	extern int __must_check td_io_sync(struct thread_data , struct fio_file );
1f440ece	197	extern int __must_check td_io_getevents(struct thread_data , unsigned int, unsigned int, const struct timespec );
dcefb588 JA	198	extern int __must_check td_io_commit(struct thread_data *);
	199	extern int __must_check td_io_open_file(struct thread_data , struct fio_file );
	200	extern int td_io_close_file(struct thread_data , struct fio_file );
38ef9c90	201	extern int td_io_unlink_file(struct thread_data , struct fio_file );
dcefb588 JA	202	extern int __must_check td_io_get_file_size(struct thread_data , struct fio_file );
	203
	204	extern struct ioengine_ops load_ioengine(struct thread_data , const char *);
	205	extern void register_ioengine(struct ioengine_ops *);
	206	extern void unregister_ioengine(struct ioengine_ops *);
de890a1e	207	extern void free_ioengine(struct thread_data *);
dcefb588 JA	208	extern void close_ioengine(struct thread_data *);
dcefb588 JA	209
de890a1e SL	210	extern int fio_show_ioengine_help(const char *engine);
de890a1e SL	211
dcefb588 JA	212	/*
	213	* io unit handling
	214	*/
dcefb588 JA	215	extern struct io_u __get_io_u(struct thread_data );
	216	extern struct io_u get_io_u(struct thread_data );
	217	extern void put_io_u(struct thread_data , struct io_u );
f2bba182	218	extern void clear_io_u(struct thread_data , struct io_u );
dcefb588	219	extern void requeue_io_u(struct thread_data , struct io_u *);
55312f9f JA	220	extern int __must_check io_u_sync_complete(struct thread_data , struct io_u );
55312f9f JA	221	extern int __must_check io_u_queued_complete(struct thread_data *, int);
dcefb588	222	extern void io_u_queued(struct thread_data , struct io_u );
0d593542	223	extern int io_u_quiesce(struct thread_data *);
dcefb588 JA	224	extern void io_u_log_error(struct thread_data , struct io_u );
dcefb588 JA	225	extern void io_u_mark_depth(struct thread_data *, unsigned int);
cc86c395	226	extern void fill_io_buffer(struct thread_data , void , unsigned int, unsigned int);
9c42684e	227	extern void io_u_fill_buffer(struct thread_data td, struct io_u , unsigned int, unsigned int);
dcefb588 JA	228	void io_u_mark_complete(struct thread_data *, unsigned int);
dcefb588 JA	229	void io_u_mark_submit(struct thread_data *, unsigned int);
e39c0676	230	bool queue_full(const struct thread_data *);
dcefb588	231
effd6ff0 JA	232	int do_io_u_sync(const struct thread_data , struct io_u );
effd6ff0 JA	233	int do_io_u_trim(const struct thread_data , struct io_u );
44f29692	234
c592b9fe JA	235	#ifdef FIO_INC_DEBUG
	236	static inline void dprint_io_u(struct io_u io_u, const char p)
	237	{
	238	struct fio_file *f = io_u->file;
	239
	240	dprint(FD_IO, "%s: io_u %p: off=%llu/len=%lu/ddir=%d", p, io_u,
	241	(unsigned long long) io_u->offset,
	242	io_u->buflen, io_u->ddir);
	243	if (fio_debug & (1 << FD_IO)) {
	244	if (f)
	245	log_info("/%s", f->file_name);
	246
	247	log_info("\n");
	248	}
	249	}
	250	#else
	251	#define dprint_io_u(io_u, p)
	252	#endif
	253
bcd5abfa JA	254	static inline enum fio_ddir acct_ddir(struct io_u *io_u)
	255	{
	256	if (io_u->acct_ddir != -1)
	257	return io_u->acct_ddir;
	258
	259	return io_u->ddir;
	260	}
	261
a9da8ab2 JA	262	static inline void io_u_clear(struct io_u *io_u, unsigned int flags)
	263	{
	264	__sync_fetch_and_and(&io_u->flags, ~flags);
	265	}
	266
	267	static inline void io_u_set(struct io_u *io_u, unsigned int flags)
	268	{
	269	__sync_fetch_and_or(&io_u->flags, flags);
	270	}
	271
dcefb588	272	#endif