[fio.git] / engines / libaio.c

/*
 * libaio engine
 *
 * IO engine using the Linux native aio interface.
 *
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <assert.h>

#include "../fio.h"
#include "../os.h"

#ifdef FIO_HAVE_LIBAIO

#define ev_to_iou(ev)	(struct io_u *) ((unsigned long) (ev)->obj)

struct libaio_data {
	io_context_t aio_ctx;
	struct io_event *aio_events;
	struct iocb **iocbs;
	struct io_u **io_us;
	int iocbs_nr;
};

static int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;

	if (io_u->ddir == DDIR_READ)
		io_prep_pread(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset);
	else if (io_u->ddir == DDIR_WRITE)
		io_prep_pwrite(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset);
	else if (io_u->ddir == DDIR_SYNC)
		io_prep_fsync(&io_u->iocb, f->fd);
	else
		return 1;

	return 0;
}

static struct io_u *fio_libaio_event(struct thread_data *td, int event)
{
	struct libaio_data *ld = td->io_ops->data;
	struct io_event *ev;
	struct io_u *io_u;

	ev = ld->aio_events + event;
	io_u = ev_to_iou(ev);

	if (ev->res != io_u->xfer_buflen) {
		if (ev->res > io_u->xfer_buflen)
			io_u->error = -ev->res;
		else
			io_u->resid = io_u->xfer_buflen - ev->res;
	} else
		io_u->error = 0;

	return io_u;
}

static int fio_libaio_getevents(struct thread_data *td, int min, int max,
				struct timespec *t)
{
	struct libaio_data *ld = td->io_ops->data;
	long r;

	do {
		r = io_getevents(ld->aio_ctx, min, max, ld->aio_events, t);
		if (r >= min)
			break;
		else if (r == -EAGAIN) {
			usleep(100);
			continue;
		} else if (r == -EINTR)
			continue;
		else if (r != 0)
			break;
	} while (1);

	return r;
}

static int fio_libaio_queue(struct thread_data *td, struct io_u *io_u)
{
	struct libaio_data *ld = td->io_ops->data;

	if (ld->iocbs_nr == (int) td->iodepth)
		return FIO_Q_BUSY;

	/*
	 * fsync is tricky, since it can fail and we need to do it
	 * serialized with other io. the reason is that linux doesn't
	 * support aio fsync yet. So return busy for the case where we
	 * have pending io, to let fio complete those first.
	 */
	if (io_u->ddir == DDIR_SYNC) {
		if (ld->iocbs_nr)
			return FIO_Q_BUSY;
		if (fsync(io_u->file->fd) < 0)
			io_u->error = errno;

		return FIO_Q_COMPLETED;
	}

	ld->iocbs[ld->iocbs_nr] = &io_u->iocb;
	ld->io_us[ld->iocbs_nr] = io_u;
	ld->iocbs_nr++;
	return FIO_Q_QUEUED;
}

static void fio_libaio_queued(struct thread_data *td, struct io_u **io_us,
			      unsigned int nr)
{
	struct timeval now;
	unsigned int i;

	fio_gettime(&now, NULL);

	for (i = 0; i < nr; i++) {
		struct io_u *io_u = io_us[i];

		memcpy(&io_u->issue_time, &now, sizeof(now));
		io_u_queued(td, io_u);
	}
}

static int fio_libaio_commit(struct thread_data *td)
{
	struct libaio_data *ld = td->io_ops->data;
	struct iocb **iocbs;
	struct io_u **io_us;
	int ret, iocbs_nr;

	if (!ld->iocbs_nr)
		return 0;

	iocbs_nr = ld->iocbs_nr;
	io_us = ld->io_us;
	iocbs = ld->iocbs;
	do {
		ret = io_submit(ld->aio_ctx, iocbs_nr, iocbs);
		if (ret == iocbs_nr) {
			fio_libaio_queued(td, io_us, ret);
			ret = 0;
			break;
		} else if (ret > 0) {
			fio_libaio_queued(td, io_us, ret);
			io_us += ret;
			iocbs += ret;
			iocbs_nr -= ret;
			continue;
		} else if (ret == -EAGAIN || !ret)
			usleep(100);
		else if (ret == -EINTR)
			continue;
		else
			break;
	} while (1);

	if (!ret)
		ld->iocbs_nr = 0;

	return ret;
}

static int fio_libaio_cancel(struct thread_data *td, struct io_u *io_u)
{
	struct libaio_data *ld = td->io_ops->data;

	return io_cancel(ld->aio_ctx, &io_u->iocb, ld->aio_events);
}

static void fio_libaio_cleanup(struct thread_data *td)
{
	struct libaio_data *ld = td->io_ops->data;

	if (ld) {
		io_destroy(ld->aio_ctx);
		free(ld->aio_events);
		free(ld->iocbs);
		free(ld->io_us);
		free(ld);
		td->io_ops->data = NULL;
	}
}

static int fio_libaio_init(struct thread_data *td)
{
	struct libaio_data *ld = malloc(sizeof(*ld));

	memset(ld, 0, sizeof(*ld));
	if (io_queue_init(td->iodepth, &ld->aio_ctx)) {
		td_verror(td, errno, "io_queue_init");
		free(ld);
		return 1;
	}

	ld->aio_events = malloc(td->iodepth * sizeof(struct io_event));
	memset(ld->aio_events, 0, td->iodepth * sizeof(struct io_event));
	ld->iocbs = malloc(td->iodepth * sizeof(struct iocb *));
	memset(ld->iocbs, 0, sizeof(struct iocb *));
	ld->io_us = malloc(td->iodepth * sizeof(struct io_u *));
	memset(ld->io_us, 0, td->iodepth * sizeof(struct io_u *));
	ld->iocbs_nr = 0;

	td->io_ops->data = ld;
	return 0;
}

static struct ioengine_ops ioengine = {
	.name		= "libaio",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_libaio_init,
	.prep		= fio_libaio_prep,
	.queue		= fio_libaio_queue,
	.commit		= fio_libaio_commit,
	.cancel		= fio_libaio_cancel,
	.getevents	= fio_libaio_getevents,
	.event		= fio_libaio_event,
	.cleanup	= fio_libaio_cleanup,
	.open_file	= generic_open_file,
	.close_file	= generic_close_file,
};

#else /* FIO_HAVE_LIBAIO */

/*
 * When we have a proper configure system in place, we simply wont build
 * and install this io engine. For now install a crippled version that
 * just complains and fails to load.
 */
static int fio_libaio_init(struct thread_data fio_unused *td)
{
	fprintf(stderr, "fio: libaio not available\n");
	return 1;
}

static struct ioengine_ops ioengine = {
	.name		= "libaio",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_libaio_init,
};

#endif

static void fio_init fio_libaio_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_libaio_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
2866c82d	1	/*
da751ca9 JA	2	* libaio engine
	3	*
	4	* IO engine using the Linux native aio interface.
2866c82d JA	5	*
	6	*/
	7	#include <stdio.h>
	8	#include <stdlib.h>
	9	#include <unistd.h>
	10	#include <errno.h>
	11	#include <assert.h>
5f350952 JA	12
	13	#include "../fio.h"
	14	#include "../os.h"
2866c82d	15
34cfcdaf JA	16	#ifdef FIO_HAVE_LIBAIO
34cfcdaf JA	17
2866c82d JA	18	#define ev_to_iou(ev) (struct io_u *) ((unsigned long) (ev)->obj)
	19
	20	struct libaio_data {
	21	io_context_t aio_ctx;
	22	struct io_event *aio_events;
755200a3	23	struct iocb **iocbs;
7e77dd02	24	struct io_u **io_us;
755200a3	25	int iocbs_nr;
2866c82d JA	26	};
2866c82d JA	27
7a16dd02	28	static int fio_libaio_prep(struct thread_data fio_unused td, struct io_u io_u)
2866c82d	29	{
53cdc686 JA	30	struct fio_file *f = io_u->file;
53cdc686 JA	31
2866c82d	32	if (io_u->ddir == DDIR_READ)
cec6b55d	33	io_prep_pread(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset);
87dc1ab1	34	else if (io_u->ddir == DDIR_WRITE)
cec6b55d	35	io_prep_pwrite(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset);
87dc1ab1 JA	36	else if (io_u->ddir == DDIR_SYNC)
	37	io_prep_fsync(&io_u->iocb, f->fd);
	38	else
	39	return 1;
2866c82d JA	40
	41	return 0;
	42	}
	43
	44	static struct io_u fio_libaio_event(struct thread_data td, int event)
	45	{
	46	struct libaio_data *ld = td->io_ops->data;
f423479d JA	47	struct io_event *ev;
f423479d JA	48	struct io_u *io_u;
2866c82d	49
f423479d JA	50	ev = ld->aio_events + event;
	51	io_u = ev_to_iou(ev);
	52
	53	if (ev->res != io_u->xfer_buflen) {
	54	if (ev->res > io_u->xfer_buflen)
	55	io_u->error = -ev->res;
	56	else
	57	io_u->resid = io_u->xfer_buflen - ev->res;
	58	} else
	59	io_u->error = 0;
	60
	61	return io_u;
2866c82d JA	62	}
	63
	64	static int fio_libaio_getevents(struct thread_data *td, int min, int max,
	65	struct timespec *t)
	66	{
	67	struct libaio_data *ld = td->io_ops->data;
	68	long r;
	69
	70	do {
	71	r = io_getevents(ld->aio_ctx, min, max, ld->aio_events, t);
ccbb91cb JA	72	if (r >= min)
	73	break;
	74	else if (r == -EAGAIN) {
2866c82d JA	75	usleep(100);
	76	continue;
	77	} else if (r == -EINTR)
	78	continue;
84585003	79	else if (r != 0)
2866c82d JA	80	break;
	81	} while (1);
	82
22819ec2	83	return r;
2866c82d JA	84	}
	85
	86	static int fio_libaio_queue(struct thread_data td, struct io_u io_u)
	87	{
	88	struct libaio_data *ld = td->io_ops->data;
2866c82d	89
755200a3 JA	90	if (ld->iocbs_nr == (int) td->iodepth)
	91	return FIO_Q_BUSY;
	92
	93	/*
	94	* fsync is tricky, since it can fail and we need to do it
	95	* serialized with other io. the reason is that linux doesn't
	96	* support aio fsync yet. So return busy for the case where we
	97	* have pending io, to let fio complete those first.
	98	*/
	99	if (io_u->ddir == DDIR_SYNC) {
	100	if (ld->iocbs_nr)
	101	return FIO_Q_BUSY;
	102	if (fsync(io_u->file->fd) < 0)
	103	io_u->error = errno;
	104
	105	return FIO_Q_COMPLETED;
	106	}
	107
	108	ld->iocbs[ld->iocbs_nr] = &io_u->iocb;
7e77dd02	109	ld->io_us[ld->iocbs_nr] = io_u;
755200a3 JA	110	ld->iocbs_nr++;
	111	return FIO_Q_QUEUED;
	112	}
	113
7e77dd02 JA	114	static void fio_libaio_queued(struct thread_data td, struct io_u *io_us,
	115	unsigned int nr)
	116	{
	117	struct timeval now;
	118	unsigned int i;
	119
	120	fio_gettime(&now, NULL);
	121
	122	for (i = 0; i < nr; i++) {
	123	struct io_u *io_u = io_us[i];
	124
	125	memcpy(&io_u->issue_time, &now, sizeof(now));
	126	io_u_queued(td, io_u);
	127	}
	128	}
	129
755200a3 JA	130	static int fio_libaio_commit(struct thread_data *td)
	131	{
	132	struct libaio_data *ld = td->io_ops->data;
	133	struct iocb **iocbs;
7e77dd02	134	struct io_u **io_us;
755200a3 JA	135	int ret, iocbs_nr;
	136
	137	if (!ld->iocbs_nr)
	138	return 0;
	139
	140	iocbs_nr = ld->iocbs_nr;
7e77dd02	141	io_us = ld->io_us;
755200a3	142	iocbs = ld->iocbs;
2866c82d	143	do {
755200a3 JA	144	ret = io_submit(ld->aio_ctx, iocbs_nr, iocbs);
755200a3 JA	145	if (ret == iocbs_nr) {
7e77dd02	146	fio_libaio_queued(td, io_us, ret);
755200a3 JA	147	ret = 0;
	148	break;
	149	} else if (ret > 0) {
7e77dd02 JA	150	fio_libaio_queued(td, io_us, ret);
7e77dd02 JA	151	io_us += ret;
755200a3 JA	152	iocbs += ret;
	153	iocbs_nr -= ret;
	154	continue;
	155	} else if (ret == -EAGAIN \|\| !ret)
2866c82d JA	156	usleep(100);
	157	else if (ret == -EINTR)
	158	continue;
755200a3	159	else
2866c82d JA	160	break;
	161	} while (1);
	162
755200a3 JA	163	if (!ret)
755200a3 JA	164	ld->iocbs_nr = 0;
2866c82d	165
36167d82	166	return ret;
2866c82d JA	167	}
	168
	169	static int fio_libaio_cancel(struct thread_data td, struct io_u io_u)
	170	{
	171	struct libaio_data *ld = td->io_ops->data;
	172
	173	return io_cancel(ld->aio_ctx, &io_u->iocb, ld->aio_events);
	174	}
	175
	176	static void fio_libaio_cleanup(struct thread_data *td)
	177	{
	178	struct libaio_data *ld = td->io_ops->data;
	179
	180	if (ld) {
	181	io_destroy(ld->aio_ctx);
7e77dd02 JA	182	free(ld->aio_events);
	183	free(ld->iocbs);
	184	free(ld->io_us);
2866c82d JA	185	free(ld);
	186	td->io_ops->data = NULL;
	187	}
	188	}
	189
	190	static int fio_libaio_init(struct thread_data *td)
	191	{
	192	struct libaio_data ld = malloc(sizeof(ld));
	193
	194	memset(ld, 0, sizeof(*ld));
	195	if (io_queue_init(td->iodepth, &ld->aio_ctx)) {
e1161c32	196	td_verror(td, errno, "io_queue_init");
cb781c75	197	free(ld);
2866c82d JA	198	return 1;
	199	}
	200
	201	ld->aio_events = malloc(td->iodepth * sizeof(struct io_event));
84585003	202	memset(ld->aio_events, 0, td->iodepth * sizeof(struct io_event));
755200a3 JA	203	ld->iocbs = malloc(td->iodepth * sizeof(struct iocb *));
755200a3 JA	204	memset(ld->iocbs, 0, sizeof(struct iocb *));
7e77dd02 JA	205	ld->io_us = malloc(td->iodepth * sizeof(struct io_u *));
7e77dd02 JA	206	memset(ld->io_us, 0, td->iodepth * sizeof(struct io_u *));
755200a3 JA	207	ld->iocbs_nr = 0;
755200a3 JA	208
2866c82d JA	209	td->io_ops->data = ld;
	210	return 0;
	211	}
	212
5f350952	213	static struct ioengine_ops ioengine = {
2866c82d JA	214	.name = "libaio",
	215	.version = FIO_IOOPS_VERSION,
	216	.init = fio_libaio_init,
	217	.prep = fio_libaio_prep,
	218	.queue = fio_libaio_queue,
755200a3	219	.commit = fio_libaio_commit,
2866c82d JA	220	.cancel = fio_libaio_cancel,
	221	.getevents = fio_libaio_getevents,
	222	.event = fio_libaio_event,
	223	.cleanup = fio_libaio_cleanup,
b5af8293 JA	224	.open_file = generic_open_file,
b5af8293 JA	225	.close_file = generic_close_file,
2866c82d	226	};
34cfcdaf JA	227
	228	#else /* FIO_HAVE_LIBAIO */
	229
	230	/*
	231	* When we have a proper configure system in place, we simply wont build
	232	* and install this io engine. For now install a crippled version that
	233	* just complains and fails to load.
	234	*/
	235	static int fio_libaio_init(struct thread_data fio_unused *td)
	236	{
	237	fprintf(stderr, "fio: libaio not available\n");
	238	return 1;
	239	}
	240
5f350952	241	static struct ioengine_ops ioengine = {
34cfcdaf JA	242	.name = "libaio",
	243	.version = FIO_IOOPS_VERSION,
	244	.init = fio_libaio_init,
	245	};
	246
	247	#endif
5f350952 JA	248
	249	static void fio_init fio_libaio_register(void)
	250	{
	251	register_ioengine(&ioengine);
	252	}
	253
	254	static void fio_exit fio_libaio_unregister(void)
	255	{
	256	unregister_ioengine(&ioengine);
	257	}