[fio.git] / engines / libaio.c

/*
 * native linux aio io engine
 *
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <assert.h>

#include "../fio.h"
#include "../os.h"

#ifdef FIO_HAVE_LIBAIO

#define ev_to_iou(ev)	(struct io_u *) ((unsigned long) (ev)->obj)

struct libaio_data {
	io_context_t aio_ctx;
	struct io_event *aio_events;
	struct iocb **iocbs;
	struct io_u **io_us;
	int iocbs_nr;
};

static int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;

	if (io_u->ddir == DDIR_READ)
		io_prep_pread(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset);
	else if (io_u->ddir == DDIR_WRITE)
		io_prep_pwrite(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset);
	else if (io_u->ddir == DDIR_SYNC)
		io_prep_fsync(&io_u->iocb, f->fd);
	else
		return 1;

	return 0;
}

static struct io_u *fio_libaio_event(struct thread_data *td, int event)
{
	struct libaio_data *ld = td->io_ops->data;

	return ev_to_iou(ld->aio_events + event);
}

static int fio_libaio_getevents(struct thread_data *td, int min, int max,
				struct timespec *t)
{
	struct libaio_data *ld = td->io_ops->data;
	long r;

	do {
		r = io_getevents(ld->aio_ctx, min, max, ld->aio_events, t);
		if (r >= min)
			break;
		else if (r == -EAGAIN) {
			usleep(100);
			continue;
		} else if (r == -EINTR)
			continue;
		else if (r != 0)
			break;
	} while (1);

	return r;
}

static int fio_libaio_queue(struct thread_data *td, struct io_u *io_u)
{
	struct libaio_data *ld = td->io_ops->data;

	if (ld->iocbs_nr == (int) td->iodepth)
		return FIO_Q_BUSY;

	/*
	 * fsync is tricky, since it can fail and we need to do it
	 * serialized with other io. the reason is that linux doesn't
	 * support aio fsync yet. So return busy for the case where we
	 * have pending io, to let fio complete those first.
	 */
	if (io_u->ddir == DDIR_SYNC) {
		if (ld->iocbs_nr)
			return FIO_Q_BUSY;
		if (fsync(io_u->file->fd) < 0)
			io_u->error = errno;

		return FIO_Q_COMPLETED;
	}

	ld->iocbs[ld->iocbs_nr] = &io_u->iocb;
	ld->io_us[ld->iocbs_nr] = io_u;
	ld->iocbs_nr++;
	return FIO_Q_QUEUED;
}

static void fio_libaio_queued(struct thread_data *td, struct io_u **io_us,
			      unsigned int nr)
{
	struct timeval now;
	unsigned int i;

	fio_gettime(&now, NULL);

	for (i = 0; i < nr; i++) {
		struct io_u *io_u = io_us[i];

		memcpy(&io_u->issue_time, &now, sizeof(now));
		io_u_queued(td, io_u);
	}
}

static int fio_libaio_commit(struct thread_data *td)
{
	struct libaio_data *ld = td->io_ops->data;
	struct iocb **iocbs;
	struct io_u **io_us;
	int ret, iocbs_nr;

	if (!ld->iocbs_nr)
		return 0;

	iocbs_nr = ld->iocbs_nr;
	io_us = ld->io_us;
	iocbs = ld->iocbs;
	do {
		ret = io_submit(ld->aio_ctx, iocbs_nr, iocbs);
		if (ret == iocbs_nr) {
			fio_libaio_queued(td, io_us, ret);
			ret = 0;
			break;
		} else if (ret > 0) {
			fio_libaio_queued(td, io_us, ret);
			io_us += ret;
			iocbs += ret;
			iocbs_nr -= ret;
			continue;
		} else if (ret == -EAGAIN || !ret)
			usleep(100);
		else if (ret == -EINTR)
			continue;
		else
			break;
	} while (1);

	if (!ret)
		ld->iocbs_nr = 0;

	return ret;
}

static int fio_libaio_cancel(struct thread_data *td, struct io_u *io_u)
{
	struct libaio_data *ld = td->io_ops->data;

	return io_cancel(ld->aio_ctx, &io_u->iocb, ld->aio_events);
}

static void fio_libaio_cleanup(struct thread_data *td)
{
	struct libaio_data *ld = td->io_ops->data;

	if (ld) {
		io_destroy(ld->aio_ctx);
		free(ld->aio_events);
		free(ld->iocbs);
		free(ld->io_us);
		free(ld);
		td->io_ops->data = NULL;
	}
}

static int fio_libaio_init(struct thread_data *td)
{
	struct libaio_data *ld = malloc(sizeof(*ld));

	memset(ld, 0, sizeof(*ld));
	if (io_queue_init(td->iodepth, &ld->aio_ctx)) {
		td_verror(td, errno, "io_queue_init");
		free(ld);
		return 1;
	}

	ld->aio_events = malloc(td->iodepth * sizeof(struct io_event));
	memset(ld->aio_events, 0, td->iodepth * sizeof(struct io_event));
	ld->iocbs = malloc(td->iodepth * sizeof(struct iocb *));
	memset(ld->iocbs, 0, sizeof(struct iocb *));
	ld->io_us = malloc(td->iodepth * sizeof(struct io_u *));
	memset(ld->io_us, 0, td->iodepth * sizeof(struct io_u *));
	ld->iocbs_nr = 0;

	td->io_ops->data = ld;
	return 0;
}

static struct ioengine_ops ioengine = {
	.name		= "libaio",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_libaio_init,
	.prep		= fio_libaio_prep,
	.queue		= fio_libaio_queue,
	.commit		= fio_libaio_commit,
	.cancel		= fio_libaio_cancel,
	.getevents	= fio_libaio_getevents,
	.event		= fio_libaio_event,
	.cleanup	= fio_libaio_cleanup,
};

#else /* FIO_HAVE_LIBAIO */

/*
 * When we have a proper configure system in place, we simply wont build
 * and install this io engine. For now install a crippled version that
 * just complains and fails to load.
 */
static int fio_libaio_init(struct thread_data fio_unused *td)
{
	fprintf(stderr, "fio: libaio not available\n");
	return 1;
}

static struct ioengine_ops ioengine = {
	.name		= "libaio",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_libaio_init,
};

#endif

static void fio_init fio_libaio_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_libaio_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
2866c82d JA	1	/*
	2	* native linux aio io engine
	3	*
	4	*/
	5	#include <stdio.h>
	6	#include <stdlib.h>
	7	#include <unistd.h>
	8	#include <errno.h>
	9	#include <assert.h>
5f350952 JA	10
	11	#include "../fio.h"
	12	#include "../os.h"
2866c82d	13
34cfcdaf JA	14	#ifdef FIO_HAVE_LIBAIO
34cfcdaf JA	15
2866c82d JA	16	#define ev_to_iou(ev) (struct io_u *) ((unsigned long) (ev)->obj)
	17
	18	struct libaio_data {
	19	io_context_t aio_ctx;
	20	struct io_event *aio_events;
755200a3	21	struct iocb **iocbs;
7e77dd02	22	struct io_u **io_us;
755200a3	23	int iocbs_nr;
2866c82d JA	24	};
2866c82d JA	25
7a16dd02	26	static int fio_libaio_prep(struct thread_data fio_unused td, struct io_u io_u)
2866c82d	27	{
53cdc686 JA	28	struct fio_file *f = io_u->file;
53cdc686 JA	29
2866c82d	30	if (io_u->ddir == DDIR_READ)
cec6b55d	31	io_prep_pread(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset);
87dc1ab1	32	else if (io_u->ddir == DDIR_WRITE)
cec6b55d	33	io_prep_pwrite(&io_u->iocb, f->fd, io_u->xfer_buf, io_u->xfer_buflen, io_u->offset);
87dc1ab1 JA	34	else if (io_u->ddir == DDIR_SYNC)
	35	io_prep_fsync(&io_u->iocb, f->fd);
	36	else
	37	return 1;
2866c82d JA	38
	39	return 0;
	40	}
	41
	42	static struct io_u fio_libaio_event(struct thread_data td, int event)
	43	{
	44	struct libaio_data *ld = td->io_ops->data;
	45
	46	return ev_to_iou(ld->aio_events + event);
	47	}
	48
	49	static int fio_libaio_getevents(struct thread_data *td, int min, int max,
	50	struct timespec *t)
	51	{
	52	struct libaio_data *ld = td->io_ops->data;
	53	long r;
	54
	55	do {
	56	r = io_getevents(ld->aio_ctx, min, max, ld->aio_events, t);
ccbb91cb JA	57	if (r >= min)
	58	break;
	59	else if (r == -EAGAIN) {
2866c82d JA	60	usleep(100);
	61	continue;
	62	} else if (r == -EINTR)
	63	continue;
84585003	64	else if (r != 0)
2866c82d JA	65	break;
	66	} while (1);
	67
22819ec2	68	return r;
2866c82d JA	69	}
	70
	71	static int fio_libaio_queue(struct thread_data td, struct io_u io_u)
	72	{
	73	struct libaio_data *ld = td->io_ops->data;
2866c82d	74
755200a3 JA	75	if (ld->iocbs_nr == (int) td->iodepth)
	76	return FIO_Q_BUSY;
	77
	78	/*
	79	* fsync is tricky, since it can fail and we need to do it
	80	* serialized with other io. the reason is that linux doesn't
	81	* support aio fsync yet. So return busy for the case where we
	82	* have pending io, to let fio complete those first.
	83	*/
	84	if (io_u->ddir == DDIR_SYNC) {
	85	if (ld->iocbs_nr)
	86	return FIO_Q_BUSY;
	87	if (fsync(io_u->file->fd) < 0)
	88	io_u->error = errno;
	89
	90	return FIO_Q_COMPLETED;
	91	}
	92
	93	ld->iocbs[ld->iocbs_nr] = &io_u->iocb;
7e77dd02	94	ld->io_us[ld->iocbs_nr] = io_u;
755200a3 JA	95	ld->iocbs_nr++;
	96	return FIO_Q_QUEUED;
	97	}
	98
7e77dd02 JA	99	static void fio_libaio_queued(struct thread_data td, struct io_u *io_us,
	100	unsigned int nr)
	101	{
	102	struct timeval now;
	103	unsigned int i;
	104
	105	fio_gettime(&now, NULL);
	106
	107	for (i = 0; i < nr; i++) {
	108	struct io_u *io_u = io_us[i];
	109
	110	memcpy(&io_u->issue_time, &now, sizeof(now));
	111	io_u_queued(td, io_u);
	112	}
	113	}
	114
755200a3 JA	115	static int fio_libaio_commit(struct thread_data *td)
	116	{
	117	struct libaio_data *ld = td->io_ops->data;
	118	struct iocb **iocbs;
7e77dd02	119	struct io_u **io_us;
755200a3 JA	120	int ret, iocbs_nr;
	121
	122	if (!ld->iocbs_nr)
	123	return 0;
	124
	125	iocbs_nr = ld->iocbs_nr;
7e77dd02	126	io_us = ld->io_us;
755200a3	127	iocbs = ld->iocbs;
2866c82d	128	do {
755200a3 JA	129	ret = io_submit(ld->aio_ctx, iocbs_nr, iocbs);
755200a3 JA	130	if (ret == iocbs_nr) {
7e77dd02	131	fio_libaio_queued(td, io_us, ret);
755200a3 JA	132	ret = 0;
	133	break;
	134	} else if (ret > 0) {
7e77dd02 JA	135	fio_libaio_queued(td, io_us, ret);
7e77dd02 JA	136	io_us += ret;
755200a3 JA	137	iocbs += ret;
	138	iocbs_nr -= ret;
	139	continue;
	140	} else if (ret == -EAGAIN \|\| !ret)
2866c82d JA	141	usleep(100);
	142	else if (ret == -EINTR)
	143	continue;
755200a3	144	else
2866c82d JA	145	break;
	146	} while (1);
	147
755200a3 JA	148	if (!ret)
755200a3 JA	149	ld->iocbs_nr = 0;
2866c82d	150
36167d82	151	return ret;
2866c82d JA	152	}
	153
	154	static int fio_libaio_cancel(struct thread_data td, struct io_u io_u)
	155	{
	156	struct libaio_data *ld = td->io_ops->data;
	157
	158	return io_cancel(ld->aio_ctx, &io_u->iocb, ld->aio_events);
	159	}
	160
	161	static void fio_libaio_cleanup(struct thread_data *td)
	162	{
	163	struct libaio_data *ld = td->io_ops->data;
	164
	165	if (ld) {
	166	io_destroy(ld->aio_ctx);
7e77dd02 JA	167	free(ld->aio_events);
	168	free(ld->iocbs);
	169	free(ld->io_us);
2866c82d JA	170	free(ld);
	171	td->io_ops->data = NULL;
	172	}
	173	}
	174
	175	static int fio_libaio_init(struct thread_data *td)
	176	{
	177	struct libaio_data ld = malloc(sizeof(ld));
	178
	179	memset(ld, 0, sizeof(*ld));
	180	if (io_queue_init(td->iodepth, &ld->aio_ctx)) {
e1161c32	181	td_verror(td, errno, "io_queue_init");
cb781c75	182	free(ld);
2866c82d JA	183	return 1;
	184	}
	185
	186	ld->aio_events = malloc(td->iodepth * sizeof(struct io_event));
84585003	187	memset(ld->aio_events, 0, td->iodepth * sizeof(struct io_event));
755200a3 JA	188	ld->iocbs = malloc(td->iodepth * sizeof(struct iocb *));
755200a3 JA	189	memset(ld->iocbs, 0, sizeof(struct iocb *));
7e77dd02 JA	190	ld->io_us = malloc(td->iodepth * sizeof(struct io_u *));
7e77dd02 JA	191	memset(ld->io_us, 0, td->iodepth * sizeof(struct io_u *));
755200a3 JA	192	ld->iocbs_nr = 0;
755200a3 JA	193
2866c82d JA	194	td->io_ops->data = ld;
	195	return 0;
	196	}
	197
5f350952	198	static struct ioengine_ops ioengine = {
2866c82d JA	199	.name = "libaio",
	200	.version = FIO_IOOPS_VERSION,
	201	.init = fio_libaio_init,
	202	.prep = fio_libaio_prep,
	203	.queue = fio_libaio_queue,
755200a3	204	.commit = fio_libaio_commit,
2866c82d JA	205	.cancel = fio_libaio_cancel,
	206	.getevents = fio_libaio_getevents,
	207	.event = fio_libaio_event,
	208	.cleanup = fio_libaio_cleanup,
2866c82d	209	};
34cfcdaf JA	210
	211	#else /* FIO_HAVE_LIBAIO */
	212
	213	/*
	214	* When we have a proper configure system in place, we simply wont build
	215	* and install this io engine. For now install a crippled version that
	216	* just complains and fails to load.
	217	*/
	218	static int fio_libaio_init(struct thread_data fio_unused *td)
	219	{
	220	fprintf(stderr, "fio: libaio not available\n");
	221	return 1;
	222	}
	223
5f350952	224	static struct ioengine_ops ioengine = {
34cfcdaf JA	225	.name = "libaio",
	226	.version = FIO_IOOPS_VERSION,
	227	.init = fio_libaio_init,
	228	};
	229
	230	#endif
5f350952 JA	231
	232	static void fio_init fio_libaio_register(void)
	233	{
	234	register_ioengine(&ioengine);
	235	}
	236
	237	static void fio_exit fio_libaio_unregister(void)
	238	{
	239	unregister_ioengine(&ioengine);
	240	}