[fio.git] / engines / syslet-rw.c

/*
 * read/write() engine that uses syslet to be async
 *
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <assert.h>

#include "../fio.h"
#include "../os.h"

#ifdef FIO_HAVE_SYSLET

struct syslet_data {
	struct io_u **events;
	unsigned int nr_events;
	
	struct async_head_user ahu;
	struct syslet_uatom **ring;

	struct syslet_uatom *head, *tail;
};

static void fio_syslet_complete_atom(struct thread_data *td,
				     struct syslet_uatom *atom)
{
	struct syslet_data *sd = td->io_ops->data;
	struct syslet_uatom *last;
	struct io_u *io_u;

	/*
	 * complete from the beginning of the sequence up to (and
	 * including) this atom
	 */
	last = atom;
	io_u = atom->private;
	atom = io_u->req.head;

	/*
	 * now complete in right order
	 */
	do {
		long ret;

		io_u = atom->private;
		ret = *atom->ret_ptr;
		if (ret > 0)
			io_u->resid = io_u->xfer_buflen - ret;
		else if (ret < 0)
			io_u->error = ret;

		assert(sd->nr_events < td->iodepth);
		sd->events[sd->nr_events++] = io_u;

		if (atom == last)
			break;

		atom = atom->next;
	} while (1);

	assert(!last->next);
}

/*
 * Inspect the ring to see if we have completed events
 */
static void fio_syslet_complete(struct thread_data *td)
{
	struct syslet_data *sd = td->io_ops->data;

	do {
		struct syslet_uatom *atom;

		atom = sd->ring[sd->ahu.user_ring_idx];
		if (!atom)
			break;

		sd->ring[sd->ahu.user_ring_idx] = NULL;
		if (++sd->ahu.user_ring_idx == td->iodepth)
			sd->ahu.user_ring_idx = 0;

		fio_syslet_complete_atom(td, atom);
	} while (1);
}

static int fio_syslet_getevents(struct thread_data *td, int min,
				int fio_unused max,
				struct timespec fio_unused *t)
{
	struct syslet_data *sd = td->io_ops->data;
	long ret;

	do {
		fio_syslet_complete(td);

		/*
		 * do we have enough immediate completions?
		 */
		if (sd->nr_events >= (unsigned int) min)
			break;

		/*
		 * OK, we need to wait for some events...
		 */
		ret = async_wait(1, sd->ahu.user_ring_idx, &sd->ahu);
		if (ret < 0)
			return -errno;
	} while (1);

	ret = sd->nr_events;
	sd->nr_events = 0;
	return ret;
}

static struct io_u *fio_syslet_event(struct thread_data *td, int event)
{
	struct syslet_data *sd = td->io_ops->data;

	return sd->events[event];
}

static void init_atom(struct syslet_uatom *atom, int nr, void *arg0,
		      void *arg1, void *arg2, void *arg3, void *ret_ptr,
		      unsigned long flags, void *priv)
{
	atom->flags = flags;
	atom->nr = nr;
	atom->ret_ptr = ret_ptr;
	atom->next = NULL;
	atom->arg_ptr[0] = arg0;
	atom->arg_ptr[1] = arg1;
	atom->arg_ptr[2] = arg2;
	atom->arg_ptr[3] = arg3;
	atom->arg_ptr[4] = atom->arg_ptr[5] = NULL;
	atom->private = priv;
}

/*
 * Use seek atom for sync
 */
static void fio_syslet_prep_sync(struct io_u *io_u, struct fio_file *f)
{
	init_atom(&io_u->req.atom, __NR_fsync, &f->fd, NULL, NULL, NULL,
		  &io_u->req.ret, 0, io_u);
}

static void fio_syslet_prep_rw(struct io_u *io_u, struct fio_file *f)
{
	int nr;

	/*
	 * prepare rw
	 */
	if (io_u->ddir == DDIR_READ)
		nr = __NR_pread64;
	else
		nr = __NR_pwrite64;

	init_atom(&io_u->req.atom, nr, &f->fd, &io_u->xfer_buf,
		  &io_u->xfer_buflen, &io_u->offset, &io_u->req.ret, 0, io_u);
}

static int fio_syslet_prep(struct thread_data fio_unused *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;

	if (io_u->ddir == DDIR_SYNC)
		fio_syslet_prep_sync(io_u, f);
	else
		fio_syslet_prep_rw(io_u, f);

	return 0;
}

static void cachemiss_thread_start(void)
{
	while (1)
		async_thread(NULL, NULL);
}

#define THREAD_STACK_SIZE (16384)

static unsigned long thread_stack_alloc()
{
	return (unsigned long) malloc(THREAD_STACK_SIZE) + THREAD_STACK_SIZE;
}

static void fio_syslet_queued(struct thread_data *td, struct syslet_data *sd)
{
	struct syslet_uatom *atom;
	struct timeval now;

	fio_gettime(&now, NULL);

	atom = sd->head;
	while (atom) {
		struct io_u *io_u = atom->private;

		memcpy(&io_u->issue_time, &now, sizeof(now));
		io_u_queued(td, io_u);
		atom = atom->next;
	}
}

static int fio_syslet_commit(struct thread_data *td)
{
	struct syslet_data *sd = td->io_ops->data;
	struct syslet_uatom *done;

	if (!sd->head)
		return 0;

	assert(!sd->tail->next);

	if (!sd->ahu.new_thread_stack)
		sd->ahu.new_thread_stack = thread_stack_alloc();

	fio_syslet_queued(td, sd);

	/*
	 * On sync completion, the atom is returned. So on NULL return
	 * it's queued asynchronously.
	 */
	done = async_exec(sd->head, &sd->ahu);

	sd->head = sd->tail = NULL;

	if (done)
		fio_syslet_complete_atom(td, done);

	return 0;
}

static int fio_syslet_queue(struct thread_data *td, struct io_u *io_u)
{
	struct syslet_data *sd = td->io_ops->data;

	if (sd->tail) {
		sd->tail->next = &io_u->req.atom;
		sd->tail = &io_u->req.atom;
	} else
		sd->head = sd->tail = &io_u->req.atom;

	io_u->req.head = sd->head;
	return FIO_Q_QUEUED;
}

static int async_head_init(struct syslet_data *sd, unsigned int depth)
{
	unsigned long ring_size;

	memset(&sd->ahu, 0, sizeof(struct async_head_user));

	ring_size = sizeof(struct syslet_uatom *) * depth;
	sd->ring = malloc(ring_size);
	memset(sd->ring, 0, ring_size);

	sd->ahu.user_ring_idx = 0;
	sd->ahu.completion_ring = sd->ring;
	sd->ahu.ring_size_bytes = ring_size;
	sd->ahu.head_stack = thread_stack_alloc();
	sd->ahu.head_eip = (unsigned long) cachemiss_thread_start;
	sd->ahu.new_thread_eip = (unsigned long) cachemiss_thread_start;

	return 0;
}

static void async_head_exit(struct syslet_data *sd)
{
	free(sd->ring);
}

static void fio_syslet_cleanup(struct thread_data *td)
{
	struct syslet_data *sd = td->io_ops->data;

	if (sd) {
		async_head_exit(sd);
		free(sd->events);
		free(sd);
		td->io_ops->data = NULL;
	}
}

static int fio_syslet_init(struct thread_data *td)
{
	struct syslet_data *sd;


	sd = malloc(sizeof(*sd));
	memset(sd, 0, sizeof(*sd));
	sd->events = malloc(sizeof(struct io_u *) * td->iodepth);
	memset(sd->events, 0, sizeof(struct io_u *) * td->iodepth);

	/*
	 * This will handily fail for kernels where syslet isn't available
	 */
	if (async_head_init(sd, td->iodepth)) {
		free(sd->events);
		free(sd);
		return 1;
	}

	td->io_ops->data = sd;
	return 0;
}

static struct ioengine_ops ioengine = {
	.name		= "syslet-rw",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_syslet_init,
	.prep		= fio_syslet_prep,
	.queue		= fio_syslet_queue,
	.commit		= fio_syslet_commit,
	.getevents	= fio_syslet_getevents,
	.event		= fio_syslet_event,
	.cleanup	= fio_syslet_cleanup,
};

#else /* FIO_HAVE_SYSLET */

/*
 * When we have a proper configure system in place, we simply wont build
 * and install this io engine. For now install a crippled version that
 * just complains and fails to load.
 */
static int fio_syslet_init(struct thread_data fio_unused *td)
{
	fprintf(stderr, "fio: syslet not available\n");
	return 1;
}

static struct ioengine_ops ioengine = {
	.name		= "syslet-rw",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_syslet_init,
};

#endif /* FIO_HAVE_SYSLET */

static void fio_init fio_syslet_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_syslet_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
a4f4fdd7 JA	1	/*
	2	* read/write() engine that uses syslet to be async
	3	*
	4	*/
	5	#include <stdio.h>
	6	#include <stdlib.h>
	7	#include <unistd.h>
	8	#include <errno.h>
	9	#include <assert.h>
	10
	11	#include "../fio.h"
	12	#include "../os.h"
	13
	14	#ifdef FIO_HAVE_SYSLET
	15
	16	struct syslet_data {
	17	struct io_u **events;
	18	unsigned int nr_events;
	19
bf0dc8fa	20	struct async_head_user ahu;
a4f4fdd7	21	struct syslet_uatom **ring;
9ff9de69 JA	22
9ff9de69 JA	23	struct syslet_uatom head, tail;
a4f4fdd7 JA	24	};
a4f4fdd7 JA	25
9ff9de69 JA	26	static void fio_syslet_complete_atom(struct thread_data *td,
	27	struct syslet_uatom *atom)
	28	{
	29	struct syslet_data *sd = td->io_ops->data;
5b38ee84	30	struct syslet_uatom *last;
9ff9de69	31	struct io_u *io_u;
9ff9de69 JA	32
9ff9de69 JA	33	/*
5b38ee84 JA	34	* complete from the beginning of the sequence up to (and
5b38ee84 JA	35	* including) this atom
9ff9de69	36	*/
5b38ee84 JA	37	last = atom;
	38	io_u = atom->private;
	39	atom = io_u->req.head;
9ff9de69 JA	40
	41	/*
	42	* now complete in right order
	43	*/
5b38ee84	44	do {
9ff9de69 JA	45	long ret;
9ff9de69 JA	46
9ff9de69 JA	47	io_u = atom->private;
	48	ret = *atom->ret_ptr;
	49	if (ret > 0)
	50	io_u->resid = io_u->xfer_buflen - ret;
	51	else if (ret < 0)
	52	io_u->error = ret;
	53
	54	assert(sd->nr_events < td->iodepth);
	55	sd->events[sd->nr_events++] = io_u;
9ff9de69	56
5b38ee84 JA	57	if (atom == last)
5b38ee84 JA	58	break;
9ff9de69	59
5b38ee84 JA	60	atom = atom->next;
	61	} while (1);
	62
	63	assert(!last->next);
9ff9de69 JA	64	}
9ff9de69 JA	65
a4f4fdd7 JA	66	/*
	67	* Inspect the ring to see if we have completed events
	68	*/
	69	static void fio_syslet_complete(struct thread_data *td)
	70	{
	71	struct syslet_data *sd = td->io_ops->data;
	72
	73	do {
	74	struct syslet_uatom *atom;
a4f4fdd7	75
bf0dc8fa	76	atom = sd->ring[sd->ahu.user_ring_idx];
a4f4fdd7 JA	77	if (!atom)
	78	break;
	79
bf0dc8fa IM	80	sd->ring[sd->ahu.user_ring_idx] = NULL;
	81	if (++sd->ahu.user_ring_idx == td->iodepth)
	82	sd->ahu.user_ring_idx = 0;
a4f4fdd7	83
9ff9de69	84	fio_syslet_complete_atom(td, atom);
a4f4fdd7 JA	85	} while (1);
	86	}
	87
	88	static int fio_syslet_getevents(struct thread_data *td, int min,
	89	int fio_unused max,
	90	struct timespec fio_unused *t)
	91	{
	92	struct syslet_data *sd = td->io_ops->data;
a4f4fdd7 JA	93	long ret;
	94
	95	do {
	96	fio_syslet_complete(td);
	97
	98	/*
	99	* do we have enough immediate completions?
	100	*/
	101	if (sd->nr_events >= (unsigned int) min)
	102	break;
	103
	104	/*
	105	* OK, we need to wait for some events...
	106	*/
9ff9de69	107	ret = async_wait(1, sd->ahu.user_ring_idx, &sd->ahu);
a4f4fdd7	108	if (ret < 0)
e49499f8	109	return -errno;
a4f4fdd7 JA	110	} while (1);
	111
	112	ret = sd->nr_events;
	113	sd->nr_events = 0;
	114	return ret;
	115	}
	116
	117	static struct io_u fio_syslet_event(struct thread_data td, int event)
	118	{
	119	struct syslet_data *sd = td->io_ops->data;
	120
	121	return sd->events[event];
	122	}
	123
	124	static void init_atom(struct syslet_uatom atom, int nr, void arg0,
a2e1b08a JA	125	void arg1, void arg2, void arg3, void ret_ptr,
a2e1b08a JA	126	unsigned long flags, void *priv)
a4f4fdd7 JA	127	{
	128	atom->flags = flags;
	129	atom->nr = nr;
	130	atom->ret_ptr = ret_ptr;
a2e1b08a	131	atom->next = NULL;
a4f4fdd7 JA	132	atom->arg_ptr[0] = arg0;
	133	atom->arg_ptr[1] = arg1;
	134	atom->arg_ptr[2] = arg2;
a2e1b08a JA	135	atom->arg_ptr[3] = arg3;
a2e1b08a JA	136	atom->arg_ptr[4] = atom->arg_ptr[5] = NULL;
a4f4fdd7 JA	137	atom->private = priv;
	138	}
	139
	140	/*
	141	* Use seek atom for sync
	142	*/
	143	static void fio_syslet_prep_sync(struct io_u io_u, struct fio_file f)
	144	{
a2e1b08a	145	init_atom(&io_u->req.atom, __NR_fsync, &f->fd, NULL, NULL, NULL,
7d44a745	146	&io_u->req.ret, 0, io_u);
a4f4fdd7 JA	147	}
	148
	149	static void fio_syslet_prep_rw(struct io_u io_u, struct fio_file f)
	150	{
	151	int nr;
	152
a4f4fdd7 JA	153	/*
	154	* prepare rw
	155	*/
	156	if (io_u->ddir == DDIR_READ)
a2e1b08a	157	nr = __NR_pread64;
a4f4fdd7	158	else
a2e1b08a	159	nr = __NR_pwrite64;
a4f4fdd7	160
a2e1b08a	161	init_atom(&io_u->req.atom, nr, &f->fd, &io_u->xfer_buf,
7d44a745	162	&io_u->xfer_buflen, &io_u->offset, &io_u->req.ret, 0, io_u);
a4f4fdd7 JA	163	}
	164
	165	static int fio_syslet_prep(struct thread_data fio_unused td, struct io_u io_u)
	166	{
	167	struct fio_file *f = io_u->file;
	168
	169	if (io_u->ddir == DDIR_SYNC)
	170	fio_syslet_prep_sync(io_u, f);
	171	else
	172	fio_syslet_prep_rw(io_u, f);
	173
	174	return 0;
	175	}
	176
bf0dc8fa IM	177	static void cachemiss_thread_start(void)
	178	{
	179	while (1)
7756b0d0	180	async_thread(NULL, NULL);
bf0dc8fa IM	181	}
	182
	183	#define THREAD_STACK_SIZE (16384)
	184
	185	static unsigned long thread_stack_alloc()
	186	{
5b38ee84	187	return (unsigned long) malloc(THREAD_STACK_SIZE) + THREAD_STACK_SIZE;
bf0dc8fa IM	188	}
bf0dc8fa IM	189
a0a930ef JA	190	static void fio_syslet_queued(struct thread_data td, struct syslet_data sd)
	191	{
	192	struct syslet_uatom *atom;
	193	struct timeval now;
	194
	195	fio_gettime(&now, NULL);
	196
	197	atom = sd->head;
	198	while (atom) {
	199	struct io_u *io_u = atom->private;
	200
	201	memcpy(&io_u->issue_time, &now, sizeof(now));
	202	io_u_queued(td, io_u);
	203	atom = atom->next;
	204	}
	205	}
	206
9ff9de69	207	static int fio_syslet_commit(struct thread_data *td)
a4f4fdd7 JA	208	{
a4f4fdd7 JA	209	struct syslet_data *sd = td->io_ops->data;
bf0dc8fa	210	struct syslet_uatom *done;
9ff9de69 JA	211
	212	if (!sd->head)
	213	return 0;
a4f4fdd7	214
5b38ee84 JA	215	assert(!sd->tail->next);
5b38ee84 JA	216
bf0dc8fa IM	217	if (!sd->ahu.new_thread_stack)
	218	sd->ahu.new_thread_stack = thread_stack_alloc();
	219
a0a930ef JA	220	fio_syslet_queued(td, sd);
a0a930ef JA	221
7d44a745 JA	222	/*
	223	* On sync completion, the atom is returned. So on NULL return
	224	* it's queued asynchronously.
	225	*/
9ff9de69	226	done = async_exec(sd->head, &sd->ahu);
bf0dc8fa	227
9ff9de69	228	sd->head = sd->tail = NULL;
a4f4fdd7	229
9ff9de69 JA	230	if (done)
9ff9de69 JA	231	fio_syslet_complete_atom(td, done);
a4f4fdd7	232
9ff9de69 JA	233	return 0;
	234	}
	235
	236	static int fio_syslet_queue(struct thread_data td, struct io_u io_u)
	237	{
	238	struct syslet_data *sd = td->io_ops->data;
bf0dc8fa	239
9ff9de69 JA	240	if (sd->tail) {
	241	sd->tail->next = &io_u->req.atom;
	242	sd->tail = &io_u->req.atom;
	243	} else
	244	sd->head = sd->tail = &io_u->req.atom;
a4f4fdd7	245
5b38ee84	246	io_u->req.head = sd->head;
9ff9de69	247	return FIO_Q_QUEUED;
a4f4fdd7 JA	248	}
a4f4fdd7 JA	249
db64e9bc	250	static int async_head_init(struct syslet_data *sd, unsigned int depth)
a4f4fdd7	251	{
a4f4fdd7 JA	252	unsigned long ring_size;
a4f4fdd7 JA	253
bf0dc8fa	254	memset(&sd->ahu, 0, sizeof(struct async_head_user));
2ca50be4	255
a4f4fdd7 JA	256	ring_size = sizeof(struct syslet_uatom ) depth;
	257	sd->ring = malloc(ring_size);
	258	memset(sd->ring, 0, ring_size);
	259
bf0dc8fa IM	260	sd->ahu.user_ring_idx = 0;
	261	sd->ahu.completion_ring = sd->ring;
	262	sd->ahu.ring_size_bytes = ring_size;
	263	sd->ahu.head_stack = thread_stack_alloc();
5b38ee84 JA	264	sd->ahu.head_eip = (unsigned long) cachemiss_thread_start;
5b38ee84 JA	265	sd->ahu.new_thread_eip = (unsigned long) cachemiss_thread_start;
db64e9bc JA	266
db64e9bc JA	267	return 0;
a4f4fdd7 JA	268	}
a4f4fdd7 JA	269
2ca50be4	270	static void async_head_exit(struct syslet_data *sd)
a4f4fdd7	271	{
7f059a76	272	free(sd->ring);
a4f4fdd7 JA	273	}
	274
	275	static void fio_syslet_cleanup(struct thread_data *td)
	276	{
	277	struct syslet_data *sd = td->io_ops->data;
	278
	279	if (sd) {
2ca50be4	280	async_head_exit(sd);
a4f4fdd7 JA	281	free(sd->events);
	282	free(sd);
	283	td->io_ops->data = NULL;
	284	}
	285	}
	286
	287	static int fio_syslet_init(struct thread_data *td)
	288	{
	289	struct syslet_data *sd;
	290
db64e9bc	291
a4f4fdd7 JA	292	sd = malloc(sizeof(*sd));
	293	memset(sd, 0, sizeof(*sd));
	294	sd->events = malloc(sizeof(struct io_u ) td->iodepth);
	295	memset(sd->events, 0, sizeof(struct io_u ) td->iodepth);
db64e9bc JA	296
	297	/*
	298	* This will handily fail for kernels where syslet isn't available
	299	*/
	300	if (async_head_init(sd, td->iodepth)) {
	301	free(sd->events);
	302	free(sd);
	303	return 1;
	304	}
	305
a4f4fdd7	306	td->io_ops->data = sd;
a4f4fdd7 JA	307	return 0;
	308	}
	309
	310	static struct ioengine_ops ioengine = {
	311	.name = "syslet-rw",
	312	.version = FIO_IOOPS_VERSION,
	313	.init = fio_syslet_init,
	314	.prep = fio_syslet_prep,
	315	.queue = fio_syslet_queue,
9ff9de69	316	.commit = fio_syslet_commit,
a4f4fdd7 JA	317	.getevents = fio_syslet_getevents,
	318	.event = fio_syslet_event,
	319	.cleanup = fio_syslet_cleanup,
	320	};
	321
	322	#else /* FIO_HAVE_SYSLET */
	323
	324	/*
	325	* When we have a proper configure system in place, we simply wont build
	326	* and install this io engine. For now install a crippled version that
	327	* just complains and fails to load.
	328	*/
	329	static int fio_syslet_init(struct thread_data fio_unused *td)
	330	{
	331	fprintf(stderr, "fio: syslet not available\n");
	332	return 1;
	333	}
	334
	335	static struct ioengine_ops ioengine = {
	336	.name = "syslet-rw",
	337	.version = FIO_IOOPS_VERSION,
	338	.init = fio_syslet_init,
	339	};
	340
	341	#endif /* FIO_HAVE_SYSLET */
	342
	343	static void fio_init fio_syslet_register(void)
	344	{
	345	register_ioengine(&ioengine);
	346	}
	347
	348	static void fio_exit fio_syslet_unregister(void)
	349	{
	350	unregister_ioengine(&ioengine);
	351	}