[fio.git] / engines / syslet-rw.c

/*
 * read/write() engine that uses syslet to be async
 *
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <assert.h>

#include "../fio.h"
#include "../os.h"

#ifdef FIO_HAVE_SYSLET

struct syslet_data {
	struct io_u **events;
	unsigned int nr_events;
	
	struct async_head_user ahu;
	struct syslet_uatom **ring;

	struct syslet_uatom *head, *tail;
	struct syslet_uatom **event_map;
	unsigned int event_map_idx;
};

static void fio_syslet_complete_atom(struct thread_data *td,
				     struct syslet_uatom *atom)
{
	struct syslet_data *sd = td->io_ops->data;
	struct io_u *io_u;
	int i, end;

	if (!sd->event_map_idx)
		return;

	/*
	 * Find the start of the string of atoms for this sequence
	 */
	for (end = sd->event_map_idx - 1; end >= 0; end--)
		if (atom == sd->event_map[end])
			break;

	if (end < 0 || atom != sd->event_map[end]) {
		printf("didn't find atom\n");
		return;
	}

	//printf("end=%d, total %d\n", end, sd->event_map_idx);

	/*
	 * now complete in right order
	 */
	for (i = 0; i <= end; i++) {
		long ret;

		atom = sd->event_map[i];
		io_u = atom->private;
		ret = *atom->ret_ptr;
		if (ret > 0)
			io_u->resid = io_u->xfer_buflen - ret;
		else if (ret < 0)
			io_u->error = ret;

		assert(sd->nr_events < td->iodepth);
		sd->events[sd->nr_events++] = io_u;
	}

	/*
	 * Move later completions to the front, if we didn't complete all
	 */
	if (end == (int) sd->event_map_idx - 1)
		sd->event_map_idx = 0;
	else {
		int nr = sd->event_map_idx - end - 1;

		memmove(sd->event_map, &sd->event_map[end + 1], nr * sizeof(struct syslet_uatom *));
		sd->event_map_idx = nr;
	}
}

/*
 * Inspect the ring to see if we have completed events
 */
static void fio_syslet_complete(struct thread_data *td)
{
	struct syslet_data *sd = td->io_ops->data;

	do {
		struct syslet_uatom *atom;

		atom = sd->ring[sd->ahu.user_ring_idx];
		if (!atom)
			break;

		sd->ring[sd->ahu.user_ring_idx] = NULL;
		if (++sd->ahu.user_ring_idx == td->iodepth)
			sd->ahu.user_ring_idx = 0;

		fio_syslet_complete_atom(td, atom);
	} while (1);
}

static int fio_syslet_getevents(struct thread_data *td, int min,
				int fio_unused max,
				struct timespec fio_unused *t)
{
	struct syslet_data *sd = td->io_ops->data;
	long ret;

	do {
		fio_syslet_complete(td);

		/*
		 * do we have enough immediate completions?
		 */
		if (sd->nr_events >= (unsigned int) min)
			break;

		/*
		 * OK, we need to wait for some events...
		 */
		ret = async_wait(1, sd->ahu.user_ring_idx, &sd->ahu);
		if (ret < 0)
			return -errno;
	} while (1);

	ret = sd->nr_events;
	sd->nr_events = 0;
	return ret;
}

static struct io_u *fio_syslet_event(struct thread_data *td, int event)
{
	struct syslet_data *sd = td->io_ops->data;

	return sd->events[event];
}

static void init_atom(struct syslet_uatom *atom, int nr, void *arg0,
		      void *arg1, void *arg2, void *arg3, void *ret_ptr,
		      unsigned long flags, void *priv)
{
	atom->flags = flags;
	atom->nr = nr;
	atom->ret_ptr = ret_ptr;
	atom->next = NULL;
	atom->arg_ptr[0] = arg0;
	atom->arg_ptr[1] = arg1;
	atom->arg_ptr[2] = arg2;
	atom->arg_ptr[3] = arg3;
	atom->arg_ptr[4] = atom->arg_ptr[5] = NULL;
	atom->private = priv;
}

/*
 * Use seek atom for sync
 */
static void fio_syslet_prep_sync(struct io_u *io_u, struct fio_file *f)
{
	init_atom(&io_u->req.atom, __NR_fsync, &f->fd, NULL, NULL, NULL,
		  &io_u->req.ret, 0, io_u);
}

static void fio_syslet_prep_rw(struct io_u *io_u, struct fio_file *f)
{
	int nr;

	/*
	 * prepare rw
	 */
	if (io_u->ddir == DDIR_READ)
		nr = __NR_pread64;
	else
		nr = __NR_pwrite64;

	init_atom(&io_u->req.atom, nr, &f->fd, &io_u->xfer_buf,
		  &io_u->xfer_buflen, &io_u->offset, &io_u->req.ret, 0, io_u);
}

static int fio_syslet_prep(struct thread_data fio_unused *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;

	if (io_u->ddir == DDIR_SYNC)
		fio_syslet_prep_sync(io_u, f);
	else
		fio_syslet_prep_rw(io_u, f);

	return 0;
}

static void cachemiss_thread_start(void)
{
	while (1)
		async_thread(NULL, NULL);
}

#define THREAD_STACK_SIZE (16384)

static unsigned long thread_stack_alloc()
{
	return (unsigned long)malloc(THREAD_STACK_SIZE) + THREAD_STACK_SIZE;
}

static int fio_syslet_commit(struct thread_data *td)
{
	struct syslet_data *sd = td->io_ops->data;
	struct syslet_uatom *done;

	if (!sd->head)
		return 0;

	if (!sd->ahu.new_thread_stack)
		sd->ahu.new_thread_stack = thread_stack_alloc();

	/*
	 * On sync completion, the atom is returned. So on NULL return
	 * it's queued asynchronously.
	 */
	done = async_exec(sd->head, &sd->ahu);

	sd->head = sd->tail = NULL;

	if (done)
		fio_syslet_complete_atom(td, done);

	return 0;
}

static int fio_syslet_queue(struct thread_data *td, struct io_u *io_u)
{
	struct syslet_data *sd = td->io_ops->data;

	if (sd->tail) {
		sd->tail->next = &io_u->req.atom;
		sd->tail = &io_u->req.atom;
	} else
		sd->head = sd->tail = &io_u->req.atom;

	sd->event_map[sd->event_map_idx++] = sd->tail;
	return FIO_Q_QUEUED;
}

static int async_head_init(struct syslet_data *sd, unsigned int depth)
{
	unsigned long ring_size;

	memset(&sd->ahu, 0, sizeof(struct async_head_user));

	ring_size = sizeof(struct syslet_uatom *) * depth;
	sd->ring = malloc(ring_size);
	memset(sd->ring, 0, ring_size);

	sd->ahu.user_ring_idx = 0;
	sd->ahu.completion_ring = sd->ring;
	sd->ahu.ring_size_bytes = ring_size;
	sd->ahu.head_stack = thread_stack_alloc();
	sd->ahu.head_eip = (unsigned long)cachemiss_thread_start;
	sd->ahu.new_thread_eip = (unsigned long)cachemiss_thread_start;

	return 0;
}

static void async_head_exit(struct syslet_data *sd)
{
	free(sd->ring);
}

static void fio_syslet_cleanup(struct thread_data *td)
{
	struct syslet_data *sd = td->io_ops->data;

	if (sd) {
		async_head_exit(sd);
		free(sd->events);
		free(sd->event_map);
		free(sd);
		td->io_ops->data = NULL;
	}
}

static int fio_syslet_init(struct thread_data *td)
{
	struct syslet_data *sd;


	sd = malloc(sizeof(*sd));
	memset(sd, 0, sizeof(*sd));
	sd->events = malloc(sizeof(struct io_u *) * td->iodepth);
	memset(sd->events, 0, sizeof(struct io_u *) * td->iodepth);
	sd->event_map = malloc(sizeof(struct syslet_uatom *) * td->iodepth);
	memset(sd->event_map, 0, sizeof(struct syslet_uatom *) * td->iodepth);

	/*
	 * This will handily fail for kernels where syslet isn't available
	 */
	if (async_head_init(sd, td->iodepth)) {
		free(sd->events);
		free(sd);
		return 1;
	}

	td->io_ops->data = sd;
	return 0;
}

static struct ioengine_ops ioengine = {
	.name		= "syslet-rw",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_syslet_init,
	.prep		= fio_syslet_prep,
	.queue		= fio_syslet_queue,
	.commit		= fio_syslet_commit,
	.getevents	= fio_syslet_getevents,
	.event		= fio_syslet_event,
	.cleanup	= fio_syslet_cleanup,
};

#else /* FIO_HAVE_SYSLET */

/*
 * When we have a proper configure system in place, we simply wont build
 * and install this io engine. For now install a crippled version that
 * just complains and fails to load.
 */
static int fio_syslet_init(struct thread_data fio_unused *td)
{
	fprintf(stderr, "fio: syslet not available\n");
	return 1;
}

static struct ioengine_ops ioengine = {
	.name		= "syslet-rw",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_syslet_init,
};

#endif /* FIO_HAVE_SYSLET */

static void fio_init fio_syslet_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_syslet_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
a4f4fdd7 JA	1	/*
	2	* read/write() engine that uses syslet to be async
	3	*
	4	*/
	5	#include <stdio.h>
	6	#include <stdlib.h>
	7	#include <unistd.h>
	8	#include <errno.h>
	9	#include <assert.h>
	10
	11	#include "../fio.h"
	12	#include "../os.h"
	13
	14	#ifdef FIO_HAVE_SYSLET
	15
	16	struct syslet_data {
	17	struct io_u **events;
	18	unsigned int nr_events;
	19
bf0dc8fa	20	struct async_head_user ahu;
a4f4fdd7	21	struct syslet_uatom **ring;
9ff9de69 JA	22
	23	struct syslet_uatom head, tail;
	24	struct syslet_uatom **event_map;
	25	unsigned int event_map_idx;
a4f4fdd7 JA	26	};
a4f4fdd7 JA	27
9ff9de69 JA	28	static void fio_syslet_complete_atom(struct thread_data *td,
	29	struct syslet_uatom *atom)
	30	{
	31	struct syslet_data *sd = td->io_ops->data;
	32	struct io_u *io_u;
	33	int i, end;
	34
	35	if (!sd->event_map_idx)
	36	return;
	37
	38	/*
	39	* Find the start of the string of atoms for this sequence
	40	*/
	41	for (end = sd->event_map_idx - 1; end >= 0; end--)
	42	if (atom == sd->event_map[end])
	43	break;
	44
	45	if (end < 0 \|\| atom != sd->event_map[end]) {
	46	printf("didn't find atom\n");
	47	return;
	48	}
	49
	50	//printf("end=%d, total %d\n", end, sd->event_map_idx);
	51
	52	/*
	53	* now complete in right order
	54	*/
	55	for (i = 0; i <= end; i++) {
	56	long ret;
	57
	58	atom = sd->event_map[i];
	59	io_u = atom->private;
	60	ret = *atom->ret_ptr;
	61	if (ret > 0)
	62	io_u->resid = io_u->xfer_buflen - ret;
	63	else if (ret < 0)
	64	io_u->error = ret;
	65
	66	assert(sd->nr_events < td->iodepth);
	67	sd->events[sd->nr_events++] = io_u;
	68	}
	69
	70	/*
	71	* Move later completions to the front, if we didn't complete all
	72	*/
	73	if (end == (int) sd->event_map_idx - 1)
	74	sd->event_map_idx = 0;
	75	else {
	76	int nr = sd->event_map_idx - end - 1;
	77
	78	memmove(sd->event_map, &sd->event_map[end + 1], nr * sizeof(struct syslet_uatom *));
	79	sd->event_map_idx = nr;
	80	}
	81	}
	82
a4f4fdd7 JA	83	/*
	84	* Inspect the ring to see if we have completed events
	85	*/
	86	static void fio_syslet_complete(struct thread_data *td)
	87	{
	88	struct syslet_data *sd = td->io_ops->data;
	89
	90	do {
	91	struct syslet_uatom *atom;
a4f4fdd7	92
bf0dc8fa	93	atom = sd->ring[sd->ahu.user_ring_idx];
a4f4fdd7 JA	94	if (!atom)
	95	break;
	96
bf0dc8fa IM	97	sd->ring[sd->ahu.user_ring_idx] = NULL;
	98	if (++sd->ahu.user_ring_idx == td->iodepth)
	99	sd->ahu.user_ring_idx = 0;
a4f4fdd7	100
9ff9de69	101	fio_syslet_complete_atom(td, atom);
a4f4fdd7 JA	102	} while (1);
	103	}
	104
	105	static int fio_syslet_getevents(struct thread_data *td, int min,
	106	int fio_unused max,
	107	struct timespec fio_unused *t)
	108	{
	109	struct syslet_data *sd = td->io_ops->data;
a4f4fdd7 JA	110	long ret;
	111
	112	do {
	113	fio_syslet_complete(td);
	114
	115	/*
	116	* do we have enough immediate completions?
	117	*/
	118	if (sd->nr_events >= (unsigned int) min)
	119	break;
	120
	121	/*
	122	* OK, we need to wait for some events...
	123	*/
9ff9de69	124	ret = async_wait(1, sd->ahu.user_ring_idx, &sd->ahu);
a4f4fdd7	125	if (ret < 0)
e49499f8	126	return -errno;
a4f4fdd7 JA	127	} while (1);
	128
	129	ret = sd->nr_events;
	130	sd->nr_events = 0;
	131	return ret;
	132	}
	133
	134	static struct io_u fio_syslet_event(struct thread_data td, int event)
	135	{
	136	struct syslet_data *sd = td->io_ops->data;
	137
	138	return sd->events[event];
	139	}
	140
	141	static void init_atom(struct syslet_uatom atom, int nr, void arg0,
a2e1b08a JA	142	void arg1, void arg2, void arg3, void ret_ptr,
a2e1b08a JA	143	unsigned long flags, void *priv)
a4f4fdd7 JA	144	{
	145	atom->flags = flags;
	146	atom->nr = nr;
	147	atom->ret_ptr = ret_ptr;
a2e1b08a	148	atom->next = NULL;
a4f4fdd7 JA	149	atom->arg_ptr[0] = arg0;
	150	atom->arg_ptr[1] = arg1;
	151	atom->arg_ptr[2] = arg2;
a2e1b08a JA	152	atom->arg_ptr[3] = arg3;
a2e1b08a JA	153	atom->arg_ptr[4] = atom->arg_ptr[5] = NULL;
a4f4fdd7 JA	154	atom->private = priv;
	155	}
	156
	157	/*
	158	* Use seek atom for sync
	159	*/
	160	static void fio_syslet_prep_sync(struct io_u io_u, struct fio_file f)
	161	{
a2e1b08a	162	init_atom(&io_u->req.atom, __NR_fsync, &f->fd, NULL, NULL, NULL,
7d44a745	163	&io_u->req.ret, 0, io_u);
a4f4fdd7 JA	164	}
	165
	166	static void fio_syslet_prep_rw(struct io_u io_u, struct fio_file f)
	167	{
	168	int nr;
	169
a4f4fdd7 JA	170	/*
	171	* prepare rw
	172	*/
	173	if (io_u->ddir == DDIR_READ)
a2e1b08a	174	nr = __NR_pread64;
a4f4fdd7	175	else
a2e1b08a	176	nr = __NR_pwrite64;
a4f4fdd7	177
a2e1b08a	178	init_atom(&io_u->req.atom, nr, &f->fd, &io_u->xfer_buf,
7d44a745	179	&io_u->xfer_buflen, &io_u->offset, &io_u->req.ret, 0, io_u);
a4f4fdd7 JA	180	}
	181
	182	static int fio_syslet_prep(struct thread_data fio_unused td, struct io_u io_u)
	183	{
	184	struct fio_file *f = io_u->file;
	185
	186	if (io_u->ddir == DDIR_SYNC)
	187	fio_syslet_prep_sync(io_u, f);
	188	else
	189	fio_syslet_prep_rw(io_u, f);
	190
	191	return 0;
	192	}
	193
bf0dc8fa IM	194	static void cachemiss_thread_start(void)
	195	{
	196	while (1)
7756b0d0	197	async_thread(NULL, NULL);
bf0dc8fa IM	198	}
	199
	200	#define THREAD_STACK_SIZE (16384)
	201
	202	static unsigned long thread_stack_alloc()
	203	{
	204	return (unsigned long)malloc(THREAD_STACK_SIZE) + THREAD_STACK_SIZE;
	205	}
	206
9ff9de69	207	static int fio_syslet_commit(struct thread_data *td)
a4f4fdd7 JA	208	{
a4f4fdd7 JA	209	struct syslet_data *sd = td->io_ops->data;
bf0dc8fa	210	struct syslet_uatom *done;
9ff9de69 JA	211
	212	if (!sd->head)
	213	return 0;
a4f4fdd7	214
bf0dc8fa IM	215	if (!sd->ahu.new_thread_stack)
	216	sd->ahu.new_thread_stack = thread_stack_alloc();
	217
7d44a745 JA	218	/*
	219	* On sync completion, the atom is returned. So on NULL return
	220	* it's queued asynchronously.
	221	*/
9ff9de69	222	done = async_exec(sd->head, &sd->ahu);
bf0dc8fa	223
9ff9de69	224	sd->head = sd->tail = NULL;
a4f4fdd7	225
9ff9de69 JA	226	if (done)
9ff9de69 JA	227	fio_syslet_complete_atom(td, done);
a4f4fdd7	228
9ff9de69 JA	229	return 0;
	230	}
	231
	232	static int fio_syslet_queue(struct thread_data td, struct io_u io_u)
	233	{
	234	struct syslet_data *sd = td->io_ops->data;
bf0dc8fa	235
9ff9de69 JA	236	if (sd->tail) {
	237	sd->tail->next = &io_u->req.atom;
	238	sd->tail = &io_u->req.atom;
	239	} else
	240	sd->head = sd->tail = &io_u->req.atom;
a4f4fdd7	241
9ff9de69 JA	242	sd->event_map[sd->event_map_idx++] = sd->tail;
9ff9de69 JA	243	return FIO_Q_QUEUED;
a4f4fdd7 JA	244	}
a4f4fdd7 JA	245
db64e9bc	246	static int async_head_init(struct syslet_data *sd, unsigned int depth)
a4f4fdd7	247	{
a4f4fdd7 JA	248	unsigned long ring_size;
a4f4fdd7 JA	249
bf0dc8fa	250	memset(&sd->ahu, 0, sizeof(struct async_head_user));
2ca50be4	251
a4f4fdd7 JA	252	ring_size = sizeof(struct syslet_uatom ) depth;
	253	sd->ring = malloc(ring_size);
	254	memset(sd->ring, 0, ring_size);
	255
bf0dc8fa IM	256	sd->ahu.user_ring_idx = 0;
	257	sd->ahu.completion_ring = sd->ring;
	258	sd->ahu.ring_size_bytes = ring_size;
	259	sd->ahu.head_stack = thread_stack_alloc();
	260	sd->ahu.head_eip = (unsigned long)cachemiss_thread_start;
	261	sd->ahu.new_thread_eip = (unsigned long)cachemiss_thread_start;
db64e9bc JA	262
db64e9bc JA	263	return 0;
a4f4fdd7 JA	264	}
a4f4fdd7 JA	265
2ca50be4	266	static void async_head_exit(struct syslet_data *sd)
a4f4fdd7	267	{
7f059a76	268	free(sd->ring);
a4f4fdd7 JA	269	}
	270
	271	static void fio_syslet_cleanup(struct thread_data *td)
	272	{
	273	struct syslet_data *sd = td->io_ops->data;
	274
	275	if (sd) {
2ca50be4	276	async_head_exit(sd);
a4f4fdd7	277	free(sd->events);
9ff9de69	278	free(sd->event_map);
a4f4fdd7 JA	279	free(sd);
	280	td->io_ops->data = NULL;
	281	}
	282	}
	283
	284	static int fio_syslet_init(struct thread_data *td)
	285	{
	286	struct syslet_data *sd;
	287
db64e9bc	288
a4f4fdd7 JA	289	sd = malloc(sizeof(*sd));
	290	memset(sd, 0, sizeof(*sd));
	291	sd->events = malloc(sizeof(struct io_u ) td->iodepth);
	292	memset(sd->events, 0, sizeof(struct io_u ) td->iodepth);
9ff9de69 JA	293	sd->event_map = malloc(sizeof(struct syslet_uatom ) td->iodepth);
9ff9de69 JA	294	memset(sd->event_map, 0, sizeof(struct syslet_uatom ) td->iodepth);
db64e9bc JA	295
	296	/*
	297	* This will handily fail for kernels where syslet isn't available
	298	*/
	299	if (async_head_init(sd, td->iodepth)) {
	300	free(sd->events);
	301	free(sd);
	302	return 1;
	303	}
	304
a4f4fdd7	305	td->io_ops->data = sd;
a4f4fdd7 JA	306	return 0;
	307	}
	308
	309	static struct ioengine_ops ioengine = {
	310	.name = "syslet-rw",
	311	.version = FIO_IOOPS_VERSION,
	312	.init = fio_syslet_init,
	313	.prep = fio_syslet_prep,
	314	.queue = fio_syslet_queue,
9ff9de69	315	.commit = fio_syslet_commit,
a4f4fdd7 JA	316	.getevents = fio_syslet_getevents,
	317	.event = fio_syslet_event,
	318	.cleanup = fio_syslet_cleanup,
	319	};
	320
	321	#else /* FIO_HAVE_SYSLET */
	322
	323	/*
	324	* When we have a proper configure system in place, we simply wont build
	325	* and install this io engine. For now install a crippled version that
	326	* just complains and fails to load.
	327	*/
	328	static int fio_syslet_init(struct thread_data fio_unused *td)
	329	{
	330	fprintf(stderr, "fio: syslet not available\n");
	331	return 1;
	332	}
	333
	334	static struct ioengine_ops ioengine = {
	335	.name = "syslet-rw",
	336	.version = FIO_IOOPS_VERSION,
	337	.init = fio_syslet_init,
	338	};
	339
	340	#endif /* FIO_HAVE_SYSLET */
	341
	342	static void fio_init fio_syslet_register(void)
	343	{
	344	register_ioengine(&ioengine);
	345	}
	346
	347	static void fio_exit fio_syslet_unregister(void)
	348	{
	349	unregister_ioengine(&ioengine);
	350	}