[fio.git] / engines / syslet-rw.c

/*
 * syslet engine
 *
 * IO engine that does regular pread(2)/pwrite(2) to transfer data, but
 * with syslets to make the execution async.
 *
 */
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <assert.h>
#include <asm/unistd.h>

#include "../fio.h"

#ifdef FIO_HAVE_SYSLET

#ifdef __NR_pread64
#define __NR_fio_pread	__NR_pread64
#define __NR_fio_pwrite	__NR_pwrite64
#else
#define __NR_fio_pread	__NR_pread
#define __NR_fio_pwrite	__NR_pwrite
#endif

struct syslet_data {
	struct io_u **events;
	unsigned int nr_events;
	
	struct async_head_user ahu;
	struct syslet_uatom **ring;

	struct syslet_uatom *head, *tail;
};

static void fio_syslet_complete_atom(struct thread_data *td,
				     struct syslet_uatom *atom)
{
	struct syslet_data *sd = td->io_ops->data;
	struct syslet_uatom *last;
	struct io_u *io_u;

	/*
	 * complete from the beginning of the sequence up to (and
	 * including) this atom
	 */
	last = atom;
	io_u = atom->private;
	atom = io_u->req.head;

	/*
	 * now complete in right order
	 */
	do {
		long ret;

		io_u = atom->private;
		ret = *atom->ret_ptr;
		if (ret >= 0)
			io_u->resid = io_u->xfer_buflen - ret;
		else if (ret < 0)
			io_u->error = ret;

		assert(sd->nr_events < td->o.iodepth);
		sd->events[sd->nr_events++] = io_u;

		if (atom == last)
			break;

		atom = atom->next;
	} while (1);

	assert(!last->next);
}

/*
 * Inspect the ring to see if we have completed events
 */
static void fio_syslet_complete(struct thread_data *td)
{
	struct syslet_data *sd = td->io_ops->data;

	do {
		struct syslet_uatom *atom;

		atom = sd->ring[sd->ahu.user_ring_idx];
		if (!atom)
			break;

		sd->ring[sd->ahu.user_ring_idx] = NULL;
		if (++sd->ahu.user_ring_idx == td->o.iodepth)
			sd->ahu.user_ring_idx = 0;

		fio_syslet_complete_atom(td, atom);
	} while (1);
}

static int fio_syslet_getevents(struct thread_data *td, int min,
				int fio_unused max,
				struct timespec fio_unused *t)
{
	struct syslet_data *sd = td->io_ops->data;
	long ret;

	do {
		fio_syslet_complete(td);

		/*
		 * do we have enough immediate completions?
		 */
		if (sd->nr_events >= (unsigned int) min)
			break;

		/*
		 * OK, we need to wait for some events...
		 */
		ret = async_wait(1, sd->ahu.user_ring_idx, &sd->ahu);
		if (ret < 0)
			return -errno;
	} while (1);

	ret = sd->nr_events;
	sd->nr_events = 0;
	return ret;
}

static struct io_u *fio_syslet_event(struct thread_data *td, int event)
{
	struct syslet_data *sd = td->io_ops->data;

	return sd->events[event];
}

static void init_atom(struct syslet_uatom *atom, int nr, void *arg0,
		      void *arg1, void *arg2, void *arg3, void *ret_ptr,
		      unsigned long flags, void *priv)
{
	atom->flags = flags;
	atom->nr = nr;
	atom->ret_ptr = ret_ptr;
	atom->next = NULL;
	atom->arg_ptr[0] = arg0;
	atom->arg_ptr[1] = arg1;
	atom->arg_ptr[2] = arg2;
	atom->arg_ptr[3] = arg3;
	atom->arg_ptr[4] = atom->arg_ptr[5] = NULL;
	atom->private = priv;
}

/*
 * Use seek atom for sync
 */
static void fio_syslet_prep_sync(struct io_u *io_u, struct fio_file *f)
{
	init_atom(&io_u->req.atom, __NR_fsync, &f->fd, NULL, NULL, NULL,
		  &io_u->req.ret, 0, io_u);
}

static void fio_syslet_prep_rw(struct io_u *io_u, struct fio_file *f)
{
	int nr;

	/*
	 * prepare rw
	 */
	if (io_u->ddir == DDIR_READ)
		nr = __NR_fio_pread;
	else
		nr = __NR_fio_pwrite;

	init_atom(&io_u->req.atom, nr, &f->fd, &io_u->xfer_buf,
		  &io_u->xfer_buflen, &io_u->offset, &io_u->req.ret, 0, io_u);
}

static int fio_syslet_prep(struct thread_data fio_unused *td, struct io_u *io_u)
{
	struct fio_file *f = io_u->file;

	if (io_u->ddir == DDIR_SYNC)
		fio_syslet_prep_sync(io_u, f);
	else
		fio_syslet_prep_rw(io_u, f);

	return 0;
}

static void cachemiss_thread_start(void)
{
	while (1)
		async_thread(NULL, NULL);
}

#define THREAD_STACK_SIZE (16384)

static unsigned long thread_stack_alloc()
{
	return (unsigned long) malloc(THREAD_STACK_SIZE) + THREAD_STACK_SIZE;
}

static void fio_syslet_queued(struct thread_data *td, struct syslet_data *sd)
{
	struct syslet_uatom *atom;
	struct timeval now;

	fio_gettime(&now, NULL);

	atom = sd->head;
	while (atom) {
		struct io_u *io_u = atom->private;

		memcpy(&io_u->issue_time, &now, sizeof(now));
		io_u_queued(td, io_u);
		atom = atom->next;
	}
}

static int fio_syslet_commit(struct thread_data *td)
{
	struct syslet_data *sd = td->io_ops->data;
	struct syslet_uatom *done;

	if (!sd->head)
		return 0;

	assert(!sd->tail->next);

	if (!sd->ahu.new_thread_stack)
		sd->ahu.new_thread_stack = thread_stack_alloc();

	fio_syslet_queued(td, sd);

	/*
	 * On sync completion, the atom is returned. So on NULL return
	 * it's queued asynchronously.
	 */
	done = async_exec(sd->head, &sd->ahu);

	if (done == (void *) -1) {
		log_err("fio: syslets don't appear to work\n");
		return -1;
	}

	sd->head = sd->tail = NULL;

	if (done)
		fio_syslet_complete_atom(td, done);

	return 0;
}

static int fio_syslet_queue(struct thread_data *td, struct io_u *io_u)
{
	struct syslet_data *sd = td->io_ops->data;

	if (sd->tail) {
		sd->tail->next = &io_u->req.atom;
		sd->tail = &io_u->req.atom;
	} else
		sd->head = sd->tail = &io_u->req.atom;

	io_u->req.head = sd->head;
	return FIO_Q_QUEUED;
}

static int async_head_init(struct syslet_data *sd, unsigned int depth)
{
	unsigned long ring_size;

	memset(&sd->ahu, 0, sizeof(struct async_head_user));

	ring_size = sizeof(struct syslet_uatom *) * depth;
	sd->ring = malloc(ring_size);
	memset(sd->ring, 0, ring_size);

	sd->ahu.user_ring_idx = 0;
	sd->ahu.completion_ring = sd->ring;
	sd->ahu.ring_size_bytes = ring_size;
	sd->ahu.head_stack = thread_stack_alloc();
	sd->ahu.head_eip = (unsigned long) cachemiss_thread_start;
	sd->ahu.new_thread_eip = (unsigned long) cachemiss_thread_start;

	return 0;
}

static void async_head_exit(struct syslet_data *sd)
{
	free(sd->ring);
}

static int check_syslet_support(struct syslet_data *sd)
{
	struct syslet_uatom atom;
	void *ret;

	init_atom(&atom, __NR_getpid, NULL, NULL, NULL, NULL, NULL, 0, NULL);
	ret = async_exec(sd->head, &sd->ahu);
	if (ret == (void *) -1)
		return 1;

	return 0;
}

static void fio_syslet_cleanup(struct thread_data *td)
{
	struct syslet_data *sd = td->io_ops->data;

	if (sd) {
		async_head_exit(sd);
		free(sd->events);
		free(sd);
		td->io_ops->data = NULL;
	}
}

static int fio_syslet_init(struct thread_data *td)
{
	struct syslet_data *sd;

	sd = malloc(sizeof(*sd));
	memset(sd, 0, sizeof(*sd));
	sd->events = malloc(sizeof(struct io_u *) * td->o.iodepth);
	memset(sd->events, 0, sizeof(struct io_u *) * td->o.iodepth);

	/*
	 * This will handily fail for kernels where syslet isn't available
	 */
	if (async_head_init(sd, td->o.iodepth)) {
		free(sd->events);
		free(sd);
		return 1;
	}

	if (check_syslet_support(sd)) {
		log_err("fio: syslets do not appear to work\n");
		free(sd->events);
		free(sd);
		return 1;
	}

	td->io_ops->data = sd;
	return 0;
}

static struct ioengine_ops ioengine = {
	.name		= "syslet-rw",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_syslet_init,
	.prep		= fio_syslet_prep,
	.queue		= fio_syslet_queue,
	.commit		= fio_syslet_commit,
	.getevents	= fio_syslet_getevents,
	.event		= fio_syslet_event,
	.cleanup	= fio_syslet_cleanup,
	.open_file	= generic_open_file,
	.close_file	= generic_close_file,
};

#else /* FIO_HAVE_SYSLET */

/*
 * When we have a proper configure system in place, we simply wont build
 * and install this io engine. For now install a crippled version that
 * just complains and fails to load.
 */
static int fio_syslet_init(struct thread_data fio_unused *td)
{
	fprintf(stderr, "fio: syslet not available\n");
	return 1;
}

static struct ioengine_ops ioengine = {
	.name		= "syslet-rw",
	.version	= FIO_IOOPS_VERSION,
	.init		= fio_syslet_init,
};

#endif /* FIO_HAVE_SYSLET */

static void fio_init fio_syslet_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_syslet_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
	1	/*
	2	* syslet engine
	3	*
	4	* IO engine that does regular pread(2)/pwrite(2) to transfer data, but
	5	* with syslets to make the execution async.
	6	*
	7	*/
	8	#include <stdio.h>
	9	#include <stdlib.h>
	10	#include <unistd.h>
	11	#include <errno.h>
	12	#include <assert.h>
	13	#include <asm/unistd.h>
	14
	15	#include "../fio.h"
	16
	17	#ifdef FIO_HAVE_SYSLET
	18
	19	#ifdef __NR_pread64
	20	#define __NR_fio_pread __NR_pread64
	21	#define __NR_fio_pwrite __NR_pwrite64
	22	#else
	23	#define __NR_fio_pread __NR_pread
	24	#define __NR_fio_pwrite __NR_pwrite
	25	#endif
	26
	27	struct syslet_data {
	28	struct io_u **events;
	29	unsigned int nr_events;
	30
	31	struct async_head_user ahu;
	32	struct syslet_uatom **ring;
	33
	34	struct syslet_uatom head, tail;
	35	};
	36
	37	static void fio_syslet_complete_atom(struct thread_data *td,
	38	struct syslet_uatom *atom)
	39	{
	40	struct syslet_data *sd = td->io_ops->data;
	41	struct syslet_uatom *last;
	42	struct io_u *io_u;
	43
	44	/*
	45	* complete from the beginning of the sequence up to (and
	46	* including) this atom
	47	*/
	48	last = atom;
	49	io_u = atom->private;
	50	atom = io_u->req.head;
	51
	52	/*
	53	* now complete in right order
	54	*/
	55	do {
	56	long ret;
	57
	58	io_u = atom->private;
	59	ret = *atom->ret_ptr;
	60	if (ret >= 0)
	61	io_u->resid = io_u->xfer_buflen - ret;
	62	else if (ret < 0)
	63	io_u->error = ret;
	64
	65	assert(sd->nr_events < td->o.iodepth);
	66	sd->events[sd->nr_events++] = io_u;
	67
	68	if (atom == last)
	69	break;
	70
	71	atom = atom->next;
	72	} while (1);
	73
	74	assert(!last->next);
	75	}
	76
	77	/*
	78	* Inspect the ring to see if we have completed events
	79	*/
	80	static void fio_syslet_complete(struct thread_data *td)
	81	{
	82	struct syslet_data *sd = td->io_ops->data;
	83
	84	do {
	85	struct syslet_uatom *atom;
	86
	87	atom = sd->ring[sd->ahu.user_ring_idx];
	88	if (!atom)
	89	break;
	90
	91	sd->ring[sd->ahu.user_ring_idx] = NULL;
	92	if (++sd->ahu.user_ring_idx == td->o.iodepth)
	93	sd->ahu.user_ring_idx = 0;
	94
	95	fio_syslet_complete_atom(td, atom);
	96	} while (1);
	97	}
	98
	99	static int fio_syslet_getevents(struct thread_data *td, int min,
	100	int fio_unused max,
	101	struct timespec fio_unused *t)
	102	{
	103	struct syslet_data *sd = td->io_ops->data;
	104	long ret;
	105
	106	do {
	107	fio_syslet_complete(td);
	108
	109	/*
	110	* do we have enough immediate completions?
	111	*/
	112	if (sd->nr_events >= (unsigned int) min)
	113	break;
	114
	115	/*
	116	* OK, we need to wait for some events...
	117	*/
	118	ret = async_wait(1, sd->ahu.user_ring_idx, &sd->ahu);
	119	if (ret < 0)
	120	return -errno;
	121	} while (1);
	122
	123	ret = sd->nr_events;
	124	sd->nr_events = 0;
	125	return ret;
	126	}
	127
	128	static struct io_u fio_syslet_event(struct thread_data td, int event)
	129	{
	130	struct syslet_data *sd = td->io_ops->data;
	131
	132	return sd->events[event];
	133	}
	134
	135	static void init_atom(struct syslet_uatom atom, int nr, void arg0,
	136	void arg1, void arg2, void arg3, void ret_ptr,
	137	unsigned long flags, void *priv)
	138	{
	139	atom->flags = flags;
	140	atom->nr = nr;
	141	atom->ret_ptr = ret_ptr;
	142	atom->next = NULL;
	143	atom->arg_ptr[0] = arg0;
	144	atom->arg_ptr[1] = arg1;
	145	atom->arg_ptr[2] = arg2;
	146	atom->arg_ptr[3] = arg3;
	147	atom->arg_ptr[4] = atom->arg_ptr[5] = NULL;
	148	atom->private = priv;
	149	}
	150
	151	/*
	152	* Use seek atom for sync
	153	*/
	154	static void fio_syslet_prep_sync(struct io_u io_u, struct fio_file f)
	155	{
	156	init_atom(&io_u->req.atom, __NR_fsync, &f->fd, NULL, NULL, NULL,
	157	&io_u->req.ret, 0, io_u);
	158	}
	159
	160	static void fio_syslet_prep_rw(struct io_u io_u, struct fio_file f)
	161	{
	162	int nr;
	163
	164	/*
	165	* prepare rw
	166	*/
	167	if (io_u->ddir == DDIR_READ)
	168	nr = __NR_fio_pread;
	169	else
	170	nr = __NR_fio_pwrite;
	171
	172	init_atom(&io_u->req.atom, nr, &f->fd, &io_u->xfer_buf,
	173	&io_u->xfer_buflen, &io_u->offset, &io_u->req.ret, 0, io_u);
	174	}
	175
	176	static int fio_syslet_prep(struct thread_data fio_unused td, struct io_u io_u)
	177	{
	178	struct fio_file *f = io_u->file;
	179
	180	if (io_u->ddir == DDIR_SYNC)
	181	fio_syslet_prep_sync(io_u, f);
	182	else
	183	fio_syslet_prep_rw(io_u, f);
	184
	185	return 0;
	186	}
	187
	188	static void cachemiss_thread_start(void)
	189	{
	190	while (1)
	191	async_thread(NULL, NULL);
	192	}
	193
	194	#define THREAD_STACK_SIZE (16384)
	195
	196	static unsigned long thread_stack_alloc()
	197	{
	198	return (unsigned long) malloc(THREAD_STACK_SIZE) + THREAD_STACK_SIZE;
	199	}
	200
	201	static void fio_syslet_queued(struct thread_data td, struct syslet_data sd)
	202	{
	203	struct syslet_uatom *atom;
	204	struct timeval now;
	205
	206	fio_gettime(&now, NULL);
	207
	208	atom = sd->head;
	209	while (atom) {
	210	struct io_u *io_u = atom->private;
	211
	212	memcpy(&io_u->issue_time, &now, sizeof(now));
	213	io_u_queued(td, io_u);
	214	atom = atom->next;
	215	}
	216	}
	217
	218	static int fio_syslet_commit(struct thread_data *td)
	219	{
	220	struct syslet_data *sd = td->io_ops->data;
	221	struct syslet_uatom *done;
	222
	223	if (!sd->head)
	224	return 0;
	225
	226	assert(!sd->tail->next);
	227
	228	if (!sd->ahu.new_thread_stack)
	229	sd->ahu.new_thread_stack = thread_stack_alloc();
	230
	231	fio_syslet_queued(td, sd);
	232
	233	/*
	234	* On sync completion, the atom is returned. So on NULL return
	235	* it's queued asynchronously.
	236	*/
	237	done = async_exec(sd->head, &sd->ahu);
	238
	239	if (done == (void *) -1) {
	240	log_err("fio: syslets don't appear to work\n");
	241	return -1;
	242	}
	243
	244	sd->head = sd->tail = NULL;
	245
	246	if (done)
	247	fio_syslet_complete_atom(td, done);
	248
	249	return 0;
	250	}
	251
	252	static int fio_syslet_queue(struct thread_data td, struct io_u io_u)
	253	{
	254	struct syslet_data *sd = td->io_ops->data;
	255
	256	if (sd->tail) {
	257	sd->tail->next = &io_u->req.atom;
	258	sd->tail = &io_u->req.atom;
	259	} else
	260	sd->head = sd->tail = &io_u->req.atom;
	261
	262	io_u->req.head = sd->head;
	263	return FIO_Q_QUEUED;
	264	}
	265
	266	static int async_head_init(struct syslet_data *sd, unsigned int depth)
	267	{
	268	unsigned long ring_size;
	269
	270	memset(&sd->ahu, 0, sizeof(struct async_head_user));
	271
	272	ring_size = sizeof(struct syslet_uatom ) depth;
	273	sd->ring = malloc(ring_size);
	274	memset(sd->ring, 0, ring_size);
	275
	276	sd->ahu.user_ring_idx = 0;
	277	sd->ahu.completion_ring = sd->ring;
	278	sd->ahu.ring_size_bytes = ring_size;
	279	sd->ahu.head_stack = thread_stack_alloc();
	280	sd->ahu.head_eip = (unsigned long) cachemiss_thread_start;
	281	sd->ahu.new_thread_eip = (unsigned long) cachemiss_thread_start;
	282
	283	return 0;
	284	}
	285
	286	static void async_head_exit(struct syslet_data *sd)
	287	{
	288	free(sd->ring);
	289	}
	290
	291	static int check_syslet_support(struct syslet_data *sd)
	292	{
	293	struct syslet_uatom atom;
	294	void *ret;
	295
	296	init_atom(&atom, __NR_getpid, NULL, NULL, NULL, NULL, NULL, 0, NULL);
	297	ret = async_exec(sd->head, &sd->ahu);
	298	if (ret == (void *) -1)
	299	return 1;
	300
	301	return 0;
	302	}
	303
	304	static void fio_syslet_cleanup(struct thread_data *td)
	305	{
	306	struct syslet_data *sd = td->io_ops->data;
	307
	308	if (sd) {
	309	async_head_exit(sd);
	310	free(sd->events);
	311	free(sd);
	312	td->io_ops->data = NULL;
	313	}
	314	}
	315
	316	static int fio_syslet_init(struct thread_data *td)
	317	{
	318	struct syslet_data *sd;
	319
	320	sd = malloc(sizeof(*sd));
	321	memset(sd, 0, sizeof(*sd));
	322	sd->events = malloc(sizeof(struct io_u ) td->o.iodepth);
	323	memset(sd->events, 0, sizeof(struct io_u ) td->o.iodepth);
	324
	325	/*
	326	* This will handily fail for kernels where syslet isn't available
	327	*/
	328	if (async_head_init(sd, td->o.iodepth)) {
	329	free(sd->events);
	330	free(sd);
	331	return 1;
	332	}
	333
	334	if (check_syslet_support(sd)) {
	335	log_err("fio: syslets do not appear to work\n");
	336	free(sd->events);
	337	free(sd);
	338	return 1;
	339	}
	340
	341	td->io_ops->data = sd;
	342	return 0;
	343	}
	344
	345	static struct ioengine_ops ioengine = {
	346	.name = "syslet-rw",
	347	.version = FIO_IOOPS_VERSION,
	348	.init = fio_syslet_init,
	349	.prep = fio_syslet_prep,
	350	.queue = fio_syslet_queue,
	351	.commit = fio_syslet_commit,
	352	.getevents = fio_syslet_getevents,
	353	.event = fio_syslet_event,
	354	.cleanup = fio_syslet_cleanup,
	355	.open_file = generic_open_file,
	356	.close_file = generic_close_file,
	357	};
	358
	359	#else /* FIO_HAVE_SYSLET */
	360
	361	/*
	362	* When we have a proper configure system in place, we simply wont build
	363	* and install this io engine. For now install a crippled version that
	364	* just complains and fails to load.
	365	*/
	366	static int fio_syslet_init(struct thread_data fio_unused *td)
	367	{
	368	fprintf(stderr, "fio: syslet not available\n");
	369	return 1;
	370	}
	371
	372	static struct ioengine_ops ioengine = {
	373	.name = "syslet-rw",
	374	.version = FIO_IOOPS_VERSION,
	375	.init = fio_syslet_init,
	376	};
	377
	378	#endif /* FIO_HAVE_SYSLET */
	379
	380	static void fio_init fio_syslet_register(void)
	381	{
	382	register_ioengine(&ioengine);
	383	}
	384
	385	static void fio_exit fio_syslet_unregister(void)
	386	{
	387	unregister_ioengine(&ioengine);
	388	}