[fio.git] / engines / rbd.c

/*
 * rbd engine
 *
 * IO engine using Ceph's librbd to test RADOS Block Devices.
 *
 */

#include <rbd/librbd.h>

#include "../fio.h"
#include "../optgroup.h"

struct fio_rbd_iou {
	struct io_u *io_u;
	rbd_completion_t completion;
	int io_seen;
	int io_complete;
};

struct rbd_data {
	rados_t cluster;
	rados_ioctx_t io_ctx;
	rbd_image_t image;
	struct io_u **aio_events;
	struct io_u **sort_events;
};

struct rbd_options {
	void *pad;
	char *cluster_name;
	char *rbd_name;
	char *pool_name;
	char *client_name;
	int busy_poll;
};

static struct fio_option options[] = {
        {
		.name		= "clustername",
		.lname		= "ceph cluster name",
		.type		= FIO_OPT_STR_STORE,
		.help		= "Cluster name for ceph",
		.off1		= offsetof(struct rbd_options, cluster_name),
		.category	= FIO_OPT_C_ENGINE,
		.group		= FIO_OPT_G_RBD,
        },
	{
		.name		= "rbdname",
		.lname		= "rbd engine rbdname",
		.type		= FIO_OPT_STR_STORE,
		.help		= "RBD name for RBD engine",
		.off1		= offsetof(struct rbd_options, rbd_name),
		.category	= FIO_OPT_C_ENGINE,
		.group		= FIO_OPT_G_RBD,
	},
	{
		.name		= "pool",
		.lname		= "rbd engine pool",
		.type		= FIO_OPT_STR_STORE,
		.help		= "Name of the pool hosting the RBD for the RBD engine",
		.off1		= offsetof(struct rbd_options, pool_name),
		.category	= FIO_OPT_C_ENGINE,
		.group		= FIO_OPT_G_RBD,
	},
	{
		.name		= "clientname",
		.lname		= "rbd engine clientname",
		.type		= FIO_OPT_STR_STORE,
		.help		= "Name of the ceph client to access the RBD for the RBD engine",
		.off1		= offsetof(struct rbd_options, client_name),
		.category	= FIO_OPT_C_ENGINE,
		.group		= FIO_OPT_G_RBD,
	},
	{
		.name		= "busy_poll",
		.lname		= "Busy poll",
		.type		= FIO_OPT_BOOL,
		.help		= "Busy poll for completions instead of sleeping",
		.off1		= offsetof(struct rbd_options, busy_poll),
		.def		= "0",
		.category	= FIO_OPT_C_ENGINE,
		.group		= FIO_OPT_G_RBD,
	},
	{
		.name = NULL,
	},
};

static int _fio_setup_rbd_data(struct thread_data *td,
			       struct rbd_data **rbd_data_ptr)
{
	struct rbd_data *rbd;

	if (td->io_ops_data)
		return 0;

	rbd = calloc(1, sizeof(struct rbd_data));
	if (!rbd)
		goto failed;

	rbd->aio_events = calloc(td->o.iodepth, sizeof(struct io_u *));
	if (!rbd->aio_events)
		goto failed;

	rbd->sort_events = calloc(td->o.iodepth, sizeof(struct io_u *));
	if (!rbd->sort_events)
		goto failed;

	*rbd_data_ptr = rbd;
	return 0;

failed:
	if (rbd) {
		if (rbd->aio_events) 
			free(rbd->aio_events);
		if (rbd->sort_events)
			free(rbd->sort_events);
		free(rbd);
	}
	return 1;

}

static int _fio_rbd_connect(struct thread_data *td)
{
	struct rbd_data *rbd = td->io_ops_data;
	struct rbd_options *o = td->eo;
	int r;

	if (o->cluster_name) {
		char *client_name = NULL; 

		/*
		 * If we specify cluser name, the rados_create2
		 * will not assume 'client.'. name is considered
		 * as a full type.id namestr
		 */
		if (o->client_name) {
			if (!index(o->client_name, '.')) {
				client_name = calloc(1, strlen("client.") +
						    strlen(o->client_name) + 1);
				strcat(client_name, "client.");
				o->client_name = strcat(client_name,
							o->client_name);
			}
		}
		r = rados_create2(&rbd->cluster, o->cluster_name,
					o->client_name, 0);
	} else
		r = rados_create(&rbd->cluster, o->client_name);
	
	if (r < 0) {
		log_err("rados_create failed.\n");
		goto failed_early;
	}

	r = rados_conf_read_file(rbd->cluster, NULL);
	if (r < 0) {
		log_err("rados_conf_read_file failed.\n");
		goto failed_early;
	}

	r = rados_connect(rbd->cluster);
	if (r < 0) {
		log_err("rados_connect failed.\n");
		goto failed_shutdown;
	}

	r = rados_ioctx_create(rbd->cluster, o->pool_name, &rbd->io_ctx);
	if (r < 0) {
		log_err("rados_ioctx_create failed.\n");
		goto failed_shutdown;
	}

	r = rbd_open(rbd->io_ctx, o->rbd_name, &rbd->image, NULL /*snap */ );
	if (r < 0) {
		log_err("rbd_open failed.\n");
		goto failed_open;
	}
	return 0;

failed_open:
	rados_ioctx_destroy(rbd->io_ctx);
	rbd->io_ctx = NULL;
failed_shutdown:
	rados_shutdown(rbd->cluster);
	rbd->cluster = NULL;
failed_early:
	return 1;
}

static void _fio_rbd_disconnect(struct rbd_data *rbd)
{
	if (!rbd)
		return;

	/* shutdown everything */
	if (rbd->image) {
		rbd_close(rbd->image);
		rbd->image = NULL;
	}

	if (rbd->io_ctx) {
		rados_ioctx_destroy(rbd->io_ctx);
		rbd->io_ctx = NULL;
	}

	if (rbd->cluster) {
		rados_shutdown(rbd->cluster);
		rbd->cluster = NULL;
	}
}

static void _fio_rbd_finish_aiocb(rbd_completion_t comp, void *data)
{
	struct fio_rbd_iou *fri = data;
	struct io_u *io_u = fri->io_u;
	ssize_t ret;

	/*
	 * Looks like return value is 0 for success, or < 0 for
	 * a specific error. So we have to assume that it can't do
	 * partial completions.
	 */
	ret = rbd_aio_get_return_value(fri->completion);
	if (ret < 0) {
		io_u->error = ret;
		io_u->resid = io_u->xfer_buflen;
	} else
		io_u->error = 0;

	fri->io_complete = 1;
}

static struct io_u *fio_rbd_event(struct thread_data *td, int event)
{
	struct rbd_data *rbd = td->io_ops_data;

	return rbd->aio_events[event];
}

static inline int fri_check_complete(struct rbd_data *rbd, struct io_u *io_u,
				     unsigned int *events)
{
	struct fio_rbd_iou *fri = io_u->engine_data;

	if (fri->io_complete) {
		fri->io_seen = 1;
		rbd->aio_events[*events] = io_u;
		(*events)++;

		rbd_aio_release(fri->completion);
		return 1;
	}

	return 0;
}

static inline int rbd_io_u_seen(struct io_u *io_u)
{
	struct fio_rbd_iou *fri = io_u->engine_data;

	return fri->io_seen;
}

static void rbd_io_u_wait_complete(struct io_u *io_u)
{
	struct fio_rbd_iou *fri = io_u->engine_data;

	rbd_aio_wait_for_complete(fri->completion);
}

static int rbd_io_u_cmp(const void *p1, const void *p2)
{
	const struct io_u **a = (const struct io_u **) p1;
	const struct io_u **b = (const struct io_u **) p2;
	uint64_t at, bt;

	at = utime_since_now(&(*a)->start_time);
	bt = utime_since_now(&(*b)->start_time);

	if (at < bt)
		return -1;
	else if (at == bt)
		return 0;
	else
		return 1;
}

static int rbd_iter_events(struct thread_data *td, unsigned int *events,
			   unsigned int min_evts, int wait)
{
	struct rbd_data *rbd = td->io_ops_data;
	unsigned int this_events = 0;
	struct io_u *io_u;
	int i, sidx;

	sidx = 0;
	io_u_qiter(&td->io_u_all, io_u, i) {
		if (!(io_u->flags & IO_U_F_FLIGHT))
			continue;
		if (rbd_io_u_seen(io_u))
			continue;

		if (fri_check_complete(rbd, io_u, events))
			this_events++;
		else if (wait)
			rbd->sort_events[sidx++] = io_u;
	}

	if (!wait || !sidx)
		return this_events;

	/*
	 * Sort events, oldest issue first, then wait on as many as we
	 * need in order of age. If we have enough events, stop waiting,
	 * and just check if any of the older ones are done.
	 */
	if (sidx > 1)
		qsort(rbd->sort_events, sidx, sizeof(struct io_u *), rbd_io_u_cmp);

	for (i = 0; i < sidx; i++) {
		io_u = rbd->sort_events[i];

		if (fri_check_complete(rbd, io_u, events)) {
			this_events++;
			continue;
		}

		/*
		 * Stop waiting when we have enough, but continue checking
		 * all pending IOs if they are complete.
		 */
		if (*events >= min_evts)
			continue;

		rbd_io_u_wait_complete(io_u);

		if (fri_check_complete(rbd, io_u, events))
			this_events++;
	}

	return this_events;
}

static int fio_rbd_getevents(struct thread_data *td, unsigned int min,
			     unsigned int max, const struct timespec *t)
{
	unsigned int this_events, events = 0;
	struct rbd_options *o = td->eo;
	int wait = 0;

	do {
		this_events = rbd_iter_events(td, &events, min, wait);

		if (events >= min)
			break;
		if (this_events)
			continue;

		if (!o->busy_poll)
			wait = 1;
		else
			nop;
	} while (1);

	return events;
}

static int fio_rbd_queue(struct thread_data *td, struct io_u *io_u)
{
	struct rbd_data *rbd = td->io_ops_data;
	struct fio_rbd_iou *fri = io_u->engine_data;
	int r = -1;

	fio_ro_check(td, io_u);

	fri->io_seen = 0;
	fri->io_complete = 0;

	r = rbd_aio_create_completion(fri, _fio_rbd_finish_aiocb,
						&fri->completion);
	if (r < 0) {
		log_err("rbd_aio_create_completion failed.\n");
		goto failed;
	}

	if (io_u->ddir == DDIR_WRITE) {
		r = rbd_aio_write(rbd->image, io_u->offset, io_u->xfer_buflen,
					 io_u->xfer_buf, fri->completion);
		if (r < 0) {
			log_err("rbd_aio_write failed.\n");
			goto failed_comp;
		}

	} else if (io_u->ddir == DDIR_READ) {
		r = rbd_aio_read(rbd->image, io_u->offset, io_u->xfer_buflen,
					io_u->xfer_buf, fri->completion);

		if (r < 0) {
			log_err("rbd_aio_read failed.\n");
			goto failed_comp;
		}
	} else if (io_u->ddir == DDIR_TRIM) {
		r = rbd_aio_discard(rbd->image, io_u->offset,
					io_u->xfer_buflen, fri->completion);
		if (r < 0) {
			log_err("rbd_aio_discard failed.\n");
			goto failed_comp;
		}
	} else if (io_u->ddir == DDIR_SYNC) {
		r = rbd_aio_flush(rbd->image, fri->completion);
		if (r < 0) {
			log_err("rbd_flush failed.\n");
			goto failed_comp;
		}
	} else {
		dprint(FD_IO, "%s: Warning: unhandled ddir: %d\n", __func__,
		       io_u->ddir);
		goto failed_comp;
	}

	return FIO_Q_QUEUED;
failed_comp:
	rbd_aio_release(fri->completion);
failed:
	io_u->error = r;
	td_verror(td, io_u->error, "xfer");
	return FIO_Q_COMPLETED;
}

static int fio_rbd_init(struct thread_data *td)
{
	int r;

	r = _fio_rbd_connect(td);
	if (r) {
		log_err("fio_rbd_connect failed, return code: %d .\n", r);
		goto failed;
	}

	return 0;

failed:
	return 1;
}

static void fio_rbd_cleanup(struct thread_data *td)
{
	struct rbd_data *rbd = td->io_ops_data;

	if (rbd) {
		_fio_rbd_disconnect(rbd);
		free(rbd->aio_events);
		free(rbd->sort_events);
		free(rbd);
	}
}

static int fio_rbd_setup(struct thread_data *td)
{
	rbd_image_info_t info;
	struct fio_file *f;
	struct rbd_data *rbd = NULL;
	int major, minor, extra;
	int r;

	/* log version of librbd. No cluster connection required. */
	rbd_version(&major, &minor, &extra);
	log_info("rbd engine: RBD version: %d.%d.%d\n", major, minor, extra);

	/* allocate engine specific structure to deal with librbd. */
	r = _fio_setup_rbd_data(td, &rbd);
	if (r) {
		log_err("fio_setup_rbd_data failed.\n");
		goto cleanup;
	}
	td->io_ops_data = rbd;

	/* librbd does not allow us to run first in the main thread and later
	 * in a fork child. It needs to be the same process context all the
	 * time. 
	 */
	td->o.use_thread = 1;

	/* connect in the main thread to determine to determine
	 * the size of the given RADOS block device. And disconnect
	 * later on.
	 */
	r = _fio_rbd_connect(td);
	if (r) {
		log_err("fio_rbd_connect failed.\n");
		goto cleanup;
	}

	/* get size of the RADOS block device */
	r = rbd_stat(rbd->image, &info, sizeof(info));
	if (r < 0) {
		log_err("rbd_status failed.\n");
		goto disconnect;
	}
	dprint(FD_IO, "rbd-engine: image size: %lu\n", info.size);

	/* taken from "net" engine. Pretend we deal with files,
	 * even if we do not have any ideas about files.
	 * The size of the RBD is set instead of a artificial file.
	 */
	if (!td->files_index) {
		add_file(td, td->o.filename ? : "rbd", 0, 0);
		td->o.nr_files = td->o.nr_files ? : 1;
		td->o.open_files++;
	}
	f = td->files[0];
	f->real_file_size = info.size;

	/* disconnect, then we were only connected to determine
	 * the size of the RBD.
	 */
	_fio_rbd_disconnect(rbd);
	return 0;

disconnect:
	_fio_rbd_disconnect(rbd);
cleanup:
	fio_rbd_cleanup(td);
	return r;
}

static int fio_rbd_open(struct thread_data *td, struct fio_file *f)
{
	return 0;
}

static int fio_rbd_invalidate(struct thread_data *td, struct fio_file *f)
{
#if defined(CONFIG_RBD_INVAL)
	struct rbd_data *rbd = td->io_ops_data;

	return rbd_invalidate_cache(rbd->image);
#else
	return 0;
#endif
}

static void fio_rbd_io_u_free(struct thread_data *td, struct io_u *io_u)
{
	struct fio_rbd_iou *fri = io_u->engine_data;

	if (fri) {
		io_u->engine_data = NULL;
		free(fri);
	}
}

static int fio_rbd_io_u_init(struct thread_data *td, struct io_u *io_u)
{
	struct fio_rbd_iou *fri;

	fri = calloc(1, sizeof(*fri));
	fri->io_u = io_u;
	io_u->engine_data = fri;
	return 0;
}

static struct ioengine_ops ioengine = {
	.name			= "rbd",
	.version		= FIO_IOOPS_VERSION,
	.setup			= fio_rbd_setup,
	.init			= fio_rbd_init,
	.queue			= fio_rbd_queue,
	.getevents		= fio_rbd_getevents,
	.event			= fio_rbd_event,
	.cleanup		= fio_rbd_cleanup,
	.open_file		= fio_rbd_open,
	.invalidate		= fio_rbd_invalidate,
	.options		= options,
	.io_u_init		= fio_rbd_io_u_init,
	.io_u_free		= fio_rbd_io_u_free,
	.option_struct_size	= sizeof(struct rbd_options),
};

static void fio_init fio_rbd_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_rbd_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
fc5c0345 DG	1	/*
	2	* rbd engine
	3	*
	4	* IO engine using Ceph's librbd to test RADOS Block Devices.
	5	*
	6	*/
	7
	8	#include <rbd/librbd.h>
	9
	10	#include "../fio.h"
d220c761	11	#include "../optgroup.h"
fc5c0345 DG	12
	13	struct fio_rbd_iou {
	14	struct io_u *io_u;
d8b64af2	15	rbd_completion_t completion;
d8b64af2	16	int io_seen;
b8ecbef6	17	int io_complete;
fc5c0345 DG	18	};
	19
	20	struct rbd_data {
	21	rados_t cluster;
	22	rados_ioctx_t io_ctx;
	23	rbd_image_t image;
	24	struct io_u **aio_events;
6f9961ac	25	struct io_u **sort_events;
fc5c0345 DG	26	};
	27
	28	struct rbd_options {
a1f871c7	29	void *pad;
6e20c6e7	30	char *cluster_name;
fc5c0345 DG	31	char *rbd_name;
	32	char *pool_name;
	33	char *client_name;
d7d702c7	34	int busy_poll;
fc5c0345 DG	35	};
	36
	37	static struct fio_option options[] = {
6e20c6e7 T	38	{
	39	.name = "clustername",
	40	.lname = "ceph cluster name",
	41	.type = FIO_OPT_STR_STORE,
	42	.help = "Cluster name for ceph",
	43	.off1 = offsetof(struct rbd_options, cluster_name),
	44	.category = FIO_OPT_C_ENGINE,
	45	.group = FIO_OPT_G_RBD,
	46	},
fc5c0345	47	{
d8b64af2 JA	48	.name = "rbdname",
	49	.lname = "rbd engine rbdname",
	50	.type = FIO_OPT_STR_STORE,
	51	.help = "RBD name for RBD engine",
	52	.off1 = offsetof(struct rbd_options, rbd_name),
	53	.category = FIO_OPT_C_ENGINE,
	54	.group = FIO_OPT_G_RBD,
	55	},
fc5c0345	56	{
d7d702c7 JA	57	.name = "pool",
	58	.lname = "rbd engine pool",
	59	.type = FIO_OPT_STR_STORE,
	60	.help = "Name of the pool hosting the RBD for the RBD engine",
	61	.off1 = offsetof(struct rbd_options, pool_name),
	62	.category = FIO_OPT_C_ENGINE,
	63	.group = FIO_OPT_G_RBD,
d8b64af2	64	},
fc5c0345	65	{
d7d702c7 JA	66	.name = "clientname",
	67	.lname = "rbd engine clientname",
	68	.type = FIO_OPT_STR_STORE,
	69	.help = "Name of the ceph client to access the RBD for the RBD engine",
	70	.off1 = offsetof(struct rbd_options, client_name),
	71	.category = FIO_OPT_C_ENGINE,
	72	.group = FIO_OPT_G_RBD,
	73	},
	74	{
	75	.name = "busy_poll",
	76	.lname = "Busy poll",
	77	.type = FIO_OPT_BOOL,
	78	.help = "Busy poll for completions instead of sleeping",
fea585d4	79	.off1 = offsetof(struct rbd_options, busy_poll),
d7d702c7 JA	80	.def = "0",
	81	.category = FIO_OPT_C_ENGINE,
	82	.group = FIO_OPT_G_RBD,
d8b64af2	83	},
fc5c0345	84	{
d8b64af2 JA	85	.name = NULL,
d8b64af2 JA	86	},
fc5c0345 DG	87	};
	88
	89	static int _fio_setup_rbd_data(struct thread_data *td,
	90	struct rbd_data **rbd_data_ptr)
	91	{
6f9961ac	92	struct rbd_data *rbd;
fc5c0345	93
565e784d	94	if (td->io_ops_data)
fc5c0345 DG	95	return 0;
fc5c0345 DG	96
6f9961ac JA	97	rbd = calloc(1, sizeof(struct rbd_data));
6f9961ac JA	98	if (!rbd)
fc5c0345 DG	99	goto failed;
fc5c0345 DG	100
6f9961ac JA	101	rbd->aio_events = calloc(td->o.iodepth, sizeof(struct io_u *));
6f9961ac JA	102	if (!rbd->aio_events)
fc5c0345 DG	103	goto failed;
fc5c0345 DG	104
6f9961ac JA	105	rbd->sort_events = calloc(td->o.iodepth, sizeof(struct io_u *));
	106	if (!rbd->sort_events)
	107	goto failed;
fc5c0345	108
6f9961ac	109	*rbd_data_ptr = rbd;
fc5c0345 DG	110	return 0;
	111
	112	failed:
5a4adfd2 JQ	113	if (rbd) {
	114	if (rbd->aio_events)
	115	free(rbd->aio_events);
	116	if (rbd->sort_events)
	117	free(rbd->sort_events);
6f9961ac	118	free(rbd);
5a4adfd2	119	}
fc5c0345 DG	120	return 1;
	121
	122	}
	123
	124	static int _fio_rbd_connect(struct thread_data *td)
	125	{
565e784d	126	struct rbd_data *rbd = td->io_ops_data;
fc5c0345 DG	127	struct rbd_options *o = td->eo;
	128	int r;
	129
6e20c6e7 T	130	if (o->cluster_name) {
	131	char *client_name = NULL;
	132
	133	/*
89e5ec96	134	* If we specify cluser name, the rados_create2
6e20c6e7 T	135	* will not assume 'client.'. name is considered
	136	* as a full type.id namestr
	137	*/
89e5ec96 JQ	138	if (o->client_name) {
	139	if (!index(o->client_name, '.')) {
	140	client_name = calloc(1, strlen("client.") +
	141	strlen(o->client_name) + 1);
	142	strcat(client_name, "client.");
	143	o->client_name = strcat(client_name,
	144	o->client_name);
	145	}
6e20c6e7 T	146	}
	147	r = rados_create2(&rbd->cluster, o->cluster_name,
	148	o->client_name, 0);
	149	} else
	150	r = rados_create(&rbd->cluster, o->client_name);
	151
fc5c0345 DG	152	if (r < 0) {
	153	log_err("rados_create failed.\n");
	154	goto failed_early;
	155	}
	156
6f9961ac	157	r = rados_conf_read_file(rbd->cluster, NULL);
fc5c0345 DG	158	if (r < 0) {
	159	log_err("rados_conf_read_file failed.\n");
	160	goto failed_early;
	161	}
	162
6f9961ac	163	r = rados_connect(rbd->cluster);
fc5c0345 DG	164	if (r < 0) {
	165	log_err("rados_connect failed.\n");
	166	goto failed_shutdown;
	167	}
	168
6f9961ac	169	r = rados_ioctx_create(rbd->cluster, o->pool_name, &rbd->io_ctx);
fc5c0345 DG	170	if (r < 0) {
	171	log_err("rados_ioctx_create failed.\n");
	172	goto failed_shutdown;
	173	}
	174
6f9961ac	175	r = rbd_open(rbd->io_ctx, o->rbd_name, &rbd->image, NULL /snap / );
fc5c0345 DG	176	if (r < 0) {
	177	log_err("rbd_open failed.\n");
	178	goto failed_open;
	179	}
	180	return 0;
	181
	182	failed_open:
6f9961ac JA	183	rados_ioctx_destroy(rbd->io_ctx);
6f9961ac JA	184	rbd->io_ctx = NULL;
fc5c0345	185	failed_shutdown:
6f9961ac JA	186	rados_shutdown(rbd->cluster);
6f9961ac JA	187	rbd->cluster = NULL;
fc5c0345 DG	188	failed_early:
	189	return 1;
	190	}
	191
6f9961ac	192	static void _fio_rbd_disconnect(struct rbd_data *rbd)
fc5c0345	193	{
6f9961ac	194	if (!rbd)
fc5c0345 DG	195	return;
	196
	197	/* shutdown everything */
6f9961ac JA	198	if (rbd->image) {
	199	rbd_close(rbd->image);
	200	rbd->image = NULL;
fc5c0345 DG	201	}
fc5c0345 DG	202
6f9961ac JA	203	if (rbd->io_ctx) {
	204	rados_ioctx_destroy(rbd->io_ctx);
	205	rbd->io_ctx = NULL;
fc5c0345 DG	206	}
fc5c0345 DG	207
6f9961ac JA	208	if (rbd->cluster) {
	209	rados_shutdown(rbd->cluster);
	210	rbd->cluster = NULL;
fc5c0345 DG	211	}
	212	}
	213
d8b64af2	214	static void _fio_rbd_finish_aiocb(rbd_completion_t comp, void *data)
fc5c0345	215	{
dbf388d2 JA	216	struct fio_rbd_iou *fri = data;
dbf388d2 JA	217	struct io_u *io_u = fri->io_u;
d8b64af2	218	ssize_t ret;
fc5c0345	219
d8b64af2 JA	220	/*
	221	* Looks like return value is 0 for success, or < 0 for
	222	* a specific error. So we have to assume that it can't do
	223	* partial completions.
	224	*/
	225	ret = rbd_aio_get_return_value(fri->completion);
	226	if (ret < 0) {
	227	io_u->error = ret;
	228	io_u->resid = io_u->xfer_buflen;
	229	} else
	230	io_u->error = 0;
20cf5aab JD	231
20cf5aab JD	232	fri->io_complete = 1;
d8b64af2	233	}
fc5c0345	234
d8b64af2 JA	235	static struct io_u fio_rbd_event(struct thread_data td, int event)
d8b64af2 JA	236	{
565e784d	237	struct rbd_data *rbd = td->io_ops_data;
fc5c0345	238
6f9961ac	239	return rbd->aio_events[event];
fc5c0345 DG	240	}
fc5c0345 DG	241
6f9961ac	242	static inline int fri_check_complete(struct rbd_data rbd, struct io_u io_u,
d8b64af2	243	unsigned int *events)
fc5c0345	244	{
d8b64af2	245	struct fio_rbd_iou *fri = io_u->engine_data;
fc5c0345	246
b8ecbef6	247	if (fri->io_complete) {
d8b64af2	248	fri->io_seen = 1;
6f9961ac	249	rbd->aio_events[*events] = io_u;
d8b64af2	250	(*events)++;
fc5c0345	251
d8b64af2 JA	252	rbd_aio_release(fri->completion);
	253	return 1;
	254	}
fc5c0345	255
d8b64af2	256	return 0;
fc5c0345 DG	257	}
fc5c0345 DG	258
6f9961ac JA	259	static inline int rbd_io_u_seen(struct io_u *io_u)
	260	{
	261	struct fio_rbd_iou *fri = io_u->engine_data;
	262
	263	return fri->io_seen;
	264	}
	265
	266	static void rbd_io_u_wait_complete(struct io_u *io_u)
	267	{
	268	struct fio_rbd_iou *fri = io_u->engine_data;
	269
	270	rbd_aio_wait_for_complete(fri->completion);
	271	}
	272
	273	static int rbd_io_u_cmp(const void p1, const void p2)
	274	{
	275	const struct io_u a = (const struct io_u ) p1;
	276	const struct io_u b = (const struct io_u ) p2;
	277	uint64_t at, bt;
	278
	279	at = utime_since_now(&(*a)->start_time);
	280	bt = utime_since_now(&(*b)->start_time);
	281
	282	if (at < bt)
	283	return -1;
	284	else if (at == bt)
	285	return 0;
	286	else
	287	return 1;
	288	}
	289
d8b64af2 JA	290	static int rbd_iter_events(struct thread_data td, unsigned int events,
d8b64af2 JA	291	unsigned int min_evts, int wait)
82340a9f	292	{
565e784d	293	struct rbd_data *rbd = td->io_ops_data;
d8b64af2 JA	294	unsigned int this_events = 0;
d8b64af2 JA	295	struct io_u *io_u;
6f9961ac	296	int i, sidx;
82340a9f	297
6f9961ac	298	sidx = 0;
d8b64af2	299	io_u_qiter(&td->io_u_all, io_u, i) {
d8b64af2 JA	300	if (!(io_u->flags & IO_U_F_FLIGHT))
d8b64af2 JA	301	continue;
6f9961ac	302	if (rbd_io_u_seen(io_u))
d8b64af2	303	continue;
82340a9f	304
6f9961ac	305	if (fri_check_complete(rbd, io_u, events))
d8b64af2	306	this_events++;
6f9961ac JA	307	else if (wait)
	308	rbd->sort_events[sidx++] = io_u;
	309	}
82340a9f	310
6f9961ac JA	311	if (!wait \|\| !sidx)
	312	return this_events;
	313
	314	/*
	315	* Sort events, oldest issue first, then wait on as many as we
	316	* need in order of age. If we have enough events, stop waiting,
	317	* and just check if any of the older ones are done.
	318	*/
	319	if (sidx > 1)
	320	qsort(rbd->sort_events, sidx, sizeof(struct io_u *), rbd_io_u_cmp);
	321
	322	for (i = 0; i < sidx; i++) {
	323	io_u = rbd->sort_events[i];
	324
	325	if (fri_check_complete(rbd, io_u, events)) {
	326	this_events++;
	327	continue;
d8b64af2	328	}
6f9961ac JA	329
	330	/*
	331	* Stop waiting when we have enough, but continue checking
	332	* all pending IOs if they are complete.
	333	*/
d8b64af2	334	if (*events >= min_evts)
6f9961ac JA	335	continue;
	336
	337	rbd_io_u_wait_complete(io_u);
	338
	339	if (fri_check_complete(rbd, io_u, events))
	340	this_events++;
d8b64af2	341	}
fc5c0345	342
d8b64af2	343	return this_events;
fc5c0345 DG	344	}
	345
	346	static int fio_rbd_getevents(struct thread_data *td, unsigned int min,
1f440ece	347	unsigned int max, const struct timespec *t)
fc5c0345	348	{
d8b64af2	349	unsigned int this_events, events = 0;
d7d702c7	350	struct rbd_options *o = td->eo;
d8b64af2	351	int wait = 0;
fc5c0345 DG	352
fc5c0345 DG	353	do {
d8b64af2	354	this_events = rbd_iter_events(td, &events, min, wait);
fc5c0345	355
d8b64af2	356	if (events >= min)
fc5c0345	357	break;
d8b64af2 JA	358	if (this_events)
d8b64af2 JA	359	continue;
fc5c0345	360
d7d702c7 JA	361	if (!o->busy_poll)
	362	wait = 1;
	363	else
	364	nop;
fc5c0345 DG	365	} while (1);
	366
	367	return events;
	368	}
	369
	370	static int fio_rbd_queue(struct thread_data td, struct io_u io_u)
	371	{
565e784d	372	struct rbd_data *rbd = td->io_ops_data;
d8b64af2 JA	373	struct fio_rbd_iou *fri = io_u->engine_data;
d8b64af2 JA	374	int r = -1;
fc5c0345 DG	375
	376	fio_ro_check(td, io_u);
	377
d8b64af2	378	fri->io_seen = 0;
b8ecbef6	379	fri->io_complete = 0;
d8b64af2	380
dbf388d2	381	r = rbd_aio_create_completion(fri, _fio_rbd_finish_aiocb,
d8b64af2	382	&fri->completion);
dbf388d2 JA	383	if (r < 0) {
	384	log_err("rbd_aio_create_completion failed.\n");
	385	goto failed;
	386	}
fc5c0345	387
dbf388d2	388	if (io_u->ddir == DDIR_WRITE) {
6f9961ac JA	389	r = rbd_aio_write(rbd->image, io_u->offset, io_u->xfer_buflen,
6f9961ac JA	390	io_u->xfer_buf, fri->completion);
fc5c0345 DG	391	if (r < 0) {
fc5c0345 DG	392	log_err("rbd_aio_write failed.\n");
dbf388d2	393	goto failed_comp;
fc5c0345 DG	394	}
	395
	396	} else if (io_u->ddir == DDIR_READ) {
6f9961ac JA	397	r = rbd_aio_read(rbd->image, io_u->offset, io_u->xfer_buflen,
6f9961ac JA	398	io_u->xfer_buf, fri->completion);
fc5c0345 DG	399
	400	if (r < 0) {
	401	log_err("rbd_aio_read failed.\n");
dbf388d2	402	goto failed_comp;
fc5c0345	403	}
dbf388d2	404	} else if (io_u->ddir == DDIR_TRIM) {
6f9961ac JA	405	r = rbd_aio_discard(rbd->image, io_u->offset,
6f9961ac JA	406	io_u->xfer_buflen, fri->completion);
82340a9f	407	if (r < 0) {
dbf388d2 JA	408	log_err("rbd_aio_discard failed.\n");
dbf388d2 JA	409	goto failed_comp;
82340a9f	410	}
dbf388d2	411	} else if (io_u->ddir == DDIR_SYNC) {
6f9961ac	412	r = rbd_aio_flush(rbd->image, fri->completion);
fc5c0345 DG	413	if (r < 0) {
fc5c0345 DG	414	log_err("rbd_flush failed.\n");
dbf388d2	415	goto failed_comp;
fc5c0345	416	}
fc5c0345 DG	417	} else {
	418	dprint(FD_IO, "%s: Warning: unhandled ddir: %d\n", __func__,
	419	io_u->ddir);
dbf388d2	420	goto failed_comp;
fc5c0345 DG	421	}
	422
	423	return FIO_Q_QUEUED;
dbf388d2 JA	424	failed_comp:
dbf388d2 JA	425	rbd_aio_release(fri->completion);
fc5c0345 DG	426	failed:
	427	io_u->error = r;
	428	td_verror(td, io_u->error, "xfer");
	429	return FIO_Q_COMPLETED;
	430	}
	431
	432	static int fio_rbd_init(struct thread_data *td)
	433	{
	434	int r;
	435
	436	r = _fio_rbd_connect(td);
	437	if (r) {
	438	log_err("fio_rbd_connect failed, return code: %d .\n", r);
	439	goto failed;
	440	}
	441
	442	return 0;
	443
	444	failed:
	445	return 1;
fc5c0345 DG	446	}
	447
	448	static void fio_rbd_cleanup(struct thread_data *td)
	449	{
565e784d	450	struct rbd_data *rbd = td->io_ops_data;
fc5c0345	451
6f9961ac JA	452	if (rbd) {
	453	_fio_rbd_disconnect(rbd);
	454	free(rbd->aio_events);
	455	free(rbd->sort_events);
	456	free(rbd);
fc5c0345	457	}
fc5c0345 DG	458	}
	459
	460	static int fio_rbd_setup(struct thread_data *td)
	461	{
fc5c0345 DG	462	rbd_image_info_t info;
fc5c0345 DG	463	struct fio_file *f;
6f9961ac	464	struct rbd_data *rbd = NULL;
fc5c0345	465	int major, minor, extra;
6f9961ac	466	int r;
fc5c0345 DG	467
	468	/* log version of librbd. No cluster connection required. */
	469	rbd_version(&major, &minor, &extra);
	470	log_info("rbd engine: RBD version: %d.%d.%d\n", major, minor, extra);
	471
	472	/* allocate engine specific structure to deal with librbd. */
6f9961ac	473	r = _fio_setup_rbd_data(td, &rbd);
fc5c0345 DG	474	if (r) {
	475	log_err("fio_setup_rbd_data failed.\n");
	476	goto cleanup;
	477	}
565e784d	478	td->io_ops_data = rbd;
fc5c0345	479
d8b64af2 JA	480	/* librbd does not allow us to run first in the main thread and later
	481	* in a fork child. It needs to be the same process context all the
	482	* time.
fc5c0345 DG	483	*/
	484	td->o.use_thread = 1;
	485
	486	/* connect in the main thread to determine to determine
	487	* the size of the given RADOS block device. And disconnect
	488	* later on.
	489	*/
	490	r = _fio_rbd_connect(td);
	491	if (r) {
	492	log_err("fio_rbd_connect failed.\n");
	493	goto cleanup;
	494	}
	495
	496	/* get size of the RADOS block device */
6f9961ac	497	r = rbd_stat(rbd->image, &info, sizeof(info));
fc5c0345 DG	498	if (r < 0) {
	499	log_err("rbd_status failed.\n");
	500	goto disconnect;
	501	}
	502	dprint(FD_IO, "rbd-engine: image size: %lu\n", info.size);
	503
	504	/* taken from "net" engine. Pretend we deal with files,
	505	* even if we do not have any ideas about files.
	506	* The size of the RBD is set instead of a artificial file.
	507	*/
	508	if (!td->files_index) {
5903e7b7	509	add_file(td, td->o.filename ? : "rbd", 0, 0);
fc5c0345	510	td->o.nr_files = td->o.nr_files ? : 1;
b53f2c54	511	td->o.open_files++;
fc5c0345 DG	512	}
	513	f = td->files[0];
	514	f->real_file_size = info.size;
	515
	516	/* disconnect, then we were only connected to determine
	517	* the size of the RBD.
	518	*/
6f9961ac	519	_fio_rbd_disconnect(rbd);
fc5c0345 DG	520	return 0;
	521
	522	disconnect:
6f9961ac	523	_fio_rbd_disconnect(rbd);
fc5c0345 DG	524	cleanup:
	525	fio_rbd_cleanup(td);
	526	return r;
	527	}
	528
	529	static int fio_rbd_open(struct thread_data td, struct fio_file f)
	530	{
	531	return 0;
	532	}
	533
d9b100fc JA	534	static int fio_rbd_invalidate(struct thread_data td, struct fio_file f)
d9b100fc JA	535	{
903b2812	536	#if defined(CONFIG_RBD_INVAL)
565e784d	537	struct rbd_data *rbd = td->io_ops_data;
903b2812	538
6f9961ac	539	return rbd_invalidate_cache(rbd->image);
903b2812	540	#else
d9b100fc	541	return 0;
903b2812	542	#endif
d9b100fc JA	543	}
d9b100fc JA	544
fc5c0345 DG	545	static void fio_rbd_io_u_free(struct thread_data td, struct io_u io_u)
fc5c0345 DG	546	{
d8b64af2	547	struct fio_rbd_iou *fri = io_u->engine_data;
fc5c0345	548
d8b64af2	549	if (fri) {
fc5c0345	550	io_u->engine_data = NULL;
d8b64af2	551	free(fri);
fc5c0345 DG	552	}
	553	}
	554
	555	static int fio_rbd_io_u_init(struct thread_data td, struct io_u io_u)
	556	{
d8b64af2	557	struct fio_rbd_iou *fri;
fc5c0345	558
d8b64af2 JA	559	fri = calloc(1, sizeof(*fri));
	560	fri->io_u = io_u;
	561	io_u->engine_data = fri;
fc5c0345 DG	562	return 0;
	563	}
	564
10aa136b	565	static struct ioengine_ops ioengine = {
d9b100fc JA	566	.name = "rbd",
	567	.version = FIO_IOOPS_VERSION,
	568	.setup = fio_rbd_setup,
	569	.init = fio_rbd_init,
	570	.queue = fio_rbd_queue,
	571	.getevents = fio_rbd_getevents,
	572	.event = fio_rbd_event,
	573	.cleanup = fio_rbd_cleanup,
	574	.open_file = fio_rbd_open,
	575	.invalidate = fio_rbd_invalidate,
	576	.options = options,
	577	.io_u_init = fio_rbd_io_u_init,
	578	.io_u_free = fio_rbd_io_u_free,
	579	.option_struct_size = sizeof(struct rbd_options),
fc5c0345 DG	580	};
	581
	582	static void fio_init fio_rbd_register(void)
	583	{
	584	register_ioengine(&ioengine);
	585	}
	586
	587	static void fio_exit fio_rbd_unregister(void)
	588	{
	589	unregister_ioengine(&ioengine);
	590	}