[fio.git] / engines / nbd.c

/*
 * NBD engine
 *
 * IO engine that talks to an NBD server.
 *
 * Copyright (C) 2019 Red Hat Inc.
 * Written by Richard W.M. Jones <rjones@redhat.com>
 *
 */

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <errno.h>

#include <libnbd.h>

#include "../fio.h"
#include "../optgroup.h"

/* Actually this differs across servers, but for nbdkit ... */
#define NBD_MAX_REQUEST_SIZE (64 * 1024 * 1024)

/* Storage for the NBD handle. */
struct nbd_data {
	struct nbd_handle *nbd;
	int debug;

	/* The list of completed io_u structs. */
	struct io_u **completed;
	size_t nr_completed;
};

/* Options. */
struct nbd_options {
	void *padding;
	char *uri;
};

static struct fio_option options[] = {
	{
		.name	= "uri",
		.lname	= "NBD URI",
		.help	= "Name of NBD URI",
		.category = FIO_OPT_C_ENGINE,
		.group	= FIO_OPT_G_NBD,
		.type	= FIO_OPT_STR_STORE,
		.off1	= offsetof(struct nbd_options, uri),
	},
	{
		.name	= NULL,
	},
};

/* Allocates nbd_data. */
static int nbd_setup(struct thread_data *td)
{
	struct nbd_data *nbd_data;
	struct nbd_options *o = td->eo;
	struct fio_file *f;
	int r;
	int64_t size;

	nbd_data = calloc(1, sizeof(*nbd_data));
	if (!nbd_data) {
		td_verror(td, errno, "calloc");
		return 1;
	}
	td->io_ops_data = nbd_data;

	/* Pretend to deal with files.	See engines/rbd.c */
	if (!td->files_index) {
		add_file(td, "nbd", 0, 0);
		td->o.nr_files = td->o.nr_files ? : 1;
		td->o.open_files++;
	}
	f = td->files[0];

	nbd_data->nbd = nbd_create();
	if (!nbd_data->nbd) {
		log_err("fio: nbd_create: %s\n", nbd_get_error());
		return 1;
	}

	/* Get the debug flag which can be set through LIBNBD_DEBUG=1. */
	nbd_data->debug = nbd_get_debug(nbd_data->nbd);

	/* Connect synchronously here so we can check for the size and
	 * in future other properties of the server.
	 */
	if (!o->uri) {
		log_err("fio: nbd: uri parameter was not specified\n");
		return 1;
	}
	r = nbd_connect_uri(nbd_data->nbd, o->uri);
	if (r == -1) {
		log_err("fio: nbd_connect_uri: %s\n", nbd_get_error());
		return 1;
	}
	size = nbd_get_size(nbd_data->nbd);
	if (size == -1) {
		log_err("fio: nbd_get_size: %s\n", nbd_get_error());
		return 1;
	}

	f->real_file_size = size;

	nbd_close (nbd_data->nbd);
	nbd_data->nbd = NULL;

	return 0;
}

/* Closes socket and frees nbd_data -- the opposite of nbd_setup. */
static void nbd_cleanup(struct thread_data *td)
{
	struct nbd_data *nbd_data = td->io_ops_data;

	if (nbd_data) {
		if (nbd_data->nbd)
			nbd_close(nbd_data->nbd);
		free(nbd_data);
	}
}

/* Connect to the server from each thread. */
static int nbd_init(struct thread_data *td)
{
	struct nbd_options *o = td->eo;
	struct nbd_data *nbd_data = td->io_ops_data;
	int r;

	if (!o->uri) {
		log_err("fio: nbd: uri parameter was not specified\n");
		return 1;
	}

	nbd_data->nbd = nbd_create();
	if (!nbd_data->nbd) {
		log_err("fio: nbd_create: %s\n", nbd_get_error());
		return 1;
	}
	/* This is actually a synchronous connect and handshake. */
	r = nbd_connect_uri(nbd_data->nbd, o->uri);
	if (r == -1) {
		log_err("fio: nbd_connect_uri: %s\n", nbd_get_error());
		return 1;
	}

	log_info("fio: connected to NBD server\n");
	return 0;
}

/* A command in flight has been completed. */
static int cmd_completed (void *vp, int *error)
{
	struct io_u *io_u;
	struct nbd_data *nbd_data;
	struct io_u **completed;

	io_u = vp;
	nbd_data = io_u->engine_data;

	if (nbd_data->debug)
		log_info("fio: nbd: command completed\n");

	if (*error != 0)
		io_u->error = *error;
	else
		io_u->error = 0;

	/* Add this completion to the list so it can be picked up
	 * later by ->event.
	 */
	completed = realloc(nbd_data->completed,
			    sizeof(struct io_u *) *
			    (nbd_data->nr_completed+1));
	if (completed == NULL) {
		io_u->error = errno;
		return 0;
	}

	nbd_data->completed = completed;
	nbd_data->completed[nbd_data->nr_completed] = io_u;
	nbd_data->nr_completed++;

	return 0;
}

/* Begin read or write request. */
static enum fio_q_status nbd_queue(struct thread_data *td,
				   struct io_u *io_u)
{
	struct nbd_data *nbd_data = td->io_ops_data;
	nbd_completion_callback completion = { .callback = cmd_completed,
					       .user_data = io_u };
	int r;

	fio_ro_check(td, io_u);

	io_u->engine_data = nbd_data;

	if (io_u->ddir == DDIR_WRITE || io_u->ddir == DDIR_READ)
		assert(io_u->xfer_buflen <= NBD_MAX_REQUEST_SIZE);

	switch (io_u->ddir) {
	case DDIR_READ:
		r = nbd_aio_pread(nbd_data->nbd,
				  io_u->xfer_buf, io_u->xfer_buflen,
				  io_u->offset, completion, 0);
		break;
	case DDIR_WRITE:
		r = nbd_aio_pwrite(nbd_data->nbd,
				   io_u->xfer_buf, io_u->xfer_buflen,
				   io_u->offset, completion, 0);
		break;
	case DDIR_TRIM:
		r = nbd_aio_trim(nbd_data->nbd, io_u->xfer_buflen,
				 io_u->offset, completion, 0);
		break;
	case DDIR_SYNC:
		/* XXX We could probably also handle
		 * DDIR_SYNC_FILE_RANGE with a bit of effort.
		 */
		r = nbd_aio_flush(nbd_data->nbd, completion, 0);
		break;
	default:
		io_u->error = EINVAL;
		return FIO_Q_COMPLETED;
	}

	if (r == -1) {
		/* errno is optional information on libnbd error path;
		 * if it's 0, set it to a default value
		 */
		io_u->error = nbd_get_errno();
		if (io_u->error == 0)
			io_u->error = EIO;
		return FIO_Q_COMPLETED;
	}

	if (nbd_data->debug)
		log_info("fio: nbd: command issued\n");
	io_u->error = 0;
	return FIO_Q_QUEUED;
}

static unsigned retire_commands(struct nbd_handle *nbd)
{
	int64_t cookie;
	unsigned r = 0;

	while ((cookie = nbd_aio_peek_command_completed(nbd)) > 0) {
		/* Ignore the return value.  cmd_completed has already
		 * checked for an error and set io_u->error.  We only
		 * have to call this to retire the command.
		 */
		nbd_aio_command_completed(nbd, cookie);
		r++;
	}

	if (nbd_get_debug(nbd))
		log_info("fio: nbd: %u commands retired\n", r);
	return r;
}

static int nbd_getevents(struct thread_data *td, unsigned int min,
			 unsigned int max, const struct timespec *t)
{
	struct nbd_data *nbd_data = td->io_ops_data;
	int r;
	unsigned events = 0;
	int timeout;

	/* XXX This handling of timeout is wrong because it will wait
	 * for up to loop iterations * timeout.
	 */
	timeout = !t ? -1 : t->tv_sec * 1000 + t->tv_nsec / 1000000;

	while (events < min) {
		r = nbd_poll(nbd_data->nbd, timeout);
		if (r == -1) {
			/* error in poll */
			log_err("fio: nbd_poll: %s\n", nbd_get_error());
			return -1;
		}
		else {
			/* poll made progress */
			events += retire_commands(nbd_data->nbd);
		}
	}

	return events;
}

static struct io_u *nbd_event(struct thread_data *td, int event)
{
	struct nbd_data *nbd_data = td->io_ops_data;

	if (nbd_data->nr_completed == 0)
		return NULL;

	/* XXX We ignore the event number and assume fio calls us
	 * exactly once for [0..nr_events-1].
	 */
	nbd_data->nr_completed--;
	return nbd_data->completed[nbd_data->nr_completed];
}

static int nbd_io_u_init(struct thread_data *td, struct io_u *io_u)
{
	io_u->engine_data = NULL;
	return 0;
}

static void nbd_io_u_free(struct thread_data *td, struct io_u *io_u)
{
	/* Nothing needs to be done. */
}

static int nbd_open_file(struct thread_data *td, struct fio_file *f)
{
	return 0;
}

static int nbd_invalidate(struct thread_data *td, struct fio_file *f)
{
	return 0;
}

FIO_STATIC struct ioengine_ops ioengine = {
	.name			= "nbd",
	.version		= FIO_IOOPS_VERSION,
	.options		= options,
	.option_struct_size	= sizeof(struct nbd_options),
	.flags			= FIO_DISKLESSIO | FIO_NOEXTEND,

	.setup			= nbd_setup,
	.init			= nbd_init,
	.cleanup		= nbd_cleanup,
	.queue			= nbd_queue,
	.getevents		= nbd_getevents,
	.event			= nbd_event,
	.io_u_init		= nbd_io_u_init,
	.io_u_free		= nbd_io_u_free,

	.open_file		= nbd_open_file,
	.invalidate		= nbd_invalidate,
};

static void fio_init fio_nbd_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_nbd_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
d643a1e2 RJ	1	/*
	2	* NBD engine
	3	*
	4	* IO engine that talks to an NBD server.
	5	*
	6	* Copyright (C) 2019 Red Hat Inc.
	7	* Written by Richard W.M. Jones <rjones@redhat.com>
	8	*
	9	*/
	10
	11	#include <stdio.h>
	12	#include <stdlib.h>
	13	#include <stdint.h>
	14	#include <errno.h>
	15
	16	#include <libnbd.h>
	17
	18	#include "../fio.h"
	19	#include "../optgroup.h"
	20
	21	/* Actually this differs across servers, but for nbdkit ... */
	22	#define NBD_MAX_REQUEST_SIZE (64 * 1024 * 1024)
	23
	24	/* Storage for the NBD handle. */
	25	struct nbd_data {
	26	struct nbd_handle *nbd;
	27	int debug;
	28
	29	/* The list of completed io_u structs. */
	30	struct io_u **completed;
	31	size_t nr_completed;
	32	};
	33
	34	/* Options. */
	35	struct nbd_options {
	36	void *padding;
	37	char *uri;
	38	};
	39
	40	static struct fio_option options[] = {
	41	{
	42	.name = "uri",
	43	.lname = "NBD URI",
	44	.help = "Name of NBD URI",
	45	.category = FIO_OPT_C_ENGINE,
	46	.group = FIO_OPT_G_NBD,
	47	.type = FIO_OPT_STR_STORE,
	48	.off1 = offsetof(struct nbd_options, uri),
	49	},
	50	{
	51	.name = NULL,
	52	},
	53	};
	54
fc002f14	55	/* Allocates nbd_data. */
d643a1e2 RJ	56	static int nbd_setup(struct thread_data *td)
	57	{
	58	struct nbd_data *nbd_data;
	59	struct nbd_options *o = td->eo;
	60	struct fio_file *f;
	61	int r;
	62	int64_t size;
	63
	64	nbd_data = calloc(1, sizeof(*nbd_data));
	65	if (!nbd_data) {
	66	td_verror(td, errno, "calloc");
	67	return 1;
	68	}
	69	td->io_ops_data = nbd_data;
	70
	71	/* Pretend to deal with files. See engines/rbd.c */
	72	if (!td->files_index) {
	73	add_file(td, "nbd", 0, 0);
	74	td->o.nr_files = td->o.nr_files ? : 1;
	75	td->o.open_files++;
	76	}
	77	f = td->files[0];
	78
	79	nbd_data->nbd = nbd_create();
	80	if (!nbd_data->nbd) {
	81	log_err("fio: nbd_create: %s\n", nbd_get_error());
	82	return 1;
	83	}
	84
	85	/* Get the debug flag which can be set through LIBNBD_DEBUG=1. */
	86	nbd_data->debug = nbd_get_debug(nbd_data->nbd);
	87
	88	/* Connect synchronously here so we can check for the size and
	89	* in future other properties of the server.
	90	*/
	91	if (!o->uri) {
	92	log_err("fio: nbd: uri parameter was not specified\n");
	93	return 1;
	94	}
	95	r = nbd_connect_uri(nbd_data->nbd, o->uri);
	96	if (r == -1) {
	97	log_err("fio: nbd_connect_uri: %s\n", nbd_get_error());
	98	return 1;
	99	}
	100	size = nbd_get_size(nbd_data->nbd);
	101	if (size == -1) {
	102	log_err("fio: nbd_get_size: %s\n", nbd_get_error());
	103	return 1;
	104	}
	105
	106	f->real_file_size = size;
	107
	108	nbd_close (nbd_data->nbd);
	109	nbd_data->nbd = NULL;
	110
	111	return 0;
	112	}
	113
	114	/* Closes socket and frees nbd_data -- the opposite of nbd_setup. */
	115	static void nbd_cleanup(struct thread_data *td)
	116	{
	117	struct nbd_data *nbd_data = td->io_ops_data;
	118
	119	if (nbd_data) {
120	if (nbd_data->nbd)
121	nbd_close(nbd_data->nbd);
122	free(nbd_data);
123	}
124	}
125
126	/* Connect to the server from each thread. */
127	static int nbd_init(struct thread_data *td)
128	{
129	struct nbd_options *o = td->eo;
130	struct nbd_data *nbd_data = td->io_ops_data;
131	int r;
132
133	if (!o->uri) {
134	log_err("fio: nbd: uri parameter was not specified\n");
135	return 1;
136	}
137
138	nbd_data->nbd = nbd_create();
139	if (!nbd_data->nbd) {
140	log_err("fio: nbd_create: %s\n", nbd_get_error());
141	return 1;
142	}
143	/* This is actually a synchronous connect and handshake. */
144	r = nbd_connect_uri(nbd_data->nbd, o->uri);
145	if (r == -1) {
146	log_err("fio: nbd_connect_uri: %s\n", nbd_get_error());
147	return 1;
148	}
149
150	log_info("fio: connected to NBD server\n");
151	return 0;
152	}
153
154	/* A command in flight has been completed. */
4e8c82b4	155	static int cmd_completed (void vp, int error)
d643a1e2 RJ	156	{
	157	struct io_u *io_u;
	158	struct nbd_data *nbd_data;
	159	struct io_u **completed;
	160
d643a1e2 RJ	161	io_u = vp;
	162	nbd_data = io_u->engine_data;
	163
	164	if (nbd_data->debug)
	165	log_info("fio: nbd: command completed\n");
	166
	167	if (*error != 0)
	168	io_u->error = *error;
	169	else
	170	io_u->error = 0;
	171
	172	/* Add this completion to the list so it can be picked up
	173	* later by ->event.
	174	*/
	175	completed = realloc(nbd_data->completed,
	176	sizeof(struct io_u )
	177	(nbd_data->nr_completed+1));
	178	if (completed == NULL) {
	179	io_u->error = errno;
	180	return 0;
	181	}
	182
	183	nbd_data->completed = completed;
	184	nbd_data->completed[nbd_data->nr_completed] = io_u;
	185	nbd_data->nr_completed++;
	186
	187	return 0;
	188	}
	189
	190	/* Begin read or write request. */
	191	static enum fio_q_status nbd_queue(struct thread_data *td,
	192	struct io_u *io_u)
	193	{
	194	struct nbd_data *nbd_data = td->io_ops_data;
4e8c82b4 RJ	195	nbd_completion_callback completion = { .callback = cmd_completed,
4e8c82b4 RJ	196	.user_data = io_u };
d643a1e2 RJ	197	int r;
	198
	199	fio_ro_check(td, io_u);
	200
	201	io_u->engine_data = nbd_data;
	202
	203	if (io_u->ddir == DDIR_WRITE \|\| io_u->ddir == DDIR_READ)
	204	assert(io_u->xfer_buflen <= NBD_MAX_REQUEST_SIZE);
	205
	206	switch (io_u->ddir) {
	207	case DDIR_READ:
4e8c82b4 RJ	208	r = nbd_aio_pread(nbd_data->nbd,
	209	io_u->xfer_buf, io_u->xfer_buflen,
	210	io_u->offset, completion, 0);
d643a1e2 RJ	211	break;
d643a1e2 RJ	212	case DDIR_WRITE:
4e8c82b4 RJ	213	r = nbd_aio_pwrite(nbd_data->nbd,
	214	io_u->xfer_buf, io_u->xfer_buflen,
	215	io_u->offset, completion, 0);
d643a1e2 RJ	216	break;
d643a1e2 RJ	217	case DDIR_TRIM:
4e8c82b4 RJ	218	r = nbd_aio_trim(nbd_data->nbd, io_u->xfer_buflen,
4e8c82b4 RJ	219	io_u->offset, completion, 0);
d643a1e2 RJ	220	break;
	221	case DDIR_SYNC:
	222	/* XXX We could probably also handle
	223	* DDIR_SYNC_FILE_RANGE with a bit of effort.
	224	*/
4e8c82b4	225	r = nbd_aio_flush(nbd_data->nbd, completion, 0);
d643a1e2 RJ	226	break;
	227	default:
	228	io_u->error = EINVAL;
	229	return FIO_Q_COMPLETED;
	230	}
	231
	232	if (r == -1) {
	233	/* errno is optional information on libnbd error path;
	234	* if it's 0, set it to a default value
	235	*/
	236	io_u->error = nbd_get_errno();
	237	if (io_u->error == 0)
	238	io_u->error = EIO;
	239	return FIO_Q_COMPLETED;
	240	}
	241
	242	if (nbd_data->debug)
	243	log_info("fio: nbd: command issued\n");
	244	io_u->error = 0;
	245	return FIO_Q_QUEUED;
	246	}
	247
	248	static unsigned retire_commands(struct nbd_handle *nbd)
	249	{
	250	int64_t cookie;
	251	unsigned r = 0;
	252
	253	while ((cookie = nbd_aio_peek_command_completed(nbd)) > 0) {
	254	/* Ignore the return value. cmd_completed has already
	255	* checked for an error and set io_u->error. We only
	256	* have to call this to retire the command.
	257	*/
	258	nbd_aio_command_completed(nbd, cookie);
	259	r++;
	260	}
	261
	262	if (nbd_get_debug(nbd))
	263	log_info("fio: nbd: %u commands retired\n", r);
	264	return r;
	265	}
	266
	267	static int nbd_getevents(struct thread_data *td, unsigned int min,
	268	unsigned int max, const struct timespec *t)
	269	{
	270	struct nbd_data *nbd_data = td->io_ops_data;
	271	int r;
	272	unsigned events = 0;
	273	int timeout;
	274
	275	/* XXX This handling of timeout is wrong because it will wait
	276	* for up to loop iterations * timeout.
	277	*/
	278	timeout = !t ? -1 : t->tv_sec * 1000 + t->tv_nsec / 1000000;
	279
	280	while (events < min) {
	281	r = nbd_poll(nbd_data->nbd, timeout);
	282	if (r == -1) {
	283	/* error in poll */
	284	log_err("fio: nbd_poll: %s\n", nbd_get_error());
	285	return -1;
	286	}
	287	else {
	288	/* poll made progress */
	289	events += retire_commands(nbd_data->nbd);
290	}
291	}
292
293	return events;
294	}
295
296	static struct io_u nbd_event(struct thread_data td, int event)
297	{
298	struct nbd_data *nbd_data = td->io_ops_data;
299
300	if (nbd_data->nr_completed == 0)
301	return NULL;
302
303	/* XXX We ignore the event number and assume fio calls us
304	* exactly once for [0..nr_events-1].
305	*/
306	nbd_data->nr_completed--;
307	return nbd_data->completed[nbd_data->nr_completed];
308	}
309
310	static int nbd_io_u_init(struct thread_data td, struct io_u io_u)
311	{
312	io_u->engine_data = NULL;
313	return 0;
314	}
315
316	static void nbd_io_u_free(struct thread_data td, struct io_u io_u)
317	{
318	/* Nothing needs to be done. */
319	}
320
321	static int nbd_open_file(struct thread_data td, struct fio_file f)
322	{
323	return 0;
324	}
325
326	static int nbd_invalidate(struct thread_data td, struct fio_file f)
327	{
328	return 0;
329	}
330
5a8a6a03	331	FIO_STATIC struct ioengine_ops ioengine = {
d643a1e2 RJ	332	.name = "nbd",
	333	.version = FIO_IOOPS_VERSION,
	334	.options = options,
	335	.option_struct_size = sizeof(struct nbd_options),
	336	.flags = FIO_DISKLESSIO \| FIO_NOEXTEND,
	337
	338	.setup = nbd_setup,
	339	.init = nbd_init,
	340	.cleanup = nbd_cleanup,
	341	.queue = nbd_queue,
	342	.getevents = nbd_getevents,
	343	.event = nbd_event,
	344	.io_u_init = nbd_io_u_init,
	345	.io_u_free = nbd_io_u_free,
	346
	347	.open_file = nbd_open_file,
	348	.invalidate = nbd_invalidate,
	349	};
	350
	351	static void fio_init fio_nbd_register(void)
	352	{
	353	register_ioengine(&ioengine);
	354	}
	355
	356	static void fio_exit fio_nbd_unregister(void)
	357	{
	358	unregister_ioengine(&ioengine);
	359	}