[fio.git] / engines / libblkio.c

/*
 * libblkio engine
 *
 * IO engine using libblkio to access various block I/O interfaces:
 * https://gitlab.com/libblkio/libblkio
 */

#include <assert.h>
#include <errno.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

#include <blkio.h>

#include "../fio.h"
#include "../optgroup.h"
#include "../options.h"
#include "../parse.h"

/* per-thread state */
struct fio_blkio_data {
	struct blkio *b;
	struct blkioq *q;

	bool has_mem_region; /* whether mem_region is valid */
	struct blkio_mem_region mem_region; /* only if allocated by libblkio */

	struct blkio_completion *completions;
};

struct fio_blkio_options {
	void *pad; /* option fields must not have offset 0 */

	char *driver;
	char *pre_connect_props;
	char *pre_start_props;
};

static struct fio_option options[] = {
	{
		.name	= "libblkio_driver",
		.lname	= "libblkio driver name",
		.type	= FIO_OPT_STR_STORE,
		.off1	= offsetof(struct fio_blkio_options, driver),
		.help	= "Name of the driver to be used by libblkio",
		.category = FIO_OPT_C_ENGINE,
		.group	= FIO_OPT_G_LIBBLKIO,
	},
	{
		.name	= "libblkio_pre_connect_props",
		.lname	= "Properties to be set before blkio_connect()",
		.type	= FIO_OPT_STR_STORE,
		.off1	= offsetof(struct fio_blkio_options, pre_connect_props),
		.help	= "",
		.category = FIO_OPT_C_ENGINE,
		.group	= FIO_OPT_G_LIBBLKIO,
	},
	{
		.name	= "libblkio_pre_start_props",
		.lname	= "Properties to be set before blkio_start()",
		.type	= FIO_OPT_STR_STORE,
		.off1	= offsetof(struct fio_blkio_options, pre_start_props),
		.help	= "",
		.category = FIO_OPT_C_ENGINE,
		.group	= FIO_OPT_G_LIBBLKIO,
	},
	{
		.name = NULL,
	},
};

static int fio_blkio_set_props_from_str(struct blkio *b, const char *opt_name,
					const char *str) {
	int ret = 0;
	char *new_str, *name, *value;

	if (!str)
		return 0;

	/* iteration can mutate string, so copy it */
	new_str = strdup(str);
	if (!new_str) {
		log_err("fio: strdup() failed\n");
		return 1;
	}

	/* iterate over property name-value pairs */
	while ((name = get_next_str(&new_str))) {
		/* split into property name and value */
		value = strchr(name, '=');
		if (!value) {
			log_err("fio: missing '=' in option %s\n", opt_name);
			ret = 1;
			break;
		}

		*value = '\0';
		++value;

		/* strip whitespace from property name */
		strip_blank_front(&name);
		strip_blank_end(name);

		if (name[0] == '\0') {
			log_err("fio: empty property name in option %s\n",
				opt_name);
			ret = 1;
			break;
		}

		/* strip whitespace from property value */
		strip_blank_front(&value);
		strip_blank_end(value);

		/* set property */
		if (blkio_set_str(b, name, value) != 0) {
			log_err("fio: error setting property '%s' to '%s': %s\n",
				name, value, blkio_get_error_msg());
			ret = 1;
			break;
		}
	}

	free(new_str);
	return ret;
}

/*
 * Log the failure of a libblkio function.
 *
 * `(void)func` is to ensure `func` exists and prevent typos
 */
#define fio_blkio_log_err(func) \
	({ \
		(void)func; \
		log_err("fio: %s() failed: %s\n", #func, \
			blkio_get_error_msg()); \
	})

static int fio_blkio_create_and_connect(struct thread_data *td,
					struct blkio **out_blkio)
{
	const struct fio_blkio_options *options = td->eo;
	struct blkio *b;
	int ret;

	if (!options->driver) {
		log_err("fio: engine libblkio requires option libblkio_driver to be set\n");
		return 1;
	}

	if (blkio_create(options->driver, &b) != 0) {
		fio_blkio_log_err(blkio_create);
		return 1;
	}

	/* don't fail if driver doesn't have a "direct" property */
	ret = blkio_set_bool(b, "direct", td->o.odirect);
	if (ret != 0 && ret != -ENOENT) {
		fio_blkio_log_err(blkio_set_bool);
		goto err_blkio_destroy;
	}

	if (blkio_set_bool(b, "read-only", read_only) != 0) {
		fio_blkio_log_err(blkio_set_bool);
		goto err_blkio_destroy;
	}

	if (fio_blkio_set_props_from_str(b, "libblkio_pre_connect_props",
					 options->pre_connect_props) != 0)
		goto err_blkio_destroy;

	if (blkio_connect(b) != 0) {
		fio_blkio_log_err(blkio_connect);
		goto err_blkio_destroy;
	}

	if (fio_blkio_set_props_from_str(b, "libblkio_pre_start_props",
					 options->pre_start_props) != 0)
		goto err_blkio_destroy;

	*out_blkio = b;
	return 0;

err_blkio_destroy:
	blkio_destroy(&b);
	return 1;
}

/*
 * This callback determines the device/file size, so it creates and connects a
 * blkio instance. But it is invoked from the main thread in the original fio
 * process, not from the processes in which jobs will actually run. It thus
 * subsequently destroys the blkio, which is recreated in the init() callback.
 */
static int fio_blkio_setup(struct thread_data *td)
{
	struct blkio *b;
	int ret = 0;
	uint64_t capacity;

	assert(td->files_index == 1);

	if (fio_blkio_create_and_connect(td, &b) != 0)
		return 1;

	if (blkio_get_uint64(b, "capacity", &capacity) != 0) {
		fio_blkio_log_err(blkio_get_uint64);
		ret = 1;
		goto out_blkio_destroy;
	}

	td->files[0]->real_file_size = capacity;
	fio_file_set_size_known(td->files[0]);

out_blkio_destroy:
	blkio_destroy(&b);
	return ret;
}

static int fio_blkio_init(struct thread_data *td)
{
	struct fio_blkio_data *data;

	/*
	 * Request enqueueing is fast, and it's not possible to know exactly
	 * when a request is submitted, so never report submission latencies.
	 */
	td->o.disable_slat = 1;

	data = calloc(1, sizeof(*data));
	if (!data) {
		log_err("fio: calloc() failed\n");
		return 1;
	}

	data->completions = calloc(td->o.iodepth, sizeof(data->completions[0]));
	if (!data->completions) {
		log_err("fio: calloc() failed\n");
		goto err_free;
	}

	if (fio_blkio_create_and_connect(td, &data->b) != 0)
		goto err_free;

	if (blkio_set_int(data->b, "num-queues", 1) != 0) {
		fio_blkio_log_err(blkio_set_int);
		goto err_blkio_destroy;
	}

	if (blkio_start(data->b) != 0) {
		fio_blkio_log_err(blkio_start);
		goto err_blkio_destroy;
	}

	data->q = blkio_get_queue(data->b, 0);

	/* Set data last so cleanup() does nothing if init() fails. */
	td->io_ops_data = data;

	return 0;

err_blkio_destroy:
	blkio_destroy(&data->b);
err_free:
	free(data->completions);
	free(data);
	return 1;
}

static int fio_blkio_post_init(struct thread_data *td)
{
	struct fio_blkio_data *data = td->io_ops_data;

	if (!data->has_mem_region) {
		/*
		 * Memory was allocated by the fio core and not iomem_alloc(),
		 * so we need to register it as a memory region here.
		 *
		 * `td->orig_buffer_size` is computed like `len` below, but then
		 * fio can add some padding to it to make sure it is
		 * sufficiently aligned to the page size and the mem_align
		 * option. However, this can make it become unaligned to the
		 * "mem-region-alignment" property in ways that the user can't
		 * control, so we essentially recompute `td->orig_buffer_size`
		 * here but without adding that padding.
		 */

		unsigned long long max_block_size;
		struct blkio_mem_region region;

		max_block_size = max(td->o.max_bs[DDIR_READ],
				     max(td->o.max_bs[DDIR_WRITE],
					 td->o.max_bs[DDIR_TRIM]));

		region = (struct blkio_mem_region) {
			.addr	= td->orig_buffer,
			.len	= (size_t)max_block_size *
					(size_t)td->o.iodepth,
			.fd	= -1,
		};

		if (blkio_map_mem_region(data->b, &region) != 0) {
			fio_blkio_log_err(blkio_map_mem_region);
			return 1;
		}
	}

	return 0;
}

static void fio_blkio_cleanup(struct thread_data *td)
{
	struct fio_blkio_data *data = td->io_ops_data;

	if (data) {
		blkio_destroy(&data->b);
		free(data->completions);
		free(data);
	}
}

#define align_up(x, y) ((((x) + (y) - 1) / (y)) * (y))

static int fio_blkio_iomem_alloc(struct thread_data *td, size_t size)
{
	struct fio_blkio_data *data = td->io_ops_data;
	int ret;
	uint64_t mem_region_alignment;

	if (blkio_get_uint64(data->b, "mem-region-alignment",
			     &mem_region_alignment) != 0) {
		fio_blkio_log_err(blkio_get_uint64);
		return 1;
	}

	/* round up size to satisfy mem-region-alignment */
	size = align_up(size, (size_t)mem_region_alignment);

	if (blkio_alloc_mem_region(data->b, &data->mem_region, size) != 0) {
		fio_blkio_log_err(blkio_alloc_mem_region);
		ret = 1;
		goto out;
	}

	if (blkio_map_mem_region(data->b, &data->mem_region) != 0) {
		fio_blkio_log_err(blkio_map_mem_region);
		ret = 1;
		goto out_free;
	}

	td->orig_buffer = data->mem_region.addr;
	data->has_mem_region = true;

	ret = 0;
	goto out;

out_free:
	blkio_free_mem_region(data->b, &data->mem_region);
out:
	return ret;
}

static void fio_blkio_iomem_free(struct thread_data *td)
{
	struct fio_blkio_data *data = td->io_ops_data;

	if (data && data->has_mem_region) {
		blkio_unmap_mem_region(data->b, &data->mem_region);
		blkio_free_mem_region(data->b, &data->mem_region);

		data->has_mem_region = false;
	}
}

static int fio_blkio_open_file(struct thread_data *td, struct fio_file *f)
{
	return 0;
}

static enum fio_q_status fio_blkio_queue(struct thread_data *td,
					 struct io_u *io_u)
{
	struct fio_blkio_data *data = td->io_ops_data;

	fio_ro_check(td, io_u);

	switch (io_u->ddir) {
		case DDIR_READ:
			blkioq_read(data->q, io_u->offset, io_u->xfer_buf,
				    (size_t)io_u->xfer_buflen, io_u, 0);
			break;
		case DDIR_WRITE:
			blkioq_write(data->q, io_u->offset, io_u->xfer_buf,
				     (size_t)io_u->xfer_buflen, io_u, 0);
			break;
		case DDIR_TRIM:
			blkioq_discard(data->q, io_u->offset, io_u->xfer_buflen,
				       io_u, 0);
		        break;
		case DDIR_SYNC:
		case DDIR_DATASYNC:
			blkioq_flush(data->q, io_u, 0);
			break;
		default:
			io_u->error = ENOTSUP;
			io_u_log_error(td, io_u);
			return FIO_Q_COMPLETED;
	}

	return FIO_Q_QUEUED;
}

static int fio_blkio_getevents(struct thread_data *td, unsigned int min,
			       unsigned int max, const struct timespec *t)
{
	struct fio_blkio_data *data = td->io_ops_data;
	int n;

	n = blkioq_do_io(data->q, data->completions, (int)min, (int)max, NULL);
	if (n < 0) {
		fio_blkio_log_err(blkioq_do_io);
		return -1;
	}

	return n;
}

static struct io_u *fio_blkio_event(struct thread_data *td, int event)
{
	struct fio_blkio_data *data = td->io_ops_data;
	struct blkio_completion *completion = &data->completions[event];
	struct io_u *io_u = completion->user_data;

	io_u->error = -completion->ret;

	return io_u;
}

FIO_STATIC struct ioengine_ops ioengine = {
	.name			= "libblkio",
	.version		= FIO_IOOPS_VERSION,
	.flags			= FIO_DISKLESSIO | FIO_NOEXTEND |
				  FIO_NO_OFFLOAD | FIO_SKIPPABLE_IOMEM_ALLOC,

	.setup			= fio_blkio_setup,
	.init			= fio_blkio_init,
	.post_init		= fio_blkio_post_init,
	.cleanup		= fio_blkio_cleanup,

	.iomem_alloc		= fio_blkio_iomem_alloc,
	.iomem_free		= fio_blkio_iomem_free,

	.open_file		= fio_blkio_open_file,

	.queue			= fio_blkio_queue,
	.getevents		= fio_blkio_getevents,
	.event			= fio_blkio_event,

	.options		= options,
	.option_struct_size	= sizeof(struct fio_blkio_options),
};

static void fio_init fio_blkio_register(void)
{
	register_ioengine(&ioengine);
}

static void fio_exit fio_blkio_unregister(void)
{
	unregister_ioengine(&ioengine);
}
Commit	Line	Data
a601337a AF	1	/*
	2	* libblkio engine
	3	*
	4	* IO engine using libblkio to access various block I/O interfaces:
	5	* https://gitlab.com/libblkio/libblkio
	6	*/
	7
	8	#include <assert.h>
	9	#include <errno.h>
	10	#include <stdbool.h>
	11	#include <stddef.h>
	12	#include <stdint.h>
	13	#include <stdlib.h>
	14	#include <string.h>
	15
	16	#include <blkio.h>
	17
	18	#include "../fio.h"
	19	#include "../optgroup.h"
	20	#include "../options.h"
	21	#include "../parse.h"
	22
	23	/* per-thread state */
	24	struct fio_blkio_data {
	25	struct blkio *b;
	26	struct blkioq *q;
	27
	28	bool has_mem_region; /* whether mem_region is valid */
ef9b6f2f	29	struct blkio_mem_region mem_region; /* only if allocated by libblkio */
a601337a AF	30
	31	struct blkio_completion *completions;
	32	};
	33
	34	struct fio_blkio_options {
	35	void pad; / option fields must not have offset 0 */
	36
	37	char *driver;
	38	char *pre_connect_props;
	39	char *pre_start_props;
	40	};
	41
	42	static struct fio_option options[] = {
	43	{
	44	.name = "libblkio_driver",
	45	.lname = "libblkio driver name",
	46	.type = FIO_OPT_STR_STORE,
	47	.off1 = offsetof(struct fio_blkio_options, driver),
	48	.help = "Name of the driver to be used by libblkio",
	49	.category = FIO_OPT_C_ENGINE,
	50	.group = FIO_OPT_G_LIBBLKIO,
	51	},
	52	{
	53	.name = "libblkio_pre_connect_props",
	54	.lname = "Properties to be set before blkio_connect()",
	55	.type = FIO_OPT_STR_STORE,
	56	.off1 = offsetof(struct fio_blkio_options, pre_connect_props),
	57	.help = "",
	58	.category = FIO_OPT_C_ENGINE,
	59	.group = FIO_OPT_G_LIBBLKIO,
	60	},
	61	{
	62	.name = "libblkio_pre_start_props",
	63	.lname = "Properties to be set before blkio_start()",
	64	.type = FIO_OPT_STR_STORE,
	65	.off1 = offsetof(struct fio_blkio_options, pre_start_props),
	66	.help = "",
	67	.category = FIO_OPT_C_ENGINE,
	68	.group = FIO_OPT_G_LIBBLKIO,
	69	},
	70	{
	71	.name = NULL,
	72	},
	73	};
	74
	75	static int fio_blkio_set_props_from_str(struct blkio b, const char opt_name,
	76	const char *str) {
	77	int ret = 0;
	78	char new_str, name, *value;
	79
	80	if (!str)
	81	return 0;
	82
	83	/* iteration can mutate string, so copy it */
	84	new_str = strdup(str);
	85	if (!new_str) {
	86	log_err("fio: strdup() failed\n");
	87	return 1;
	88	}
	89
	90	/* iterate over property name-value pairs */
	91	while ((name = get_next_str(&new_str))) {
	92	/* split into property name and value */
	93	value = strchr(name, '=');
94	if (!value) {
95	log_err("fio: missing '=' in option %s\n", opt_name);
96	ret = 1;
97	break;
98	}
99
100	*value = '\0';
101	++value;
102
103	/* strip whitespace from property name */
104	strip_blank_front(&name);
105	strip_blank_end(name);
106
107	if (name[0] == '\0') {
108	log_err("fio: empty property name in option %s\n",
109	opt_name);
110	ret = 1;
111	break;
112	}
113
114	/* strip whitespace from property value */
115	strip_blank_front(&value);
116	strip_blank_end(value);
117
118	/* set property */
119	if (blkio_set_str(b, name, value) != 0) {
120	log_err("fio: error setting property '%s' to '%s': %s\n",
121	name, value, blkio_get_error_msg());
122	ret = 1;
123	break;
124	}
125	}
126
127	free(new_str);
128	return ret;
129	}
130
131	/*
132	* Log the failure of a libblkio function.
133	*
134	* `(void)func` is to ensure `func` exists and prevent typos
135	*/
136	#define fio_blkio_log_err(func) \
137	({ \
138	(void)func; \
139	log_err("fio: %s() failed: %s\n", #func, \
140	blkio_get_error_msg()); \
141	})
142
143	static int fio_blkio_create_and_connect(struct thread_data *td,
144	struct blkio **out_blkio)
145	{
146	const struct fio_blkio_options *options = td->eo;
147	struct blkio *b;
148	int ret;
149
150	if (!options->driver) {
151	log_err("fio: engine libblkio requires option libblkio_driver to be set\n");
152	return 1;
153	}
154
155	if (blkio_create(options->driver, &b) != 0) {
156	fio_blkio_log_err(blkio_create);
157	return 1;
158	}
159
160	/* don't fail if driver doesn't have a "direct" property */
161	ret = blkio_set_bool(b, "direct", td->o.odirect);
162	if (ret != 0 && ret != -ENOENT) {
163	fio_blkio_log_err(blkio_set_bool);
164	goto err_blkio_destroy;
165	}
166
167	if (blkio_set_bool(b, "read-only", read_only) != 0) {
168	fio_blkio_log_err(blkio_set_bool);
169	goto err_blkio_destroy;
170	}
171
172	if (fio_blkio_set_props_from_str(b, "libblkio_pre_connect_props",
173	options->pre_connect_props) != 0)
174	goto err_blkio_destroy;
175
176	if (blkio_connect(b) != 0) {
177	fio_blkio_log_err(blkio_connect);
178	goto err_blkio_destroy;
179	}
180
181	if (fio_blkio_set_props_from_str(b, "libblkio_pre_start_props",
182	options->pre_start_props) != 0)
183	goto err_blkio_destroy;
184
185	*out_blkio = b;
186	return 0;
187
188	err_blkio_destroy:
189	blkio_destroy(&b);
190	return 1;
191	}
192
193	/*
194	* This callback determines the device/file size, so it creates and connects a
195	* blkio instance. But it is invoked from the main thread in the original fio
196	* process, not from the processes in which jobs will actually run. It thus
197	* subsequently destroys the blkio, which is recreated in the init() callback.
198	*/
199	static int fio_blkio_setup(struct thread_data *td)
200	{
201	struct blkio *b;
202	int ret = 0;
203	uint64_t capacity;
204
205	assert(td->files_index == 1);
206
207	if (fio_blkio_create_and_connect(td, &b) != 0)
208	return 1;
209
210	if (blkio_get_uint64(b, "capacity", &capacity) != 0) {
211	fio_blkio_log_err(blkio_get_uint64);
212	ret = 1;
213	goto out_blkio_destroy;
214	}
215
216	td->files[0]->real_file_size = capacity;
217	fio_file_set_size_known(td->files[0]);
218
219	out_blkio_destroy:
220	blkio_destroy(&b);
221	return ret;
222	}
223
224	static int fio_blkio_init(struct thread_data *td)
225	{
226	struct fio_blkio_data *data;
227
228	/*
229	* Request enqueueing is fast, and it's not possible to know exactly
230	* when a request is submitted, so never report submission latencies.
231	*/
232	td->o.disable_slat = 1;
233
234	data = calloc(1, sizeof(*data));
235	if (!data) {
236	log_err("fio: calloc() failed\n");
237	return 1;
238	}
239
240	data->completions = calloc(td->o.iodepth, sizeof(data->completions[0]));
241	if (!data->completions) {
242	log_err("fio: calloc() failed\n");
243	goto err_free;
244	}
245
246	if (fio_blkio_create_and_connect(td, &data->b) != 0)
247	goto err_free;
248
249	if (blkio_set_int(data->b, "num-queues", 1) != 0) {
250	fio_blkio_log_err(blkio_set_int);
251	goto err_blkio_destroy;
252	}
253
254	if (blkio_start(data->b) != 0) {
255	fio_blkio_log_err(blkio_start);
256	goto err_blkio_destroy;
257	}
258
259	data->q = blkio_get_queue(data->b, 0);
260
261	/* Set data last so cleanup() does nothing if init() fails. */
262	td->io_ops_data = data;
263
264	return 0;
265
266	err_blkio_destroy:
267	blkio_destroy(&data->b);
268	err_free:
269	free(data->completions);
270	free(data);
271	return 1;
272	}
273
ef9b6f2f AF	274	static int fio_blkio_post_init(struct thread_data *td)
	275	{
	276	struct fio_blkio_data *data = td->io_ops_data;
	277
	278	if (!data->has_mem_region) {
	279	/*
	280	* Memory was allocated by the fio core and not iomem_alloc(),
	281	* so we need to register it as a memory region here.
	282	*
	283	* `td->orig_buffer_size` is computed like `len` below, but then
	284	* fio can add some padding to it to make sure it is
	285	* sufficiently aligned to the page size and the mem_align
	286	* option. However, this can make it become unaligned to the
	287	* "mem-region-alignment" property in ways that the user can't
	288	* control, so we essentially recompute `td->orig_buffer_size`
	289	* here but without adding that padding.
	290	*/
	291
	292	unsigned long long max_block_size;
	293	struct blkio_mem_region region;
	294
	295	max_block_size = max(td->o.max_bs[DDIR_READ],
	296	max(td->o.max_bs[DDIR_WRITE],
	297	td->o.max_bs[DDIR_TRIM]));
	298
	299	region = (struct blkio_mem_region) {
	300	.addr = td->orig_buffer,
	301	.len = (size_t)max_block_size *
	302	(size_t)td->o.iodepth,
	303	.fd = -1,
	304	};
	305
	306	if (blkio_map_mem_region(data->b, &region) != 0) {
	307	fio_blkio_log_err(blkio_map_mem_region);
	308	return 1;
	309	}
	310	}
	311
	312	return 0;
	313	}
	314
a601337a AF	315	static void fio_blkio_cleanup(struct thread_data *td)
	316	{
	317	struct fio_blkio_data *data = td->io_ops_data;
	318
	319	if (data) {
	320	blkio_destroy(&data->b);
	321	free(data->completions);
	322	free(data);
	323	}
	324	}
	325
	326	#define align_up(x, y) ((((x) + (y) - 1) / (y)) * (y))
	327
	328	static int fio_blkio_iomem_alloc(struct thread_data *td, size_t size)
	329	{
	330	struct fio_blkio_data *data = td->io_ops_data;
	331	int ret;
	332	uint64_t mem_region_alignment;
	333
	334	if (blkio_get_uint64(data->b, "mem-region-alignment",
	335	&mem_region_alignment) != 0) {
	336	fio_blkio_log_err(blkio_get_uint64);
	337	return 1;
	338	}
	339
	340	/* round up size to satisfy mem-region-alignment */
	341	size = align_up(size, (size_t)mem_region_alignment);
	342
	343	if (blkio_alloc_mem_region(data->b, &data->mem_region, size) != 0) {
	344	fio_blkio_log_err(blkio_alloc_mem_region);
	345	ret = 1;
	346	goto out;
	347	}
	348
	349	if (blkio_map_mem_region(data->b, &data->mem_region) != 0) {
	350	fio_blkio_log_err(blkio_map_mem_region);
	351	ret = 1;
	352	goto out_free;
	353	}
	354
	355	td->orig_buffer = data->mem_region.addr;
	356	data->has_mem_region = true;
	357
	358	ret = 0;
	359	goto out;
	360
	361	out_free:
	362	blkio_free_mem_region(data->b, &data->mem_region);
	363	out:
	364	return ret;
	365	}
	366
	367	static void fio_blkio_iomem_free(struct thread_data *td)
	368	{
	369	struct fio_blkio_data *data = td->io_ops_data;
	370
	371	if (data && data->has_mem_region) {
	372	blkio_unmap_mem_region(data->b, &data->mem_region);
	373	blkio_free_mem_region(data->b, &data->mem_region);
	374
	375	data->has_mem_region = false;
	376	}
	377	}
	378
379	static int fio_blkio_open_file(struct thread_data td, struct fio_file f)
380	{
381	return 0;
382	}
383
384	static enum fio_q_status fio_blkio_queue(struct thread_data *td,
385	struct io_u *io_u)
386	{
387	struct fio_blkio_data *data = td->io_ops_data;
388
389	fio_ro_check(td, io_u);
390
391	switch (io_u->ddir) {
392	case DDIR_READ:
393	blkioq_read(data->q, io_u->offset, io_u->xfer_buf,
394	(size_t)io_u->xfer_buflen, io_u, 0);
395	break;
396	case DDIR_WRITE:
397	blkioq_write(data->q, io_u->offset, io_u->xfer_buf,
398	(size_t)io_u->xfer_buflen, io_u, 0);
399	break;
400	case DDIR_TRIM:
401	blkioq_discard(data->q, io_u->offset, io_u->xfer_buflen,
402	io_u, 0);
403	break;
404	case DDIR_SYNC:
405	case DDIR_DATASYNC:
406	blkioq_flush(data->q, io_u, 0);
407	break;
408	default:
409	io_u->error = ENOTSUP;
410	io_u_log_error(td, io_u);
411	return FIO_Q_COMPLETED;
412	}
413
414	return FIO_Q_QUEUED;
415	}
416
417	static int fio_blkio_getevents(struct thread_data *td, unsigned int min,
418	unsigned int max, const struct timespec *t)
419	{
420	struct fio_blkio_data *data = td->io_ops_data;
421	int n;
422
423	n = blkioq_do_io(data->q, data->completions, (int)min, (int)max, NULL);
424	if (n < 0) {
425	fio_blkio_log_err(blkioq_do_io);
426	return -1;
427	}
428
429	return n;
430	}
431
432	static struct io_u fio_blkio_event(struct thread_data td, int event)
433	{
434	struct fio_blkio_data *data = td->io_ops_data;
435	struct blkio_completion *completion = &data->completions[event];
436	struct io_u *io_u = completion->user_data;
437
438	io_u->error = -completion->ret;
439
440	return io_u;
441	}
442
443	FIO_STATIC struct ioengine_ops ioengine = {
444	.name = "libblkio",
445	.version = FIO_IOOPS_VERSION,
446	.flags = FIO_DISKLESSIO \| FIO_NOEXTEND \|
ef9b6f2f	447	FIO_NO_OFFLOAD \| FIO_SKIPPABLE_IOMEM_ALLOC,
a601337a AF	448
	449	.setup = fio_blkio_setup,
	450	.init = fio_blkio_init,
ef9b6f2f	451	.post_init = fio_blkio_post_init,
a601337a AF	452	.cleanup = fio_blkio_cleanup,
	453
	454	.iomem_alloc = fio_blkio_iomem_alloc,
	455	.iomem_free = fio_blkio_iomem_free,
	456
	457	.open_file = fio_blkio_open_file,
	458
	459	.queue = fio_blkio_queue,
	460	.getevents = fio_blkio_getevents,
	461	.event = fio_blkio_event,
	462
	463	.options = options,
	464	.option_struct_size = sizeof(struct fio_blkio_options),
	465	};
	466
	467	static void fio_init fio_blkio_register(void)
	468	{
	469	register_ioengine(&ioengine);
	470	}
	471
	472	static void fio_exit fio_blkio_unregister(void)
	473	{
	474	unregister_ioengine(&ioengine);
	475	}