From 2041bd343da1c1e955253f62374588718c64f0f3 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 27 Nov 2018 16:01:44 -0700 Subject: [PATCH] engines/libaio: add preliminary support for pre-mapped IO buffers Experimental kernel features that allows us to register IO buffers when the io_context is setup, eliminating the need to perform get_user_pages() + put_page() for each IO. This dramatically increases performance and lowers latency. Signed-off-by: Jens Axboe --- backend.c | 3 ++ engines/libaio.c | 73 ++++++++++++++++++++++++++++++++++-------------- ioengines.h | 3 +- 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/backend.c b/backend.c index 2f103b3a..2f463293 100644 --- a/backend.c +++ b/backend.c @@ -1704,6 +1704,9 @@ static void *thread_main(void *data) if (init_io_u(td)) goto err; + if (td->io_ops->post_init && td->io_ops->post_init(td)) + goto err; + if (o->verify_async && verify_async_init(td)) goto err; diff --git a/engines/libaio.c b/engines/libaio.c index 74238e4e..e0a073bc 100644 --- a/engines/libaio.c +++ b/engines/libaio.c @@ -24,6 +24,9 @@ #ifndef IOCTX_FLAG_IOPOLL #define IOCTX_FLAG_IOPOLL (1 << 1) #endif +#ifndef IOCTX_FLAG_FIXEDBUFS +#define IOCTX_FLAG_FIXEDBUFS (1 << 2) +#endif static int fio_libaio_commit(struct thread_data *td); @@ -56,6 +59,7 @@ struct libaio_options { unsigned int userspace_reap; unsigned int hipri; unsigned int useriocb; + unsigned int fixedbufs; }; static struct fio_option options[] = { @@ -86,6 +90,15 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_LIBAIO, }, + { + .name = "fixedbufs", + .lname = "Fixed (pre-mapped) IO buffers", + .type = FIO_OPT_STR_SET, + .off1 = offsetof(struct libaio_options, fixedbufs), + .help = "Pre map IO buffers", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_LIBAIO, + }, { .name = NULL, }, @@ -399,7 +412,7 @@ static void fio_libaio_cleanup(struct thread_data *td) } static int fio_libaio_old_queue_init(struct libaio_data *ld, unsigned int depth, - bool hipri, bool useriocb) + bool hipri, bool useriocb, bool fixedbufs) { if (hipri) { log_err("fio: polled aio not available on your platform\n"); @@ -409,12 +422,16 @@ static int fio_libaio_old_queue_init(struct libaio_data *ld, unsigned int depth, log_err("fio: user mapped iocbs not available on your platform\n"); return 1; } + if (fixedbufs) { + log_err("fio: fixed buffers not available on your platform\n"); + return 1; + } return io_queue_init(depth, &ld->aio_ctx); } static int fio_libaio_queue_init(struct libaio_data *ld, unsigned int depth, - bool hipri, bool useriocb) + bool hipri, bool useriocb, bool fixedbufs) { #ifdef __NR_sys_io_setup2 int ret, flags = 0; @@ -423,6 +440,8 @@ static int fio_libaio_queue_init(struct libaio_data *ld, unsigned int depth, flags |= IOCTX_FLAG_IOPOLL; if (useriocb) flags |= IOCTX_FLAG_USERIOCB; + if (fixedbufs) + flags |= IOCTX_FLAG_FIXEDBUFS; ret = syscall(__NR_sys_io_setup2, depth, flags, ld->user_iocbs, &ld->aio_ctx); @@ -430,14 +449,42 @@ static int fio_libaio_queue_init(struct libaio_data *ld, unsigned int depth, return 0; /* fall through to old syscall */ #endif - return fio_libaio_old_queue_init(ld, depth, hipri, useriocb); + return fio_libaio_old_queue_init(ld, depth, hipri, useriocb, fixedbufs); +} + +static int fio_libaio_post_init(struct thread_data *td) +{ + struct libaio_data *ld = td->io_ops_data; + struct libaio_options *o = td->eo; + struct io_u *io_u; + struct iocb *iocb; + int err = 0; + + if (o->fixedbufs) { + int i; + + for (i = 0; i < td->o.iodepth; i++) { + io_u = ld->io_u_index[i]; + iocb = &ld->user_iocbs[i]; + iocb->u.c.buf = io_u->buf; + iocb->u.c.nbytes = io_u->buflen; + } + } + + err = fio_libaio_queue_init(ld, td->o.iodepth, o->hipri, o->useriocb, + o->fixedbufs); + if (err) { + td_verror(td, -err, "io_queue_init"); + return 1; + } + + return 0; } static int fio_libaio_init(struct thread_data *td) { struct libaio_options *o = td->eo; struct libaio_data *ld; - int err = 0; ld = calloc(1, sizeof(*ld)); @@ -450,23 +497,6 @@ static int fio_libaio_init(struct thread_data *td) memset(ld->user_iocbs, 0, size); } - /* - * First try passing in 0 for queue depth, since we don't - * care about the user ring. If that fails, the kernel is too old - * and we need the right depth. - */ - err = fio_libaio_queue_init(ld, td->o.iodepth, o->hipri, o->useriocb); - if (err) { - td_verror(td, -err, "io_queue_init"); - log_err("fio: check /proc/sys/fs/aio-max-nr\n"); - if (ld->user_iocbs) { - size_t size = td->o.iodepth * sizeof(struct iocb); - fio_memfree(ld->user_iocbs, size, false); - } - free(ld); - return 1; - } - ld->entries = td->o.iodepth; ld->is_pow2 = is_power_of_2(ld->entries); ld->aio_events = calloc(ld->entries, sizeof(struct io_event)); @@ -494,6 +524,7 @@ static struct ioengine_ops ioengine = { .name = "libaio", .version = FIO_IOOPS_VERSION, .init = fio_libaio_init, + .post_init = fio_libaio_post_init, .io_u_init = fio_libaio_io_u_init, .prep = fio_libaio_prep, .queue = fio_libaio_queue, diff --git a/ioengines.h b/ioengines.h index feb21db8..b9cd33d5 100644 --- a/ioengines.h +++ b/ioengines.h @@ -7,7 +7,7 @@ #include "flist.h" #include "io_u.h" -#define FIO_IOOPS_VERSION 24 +#define FIO_IOOPS_VERSION 25 /* * io_ops->queue() return values @@ -25,6 +25,7 @@ struct ioengine_ops { int flags; int (*setup)(struct thread_data *); int (*init)(struct thread_data *); + int (*post_init)(struct thread_data *); int (*prep)(struct thread_data *, struct io_u *); enum fio_q_status (*queue)(struct thread_data *, struct io_u *); int (*commit)(struct thread_data *); -- 2.25.1