From: Jens Axboe Date: Thu, 5 Sep 2019 14:59:29 +0000 (-0600) Subject: engines/io_uring: add support for registered files X-Git-Tag: fio-3.16~9 X-Git-Url: https://git.kernel.dk/?p=fio.git;a=commitdiff_plain;h=5ffd562645eceb5af6c8d068014538320d8a93e9 engines/io_uring: add support for registered files This feature is exposed as a separate option, like fixedbufs, and provides a way for fio to register a set of files with the kernel. This improves IO efficiency. It is also a requirement to be able to use sqthread_poll, as that feature requires fixed files on the kernel side. Signed-off-by: Jens Axboe --- diff --git a/HOWTO b/HOWTO index 4201e2e9..6b449e97 100644 --- a/HOWTO +++ b/HOWTO @@ -2033,6 +2033,12 @@ with the caveat that when used on the command line, they must come after the map and release for each IO. This is more efficient, and reduces the IO latency as well. +.. option:: registerfiles : [io_uring] + With this option, fio registers the set of files being used with the + kernel. This avoids the overhead of managing file counts in the kernel, + making the submission and completion part more lightweight. Required + for the below :option:`sqthread_poll` option. + .. option:: sqthread_poll : [io_uring] Normally fio will submit IO by issuing a system call to notify the diff --git a/engines/io_uring.c b/engines/io_uring.c index 9bcfec17..53bd39c8 100644 --- a/engines/io_uring.c +++ b/engines/io_uring.c @@ -50,6 +50,8 @@ struct ioring_data { struct io_u **io_u_index; + int *fds; + struct io_sq_ring sq_ring; struct io_uring_sqe *sqes; struct iovec *iovecs; @@ -69,6 +71,7 @@ struct ioring_options { void *pad; unsigned int hipri; unsigned int fixedbufs; + unsigned int registerfiles; unsigned int sqpoll_thread; unsigned int sqpoll_set; unsigned int sqpoll_cpu; @@ -102,6 +105,15 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_LIBAIO, }, + { + .name = "registerfiles", + .lname = "Register file set", + .type = FIO_OPT_STR_SET, + .off1 = offsetof(struct ioring_options, registerfiles), + .help = "Pre-open/register files", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_LIBAIO, + }, { .name = "sqthread_poll", .lname = "Kernel SQ thread polling", @@ -140,8 +152,13 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) struct io_uring_sqe *sqe; sqe = &ld->sqes[io_u->index]; - sqe->fd = f->fd; - sqe->flags = 0; + if (o->registerfiles) { + sqe->fd = f->engine_pos; + sqe->flags = IOSQE_FIXED_FILE; + } else { + sqe->fd = f->fd; + sqe->flags = 0; + } sqe->ioprio = 0; sqe->buf_index = 0; @@ -388,6 +405,7 @@ static void fio_ioring_cleanup(struct thread_data *td) free(ld->io_u_index); free(ld->iovecs); + free(ld->fds); free(ld); } } @@ -476,9 +494,50 @@ static int fio_ioring_queue_init(struct thread_data *td) return fio_ioring_mmap(ld, &p); } +static int fio_ioring_register_files(struct thread_data *td) +{ + struct ioring_data *ld = td->io_ops_data; + struct fio_file *f; + unsigned int i; + int ret; + + ld->fds = calloc(td->o.nr_files, sizeof(int)); + + for_each_file(td, f, i) { + ret = generic_open_file(td, f); + if (ret) + goto err; + ld->fds[i] = f->fd; + f->engine_pos = i; + } + + ret = syscall(__NR_sys_io_uring_register, ld->ring_fd, + IORING_REGISTER_FILES, ld->fds, td->o.nr_files); + if (ret) { +err: + free(ld->fds); + ld->fds = NULL; + } + + /* + * Pretend the file is closed again, and really close it if we hit + * an error. + */ + for_each_file(td, f, i) { + if (ret) { + int fio_unused ret2; + ret2 = generic_close_file(td, f); + } else + f->fd = -1; + } + + return ret; +} + static int fio_ioring_post_init(struct thread_data *td) { struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; struct io_u *io_u; int err, i; @@ -496,6 +555,14 @@ static int fio_ioring_post_init(struct thread_data *td) return 1; } + if (o->registerfiles) { + err = fio_ioring_register_files(td); + if (err) { + td_verror(td, errno, "ioring_register_files"); + return 1; + } + } + return 0; } @@ -506,8 +573,19 @@ static unsigned roundup_pow2(unsigned depth) static int fio_ioring_init(struct thread_data *td) { + struct ioring_options *o = td->eo; struct ioring_data *ld; + /* sqthread submission requires registered files */ + if (o->sqpoll_thread) + o->registerfiles = 1; + + if (o->registerfiles && td->o.nr_files != td->o.open_files) { + log_err("fio: io_uring registered files require nr_files to " + "be identical to open_files\n"); + return 1; + } + ld = calloc(1, sizeof(*ld)); /* ring depth must be a power-of-2 */ @@ -530,6 +608,29 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u) return 0; } +static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + + if (!o->registerfiles) + return generic_open_file(td, f); + + f->fd = ld->fds[f->engine_pos]; + return 0; +} + +static int fio_ioring_close_file(struct thread_data *td, struct fio_file *f) +{ + struct ioring_options *o = td->eo; + + if (!o->registerfiles) + return generic_close_file(td, f); + + f->fd = -1; + return 0; +} + static struct ioengine_ops ioengine = { .name = "io_uring", .version = FIO_IOOPS_VERSION, @@ -543,8 +644,8 @@ static struct ioengine_ops ioengine = { .getevents = fio_ioring_getevents, .event = fio_ioring_event, .cleanup = fio_ioring_cleanup, - .open_file = generic_open_file, - .close_file = generic_close_file, + .open_file = fio_ioring_open_file, + .close_file = fio_ioring_close_file, .get_file_size = generic_get_file_size, .options = options, .option_struct_size = sizeof(struct ioring_options), diff --git a/fio.1 b/fio.1 index 3e872bce..e0283f7f 100644 --- a/fio.1 +++ b/fio.1 @@ -1791,6 +1791,12 @@ release them when IO is done. If this option is set, the pages are pre-mapped before IO is started. This eliminates the need to map and release for each IO. This is more efficient, and reduces the IO latency as well. .TP +.BI (io_uring)registerfiles +With this option, fio registers the set of files being used with the kernel. +This avoids the overhead of managing file counts in the kernel, making the +submission and completion part more lightweight. Required for the below +sqthread_poll option. +.TP .BI (io_uring)sqthread_poll Normally fio will submit IO by issuing a system call to notify the kernel of available items in the SQ ring. If this option is set, the act of submitting IO