X-Git-Url: https://git.kernel.dk/?p=fio.git;a=blobdiff_plain;f=engines%2Fio_uring.c;h=10cfe9f23e2476ffed8500ce13e46d2ad2e8ed90;hp=ca3e157f8f1c3d648d37d2424453a229924c3212;hb=27f436d9f72a9d2d3da3adfdf712757152eab29e;hpb=2a09811cf987fce7f125178ba444aa31f1a57ab4 diff --git a/engines/io_uring.c b/engines/io_uring.c index ca3e157f..10cfe9f2 100644 --- a/engines/io_uring.c +++ b/engines/io_uring.c @@ -21,7 +21,7 @@ #ifdef ARCH_HAVE_IOURING #include "../lib/types.h" -#include "../os/io_uring.h" +#include "../os/linux/io_uring.h" struct io_sq_ring { unsigned *head; @@ -50,6 +50,8 @@ struct ioring_data { struct io_u **io_u_index; + int *fds; + struct io_sq_ring sq_ring; struct io_uring_sqe *sqes; struct iovec *iovecs; @@ -62,9 +64,6 @@ struct ioring_data { int cq_ring_off; unsigned iodepth; - uint64_t cachehit; - uint64_t cachemiss; - struct ioring_mmap mmap[3]; }; @@ -72,6 +71,7 @@ struct ioring_options { void *pad; unsigned int hipri; unsigned int fixedbufs; + unsigned int registerfiles; unsigned int sqpoll_thread; unsigned int sqpoll_set; unsigned int sqpoll_cpu; @@ -94,7 +94,7 @@ static struct fio_option options[] = { .off1 = offsetof(struct ioring_options, hipri), .help = "Use polled IO completions", .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, + .group = FIO_OPT_G_IOURING, }, { .name = "fixedbufs", @@ -103,7 +103,16 @@ static struct fio_option options[] = { .off1 = offsetof(struct ioring_options, fixedbufs), .help = "Pre map IO buffers", .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, + .group = FIO_OPT_G_IOURING, + }, + { + .name = "registerfiles", + .lname = "Register file set", + .type = FIO_OPT_STR_SET, + .off1 = offsetof(struct ioring_options, registerfiles), + .help = "Pre-open/register files", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_IOURING, }, { .name = "sqthread_poll", @@ -112,7 +121,7 @@ static struct fio_option options[] = { .off1 = offsetof(struct ioring_options, sqpoll_thread), .help = "Offload submission/completion to kernel thread", .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, + .group = FIO_OPT_G_IOURING, }, { .name = "sqthread_poll_cpu", @@ -121,7 +130,7 @@ static struct fio_option options[] = { .cb = fio_ioring_sqpoll_cb, .help = "What CPU to run SQ thread polling on", .category = FIO_OPT_C_ENGINE, - .group = FIO_OPT_G_LIBAIO, + .group = FIO_OPT_G_IOURING, }, { .name = NULL, @@ -132,7 +141,7 @@ static int io_uring_enter(struct ioring_data *ld, unsigned int to_submit, unsigned int min_complete, unsigned int flags) { return syscall(__NR_sys_io_uring_enter, ld->ring_fd, to_submit, - min_complete, flags); + min_complete, flags, NULL, 0); } static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) @@ -143,8 +152,13 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) struct io_uring_sqe *sqe; sqe = &ld->sqes[io_u->index]; - sqe->fd = f->fd; - sqe->flags = 0; + if (o->registerfiles) { + sqe->fd = f->engine_pos; + sqe->flags = IOSQE_FIXED_FILE; + } else { + sqe->fd = f->fd; + sqe->flags = 0; + } sqe->ioprio = 0; sqe->buf_index = 0; @@ -154,7 +168,7 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) sqe->opcode = IORING_OP_READ_FIXED; else sqe->opcode = IORING_OP_WRITE_FIXED; - sqe->addr = io_u->xfer_buf; + sqe->addr = (unsigned long) io_u->xfer_buf; sqe->len = io_u->xfer_buflen; sqe->buf_index = io_u->index; } else { @@ -162,7 +176,7 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u) sqe->opcode = IORING_OP_READV; else sqe->opcode = IORING_OP_WRITEV; - sqe->addr = &ld->iovecs[io_u->index]; + sqe->addr = (unsigned long) &ld->iovecs[io_u->index]; sqe->len = 1; } sqe->off = io_u->offset; @@ -187,7 +201,7 @@ static struct io_u *fio_ioring_event(struct thread_data *td, int event) index = (event + ld->cq_ring_off) & ld->cq_ring_mask; cqe = &ld->cq_ring.cqes[index]; - io_u = (struct io_u *) cqe->user_data; + io_u = (struct io_u *) (uintptr_t) cqe->user_data; if (cqe->res != io_u->xfer_buflen) { if (cqe->res > io_u->xfer_buflen) @@ -197,13 +211,6 @@ static struct io_u *fio_ioring_event(struct thread_data *td, int event) } else io_u->error = 0; - if (io_u->ddir == DDIR_READ) { - if (cqe->flags & IOCQE_FLAG_CACHEHIT) - ld->cachehit++; - else - ld->cachemiss++; - } - return io_u; } @@ -243,6 +250,8 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min, r = fio_ioring_cqring_reap(td, events, max); if (r) { events += r; + if (actual_min != 0) + actual_min -= r; continue; } @@ -289,6 +298,8 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td, if (next_tail == *ring->head) return FIO_Q_BUSY; + /* ensure sqe stores are ordered with tail update */ + write_barrier(); ring->array[tail & ld->sq_ring_mask] = io_u->index; *ring->tail = next_tail; write_barrier(); @@ -338,7 +349,8 @@ static int fio_ioring_commit(struct thread_data *td) read_barrier(); if (*ring->flags & IORING_SQ_NEED_WAKEUP) - io_uring_enter(ld, ld->queued, 0, 0); + io_uring_enter(ld, ld->queued, 0, + IORING_ENTER_SQ_WAKEUP); ld->queued = 0; return 0; } @@ -388,14 +400,12 @@ static void fio_ioring_cleanup(struct thread_data *td) struct ioring_data *ld = td->io_ops_data; if (ld) { - td->ts.cachehit += ld->cachehit; - td->ts.cachemiss += ld->cachemiss; - if (!(td->flags & TD_F_CHILD)) fio_ioring_unmap(ld); free(ld->io_u_index); free(ld->iovecs); + free(ld->fds); free(ld); } } @@ -460,15 +470,6 @@ static int fio_ioring_queue_init(struct thread_data *td) } } - if (o->fixedbufs) { - struct rlimit rlim = { - .rlim_cur = RLIM_INFINITY, - .rlim_max = RLIM_INFINITY, - }; - - setrlimit(RLIMIT_MEMLOCK, &rlim); - } - ret = syscall(__NR_sys_io_uring_setup, depth, &p); if (ret < 0) return ret; @@ -476,13 +477,16 @@ static int fio_ioring_queue_init(struct thread_data *td) ld->ring_fd = ret; if (o->fixedbufs) { - struct io_uring_register_buffers reg = { - .iovecs = ld->iovecs, - .nr_iovecs = depth + struct rlimit rlim = { + .rlim_cur = RLIM_INFINITY, + .rlim_max = RLIM_INFINITY, }; + if (setrlimit(RLIMIT_MEMLOCK, &rlim) < 0) + return -1; + ret = syscall(__NR_sys_io_uring_register, ld->ring_fd, - IORING_REGISTER_BUFFERS, ®); + IORING_REGISTER_BUFFERS, ld->iovecs, depth); if (ret < 0) return ret; } @@ -490,9 +494,50 @@ static int fio_ioring_queue_init(struct thread_data *td) return fio_ioring_mmap(ld, &p); } +static int fio_ioring_register_files(struct thread_data *td) +{ + struct ioring_data *ld = td->io_ops_data; + struct fio_file *f; + unsigned int i; + int ret; + + ld->fds = calloc(td->o.nr_files, sizeof(int)); + + for_each_file(td, f, i) { + ret = generic_open_file(td, f); + if (ret) + goto err; + ld->fds[i] = f->fd; + f->engine_pos = i; + } + + ret = syscall(__NR_sys_io_uring_register, ld->ring_fd, + IORING_REGISTER_FILES, ld->fds, td->o.nr_files); + if (ret) { +err: + free(ld->fds); + ld->fds = NULL; + } + + /* + * Pretend the file is closed again, and really close it if we hit + * an error. + */ + for_each_file(td, f, i) { + if (ret) { + int fio_unused ret2; + ret2 = generic_close_file(td, f); + } else + f->fd = -1; + } + + return ret; +} + static int fio_ioring_post_init(struct thread_data *td) { struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; struct io_u *io_u; int err, i; @@ -510,6 +555,14 @@ static int fio_ioring_post_init(struct thread_data *td) return 1; } + if (o->registerfiles) { + err = fio_ioring_register_files(td); + if (err) { + td_verror(td, errno, "ioring_register_files"); + return 1; + } + } + return 0; } @@ -520,8 +573,19 @@ static unsigned roundup_pow2(unsigned depth) static int fio_ioring_init(struct thread_data *td) { + struct ioring_options *o = td->eo; struct ioring_data *ld; + /* sqthread submission requires registered files */ + if (o->sqpoll_thread) + o->registerfiles = 1; + + if (o->registerfiles && td->o.nr_files != td->o.open_files) { + log_err("fio: io_uring registered files require nr_files to " + "be identical to open_files\n"); + return 1; + } + ld = calloc(1, sizeof(*ld)); /* ring depth must be a power-of-2 */ @@ -544,9 +608,33 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u) return 0; } +static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f) +{ + struct ioring_data *ld = td->io_ops_data; + struct ioring_options *o = td->eo; + + if (!o->registerfiles) + return generic_open_file(td, f); + + f->fd = ld->fds[f->engine_pos]; + return 0; +} + +static int fio_ioring_close_file(struct thread_data *td, struct fio_file *f) +{ + struct ioring_options *o = td->eo; + + if (!o->registerfiles) + return generic_close_file(td, f); + + f->fd = -1; + return 0; +} + static struct ioengine_ops ioengine = { .name = "io_uring", .version = FIO_IOOPS_VERSION, + .flags = FIO_ASYNCIO_SYNC_TRIM, .init = fio_ioring_init, .post_init = fio_ioring_post_init, .io_u_init = fio_ioring_io_u_init, @@ -556,8 +644,8 @@ static struct ioengine_ops ioengine = { .getevents = fio_ioring_getevents, .event = fio_ioring_event, .cleanup = fio_ioring_cleanup, - .open_file = generic_open_file, - .close_file = generic_close_file, + .open_file = fio_ioring_open_file, + .close_file = fio_ioring_close_file, .get_file_size = generic_get_file_size, .options = options, .option_struct_size = sizeof(struct ioring_options),