io_uring/epoll: add multishot support for IORING_OP_EPOLL_WAIT

As with other multishot requests, submitting a multishot epoll wait
request will keep it re-armed post the initial trigger. This allows
multiple epoll wait completions per request submitted, every time
events are available. If more completions are expected for this
epoll wait request, then IORING_CQE_F_MORE will be set in the posted
cqe->flags.

For multishot, the request remains on the epoll callback waitqueue
head. This means that epoll doesn't need to juggle the ep->lock
writelock (and disable/enable IRQs) for each invocation of the
reaping loop. That should translate into nice efficiency gains.

Use by setting IORING_EPOLL_WAIT_MULTISHOT in the sqe->epoll_flags
member. Must be used with provided buffers.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index a559e1e1544a2afe4a317b40d491659802498f8f..93f504b6d4eca8244d22fef6bcf02c446f646920 100644 (file)
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -73,6 +73,7 @@ struct io_uring_sqe {
                __u32           futex_flags;
                __u32           install_fd_flags;
                __u32           nop_flags;
+               __u32           epoll_flags;
        };
        __u64   user_data;      /* data to be passed back at completion time */
        /* pack this to avoid bogus arm OABI complaints */
@@ -405,6 +406,11 @@ enum io_uring_op {
 #define IORING_ACCEPT_DONTWAIT (1U << 1)
 #define IORING_ACCEPT_POLL_FIRST       (1U << 2)
 
+/*
+ * epoll_wait flags, stored in sqe->epoll_flags
+ */
+#define IORING_EPOLL_WAIT_MULTISHOT    (1U << 0)
+
 /*
  * IORING_OP_MSG_RING command types, stored in sqe->addr
  */

diff --git a/io_uring/epoll.c b/io_uring/epoll.c
index 0a2949c2b6b2c138fdb468d8bf9f37b4db4c1f59..edd62b1a8a83007fc8a53779512b6e9c3b6048bd 100644 (file)
--- a/io_uring/epoll.c
+++ b/io_uring/epoll.c
@@ -25,6 +25,7 @@ struct io_epoll {
 struct io_epoll_wait {
        struct file                     *file;
        int                             maxevents;
+       int                             flags;
        struct epoll_event __user       *events;
        struct wait_queue_entry         wait;
 };
@@ -127,11 +128,12 @@ static void io_epoll_retry(struct io_kiocb *req, struct io_tw_state *ts)
        io_req_task_submit(req, ts);
 }
 
-static int io_epoll_execute(struct io_kiocb *req)
+static int io_epoll_execute(struct io_kiocb *req, __poll_t mask)
 {
        struct io_epoll_wait *iew = io_kiocb_to_cmd(req, struct io_epoll_wait);
 
-       list_del_init_careful(&iew->wait.entry);
+       if (mask & EPOLL_URING_WAKE || !(req->flags & REQ_F_APOLL_MULTISHOT))
+               list_del_init_careful(&iew->wait.entry);
        if (io_poll_get_ownership(req)) {
                req->io_task_work.func = io_epoll_retry;
                io_req_task_work_add(req);
@@ -140,13 +142,13 @@ static int io_epoll_execute(struct io_kiocb *req)
        return 1;
 }
 
-static __cold int io_epoll_pollfree_wake(struct io_kiocb *req)
+static __cold int io_epoll_pollfree_wake(struct io_kiocb *req, __poll_t mask)
 {
        struct io_epoll_wait *iew = io_kiocb_to_cmd(req, struct io_epoll_wait);
 
        io_poll_mark_cancelled(req);
        list_del_init_careful(&iew->wait.entry);
-       io_epoll_execute(req);
+       io_epoll_execute(req, mask);
        return 1;
 }
 
@@ -156,21 +158,31 @@ static int io_epoll_wait_fn(struct wait_queue_entry *wait, unsigned mode,
        struct io_kiocb *req = wait->private;
        __poll_t mask = key_to_poll(key);
 
+       if (mask & EPOLL_SCAN_WAKE)
+               return 0;
        if (unlikely(mask & POLLFREE))
-               return io_epoll_pollfree_wake(req);
+               return io_epoll_pollfree_wake(req, mask);
 
-       return io_epoll_execute(req);
+       return io_epoll_execute(req, mask);
 }
 
 int io_epoll_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
        struct io_epoll_wait *iew = io_kiocb_to_cmd(req, struct io_epoll_wait);
 
-       if (sqe->off || sqe->rw_flags || sqe->splice_fd_in)
+       if (sqe->off || sqe->splice_fd_in)
                return -EINVAL;
 
        iew->maxevents = READ_ONCE(sqe->len);
        iew->events = u64_to_user_ptr(READ_ONCE(sqe->addr));
+       iew->flags = READ_ONCE(sqe->epoll_flags);
+       if (iew->flags & ~IORING_EPOLL_WAIT_MULTISHOT) {
+               return -EINVAL;
+       } else if (iew->flags & IORING_EPOLL_WAIT_MULTISHOT) {
+               if (!(req->flags & REQ_F_BUFFER_SELECT))
+                       return -EINVAL;
+               req->flags |= REQ_F_APOLL_MULTISHOT;
+       }
        if (req->flags & REQ_F_BUFFER_SELECT && iew->events)
                return -EINVAL;
 
@@ -193,7 +205,7 @@ int io_epoll_wait(struct io_kiocb *req, unsigned int issue_flags)
        int ret;
 
        io_ring_submit_lock(ctx, issue_flags);
-
+retry:
        if (io_do_buffer_select(req)) {
                size_t len = maxevents * sizeof(*evs);
 
@@ -204,22 +216,42 @@ int io_epoll_wait(struct io_kiocb *req, unsigned int issue_flags)
                maxevents = len / sizeof(*evs);
        }
 
-       ret = epoll_queue(req->file, evs, maxevents, &iew->wait, false);
+       ret = epoll_queue(req->file, evs, maxevents, &iew->wait,
+                               req->flags & REQ_F_APOLL_MULTISHOT);
        if (ret == -EIOCBQUEUED) {
                io_kbuf_recycle(req, 0);
+skip_complete:
                if (hlist_unhashed(&req->hash_node))
                        hlist_add_head(&req->hash_node, &ctx->epoll_list);
                io_ring_submit_unlock(ctx, issue_flags);
                return IOU_ISSUE_SKIP_COMPLETE;
        } else if (ret > 0) {
                cflags = io_put_kbuf(req, ret * sizeof(*evs), 0);
+               if (req->flags & REQ_F_BL_EMPTY)
+                       goto stop_multi;
+               if (req->flags & REQ_F_APOLL_MULTISHOT) {
+                       if (io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) {
+                               if (ret == maxevents * sizeof(*evs))
+                                       goto retry;
+                               goto skip_complete;
+                       }
+                       goto stop_multi;
+               }
        } else if (!ret) {
                io_kbuf_recycle(req, 0);
        } else {
 err:
                req_set_fail(req);
+               if (req->flags & REQ_F_APOLL_MULTISHOT) {
+stop_multi:
+                       atomic_or(IO_POLL_FINISH_FLAG, &req->poll_refs);
+                       io_poll_multishot_retry(req);
+                       epoll_wait_remove(req->file, &iew->wait);
+                       req->flags &= ~REQ_F_APOLL_MULTISHOT;
+               }
        }
-       hlist_del_init(&req->hash_node);
+       if (!(req->flags & REQ_F_APOLL_MULTISHOT))
+               hlist_del_init(&req->hash_node);
        io_ring_submit_unlock(ctx, issue_flags);
        io_req_set_res(req, ret, cflags);
        return IOU_OK;