io_uring: use EPOLLEXCLUSIVE flag to aoid thundering herd type behavior

Applications can pass this flag in to avoid accept thundering herd.

Signed-off-by: Jiufei Xue <jiufei.xue@linux.alibaba.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 0eb063daa9b5feb47d0e8230bda1e466b885c311..311e8038ae5855ad8f5842295cad9a050138b1b2 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -4245,7 +4245,11 @@ static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
 
        pt->error = 0;
        poll->head = head;
-       add_wait_queue(head, &poll->wait);
+
+       if (poll->events & EPOLLEXCLUSIVE)
+               add_wait_queue_exclusive(head, &poll->wait);
+       else
+               add_wait_queue(head, &poll->wait);
 }
 
 static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
@@ -4602,7 +4606,8 @@ static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe
 #ifdef __BIG_ENDIAN
        events = swahw32(events);
 #endif
-       poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
+       poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP |
+                      (events & EPOLLEXCLUSIVE);
 
        io_get_req_task(req);
        return 0;