io_uring: add lazy poll_wq activation

[linux-block.git] / io_uring / io_uring.h

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index c75bbb94703c0bfb4bdf0d962fdc2b834b259ffc..5113e0ddb01db3fad6b922f8201ea1e35167d8c3 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -222,7 +222,7 @@ static inline void io_commit_cqring(struct io_ring_ctx *ctx)
 
 static inline void io_poll_wq_wake(struct io_ring_ctx *ctx)
 {
-       if (waitqueue_active(&ctx->poll_wq))
+       if (wq_has_sleeper(&ctx->poll_wq))
                __wake_up(&ctx->poll_wq, TASK_NORMAL, 0,
                                poll_to_key(EPOLL_URING_WAKE | EPOLLIN));
 }
@@ -230,8 +230,6 @@ static inline void io_poll_wq_wake(struct io_ring_ctx *ctx)
 /* requires smb_mb() prior, see wq_has_sleeper() */
 static inline void __io_cqring_wake(struct io_ring_ctx *ctx)
 {
-       io_poll_wq_wake(ctx);
-
        /*
         * Trigger waitqueue handler on all waiters on our waitqueue. This
         * won't necessarily wake up all the tasks, io_should_wake() will make
@@ -316,7 +314,8 @@ static inline void io_req_complete_defer(struct io_kiocb *req)
 
 static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx)
 {
-       if (unlikely(ctx->off_timeout_used || ctx->drain_active || ctx->has_evfd))
+       if (unlikely(ctx->off_timeout_used || ctx->drain_active ||
+                    ctx->has_evfd || ctx->poll_activated))
                __io_commit_cqring_flush(ctx);
 }