io_uring: wake up optimisations

[linux-block.git] / io_uring / io_uring.c

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index de71730d9051f9b3baf71e969519c6708ebb503d..f2e1dd076d9828f987271f093d5ee8e6a1c22191 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -621,6 +621,25 @@ static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
        io_cqring_wake(ctx);
 }
 
+static inline void __io_cq_unlock_post_flush(struct io_ring_ctx *ctx)
+       __releases(ctx->completion_lock)
+{
+       io_commit_cqring(ctx);
+       __io_cq_unlock(ctx);
+       io_commit_cqring_flush(ctx);
+
+       /*
+        * As ->task_complete implies that the ring is single tasked, cq_wait
+        * may only be waited on by the current in io_cqring_wait(), but since
+        * it will re-check the wakeup conditions once we return we can safely
+        * skip waking it up.
+        */
+       if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) {
+               smp_mb();
+               __io_cqring_wake(ctx);
+       }
+}
+
 void io_cq_unlock_post(struct io_ring_ctx *ctx)
        __releases(ctx->completion_lock)
 {
@@ -1480,7 +1499,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx)
                        }
                }
        }
-       __io_cq_unlock_post(ctx);
+       __io_cq_unlock_post_flush(ctx);
 
        if (!wq_list_empty(&ctx->submit_state.compl_reqs)) {
                io_free_batch_list(ctx, state->compl_reqs.first);