io_uring: move schedule wait logic into helper

In preparation for expanding how we handle waits, move the actual
schedule and schedule_timeout() handling into a helper.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 37053d32c6683f0553dda9da7b2a6919851b65f8..9e2b8d4c05dbb700e78886ff071ceabe7f0fcb22 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2350,22 +2350,10 @@ static bool current_pending_io(void)
        return percpu_counter_read_positive(&tctx->inflight);
 }
 
-/* when returns >0, the caller should retry */
-static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
-                                         struct io_wait_queue *iowq)
+static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
+                                    struct io_wait_queue *iowq)
 {
-       int ret;
-
-       if (unlikely(READ_ONCE(ctx->check_cq)))
-               return 1;
-       if (unlikely(!llist_empty(&ctx->work_llist)))
-               return 1;
-       if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL)))
-               return 1;
-       if (unlikely(task_sigpending(current)))
-               return -EINTR;
-       if (unlikely(io_should_wake(iowq)))
-               return 0;
+       int ret = 0;
 
        /*
         * Mark us as being in io_wait if we have pending requests, so cpufreq
@@ -2374,7 +2362,6 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
         */
        if (current_pending_io())
                current->in_iowait = 1;
-       ret = 0;
        if (iowq->timeout == KTIME_MAX)
                schedule();
        else if (!schedule_hrtimeout_range_clock(&iowq->timeout, 0,
@@ -2384,6 +2371,24 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
        return ret;
 }
 
+/* If this returns > 0, the caller should retry */
+static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
+                                         struct io_wait_queue *iowq)
+{
+       if (unlikely(READ_ONCE(ctx->check_cq)))
+               return 1;
+       if (unlikely(!llist_empty(&ctx->work_llist)))
+               return 1;
+       if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL)))
+               return 1;
+       if (unlikely(task_sigpending(current)))
+               return -EINTR;
+       if (unlikely(io_should_wake(iowq)))
+               return 0;
+
+       return __io_cqring_wait_schedule(ctx, iowq);
+}
+
 struct ext_arg {
        size_t argsz;
        struct __kernel_timespec __user *ts;