io_uring: support deferring submissions to schedule out time

If IORING_SETUP_SCHED_SUBMIT is enabled for !SQPOLL rings,
io_uring_enter(2) syscalls that submit requests will merely prepare the
selected requests and not go through the submission of them. Submission
will instead be performed when the task that prepared the IO is
scheduled out. This allows for fairly trivial batching of submission
events, even across multiple io_uring_enter(2) syscalls, and for
submission of already prepared IO by the time that the task is scheduled
out anyway.

This is a WIP patch, more of a proof of concept than anything else. The
schedule out handling isn't particularly nice, as submission can
potentially cause a blocking event. If the flow to get there is through
schedule, then we could recurse. This is handled by ensuring that the
pending submission list is fully cleared before any of them are
submitted. Hence if an unlikely recursion event into schedule were to
happen, it will be a no-op as nothing is pending.

Outside of doing lazy batch submissions when the task schedules out
anyway, it also allows more precise control of the full list of
pending requests to issue, as they have all been prepared upfront. This
can be advantageous in situations where better decisions can be made
when the kernel knows exactly what is pending on the submission side.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 3a3f651a8a8f972ada95ee3ee6576f81adff9987..e7e7efc6370093a952bd57effece6db4c7abab33 100644 (file)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -89,6 +89,7 @@ struct io_uring_task {
        struct io_wq                    *io_wq;
 
        struct wait_queue_head          wait;
+       struct io_wq_work_list          sched_submit_list;
 
        struct xarray                   xa;
 

diff --git a/include/trace/events/io_uring.h b/include/trace/events/io_uring.h
index 412c9c210a329a2675d88278f2753d80755b8136..d819fabe29b4516f53fc152fc80ad6f519d0ee8f 100644 (file)
--- a/include/trace/events/io_uring.h
+++ b/include/trace/events/io_uring.h
@@ -401,6 +401,34 @@ TRACE_EVENT(io_uring_submit_req,
                  __entry->sq_thread)
 );
 
+/**
+ * io_uring_sched_submit - called on schedule out submit
+ *
+ * @req:               pointer to a submitted request
+ *
+ */
+TRACE_EVENT(io_uring_sched_submit,
+
+       TP_PROTO(struct io_kiocb *req),
+
+       TP_ARGS(req),
+
+       TP_STRUCT__entry (
+               __field(  void *,               ctx             )
+               __field(  void *,               req             )
+               __field(  unsigned long long,   user_data       )
+       ),
+
+       TP_fast_assign(
+               __entry->ctx            = req->ctx;
+               __entry->req            = req;
+               __entry->user_data      = req->cqe.user_data;
+       ),
+
+       TP_printk("ring %p, req %p, user_data 0x%llx",
+                 __entry->ctx, __entry->req, __entry->user_data)
+);
+
 /*
  * io_uring_poll_arm - called after arming a poll wait if successful
  *

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 78cdc57f53eae612f9b895366401c168c490d135..537c52774b085742ba9d399ce6cabcf8daa30201 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -148,7 +148,7 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
                                         bool cancel_all);
 
 static void io_queue_sqe(struct io_kiocb *req);
-static void io_uring_flush_pending_submits(void);
+static void io_uring_flush_pending_submits(struct io_ring_ctx *ctx);
 
 struct kmem_cache *req_cachep;
 static struct workqueue_struct *iou_wq __ro_after_init;
@@ -1533,7 +1533,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
                return 0;
 
        if (ctx->flags & IORING_SETUP_SCHED_SUBMIT)
-               io_uring_flush_pending_submits();
+               io_uring_flush_pending_submits(ctx);
 
        do {
                int ret = 0;
@@ -2178,6 +2178,12 @@ static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,
        return 0;
 }
 
+/*
+ * If we have this much queued before schedule, flush it out at submission
+ * time.
+ */
+#define IOU_MAX_DEFERRED       32
+
 static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
                         const struct io_uring_sqe *sqe)
        __must_hold(&ctx->uring_lock)
@@ -2223,7 +2229,25 @@ fallback:
                return 0;
        }
 
-       io_queue_sqe(req);
+       if (io_uring_inline_issue(ctx)) {
+               io_queue_sqe(req);
+       } else {
+               struct io_uring_task *tctx = current->io_uring;
+
+               /*
+                * If IORING_SETUP_SCHED_SUBMIT is set, after the request has
+                * been prepared (and is stable in terms of potential user
+                * data), stash it away in our task private submit list.
+                * Once the task is scheduled out, we'll submit whatever was
+                * deferred here. If we "plenty" stashed away already, flush
+                * it out.
+                */
+               wq_list_add_tail(&req->comp_list, &tctx->sched_submit_list);
+               ctx->submit_state.sched_plug_submit_nr++;
+               if (ctx->submit_state.sched_plug_submit_nr >= IOU_MAX_DEFERRED)
+                       io_uring_flush_pending_submits(ctx);
+
+       }
        return 0;
 }
 
@@ -3175,6 +3199,7 @@ __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
 
        if (!current->io_uring)
                return;
+       __io_uring_submit_on_sched(tctx);
        if (tctx->io_wq)
                io_wq_exit_start(tctx->io_wq);
 
@@ -3265,20 +3290,98 @@ static void io_uring_submit_on_sched_sqpoll(struct io_uring_task *tctx)
        }
 }
 
-static void io_uring_flush_pending_submits(void)
+static void io_uring_start_ctx_flush(struct io_ring_ctx *ctx)
+{
+       struct io_submit_state *state = &ctx->submit_state;
+
+       io_submit_state_start(state, state->sched_plug_submit_nr);
+       if (state->need_plug) {
+               blk_start_plug_nr_ios(&state->plug, state->sched_plug_submit_nr);
+               state->plug_started = true;
+               state->need_plug = false;
+       }
+       state->sched_plug_submit_nr = 0;
+}
+
+static void io_uring_swap_ctx(struct io_kiocb *req, struct io_ring_ctx **ctxp)
+{
+       struct io_ring_ctx *ctx = *ctxp;
+
+       if (ctx == req->ctx)
+               return;
+       if (ctx) {
+               io_submit_state_end(ctx);
+               mutex_unlock(&ctx->uring_lock);
+       }
+       *ctxp = ctx = req->ctx;
+       mutex_lock(&ctx->uring_lock);
+       io_uring_start_ctx_flush(ctx);
+}
+
+static void __io_uring_flush_pending_submits(struct io_wq_work_node *node,
+                                            struct io_ring_ctx **cur_ctx)
+{
+       io_uring_start_ctx_flush(*cur_ctx);
+
+       while (node) {
+               struct io_wq_work_node *nxt = node->next;
+               struct io_kiocb *req;
+
+               req = container_of(node, struct io_kiocb, comp_list);
+               io_uring_swap_ctx(req, cur_ctx);
+               trace_io_uring_sched_submit(req);
+               node->next = NULL;
+               io_queue_sqe(req);
+               node = nxt;
+       }
+       io_submit_state_end(*cur_ctx);
+}
+
+static void io_uring_flush_pending_submits(struct io_ring_ctx *ctx)
 {
        struct io_uring_task *tctx = current->io_uring;
+       struct io_wq_work_node *node = tctx->sched_submit_list.first;
+
+       if (node) {
+               struct io_ring_ctx *org_ctx = ctx;
 
+               INIT_WQ_LIST(&tctx->sched_submit_list);
+               __io_uring_flush_pending_submits(node, &ctx);
+               if (ctx != org_ctx) {
+                       mutex_unlock(&ctx->uring_lock);
+                       mutex_lock(&org_ctx->uring_lock);
+               }
+       }
        if (tctx->sq_poll_iter)
                io_uring_submit_on_sched_sqpoll(tctx);
 }
 
+static void io_uring_sched_submit(struct io_wq_work_node *node)
+{
+       struct io_ring_ctx *ctx;
+       struct io_kiocb *req;
+
+       __set_current_state(TASK_RUNNING);
+
+       req = container_of(node, struct io_kiocb, comp_list);
+       ctx = req->ctx;
+       mutex_lock(&ctx->uring_lock);
+       __io_uring_flush_pending_submits(node, &ctx);
+       mutex_unlock(&ctx->uring_lock);
+}
+
 /*
  * Called on schedule out if the task has a ring that was setup with
  * IORING_SETUP_SCHED_SUBMIT. Flushes pending submits.
  */
 void __io_uring_submit_on_sched(struct io_uring_task *tctx)
 {
+       struct io_wq_work_node *node = tctx->sched_submit_list.first;
+
+       if (node) {
+               INIT_WQ_LIST(&tctx->sched_submit_list);
+               io_uring_sched_submit(node);
+       }
        if (tctx->sq_poll_iter)
                io_uring_submit_on_sched_sqpoll(tctx);
 }

diff --git a/io_uring/tctx.c b/io_uring/tctx.c
index 371b9a6908288e73c24279b750e15a0ecf67f003..66643cb19675690d26b1851a8643ae9c88cf01c4 100644 (file)
--- a/io_uring/tctx.c
+++ b/io_uring/tctx.c
@@ -51,6 +51,7 @@ void __io_uring_free(struct task_struct *tsk)
        WARN_ON_ONCE(!xa_empty(&tctx->xa));
        WARN_ON_ONCE(tctx->io_wq);
        WARN_ON_ONCE(tctx->cached_refs);
+       WARN_ON_ONCE(!wq_list_empty(&tctx->sched_submit_list));
 
        percpu_counter_destroy(&tctx->inflight);
        kfree(tctx);
@@ -84,6 +85,7 @@ __cold int io_uring_alloc_task_context(struct task_struct *task,
        if (ctx->flags & IORING_SETUP_SCHED_SUBMIT)
                tctx->sched_submit = true;
 
+       INIT_WQ_LIST(&tctx->sched_submit_list);
        xa_init(&tctx->xa);
        init_waitqueue_head(&tctx->wait);
        atomic_set(&tctx->in_cancel, 0);