no-ipi

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index 35d8c2b466994e6459fa9382a2352d6803afa4b1..8622e21996f86f80c7fe8122916b94c257725092 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -872,7 +872,7 @@ static bool io_wq_for_each_worker(struct io_wqe *wqe,
 
 static bool io_wq_worker_wake(struct io_worker *worker, void *data)
 {
-       set_notify_signal(worker->task);
+       __set_notify_signal(worker->task);
        wake_up_process(worker->task);
        return false;
 }
@@ -992,7 +992,7 @@ static bool __io_wq_worker_cancel(struct io_worker *worker,
 {
        if (work && match->fn(work, match->data)) {
                work->flags |= IO_WQ_WORK_CANCEL;
-               set_notify_signal(worker->task);
+               __set_notify_signal(worker->task);
                return true;
        }
 

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 1e0f931bc9ad5babdd2c7b015808712c61cd9db3..8fb041c37e8f8823de61f1037d481071384d4098 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2501,6 +2501,13 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)
 {
        if (!ctx)
                return;
+       if (ctx->flags & IORING_SETUP_TW_FLAG &&
+           ctx->rings->sq_flags & IORING_SQ_TW) {
+               spin_lock(&ctx->completion_lock);
+               WRITE_ONCE(ctx->rings->sq_flags,
+                               ctx->rings->sq_flags & ~IORING_SQ_TW);
+               spin_unlock(&ctx->completion_lock);
+       }
        if (*locked) {
                io_submit_flush_completions(ctx);
                mutex_unlock(&ctx->uring_lock);
@@ -2586,6 +2593,7 @@ void io_uring_task_work_run(void)
 static void io_req_task_work_add(struct io_kiocb *req, bool priority)
 {
        struct task_struct *tsk = req->task;
+       struct io_ring_ctx *ctx = req->ctx;
        struct io_uring_task *tctx = tsk->io_uring;
        struct io_task_work *head;
 
@@ -2606,6 +2614,14 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority)
                req->io_task_work.next = head;
        } while (cmpxchg(&tctx->task_list, head, &req->io_task_work) != head);
 
+       if (ctx->flags & IORING_SETUP_TW_FLAG &&
+           !(ctx->rings->sq_flags & IORING_SQ_TW)) {
+               spin_lock(&ctx->completion_lock);
+               WRITE_ONCE(ctx->rings->sq_flags,
+                               ctx->rings->sq_flags | IORING_SQ_TW);
+               spin_unlock(&ctx->completion_lock);
+       }
+
        if (!head) {
                /*
                 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
@@ -2613,8 +2629,10 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority)
                 * processing task_work. There's no reliable way to tell if TWA_RESUME
                 * will do the job.
                 */
-               if (req->ctx->flags & IORING_SETUP_SQPOLL)
+               if (ctx->flags & IORING_SETUP_SQPOLL)
                        wake_up_process(tsk);
+               else if (ctx->flags & IORING_SETUP_NOIPI)
+                       task_work_notify(tsk, TWA_SIGNAL_NOIPI);
                else
                        task_work_notify(tsk, TWA_SIGNAL);
        }
@@ -11364,7 +11382,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
        if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
                        IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
                        IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
-                       IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL))
+                       IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |
+                       IORING_SETUP_NOIPI | IORING_SETUP_TW_FLAG))
                return -EINVAL;
 
        return  io_uring_create(entries, &p, params);

diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h
index 3c8b34876744bda628be4c1084e39fc06dbdb938..66b689f6cfcb01afeb02f004f2835d7a1c7f9ee8 100644 (file)
--- a/include/linux/sched/signal.h
+++ b/include/linux/sched/signal.h
@@ -355,14 +355,23 @@ static inline void clear_notify_signal(void)
        smp_mb__after_atomic();
 }
 
+/*
+ * Returns 'true' if kick_process() is needed to force a transition from
+ * user -> kernel to guarantee expedient run of TWA_SIGNAL based task_work.
+ */
+static inline bool __set_notify_signal(struct task_struct *task)
+{
+       return !test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) &&
+              !wake_up_state(task, TASK_INTERRUPTIBLE);
+}
+
 /*
  * Called to break out of interruptible wait loops, and enter the
  * exit_to_user_mode_loop().
  */
 static inline void set_notify_signal(struct task_struct *task)
 {
-       if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) &&
-           !wake_up_state(task, TASK_INTERRUPTIBLE))
+       if (__set_notify_signal(task))
                kick_process(task);
 }
 

diff --git a/include/linux/task_work.h b/include/linux/task_work.h
index 66852f4a2ca034e13104493524fe99061cf9ccaa..9ed954fc2b34ac8651baebdc6113362a74321ba8 100644 (file)
--- a/include/linux/task_work.h
+++ b/include/linux/task_work.h
@@ -18,6 +18,7 @@ enum task_work_notify_mode {
        TWA_NONE,
        TWA_RESUME,
        TWA_SIGNAL,
+       TWA_SIGNAL_NOIPI,
 };
 
 static inline bool task_work_pending(struct task_struct *task)

diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 980d82eb196e833a9c2375c8cfb1d17896567229..28b6cce9a45fb17acfff8b0e3c212e3d69ece181 100644 (file)
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -102,6 +102,8 @@ enum {
 #define IORING_SETUP_ATTACH_WQ (1U << 5)       /* attach to existing wq */
 #define IORING_SETUP_R_DISABLED        (1U << 6)       /* start with ring disabled */
 #define IORING_SETUP_SUBMIT_ALL        (1U << 7)       /* continue submit on error */
+#define IORING_SETUP_NOIPI     (1U << 8)       /* no IPI needed */
+#define IORING_SETUP_TW_FLAG   (1U << 9)       /* set IORING_SQ_TW */
 
 enum {
        IORING_OP_NOP,
@@ -248,6 +250,7 @@ struct io_sqring_offsets {
  */
 #define IORING_SQ_NEED_WAKEUP  (1U << 0) /* needs io_uring_enter wakeup */
 #define IORING_SQ_CQ_OVERFLOW  (1U << 1) /* CQ ring is overflown */
+#define IORING_SQ_TW           (1U << 2)
 
 struct io_cqring_offsets {
        __u32 head;

diff --git a/kernel/task_work.c b/kernel/task_work.c
index cce8a7ca228f70d329604fc28d421ede3213faf7..ea3b6d2e02f326c83b7cda0a7cbed39df03e1505 100644 (file)
--- a/kernel/task_work.c
+++ b/kernel/task_work.c
@@ -17,6 +17,9 @@ void task_work_notify(struct task_struct *task,
        case TWA_SIGNAL:
                set_notify_signal(task);
                break;
+       case TWA_SIGNAL_NOIPI:
+               __set_notify_signal(task);
+               break;
        default:
                WARN_ON_ONCE(1);
                break;