From e5b6516251a2c7ef733eefa2cd02488235333d84 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 21 Apr 2022 08:42:13 -0600 Subject: [PATCH] no-ipi Signed-off-by: Jens Axboe --- fs/io-wq.c | 4 ++-- fs/io_uring.c | 23 +++++++++++++++++++++-- include/linux/sched/signal.h | 13 +++++++++++-- include/linux/task_work.h | 1 + include/uapi/linux/io_uring.h | 3 +++ kernel/task_work.c | 3 +++ 6 files changed, 41 insertions(+), 6 deletions(-) diff --git a/fs/io-wq.c b/fs/io-wq.c index 35d8c2b46699..8622e21996f8 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -872,7 +872,7 @@ static bool io_wq_for_each_worker(struct io_wqe *wqe, static bool io_wq_worker_wake(struct io_worker *worker, void *data) { - set_notify_signal(worker->task); + __set_notify_signal(worker->task); wake_up_process(worker->task); return false; } @@ -992,7 +992,7 @@ static bool __io_wq_worker_cancel(struct io_worker *worker, { if (work && match->fn(work, match->data)) { work->flags |= IO_WQ_WORK_CANCEL; - set_notify_signal(worker->task); + __set_notify_signal(worker->task); return true; } diff --git a/fs/io_uring.c b/fs/io_uring.c index 1e0f931bc9ad..8fb041c37e8f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2501,6 +2501,13 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked) { if (!ctx) return; + if (ctx->flags & IORING_SETUP_TW_FLAG && + ctx->rings->sq_flags & IORING_SQ_TW) { + spin_lock(&ctx->completion_lock); + WRITE_ONCE(ctx->rings->sq_flags, + ctx->rings->sq_flags & ~IORING_SQ_TW); + spin_unlock(&ctx->completion_lock); + } if (*locked) { io_submit_flush_completions(ctx); mutex_unlock(&ctx->uring_lock); @@ -2586,6 +2593,7 @@ void io_uring_task_work_run(void) static void io_req_task_work_add(struct io_kiocb *req, bool priority) { struct task_struct *tsk = req->task; + struct io_ring_ctx *ctx = req->ctx; struct io_uring_task *tctx = tsk->io_uring; struct io_task_work *head; @@ -2606,6 +2614,14 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority) req->io_task_work.next = head; } while (cmpxchg(&tctx->task_list, head, &req->io_task_work) != head); + if (ctx->flags & IORING_SETUP_TW_FLAG && + !(ctx->rings->sq_flags & IORING_SQ_TW)) { + spin_lock(&ctx->completion_lock); + WRITE_ONCE(ctx->rings->sq_flags, + ctx->rings->sq_flags | IORING_SQ_TW); + spin_unlock(&ctx->completion_lock); + } + if (!head) { /* * SQPOLL kernel thread doesn't need notification, just a wakeup. For @@ -2613,8 +2629,10 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority) * processing task_work. There's no reliable way to tell if TWA_RESUME * will do the job. */ - if (req->ctx->flags & IORING_SETUP_SQPOLL) + if (ctx->flags & IORING_SETUP_SQPOLL) wake_up_process(tsk); + else if (ctx->flags & IORING_SETUP_NOIPI) + task_work_notify(tsk, TWA_SIGNAL_NOIPI); else task_work_notify(tsk, TWA_SIGNAL); } @@ -11364,7 +11382,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL | IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ | - IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL)) + IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL | + IORING_SETUP_NOIPI | IORING_SETUP_TW_FLAG)) return -EINVAL; return io_uring_create(entries, &p, params); diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 3c8b34876744..66b689f6cfcb 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -355,14 +355,23 @@ static inline void clear_notify_signal(void) smp_mb__after_atomic(); } +/* + * Returns 'true' if kick_process() is needed to force a transition from + * user -> kernel to guarantee expedient run of TWA_SIGNAL based task_work. + */ +static inline bool __set_notify_signal(struct task_struct *task) +{ + return !test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) && + !wake_up_state(task, TASK_INTERRUPTIBLE); +} + /* * Called to break out of interruptible wait loops, and enter the * exit_to_user_mode_loop(). */ static inline void set_notify_signal(struct task_struct *task) { - if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) && - !wake_up_state(task, TASK_INTERRUPTIBLE)) + if (__set_notify_signal(task)) kick_process(task); } diff --git a/include/linux/task_work.h b/include/linux/task_work.h index 66852f4a2ca0..9ed954fc2b34 100644 --- a/include/linux/task_work.h +++ b/include/linux/task_work.h @@ -18,6 +18,7 @@ enum task_work_notify_mode { TWA_NONE, TWA_RESUME, TWA_SIGNAL, + TWA_SIGNAL_NOIPI, }; static inline bool task_work_pending(struct task_struct *task) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 980d82eb196e..28b6cce9a45f 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -102,6 +102,8 @@ enum { #define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */ #define IORING_SETUP_R_DISABLED (1U << 6) /* start with ring disabled */ #define IORING_SETUP_SUBMIT_ALL (1U << 7) /* continue submit on error */ +#define IORING_SETUP_NOIPI (1U << 8) /* no IPI needed */ +#define IORING_SETUP_TW_FLAG (1U << 9) /* set IORING_SQ_TW */ enum { IORING_OP_NOP, @@ -248,6 +250,7 @@ struct io_sqring_offsets { */ #define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */ #define IORING_SQ_CQ_OVERFLOW (1U << 1) /* CQ ring is overflown */ +#define IORING_SQ_TW (1U << 2) struct io_cqring_offsets { __u32 head; diff --git a/kernel/task_work.c b/kernel/task_work.c index cce8a7ca228f..ea3b6d2e02f3 100644 --- a/kernel/task_work.c +++ b/kernel/task_work.c @@ -17,6 +17,9 @@ void task_work_notify(struct task_struct *task, case TWA_SIGNAL: set_notify_signal(task); break; + case TWA_SIGNAL_NOIPI: + __set_notify_signal(task); + break; default: WARN_ON_ONCE(1); break; -- 2.25.1