no-ipitw-test

Signed-off-by: Jens Axboe <axboe@kernel.dk>

6 files changed, 41 insertions, 6 deletions

diff --git a/fs/io-wq.c b/fs/io-wq.c
index 35d8c2b46699..8622e21996f8 100644
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -872,7 +872,7 @@ static bool io_wq_for_each_worker(struct io_wqe *wqe,
 
 static bool io_wq_worker_wake(struct io_worker *worker, void *data)
 {
-	set_notify_signal(worker->task);
+	__set_notify_signal(worker->task);
 	wake_up_process(worker->task);
 	return false;
 }
@@ -992,7 +992,7 @@ static bool __io_wq_worker_cancel(struct io_worker *worker,
 {
 	if (work && match->fn(work, match->data)) {
 		work->flags |= IO_WQ_WORK_CANCEL;
-		set_notify_signal(worker->task);
+		__set_notify_signal(worker->task);
 		return true;
 	}
 
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 1e0f931bc9ad..8fb041c37e8f 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2501,6 +2501,13 @@ static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)
 {
 	if (!ctx)
 		return;
+	if (ctx->flags & IORING_SETUP_TW_FLAG &&
+	    ctx->rings->sq_flags & IORING_SQ_TW) {
+		spin_lock(&ctx->completion_lock);
+		WRITE_ONCE(ctx->rings->sq_flags,
+				ctx->rings->sq_flags & ~IORING_SQ_TW);
+		spin_unlock(&ctx->completion_lock);
+	}
 	if (*locked) {
 		io_submit_flush_completions(ctx);
 		mutex_unlock(&ctx->uring_lock);
@@ -2586,6 +2593,7 @@ void io_uring_task_work_run(void)
 static void io_req_task_work_add(struct io_kiocb *req, bool priority)
 {
 	struct task_struct *tsk = req->task;
+	struct io_ring_ctx *ctx = req->ctx;
 	struct io_uring_task *tctx = tsk->io_uring;
 	struct io_task_work *head;
 
@@ -2606,6 +2614,14 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority)
 		req->io_task_work.next = head;
 	} while (cmpxchg(&tctx->task_list, head, &req->io_task_work) != head);
 
+	if (ctx->flags & IORING_SETUP_TW_FLAG &&
+	    !(ctx->rings->sq_flags & IORING_SQ_TW)) {
+		spin_lock(&ctx->completion_lock);
+		WRITE_ONCE(ctx->rings->sq_flags,
+				ctx->rings->sq_flags | IORING_SQ_TW);
+		spin_unlock(&ctx->completion_lock);
+	}
+
 	if (!head) {
 		/*
 		 * SQPOLL kernel thread doesn't need notification, just a wakeup. For
@@ -2613,8 +2629,10 @@ static void io_req_task_work_add(struct io_kiocb *req, bool priority)
 		 * processing task_work. There's no reliable way to tell if TWA_RESUME
 		 * will do the job.
 		 */
-		if (req->ctx->flags & IORING_SETUP_SQPOLL)
+		if (ctx->flags & IORING_SETUP_SQPOLL)
 			wake_up_process(tsk);
+		else if (ctx->flags & IORING_SETUP_NOIPI)
+			task_work_notify(tsk, TWA_SIGNAL_NOIPI);
 		else
 			task_work_notify(tsk, TWA_SIGNAL);
 	}
@@ -11364,7 +11382,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
 			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
 			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
-			IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL))
+			IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |
+			IORING_SETUP_NOIPI | IORING_SETUP_TW_FLAG))
 		return -EINVAL;
 
 	return  io_uring_create(entries, &p, params);
diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h
index 3c8b34876744..66b689f6cfcb 100644
--- a/include/linux/sched/signal.h
+++ b/include/linux/sched/signal.h
@@ -356,13 +356,22 @@ static inline void clear_notify_signal(void)
 }
 
 /*
+ * Returns 'true' if kick_process() is needed to force a transition from
+ * user -> kernel to guarantee expedient run of TWA_SIGNAL based task_work.
+ */
+static inline bool __set_notify_signal(struct task_struct *task)
+{
+	return !test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) &&
+	       !wake_up_state(task, TASK_INTERRUPTIBLE);
+}
+
+/*
  * Called to break out of interruptible wait loops, and enter the
  * exit_to_user_mode_loop().
  */
 static inline void set_notify_signal(struct task_struct *task)
 {
-	if (!test_and_set_tsk_thread_flag(task, TIF_NOTIFY_SIGNAL) &&
-	    !wake_up_state(task, TASK_INTERRUPTIBLE))
+	if (__set_notify_signal(task))
 		kick_process(task);
 }
 
diff --git a/include/linux/task_work.h b/include/linux/task_work.h
index 66852f4a2ca0..9ed954fc2b34 100644
--- a/include/linux/task_work.h
+++ b/include/linux/task_work.h
@@ -18,6 +18,7 @@ enum task_work_notify_mode {
 	TWA_NONE,
 	TWA_RESUME,
 	TWA_SIGNAL,
+	TWA_SIGNAL_NOIPI,
 };
 
 static inline bool task_work_pending(struct task_struct *task)
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 980d82eb196e..28b6cce9a45f 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -102,6 +102,8 @@ enum {
 #define IORING_SETUP_ATTACH_WQ	(1U << 5)	/* attach to existing wq */
 #define IORING_SETUP_R_DISABLED	(1U << 6)	/* start with ring disabled */
 #define IORING_SETUP_SUBMIT_ALL	(1U << 7)	/* continue submit on error */
+#define IORING_SETUP_NOIPI	(1U << 8)	/* no IPI needed */
+#define IORING_SETUP_TW_FLAG	(1U << 9)	/* set IORING_SQ_TW */
 
 enum {
 	IORING_OP_NOP,
@@ -248,6 +250,7 @@ struct io_sqring_offsets {
  */
 #define IORING_SQ_NEED_WAKEUP	(1U << 0) /* needs io_uring_enter wakeup */
 #define IORING_SQ_CQ_OVERFLOW	(1U << 1) /* CQ ring is overflown */
+#define IORING_SQ_TW		(1U << 2)
 
 struct io_cqring_offsets {
 	__u32 head;
diff --git a/kernel/task_work.c b/kernel/task_work.c
index cce8a7ca228f..ea3b6d2e02f3 100644
--- a/kernel/task_work.c
+++ b/kernel/task_work.c
@@ -17,6 +17,9 @@ void task_work_notify(struct task_struct *task,
 	case TWA_SIGNAL:
 		set_notify_signal(task);
 		break;
+	case TWA_SIGNAL_NOIPI:
+		__set_notify_signal(task);
+		break;
 	default:
 		WARN_ON_ONCE(1);
 		break;