[linux-2.6-block.git] / kernel / entry / common.c

// SPDX-License-Identifier: GPL-2.0

#include <linux/context_tracking.h>
#include <linux/entry-common.h>
#include <linux/highmem.h>
#include <linux/livepatch.h>
#include <linux/audit.h>
#include <linux/tick.h>

#include "common.h"

#define CREATE_TRACE_POINTS
#include <trace/events/syscalls.h>

/* See comment for enter_from_user_mode() in entry-common.h */
static __always_inline void __enter_from_user_mode(struct pt_regs *regs)
{
	arch_check_user_regs(regs);
	lockdep_hardirqs_off(CALLER_ADDR0);

	CT_WARN_ON(ct_state() != CONTEXT_USER);
	user_exit_irqoff();

	instrumentation_begin();
	trace_hardirqs_off_finish();
	instrumentation_end();
}

void noinstr enter_from_user_mode(struct pt_regs *regs)
{
	__enter_from_user_mode(regs);
}

static inline void syscall_enter_audit(struct pt_regs *regs, long syscall)
{
	if (unlikely(audit_context())) {
		unsigned long args[6];

		syscall_get_arguments(current, regs, args);
		audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]);
	}
}

static long syscall_trace_enter(struct pt_regs *regs, long syscall,
				unsigned long work)
{
	long ret = 0;

	/*
	 * Handle Syscall User Dispatch.  This must comes first, since
	 * the ABI here can be something that doesn't make sense for
	 * other syscall_work features.
	 */
	if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) {
		if (syscall_user_dispatch(regs))
			return -1L;
	}

	/* Handle ptrace */
	if (work & (SYSCALL_WORK_SYSCALL_TRACE | SYSCALL_WORK_SYSCALL_EMU)) {
		ret = arch_syscall_enter_tracehook(regs);
		if (ret || (work & SYSCALL_WORK_SYSCALL_EMU))
			return -1L;
	}

	/* Do seccomp after ptrace, to catch any tracer changes. */
	if (work & SYSCALL_WORK_SECCOMP) {
		ret = __secure_computing(NULL);
		if (ret == -1L)
			return ret;
	}

	/* Either of the above might have changed the syscall number */
	syscall = syscall_get_nr(current, regs);

	if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT))
		trace_sys_enter(regs, syscall);

	syscall_enter_audit(regs, syscall);

	return ret ? : syscall;
}

static __always_inline long
__syscall_enter_from_user_work(struct pt_regs *regs, long syscall)
{
	unsigned long work = READ_ONCE(current_thread_info()->syscall_work);

	if (work & SYSCALL_WORK_ENTER)
		syscall = syscall_trace_enter(regs, syscall, work);

	return syscall;
}

long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)
{
	return __syscall_enter_from_user_work(regs, syscall);
}

noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
{
	long ret;

	__enter_from_user_mode(regs);

	instrumentation_begin();
	local_irq_enable();
	ret = __syscall_enter_from_user_work(regs, syscall);
	instrumentation_end();

	return ret;
}

noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)
{
	__enter_from_user_mode(regs);
	instrumentation_begin();
	local_irq_enable();
	instrumentation_end();
}

/* See comment for exit_to_user_mode() in entry-common.h */
static __always_inline void __exit_to_user_mode(void)
{
	instrumentation_begin();
	trace_hardirqs_on_prepare();
	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
	instrumentation_end();

	user_enter_irqoff();
	arch_exit_to_user_mode();
	lockdep_hardirqs_on(CALLER_ADDR0);
}

void noinstr exit_to_user_mode(void)
{
	__exit_to_user_mode();
}

/* Workaround to allow gradual conversion of architecture code */
void __weak arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) { }

static void handle_signal_work(struct pt_regs *regs, unsigned long ti_work)
{
	if (ti_work & _TIF_NOTIFY_SIGNAL)
		tracehook_notify_signal();

	arch_do_signal_or_restart(regs, ti_work & _TIF_SIGPENDING);
}

static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
					    unsigned long ti_work)
{
	/*
	 * Before returning to user space ensure that all pending work
	 * items have been completed.
	 */
	while (ti_work & EXIT_TO_USER_MODE_WORK) {

		local_irq_enable_exit_to_user(ti_work);

		if (ti_work & _TIF_NEED_RESCHED)
			schedule();

		if (ti_work & _TIF_UPROBE)
			uprobe_notify_resume(regs);

		if (ti_work & _TIF_PATCH_PENDING)
			klp_update_patch_state(current);

		if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
			handle_signal_work(regs, ti_work);

		if (ti_work & _TIF_NOTIFY_RESUME) {
			tracehook_notify_resume(regs);
			rseq_handle_notify_resume(NULL, regs);
		}

		/* Architecture specific TIF work */
		arch_exit_to_user_mode_work(regs, ti_work);

		/*
		 * Disable interrupts and reevaluate the work flags as they
		 * might have changed while interrupts and preemption was
		 * enabled above.
		 */
		local_irq_disable_exit_to_user();

		/* Check if any of the above work has queued a deferred wakeup */
		tick_nohz_user_enter_prepare();

		ti_work = READ_ONCE(current_thread_info()->flags);
	}

	/* Return the latest work state for arch_exit_to_user_mode() */
	return ti_work;
}

static void exit_to_user_mode_prepare(struct pt_regs *regs)
{
	unsigned long ti_work = READ_ONCE(current_thread_info()->flags);

	lockdep_assert_irqs_disabled();

	/* Flush pending rcuog wakeup before the last need_resched() check */
	tick_nohz_user_enter_prepare();

	if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
		ti_work = exit_to_user_mode_loop(regs, ti_work);

	arch_exit_to_user_mode_prepare(regs, ti_work);

	/* Ensure that the address limit is intact and no locks are held */
	addr_limit_user_check();
	kmap_assert_nomap();
	lockdep_assert_irqs_disabled();
	lockdep_sys_exit();
}

/*
 * If SYSCALL_EMU is set, then the only reason to report is when
 * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP).  This syscall
 * instruction has been already reported in syscall_enter_from_user_mode().
 */
static inline bool report_single_step(unsigned long work)
{
	if (work & SYSCALL_WORK_SYSCALL_EMU)
		return false;

	return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP;
}

static void syscall_exit_work(struct pt_regs *regs, unsigned long work)
{
	bool step;

	/*
	 * If the syscall was rolled back due to syscall user dispatching,
	 * then the tracers below are not invoked for the same reason as
	 * the entry side was not invoked in syscall_trace_enter(): The ABI
	 * of these syscalls is unknown.
	 */
	if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) {
		if (unlikely(current->syscall_dispatch.on_dispatch)) {
			current->syscall_dispatch.on_dispatch = false;
			return;
		}
	}

	audit_syscall_exit(regs);

	if (work & SYSCALL_WORK_SYSCALL_TRACEPOINT)
		trace_sys_exit(regs, syscall_get_return_value(current, regs));

	step = report_single_step(work);
	if (step || work & SYSCALL_WORK_SYSCALL_TRACE)
		arch_syscall_exit_tracehook(regs, step);
}

/*
 * Syscall specific exit to user mode preparation. Runs with interrupts
 * enabled.
 */
static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
{
	unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
	unsigned long nr = syscall_get_nr(current, regs);

	CT_WARN_ON(ct_state() != CONTEXT_KERNEL);

	if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
		if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
			local_irq_enable();
	}

	rseq_syscall(regs);

	/*
	 * Do one-time syscall specific work. If these work items are
	 * enabled, we want to run them exactly once per syscall exit with
	 * interrupts enabled.
	 */
	if (unlikely(work & SYSCALL_WORK_EXIT))
		syscall_exit_work(regs, work);
}

static __always_inline void __syscall_exit_to_user_mode_work(struct pt_regs *regs)
{
	syscall_exit_to_user_mode_prepare(regs);
	local_irq_disable_exit_to_user();
	exit_to_user_mode_prepare(regs);
}

void syscall_exit_to_user_mode_work(struct pt_regs *regs)
{
	__syscall_exit_to_user_mode_work(regs);
}

__visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)
{
	instrumentation_begin();
	__syscall_exit_to_user_mode_work(regs);
	instrumentation_end();
	__exit_to_user_mode();
}

noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
{
	__enter_from_user_mode(regs);
}

noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)
{
	instrumentation_begin();
	exit_to_user_mode_prepare(regs);
	instrumentation_end();
	__exit_to_user_mode();
}

noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
{
	irqentry_state_t ret = {
		.exit_rcu = false,
	};

	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);
		return ret;
	}

	/*
	 * If this entry hit the idle task invoke rcu_irq_enter() whether
	 * RCU is watching or not.
	 *
	 * Interrupts can nest when the first interrupt invokes softirq
	 * processing on return which enables interrupts.
	 *
	 * Scheduler ticks in the idle task can mark quiescent state and
	 * terminate a grace period, if and only if the timer interrupt is
	 * not nested into another interrupt.
	 *
	 * Checking for rcu_is_watching() here would prevent the nesting
	 * interrupt to invoke rcu_irq_enter(). If that nested interrupt is
	 * the tick then rcu_flavor_sched_clock_irq() would wrongfully
	 * assume that it is the first interrupt and eventually claim
	 * quiescent state and end grace periods prematurely.
	 *
	 * Unconditionally invoke rcu_irq_enter() so RCU state stays
	 * consistent.
	 *
	 * TINY_RCU does not support EQS, so let the compiler eliminate
	 * this part when enabled.
	 */
	if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
		/*
		 * If RCU is not watching then the same careful
		 * sequence vs. lockdep and tracing is required
		 * as in irqentry_enter_from_user_mode().
		 */
		lockdep_hardirqs_off(CALLER_ADDR0);
		rcu_irq_enter();
		instrumentation_begin();
		trace_hardirqs_off_finish();
		instrumentation_end();

		ret.exit_rcu = true;
		return ret;
	}

	/*
	 * If RCU is watching then RCU only wants to check whether it needs
	 * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick()
	 * already contains a warning when RCU is not watching, so no point
	 * in having another one here.
	 */
	lockdep_hardirqs_off(CALLER_ADDR0);
	instrumentation_begin();
	rcu_irq_enter_check_tick();
	trace_hardirqs_off_finish();
	instrumentation_end();

	return ret;
}

void irqentry_exit_cond_resched(void)
{
	if (!preempt_count()) {
		/* Sanity check RCU and thread stack */
		rcu_irq_exit_check_preempt();
		if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
			WARN_ON_ONCE(!on_thread_stack());
		if (need_resched())
			preempt_schedule_irq();
	}
}
#ifdef CONFIG_PREEMPT_DYNAMIC
DEFINE_STATIC_CALL(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
#endif

noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
{
	lockdep_assert_irqs_disabled();

	/* Check whether this returns to user mode */
	if (user_mode(regs)) {
		irqentry_exit_to_user_mode(regs);
	} else if (!regs_irqs_disabled(regs)) {
		/*
		 * If RCU was not watching on entry this needs to be done
		 * carefully and needs the same ordering of lockdep/tracing
		 * and RCU as the return to user mode path.
		 */
		if (state.exit_rcu) {
			instrumentation_begin();
			/* Tell the tracer that IRET will enable interrupts */
			trace_hardirqs_on_prepare();
			lockdep_hardirqs_on_prepare(CALLER_ADDR0);
			instrumentation_end();
			rcu_irq_exit();
			lockdep_hardirqs_on(CALLER_ADDR0);
			return;
		}

		instrumentation_begin();
		if (IS_ENABLED(CONFIG_PREEMPTION)) {
#ifdef CONFIG_PREEMPT_DYNAMIC
			static_call(irqentry_exit_cond_resched)();
#else
			irqentry_exit_cond_resched();
#endif
		}
		/* Covers both tracing and lockdep */
		trace_hardirqs_on();
		instrumentation_end();
	} else {
		/*
		 * IRQ flags state is correct already. Just tell RCU if it
		 * was not watching on entry.
		 */
		if (state.exit_rcu)
			rcu_irq_exit();
	}
}

irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)
{
	irqentry_state_t irq_state;

	irq_state.lockdep = lockdep_hardirqs_enabled();

	__nmi_enter();
	lockdep_hardirqs_off(CALLER_ADDR0);
	lockdep_hardirq_enter();
	rcu_nmi_enter();

	instrumentation_begin();
	trace_hardirqs_off_finish();
	ftrace_nmi_enter();
	instrumentation_end();

	return irq_state;
}

void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)
{
	instrumentation_begin();
	ftrace_nmi_exit();
	if (irq_state.lockdep) {
		trace_hardirqs_on_prepare();
		lockdep_hardirqs_on_prepare(CALLER_ADDR0);
	}
	instrumentation_end();

	rcu_nmi_exit();
	lockdep_hardirq_exit();
	if (irq_state.lockdep)
		lockdep_hardirqs_on(CALLER_ADDR0);
	__nmi_exit();
}
Commit	Line	Data
142781e1 TG	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <linux/context_tracking.h>
	4	#include <linux/entry-common.h>
5fbda3ec	5	#include <linux/highmem.h>
a9f3a74a TG	6	#include <linux/livepatch.h>
a9f3a74a TG	7	#include <linux/audit.h>
f268c373	8	#include <linux/tick.h>
142781e1	9
11894468 GKB	10	#include "common.h"
11894468 GKB	11
142781e1 TG	12	#define CREATE_TRACE_POINTS
	13	#include <trace/events/syscalls.h>
	14
96e2fbcc	15	/* See comment for enter_from_user_mode() in entry-common.h */
6666bb71	16	static __always_inline void __enter_from_user_mode(struct pt_regs *regs)
142781e1 TG	17	{
	18	arch_check_user_regs(regs);
	19	lockdep_hardirqs_off(CALLER_ADDR0);
	20
	21	CT_WARN_ON(ct_state() != CONTEXT_USER);
	22	user_exit_irqoff();
	23
	24	instrumentation_begin();
	25	trace_hardirqs_off_finish();
	26	instrumentation_end();
	27	}
	28
96e2fbcc SS	29	void noinstr enter_from_user_mode(struct pt_regs *regs)
	30	{
	31	__enter_from_user_mode(regs);
	32	}
	33
142781e1 TG	34	static inline void syscall_enter_audit(struct pt_regs *regs, long syscall)
	35	{
	36	if (unlikely(audit_context())) {
	37	unsigned long args[6];
	38
	39	syscall_get_arguments(current, regs, args);
	40	audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]);
	41	}
	42	}
	43
	44	static long syscall_trace_enter(struct pt_regs *regs, long syscall,
29915524	45	unsigned long work)
142781e1 TG	46	{
	47	long ret = 0;
	48
11894468 GKB	49	/*
	50	* Handle Syscall User Dispatch. This must comes first, since
	51	* the ABI here can be something that doesn't make sense for
	52	* other syscall_work features.
	53	*/
	54	if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) {
	55	if (syscall_user_dispatch(regs))
	56	return -1L;
	57	}
	58
142781e1	59	/* Handle ptrace */
64eb35f7	60	if (work & (SYSCALL_WORK_SYSCALL_TRACE \| SYSCALL_WORK_SYSCALL_EMU)) {
142781e1	61	ret = arch_syscall_enter_tracehook(regs);
64eb35f7	62	if (ret \|\| (work & SYSCALL_WORK_SYSCALL_EMU))
142781e1 TG	63	return -1L;
	64	}
	65
	66	/* Do seccomp after ptrace, to catch any tracer changes. */
23d67a54	67	if (work & SYSCALL_WORK_SECCOMP) {
142781e1 TG	68	ret = __secure_computing(NULL);
	69	if (ret == -1L)
	70	return ret;
	71	}
	72
b6ec4134 KC	73	/* Either of the above might have changed the syscall number */
	74	syscall = syscall_get_nr(current, regs);
	75
524666cb	76	if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT))
142781e1 TG	77	trace_sys_enter(regs, syscall);
	78
	79	syscall_enter_audit(regs, syscall);
	80
	81	return ret ? : syscall;
	82	}
	83
4facb95b TG	84	static __always_inline long
4facb95b TG	85	__syscall_enter_from_user_work(struct pt_regs *regs, long syscall)
142781e1	86	{
b86678cf	87	unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
142781e1	88
29915524 GKB	89	if (work & SYSCALL_WORK_ENTER)
29915524 GKB	90	syscall = syscall_trace_enter(regs, syscall, work);
142781e1 TG	91
	92	return syscall;
	93	}
	94
4facb95b TG	95	long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall)
	96	{
	97	return __syscall_enter_from_user_work(regs, syscall);
	98	}
	99
	100	noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall)
	101	{
	102	long ret;
	103
6666bb71	104	__enter_from_user_mode(regs);
4facb95b TG	105
	106	instrumentation_begin();
	107	local_irq_enable();
	108	ret = __syscall_enter_from_user_work(regs, syscall);
	109	instrumentation_end();
	110
	111	return ret;
	112	}
	113
	114	noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs)
	115	{
6666bb71	116	__enter_from_user_mode(regs);
4facb95b TG	117	instrumentation_begin();
	118	local_irq_enable();
	119	instrumentation_end();
	120	}
	121
310de1a6	122	/* See comment for exit_to_user_mode() in entry-common.h */
bb793562	123	static __always_inline void __exit_to_user_mode(void)
a9f3a74a TG	124	{
	125	instrumentation_begin();
	126	trace_hardirqs_on_prepare();
	127	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
	128	instrumentation_end();
	129
	130	user_enter_irqoff();
	131	arch_exit_to_user_mode();
	132	lockdep_hardirqs_on(CALLER_ADDR0);
	133	}
	134
310de1a6 SS	135	void noinstr exit_to_user_mode(void)
	136	{
	137	__exit_to_user_mode();
	138	}
	139
a9f3a74a	140	/* Workaround to allow gradual conversion of architecture code */
12db8b69 JA	141	void __weak arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) { }
	142
	143	static void handle_signal_work(struct pt_regs *regs, unsigned long ti_work)
	144	{
	145	if (ti_work & _TIF_NOTIFY_SIGNAL)
	146	tracehook_notify_signal();
	147
	148	arch_do_signal_or_restart(regs, ti_work & _TIF_SIGPENDING);
	149	}
a9f3a74a TG	150
	151	static unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
	152	unsigned long ti_work)
	153	{
	154	/*
	155	* Before returning to user space ensure that all pending work
	156	* items have been completed.
	157	*/
	158	while (ti_work & EXIT_TO_USER_MODE_WORK) {
	159
	160	local_irq_enable_exit_to_user(ti_work);
	161
	162	if (ti_work & _TIF_NEED_RESCHED)
	163	schedule();
	164
	165	if (ti_work & _TIF_UPROBE)
	166	uprobe_notify_resume(regs);
	167
	168	if (ti_work & _TIF_PATCH_PENDING)
	169	klp_update_patch_state(current);
	170
12db8b69 JA	171	if (ti_work & (_TIF_SIGPENDING \| _TIF_NOTIFY_SIGNAL))
12db8b69 JA	172	handle_signal_work(regs, ti_work);
a9f3a74a TG	173
a9f3a74a TG	174	if (ti_work & _TIF_NOTIFY_RESUME) {
a9f3a74a TG	175	tracehook_notify_resume(regs);
	176	rseq_handle_notify_resume(NULL, regs);
	177	}
	178
	179	/* Architecture specific TIF work */
	180	arch_exit_to_user_mode_work(regs, ti_work);
	181
	182	/*
	183	* Disable interrupts and reevaluate the work flags as they
	184	* might have changed while interrupts and preemption was
	185	* enabled above.
	186	*/
	187	local_irq_disable_exit_to_user();
47b8ff19 FW	188
47b8ff19 FW	189	/* Check if any of the above work has queued a deferred wakeup */
f268c373	190	tick_nohz_user_enter_prepare();
47b8ff19	191
a9f3a74a TG	192	ti_work = READ_ONCE(current_thread_info()->flags);
	193	}
	194
	195	/* Return the latest work state for arch_exit_to_user_mode() */
	196	return ti_work;
	197	}
	198
	199	static void exit_to_user_mode_prepare(struct pt_regs *regs)
	200	{
	201	unsigned long ti_work = READ_ONCE(current_thread_info()->flags);
	202
	203	lockdep_assert_irqs_disabled();
	204
47b8ff19	205	/* Flush pending rcuog wakeup before the last need_resched() check */
f268c373	206	tick_nohz_user_enter_prepare();
47b8ff19	207
a9f3a74a TG	208	if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK))
	209	ti_work = exit_to_user_mode_loop(regs, ti_work);
	210
	211	arch_exit_to_user_mode_prepare(regs, ti_work);
	212
	213	/* Ensure that the address limit is intact and no locks are held */
	214	addr_limit_user_check();
5fbda3ec	215	kmap_assert_nomap();
a9f3a74a TG	216	lockdep_assert_irqs_disabled();
	217	lockdep_sys_exit();
	218	}
	219
a9f3a74a	220	/*
64eb35f7	221	* If SYSCALL_EMU is set, then the only reason to report is when
6342adca	222	* SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall
900ffe39	223	* instruction has been already reported in syscall_enter_from_user_mode().
a9f3a74a	224	*/
64eb35f7	225	static inline bool report_single_step(unsigned long work)
a9f3a74a	226	{
41c1a06d	227	if (work & SYSCALL_WORK_SYSCALL_EMU)
64eb35f7 GKB	228	return false;
64eb35f7 GKB	229
6342adca	230	return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP;
a9f3a74a	231	}
29915524 GKB	232
29915524 GKB	233	static void syscall_exit_work(struct pt_regs *regs, unsigned long work)
a9f3a74a TG	234	{
	235	bool step;
	236
11894468 GKB	237	/*
	238	* If the syscall was rolled back due to syscall user dispatching,
	239	* then the tracers below are not invoked for the same reason as
	240	* the entry side was not invoked in syscall_trace_enter(): The ABI
	241	* of these syscalls is unknown.
	242	*/
	243	if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) {
	244	if (unlikely(current->syscall_dispatch.on_dispatch)) {
	245	current->syscall_dispatch.on_dispatch = false;
	246	return;
	247	}
	248	}
	249
a9f3a74a TG	250	audit_syscall_exit(regs);
a9f3a74a TG	251
524666cb	252	if (work & SYSCALL_WORK_SYSCALL_TRACEPOINT)
a9f3a74a TG	253	trace_sys_exit(regs, syscall_get_return_value(current, regs));
a9f3a74a TG	254
64eb35f7	255	step = report_single_step(work);
64c19ba2	256	if (step \|\| work & SYSCALL_WORK_SYSCALL_TRACE)
a9f3a74a TG	257	arch_syscall_exit_tracehook(regs, step);
	258	}
	259
	260	/*
	261	* Syscall specific exit to user mode preparation. Runs with interrupts
	262	* enabled.
	263	*/
	264	static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs)
	265	{
b86678cf	266	unsigned long work = READ_ONCE(current_thread_info()->syscall_work);
a9f3a74a TG	267	unsigned long nr = syscall_get_nr(current, regs);
	268
	269	CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
	270
	271	if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
	272	if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr))
	273	local_irq_enable();
	274	}
	275
	276	rseq_syscall(regs);
	277
	278	/*
	279	* Do one-time syscall specific work. If these work items are
	280	* enabled, we want to run them exactly once per syscall exit with
	281	* interrupts enabled.
	282	*/
29915524 GKB	283	if (unlikely(work & SYSCALL_WORK_EXIT))
29915524 GKB	284	syscall_exit_work(regs, work);
a9f3a74a TG	285	}
a9f3a74a TG	286
c6156e1d	287	static __always_inline void __syscall_exit_to_user_mode_work(struct pt_regs *regs)
a9f3a74a	288	{
a9f3a74a TG	289	syscall_exit_to_user_mode_prepare(regs);
	290	local_irq_disable_exit_to_user();
	291	exit_to_user_mode_prepare(regs);
c6156e1d SS	292	}
	293
	294	void syscall_exit_to_user_mode_work(struct pt_regs *regs)
	295	{
	296	__syscall_exit_to_user_mode_work(regs);
	297	}
	298
	299	__visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs)
	300	{
	301	instrumentation_begin();
	302	__syscall_exit_to_user_mode_work(regs);
a9f3a74a	303	instrumentation_end();
bb793562	304	__exit_to_user_mode();
a9f3a74a TG	305	}
a9f3a74a TG	306
142781e1 TG	307	noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs)
142781e1 TG	308	{
6666bb71	309	__enter_from_user_mode(regs);
142781e1	310	}
a9f3a74a TG	311
	312	noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs)
	313	{
	314	instrumentation_begin();
	315	exit_to_user_mode_prepare(regs);
	316	instrumentation_end();
bb793562	317	__exit_to_user_mode();
a9f3a74a	318	}
a5497bab	319
aadfc2f9	320	noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
a5497bab TG	321	{
	322	irqentry_state_t ret = {
	323	.exit_rcu = false,
	324	};
	325
	326	if (user_mode(regs)) {
	327	irqentry_enter_from_user_mode(regs);
	328	return ret;
	329	}
	330
	331	/*
	332	* If this entry hit the idle task invoke rcu_irq_enter() whether
	333	* RCU is watching or not.
	334	*
78a56e04	335	* Interrupts can nest when the first interrupt invokes softirq
a5497bab TG	336	* processing on return which enables interrupts.
	337	*
	338	* Scheduler ticks in the idle task can mark quiescent state and
	339	* terminate a grace period, if and only if the timer interrupt is
	340	* not nested into another interrupt.
	341	*
7f2a53c2	342	* Checking for rcu_is_watching() here would prevent the nesting
a5497bab TG	343	* interrupt to invoke rcu_irq_enter(). If that nested interrupt is
a5497bab TG	344	* the tick then rcu_flavor_sched_clock_irq() would wrongfully
97258ce9	345	* assume that it is the first interrupt and eventually claim
78a56e04	346	* quiescent state and end grace periods prematurely.
a5497bab TG	347	*
	348	* Unconditionally invoke rcu_irq_enter() so RCU state stays
	349	* consistent.
	350	*
	351	* TINY_RCU does not support EQS, so let the compiler eliminate
	352	* this part when enabled.
	353	*/
	354	if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
	355	/*
	356	* If RCU is not watching then the same careful
	357	* sequence vs. lockdep and tracing is required
45ff5105	358	* as in irqentry_enter_from_user_mode().
a5497bab TG	359	*/
	360	lockdep_hardirqs_off(CALLER_ADDR0);
	361	rcu_irq_enter();
	362	instrumentation_begin();
	363	trace_hardirqs_off_finish();
	364	instrumentation_end();
	365
	366	ret.exit_rcu = true;
	367	return ret;
	368	}
	369
	370	/*
	371	* If RCU is watching then RCU only wants to check whether it needs
	372	* to restart the tick in NOHZ mode. rcu_irq_enter_check_tick()
	373	* already contains a warning when RCU is not watching, so no point
	374	* in having another one here.
	375	*/
9d820f68	376	lockdep_hardirqs_off(CALLER_ADDR0);
a5497bab TG	377	instrumentation_begin();
a5497bab TG	378	rcu_irq_enter_check_tick();
9d820f68	379	trace_hardirqs_off_finish();
a5497bab TG	380	instrumentation_end();
	381
	382	return ret;
	383	}
	384
	385	void irqentry_exit_cond_resched(void)
	386	{
	387	if (!preempt_count()) {
	388	/* Sanity check RCU and thread stack */
	389	rcu_irq_exit_check_preempt();
	390	if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
	391	WARN_ON_ONCE(!on_thread_stack());
	392	if (need_resched())
	393	preempt_schedule_irq();
	394	}
	395	}
40607ee9 PZI	396	#ifdef CONFIG_PREEMPT_DYNAMIC
	397	DEFINE_STATIC_CALL(irqentry_exit_cond_resched, irqentry_exit_cond_resched);
	398	#endif
a5497bab	399
aadfc2f9	400	noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
a5497bab TG	401	{
	402	lockdep_assert_irqs_disabled();
	403
	404	/* Check whether this returns to user mode */
	405	if (user_mode(regs)) {
	406	irqentry_exit_to_user_mode(regs);
	407	} else if (!regs_irqs_disabled(regs)) {
	408	/*
	409	* If RCU was not watching on entry this needs to be done
	410	* carefully and needs the same ordering of lockdep/tracing
	411	* and RCU as the return to user mode path.
	412	*/
	413	if (state.exit_rcu) {
	414	instrumentation_begin();
	415	/* Tell the tracer that IRET will enable interrupts */
	416	trace_hardirqs_on_prepare();
	417	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
	418	instrumentation_end();
	419	rcu_irq_exit();
	420	lockdep_hardirqs_on(CALLER_ADDR0);
	421	return;
	422	}
	423
	424	instrumentation_begin();
40607ee9	425	if (IS_ENABLED(CONFIG_PREEMPTION)) {
0c89d87d	426	#ifdef CONFIG_PREEMPT_DYNAMIC
40607ee9 PZI	427	static_call(irqentry_exit_cond_resched)();
40607ee9 PZI	428	#else
a5497bab	429	irqentry_exit_cond_resched();
40607ee9 PZI	430	#endif
40607ee9 PZI	431	}
a5497bab TG	432	/* Covers both tracing and lockdep */
	433	trace_hardirqs_on();
	434	instrumentation_end();
	435	} else {
	436	/*
	437	* IRQ flags state is correct already. Just tell RCU if it
	438	* was not watching on entry.
	439	*/
	440	if (state.exit_rcu)
	441	rcu_irq_exit();
	442	}
	443	}
b6be002b TG	444
	445	irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)
	446	{
	447	irqentry_state_t irq_state;
	448
	449	irq_state.lockdep = lockdep_hardirqs_enabled();
	450
	451	__nmi_enter();
	452	lockdep_hardirqs_off(CALLER_ADDR0);
	453	lockdep_hardirq_enter();
	454	rcu_nmi_enter();
	455
	456	instrumentation_begin();
	457	trace_hardirqs_off_finish();
	458	ftrace_nmi_enter();
	459	instrumentation_end();
	460
	461	return irq_state;
	462	}
	463
	464	void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)
	465	{
	466	instrumentation_begin();
	467	ftrace_nmi_exit();
	468	if (irq_state.lockdep) {
	469	trace_hardirqs_on_prepare();
	470	lockdep_hardirqs_on_prepare(CALLER_ADDR0);
	471	}
	472	instrumentation_end();
	473
	474	rcu_nmi_exit();
	475	lockdep_hardirq_exit();
	476	if (irq_state.lockdep)
	477	lockdep_hardirqs_on(CALLER_ADDR0);
	478	__nmi_exit();
	479	}