Commit | Line | Data |
---|---|---|
142781e1 TG |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | #include <linux/context_tracking.h> | |
4 | #include <linux/entry-common.h> | |
a9f3a74a TG |
5 | #include <linux/livepatch.h> |
6 | #include <linux/audit.h> | |
142781e1 TG |
7 | |
8 | #define CREATE_TRACE_POINTS | |
9 | #include <trace/events/syscalls.h> | |
10 | ||
11 | /** | |
12 | * enter_from_user_mode - Establish state when coming from user mode | |
13 | * | |
14 | * Syscall/interrupt entry disables interrupts, but user mode is traced as | |
15 | * interrupts enabled. Also with NO_HZ_FULL RCU might be idle. | |
16 | * | |
17 | * 1) Tell lockdep that interrupts are disabled | |
18 | * 2) Invoke context tracking if enabled to reactivate RCU | |
19 | * 3) Trace interrupts off state | |
20 | */ | |
21 | static __always_inline void enter_from_user_mode(struct pt_regs *regs) | |
22 | { | |
23 | arch_check_user_regs(regs); | |
24 | lockdep_hardirqs_off(CALLER_ADDR0); | |
25 | ||
26 | CT_WARN_ON(ct_state() != CONTEXT_USER); | |
27 | user_exit_irqoff(); | |
28 | ||
29 | instrumentation_begin(); | |
30 | trace_hardirqs_off_finish(); | |
31 | instrumentation_end(); | |
32 | } | |
33 | ||
34 | static inline void syscall_enter_audit(struct pt_regs *regs, long syscall) | |
35 | { | |
36 | if (unlikely(audit_context())) { | |
37 | unsigned long args[6]; | |
38 | ||
39 | syscall_get_arguments(current, regs, args); | |
40 | audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]); | |
41 | } | |
42 | } | |
43 | ||
44 | static long syscall_trace_enter(struct pt_regs *regs, long syscall, | |
45 | unsigned long ti_work) | |
46 | { | |
47 | long ret = 0; | |
48 | ||
49 | /* Handle ptrace */ | |
50 | if (ti_work & (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU)) { | |
51 | ret = arch_syscall_enter_tracehook(regs); | |
52 | if (ret || (ti_work & _TIF_SYSCALL_EMU)) | |
53 | return -1L; | |
54 | } | |
55 | ||
56 | /* Do seccomp after ptrace, to catch any tracer changes. */ | |
57 | if (ti_work & _TIF_SECCOMP) { | |
58 | ret = __secure_computing(NULL); | |
59 | if (ret == -1L) | |
60 | return ret; | |
61 | } | |
62 | ||
63 | if (unlikely(ti_work & _TIF_SYSCALL_TRACEPOINT)) | |
64 | trace_sys_enter(regs, syscall); | |
65 | ||
66 | syscall_enter_audit(regs, syscall); | |
67 | ||
68 | return ret ? : syscall; | |
69 | } | |
70 | ||
71 | noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall) | |
72 | { | |
73 | unsigned long ti_work; | |
74 | ||
75 | enter_from_user_mode(regs); | |
76 | instrumentation_begin(); | |
77 | ||
78 | local_irq_enable(); | |
79 | ti_work = READ_ONCE(current_thread_info()->flags); | |
80 | if (ti_work & SYSCALL_ENTER_WORK) | |
81 | syscall = syscall_trace_enter(regs, syscall, ti_work); | |
82 | instrumentation_end(); | |
83 | ||
84 | return syscall; | |
85 | } | |
86 | ||
a9f3a74a TG |
87 | /** |
88 | * exit_to_user_mode - Fixup state when exiting to user mode | |
89 | * | |
90 | * Syscall/interupt exit enables interrupts, but the kernel state is | |
91 | * interrupts disabled when this is invoked. Also tell RCU about it. | |
92 | * | |
93 | * 1) Trace interrupts on state | |
94 | * 2) Invoke context tracking if enabled to adjust RCU state | |
95 | * 3) Invoke architecture specific last minute exit code, e.g. speculation | |
96 | * mitigations, etc. | |
97 | * 4) Tell lockdep that interrupts are enabled | |
98 | */ | |
99 | static __always_inline void exit_to_user_mode(void) | |
100 | { | |
101 | instrumentation_begin(); | |
102 | trace_hardirqs_on_prepare(); | |
103 | lockdep_hardirqs_on_prepare(CALLER_ADDR0); | |
104 | instrumentation_end(); | |
105 | ||
106 | user_enter_irqoff(); | |
107 | arch_exit_to_user_mode(); | |
108 | lockdep_hardirqs_on(CALLER_ADDR0); | |
109 | } | |
110 | ||
111 | /* Workaround to allow gradual conversion of architecture code */ | |
112 | void __weak arch_do_signal(struct pt_regs *regs) { } | |
113 | ||
114 | static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, | |
115 | unsigned long ti_work) | |
116 | { | |
117 | /* | |
118 | * Before returning to user space ensure that all pending work | |
119 | * items have been completed. | |
120 | */ | |
121 | while (ti_work & EXIT_TO_USER_MODE_WORK) { | |
122 | ||
123 | local_irq_enable_exit_to_user(ti_work); | |
124 | ||
125 | if (ti_work & _TIF_NEED_RESCHED) | |
126 | schedule(); | |
127 | ||
128 | if (ti_work & _TIF_UPROBE) | |
129 | uprobe_notify_resume(regs); | |
130 | ||
131 | if (ti_work & _TIF_PATCH_PENDING) | |
132 | klp_update_patch_state(current); | |
133 | ||
134 | if (ti_work & _TIF_SIGPENDING) | |
135 | arch_do_signal(regs); | |
136 | ||
137 | if (ti_work & _TIF_NOTIFY_RESUME) { | |
138 | clear_thread_flag(TIF_NOTIFY_RESUME); | |
139 | tracehook_notify_resume(regs); | |
140 | rseq_handle_notify_resume(NULL, regs); | |
141 | } | |
142 | ||
143 | /* Architecture specific TIF work */ | |
144 | arch_exit_to_user_mode_work(regs, ti_work); | |
145 | ||
146 | /* | |
147 | * Disable interrupts and reevaluate the work flags as they | |
148 | * might have changed while interrupts and preemption was | |
149 | * enabled above. | |
150 | */ | |
151 | local_irq_disable_exit_to_user(); | |
152 | ti_work = READ_ONCE(current_thread_info()->flags); | |
153 | } | |
154 | ||
155 | /* Return the latest work state for arch_exit_to_user_mode() */ | |
156 | return ti_work; | |
157 | } | |
158 | ||
159 | static void exit_to_user_mode_prepare(struct pt_regs *regs) | |
160 | { | |
161 | unsigned long ti_work = READ_ONCE(current_thread_info()->flags); | |
162 | ||
163 | lockdep_assert_irqs_disabled(); | |
164 | ||
165 | if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) | |
166 | ti_work = exit_to_user_mode_loop(regs, ti_work); | |
167 | ||
168 | arch_exit_to_user_mode_prepare(regs, ti_work); | |
169 | ||
170 | /* Ensure that the address limit is intact and no locks are held */ | |
171 | addr_limit_user_check(); | |
172 | lockdep_assert_irqs_disabled(); | |
173 | lockdep_sys_exit(); | |
174 | } | |
175 | ||
176 | #ifndef _TIF_SINGLESTEP | |
177 | static inline bool report_single_step(unsigned long ti_work) | |
178 | { | |
179 | return false; | |
180 | } | |
181 | #else | |
182 | /* | |
183 | * If TIF_SYSCALL_EMU is set, then the only reason to report is when | |
184 | * TIF_SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall | |
185 | * instruction has been already reported in syscall_enter_from_usermode(). | |
186 | */ | |
187 | #define SYSEMU_STEP (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU) | |
188 | ||
189 | static inline bool report_single_step(unsigned long ti_work) | |
190 | { | |
191 | return (ti_work & SYSEMU_STEP) == _TIF_SINGLESTEP; | |
192 | } | |
193 | #endif | |
194 | ||
195 | static void syscall_exit_work(struct pt_regs *regs, unsigned long ti_work) | |
196 | { | |
197 | bool step; | |
198 | ||
199 | audit_syscall_exit(regs); | |
200 | ||
201 | if (ti_work & _TIF_SYSCALL_TRACEPOINT) | |
202 | trace_sys_exit(regs, syscall_get_return_value(current, regs)); | |
203 | ||
204 | step = report_single_step(ti_work); | |
205 | if (step || ti_work & _TIF_SYSCALL_TRACE) | |
206 | arch_syscall_exit_tracehook(regs, step); | |
207 | } | |
208 | ||
209 | /* | |
210 | * Syscall specific exit to user mode preparation. Runs with interrupts | |
211 | * enabled. | |
212 | */ | |
213 | static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs) | |
214 | { | |
215 | u32 cached_flags = READ_ONCE(current_thread_info()->flags); | |
216 | unsigned long nr = syscall_get_nr(current, regs); | |
217 | ||
218 | CT_WARN_ON(ct_state() != CONTEXT_KERNEL); | |
219 | ||
220 | if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { | |
221 | if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr)) | |
222 | local_irq_enable(); | |
223 | } | |
224 | ||
225 | rseq_syscall(regs); | |
226 | ||
227 | /* | |
228 | * Do one-time syscall specific work. If these work items are | |
229 | * enabled, we want to run them exactly once per syscall exit with | |
230 | * interrupts enabled. | |
231 | */ | |
232 | if (unlikely(cached_flags & SYSCALL_EXIT_WORK)) | |
233 | syscall_exit_work(regs, cached_flags); | |
234 | } | |
235 | ||
236 | __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs) | |
237 | { | |
238 | instrumentation_begin(); | |
239 | syscall_exit_to_user_mode_prepare(regs); | |
240 | local_irq_disable_exit_to_user(); | |
241 | exit_to_user_mode_prepare(regs); | |
242 | instrumentation_end(); | |
243 | exit_to_user_mode(); | |
244 | } | |
245 | ||
142781e1 TG |
246 | noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs) |
247 | { | |
248 | enter_from_user_mode(regs); | |
249 | } | |
a9f3a74a TG |
250 | |
251 | noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs) | |
252 | { | |
253 | instrumentation_begin(); | |
254 | exit_to_user_mode_prepare(regs); | |
255 | instrumentation_end(); | |
256 | exit_to_user_mode(); | |
257 | } | |
a5497bab TG |
258 | |
259 | irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs) | |
260 | { | |
261 | irqentry_state_t ret = { | |
262 | .exit_rcu = false, | |
263 | }; | |
264 | ||
265 | if (user_mode(regs)) { | |
266 | irqentry_enter_from_user_mode(regs); | |
267 | return ret; | |
268 | } | |
269 | ||
270 | /* | |
271 | * If this entry hit the idle task invoke rcu_irq_enter() whether | |
272 | * RCU is watching or not. | |
273 | * | |
274 | * Interupts can nest when the first interrupt invokes softirq | |
275 | * processing on return which enables interrupts. | |
276 | * | |
277 | * Scheduler ticks in the idle task can mark quiescent state and | |
278 | * terminate a grace period, if and only if the timer interrupt is | |
279 | * not nested into another interrupt. | |
280 | * | |
281 | * Checking for __rcu_is_watching() here would prevent the nesting | |
282 | * interrupt to invoke rcu_irq_enter(). If that nested interrupt is | |
283 | * the tick then rcu_flavor_sched_clock_irq() would wrongfully | |
284 | * assume that it is the first interupt and eventually claim | |
285 | * quiescient state and end grace periods prematurely. | |
286 | * | |
287 | * Unconditionally invoke rcu_irq_enter() so RCU state stays | |
288 | * consistent. | |
289 | * | |
290 | * TINY_RCU does not support EQS, so let the compiler eliminate | |
291 | * this part when enabled. | |
292 | */ | |
293 | if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { | |
294 | /* | |
295 | * If RCU is not watching then the same careful | |
296 | * sequence vs. lockdep and tracing is required | |
297 | * as in irq_enter_from_user_mode(). | |
298 | */ | |
299 | lockdep_hardirqs_off(CALLER_ADDR0); | |
300 | rcu_irq_enter(); | |
301 | instrumentation_begin(); | |
302 | trace_hardirqs_off_finish(); | |
303 | instrumentation_end(); | |
304 | ||
305 | ret.exit_rcu = true; | |
306 | return ret; | |
307 | } | |
308 | ||
309 | /* | |
310 | * If RCU is watching then RCU only wants to check whether it needs | |
311 | * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick() | |
312 | * already contains a warning when RCU is not watching, so no point | |
313 | * in having another one here. | |
314 | */ | |
315 | instrumentation_begin(); | |
316 | rcu_irq_enter_check_tick(); | |
317 | /* Use the combo lockdep/tracing function */ | |
318 | trace_hardirqs_off(); | |
319 | instrumentation_end(); | |
320 | ||
321 | return ret; | |
322 | } | |
323 | ||
324 | void irqentry_exit_cond_resched(void) | |
325 | { | |
326 | if (!preempt_count()) { | |
327 | /* Sanity check RCU and thread stack */ | |
328 | rcu_irq_exit_check_preempt(); | |
329 | if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) | |
330 | WARN_ON_ONCE(!on_thread_stack()); | |
331 | if (need_resched()) | |
332 | preempt_schedule_irq(); | |
333 | } | |
334 | } | |
335 | ||
336 | void noinstr irqentry_exit(struct pt_regs *regs, irqentry_state_t state) | |
337 | { | |
338 | lockdep_assert_irqs_disabled(); | |
339 | ||
340 | /* Check whether this returns to user mode */ | |
341 | if (user_mode(regs)) { | |
342 | irqentry_exit_to_user_mode(regs); | |
343 | } else if (!regs_irqs_disabled(regs)) { | |
344 | /* | |
345 | * If RCU was not watching on entry this needs to be done | |
346 | * carefully and needs the same ordering of lockdep/tracing | |
347 | * and RCU as the return to user mode path. | |
348 | */ | |
349 | if (state.exit_rcu) { | |
350 | instrumentation_begin(); | |
351 | /* Tell the tracer that IRET will enable interrupts */ | |
352 | trace_hardirqs_on_prepare(); | |
353 | lockdep_hardirqs_on_prepare(CALLER_ADDR0); | |
354 | instrumentation_end(); | |
355 | rcu_irq_exit(); | |
356 | lockdep_hardirqs_on(CALLER_ADDR0); | |
357 | return; | |
358 | } | |
359 | ||
360 | instrumentation_begin(); | |
361 | if (IS_ENABLED(CONFIG_PREEMPTION)) | |
362 | irqentry_exit_cond_resched(); | |
363 | /* Covers both tracing and lockdep */ | |
364 | trace_hardirqs_on(); | |
365 | instrumentation_end(); | |
366 | } else { | |
367 | /* | |
368 | * IRQ flags state is correct already. Just tell RCU if it | |
369 | * was not watching on entry. | |
370 | */ | |
371 | if (state.exit_rcu) | |
372 | rcu_irq_exit(); | |
373 | } | |
374 | } |