Commit | Line | Data |
---|---|---|
4141c857 MR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
f37099b6 MR |
3 | #include <linux/compiler.h> |
4 | #include <linux/context_tracking.h> | |
4141c857 MR |
5 | #include <linux/errno.h> |
6 | #include <linux/nospec.h> | |
7 | #include <linux/ptrace.h> | |
8 | #include <linux/syscalls.h> | |
9 | ||
f37099b6 | 10 | #include <asm/daifflags.h> |
969f5ea6 | 11 | #include <asm/debug-monitors.h> |
3b714275 | 12 | #include <asm/fpsimd.h> |
4141c857 | 13 | #include <asm/syscall.h> |
f37099b6 | 14 | #include <asm/thread_info.h> |
3b714275 | 15 | #include <asm/unistd.h> |
4141c857 | 16 | |
53290432 | 17 | long compat_arm_syscall(struct pt_regs *regs, int scno); |
4378a7d4 MR |
18 | long sys_ni_syscall(void); |
19 | ||
53290432 | 20 | static long do_ni_syscall(struct pt_regs *regs, int scno) |
4141c857 MR |
21 | { |
22 | #ifdef CONFIG_COMPAT | |
23 | long ret; | |
24 | if (is_compat_task()) { | |
53290432 | 25 | ret = compat_arm_syscall(regs, scno); |
4141c857 MR |
26 | if (ret != -ENOSYS) |
27 | return ret; | |
28 | } | |
29 | #endif | |
30 | ||
31 | return sys_ni_syscall(); | |
32 | } | |
33 | ||
34 | static long __invoke_syscall(struct pt_regs *regs, syscall_fn_t syscall_fn) | |
35 | { | |
4378a7d4 | 36 | return syscall_fn(regs); |
4141c857 MR |
37 | } |
38 | ||
f37099b6 MR |
39 | static void invoke_syscall(struct pt_regs *regs, unsigned int scno, |
40 | unsigned int sc_nr, | |
41 | const syscall_fn_t syscall_table[]) | |
4141c857 MR |
42 | { |
43 | long ret; | |
44 | ||
45 | if (scno < sc_nr) { | |
46 | syscall_fn_t syscall_fn; | |
47 | syscall_fn = syscall_table[array_index_nospec(scno, sc_nr)]; | |
48 | ret = __invoke_syscall(regs, syscall_fn); | |
49 | } else { | |
53290432 | 50 | ret = do_ni_syscall(regs, scno); |
4141c857 MR |
51 | } |
52 | ||
53 | regs->regs[0] = ret; | |
54 | } | |
f37099b6 MR |
55 | |
56 | static inline bool has_syscall_work(unsigned long flags) | |
57 | { | |
58 | return unlikely(flags & _TIF_SYSCALL_WORK); | |
59 | } | |
60 | ||
61 | int syscall_trace_enter(struct pt_regs *regs); | |
62 | void syscall_trace_exit(struct pt_regs *regs); | |
63 | ||
969f5ea6 WD |
64 | #ifdef CONFIG_ARM64_ERRATUM_1463225 |
65 | DECLARE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); | |
66 | ||
67 | static void cortex_a76_erratum_1463225_svc_handler(void) | |
68 | { | |
69 | u32 reg, val; | |
70 | ||
71 | if (!unlikely(test_thread_flag(TIF_SINGLESTEP))) | |
72 | return; | |
73 | ||
74 | if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225))) | |
75 | return; | |
76 | ||
77 | __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1); | |
78 | reg = read_sysreg(mdscr_el1); | |
79 | val = reg | DBG_MDSCR_SS | DBG_MDSCR_KDE; | |
80 | write_sysreg(val, mdscr_el1); | |
81 | asm volatile("msr daifclr, #8"); | |
82 | isb(); | |
83 | ||
84 | /* We will have taken a single-step exception by this point */ | |
85 | ||
86 | write_sysreg(reg, mdscr_el1); | |
87 | __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0); | |
88 | } | |
89 | #else | |
90 | static void cortex_a76_erratum_1463225_svc_handler(void) { } | |
91 | #endif /* CONFIG_ARM64_ERRATUM_1463225 */ | |
92 | ||
3b714275 MR |
93 | static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, |
94 | const syscall_fn_t syscall_table[]) | |
f37099b6 MR |
95 | { |
96 | unsigned long flags = current_thread_info()->flags; | |
97 | ||
98 | regs->orig_x0 = regs->regs[0]; | |
99 | regs->syscallno = scno; | |
100 | ||
969f5ea6 | 101 | cortex_a76_erratum_1463225_svc_handler(); |
f37099b6 MR |
102 | local_daif_restore(DAIF_PROCCTX); |
103 | user_exit(); | |
104 | ||
105 | if (has_syscall_work(flags)) { | |
106 | /* set default errno for user-issued syscall(-1) */ | |
107 | if (scno == NO_SYSCALL) | |
108 | regs->regs[0] = -ENOSYS; | |
109 | scno = syscall_trace_enter(regs); | |
110 | if (scno == NO_SYSCALL) | |
111 | goto trace_exit; | |
112 | } | |
113 | ||
114 | invoke_syscall(regs, scno, sc_nr, syscall_table); | |
115 | ||
116 | /* | |
117 | * The tracing status may have changed under our feet, so we have to | |
118 | * check again. However, if we were tracing entry, then we always trace | |
119 | * exit regardless, as the old entry assembly did. | |
120 | */ | |
121 | if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) { | |
122 | local_daif_mask(); | |
123 | flags = current_thread_info()->flags; | |
efd11235 WD |
124 | if (!has_syscall_work(flags)) { |
125 | /* | |
126 | * We're off to userspace, where interrupts are | |
127 | * always enabled after we restore the flags from | |
128 | * the SPSR. | |
129 | */ | |
130 | trace_hardirqs_on(); | |
f37099b6 | 131 | return; |
efd11235 | 132 | } |
f37099b6 MR |
133 | local_daif_restore(DAIF_PROCCTX); |
134 | } | |
135 | ||
136 | trace_exit: | |
137 | syscall_trace_exit(regs); | |
138 | } | |
3b714275 MR |
139 | |
140 | static inline void sve_user_discard(void) | |
141 | { | |
142 | if (!system_supports_sve()) | |
143 | return; | |
144 | ||
145 | clear_thread_flag(TIF_SVE); | |
146 | ||
147 | /* | |
148 | * task_fpsimd_load() won't be called to update CPACR_EL1 in | |
149 | * ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only | |
150 | * happens if a context switch or kernel_neon_begin() or context | |
151 | * modification (sigreturn, ptrace) intervenes. | |
152 | * So, ensure that CPACR_EL1 is already correct for the fast-path case. | |
153 | */ | |
154 | sve_user_disable(); | |
155 | } | |
156 | ||
afa7c0e5 | 157 | void el0_svc_handler(struct pt_regs *regs) |
3b714275 MR |
158 | { |
159 | sve_user_discard(); | |
160 | el0_svc_common(regs, regs->regs[8], __NR_syscalls, sys_call_table); | |
161 | } | |
162 | ||
163 | #ifdef CONFIG_COMPAT | |
afa7c0e5 | 164 | void el0_svc_compat_handler(struct pt_regs *regs) |
3b714275 MR |
165 | { |
166 | el0_svc_common(regs, regs->regs[7], __NR_compat_syscalls, | |
167 | compat_sys_call_table); | |
168 | } | |
169 | #endif |