Commit | Line | Data |
---|---|---|
50acfb2b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
76d2a049 PD |
2 | /* |
3 | * Copyright (C) 2012 Regents of the University of California | |
76d2a049 PD |
4 | */ |
5 | ||
5ed881bc | 6 | #include <linux/cpu.h> |
76d2a049 PD |
7 | #include <linux/kernel.h> |
8 | #include <linux/init.h> | |
9 | #include <linux/sched.h> | |
10 | #include <linux/sched/debug.h> | |
11 | #include <linux/sched/signal.h> | |
12 | #include <linux/signal.h> | |
13 | #include <linux/kdebug.h> | |
14 | #include <linux/uaccess.h> | |
c22b0bcb | 15 | #include <linux/kprobes.h> |
9f564b92 BT |
16 | #include <linux/uprobes.h> |
17 | #include <asm/uprobes.h> | |
76d2a049 PD |
18 | #include <linux/mm.h> |
19 | #include <linux/module.h> | |
20 | #include <linux/irq.h> | |
3f190111 | 21 | #include <linux/kexec.h> |
f0bddf50 | 22 | #include <linux/entry-common.h> |
76d2a049 | 23 | |
030f1dfa | 24 | #include <asm/asm-prototypes.h> |
091b9450 | 25 | #include <asm/bug.h> |
af0ead42 | 26 | #include <asm/cfi.h> |
d951b20b | 27 | #include <asm/csr.h> |
76d2a049 PD |
28 | #include <asm/processor.h> |
29 | #include <asm/ptrace.h> | |
f0bddf50 | 30 | #include <asm/syscall.h> |
d951b20b | 31 | #include <asm/thread_info.h> |
cd054837 | 32 | #include <asm/vector.h> |
163e76cc | 33 | #include <asm/irq_stack.h> |
76d2a049 PD |
34 | |
35 | int show_unhandled_signals = 1; | |
36 | ||
76d2a049 PD |
37 | static DEFINE_SPINLOCK(die_lock); |
38 | ||
eb165bfa BT |
39 | static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs) |
40 | { | |
41 | char str[sizeof("0000 ") * 12 + 2 + 1], *p = str; | |
42 | const u16 *insns = (u16 *)instruction_pointer(regs); | |
43 | long bad; | |
44 | u16 val; | |
45 | int i; | |
46 | ||
47 | for (i = -10; i < 2; i++) { | |
48 | bad = get_kernel_nofault(val, &insns[i]); | |
49 | if (!bad) { | |
50 | p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val); | |
51 | } else { | |
52 | printk("%sCode: Unable to access instruction at 0x%px.\n", | |
53 | loglvl, &insns[i]); | |
54 | return; | |
55 | } | |
56 | } | |
57 | printk("%sCode: %s\n", loglvl, str); | |
58 | } | |
59 | ||
76d2a049 PD |
60 | void die(struct pt_regs *regs, const char *str) |
61 | { | |
62 | static int die_counter; | |
63 | int ret; | |
f2913d00 | 64 | long cause; |
130aee3f | 65 | unsigned long flags; |
76d2a049 PD |
66 | |
67 | oops_enter(); | |
68 | ||
130aee3f | 69 | spin_lock_irqsave(&die_lock, flags); |
76d2a049 PD |
70 | console_verbose(); |
71 | bust_spinlocks(1); | |
72 | ||
73 | pr_emerg("%s [#%d]\n", str, ++die_counter); | |
74 | print_modules(); | |
eb165bfa | 75 | if (regs) { |
f2913d00 | 76 | show_regs(regs); |
eb165bfa BT |
77 | dump_kernel_instr(KERN_EMERG, regs); |
78 | } | |
76d2a049 | 79 | |
f2913d00 PD |
80 | cause = regs ? regs->cause : -1; |
81 | ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV); | |
76d2a049 | 82 | |
f2913d00 | 83 | if (kexec_should_crash(current)) |
3f190111 XT |
84 | crash_kexec(regs); |
85 | ||
76d2a049 PD |
86 | bust_spinlocks(0); |
87 | add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); | |
130aee3f | 88 | spin_unlock_irqrestore(&die_lock, flags); |
76d2a049 PD |
89 | oops_exit(); |
90 | ||
91 | if (in_interrupt()) | |
92 | panic("Fatal exception in interrupt"); | |
93 | if (panic_on_oops) | |
94 | panic("Fatal exception"); | |
95 | if (ret != NOTIFY_STOP) | |
0e25498f | 96 | make_task_dead(SIGSEGV); |
76d2a049 PD |
97 | } |
98 | ||
6f25a967 | 99 | void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) |
76d2a049 | 100 | { |
6f25a967 EB |
101 | struct task_struct *tsk = current; |
102 | ||
76d2a049 PD |
103 | if (show_unhandled_signals && unhandled_signal(tsk, signo) |
104 | && printk_ratelimit()) { | |
105 | pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT, | |
106 | tsk->comm, task_pid_nr(tsk), signo, code, addr); | |
6ab77af4 | 107 | print_vma_addr(KERN_CONT " in ", instruction_pointer(regs)); |
76d2a049 | 108 | pr_cont("\n"); |
091b9450 | 109 | __show_regs(regs); |
76d2a049 PD |
110 | } |
111 | ||
2e1661d2 | 112 | force_sig_fault(signo, code, (void __user *)addr); |
76d2a049 PD |
113 | } |
114 | ||
115 | static void do_trap_error(struct pt_regs *regs, int signo, int code, | |
116 | unsigned long addr, const char *str) | |
117 | { | |
74784081 GR |
118 | current->thread.bad_cause = regs->cause; |
119 | ||
76d2a049 | 120 | if (user_mode(regs)) { |
6f25a967 | 121 | do_trap(regs, signo, code, addr); |
76d2a049 PD |
122 | } else { |
123 | if (!fixup_exception(regs)) | |
124 | die(regs, str); | |
125 | } | |
126 | } | |
127 | ||
e64f737a | 128 | #if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE) |
d0db02c6 | 129 | #define __trap_section __noinstr_section(".xip.traps") |
5e63215c | 130 | #else |
d0db02c6 | 131 | #define __trap_section noinstr |
5e63215c | 132 | #endif |
f0bddf50 GR |
133 | #define DO_ERROR_INFO(name, signo, code, str) \ |
134 | asmlinkage __visible __trap_section void name(struct pt_regs *regs) \ | |
135 | { \ | |
136 | if (user_mode(regs)) { \ | |
137 | irqentry_enter_from_user_mode(regs); \ | |
138 | do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ | |
139 | irqentry_exit_to_user_mode(regs); \ | |
140 | } else { \ | |
141 | irqentry_state_t state = irqentry_nmi_enter(regs); \ | |
142 | do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \ | |
143 | irqentry_nmi_exit(regs, state); \ | |
144 | } \ | |
76d2a049 PD |
145 | } |
146 | ||
147 | DO_ERROR_INFO(do_trap_unknown, | |
148 | SIGILL, ILL_ILLTRP, "unknown exception"); | |
149 | DO_ERROR_INFO(do_trap_insn_misaligned, | |
150 | SIGBUS, BUS_ADRALN, "instruction address misaligned"); | |
151 | DO_ERROR_INFO(do_trap_insn_fault, | |
152 | SIGSEGV, SEGV_ACCERR, "instruction access fault"); | |
cd054837 AC |
153 | |
154 | asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs) | |
155 | { | |
26c38cd8 AC |
156 | bool handled; |
157 | ||
cd054837 AC |
158 | if (user_mode(regs)) { |
159 | irqentry_enter_from_user_mode(regs); | |
160 | ||
161 | local_irq_enable(); | |
162 | ||
26c38cd8 AC |
163 | handled = riscv_v_first_use_handler(regs); |
164 | ||
165 | local_irq_disable(); | |
166 | ||
167 | if (!handled) | |
cd054837 AC |
168 | do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, |
169 | "Oops - illegal instruction"); | |
170 | ||
171 | irqentry_exit_to_user_mode(regs); | |
172 | } else { | |
173 | irqentry_state_t state = irqentry_nmi_enter(regs); | |
174 | ||
175 | do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc, | |
176 | "Oops - illegal instruction"); | |
177 | ||
178 | irqentry_nmi_exit(regs, state); | |
179 | } | |
180 | } | |
181 | ||
76d2a049 PD |
182 | DO_ERROR_INFO(do_trap_load_fault, |
183 | SIGSEGV, SEGV_ACCERR, "load access fault"); | |
956d705d DLM |
184 | #ifndef CONFIG_RISCV_M_MODE |
185 | DO_ERROR_INFO(do_trap_load_misaligned, | |
186 | SIGBUS, BUS_ADRALN, "Oops - load address misaligned"); | |
76d2a049 | 187 | DO_ERROR_INFO(do_trap_store_misaligned, |
956d705d DLM |
188 | SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned"); |
189 | #else | |
190 | int handle_misaligned_load(struct pt_regs *regs); | |
191 | int handle_misaligned_store(struct pt_regs *regs); | |
192 | ||
f0bddf50 | 193 | asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs) |
956d705d | 194 | { |
f0bddf50 GR |
195 | if (user_mode(regs)) { |
196 | irqentry_enter_from_user_mode(regs); | |
197 | ||
198 | if (handle_misaligned_load(regs)) | |
199 | do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, | |
200 | "Oops - load address misaligned"); | |
201 | ||
202 | irqentry_exit_to_user_mode(regs); | |
203 | } else { | |
204 | irqentry_state_t state = irqentry_nmi_enter(regs); | |
205 | ||
206 | if (handle_misaligned_load(regs)) | |
207 | do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, | |
208 | "Oops - load address misaligned"); | |
209 | ||
210 | irqentry_nmi_exit(regs, state); | |
211 | } | |
956d705d DLM |
212 | } |
213 | ||
f0bddf50 | 214 | asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs) |
956d705d | 215 | { |
f0bddf50 GR |
216 | if (user_mode(regs)) { |
217 | irqentry_enter_from_user_mode(regs); | |
218 | ||
219 | if (handle_misaligned_store(regs)) | |
220 | do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, | |
221 | "Oops - store (or AMO) address misaligned"); | |
222 | ||
223 | irqentry_exit_to_user_mode(regs); | |
224 | } else { | |
225 | irqentry_state_t state = irqentry_nmi_enter(regs); | |
226 | ||
227 | if (handle_misaligned_store(regs)) | |
228 | do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc, | |
229 | "Oops - store (or AMO) address misaligned"); | |
230 | ||
231 | irqentry_nmi_exit(regs, state); | |
232 | } | |
956d705d DLM |
233 | } |
234 | #endif | |
76d2a049 PD |
235 | DO_ERROR_INFO(do_trap_store_fault, |
236 | SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault"); | |
76d2a049 PD |
237 | DO_ERROR_INFO(do_trap_ecall_s, |
238 | SIGILL, ILL_ILLTRP, "environment call from S-mode"); | |
239 | DO_ERROR_INFO(do_trap_ecall_m, | |
240 | SIGILL, ILL_ILLTRP, "environment call from M-mode"); | |
241 | ||
ee72e0e7 VC |
242 | static inline unsigned long get_break_insn_length(unsigned long pc) |
243 | { | |
244 | bug_insn_t insn; | |
245 | ||
25f12ae4 | 246 | if (get_kernel_nofault(insn, (bug_insn_t *)pc)) |
ee72e0e7 | 247 | return 0; |
b42d763a ZL |
248 | |
249 | return GET_INSN_LENGTH(insn); | |
ee72e0e7 | 250 | } |
ee72e0e7 | 251 | |
9f564b92 BT |
252 | static bool probe_single_step_handler(struct pt_regs *regs) |
253 | { | |
254 | bool user = user_mode(regs); | |
255 | ||
256 | return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs); | |
257 | } | |
258 | ||
259 | static bool probe_breakpoint_handler(struct pt_regs *regs) | |
260 | { | |
261 | bool user = user_mode(regs); | |
262 | ||
263 | return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs); | |
264 | } | |
265 | ||
f0bddf50 | 266 | void handle_break(struct pt_regs *regs) |
76d2a049 | 267 | { |
9f564b92 | 268 | if (probe_single_step_handler(regs)) |
c22b0bcb GR |
269 | return; |
270 | ||
9f564b92 | 271 | if (probe_breakpoint_handler(regs)) |
74784081 GR |
272 | return; |
273 | ||
74784081 | 274 | current->thread.bad_cause = regs->cause; |
c22b0bcb | 275 | |
e8f44c50 | 276 | if (user_mode(regs)) |
a4c3733d | 277 | force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc); |
fe89bd2b VC |
278 | #ifdef CONFIG_KGDB |
279 | else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP) | |
280 | == NOTIFY_STOP) | |
281 | return; | |
282 | #endif | |
af0ead42 ST |
283 | else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN || |
284 | handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) | |
a4c3733d | 285 | regs->epc += get_break_insn_length(regs->epc); |
e8f44c50 CH |
286 | else |
287 | die(regs, "Kernel BUG"); | |
76d2a049 | 288 | } |
f0bddf50 GR |
289 | |
290 | asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs) | |
291 | { | |
292 | if (user_mode(regs)) { | |
293 | irqentry_enter_from_user_mode(regs); | |
294 | ||
295 | handle_break(regs); | |
296 | ||
297 | irqentry_exit_to_user_mode(regs); | |
298 | } else { | |
299 | irqentry_state_t state = irqentry_nmi_enter(regs); | |
300 | ||
301 | handle_break(regs); | |
302 | ||
303 | irqentry_nmi_exit(regs, state); | |
304 | } | |
305 | } | |
306 | ||
307 | asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs) | |
308 | { | |
309 | if (user_mode(regs)) { | |
52449c17 | 310 | long syscall = regs->a7; |
f0bddf50 | 311 | |
f0bddf50 GR |
312 | regs->epc += 4; |
313 | regs->orig_a0 = regs->a0; | |
314 | ||
9657e9b7 BT |
315 | riscv_v_vstate_discard(regs); |
316 | ||
9c2598d4 BT |
317 | syscall = syscall_enter_from_user_mode(regs, syscall); |
318 | ||
52449c17 | 319 | if (syscall >= 0 && syscall < NR_syscalls) |
f0bddf50 | 320 | syscall_handler(regs, syscall); |
52449c17 | 321 | else if (syscall != -1) |
f0bddf50 GR |
322 | regs->a0 = -ENOSYS; |
323 | ||
324 | syscall_exit_to_user_mode(regs); | |
325 | } else { | |
326 | irqentry_state_t state = irqentry_nmi_enter(regs); | |
327 | ||
328 | do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc, | |
329 | "Oops - environment call from U-mode"); | |
330 | ||
331 | irqentry_nmi_exit(regs, state); | |
332 | } | |
333 | ||
334 | } | |
335 | ||
336 | #ifdef CONFIG_MMU | |
337 | asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs) | |
338 | { | |
339 | irqentry_state_t state = irqentry_enter(regs); | |
340 | ||
341 | handle_page_fault(regs); | |
342 | ||
343 | local_irq_disable(); | |
344 | ||
345 | irqentry_exit(regs, state); | |
346 | } | |
347 | #endif | |
348 | ||
163e76cc | 349 | static void noinstr handle_riscv_irq(struct pt_regs *regs) |
f0bddf50 GR |
350 | { |
351 | struct pt_regs *old_regs; | |
f0bddf50 GR |
352 | |
353 | irq_enter_rcu(); | |
354 | old_regs = set_irq_regs(regs); | |
355 | handle_arch_irq(regs); | |
356 | set_irq_regs(old_regs); | |
357 | irq_exit_rcu(); | |
163e76cc GR |
358 | } |
359 | ||
360 | asmlinkage void noinstr do_irq(struct pt_regs *regs) | |
361 | { | |
362 | irqentry_state_t state = irqentry_enter(regs); | |
363 | #ifdef CONFIG_IRQ_STACKS | |
364 | if (on_thread_stack()) { | |
365 | ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id()) | |
366 | + IRQ_STACK_SIZE/sizeof(ulong); | |
367 | __asm__ __volatile( | |
368 | "addi sp, sp, -"RISCV_SZPTR "\n" | |
369 | REG_S" ra, (sp) \n" | |
370 | "addi sp, sp, -"RISCV_SZPTR "\n" | |
371 | REG_S" s0, (sp) \n" | |
372 | "addi s0, sp, 2*"RISCV_SZPTR "\n" | |
373 | "move sp, %[sp] \n" | |
374 | "move a0, %[regs] \n" | |
375 | "call handle_riscv_irq \n" | |
376 | "addi sp, s0, -2*"RISCV_SZPTR"\n" | |
377 | REG_L" s0, (sp) \n" | |
378 | "addi sp, sp, "RISCV_SZPTR "\n" | |
379 | REG_L" ra, (sp) \n" | |
380 | "addi sp, sp, "RISCV_SZPTR "\n" | |
381 | : | |
382 | : [sp] "r" (sp), [regs] "r" (regs) | |
383 | : "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", | |
384 | "t0", "t1", "t2", "t3", "t4", "t5", "t6", | |
8d0be641 GR |
385 | #ifndef CONFIG_FRAME_POINTER |
386 | "s0", | |
387 | #endif | |
163e76cc GR |
388 | "memory"); |
389 | } else | |
390 | #endif | |
391 | handle_riscv_irq(regs); | |
f0bddf50 GR |
392 | |
393 | irqentry_exit(regs, state); | |
394 | } | |
76d2a049 PD |
395 | |
396 | #ifdef CONFIG_GENERIC_BUG | |
397 | int is_valid_bugaddr(unsigned long pc) | |
398 | { | |
399 | bug_insn_t insn; | |
400 | ||
9a6e7af0 | 401 | if (pc < VMALLOC_START) |
76d2a049 | 402 | return 0; |
25f12ae4 | 403 | if (get_kernel_nofault(insn, (bug_insn_t *)pc)) |
76d2a049 | 404 | return 0; |
ee72e0e7 VC |
405 | if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) |
406 | return (insn == __BUG_INSN_32); | |
407 | else | |
408 | return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16); | |
76d2a049 PD |
409 | } |
410 | #endif /* CONFIG_GENERIC_BUG */ | |
411 | ||
31da94c2 | 412 | #ifdef CONFIG_VMAP_STACK |
de57ecc4 PD |
413 | /* |
414 | * Extra stack space that allows us to provide panic messages when the kernel | |
415 | * has overflowed its stack. | |
416 | */ | |
31da94c2 TT |
417 | static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], |
418 | overflow_stack)__aligned(16); | |
419 | /* | |
de57ecc4 PD |
420 | * A temporary stack for use by handle_kernel_stack_overflow. This is used so |
421 | * we can call into C code to get the per-hart overflow stack. Usage of this | |
422 | * stack must be protected by spin_shadow_stack. | |
31da94c2 | 423 | */ |
b003b3b7 | 424 | long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16); |
31da94c2 | 425 | |
7e186433 JZ |
426 | /* |
427 | * A pseudo spinlock to protect the shadow stack from being used by multiple | |
428 | * harts concurrently. This isn't a real spinlock because the lock side must | |
429 | * be taken without a valid stack and only a single register, it's only taken | |
430 | * while in the process of panicing anyway so the performance and error | |
431 | * checking a proper spinlock gives us doesn't matter. | |
432 | */ | |
433 | unsigned long spin_shadow_stack; | |
434 | ||
31da94c2 TT |
435 | asmlinkage unsigned long get_overflow_stack(void) |
436 | { | |
437 | return (unsigned long)this_cpu_ptr(overflow_stack) + | |
438 | OVERFLOW_STACK_SIZE; | |
439 | } | |
440 | ||
441 | asmlinkage void handle_bad_stack(struct pt_regs *regs) | |
442 | { | |
443 | unsigned long tsk_stk = (unsigned long)current->stack; | |
444 | unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); | |
445 | ||
7e186433 JZ |
446 | /* |
447 | * We're done with the shadow stack by this point, as we're on the | |
448 | * overflow stack. Tell any other concurrent overflowing harts that | |
449 | * they can proceed with panicing by releasing the pseudo-spinlock. | |
450 | * | |
451 | * This pairs with an amoswap.aq in handle_kernel_stack_overflow. | |
452 | */ | |
453 | smp_store_release(&spin_shadow_stack, 0); | |
454 | ||
31da94c2 TT |
455 | console_verbose(); |
456 | ||
457 | pr_emerg("Insufficient stack space to handle exception!\n"); | |
458 | pr_emerg("Task stack: [0x%016lx..0x%016lx]\n", | |
459 | tsk_stk, tsk_stk + THREAD_SIZE); | |
460 | pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n", | |
461 | ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE); | |
462 | ||
463 | __show_regs(regs); | |
464 | panic("Kernel stack overflow"); | |
465 | ||
466 | for (;;) | |
467 | wait_for_interrupt(); | |
468 | } | |
469 | #endif |