[linux-2.6-block.git] / arch / riscv / kernel / traps.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2012 Regents of the University of California
 */

#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/kdebug.h>
#include <linux/uaccess.h>
#include <linux/kprobes.h>
#include <linux/uprobes.h>
#include <asm/uprobes.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/irq.h>
#include <linux/kexec.h>
#include <linux/entry-common.h>

#include <asm/asm-prototypes.h>
#include <asm/bug.h>
#include <asm/cfi.h>
#include <asm/csr.h>
#include <asm/processor.h>
#include <asm/ptrace.h>
#include <asm/syscall.h>
#include <asm/thread_info.h>
#include <asm/vector.h>
#include <asm/irq_stack.h>

int show_unhandled_signals = 1;

static DEFINE_SPINLOCK(die_lock);

static void dump_kernel_instr(const char *loglvl, struct pt_regs *regs)
{
	char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
	const u16 *insns = (u16 *)instruction_pointer(regs);
	long bad;
	u16 val;
	int i;

	for (i = -10; i < 2; i++) {
		bad = get_kernel_nofault(val, &insns[i]);
		if (!bad) {
			p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
		} else {
			printk("%sCode: Unable to access instruction at 0x%px.\n",
			       loglvl, &insns[i]);
			return;
		}
	}
	printk("%sCode: %s\n", loglvl, str);
}

void die(struct pt_regs *regs, const char *str)
{
	static int die_counter;
	int ret;
	long cause;
	unsigned long flags;

	oops_enter();

	spin_lock_irqsave(&die_lock, flags);
	console_verbose();
	bust_spinlocks(1);

	pr_emerg("%s [#%d]\n", str, ++die_counter);
	print_modules();
	if (regs) {
		show_regs(regs);
		dump_kernel_instr(KERN_EMERG, regs);
	}

	cause = regs ? regs->cause : -1;
	ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);

	if (kexec_should_crash(current))
		crash_kexec(regs);

	bust_spinlocks(0);
	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
	spin_unlock_irqrestore(&die_lock, flags);
	oops_exit();

	if (in_interrupt())
		panic("Fatal exception in interrupt");
	if (panic_on_oops)
		panic("Fatal exception");
	if (ret != NOTIFY_STOP)
		make_task_dead(SIGSEGV);
}

void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
{
	struct task_struct *tsk = current;

	if (show_unhandled_signals && unhandled_signal(tsk, signo)
	    && printk_ratelimit()) {
		pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
			tsk->comm, task_pid_nr(tsk), signo, code, addr);
		print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
		pr_cont("\n");
		__show_regs(regs);
	}

	force_sig_fault(signo, code, (void __user *)addr);
}

static void do_trap_error(struct pt_regs *regs, int signo, int code,
	unsigned long addr, const char *str)
{
	current->thread.bad_cause = regs->cause;

	if (user_mode(regs)) {
		do_trap(regs, signo, code, addr);
	} else {
		if (!fixup_exception(regs))
			die(regs, str);
	}
}

#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
#define __trap_section __noinstr_section(".xip.traps")
#else
#define __trap_section noinstr
#endif
#define DO_ERROR_INFO(name, signo, code, str)					\
asmlinkage __visible __trap_section void name(struct pt_regs *regs)		\
{										\
	if (user_mode(regs)) {							\
		irqentry_enter_from_user_mode(regs);				\
		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
		irqentry_exit_to_user_mode(regs);				\
	} else {								\
		irqentry_state_t state = irqentry_nmi_enter(regs);		\
		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
		irqentry_nmi_exit(regs, state);					\
	}									\
}

DO_ERROR_INFO(do_trap_unknown,
	SIGILL, ILL_ILLTRP, "unknown exception");
DO_ERROR_INFO(do_trap_insn_misaligned,
	SIGBUS, BUS_ADRALN, "instruction address misaligned");
DO_ERROR_INFO(do_trap_insn_fault,
	SIGSEGV, SEGV_ACCERR, "instruction access fault");

asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
{
	bool handled;

	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);

		local_irq_enable();

		handled = riscv_v_first_use_handler(regs);

		local_irq_disable();

		if (!handled)
			do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
				      "Oops - illegal instruction");

		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
			      "Oops - illegal instruction");

		irqentry_nmi_exit(regs, state);
	}
}

DO_ERROR_INFO(do_trap_load_fault,
	SIGSEGV, SEGV_ACCERR, "load access fault");
#ifndef CONFIG_RISCV_M_MODE
DO_ERROR_INFO(do_trap_load_misaligned,
	SIGBUS, BUS_ADRALN, "Oops - load address misaligned");
DO_ERROR_INFO(do_trap_store_misaligned,
	SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned");
#else
int handle_misaligned_load(struct pt_regs *regs);
int handle_misaligned_store(struct pt_regs *regs);

asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);

		if (handle_misaligned_load(regs))
			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
			      "Oops - load address misaligned");

		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		if (handle_misaligned_load(regs))
			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
			      "Oops - load address misaligned");

		irqentry_nmi_exit(regs, state);
	}
}

asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);

		if (handle_misaligned_store(regs))
			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
				"Oops - store (or AMO) address misaligned");

		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		if (handle_misaligned_store(regs))
			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
				"Oops - store (or AMO) address misaligned");

		irqentry_nmi_exit(regs, state);
	}
}
#endif
DO_ERROR_INFO(do_trap_store_fault,
	SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
DO_ERROR_INFO(do_trap_ecall_s,
	SIGILL, ILL_ILLTRP, "environment call from S-mode");
DO_ERROR_INFO(do_trap_ecall_m,
	SIGILL, ILL_ILLTRP, "environment call from M-mode");

static inline unsigned long get_break_insn_length(unsigned long pc)
{
	bug_insn_t insn;

	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
		return 0;

	return GET_INSN_LENGTH(insn);
}

static bool probe_single_step_handler(struct pt_regs *regs)
{
	bool user = user_mode(regs);

	return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs);
}

static bool probe_breakpoint_handler(struct pt_regs *regs)
{
	bool user = user_mode(regs);

	return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs);
}

void handle_break(struct pt_regs *regs)
{
	if (probe_single_step_handler(regs))
		return;

	if (probe_breakpoint_handler(regs))
		return;

	current->thread.bad_cause = regs->cause;

	if (user_mode(regs))
		force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
#ifdef CONFIG_KGDB
	else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
								== NOTIFY_STOP)
		return;
#endif
	else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN ||
		 handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
		regs->epc += get_break_insn_length(regs->epc);
	else
		die(regs, "Kernel BUG");
}

asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);

		handle_break(regs);

		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		handle_break(regs);

		irqentry_nmi_exit(regs, state);
	}
}

asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		long syscall = regs->a7;

		regs->epc += 4;
		regs->orig_a0 = regs->a0;

		riscv_v_vstate_discard(regs);

		syscall = syscall_enter_from_user_mode(regs, syscall);

		if (syscall >= 0 && syscall < NR_syscalls)
			syscall_handler(regs, syscall);
		else if (syscall != -1)
			regs->a0 = -ENOSYS;

		syscall_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
			"Oops - environment call from U-mode");

		irqentry_nmi_exit(regs, state);
	}

}

#ifdef CONFIG_MMU
asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
{
	irqentry_state_t state = irqentry_enter(regs);

	handle_page_fault(regs);

	local_irq_disable();

	irqentry_exit(regs, state);
}
#endif

static void noinstr handle_riscv_irq(struct pt_regs *regs)
{
	struct pt_regs *old_regs;

	irq_enter_rcu();
	old_regs = set_irq_regs(regs);
	handle_arch_irq(regs);
	set_irq_regs(old_regs);
	irq_exit_rcu();
}

asmlinkage void noinstr do_irq(struct pt_regs *regs)
{
	irqentry_state_t state = irqentry_enter(regs);
#ifdef CONFIG_IRQ_STACKS
	if (on_thread_stack()) {
		ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
					+ IRQ_STACK_SIZE/sizeof(ulong);
		__asm__ __volatile(
		"addi	sp, sp, -"RISCV_SZPTR  "\n"
		REG_S"  ra, (sp)		\n"
		"addi	sp, sp, -"RISCV_SZPTR  "\n"
		REG_S"  s0, (sp)		\n"
		"addi	s0, sp, 2*"RISCV_SZPTR "\n"
		"move	sp, %[sp]		\n"
		"move	a0, %[regs]		\n"
		"call	handle_riscv_irq	\n"
		"addi	sp, s0, -2*"RISCV_SZPTR"\n"
		REG_L"  s0, (sp)		\n"
		"addi	sp, sp, "RISCV_SZPTR   "\n"
		REG_L"  ra, (sp)		\n"
		"addi	sp, sp, "RISCV_SZPTR   "\n"
		:
		: [sp] "r" (sp), [regs] "r" (regs)
		: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
		  "t0", "t1", "t2", "t3", "t4", "t5", "t6",
#ifndef CONFIG_FRAME_POINTER
		  "s0",
#endif
		  "memory");
	} else
#endif
		handle_riscv_irq(regs);

	irqentry_exit(regs, state);
}

#ifdef CONFIG_GENERIC_BUG
int is_valid_bugaddr(unsigned long pc)
{
	bug_insn_t insn;

	if (pc < VMALLOC_START)
		return 0;
	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
		return 0;
	if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
		return (insn == __BUG_INSN_32);
	else
		return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
}
#endif /* CONFIG_GENERIC_BUG */

#ifdef CONFIG_VMAP_STACK
/*
 * Extra stack space that allows us to provide panic messages when the kernel
 * has overflowed its stack.
 */
static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
		overflow_stack)__aligned(16);
/*
 * A temporary stack for use by handle_kernel_stack_overflow.  This is used so
 * we can call into C code to get the per-hart overflow stack.  Usage of this
 * stack must be protected by spin_shadow_stack.
 */
long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16);

/*
 * A pseudo spinlock to protect the shadow stack from being used by multiple
 * harts concurrently.  This isn't a real spinlock because the lock side must
 * be taken without a valid stack and only a single register, it's only taken
 * while in the process of panicing anyway so the performance and error
 * checking a proper spinlock gives us doesn't matter.
 */
unsigned long spin_shadow_stack;

asmlinkage unsigned long get_overflow_stack(void)
{
	return (unsigned long)this_cpu_ptr(overflow_stack) +
		OVERFLOW_STACK_SIZE;
}

asmlinkage void handle_bad_stack(struct pt_regs *regs)
{
	unsigned long tsk_stk = (unsigned long)current->stack;
	unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);

	/*
	 * We're done with the shadow stack by this point, as we're on the
	 * overflow stack.  Tell any other concurrent overflowing harts that
	 * they can proceed with panicing by releasing the pseudo-spinlock.
	 *
	 * This pairs with an amoswap.aq in handle_kernel_stack_overflow.
	 */
	smp_store_release(&spin_shadow_stack, 0);

	console_verbose();

	pr_emerg("Insufficient stack space to handle exception!\n");
	pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
			tsk_stk, tsk_stk + THREAD_SIZE);
	pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
			ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);

	__show_regs(regs);
	panic("Kernel stack overflow");

	for (;;)
		wait_for_interrupt();
}
#endif
Commit	Line	Data
50acfb2b	1	// SPDX-License-Identifier: GPL-2.0-only
76d2a049 PD	2	/*
76d2a049 PD	3	* Copyright (C) 2012 Regents of the University of California
76d2a049 PD	4	*/
76d2a049 PD	5
5ed881bc	6	#include <linux/cpu.h>
76d2a049 PD	7	#include <linux/kernel.h>
	8	#include <linux/init.h>
	9	#include <linux/sched.h>
	10	#include <linux/sched/debug.h>
	11	#include <linux/sched/signal.h>
	12	#include <linux/signal.h>
	13	#include <linux/kdebug.h>
	14	#include <linux/uaccess.h>
c22b0bcb	15	#include <linux/kprobes.h>
9f564b92 BT	16	#include <linux/uprobes.h>
9f564b92 BT	17	#include <asm/uprobes.h>
76d2a049 PD	18	#include <linux/mm.h>
	19	#include <linux/module.h>
	20	#include <linux/irq.h>
3f190111	21	#include <linux/kexec.h>
f0bddf50	22	#include <linux/entry-common.h>
76d2a049	23
030f1dfa	24	#include <asm/asm-prototypes.h>
091b9450	25	#include <asm/bug.h>
af0ead42	26	#include <asm/cfi.h>
d951b20b	27	#include <asm/csr.h>
76d2a049 PD	28	#include <asm/processor.h>
76d2a049 PD	29	#include <asm/ptrace.h>
f0bddf50	30	#include <asm/syscall.h>
d951b20b	31	#include <asm/thread_info.h>
cd054837	32	#include <asm/vector.h>
163e76cc	33	#include <asm/irq_stack.h>
76d2a049 PD	34
	35	int show_unhandled_signals = 1;
	36
76d2a049 PD	37	static DEFINE_SPINLOCK(die_lock);
76d2a049 PD	38
eb165bfa BT	39	static void dump_kernel_instr(const char loglvl, struct pt_regs regs)
	40	{
	41	char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
	42	const u16 insns = (u16 )instruction_pointer(regs);
	43	long bad;
	44	u16 val;
	45	int i;
	46
	47	for (i = -10; i < 2; i++) {
	48	bad = get_kernel_nofault(val, &insns[i]);
	49	if (!bad) {
	50	p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
	51	} else {
	52	printk("%sCode: Unable to access instruction at 0x%px.\n",
	53	loglvl, &insns[i]);
	54	return;
	55	}
	56	}
	57	printk("%sCode: %s\n", loglvl, str);
	58	}
	59
76d2a049 PD	60	void die(struct pt_regs regs, const char str)
	61	{
	62	static int die_counter;
	63	int ret;
f2913d00	64	long cause;
130aee3f	65	unsigned long flags;
76d2a049 PD	66
	67	oops_enter();
	68
130aee3f	69	spin_lock_irqsave(&die_lock, flags);
76d2a049 PD	70	console_verbose();
	71	bust_spinlocks(1);
	72
	73	pr_emerg("%s [#%d]\n", str, ++die_counter);
	74	print_modules();
eb165bfa	75	if (regs) {
f2913d00	76	show_regs(regs);
eb165bfa BT	77	dump_kernel_instr(KERN_EMERG, regs);
eb165bfa BT	78	}
76d2a049	79
f2913d00 PD	80	cause = regs ? regs->cause : -1;
f2913d00 PD	81	ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
76d2a049	82
f2913d00	83	if (kexec_should_crash(current))
3f190111 XT	84	crash_kexec(regs);
3f190111 XT	85
76d2a049 PD	86	bust_spinlocks(0);
76d2a049 PD	87	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
130aee3f	88	spin_unlock_irqrestore(&die_lock, flags);
76d2a049 PD	89	oops_exit();
	90
	91	if (in_interrupt())
	92	panic("Fatal exception in interrupt");
	93	if (panic_on_oops)
	94	panic("Fatal exception");
	95	if (ret != NOTIFY_STOP)
0e25498f	96	make_task_dead(SIGSEGV);
76d2a049 PD	97	}
76d2a049 PD	98
6f25a967	99	void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
76d2a049	100	{
6f25a967 EB	101	struct task_struct *tsk = current;
6f25a967 EB	102
76d2a049 PD	103	if (show_unhandled_signals && unhandled_signal(tsk, signo)
	104	&& printk_ratelimit()) {
	105	pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
	106	tsk->comm, task_pid_nr(tsk), signo, code, addr);
6ab77af4	107	print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
76d2a049	108	pr_cont("\n");
091b9450	109	__show_regs(regs);
76d2a049 PD	110	}
76d2a049 PD	111
2e1661d2	112	force_sig_fault(signo, code, (void __user *)addr);
76d2a049 PD	113	}
	114
	115	static void do_trap_error(struct pt_regs *regs, int signo, int code,
	116	unsigned long addr, const char *str)
	117	{
74784081 GR	118	current->thread.bad_cause = regs->cause;
74784081 GR	119
76d2a049	120	if (user_mode(regs)) {
6f25a967	121	do_trap(regs, signo, code, addr);
76d2a049 PD	122	} else {
	123	if (!fixup_exception(regs))
	124	die(regs, str);
	125	}
	126	}
	127
e64f737a	128	#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
d0db02c6	129	#define __trap_section __noinstr_section(".xip.traps")
5e63215c	130	#else
d0db02c6	131	#define __trap_section noinstr
5e63215c	132	#endif
f0bddf50 GR	133	#define DO_ERROR_INFO(name, signo, code, str) \
	134	asmlinkage __visible __trap_section void name(struct pt_regs *regs) \
	135	{ \
	136	if (user_mode(regs)) { \
	137	irqentry_enter_from_user_mode(regs); \
	138	do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
	139	irqentry_exit_to_user_mode(regs); \
	140	} else { \
	141	irqentry_state_t state = irqentry_nmi_enter(regs); \
	142	do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
	143	irqentry_nmi_exit(regs, state); \
	144	} \
76d2a049 PD	145	}
	146
	147	DO_ERROR_INFO(do_trap_unknown,
	148	SIGILL, ILL_ILLTRP, "unknown exception");
	149	DO_ERROR_INFO(do_trap_insn_misaligned,
	150	SIGBUS, BUS_ADRALN, "instruction address misaligned");
	151	DO_ERROR_INFO(do_trap_insn_fault,
	152	SIGSEGV, SEGV_ACCERR, "instruction access fault");
cd054837 AC	153
	154	asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
	155	{
26c38cd8 AC	156	bool handled;
26c38cd8 AC	157
cd054837 AC	158	if (user_mode(regs)) {
	159	irqentry_enter_from_user_mode(regs);
	160
	161	local_irq_enable();
	162
26c38cd8 AC	163	handled = riscv_v_first_use_handler(regs);
	164
	165	local_irq_disable();
	166
	167	if (!handled)
cd054837 AC	168	do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
	169	"Oops - illegal instruction");
	170
	171	irqentry_exit_to_user_mode(regs);
	172	} else {
	173	irqentry_state_t state = irqentry_nmi_enter(regs);
	174
	175	do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
	176	"Oops - illegal instruction");
	177
	178	irqentry_nmi_exit(regs, state);
	179	}
	180	}
	181
76d2a049 PD	182	DO_ERROR_INFO(do_trap_load_fault,
76d2a049 PD	183	SIGSEGV, SEGV_ACCERR, "load access fault");
956d705d DLM	184	#ifndef CONFIG_RISCV_M_MODE
	185	DO_ERROR_INFO(do_trap_load_misaligned,
	186	SIGBUS, BUS_ADRALN, "Oops - load address misaligned");
76d2a049	187	DO_ERROR_INFO(do_trap_store_misaligned,
956d705d DLM	188	SIGBUS, BUS_ADRALN, "Oops - store (or AMO) address misaligned");
	189	#else
	190	int handle_misaligned_load(struct pt_regs *regs);
	191	int handle_misaligned_store(struct pt_regs *regs);
	192
f0bddf50	193	asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
956d705d	194	{
f0bddf50 GR	195	if (user_mode(regs)) {
	196	irqentry_enter_from_user_mode(regs);
	197
	198	if (handle_misaligned_load(regs))
	199	do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
	200	"Oops - load address misaligned");
	201
	202	irqentry_exit_to_user_mode(regs);
	203	} else {
	204	irqentry_state_t state = irqentry_nmi_enter(regs);
	205
	206	if (handle_misaligned_load(regs))
	207	do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
	208	"Oops - load address misaligned");
	209
	210	irqentry_nmi_exit(regs, state);
	211	}
956d705d DLM	212	}
956d705d DLM	213
f0bddf50	214	asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
956d705d	215	{
f0bddf50 GR	216	if (user_mode(regs)) {
	217	irqentry_enter_from_user_mode(regs);
	218
	219	if (handle_misaligned_store(regs))
	220	do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
	221	"Oops - store (or AMO) address misaligned");
	222
	223	irqentry_exit_to_user_mode(regs);
	224	} else {
	225	irqentry_state_t state = irqentry_nmi_enter(regs);
	226
	227	if (handle_misaligned_store(regs))
	228	do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
	229	"Oops - store (or AMO) address misaligned");
	230
	231	irqentry_nmi_exit(regs, state);
	232	}
956d705d DLM	233	}
956d705d DLM	234	#endif
76d2a049 PD	235	DO_ERROR_INFO(do_trap_store_fault,
76d2a049 PD	236	SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
76d2a049 PD	237	DO_ERROR_INFO(do_trap_ecall_s,
	238	SIGILL, ILL_ILLTRP, "environment call from S-mode");
	239	DO_ERROR_INFO(do_trap_ecall_m,
	240	SIGILL, ILL_ILLTRP, "environment call from M-mode");
	241
ee72e0e7 VC	242	static inline unsigned long get_break_insn_length(unsigned long pc)
	243	{
	244	bug_insn_t insn;
	245
25f12ae4	246	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
ee72e0e7	247	return 0;
b42d763a ZL	248
b42d763a ZL	249	return GET_INSN_LENGTH(insn);
ee72e0e7	250	}
ee72e0e7	251
9f564b92 BT	252	static bool probe_single_step_handler(struct pt_regs *regs)
	253	{
	254	bool user = user_mode(regs);
	255
	256	return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs);
	257	}
	258
	259	static bool probe_breakpoint_handler(struct pt_regs *regs)
	260	{
	261	bool user = user_mode(regs);
	262
	263	return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs);
	264	}
	265
f0bddf50	266	void handle_break(struct pt_regs *regs)
76d2a049	267	{
9f564b92	268	if (probe_single_step_handler(regs))
c22b0bcb GR	269	return;
c22b0bcb GR	270
9f564b92	271	if (probe_breakpoint_handler(regs))
74784081 GR	272	return;
74784081 GR	273
74784081	274	current->thread.bad_cause = regs->cause;
c22b0bcb	275
e8f44c50	276	if (user_mode(regs))
a4c3733d	277	force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
fe89bd2b VC	278	#ifdef CONFIG_KGDB
	279	else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
	280	== NOTIFY_STOP)
	281	return;
	282	#endif
af0ead42 ST	283	else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN \|\|
af0ead42 ST	284	handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
a4c3733d	285	regs->epc += get_break_insn_length(regs->epc);
e8f44c50 CH	286	else
e8f44c50 CH	287	die(regs, "Kernel BUG");
76d2a049	288	}
f0bddf50 GR	289
	290	asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
	291	{
	292	if (user_mode(regs)) {
	293	irqentry_enter_from_user_mode(regs);
	294
	295	handle_break(regs);
	296
	297	irqentry_exit_to_user_mode(regs);
	298	} else {
	299	irqentry_state_t state = irqentry_nmi_enter(regs);
	300
	301	handle_break(regs);
	302
	303	irqentry_nmi_exit(regs, state);
	304	}
	305	}
	306
	307	asmlinkage __visible __trap_section void do_trap_ecall_u(struct pt_regs *regs)
	308	{
	309	if (user_mode(regs)) {
52449c17	310	long syscall = regs->a7;
f0bddf50	311
f0bddf50 GR	312	regs->epc += 4;
	313	regs->orig_a0 = regs->a0;
	314
9657e9b7 BT	315	riscv_v_vstate_discard(regs);
9657e9b7 BT	316
9c2598d4 BT	317	syscall = syscall_enter_from_user_mode(regs, syscall);
9c2598d4 BT	318
52449c17	319	if (syscall >= 0 && syscall < NR_syscalls)
f0bddf50	320	syscall_handler(regs, syscall);
52449c17	321	else if (syscall != -1)
f0bddf50 GR	322	regs->a0 = -ENOSYS;
	323
	324	syscall_exit_to_user_mode(regs);
	325	} else {
	326	irqentry_state_t state = irqentry_nmi_enter(regs);
	327
	328	do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
	329	"Oops - environment call from U-mode");
	330
	331	irqentry_nmi_exit(regs, state);
	332	}
	333
	334	}
	335
	336	#ifdef CONFIG_MMU
	337	asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
	338	{
	339	irqentry_state_t state = irqentry_enter(regs);
	340
	341	handle_page_fault(regs);
	342
	343	local_irq_disable();
	344
	345	irqentry_exit(regs, state);
	346	}
	347	#endif
	348
163e76cc	349	static void noinstr handle_riscv_irq(struct pt_regs *regs)
f0bddf50 GR	350	{
f0bddf50 GR	351	struct pt_regs *old_regs;
f0bddf50 GR	352
	353	irq_enter_rcu();
	354	old_regs = set_irq_regs(regs);
	355	handle_arch_irq(regs);
	356	set_irq_regs(old_regs);
	357	irq_exit_rcu();
163e76cc GR	358	}
	359
	360	asmlinkage void noinstr do_irq(struct pt_regs *regs)
	361	{
	362	irqentry_state_t state = irqentry_enter(regs);
	363	#ifdef CONFIG_IRQ_STACKS
	364	if (on_thread_stack()) {
	365	ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
	366	+ IRQ_STACK_SIZE/sizeof(ulong);
	367	__asm__ __volatile(
	368	"addi sp, sp, -"RISCV_SZPTR "\n"
	369	REG_S" ra, (sp) \n"
	370	"addi sp, sp, -"RISCV_SZPTR "\n"
	371	REG_S" s0, (sp) \n"
	372	"addi s0, sp, 2*"RISCV_SZPTR "\n"
	373	"move sp, %[sp] \n"
	374	"move a0, %[regs] \n"
	375	"call handle_riscv_irq \n"
	376	"addi sp, s0, -2*"RISCV_SZPTR"\n"
	377	REG_L" s0, (sp) \n"
	378	"addi sp, sp, "RISCV_SZPTR "\n"
	379	REG_L" ra, (sp) \n"
	380	"addi sp, sp, "RISCV_SZPTR "\n"
	381	:
	382	: [sp] "r" (sp), [regs] "r" (regs)
	383	: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
	384	"t0", "t1", "t2", "t3", "t4", "t5", "t6",
8d0be641 GR	385	#ifndef CONFIG_FRAME_POINTER
	386	"s0",
	387	#endif
163e76cc GR	388	"memory");
	389	} else
	390	#endif
	391	handle_riscv_irq(regs);
f0bddf50 GR	392
	393	irqentry_exit(regs, state);
	394	}
76d2a049 PD	395
	396	#ifdef CONFIG_GENERIC_BUG
	397	int is_valid_bugaddr(unsigned long pc)
	398	{
	399	bug_insn_t insn;
	400
9a6e7af0	401	if (pc < VMALLOC_START)
76d2a049	402	return 0;
25f12ae4	403	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
76d2a049	404	return 0;
ee72e0e7 VC	405	if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
	406	return (insn == __BUG_INSN_32);
	407	else
	408	return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
76d2a049 PD	409	}
	410	#endif /* CONFIG_GENERIC_BUG */
	411
31da94c2	412	#ifdef CONFIG_VMAP_STACK
de57ecc4 PD	413	/*
	414	* Extra stack space that allows us to provide panic messages when the kernel
	415	* has overflowed its stack.
	416	*/
31da94c2 TT	417	static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
	418	overflow_stack)__aligned(16);
	419	/*
de57ecc4 PD	420	* A temporary stack for use by handle_kernel_stack_overflow. This is used so
	421	* we can call into C code to get the per-hart overflow stack. Usage of this
	422	* stack must be protected by spin_shadow_stack.
31da94c2	423	*/
b003b3b7	424	long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16);
31da94c2	425
7e186433 JZ	426	/*
	427	* A pseudo spinlock to protect the shadow stack from being used by multiple
	428	* harts concurrently. This isn't a real spinlock because the lock side must
	429	* be taken without a valid stack and only a single register, it's only taken
	430	* while in the process of panicing anyway so the performance and error
	431	* checking a proper spinlock gives us doesn't matter.
	432	*/
	433	unsigned long spin_shadow_stack;
	434
31da94c2 TT	435	asmlinkage unsigned long get_overflow_stack(void)
	436	{
	437	return (unsigned long)this_cpu_ptr(overflow_stack) +
	438	OVERFLOW_STACK_SIZE;
	439	}
	440
	441	asmlinkage void handle_bad_stack(struct pt_regs *regs)
	442	{
	443	unsigned long tsk_stk = (unsigned long)current->stack;
	444	unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
	445
7e186433 JZ	446	/*
	447	* We're done with the shadow stack by this point, as we're on the
	448	* overflow stack. Tell any other concurrent overflowing harts that
	449	* they can proceed with panicing by releasing the pseudo-spinlock.
	450	*
	451	* This pairs with an amoswap.aq in handle_kernel_stack_overflow.
	452	*/
	453	smp_store_release(&spin_shadow_stack, 0);
	454
31da94c2 TT	455	console_verbose();
	456
	457	pr_emerg("Insufficient stack space to handle exception!\n");
	458	pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
	459	tsk_stk, tsk_stk + THREAD_SIZE);
	460	pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
	461	ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
	462
	463	__show_regs(regs);
	464	panic("Kernel stack overflow");
	465
	466	for (;;)
	467	wait_for_interrupt();
	468	}
	469	#endif