[linux-block.git] / arch / riscv / kernel / traps.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2012 Regents of the University of California
 */

#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/randomize_kstack.h>
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/sched/signal.h>
#include <linux/signal.h>
#include <linux/kdebug.h>
#include <linux/uaccess.h>
#include <linux/kprobes.h>
#include <linux/uprobes.h>
#include <asm/uprobes.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/irq.h>
#include <linux/kexec.h>
#include <linux/entry-common.h>

#include <asm/asm-prototypes.h>
#include <asm/bug.h>
#include <asm/cfi.h>
#include <asm/csr.h>
#include <asm/processor.h>
#include <asm/ptrace.h>
#include <asm/syscall.h>
#include <asm/thread_info.h>
#include <asm/vector.h>
#include <asm/irq_stack.h>

int show_unhandled_signals = 1;

static DEFINE_SPINLOCK(die_lock);

static int copy_code(struct pt_regs *regs, u16 *val, const u16 *insns)
{
	const void __user *uaddr = (__force const void __user *)insns;

	if (!user_mode(regs))
		return get_kernel_nofault(*val, insns);

	/* The user space code from other tasks cannot be accessed. */
	if (regs != task_pt_regs(current))
		return -EPERM;

	return copy_from_user_nofault(val, uaddr, sizeof(*val));
}

static void dump_instr(const char *loglvl, struct pt_regs *regs)
{
	char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
	const u16 *insns = (u16 *)instruction_pointer(regs);
	long bad;
	u16 val;
	int i;

	for (i = -10; i < 2; i++) {
		bad = copy_code(regs, &val, &insns[i]);
		if (!bad) {
			p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
		} else {
			printk("%sCode: Unable to access instruction at 0x%px.\n",
			       loglvl, &insns[i]);
			return;
		}
	}
	printk("%sCode: %s\n", loglvl, str);
}

void die(struct pt_regs *regs, const char *str)
{
	static int die_counter;
	int ret;
	long cause;
	unsigned long flags;

	oops_enter();

	spin_lock_irqsave(&die_lock, flags);
	console_verbose();
	bust_spinlocks(1);

	pr_emerg("%s [#%d]\n", str, ++die_counter);
	print_modules();
	if (regs) {
		show_regs(regs);
		dump_instr(KERN_EMERG, regs);
	}

	cause = regs ? regs->cause : -1;
	ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);

	if (kexec_should_crash(current))
		crash_kexec(regs);

	bust_spinlocks(0);
	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
	spin_unlock_irqrestore(&die_lock, flags);
	oops_exit();

	if (in_interrupt())
		panic("Fatal exception in interrupt");
	if (panic_on_oops)
		panic("Fatal exception");
	if (ret != NOTIFY_STOP)
		make_task_dead(SIGSEGV);
}

void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
{
	struct task_struct *tsk = current;

	if (show_unhandled_signals && unhandled_signal(tsk, signo)
	    && printk_ratelimit()) {
		pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
			tsk->comm, task_pid_nr(tsk), signo, code, addr);
		print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
		pr_cont("\n");
		__show_regs(regs);
		dump_instr(KERN_INFO, regs);
	}

	force_sig_fault(signo, code, (void __user *)addr);
}

static void do_trap_error(struct pt_regs *regs, int signo, int code,
	unsigned long addr, const char *str)
{
	current->thread.bad_cause = regs->cause;

	if (user_mode(regs)) {
		do_trap(regs, signo, code, addr);
	} else {
		if (!fixup_exception(regs))
			die(regs, str);
	}
}

#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
#define __trap_section __noinstr_section(".xip.traps")
#else
#define __trap_section noinstr
#endif
#define DO_ERROR_INFO(name, signo, code, str)					\
asmlinkage __visible __trap_section void name(struct pt_regs *regs)		\
{										\
	if (user_mode(regs)) {							\
		irqentry_enter_from_user_mode(regs);				\
		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
		irqentry_exit_to_user_mode(regs);				\
	} else {								\
		irqentry_state_t state = irqentry_nmi_enter(regs);		\
		do_trap_error(regs, signo, code, regs->epc, "Oops - " str);	\
		irqentry_nmi_exit(regs, state);					\
	}									\
}

DO_ERROR_INFO(do_trap_unknown,
	SIGILL, ILL_ILLTRP, "unknown exception");
DO_ERROR_INFO(do_trap_insn_misaligned,
	SIGBUS, BUS_ADRALN, "instruction address misaligned");
DO_ERROR_INFO(do_trap_insn_fault,
	SIGSEGV, SEGV_ACCERR, "instruction access fault");

asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
{
	bool handled;

	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);

		local_irq_enable();

		handled = riscv_v_first_use_handler(regs);

		local_irq_disable();

		if (!handled)
			do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
				      "Oops - illegal instruction");

		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
			      "Oops - illegal instruction");

		irqentry_nmi_exit(regs, state);
	}
}

DO_ERROR_INFO(do_trap_load_fault,
	SIGSEGV, SEGV_ACCERR, "load access fault");

asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);

		if (handle_misaligned_load(regs))
			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
			      "Oops - load address misaligned");

		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		if (handle_misaligned_load(regs))
			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
			      "Oops - load address misaligned");

		irqentry_nmi_exit(regs, state);
	}
}

asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);

		if (handle_misaligned_store(regs))
			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
				"Oops - store (or AMO) address misaligned");

		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		if (handle_misaligned_store(regs))
			do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
				"Oops - store (or AMO) address misaligned");

		irqentry_nmi_exit(regs, state);
	}
}
DO_ERROR_INFO(do_trap_store_fault,
	SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
DO_ERROR_INFO(do_trap_ecall_s,
	SIGILL, ILL_ILLTRP, "environment call from S-mode");
DO_ERROR_INFO(do_trap_ecall_m,
	SIGILL, ILL_ILLTRP, "environment call from M-mode");

static inline unsigned long get_break_insn_length(unsigned long pc)
{
	bug_insn_t insn;

	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
		return 0;

	return GET_INSN_LENGTH(insn);
}

static bool probe_single_step_handler(struct pt_regs *regs)
{
	bool user = user_mode(regs);

	return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs);
}

static bool probe_breakpoint_handler(struct pt_regs *regs)
{
	bool user = user_mode(regs);

	return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs);
}

void handle_break(struct pt_regs *regs)
{
	if (probe_single_step_handler(regs))
		return;

	if (probe_breakpoint_handler(regs))
		return;

	current->thread.bad_cause = regs->cause;

	if (user_mode(regs))
		force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
#ifdef CONFIG_KGDB
	else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
								== NOTIFY_STOP)
		return;
#endif
	else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN ||
		 handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
		regs->epc += get_break_insn_length(regs->epc);
	else
		die(regs, "Kernel BUG");
}

asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		irqentry_enter_from_user_mode(regs);

		handle_break(regs);

		irqentry_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		handle_break(regs);

		irqentry_nmi_exit(regs, state);
	}
}

asmlinkage __visible __trap_section  __no_stack_protector
void do_trap_ecall_u(struct pt_regs *regs)
{
	if (user_mode(regs)) {
		long syscall = regs->a7;

		regs->epc += 4;
		regs->orig_a0 = regs->a0;
		regs->a0 = -ENOSYS;

		riscv_v_vstate_discard(regs);

		syscall = syscall_enter_from_user_mode(regs, syscall);

		add_random_kstack_offset();

		if (syscall >= 0 && syscall < NR_syscalls)
			syscall_handler(regs, syscall);

		/*
		 * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(),
		 * so the maximum stack offset is 1k bytes (10 bits).
		 *
		 * The actual entropy will be further reduced by the compiler when
		 * applying stack alignment constraints: 16-byte (i.e. 4-bit) aligned
		 * for RV32I or RV64I.
		 *
		 * The resulting 6 bits of entropy is seen in SP[9:4].
		 */
		choose_random_kstack_offset(get_random_u16());

		syscall_exit_to_user_mode(regs);
	} else {
		irqentry_state_t state = irqentry_nmi_enter(regs);

		do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
			"Oops - environment call from U-mode");

		irqentry_nmi_exit(regs, state);
	}

}

#ifdef CONFIG_MMU
asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
{
	irqentry_state_t state = irqentry_enter(regs);

	handle_page_fault(regs);

	local_irq_disable();

	irqentry_exit(regs, state);
}
#endif

static void noinstr handle_riscv_irq(struct pt_regs *regs)
{
	struct pt_regs *old_regs;

	irq_enter_rcu();
	old_regs = set_irq_regs(regs);
	handle_arch_irq(regs);
	set_irq_regs(old_regs);
	irq_exit_rcu();
}

asmlinkage void noinstr do_irq(struct pt_regs *regs)
{
	irqentry_state_t state = irqentry_enter(regs);

	if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack())
		call_on_irq_stack(regs, handle_riscv_irq);
	else
		handle_riscv_irq(regs);

	irqentry_exit(regs, state);
}

#ifdef CONFIG_GENERIC_BUG
int is_valid_bugaddr(unsigned long pc)
{
	bug_insn_t insn;

	if (pc < VMALLOC_START)
		return 0;
	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
		return 0;
	if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
		return (insn == __BUG_INSN_32);
	else
		return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
}
#endif /* CONFIG_GENERIC_BUG */

#ifdef CONFIG_VMAP_STACK
DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
		overflow_stack)__aligned(16);

asmlinkage void handle_bad_stack(struct pt_regs *regs)
{
	unsigned long tsk_stk = (unsigned long)current->stack;
	unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);

	console_verbose();

	pr_emerg("Insufficient stack space to handle exception!\n");
	pr_emerg("Task stack:     [0x%016lx..0x%016lx]\n",
			tsk_stk, tsk_stk + THREAD_SIZE);
	pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
			ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);

	__show_regs(regs);
	panic("Kernel stack overflow");

	for (;;)
		wait_for_interrupt();
}
#endif
Commit	Line	Data
50acfb2b	1	// SPDX-License-Identifier: GPL-2.0-only
76d2a049 PD	2	/*
76d2a049 PD	3	* Copyright (C) 2012 Regents of the University of California
76d2a049 PD	4	*/
76d2a049 PD	5
5ed881bc	6	#include <linux/cpu.h>
76d2a049 PD	7	#include <linux/kernel.h>
76d2a049 PD	8	#include <linux/init.h>
05d450aa	9	#include <linux/randomize_kstack.h>
76d2a049 PD	10	#include <linux/sched.h>
	11	#include <linux/sched/debug.h>
	12	#include <linux/sched/signal.h>
	13	#include <linux/signal.h>
	14	#include <linux/kdebug.h>
	15	#include <linux/uaccess.h>
c22b0bcb	16	#include <linux/kprobes.h>
9f564b92 BT	17	#include <linux/uprobes.h>
9f564b92 BT	18	#include <asm/uprobes.h>
76d2a049 PD	19	#include <linux/mm.h>
	20	#include <linux/module.h>
	21	#include <linux/irq.h>
3f190111	22	#include <linux/kexec.h>
f0bddf50	23	#include <linux/entry-common.h>
76d2a049	24
030f1dfa	25	#include <asm/asm-prototypes.h>
091b9450	26	#include <asm/bug.h>
af0ead42	27	#include <asm/cfi.h>
d951b20b	28	#include <asm/csr.h>
76d2a049 PD	29	#include <asm/processor.h>
76d2a049 PD	30	#include <asm/ptrace.h>
f0bddf50	31	#include <asm/syscall.h>
d951b20b	32	#include <asm/thread_info.h>
cd054837	33	#include <asm/vector.h>
163e76cc	34	#include <asm/irq_stack.h>
76d2a049 PD	35
	36	int show_unhandled_signals = 1;
	37
76d2a049 PD	38	static DEFINE_SPINLOCK(die_lock);
76d2a049 PD	39
b8a03a63 YC	40	static int copy_code(struct pt_regs regs, u16 val, const u16 *insns)
	41	{
	42	const void __user uaddr = (__force const void __user )insns;
	43
	44	if (!user_mode(regs))
	45	return get_kernel_nofault(*val, insns);
	46
	47	/* The user space code from other tasks cannot be accessed. */
	48	if (regs != task_pt_regs(current))
	49	return -EPERM;
	50
	51	return copy_from_user_nofault(val, uaddr, sizeof(*val));
	52	}
	53
	54	static void dump_instr(const char loglvl, struct pt_regs regs)
eb165bfa BT	55	{
	56	char str[sizeof("0000 ") * 12 + 2 + 1], *p = str;
	57	const u16 insns = (u16 )instruction_pointer(regs);
	58	long bad;
	59	u16 val;
	60	int i;
	61
	62	for (i = -10; i < 2; i++) {
b8a03a63	63	bad = copy_code(regs, &val, &insns[i]);
eb165bfa BT	64	if (!bad) {
	65	p += sprintf(p, i == 0 ? "(%04hx) " : "%04hx ", val);
	66	} else {
	67	printk("%sCode: Unable to access instruction at 0x%px.\n",
	68	loglvl, &insns[i]);
	69	return;
	70	}
	71	}
	72	printk("%sCode: %s\n", loglvl, str);
	73	}
	74
76d2a049 PD	75	void die(struct pt_regs regs, const char str)
	76	{
	77	static int die_counter;
	78	int ret;
f2913d00	79	long cause;
130aee3f	80	unsigned long flags;
76d2a049 PD	81
	82	oops_enter();
	83
130aee3f	84	spin_lock_irqsave(&die_lock, flags);
76d2a049 PD	85	console_verbose();
	86	bust_spinlocks(1);
	87
	88	pr_emerg("%s [#%d]\n", str, ++die_counter);
	89	print_modules();
eb165bfa	90	if (regs) {
f2913d00	91	show_regs(regs);
b8a03a63	92	dump_instr(KERN_EMERG, regs);
eb165bfa	93	}
76d2a049	94
f2913d00 PD	95	cause = regs ? regs->cause : -1;
f2913d00 PD	96	ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV);
76d2a049	97
f2913d00	98	if (kexec_should_crash(current))
3f190111 XT	99	crash_kexec(regs);
3f190111 XT	100
76d2a049 PD	101	bust_spinlocks(0);
76d2a049 PD	102	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
130aee3f	103	spin_unlock_irqrestore(&die_lock, flags);
76d2a049 PD	104	oops_exit();
	105
	106	if (in_interrupt())
	107	panic("Fatal exception in interrupt");
	108	if (panic_on_oops)
	109	panic("Fatal exception");
	110	if (ret != NOTIFY_STOP)
0e25498f	111	make_task_dead(SIGSEGV);
76d2a049 PD	112	}
76d2a049 PD	113
6f25a967	114	void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr)
76d2a049	115	{
6f25a967 EB	116	struct task_struct *tsk = current;
6f25a967 EB	117
76d2a049 PD	118	if (show_unhandled_signals && unhandled_signal(tsk, signo)
	119	&& printk_ratelimit()) {
	120	pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT,
	121	tsk->comm, task_pid_nr(tsk), signo, code, addr);
6ab77af4	122	print_vma_addr(KERN_CONT " in ", instruction_pointer(regs));
76d2a049	123	pr_cont("\n");
091b9450	124	__show_regs(regs);
dd33e5dc	125	dump_instr(KERN_INFO, regs);
76d2a049 PD	126	}
76d2a049 PD	127
2e1661d2	128	force_sig_fault(signo, code, (void __user *)addr);
76d2a049 PD	129	}
	130
	131	static void do_trap_error(struct pt_regs *regs, int signo, int code,
	132	unsigned long addr, const char *str)
	133	{
74784081 GR	134	current->thread.bad_cause = regs->cause;
74784081 GR	135
76d2a049	136	if (user_mode(regs)) {
6f25a967	137	do_trap(regs, signo, code, addr);
76d2a049 PD	138	} else {
	139	if (!fixup_exception(regs))
	140	die(regs, str);
	141	}
	142	}
	143
e64f737a	144	#if defined(CONFIG_XIP_KERNEL) && defined(CONFIG_RISCV_ALTERNATIVE)
d0db02c6	145	#define __trap_section __noinstr_section(".xip.traps")
5e63215c	146	#else
d0db02c6	147	#define __trap_section noinstr
5e63215c	148	#endif
f0bddf50 GR	149	#define DO_ERROR_INFO(name, signo, code, str) \
	150	asmlinkage __visible __trap_section void name(struct pt_regs *regs) \
	151	{ \
	152	if (user_mode(regs)) { \
	153	irqentry_enter_from_user_mode(regs); \
	154	do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
	155	irqentry_exit_to_user_mode(regs); \
	156	} else { \
	157	irqentry_state_t state = irqentry_nmi_enter(regs); \
	158	do_trap_error(regs, signo, code, regs->epc, "Oops - " str); \
	159	irqentry_nmi_exit(regs, state); \
	160	} \
76d2a049 PD	161	}
	162
	163	DO_ERROR_INFO(do_trap_unknown,
	164	SIGILL, ILL_ILLTRP, "unknown exception");
	165	DO_ERROR_INFO(do_trap_insn_misaligned,
	166	SIGBUS, BUS_ADRALN, "instruction address misaligned");
	167	DO_ERROR_INFO(do_trap_insn_fault,
	168	SIGSEGV, SEGV_ACCERR, "instruction access fault");
cd054837 AC	169
	170	asmlinkage __visible __trap_section void do_trap_insn_illegal(struct pt_regs *regs)
	171	{
26c38cd8 AC	172	bool handled;
26c38cd8 AC	173
cd054837 AC	174	if (user_mode(regs)) {
	175	irqentry_enter_from_user_mode(regs);
	176
	177	local_irq_enable();
	178
26c38cd8 AC	179	handled = riscv_v_first_use_handler(regs);
	180
	181	local_irq_disable();
	182
	183	if (!handled)
cd054837 AC	184	do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
	185	"Oops - illegal instruction");
	186
	187	irqentry_exit_to_user_mode(regs);
	188	} else {
	189	irqentry_state_t state = irqentry_nmi_enter(regs);
	190
	191	do_trap_error(regs, SIGILL, ILL_ILLOPC, regs->epc,
	192	"Oops - illegal instruction");
	193
	194	irqentry_nmi_exit(regs, state);
	195	}
	196	}
	197
76d2a049 PD	198	DO_ERROR_INFO(do_trap_load_fault,
76d2a049 PD	199	SIGSEGV, SEGV_ACCERR, "load access fault");
956d705d	200
f0bddf50	201	asmlinkage __visible __trap_section void do_trap_load_misaligned(struct pt_regs *regs)
956d705d	202	{
f0bddf50 GR	203	if (user_mode(regs)) {
	204	irqentry_enter_from_user_mode(regs);
	205
	206	if (handle_misaligned_load(regs))
	207	do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
	208	"Oops - load address misaligned");
	209
	210	irqentry_exit_to_user_mode(regs);
	211	} else {
	212	irqentry_state_t state = irqentry_nmi_enter(regs);
	213
	214	if (handle_misaligned_load(regs))
	215	do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
	216	"Oops - load address misaligned");
	217
	218	irqentry_nmi_exit(regs, state);
	219	}
956d705d DLM	220	}
956d705d DLM	221
f0bddf50	222	asmlinkage __visible __trap_section void do_trap_store_misaligned(struct pt_regs *regs)
956d705d	223	{
f0bddf50 GR	224	if (user_mode(regs)) {
	225	irqentry_enter_from_user_mode(regs);
	226
	227	if (handle_misaligned_store(regs))
	228	do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
	229	"Oops - store (or AMO) address misaligned");
	230
	231	irqentry_exit_to_user_mode(regs);
	232	} else {
	233	irqentry_state_t state = irqentry_nmi_enter(regs);
	234
	235	if (handle_misaligned_store(regs))
	236	do_trap_error(regs, SIGBUS, BUS_ADRALN, regs->epc,
	237	"Oops - store (or AMO) address misaligned");
	238
	239	irqentry_nmi_exit(regs, state);
	240	}
956d705d	241	}
76d2a049 PD	242	DO_ERROR_INFO(do_trap_store_fault,
76d2a049 PD	243	SIGSEGV, SEGV_ACCERR, "store (or AMO) access fault");
76d2a049 PD	244	DO_ERROR_INFO(do_trap_ecall_s,
	245	SIGILL, ILL_ILLTRP, "environment call from S-mode");
	246	DO_ERROR_INFO(do_trap_ecall_m,
	247	SIGILL, ILL_ILLTRP, "environment call from M-mode");
	248
ee72e0e7 VC	249	static inline unsigned long get_break_insn_length(unsigned long pc)
	250	{
	251	bug_insn_t insn;
	252
25f12ae4	253	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
ee72e0e7	254	return 0;
b42d763a ZL	255
b42d763a ZL	256	return GET_INSN_LENGTH(insn);
ee72e0e7	257	}
ee72e0e7	258
9f564b92 BT	259	static bool probe_single_step_handler(struct pt_regs *regs)
	260	{
	261	bool user = user_mode(regs);
	262
	263	return user ? uprobe_single_step_handler(regs) : kprobe_single_step_handler(regs);
	264	}
	265
	266	static bool probe_breakpoint_handler(struct pt_regs *regs)
	267	{
	268	bool user = user_mode(regs);
	269
	270	return user ? uprobe_breakpoint_handler(regs) : kprobe_breakpoint_handler(regs);
	271	}
	272
f0bddf50	273	void handle_break(struct pt_regs *regs)
76d2a049	274	{
9f564b92	275	if (probe_single_step_handler(regs))
c22b0bcb GR	276	return;
c22b0bcb GR	277
9f564b92	278	if (probe_breakpoint_handler(regs))
74784081 GR	279	return;
74784081 GR	280
74784081	281	current->thread.bad_cause = regs->cause;
c22b0bcb	282
e8f44c50	283	if (user_mode(regs))
a4c3733d	284	force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc);
fe89bd2b VC	285	#ifdef CONFIG_KGDB
	286	else if (notify_die(DIE_TRAP, "EBREAK", regs, 0, regs->cause, SIGTRAP)
	287	== NOTIFY_STOP)
	288	return;
	289	#endif
af0ead42 ST	290	else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN \|\|
af0ead42 ST	291	handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
a4c3733d	292	regs->epc += get_break_insn_length(regs->epc);
e8f44c50 CH	293	else
e8f44c50 CH	294	die(regs, "Kernel BUG");
76d2a049	295	}
f0bddf50 GR	296
	297	asmlinkage __visible __trap_section void do_trap_break(struct pt_regs *regs)
	298	{
	299	if (user_mode(regs)) {
	300	irqentry_enter_from_user_mode(regs);
	301
	302	handle_break(regs);
	303
	304	irqentry_exit_to_user_mode(regs);
	305	} else {
	306	irqentry_state_t state = irqentry_nmi_enter(regs);
	307
	308	handle_break(regs);
	309
	310	irqentry_nmi_exit(regs, state);
	311	}
	312	}
	313
05d450aa SS	314	asmlinkage __visible __trap_section __no_stack_protector
05d450aa SS	315	void do_trap_ecall_u(struct pt_regs *regs)
f0bddf50 GR	316	{
f0bddf50 GR	317	if (user_mode(regs)) {
52449c17	318	long syscall = regs->a7;
f0bddf50	319
f0bddf50 GR	320	regs->epc += 4;
f0bddf50 GR	321	regs->orig_a0 = regs->a0;
61119394	322	regs->a0 = -ENOSYS;
f0bddf50	323
9657e9b7 BT	324	riscv_v_vstate_discard(regs);
9657e9b7 BT	325
9c2598d4 BT	326	syscall = syscall_enter_from_user_mode(regs, syscall);
9c2598d4 BT	327
05d450aa SS	328	add_random_kstack_offset();
05d450aa SS	329
52449c17	330	if (syscall >= 0 && syscall < NR_syscalls)
f0bddf50	331	syscall_handler(regs, syscall);
61119394	332
05d450aa SS	333	/*
	334	* Ultimately, this value will get limited by KSTACK_OFFSET_MAX(),
	335	* so the maximum stack offset is 1k bytes (10 bits).
	336	*
	337	* The actual entropy will be further reduced by the compiler when
	338	* applying stack alignment constraints: 16-byte (i.e. 4-bit) aligned
	339	* for RV32I or RV64I.
	340	*
	341	* The resulting 6 bits of entropy is seen in SP[9:4].
	342	*/
	343	choose_random_kstack_offset(get_random_u16());
f0bddf50 GR	344
	345	syscall_exit_to_user_mode(regs);
	346	} else {
	347	irqentry_state_t state = irqentry_nmi_enter(regs);
	348
	349	do_trap_error(regs, SIGILL, ILL_ILLTRP, regs->epc,
	350	"Oops - environment call from U-mode");
	351
	352	irqentry_nmi_exit(regs, state);
	353	}
	354
	355	}
	356
	357	#ifdef CONFIG_MMU
	358	asmlinkage __visible noinstr void do_page_fault(struct pt_regs *regs)
	359	{
	360	irqentry_state_t state = irqentry_enter(regs);
	361
	362	handle_page_fault(regs);
	363
	364	local_irq_disable();
	365
	366	irqentry_exit(regs, state);
	367	}
	368	#endif
	369
163e76cc	370	static void noinstr handle_riscv_irq(struct pt_regs *regs)
f0bddf50 GR	371	{
f0bddf50 GR	372	struct pt_regs *old_regs;
f0bddf50 GR	373
	374	irq_enter_rcu();
	375	old_regs = set_irq_regs(regs);
	376	handle_arch_irq(regs);
	377	set_irq_regs(old_regs);
	378	irq_exit_rcu();
163e76cc GR	379	}
	380
	381	asmlinkage void noinstr do_irq(struct pt_regs *regs)
	382	{
	383	irqentry_state_t state = irqentry_enter(regs);
82982fdd ST	384
	385	if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack())
	386	call_on_irq_stack(regs, handle_riscv_irq);
	387	else
163e76cc	388	handle_riscv_irq(regs);
f0bddf50 GR	389
	390	irqentry_exit(regs, state);
	391	}
76d2a049 PD	392
	393	#ifdef CONFIG_GENERIC_BUG
	394	int is_valid_bugaddr(unsigned long pc)
	395	{
	396	bug_insn_t insn;
	397
9a6e7af0	398	if (pc < VMALLOC_START)
76d2a049	399	return 0;
25f12ae4	400	if (get_kernel_nofault(insn, (bug_insn_t *)pc))
76d2a049	401	return 0;
ee72e0e7 VC	402	if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32)
	403	return (insn == __BUG_INSN_32);
	404	else
	405	return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16);
76d2a049 PD	406	}
	407	#endif /* CONFIG_GENERIC_BUG */
	408
31da94c2	409	#ifdef CONFIG_VMAP_STACK
be97d0db	410	DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
31da94c2	411	overflow_stack)__aligned(16);
31da94c2 TT	412
	413	asmlinkage void handle_bad_stack(struct pt_regs *regs)
	414	{
	415	unsigned long tsk_stk = (unsigned long)current->stack;
	416	unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
	417
	418	console_verbose();
	419
	420	pr_emerg("Insufficient stack space to handle exception!\n");
	421	pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
	422	tsk_stk, tsk_stk + THREAD_SIZE);
	423	pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
	424	ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
	425
	426	__show_regs(regs);
	427	panic("Kernel stack overflow");
	428
	429	for (;;)
	430	wait_for_interrupt();
	431	}
	432	#endif