[linux-block.git] / arch / loongarch / kernel / ftrace_dyn.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Based on arch/arm64/kernel/ftrace.c
 *
 * Copyright (C) 2022 Loongson Technology Corporation Limited
 */

#include <linux/ftrace.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>

#include <asm/inst.h>
#include <asm/module.h>

static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, bool validate)
{
	u32 replaced;

	if (validate) {
		if (larch_insn_read((void *)pc, &replaced))
			return -EFAULT;

		if (replaced != old)
			return -EINVAL;
	}

	if (larch_insn_patch_text((void *)pc, new))
		return -EPERM;

	return 0;
}

#ifdef CONFIG_MODULES
static bool reachable_by_bl(unsigned long addr, unsigned long pc)
{
	long offset = (long)addr - (long)pc;

	return offset >= -SZ_128M && offset < SZ_128M;
}

static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr)
{
	struct plt_entry *plt = mod->arch.ftrace_trampolines;

	if (addr == FTRACE_ADDR)
		return &plt[FTRACE_PLT_IDX];
	if (addr == FTRACE_REGS_ADDR &&
			IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
		return &plt[FTRACE_REGS_PLT_IDX];

	return NULL;
}

/*
 * Find the address the callsite must branch to in order to reach '*addr'.
 *
 * Due to the limited range of 'bl' instruction, modules may be placed too far
 * away to branch directly and we must use a PLT.
 *
 * Returns true when '*addr' contains a reachable target address, or has been
 * modified to contain a PLT address. Returns false otherwise.
 */
static bool ftrace_find_callable_addr(struct dyn_ftrace *rec, struct module *mod, unsigned long *addr)
{
	unsigned long pc = rec->ip + LOONGARCH_INSN_SIZE;
	struct plt_entry *plt;

	/*
	 * If a custom trampoline is unreachable, rely on the ftrace_regs_caller
	 * trampoline which knows how to indirectly reach that trampoline through
	 * ops->direct_call.
	 */
	if (*addr != FTRACE_ADDR && *addr != FTRACE_REGS_ADDR && !reachable_by_bl(*addr, pc))
		*addr = FTRACE_REGS_ADDR;

	/*
	 * When the target is within range of the 'bl' instruction, use 'addr'
	 * as-is and branch to that directly.
	 */
	if (reachable_by_bl(*addr, pc))
		return true;

	/*
	 * 'mod' is only set at module load time, but if we end up
	 * dealing with an out-of-range condition, we can assume it
	 * is due to a module being loaded far away from the kernel.
	 *
	 * NOTE: __module_text_address() must be called with preemption
	 * disabled, but we can rely on ftrace_lock to ensure that 'mod'
	 * retains its validity throughout the remainder of this code.
	 */
	if (!mod) {
		preempt_disable();
		mod = __module_text_address(pc);
		preempt_enable();
	}

	if (WARN_ON(!mod))
		return false;

	plt = get_ftrace_plt(mod, *addr);
	if (!plt) {
		pr_err("ftrace: no module PLT for %ps\n", (void *)*addr);
		return false;
	}

	*addr = (unsigned long)plt;
	return true;
}
#else /* !CONFIG_MODULES */
static bool ftrace_find_callable_addr(struct dyn_ftrace *rec, struct module *mod, unsigned long *addr)
{
	return true;
}
#endif

#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr)
{
	u32 old, new;
	unsigned long pc;

	pc = rec->ip + LOONGARCH_INSN_SIZE;

	if (!ftrace_find_callable_addr(rec, NULL, &addr))
		return -EINVAL;

	if (!ftrace_find_callable_addr(rec, NULL, &old_addr))
		return -EINVAL;

	new = larch_insn_gen_bl(pc, addr);
	old = larch_insn_gen_bl(pc, old_addr);

	return ftrace_modify_code(pc, old, new, true);
}
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */

int ftrace_update_ftrace_func(ftrace_func_t func)
{
	u32 new;
	unsigned long pc;

	pc = (unsigned long)&ftrace_call;
	new = larch_insn_gen_bl(pc, (unsigned long)func);

	return ftrace_modify_code(pc, 0, new, false);
}

/*
 * The compiler has inserted 2 NOPs before the regular function prologue.
 * T series registers are available and safe because of LoongArch's psABI.
 *
 * At runtime, we can replace nop with bl to enable ftrace call and replace bl
 * with nop to disable ftrace call. The bl requires us to save the original RA
 * value, so it saves RA at t0 here.
 *
 * Details are:
 *
 * | Compiled   |       Disabled         |        Enabled         |
 * +------------+------------------------+------------------------+
 * | nop        | move     t0, ra        | move     t0, ra        |
 * | nop        | nop                    | bl       ftrace_caller |
 * | func_body  | func_body              | func_body              |
 *
 * The RA value will be recovered by ftrace_regs_entry, and restored into RA
 * before returning to the regular function prologue. When a function is not
 * being traced, the "move t0, ra" is not harmful.
 */

int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
{
	u32 old, new;
	unsigned long pc;

	pc = rec->ip;
	old = larch_insn_gen_nop();
	new = larch_insn_gen_move(LOONGARCH_GPR_T0, LOONGARCH_GPR_RA);

	return ftrace_modify_code(pc, old, new, true);
}

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
	u32 old, new;
	unsigned long pc;

	pc = rec->ip + LOONGARCH_INSN_SIZE;

	if (!ftrace_find_callable_addr(rec, NULL, &addr))
		return -EINVAL;

	old = larch_insn_gen_nop();
	new = larch_insn_gen_bl(pc, addr);

	return ftrace_modify_code(pc, old, new, true);
}

int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr)
{
	u32 old, new;
	unsigned long pc;

	pc = rec->ip + LOONGARCH_INSN_SIZE;

	if (!ftrace_find_callable_addr(rec, NULL, &addr))
		return -EINVAL;

	new = larch_insn_gen_nop();
	old = larch_insn_gen_bl(pc, addr);

	return ftrace_modify_code(pc, old, new, true);
}

void arch_ftrace_update_code(int command)
{
	command |= FTRACE_MAY_SLEEP;
	ftrace_modify_all_code(command);
}

int __init ftrace_dyn_arch_init(void)
{
	return 0;
}

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent)
{
	unsigned long old;
	unsigned long return_hooker = (unsigned long)&return_to_handler;

	if (unlikely(atomic_read(&current->tracing_graph_pause)))
		return;

	old = *parent;

	if (!function_graph_enter(old, self_addr, 0, parent))
		*parent = return_hooker;
}

#ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
		       struct ftrace_ops *op, struct ftrace_regs *fregs)
{
	struct pt_regs *regs = &fregs->regs;
	unsigned long *parent = (unsigned long *)&regs->regs[1];

	prepare_ftrace_return(ip, (unsigned long *)parent);
}
#else
static int ftrace_modify_graph_caller(bool enable)
{
	u32 branch, nop;
	unsigned long pc, func;
	extern void ftrace_graph_call(void);

	pc = (unsigned long)&ftrace_graph_call;
	func = (unsigned long)&ftrace_graph_caller;

	nop = larch_insn_gen_nop();
	branch = larch_insn_gen_b(pc, func);

	if (enable)
		return ftrace_modify_code(pc, nop, branch, true);
	else
		return ftrace_modify_code(pc, branch, nop, true);
}

int ftrace_enable_ftrace_graph_caller(void)
{
	return ftrace_modify_graph_caller(true);
}

int ftrace_disable_ftrace_graph_caller(void)
{
	return ftrace_modify_graph_caller(false);
}
#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

#ifdef CONFIG_KPROBES_ON_FTRACE
/* Ftrace callback handler for kprobes -- called under preepmt disabled */
void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
			   struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
	int bit;
	struct pt_regs *regs;
	struct kprobe *p;
	struct kprobe_ctlblk *kcb;

	if (unlikely(kprobe_ftrace_disabled))
		return;

	bit = ftrace_test_recursion_trylock(ip, parent_ip);
	if (bit < 0)
		return;

	p = get_kprobe((kprobe_opcode_t *)ip);
	if (unlikely(!p) || kprobe_disabled(p))
		goto out;

	regs = ftrace_get_regs(fregs);
	if (!regs)
		goto out;

	kcb = get_kprobe_ctlblk();
	if (kprobe_running()) {
		kprobes_inc_nmissed_count(p);
	} else {
		unsigned long orig_ip = instruction_pointer(regs);

		instruction_pointer_set(regs, ip);

		__this_cpu_write(current_kprobe, p);
		kcb->kprobe_status = KPROBE_HIT_ACTIVE;
		if (!p->pre_handler || !p->pre_handler(p, regs)) {
			/*
			 * Emulate singlestep (and also recover regs->csr_era)
			 * as if there is a nop
			 */
			instruction_pointer_set(regs, (unsigned long)p->addr + MCOUNT_INSN_SIZE);
			if (unlikely(p->post_handler)) {
				kcb->kprobe_status = KPROBE_HIT_SSDONE;
				p->post_handler(p, regs, 0);
			}
			instruction_pointer_set(regs, orig_ip);
		}

		/*
		 * If pre_handler returns !0, it changes regs->csr_era. We have to
		 * skip emulating post_handler.
		 */
		__this_cpu_write(current_kprobe, NULL);
	}
out:
	ftrace_test_recursion_unlock(bit);
}
NOKPROBE_SYMBOL(kprobe_ftrace_handler);

int arch_prepare_kprobe_ftrace(struct kprobe *p)
{
	p->ainsn.insn = NULL;
	return 0;
}
#endif /* CONFIG_KPROBES_ON_FTRACE */
Commit	Line	Data
4733f09d QZ	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Based on arch/arm64/kernel/ftrace.c
	4	*
	5	* Copyright (C) 2022 Loongson Technology Corporation Limited
	6	*/
	7
	8	#include <linux/ftrace.h>
09e679c2	9	#include <linux/kprobes.h>
4733f09d QZ	10	#include <linux/uaccess.h>
	11
	12	#include <asm/inst.h>
28ac0a9e	13	#include <asm/module.h>
4733f09d QZ	14
	15	static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, bool validate)
	16	{
	17	u32 replaced;
	18
	19	if (validate) {
	20	if (larch_insn_read((void *)pc, &replaced))
	21	return -EFAULT;
	22
	23	if (replaced != old)
	24	return -EINVAL;
	25	}
	26
	27	if (larch_insn_patch_text((void *)pc, new))
	28	return -EPERM;
	29
	30	return 0;
	31	}
	32
28ac0a9e	33	#ifdef CONFIG_MODULES
24d4f527	34	static bool reachable_by_bl(unsigned long addr, unsigned long pc)
28ac0a9e	35	{
24d4f527	36	long offset = (long)addr - (long)pc;
28ac0a9e	37
24d4f527	38	return offset >= -SZ_128M && offset < SZ_128M;
28ac0a9e QZ	39	}
	40
	41	static struct plt_entry get_ftrace_plt(struct module mod, unsigned long addr)
	42	{
	43	struct plt_entry *plt = mod->arch.ftrace_trampolines;
	44
	45	if (addr == FTRACE_ADDR)
	46	return &plt[FTRACE_PLT_IDX];
	47	if (addr == FTRACE_REGS_ADDR &&
	48	IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
	49	return &plt[FTRACE_REGS_PLT_IDX];
	50
	51	return NULL;
	52	}
	53
24d4f527 YT	54	/*
	55	* Find the address the callsite must branch to in order to reach '*addr'.
	56	*
	57	* Due to the limited range of 'bl' instruction, modules may be placed too far
	58	* away to branch directly and we must use a PLT.
	59	*
	60	* Returns true when '*addr' contains a reachable target address, or has been
	61	* modified to contain a PLT address. Returns false otherwise.
	62	*/
	63	static bool ftrace_find_callable_addr(struct dyn_ftrace rec, struct module mod, unsigned long *addr)
28ac0a9e	64	{
24d4f527	65	unsigned long pc = rec->ip + LOONGARCH_INSN_SIZE;
28ac0a9e QZ	66	struct plt_entry *plt;
28ac0a9e QZ	67
9cdc3b6a YT	68	/*
	69	* If a custom trampoline is unreachable, rely on the ftrace_regs_caller
	70	* trampoline which knows how to indirectly reach that trampoline through
	71	* ops->direct_call.
	72	*/
	73	if (addr != FTRACE_ADDR && addr != FTRACE_REGS_ADDR && !reachable_by_bl(*addr, pc))
	74	*addr = FTRACE_REGS_ADDR;
	75
24d4f527 YT	76	/*
	77	* When the target is within range of the 'bl' instruction, use 'addr'
	78	* as-is and branch to that directly.
	79	*/
	80	if (reachable_by_bl(*addr, pc))
	81	return true;
	82
	83	/*
	84	* 'mod' is only set at module load time, but if we end up
	85	* dealing with an out-of-range condition, we can assume it
	86	* is due to a module being loaded far away from the kernel.
	87	*
	88	* NOTE: __module_text_address() must be called with preemption
	89	* disabled, but we can rely on ftrace_lock to ensure that 'mod'
	90	* retains its validity throughout the remainder of this code.
	91	*/
	92	if (!mod) {
	93	preempt_disable();
	94	mod = __module_text_address(pc);
	95	preempt_enable();
	96	}
	97
	98	if (WARN_ON(!mod))
	99	return false;
	100
	101	plt = get_ftrace_plt(mod, *addr);
28ac0a9e	102	if (!plt) {
24d4f527 YT	103	pr_err("ftrace: no module PLT for %ps\n", (void )addr);
24d4f527 YT	104	return false;
28ac0a9e QZ	105	}
28ac0a9e QZ	106
24d4f527 YT	107	*addr = (unsigned long)plt;
	108	return true;
	109	}
	110	#else /* !CONFIG_MODULES */
	111	static bool ftrace_find_callable_addr(struct dyn_ftrace rec, struct module mod, unsigned long *addr)
	112	{
	113	return true;
28ac0a9e QZ	114	}
	115	#endif
	116
819cf655	117	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
8778ba2c QZ	118	int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr)
	119	{
	120	u32 old, new;
	121	unsigned long pc;
	122
	123	pc = rec->ip + LOONGARCH_INSN_SIZE;
	124
24d4f527 YT	125	if (!ftrace_find_callable_addr(rec, NULL, &addr))
24d4f527 YT	126	return -EINVAL;
28ac0a9e	127
24d4f527 YT	128	if (!ftrace_find_callable_addr(rec, NULL, &old_addr))
24d4f527 YT	129	return -EINVAL;
28ac0a9e	130
8778ba2c QZ	131	new = larch_insn_gen_bl(pc, addr);
	132	old = larch_insn_gen_bl(pc, old_addr);
	133
	134	return ftrace_modify_code(pc, old, new, true);
	135	}
	136	#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
	137
4733f09d QZ	138	int ftrace_update_ftrace_func(ftrace_func_t func)
	139	{
	140	u32 new;
	141	unsigned long pc;
	142
	143	pc = (unsigned long)&ftrace_call;
	144	new = larch_insn_gen_bl(pc, (unsigned long)func);
	145
	146	return ftrace_modify_code(pc, 0, new, false);
	147	}
	148
	149	/*
	150	* The compiler has inserted 2 NOPs before the regular function prologue.
	151	* T series registers are available and safe because of LoongArch's psABI.
	152	*
	153	* At runtime, we can replace nop with bl to enable ftrace call and replace bl
	154	* with nop to disable ftrace call. The bl requires us to save the original RA
	155	* value, so it saves RA at t0 here.
	156	*
	157	* Details are:
	158	*
	159	* \| Compiled \| Disabled \| Enabled \|
	160	* +------------+------------------------+------------------------+
	161	* \| nop \| move t0, ra \| move t0, ra \|
	162	* \| nop \| nop \| bl ftrace_caller \|
	163	* \| func_body \| func_body \| func_body \|
	164	*
	165	* The RA value will be recovered by ftrace_regs_entry, and restored into RA
	166	* before returning to the regular function prologue. When a function is not
	167	* being traced, the "move t0, ra" is not harmful.
	168	*/
	169
	170	int ftrace_init_nop(struct module mod, struct dyn_ftrace rec)
	171	{
	172	u32 old, new;
	173	unsigned long pc;
	174
	175	pc = rec->ip;
	176	old = larch_insn_gen_nop();
	177	new = larch_insn_gen_move(LOONGARCH_GPR_T0, LOONGARCH_GPR_RA);
	178
	179	return ftrace_modify_code(pc, old, new, true);
	180	}
	181
	182	int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
	183	{
	184	u32 old, new;
	185	unsigned long pc;
	186
	187	pc = rec->ip + LOONGARCH_INSN_SIZE;
	188
24d4f527 YT	189	if (!ftrace_find_callable_addr(rec, NULL, &addr))
24d4f527 YT	190	return -EINVAL;
28ac0a9e	191
4733f09d QZ	192	old = larch_insn_gen_nop();
	193	new = larch_insn_gen_bl(pc, addr);
	194
	195	return ftrace_modify_code(pc, old, new, true);
	196	}
	197
	198	int ftrace_make_nop(struct module mod, struct dyn_ftrace rec, unsigned long addr)
	199	{
	200	u32 old, new;
	201	unsigned long pc;
	202
	203	pc = rec->ip + LOONGARCH_INSN_SIZE;
	204
24d4f527 YT	205	if (!ftrace_find_callable_addr(rec, NULL, &addr))
24d4f527 YT	206	return -EINVAL;
28ac0a9e	207
4733f09d QZ	208	new = larch_insn_gen_nop();
	209	old = larch_insn_gen_bl(pc, addr);
	210
	211	return ftrace_modify_code(pc, old, new, true);
	212	}
	213
	214	void arch_ftrace_update_code(int command)
	215	{
	216	command \|= FTRACE_MAY_SLEEP;
	217	ftrace_modify_all_code(command);
	218	}
	219
	220	int __init ftrace_dyn_arch_init(void)
	221	{
	222	return 0;
	223	}
5fcfad3d QZ	224
	225	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	226	void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent)
	227	{
	228	unsigned long old;
	229	unsigned long return_hooker = (unsigned long)&return_to_handler;
	230
	231	if (unlikely(atomic_read(&current->tracing_graph_pause)))
	232	return;
	233
	234	old = *parent;
	235
a51ac524	236	if (!function_graph_enter(old, self_addr, 0, parent))
5fcfad3d QZ	237	*parent = return_hooker;
	238	}
	239
ac7127e1 QZ	240	#ifdef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
	241	void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
	242	struct ftrace_ops op, struct ftrace_regs fregs)
	243	{
	244	struct pt_regs *regs = &fregs->regs;
	245	unsigned long parent = (unsigned long )&regs->regs[1];
	246
	247	prepare_ftrace_return(ip, (unsigned long *)parent);
	248	}
	249	#else
5fcfad3d QZ	250	static int ftrace_modify_graph_caller(bool enable)
	251	{
	252	u32 branch, nop;
	253	unsigned long pc, func;
	254	extern void ftrace_graph_call(void);
	255
	256	pc = (unsigned long)&ftrace_graph_call;
	257	func = (unsigned long)&ftrace_graph_caller;
	258
	259	nop = larch_insn_gen_nop();
	260	branch = larch_insn_gen_b(pc, func);
	261
	262	if (enable)
	263	return ftrace_modify_code(pc, nop, branch, true);
	264	else
	265	return ftrace_modify_code(pc, branch, nop, true);
	266	}
	267
	268	int ftrace_enable_ftrace_graph_caller(void)
	269	{
	270	return ftrace_modify_graph_caller(true);
	271	}
	272
	273	int ftrace_disable_ftrace_graph_caller(void)
	274	{
	275	return ftrace_modify_graph_caller(false);
	276	}
ac7127e1	277	#endif /* CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS */
5fcfad3d	278	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
09e679c2 TY	279
	280	#ifdef CONFIG_KPROBES_ON_FTRACE
	281	/* Ftrace callback handler for kprobes -- called under preepmt disabled */
	282	void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
	283	struct ftrace_ops ops, struct ftrace_regs fregs)
	284	{
	285	int bit;
	286	struct pt_regs *regs;
	287	struct kprobe *p;
	288	struct kprobe_ctlblk *kcb;
	289
1a7d0890 SB	290	if (unlikely(kprobe_ftrace_disabled))
	291	return;
	292
09e679c2 TY	293	bit = ftrace_test_recursion_trylock(ip, parent_ip);
	294	if (bit < 0)
	295	return;
	296
	297	p = get_kprobe((kprobe_opcode_t *)ip);
	298	if (unlikely(!p) \|\| kprobe_disabled(p))
	299	goto out;
	300
	301	regs = ftrace_get_regs(fregs);
	302	if (!regs)
	303	goto out;
	304
	305	kcb = get_kprobe_ctlblk();
	306	if (kprobe_running()) {
	307	kprobes_inc_nmissed_count(p);
	308	} else {
	309	unsigned long orig_ip = instruction_pointer(regs);
	310
	311	instruction_pointer_set(regs, ip);
	312
	313	__this_cpu_write(current_kprobe, p);
	314	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
	315	if (!p->pre_handler \|\| !p->pre_handler(p, regs)) {
	316	/*
	317	* Emulate singlestep (and also recover regs->csr_era)
	318	* as if there is a nop
	319	*/
	320	instruction_pointer_set(regs, (unsigned long)p->addr + MCOUNT_INSN_SIZE);
	321	if (unlikely(p->post_handler)) {
	322	kcb->kprobe_status = KPROBE_HIT_SSDONE;
	323	p->post_handler(p, regs, 0);
	324	}
	325	instruction_pointer_set(regs, orig_ip);
	326	}
	327
	328	/*
	329	* If pre_handler returns !0, it changes regs->csr_era. We have to
	330	* skip emulating post_handler.
	331	*/
	332	__this_cpu_write(current_kprobe, NULL);
	333	}
	334	out:
	335	ftrace_test_recursion_unlock(bit);
	336	}
	337	NOKPROBE_SYMBOL(kprobe_ftrace_handler);
	338
	339	int arch_prepare_kprobe_ftrace(struct kprobe *p)
	340	{
	341	p->ainsn.insn = NULL;
	342	return 0;
	343	}
	344	#endif /* CONFIG_KPROBES_ON_FTRACE */