[linux-block.git] / arch / s390 / kernel / ftrace.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Dynamic function tracer architecture backend.
 *
 * Copyright IBM Corp. 2009,2014
 *
 *   Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
 */

#include <linux/moduleloader.h>
#include <linux/hardirq.h>
#include <linux/uaccess.h>
#include <linux/ftrace.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/kprobes.h>
#include <trace/syscall.h>
#include <asm/asm-offsets.h>
#include <asm/text-patching.h>
#include <asm/cacheflush.h>
#include <asm/ftrace.lds.h>
#include <asm/nospec-branch.h>
#include <asm/set_memory.h>
#include "entry.h"
#include "ftrace.h"

/*
 * To generate function prologue either gcc's hotpatch feature (since gcc 4.8)
 * or a combination of -pg -mrecord-mcount -mnop-mcount -mfentry flags
 * (since gcc 9 / clang 10) is used.
 * In both cases the original and also the disabled function prologue contains
 * only a single six byte instruction and looks like this:
 * >	brcl	0,0			# offset 0
 * To enable ftrace the code gets patched like above and afterwards looks
 * like this:
 * >	brasl	%r0,ftrace_caller	# offset 0
 *
 * The instruction will be patched by ftrace_make_call / ftrace_make_nop.
 * The ftrace function gets called with a non-standard C function call ABI
 * where r0 contains the return address. It is also expected that the called
 * function only clobbers r0 and r1, but restores r2-r15.
 * For module code we can't directly jump to ftrace caller, but need a
 * trampoline (ftrace_plt), which clobbers also r1.
 */

void *ftrace_func __read_mostly = ftrace_stub;
struct ftrace_insn {
	u16 opc;
	s32 disp;
} __packed;

#ifdef CONFIG_MODULES
static char *ftrace_plt;
#endif /* CONFIG_MODULES */

static const char *ftrace_shared_hotpatch_trampoline(const char **end)
{
	const char *tstart, *tend;

	tstart = ftrace_shared_hotpatch_trampoline_br;
	tend = ftrace_shared_hotpatch_trampoline_br_end;
#ifdef CONFIG_EXPOLINE
	if (!nospec_disable) {
		tstart = ftrace_shared_hotpatch_trampoline_exrl;
		tend = ftrace_shared_hotpatch_trampoline_exrl_end;
	}
#endif /* CONFIG_EXPOLINE */
	if (end)
		*end = tend;
	return tstart;
}

bool ftrace_need_init_nop(void)
{
	return true;
}

int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
{
	static struct ftrace_hotpatch_trampoline *next_vmlinux_trampoline =
		__ftrace_hotpatch_trampolines_start;
	static const char orig[6] = { 0xc0, 0x04, 0x00, 0x00, 0x00, 0x00 };
	static struct ftrace_hotpatch_trampoline *trampoline;
	struct ftrace_hotpatch_trampoline **next_trampoline;
	struct ftrace_hotpatch_trampoline *trampolines_end;
	struct ftrace_hotpatch_trampoline tmp;
	struct ftrace_insn *insn;
	const char *shared;
	s32 disp;

	BUILD_BUG_ON(sizeof(struct ftrace_hotpatch_trampoline) !=
		     SIZEOF_FTRACE_HOTPATCH_TRAMPOLINE);

	next_trampoline = &next_vmlinux_trampoline;
	trampolines_end = __ftrace_hotpatch_trampolines_end;
	shared = ftrace_shared_hotpatch_trampoline(NULL);
#ifdef CONFIG_MODULES
	if (mod) {
		next_trampoline = &mod->arch.next_trampoline;
		trampolines_end = mod->arch.trampolines_end;
		shared = ftrace_plt;
	}
#endif

	if (WARN_ON_ONCE(*next_trampoline >= trampolines_end))
		return -ENOMEM;
	trampoline = (*next_trampoline)++;

	/* Check for the compiler-generated fentry nop (brcl 0, .). */
	if (WARN_ON_ONCE(memcmp((const void *)rec->ip, &orig, sizeof(orig))))
		return -EINVAL;

	/* Generate the trampoline. */
	tmp.brasl_opc = 0xc015; /* brasl %r1, shared */
	tmp.brasl_disp = (shared - (const char *)&trampoline->brasl_opc) / 2;
	tmp.interceptor = FTRACE_ADDR;
	tmp.rest_of_intercepted_function = rec->ip + sizeof(struct ftrace_insn);
	s390_kernel_write(trampoline, &tmp, sizeof(tmp));

	/* Generate a jump to the trampoline. */
	disp = ((char *)trampoline - (char *)rec->ip) / 2;
	insn = (struct ftrace_insn *)rec->ip;
	s390_kernel_write(&insn->disp, &disp, sizeof(disp));

	return 0;
}

static struct ftrace_hotpatch_trampoline *ftrace_get_trampoline(struct dyn_ftrace *rec)
{
	struct ftrace_hotpatch_trampoline *trampoline;
	struct ftrace_insn insn;
	s64 disp;
	u16 opc;

	if (copy_from_kernel_nofault(&insn, (void *)rec->ip, sizeof(insn)))
		return ERR_PTR(-EFAULT);
	disp = (s64)insn.disp * 2;
	trampoline = (void *)(rec->ip + disp);
	if (get_kernel_nofault(opc, &trampoline->brasl_opc))
		return ERR_PTR(-EFAULT);
	if (opc != 0xc015)
		return ERR_PTR(-EINVAL);
	return trampoline;
}

int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
		       unsigned long addr)
{
	struct ftrace_hotpatch_trampoline *trampoline;
	u64 old;

	trampoline = ftrace_get_trampoline(rec);
	if (IS_ERR(trampoline))
		return PTR_ERR(trampoline);
	if (get_kernel_nofault(old, &trampoline->interceptor))
		return -EFAULT;
	if (old != old_addr)
		return -EINVAL;
	s390_kernel_write(&trampoline->interceptor, &addr, sizeof(addr));
	return 0;
}

static int ftrace_patch_branch_mask(void *addr, u16 expected, bool enable)
{
	u16 old;
	u8 op;

	if (get_kernel_nofault(old, addr))
		return -EFAULT;
	if (old != expected)
		return -EINVAL;
	/* set mask field to all ones or zeroes */
	op = enable ? 0xf4 : 0x04;
	s390_kernel_write((char *)addr + 1, &op, sizeof(op));
	return 0;
}

int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
		    unsigned long addr)
{
	/* Expect brcl 0xf,... */
	return ftrace_patch_branch_mask((void *)rec->ip, 0xc0f4, false);
}

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
	struct ftrace_hotpatch_trampoline *trampoline;

	trampoline = ftrace_get_trampoline(rec);
	if (IS_ERR(trampoline))
		return PTR_ERR(trampoline);
	s390_kernel_write(&trampoline->interceptor, &addr, sizeof(addr));
	/* Expect brcl 0x0,... */
	return ftrace_patch_branch_mask((void *)rec->ip, 0xc004, true);
}

int ftrace_update_ftrace_func(ftrace_func_t func)
{
	ftrace_func = func;
	return 0;
}

void arch_ftrace_update_code(int command)
{
	ftrace_modify_all_code(command);
}

void ftrace_arch_code_modify_post_process(void)
{
	/*
	 * Flush any pre-fetched instructions on all
	 * CPUs to make the new code visible.
	 */
	text_poke_sync_lock();
}

#ifdef CONFIG_MODULES

static int __init ftrace_plt_init(void)
{
	const char *start, *end;

	ftrace_plt = module_alloc(PAGE_SIZE);
	if (!ftrace_plt)
		panic("cannot allocate ftrace plt\n");

	start = ftrace_shared_hotpatch_trampoline(&end);
	memcpy(ftrace_plt, start, end - start);
	set_memory_rox((unsigned long)ftrace_plt, 1);
	return 0;
}
device_initcall(ftrace_plt_init);

#endif /* CONFIG_MODULES */

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/*
 * Hook the return address and push it in the stack of return addresses
 * in current thread info.
 */
unsigned long prepare_ftrace_return(unsigned long ra, unsigned long sp,
				    unsigned long ip)
{
	if (unlikely(ftrace_graph_is_dead()))
		goto out;
	if (unlikely(atomic_read(&current->tracing_graph_pause)))
		goto out;
	ip -= MCOUNT_INSN_SIZE;
	if (!function_graph_enter(ra, ip, 0, (void *) sp))
		ra = (unsigned long) return_to_handler;
out:
	return ra;
}
NOKPROBE_SYMBOL(prepare_ftrace_return);

/*
 * Patch the kernel code at ftrace_graph_caller location. The instruction
 * there is branch relative on condition. To enable the ftrace graph code
 * block, we simply patch the mask field of the instruction to zero and
 * turn the instruction into a nop.
 * To disable the ftrace graph code the mask field will be patched to
 * all ones, which turns the instruction into an unconditional branch.
 */
int ftrace_enable_ftrace_graph_caller(void)
{
	int rc;

	/* Expect brc 0xf,... */
	rc = ftrace_patch_branch_mask(ftrace_graph_caller, 0xa7f4, false);
	if (rc)
		return rc;
	text_poke_sync_lock();
	return 0;
}

int ftrace_disable_ftrace_graph_caller(void)
{
	int rc;

	/* Expect brc 0x0,... */
	rc = ftrace_patch_branch_mask(ftrace_graph_caller, 0xa704, true);
	if (rc)
		return rc;
	text_poke_sync_lock();
	return 0;
}

#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

#ifdef CONFIG_KPROBES_ON_FTRACE
void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
		struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
	struct kprobe_ctlblk *kcb;
	struct pt_regs *regs;
	struct kprobe *p;
	int bit;

	bit = ftrace_test_recursion_trylock(ip, parent_ip);
	if (bit < 0)
		return;

	regs = ftrace_get_regs(fregs);
	p = get_kprobe((kprobe_opcode_t *)ip);
	if (!regs || unlikely(!p) || kprobe_disabled(p))
		goto out;

	if (kprobe_running()) {
		kprobes_inc_nmissed_count(p);
		goto out;
	}

	__this_cpu_write(current_kprobe, p);

	kcb = get_kprobe_ctlblk();
	kcb->kprobe_status = KPROBE_HIT_ACTIVE;

	instruction_pointer_set(regs, ip);

	if (!p->pre_handler || !p->pre_handler(p, regs)) {

		instruction_pointer_set(regs, ip + MCOUNT_INSN_SIZE);

		if (unlikely(p->post_handler)) {
			kcb->kprobe_status = KPROBE_HIT_SSDONE;
			p->post_handler(p, regs, 0);
		}
	}
	__this_cpu_write(current_kprobe, NULL);
out:
	ftrace_test_recursion_unlock(bit);
}
NOKPROBE_SYMBOL(kprobe_ftrace_handler);

int arch_prepare_kprobe_ftrace(struct kprobe *p)
{
	p->ainsn.insn = NULL;
	return 0;
}
#endif
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
dfd9f7ab HC	2	/*
	3	* Dynamic function tracer architecture backend.
	4	*
3d1e220d	5	* Copyright IBM Corp. 2009,2014
dfd9f7ab	6	*
f36e7c98	7	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
dfd9f7ab HC	8	*/
dfd9f7ab HC	9
c933146a	10	#include <linux/moduleloader.h>
88dbd203	11	#include <linux/hardirq.h>
dfd9f7ab HC	12	#include <linux/uaccess.h>
	13	#include <linux/ftrace.h>
	14	#include <linux/kernel.h>
	15	#include <linux/types.h>
4cc9bed0	16	#include <linux/kprobes.h>
9bf1226b	17	#include <trace/syscall.h>
cbb870c8	18	#include <asm/asm-offsets.h>
ae2b9a11	19	#include <asm/text-patching.h>
c933146a	20	#include <asm/cacheflush.h>
de5012b4 IL	21	#include <asm/ftrace.lds.h>
de5012b4 IL	22	#include <asm/nospec-branch.h>
e6c7c630	23	#include <asm/set_memory.h>
63df41d6	24	#include "entry.h"
de5012b4	25	#include "ftrace.h"
dfd9f7ab	26
4cc9bed0	27	/*
c9343637 VG	28	* To generate function prologue either gcc's hotpatch feature (since gcc 4.8)
	29	* or a combination of -pg -mrecord-mcount -mnop-mcount -mfentry flags
	30	* (since gcc 9 / clang 10) is used.
	31	* In both cases the original and also the disabled function prologue contains
	32	* only a single six byte instruction and looks like this:
	33	* > brcl 0,0 # offset 0
	34	* To enable ftrace the code gets patched like above and afterwards looks
	35	* like this:
c933146a	36	* > brasl %r0,ftrace_caller # offset 0
c9343637 VG	37	*
c9343637 VG	38	* The instruction will be patched by ftrace_make_call / ftrace_make_nop.
3d1e220d HC	39	* The ftrace function gets called with a non-standard C function call ABI
	40	* where r0 contains the return address. It is also expected that the called
	41	* function only clobbers r0 and r1, but restores r2-r15.
c933146a HC	42	* For module code we can't directly jump to ftrace caller, but need a
c933146a HC	43	* trampoline (ftrace_plt), which clobbers also r1.
4cc9bed0	44	*/
dfd9f7ab	45
f8c26027	46	void *ftrace_func __read_mostly = ftrace_stub;
de5012b4 IL	47	struct ftrace_insn {
	48	u16 opc;
	49	s32 disp;
	50	} __packed;
	51
de5012b4 IL	52	#ifdef CONFIG_MODULES
de5012b4 IL	53	static char *ftrace_plt;
de5012b4 IL	54	#endif /* CONFIG_MODULES */
	55
	56	static const char ftrace_shared_hotpatch_trampoline(const char *end)
	57	{
	58	const char tstart, tend;
	59
	60	tstart = ftrace_shared_hotpatch_trampoline_br;
	61	tend = ftrace_shared_hotpatch_trampoline_br_end;
	62	#ifdef CONFIG_EXPOLINE
	63	if (!nospec_disable) {
4efd417f VG	64	tstart = ftrace_shared_hotpatch_trampoline_exrl;
4efd417f VG	65	tend = ftrace_shared_hotpatch_trampoline_exrl_end;
de5012b4 IL	66	}
	67	#endif /* CONFIG_EXPOLINE */
	68	if (end)
	69	*end = tend;
	70	return tstart;
	71	}
	72
	73	bool ftrace_need_init_nop(void)
	74	{
b860b934	75	return true;
de5012b4 IL	76	}
	77
	78	int ftrace_init_nop(struct module mod, struct dyn_ftrace rec)
	79	{
	80	static struct ftrace_hotpatch_trampoline *next_vmlinux_trampoline =
	81	__ftrace_hotpatch_trampolines_start;
	82	static const char orig[6] = { 0xc0, 0x04, 0x00, 0x00, 0x00, 0x00 };
	83	static struct ftrace_hotpatch_trampoline *trampoline;
	84	struct ftrace_hotpatch_trampoline **next_trampoline;
	85	struct ftrace_hotpatch_trampoline *trampolines_end;
	86	struct ftrace_hotpatch_trampoline tmp;
	87	struct ftrace_insn *insn;
	88	const char *shared;
	89	s32 disp;
	90
	91	BUILD_BUG_ON(sizeof(struct ftrace_hotpatch_trampoline) !=
	92	SIZEOF_FTRACE_HOTPATCH_TRAMPOLINE);
	93
	94	next_trampoline = &next_vmlinux_trampoline;
	95	trampolines_end = __ftrace_hotpatch_trampolines_end;
	96	shared = ftrace_shared_hotpatch_trampoline(NULL);
	97	#ifdef CONFIG_MODULES
	98	if (mod) {
	99	next_trampoline = &mod->arch.next_trampoline;
	100	trampolines_end = mod->arch.trampolines_end;
	101	shared = ftrace_plt;
	102	}
	103	#endif
	104
	105	if (WARN_ON_ONCE(*next_trampoline >= trampolines_end))
	106	return -ENOMEM;
	107	trampoline = (*next_trampoline)++;
	108
	109	/* Check for the compiler-generated fentry nop (brcl 0, .). */
	110	if (WARN_ON_ONCE(memcmp((const void *)rec->ip, &orig, sizeof(orig))))
	111	return -EINVAL;
	112
	113	/* Generate the trampoline. */
	114	tmp.brasl_opc = 0xc015; /* brasl %r1, shared */
	115	tmp.brasl_disp = (shared - (const char *)&trampoline->brasl_opc) / 2;
	116	tmp.interceptor = FTRACE_ADDR;
	117	tmp.rest_of_intercepted_function = rec->ip + sizeof(struct ftrace_insn);
	118	s390_kernel_write(trampoline, &tmp, sizeof(tmp));
	119
	120	/* Generate a jump to the trampoline. */
	121	disp = ((char )trampoline - (char )rec->ip) / 2;
	122	insn = (struct ftrace_insn *)rec->ip;
	123	s390_kernel_write(&insn->disp, &disp, sizeof(disp));
	124
	125	return 0;
	126	}
dfd9f7ab	127
9fa881f7 HC	128	static struct ftrace_hotpatch_trampoline ftrace_get_trampoline(struct dyn_ftrace rec)
	129	{
	130	struct ftrace_hotpatch_trampoline *trampoline;
	131	struct ftrace_insn insn;
	132	s64 disp;
	133	u16 opc;
	134
	135	if (copy_from_kernel_nofault(&insn, (void *)rec->ip, sizeof(insn)))
	136	return ERR_PTR(-EFAULT);
	137	disp = (s64)insn.disp * 2;
	138	trampoline = (void *)(rec->ip + disp);
	139	if (get_kernel_nofault(opc, &trampoline->brasl_opc))
	140	return ERR_PTR(-EFAULT);
	141	if (opc != 0xc015)
	142	return ERR_PTR(-EINVAL);
	143	return trampoline;
	144	}
	145
10dec7db HC	146	int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
	147	unsigned long addr)
	148	{
9fa881f7 HC	149	struct ftrace_hotpatch_trampoline *trampoline;
	150	u64 old;
	151
	152	trampoline = ftrace_get_trampoline(rec);
	153	if (IS_ERR(trampoline))
	154	return PTR_ERR(trampoline);
	155	if (get_kernel_nofault(old, &trampoline->interceptor))
	156	return -EFAULT;
	157	if (old != old_addr)
	158	return -EINVAL;
	159	s390_kernel_write(&trampoline->interceptor, &addr, sizeof(addr));
10dec7db HC	160	return 0;
10dec7db HC	161	}
dfd9f7ab	162
98c0d24d	163	static int ftrace_patch_branch_mask(void *addr, u16 expected, bool enable)
de5012b4	164	{
98c0d24d SS	165	u16 old;
98c0d24d SS	166	u8 op;
de5012b4	167
98c0d24d SS	168	if (get_kernel_nofault(old, addr))
	169	return -EFAULT;
	170	if (old != expected)
	171	return -EINVAL;
	172	/* set mask field to all ones or zeroes */
	173	op = enable ? 0xf4 : 0x04;
	174	s390_kernel_write((char *)addr + 1, &op, sizeof(op));
	175	return 0;
de5012b4 IL	176	}
de5012b4 IL	177
dfd9f7ab HC	178	int ftrace_make_nop(struct module mod, struct dyn_ftrace rec,
	179	unsigned long addr)
	180	{
98c0d24d SS	181	/* Expect brcl 0xf,... */
98c0d24d SS	182	return ftrace_patch_branch_mask((void *)rec->ip, 0xc0f4, false);
de5012b4 IL	183	}
de5012b4 IL	184
dfd9f7ab HC	185	int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
dfd9f7ab HC	186	{
9fa881f7 HC	187	struct ftrace_hotpatch_trampoline *trampoline;
	188
	189	trampoline = ftrace_get_trampoline(rec);
	190	if (IS_ERR(trampoline))
	191	return PTR_ERR(trampoline);
	192	s390_kernel_write(&trampoline->interceptor, &addr, sizeof(addr));
98c0d24d SS	193	/* Expect brcl 0x0,... */
98c0d24d SS	194	return ftrace_patch_branch_mask((void *)rec->ip, 0xc004, true);
dfd9f7ab HC	195	}
	196
	197	int ftrace_update_ftrace_func(ftrace_func_t func)
	198	{
f8c26027	199	ftrace_func = func;
dfd9f7ab HC	200	return 0;
	201	}
	202
de5012b4 IL	203	void arch_ftrace_update_code(int command)
de5012b4 IL	204	{
b860b934	205	ftrace_modify_all_code(command);
de5012b4 IL	206	}
de5012b4 IL	207
3a2bfec0	208	void ftrace_arch_code_modify_post_process(void)
de5012b4	209	{
ae2b9a11 HC	210	/*
	211	* Flush any pre-fetched instructions on all
	212	* CPUs to make the new code visible.
	213	*/
	214	text_poke_sync_lock();
de5012b4 IL	215	}
de5012b4 IL	216
085b6ba0 HC	217	#ifdef CONFIG_MODULES
085b6ba0 HC	218
c933146a HC	219	static int __init ftrace_plt_init(void)
c933146a HC	220	{
de5012b4	221	const char start, end;
c933146a	222
de5012b4	223	ftrace_plt = module_alloc(PAGE_SIZE);
c933146a HC	224	if (!ftrace_plt)
c933146a HC	225	panic("cannot allocate ftrace plt\n");
de5012b4 IL	226
de5012b4 IL	227	start = ftrace_shared_hotpatch_trampoline(&end);
de5012b4	228	memcpy(ftrace_plt, start, end - start);
7c7ab788	229	set_memory_rox((unsigned long)ftrace_plt, 1);
c933146a HC	230	return 0;
	231	}
	232	device_initcall(ftrace_plt_init);
	233
085b6ba0 HC	234	#endif /* CONFIG_MODULES */
085b6ba0 HC	235
88dbd203	236	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
88dbd203 HC	237	/*
	238	* Hook the return address and push it in the stack of return addresses
	239	* in current thread info.
	240	*/
ec7bf478 MS	241	unsigned long prepare_ftrace_return(unsigned long ra, unsigned long sp,
ec7bf478 MS	242	unsigned long ip)
88dbd203	243	{
6ed15ea6 HC	244	if (unlikely(ftrace_graph_is_dead()))
6ed15ea6 HC	245	goto out;
88dbd203 HC	246	if (unlikely(atomic_read(&current->tracing_graph_pause)))
88dbd203 HC	247	goto out;
9cb1ccec	248	ip -= MCOUNT_INSN_SIZE;
ec7bf478 MS	249	if (!function_graph_enter(ra, ip, 0, (void *) sp))
ec7bf478 MS	250	ra = (unsigned long) return_to_handler;
88dbd203	251	out:
ec7bf478	252	return ra;
88dbd203	253	}
7a5388de	254	NOKPROBE_SYMBOL(prepare_ftrace_return);
4cc9bed0	255
4cc9bed0 MS	256	/*
4cc9bed0 MS	257	* Patch the kernel code at ftrace_graph_caller location. The instruction
0cccdda8 HC	258	* there is branch relative on condition. To enable the ftrace graph code
	259	* block, we simply patch the mask field of the instruction to zero and
	260	* turn the instruction into a nop.
	261	* To disable the ftrace graph code the mask field will be patched to
	262	* all ones, which turns the instruction into an unconditional branch.
4cc9bed0	263	*/
2481a87b HC	264	int ftrace_enable_ftrace_graph_caller(void)
2481a87b HC	265	{
98c0d24d SS	266	int rc;
	267
	268	/* Expect brc 0xf,... */
	269	rc = ftrace_patch_branch_mask(ftrace_graph_caller, 0xa7f4, false);
	270	if (rc)
	271	return rc;
e5873d6f	272	text_poke_sync_lock();
8a5d8473	273	return 0;
2481a87b HC	274	}
	275
	276	int ftrace_disable_ftrace_graph_caller(void)
	277	{
98c0d24d SS	278	int rc;
	279
	280	/* Expect brc 0x0,... */
	281	rc = ftrace_patch_branch_mask(ftrace_graph_caller, 0xa704, true);
	282	if (rc)
	283	return rc;
e5873d6f	284	text_poke_sync_lock();
8a5d8473	285	return 0;
2481a87b HC	286	}
2481a87b HC	287
88dbd203	288	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
657480d9 SS	289
	290	#ifdef CONFIG_KPROBES_ON_FTRACE
	291	void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
d19ad077	292	struct ftrace_ops ops, struct ftrace_regs fregs)
657480d9 SS	293	{
657480d9 SS	294	struct kprobe_ctlblk *kcb;
d19ad077	295	struct pt_regs *regs;
c536aa1c SRV	296	struct kprobe *p;
c536aa1c SRV	297	int bit;
657480d9	298
773c1670	299	bit = ftrace_test_recursion_trylock(ip, parent_ip);
c536aa1c	300	if (bit < 0)
657480d9 SS	301	return;
657480d9 SS	302
d19ad077	303	regs = ftrace_get_regs(fregs);
c536aa1c	304	p = get_kprobe((kprobe_opcode_t *)ip);
1389f179	305	if (!regs \|\| unlikely(!p) \|\| kprobe_disabled(p))
c536aa1c SRV	306	goto out;
c536aa1c SRV	307
657480d9 SS	308	if (kprobe_running()) {
657480d9 SS	309	kprobes_inc_nmissed_count(p);
c536aa1c	310	goto out;
657480d9 SS	311	}
	312
	313	__this_cpu_write(current_kprobe, p);
	314
	315	kcb = get_kprobe_ctlblk();
	316	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
	317
	318	instruction_pointer_set(regs, ip);
	319
	320	if (!p->pre_handler \|\| !p->pre_handler(p, regs)) {
	321
	322	instruction_pointer_set(regs, ip + MCOUNT_INSN_SIZE);
	323
	324	if (unlikely(p->post_handler)) {
	325	kcb->kprobe_status = KPROBE_HIT_SSDONE;
	326	p->post_handler(p, regs, 0);
	327	}
	328	}
	329	__this_cpu_write(current_kprobe, NULL);
c536aa1c	330	out:
c536aa1c	331	ftrace_test_recursion_unlock(bit);
657480d9 SS	332	}
	333	NOKPROBE_SYMBOL(kprobe_ftrace_handler);
	334
	335	int arch_prepare_kprobe_ftrace(struct kprobe *p)
	336	{
	337	p->ainsn.insn = NULL;
	338	return 0;
	339	}
	340	#endif