[linux-2.6-block.git] / arch / sh / kernel / ftrace.c

/*
 * Copyright (C) 2008 Matt Fleming <matt@console-pimps.org>
 * Copyright (C) 2008 Paul Mundt <lethal@linux-sh.org>
 *
 * Code for replacing ftrace calls with jumps.
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 *
 * Thanks goes to Ingo Molnar, for suggesting the idea.
 * Mathieu Desnoyers, for suggesting postponing the modifications.
 * Arjan van de Ven, for keeping me straight, and explaining to me
 * the dangers of modifying code on the run.
 */
#include <linux/uaccess.h>
#include <linux/ftrace.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/io.h>
#include <linux/kernel.h>
#include <asm/ftrace.h>
#include <asm/cacheflush.h>
#include <asm/unistd.h>
#include <trace/syscall.h>

#ifdef CONFIG_DYNAMIC_FTRACE
static unsigned char ftrace_replaced_code[MCOUNT_INSN_SIZE];

static unsigned char ftrace_nop[4];
/*
 * If we're trying to nop out a call to a function, we instead
 * place a call to the address after the memory table.
 *
 * 8c011060 <a>:
 * 8c011060:       02 d1           mov.l   8c01106c <a+0xc>,r1
 * 8c011062:       22 4f           sts.l   pr,@-r15
 * 8c011064:       02 c7           mova    8c011070 <a+0x10>,r0
 * 8c011066:       2b 41           jmp     @r1
 * 8c011068:       2a 40           lds     r0,pr
 * 8c01106a:       09 00           nop
 * 8c01106c:       68 24           .word 0x2468     <--- ip
 * 8c01106e:       1d 8c           .word 0x8c1d
 * 8c011070:       26 4f           lds.l   @r15+,pr <--- ip + MCOUNT_INSN_SIZE
 *
 * We write 0x8c011070 to 0x8c01106c so that on entry to a() we branch
 * past the _mcount call and continue executing code like normal.
 */
static unsigned char *ftrace_nop_replace(unsigned long ip)
{
	__raw_writel(ip + MCOUNT_INSN_SIZE, ftrace_nop);
	return ftrace_nop;
}

static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
{
	/* Place the address in the memory table. */
	__raw_writel(addr, ftrace_replaced_code);

	/*
	 * No locking needed, this must be called via kstop_machine
	 * which in essence is like running on a uniprocessor machine.
	 */
	return ftrace_replaced_code;
}

/*
 * Modifying code must take extra care. On an SMP machine, if
 * the code being modified is also being executed on another CPU
 * that CPU will have undefined results and possibly take a GPF.
 * We use kstop_machine to stop other CPUS from exectuing code.
 * But this does not stop NMIs from happening. We still need
 * to protect against that. We separate out the modification of
 * the code to take care of this.
 *
 * Two buffers are added: An IP buffer and a "code" buffer.
 *
 * 1) Put the instruction pointer into the IP buffer
 *    and the new code into the "code" buffer.
 * 2) Wait for any running NMIs to finish and set a flag that says
 *    we are modifying code, it is done in an atomic operation.
 * 3) Write the code
 * 4) clear the flag.
 * 5) Wait for any running NMIs to finish.
 *
 * If an NMI is executed, the first thing it does is to call
 * "ftrace_nmi_enter". This will check if the flag is set to write
 * and if it is, it will write what is in the IP and "code" buffers.
 *
 * The trick is, it does not matter if everyone is writing the same
 * content to the code location. Also, if a CPU is executing code
 * it is OK to write to that code location if the contents being written
 * are the same as what exists.
 */
#define MOD_CODE_WRITE_FLAG (1 << 31)	/* set when NMI should do the write */
static atomic_t nmi_running = ATOMIC_INIT(0);
static int mod_code_status;		/* holds return value of text write */
static void *mod_code_ip;		/* holds the IP to write to */
static void *mod_code_newcode;		/* holds the text to write to the IP */

static unsigned nmi_wait_count;
static atomic_t nmi_update_count = ATOMIC_INIT(0);

int ftrace_arch_read_dyn_info(char *buf, int size)
{
	int r;

	r = snprintf(buf, size, "%u %u",
		     nmi_wait_count,
		     atomic_read(&nmi_update_count));
	return r;
}

static void clear_mod_flag(void)
{
	int old = atomic_read(&nmi_running);

	for (;;) {
		int new = old & ~MOD_CODE_WRITE_FLAG;

		if (old == new)
			break;

		old = atomic_cmpxchg(&nmi_running, old, new);
	}
}

static void ftrace_mod_code(void)
{
	/*
	 * Yes, more than one CPU process can be writing to mod_code_status.
	 *    (and the code itself)
	 * But if one were to fail, then they all should, and if one were
	 * to succeed, then they all should.
	 */
	mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
					     MCOUNT_INSN_SIZE);

	/* if we fail, then kill any new writers */
	if (mod_code_status)
		clear_mod_flag();
}

void ftrace_nmi_enter(void)
{
	if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) {
		smp_rmb();
		ftrace_mod_code();
		atomic_inc(&nmi_update_count);
	}
	/* Must have previous changes seen before executions */
	smp_mb();
}

void ftrace_nmi_exit(void)
{
	/* Finish all executions before clearing nmi_running */
	smp_mb();
	atomic_dec(&nmi_running);
}

static void wait_for_nmi_and_set_mod_flag(void)
{
	if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG))
		return;

	do {
		cpu_relax();
	} while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG));

	nmi_wait_count++;
}

static void wait_for_nmi(void)
{
	if (!atomic_read(&nmi_running))
		return;

	do {
		cpu_relax();
	} while (atomic_read(&nmi_running));

	nmi_wait_count++;
}

static int
do_ftrace_mod_code(unsigned long ip, void *new_code)
{
	mod_code_ip = (void *)ip;
	mod_code_newcode = new_code;

	/* The buffers need to be visible before we let NMIs write them */
	smp_mb();

	wait_for_nmi_and_set_mod_flag();

	/* Make sure all running NMIs have finished before we write the code */
	smp_mb();

	ftrace_mod_code();

	/* Make sure the write happens before clearing the bit */
	smp_mb();

	clear_mod_flag();
	wait_for_nmi();

	return mod_code_status;
}

static int ftrace_modify_code(unsigned long ip, unsigned char *old_code,
		       unsigned char *new_code)
{
	unsigned char replaced[MCOUNT_INSN_SIZE];

	/*
	 * Note: Due to modules and __init, code can
	 *  disappear and change, we need to protect against faulting
	 *  as well as code changing. We do this by using the
	 *  probe_kernel_* functions.
	 *
	 * No real locking needed, this code is run through
	 * kstop_machine, or before SMP starts.
	 */

	/* read the text we want to modify */
	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	/* Make sure it is what we expect it to be */
	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
		return -EINVAL;

	/* replace the text with the new text */
	if (do_ftrace_mod_code(ip, new_code))
		return -EPERM;

	flush_icache_range(ip, ip + MCOUNT_INSN_SIZE);

	return 0;
}

int ftrace_update_ftrace_func(ftrace_func_t func)
{
	unsigned long ip = (unsigned long)(&ftrace_call) + MCOUNT_INSN_OFFSET;
	unsigned char old[MCOUNT_INSN_SIZE], *new;

	memcpy(old, (unsigned char *)ip, MCOUNT_INSN_SIZE);
	new = ftrace_call_replace(ip, (unsigned long)func);

	return ftrace_modify_code(ip, old, new);
}

int ftrace_make_nop(struct module *mod,
		    struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned char *new, *old;
	unsigned long ip = rec->ip;

	old = ftrace_call_replace(ip, addr);
	new = ftrace_nop_replace(ip);

	return ftrace_modify_code(rec->ip, old, new);
}

int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
{
	unsigned char *new, *old;
	unsigned long ip = rec->ip;

	old = ftrace_nop_replace(ip);
	new = ftrace_call_replace(ip, addr);

	return ftrace_modify_code(rec->ip, old, new);
}

int __init ftrace_dyn_arch_init(void *data)
{
	/* The return code is retured via data */
	__raw_writel(0, (unsigned long)data);

	return 0;
}
#endif /* CONFIG_DYNAMIC_FTRACE */

#ifdef CONFIG_FUNCTION_GRAPH_TRACER
#ifdef CONFIG_DYNAMIC_FTRACE
extern void ftrace_graph_call(void);

static int ftrace_mod(unsigned long ip, unsigned long old_addr,
		      unsigned long new_addr)
{
	unsigned char code[MCOUNT_INSN_SIZE];

	if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
		return -EFAULT;

	if (old_addr != __raw_readl((unsigned long *)code))
		return -EINVAL;

	__raw_writel(new_addr, ip);
	return 0;
}

int ftrace_enable_ftrace_graph_caller(void)
{
	unsigned long ip, old_addr, new_addr;

	ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET;
	old_addr = (unsigned long)(&skip_trace);
	new_addr = (unsigned long)(&ftrace_graph_caller);

	return ftrace_mod(ip, old_addr, new_addr);
}

int ftrace_disable_ftrace_graph_caller(void)
{
	unsigned long ip, old_addr, new_addr;

	ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET;
	old_addr = (unsigned long)(&ftrace_graph_caller);
	new_addr = (unsigned long)(&skip_trace);

	return ftrace_mod(ip, old_addr, new_addr);
}
#endif /* CONFIG_DYNAMIC_FTRACE */

/*
 * Hook the return address and push it in the stack of return addrs
 * in the current thread info.
 *
 * This is the main routine for the function graph tracer. The function
 * graph tracer essentially works like this:
 *
 * parent is the stack address containing self_addr's return address.
 * We pull the real return address out of parent and store it in
 * current's ret_stack. Then, we replace the return address on the stack
 * with the address of return_to_handler. self_addr is the function that
 * called mcount.
 *
 * When self_addr returns, it will jump to return_to_handler which calls
 * ftrace_return_to_handler. ftrace_return_to_handler will pull the real
 * return address off of current's ret_stack and jump to it.
 */
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
{
	unsigned long old;
	int faulted, err;
	struct ftrace_graph_ent trace;
	unsigned long return_hooker = (unsigned long)&return_to_handler;

	if (unlikely(atomic_read(&current->tracing_graph_pause)))
		return;

	/*
	 * Protect against fault, even if it shouldn't
	 * happen. This tool is too much intrusive to
	 * ignore such a protection.
	 */
	__asm__ __volatile__(
		"1:						\n\t"
		"mov.l		@%2, %0				\n\t"
		"2:						\n\t"
		"mov.l		%3, @%2				\n\t"
		"mov		#0, %1				\n\t"
		"3:						\n\t"
		".section .fixup, \"ax\"			\n\t"
		"4:						\n\t"
		"mov.l		5f, %0				\n\t"
		"jmp		@%0				\n\t"
		" mov		#1, %1				\n\t"
		".balign 4					\n\t"
		"5:	.long 3b				\n\t"
		".previous					\n\t"
		".section __ex_table,\"a\"			\n\t"
		".long 1b, 4b					\n\t"
		".long 2b, 4b					\n\t"
		".previous					\n\t"
		: "=&r" (old), "=r" (faulted)
		: "r" (parent), "r" (return_hooker)
	);

	if (unlikely(faulted)) {
		ftrace_graph_stop();
		WARN_ON(1);
		return;
	}

	err = ftrace_push_return_trace(old, self_addr, &trace.depth, 0);
	if (err == -EBUSY) {
		__raw_writel(old, parent);
		return;
	}

	trace.func = self_addr;

	/* Only trace if the calling function expects to */
	if (!ftrace_graph_entry(&trace)) {
		current->curr_ret_stack--;
		__raw_writel(old, parent);
	}
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
Commit	Line	Data
fad57feb	1	/*
7780b6a2	2	* Copyright (C) 2008 Matt Fleming <matt@console-pimps.org>
b5cfeac9	3	* Copyright (C) 2008 Paul Mundt <lethal@linux-sh.org>
fad57feb MF	4	*
	5	* Code for replacing ftrace calls with jumps.
	6	*
	7	* Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
	8	*
	9	* Thanks goes to Ingo Molnar, for suggesting the idea.
	10	* Mathieu Desnoyers, for suggesting postponing the modifications.
	11	* Arjan van de Ven, for keeping me straight, and explaining to me
	12	* the dangers of modifying code on the run.
	13	*/
	14	#include <linux/uaccess.h>
	15	#include <linux/ftrace.h>
	16	#include <linux/string.h>
	17	#include <linux/init.h>
	18	#include <linux/io.h>
327933f5	19	#include <linux/kernel.h>
fad57feb MF	20	#include <asm/ftrace.h>
fad57feb MF	21	#include <asm/cacheflush.h>
c652d780 MF	22	#include <asm/unistd.h>
c652d780 MF	23	#include <trace/syscall.h>
fad57feb	24
327933f5	25	#ifdef CONFIG_DYNAMIC_FTRACE
fad57feb MF	26	static unsigned char ftrace_replaced_code[MCOUNT_INSN_SIZE];
fad57feb MF	27
9e28c46b MF	28	static unsigned char ftrace_nop[4];
	29	/*
	30	* If we're trying to nop out a call to a function, we instead
	31	* place a call to the address after the memory table.
	32	*
	33	* 8c011060 <a>:
	34	* 8c011060: 02 d1 mov.l 8c01106c <a+0xc>,r1
	35	* 8c011062: 22 4f sts.l pr,@-r15
	36	* 8c011064: 02 c7 mova 8c011070 <a+0x10>,r0
	37	* 8c011066: 2b 41 jmp @r1
	38	* 8c011068: 2a 40 lds r0,pr
	39	* 8c01106a: 09 00 nop
	40	* 8c01106c: 68 24 .word 0x2468 <--- ip
	41	* 8c01106e: 1d 8c .word 0x8c1d
	42	* 8c011070: 26 4f lds.l @r15+,pr <--- ip + MCOUNT_INSN_SIZE
	43	*
	44	* We write 0x8c011070 to 0x8c01106c so that on entry to a() we branch
	45	* past the _mcount call and continue executing code like normal.
	46	*/
	47	static unsigned char *ftrace_nop_replace(unsigned long ip)
fad57feb	48	{
9e28c46b	49	__raw_writel(ip + MCOUNT_INSN_SIZE, ftrace_nop);
fad57feb MF	50	return ftrace_nop;
	51	}
	52
9e28c46b	53	static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
fad57feb MF	54	{
fad57feb MF	55	/* Place the address in the memory table. */
9e28c46b	56	__raw_writel(addr, ftrace_replaced_code);
fad57feb MF	57
	58	/*
	59	* No locking needed, this must be called via kstop_machine
	60	* which in essence is like running on a uniprocessor machine.
	61	*/
	62	return ftrace_replaced_code;
	63	}
	64
e4b053d9 PM	65	/*
	66	* Modifying code must take extra care. On an SMP machine, if
	67	* the code being modified is also being executed on another CPU
	68	* that CPU will have undefined results and possibly take a GPF.
	69	* We use kstop_machine to stop other CPUS from exectuing code.
	70	* But this does not stop NMIs from happening. We still need
	71	* to protect against that. We separate out the modification of
	72	* the code to take care of this.
	73	*
	74	* Two buffers are added: An IP buffer and a "code" buffer.
	75	*
	76	* 1) Put the instruction pointer into the IP buffer
	77	* and the new code into the "code" buffer.
	78	* 2) Wait for any running NMIs to finish and set a flag that says
	79	* we are modifying code, it is done in an atomic operation.
	80	* 3) Write the code
	81	* 4) clear the flag.
	82	* 5) Wait for any running NMIs to finish.
	83	*
	84	* If an NMI is executed, the first thing it does is to call
	85	* "ftrace_nmi_enter". This will check if the flag is set to write
	86	* and if it is, it will write what is in the IP and "code" buffers.
	87	*
	88	* The trick is, it does not matter if everyone is writing the same
	89	* content to the code location. Also, if a CPU is executing code
	90	* it is OK to write to that code location if the contents being written
	91	* are the same as what exists.
	92	*/
	93	#define MOD_CODE_WRITE_FLAG (1 << 31) /* set when NMI should do the write */
	94	static atomic_t nmi_running = ATOMIC_INIT(0);
	95	static int mod_code_status; /* holds return value of text write */
	96	static void mod_code_ip; / holds the IP to write to */
	97	static void mod_code_newcode; / holds the text to write to the IP */
	98
	99	static unsigned nmi_wait_count;
	100	static atomic_t nmi_update_count = ATOMIC_INIT(0);
	101
	102	int ftrace_arch_read_dyn_info(char *buf, int size)
	103	{
	104	int r;
	105
	106	r = snprintf(buf, size, "%u %u",
	107	nmi_wait_count,
	108	atomic_read(&nmi_update_count));
	109	return r;
	110	}
	111
	112	static void clear_mod_flag(void)
	113	{
	114	int old = atomic_read(&nmi_running);
	115
	116	for (;;) {
	117	int new = old & ~MOD_CODE_WRITE_FLAG;
	118
	119	if (old == new)
	120	break;
	121
	122	old = atomic_cmpxchg(&nmi_running, old, new);
	123	}
	124	}
	125
	126	static void ftrace_mod_code(void)
	127	{
	128	/*
129	* Yes, more than one CPU process can be writing to mod_code_status.
130	* (and the code itself)
131	* But if one were to fail, then they all should, and if one were
132	* to succeed, then they all should.
133	*/
134	mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
135	MCOUNT_INSN_SIZE);
136
137	/* if we fail, then kill any new writers */
138	if (mod_code_status)
139	clear_mod_flag();
140	}
141
142	void ftrace_nmi_enter(void)
143	{
144	if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) {
145	smp_rmb();
146	ftrace_mod_code();
147	atomic_inc(&nmi_update_count);
148	}
149	/* Must have previous changes seen before executions */
150	smp_mb();
151	}
152
153	void ftrace_nmi_exit(void)
154	{
155	/* Finish all executions before clearing nmi_running */
156	smp_mb();
157	atomic_dec(&nmi_running);
158	}
159
160	static void wait_for_nmi_and_set_mod_flag(void)
161	{
162	if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG))
163	return;
164
165	do {
166	cpu_relax();
167	} while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG));
168
169	nmi_wait_count++;
170	}
171
172	static void wait_for_nmi(void)
173	{
174	if (!atomic_read(&nmi_running))
175	return;
176
177	do {
178	cpu_relax();
179	} while (atomic_read(&nmi_running));
180
181	nmi_wait_count++;
182	}
183
184	static int
185	do_ftrace_mod_code(unsigned long ip, void *new_code)
186	{
187	mod_code_ip = (void *)ip;
188	mod_code_newcode = new_code;
189
190	/* The buffers need to be visible before we let NMIs write them */
191	smp_mb();
192
193	wait_for_nmi_and_set_mod_flag();
194
195	/* Make sure all running NMIs have finished before we write the code */
196	smp_mb();
197
198	ftrace_mod_code();
199
200	/* Make sure the write happens before clearing the bit */
201	smp_mb();
202
203	clear_mod_flag();
204	wait_for_nmi();
205
206	return mod_code_status;
207	}
208
9e28c46b	209	static int ftrace_modify_code(unsigned long ip, unsigned char *old_code,
fad57feb MF	210	unsigned char *new_code)
	211	{
	212	unsigned char replaced[MCOUNT_INSN_SIZE];
	213
	214	/*
	215	* Note: Due to modules and __init, code can
	216	* disappear and change, we need to protect against faulting
	217	* as well as code changing. We do this by using the
	218	* probe_kernel_* functions.
	219	*
	220	* No real locking needed, this code is run through
	221	* kstop_machine, or before SMP starts.
	222	*/
	223
fad57feb MF	224	/* read the text we want to modify */
	225	if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
	226	return -EFAULT;
	227
	228	/* Make sure it is what we expect it to be */
	229	if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
	230	return -EINVAL;
	231
	232	/* replace the text with the new text */
e4b053d9	233	if (do_ftrace_mod_code(ip, new_code))
fad57feb MF	234	return -EPERM;
	235
	236	flush_icache_range(ip, ip + MCOUNT_INSN_SIZE);
	237
	238	return 0;
	239	}
	240
	241	int ftrace_update_ftrace_func(ftrace_func_t func)
	242	{
9e28c46b	243	unsigned long ip = (unsigned long)(&ftrace_call) + MCOUNT_INSN_OFFSET;
fad57feb MF	244	unsigned char old[MCOUNT_INSN_SIZE], *new;
fad57feb MF	245
9e28c46b	246	memcpy(old, (unsigned char *)ip, MCOUNT_INSN_SIZE);
fad57feb MF	247	new = ftrace_call_replace(ip, (unsigned long)func);
fad57feb MF	248
9e28c46b	249	return ftrace_modify_code(ip, old, new);
fad57feb MF	250	}
fad57feb MF	251
b5cfeac9 PM	252	int ftrace_make_nop(struct module *mod,
	253	struct dyn_ftrace *rec, unsigned long addr)
	254	{
	255	unsigned char new, old;
	256	unsigned long ip = rec->ip;
	257
	258	old = ftrace_call_replace(ip, addr);
9e28c46b	259	new = ftrace_nop_replace(ip);
b5cfeac9 PM	260
	261	return ftrace_modify_code(rec->ip, old, new);
	262	}
	263
	264	int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
	265	{
	266	unsigned char new, old;
	267	unsigned long ip = rec->ip;
	268
9e28c46b	269	old = ftrace_nop_replace(ip);
b5cfeac9 PM	270	new = ftrace_call_replace(ip, addr);
	271
	272	return ftrace_modify_code(rec->ip, old, new);
	273	}
	274
fad57feb MF	275	int __init ftrace_dyn_arch_init(void *data)
	276	{
	277	/* The return code is retured via data */
	278	__raw_writel(0, (unsigned long)data);
	279
	280	return 0;
	281	}
327933f5 MF	282	#endif /* CONFIG_DYNAMIC_FTRACE */
	283
	284	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	285	#ifdef CONFIG_DYNAMIC_FTRACE
	286	extern void ftrace_graph_call(void);
	287
	288	static int ftrace_mod(unsigned long ip, unsigned long old_addr,
	289	unsigned long new_addr)
	290	{
	291	unsigned char code[MCOUNT_INSN_SIZE];
	292
	293	if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
	294	return -EFAULT;
	295
	296	if (old_addr != __raw_readl((unsigned long *)code))
	297	return -EINVAL;
	298
	299	__raw_writel(new_addr, ip);
	300	return 0;
	301	}
	302
	303	int ftrace_enable_ftrace_graph_caller(void)
	304	{
	305	unsigned long ip, old_addr, new_addr;
	306
	307	ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET;
	308	old_addr = (unsigned long)(&skip_trace);
	309	new_addr = (unsigned long)(&ftrace_graph_caller);
	310
	311	return ftrace_mod(ip, old_addr, new_addr);
	312	}
	313
	314	int ftrace_disable_ftrace_graph_caller(void)
	315	{
	316	unsigned long ip, old_addr, new_addr;
	317
	318	ip = (unsigned long)(&ftrace_graph_call) + GRAPH_INSN_OFFSET;
	319	old_addr = (unsigned long)(&ftrace_graph_caller);
	320	new_addr = (unsigned long)(&skip_trace);
	321
	322	return ftrace_mod(ip, old_addr, new_addr);
	323	}
	324	#endif /* CONFIG_DYNAMIC_FTRACE */
	325
	326	/*
	327	* Hook the return address and push it in the stack of return addrs
	328	* in the current thread info.
	329	*
	330	* This is the main routine for the function graph tracer. The function
	331	* graph tracer essentially works like this:
	332	*
	333	* parent is the stack address containing self_addr's return address.
	334	* We pull the real return address out of parent and store it in
	335	* current's ret_stack. Then, we replace the return address on the stack
	336	* with the address of return_to_handler. self_addr is the function that
	337	* called mcount.
	338	*
	339	* When self_addr returns, it will jump to return_to_handler which calls
	340	* ftrace_return_to_handler. ftrace_return_to_handler will pull the real
	341	* return address off of current's ret_stack and jump to it.
	342	*/
	343	void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
	344	{
	345	unsigned long old;
346	int faulted, err;
347	struct ftrace_graph_ent trace;
348	unsigned long return_hooker = (unsigned long)&return_to_handler;
349
350	if (unlikely(atomic_read(&current->tracing_graph_pause)))
351	return;
352
353	/*
354	* Protect against fault, even if it shouldn't
355	* happen. This tool is too much intrusive to
356	* ignore such a protection.
357	*/
358	__asm__ __volatile__(
359	"1: \n\t"
360	"mov.l @%2, %0 \n\t"
361	"2: \n\t"
362	"mov.l %3, @%2 \n\t"
363	"mov #0, %1 \n\t"
364	"3: \n\t"
365	".section .fixup, \"ax\" \n\t"
366	"4: \n\t"
367	"mov.l 5f, %0 \n\t"
368	"jmp @%0 \n\t"
369	" mov #1, %1 \n\t"
370	".balign 4 \n\t"
371	"5: .long 3b \n\t"
372	".previous \n\t"
373	".section __ex_table,\"a\" \n\t"
374	".long 1b, 4b \n\t"
375	".long 2b, 4b \n\t"
376	".previous \n\t"
377	: "=&r" (old), "=r" (faulted)
378	: "r" (parent), "r" (return_hooker)
379	);
380
381	if (unlikely(faulted)) {
382	ftrace_graph_stop();
383	WARN_ON(1);
384	return;
385	}
386
387	err = ftrace_push_return_trace(old, self_addr, &trace.depth, 0);
388	if (err == -EBUSY) {
389	__raw_writel(old, parent);
390	return;
391	}
392
393	trace.func = self_addr;
394
395	/* Only trace if the calling function expects to */
396	if (!ftrace_graph_entry(&trace)) {
397	current->curr_ret_stack--;
398	__raw_writel(old, parent);
399	}
400	}
401	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */