1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/ftrace.h>
4 #include <linux/uaccess.h>
5 #include <asm/cacheflush.h>
7 #ifndef CONFIG_DYNAMIC_FTRACE
8 extern void (*ftrace_trace_function)(unsigned long, unsigned long,
9 struct ftrace_ops*, struct pt_regs*);
10 extern void ftrace_graph_caller(void);
12 noinline void __naked ftrace_stub(unsigned long ip, unsigned long parent_ip,
13 struct ftrace_ops *op, struct pt_regs *regs)
15 __asm__ (""); /* avoid to optimize as pure function */
18 noinline void _mcount(unsigned long parent_ip)
20 /* save all state by the compiler prologue */
22 unsigned long ip = (unsigned long)__builtin_return_address(0);
24 if (ftrace_trace_function != ftrace_stub)
25 ftrace_trace_function(ip - MCOUNT_INSN_SIZE, parent_ip,
28 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
29 if (ftrace_graph_return != (trace_func_graph_ret_t)ftrace_stub
30 || ftrace_graph_entry != ftrace_graph_entry_stub)
31 ftrace_graph_caller();
34 /* restore all state by the compiler epilogue */
36 EXPORT_SYMBOL(_mcount);
38 #else /* CONFIG_DYNAMIC_FTRACE */
40 noinline void __naked ftrace_stub(unsigned long ip, unsigned long parent_ip,
41 struct ftrace_ops *op, struct pt_regs *regs)
43 __asm__ (""); /* avoid to optimize as pure function */
46 noinline void __naked _mcount(unsigned long parent_ip)
48 __asm__ (""); /* avoid to optimize as pure function */
50 EXPORT_SYMBOL(_mcount);
52 #define XSTR(s) STR(s)
54 void _ftrace_caller(unsigned long parent_ip)
56 /* save all state needed by the compiler prologue */
59 * prepare arguments for real tracing function
60 * first arg : __builtin_return_address(0) - MCOUNT_INSN_SIZE
61 * second arg : parent_ip
63 __asm__ __volatile__ (
65 "addi $r0, %1, #-" XSTR(MCOUNT_INSN_SIZE) "\n\t"
67 : "r" (parent_ip), "r" (__builtin_return_address(0)));
69 /* a placeholder for the call to a real tracing function */
70 __asm__ __volatile__ (
76 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
77 /* a placeholder for the call to ftrace_graph_caller */
78 __asm__ __volatile__ (
79 "ftrace_graph_call: \n\t"
84 /* restore all state needed by the compiler epilogue */
87 int __init ftrace_dyn_arch_init(void)
92 int ftrace_arch_code_modify_prepare(void)
94 set_all_modules_text_rw();
98 int ftrace_arch_code_modify_post_process(void)
100 set_all_modules_text_ro();
104 static unsigned long gen_sethi_insn(unsigned long addr)
106 unsigned long opcode = 0x46000000;
107 unsigned long imm = addr >> 12;
108 unsigned long rt_num = 0xf << 20;
110 return ENDIAN_CONVERT(opcode | rt_num | imm);
113 static unsigned long gen_ori_insn(unsigned long addr)
115 unsigned long opcode = 0x58000000;
116 unsigned long imm = addr & 0x0000fff;
117 unsigned long rt_num = 0xf << 20;
118 unsigned long ra_num = 0xf << 15;
120 return ENDIAN_CONVERT(opcode | rt_num | ra_num | imm);
123 static unsigned long gen_jral_insn(unsigned long addr)
125 unsigned long opcode = 0x4a000001;
126 unsigned long rt_num = 0x1e << 20;
127 unsigned long rb_num = 0xf << 10;
129 return ENDIAN_CONVERT(opcode | rt_num | rb_num);
132 static void ftrace_gen_call_insn(unsigned long *call_insns,
135 call_insns[0] = gen_sethi_insn(addr); /* sethi $r15, imm20u */
136 call_insns[1] = gen_ori_insn(addr); /* ori $r15, $r15, imm15u */
137 call_insns[2] = gen_jral_insn(addr); /* jral $lp, $r15 */
140 static int __ftrace_modify_code(unsigned long pc, unsigned long *old_insn,
141 unsigned long *new_insn, bool validate)
143 unsigned long orig_insn[3];
146 if (probe_kernel_read(orig_insn, (void *)pc, MCOUNT_INSN_SIZE))
148 if (memcmp(orig_insn, old_insn, MCOUNT_INSN_SIZE))
152 if (probe_kernel_write((void *)pc, new_insn, MCOUNT_INSN_SIZE))
158 static int ftrace_modify_code(unsigned long pc, unsigned long *old_insn,
159 unsigned long *new_insn, bool validate)
163 ret = __ftrace_modify_code(pc, old_insn, new_insn, validate);
167 flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
172 int ftrace_update_ftrace_func(ftrace_func_t func)
174 unsigned long pc = (unsigned long)&ftrace_call;
175 unsigned long old_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
176 unsigned long new_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
178 if (func != ftrace_stub)
179 ftrace_gen_call_insn(new_insn, (unsigned long)func);
181 return ftrace_modify_code(pc, old_insn, new_insn, false);
184 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
186 unsigned long pc = rec->ip;
187 unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
188 unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
190 ftrace_gen_call_insn(call_insn, addr);
192 return ftrace_modify_code(pc, nop_insn, call_insn, true);
195 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
198 unsigned long pc = rec->ip;
199 unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
200 unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
202 ftrace_gen_call_insn(call_insn, addr);
204 return ftrace_modify_code(pc, call_insn, nop_insn, true);
206 #endif /* CONFIG_DYNAMIC_FTRACE */
208 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
209 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
210 unsigned long frame_pointer)
212 unsigned long return_hooker = (unsigned long)&return_to_handler;
215 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
220 if (!function_graph_enter(old, self_addr, frame_pointer, NULL))
221 *parent = return_hooker;
224 noinline void ftrace_graph_caller(void)
226 unsigned long *parent_ip =
227 (unsigned long *)(__builtin_frame_address(2) - 4);
229 unsigned long selfpc =
230 (unsigned long)(__builtin_return_address(1) - MCOUNT_INSN_SIZE);
232 unsigned long frame_pointer =
233 (unsigned long)__builtin_frame_address(3);
235 prepare_ftrace_return(parent_ip, selfpc, frame_pointer);
238 extern unsigned long ftrace_return_to_handler(unsigned long frame_pointer);
239 void __naked return_to_handler(void)
241 __asm__ __volatile__ (
242 /* save state needed by the ABI */
243 "smw.adm $r0,[$sp],$r1,#0x0 \n\t"
245 /* get original return address */
247 "bal ftrace_return_to_handler\n\t"
250 /* restore state nedded by the ABI */
251 "lmw.bim $r0,[$sp],$r1,#0x0 \n\t");
254 #ifdef CONFIG_DYNAMIC_FTRACE
255 extern unsigned long ftrace_graph_call;
257 static int ftrace_modify_graph_caller(bool enable)
259 unsigned long pc = (unsigned long)&ftrace_graph_call;
260 unsigned long nop_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
261 unsigned long call_insn[3] = {INSN_NOP, INSN_NOP, INSN_NOP};
263 ftrace_gen_call_insn(call_insn, (unsigned long)ftrace_graph_caller);
266 return ftrace_modify_code(pc, nop_insn, call_insn, true);
268 return ftrace_modify_code(pc, call_insn, nop_insn, true);
271 int ftrace_enable_ftrace_graph_caller(void)
273 return ftrace_modify_graph_caller(true);
276 int ftrace_disable_ftrace_graph_caller(void)
278 return ftrace_modify_graph_caller(false);
280 #endif /* CONFIG_DYNAMIC_FTRACE */
282 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
285 #ifdef CONFIG_TRACE_IRQFLAGS
286 noinline void __trace_hardirqs_off(void)
288 trace_hardirqs_off();
290 noinline void __trace_hardirqs_on(void)
294 #endif /* CONFIG_TRACE_IRQFLAGS */