Commit | Line | Data |
---|---|---|
d2912cb1 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
819e50e2 AT |
2 | /* |
3 | * arch/arm64/kernel/ftrace.c | |
4 | * | |
5 | * Copyright (C) 2013 Linaro Limited | |
6 | * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> | |
819e50e2 AT |
7 | */ |
8 | ||
9 | #include <linux/ftrace.h> | |
e71a4e1b | 10 | #include <linux/module.h> |
819e50e2 AT |
11 | #include <linux/swab.h> |
12 | #include <linux/uaccess.h> | |
13 | ||
14 | #include <asm/cacheflush.h> | |
e71a4e1b | 15 | #include <asm/debug-monitors.h> |
819e50e2 AT |
16 | #include <asm/ftrace.h> |
17 | #include <asm/insn.h> | |
18 | ||
bd7d38db AT |
19 | #ifdef CONFIG_DYNAMIC_FTRACE |
20 | /* | |
21 | * Replace a single instruction, which may be a branch or NOP. | |
22 | * If @validate == true, a replaced instruction is checked against 'old'. | |
23 | */ | |
24 | static int ftrace_modify_code(unsigned long pc, u32 old, u32 new, | |
25 | bool validate) | |
26 | { | |
27 | u32 replaced; | |
28 | ||
29 | /* | |
30 | * Note: | |
004ab584 LB |
31 | * We are paranoid about modifying text, as if a bug were to happen, it |
32 | * could cause us to read or write to someplace that could cause harm. | |
33 | * Carefully read and modify the code with aarch64_insn_*() which uses | |
34 | * probe_kernel_*(), and make sure what we read is what we expected it | |
35 | * to be before modifying it. | |
bd7d38db AT |
36 | */ |
37 | if (validate) { | |
38 | if (aarch64_insn_read((void *)pc, &replaced)) | |
39 | return -EFAULT; | |
40 | ||
41 | if (replaced != old) | |
42 | return -EINVAL; | |
43 | } | |
44 | if (aarch64_insn_patch_text_nosync((void *)pc, new)) | |
45 | return -EPERM; | |
46 | ||
47 | return 0; | |
48 | } | |
49 | ||
50 | /* | |
51 | * Replace tracer function in ftrace_caller() | |
52 | */ | |
53 | int ftrace_update_ftrace_func(ftrace_func_t func) | |
54 | { | |
55 | unsigned long pc; | |
56 | u32 new; | |
57 | ||
58 | pc = (unsigned long)&ftrace_call; | |
9f1ae759 CM |
59 | new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func, |
60 | AARCH64_INSN_BRANCH_LINK); | |
bd7d38db AT |
61 | |
62 | return ftrace_modify_code(pc, 0, new, false); | |
63 | } | |
64 | ||
65 | /* | |
66 | * Turn on the call to ftrace_caller() in instrumented function | |
67 | */ | |
68 | int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) | |
69 | { | |
70 | unsigned long pc = rec->ip; | |
71 | u32 old, new; | |
68764420 WD |
72 | long offset = (long)pc - (long)addr; |
73 | ||
74 | if (offset < -SZ_128M || offset >= SZ_128M) { | |
8486e54d | 75 | #ifdef CONFIG_ARM64_MODULE_PLTS |
b6143d10 | 76 | struct plt_entry trampoline, *dst; |
e71a4e1b AB |
77 | struct module *mod; |
78 | ||
79 | /* | |
80 | * On kernels that support module PLTs, the offset between the | |
81 | * branch instruction and its target may legally exceed the | |
82 | * range of an ordinary relative 'bl' opcode. In this case, we | |
83 | * need to branch via a trampoline in the module. | |
84 | * | |
85 | * NOTE: __module_text_address() must be called with preemption | |
86 | * disabled, but we can rely on ftrace_lock to ensure that 'mod' | |
87 | * retains its validity throughout the remainder of this code. | |
88 | */ | |
89 | preempt_disable(); | |
90 | mod = __module_text_address(pc); | |
91 | preempt_enable(); | |
92 | ||
93 | if (WARN_ON(!mod)) | |
94 | return -EINVAL; | |
95 | ||
96 | /* | |
97 | * There is only one ftrace trampoline per module. For now, | |
98 | * this is not a problem since on arm64, all dynamic ftrace | |
99 | * invocations are routed via ftrace_caller(). This will need | |
100 | * to be revisited if support for multiple ftrace entry points | |
101 | * is added in the future, but for now, the pr_err() below | |
102 | * deals with a theoretical issue only. | |
4e69ecf4 AB |
103 | * |
104 | * Note that PLTs are place relative, and plt_entries_equal() | |
105 | * checks whether they point to the same target. Here, we need | |
106 | * to check if the actual opcodes are in fact identical, | |
107 | * regardless of the offset in memory so use memcmp() instead. | |
e71a4e1b | 108 | */ |
b6143d10 WD |
109 | dst = mod->arch.ftrace_trampoline; |
110 | trampoline = get_plt_entry(addr, dst); | |
111 | if (memcmp(dst, &trampoline, sizeof(trampoline))) { | |
112 | if (plt_entry_is_initialized(dst)) { | |
e71a4e1b AB |
113 | pr_err("ftrace: far branches to multiple entry points unsupported inside a single module\n"); |
114 | return -EINVAL; | |
115 | } | |
116 | ||
117 | /* point the trampoline to our ftrace entry point */ | |
118 | module_disable_ro(mod); | |
b6143d10 | 119 | *dst = trampoline; |
e71a4e1b AB |
120 | module_enable_ro(mod, true); |
121 | ||
b6143d10 WD |
122 | /* |
123 | * Ensure updated trampoline is visible to instruction | |
124 | * fetch before we patch in the branch. | |
125 | */ | |
126 | __flush_icache_range((unsigned long)&dst[0], | |
127 | (unsigned long)&dst[1]); | |
e71a4e1b | 128 | } |
b6143d10 | 129 | addr = (unsigned long)dst; |
8486e54d MR |
130 | #else /* CONFIG_ARM64_MODULE_PLTS */ |
131 | return -EINVAL; | |
68764420 | 132 | #endif /* CONFIG_ARM64_MODULE_PLTS */ |
8486e54d | 133 | } |
e71a4e1b | 134 | |
bd7d38db | 135 | old = aarch64_insn_gen_nop(); |
9f1ae759 | 136 | new = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK); |
bd7d38db AT |
137 | |
138 | return ftrace_modify_code(pc, old, new, true); | |
139 | } | |
140 | ||
141 | /* | |
142 | * Turn off the call to ftrace_caller() in instrumented function | |
143 | */ | |
144 | int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, | |
145 | unsigned long addr) | |
146 | { | |
147 | unsigned long pc = rec->ip; | |
f8af0b36 AB |
148 | bool validate = true; |
149 | u32 old = 0, new; | |
68764420 WD |
150 | long offset = (long)pc - (long)addr; |
151 | ||
152 | if (offset < -SZ_128M || offset >= SZ_128M) { | |
8486e54d | 153 | #ifdef CONFIG_ARM64_MODULE_PLTS |
f8af0b36 AB |
154 | u32 replaced; |
155 | ||
156 | /* | |
157 | * 'mod' is only set at module load time, but if we end up | |
158 | * dealing with an out-of-range condition, we can assume it | |
159 | * is due to a module being loaded far away from the kernel. | |
160 | */ | |
161 | if (!mod) { | |
162 | preempt_disable(); | |
163 | mod = __module_text_address(pc); | |
164 | preempt_enable(); | |
165 | ||
166 | if (WARN_ON(!mod)) | |
167 | return -EINVAL; | |
168 | } | |
169 | ||
170 | /* | |
171 | * The instruction we are about to patch may be a branch and | |
172 | * link instruction that was redirected via a PLT entry. In | |
173 | * this case, the normal validation will fail, but we can at | |
174 | * least check that we are dealing with a branch and link | |
175 | * instruction that points into the right module. | |
176 | */ | |
177 | if (aarch64_insn_read((void *)pc, &replaced)) | |
178 | return -EFAULT; | |
179 | ||
180 | if (!aarch64_insn_is_bl(replaced) || | |
181 | !within_module(pc + aarch64_get_branch_offset(replaced), | |
182 | mod)) | |
183 | return -EINVAL; | |
184 | ||
185 | validate = false; | |
8486e54d MR |
186 | #else /* CONFIG_ARM64_MODULE_PLTS */ |
187 | return -EINVAL; | |
188 | #endif /* CONFIG_ARM64_MODULE_PLTS */ | |
f8af0b36 AB |
189 | } else { |
190 | old = aarch64_insn_gen_branch_imm(pc, addr, | |
191 | AARCH64_INSN_BRANCH_LINK); | |
192 | } | |
bd7d38db | 193 | |
bd7d38db AT |
194 | new = aarch64_insn_gen_nop(); |
195 | ||
f8af0b36 | 196 | return ftrace_modify_code(pc, old, new, validate); |
bd7d38db AT |
197 | } |
198 | ||
81a6a146 LB |
199 | void arch_ftrace_update_code(int command) |
200 | { | |
e4c07bf9 | 201 | command |= FTRACE_MAY_SLEEP; |
81a6a146 LB |
202 | ftrace_modify_all_code(command); |
203 | } | |
204 | ||
bd7d38db AT |
205 | int __init ftrace_dyn_arch_init(void) |
206 | { | |
207 | return 0; | |
208 | } | |
209 | #endif /* CONFIG_DYNAMIC_FTRACE */ | |
210 | ||
819e50e2 AT |
211 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
212 | /* | |
213 | * function_graph tracer expects ftrace_return_to_handler() to be called | |
214 | * on the way back to parent. For this purpose, this function is called | |
215 | * in _mcount() or ftrace_caller() to replace return address (*parent) on | |
216 | * the call stack to return_to_handler. | |
217 | * | |
218 | * Note that @frame_pointer is used only for sanity check later. | |
219 | */ | |
7dc48bf9 | 220 | void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent, |
819e50e2 AT |
221 | unsigned long frame_pointer) |
222 | { | |
223 | unsigned long return_hooker = (unsigned long)&return_to_handler; | |
224 | unsigned long old; | |
819e50e2 AT |
225 | |
226 | if (unlikely(atomic_read(¤t->tracing_graph_pause))) | |
227 | return; | |
228 | ||
229 | /* | |
230 | * Note: | |
231 | * No protection against faulting at *parent, which may be seen | |
232 | * on other archs. It's unlikely on AArch64. | |
233 | */ | |
234 | old = *parent; | |
819e50e2 | 235 | |
01e0ab2c | 236 | if (!function_graph_enter(old, self_addr, frame_pointer, NULL)) |
79fdee9b | 237 | *parent = return_hooker; |
819e50e2 | 238 | } |
bd7d38db AT |
239 | |
240 | #ifdef CONFIG_DYNAMIC_FTRACE | |
241 | /* | |
242 | * Turn on/off the call to ftrace_graph_caller() in ftrace_caller() | |
243 | * depending on @enable. | |
244 | */ | |
245 | static int ftrace_modify_graph_caller(bool enable) | |
246 | { | |
247 | unsigned long pc = (unsigned long)&ftrace_graph_call; | |
248 | u32 branch, nop; | |
249 | ||
250 | branch = aarch64_insn_gen_branch_imm(pc, | |
9f1ae759 | 251 | (unsigned long)ftrace_graph_caller, |
d0d62230 | 252 | AARCH64_INSN_BRANCH_NOLINK); |
bd7d38db AT |
253 | nop = aarch64_insn_gen_nop(); |
254 | ||
255 | if (enable) | |
256 | return ftrace_modify_code(pc, nop, branch, true); | |
257 | else | |
258 | return ftrace_modify_code(pc, branch, nop, true); | |
259 | } | |
260 | ||
261 | int ftrace_enable_ftrace_graph_caller(void) | |
262 | { | |
263 | return ftrace_modify_graph_caller(true); | |
264 | } | |
265 | ||
266 | int ftrace_disable_ftrace_graph_caller(void) | |
267 | { | |
268 | return ftrace_modify_graph_caller(false); | |
269 | } | |
270 | #endif /* CONFIG_DYNAMIC_FTRACE */ | |
819e50e2 | 271 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |