Commit | Line | Data |
---|---|---|
cad9931f MH |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * fprobe - Simple ftrace probe wrapper for function entry. | |
4 | */ | |
5 | #define pr_fmt(fmt) "fprobe: " fmt | |
6 | ||
7 | #include <linux/err.h> | |
8 | #include <linux/fprobe.h> | |
9 | #include <linux/kallsyms.h> | |
10 | #include <linux/kprobes.h> | |
5b0ab789 | 11 | #include <linux/rethook.h> |
cad9931f MH |
12 | #include <linux/slab.h> |
13 | #include <linux/sort.h> | |
14 | ||
5b0ab789 MH |
15 | #include "trace.h" |
16 | ||
17 | struct fprobe_rethook_node { | |
18 | struct rethook_node node; | |
19 | unsigned long entry_ip; | |
27527410 | 20 | unsigned long entry_parent_ip; |
76d0de57 | 21 | char data[]; |
5b0ab789 MH |
22 | }; |
23 | ||
3cc4e2c5 ZG |
24 | static inline void __fprobe_handler(unsigned long ip, unsigned long parent_ip, |
25 | struct ftrace_ops *ops, struct ftrace_regs *fregs) | |
cad9931f | 26 | { |
5b0ab789 | 27 | struct fprobe_rethook_node *fpr; |
76d0de57 | 28 | struct rethook_node *rh = NULL; |
cad9931f | 29 | struct fprobe *fp; |
76d0de57 | 30 | void *entry_data = NULL; |
3cc4e2c5 | 31 | int ret = 0; |
cad9931f MH |
32 | |
33 | fp = container_of(ops, struct fprobe, ops); | |
cad9931f | 34 | |
5b0ab789 MH |
35 | if (fp->exit_handler) { |
36 | rh = rethook_try_get(fp->rethook); | |
37 | if (!rh) { | |
38 | fp->nmissed++; | |
3cc4e2c5 | 39 | return; |
5b0ab789 MH |
40 | } |
41 | fpr = container_of(rh, struct fprobe_rethook_node, node); | |
42 | fpr->entry_ip = ip; | |
27527410 | 43 | fpr->entry_parent_ip = parent_ip; |
76d0de57 MHG |
44 | if (fp->entry_data_size) |
45 | entry_data = fpr->data; | |
5b0ab789 MH |
46 | } |
47 | ||
76d0de57 | 48 | if (fp->entry_handler) |
cb16330d | 49 | ret = fp->entry_handler(fp, ip, parent_ip, ftrace_get_regs(fregs), entry_data); |
76d0de57 | 50 | |
39d95420 MHG |
51 | /* If entry_handler returns !0, nmissed is not counted. */ |
52 | if (rh) { | |
53 | if (ret) | |
54 | rethook_recycle(rh); | |
55 | else | |
56 | rethook_hook(rh, ftrace_get_regs(fregs), true); | |
57 | } | |
3cc4e2c5 ZG |
58 | } |
59 | ||
60 | static void fprobe_handler(unsigned long ip, unsigned long parent_ip, | |
61 | struct ftrace_ops *ops, struct ftrace_regs *fregs) | |
62 | { | |
63 | struct fprobe *fp; | |
64 | int bit; | |
65 | ||
66 | fp = container_of(ops, struct fprobe, ops); | |
67 | if (fprobe_disabled(fp)) | |
68 | return; | |
69 | ||
70 | /* recursion detection has to go before any traceable function and | |
71 | * all functions before this point should be marked as notrace | |
72 | */ | |
73 | bit = ftrace_test_recursion_trylock(ip, parent_ip); | |
74 | if (bit < 0) { | |
75 | fp->nmissed++; | |
76 | return; | |
77 | } | |
78 | __fprobe_handler(ip, parent_ip, ops, fregs); | |
cad9931f | 79 | ftrace_test_recursion_unlock(bit); |
3cc4e2c5 | 80 | |
cad9931f MH |
81 | } |
82 | NOKPROBE_SYMBOL(fprobe_handler); | |
83 | ||
ab51e15d MH |
84 | static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, |
85 | struct ftrace_ops *ops, struct ftrace_regs *fregs) | |
86 | { | |
3cc4e2c5 ZG |
87 | struct fprobe *fp; |
88 | int bit; | |
89 | ||
90 | fp = container_of(ops, struct fprobe, ops); | |
91 | if (fprobe_disabled(fp)) | |
92 | return; | |
93 | ||
94 | /* recursion detection has to go before any traceable function and | |
95 | * all functions called before this point should be marked as notrace | |
96 | */ | |
97 | bit = ftrace_test_recursion_trylock(ip, parent_ip); | |
98 | if (bit < 0) { | |
99 | fp->nmissed++; | |
100 | return; | |
101 | } | |
ab51e15d | 102 | |
d5f28bb1 MHG |
103 | /* |
104 | * This user handler is shared with other kprobes and is not expected to be | |
105 | * called recursively. So if any other kprobe handler is running, this will | |
106 | * exit as kprobe does. See the section 'Share the callbacks with kprobes' | |
107 | * in Documentation/trace/fprobe.rst for more information. | |
108 | */ | |
ab51e15d MH |
109 | if (unlikely(kprobe_running())) { |
110 | fp->nmissed++; | |
5f0c584d | 111 | goto recursion_unlock; |
ab51e15d | 112 | } |
3cc4e2c5 | 113 | |
ab51e15d | 114 | kprobe_busy_begin(); |
3cc4e2c5 | 115 | __fprobe_handler(ip, parent_ip, ops, fregs); |
ab51e15d | 116 | kprobe_busy_end(); |
5f0c584d ZG |
117 | |
118 | recursion_unlock: | |
3cc4e2c5 | 119 | ftrace_test_recursion_unlock(bit); |
ab51e15d MH |
120 | } |
121 | ||
5b0ab789 | 122 | static void fprobe_exit_handler(struct rethook_node *rh, void *data, |
cb16330d | 123 | unsigned long ret_ip, struct pt_regs *regs) |
5b0ab789 MH |
124 | { |
125 | struct fprobe *fp = (struct fprobe *)data; | |
126 | struct fprobe_rethook_node *fpr; | |
27527410 | 127 | int bit; |
5b0ab789 MH |
128 | |
129 | if (!fp || fprobe_disabled(fp)) | |
130 | return; | |
131 | ||
132 | fpr = container_of(rh, struct fprobe_rethook_node, node); | |
133 | ||
27527410 ZG |
134 | /* |
135 | * we need to assure no calls to traceable functions in-between the | |
136 | * end of fprobe_handler and the beginning of fprobe_exit_handler. | |
137 | */ | |
138 | bit = ftrace_test_recursion_trylock(fpr->entry_ip, fpr->entry_parent_ip); | |
139 | if (bit < 0) { | |
140 | fp->nmissed++; | |
141 | return; | |
142 | } | |
143 | ||
cb16330d | 144 | fp->exit_handler(fp, fpr->entry_ip, ret_ip, regs, |
76d0de57 | 145 | fp->entry_data_size ? (void *)fpr->data : NULL); |
27527410 | 146 | ftrace_test_recursion_unlock(bit); |
5b0ab789 MH |
147 | } |
148 | NOKPROBE_SYMBOL(fprobe_exit_handler); | |
149 | ||
8be92533 JO |
150 | static int symbols_cmp(const void *a, const void *b) |
151 | { | |
152 | const char **str_a = (const char **) a; | |
153 | const char **str_b = (const char **) b; | |
154 | ||
155 | return strcmp(*str_a, *str_b); | |
156 | } | |
157 | ||
cad9931f MH |
158 | /* Convert ftrace location address from symbols */ |
159 | static unsigned long *get_ftrace_locations(const char **syms, int num) | |
160 | { | |
cad9931f | 161 | unsigned long *addrs; |
cad9931f MH |
162 | |
163 | /* Convert symbols to symbol address */ | |
164 | addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL); | |
165 | if (!addrs) | |
166 | return ERR_PTR(-ENOMEM); | |
167 | ||
8be92533 JO |
168 | /* ftrace_lookup_symbols expects sorted symbols */ |
169 | sort(syms, num, sizeof(*syms), symbols_cmp, NULL); | |
cad9931f | 170 | |
8be92533 JO |
171 | if (!ftrace_lookup_symbols(syms, num, addrs)) |
172 | return addrs; | |
cad9931f | 173 | |
cad9931f | 174 | kfree(addrs); |
cad9931f MH |
175 | return ERR_PTR(-ENOENT); |
176 | } | |
177 | ||
178 | static void fprobe_init(struct fprobe *fp) | |
179 | { | |
180 | fp->nmissed = 0; | |
ab51e15d MH |
181 | if (fprobe_shared_with_kprobes(fp)) |
182 | fp->ops.func = fprobe_kprobe_handler; | |
183 | else | |
184 | fp->ops.func = fprobe_handler; | |
cad9931f MH |
185 | fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS; |
186 | } | |
187 | ||
5b0ab789 MH |
188 | static int fprobe_init_rethook(struct fprobe *fp, int num) |
189 | { | |
4bbd9345 | 190 | int size; |
5b0ab789 | 191 | |
700b2b43 | 192 | if (num <= 0) |
5b0ab789 MH |
193 | return -EINVAL; |
194 | ||
195 | if (!fp->exit_handler) { | |
196 | fp->rethook = NULL; | |
197 | return 0; | |
198 | } | |
199 | ||
200 | /* Initialize rethook if needed */ | |
59a7a298 MHG |
201 | if (fp->nr_maxactive) |
202 | size = fp->nr_maxactive; | |
203 | else | |
204 | size = num * num_possible_cpus() * 2; | |
700b2b43 MHG |
205 | if (size <= 0) |
206 | return -EINVAL; | |
5b0ab789 | 207 | |
4bbd9345 | 208 | /* Initialize rethook */ |
209 | fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler, | |
210 | sizeof(struct fprobe_rethook_node), size); | |
211 | if (IS_ERR(fp->rethook)) | |
212 | return PTR_ERR(fp->rethook); | |
213 | ||
5b0ab789 MH |
214 | return 0; |
215 | } | |
216 | ||
217 | static void fprobe_fail_cleanup(struct fprobe *fp) | |
218 | { | |
4bbd9345 | 219 | if (!IS_ERR_OR_NULL(fp->rethook)) { |
5b0ab789 MH |
220 | /* Don't need to cleanup rethook->handler because this is not used. */ |
221 | rethook_free(fp->rethook); | |
222 | fp->rethook = NULL; | |
223 | } | |
224 | ftrace_free_filter(&fp->ops); | |
225 | } | |
226 | ||
cad9931f MH |
227 | /** |
228 | * register_fprobe() - Register fprobe to ftrace by pattern. | |
229 | * @fp: A fprobe data structure to be registered. | |
230 | * @filter: A wildcard pattern of probed symbols. | |
231 | * @notfilter: A wildcard pattern of NOT probed symbols. | |
232 | * | |
233 | * Register @fp to ftrace for enabling the probe on the symbols matched to @filter. | |
234 | * If @notfilter is not NULL, the symbols matched the @notfilter are not probed. | |
235 | * | |
236 | * Return 0 if @fp is registered successfully, -errno if not. | |
237 | */ | |
238 | int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter) | |
239 | { | |
5b0ab789 | 240 | struct ftrace_hash *hash; |
cad9931f MH |
241 | unsigned char *str; |
242 | int ret, len; | |
243 | ||
244 | if (!fp || !filter) | |
245 | return -EINVAL; | |
246 | ||
247 | fprobe_init(fp); | |
248 | ||
249 | len = strlen(filter); | |
250 | str = kstrdup(filter, GFP_KERNEL); | |
251 | ret = ftrace_set_filter(&fp->ops, str, len, 0); | |
252 | kfree(str); | |
253 | if (ret) | |
254 | return ret; | |
255 | ||
256 | if (notfilter) { | |
257 | len = strlen(notfilter); | |
258 | str = kstrdup(notfilter, GFP_KERNEL); | |
259 | ret = ftrace_set_notrace(&fp->ops, str, len, 0); | |
260 | kfree(str); | |
261 | if (ret) | |
262 | goto out; | |
263 | } | |
264 | ||
5b0ab789 MH |
265 | /* TODO: |
266 | * correctly calculate the total number of filtered symbols | |
267 | * from both filter and notfilter. | |
268 | */ | |
261608f3 | 269 | hash = rcu_access_pointer(fp->ops.local_hash.filter_hash); |
5b0ab789 MH |
270 | if (WARN_ON_ONCE(!hash)) |
271 | goto out; | |
272 | ||
273 | ret = fprobe_init_rethook(fp, (int)hash->count); | |
274 | if (!ret) | |
275 | ret = register_ftrace_function(&fp->ops); | |
276 | ||
cad9931f MH |
277 | out: |
278 | if (ret) | |
5b0ab789 | 279 | fprobe_fail_cleanup(fp); |
cad9931f MH |
280 | return ret; |
281 | } | |
282 | EXPORT_SYMBOL_GPL(register_fprobe); | |
283 | ||
284 | /** | |
285 | * register_fprobe_ips() - Register fprobe to ftrace by address. | |
286 | * @fp: A fprobe data structure to be registered. | |
287 | * @addrs: An array of target ftrace location addresses. | |
288 | * @num: The number of entries of @addrs. | |
289 | * | |
290 | * Register @fp to ftrace for enabling the probe on the address given by @addrs. | |
291 | * The @addrs must be the addresses of ftrace location address, which may be | |
292 | * the symbol address + arch-dependent offset. | |
293 | * If you unsure what this mean, please use other registration functions. | |
294 | * | |
295 | * Return 0 if @fp is registered successfully, -errno if not. | |
296 | */ | |
297 | int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num) | |
298 | { | |
299 | int ret; | |
300 | ||
301 | if (!fp || !addrs || num <= 0) | |
302 | return -EINVAL; | |
303 | ||
304 | fprobe_init(fp); | |
305 | ||
306 | ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0); | |
5b0ab789 MH |
307 | if (ret) |
308 | return ret; | |
309 | ||
310 | ret = fprobe_init_rethook(fp, num); | |
cad9931f MH |
311 | if (!ret) |
312 | ret = register_ftrace_function(&fp->ops); | |
313 | ||
314 | if (ret) | |
5b0ab789 | 315 | fprobe_fail_cleanup(fp); |
cad9931f MH |
316 | return ret; |
317 | } | |
318 | EXPORT_SYMBOL_GPL(register_fprobe_ips); | |
319 | ||
320 | /** | |
321 | * register_fprobe_syms() - Register fprobe to ftrace by symbols. | |
322 | * @fp: A fprobe data structure to be registered. | |
323 | * @syms: An array of target symbols. | |
324 | * @num: The number of entries of @syms. | |
325 | * | |
326 | * Register @fp to the symbols given by @syms array. This will be useful if | |
327 | * you are sure the symbols exist in the kernel. | |
328 | * | |
329 | * Return 0 if @fp is registered successfully, -errno if not. | |
330 | */ | |
331 | int register_fprobe_syms(struct fprobe *fp, const char **syms, int num) | |
332 | { | |
333 | unsigned long *addrs; | |
334 | int ret; | |
335 | ||
336 | if (!fp || !syms || num <= 0) | |
337 | return -EINVAL; | |
338 | ||
339 | addrs = get_ftrace_locations(syms, num); | |
340 | if (IS_ERR(addrs)) | |
341 | return PTR_ERR(addrs); | |
342 | ||
343 | ret = register_fprobe_ips(fp, addrs, num); | |
344 | ||
345 | kfree(addrs); | |
346 | ||
347 | return ret; | |
348 | } | |
349 | EXPORT_SYMBOL_GPL(register_fprobe_syms); | |
350 | ||
334e5519 MHG |
351 | bool fprobe_is_registered(struct fprobe *fp) |
352 | { | |
353 | if (!fp || (fp->ops.saved_func != fprobe_handler && | |
354 | fp->ops.saved_func != fprobe_kprobe_handler)) | |
355 | return false; | |
356 | return true; | |
357 | } | |
358 | ||
cad9931f MH |
359 | /** |
360 | * unregister_fprobe() - Unregister fprobe from ftrace | |
361 | * @fp: A fprobe data structure to be unregistered. | |
362 | * | |
363 | * Unregister fprobe (and remove ftrace hooks from the function entries). | |
364 | * | |
365 | * Return 0 if @fp is unregistered successfully, -errno if not. | |
366 | */ | |
367 | int unregister_fprobe(struct fprobe *fp) | |
368 | { | |
369 | int ret; | |
370 | ||
334e5519 | 371 | if (!fprobe_is_registered(fp)) |
cad9931f MH |
372 | return -EINVAL; |
373 | ||
4bbd9345 | 374 | if (!IS_ERR_OR_NULL(fp->rethook)) |
195b9cb5 | 375 | rethook_stop(fp->rethook); |
5b0ab789 | 376 | |
cad9931f | 377 | ret = unregister_ftrace_function(&fp->ops); |
5b0ab789 MH |
378 | if (ret < 0) |
379 | return ret; | |
cad9931f | 380 | |
4bbd9345 | 381 | if (!IS_ERR_OR_NULL(fp->rethook)) |
5f810187 JO |
382 | rethook_free(fp->rethook); |
383 | ||
5b0ab789 | 384 | ftrace_free_filter(&fp->ops); |
cad9931f MH |
385 | |
386 | return ret; | |
387 | } | |
388 | EXPORT_SYMBOL_GPL(unregister_fprobe); |