Commit | Line | Data |
---|---|---|
cad9931f MH |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * fprobe - Simple ftrace probe wrapper for function entry. | |
4 | */ | |
5 | #define pr_fmt(fmt) "fprobe: " fmt | |
6 | ||
7 | #include <linux/err.h> | |
8 | #include <linux/fprobe.h> | |
9 | #include <linux/kallsyms.h> | |
10 | #include <linux/kprobes.h> | |
5b0ab789 | 11 | #include <linux/rethook.h> |
cad9931f MH |
12 | #include <linux/slab.h> |
13 | #include <linux/sort.h> | |
14 | ||
5b0ab789 MH |
15 | #include "trace.h" |
16 | ||
17 | struct fprobe_rethook_node { | |
18 | struct rethook_node node; | |
19 | unsigned long entry_ip; | |
27527410 | 20 | unsigned long entry_parent_ip; |
76d0de57 | 21 | char data[]; |
5b0ab789 MH |
22 | }; |
23 | ||
3cc4e2c5 ZG |
24 | static inline void __fprobe_handler(unsigned long ip, unsigned long parent_ip, |
25 | struct ftrace_ops *ops, struct ftrace_regs *fregs) | |
cad9931f | 26 | { |
5b0ab789 | 27 | struct fprobe_rethook_node *fpr; |
76d0de57 | 28 | struct rethook_node *rh = NULL; |
cad9931f | 29 | struct fprobe *fp; |
76d0de57 | 30 | void *entry_data = NULL; |
3cc4e2c5 | 31 | int ret = 0; |
cad9931f MH |
32 | |
33 | fp = container_of(ops, struct fprobe, ops); | |
cad9931f | 34 | |
5b0ab789 MH |
35 | if (fp->exit_handler) { |
36 | rh = rethook_try_get(fp->rethook); | |
37 | if (!rh) { | |
38 | fp->nmissed++; | |
3cc4e2c5 | 39 | return; |
5b0ab789 MH |
40 | } |
41 | fpr = container_of(rh, struct fprobe_rethook_node, node); | |
42 | fpr->entry_ip = ip; | |
27527410 | 43 | fpr->entry_parent_ip = parent_ip; |
76d0de57 MHG |
44 | if (fp->entry_data_size) |
45 | entry_data = fpr->data; | |
5b0ab789 MH |
46 | } |
47 | ||
76d0de57 | 48 | if (fp->entry_handler) |
cb16330d | 49 | ret = fp->entry_handler(fp, ip, parent_ip, ftrace_get_regs(fregs), entry_data); |
76d0de57 | 50 | |
39d95420 MHG |
51 | /* If entry_handler returns !0, nmissed is not counted. */ |
52 | if (rh) { | |
53 | if (ret) | |
54 | rethook_recycle(rh); | |
55 | else | |
56 | rethook_hook(rh, ftrace_get_regs(fregs), true); | |
57 | } | |
3cc4e2c5 ZG |
58 | } |
59 | ||
60 | static void fprobe_handler(unsigned long ip, unsigned long parent_ip, | |
61 | struct ftrace_ops *ops, struct ftrace_regs *fregs) | |
62 | { | |
63 | struct fprobe *fp; | |
64 | int bit; | |
65 | ||
66 | fp = container_of(ops, struct fprobe, ops); | |
67 | if (fprobe_disabled(fp)) | |
68 | return; | |
69 | ||
70 | /* recursion detection has to go before any traceable function and | |
71 | * all functions before this point should be marked as notrace | |
72 | */ | |
73 | bit = ftrace_test_recursion_trylock(ip, parent_ip); | |
74 | if (bit < 0) { | |
75 | fp->nmissed++; | |
76 | return; | |
77 | } | |
78 | __fprobe_handler(ip, parent_ip, ops, fregs); | |
cad9931f | 79 | ftrace_test_recursion_unlock(bit); |
3cc4e2c5 | 80 | |
cad9931f MH |
81 | } |
82 | NOKPROBE_SYMBOL(fprobe_handler); | |
83 | ||
ab51e15d MH |
84 | static void fprobe_kprobe_handler(unsigned long ip, unsigned long parent_ip, |
85 | struct ftrace_ops *ops, struct ftrace_regs *fregs) | |
86 | { | |
3cc4e2c5 ZG |
87 | struct fprobe *fp; |
88 | int bit; | |
89 | ||
90 | fp = container_of(ops, struct fprobe, ops); | |
91 | if (fprobe_disabled(fp)) | |
92 | return; | |
93 | ||
94 | /* recursion detection has to go before any traceable function and | |
95 | * all functions called before this point should be marked as notrace | |
96 | */ | |
97 | bit = ftrace_test_recursion_trylock(ip, parent_ip); | |
98 | if (bit < 0) { | |
99 | fp->nmissed++; | |
100 | return; | |
101 | } | |
ab51e15d | 102 | |
d5f28bb1 MHG |
103 | /* |
104 | * This user handler is shared with other kprobes and is not expected to be | |
105 | * called recursively. So if any other kprobe handler is running, this will | |
106 | * exit as kprobe does. See the section 'Share the callbacks with kprobes' | |
107 | * in Documentation/trace/fprobe.rst for more information. | |
108 | */ | |
ab51e15d MH |
109 | if (unlikely(kprobe_running())) { |
110 | fp->nmissed++; | |
5f0c584d | 111 | goto recursion_unlock; |
ab51e15d | 112 | } |
3cc4e2c5 | 113 | |
ab51e15d | 114 | kprobe_busy_begin(); |
3cc4e2c5 | 115 | __fprobe_handler(ip, parent_ip, ops, fregs); |
ab51e15d | 116 | kprobe_busy_end(); |
5f0c584d ZG |
117 | |
118 | recursion_unlock: | |
3cc4e2c5 | 119 | ftrace_test_recursion_unlock(bit); |
ab51e15d MH |
120 | } |
121 | ||
5b0ab789 | 122 | static void fprobe_exit_handler(struct rethook_node *rh, void *data, |
cb16330d | 123 | unsigned long ret_ip, struct pt_regs *regs) |
5b0ab789 MH |
124 | { |
125 | struct fprobe *fp = (struct fprobe *)data; | |
126 | struct fprobe_rethook_node *fpr; | |
27527410 | 127 | int bit; |
5b0ab789 MH |
128 | |
129 | if (!fp || fprobe_disabled(fp)) | |
130 | return; | |
131 | ||
132 | fpr = container_of(rh, struct fprobe_rethook_node, node); | |
133 | ||
27527410 ZG |
134 | /* |
135 | * we need to assure no calls to traceable functions in-between the | |
136 | * end of fprobe_handler and the beginning of fprobe_exit_handler. | |
137 | */ | |
138 | bit = ftrace_test_recursion_trylock(fpr->entry_ip, fpr->entry_parent_ip); | |
139 | if (bit < 0) { | |
140 | fp->nmissed++; | |
141 | return; | |
142 | } | |
143 | ||
cb16330d | 144 | fp->exit_handler(fp, fpr->entry_ip, ret_ip, regs, |
76d0de57 | 145 | fp->entry_data_size ? (void *)fpr->data : NULL); |
27527410 | 146 | ftrace_test_recursion_unlock(bit); |
5b0ab789 MH |
147 | } |
148 | NOKPROBE_SYMBOL(fprobe_exit_handler); | |
149 | ||
8be92533 JO |
150 | static int symbols_cmp(const void *a, const void *b) |
151 | { | |
152 | const char **str_a = (const char **) a; | |
153 | const char **str_b = (const char **) b; | |
154 | ||
155 | return strcmp(*str_a, *str_b); | |
156 | } | |
157 | ||
cad9931f MH |
158 | /* Convert ftrace location address from symbols */ |
159 | static unsigned long *get_ftrace_locations(const char **syms, int num) | |
160 | { | |
cad9931f | 161 | unsigned long *addrs; |
cad9931f MH |
162 | |
163 | /* Convert symbols to symbol address */ | |
164 | addrs = kcalloc(num, sizeof(*addrs), GFP_KERNEL); | |
165 | if (!addrs) | |
166 | return ERR_PTR(-ENOMEM); | |
167 | ||
8be92533 JO |
168 | /* ftrace_lookup_symbols expects sorted symbols */ |
169 | sort(syms, num, sizeof(*syms), symbols_cmp, NULL); | |
cad9931f | 170 | |
8be92533 JO |
171 | if (!ftrace_lookup_symbols(syms, num, addrs)) |
172 | return addrs; | |
cad9931f | 173 | |
cad9931f | 174 | kfree(addrs); |
cad9931f MH |
175 | return ERR_PTR(-ENOENT); |
176 | } | |
177 | ||
178 | static void fprobe_init(struct fprobe *fp) | |
179 | { | |
180 | fp->nmissed = 0; | |
ab51e15d MH |
181 | if (fprobe_shared_with_kprobes(fp)) |
182 | fp->ops.func = fprobe_kprobe_handler; | |
183 | else | |
184 | fp->ops.func = fprobe_handler; | |
cad9931f MH |
185 | fp->ops.flags |= FTRACE_OPS_FL_SAVE_REGS; |
186 | } | |
187 | ||
5b0ab789 MH |
188 | static int fprobe_init_rethook(struct fprobe *fp, int num) |
189 | { | |
4bbd9345 | 190 | int size; |
5b0ab789 | 191 | |
5b0ab789 MH |
192 | if (!fp->exit_handler) { |
193 | fp->rethook = NULL; | |
194 | return 0; | |
195 | } | |
196 | ||
197 | /* Initialize rethook if needed */ | |
59a7a298 | 198 | if (fp->nr_maxactive) |
65727860 | 199 | num = fp->nr_maxactive; |
59a7a298 | 200 | else |
65727860 MHG |
201 | num *= num_possible_cpus() * 2; |
202 | if (num <= 0) | |
700b2b43 | 203 | return -EINVAL; |
5b0ab789 | 204 | |
65727860 MHG |
205 | size = sizeof(struct fprobe_rethook_node) + fp->entry_data_size; |
206 | ||
4bbd9345 | 207 | /* Initialize rethook */ |
65727860 | 208 | fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler, size, num); |
4bbd9345 | 209 | if (IS_ERR(fp->rethook)) |
210 | return PTR_ERR(fp->rethook); | |
211 | ||
5b0ab789 MH |
212 | return 0; |
213 | } | |
214 | ||
215 | static void fprobe_fail_cleanup(struct fprobe *fp) | |
216 | { | |
4bbd9345 | 217 | if (!IS_ERR_OR_NULL(fp->rethook)) { |
5b0ab789 MH |
218 | /* Don't need to cleanup rethook->handler because this is not used. */ |
219 | rethook_free(fp->rethook); | |
220 | fp->rethook = NULL; | |
221 | } | |
222 | ftrace_free_filter(&fp->ops); | |
223 | } | |
224 | ||
cad9931f MH |
225 | /** |
226 | * register_fprobe() - Register fprobe to ftrace by pattern. | |
227 | * @fp: A fprobe data structure to be registered. | |
228 | * @filter: A wildcard pattern of probed symbols. | |
229 | * @notfilter: A wildcard pattern of NOT probed symbols. | |
230 | * | |
231 | * Register @fp to ftrace for enabling the probe on the symbols matched to @filter. | |
232 | * If @notfilter is not NULL, the symbols matched the @notfilter are not probed. | |
233 | * | |
234 | * Return 0 if @fp is registered successfully, -errno if not. | |
235 | */ | |
236 | int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter) | |
237 | { | |
5b0ab789 | 238 | struct ftrace_hash *hash; |
cad9931f MH |
239 | unsigned char *str; |
240 | int ret, len; | |
241 | ||
242 | if (!fp || !filter) | |
243 | return -EINVAL; | |
244 | ||
245 | fprobe_init(fp); | |
246 | ||
247 | len = strlen(filter); | |
248 | str = kstrdup(filter, GFP_KERNEL); | |
249 | ret = ftrace_set_filter(&fp->ops, str, len, 0); | |
250 | kfree(str); | |
251 | if (ret) | |
252 | return ret; | |
253 | ||
254 | if (notfilter) { | |
255 | len = strlen(notfilter); | |
256 | str = kstrdup(notfilter, GFP_KERNEL); | |
257 | ret = ftrace_set_notrace(&fp->ops, str, len, 0); | |
258 | kfree(str); | |
259 | if (ret) | |
260 | goto out; | |
261 | } | |
262 | ||
5b0ab789 MH |
263 | /* TODO: |
264 | * correctly calculate the total number of filtered symbols | |
265 | * from both filter and notfilter. | |
266 | */ | |
261608f3 | 267 | hash = rcu_access_pointer(fp->ops.local_hash.filter_hash); |
5b0ab789 MH |
268 | if (WARN_ON_ONCE(!hash)) |
269 | goto out; | |
270 | ||
271 | ret = fprobe_init_rethook(fp, (int)hash->count); | |
272 | if (!ret) | |
273 | ret = register_ftrace_function(&fp->ops); | |
274 | ||
cad9931f MH |
275 | out: |
276 | if (ret) | |
5b0ab789 | 277 | fprobe_fail_cleanup(fp); |
cad9931f MH |
278 | return ret; |
279 | } | |
280 | EXPORT_SYMBOL_GPL(register_fprobe); | |
281 | ||
282 | /** | |
283 | * register_fprobe_ips() - Register fprobe to ftrace by address. | |
284 | * @fp: A fprobe data structure to be registered. | |
285 | * @addrs: An array of target ftrace location addresses. | |
286 | * @num: The number of entries of @addrs. | |
287 | * | |
288 | * Register @fp to ftrace for enabling the probe on the address given by @addrs. | |
289 | * The @addrs must be the addresses of ftrace location address, which may be | |
290 | * the symbol address + arch-dependent offset. | |
291 | * If you unsure what this mean, please use other registration functions. | |
292 | * | |
293 | * Return 0 if @fp is registered successfully, -errno if not. | |
294 | */ | |
295 | int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num) | |
296 | { | |
297 | int ret; | |
298 | ||
299 | if (!fp || !addrs || num <= 0) | |
300 | return -EINVAL; | |
301 | ||
302 | fprobe_init(fp); | |
303 | ||
304 | ret = ftrace_set_filter_ips(&fp->ops, addrs, num, 0, 0); | |
5b0ab789 MH |
305 | if (ret) |
306 | return ret; | |
307 | ||
308 | ret = fprobe_init_rethook(fp, num); | |
cad9931f MH |
309 | if (!ret) |
310 | ret = register_ftrace_function(&fp->ops); | |
311 | ||
312 | if (ret) | |
5b0ab789 | 313 | fprobe_fail_cleanup(fp); |
cad9931f MH |
314 | return ret; |
315 | } | |
316 | EXPORT_SYMBOL_GPL(register_fprobe_ips); | |
317 | ||
318 | /** | |
319 | * register_fprobe_syms() - Register fprobe to ftrace by symbols. | |
320 | * @fp: A fprobe data structure to be registered. | |
321 | * @syms: An array of target symbols. | |
322 | * @num: The number of entries of @syms. | |
323 | * | |
324 | * Register @fp to the symbols given by @syms array. This will be useful if | |
325 | * you are sure the symbols exist in the kernel. | |
326 | * | |
327 | * Return 0 if @fp is registered successfully, -errno if not. | |
328 | */ | |
329 | int register_fprobe_syms(struct fprobe *fp, const char **syms, int num) | |
330 | { | |
331 | unsigned long *addrs; | |
332 | int ret; | |
333 | ||
334 | if (!fp || !syms || num <= 0) | |
335 | return -EINVAL; | |
336 | ||
337 | addrs = get_ftrace_locations(syms, num); | |
338 | if (IS_ERR(addrs)) | |
339 | return PTR_ERR(addrs); | |
340 | ||
341 | ret = register_fprobe_ips(fp, addrs, num); | |
342 | ||
343 | kfree(addrs); | |
344 | ||
345 | return ret; | |
346 | } | |
347 | EXPORT_SYMBOL_GPL(register_fprobe_syms); | |
348 | ||
334e5519 MHG |
349 | bool fprobe_is_registered(struct fprobe *fp) |
350 | { | |
351 | if (!fp || (fp->ops.saved_func != fprobe_handler && | |
352 | fp->ops.saved_func != fprobe_kprobe_handler)) | |
353 | return false; | |
354 | return true; | |
355 | } | |
356 | ||
cad9931f MH |
357 | /** |
358 | * unregister_fprobe() - Unregister fprobe from ftrace | |
359 | * @fp: A fprobe data structure to be unregistered. | |
360 | * | |
361 | * Unregister fprobe (and remove ftrace hooks from the function entries). | |
362 | * | |
363 | * Return 0 if @fp is unregistered successfully, -errno if not. | |
364 | */ | |
365 | int unregister_fprobe(struct fprobe *fp) | |
366 | { | |
367 | int ret; | |
368 | ||
334e5519 | 369 | if (!fprobe_is_registered(fp)) |
cad9931f MH |
370 | return -EINVAL; |
371 | ||
4bbd9345 | 372 | if (!IS_ERR_OR_NULL(fp->rethook)) |
195b9cb5 | 373 | rethook_stop(fp->rethook); |
5b0ab789 | 374 | |
cad9931f | 375 | ret = unregister_ftrace_function(&fp->ops); |
5b0ab789 MH |
376 | if (ret < 0) |
377 | return ret; | |
cad9931f | 378 | |
4bbd9345 | 379 | if (!IS_ERR_OR_NULL(fp->rethook)) |
5f810187 JO |
380 | rethook_free(fp->rethook); |
381 | ||
5b0ab789 | 382 | ftrace_free_filter(&fp->ops); |
cad9931f MH |
383 | |
384 | return ret; | |
385 | } | |
386 | EXPORT_SYMBOL_GPL(unregister_fprobe); |