Commit | Line | Data |
---|---|---|
bcea3f96 | 1 | // SPDX-License-Identifier: GPL-2.0 |
413d37d1 | 2 | /* |
77b44d1b | 3 | * Kprobes-based tracing events |
413d37d1 MH |
4 | * |
5 | * Created by Masami Hiramatsu <mhiramat@redhat.com> | |
6 | * | |
413d37d1 | 7 | */ |
72576341 | 8 | #define pr_fmt(fmt) "trace_kprobe: " fmt |
413d37d1 MH |
9 | |
10 | #include <linux/module.h> | |
11 | #include <linux/uaccess.h> | |
b2d09103 | 12 | #include <linux/rculist.h> |
540adea3 | 13 | #include <linux/error-injection.h> |
a94549dd | 14 | #include <linux/security.h> |
413d37d1 | 15 | |
970988e1 MH |
16 | #include <asm/setup.h> /* for COMMAND_LINE_SIZE */ |
17 | ||
6212dd29 | 18 | #include "trace_dynevent.h" |
d899926f | 19 | #include "trace_kprobe_selftest.h" |
8ab83f56 | 20 | #include "trace_probe.h" |
53305928 | 21 | #include "trace_probe_tmpl.h" |
1ff511e3 | 22 | |
8ab83f56 | 23 | #define KPROBE_EVENT_SYSTEM "kprobes" |
696ced4f | 24 | #define KRETPROBE_MAXACTIVE_MAX 4096 |
970988e1 MH |
25 | #define MAX_KPROBE_CMDLINE_SIZE 1024 |
26 | ||
27 | /* Kprobe early definition from command line */ | |
28 | static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata; | |
b6399cc7 | 29 | static bool kprobe_boot_events_enabled __initdata; |
970988e1 MH |
30 | |
31 | static int __init set_kprobe_boot_events(char *str) | |
32 | { | |
33 | strlcpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE); | |
34 | return 0; | |
35 | } | |
36 | __setup("kprobe_event=", set_kprobe_boot_events); | |
e09c8614 | 37 | |
6212dd29 MH |
38 | static int trace_kprobe_create(int argc, const char **argv); |
39 | static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev); | |
40 | static int trace_kprobe_release(struct dyn_event *ev); | |
41 | static bool trace_kprobe_is_busy(struct dyn_event *ev); | |
42 | static bool trace_kprobe_match(const char *system, const char *event, | |
30199137 | 43 | int argc, const char **argv, struct dyn_event *ev); |
6212dd29 MH |
44 | |
45 | static struct dyn_event_operations trace_kprobe_ops = { | |
46 | .create = trace_kprobe_create, | |
47 | .show = trace_kprobe_show, | |
48 | .is_busy = trace_kprobe_is_busy, | |
49 | .free = trace_kprobe_release, | |
50 | .match = trace_kprobe_match, | |
51 | }; | |
52 | ||
cede666e | 53 | /* |
77b44d1b | 54 | * Kprobe event core functions |
413d37d1 | 55 | */ |
c31ffb3f | 56 | struct trace_kprobe { |
6212dd29 | 57 | struct dyn_event devent; |
4a846b44 | 58 | struct kretprobe rp; /* Use rp.kp for kprobe use */ |
a7636d9e | 59 | unsigned long __percpu *nhit; |
413d37d1 | 60 | const char *symbol; /* symbol name */ |
c31ffb3f | 61 | struct trace_probe tp; |
413d37d1 MH |
62 | }; |
63 | ||
6212dd29 MH |
64 | static bool is_trace_kprobe(struct dyn_event *ev) |
65 | { | |
66 | return ev->ops == &trace_kprobe_ops; | |
67 | } | |
68 | ||
69 | static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev) | |
70 | { | |
71 | return container_of(ev, struct trace_kprobe, devent); | |
72 | } | |
73 | ||
74 | /** | |
75 | * for_each_trace_kprobe - iterate over the trace_kprobe list | |
76 | * @pos: the struct trace_kprobe * for each entry | |
77 | * @dpos: the struct dyn_event * to use as a loop cursor | |
78 | */ | |
79 | #define for_each_trace_kprobe(pos, dpos) \ | |
80 | for_each_dyn_event(dpos) \ | |
81 | if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos))) | |
82 | ||
c31ffb3f NK |
83 | #define SIZEOF_TRACE_KPROBE(n) \ |
84 | (offsetof(struct trace_kprobe, tp.args) + \ | |
eca0d916 | 85 | (sizeof(struct probe_arg) * (n))) |
a82378d8 | 86 | |
3da0f180 | 87 | static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk) |
413d37d1 | 88 | { |
c31ffb3f | 89 | return tk->rp.handler != NULL; |
413d37d1 MH |
90 | } |
91 | ||
3da0f180 | 92 | static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk) |
413d37d1 | 93 | { |
c31ffb3f | 94 | return tk->symbol ? tk->symbol : "unknown"; |
413d37d1 MH |
95 | } |
96 | ||
3da0f180 | 97 | static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk) |
61424318 | 98 | { |
c31ffb3f | 99 | return tk->rp.kp.offset; |
61424318 MH |
100 | } |
101 | ||
3da0f180 | 102 | static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk) |
61424318 | 103 | { |
c31ffb3f | 104 | return !!(kprobe_gone(&tk->rp.kp)); |
61424318 MH |
105 | } |
106 | ||
3da0f180 | 107 | static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk, |
c31ffb3f | 108 | struct module *mod) |
61424318 MH |
109 | { |
110 | int len = strlen(mod->name); | |
c31ffb3f | 111 | const char *name = trace_kprobe_symbol(tk); |
61424318 MH |
112 | return strncmp(mod->name, name, len) == 0 && name[len] == ':'; |
113 | } | |
114 | ||
59158ec4 | 115 | static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk) |
61424318 | 116 | { |
59158ec4 MH |
117 | char *p; |
118 | bool ret; | |
119 | ||
120 | if (!tk->symbol) | |
121 | return false; | |
122 | p = strchr(tk->symbol, ':'); | |
123 | if (!p) | |
124 | return true; | |
125 | *p = '\0'; | |
126 | mutex_lock(&module_mutex); | |
127 | ret = !!find_module(tk->symbol); | |
128 | mutex_unlock(&module_mutex); | |
129 | *p = ':'; | |
130 | ||
131 | return ret; | |
61424318 MH |
132 | } |
133 | ||
6212dd29 MH |
134 | static bool trace_kprobe_is_busy(struct dyn_event *ev) |
135 | { | |
136 | struct trace_kprobe *tk = to_trace_kprobe(ev); | |
137 | ||
138 | return trace_probe_is_enabled(&tk->tp); | |
139 | } | |
140 | ||
eb5bf813 MH |
141 | static bool trace_kprobe_match_command_head(struct trace_kprobe *tk, |
142 | int argc, const char **argv) | |
143 | { | |
144 | char buf[MAX_ARGSTR_LEN + 1]; | |
145 | ||
146 | if (!argc) | |
147 | return true; | |
148 | ||
149 | if (!tk->symbol) | |
150 | snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr); | |
151 | else if (tk->rp.kp.offset) | |
152 | snprintf(buf, sizeof(buf), "%s+%u", | |
153 | trace_kprobe_symbol(tk), tk->rp.kp.offset); | |
154 | else | |
155 | snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk)); | |
156 | if (strcmp(buf, argv[0])) | |
157 | return false; | |
158 | argc--; argv++; | |
159 | ||
160 | return trace_probe_match_command_args(&tk->tp, argc, argv); | |
161 | } | |
162 | ||
6212dd29 | 163 | static bool trace_kprobe_match(const char *system, const char *event, |
30199137 | 164 | int argc, const char **argv, struct dyn_event *ev) |
6212dd29 MH |
165 | { |
166 | struct trace_kprobe *tk = to_trace_kprobe(ev); | |
167 | ||
b55ce203 | 168 | return strcmp(trace_probe_name(&tk->tp), event) == 0 && |
eb5bf813 MH |
169 | (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) && |
170 | trace_kprobe_match_command_head(tk, argc, argv); | |
6212dd29 MH |
171 | } |
172 | ||
f18f97ac MN |
173 | static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk) |
174 | { | |
175 | unsigned long nhit = 0; | |
176 | int cpu; | |
177 | ||
178 | for_each_possible_cpu(cpu) | |
179 | nhit += *per_cpu_ptr(tk->nhit, cpu); | |
180 | ||
181 | return nhit; | |
182 | } | |
183 | ||
715fa2fd MH |
184 | static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk) |
185 | { | |
186 | return !(list_empty(&tk->rp.kp.list) && | |
187 | hlist_unhashed(&tk->rp.kp.hlist)); | |
188 | } | |
189 | ||
6bc6c77c | 190 | /* Return 0 if it fails to find the symbol address */ |
45408c4f MH |
191 | static nokprobe_inline |
192 | unsigned long trace_kprobe_address(struct trace_kprobe *tk) | |
193 | { | |
194 | unsigned long addr; | |
195 | ||
196 | if (tk->symbol) { | |
197 | addr = (unsigned long) | |
198 | kallsyms_lookup_name(trace_kprobe_symbol(tk)); | |
6bc6c77c MH |
199 | if (addr) |
200 | addr += tk->rp.kp.offset; | |
45408c4f MH |
201 | } else { |
202 | addr = (unsigned long)tk->rp.kp.addr; | |
203 | } | |
204 | return addr; | |
205 | } | |
206 | ||
60d53e2c MH |
207 | static nokprobe_inline struct trace_kprobe * |
208 | trace_kprobe_primary_from_call(struct trace_event_call *call) | |
209 | { | |
210 | struct trace_probe *tp; | |
211 | ||
212 | tp = trace_probe_primary_from_call(call); | |
213 | if (WARN_ON_ONCE(!tp)) | |
214 | return NULL; | |
215 | ||
216 | return container_of(tp, struct trace_kprobe, tp); | |
217 | } | |
218 | ||
b4da3340 | 219 | bool trace_kprobe_on_func_entry(struct trace_event_call *call) |
9802d865 | 220 | { |
60d53e2c | 221 | struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); |
b4da3340 | 222 | |
60d53e2c | 223 | return tk ? kprobe_on_func_entry(tk->rp.kp.addr, |
b4da3340 | 224 | tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name, |
60d53e2c | 225 | tk->rp.kp.addr ? 0 : tk->rp.kp.offset) : false; |
9802d865 JB |
226 | } |
227 | ||
b4da3340 | 228 | bool trace_kprobe_error_injectable(struct trace_event_call *call) |
9802d865 | 229 | { |
60d53e2c | 230 | struct trace_kprobe *tk = trace_kprobe_primary_from_call(call); |
9802d865 | 231 | |
60d53e2c MH |
232 | return tk ? within_error_injection_list(trace_kprobe_address(tk)) : |
233 | false; | |
9802d865 JB |
234 | } |
235 | ||
c31ffb3f NK |
236 | static int register_kprobe_event(struct trace_kprobe *tk); |
237 | static int unregister_kprobe_event(struct trace_kprobe *tk); | |
413d37d1 | 238 | |
50d78056 MH |
239 | static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); |
240 | static int kretprobe_dispatcher(struct kretprobe_instance *ri, | |
241 | struct pt_regs *regs); | |
242 | ||
455b2899 MH |
243 | static void free_trace_kprobe(struct trace_kprobe *tk) |
244 | { | |
245 | if (tk) { | |
246 | trace_probe_cleanup(&tk->tp); | |
247 | kfree(tk->symbol); | |
248 | free_percpu(tk->nhit); | |
249 | kfree(tk); | |
250 | } | |
251 | } | |
252 | ||
4a846b44 MH |
253 | /* |
254 | * Allocate new trace_probe and initialize it (including kprobes). | |
255 | */ | |
c31ffb3f | 256 | static struct trace_kprobe *alloc_trace_kprobe(const char *group, |
f52487e9 | 257 | const char *event, |
4a846b44 MH |
258 | void *addr, |
259 | const char *symbol, | |
260 | unsigned long offs, | |
696ced4f | 261 | int maxactive, |
3a6b7666 | 262 | int nargs, bool is_return) |
413d37d1 | 263 | { |
c31ffb3f | 264 | struct trace_kprobe *tk; |
6f3cf440 | 265 | int ret = -ENOMEM; |
413d37d1 | 266 | |
c31ffb3f NK |
267 | tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL); |
268 | if (!tk) | |
6f3cf440 | 269 | return ERR_PTR(ret); |
413d37d1 | 270 | |
a7636d9e MKL |
271 | tk->nhit = alloc_percpu(unsigned long); |
272 | if (!tk->nhit) | |
273 | goto error; | |
274 | ||
413d37d1 | 275 | if (symbol) { |
c31ffb3f NK |
276 | tk->symbol = kstrdup(symbol, GFP_KERNEL); |
277 | if (!tk->symbol) | |
413d37d1 | 278 | goto error; |
c31ffb3f NK |
279 | tk->rp.kp.symbol_name = tk->symbol; |
280 | tk->rp.kp.offset = offs; | |
4a846b44 | 281 | } else |
c31ffb3f | 282 | tk->rp.kp.addr = addr; |
4a846b44 MH |
283 | |
284 | if (is_return) | |
c31ffb3f | 285 | tk->rp.handler = kretprobe_dispatcher; |
4a846b44 | 286 | else |
c31ffb3f | 287 | tk->rp.kp.pre_handler = kprobe_dispatcher; |
4a846b44 | 288 | |
696ced4f | 289 | tk->rp.maxactive = maxactive; |
715fa2fd MH |
290 | INIT_HLIST_NODE(&tk->rp.kp.hlist); |
291 | INIT_LIST_HEAD(&tk->rp.kp.list); | |
696ced4f | 292 | |
455b2899 MH |
293 | ret = trace_probe_init(&tk->tp, event, group); |
294 | if (ret < 0) | |
f52487e9 MH |
295 | goto error; |
296 | ||
6212dd29 | 297 | dyn_event_init(&tk->devent, &trace_kprobe_ops); |
c31ffb3f | 298 | return tk; |
413d37d1 | 299 | error: |
455b2899 | 300 | free_trace_kprobe(tk); |
6f3cf440 | 301 | return ERR_PTR(ret); |
413d37d1 MH |
302 | } |
303 | ||
c31ffb3f NK |
304 | static struct trace_kprobe *find_trace_kprobe(const char *event, |
305 | const char *group) | |
413d37d1 | 306 | { |
6212dd29 | 307 | struct dyn_event *pos; |
c31ffb3f | 308 | struct trace_kprobe *tk; |
413d37d1 | 309 | |
6212dd29 | 310 | for_each_trace_kprobe(tk, pos) |
b55ce203 MH |
311 | if (strcmp(trace_probe_name(&tk->tp), event) == 0 && |
312 | strcmp(trace_probe_group_name(&tk->tp), group) == 0) | |
c31ffb3f | 313 | return tk; |
413d37d1 MH |
314 | return NULL; |
315 | } | |
316 | ||
87107a25 SRV |
317 | static inline int __enable_trace_kprobe(struct trace_kprobe *tk) |
318 | { | |
319 | int ret = 0; | |
320 | ||
715fa2fd | 321 | if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) { |
87107a25 SRV |
322 | if (trace_kprobe_is_return(tk)) |
323 | ret = enable_kretprobe(&tk->rp); | |
324 | else | |
325 | ret = enable_kprobe(&tk->rp.kp); | |
326 | } | |
327 | ||
328 | return ret; | |
329 | } | |
330 | ||
60d53e2c MH |
331 | static void __disable_trace_kprobe(struct trace_probe *tp) |
332 | { | |
333 | struct trace_probe *pos; | |
334 | struct trace_kprobe *tk; | |
335 | ||
336 | list_for_each_entry(pos, trace_probe_probe_list(tp), list) { | |
337 | tk = container_of(pos, struct trace_kprobe, tp); | |
338 | if (!trace_kprobe_is_registered(tk)) | |
339 | continue; | |
340 | if (trace_kprobe_is_return(tk)) | |
341 | disable_kretprobe(&tk->rp); | |
342 | else | |
343 | disable_kprobe(&tk->rp.kp); | |
344 | } | |
345 | } | |
346 | ||
41a7dd42 MH |
347 | /* |
348 | * Enable trace_probe | |
349 | * if the file is NULL, enable "perf" handler, or enable "trace" handler. | |
350 | */ | |
60d53e2c MH |
351 | static int enable_trace_kprobe(struct trace_event_call *call, |
352 | struct trace_event_file *file) | |
1538f888 | 353 | { |
60d53e2c MH |
354 | struct trace_probe *pos, *tp; |
355 | struct trace_kprobe *tk; | |
356 | bool enabled; | |
1538f888 MH |
357 | int ret = 0; |
358 | ||
60d53e2c MH |
359 | tp = trace_probe_primary_from_call(call); |
360 | if (WARN_ON_ONCE(!tp)) | |
361 | return -ENODEV; | |
362 | enabled = trace_probe_is_enabled(tp); | |
363 | ||
364 | /* This also changes "enabled" state */ | |
41a7dd42 | 365 | if (file) { |
60d53e2c | 366 | ret = trace_probe_add_file(tp, file); |
b5f935ee MH |
367 | if (ret) |
368 | return ret; | |
369 | } else | |
60d53e2c | 370 | trace_probe_set_flag(tp, TP_FLAG_PROFILE); |
41a7dd42 | 371 | |
b5f935ee MH |
372 | if (enabled) |
373 | return 0; | |
87107a25 | 374 | |
60d53e2c MH |
375 | list_for_each_entry(pos, trace_probe_probe_list(tp), list) { |
376 | tk = container_of(pos, struct trace_kprobe, tp); | |
377 | if (trace_kprobe_has_gone(tk)) | |
378 | continue; | |
379 | ret = __enable_trace_kprobe(tk); | |
44d00dc7 | 380 | if (ret) |
60d53e2c | 381 | break; |
60d53e2c MH |
382 | enabled = true; |
383 | } | |
384 | ||
44d00dc7 MH |
385 | if (ret) { |
386 | /* Failed to enable one of them. Roll back all */ | |
387 | if (enabled) | |
388 | __disable_trace_kprobe(tp); | |
b5f935ee | 389 | if (file) |
60d53e2c | 390 | trace_probe_remove_file(tp, file); |
b5f935ee | 391 | else |
60d53e2c | 392 | trace_probe_clear_flag(tp, TP_FLAG_PROFILE); |
57ea2a34 | 393 | } |
b5f935ee | 394 | |
1538f888 MH |
395 | return ret; |
396 | } | |
397 | ||
41a7dd42 MH |
398 | /* |
399 | * Disable trace_probe | |
400 | * if the file is NULL, disable "perf" handler, or disable "trace" handler. | |
401 | */ | |
60d53e2c MH |
402 | static int disable_trace_kprobe(struct trace_event_call *call, |
403 | struct trace_event_file *file) | |
1538f888 | 404 | { |
60d53e2c MH |
405 | struct trace_probe *tp; |
406 | ||
407 | tp = trace_probe_primary_from_call(call); | |
408 | if (WARN_ON_ONCE(!tp)) | |
409 | return -ENODEV; | |
41a7dd42 | 410 | |
41a7dd42 | 411 | if (file) { |
b5f935ee MH |
412 | if (!trace_probe_get_file_link(tp, file)) |
413 | return -ENOENT; | |
414 | if (!trace_probe_has_single_file(tp)) | |
b04d52e3 | 415 | goto out; |
747774d6 | 416 | trace_probe_clear_flag(tp, TP_FLAG_TRACE); |
41a7dd42 | 417 | } else |
747774d6 | 418 | trace_probe_clear_flag(tp, TP_FLAG_PROFILE); |
41a7dd42 | 419 | |
60d53e2c MH |
420 | if (!trace_probe_is_enabled(tp)) |
421 | __disable_trace_kprobe(tp); | |
e12f03d7 | 422 | |
3fe3d619 | 423 | out: |
b5f935ee | 424 | if (file) |
a232e270 | 425 | /* |
b5f935ee MH |
426 | * Synchronization is done in below function. For perf event, |
427 | * file == NULL and perf_trace_event_unreg() calls | |
428 | * tracepoint_synchronize_unregister() to ensure synchronize | |
429 | * event. We don't need to care about it. | |
a232e270 | 430 | */ |
b5f935ee | 431 | trace_probe_remove_file(tp, file); |
a232e270 | 432 | |
60d53e2c | 433 | return 0; |
1538f888 MH |
434 | } |
435 | ||
45408c4f MH |
436 | #if defined(CONFIG_KPROBES_ON_FTRACE) && \ |
437 | !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE) | |
438 | static bool within_notrace_func(struct trace_kprobe *tk) | |
439 | { | |
440 | unsigned long offset, size, addr; | |
441 | ||
442 | addr = trace_kprobe_address(tk); | |
6bc6c77c MH |
443 | if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset)) |
444 | return false; | |
45408c4f | 445 | |
9161a864 MH |
446 | /* Get the entry address of the target function */ |
447 | addr -= offset; | |
448 | ||
449 | /* | |
450 | * Since ftrace_location_range() does inclusive range check, we need | |
451 | * to subtract 1 byte from the end address. | |
452 | */ | |
453 | return !ftrace_location_range(addr, addr + size - 1); | |
45408c4f MH |
454 | } |
455 | #else | |
456 | #define within_notrace_func(tk) (false) | |
457 | #endif | |
458 | ||
61424318 | 459 | /* Internal register function - just handle k*probes and flags */ |
c31ffb3f | 460 | static int __register_trace_kprobe(struct trace_kprobe *tk) |
413d37d1 | 461 | { |
a6682814 | 462 | int i, ret; |
61424318 | 463 | |
a94549dd DH |
464 | ret = security_locked_down(LOCKDOWN_KPROBES); |
465 | if (ret) | |
466 | return ret; | |
467 | ||
715fa2fd | 468 | if (trace_kprobe_is_registered(tk)) |
61424318 MH |
469 | return -EINVAL; |
470 | ||
45408c4f MH |
471 | if (within_notrace_func(tk)) { |
472 | pr_warn("Could not probe notrace function %s\n", | |
473 | trace_kprobe_symbol(tk)); | |
474 | return -EINVAL; | |
475 | } | |
476 | ||
a6682814 MH |
477 | for (i = 0; i < tk->tp.nr_args; i++) { |
478 | ret = traceprobe_update_arg(&tk->tp.args[i]); | |
479 | if (ret) | |
480 | return ret; | |
481 | } | |
482 | ||
61424318 | 483 | /* Set/clear disabled flag according to tp->flag */ |
c31ffb3f NK |
484 | if (trace_probe_is_enabled(&tk->tp)) |
485 | tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED; | |
61424318 | 486 | else |
c31ffb3f | 487 | tk->rp.kp.flags |= KPROBE_FLAG_DISABLED; |
61424318 | 488 | |
c31ffb3f NK |
489 | if (trace_kprobe_is_return(tk)) |
490 | ret = register_kretprobe(&tk->rp); | |
413d37d1 | 491 | else |
c31ffb3f | 492 | ret = register_kprobe(&tk->rp.kp); |
61424318 | 493 | |
61424318 MH |
494 | return ret; |
495 | } | |
496 | ||
497 | /* Internal unregister function - just handle k*probes and flags */ | |
c31ffb3f | 498 | static void __unregister_trace_kprobe(struct trace_kprobe *tk) |
61424318 | 499 | { |
715fa2fd | 500 | if (trace_kprobe_is_registered(tk)) { |
c31ffb3f NK |
501 | if (trace_kprobe_is_return(tk)) |
502 | unregister_kretprobe(&tk->rp); | |
61424318 | 503 | else |
c31ffb3f | 504 | unregister_kprobe(&tk->rp.kp); |
715fa2fd MH |
505 | /* Cleanup kprobe for reuse and mark it unregistered */ |
506 | INIT_HLIST_NODE(&tk->rp.kp.hlist); | |
507 | INIT_LIST_HEAD(&tk->rp.kp.list); | |
c31ffb3f NK |
508 | if (tk->rp.kp.symbol_name) |
509 | tk->rp.kp.addr = NULL; | |
61424318 MH |
510 | } |
511 | } | |
512 | ||
6212dd29 | 513 | /* Unregister a trace_probe and probe_event */ |
c31ffb3f | 514 | static int unregister_trace_kprobe(struct trace_kprobe *tk) |
61424318 | 515 | { |
ca89bc07 MH |
516 | /* If other probes are on the event, just unregister kprobe */ |
517 | if (trace_probe_has_sibling(&tk->tp)) | |
518 | goto unreg; | |
519 | ||
02ca1521 | 520 | /* Enabled event can not be unregistered */ |
c31ffb3f | 521 | if (trace_probe_is_enabled(&tk->tp)) |
02ca1521 MH |
522 | return -EBUSY; |
523 | ||
40c32592 | 524 | /* Will fail if probe is being used by ftrace or perf */ |
c31ffb3f | 525 | if (unregister_kprobe_event(tk)) |
40c32592 SRRH |
526 | return -EBUSY; |
527 | ||
ca89bc07 | 528 | unreg: |
c31ffb3f | 529 | __unregister_trace_kprobe(tk); |
6212dd29 | 530 | dyn_event_remove(&tk->devent); |
ca89bc07 | 531 | trace_probe_unlink(&tk->tp); |
02ca1521 MH |
532 | |
533 | return 0; | |
413d37d1 MH |
534 | } |
535 | ||
fe60b0ce MH |
536 | static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig, |
537 | struct trace_kprobe *comp) | |
538 | { | |
539 | struct trace_probe_event *tpe = orig->tp.event; | |
540 | struct trace_probe *pos; | |
541 | int i; | |
542 | ||
543 | list_for_each_entry(pos, &tpe->probes, list) { | |
544 | orig = container_of(pos, struct trace_kprobe, tp); | |
545 | if (strcmp(trace_kprobe_symbol(orig), | |
546 | trace_kprobe_symbol(comp)) || | |
547 | trace_kprobe_offset(orig) != trace_kprobe_offset(comp)) | |
548 | continue; | |
549 | ||
550 | /* | |
551 | * trace_probe_compare_arg_type() ensured that nr_args and | |
552 | * each argument name and type are same. Let's compare comm. | |
553 | */ | |
554 | for (i = 0; i < orig->tp.nr_args; i++) { | |
555 | if (strcmp(orig->tp.args[i].comm, | |
556 | comp->tp.args[i].comm)) | |
f8d7ab2b | 557 | break; |
fe60b0ce MH |
558 | } |
559 | ||
f8d7ab2b SD |
560 | if (i == orig->tp.nr_args) |
561 | return true; | |
fe60b0ce MH |
562 | } |
563 | ||
564 | return false; | |
565 | } | |
566 | ||
ca89bc07 MH |
567 | static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to) |
568 | { | |
569 | int ret; | |
570 | ||
fe60b0ce MH |
571 | ret = trace_probe_compare_arg_type(&tk->tp, &to->tp); |
572 | if (ret) { | |
573 | /* Note that argument starts index = 2 */ | |
574 | trace_probe_log_set_index(ret + 1); | |
575 | trace_probe_log_err(0, DIFF_ARG_TYPE); | |
576 | return -EEXIST; | |
577 | } | |
578 | if (trace_kprobe_has_same_kprobe(to, tk)) { | |
579 | trace_probe_log_set_index(0); | |
580 | trace_probe_log_err(0, SAME_PROBE); | |
581 | return -EEXIST; | |
582 | } | |
583 | ||
ca89bc07 MH |
584 | /* Append to existing event */ |
585 | ret = trace_probe_append(&tk->tp, &to->tp); | |
586 | if (ret) | |
587 | return ret; | |
588 | ||
589 | /* Register k*probe */ | |
590 | ret = __register_trace_kprobe(tk); | |
591 | if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { | |
592 | pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); | |
593 | ret = 0; | |
594 | } | |
595 | ||
596 | if (ret) | |
597 | trace_probe_unlink(&tk->tp); | |
598 | else | |
599 | dyn_event_add(&tk->devent); | |
600 | ||
601 | return ret; | |
602 | } | |
603 | ||
413d37d1 | 604 | /* Register a trace_probe and probe_event */ |
c31ffb3f | 605 | static int register_trace_kprobe(struct trace_kprobe *tk) |
413d37d1 | 606 | { |
c31ffb3f | 607 | struct trace_kprobe *old_tk; |
413d37d1 MH |
608 | int ret; |
609 | ||
6212dd29 | 610 | mutex_lock(&event_mutex); |
413d37d1 | 611 | |
b55ce203 MH |
612 | old_tk = find_trace_kprobe(trace_probe_name(&tk->tp), |
613 | trace_probe_group_name(&tk->tp)); | |
c31ffb3f | 614 | if (old_tk) { |
ca89bc07 MH |
615 | if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) { |
616 | trace_probe_log_set_index(0); | |
617 | trace_probe_log_err(0, DIFF_PROBE_TYPE); | |
618 | ret = -EEXIST; | |
619 | } else { | |
fe60b0ce | 620 | ret = append_trace_kprobe(tk, old_tk); |
ca89bc07 MH |
621 | } |
622 | goto end; | |
2d5e067e | 623 | } |
61424318 MH |
624 | |
625 | /* Register new event */ | |
c31ffb3f | 626 | ret = register_kprobe_event(tk); |
2d5e067e | 627 | if (ret) { |
a395d6a7 | 628 | pr_warn("Failed to register probe event(%d)\n", ret); |
2d5e067e MH |
629 | goto end; |
630 | } | |
631 | ||
61424318 | 632 | /* Register k*probe */ |
c31ffb3f | 633 | ret = __register_trace_kprobe(tk); |
59158ec4 MH |
634 | if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) { |
635 | pr_warn("This probe might be able to register after target module is loaded. Continue.\n"); | |
636 | ret = 0; | |
637 | } | |
638 | ||
61424318 | 639 | if (ret < 0) |
c31ffb3f | 640 | unregister_kprobe_event(tk); |
61424318 | 641 | else |
6212dd29 | 642 | dyn_event_add(&tk->devent); |
61424318 | 643 | |
413d37d1 | 644 | end: |
6212dd29 | 645 | mutex_unlock(&event_mutex); |
413d37d1 MH |
646 | return ret; |
647 | } | |
648 | ||
61424318 | 649 | /* Module notifier call back, checking event on the module */ |
c31ffb3f | 650 | static int trace_kprobe_module_callback(struct notifier_block *nb, |
61424318 MH |
651 | unsigned long val, void *data) |
652 | { | |
653 | struct module *mod = data; | |
6212dd29 | 654 | struct dyn_event *pos; |
c31ffb3f | 655 | struct trace_kprobe *tk; |
61424318 MH |
656 | int ret; |
657 | ||
658 | if (val != MODULE_STATE_COMING) | |
659 | return NOTIFY_DONE; | |
660 | ||
661 | /* Update probes on coming module */ | |
6212dd29 MH |
662 | mutex_lock(&event_mutex); |
663 | for_each_trace_kprobe(tk, pos) { | |
c31ffb3f | 664 | if (trace_kprobe_within_module(tk, mod)) { |
02ca1521 | 665 | /* Don't need to check busy - this should have gone. */ |
c31ffb3f NK |
666 | __unregister_trace_kprobe(tk); |
667 | ret = __register_trace_kprobe(tk); | |
61424318 | 668 | if (ret) |
a395d6a7 | 669 | pr_warn("Failed to re-register probe %s on %s: %d\n", |
b55ce203 | 670 | trace_probe_name(&tk->tp), |
a395d6a7 | 671 | mod->name, ret); |
61424318 MH |
672 | } |
673 | } | |
6212dd29 | 674 | mutex_unlock(&event_mutex); |
61424318 MH |
675 | |
676 | return NOTIFY_DONE; | |
677 | } | |
678 | ||
c31ffb3f NK |
679 | static struct notifier_block trace_kprobe_module_nb = { |
680 | .notifier_call = trace_kprobe_module_callback, | |
61424318 MH |
681 | .priority = 1 /* Invoked after kprobe module callback */ |
682 | }; | |
683 | ||
fca18a47 NR |
684 | /* Convert certain expected symbols into '_' when generating event names */ |
685 | static inline void sanitize_event_name(char *name) | |
686 | { | |
687 | while (*name++ != '\0') | |
688 | if (*name == ':' || *name == '.') | |
689 | *name = '_'; | |
690 | } | |
691 | ||
6212dd29 | 692 | static int trace_kprobe_create(int argc, const char *argv[]) |
413d37d1 MH |
693 | { |
694 | /* | |
695 | * Argument syntax: | |
696ced4f AC |
696 | * - Add kprobe: |
697 | * p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS] | |
698 | * - Add kretprobe: | |
699 | * r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS] | |
413d37d1 | 700 | * Fetch args: |
2e06ff63 MH |
701 | * $retval : fetch return value |
702 | * $stack : fetch stack address | |
703 | * $stackN : fetch Nth of stack (N:0-) | |
35abb67d | 704 | * $comm : fetch current task comm |
413d37d1 MH |
705 | * @ADDR : fetch memory at ADDR (ADDR should be in kernel) |
706 | * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) | |
707 | * %REG : fetch register REG | |
93ccae7a | 708 | * Dereferencing memory fetch: |
413d37d1 | 709 | * +|-offs(ARG) : fetch memory at ARG +|- offs address. |
eca0d916 MH |
710 | * Alias name of args: |
711 | * NAME=FETCHARG : set NAME as alias of FETCHARG. | |
93ccae7a MH |
712 | * Type of args: |
713 | * FETCHARG:TYPE : use TYPE instead of unsigned long. | |
413d37d1 | 714 | */ |
ab105a4f | 715 | struct trace_kprobe *tk = NULL; |
6212dd29 MH |
716 | int i, len, ret = 0; |
717 | bool is_return = false; | |
718 | char *symbol = NULL, *tmp = NULL; | |
719 | const char *event = NULL, *group = KPROBE_EVENT_SYSTEM; | |
696ced4f | 720 | int maxactive = 0; |
c5d343b6 | 721 | long offset = 0; |
413d37d1 | 722 | void *addr = NULL; |
4a846b44 | 723 | char buf[MAX_EVENT_NAME_LEN]; |
a1303af5 | 724 | unsigned int flags = TPARG_FL_KERNEL; |
413d37d1 | 725 | |
8b05a3a7 AR |
726 | switch (argv[0][0]) { |
727 | case 'r': | |
3a6b7666 | 728 | is_return = true; |
a1303af5 | 729 | flags |= TPARG_FL_RETURN; |
8b05a3a7 AR |
730 | break; |
731 | case 'p': | |
732 | break; | |
733 | default: | |
734 | return -ECANCELED; | |
735 | } | |
736 | if (argc < 2) | |
6212dd29 | 737 | return -ECANCELED; |
413d37d1 | 738 | |
ab105a4f MH |
739 | trace_probe_log_init("trace_kprobe", argc, argv); |
740 | ||
696ced4f | 741 | event = strchr(&argv[0][1], ':'); |
6212dd29 | 742 | if (event) |
696ced4f | 743 | event++; |
6212dd29 | 744 | |
287c038c MH |
745 | if (isdigit(argv[0][1])) { |
746 | if (!is_return) { | |
ab105a4f MH |
747 | trace_probe_log_err(1, MAXACT_NO_KPROBE); |
748 | goto parse_error; | |
287c038c | 749 | } |
6212dd29 MH |
750 | if (event) |
751 | len = event - &argv[0][1] - 1; | |
752 | else | |
753 | len = strlen(&argv[0][1]); | |
ab105a4f MH |
754 | if (len > MAX_EVENT_NAME_LEN - 1) { |
755 | trace_probe_log_err(1, BAD_MAXACT); | |
756 | goto parse_error; | |
757 | } | |
6212dd29 MH |
758 | memcpy(buf, &argv[0][1], len); |
759 | buf[len] = '\0'; | |
760 | ret = kstrtouint(buf, 0, &maxactive); | |
287c038c | 761 | if (ret || !maxactive) { |
ab105a4f MH |
762 | trace_probe_log_err(1, BAD_MAXACT); |
763 | goto parse_error; | |
696ced4f AC |
764 | } |
765 | /* kretprobes instances are iterated over via a list. The | |
766 | * maximum should stay reasonable. | |
767 | */ | |
768 | if (maxactive > KRETPROBE_MAXACTIVE_MAX) { | |
ab105a4f MH |
769 | trace_probe_log_err(1, MAXACT_TOO_BIG); |
770 | goto parse_error; | |
696ced4f AC |
771 | } |
772 | } | |
773 | ||
9e52b325 SD |
774 | /* try to parse an address. if that fails, try to read the |
775 | * input as a symbol. */ | |
776 | if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) { | |
ab105a4f | 777 | trace_probe_log_set_index(1); |
6212dd29 | 778 | /* Check whether uprobe event specified */ |
ab105a4f MH |
779 | if (strchr(argv[1], '/') && strchr(argv[1], ':')) { |
780 | ret = -ECANCELED; | |
781 | goto error; | |
782 | } | |
413d37d1 | 783 | /* a symbol specified */ |
6212dd29 MH |
784 | symbol = kstrdup(argv[1], GFP_KERNEL); |
785 | if (!symbol) | |
786 | return -ENOMEM; | |
413d37d1 | 787 | /* TODO: support .init module functions */ |
8ab83f56 | 788 | ret = traceprobe_split_symbol_offset(symbol, &offset); |
c5d343b6 | 789 | if (ret || offset < 0 || offset > UINT_MAX) { |
ab105a4f MH |
790 | trace_probe_log_err(0, BAD_PROBE_ADDR); |
791 | goto parse_error; | |
e63cc239 | 792 | } |
a1303af5 MH |
793 | if (kprobe_on_func_entry(NULL, symbol, offset)) |
794 | flags |= TPARG_FL_FENTRY; | |
795 | if (offset && is_return && !(flags & TPARG_FL_FENTRY)) { | |
ab105a4f MH |
796 | trace_probe_log_err(0, BAD_RETPROBE); |
797 | goto parse_error; | |
e63cc239 | 798 | } |
413d37d1 MH |
799 | } |
800 | ||
ab105a4f | 801 | trace_probe_log_set_index(0); |
6212dd29 | 802 | if (event) { |
ab105a4f MH |
803 | ret = traceprobe_parse_event_name(&event, &group, buf, |
804 | event - argv[0]); | |
6212dd29 | 805 | if (ret) |
ab105a4f | 806 | goto parse_error; |
6212dd29 | 807 | } else { |
4263565d | 808 | /* Make a new event name */ |
4263565d | 809 | if (symbol) |
6f3cf440 | 810 | snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld", |
4263565d MH |
811 | is_return ? 'r' : 'p', symbol, offset); |
812 | else | |
6f3cf440 | 813 | snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p", |
4263565d | 814 | is_return ? 'r' : 'p', addr); |
fca18a47 | 815 | sanitize_event_name(buf); |
4a846b44 MH |
816 | event = buf; |
817 | } | |
6212dd29 MH |
818 | |
819 | /* setup a probe */ | |
696ced4f | 820 | tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive, |
ab105a4f | 821 | argc - 2, is_return); |
c31ffb3f | 822 | if (IS_ERR(tk)) { |
6212dd29 | 823 | ret = PTR_ERR(tk); |
ab105a4f | 824 | /* This must return -ENOMEM, else there is a bug */ |
a039480e | 825 | WARN_ON_ONCE(ret != -ENOMEM); |
ab105a4f | 826 | goto out; /* We know tk is not allocated */ |
e63cc239 | 827 | } |
ab105a4f | 828 | argc -= 2; argv += 2; |
413d37d1 | 829 | |
413d37d1 | 830 | /* parse arguments */ |
a82378d8 | 831 | for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { |
6212dd29 MH |
832 | tmp = kstrdup(argv[i], GFP_KERNEL); |
833 | if (!tmp) { | |
ba8665d7 | 834 | ret = -ENOMEM; |
413d37d1 MH |
835 | goto error; |
836 | } | |
da34634f | 837 | |
ab105a4f | 838 | trace_probe_log_set_index(i + 2); |
6212dd29 MH |
839 | ret = traceprobe_parse_probe_arg(&tk->tp, i, tmp, flags); |
840 | kfree(tmp); | |
d00bbea9 | 841 | if (ret) |
ab105a4f | 842 | goto error; /* This can be -ENOMEM */ |
413d37d1 | 843 | } |
413d37d1 | 844 | |
f730e0f2 MH |
845 | ret = traceprobe_set_print_fmt(&tk->tp, is_return); |
846 | if (ret < 0) | |
847 | goto error; | |
848 | ||
c31ffb3f | 849 | ret = register_trace_kprobe(tk); |
ab105a4f MH |
850 | if (ret) { |
851 | trace_probe_log_set_index(1); | |
852 | if (ret == -EILSEQ) | |
853 | trace_probe_log_err(0, BAD_INSN_BNDRY); | |
854 | else if (ret == -ENOENT) | |
855 | trace_probe_log_err(0, BAD_PROBE_ADDR); | |
ca89bc07 | 856 | else if (ret != -ENOMEM && ret != -EEXIST) |
ab105a4f | 857 | trace_probe_log_err(0, FAIL_REG_PROBE); |
413d37d1 | 858 | goto error; |
ab105a4f MH |
859 | } |
860 | ||
6212dd29 | 861 | out: |
ab105a4f | 862 | trace_probe_log_clear(); |
6212dd29 MH |
863 | kfree(symbol); |
864 | return ret; | |
413d37d1 | 865 | |
ab105a4f MH |
866 | parse_error: |
867 | ret = -EINVAL; | |
413d37d1 | 868 | error: |
c31ffb3f | 869 | free_trace_kprobe(tk); |
6212dd29 | 870 | goto out; |
413d37d1 MH |
871 | } |
872 | ||
6212dd29 | 873 | static int create_or_delete_trace_kprobe(int argc, char **argv) |
413d37d1 | 874 | { |
6212dd29 | 875 | int ret; |
02ca1521 | 876 | |
6212dd29 MH |
877 | if (argv[0][0] == '-') |
878 | return dyn_event_release(argc, argv, &trace_kprobe_ops); | |
413d37d1 | 879 | |
6212dd29 MH |
880 | ret = trace_kprobe_create(argc, (const char **)argv); |
881 | return ret == -ECANCELED ? -EINVAL : ret; | |
413d37d1 MH |
882 | } |
883 | ||
6212dd29 | 884 | static int trace_kprobe_release(struct dyn_event *ev) |
413d37d1 | 885 | { |
6212dd29 MH |
886 | struct trace_kprobe *tk = to_trace_kprobe(ev); |
887 | int ret = unregister_trace_kprobe(tk); | |
413d37d1 | 888 | |
6212dd29 MH |
889 | if (!ret) |
890 | free_trace_kprobe(tk); | |
891 | return ret; | |
413d37d1 MH |
892 | } |
893 | ||
6212dd29 | 894 | static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev) |
413d37d1 | 895 | { |
6212dd29 | 896 | struct trace_kprobe *tk = to_trace_kprobe(ev); |
93ccae7a | 897 | int i; |
413d37d1 | 898 | |
fa6f0cc7 | 899 | seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p'); |
b55ce203 MH |
900 | seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp), |
901 | trace_probe_name(&tk->tp)); | |
413d37d1 | 902 | |
c31ffb3f NK |
903 | if (!tk->symbol) |
904 | seq_printf(m, " 0x%p", tk->rp.kp.addr); | |
905 | else if (tk->rp.kp.offset) | |
906 | seq_printf(m, " %s+%u", trace_kprobe_symbol(tk), | |
907 | tk->rp.kp.offset); | |
413d37d1 | 908 | else |
c31ffb3f | 909 | seq_printf(m, " %s", trace_kprobe_symbol(tk)); |
413d37d1 | 910 | |
c31ffb3f NK |
911 | for (i = 0; i < tk->tp.nr_args; i++) |
912 | seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm); | |
fa6f0cc7 | 913 | seq_putc(m, '\n'); |
93ccae7a | 914 | |
413d37d1 MH |
915 | return 0; |
916 | } | |
917 | ||
6212dd29 MH |
918 | static int probes_seq_show(struct seq_file *m, void *v) |
919 | { | |
920 | struct dyn_event *ev = v; | |
921 | ||
922 | if (!is_trace_kprobe(ev)) | |
923 | return 0; | |
924 | ||
925 | return trace_kprobe_show(m, ev); | |
926 | } | |
927 | ||
413d37d1 | 928 | static const struct seq_operations probes_seq_op = { |
6212dd29 MH |
929 | .start = dyn_event_seq_start, |
930 | .next = dyn_event_seq_next, | |
931 | .stop = dyn_event_seq_stop, | |
413d37d1 MH |
932 | .show = probes_seq_show |
933 | }; | |
934 | ||
935 | static int probes_open(struct inode *inode, struct file *file) | |
936 | { | |
02ca1521 MH |
937 | int ret; |
938 | ||
939 | if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { | |
6212dd29 | 940 | ret = dyn_events_release_all(&trace_kprobe_ops); |
02ca1521 MH |
941 | if (ret < 0) |
942 | return ret; | |
943 | } | |
413d37d1 MH |
944 | |
945 | return seq_open(file, &probes_seq_op); | |
946 | } | |
947 | ||
413d37d1 MH |
948 | static ssize_t probes_write(struct file *file, const char __user *buffer, |
949 | size_t count, loff_t *ppos) | |
950 | { | |
7e465baa | 951 | return trace_parse_run_command(file, buffer, count, ppos, |
6212dd29 | 952 | create_or_delete_trace_kprobe); |
413d37d1 MH |
953 | } |
954 | ||
955 | static const struct file_operations kprobe_events_ops = { | |
956 | .owner = THIS_MODULE, | |
957 | .open = probes_open, | |
958 | .read = seq_read, | |
959 | .llseek = seq_lseek, | |
960 | .release = seq_release, | |
961 | .write = probes_write, | |
962 | }; | |
963 | ||
cd7e7bd5 MH |
964 | /* Probes profiling interfaces */ |
965 | static int probes_profile_seq_show(struct seq_file *m, void *v) | |
966 | { | |
6212dd29 MH |
967 | struct dyn_event *ev = v; |
968 | struct trace_kprobe *tk; | |
cd7e7bd5 | 969 | |
6212dd29 MH |
970 | if (!is_trace_kprobe(ev)) |
971 | return 0; | |
cd7e7bd5 | 972 | |
6212dd29 | 973 | tk = to_trace_kprobe(ev); |
de7b2973 | 974 | seq_printf(m, " %-44s %15lu %15lu\n", |
b55ce203 | 975 | trace_probe_name(&tk->tp), |
f18f97ac | 976 | trace_kprobe_nhit(tk), |
c31ffb3f | 977 | tk->rp.kp.nmissed); |
cd7e7bd5 MH |
978 | |
979 | return 0; | |
980 | } | |
981 | ||
982 | static const struct seq_operations profile_seq_op = { | |
6212dd29 MH |
983 | .start = dyn_event_seq_start, |
984 | .next = dyn_event_seq_next, | |
985 | .stop = dyn_event_seq_stop, | |
cd7e7bd5 MH |
986 | .show = probes_profile_seq_show |
987 | }; | |
988 | ||
989 | static int profile_open(struct inode *inode, struct file *file) | |
990 | { | |
991 | return seq_open(file, &profile_seq_op); | |
992 | } | |
993 | ||
994 | static const struct file_operations kprobe_profile_ops = { | |
995 | .owner = THIS_MODULE, | |
996 | .open = profile_open, | |
997 | .read = seq_read, | |
998 | .llseek = seq_lseek, | |
999 | .release = seq_release, | |
1000 | }; | |
1001 | ||
53305928 MH |
1002 | /* Kprobe specific fetch functions */ |
1003 | ||
1004 | /* Return the length of string -- including null terminal byte */ | |
9178412d MH |
1005 | static nokprobe_inline int |
1006 | fetch_store_strlen(unsigned long addr) | |
53305928 | 1007 | { |
53305928 MH |
1008 | int ret, len = 0; |
1009 | u8 c; | |
1010 | ||
53305928 | 1011 | do { |
49ef5f45 | 1012 | ret = probe_kernel_read(&c, (u8 *)addr + len, 1); |
53305928 MH |
1013 | len++; |
1014 | } while (c && ret == 0 && len < MAX_STRING_SIZE); | |
1015 | ||
9178412d | 1016 | return (ret < 0) ? ret : len; |
53305928 MH |
1017 | } |
1018 | ||
88903c46 MH |
1019 | /* Return the length of string -- including null terminal byte */ |
1020 | static nokprobe_inline int | |
1021 | fetch_store_strlen_user(unsigned long addr) | |
1022 | { | |
1023 | const void __user *uaddr = (__force const void __user *)addr; | |
1024 | ||
1025 | return strnlen_unsafe_user(uaddr, MAX_STRING_SIZE); | |
1026 | } | |
1027 | ||
53305928 MH |
1028 | /* |
1029 | * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max | |
1030 | * length and relative data location. | |
1031 | */ | |
9178412d MH |
1032 | static nokprobe_inline int |
1033 | fetch_store_string(unsigned long addr, void *dest, void *base) | |
53305928 | 1034 | { |
9178412d | 1035 | int maxlen = get_loc_len(*(u32 *)dest); |
88903c46 | 1036 | void *__dest; |
53305928 MH |
1037 | long ret; |
1038 | ||
9178412d MH |
1039 | if (unlikely(!maxlen)) |
1040 | return -ENOMEM; | |
88903c46 MH |
1041 | |
1042 | __dest = get_loc_data(dest, base); | |
1043 | ||
53305928 MH |
1044 | /* |
1045 | * Try to get string again, since the string can be changed while | |
1046 | * probing. | |
1047 | */ | |
88903c46 MH |
1048 | ret = strncpy_from_unsafe(__dest, (void *)addr, maxlen); |
1049 | if (ret >= 0) | |
1050 | *(u32 *)dest = make_data_loc(ret, __dest - base); | |
1051 | ||
1052 | return ret; | |
1053 | } | |
53305928 | 1054 | |
88903c46 MH |
1055 | /* |
1056 | * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf | |
1057 | * with max length and relative data location. | |
1058 | */ | |
1059 | static nokprobe_inline int | |
1060 | fetch_store_string_user(unsigned long addr, void *dest, void *base) | |
1061 | { | |
1062 | const void __user *uaddr = (__force const void __user *)addr; | |
1063 | int maxlen = get_loc_len(*(u32 *)dest); | |
1064 | void *__dest; | |
1065 | long ret; | |
1066 | ||
1067 | if (unlikely(!maxlen)) | |
1068 | return -ENOMEM; | |
1069 | ||
1070 | __dest = get_loc_data(dest, base); | |
1071 | ||
1072 | ret = strncpy_from_unsafe_user(__dest, uaddr, maxlen); | |
9178412d | 1073 | if (ret >= 0) |
88903c46 MH |
1074 | *(u32 *)dest = make_data_loc(ret, __dest - base); |
1075 | ||
9178412d | 1076 | return ret; |
53305928 MH |
1077 | } |
1078 | ||
9b960a38 MH |
1079 | static nokprobe_inline int |
1080 | probe_mem_read(void *dest, void *src, size_t size) | |
1081 | { | |
1082 | return probe_kernel_read(dest, src, size); | |
1083 | } | |
1084 | ||
e65f7ae7 MH |
1085 | static nokprobe_inline int |
1086 | probe_mem_read_user(void *dest, void *src, size_t size) | |
1087 | { | |
539b75b2 MH |
1088 | const void __user *uaddr = (__force const void __user *)src; |
1089 | ||
1090 | return probe_user_read(dest, uaddr, size); | |
e65f7ae7 MH |
1091 | } |
1092 | ||
53305928 MH |
1093 | /* Note that we don't verify it, since the code does not come from user space */ |
1094 | static int | |
1095 | process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest, | |
9178412d | 1096 | void *base) |
53305928 MH |
1097 | { |
1098 | unsigned long val; | |
53305928 | 1099 | |
a6682814 | 1100 | retry: |
53305928 MH |
1101 | /* 1st stage: get value from context */ |
1102 | switch (code->op) { | |
1103 | case FETCH_OP_REG: | |
1104 | val = regs_get_register(regs, code->param); | |
1105 | break; | |
1106 | case FETCH_OP_STACK: | |
1107 | val = regs_get_kernel_stack_nth(regs, code->param); | |
1108 | break; | |
1109 | case FETCH_OP_STACKP: | |
1110 | val = kernel_stack_pointer(regs); | |
1111 | break; | |
1112 | case FETCH_OP_RETVAL: | |
1113 | val = regs_return_value(regs); | |
1114 | break; | |
1115 | case FETCH_OP_IMM: | |
1116 | val = code->immediate; | |
1117 | break; | |
1118 | case FETCH_OP_COMM: | |
1119 | val = (unsigned long)current->comm; | |
1120 | break; | |
a42e3c4d MH |
1121 | case FETCH_OP_DATA: |
1122 | val = (unsigned long)code->data; | |
1123 | break; | |
a1303af5 MH |
1124 | #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API |
1125 | case FETCH_OP_ARG: | |
1126 | val = regs_get_kernel_argument(regs, code->param); | |
1127 | break; | |
1128 | #endif | |
a6682814 MH |
1129 | case FETCH_NOP_SYMBOL: /* Ignore a place holder */ |
1130 | code++; | |
1131 | goto retry; | |
53305928 MH |
1132 | default: |
1133 | return -EILSEQ; | |
1134 | } | |
1135 | code++; | |
1136 | ||
9b960a38 | 1137 | return process_fetch_insn_bottom(code, val, dest, base); |
53305928 MH |
1138 | } |
1139 | NOKPROBE_SYMBOL(process_fetch_insn) | |
1140 | ||
413d37d1 | 1141 | /* Kprobe handler */ |
3da0f180 | 1142 | static nokprobe_inline void |
c31ffb3f | 1143 | __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, |
7f1d2f82 | 1144 | struct trace_event_file *trace_file) |
413d37d1 | 1145 | { |
93ccae7a | 1146 | struct kprobe_trace_entry_head *entry; |
413d37d1 | 1147 | struct ring_buffer_event *event; |
8f8ffe24 | 1148 | struct ring_buffer *buffer; |
e09c8614 | 1149 | int size, dsize, pc; |
413d37d1 | 1150 | unsigned long irq_flags; |
e3dc9f89 | 1151 | struct trace_event_call *call = trace_probe_event_call(&tk->tp); |
413d37d1 | 1152 | |
7f1d2f82 | 1153 | WARN_ON(call != trace_file->event_call); |
41a7dd42 | 1154 | |
09a5059a | 1155 | if (trace_trigger_soft_disabled(trace_file)) |
13a1e4ae | 1156 | return; |
b8820084 | 1157 | |
413d37d1 MH |
1158 | local_save_flags(irq_flags); |
1159 | pc = preempt_count(); | |
1160 | ||
c31ffb3f NK |
1161 | dsize = __get_data_size(&tk->tp, regs); |
1162 | size = sizeof(*entry) + tk->tp.size + dsize; | |
413d37d1 | 1163 | |
7f1d2f82 | 1164 | event = trace_event_buffer_lock_reserve(&buffer, trace_file, |
41a7dd42 MH |
1165 | call->event.type, |
1166 | size, irq_flags, pc); | |
413d37d1 | 1167 | if (!event) |
1e12a4a7 | 1168 | return; |
413d37d1 MH |
1169 | |
1170 | entry = ring_buffer_event_data(event); | |
c31ffb3f | 1171 | entry->ip = (unsigned long)tk->rp.kp.addr; |
9178412d | 1172 | store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize); |
413d37d1 | 1173 | |
7f1d2f82 | 1174 | event_trigger_unlock_commit_regs(trace_file, buffer, event, |
13a1e4ae | 1175 | entry, irq_flags, pc, regs); |
413d37d1 MH |
1176 | } |
1177 | ||
3da0f180 | 1178 | static void |
c31ffb3f | 1179 | kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs) |
41a7dd42 | 1180 | { |
b04d52e3 | 1181 | struct event_file_link *link; |
41a7dd42 | 1182 | |
b5f935ee | 1183 | trace_probe_for_each_link_rcu(link, &tk->tp) |
c31ffb3f | 1184 | __kprobe_trace_func(tk, regs, link->file); |
41a7dd42 | 1185 | } |
3da0f180 | 1186 | NOKPROBE_SYMBOL(kprobe_trace_func); |
41a7dd42 | 1187 | |
413d37d1 | 1188 | /* Kretprobe handler */ |
3da0f180 | 1189 | static nokprobe_inline void |
c31ffb3f | 1190 | __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, |
41a7dd42 | 1191 | struct pt_regs *regs, |
7f1d2f82 | 1192 | struct trace_event_file *trace_file) |
413d37d1 | 1193 | { |
93ccae7a | 1194 | struct kretprobe_trace_entry_head *entry; |
413d37d1 | 1195 | struct ring_buffer_event *event; |
8f8ffe24 | 1196 | struct ring_buffer *buffer; |
e09c8614 | 1197 | int size, pc, dsize; |
413d37d1 | 1198 | unsigned long irq_flags; |
e3dc9f89 | 1199 | struct trace_event_call *call = trace_probe_event_call(&tk->tp); |
413d37d1 | 1200 | |
7f1d2f82 | 1201 | WARN_ON(call != trace_file->event_call); |
41a7dd42 | 1202 | |
09a5059a | 1203 | if (trace_trigger_soft_disabled(trace_file)) |
13a1e4ae | 1204 | return; |
b8820084 | 1205 | |
413d37d1 MH |
1206 | local_save_flags(irq_flags); |
1207 | pc = preempt_count(); | |
1208 | ||
c31ffb3f NK |
1209 | dsize = __get_data_size(&tk->tp, regs); |
1210 | size = sizeof(*entry) + tk->tp.size + dsize; | |
413d37d1 | 1211 | |
7f1d2f82 | 1212 | event = trace_event_buffer_lock_reserve(&buffer, trace_file, |
41a7dd42 MH |
1213 | call->event.type, |
1214 | size, irq_flags, pc); | |
413d37d1 | 1215 | if (!event) |
1e12a4a7 | 1216 | return; |
413d37d1 MH |
1217 | |
1218 | entry = ring_buffer_event_data(event); | |
c31ffb3f | 1219 | entry->func = (unsigned long)tk->rp.kp.addr; |
413d37d1 | 1220 | entry->ret_ip = (unsigned long)ri->ret_addr; |
9178412d | 1221 | store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize); |
413d37d1 | 1222 | |
7f1d2f82 | 1223 | event_trigger_unlock_commit_regs(trace_file, buffer, event, |
13a1e4ae | 1224 | entry, irq_flags, pc, regs); |
413d37d1 MH |
1225 | } |
1226 | ||
3da0f180 | 1227 | static void |
c31ffb3f | 1228 | kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, |
41a7dd42 MH |
1229 | struct pt_regs *regs) |
1230 | { | |
b04d52e3 | 1231 | struct event_file_link *link; |
41a7dd42 | 1232 | |
b5f935ee | 1233 | trace_probe_for_each_link_rcu(link, &tk->tp) |
c31ffb3f | 1234 | __kretprobe_trace_func(tk, ri, regs, link->file); |
41a7dd42 | 1235 | } |
3da0f180 | 1236 | NOKPROBE_SYMBOL(kretprobe_trace_func); |
41a7dd42 | 1237 | |
413d37d1 | 1238 | /* Event entry printers */ |
b62fdd97 | 1239 | static enum print_line_t |
a9a57763 SR |
1240 | print_kprobe_event(struct trace_iterator *iter, int flags, |
1241 | struct trace_event *event) | |
413d37d1 | 1242 | { |
93ccae7a | 1243 | struct kprobe_trace_entry_head *field; |
413d37d1 | 1244 | struct trace_seq *s = &iter->seq; |
eca0d916 | 1245 | struct trace_probe *tp; |
413d37d1 | 1246 | |
93ccae7a | 1247 | field = (struct kprobe_trace_entry_head *)iter->ent; |
60d53e2c MH |
1248 | tp = trace_probe_primary_from_call( |
1249 | container_of(event, struct trace_event_call, event)); | |
1250 | if (WARN_ON_ONCE(!tp)) | |
1251 | goto out; | |
413d37d1 | 1252 | |
b55ce203 | 1253 | trace_seq_printf(s, "%s: (", trace_probe_name(tp)); |
6e9f23d1 | 1254 | |
413d37d1 | 1255 | if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) |
85224da0 | 1256 | goto out; |
413d37d1 | 1257 | |
85224da0 | 1258 | trace_seq_putc(s, ')'); |
413d37d1 | 1259 | |
56de7630 MH |
1260 | if (print_probe_args(s, tp->args, tp->nr_args, |
1261 | (u8 *)&field[1], field) < 0) | |
1262 | goto out; | |
413d37d1 | 1263 | |
85224da0 SRRH |
1264 | trace_seq_putc(s, '\n'); |
1265 | out: | |
1266 | return trace_handle_return(s); | |
413d37d1 MH |
1267 | } |
1268 | ||
b62fdd97 | 1269 | static enum print_line_t |
a9a57763 SR |
1270 | print_kretprobe_event(struct trace_iterator *iter, int flags, |
1271 | struct trace_event *event) | |
413d37d1 | 1272 | { |
93ccae7a | 1273 | struct kretprobe_trace_entry_head *field; |
413d37d1 | 1274 | struct trace_seq *s = &iter->seq; |
eca0d916 | 1275 | struct trace_probe *tp; |
413d37d1 | 1276 | |
93ccae7a | 1277 | field = (struct kretprobe_trace_entry_head *)iter->ent; |
60d53e2c MH |
1278 | tp = trace_probe_primary_from_call( |
1279 | container_of(event, struct trace_event_call, event)); | |
1280 | if (WARN_ON_ONCE(!tp)) | |
1281 | goto out; | |
413d37d1 | 1282 | |
b55ce203 | 1283 | trace_seq_printf(s, "%s: (", trace_probe_name(tp)); |
6e9f23d1 | 1284 | |
413d37d1 | 1285 | if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) |
85224da0 | 1286 | goto out; |
413d37d1 | 1287 | |
85224da0 | 1288 | trace_seq_puts(s, " <- "); |
413d37d1 MH |
1289 | |
1290 | if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) | |
85224da0 | 1291 | goto out; |
413d37d1 | 1292 | |
85224da0 | 1293 | trace_seq_putc(s, ')'); |
413d37d1 | 1294 | |
56de7630 MH |
1295 | if (print_probe_args(s, tp->args, tp->nr_args, |
1296 | (u8 *)&field[1], field) < 0) | |
1297 | goto out; | |
413d37d1 | 1298 | |
85224da0 | 1299 | trace_seq_putc(s, '\n'); |
413d37d1 | 1300 | |
85224da0 SRRH |
1301 | out: |
1302 | return trace_handle_return(s); | |
413d37d1 MH |
1303 | } |
1304 | ||
413d37d1 | 1305 | |
2425bcb9 | 1306 | static int kprobe_event_define_fields(struct trace_event_call *event_call) |
413d37d1 | 1307 | { |
eeb07b06 | 1308 | int ret; |
93ccae7a | 1309 | struct kprobe_trace_entry_head field; |
60d53e2c MH |
1310 | struct trace_probe *tp; |
1311 | ||
1312 | tp = trace_probe_primary_from_call(event_call); | |
1313 | if (WARN_ON_ONCE(!tp)) | |
1314 | return -ENOENT; | |
413d37d1 | 1315 | |
a703d946 | 1316 | DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); |
c31ffb3f | 1317 | |
60d53e2c | 1318 | return traceprobe_define_arg_fields(event_call, sizeof(field), tp); |
413d37d1 MH |
1319 | } |
1320 | ||
2425bcb9 | 1321 | static int kretprobe_event_define_fields(struct trace_event_call *event_call) |
413d37d1 | 1322 | { |
eeb07b06 | 1323 | int ret; |
93ccae7a | 1324 | struct kretprobe_trace_entry_head field; |
60d53e2c MH |
1325 | struct trace_probe *tp; |
1326 | ||
1327 | tp = trace_probe_primary_from_call(event_call); | |
1328 | if (WARN_ON_ONCE(!tp)) | |
1329 | return -ENOENT; | |
413d37d1 | 1330 | |
a703d946 MH |
1331 | DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); |
1332 | DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); | |
c31ffb3f | 1333 | |
60d53e2c | 1334 | return traceprobe_define_arg_fields(event_call, sizeof(field), tp); |
413d37d1 MH |
1335 | } |
1336 | ||
07b139c8 | 1337 | #ifdef CONFIG_PERF_EVENTS |
e08d1c65 MH |
1338 | |
1339 | /* Kprobe profile handler */ | |
9802d865 | 1340 | static int |
c31ffb3f | 1341 | kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs) |
e08d1c65 | 1342 | { |
e3dc9f89 | 1343 | struct trace_event_call *call = trace_probe_event_call(&tk->tp); |
93ccae7a | 1344 | struct kprobe_trace_entry_head *entry; |
1c024eca | 1345 | struct hlist_head *head; |
e09c8614 | 1346 | int size, __size, dsize; |
4ed7c92d | 1347 | int rctx; |
e08d1c65 | 1348 | |
9802d865 | 1349 | if (bpf_prog_array_valid(call)) { |
66665ad2 | 1350 | unsigned long orig_ip = instruction_pointer(regs); |
9802d865 JB |
1351 | int ret; |
1352 | ||
1353 | ret = trace_call_bpf(call, regs); | |
1354 | ||
1355 | /* | |
1356 | * We need to check and see if we modified the pc of the | |
cce188bd MH |
1357 | * pt_regs, and if so return 1 so that we don't do the |
1358 | * single stepping. | |
9802d865 | 1359 | */ |
cce188bd | 1360 | if (orig_ip != instruction_pointer(regs)) |
9802d865 | 1361 | return 1; |
9802d865 JB |
1362 | if (!ret) |
1363 | return 0; | |
1364 | } | |
2541517c | 1365 | |
288e984e ON |
1366 | head = this_cpu_ptr(call->perf_events); |
1367 | if (hlist_empty(head)) | |
9802d865 | 1368 | return 0; |
288e984e | 1369 | |
c31ffb3f NK |
1370 | dsize = __get_data_size(&tk->tp, regs); |
1371 | __size = sizeof(*entry) + tk->tp.size + dsize; | |
74ebb63e MH |
1372 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); |
1373 | size -= sizeof(u32); | |
ce71b9df | 1374 | |
1e1dcd93 | 1375 | entry = perf_trace_buf_alloc(size, NULL, &rctx); |
430ad5a6 | 1376 | if (!entry) |
9802d865 | 1377 | return 0; |
a1a138d0 | 1378 | |
c31ffb3f | 1379 | entry->ip = (unsigned long)tk->rp.kp.addr; |
e09c8614 | 1380 | memset(&entry[1], 0, dsize); |
9178412d | 1381 | store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize); |
1e1dcd93 | 1382 | perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, |
8fd0fbbe | 1383 | head, NULL); |
9802d865 | 1384 | return 0; |
e08d1c65 | 1385 | } |
3da0f180 | 1386 | NOKPROBE_SYMBOL(kprobe_perf_func); |
e08d1c65 MH |
1387 | |
1388 | /* Kretprobe profile handler */ | |
3da0f180 | 1389 | static void |
c31ffb3f | 1390 | kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, |
2b106aab | 1391 | struct pt_regs *regs) |
e08d1c65 | 1392 | { |
e3dc9f89 | 1393 | struct trace_event_call *call = trace_probe_event_call(&tk->tp); |
93ccae7a | 1394 | struct kretprobe_trace_entry_head *entry; |
1c024eca | 1395 | struct hlist_head *head; |
e09c8614 | 1396 | int size, __size, dsize; |
4ed7c92d | 1397 | int rctx; |
e08d1c65 | 1398 | |
e87c6bc3 | 1399 | if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs)) |
2541517c AS |
1400 | return; |
1401 | ||
288e984e ON |
1402 | head = this_cpu_ptr(call->perf_events); |
1403 | if (hlist_empty(head)) | |
1404 | return; | |
1405 | ||
c31ffb3f NK |
1406 | dsize = __get_data_size(&tk->tp, regs); |
1407 | __size = sizeof(*entry) + tk->tp.size + dsize; | |
74ebb63e MH |
1408 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); |
1409 | size -= sizeof(u32); | |
444a2a3b | 1410 | |
1e1dcd93 | 1411 | entry = perf_trace_buf_alloc(size, NULL, &rctx); |
430ad5a6 | 1412 | if (!entry) |
1e12a4a7 | 1413 | return; |
e08d1c65 | 1414 | |
c31ffb3f | 1415 | entry->func = (unsigned long)tk->rp.kp.addr; |
a1a138d0 | 1416 | entry->ret_ip = (unsigned long)ri->ret_addr; |
9178412d | 1417 | store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize); |
1e1dcd93 | 1418 | perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, |
8fd0fbbe | 1419 | head, NULL); |
e08d1c65 | 1420 | } |
3da0f180 | 1421 | NOKPROBE_SYMBOL(kretprobe_perf_func); |
41bdc4b4 YS |
1422 | |
1423 | int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type, | |
1424 | const char **symbol, u64 *probe_offset, | |
1425 | u64 *probe_addr, bool perf_type_tracepoint) | |
1426 | { | |
1427 | const char *pevent = trace_event_name(event->tp_event); | |
1428 | const char *group = event->tp_event->class->system; | |
1429 | struct trace_kprobe *tk; | |
1430 | ||
1431 | if (perf_type_tracepoint) | |
1432 | tk = find_trace_kprobe(pevent, group); | |
1433 | else | |
1434 | tk = event->tp_event->data; | |
1435 | if (!tk) | |
1436 | return -EINVAL; | |
1437 | ||
1438 | *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE | |
1439 | : BPF_FD_TYPE_KPROBE; | |
1440 | if (tk->symbol) { | |
1441 | *symbol = tk->symbol; | |
1442 | *probe_offset = tk->rp.kp.offset; | |
1443 | *probe_addr = 0; | |
1444 | } else { | |
1445 | *symbol = NULL; | |
1446 | *probe_offset = 0; | |
1447 | *probe_addr = (unsigned long)tk->rp.kp.addr; | |
1448 | } | |
1449 | return 0; | |
1450 | } | |
07b139c8 | 1451 | #endif /* CONFIG_PERF_EVENTS */ |
50d78056 | 1452 | |
3fe3d619 ON |
1453 | /* |
1454 | * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex. | |
1455 | * | |
1456 | * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe | |
1457 | * lockless, but we can't race with this __init function. | |
1458 | */ | |
2425bcb9 | 1459 | static int kprobe_register(struct trace_event_call *event, |
fbc1963d | 1460 | enum trace_reg type, void *data) |
2239291a | 1461 | { |
7f1d2f82 | 1462 | struct trace_event_file *file = data; |
1538f888 | 1463 | |
2239291a SR |
1464 | switch (type) { |
1465 | case TRACE_REG_REGISTER: | |
60d53e2c | 1466 | return enable_trace_kprobe(event, file); |
2239291a | 1467 | case TRACE_REG_UNREGISTER: |
60d53e2c | 1468 | return disable_trace_kprobe(event, file); |
2239291a SR |
1469 | |
1470 | #ifdef CONFIG_PERF_EVENTS | |
1471 | case TRACE_REG_PERF_REGISTER: | |
60d53e2c | 1472 | return enable_trace_kprobe(event, NULL); |
2239291a | 1473 | case TRACE_REG_PERF_UNREGISTER: |
60d53e2c | 1474 | return disable_trace_kprobe(event, NULL); |
ceec0b6f JO |
1475 | case TRACE_REG_PERF_OPEN: |
1476 | case TRACE_REG_PERF_CLOSE: | |
489c75c3 JO |
1477 | case TRACE_REG_PERF_ADD: |
1478 | case TRACE_REG_PERF_DEL: | |
ceec0b6f | 1479 | return 0; |
2239291a SR |
1480 | #endif |
1481 | } | |
1482 | return 0; | |
1483 | } | |
50d78056 | 1484 | |
3da0f180 | 1485 | static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) |
50d78056 | 1486 | { |
c31ffb3f | 1487 | struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp); |
9802d865 | 1488 | int ret = 0; |
e08d1c65 | 1489 | |
a7636d9e | 1490 | raw_cpu_inc(*tk->nhit); |
48182bd2 | 1491 | |
747774d6 | 1492 | if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) |
c31ffb3f | 1493 | kprobe_trace_func(tk, regs); |
07b139c8 | 1494 | #ifdef CONFIG_PERF_EVENTS |
747774d6 | 1495 | if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) |
9802d865 | 1496 | ret = kprobe_perf_func(tk, regs); |
07b139c8 | 1497 | #endif |
9802d865 | 1498 | return ret; |
50d78056 | 1499 | } |
3da0f180 | 1500 | NOKPROBE_SYMBOL(kprobe_dispatcher); |
50d78056 | 1501 | |
3da0f180 MH |
1502 | static int |
1503 | kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) | |
50d78056 | 1504 | { |
c31ffb3f | 1505 | struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp); |
50d78056 | 1506 | |
a7636d9e | 1507 | raw_cpu_inc(*tk->nhit); |
48182bd2 | 1508 | |
747774d6 | 1509 | if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE)) |
c31ffb3f | 1510 | kretprobe_trace_func(tk, ri, regs); |
07b139c8 | 1511 | #ifdef CONFIG_PERF_EVENTS |
747774d6 | 1512 | if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE)) |
c31ffb3f | 1513 | kretprobe_perf_func(tk, ri, regs); |
07b139c8 | 1514 | #endif |
50d78056 MH |
1515 | return 0; /* We don't tweek kernel, so just return 0 */ |
1516 | } | |
3da0f180 | 1517 | NOKPROBE_SYMBOL(kretprobe_dispatcher); |
e08d1c65 | 1518 | |
a9a57763 SR |
1519 | static struct trace_event_functions kretprobe_funcs = { |
1520 | .trace = print_kretprobe_event | |
1521 | }; | |
1522 | ||
1523 | static struct trace_event_functions kprobe_funcs = { | |
1524 | .trace = print_kprobe_event | |
1525 | }; | |
1526 | ||
e3dc9f89 | 1527 | static inline void init_trace_event_call(struct trace_kprobe *tk) |
413d37d1 | 1528 | { |
e3dc9f89 MH |
1529 | struct trace_event_call *call = trace_probe_event_call(&tk->tp); |
1530 | ||
c31ffb3f | 1531 | if (trace_kprobe_is_return(tk)) { |
80decc70 | 1532 | call->event.funcs = &kretprobe_funcs; |
2e33af02 | 1533 | call->class->define_fields = kretprobe_event_define_fields; |
413d37d1 | 1534 | } else { |
80decc70 | 1535 | call->event.funcs = &kprobe_funcs; |
2e33af02 | 1536 | call->class->define_fields = kprobe_event_define_fields; |
413d37d1 | 1537 | } |
e12f03d7 SL |
1538 | |
1539 | call->flags = TRACE_EVENT_FL_KPROBE; | |
1540 | call->class->reg = kprobe_register; | |
e12f03d7 SL |
1541 | } |
1542 | ||
1543 | static int register_kprobe_event(struct trace_kprobe *tk) | |
1544 | { | |
e3dc9f89 | 1545 | init_trace_event_call(tk); |
f730e0f2 | 1546 | |
46e5376d | 1547 | return trace_probe_register_event_call(&tk->tp); |
413d37d1 MH |
1548 | } |
1549 | ||
c31ffb3f | 1550 | static int unregister_kprobe_event(struct trace_kprobe *tk) |
413d37d1 | 1551 | { |
46e5376d | 1552 | return trace_probe_unregister_event_call(&tk->tp); |
413d37d1 MH |
1553 | } |
1554 | ||
e12f03d7 SL |
1555 | #ifdef CONFIG_PERF_EVENTS |
1556 | /* create a trace_kprobe, but don't add it to global lists */ | |
1557 | struct trace_event_call * | |
1558 | create_local_trace_kprobe(char *func, void *addr, unsigned long offs, | |
1559 | bool is_return) | |
1560 | { | |
1561 | struct trace_kprobe *tk; | |
1562 | int ret; | |
1563 | char *event; | |
1564 | ||
1565 | /* | |
6212dd29 | 1566 | * local trace_kprobes are not added to dyn_event, so they are never |
e12f03d7 SL |
1567 | * searched in find_trace_kprobe(). Therefore, there is no concern of |
1568 | * duplicated name here. | |
1569 | */ | |
1570 | event = func ? func : "DUMMY_EVENT"; | |
1571 | ||
1572 | tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func, | |
1573 | offs, 0 /* maxactive */, 0 /* nargs */, | |
1574 | is_return); | |
1575 | ||
1576 | if (IS_ERR(tk)) { | |
1577 | pr_info("Failed to allocate trace_probe.(%d)\n", | |
1578 | (int)PTR_ERR(tk)); | |
1579 | return ERR_CAST(tk); | |
1580 | } | |
1581 | ||
e3dc9f89 | 1582 | init_trace_event_call(tk); |
e12f03d7 | 1583 | |
0a46c854 | 1584 | if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) { |
e12f03d7 SL |
1585 | ret = -ENOMEM; |
1586 | goto error; | |
1587 | } | |
1588 | ||
1589 | ret = __register_trace_kprobe(tk); | |
f730e0f2 | 1590 | if (ret < 0) |
e12f03d7 SL |
1591 | goto error; |
1592 | ||
e3dc9f89 | 1593 | return trace_probe_event_call(&tk->tp); |
e12f03d7 SL |
1594 | error: |
1595 | free_trace_kprobe(tk); | |
1596 | return ERR_PTR(ret); | |
1597 | } | |
1598 | ||
1599 | void destroy_local_trace_kprobe(struct trace_event_call *event_call) | |
1600 | { | |
1601 | struct trace_kprobe *tk; | |
1602 | ||
60d53e2c MH |
1603 | tk = trace_kprobe_primary_from_call(event_call); |
1604 | if (unlikely(!tk)) | |
1605 | return; | |
e12f03d7 SL |
1606 | |
1607 | if (trace_probe_is_enabled(&tk->tp)) { | |
1608 | WARN_ON(1); | |
1609 | return; | |
1610 | } | |
1611 | ||
1612 | __unregister_trace_kprobe(tk); | |
0fc8c358 | 1613 | |
e12f03d7 SL |
1614 | free_trace_kprobe(tk); |
1615 | } | |
1616 | #endif /* CONFIG_PERF_EVENTS */ | |
1617 | ||
970988e1 MH |
1618 | static __init void enable_boot_kprobe_events(void) |
1619 | { | |
1620 | struct trace_array *tr = top_trace_array(); | |
1621 | struct trace_event_file *file; | |
1622 | struct trace_kprobe *tk; | |
1623 | struct dyn_event *pos; | |
1624 | ||
1625 | mutex_lock(&event_mutex); | |
1626 | for_each_trace_kprobe(tk, pos) { | |
1627 | list_for_each_entry(file, &tr->events, list) | |
e3dc9f89 | 1628 | if (file->event_call == trace_probe_event_call(&tk->tp)) |
970988e1 MH |
1629 | trace_event_enable_disable(file, 1, 0); |
1630 | } | |
1631 | mutex_unlock(&event_mutex); | |
1632 | } | |
1633 | ||
1634 | static __init void setup_boot_kprobe_events(void) | |
1635 | { | |
1636 | char *p, *cmd = kprobe_boot_events_buf; | |
1637 | int ret; | |
1638 | ||
1639 | strreplace(kprobe_boot_events_buf, ',', ' '); | |
1640 | ||
1641 | while (cmd && *cmd != '\0') { | |
1642 | p = strchr(cmd, ';'); | |
1643 | if (p) | |
1644 | *p++ = '\0'; | |
1645 | ||
1646 | ret = trace_run_command(cmd, create_or_delete_trace_kprobe); | |
1647 | if (ret) | |
1648 | pr_warn("Failed to add event(%d): %s\n", ret, cmd); | |
b6399cc7 SRV |
1649 | else |
1650 | kprobe_boot_events_enabled = true; | |
970988e1 MH |
1651 | |
1652 | cmd = p; | |
1653 | } | |
1654 | ||
1655 | enable_boot_kprobe_events(); | |
1656 | } | |
1657 | ||
8434dc93 | 1658 | /* Make a tracefs interface for controlling probe points */ |
413d37d1 MH |
1659 | static __init int init_kprobe_trace(void) |
1660 | { | |
1661 | struct dentry *d_tracer; | |
1662 | struct dentry *entry; | |
6212dd29 MH |
1663 | int ret; |
1664 | ||
1665 | ret = dyn_event_register(&trace_kprobe_ops); | |
1666 | if (ret) | |
1667 | return ret; | |
413d37d1 | 1668 | |
c31ffb3f | 1669 | if (register_module_notifier(&trace_kprobe_module_nb)) |
61424318 MH |
1670 | return -EINVAL; |
1671 | ||
413d37d1 | 1672 | d_tracer = tracing_init_dentry(); |
14a5ae40 | 1673 | if (IS_ERR(d_tracer)) |
413d37d1 MH |
1674 | return 0; |
1675 | ||
8434dc93 | 1676 | entry = tracefs_create_file("kprobe_events", 0644, d_tracer, |
413d37d1 MH |
1677 | NULL, &kprobe_events_ops); |
1678 | ||
cd7e7bd5 | 1679 | /* Event list interface */ |
413d37d1 | 1680 | if (!entry) |
a395d6a7 | 1681 | pr_warn("Could not create tracefs 'kprobe_events' entry\n"); |
cd7e7bd5 MH |
1682 | |
1683 | /* Profile interface */ | |
8434dc93 | 1684 | entry = tracefs_create_file("kprobe_profile", 0444, d_tracer, |
cd7e7bd5 MH |
1685 | NULL, &kprobe_profile_ops); |
1686 | ||
1687 | if (!entry) | |
a395d6a7 | 1688 | pr_warn("Could not create tracefs 'kprobe_profile' entry\n"); |
970988e1 MH |
1689 | |
1690 | setup_boot_kprobe_events(); | |
1691 | ||
413d37d1 MH |
1692 | return 0; |
1693 | } | |
1694 | fs_initcall(init_kprobe_trace); | |
1695 | ||
1696 | ||
1697 | #ifdef CONFIG_FTRACE_STARTUP_TEST | |
26a346f2 | 1698 | static __init struct trace_event_file * |
c31ffb3f | 1699 | find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr) |
41a7dd42 | 1700 | { |
7f1d2f82 | 1701 | struct trace_event_file *file; |
41a7dd42 MH |
1702 | |
1703 | list_for_each_entry(file, &tr->events, list) | |
e3dc9f89 | 1704 | if (file->event_call == trace_probe_event_call(&tk->tp)) |
41a7dd42 MH |
1705 | return file; |
1706 | ||
1707 | return NULL; | |
1708 | } | |
1709 | ||
3fe3d619 | 1710 | /* |
c31ffb3f | 1711 | * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this |
3fe3d619 ON |
1712 | * stage, we can do this lockless. |
1713 | */ | |
413d37d1 MH |
1714 | static __init int kprobe_trace_self_tests_init(void) |
1715 | { | |
231e36f4 | 1716 | int ret, warn = 0; |
413d37d1 | 1717 | int (*target)(int, int, int, int, int, int); |
c31ffb3f | 1718 | struct trace_kprobe *tk; |
7f1d2f82 | 1719 | struct trace_event_file *file; |
413d37d1 | 1720 | |
748ec3a2 YY |
1721 | if (tracing_is_disabled()) |
1722 | return -ENODEV; | |
1723 | ||
b6399cc7 SRV |
1724 | if (kprobe_boot_events_enabled) { |
1725 | pr_info("Skipping kprobe tests due to kprobe_event on cmdline\n"); | |
1726 | return 0; | |
1727 | } | |
1728 | ||
413d37d1 MH |
1729 | target = kprobe_trace_selftest_target; |
1730 | ||
1731 | pr_info("Testing kprobe tracing: "); | |
1732 | ||
6212dd29 MH |
1733 | ret = trace_run_command("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)", |
1734 | create_or_delete_trace_kprobe); | |
231e36f4 | 1735 | if (WARN_ON_ONCE(ret)) { |
41a7dd42 | 1736 | pr_warn("error on probing function entry.\n"); |
231e36f4 MH |
1737 | warn++; |
1738 | } else { | |
1739 | /* Enable trace point */ | |
c31ffb3f NK |
1740 | tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); |
1741 | if (WARN_ON_ONCE(tk == NULL)) { | |
41a7dd42 | 1742 | pr_warn("error on getting new probe.\n"); |
231e36f4 | 1743 | warn++; |
41a7dd42 | 1744 | } else { |
c31ffb3f | 1745 | file = find_trace_probe_file(tk, top_trace_array()); |
41a7dd42 MH |
1746 | if (WARN_ON_ONCE(file == NULL)) { |
1747 | pr_warn("error on getting probe file.\n"); | |
1748 | warn++; | |
1749 | } else | |
60d53e2c MH |
1750 | enable_trace_kprobe( |
1751 | trace_probe_event_call(&tk->tp), file); | |
41a7dd42 | 1752 | } |
231e36f4 | 1753 | } |
413d37d1 | 1754 | |
6212dd29 MH |
1755 | ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target $retval", |
1756 | create_or_delete_trace_kprobe); | |
231e36f4 | 1757 | if (WARN_ON_ONCE(ret)) { |
41a7dd42 | 1758 | pr_warn("error on probing function return.\n"); |
231e36f4 MH |
1759 | warn++; |
1760 | } else { | |
1761 | /* Enable trace point */ | |
c31ffb3f NK |
1762 | tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); |
1763 | if (WARN_ON_ONCE(tk == NULL)) { | |
41a7dd42 | 1764 | pr_warn("error on getting 2nd new probe.\n"); |
231e36f4 | 1765 | warn++; |
41a7dd42 | 1766 | } else { |
c31ffb3f | 1767 | file = find_trace_probe_file(tk, top_trace_array()); |
41a7dd42 MH |
1768 | if (WARN_ON_ONCE(file == NULL)) { |
1769 | pr_warn("error on getting probe file.\n"); | |
1770 | warn++; | |
1771 | } else | |
60d53e2c MH |
1772 | enable_trace_kprobe( |
1773 | trace_probe_event_call(&tk->tp), file); | |
41a7dd42 | 1774 | } |
231e36f4 MH |
1775 | } |
1776 | ||
1777 | if (warn) | |
1778 | goto end; | |
413d37d1 MH |
1779 | |
1780 | ret = target(1, 2, 3, 4, 5, 6); | |
1781 | ||
d4d7ccc8 MN |
1782 | /* |
1783 | * Not expecting an error here, the check is only to prevent the | |
1784 | * optimizer from removing the call to target() as otherwise there | |
1785 | * are no side-effects and the call is never performed. | |
1786 | */ | |
1787 | if (ret != 21) | |
1788 | warn++; | |
1789 | ||
02ca1521 | 1790 | /* Disable trace points before removing it */ |
c31ffb3f NK |
1791 | tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM); |
1792 | if (WARN_ON_ONCE(tk == NULL)) { | |
41a7dd42 | 1793 | pr_warn("error on getting test probe.\n"); |
02ca1521 | 1794 | warn++; |
41a7dd42 | 1795 | } else { |
d4d7ccc8 MN |
1796 | if (trace_kprobe_nhit(tk) != 1) { |
1797 | pr_warn("incorrect number of testprobe hits\n"); | |
1798 | warn++; | |
1799 | } | |
1800 | ||
c31ffb3f | 1801 | file = find_trace_probe_file(tk, top_trace_array()); |
41a7dd42 MH |
1802 | if (WARN_ON_ONCE(file == NULL)) { |
1803 | pr_warn("error on getting probe file.\n"); | |
1804 | warn++; | |
1805 | } else | |
60d53e2c MH |
1806 | disable_trace_kprobe( |
1807 | trace_probe_event_call(&tk->tp), file); | |
41a7dd42 | 1808 | } |
02ca1521 | 1809 | |
c31ffb3f NK |
1810 | tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM); |
1811 | if (WARN_ON_ONCE(tk == NULL)) { | |
41a7dd42 | 1812 | pr_warn("error on getting 2nd test probe.\n"); |
02ca1521 | 1813 | warn++; |
41a7dd42 | 1814 | } else { |
d4d7ccc8 MN |
1815 | if (trace_kprobe_nhit(tk) != 1) { |
1816 | pr_warn("incorrect number of testprobe2 hits\n"); | |
1817 | warn++; | |
1818 | } | |
1819 | ||
c31ffb3f | 1820 | file = find_trace_probe_file(tk, top_trace_array()); |
41a7dd42 MH |
1821 | if (WARN_ON_ONCE(file == NULL)) { |
1822 | pr_warn("error on getting probe file.\n"); | |
1823 | warn++; | |
1824 | } else | |
60d53e2c MH |
1825 | disable_trace_kprobe( |
1826 | trace_probe_event_call(&tk->tp), file); | |
41a7dd42 | 1827 | } |
02ca1521 | 1828 | |
6212dd29 | 1829 | ret = trace_run_command("-:testprobe", create_or_delete_trace_kprobe); |
231e36f4 | 1830 | if (WARN_ON_ONCE(ret)) { |
41a7dd42 | 1831 | pr_warn("error on deleting a probe.\n"); |
231e36f4 MH |
1832 | warn++; |
1833 | } | |
1834 | ||
6212dd29 | 1835 | ret = trace_run_command("-:testprobe2", create_or_delete_trace_kprobe); |
231e36f4 | 1836 | if (WARN_ON_ONCE(ret)) { |
41a7dd42 | 1837 | pr_warn("error on deleting a probe.\n"); |
231e36f4 MH |
1838 | warn++; |
1839 | } | |
413d37d1 | 1840 | |
231e36f4 | 1841 | end: |
6212dd29 MH |
1842 | ret = dyn_events_release_all(&trace_kprobe_ops); |
1843 | if (WARN_ON_ONCE(ret)) { | |
1844 | pr_warn("error on cleaning up probes.\n"); | |
1845 | warn++; | |
1846 | } | |
30e7d894 TG |
1847 | /* |
1848 | * Wait for the optimizer work to finish. Otherwise it might fiddle | |
1849 | * with probes in already freed __init text. | |
1850 | */ | |
1851 | wait_for_kprobe_optimizer(); | |
231e36f4 MH |
1852 | if (warn) |
1853 | pr_cont("NG: Some tests are failed. Please check them.\n"); | |
1854 | else | |
1855 | pr_cont("OK\n"); | |
413d37d1 MH |
1856 | return 0; |
1857 | } | |
1858 | ||
1859 | late_initcall(kprobe_trace_self_tests_init); | |
1860 | ||
1861 | #endif |