projects / linux-2.6-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
ftrace: Simplify ftrace hash lookup code in clear_func_from_hash()
[linux-2.6-block.git] / kernel / trace / trace_kprobe.c
... / ...
CommitLineData
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Kprobes-based tracing events
4 *
5 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 *
7 */
8#define pr_fmt(fmt) "trace_kprobe: " fmt
9
10#include <linux/module.h>
11#include <linux/uaccess.h>
12#include <linux/rculist.h>
13#include <linux/error-injection.h>
14
15#include <asm/setup.h> /* for COMMAND_LINE_SIZE */
16
17#include "trace_dynevent.h"
18#include "trace_kprobe_selftest.h"
19#include "trace_probe.h"
20#include "trace_probe_tmpl.h"
21
22#define KPROBE_EVENT_SYSTEM "kprobes"
23#define KRETPROBE_MAXACTIVE_MAX 4096
24#define MAX_KPROBE_CMDLINE_SIZE 1024
25
26/* Kprobe early definition from command line */
27static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata;
28static bool kprobe_boot_events_enabled __initdata;
29
30static int __init set_kprobe_boot_events(char *str)
31{
32 strlcpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE);
33 return 0;
34}
35__setup("kprobe_event=", set_kprobe_boot_events);
36
37static int trace_kprobe_create(int argc, const char **argv);
38static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev);
39static int trace_kprobe_release(struct dyn_event *ev);
40static bool trace_kprobe_is_busy(struct dyn_event *ev);
41static bool trace_kprobe_match(const char *system, const char *event,
42 int argc, const char **argv, struct dyn_event *ev);
43
44static struct dyn_event_operations trace_kprobe_ops = {
45 .create = trace_kprobe_create,
46 .show = trace_kprobe_show,
47 .is_busy = trace_kprobe_is_busy,
48 .free = trace_kprobe_release,
49 .match = trace_kprobe_match,
50};
51
52/*
53 * Kprobe event core functions
54 */
55struct trace_kprobe {
56 struct dyn_event devent;
57 struct kretprobe rp; /* Use rp.kp for kprobe use */
58 unsigned long __percpu *nhit;
59 const char *symbol; /* symbol name */
60 struct trace_probe tp;
61};
62
63static bool is_trace_kprobe(struct dyn_event *ev)
64{
65 return ev->ops == &trace_kprobe_ops;
66}
67
68static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev)
69{
70 return container_of(ev, struct trace_kprobe, devent);
71}
72
73/**
74 * for_each_trace_kprobe - iterate over the trace_kprobe list
75 * @pos: the struct trace_kprobe * for each entry
76 * @dpos: the struct dyn_event * to use as a loop cursor
77 */
78#define for_each_trace_kprobe(pos, dpos) \
79 for_each_dyn_event(dpos) \
80 if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos)))
81
82#define SIZEOF_TRACE_KPROBE(n) \
83 (offsetof(struct trace_kprobe, tp.args) + \
84 (sizeof(struct probe_arg) * (n)))
85
86static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
87{
88 return tk->rp.handler != NULL;
89}
90
91static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
92{
93 return tk->symbol ? tk->symbol : "unknown";
94}
95
96static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
97{
98 return tk->rp.kp.offset;
99}
100
101static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
102{
103 return !!(kprobe_gone(&tk->rp.kp));
104}
105
106static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
107 struct module *mod)
108{
109 int len = strlen(mod->name);
110 const char *name = trace_kprobe_symbol(tk);
111 return strncmp(mod->name, name, len) == 0 && name[len] == ':';
112}
113
114static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
115{
116 char *p;
117 bool ret;
118
119 if (!tk->symbol)
120 return false;
121 p = strchr(tk->symbol, ':');
122 if (!p)
123 return true;
124 *p = '\0';
125 mutex_lock(&module_mutex);
126 ret = !!find_module(tk->symbol);
127 mutex_unlock(&module_mutex);
128 *p = ':';
129
130 return ret;
131}
132
133static bool trace_kprobe_is_busy(struct dyn_event *ev)
134{
135 struct trace_kprobe *tk = to_trace_kprobe(ev);
136
137 return trace_probe_is_enabled(&tk->tp);
138}
139
140static bool trace_kprobe_match_command_head(struct trace_kprobe *tk,
141 int argc, const char **argv)
142{
143 char buf[MAX_ARGSTR_LEN + 1];
144
145 if (!argc)
146 return true;
147
148 if (!tk->symbol)
149 snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr);
150 else if (tk->rp.kp.offset)
151 snprintf(buf, sizeof(buf), "%s+%u",
152 trace_kprobe_symbol(tk), tk->rp.kp.offset);
153 else
154 snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk));
155 if (strcmp(buf, argv[0]))
156 return false;
157 argc--; argv++;
158
159 return trace_probe_match_command_args(&tk->tp, argc, argv);
160}
161
162static bool trace_kprobe_match(const char *system, const char *event,
163 int argc, const char **argv, struct dyn_event *ev)
164{
165 struct trace_kprobe *tk = to_trace_kprobe(ev);
166
167 return strcmp(trace_probe_name(&tk->tp), event) == 0 &&
168 (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) &&
169 trace_kprobe_match_command_head(tk, argc, argv);
170}
171
172static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
173{
174 unsigned long nhit = 0;
175 int cpu;
176
177 for_each_possible_cpu(cpu)
178 nhit += *per_cpu_ptr(tk->nhit, cpu);
179
180 return nhit;
181}
182
183static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk)
184{
185 return !(list_empty(&tk->rp.kp.list) &&
186 hlist_unhashed(&tk->rp.kp.hlist));
187}
188
189/* Return 0 if it fails to find the symbol address */
190static nokprobe_inline
191unsigned long trace_kprobe_address(struct trace_kprobe *tk)
192{
193 unsigned long addr;
194
195 if (tk->symbol) {
196 addr = (unsigned long)
197 kallsyms_lookup_name(trace_kprobe_symbol(tk));
198 if (addr)
199 addr += tk->rp.kp.offset;
200 } else {
201 addr = (unsigned long)tk->rp.kp.addr;
202 }
203 return addr;
204}
205
206static nokprobe_inline struct trace_kprobe *
207trace_kprobe_primary_from_call(struct trace_event_call *call)
208{
209 struct trace_probe *tp;
210
211 tp = trace_probe_primary_from_call(call);
212 if (WARN_ON_ONCE(!tp))
213 return NULL;
214
215 return container_of(tp, struct trace_kprobe, tp);
216}
217
218bool trace_kprobe_on_func_entry(struct trace_event_call *call)
219{
220 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
221
222 return tk ? kprobe_on_func_entry(tk->rp.kp.addr,
223 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,
224 tk->rp.kp.addr ? 0 : tk->rp.kp.offset) : false;
225}
226
227bool trace_kprobe_error_injectable(struct trace_event_call *call)
228{
229 struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
230
231 return tk ? within_error_injection_list(trace_kprobe_address(tk)) :
232 false;
233}
234
235static int register_kprobe_event(struct trace_kprobe *tk);
236static int unregister_kprobe_event(struct trace_kprobe *tk);
237
238static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
239static int kretprobe_dispatcher(struct kretprobe_instance *ri,
240 struct pt_regs *regs);
241
242static void free_trace_kprobe(struct trace_kprobe *tk)
243{
244 if (tk) {
245 trace_probe_cleanup(&tk->tp);
246 kfree(tk->symbol);
247 free_percpu(tk->nhit);
248 kfree(tk);
249 }
250}
251
252/*
253 * Allocate new trace_probe and initialize it (including kprobes).
254 */
255static struct trace_kprobe *alloc_trace_kprobe(const char *group,
256 const char *event,
257 void *addr,
258 const char *symbol,
259 unsigned long offs,
260 int maxactive,
261 int nargs, bool is_return)
262{
263 struct trace_kprobe *tk;
264 int ret = -ENOMEM;
265
266 tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);
267 if (!tk)
268 return ERR_PTR(ret);
269
270 tk->nhit = alloc_percpu(unsigned long);
271 if (!tk->nhit)
272 goto error;
273
274 if (symbol) {
275 tk->symbol = kstrdup(symbol, GFP_KERNEL);
276 if (!tk->symbol)
277 goto error;
278 tk->rp.kp.symbol_name = tk->symbol;
279 tk->rp.kp.offset = offs;
280 } else
281 tk->rp.kp.addr = addr;
282
283 if (is_return)
284 tk->rp.handler = kretprobe_dispatcher;
285 else
286 tk->rp.kp.pre_handler = kprobe_dispatcher;
287
288 tk->rp.maxactive = maxactive;
289 INIT_HLIST_NODE(&tk->rp.kp.hlist);
290 INIT_LIST_HEAD(&tk->rp.kp.list);
291
292 ret = trace_probe_init(&tk->tp, event, group);
293 if (ret < 0)
294 goto error;
295
296 dyn_event_init(&tk->devent, &trace_kprobe_ops);
297 return tk;
298error:
299 free_trace_kprobe(tk);
300 return ERR_PTR(ret);
301}
302
303static struct trace_kprobe *find_trace_kprobe(const char *event,
304 const char *group)
305{
306 struct dyn_event *pos;
307 struct trace_kprobe *tk;
308
309 for_each_trace_kprobe(tk, pos)
310 if (strcmp(trace_probe_name(&tk->tp), event) == 0 &&
311 strcmp(trace_probe_group_name(&tk->tp), group) == 0)
312 return tk;
313 return NULL;
314}
315
316static inline int __enable_trace_kprobe(struct trace_kprobe *tk)
317{
318 int ret = 0;
319
320 if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) {
321 if (trace_kprobe_is_return(tk))
322 ret = enable_kretprobe(&tk->rp);
323 else
324 ret = enable_kprobe(&tk->rp.kp);
325 }
326
327 return ret;
328}
329
330static void __disable_trace_kprobe(struct trace_probe *tp)
331{
332 struct trace_probe *pos;
333 struct trace_kprobe *tk;
334
335 list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
336 tk = container_of(pos, struct trace_kprobe, tp);
337 if (!trace_kprobe_is_registered(tk))
338 continue;
339 if (trace_kprobe_is_return(tk))
340 disable_kretprobe(&tk->rp);
341 else
342 disable_kprobe(&tk->rp.kp);
343 }
344}
345
346/*
347 * Enable trace_probe
348 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
349 */
350static int enable_trace_kprobe(struct trace_event_call *call,
351 struct trace_event_file *file)
352{
353 struct trace_probe *pos, *tp;
354 struct trace_kprobe *tk;
355 bool enabled;
356 int ret = 0;
357
358 tp = trace_probe_primary_from_call(call);
359 if (WARN_ON_ONCE(!tp))
360 return -ENODEV;
361 enabled = trace_probe_is_enabled(tp);
362
363 /* This also changes "enabled" state */
364 if (file) {
365 ret = trace_probe_add_file(tp, file);
366 if (ret)
367 return ret;
368 } else
369 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
370
371 if (enabled)
372 return 0;
373
374 enabled = false;
375 list_for_each_entry(pos, trace_probe_probe_list(tp), list) {
376 tk = container_of(pos, struct trace_kprobe, tp);
377 if (trace_kprobe_has_gone(tk))
378 continue;
379 ret = __enable_trace_kprobe(tk);
380 if (ret) {
381 if (enabled) {
382 __disable_trace_kprobe(tp);
383 enabled = false;
384 }
385 break;
386 }
387 enabled = true;
388 }
389
390 if (!enabled) {
391 /* No probe is enabled. Roll back */
392 if (file)
393 trace_probe_remove_file(tp, file);
394 else
395 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
396 if (!ret)
397 /* Since all probes are gone, this is not available */
398 ret = -EADDRNOTAVAIL;
399 }
400
401 return ret;
402}
403
404/*
405 * Disable trace_probe
406 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
407 */
408static int disable_trace_kprobe(struct trace_event_call *call,
409 struct trace_event_file *file)
410{
411 struct trace_probe *tp;
412
413 tp = trace_probe_primary_from_call(call);
414 if (WARN_ON_ONCE(!tp))
415 return -ENODEV;
416
417 if (file) {
418 if (!trace_probe_get_file_link(tp, file))
419 return -ENOENT;
420 if (!trace_probe_has_single_file(tp))
421 goto out;
422 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
423 } else
424 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
425
426 if (!trace_probe_is_enabled(tp))
427 __disable_trace_kprobe(tp);
428
429 out:
430 if (file)
431 /*
432 * Synchronization is done in below function. For perf event,
433 * file == NULL and perf_trace_event_unreg() calls
434 * tracepoint_synchronize_unregister() to ensure synchronize
435 * event. We don't need to care about it.
436 */
437 trace_probe_remove_file(tp, file);
438
439 return 0;
440}
441
442#if defined(CONFIG_KPROBES_ON_FTRACE) && \
443 !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE)
444static bool within_notrace_func(struct trace_kprobe *tk)
445{
446 unsigned long offset, size, addr;
447
448 addr = trace_kprobe_address(tk);
449 if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset))
450 return false;
451
452 /* Get the entry address of the target function */
453 addr -= offset;
454
455 /*
456 * Since ftrace_location_range() does inclusive range check, we need
457 * to subtract 1 byte from the end address.
458 */
459 return !ftrace_location_range(addr, addr + size - 1);
460}
461#else
462#define within_notrace_func(tk) (false)
463#endif
464
465/* Internal register function - just handle k*probes and flags */
466static int __register_trace_kprobe(struct trace_kprobe *tk)
467{
468 int i, ret;
469
470 if (trace_kprobe_is_registered(tk))
471 return -EINVAL;
472
473 if (within_notrace_func(tk)) {
474 pr_warn("Could not probe notrace function %s\n",
475 trace_kprobe_symbol(tk));
476 return -EINVAL;
477 }
478
479 for (i = 0; i < tk->tp.nr_args; i++) {
480 ret = traceprobe_update_arg(&tk->tp.args[i]);
481 if (ret)
482 return ret;
483 }
484
485 /* Set/clear disabled flag according to tp->flag */
486 if (trace_probe_is_enabled(&tk->tp))
487 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
488 else
489 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
490
491 if (trace_kprobe_is_return(tk))
492 ret = register_kretprobe(&tk->rp);
493 else
494 ret = register_kprobe(&tk->rp.kp);
495
496 return ret;
497}
498
499/* Internal unregister function - just handle k*probes and flags */
500static void __unregister_trace_kprobe(struct trace_kprobe *tk)
501{
502 if (trace_kprobe_is_registered(tk)) {
503 if (trace_kprobe_is_return(tk))
504 unregister_kretprobe(&tk->rp);
505 else
506 unregister_kprobe(&tk->rp.kp);
507 /* Cleanup kprobe for reuse and mark it unregistered */
508 INIT_HLIST_NODE(&tk->rp.kp.hlist);
509 INIT_LIST_HEAD(&tk->rp.kp.list);
510 if (tk->rp.kp.symbol_name)
511 tk->rp.kp.addr = NULL;
512 }
513}
514
515/* Unregister a trace_probe and probe_event */
516static int unregister_trace_kprobe(struct trace_kprobe *tk)
517{
518 /* If other probes are on the event, just unregister kprobe */
519 if (trace_probe_has_sibling(&tk->tp))
520 goto unreg;
521
522 /* Enabled event can not be unregistered */
523 if (trace_probe_is_enabled(&tk->tp))
524 return -EBUSY;
525
526 /* Will fail if probe is being used by ftrace or perf */
527 if (unregister_kprobe_event(tk))
528 return -EBUSY;
529
530unreg:
531 __unregister_trace_kprobe(tk);
532 dyn_event_remove(&tk->devent);
533 trace_probe_unlink(&tk->tp);
534
535 return 0;
536}
537
538static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to)
539{
540 int ret;
541
542 /* Append to existing event */
543 ret = trace_probe_append(&tk->tp, &to->tp);
544 if (ret)
545 return ret;
546
547 /* Register k*probe */
548 ret = __register_trace_kprobe(tk);
549 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
550 pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
551 ret = 0;
552 }
553
554 if (ret)
555 trace_probe_unlink(&tk->tp);
556 else
557 dyn_event_add(&tk->devent);
558
559 return ret;
560}
561
562/* Register a trace_probe and probe_event */
563static int register_trace_kprobe(struct trace_kprobe *tk)
564{
565 struct trace_kprobe *old_tk;
566 int ret;
567
568 mutex_lock(&event_mutex);
569
570 old_tk = find_trace_kprobe(trace_probe_name(&tk->tp),
571 trace_probe_group_name(&tk->tp));
572 if (old_tk) {
573 if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) {
574 trace_probe_log_set_index(0);
575 trace_probe_log_err(0, DIFF_PROBE_TYPE);
576 ret = -EEXIST;
577 } else {
578 ret = trace_probe_compare_arg_type(&tk->tp, &old_tk->tp);
579 if (ret) {
580 /* Note that argument starts index = 2 */
581 trace_probe_log_set_index(ret + 1);
582 trace_probe_log_err(0, DIFF_ARG_TYPE);
583 ret = -EEXIST;
584 } else
585 ret = append_trace_kprobe(tk, old_tk);
586 }
587 goto end;
588 }
589
590 /* Register new event */
591 ret = register_kprobe_event(tk);
592 if (ret) {
593 pr_warn("Failed to register probe event(%d)\n", ret);
594 goto end;
595 }
596
597 /* Register k*probe */
598 ret = __register_trace_kprobe(tk);
599 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
600 pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
601 ret = 0;
602 }
603
604 if (ret < 0)
605 unregister_kprobe_event(tk);
606 else
607 dyn_event_add(&tk->devent);
608
609end:
610 mutex_unlock(&event_mutex);
611 return ret;
612}
613
614/* Module notifier call back, checking event on the module */
615static int trace_kprobe_module_callback(struct notifier_block *nb,
616 unsigned long val, void *data)
617{
618 struct module *mod = data;
619 struct dyn_event *pos;
620 struct trace_kprobe *tk;
621 int ret;
622
623 if (val != MODULE_STATE_COMING)
624 return NOTIFY_DONE;
625
626 /* Update probes on coming module */
627 mutex_lock(&event_mutex);
628 for_each_trace_kprobe(tk, pos) {
629 if (trace_kprobe_within_module(tk, mod)) {
630 /* Don't need to check busy - this should have gone. */
631 __unregister_trace_kprobe(tk);
632 ret = __register_trace_kprobe(tk);
633 if (ret)
634 pr_warn("Failed to re-register probe %s on %s: %d\n",
635 trace_probe_name(&tk->tp),
636 mod->name, ret);
637 }
638 }
639 mutex_unlock(&event_mutex);
640
641 return NOTIFY_DONE;
642}
643
644static struct notifier_block trace_kprobe_module_nb = {
645 .notifier_call = trace_kprobe_module_callback,
646 .priority = 1 /* Invoked after kprobe module callback */
647};
648
649/* Convert certain expected symbols into '_' when generating event names */
650static inline void sanitize_event_name(char *name)
651{
652 while (*name++ != '\0')
653 if (*name == ':' || *name == '.')
654 *name = '_';
655}
656
657static int trace_kprobe_create(int argc, const char *argv[])
658{
659 /*
660 * Argument syntax:
661 * - Add kprobe:
662 * p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
663 * - Add kretprobe:
664 * r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
665 * Fetch args:
666 * $retval : fetch return value
667 * $stack : fetch stack address
668 * $stackN : fetch Nth of stack (N:0-)
669 * $comm : fetch current task comm
670 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
671 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
672 * %REG : fetch register REG
673 * Dereferencing memory fetch:
674 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
675 * Alias name of args:
676 * NAME=FETCHARG : set NAME as alias of FETCHARG.
677 * Type of args:
678 * FETCHARG:TYPE : use TYPE instead of unsigned long.
679 */
680 struct trace_kprobe *tk = NULL;
681 int i, len, ret = 0;
682 bool is_return = false;
683 char *symbol = NULL, *tmp = NULL;
684 const char *event = NULL, *group = KPROBE_EVENT_SYSTEM;
685 int maxactive = 0;
686 long offset = 0;
687 void *addr = NULL;
688 char buf[MAX_EVENT_NAME_LEN];
689 unsigned int flags = TPARG_FL_KERNEL;
690
691 switch (argv[0][0]) {
692 case 'r':
693 is_return = true;
694 flags |= TPARG_FL_RETURN;
695 break;
696 case 'p':
697 break;
698 default:
699 return -ECANCELED;
700 }
701 if (argc < 2)
702 return -ECANCELED;
703
704 trace_probe_log_init("trace_kprobe", argc, argv);
705
706 event = strchr(&argv[0][1], ':');
707 if (event)
708 event++;
709
710 if (isdigit(argv[0][1])) {
711 if (!is_return) {
712 trace_probe_log_err(1, MAXACT_NO_KPROBE);
713 goto parse_error;
714 }
715 if (event)
716 len = event - &argv[0][1] - 1;
717 else
718 len = strlen(&argv[0][1]);
719 if (len > MAX_EVENT_NAME_LEN - 1) {
720 trace_probe_log_err(1, BAD_MAXACT);
721 goto parse_error;
722 }
723 memcpy(buf, &argv[0][1], len);
724 buf[len] = '\0';
725 ret = kstrtouint(buf, 0, &maxactive);
726 if (ret || !maxactive) {
727 trace_probe_log_err(1, BAD_MAXACT);
728 goto parse_error;
729 }
730 /* kretprobes instances are iterated over via a list. The
731 * maximum should stay reasonable.
732 */
733 if (maxactive > KRETPROBE_MAXACTIVE_MAX) {
734 trace_probe_log_err(1, MAXACT_TOO_BIG);
735 goto parse_error;
736 }
737 }
738
739 /* try to parse an address. if that fails, try to read the
740 * input as a symbol. */
741 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
742 trace_probe_log_set_index(1);
743 /* Check whether uprobe event specified */
744 if (strchr(argv[1], '/') && strchr(argv[1], ':')) {
745 ret = -ECANCELED;
746 goto error;
747 }
748 /* a symbol specified */
749 symbol = kstrdup(argv[1], GFP_KERNEL);
750 if (!symbol)
751 return -ENOMEM;
752 /* TODO: support .init module functions */
753 ret = traceprobe_split_symbol_offset(symbol, &offset);
754 if (ret || offset < 0 || offset > UINT_MAX) {
755 trace_probe_log_err(0, BAD_PROBE_ADDR);
756 goto parse_error;
757 }
758 if (kprobe_on_func_entry(NULL, symbol, offset))
759 flags |= TPARG_FL_FENTRY;
760 if (offset && is_return && !(flags & TPARG_FL_FENTRY)) {
761 trace_probe_log_err(0, BAD_RETPROBE);
762 goto parse_error;
763 }
764 }
765
766 trace_probe_log_set_index(0);
767 if (event) {
768 ret = traceprobe_parse_event_name(&event, &group, buf,
769 event - argv[0]);
770 if (ret)
771 goto parse_error;
772 } else {
773 /* Make a new event name */
774 if (symbol)
775 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
776 is_return ? 'r' : 'p', symbol, offset);
777 else
778 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
779 is_return ? 'r' : 'p', addr);
780 sanitize_event_name(buf);
781 event = buf;
782 }
783
784 /* setup a probe */
785 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
786 argc - 2, is_return);
787 if (IS_ERR(tk)) {
788 ret = PTR_ERR(tk);
789 /* This must return -ENOMEM, else there is a bug */
790 WARN_ON_ONCE(ret != -ENOMEM);
791 goto out; /* We know tk is not allocated */
792 }
793 argc -= 2; argv += 2;
794
795 /* parse arguments */
796 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
797 tmp = kstrdup(argv[i], GFP_KERNEL);
798 if (!tmp) {
799 ret = -ENOMEM;
800 goto error;
801 }
802
803 trace_probe_log_set_index(i + 2);
804 ret = traceprobe_parse_probe_arg(&tk->tp, i, tmp, flags);
805 kfree(tmp);
806 if (ret)
807 goto error; /* This can be -ENOMEM */
808 }
809
810 ret = traceprobe_set_print_fmt(&tk->tp, is_return);
811 if (ret < 0)
812 goto error;
813
814 ret = register_trace_kprobe(tk);
815 if (ret) {
816 trace_probe_log_set_index(1);
817 if (ret == -EILSEQ)
818 trace_probe_log_err(0, BAD_INSN_BNDRY);
819 else if (ret == -ENOENT)
820 trace_probe_log_err(0, BAD_PROBE_ADDR);
821 else if (ret != -ENOMEM && ret != -EEXIST)
822 trace_probe_log_err(0, FAIL_REG_PROBE);
823 goto error;
824 }
825
826out:
827 trace_probe_log_clear();
828 kfree(symbol);
829 return ret;
830
831parse_error:
832 ret = -EINVAL;
833error:
834 free_trace_kprobe(tk);
835 goto out;
836}
837
838static int create_or_delete_trace_kprobe(int argc, char **argv)
839{
840 int ret;
841
842 if (argv[0][0] == '-')
843 return dyn_event_release(argc, argv, &trace_kprobe_ops);
844
845 ret = trace_kprobe_create(argc, (const char **)argv);
846 return ret == -ECANCELED ? -EINVAL : ret;
847}
848
849static int trace_kprobe_release(struct dyn_event *ev)
850{
851 struct trace_kprobe *tk = to_trace_kprobe(ev);
852 int ret = unregister_trace_kprobe(tk);
853
854 if (!ret)
855 free_trace_kprobe(tk);
856 return ret;
857}
858
859static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev)
860{
861 struct trace_kprobe *tk = to_trace_kprobe(ev);
862 int i;
863
864 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
865 seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp),
866 trace_probe_name(&tk->tp));
867
868 if (!tk->symbol)
869 seq_printf(m, " 0x%p", tk->rp.kp.addr);
870 else if (tk->rp.kp.offset)
871 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
872 tk->rp.kp.offset);
873 else
874 seq_printf(m, " %s", trace_kprobe_symbol(tk));
875
876 for (i = 0; i < tk->tp.nr_args; i++)
877 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
878 seq_putc(m, '\n');
879
880 return 0;
881}
882
883static int probes_seq_show(struct seq_file *m, void *v)
884{
885 struct dyn_event *ev = v;
886
887 if (!is_trace_kprobe(ev))
888 return 0;
889
890 return trace_kprobe_show(m, ev);
891}
892
893static const struct seq_operations probes_seq_op = {
894 .start = dyn_event_seq_start,
895 .next = dyn_event_seq_next,
896 .stop = dyn_event_seq_stop,
897 .show = probes_seq_show
898};
899
900static int probes_open(struct inode *inode, struct file *file)
901{
902 int ret;
903
904 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
905 ret = dyn_events_release_all(&trace_kprobe_ops);
906 if (ret < 0)
907 return ret;
908 }
909
910 return seq_open(file, &probes_seq_op);
911}
912
913static ssize_t probes_write(struct file *file, const char __user *buffer,
914 size_t count, loff_t *ppos)
915{
916 return trace_parse_run_command(file, buffer, count, ppos,
917 create_or_delete_trace_kprobe);
918}
919
920static const struct file_operations kprobe_events_ops = {
921 .owner = THIS_MODULE,
922 .open = probes_open,
923 .read = seq_read,
924 .llseek = seq_lseek,
925 .release = seq_release,
926 .write = probes_write,
927};
928
929/* Probes profiling interfaces */
930static int probes_profile_seq_show(struct seq_file *m, void *v)
931{
932 struct dyn_event *ev = v;
933 struct trace_kprobe *tk;
934
935 if (!is_trace_kprobe(ev))
936 return 0;
937
938 tk = to_trace_kprobe(ev);
939 seq_printf(m, " %-44s %15lu %15lu\n",
940 trace_probe_name(&tk->tp),
941 trace_kprobe_nhit(tk),
942 tk->rp.kp.nmissed);
943
944 return 0;
945}
946
947static const struct seq_operations profile_seq_op = {
948 .start = dyn_event_seq_start,
949 .next = dyn_event_seq_next,
950 .stop = dyn_event_seq_stop,
951 .show = probes_profile_seq_show
952};
953
954static int profile_open(struct inode *inode, struct file *file)
955{
956 return seq_open(file, &profile_seq_op);
957}
958
959static const struct file_operations kprobe_profile_ops = {
960 .owner = THIS_MODULE,
961 .open = profile_open,
962 .read = seq_read,
963 .llseek = seq_lseek,
964 .release = seq_release,
965};
966
967/* Kprobe specific fetch functions */
968
969/* Return the length of string -- including null terminal byte */
970static nokprobe_inline int
971fetch_store_strlen(unsigned long addr)
972{
973 int ret, len = 0;
974 u8 c;
975
976 do {
977 ret = probe_kernel_read(&c, (u8 *)addr + len, 1);
978 len++;
979 } while (c && ret == 0 && len < MAX_STRING_SIZE);
980
981 return (ret < 0) ? ret : len;
982}
983
984/* Return the length of string -- including null terminal byte */
985static nokprobe_inline int
986fetch_store_strlen_user(unsigned long addr)
987{
988 const void __user *uaddr = (__force const void __user *)addr;
989
990 return strnlen_unsafe_user(uaddr, MAX_STRING_SIZE);
991}
992
993/*
994 * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max
995 * length and relative data location.
996 */
997static nokprobe_inline int
998fetch_store_string(unsigned long addr, void *dest, void *base)
999{
1000 int maxlen = get_loc_len(*(u32 *)dest);
1001 void *__dest;
1002 long ret;
1003
1004 if (unlikely(!maxlen))
1005 return -ENOMEM;
1006
1007 __dest = get_loc_data(dest, base);
1008
1009 /*
1010 * Try to get string again, since the string can be changed while
1011 * probing.
1012 */
1013 ret = strncpy_from_unsafe(__dest, (void *)addr, maxlen);
1014 if (ret >= 0)
1015 *(u32 *)dest = make_data_loc(ret, __dest - base);
1016
1017 return ret;
1018}
1019
1020/*
1021 * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf
1022 * with max length and relative data location.
1023 */
1024static nokprobe_inline int
1025fetch_store_string_user(unsigned long addr, void *dest, void *base)
1026{
1027 const void __user *uaddr = (__force const void __user *)addr;
1028 int maxlen = get_loc_len(*(u32 *)dest);
1029 void *__dest;
1030 long ret;
1031
1032 if (unlikely(!maxlen))
1033 return -ENOMEM;
1034
1035 __dest = get_loc_data(dest, base);
1036
1037 ret = strncpy_from_unsafe_user(__dest, uaddr, maxlen);
1038 if (ret >= 0)
1039 *(u32 *)dest = make_data_loc(ret, __dest - base);
1040
1041 return ret;
1042}
1043
1044static nokprobe_inline int
1045probe_mem_read(void *dest, void *src, size_t size)
1046{
1047 return probe_kernel_read(dest, src, size);
1048}
1049
1050static nokprobe_inline int
1051probe_mem_read_user(void *dest, void *src, size_t size)
1052{
1053 const void __user *uaddr = (__force const void __user *)src;
1054
1055 return probe_user_read(dest, uaddr, size);
1056}
1057
1058/* Note that we don't verify it, since the code does not come from user space */
1059static int
1060process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest,
1061 void *base)
1062{
1063 unsigned long val;
1064
1065retry:
1066 /* 1st stage: get value from context */
1067 switch (code->op) {
1068 case FETCH_OP_REG:
1069 val = regs_get_register(regs, code->param);
1070 break;
1071 case FETCH_OP_STACK:
1072 val = regs_get_kernel_stack_nth(regs, code->param);
1073 break;
1074 case FETCH_OP_STACKP:
1075 val = kernel_stack_pointer(regs);
1076 break;
1077 case FETCH_OP_RETVAL:
1078 val = regs_return_value(regs);
1079 break;
1080 case FETCH_OP_IMM:
1081 val = code->immediate;
1082 break;
1083 case FETCH_OP_COMM:
1084 val = (unsigned long)current->comm;
1085 break;
1086 case FETCH_OP_DATA:
1087 val = (unsigned long)code->data;
1088 break;
1089#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
1090 case FETCH_OP_ARG:
1091 val = regs_get_kernel_argument(regs, code->param);
1092 break;
1093#endif
1094 case FETCH_NOP_SYMBOL: /* Ignore a place holder */
1095 code++;
1096 goto retry;
1097 default:
1098 return -EILSEQ;
1099 }
1100 code++;
1101
1102 return process_fetch_insn_bottom(code, val, dest, base);
1103}
1104NOKPROBE_SYMBOL(process_fetch_insn)
1105
1106/* Kprobe handler */
1107static nokprobe_inline void
1108__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
1109 struct trace_event_file *trace_file)
1110{
1111 struct kprobe_trace_entry_head *entry;
1112 struct ring_buffer_event *event;
1113 struct ring_buffer *buffer;
1114 int size, dsize, pc;
1115 unsigned long irq_flags;
1116 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1117
1118 WARN_ON(call != trace_file->event_call);
1119
1120 if (trace_trigger_soft_disabled(trace_file))
1121 return;
1122
1123 local_save_flags(irq_flags);
1124 pc = preempt_count();
1125
1126 dsize = __get_data_size(&tk->tp, regs);
1127 size = sizeof(*entry) + tk->tp.size + dsize;
1128
1129 event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1130 call->event.type,
1131 size, irq_flags, pc);
1132 if (!event)
1133 return;
1134
1135 entry = ring_buffer_event_data(event);
1136 entry->ip = (unsigned long)tk->rp.kp.addr;
1137 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1138
1139 event_trigger_unlock_commit_regs(trace_file, buffer, event,
1140 entry, irq_flags, pc, regs);
1141}
1142
1143static void
1144kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
1145{
1146 struct event_file_link *link;
1147
1148 trace_probe_for_each_link_rcu(link, &tk->tp)
1149 __kprobe_trace_func(tk, regs, link->file);
1150}
1151NOKPROBE_SYMBOL(kprobe_trace_func);
1152
1153/* Kretprobe handler */
1154static nokprobe_inline void
1155__kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1156 struct pt_regs *regs,
1157 struct trace_event_file *trace_file)
1158{
1159 struct kretprobe_trace_entry_head *entry;
1160 struct ring_buffer_event *event;
1161 struct ring_buffer *buffer;
1162 int size, pc, dsize;
1163 unsigned long irq_flags;
1164 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1165
1166 WARN_ON(call != trace_file->event_call);
1167
1168 if (trace_trigger_soft_disabled(trace_file))
1169 return;
1170
1171 local_save_flags(irq_flags);
1172 pc = preempt_count();
1173
1174 dsize = __get_data_size(&tk->tp, regs);
1175 size = sizeof(*entry) + tk->tp.size + dsize;
1176
1177 event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1178 call->event.type,
1179 size, irq_flags, pc);
1180 if (!event)
1181 return;
1182
1183 entry = ring_buffer_event_data(event);
1184 entry->func = (unsigned long)tk->rp.kp.addr;
1185 entry->ret_ip = (unsigned long)ri->ret_addr;
1186 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1187
1188 event_trigger_unlock_commit_regs(trace_file, buffer, event,
1189 entry, irq_flags, pc, regs);
1190}
1191
1192static void
1193kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1194 struct pt_regs *regs)
1195{
1196 struct event_file_link *link;
1197
1198 trace_probe_for_each_link_rcu(link, &tk->tp)
1199 __kretprobe_trace_func(tk, ri, regs, link->file);
1200}
1201NOKPROBE_SYMBOL(kretprobe_trace_func);
1202
1203/* Event entry printers */
1204static enum print_line_t
1205print_kprobe_event(struct trace_iterator *iter, int flags,
1206 struct trace_event *event)
1207{
1208 struct kprobe_trace_entry_head *field;
1209 struct trace_seq *s = &iter->seq;
1210 struct trace_probe *tp;
1211
1212 field = (struct kprobe_trace_entry_head *)iter->ent;
1213 tp = trace_probe_primary_from_call(
1214 container_of(event, struct trace_event_call, event));
1215 if (WARN_ON_ONCE(!tp))
1216 goto out;
1217
1218 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
1219
1220 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
1221 goto out;
1222
1223 trace_seq_putc(s, ')');
1224
1225 if (print_probe_args(s, tp->args, tp->nr_args,
1226 (u8 *)&field[1], field) < 0)
1227 goto out;
1228
1229 trace_seq_putc(s, '\n');
1230 out:
1231 return trace_handle_return(s);
1232}
1233
1234static enum print_line_t
1235print_kretprobe_event(struct trace_iterator *iter, int flags,
1236 struct trace_event *event)
1237{
1238 struct kretprobe_trace_entry_head *field;
1239 struct trace_seq *s = &iter->seq;
1240 struct trace_probe *tp;
1241
1242 field = (struct kretprobe_trace_entry_head *)iter->ent;
1243 tp = trace_probe_primary_from_call(
1244 container_of(event, struct trace_event_call, event));
1245 if (WARN_ON_ONCE(!tp))
1246 goto out;
1247
1248 trace_seq_printf(s, "%s: (", trace_probe_name(tp));
1249
1250 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
1251 goto out;
1252
1253 trace_seq_puts(s, " <- ");
1254
1255 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
1256 goto out;
1257
1258 trace_seq_putc(s, ')');
1259
1260 if (print_probe_args(s, tp->args, tp->nr_args,
1261 (u8 *)&field[1], field) < 0)
1262 goto out;
1263
1264 trace_seq_putc(s, '\n');
1265
1266 out:
1267 return trace_handle_return(s);
1268}
1269
1270
1271static int kprobe_event_define_fields(struct trace_event_call *event_call)
1272{
1273 int ret;
1274 struct kprobe_trace_entry_head field;
1275 struct trace_probe *tp;
1276
1277 tp = trace_probe_primary_from_call(event_call);
1278 if (WARN_ON_ONCE(!tp))
1279 return -ENOENT;
1280
1281 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1282
1283 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1284}
1285
1286static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1287{
1288 int ret;
1289 struct kretprobe_trace_entry_head field;
1290 struct trace_probe *tp;
1291
1292 tp = trace_probe_primary_from_call(event_call);
1293 if (WARN_ON_ONCE(!tp))
1294 return -ENOENT;
1295
1296 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1297 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1298
1299 return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1300}
1301
1302#ifdef CONFIG_PERF_EVENTS
1303
1304/* Kprobe profile handler */
1305static int
1306kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1307{
1308 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1309 struct kprobe_trace_entry_head *entry;
1310 struct hlist_head *head;
1311 int size, __size, dsize;
1312 int rctx;
1313
1314 if (bpf_prog_array_valid(call)) {
1315 unsigned long orig_ip = instruction_pointer(regs);
1316 int ret;
1317
1318 ret = trace_call_bpf(call, regs);
1319
1320 /*
1321 * We need to check and see if we modified the pc of the
1322 * pt_regs, and if so return 1 so that we don't do the
1323 * single stepping.
1324 */
1325 if (orig_ip != instruction_pointer(regs))
1326 return 1;
1327 if (!ret)
1328 return 0;
1329 }
1330
1331 head = this_cpu_ptr(call->perf_events);
1332 if (hlist_empty(head))
1333 return 0;
1334
1335 dsize = __get_data_size(&tk->tp, regs);
1336 __size = sizeof(*entry) + tk->tp.size + dsize;
1337 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1338 size -= sizeof(u32);
1339
1340 entry = perf_trace_buf_alloc(size, NULL, &rctx);
1341 if (!entry)
1342 return 0;
1343
1344 entry->ip = (unsigned long)tk->rp.kp.addr;
1345 memset(&entry[1], 0, dsize);
1346 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1347 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1348 head, NULL);
1349 return 0;
1350}
1351NOKPROBE_SYMBOL(kprobe_perf_func);
1352
1353/* Kretprobe profile handler */
1354static void
1355kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1356 struct pt_regs *regs)
1357{
1358 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1359 struct kretprobe_trace_entry_head *entry;
1360 struct hlist_head *head;
1361 int size, __size, dsize;
1362 int rctx;
1363
1364 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
1365 return;
1366
1367 head = this_cpu_ptr(call->perf_events);
1368 if (hlist_empty(head))
1369 return;
1370
1371 dsize = __get_data_size(&tk->tp, regs);
1372 __size = sizeof(*entry) + tk->tp.size + dsize;
1373 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1374 size -= sizeof(u32);
1375
1376 entry = perf_trace_buf_alloc(size, NULL, &rctx);
1377 if (!entry)
1378 return;
1379
1380 entry->func = (unsigned long)tk->rp.kp.addr;
1381 entry->ret_ip = (unsigned long)ri->ret_addr;
1382 store_trace_args(&entry[1], &tk->tp, regs, sizeof(*entry), dsize);
1383 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1384 head, NULL);
1385}
1386NOKPROBE_SYMBOL(kretprobe_perf_func);
1387
1388int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type,
1389 const char **symbol, u64 *probe_offset,
1390 u64 *probe_addr, bool perf_type_tracepoint)
1391{
1392 const char *pevent = trace_event_name(event->tp_event);
1393 const char *group = event->tp_event->class->system;
1394 struct trace_kprobe *tk;
1395
1396 if (perf_type_tracepoint)
1397 tk = find_trace_kprobe(pevent, group);
1398 else
1399 tk = event->tp_event->data;
1400 if (!tk)
1401 return -EINVAL;
1402
1403 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE
1404 : BPF_FD_TYPE_KPROBE;
1405 if (tk->symbol) {
1406 *symbol = tk->symbol;
1407 *probe_offset = tk->rp.kp.offset;
1408 *probe_addr = 0;
1409 } else {
1410 *symbol = NULL;
1411 *probe_offset = 0;
1412 *probe_addr = (unsigned long)tk->rp.kp.addr;
1413 }
1414 return 0;
1415}
1416#endif /* CONFIG_PERF_EVENTS */
1417
1418/*
1419 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1420 *
1421 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1422 * lockless, but we can't race with this __init function.
1423 */
1424static int kprobe_register(struct trace_event_call *event,
1425 enum trace_reg type, void *data)
1426{
1427 struct trace_event_file *file = data;
1428
1429 switch (type) {
1430 case TRACE_REG_REGISTER:
1431 return enable_trace_kprobe(event, file);
1432 case TRACE_REG_UNREGISTER:
1433 return disable_trace_kprobe(event, file);
1434
1435#ifdef CONFIG_PERF_EVENTS
1436 case TRACE_REG_PERF_REGISTER:
1437 return enable_trace_kprobe(event, NULL);
1438 case TRACE_REG_PERF_UNREGISTER:
1439 return disable_trace_kprobe(event, NULL);
1440 case TRACE_REG_PERF_OPEN:
1441 case TRACE_REG_PERF_CLOSE:
1442 case TRACE_REG_PERF_ADD:
1443 case TRACE_REG_PERF_DEL:
1444 return 0;
1445#endif
1446 }
1447 return 0;
1448}
1449
1450static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1451{
1452 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1453 int ret = 0;
1454
1455 raw_cpu_inc(*tk->nhit);
1456
1457 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))
1458 kprobe_trace_func(tk, regs);
1459#ifdef CONFIG_PERF_EVENTS
1460 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))
1461 ret = kprobe_perf_func(tk, regs);
1462#endif
1463 return ret;
1464}
1465NOKPROBE_SYMBOL(kprobe_dispatcher);
1466
1467static int
1468kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1469{
1470 struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
1471
1472 raw_cpu_inc(*tk->nhit);
1473
1474 if (trace_probe_test_flag(&tk->tp, TP_FLAG_TRACE))
1475 kretprobe_trace_func(tk, ri, regs);
1476#ifdef CONFIG_PERF_EVENTS
1477 if (trace_probe_test_flag(&tk->tp, TP_FLAG_PROFILE))
1478 kretprobe_perf_func(tk, ri, regs);
1479#endif
1480 return 0; /* We don't tweek kernel, so just return 0 */
1481}
1482NOKPROBE_SYMBOL(kretprobe_dispatcher);
1483
1484static struct trace_event_functions kretprobe_funcs = {
1485 .trace = print_kretprobe_event
1486};
1487
1488static struct trace_event_functions kprobe_funcs = {
1489 .trace = print_kprobe_event
1490};
1491
1492static inline void init_trace_event_call(struct trace_kprobe *tk)
1493{
1494 struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1495
1496 if (trace_kprobe_is_return(tk)) {
1497 call->event.funcs = &kretprobe_funcs;
1498 call->class->define_fields = kretprobe_event_define_fields;
1499 } else {
1500 call->event.funcs = &kprobe_funcs;
1501 call->class->define_fields = kprobe_event_define_fields;
1502 }
1503
1504 call->flags = TRACE_EVENT_FL_KPROBE;
1505 call->class->reg = kprobe_register;
1506}
1507
1508static int register_kprobe_event(struct trace_kprobe *tk)
1509{
1510 init_trace_event_call(tk);
1511
1512 return trace_probe_register_event_call(&tk->tp);
1513}
1514
1515static int unregister_kprobe_event(struct trace_kprobe *tk)
1516{
1517 return trace_probe_unregister_event_call(&tk->tp);
1518}
1519
1520#ifdef CONFIG_PERF_EVENTS
1521/* create a trace_kprobe, but don't add it to global lists */
1522struct trace_event_call *
1523create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1524 bool is_return)
1525{
1526 struct trace_kprobe *tk;
1527 int ret;
1528 char *event;
1529
1530 /*
1531 * local trace_kprobes are not added to dyn_event, so they are never
1532 * searched in find_trace_kprobe(). Therefore, there is no concern of
1533 * duplicated name here.
1534 */
1535 event = func ? func : "DUMMY_EVENT";
1536
1537 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func,
1538 offs, 0 /* maxactive */, 0 /* nargs */,
1539 is_return);
1540
1541 if (IS_ERR(tk)) {
1542 pr_info("Failed to allocate trace_probe.(%d)\n",
1543 (int)PTR_ERR(tk));
1544 return ERR_CAST(tk);
1545 }
1546
1547 init_trace_event_call(tk);
1548
1549 if (traceprobe_set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) {
1550 ret = -ENOMEM;
1551 goto error;
1552 }
1553
1554 ret = __register_trace_kprobe(tk);
1555 if (ret < 0)
1556 goto error;
1557
1558 return trace_probe_event_call(&tk->tp);
1559error:
1560 free_trace_kprobe(tk);
1561 return ERR_PTR(ret);
1562}
1563
1564void destroy_local_trace_kprobe(struct trace_event_call *event_call)
1565{
1566 struct trace_kprobe *tk;
1567
1568 tk = trace_kprobe_primary_from_call(event_call);
1569 if (unlikely(!tk))
1570 return;
1571
1572 if (trace_probe_is_enabled(&tk->tp)) {
1573 WARN_ON(1);
1574 return;
1575 }
1576
1577 __unregister_trace_kprobe(tk);
1578
1579 free_trace_kprobe(tk);
1580}
1581#endif /* CONFIG_PERF_EVENTS */
1582
1583static __init void enable_boot_kprobe_events(void)
1584{
1585 struct trace_array *tr = top_trace_array();
1586 struct trace_event_file *file;
1587 struct trace_kprobe *tk;
1588 struct dyn_event *pos;
1589
1590 mutex_lock(&event_mutex);
1591 for_each_trace_kprobe(tk, pos) {
1592 list_for_each_entry(file, &tr->events, list)
1593 if (file->event_call == trace_probe_event_call(&tk->tp))
1594 trace_event_enable_disable(file, 1, 0);
1595 }
1596 mutex_unlock(&event_mutex);
1597}
1598
1599static __init void setup_boot_kprobe_events(void)
1600{
1601 char *p, *cmd = kprobe_boot_events_buf;
1602 int ret;
1603
1604 strreplace(kprobe_boot_events_buf, ',', ' ');
1605
1606 while (cmd && *cmd != '\0') {
1607 p = strchr(cmd, ';');
1608 if (p)
1609 *p++ = '\0';
1610
1611 ret = trace_run_command(cmd, create_or_delete_trace_kprobe);
1612 if (ret)
1613 pr_warn("Failed to add event(%d): %s\n", ret, cmd);
1614 else
1615 kprobe_boot_events_enabled = true;
1616
1617 cmd = p;
1618 }
1619
1620 enable_boot_kprobe_events();
1621}
1622
1623/* Make a tracefs interface for controlling probe points */
1624static __init int init_kprobe_trace(void)
1625{
1626 struct dentry *d_tracer;
1627 struct dentry *entry;
1628 int ret;
1629
1630 ret = dyn_event_register(&trace_kprobe_ops);
1631 if (ret)
1632 return ret;
1633
1634 if (register_module_notifier(&trace_kprobe_module_nb))
1635 return -EINVAL;
1636
1637 d_tracer = tracing_init_dentry();
1638 if (IS_ERR(d_tracer))
1639 return 0;
1640
1641 entry = tracefs_create_file("kprobe_events", 0644, d_tracer,
1642 NULL, &kprobe_events_ops);
1643
1644 /* Event list interface */
1645 if (!entry)
1646 pr_warn("Could not create tracefs 'kprobe_events' entry\n");
1647
1648 /* Profile interface */
1649 entry = tracefs_create_file("kprobe_profile", 0444, d_tracer,
1650 NULL, &kprobe_profile_ops);
1651
1652 if (!entry)
1653 pr_warn("Could not create tracefs 'kprobe_profile' entry\n");
1654
1655 setup_boot_kprobe_events();
1656
1657 return 0;
1658}
1659fs_initcall(init_kprobe_trace);
1660
1661
1662#ifdef CONFIG_FTRACE_STARTUP_TEST
1663static __init struct trace_event_file *
1664find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
1665{
1666 struct trace_event_file *file;
1667
1668 list_for_each_entry(file, &tr->events, list)
1669 if (file->event_call == trace_probe_event_call(&tk->tp))
1670 return file;
1671
1672 return NULL;
1673}
1674
1675/*
1676 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
1677 * stage, we can do this lockless.
1678 */
1679static __init int kprobe_trace_self_tests_init(void)
1680{
1681 int ret, warn = 0;
1682 int (*target)(int, int, int, int, int, int);
1683 struct trace_kprobe *tk;
1684 struct trace_event_file *file;
1685
1686 if (tracing_is_disabled())
1687 return -ENODEV;
1688
1689 if (kprobe_boot_events_enabled) {
1690 pr_info("Skipping kprobe tests due to kprobe_event on cmdline\n");
1691 return 0;
1692 }
1693
1694 target = kprobe_trace_selftest_target;
1695
1696 pr_info("Testing kprobe tracing: ");
1697
1698 ret = trace_run_command("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)",
1699 create_or_delete_trace_kprobe);
1700 if (WARN_ON_ONCE(ret)) {
1701 pr_warn("error on probing function entry.\n");
1702 warn++;
1703 } else {
1704 /* Enable trace point */
1705 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1706 if (WARN_ON_ONCE(tk == NULL)) {
1707 pr_warn("error on getting new probe.\n");
1708 warn++;
1709 } else {
1710 file = find_trace_probe_file(tk, top_trace_array());
1711 if (WARN_ON_ONCE(file == NULL)) {
1712 pr_warn("error on getting probe file.\n");
1713 warn++;
1714 } else
1715 enable_trace_kprobe(
1716 trace_probe_event_call(&tk->tp), file);
1717 }
1718 }
1719
1720 ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target $retval",
1721 create_or_delete_trace_kprobe);
1722 if (WARN_ON_ONCE(ret)) {
1723 pr_warn("error on probing function return.\n");
1724 warn++;
1725 } else {
1726 /* Enable trace point */
1727 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1728 if (WARN_ON_ONCE(tk == NULL)) {
1729 pr_warn("error on getting 2nd new probe.\n");
1730 warn++;
1731 } else {
1732 file = find_trace_probe_file(tk, top_trace_array());
1733 if (WARN_ON_ONCE(file == NULL)) {
1734 pr_warn("error on getting probe file.\n");
1735 warn++;
1736 } else
1737 enable_trace_kprobe(
1738 trace_probe_event_call(&tk->tp), file);
1739 }
1740 }
1741
1742 if (warn)
1743 goto end;
1744
1745 ret = target(1, 2, 3, 4, 5, 6);
1746
1747 /*
1748 * Not expecting an error here, the check is only to prevent the
1749 * optimizer from removing the call to target() as otherwise there
1750 * are no side-effects and the call is never performed.
1751 */
1752 if (ret != 21)
1753 warn++;
1754
1755 /* Disable trace points before removing it */
1756 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1757 if (WARN_ON_ONCE(tk == NULL)) {
1758 pr_warn("error on getting test probe.\n");
1759 warn++;
1760 } else {
1761 if (trace_kprobe_nhit(tk) != 1) {
1762 pr_warn("incorrect number of testprobe hits\n");
1763 warn++;
1764 }
1765
1766 file = find_trace_probe_file(tk, top_trace_array());
1767 if (WARN_ON_ONCE(file == NULL)) {
1768 pr_warn("error on getting probe file.\n");
1769 warn++;
1770 } else
1771 disable_trace_kprobe(
1772 trace_probe_event_call(&tk->tp), file);
1773 }
1774
1775 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1776 if (WARN_ON_ONCE(tk == NULL)) {
1777 pr_warn("error on getting 2nd test probe.\n");
1778 warn++;
1779 } else {
1780 if (trace_kprobe_nhit(tk) != 1) {
1781 pr_warn("incorrect number of testprobe2 hits\n");
1782 warn++;
1783 }
1784
1785 file = find_trace_probe_file(tk, top_trace_array());
1786 if (WARN_ON_ONCE(file == NULL)) {
1787 pr_warn("error on getting probe file.\n");
1788 warn++;
1789 } else
1790 disable_trace_kprobe(
1791 trace_probe_event_call(&tk->tp), file);
1792 }
1793
1794 ret = trace_run_command("-:testprobe", create_or_delete_trace_kprobe);
1795 if (WARN_ON_ONCE(ret)) {
1796 pr_warn("error on deleting a probe.\n");
1797 warn++;
1798 }
1799
1800 ret = trace_run_command("-:testprobe2", create_or_delete_trace_kprobe);
1801 if (WARN_ON_ONCE(ret)) {
1802 pr_warn("error on deleting a probe.\n");
1803 warn++;
1804 }
1805
1806end:
1807 ret = dyn_events_release_all(&trace_kprobe_ops);
1808 if (WARN_ON_ONCE(ret)) {
1809 pr_warn("error on cleaning up probes.\n");
1810 warn++;
1811 }
1812 /*
1813 * Wait for the optimizer work to finish. Otherwise it might fiddle
1814 * with probes in already freed __init text.
1815 */
1816 wait_for_kprobe_optimizer();
1817 if (warn)
1818 pr_cont("NG: Some tests are failed. Please check them.\n");
1819 else
1820 pr_cont("OK\n");
1821 return 0;
1822}
1823
1824late_initcall(kprobe_trace_self_tests_init);
1825
1826#endif
Linux 6.x block layer and io_uring tree(s)