Commit | Line | Data |
---|---|---|
7491e2c4 TSV |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * event probes | |
4 | * | |
5 | * Part of this code was copied from kernel/trace/trace_kprobe.c written by | |
6 | * Masami Hiramatsu <mhiramat@kernel.org> | |
7 | * | |
8 | * Copyright (C) 2021, VMware Inc, Steven Rostedt <rostedt@goodmis.org> | |
9 | * Copyright (C) 2021, VMware Inc, Tzvetomir Stoyanov tz.stoyanov@gmail.com> | |
10 | * | |
11 | */ | |
12 | #include <linux/module.h> | |
13 | #include <linux/mutex.h> | |
14 | #include <linux/ftrace.h> | |
15 | ||
16 | #include "trace_dynevent.h" | |
17 | #include "trace_probe.h" | |
18 | #include "trace_probe_tmpl.h" | |
f1d3cbfa | 19 | #include "trace_probe_kernel.h" |
7491e2c4 TSV |
20 | |
21 | #define EPROBE_EVENT_SYSTEM "eprobes" | |
22 | ||
23 | struct trace_eprobe { | |
24 | /* tracepoint system */ | |
25 | const char *event_system; | |
26 | ||
27 | /* tracepoint event */ | |
28 | const char *event_name; | |
29 | ||
752be5c5 MHG |
30 | /* filter string for the tracepoint */ |
31 | char *filter_str; | |
32 | ||
7491e2c4 TSV |
33 | struct trace_event_call *event; |
34 | ||
35 | struct dyn_event devent; | |
36 | struct trace_probe tp; | |
37 | }; | |
38 | ||
39 | struct eprobe_data { | |
40 | struct trace_event_file *file; | |
41 | struct trace_eprobe *ep; | |
42 | }; | |
43 | ||
44 | static int __trace_eprobe_create(int argc, const char *argv[]); | |
45 | ||
46 | static void trace_event_probe_cleanup(struct trace_eprobe *ep) | |
47 | { | |
48 | if (!ep) | |
49 | return; | |
50 | trace_probe_cleanup(&ep->tp); | |
51 | kfree(ep->event_name); | |
52 | kfree(ep->event_system); | |
53 | if (ep->event) | |
54 | trace_event_put_ref(ep->event); | |
d1776c02 | 55 | kfree(ep->filter_str); |
7491e2c4 TSV |
56 | kfree(ep); |
57 | } | |
58 | ||
59 | static struct trace_eprobe *to_trace_eprobe(struct dyn_event *ev) | |
60 | { | |
61 | return container_of(ev, struct trace_eprobe, devent); | |
62 | } | |
63 | ||
64 | static int eprobe_dyn_event_create(const char *raw_command) | |
65 | { | |
66 | return trace_probe_create(raw_command, __trace_eprobe_create); | |
67 | } | |
68 | ||
69 | static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev) | |
70 | { | |
71 | struct trace_eprobe *ep = to_trace_eprobe(ev); | |
72 | int i; | |
73 | ||
74 | seq_printf(m, "e:%s/%s", trace_probe_group_name(&ep->tp), | |
75 | trace_probe_name(&ep->tp)); | |
76 | seq_printf(m, " %s.%s", ep->event_system, ep->event_name); | |
77 | ||
78 | for (i = 0; i < ep->tp.nr_args; i++) | |
79 | seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm); | |
80 | seq_putc(m, '\n'); | |
81 | ||
82 | return 0; | |
83 | } | |
84 | ||
85 | static int unregister_trace_eprobe(struct trace_eprobe *ep) | |
86 | { | |
87 | /* If other probes are on the event, just unregister eprobe */ | |
88 | if (trace_probe_has_sibling(&ep->tp)) | |
89 | goto unreg; | |
90 | ||
91 | /* Enabled event can not be unregistered */ | |
92 | if (trace_probe_is_enabled(&ep->tp)) | |
93 | return -EBUSY; | |
94 | ||
95 | /* Will fail if probe is being used by ftrace or perf */ | |
96 | if (trace_probe_unregister_event_call(&ep->tp)) | |
97 | return -EBUSY; | |
98 | ||
99 | unreg: | |
100 | dyn_event_remove(&ep->devent); | |
101 | trace_probe_unlink(&ep->tp); | |
102 | ||
103 | return 0; | |
104 | } | |
105 | ||
106 | static int eprobe_dyn_event_release(struct dyn_event *ev) | |
107 | { | |
108 | struct trace_eprobe *ep = to_trace_eprobe(ev); | |
109 | int ret = unregister_trace_eprobe(ep); | |
110 | ||
111 | if (!ret) | |
112 | trace_event_probe_cleanup(ep); | |
113 | return ret; | |
114 | } | |
115 | ||
116 | static bool eprobe_dyn_event_is_busy(struct dyn_event *ev) | |
117 | { | |
118 | struct trace_eprobe *ep = to_trace_eprobe(ev); | |
119 | ||
120 | return trace_probe_is_enabled(&ep->tp); | |
121 | } | |
122 | ||
123 | static bool eprobe_dyn_event_match(const char *system, const char *event, | |
124 | int argc, const char **argv, struct dyn_event *ev) | |
125 | { | |
126 | struct trace_eprobe *ep = to_trace_eprobe(ev); | |
7d5fda1c SRV |
127 | const char *slash; |
128 | ||
129 | /* | |
130 | * We match the following: | |
131 | * event only - match all eprobes with event name | |
132 | * system and event only - match all system/event probes | |
95c104c3 | 133 | * system only - match all system probes |
7d5fda1c SRV |
134 | * |
135 | * The below has the above satisfied with more arguments: | |
136 | * | |
137 | * attached system/event - If the arg has the system and event | |
138 | * the probe is attached to, match | |
139 | * probes with the attachment. | |
140 | * | |
141 | * If any more args are given, then it requires a full match. | |
142 | */ | |
143 | ||
144 | /* | |
145 | * If system exists, but this probe is not part of that system | |
146 | * do not match. | |
147 | */ | |
148 | if (system && strcmp(trace_probe_group_name(&ep->tp), system) != 0) | |
149 | return false; | |
150 | ||
151 | /* Must match the event name */ | |
95c104c3 | 152 | if (event[0] != '\0' && strcmp(trace_probe_name(&ep->tp), event) != 0) |
7d5fda1c SRV |
153 | return false; |
154 | ||
155 | /* No arguments match all */ | |
156 | if (argc < 1) | |
157 | return true; | |
158 | ||
159 | /* First argument is the system/event the probe is attached to */ | |
160 | ||
161 | slash = strchr(argv[0], '/'); | |
162 | if (!slash) | |
163 | slash = strchr(argv[0], '.'); | |
164 | if (!slash) | |
165 | return false; | |
166 | ||
167 | if (strncmp(ep->event_system, argv[0], slash - argv[0])) | |
168 | return false; | |
169 | if (strcmp(ep->event_name, slash + 1)) | |
170 | return false; | |
171 | ||
172 | argc--; | |
173 | argv++; | |
174 | ||
175 | /* If there are no other args, then match */ | |
176 | if (argc < 1) | |
177 | return true; | |
178 | ||
179 | return trace_probe_match_command_args(&ep->tp, argc, argv); | |
7491e2c4 TSV |
180 | } |
181 | ||
182 | static struct dyn_event_operations eprobe_dyn_event_ops = { | |
183 | .create = eprobe_dyn_event_create, | |
184 | .show = eprobe_dyn_event_show, | |
185 | .is_busy = eprobe_dyn_event_is_busy, | |
186 | .free = eprobe_dyn_event_release, | |
187 | .match = eprobe_dyn_event_match, | |
188 | }; | |
189 | ||
190 | static struct trace_eprobe *alloc_event_probe(const char *group, | |
191 | const char *this_event, | |
192 | struct trace_event_call *event, | |
193 | int nargs) | |
194 | { | |
195 | struct trace_eprobe *ep; | |
196 | const char *event_name; | |
197 | const char *sys_name; | |
198 | int ret = -ENOMEM; | |
199 | ||
200 | if (!event) | |
201 | return ERR_PTR(-ENODEV); | |
202 | ||
203 | sys_name = event->class->system; | |
204 | event_name = trace_event_name(event); | |
205 | ||
206 | ep = kzalloc(struct_size(ep, tp.args, nargs), GFP_KERNEL); | |
207 | if (!ep) { | |
5615e088 | 208 | trace_event_put_ref(event); |
7491e2c4 TSV |
209 | goto error; |
210 | } | |
211 | ep->event = event; | |
212 | ep->event_name = kstrdup(event_name, GFP_KERNEL); | |
213 | if (!ep->event_name) | |
214 | goto error; | |
215 | ep->event_system = kstrdup(sys_name, GFP_KERNEL); | |
216 | if (!ep->event_system) | |
217 | goto error; | |
218 | ||
219 | ret = trace_probe_init(&ep->tp, this_event, group, false); | |
220 | if (ret < 0) | |
221 | goto error; | |
222 | ||
223 | dyn_event_init(&ep->devent, &eprobe_dyn_event_ops); | |
224 | return ep; | |
225 | error: | |
226 | trace_event_probe_cleanup(ep); | |
227 | return ERR_PTR(ret); | |
228 | } | |
229 | ||
230 | static int trace_eprobe_tp_arg_update(struct trace_eprobe *ep, int i) | |
231 | { | |
232 | struct probe_arg *parg = &ep->tp.args[i]; | |
233 | struct ftrace_event_field *field; | |
234 | struct list_head *head; | |
6a832ec3 | 235 | int ret = -ENOENT; |
7491e2c4 TSV |
236 | |
237 | head = trace_get_fields(ep->event); | |
238 | list_for_each_entry(field, head, link) { | |
239 | if (!strcmp(parg->code->data, field->name)) { | |
240 | kfree(parg->code->data); | |
241 | parg->code->data = field; | |
242 | return 0; | |
243 | } | |
244 | } | |
6a832ec3 SRG |
245 | |
246 | /* | |
247 | * Argument not found on event. But allow for comm and COMM | |
248 | * to be used to get the current->comm. | |
249 | */ | |
250 | if (strcmp(parg->code->data, "COMM") == 0 || | |
251 | strcmp(parg->code->data, "comm") == 0) { | |
252 | parg->code->op = FETCH_OP_COMM; | |
253 | ret = 0; | |
254 | } | |
255 | ||
7491e2c4 TSV |
256 | kfree(parg->code->data); |
257 | parg->code->data = NULL; | |
6a832ec3 | 258 | return ret; |
7491e2c4 TSV |
259 | } |
260 | ||
261 | static int eprobe_event_define_fields(struct trace_event_call *event_call) | |
262 | { | |
7491e2c4 TSV |
263 | struct eprobe_trace_entry_head field; |
264 | struct trace_probe *tp; | |
265 | ||
266 | tp = trace_probe_primary_from_call(event_call); | |
267 | if (WARN_ON_ONCE(!tp)) | |
268 | return -ENOENT; | |
269 | ||
7491e2c4 TSV |
270 | return traceprobe_define_arg_fields(event_call, sizeof(field), tp); |
271 | } | |
272 | ||
273 | static struct trace_event_fields eprobe_fields_array[] = { | |
274 | { .type = TRACE_FUNCTION_TYPE, | |
275 | .define_fields = eprobe_event_define_fields }, | |
276 | {} | |
277 | }; | |
278 | ||
279 | /* Event entry printers */ | |
280 | static enum print_line_t | |
281 | print_eprobe_event(struct trace_iterator *iter, int flags, | |
282 | struct trace_event *event) | |
283 | { | |
284 | struct eprobe_trace_entry_head *field; | |
285 | struct trace_event_call *pevent; | |
286 | struct trace_event *probed_event; | |
287 | struct trace_seq *s = &iter->seq; | |
b61edd57 | 288 | struct trace_eprobe *ep; |
7491e2c4 | 289 | struct trace_probe *tp; |
b61edd57 | 290 | unsigned int type; |
7491e2c4 TSV |
291 | |
292 | field = (struct eprobe_trace_entry_head *)iter->ent; | |
293 | tp = trace_probe_primary_from_call( | |
294 | container_of(event, struct trace_event_call, event)); | |
295 | if (WARN_ON_ONCE(!tp)) | |
296 | goto out; | |
297 | ||
b61edd57 SRG |
298 | ep = container_of(tp, struct trace_eprobe, tp); |
299 | type = ep->event->event.type; | |
300 | ||
7491e2c4 TSV |
301 | trace_seq_printf(s, "%s: (", trace_probe_name(tp)); |
302 | ||
b61edd57 | 303 | probed_event = ftrace_find_event(type); |
7491e2c4 TSV |
304 | if (probed_event) { |
305 | pevent = container_of(probed_event, struct trace_event_call, event); | |
306 | trace_seq_printf(s, "%s.%s", pevent->class->system, | |
307 | trace_event_name(pevent)); | |
308 | } else { | |
b61edd57 | 309 | trace_seq_printf(s, "%u", type); |
7491e2c4 TSV |
310 | } |
311 | ||
312 | trace_seq_putc(s, ')'); | |
313 | ||
196b6389 | 314 | if (trace_probe_print_args(s, tp->args, tp->nr_args, |
7491e2c4 TSV |
315 | (u8 *)&field[1], field) < 0) |
316 | goto out; | |
317 | ||
318 | trace_seq_putc(s, '\n'); | |
319 | out: | |
320 | return trace_handle_return(s); | |
321 | } | |
322 | ||
323 | static unsigned long get_event_field(struct fetch_insn *code, void *rec) | |
324 | { | |
325 | struct ftrace_event_field *field = code->data; | |
326 | unsigned long val; | |
327 | void *addr; | |
328 | ||
329 | addr = rec + field->offset; | |
330 | ||
f04dec93 SRG |
331 | if (is_string_field(field)) { |
332 | switch (field->filter_type) { | |
333 | case FILTER_DYN_STRING: | |
334 | val = (unsigned long)(rec + (*(unsigned int *)addr & 0xffff)); | |
335 | break; | |
336 | case FILTER_RDYN_STRING: | |
337 | val = (unsigned long)(addr + (*(unsigned int *)addr & 0xffff)); | |
338 | break; | |
339 | case FILTER_STATIC_STRING: | |
340 | val = (unsigned long)addr; | |
341 | break; | |
342 | case FILTER_PTR_STRING: | |
343 | val = (unsigned long)(*(char *)addr); | |
344 | break; | |
345 | default: | |
346 | WARN_ON_ONCE(1); | |
347 | return 0; | |
348 | } | |
349 | return val; | |
350 | } | |
351 | ||
7491e2c4 TSV |
352 | switch (field->size) { |
353 | case 1: | |
354 | if (field->is_signed) | |
355 | val = *(char *)addr; | |
356 | else | |
357 | val = *(unsigned char *)addr; | |
358 | break; | |
359 | case 2: | |
360 | if (field->is_signed) | |
361 | val = *(short *)addr; | |
362 | else | |
363 | val = *(unsigned short *)addr; | |
364 | break; | |
365 | case 4: | |
366 | if (field->is_signed) | |
367 | val = *(int *)addr; | |
368 | else | |
369 | val = *(unsigned int *)addr; | |
370 | break; | |
371 | default: | |
372 | if (field->is_signed) | |
373 | val = *(long *)addr; | |
374 | else | |
375 | val = *(unsigned long *)addr; | |
376 | break; | |
377 | } | |
378 | return val; | |
379 | } | |
380 | ||
381 | static int get_eprobe_size(struct trace_probe *tp, void *rec) | |
382 | { | |
6a832ec3 | 383 | struct fetch_insn *code; |
7491e2c4 TSV |
384 | struct probe_arg *arg; |
385 | int i, len, ret = 0; | |
386 | ||
387 | for (i = 0; i < tp->nr_args; i++) { | |
388 | arg = tp->args + i; | |
6a832ec3 | 389 | if (arg->dynamic) { |
7491e2c4 TSV |
390 | unsigned long val; |
391 | ||
6a832ec3 SRG |
392 | code = arg->code; |
393 | retry: | |
394 | switch (code->op) { | |
395 | case FETCH_OP_TP_ARG: | |
396 | val = get_event_field(code, rec); | |
397 | break; | |
398 | case FETCH_OP_IMM: | |
399 | val = code->immediate; | |
400 | break; | |
401 | case FETCH_OP_COMM: | |
402 | val = (unsigned long)current->comm; | |
403 | break; | |
404 | case FETCH_OP_DATA: | |
405 | val = (unsigned long)code->data; | |
406 | break; | |
407 | case FETCH_NOP_SYMBOL: /* Ignore a place holder */ | |
408 | code++; | |
409 | goto retry; | |
410 | default: | |
411 | continue; | |
412 | } | |
413 | code++; | |
414 | len = process_fetch_insn_bottom(code, val, NULL, NULL); | |
7491e2c4 TSV |
415 | if (len > 0) |
416 | ret += len; | |
417 | } | |
418 | } | |
419 | ||
420 | return ret; | |
421 | } | |
422 | ||
423 | /* Kprobe specific fetch functions */ | |
424 | ||
425 | /* Note that we don't verify it, since the code does not come from user space */ | |
426 | static int | |
427 | process_fetch_insn(struct fetch_insn *code, void *rec, void *dest, | |
428 | void *base) | |
429 | { | |
430 | unsigned long val; | |
431 | ||
6a832ec3 SRG |
432 | retry: |
433 | switch (code->op) { | |
434 | case FETCH_OP_TP_ARG: | |
435 | val = get_event_field(code, rec); | |
436 | break; | |
437 | case FETCH_OP_IMM: | |
438 | val = code->immediate; | |
439 | break; | |
440 | case FETCH_OP_COMM: | |
441 | val = (unsigned long)current->comm; | |
442 | break; | |
443 | case FETCH_OP_DATA: | |
444 | val = (unsigned long)code->data; | |
445 | break; | |
446 | case FETCH_NOP_SYMBOL: /* Ignore a place holder */ | |
447 | code++; | |
448 | goto retry; | |
449 | default: | |
450 | return -EILSEQ; | |
451 | } | |
452 | code++; | |
453 | return process_fetch_insn_bottom(code, val, dest, base); | |
7491e2c4 TSV |
454 | } |
455 | NOKPROBE_SYMBOL(process_fetch_insn) | |
456 | ||
457 | /* Return the length of string -- including null terminal byte */ | |
458 | static nokprobe_inline int | |
459 | fetch_store_strlen_user(unsigned long addr) | |
460 | { | |
f1d3cbfa | 461 | return kern_fetch_store_strlen_user(addr); |
7491e2c4 TSV |
462 | } |
463 | ||
464 | /* Return the length of string -- including null terminal byte */ | |
465 | static nokprobe_inline int | |
466 | fetch_store_strlen(unsigned long addr) | |
467 | { | |
f1d3cbfa | 468 | return kern_fetch_store_strlen(addr); |
7491e2c4 TSV |
469 | } |
470 | ||
471 | /* | |
472 | * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf | |
473 | * with max length and relative data location. | |
474 | */ | |
475 | static nokprobe_inline int | |
476 | fetch_store_string_user(unsigned long addr, void *dest, void *base) | |
477 | { | |
f1d3cbfa | 478 | return kern_fetch_store_string_user(addr, dest, base); |
7491e2c4 TSV |
479 | } |
480 | ||
481 | /* | |
482 | * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max | |
483 | * length and relative data location. | |
484 | */ | |
485 | static nokprobe_inline int | |
486 | fetch_store_string(unsigned long addr, void *dest, void *base) | |
487 | { | |
f1d3cbfa | 488 | return kern_fetch_store_string(addr, dest, base); |
7491e2c4 TSV |
489 | } |
490 | ||
491 | static nokprobe_inline int | |
492 | probe_mem_read_user(void *dest, void *src, size_t size) | |
493 | { | |
494 | const void __user *uaddr = (__force const void __user *)src; | |
495 | ||
496 | return copy_from_user_nofault(dest, uaddr, size); | |
497 | } | |
498 | ||
499 | static nokprobe_inline int | |
500 | probe_mem_read(void *dest, void *src, size_t size) | |
501 | { | |
502 | #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE | |
503 | if ((unsigned long)src < TASK_SIZE) | |
504 | return probe_mem_read_user(dest, src, size); | |
505 | #endif | |
506 | return copy_from_kernel_nofault(dest, src, size); | |
507 | } | |
508 | ||
509 | /* eprobe handler */ | |
510 | static inline void | |
511 | __eprobe_trace_func(struct eprobe_data *edata, void *rec) | |
512 | { | |
513 | struct eprobe_trace_entry_head *entry; | |
514 | struct trace_event_call *call = trace_probe_event_call(&edata->ep->tp); | |
515 | struct trace_event_buffer fbuffer; | |
516 | int dsize; | |
517 | ||
518 | if (WARN_ON_ONCE(call != edata->file->event_call)) | |
519 | return; | |
520 | ||
521 | if (trace_trigger_soft_disabled(edata->file)) | |
522 | return; | |
523 | ||
7491e2c4 | 524 | dsize = get_eprobe_size(&edata->ep->tp, rec); |
3e8b1a29 SRV |
525 | |
526 | entry = trace_event_buffer_reserve(&fbuffer, edata->file, | |
527 | sizeof(*entry) + edata->ep->tp.size + dsize); | |
528 | ||
529 | if (!entry) | |
7491e2c4 TSV |
530 | return; |
531 | ||
532 | entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event); | |
7491e2c4 TSV |
533 | store_trace_args(&entry[1], &edata->ep->tp, rec, sizeof(*entry), dsize); |
534 | ||
535 | trace_event_buffer_commit(&fbuffer); | |
536 | } | |
537 | ||
538 | /* | |
539 | * The event probe implementation uses event triggers to get access to | |
540 | * the event it is attached to, but is not an actual trigger. The below | |
541 | * functions are just stubs to fulfill what is needed to use the trigger | |
542 | * infrastructure. | |
543 | */ | |
47670541 | 544 | static int eprobe_trigger_init(struct event_trigger_data *data) |
7491e2c4 TSV |
545 | { |
546 | return 0; | |
547 | } | |
548 | ||
47670541 | 549 | static void eprobe_trigger_free(struct event_trigger_data *data) |
7491e2c4 TSV |
550 | { |
551 | ||
552 | } | |
553 | ||
554 | static int eprobe_trigger_print(struct seq_file *m, | |
7491e2c4 TSV |
555 | struct event_trigger_data *data) |
556 | { | |
557 | /* Do not print eprobe event triggers */ | |
558 | return 0; | |
559 | } | |
560 | ||
561 | static void eprobe_trigger_func(struct event_trigger_data *data, | |
562 | struct trace_buffer *buffer, void *rec, | |
563 | struct ring_buffer_event *rbe) | |
564 | { | |
565 | struct eprobe_data *edata = data->private_data; | |
566 | ||
94eedf3d SRG |
567 | if (unlikely(!rec)) |
568 | return; | |
569 | ||
7491e2c4 TSV |
570 | __eprobe_trace_func(edata, rec); |
571 | } | |
572 | ||
573 | static struct event_trigger_ops eprobe_trigger_ops = { | |
fb339e53 | 574 | .trigger = eprobe_trigger_func, |
7491e2c4 TSV |
575 | .print = eprobe_trigger_print, |
576 | .init = eprobe_trigger_init, | |
577 | .free = eprobe_trigger_free, | |
578 | }; | |
579 | ||
9ec5a7d1 TZ |
580 | static int eprobe_trigger_cmd_parse(struct event_command *cmd_ops, |
581 | struct trace_event_file *file, | |
e1f187d0 TZ |
582 | char *glob, char *cmd, |
583 | char *param_and_filter) | |
7491e2c4 TSV |
584 | { |
585 | return -1; | |
586 | } | |
587 | ||
2378a2d6 TZ |
588 | static int eprobe_trigger_reg_func(char *glob, |
589 | struct event_trigger_data *data, | |
590 | struct trace_event_file *file) | |
7491e2c4 TSV |
591 | { |
592 | return -1; | |
593 | } | |
594 | ||
2378a2d6 TZ |
595 | static void eprobe_trigger_unreg_func(char *glob, |
596 | struct event_trigger_data *data, | |
597 | struct trace_event_file *file) | |
7491e2c4 TSV |
598 | { |
599 | ||
600 | } | |
601 | ||
602 | static struct event_trigger_ops *eprobe_trigger_get_ops(char *cmd, | |
603 | char *param) | |
604 | { | |
605 | return &eprobe_trigger_ops; | |
606 | } | |
607 | ||
608 | static struct event_command event_trigger_cmd = { | |
609 | .name = "eprobe", | |
610 | .trigger_type = ETT_EVENT_EPROBE, | |
611 | .flags = EVENT_CMD_FL_NEEDS_REC, | |
9ec5a7d1 | 612 | .parse = eprobe_trigger_cmd_parse, |
7491e2c4 TSV |
613 | .reg = eprobe_trigger_reg_func, |
614 | .unreg = eprobe_trigger_unreg_func, | |
615 | .unreg_all = NULL, | |
616 | .get_trigger_ops = eprobe_trigger_get_ops, | |
617 | .set_filter = NULL, | |
618 | }; | |
619 | ||
620 | static struct event_trigger_data * | |
621 | new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file) | |
622 | { | |
623 | struct event_trigger_data *trigger; | |
752be5c5 | 624 | struct event_filter *filter = NULL; |
7491e2c4 | 625 | struct eprobe_data *edata; |
752be5c5 | 626 | int ret; |
7491e2c4 TSV |
627 | |
628 | edata = kzalloc(sizeof(*edata), GFP_KERNEL); | |
629 | trigger = kzalloc(sizeof(*trigger), GFP_KERNEL); | |
630 | if (!trigger || !edata) { | |
752be5c5 MHG |
631 | ret = -ENOMEM; |
632 | goto error; | |
7491e2c4 TSV |
633 | } |
634 | ||
635 | trigger->flags = EVENT_TRIGGER_FL_PROBE; | |
636 | trigger->count = -1; | |
637 | trigger->ops = &eprobe_trigger_ops; | |
638 | ||
639 | /* | |
640 | * EVENT PROBE triggers are not registered as commands with | |
641 | * register_event_command(), as they are not controlled by the user | |
642 | * from the trigger file | |
643 | */ | |
644 | trigger->cmd_ops = &event_trigger_cmd; | |
645 | ||
646 | INIT_LIST_HEAD(&trigger->list); | |
752be5c5 MHG |
647 | |
648 | if (ep->filter_str) { | |
40adaf51 | 649 | ret = create_event_filter(file->tr, ep->event, |
752be5c5 MHG |
650 | ep->filter_str, false, &filter); |
651 | if (ret) | |
652 | goto error; | |
653 | } | |
654 | RCU_INIT_POINTER(trigger->filter, filter); | |
7491e2c4 TSV |
655 | |
656 | edata->file = file; | |
657 | edata->ep = ep; | |
658 | trigger->private_data = edata; | |
659 | ||
660 | return trigger; | |
752be5c5 MHG |
661 | error: |
662 | free_event_filter(filter); | |
663 | kfree(edata); | |
664 | kfree(trigger); | |
665 | return ERR_PTR(ret); | |
7491e2c4 TSV |
666 | } |
667 | ||
668 | static int enable_eprobe(struct trace_eprobe *ep, | |
669 | struct trace_event_file *eprobe_file) | |
670 | { | |
671 | struct event_trigger_data *trigger; | |
672 | struct trace_event_file *file; | |
673 | struct trace_array *tr = eprobe_file->tr; | |
674 | ||
675 | file = find_event_file(tr, ep->event_system, ep->event_name); | |
676 | if (!file) | |
677 | return -ENOENT; | |
678 | trigger = new_eprobe_trigger(ep, eprobe_file); | |
679 | if (IS_ERR(trigger)) | |
680 | return PTR_ERR(trigger); | |
681 | ||
682 | list_add_tail_rcu(&trigger->list, &file->triggers); | |
683 | ||
684 | trace_event_trigger_enable_disable(file, 1); | |
685 | update_cond_flag(file); | |
686 | ||
687 | return 0; | |
688 | } | |
689 | ||
690 | static struct trace_event_functions eprobe_funcs = { | |
691 | .trace = print_eprobe_event | |
692 | }; | |
693 | ||
694 | static int disable_eprobe(struct trace_eprobe *ep, | |
695 | struct trace_array *tr) | |
696 | { | |
ba27d855 | 697 | struct event_trigger_data *trigger = NULL, *iter; |
7491e2c4 | 698 | struct trace_event_file *file; |
752be5c5 | 699 | struct event_filter *filter; |
7491e2c4 TSV |
700 | struct eprobe_data *edata; |
701 | ||
702 | file = find_event_file(tr, ep->event_system, ep->event_name); | |
703 | if (!file) | |
704 | return -ENOENT; | |
705 | ||
ba27d855 JK |
706 | list_for_each_entry(iter, &file->triggers, list) { |
707 | if (!(iter->flags & EVENT_TRIGGER_FL_PROBE)) | |
7491e2c4 | 708 | continue; |
ba27d855 JK |
709 | edata = iter->private_data; |
710 | if (edata->ep == ep) { | |
711 | trigger = iter; | |
7491e2c4 | 712 | break; |
ba27d855 | 713 | } |
7491e2c4 | 714 | } |
ba27d855 | 715 | if (!trigger) |
7491e2c4 TSV |
716 | return -ENODEV; |
717 | ||
718 | list_del_rcu(&trigger->list); | |
719 | ||
720 | trace_event_trigger_enable_disable(file, 0); | |
721 | update_cond_flag(file); | |
6675880f VS |
722 | |
723 | /* Make sure nothing is using the edata or trigger */ | |
724 | tracepoint_synchronize_unregister(); | |
725 | ||
752be5c5 MHG |
726 | filter = rcu_access_pointer(trigger->filter); |
727 | ||
728 | if (filter) | |
729 | free_event_filter(filter); | |
6675880f VS |
730 | kfree(edata); |
731 | kfree(trigger); | |
732 | ||
7491e2c4 TSV |
733 | return 0; |
734 | } | |
735 | ||
736 | static int enable_trace_eprobe(struct trace_event_call *call, | |
737 | struct trace_event_file *file) | |
738 | { | |
739 | struct trace_probe *pos, *tp; | |
740 | struct trace_eprobe *ep; | |
741 | bool enabled; | |
742 | int ret = 0; | |
743 | ||
744 | tp = trace_probe_primary_from_call(call); | |
745 | if (WARN_ON_ONCE(!tp)) | |
746 | return -ENODEV; | |
747 | enabled = trace_probe_is_enabled(tp); | |
748 | ||
749 | /* This also changes "enabled" state */ | |
750 | if (file) { | |
751 | ret = trace_probe_add_file(tp, file); | |
752 | if (ret) | |
753 | return ret; | |
754 | } else | |
755 | trace_probe_set_flag(tp, TP_FLAG_PROFILE); | |
756 | ||
757 | if (enabled) | |
758 | return 0; | |
759 | ||
760 | list_for_each_entry(pos, trace_probe_probe_list(tp), list) { | |
761 | ep = container_of(pos, struct trace_eprobe, tp); | |
762 | ret = enable_eprobe(ep, file); | |
763 | if (ret) | |
764 | break; | |
765 | enabled = true; | |
766 | } | |
767 | ||
768 | if (ret) { | |
769 | /* Failed to enable one of them. Roll back all */ | |
770 | if (enabled) | |
771 | disable_eprobe(ep, file->tr); | |
772 | if (file) | |
773 | trace_probe_remove_file(tp, file); | |
774 | else | |
775 | trace_probe_clear_flag(tp, TP_FLAG_PROFILE); | |
776 | } | |
777 | ||
778 | return ret; | |
779 | } | |
780 | ||
781 | static int disable_trace_eprobe(struct trace_event_call *call, | |
782 | struct trace_event_file *file) | |
783 | { | |
784 | struct trace_probe *pos, *tp; | |
785 | struct trace_eprobe *ep; | |
786 | ||
787 | tp = trace_probe_primary_from_call(call); | |
788 | if (WARN_ON_ONCE(!tp)) | |
789 | return -ENODEV; | |
790 | ||
791 | if (file) { | |
792 | if (!trace_probe_get_file_link(tp, file)) | |
793 | return -ENOENT; | |
794 | if (!trace_probe_has_single_file(tp)) | |
795 | goto out; | |
796 | trace_probe_clear_flag(tp, TP_FLAG_TRACE); | |
797 | } else | |
798 | trace_probe_clear_flag(tp, TP_FLAG_PROFILE); | |
799 | ||
800 | if (!trace_probe_is_enabled(tp)) { | |
801 | list_for_each_entry(pos, trace_probe_probe_list(tp), list) { | |
802 | ep = container_of(pos, struct trace_eprobe, tp); | |
803 | disable_eprobe(ep, file->tr); | |
804 | } | |
805 | } | |
806 | ||
807 | out: | |
808 | if (file) | |
809 | /* | |
810 | * Synchronization is done in below function. For perf event, | |
811 | * file == NULL and perf_trace_event_unreg() calls | |
812 | * tracepoint_synchronize_unregister() to ensure synchronize | |
813 | * event. We don't need to care about it. | |
814 | */ | |
815 | trace_probe_remove_file(tp, file); | |
816 | ||
817 | return 0; | |
818 | } | |
819 | ||
820 | static int eprobe_register(struct trace_event_call *event, | |
821 | enum trace_reg type, void *data) | |
822 | { | |
823 | struct trace_event_file *file = data; | |
824 | ||
825 | switch (type) { | |
826 | case TRACE_REG_REGISTER: | |
827 | return enable_trace_eprobe(event, file); | |
828 | case TRACE_REG_UNREGISTER: | |
829 | return disable_trace_eprobe(event, file); | |
830 | #ifdef CONFIG_PERF_EVENTS | |
831 | case TRACE_REG_PERF_REGISTER: | |
832 | case TRACE_REG_PERF_UNREGISTER: | |
833 | case TRACE_REG_PERF_OPEN: | |
834 | case TRACE_REG_PERF_CLOSE: | |
835 | case TRACE_REG_PERF_ADD: | |
836 | case TRACE_REG_PERF_DEL: | |
837 | return 0; | |
838 | #endif | |
839 | } | |
840 | return 0; | |
841 | } | |
842 | ||
843 | static inline void init_trace_eprobe_call(struct trace_eprobe *ep) | |
844 | { | |
845 | struct trace_event_call *call = trace_probe_event_call(&ep->tp); | |
846 | ||
847 | call->flags = TRACE_EVENT_FL_EPROBE; | |
848 | call->event.funcs = &eprobe_funcs; | |
849 | call->class->fields_array = eprobe_fields_array; | |
850 | call->class->reg = eprobe_register; | |
851 | } | |
852 | ||
853 | static struct trace_event_call * | |
854 | find_and_get_event(const char *system, const char *event_name) | |
855 | { | |
856 | struct trace_event_call *tp_event; | |
857 | const char *name; | |
858 | ||
859 | list_for_each_entry(tp_event, &ftrace_events, list) { | |
860 | /* Skip other probes and ftrace events */ | |
861 | if (tp_event->flags & | |
862 | (TRACE_EVENT_FL_IGNORE_ENABLE | | |
863 | TRACE_EVENT_FL_KPROBE | | |
864 | TRACE_EVENT_FL_UPROBE | | |
865 | TRACE_EVENT_FL_EPROBE)) | |
866 | continue; | |
867 | if (!tp_event->class->system || | |
868 | strcmp(system, tp_event->class->system)) | |
869 | continue; | |
870 | name = trace_event_name(tp_event); | |
871 | if (!name || strcmp(event_name, name)) | |
872 | continue; | |
873 | if (!trace_event_try_get_ref(tp_event)) { | |
874 | return NULL; | |
875 | break; | |
876 | } | |
877 | return tp_event; | |
878 | break; | |
879 | } | |
880 | return NULL; | |
881 | } | |
882 | ||
883 | static int trace_eprobe_tp_update_arg(struct trace_eprobe *ep, const char *argv[], int i) | |
884 | { | |
885 | unsigned int flags = TPARG_FL_KERNEL | TPARG_FL_TPOINT; | |
886 | int ret; | |
887 | ||
888 | ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], flags); | |
889 | if (ret) | |
890 | return ret; | |
891 | ||
2f63e5d2 | 892 | if (ep->tp.args[i].code->op == FETCH_OP_TP_ARG) { |
7491e2c4 | 893 | ret = trace_eprobe_tp_arg_update(ep, i); |
2f63e5d2 MHG |
894 | if (ret) |
895 | trace_probe_log_err(0, BAD_ATTACH_ARG); | |
896 | } | |
7491e2c4 | 897 | |
6a832ec3 SRG |
898 | /* Handle symbols "@" */ |
899 | if (!ret) | |
900 | ret = traceprobe_update_arg(&ep->tp.args[i]); | |
901 | ||
7491e2c4 TSV |
902 | return ret; |
903 | } | |
904 | ||
752be5c5 MHG |
905 | static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const char *argv[]) |
906 | { | |
342a4a2f | 907 | struct event_filter *dummy = NULL; |
752be5c5 MHG |
908 | int i, ret, len = 0; |
909 | char *p; | |
910 | ||
911 | if (argc == 0) { | |
912 | trace_probe_log_err(0, NO_EP_FILTER); | |
913 | return -EINVAL; | |
914 | } | |
915 | ||
916 | /* Recover the filter string */ | |
917 | for (i = 0; i < argc; i++) | |
918 | len += strlen(argv[i]) + 1; | |
919 | ||
920 | ep->filter_str = kzalloc(len, GFP_KERNEL); | |
921 | if (!ep->filter_str) | |
922 | return -ENOMEM; | |
923 | ||
924 | p = ep->filter_str; | |
925 | for (i = 0; i < argc; i++) { | |
c96abaec QF |
926 | if (i) |
927 | ret = snprintf(p, len, " %s", argv[i]); | |
928 | else | |
929 | ret = snprintf(p, len, "%s", argv[i]); | |
752be5c5 MHG |
930 | p += ret; |
931 | len -= ret; | |
932 | } | |
752be5c5 MHG |
933 | |
934 | /* | |
935 | * Ensure the filter string can be parsed correctly. Note, this | |
936 | * filter string is for the original event, not for the eprobe. | |
937 | */ | |
938 | ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str, | |
939 | true, &dummy); | |
940 | free_event_filter(dummy); | |
941 | if (ret) | |
942 | goto error; | |
943 | ||
944 | return 0; | |
945 | error: | |
946 | kfree(ep->filter_str); | |
947 | ep->filter_str = NULL; | |
948 | return ret; | |
949 | } | |
950 | ||
7491e2c4 TSV |
951 | static int __trace_eprobe_create(int argc, const char *argv[]) |
952 | { | |
953 | /* | |
954 | * Argument syntax: | |
752be5c5 MHG |
955 | * e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS] [if FILTER] |
956 | * Fetch args (no space): | |
7491e2c4 TSV |
957 | * <name>=$<field>[:TYPE] |
958 | */ | |
959 | const char *event = NULL, *group = EPROBE_EVENT_SYSTEM; | |
960 | const char *sys_event = NULL, *sys_name = NULL; | |
961 | struct trace_event_call *event_call; | |
962 | struct trace_eprobe *ep = NULL; | |
963 | char buf1[MAX_EVENT_NAME_LEN]; | |
964 | char buf2[MAX_EVENT_NAME_LEN]; | |
95c104c3 | 965 | char gbuf[MAX_EVENT_NAME_LEN]; |
752be5c5 MHG |
966 | int ret = 0, filter_idx = 0; |
967 | int i, filter_cnt; | |
7491e2c4 TSV |
968 | |
969 | if (argc < 2 || argv[0][0] != 'e') | |
970 | return -ECANCELED; | |
971 | ||
972 | trace_probe_log_init("event_probe", argc, argv); | |
973 | ||
974 | event = strchr(&argv[0][1], ':'); | |
975 | if (event) { | |
976 | event++; | |
95c104c3 | 977 | ret = traceprobe_parse_event_name(&event, &group, gbuf, |
7491e2c4 TSV |
978 | event - argv[0]); |
979 | if (ret) | |
980 | goto parse_error; | |
7491e2c4 | 981 | } |
7491e2c4 | 982 | |
b774926c | 983 | trace_probe_log_set_index(1); |
7491e2c4 | 984 | sys_event = argv[1]; |
95c104c3 | 985 | ret = traceprobe_parse_event_name(&sys_event, &sys_name, buf2, 0); |
d8a64313 | 986 | if (ret || !sys_event || !sys_name) { |
2f63e5d2 | 987 | trace_probe_log_err(0, NO_EVENT_INFO); |
7491e2c4 | 988 | goto parse_error; |
2f63e5d2 | 989 | } |
7491e2c4 | 990 | |
95c104c3 | 991 | if (!event) { |
dc399ade | 992 | strscpy(buf1, sys_event, MAX_EVENT_NAME_LEN); |
95c104c3 LY |
993 | event = buf1; |
994 | } | |
995 | ||
752be5c5 MHG |
996 | for (i = 2; i < argc; i++) { |
997 | if (!strcmp(argv[i], "if")) { | |
998 | filter_idx = i + 1; | |
999 | filter_cnt = argc - filter_idx; | |
1000 | argc = i; | |
1001 | break; | |
1002 | } | |
1003 | } | |
1004 | ||
7491e2c4 TSV |
1005 | mutex_lock(&event_mutex); |
1006 | event_call = find_and_get_event(sys_name, sys_event); | |
1007 | ep = alloc_event_probe(group, event, event_call, argc - 2); | |
1008 | mutex_unlock(&event_mutex); | |
1009 | ||
1010 | if (IS_ERR(ep)) { | |
1011 | ret = PTR_ERR(ep); | |
2f63e5d2 MHG |
1012 | if (ret == -ENODEV) |
1013 | trace_probe_log_err(0, BAD_ATTACH_EVENT); | |
ddcf906f | 1014 | /* This must return -ENOMEM or missing event, else there is a bug */ |
7fa598f9 | 1015 | WARN_ON_ONCE(ret != -ENOMEM && ret != -ENODEV); |
5615e088 DC |
1016 | ep = NULL; |
1017 | goto error; | |
7491e2c4 TSV |
1018 | } |
1019 | ||
752be5c5 MHG |
1020 | if (filter_idx) { |
1021 | trace_probe_log_set_index(filter_idx); | |
1022 | ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx); | |
1023 | if (ret) | |
1024 | goto parse_error; | |
1025 | } else | |
1026 | ep->filter_str = NULL; | |
1027 | ||
7491e2c4 TSV |
1028 | argc -= 2; argv += 2; |
1029 | /* parse arguments */ | |
1030 | for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { | |
1031 | trace_probe_log_set_index(i + 2); | |
1032 | ret = trace_eprobe_tp_update_arg(ep, argv, i); | |
1033 | if (ret) | |
1034 | goto error; | |
1035 | } | |
1036 | ret = traceprobe_set_print_fmt(&ep->tp, PROBE_PRINT_EVENT); | |
1037 | if (ret < 0) | |
1038 | goto error; | |
1039 | init_trace_eprobe_call(ep); | |
1040 | mutex_lock(&event_mutex); | |
1041 | ret = trace_probe_register_event_call(&ep->tp); | |
1042 | if (ret) { | |
1043 | if (ret == -EEXIST) { | |
1044 | trace_probe_log_set_index(0); | |
1045 | trace_probe_log_err(0, EVENT_EXIST); | |
1046 | } | |
1047 | mutex_unlock(&event_mutex); | |
1048 | goto error; | |
1049 | } | |
1050 | ret = dyn_event_add(&ep->devent, &ep->tp.event->call); | |
1051 | mutex_unlock(&event_mutex); | |
1052 | return ret; | |
1053 | parse_error: | |
1054 | ret = -EINVAL; | |
1055 | error: | |
1056 | trace_event_probe_cleanup(ep); | |
1057 | return ret; | |
1058 | } | |
1059 | ||
1060 | /* | |
1061 | * Register dynevent at core_initcall. This allows kernel to setup eprobe | |
1062 | * events in postcore_initcall without tracefs. | |
1063 | */ | |
1064 | static __init int trace_events_eprobe_init_early(void) | |
1065 | { | |
1066 | int err = 0; | |
1067 | ||
1068 | err = dyn_event_register(&eprobe_dyn_event_ops); | |
1069 | if (err) | |
1070 | pr_warn("Could not register eprobe_dyn_event_ops\n"); | |
1071 | ||
1072 | return err; | |
1073 | } | |
1074 | core_initcall(trace_events_eprobe_init_early); |