Commit | Line | Data |
---|---|---|
bcea3f96 | 1 | // SPDX-License-Identifier: GPL-2.0 |
f3f096cf SD |
2 | /* |
3 | * uprobes-based tracing events | |
4 | * | |
f3f096cf SD |
5 | * Copyright (C) IBM Corporation, 2010-2012 |
6 | * Author: Srikar Dronamraju <srikar@linux.vnet.ibm.com> | |
7 | */ | |
ea6eb5e7 | 8 | #define pr_fmt(fmt) "trace_uprobe: " fmt |
f3f096cf | 9 | |
0597c49c | 10 | #include <linux/ctype.h> |
f3f096cf SD |
11 | #include <linux/module.h> |
12 | #include <linux/uaccess.h> | |
13 | #include <linux/uprobes.h> | |
14 | #include <linux/namei.h> | |
b2e902f0 | 15 | #include <linux/string.h> |
b2d09103 | 16 | #include <linux/rculist.h> |
f3f096cf | 17 | |
0597c49c | 18 | #include "trace_dynevent.h" |
f3f096cf | 19 | #include "trace_probe.h" |
53305928 | 20 | #include "trace_probe_tmpl.h" |
f3f096cf SD |
21 | |
22 | #define UPROBE_EVENT_SYSTEM "uprobes" | |
23 | ||
457d1772 ON |
24 | struct uprobe_trace_entry_head { |
25 | struct trace_entry ent; | |
26 | unsigned long vaddr[]; | |
27 | }; | |
28 | ||
29 | #define SIZEOF_TRACE_ENTRY(is_return) \ | |
30 | (sizeof(struct uprobe_trace_entry_head) + \ | |
31 | sizeof(unsigned long) * (is_return ? 2 : 1)) | |
32 | ||
33 | #define DATAOF_TRACE_ENTRY(entry, is_return) \ | |
34 | ((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return)) | |
35 | ||
736288ba ON |
36 | struct trace_uprobe_filter { |
37 | rwlock_t rwlock; | |
38 | int nr_systemwide; | |
39 | struct list_head perf_events; | |
40 | }; | |
41 | ||
0597c49c MH |
42 | static int trace_uprobe_create(int argc, const char **argv); |
43 | static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev); | |
44 | static int trace_uprobe_release(struct dyn_event *ev); | |
45 | static bool trace_uprobe_is_busy(struct dyn_event *ev); | |
46 | static bool trace_uprobe_match(const char *system, const char *event, | |
47 | struct dyn_event *ev); | |
48 | ||
49 | static struct dyn_event_operations trace_uprobe_ops = { | |
50 | .create = trace_uprobe_create, | |
51 | .show = trace_uprobe_show, | |
52 | .is_busy = trace_uprobe_is_busy, | |
53 | .free = trace_uprobe_release, | |
54 | .match = trace_uprobe_match, | |
55 | }; | |
56 | ||
f3f096cf SD |
57 | /* |
58 | * uprobe event core functions | |
59 | */ | |
f3f096cf | 60 | struct trace_uprobe { |
0597c49c | 61 | struct dyn_event devent; |
736288ba | 62 | struct trace_uprobe_filter filter; |
a932b738 | 63 | struct uprobe_consumer consumer; |
0c92c7a3 | 64 | struct path path; |
f3f096cf SD |
65 | struct inode *inode; |
66 | char *filename; | |
67 | unsigned long offset; | |
1cc33161 | 68 | unsigned long ref_ctr_offset; |
f3f096cf | 69 | unsigned long nhit; |
14577c39 | 70 | struct trace_probe tp; |
f3f096cf SD |
71 | }; |
72 | ||
0597c49c MH |
73 | static bool is_trace_uprobe(struct dyn_event *ev) |
74 | { | |
75 | return ev->ops == &trace_uprobe_ops; | |
76 | } | |
77 | ||
78 | static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev) | |
79 | { | |
80 | return container_of(ev, struct trace_uprobe, devent); | |
81 | } | |
82 | ||
83 | /** | |
84 | * for_each_trace_uprobe - iterate over the trace_uprobe list | |
85 | * @pos: the struct trace_uprobe * for each entry | |
86 | * @dpos: the struct dyn_event * to use as a loop cursor | |
87 | */ | |
88 | #define for_each_trace_uprobe(pos, dpos) \ | |
89 | for_each_dyn_event(dpos) \ | |
90 | if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos))) | |
91 | ||
14577c39 NK |
92 | #define SIZEOF_TRACE_UPROBE(n) \ |
93 | (offsetof(struct trace_uprobe, tp.args) + \ | |
f3f096cf SD |
94 | (sizeof(struct probe_arg) * (n))) |
95 | ||
96 | static int register_uprobe_event(struct trace_uprobe *tu); | |
c6c2401d | 97 | static int unregister_uprobe_event(struct trace_uprobe *tu); |
f3f096cf | 98 | |
b7e0bf34 NK |
99 | struct uprobe_dispatch_data { |
100 | struct trace_uprobe *tu; | |
101 | unsigned long bp_addr; | |
102 | }; | |
103 | ||
f3f096cf | 104 | static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs); |
c1ae5c75 ON |
105 | static int uretprobe_dispatcher(struct uprobe_consumer *con, |
106 | unsigned long func, struct pt_regs *regs); | |
f3f096cf | 107 | |
3fd996a2 NK |
108 | #ifdef CONFIG_STACK_GROWSUP |
109 | static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n) | |
110 | { | |
111 | return addr - (n * sizeof(long)); | |
112 | } | |
113 | #else | |
114 | static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n) | |
115 | { | |
116 | return addr + (n * sizeof(long)); | |
117 | } | |
118 | #endif | |
119 | ||
120 | static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n) | |
121 | { | |
122 | unsigned long ret; | |
123 | unsigned long addr = user_stack_pointer(regs); | |
124 | ||
125 | addr = adjust_stack_addr(addr, n); | |
126 | ||
127 | if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret))) | |
128 | return 0; | |
129 | ||
130 | return ret; | |
131 | } | |
132 | ||
133 | /* | |
134 | * Uprobes-specific fetch functions | |
135 | */ | |
53305928 | 136 | static nokprobe_inline int |
9b960a38 | 137 | probe_mem_read(void *dest, void *src, size_t size) |
53305928 MH |
138 | { |
139 | void __user *vaddr = (void __force __user *)src; | |
140 | ||
f3f58935 | 141 | return copy_from_user(dest, vaddr, size) ? -EFAULT : 0; |
5baaa59e | 142 | } |
5baaa59e NK |
143 | /* |
144 | * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max | |
145 | * length and relative data location. | |
146 | */ | |
9178412d MH |
147 | static nokprobe_inline int |
148 | fetch_store_string(unsigned long addr, void *dest, void *base) | |
5baaa59e NK |
149 | { |
150 | long ret; | |
9178412d MH |
151 | u32 loc = *(u32 *)dest; |
152 | int maxlen = get_loc_len(loc); | |
153 | u8 *dst = get_loc_data(dest, base); | |
5baaa59e NK |
154 | void __user *src = (void __force __user *) addr; |
155 | ||
9178412d MH |
156 | if (unlikely(!maxlen)) |
157 | return -ENOMEM; | |
5baaa59e | 158 | |
4dd537ac MH |
159 | if (addr == FETCH_TOKEN_COMM) |
160 | ret = strlcpy(dst, current->comm, maxlen); | |
161 | else | |
162 | ret = strncpy_from_user(dst, src, maxlen); | |
9178412d MH |
163 | if (ret >= 0) { |
164 | if (ret == maxlen) | |
165 | dst[ret - 1] = '\0'; | |
0722069a AZ |
166 | else |
167 | /* | |
168 | * Include the terminating null byte. In this case it | |
169 | * was copied by strncpy_from_user but not accounted | |
170 | * for in ret. | |
171 | */ | |
172 | ret++; | |
9178412d | 173 | *(u32 *)dest = make_data_loc(ret, (void *)dst - base); |
5baaa59e | 174 | } |
9178412d MH |
175 | |
176 | return ret; | |
5baaa59e NK |
177 | } |
178 | ||
53305928 | 179 | /* Return the length of string -- including null terminal byte */ |
9178412d MH |
180 | static nokprobe_inline int |
181 | fetch_store_strlen(unsigned long addr) | |
5baaa59e NK |
182 | { |
183 | int len; | |
184 | void __user *vaddr = (void __force __user *) addr; | |
185 | ||
4dd537ac MH |
186 | if (addr == FETCH_TOKEN_COMM) |
187 | len = strlen(current->comm) + 1; | |
188 | else | |
189 | len = strnlen_user(vaddr, MAX_STRING_SIZE); | |
5baaa59e | 190 | |
9178412d | 191 | return (len > MAX_STRING_SIZE) ? 0 : len; |
5baaa59e | 192 | } |
3fd996a2 | 193 | |
53305928 | 194 | static unsigned long translate_user_vaddr(unsigned long file_offset) |
b7e0bf34 NK |
195 | { |
196 | unsigned long base_addr; | |
197 | struct uprobe_dispatch_data *udd; | |
198 | ||
199 | udd = (void *) current->utask->vaddr; | |
200 | ||
201 | base_addr = udd->bp_addr - udd->tu->offset; | |
53305928 | 202 | return base_addr + file_offset; |
b7e0bf34 | 203 | } |
b7e0bf34 | 204 | |
53305928 MH |
205 | /* Note that we don't verify it, since the code does not come from user space */ |
206 | static int | |
207 | process_fetch_insn(struct fetch_insn *code, struct pt_regs *regs, void *dest, | |
9178412d | 208 | void *base) |
53305928 MH |
209 | { |
210 | unsigned long val; | |
53305928 MH |
211 | |
212 | /* 1st stage: get value from context */ | |
213 | switch (code->op) { | |
214 | case FETCH_OP_REG: | |
215 | val = regs_get_register(regs, code->param); | |
216 | break; | |
217 | case FETCH_OP_STACK: | |
218 | val = get_user_stack_nth(regs, code->param); | |
219 | break; | |
220 | case FETCH_OP_STACKP: | |
221 | val = user_stack_pointer(regs); | |
222 | break; | |
223 | case FETCH_OP_RETVAL: | |
224 | val = regs_return_value(regs); | |
225 | break; | |
226 | case FETCH_OP_IMM: | |
227 | val = code->immediate; | |
228 | break; | |
4dd537ac MH |
229 | case FETCH_OP_COMM: |
230 | val = FETCH_TOKEN_COMM; | |
231 | break; | |
53305928 MH |
232 | case FETCH_OP_FOFFS: |
233 | val = translate_user_vaddr(code->immediate); | |
234 | break; | |
235 | default: | |
236 | return -EILSEQ; | |
237 | } | |
238 | code++; | |
239 | ||
9b960a38 | 240 | return process_fetch_insn_bottom(code, val, dest, base); |
53305928 MH |
241 | } |
242 | NOKPROBE_SYMBOL(process_fetch_insn) | |
243 | ||
736288ba ON |
244 | static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter) |
245 | { | |
246 | rwlock_init(&filter->rwlock); | |
247 | filter->nr_systemwide = 0; | |
248 | INIT_LIST_HEAD(&filter->perf_events); | |
249 | } | |
250 | ||
251 | static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter) | |
252 | { | |
253 | return !filter->nr_systemwide && list_empty(&filter->perf_events); | |
254 | } | |
255 | ||
c1ae5c75 ON |
256 | static inline bool is_ret_probe(struct trace_uprobe *tu) |
257 | { | |
258 | return tu->consumer.ret_handler != NULL; | |
259 | } | |
260 | ||
0597c49c MH |
261 | static bool trace_uprobe_is_busy(struct dyn_event *ev) |
262 | { | |
263 | struct trace_uprobe *tu = to_trace_uprobe(ev); | |
264 | ||
265 | return trace_probe_is_enabled(&tu->tp); | |
266 | } | |
267 | ||
268 | static bool trace_uprobe_match(const char *system, const char *event, | |
269 | struct dyn_event *ev) | |
270 | { | |
271 | struct trace_uprobe *tu = to_trace_uprobe(ev); | |
272 | ||
273 | return strcmp(trace_event_name(&tu->tp.call), event) == 0 && | |
274 | (!system || strcmp(tu->tp.call.class->system, system) == 0); | |
275 | } | |
276 | ||
f3f096cf SD |
277 | /* |
278 | * Allocate new trace_uprobe and initialize it (including uprobes). | |
279 | */ | |
280 | static struct trace_uprobe * | |
c1ae5c75 | 281 | alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret) |
f3f096cf SD |
282 | { |
283 | struct trace_uprobe *tu; | |
284 | ||
5b7a9622 | 285 | if (!event || !group) |
f3f096cf SD |
286 | return ERR_PTR(-EINVAL); |
287 | ||
288 | tu = kzalloc(SIZEOF_TRACE_UPROBE(nargs), GFP_KERNEL); | |
289 | if (!tu) | |
290 | return ERR_PTR(-ENOMEM); | |
291 | ||
14577c39 NK |
292 | tu->tp.call.class = &tu->tp.class; |
293 | tu->tp.call.name = kstrdup(event, GFP_KERNEL); | |
294 | if (!tu->tp.call.name) | |
f3f096cf SD |
295 | goto error; |
296 | ||
14577c39 NK |
297 | tu->tp.class.system = kstrdup(group, GFP_KERNEL); |
298 | if (!tu->tp.class.system) | |
f3f096cf SD |
299 | goto error; |
300 | ||
0597c49c | 301 | dyn_event_init(&tu->devent, &trace_uprobe_ops); |
70ed91c6 | 302 | INIT_LIST_HEAD(&tu->tp.files); |
a932b738 | 303 | tu->consumer.handler = uprobe_dispatcher; |
c1ae5c75 ON |
304 | if (is_ret) |
305 | tu->consumer.ret_handler = uretprobe_dispatcher; | |
736288ba | 306 | init_trace_uprobe_filter(&tu->filter); |
f3f096cf SD |
307 | return tu; |
308 | ||
309 | error: | |
14577c39 | 310 | kfree(tu->tp.call.name); |
f3f096cf SD |
311 | kfree(tu); |
312 | ||
313 | return ERR_PTR(-ENOMEM); | |
314 | } | |
315 | ||
316 | static void free_trace_uprobe(struct trace_uprobe *tu) | |
317 | { | |
318 | int i; | |
319 | ||
0597c49c MH |
320 | if (!tu) |
321 | return; | |
322 | ||
14577c39 NK |
323 | for (i = 0; i < tu->tp.nr_args; i++) |
324 | traceprobe_free_probe_arg(&tu->tp.args[i]); | |
f3f096cf | 325 | |
0c92c7a3 | 326 | path_put(&tu->path); |
14577c39 NK |
327 | kfree(tu->tp.call.class->system); |
328 | kfree(tu->tp.call.name); | |
f3f096cf SD |
329 | kfree(tu->filename); |
330 | kfree(tu); | |
331 | } | |
332 | ||
333 | static struct trace_uprobe *find_probe_event(const char *event, const char *group) | |
334 | { | |
0597c49c | 335 | struct dyn_event *pos; |
f3f096cf SD |
336 | struct trace_uprobe *tu; |
337 | ||
0597c49c | 338 | for_each_trace_uprobe(tu, pos) |
687fcc4a | 339 | if (strcmp(trace_event_name(&tu->tp.call), event) == 0 && |
14577c39 | 340 | strcmp(tu->tp.call.class->system, group) == 0) |
f3f096cf SD |
341 | return tu; |
342 | ||
343 | return NULL; | |
344 | } | |
345 | ||
0597c49c | 346 | /* Unregister a trace_uprobe and probe_event */ |
c6c2401d | 347 | static int unregister_trace_uprobe(struct trace_uprobe *tu) |
f3f096cf | 348 | { |
c6c2401d SRRH |
349 | int ret; |
350 | ||
351 | ret = unregister_uprobe_event(tu); | |
352 | if (ret) | |
353 | return ret; | |
354 | ||
0597c49c | 355 | dyn_event_remove(&tu->devent); |
f3f096cf | 356 | free_trace_uprobe(tu); |
c6c2401d | 357 | return 0; |
f3f096cf SD |
358 | } |
359 | ||
ccea8727 RB |
360 | /* |
361 | * Uprobe with multiple reference counter is not allowed. i.e. | |
362 | * If inode and offset matches, reference counter offset *must* | |
363 | * match as well. Though, there is one exception: If user is | |
364 | * replacing old trace_uprobe with new one(same group/event), | |
365 | * then we allow same uprobe with new reference counter as far | |
366 | * as the new one does not conflict with any other existing | |
367 | * ones. | |
368 | */ | |
369 | static struct trace_uprobe *find_old_trace_uprobe(struct trace_uprobe *new) | |
370 | { | |
0597c49c | 371 | struct dyn_event *pos; |
ccea8727 RB |
372 | struct trace_uprobe *tmp, *old = NULL; |
373 | struct inode *new_inode = d_real_inode(new->path.dentry); | |
374 | ||
375 | old = find_probe_event(trace_event_name(&new->tp.call), | |
376 | new->tp.call.class->system); | |
377 | ||
0597c49c | 378 | for_each_trace_uprobe(tmp, pos) { |
ccea8727 RB |
379 | if ((old ? old != tmp : true) && |
380 | new_inode == d_real_inode(tmp->path.dentry) && | |
381 | new->offset == tmp->offset && | |
382 | new->ref_ctr_offset != tmp->ref_ctr_offset) { | |
383 | pr_warn("Reference counter offset mismatch."); | |
384 | return ERR_PTR(-EINVAL); | |
385 | } | |
386 | } | |
387 | return old; | |
388 | } | |
389 | ||
f3f096cf SD |
390 | /* Register a trace_uprobe and probe_event */ |
391 | static int register_trace_uprobe(struct trace_uprobe *tu) | |
392 | { | |
14577c39 | 393 | struct trace_uprobe *old_tu; |
f3f096cf SD |
394 | int ret; |
395 | ||
0597c49c | 396 | mutex_lock(&event_mutex); |
f3f096cf SD |
397 | |
398 | /* register as an event */ | |
ccea8727 RB |
399 | old_tu = find_old_trace_uprobe(tu); |
400 | if (IS_ERR(old_tu)) { | |
401 | ret = PTR_ERR(old_tu); | |
402 | goto end; | |
403 | } | |
404 | ||
14577c39 | 405 | if (old_tu) { |
f3f096cf | 406 | /* delete old event */ |
14577c39 | 407 | ret = unregister_trace_uprobe(old_tu); |
c6c2401d SRRH |
408 | if (ret) |
409 | goto end; | |
410 | } | |
f3f096cf SD |
411 | |
412 | ret = register_uprobe_event(tu); | |
413 | if (ret) { | |
a395d6a7 | 414 | pr_warn("Failed to register probe event(%d)\n", ret); |
f3f096cf SD |
415 | goto end; |
416 | } | |
417 | ||
0597c49c | 418 | dyn_event_add(&tu->devent); |
f3f096cf SD |
419 | |
420 | end: | |
0597c49c | 421 | mutex_unlock(&event_mutex); |
f3f096cf SD |
422 | |
423 | return ret; | |
424 | } | |
425 | ||
426 | /* | |
427 | * Argument syntax: | |
306cfe20 | 428 | * - Add uprobe: p|r[:[GRP/]EVENT] PATH:OFFSET [FETCHARGS] |
f3f096cf | 429 | */ |
0597c49c | 430 | static int trace_uprobe_create(int argc, const char **argv) |
f3f096cf SD |
431 | { |
432 | struct trace_uprobe *tu; | |
0597c49c MH |
433 | const char *event = NULL, *group = UPROBE_EVENT_SYSTEM; |
434 | char *arg, *filename, *rctr, *rctr_end, *tmp; | |
f3f096cf SD |
435 | char buf[MAX_EVENT_NAME_LEN]; |
436 | struct path path; | |
1cc33161 | 437 | unsigned long offset, ref_ctr_offset; |
0597c49c | 438 | bool is_return = false; |
f3f096cf SD |
439 | int i, ret; |
440 | ||
f3f096cf | 441 | ret = 0; |
1cc33161 | 442 | ref_ctr_offset = 0; |
f3f096cf | 443 | |
f01098c7 ET |
444 | switch (argv[0][0]) { |
445 | case 'r': | |
4ee5a52e | 446 | is_return = true; |
f01098c7 ET |
447 | break; |
448 | case 'p': | |
449 | break; | |
450 | default: | |
451 | return -ECANCELED; | |
452 | } | |
453 | ||
454 | if (argc < 2) | |
0597c49c | 455 | return -ECANCELED; |
f3f096cf | 456 | |
0597c49c | 457 | if (argv[0][1] == ':') |
f3f096cf | 458 | event = &argv[0][2]; |
f3f096cf | 459 | |
0597c49c MH |
460 | if (!strchr(argv[1], '/')) |
461 | return -ECANCELED; | |
f3f096cf | 462 | |
0597c49c MH |
463 | filename = kstrdup(argv[1], GFP_KERNEL); |
464 | if (!filename) | |
465 | return -ENOMEM; | |
f3f096cf | 466 | |
6496bb72 | 467 | /* Find the last occurrence, in case the path contains ':' too. */ |
0597c49c MH |
468 | arg = strrchr(filename, ':'); |
469 | if (!arg || !isdigit(arg[1])) { | |
470 | kfree(filename); | |
471 | return -ECANCELED; | |
472 | } | |
f3f096cf | 473 | |
ab105a4f MH |
474 | trace_probe_log_init("trace_uprobe", argc, argv); |
475 | trace_probe_log_set_index(1); /* filename is the 2nd argument */ | |
476 | ||
f3f096cf | 477 | *arg++ = '\0'; |
f3f096cf | 478 | ret = kern_path(filename, LOOKUP_FOLLOW, &path); |
0597c49c | 479 | if (ret) { |
ab105a4f | 480 | trace_probe_log_err(0, FILE_NOT_FOUND); |
0597c49c | 481 | kfree(filename); |
ab105a4f | 482 | trace_probe_log_clear(); |
0c92c7a3 | 483 | return ret; |
0597c49c | 484 | } |
0c92c7a3 | 485 | if (!d_is_reg(path.dentry)) { |
ab105a4f | 486 | trace_probe_log_err(0, NO_REGULAR_FILE); |
d24d7dbf JZ |
487 | ret = -EINVAL; |
488 | goto fail_address_parse; | |
489 | } | |
f3f096cf | 490 | |
1cc33161 RB |
491 | /* Parse reference counter offset if specified. */ |
492 | rctr = strchr(arg, '('); | |
493 | if (rctr) { | |
494 | rctr_end = strchr(rctr, ')'); | |
ab105a4f MH |
495 | if (!rctr_end) { |
496 | ret = -EINVAL; | |
497 | rctr_end = rctr + strlen(rctr); | |
498 | trace_probe_log_err(rctr_end - filename, | |
499 | REFCNT_OPEN_BRACE); | |
500 | goto fail_address_parse; | |
501 | } else if (rctr_end[1] != '\0') { | |
1cc33161 | 502 | ret = -EINVAL; |
ab105a4f MH |
503 | trace_probe_log_err(rctr_end + 1 - filename, |
504 | BAD_REFCNT_SUFFIX); | |
1cc33161 RB |
505 | goto fail_address_parse; |
506 | } | |
507 | ||
508 | *rctr++ = '\0'; | |
509 | *rctr_end = '\0'; | |
510 | ret = kstrtoul(rctr, 0, &ref_ctr_offset); | |
511 | if (ret) { | |
ab105a4f | 512 | trace_probe_log_err(rctr - filename, BAD_REFCNT); |
1cc33161 RB |
513 | goto fail_address_parse; |
514 | } | |
515 | } | |
516 | ||
517 | /* Parse uprobe offset. */ | |
84d7ed79 | 518 | ret = kstrtoul(arg, 0, &offset); |
ab105a4f MH |
519 | if (ret) { |
520 | trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS); | |
84d7ed79 | 521 | goto fail_address_parse; |
ab105a4f | 522 | } |
f3f096cf SD |
523 | |
524 | /* setup a probe */ | |
ab105a4f | 525 | trace_probe_log_set_index(0); |
0597c49c | 526 | if (event) { |
ab105a4f MH |
527 | ret = traceprobe_parse_event_name(&event, &group, buf, |
528 | event - argv[0]); | |
0597c49c MH |
529 | if (ret) |
530 | goto fail_address_parse; | |
531 | } else { | |
b2e902f0 | 532 | char *tail; |
f3f096cf SD |
533 | char *ptr; |
534 | ||
b2e902f0 AS |
535 | tail = kstrdup(kbasename(filename), GFP_KERNEL); |
536 | if (!tail) { | |
f3f096cf SD |
537 | ret = -ENOMEM; |
538 | goto fail_address_parse; | |
539 | } | |
540 | ||
f3f096cf SD |
541 | ptr = strpbrk(tail, ".-_"); |
542 | if (ptr) | |
543 | *ptr = '\0'; | |
544 | ||
545 | snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset); | |
546 | event = buf; | |
547 | kfree(tail); | |
548 | } | |
549 | ||
ab105a4f MH |
550 | argc -= 2; |
551 | argv += 2; | |
552 | ||
4ee5a52e | 553 | tu = alloc_trace_uprobe(group, event, argc, is_return); |
f3f096cf | 554 | if (IS_ERR(tu)) { |
f3f096cf | 555 | ret = PTR_ERR(tu); |
a039480e MH |
556 | /* This must return -ENOMEM otherwise there is a bug */ |
557 | WARN_ON_ONCE(ret != -ENOMEM); | |
f3f096cf SD |
558 | goto fail_address_parse; |
559 | } | |
560 | tu->offset = offset; | |
1cc33161 | 561 | tu->ref_ctr_offset = ref_ctr_offset; |
0c92c7a3 | 562 | tu->path = path; |
0597c49c | 563 | tu->filename = filename; |
f3f096cf SD |
564 | |
565 | /* parse arguments */ | |
f3f096cf | 566 | for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { |
0597c49c MH |
567 | tmp = kstrdup(argv[i], GFP_KERNEL); |
568 | if (!tmp) { | |
569 | ret = -ENOMEM; | |
570 | goto error; | |
571 | } | |
572 | ||
ab105a4f | 573 | trace_probe_log_set_index(i + 2); |
0597c49c | 574 | ret = traceprobe_parse_probe_arg(&tu->tp, i, tmp, |
a1303af5 | 575 | is_return ? TPARG_FL_RETURN : 0); |
0597c49c | 576 | kfree(tmp); |
d00bbea9 | 577 | if (ret) |
f3f096cf | 578 | goto error; |
f3f096cf SD |
579 | } |
580 | ||
581 | ret = register_trace_uprobe(tu); | |
ab105a4f MH |
582 | if (!ret) |
583 | goto out; | |
f3f096cf SD |
584 | |
585 | error: | |
586 | free_trace_uprobe(tu); | |
ab105a4f MH |
587 | out: |
588 | trace_probe_log_clear(); | |
f3f096cf SD |
589 | return ret; |
590 | ||
591 | fail_address_parse: | |
ab105a4f | 592 | trace_probe_log_clear(); |
0c92c7a3 | 593 | path_put(&path); |
0597c49c | 594 | kfree(filename); |
f3f096cf | 595 | |
f3f096cf SD |
596 | return ret; |
597 | } | |
598 | ||
0597c49c | 599 | static int create_or_delete_trace_uprobe(int argc, char **argv) |
f3f096cf | 600 | { |
0597c49c | 601 | int ret; |
f3f096cf | 602 | |
0597c49c MH |
603 | if (argv[0][0] == '-') |
604 | return dyn_event_release(argc, argv, &trace_uprobe_ops); | |
f3f096cf | 605 | |
0597c49c MH |
606 | ret = trace_uprobe_create(argc, (const char **)argv); |
607 | return ret == -ECANCELED ? -EINVAL : ret; | |
f3f096cf SD |
608 | } |
609 | ||
0597c49c | 610 | static int trace_uprobe_release(struct dyn_event *ev) |
f3f096cf | 611 | { |
0597c49c | 612 | struct trace_uprobe *tu = to_trace_uprobe(ev); |
f3f096cf | 613 | |
0597c49c | 614 | return unregister_trace_uprobe(tu); |
f3f096cf SD |
615 | } |
616 | ||
0597c49c MH |
617 | /* Probes listing interfaces */ |
618 | static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev) | |
f3f096cf | 619 | { |
0597c49c | 620 | struct trace_uprobe *tu = to_trace_uprobe(ev); |
3ede82dd | 621 | char c = is_ret_probe(tu) ? 'r' : 'p'; |
f3f096cf SD |
622 | int i; |
623 | ||
a64b2c01 RB |
624 | seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, tu->tp.call.class->system, |
625 | trace_event_name(&tu->tp.call), tu->filename, | |
626 | (int)(sizeof(void *) * 2), tu->offset); | |
f3f096cf | 627 | |
1cc33161 RB |
628 | if (tu->ref_ctr_offset) |
629 | seq_printf(m, "(0x%lx)", tu->ref_ctr_offset); | |
630 | ||
14577c39 NK |
631 | for (i = 0; i < tu->tp.nr_args; i++) |
632 | seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm); | |
f3f096cf | 633 | |
fa6f0cc7 | 634 | seq_putc(m, '\n'); |
f3f096cf SD |
635 | return 0; |
636 | } | |
637 | ||
0597c49c MH |
638 | static int probes_seq_show(struct seq_file *m, void *v) |
639 | { | |
640 | struct dyn_event *ev = v; | |
641 | ||
642 | if (!is_trace_uprobe(ev)) | |
643 | return 0; | |
644 | ||
645 | return trace_uprobe_show(m, ev); | |
646 | } | |
647 | ||
f3f096cf | 648 | static const struct seq_operations probes_seq_op = { |
0597c49c MH |
649 | .start = dyn_event_seq_start, |
650 | .next = dyn_event_seq_next, | |
651 | .stop = dyn_event_seq_stop, | |
652 | .show = probes_seq_show | |
f3f096cf SD |
653 | }; |
654 | ||
655 | static int probes_open(struct inode *inode, struct file *file) | |
656 | { | |
c6c2401d SRRH |
657 | int ret; |
658 | ||
659 | if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { | |
0597c49c | 660 | ret = dyn_events_release_all(&trace_uprobe_ops); |
c6c2401d SRRH |
661 | if (ret) |
662 | return ret; | |
663 | } | |
f3f096cf SD |
664 | |
665 | return seq_open(file, &probes_seq_op); | |
666 | } | |
667 | ||
668 | static ssize_t probes_write(struct file *file, const char __user *buffer, | |
669 | size_t count, loff_t *ppos) | |
670 | { | |
0597c49c MH |
671 | return trace_parse_run_command(file, buffer, count, ppos, |
672 | create_or_delete_trace_uprobe); | |
f3f096cf SD |
673 | } |
674 | ||
675 | static const struct file_operations uprobe_events_ops = { | |
676 | .owner = THIS_MODULE, | |
677 | .open = probes_open, | |
678 | .read = seq_read, | |
679 | .llseek = seq_lseek, | |
680 | .release = seq_release, | |
681 | .write = probes_write, | |
682 | }; | |
683 | ||
684 | /* Probes profiling interfaces */ | |
685 | static int probes_profile_seq_show(struct seq_file *m, void *v) | |
686 | { | |
0597c49c MH |
687 | struct dyn_event *ev = v; |
688 | struct trace_uprobe *tu; | |
689 | ||
690 | if (!is_trace_uprobe(ev)) | |
691 | return 0; | |
f3f096cf | 692 | |
0597c49c | 693 | tu = to_trace_uprobe(ev); |
de7b2973 | 694 | seq_printf(m, " %s %-44s %15lu\n", tu->filename, |
687fcc4a | 695 | trace_event_name(&tu->tp.call), tu->nhit); |
f3f096cf SD |
696 | return 0; |
697 | } | |
698 | ||
699 | static const struct seq_operations profile_seq_op = { | |
0597c49c MH |
700 | .start = dyn_event_seq_start, |
701 | .next = dyn_event_seq_next, | |
702 | .stop = dyn_event_seq_stop, | |
f3f096cf SD |
703 | .show = probes_profile_seq_show |
704 | }; | |
705 | ||
706 | static int profile_open(struct inode *inode, struct file *file) | |
707 | { | |
708 | return seq_open(file, &profile_seq_op); | |
709 | } | |
710 | ||
711 | static const struct file_operations uprobe_profile_ops = { | |
712 | .owner = THIS_MODULE, | |
713 | .open = profile_open, | |
714 | .read = seq_read, | |
715 | .llseek = seq_lseek, | |
716 | .release = seq_release, | |
717 | }; | |
718 | ||
dcad1a20 NK |
719 | struct uprobe_cpu_buffer { |
720 | struct mutex mutex; | |
721 | void *buf; | |
722 | }; | |
723 | static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer; | |
724 | static int uprobe_buffer_refcnt; | |
725 | ||
726 | static int uprobe_buffer_init(void) | |
727 | { | |
728 | int cpu, err_cpu; | |
729 | ||
730 | uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer); | |
731 | if (uprobe_cpu_buffer == NULL) | |
732 | return -ENOMEM; | |
733 | ||
734 | for_each_possible_cpu(cpu) { | |
735 | struct page *p = alloc_pages_node(cpu_to_node(cpu), | |
736 | GFP_KERNEL, 0); | |
737 | if (p == NULL) { | |
738 | err_cpu = cpu; | |
739 | goto err; | |
740 | } | |
741 | per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p); | |
742 | mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex); | |
743 | } | |
744 | ||
745 | return 0; | |
746 | ||
747 | err: | |
748 | for_each_possible_cpu(cpu) { | |
749 | if (cpu == err_cpu) | |
750 | break; | |
751 | free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf); | |
752 | } | |
753 | ||
754 | free_percpu(uprobe_cpu_buffer); | |
755 | return -ENOMEM; | |
756 | } | |
757 | ||
758 | static int uprobe_buffer_enable(void) | |
759 | { | |
760 | int ret = 0; | |
761 | ||
762 | BUG_ON(!mutex_is_locked(&event_mutex)); | |
763 | ||
764 | if (uprobe_buffer_refcnt++ == 0) { | |
765 | ret = uprobe_buffer_init(); | |
766 | if (ret < 0) | |
767 | uprobe_buffer_refcnt--; | |
768 | } | |
769 | ||
770 | return ret; | |
771 | } | |
772 | ||
773 | static void uprobe_buffer_disable(void) | |
774 | { | |
6ea6215f J |
775 | int cpu; |
776 | ||
dcad1a20 NK |
777 | BUG_ON(!mutex_is_locked(&event_mutex)); |
778 | ||
779 | if (--uprobe_buffer_refcnt == 0) { | |
6ea6215f J |
780 | for_each_possible_cpu(cpu) |
781 | free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, | |
782 | cpu)->buf); | |
783 | ||
dcad1a20 NK |
784 | free_percpu(uprobe_cpu_buffer); |
785 | uprobe_cpu_buffer = NULL; | |
786 | } | |
787 | } | |
788 | ||
789 | static struct uprobe_cpu_buffer *uprobe_buffer_get(void) | |
790 | { | |
791 | struct uprobe_cpu_buffer *ucb; | |
792 | int cpu; | |
793 | ||
794 | cpu = raw_smp_processor_id(); | |
795 | ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu); | |
796 | ||
797 | /* | |
798 | * Use per-cpu buffers for fastest access, but we might migrate | |
799 | * so the mutex makes sure we have sole access to it. | |
800 | */ | |
801 | mutex_lock(&ucb->mutex); | |
802 | ||
803 | return ucb; | |
804 | } | |
805 | ||
806 | static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb) | |
807 | { | |
808 | mutex_unlock(&ucb->mutex); | |
809 | } | |
810 | ||
a43b9704 | 811 | static void __uprobe_trace_func(struct trace_uprobe *tu, |
dd9fa555 | 812 | unsigned long func, struct pt_regs *regs, |
70ed91c6 | 813 | struct uprobe_cpu_buffer *ucb, int dsize, |
7f1d2f82 | 814 | struct trace_event_file *trace_file) |
f3f096cf SD |
815 | { |
816 | struct uprobe_trace_entry_head *entry; | |
817 | struct ring_buffer_event *event; | |
818 | struct ring_buffer *buffer; | |
457d1772 | 819 | void *data; |
dd9fa555 | 820 | int size, esize; |
2425bcb9 | 821 | struct trace_event_call *call = &tu->tp.call; |
f3f096cf | 822 | |
7f1d2f82 | 823 | WARN_ON(call != trace_file->event_call); |
70ed91c6 | 824 | |
dd9fa555 | 825 | if (WARN_ON_ONCE(tu->tp.size + dsize > PAGE_SIZE)) |
dcad1a20 NK |
826 | return; |
827 | ||
09a5059a | 828 | if (trace_trigger_soft_disabled(trace_file)) |
ca3b1620 NK |
829 | return; |
830 | ||
dd9fa555 | 831 | esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); |
dcad1a20 | 832 | size = esize + tu->tp.size + dsize; |
7f1d2f82 | 833 | event = trace_event_buffer_lock_reserve(&buffer, trace_file, |
70ed91c6 | 834 | call->event.type, size, 0, 0); |
f3f096cf | 835 | if (!event) |
dd9fa555 | 836 | return; |
f3f096cf SD |
837 | |
838 | entry = ring_buffer_event_data(event); | |
393a736c ON |
839 | if (is_ret_probe(tu)) { |
840 | entry->vaddr[0] = func; | |
841 | entry->vaddr[1] = instruction_pointer(regs); | |
842 | data = DATAOF_TRACE_ENTRY(entry, true); | |
843 | } else { | |
844 | entry->vaddr[0] = instruction_pointer(regs); | |
845 | data = DATAOF_TRACE_ENTRY(entry, false); | |
846 | } | |
847 | ||
dcad1a20 | 848 | memcpy(data, ucb->buf, tu->tp.size + dsize); |
f3f096cf | 849 | |
7f1d2f82 | 850 | event_trigger_unlock_commit(trace_file, buffer, event, entry, 0, 0); |
a51cc604 | 851 | } |
f42d24a1 | 852 | |
a51cc604 | 853 | /* uprobe handler */ |
dd9fa555 NK |
854 | static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs, |
855 | struct uprobe_cpu_buffer *ucb, int dsize) | |
a51cc604 | 856 | { |
70ed91c6 J |
857 | struct event_file_link *link; |
858 | ||
859 | if (is_ret_probe(tu)) | |
860 | return 0; | |
861 | ||
862 | rcu_read_lock(); | |
863 | list_for_each_entry_rcu(link, &tu->tp.files, list) | |
864 | __uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file); | |
865 | rcu_read_unlock(); | |
866 | ||
f42d24a1 | 867 | return 0; |
f3f096cf SD |
868 | } |
869 | ||
c1ae5c75 | 870 | static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func, |
dd9fa555 NK |
871 | struct pt_regs *regs, |
872 | struct uprobe_cpu_buffer *ucb, int dsize) | |
c1ae5c75 | 873 | { |
70ed91c6 J |
874 | struct event_file_link *link; |
875 | ||
876 | rcu_read_lock(); | |
877 | list_for_each_entry_rcu(link, &tu->tp.files, list) | |
878 | __uprobe_trace_func(tu, func, regs, ucb, dsize, link->file); | |
879 | rcu_read_unlock(); | |
c1ae5c75 ON |
880 | } |
881 | ||
f3f096cf SD |
882 | /* Event entry printers */ |
883 | static enum print_line_t | |
884 | print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event) | |
885 | { | |
457d1772 | 886 | struct uprobe_trace_entry_head *entry; |
f3f096cf SD |
887 | struct trace_seq *s = &iter->seq; |
888 | struct trace_uprobe *tu; | |
889 | u8 *data; | |
f3f096cf | 890 | |
457d1772 | 891 | entry = (struct uprobe_trace_entry_head *)iter->ent; |
14577c39 | 892 | tu = container_of(event, struct trace_uprobe, tp.call.event); |
f3f096cf | 893 | |
3ede82dd | 894 | if (is_ret_probe(tu)) { |
8579a107 | 895 | trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)", |
687fcc4a | 896 | trace_event_name(&tu->tp.call), |
8579a107 | 897 | entry->vaddr[1], entry->vaddr[0]); |
3ede82dd ON |
898 | data = DATAOF_TRACE_ENTRY(entry, true); |
899 | } else { | |
8579a107 | 900 | trace_seq_printf(s, "%s: (0x%lx)", |
687fcc4a | 901 | trace_event_name(&tu->tp.call), |
8579a107 | 902 | entry->vaddr[0]); |
3ede82dd ON |
903 | data = DATAOF_TRACE_ENTRY(entry, false); |
904 | } | |
f3f096cf | 905 | |
56de7630 MH |
906 | if (print_probe_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0) |
907 | goto out; | |
f3f096cf | 908 | |
8579a107 | 909 | trace_seq_putc(s, '\n'); |
f3f096cf | 910 | |
8579a107 SRRH |
911 | out: |
912 | return trace_handle_return(s); | |
f3f096cf SD |
913 | } |
914 | ||
31ba3348 ON |
915 | typedef bool (*filter_func_t)(struct uprobe_consumer *self, |
916 | enum uprobe_filter_ctx ctx, | |
917 | struct mm_struct *mm); | |
918 | ||
919 | static int | |
7f1d2f82 | 920 | probe_event_enable(struct trace_uprobe *tu, struct trace_event_file *file, |
70ed91c6 | 921 | filter_func_t filter) |
f3f096cf | 922 | { |
70ed91c6 J |
923 | bool enabled = trace_probe_is_enabled(&tu->tp); |
924 | struct event_file_link *link = NULL; | |
925 | int ret; | |
926 | ||
927 | if (file) { | |
48212542 ON |
928 | if (tu->tp.flags & TP_FLAG_PROFILE) |
929 | return -EINTR; | |
930 | ||
70ed91c6 J |
931 | link = kmalloc(sizeof(*link), GFP_KERNEL); |
932 | if (!link) | |
933 | return -ENOMEM; | |
934 | ||
935 | link->file = file; | |
936 | list_add_tail_rcu(&link->list, &tu->tp.files); | |
937 | ||
938 | tu->tp.flags |= TP_FLAG_TRACE; | |
48212542 ON |
939 | } else { |
940 | if (tu->tp.flags & TP_FLAG_TRACE) | |
941 | return -EINTR; | |
942 | ||
70ed91c6 | 943 | tu->tp.flags |= TP_FLAG_PROFILE; |
48212542 | 944 | } |
f3f096cf | 945 | |
736288ba ON |
946 | WARN_ON(!uprobe_filter_is_empty(&tu->filter)); |
947 | ||
70ed91c6 J |
948 | if (enabled) |
949 | return 0; | |
950 | ||
fb6bab6a ON |
951 | ret = uprobe_buffer_enable(); |
952 | if (ret) | |
953 | goto err_flags; | |
954 | ||
31ba3348 | 955 | tu->consumer.filter = filter; |
0c92c7a3 | 956 | tu->inode = d_real_inode(tu->path.dentry); |
1cc33161 RB |
957 | if (tu->ref_ctr_offset) { |
958 | ret = uprobe_register_refctr(tu->inode, tu->offset, | |
959 | tu->ref_ctr_offset, &tu->consumer); | |
960 | } else { | |
961 | ret = uprobe_register(tu->inode, tu->offset, &tu->consumer); | |
962 | } | |
963 | ||
fb6bab6a ON |
964 | if (ret) |
965 | goto err_buffer; | |
966 | ||
967 | return 0; | |
968 | ||
969 | err_buffer: | |
970 | uprobe_buffer_disable(); | |
f3f096cf | 971 | |
fb6bab6a ON |
972 | err_flags: |
973 | if (file) { | |
974 | list_del(&link->list); | |
975 | kfree(link); | |
976 | tu->tp.flags &= ~TP_FLAG_TRACE; | |
977 | } else { | |
978 | tu->tp.flags &= ~TP_FLAG_PROFILE; | |
979 | } | |
4161824f | 980 | return ret; |
f3f096cf SD |
981 | } |
982 | ||
70ed91c6 | 983 | static void |
7f1d2f82 | 984 | probe_event_disable(struct trace_uprobe *tu, struct trace_event_file *file) |
f3f096cf | 985 | { |
14577c39 | 986 | if (!trace_probe_is_enabled(&tu->tp)) |
f3f096cf SD |
987 | return; |
988 | ||
70ed91c6 J |
989 | if (file) { |
990 | struct event_file_link *link; | |
991 | ||
992 | link = find_event_file_link(&tu->tp, file); | |
993 | if (!link) | |
994 | return; | |
995 | ||
996 | list_del_rcu(&link->list); | |
997 | /* synchronize with u{,ret}probe_trace_func */ | |
016f8ffc | 998 | synchronize_rcu(); |
70ed91c6 J |
999 | kfree(link); |
1000 | ||
1001 | if (!list_empty(&tu->tp.files)) | |
1002 | return; | |
1003 | } | |
1004 | ||
736288ba ON |
1005 | WARN_ON(!uprobe_filter_is_empty(&tu->filter)); |
1006 | ||
a932b738 | 1007 | uprobe_unregister(tu->inode, tu->offset, &tu->consumer); |
0c92c7a3 | 1008 | tu->inode = NULL; |
70ed91c6 | 1009 | tu->tp.flags &= file ? ~TP_FLAG_TRACE : ~TP_FLAG_PROFILE; |
dcad1a20 NK |
1010 | |
1011 | uprobe_buffer_disable(); | |
f3f096cf SD |
1012 | } |
1013 | ||
2425bcb9 | 1014 | static int uprobe_event_define_fields(struct trace_event_call *event_call) |
f3f096cf | 1015 | { |
eeb07b06 | 1016 | int ret, size; |
f3f096cf | 1017 | struct uprobe_trace_entry_head field; |
457d1772 | 1018 | struct trace_uprobe *tu = event_call->data; |
f3f096cf | 1019 | |
4d1298e2 ON |
1020 | if (is_ret_probe(tu)) { |
1021 | DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0); | |
1022 | DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0); | |
1023 | size = SIZEOF_TRACE_ENTRY(true); | |
1024 | } else { | |
1025 | DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0); | |
1026 | size = SIZEOF_TRACE_ENTRY(false); | |
1027 | } | |
f3f096cf | 1028 | |
eeb07b06 | 1029 | return traceprobe_define_arg_fields(event_call, size, &tu->tp); |
f3f096cf SD |
1030 | } |
1031 | ||
f3f096cf | 1032 | #ifdef CONFIG_PERF_EVENTS |
31ba3348 ON |
1033 | static bool |
1034 | __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm) | |
1035 | { | |
1036 | struct perf_event *event; | |
1037 | ||
1038 | if (filter->nr_systemwide) | |
1039 | return true; | |
1040 | ||
1041 | list_for_each_entry(event, &filter->perf_events, hw.tp_list) { | |
50f16a8b | 1042 | if (event->hw.target->mm == mm) |
31ba3348 ON |
1043 | return true; |
1044 | } | |
1045 | ||
1046 | return false; | |
1047 | } | |
1048 | ||
b2fe8ba6 ON |
1049 | static inline bool |
1050 | uprobe_filter_event(struct trace_uprobe *tu, struct perf_event *event) | |
1051 | { | |
50f16a8b | 1052 | return __uprobe_perf_filter(&tu->filter, event->hw.target->mm); |
b2fe8ba6 ON |
1053 | } |
1054 | ||
ce5f36a5 | 1055 | static int uprobe_perf_close(struct trace_uprobe *tu, struct perf_event *event) |
736288ba | 1056 | { |
b2fe8ba6 ON |
1057 | bool done; |
1058 | ||
736288ba | 1059 | write_lock(&tu->filter.rwlock); |
50f16a8b | 1060 | if (event->hw.target) { |
ce5f36a5 | 1061 | list_del(&event->hw.tp_list); |
b2fe8ba6 | 1062 | done = tu->filter.nr_systemwide || |
50f16a8b | 1063 | (event->hw.target->flags & PF_EXITING) || |
b2fe8ba6 | 1064 | uprobe_filter_event(tu, event); |
b2fe8ba6 | 1065 | } else { |
ce5f36a5 | 1066 | tu->filter.nr_systemwide--; |
b2fe8ba6 | 1067 | done = tu->filter.nr_systemwide; |
b2fe8ba6 | 1068 | } |
736288ba ON |
1069 | write_unlock(&tu->filter.rwlock); |
1070 | ||
b2fe8ba6 | 1071 | if (!done) |
927d6874 | 1072 | return uprobe_apply(tu->inode, tu->offset, &tu->consumer, false); |
31ba3348 | 1073 | |
736288ba ON |
1074 | return 0; |
1075 | } | |
1076 | ||
ce5f36a5 | 1077 | static int uprobe_perf_open(struct trace_uprobe *tu, struct perf_event *event) |
736288ba | 1078 | { |
b2fe8ba6 | 1079 | bool done; |
927d6874 | 1080 | int err; |
b2fe8ba6 | 1081 | |
736288ba | 1082 | write_lock(&tu->filter.rwlock); |
50f16a8b | 1083 | if (event->hw.target) { |
ce5f36a5 ON |
1084 | /* |
1085 | * event->parent != NULL means copy_process(), we can avoid | |
1086 | * uprobe_apply(). current->mm must be probed and we can rely | |
1087 | * on dup_mmap() which preserves the already installed bp's. | |
1088 | * | |
1089 | * attr.enable_on_exec means that exec/mmap will install the | |
1090 | * breakpoints we need. | |
1091 | */ | |
b2fe8ba6 | 1092 | done = tu->filter.nr_systemwide || |
ce5f36a5 | 1093 | event->parent || event->attr.enable_on_exec || |
b2fe8ba6 | 1094 | uprobe_filter_event(tu, event); |
ce5f36a5 | 1095 | list_add(&event->hw.tp_list, &tu->filter.perf_events); |
b2fe8ba6 | 1096 | } else { |
b2fe8ba6 | 1097 | done = tu->filter.nr_systemwide; |
ce5f36a5 | 1098 | tu->filter.nr_systemwide++; |
b2fe8ba6 | 1099 | } |
736288ba ON |
1100 | write_unlock(&tu->filter.rwlock); |
1101 | ||
927d6874 ON |
1102 | err = 0; |
1103 | if (!done) { | |
1104 | err = uprobe_apply(tu->inode, tu->offset, &tu->consumer, true); | |
1105 | if (err) | |
1106 | uprobe_perf_close(tu, event); | |
1107 | } | |
1108 | return err; | |
736288ba ON |
1109 | } |
1110 | ||
31ba3348 ON |
1111 | static bool uprobe_perf_filter(struct uprobe_consumer *uc, |
1112 | enum uprobe_filter_ctx ctx, struct mm_struct *mm) | |
1113 | { | |
1114 | struct trace_uprobe *tu; | |
1115 | int ret; | |
1116 | ||
1117 | tu = container_of(uc, struct trace_uprobe, consumer); | |
1118 | read_lock(&tu->filter.rwlock); | |
1119 | ret = __uprobe_perf_filter(&tu->filter, mm); | |
1120 | read_unlock(&tu->filter.rwlock); | |
1121 | ||
1122 | return ret; | |
1123 | } | |
1124 | ||
a43b9704 | 1125 | static void __uprobe_perf_func(struct trace_uprobe *tu, |
dd9fa555 NK |
1126 | unsigned long func, struct pt_regs *regs, |
1127 | struct uprobe_cpu_buffer *ucb, int dsize) | |
f3f096cf | 1128 | { |
2425bcb9 | 1129 | struct trace_event_call *call = &tu->tp.call; |
f3f096cf SD |
1130 | struct uprobe_trace_entry_head *entry; |
1131 | struct hlist_head *head; | |
457d1772 | 1132 | void *data; |
dd9fa555 | 1133 | int size, esize; |
dcad1a20 NK |
1134 | int rctx; |
1135 | ||
e87c6bc3 | 1136 | if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs)) |
04a22fae WN |
1137 | return; |
1138 | ||
dcad1a20 | 1139 | esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); |
f3f096cf | 1140 | |
dcad1a20 NK |
1141 | size = esize + tu->tp.size + dsize; |
1142 | size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32); | |
1143 | if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough")) | |
1144 | return; | |
1145 | ||
f3f096cf | 1146 | preempt_disable(); |
515619f2 ON |
1147 | head = this_cpu_ptr(call->perf_events); |
1148 | if (hlist_empty(head)) | |
1149 | goto out; | |
1150 | ||
1e1dcd93 | 1151 | entry = perf_trace_buf_alloc(size, NULL, &rctx); |
f3f096cf SD |
1152 | if (!entry) |
1153 | goto out; | |
1154 | ||
393a736c ON |
1155 | if (is_ret_probe(tu)) { |
1156 | entry->vaddr[0] = func; | |
32520b2c | 1157 | entry->vaddr[1] = instruction_pointer(regs); |
393a736c ON |
1158 | data = DATAOF_TRACE_ENTRY(entry, true); |
1159 | } else { | |
32520b2c | 1160 | entry->vaddr[0] = instruction_pointer(regs); |
393a736c ON |
1161 | data = DATAOF_TRACE_ENTRY(entry, false); |
1162 | } | |
1163 | ||
dcad1a20 NK |
1164 | memcpy(data, ucb->buf, tu->tp.size + dsize); |
1165 | ||
1166 | if (size - esize > tu->tp.size + dsize) { | |
1167 | int len = tu->tp.size + dsize; | |
14577c39 | 1168 | |
dcad1a20 | 1169 | memset(data + len, 0, size - esize - len); |
14577c39 | 1170 | } |
f3f096cf | 1171 | |
1e1dcd93 | 1172 | perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs, |
8fd0fbbe | 1173 | head, NULL); |
f3f096cf SD |
1174 | out: |
1175 | preempt_enable(); | |
a51cc604 ON |
1176 | } |
1177 | ||
1178 | /* uprobe profile handler */ | |
dd9fa555 NK |
1179 | static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs, |
1180 | struct uprobe_cpu_buffer *ucb, int dsize) | |
a51cc604 ON |
1181 | { |
1182 | if (!uprobe_perf_filter(&tu->consumer, 0, current->mm)) | |
1183 | return UPROBE_HANDLER_REMOVE; | |
1184 | ||
393a736c | 1185 | if (!is_ret_probe(tu)) |
dd9fa555 | 1186 | __uprobe_perf_func(tu, 0, regs, ucb, dsize); |
f42d24a1 | 1187 | return 0; |
f3f096cf | 1188 | } |
c1ae5c75 ON |
1189 | |
1190 | static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func, | |
dd9fa555 NK |
1191 | struct pt_regs *regs, |
1192 | struct uprobe_cpu_buffer *ucb, int dsize) | |
c1ae5c75 | 1193 | { |
dd9fa555 | 1194 | __uprobe_perf_func(tu, func, regs, ucb, dsize); |
c1ae5c75 | 1195 | } |
41bdc4b4 YS |
1196 | |
1197 | int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type, | |
1198 | const char **filename, u64 *probe_offset, | |
1199 | bool perf_type_tracepoint) | |
1200 | { | |
1201 | const char *pevent = trace_event_name(event->tp_event); | |
1202 | const char *group = event->tp_event->class->system; | |
1203 | struct trace_uprobe *tu; | |
1204 | ||
1205 | if (perf_type_tracepoint) | |
1206 | tu = find_probe_event(pevent, group); | |
1207 | else | |
1208 | tu = event->tp_event->data; | |
1209 | if (!tu) | |
1210 | return -EINVAL; | |
1211 | ||
1212 | *fd_type = is_ret_probe(tu) ? BPF_FD_TYPE_URETPROBE | |
1213 | : BPF_FD_TYPE_UPROBE; | |
1214 | *filename = tu->filename; | |
1215 | *probe_offset = tu->offset; | |
1216 | return 0; | |
1217 | } | |
f3f096cf SD |
1218 | #endif /* CONFIG_PERF_EVENTS */ |
1219 | ||
70ed91c6 | 1220 | static int |
2425bcb9 | 1221 | trace_uprobe_register(struct trace_event_call *event, enum trace_reg type, |
70ed91c6 | 1222 | void *data) |
f3f096cf | 1223 | { |
457d1772 | 1224 | struct trace_uprobe *tu = event->data; |
7f1d2f82 | 1225 | struct trace_event_file *file = data; |
f3f096cf SD |
1226 | |
1227 | switch (type) { | |
1228 | case TRACE_REG_REGISTER: | |
70ed91c6 | 1229 | return probe_event_enable(tu, file, NULL); |
f3f096cf SD |
1230 | |
1231 | case TRACE_REG_UNREGISTER: | |
70ed91c6 | 1232 | probe_event_disable(tu, file); |
f3f096cf SD |
1233 | return 0; |
1234 | ||
1235 | #ifdef CONFIG_PERF_EVENTS | |
1236 | case TRACE_REG_PERF_REGISTER: | |
70ed91c6 | 1237 | return probe_event_enable(tu, NULL, uprobe_perf_filter); |
f3f096cf SD |
1238 | |
1239 | case TRACE_REG_PERF_UNREGISTER: | |
70ed91c6 | 1240 | probe_event_disable(tu, NULL); |
f3f096cf | 1241 | return 0; |
736288ba ON |
1242 | |
1243 | case TRACE_REG_PERF_OPEN: | |
1244 | return uprobe_perf_open(tu, data); | |
1245 | ||
1246 | case TRACE_REG_PERF_CLOSE: | |
1247 | return uprobe_perf_close(tu, data); | |
1248 | ||
f3f096cf SD |
1249 | #endif |
1250 | default: | |
1251 | return 0; | |
1252 | } | |
1253 | return 0; | |
1254 | } | |
1255 | ||
1256 | static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs) | |
1257 | { | |
f3f096cf | 1258 | struct trace_uprobe *tu; |
b7e0bf34 | 1259 | struct uprobe_dispatch_data udd; |
dd9fa555 NK |
1260 | struct uprobe_cpu_buffer *ucb; |
1261 | int dsize, esize; | |
f42d24a1 | 1262 | int ret = 0; |
f3f096cf | 1263 | |
dd9fa555 | 1264 | |
a932b738 | 1265 | tu = container_of(con, struct trace_uprobe, consumer); |
1b47aefd | 1266 | tu->nhit++; |
f3f096cf | 1267 | |
b7e0bf34 NK |
1268 | udd.tu = tu; |
1269 | udd.bp_addr = instruction_pointer(regs); | |
1270 | ||
1271 | current->utask->vaddr = (unsigned long) &udd; | |
1272 | ||
dd9fa555 NK |
1273 | if (WARN_ON_ONCE(!uprobe_cpu_buffer)) |
1274 | return 0; | |
1275 | ||
1276 | dsize = __get_data_size(&tu->tp, regs); | |
1277 | esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); | |
1278 | ||
1279 | ucb = uprobe_buffer_get(); | |
9178412d | 1280 | store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize); |
dd9fa555 | 1281 | |
14577c39 | 1282 | if (tu->tp.flags & TP_FLAG_TRACE) |
dd9fa555 | 1283 | ret |= uprobe_trace_func(tu, regs, ucb, dsize); |
f3f096cf SD |
1284 | |
1285 | #ifdef CONFIG_PERF_EVENTS | |
14577c39 | 1286 | if (tu->tp.flags & TP_FLAG_PROFILE) |
dd9fa555 | 1287 | ret |= uprobe_perf_func(tu, regs, ucb, dsize); |
f3f096cf | 1288 | #endif |
dd9fa555 | 1289 | uprobe_buffer_put(ucb); |
f42d24a1 | 1290 | return ret; |
f3f096cf SD |
1291 | } |
1292 | ||
c1ae5c75 ON |
1293 | static int uretprobe_dispatcher(struct uprobe_consumer *con, |
1294 | unsigned long func, struct pt_regs *regs) | |
1295 | { | |
1296 | struct trace_uprobe *tu; | |
b7e0bf34 | 1297 | struct uprobe_dispatch_data udd; |
dd9fa555 NK |
1298 | struct uprobe_cpu_buffer *ucb; |
1299 | int dsize, esize; | |
c1ae5c75 ON |
1300 | |
1301 | tu = container_of(con, struct trace_uprobe, consumer); | |
1302 | ||
b7e0bf34 NK |
1303 | udd.tu = tu; |
1304 | udd.bp_addr = func; | |
1305 | ||
1306 | current->utask->vaddr = (unsigned long) &udd; | |
1307 | ||
dd9fa555 NK |
1308 | if (WARN_ON_ONCE(!uprobe_cpu_buffer)) |
1309 | return 0; | |
1310 | ||
1311 | dsize = __get_data_size(&tu->tp, regs); | |
1312 | esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); | |
1313 | ||
1314 | ucb = uprobe_buffer_get(); | |
9178412d | 1315 | store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize); |
dd9fa555 | 1316 | |
14577c39 | 1317 | if (tu->tp.flags & TP_FLAG_TRACE) |
dd9fa555 | 1318 | uretprobe_trace_func(tu, func, regs, ucb, dsize); |
c1ae5c75 ON |
1319 | |
1320 | #ifdef CONFIG_PERF_EVENTS | |
14577c39 | 1321 | if (tu->tp.flags & TP_FLAG_PROFILE) |
dd9fa555 | 1322 | uretprobe_perf_func(tu, func, regs, ucb, dsize); |
c1ae5c75 | 1323 | #endif |
dd9fa555 | 1324 | uprobe_buffer_put(ucb); |
c1ae5c75 ON |
1325 | return 0; |
1326 | } | |
1327 | ||
f3f096cf SD |
1328 | static struct trace_event_functions uprobe_funcs = { |
1329 | .trace = print_uprobe_event | |
1330 | }; | |
1331 | ||
33ea4b24 SL |
1332 | static inline void init_trace_event_call(struct trace_uprobe *tu, |
1333 | struct trace_event_call *call) | |
f3f096cf | 1334 | { |
f3f096cf SD |
1335 | INIT_LIST_HEAD(&call->class->fields); |
1336 | call->event.funcs = &uprobe_funcs; | |
1337 | call->class->define_fields = uprobe_event_define_fields; | |
1338 | ||
33ea4b24 SL |
1339 | call->flags = TRACE_EVENT_FL_UPROBE; |
1340 | call->class->reg = trace_uprobe_register; | |
1341 | call->data = tu; | |
1342 | } | |
1343 | ||
1344 | static int register_uprobe_event(struct trace_uprobe *tu) | |
1345 | { | |
1346 | struct trace_event_call *call = &tu->tp.call; | |
1347 | int ret = 0; | |
1348 | ||
1349 | init_trace_event_call(tu, call); | |
1350 | ||
0a46c854 | 1351 | if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) |
f3f096cf SD |
1352 | return -ENOMEM; |
1353 | ||
9023c930 | 1354 | ret = register_trace_event(&call->event); |
f3f096cf SD |
1355 | if (!ret) { |
1356 | kfree(call->print_fmt); | |
1357 | return -ENODEV; | |
1358 | } | |
ede392a7 | 1359 | |
7e1413ed | 1360 | ret = trace_add_event_call(call); |
f3f096cf SD |
1361 | |
1362 | if (ret) { | |
de7b2973 | 1363 | pr_info("Failed to register uprobe event: %s\n", |
687fcc4a | 1364 | trace_event_name(call)); |
f3f096cf | 1365 | kfree(call->print_fmt); |
9023c930 | 1366 | unregister_trace_event(&call->event); |
f3f096cf SD |
1367 | } |
1368 | ||
1369 | return ret; | |
1370 | } | |
1371 | ||
c6c2401d | 1372 | static int unregister_uprobe_event(struct trace_uprobe *tu) |
f3f096cf | 1373 | { |
c6c2401d SRRH |
1374 | int ret; |
1375 | ||
f3f096cf | 1376 | /* tu->event is unregistered in trace_remove_event_call() */ |
7e1413ed | 1377 | ret = trace_remove_event_call(&tu->tp.call); |
c6c2401d SRRH |
1378 | if (ret) |
1379 | return ret; | |
14577c39 NK |
1380 | kfree(tu->tp.call.print_fmt); |
1381 | tu->tp.call.print_fmt = NULL; | |
c6c2401d | 1382 | return 0; |
f3f096cf SD |
1383 | } |
1384 | ||
33ea4b24 SL |
1385 | #ifdef CONFIG_PERF_EVENTS |
1386 | struct trace_event_call * | |
a6ca88b2 SL |
1387 | create_local_trace_uprobe(char *name, unsigned long offs, |
1388 | unsigned long ref_ctr_offset, bool is_return) | |
33ea4b24 SL |
1389 | { |
1390 | struct trace_uprobe *tu; | |
33ea4b24 SL |
1391 | struct path path; |
1392 | int ret; | |
1393 | ||
1394 | ret = kern_path(name, LOOKUP_FOLLOW, &path); | |
1395 | if (ret) | |
1396 | return ERR_PTR(ret); | |
1397 | ||
0c92c7a3 SL |
1398 | if (!d_is_reg(path.dentry)) { |
1399 | path_put(&path); | |
33ea4b24 SL |
1400 | return ERR_PTR(-EINVAL); |
1401 | } | |
1402 | ||
1403 | /* | |
0597c49c | 1404 | * local trace_kprobes are not added to dyn_event, so they are never |
33ea4b24 SL |
1405 | * searched in find_trace_kprobe(). Therefore, there is no concern of |
1406 | * duplicated name "DUMMY_EVENT" here. | |
1407 | */ | |
1408 | tu = alloc_trace_uprobe(UPROBE_EVENT_SYSTEM, "DUMMY_EVENT", 0, | |
1409 | is_return); | |
1410 | ||
1411 | if (IS_ERR(tu)) { | |
1412 | pr_info("Failed to allocate trace_uprobe.(%d)\n", | |
1413 | (int)PTR_ERR(tu)); | |
0c92c7a3 | 1414 | path_put(&path); |
33ea4b24 SL |
1415 | return ERR_CAST(tu); |
1416 | } | |
1417 | ||
1418 | tu->offset = offs; | |
0c92c7a3 | 1419 | tu->path = path; |
a6ca88b2 | 1420 | tu->ref_ctr_offset = ref_ctr_offset; |
33ea4b24 SL |
1421 | tu->filename = kstrdup(name, GFP_KERNEL); |
1422 | init_trace_event_call(tu, &tu->tp.call); | |
1423 | ||
0a46c854 | 1424 | if (traceprobe_set_print_fmt(&tu->tp, is_ret_probe(tu)) < 0) { |
33ea4b24 SL |
1425 | ret = -ENOMEM; |
1426 | goto error; | |
1427 | } | |
1428 | ||
1429 | return &tu->tp.call; | |
1430 | error: | |
1431 | free_trace_uprobe(tu); | |
1432 | return ERR_PTR(ret); | |
1433 | } | |
1434 | ||
1435 | void destroy_local_trace_uprobe(struct trace_event_call *event_call) | |
1436 | { | |
1437 | struct trace_uprobe *tu; | |
1438 | ||
1439 | tu = container_of(event_call, struct trace_uprobe, tp.call); | |
1440 | ||
1441 | kfree(tu->tp.call.print_fmt); | |
1442 | tu->tp.call.print_fmt = NULL; | |
1443 | ||
1444 | free_trace_uprobe(tu); | |
1445 | } | |
1446 | #endif /* CONFIG_PERF_EVENTS */ | |
1447 | ||
f3f096cf SD |
1448 | /* Make a trace interface for controling probe points */ |
1449 | static __init int init_uprobe_trace(void) | |
1450 | { | |
1451 | struct dentry *d_tracer; | |
0597c49c MH |
1452 | int ret; |
1453 | ||
1454 | ret = dyn_event_register(&trace_uprobe_ops); | |
1455 | if (ret) | |
1456 | return ret; | |
f3f096cf SD |
1457 | |
1458 | d_tracer = tracing_init_dentry(); | |
14a5ae40 | 1459 | if (IS_ERR(d_tracer)) |
f3f096cf SD |
1460 | return 0; |
1461 | ||
1462 | trace_create_file("uprobe_events", 0644, d_tracer, | |
1463 | NULL, &uprobe_events_ops); | |
1464 | /* Profile interface */ | |
1465 | trace_create_file("uprobe_profile", 0444, d_tracer, | |
1466 | NULL, &uprobe_profile_ops); | |
1467 | return 0; | |
1468 | } | |
1469 | ||
1470 | fs_initcall(init_uprobe_trace); |