Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1b29b018 SR |
2 | /* |
3 | * ring buffer based function tracer | |
4 | * | |
5 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
6 | * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> | |
7 | * | |
8 | * Based on code from the latency_tracer, that is: | |
9 | * | |
10 | * Copyright (C) 2004-2006 Ingo Molnar | |
6d49e352 | 11 | * Copyright (C) 2004 Nadia Yvette Chambers |
1b29b018 | 12 | */ |
23b4ff3a | 13 | #include <linux/ring_buffer.h> |
1b29b018 SR |
14 | #include <linux/debugfs.h> |
15 | #include <linux/uaccess.h> | |
16 | #include <linux/ftrace.h> | |
f20a5806 | 17 | #include <linux/slab.h> |
2e0f5761 | 18 | #include <linux/fs.h> |
1b29b018 SR |
19 | |
20 | #include "trace.h" | |
21 | ||
f20a5806 SRRH |
22 | static void tracing_start_function_trace(struct trace_array *tr); |
23 | static void tracing_stop_function_trace(struct trace_array *tr); | |
24 | static void | |
25 | function_trace_call(unsigned long ip, unsigned long parent_ip, | |
26 | struct ftrace_ops *op, struct pt_regs *pt_regs); | |
27 | static void | |
28 | function_stack_trace_call(unsigned long ip, unsigned long parent_ip, | |
29 | struct ftrace_ops *op, struct pt_regs *pt_regs); | |
f20a5806 SRRH |
30 | static struct tracer_flags func_flags; |
31 | ||
32 | /* Our option */ | |
33 | enum { | |
34 | TRACE_FUNC_OPT_STACK = 0x1, | |
35 | }; | |
36 | ||
37 | static int allocate_ftrace_ops(struct trace_array *tr) | |
38 | { | |
39 | struct ftrace_ops *ops; | |
a225cdd2 | 40 | |
f20a5806 SRRH |
41 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); |
42 | if (!ops) | |
43 | return -ENOMEM; | |
53614991 | 44 | |
f20a5806 SRRH |
45 | /* Currently only the non stack verision is supported */ |
46 | ops->func = function_trace_call; | |
345ddcc8 | 47 | ops->flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_PID; |
f20a5806 SRRH |
48 | |
49 | tr->ops = ops; | |
50 | ops->private = tr; | |
51 | return 0; | |
52 | } | |
a225cdd2 | 53 | |
591dffda SRRH |
54 | |
55 | int ftrace_create_function_files(struct trace_array *tr, | |
56 | struct dentry *parent) | |
57 | { | |
58 | int ret; | |
59 | ||
5d6c97c5 SRRH |
60 | /* |
61 | * The top level array uses the "global_ops", and the files are | |
62 | * created on boot up. | |
63 | */ | |
64 | if (tr->flags & TRACE_ARRAY_FL_GLOBAL) | |
65 | return 0; | |
66 | ||
67 | ret = allocate_ftrace_ops(tr); | |
68 | if (ret) | |
69 | return ret; | |
591dffda SRRH |
70 | |
71 | ftrace_create_filter_files(tr->ops, parent); | |
72 | ||
73 | return 0; | |
74 | } | |
75 | ||
76 | void ftrace_destroy_function_files(struct trace_array *tr) | |
77 | { | |
78 | ftrace_destroy_filter_files(tr->ops); | |
79 | kfree(tr->ops); | |
80 | tr->ops = NULL; | |
81 | } | |
82 | ||
b6f11df2 | 83 | static int function_trace_init(struct trace_array *tr) |
1b29b018 | 84 | { |
4104d326 | 85 | ftrace_func_t func; |
f20a5806 | 86 | |
4104d326 SRRH |
87 | /* |
88 | * Instance trace_arrays get their ops allocated | |
89 | * at instance creation. Unless it failed | |
90 | * the allocation. | |
91 | */ | |
92 | if (!tr->ops) | |
591dffda | 93 | return -ENOMEM; |
4104d326 SRRH |
94 | |
95 | /* Currently only the global instance can do stack tracing */ | |
96 | if (tr->flags & TRACE_ARRAY_FL_GLOBAL && | |
97 | func_flags.val & TRACE_FUNC_OPT_STACK) | |
98 | func = function_stack_trace_call; | |
99 | else | |
100 | func = function_trace_call; | |
101 | ||
102 | ftrace_init_array_ops(tr, func); | |
f20a5806 | 103 | |
12883efb | 104 | tr->trace_buffer.cpu = get_cpu(); |
26bc83f4 SR |
105 | put_cpu(); |
106 | ||
41bc8144 | 107 | tracing_start_cmdline_record(); |
f20a5806 | 108 | tracing_start_function_trace(tr); |
1c80025a | 109 | return 0; |
1b29b018 SR |
110 | } |
111 | ||
e309b41d | 112 | static void function_trace_reset(struct trace_array *tr) |
1b29b018 | 113 | { |
f20a5806 | 114 | tracing_stop_function_trace(tr); |
b6f11df2 | 115 | tracing_stop_cmdline_record(); |
4104d326 | 116 | ftrace_reset_array_ops(tr); |
1b29b018 SR |
117 | } |
118 | ||
9036990d SR |
119 | static void function_trace_start(struct trace_array *tr) |
120 | { | |
12883efb | 121 | tracing_reset_online_cpus(&tr->trace_buffer); |
9036990d SR |
122 | } |
123 | ||
bb3c3c95 | 124 | static void |
2f5f6ad9 | 125 | function_trace_call(unsigned long ip, unsigned long parent_ip, |
a1e2e31d | 126 | struct ftrace_ops *op, struct pt_regs *pt_regs) |
bb3c3c95 | 127 | { |
f20a5806 | 128 | struct trace_array *tr = op->private; |
bb3c3c95 SR |
129 | struct trace_array_cpu *data; |
130 | unsigned long flags; | |
d41032a8 | 131 | int bit; |
bb3c3c95 SR |
132 | int cpu; |
133 | int pc; | |
134 | ||
f20a5806 | 135 | if (unlikely(!tr->function_enabled)) |
bb3c3c95 SR |
136 | return; |
137 | ||
897f68a4 SR |
138 | pc = preempt_count(); |
139 | preempt_disable_notrace(); | |
bb3c3c95 | 140 | |
897f68a4 SR |
141 | bit = trace_test_and_set_recursion(TRACE_FTRACE_START, TRACE_FTRACE_MAX); |
142 | if (bit < 0) | |
143 | goto out; | |
144 | ||
145 | cpu = smp_processor_id(); | |
12883efb | 146 | data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
897f68a4 SR |
147 | if (!atomic_read(&data->disabled)) { |
148 | local_save_flags(flags); | |
7be42151 | 149 | trace_function(tr, ip, parent_ip, flags, pc); |
bb3c3c95 | 150 | } |
897f68a4 | 151 | trace_clear_recursion(bit); |
bb3c3c95 | 152 | |
897f68a4 SR |
153 | out: |
154 | preempt_enable_notrace(); | |
bb3c3c95 SR |
155 | } |
156 | ||
2ee5b92a SRV |
157 | #ifdef CONFIG_UNWINDER_ORC |
158 | /* | |
159 | * Skip 2: | |
160 | * | |
161 | * function_stack_trace_call() | |
162 | * ftrace_call() | |
163 | */ | |
164 | #define STACK_SKIP 2 | |
165 | #else | |
166 | /* | |
167 | * Skip 3: | |
168 | * __trace_stack() | |
169 | * function_stack_trace_call() | |
170 | * ftrace_call() | |
171 | */ | |
172 | #define STACK_SKIP 3 | |
173 | #endif | |
174 | ||
53614991 | 175 | static void |
2f5f6ad9 | 176 | function_stack_trace_call(unsigned long ip, unsigned long parent_ip, |
a1e2e31d | 177 | struct ftrace_ops *op, struct pt_regs *pt_regs) |
53614991 | 178 | { |
f20a5806 | 179 | struct trace_array *tr = op->private; |
53614991 SR |
180 | struct trace_array_cpu *data; |
181 | unsigned long flags; | |
182 | long disabled; | |
183 | int cpu; | |
184 | int pc; | |
185 | ||
f20a5806 | 186 | if (unlikely(!tr->function_enabled)) |
53614991 SR |
187 | return; |
188 | ||
189 | /* | |
190 | * Need to use raw, since this must be called before the | |
191 | * recursive protection is performed. | |
192 | */ | |
193 | local_irq_save(flags); | |
194 | cpu = raw_smp_processor_id(); | |
12883efb | 195 | data = per_cpu_ptr(tr->trace_buffer.data, cpu); |
53614991 SR |
196 | disabled = atomic_inc_return(&data->disabled); |
197 | ||
198 | if (likely(disabled == 1)) { | |
199 | pc = preempt_count(); | |
7be42151 | 200 | trace_function(tr, ip, parent_ip, flags, pc); |
2ee5b92a | 201 | __trace_stack(tr, flags, STACK_SKIP, pc); |
53614991 SR |
202 | } |
203 | ||
204 | atomic_dec(&data->disabled); | |
205 | local_irq_restore(flags); | |
206 | } | |
207 | ||
53614991 SR |
208 | static struct tracer_opt func_opts[] = { |
209 | #ifdef CONFIG_STACKTRACE | |
210 | { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) }, | |
211 | #endif | |
212 | { } /* Always set a last empty entry */ | |
213 | }; | |
214 | ||
215 | static struct tracer_flags func_flags = { | |
216 | .val = 0, /* By default: all flags disabled */ | |
217 | .opts = func_opts | |
218 | }; | |
219 | ||
f20a5806 | 220 | static void tracing_start_function_trace(struct trace_array *tr) |
3eb36aa0 | 221 | { |
f20a5806 SRRH |
222 | tr->function_enabled = 0; |
223 | register_ftrace_function(tr->ops); | |
224 | tr->function_enabled = 1; | |
3eb36aa0 SR |
225 | } |
226 | ||
f20a5806 | 227 | static void tracing_stop_function_trace(struct trace_array *tr) |
3eb36aa0 | 228 | { |
f20a5806 SRRH |
229 | tr->function_enabled = 0; |
230 | unregister_ftrace_function(tr->ops); | |
3eb36aa0 SR |
231 | } |
232 | ||
d39cdd20 CH |
233 | static struct tracer function_trace; |
234 | ||
8c1a49ae SRRH |
235 | static int |
236 | func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) | |
53614991 | 237 | { |
f555f123 AV |
238 | switch (bit) { |
239 | case TRACE_FUNC_OPT_STACK: | |
53614991 SR |
240 | /* do nothing if already set */ |
241 | if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK)) | |
f555f123 | 242 | break; |
53614991 | 243 | |
d39cdd20 CH |
244 | /* We can change this flag when not running. */ |
245 | if (tr->current_trace != &function_trace) | |
246 | break; | |
247 | ||
f20a5806 SRRH |
248 | unregister_ftrace_function(tr->ops); |
249 | ||
3eb36aa0 | 250 | if (set) { |
4104d326 | 251 | tr->ops->func = function_stack_trace_call; |
f20a5806 | 252 | register_ftrace_function(tr->ops); |
3eb36aa0 | 253 | } else { |
4104d326 | 254 | tr->ops->func = function_trace_call; |
f20a5806 | 255 | register_ftrace_function(tr->ops); |
3eb36aa0 | 256 | } |
53614991 | 257 | |
f555f123 AV |
258 | break; |
259 | default: | |
260 | return -EINVAL; | |
53614991 SR |
261 | } |
262 | ||
f555f123 | 263 | return 0; |
53614991 SR |
264 | } |
265 | ||
8f768993 | 266 | static struct tracer function_trace __tracer_data = |
1b29b018 | 267 | { |
3eb36aa0 SR |
268 | .name = "function", |
269 | .init = function_trace_init, | |
270 | .reset = function_trace_reset, | |
271 | .start = function_trace_start, | |
53614991 SR |
272 | .flags = &func_flags, |
273 | .set_flag = func_set_flag, | |
f20a5806 | 274 | .allow_instances = true, |
60a11774 | 275 | #ifdef CONFIG_FTRACE_SELFTEST |
3eb36aa0 | 276 | .selftest = trace_selftest_startup_function, |
60a11774 | 277 | #endif |
1b29b018 SR |
278 | }; |
279 | ||
23b4ff3a | 280 | #ifdef CONFIG_DYNAMIC_FTRACE |
fe014e24 | 281 | static void update_traceon_count(struct ftrace_probe_ops *ops, |
2290f2c5 SRV |
282 | unsigned long ip, |
283 | struct trace_array *tr, bool on, | |
6e444319 | 284 | void *data) |
23b4ff3a | 285 | { |
6e444319 | 286 | struct ftrace_func_mapper *mapper = data; |
fe014e24 SRV |
287 | long *count; |
288 | long old_count; | |
23b4ff3a | 289 | |
a9ce7c36 SRRH |
290 | /* |
291 | * Tracing gets disabled (or enabled) once per count. | |
0af26492 | 292 | * This function can be called at the same time on multiple CPUs. |
a9ce7c36 SRRH |
293 | * It is fine if both disable (or enable) tracing, as disabling |
294 | * (or enabling) the second time doesn't do anything as the | |
295 | * state of the tracer is already disabled (or enabled). | |
296 | * What needs to be synchronized in this case is that the count | |
297 | * only gets decremented once, even if the tracer is disabled | |
298 | * (or enabled) twice, as the second one is really a nop. | |
299 | * | |
300 | * The memory barriers guarantee that we only decrement the | |
301 | * counter once. First the count is read to a local variable | |
302 | * and a read barrier is used to make sure that it is loaded | |
303 | * before checking if the tracer is in the state we want. | |
304 | * If the tracer is not in the state we want, then the count | |
305 | * is guaranteed to be the old count. | |
306 | * | |
307 | * Next the tracer is set to the state we want (disabled or enabled) | |
308 | * then a write memory barrier is used to make sure that | |
309 | * the new state is visible before changing the counter by | |
310 | * one minus the old counter. This guarantees that another CPU | |
311 | * executing this code will see the new state before seeing | |
0af26492 | 312 | * the new counter value, and would not do anything if the new |
a9ce7c36 SRRH |
313 | * counter is seen. |
314 | * | |
315 | * Note, there is no synchronization between this and a user | |
316 | * setting the tracing_on file. But we currently don't care | |
317 | * about that. | |
318 | */ | |
fe014e24 SRV |
319 | count = (long *)ftrace_func_mapper_find_ip(mapper, ip); |
320 | old_count = *count; | |
321 | ||
322 | if (old_count <= 0) | |
a9ce7c36 | 323 | return; |
23b4ff3a | 324 | |
a9ce7c36 SRRH |
325 | /* Make sure we see count before checking tracing state */ |
326 | smp_rmb(); | |
23b4ff3a | 327 | |
2290f2c5 | 328 | if (on == !!tracer_tracing_is_on(tr)) |
a9ce7c36 SRRH |
329 | return; |
330 | ||
331 | if (on) | |
2290f2c5 | 332 | tracer_tracing_on(tr); |
a9ce7c36 | 333 | else |
2290f2c5 | 334 | tracer_tracing_off(tr); |
a9ce7c36 | 335 | |
a9ce7c36 SRRH |
336 | /* Make sure tracing state is visible before updating count */ |
337 | smp_wmb(); | |
338 | ||
339 | *count = old_count - 1; | |
23b4ff3a SR |
340 | } |
341 | ||
342 | static void | |
bca6c8d0 | 343 | ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 344 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 345 | void *data) |
23b4ff3a | 346 | { |
2290f2c5 | 347 | update_traceon_count(ops, ip, tr, 1, data); |
1c317143 | 348 | } |
23b4ff3a | 349 | |
1c317143 | 350 | static void |
bca6c8d0 | 351 | ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 352 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 353 | void *data) |
1c317143 | 354 | { |
2290f2c5 | 355 | update_traceon_count(ops, ip, tr, 0, data); |
23b4ff3a SR |
356 | } |
357 | ||
8380d248 | 358 | static void |
bca6c8d0 | 359 | ftrace_traceon(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 360 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 361 | void *data) |
8380d248 | 362 | { |
2290f2c5 | 363 | if (tracer_tracing_is_on(tr)) |
8380d248 SRRH |
364 | return; |
365 | ||
2290f2c5 | 366 | tracer_tracing_on(tr); |
8380d248 SRRH |
367 | } |
368 | ||
369 | static void | |
bca6c8d0 | 370 | ftrace_traceoff(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 371 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 372 | void *data) |
8380d248 | 373 | { |
2290f2c5 | 374 | if (!tracer_tracing_is_on(tr)) |
8380d248 SRRH |
375 | return; |
376 | ||
2290f2c5 | 377 | tracer_tracing_off(tr); |
8380d248 SRRH |
378 | } |
379 | ||
2ee5b92a SRV |
380 | #ifdef CONFIG_UNWINDER_ORC |
381 | /* | |
382 | * Skip 3: | |
383 | * | |
384 | * function_trace_probe_call() | |
385 | * ftrace_ops_assist_func() | |
386 | * ftrace_call() | |
387 | */ | |
388 | #define FTRACE_STACK_SKIP 3 | |
389 | #else | |
dd42cd3e | 390 | /* |
2ee5b92a SRV |
391 | * Skip 5: |
392 | * | |
393 | * __trace_stack() | |
dd42cd3e SRRH |
394 | * ftrace_stacktrace() |
395 | * function_trace_probe_call() | |
2ee5b92a | 396 | * ftrace_ops_assist_func() |
dd42cd3e SRRH |
397 | * ftrace_call() |
398 | */ | |
2ee5b92a SRV |
399 | #define FTRACE_STACK_SKIP 5 |
400 | #endif | |
dd42cd3e | 401 | |
dcc19d28 SRV |
402 | static __always_inline void trace_stack(struct trace_array *tr) |
403 | { | |
404 | unsigned long flags; | |
405 | int pc; | |
406 | ||
407 | local_save_flags(flags); | |
408 | pc = preempt_count(); | |
409 | ||
2ee5b92a | 410 | __trace_stack(tr, flags, FTRACE_STACK_SKIP, pc); |
dcc19d28 SRV |
411 | } |
412 | ||
dd42cd3e | 413 | static void |
bca6c8d0 | 414 | ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 415 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 416 | void *data) |
dd42cd3e | 417 | { |
dcc19d28 | 418 | trace_stack(tr); |
dd42cd3e SRRH |
419 | } |
420 | ||
421 | static void | |
bca6c8d0 | 422 | ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 423 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 424 | void *data) |
dd42cd3e | 425 | { |
6e444319 | 426 | struct ftrace_func_mapper *mapper = data; |
fe014e24 | 427 | long *count; |
a9ce7c36 SRRH |
428 | long old_count; |
429 | long new_count; | |
430 | ||
fe014e24 SRV |
431 | if (!tracing_is_on()) |
432 | return; | |
433 | ||
434 | /* unlimited? */ | |
435 | if (!mapper) { | |
dcc19d28 | 436 | trace_stack(tr); |
fe014e24 SRV |
437 | return; |
438 | } | |
439 | ||
440 | count = (long *)ftrace_func_mapper_find_ip(mapper, ip); | |
441 | ||
a9ce7c36 SRRH |
442 | /* |
443 | * Stack traces should only execute the number of times the | |
444 | * user specified in the counter. | |
445 | */ | |
446 | do { | |
a9ce7c36 SRRH |
447 | old_count = *count; |
448 | ||
449 | if (!old_count) | |
450 | return; | |
451 | ||
a9ce7c36 SRRH |
452 | new_count = old_count - 1; |
453 | new_count = cmpxchg(count, old_count, new_count); | |
454 | if (new_count == old_count) | |
dcc19d28 | 455 | trace_stack(tr); |
a9ce7c36 | 456 | |
fe014e24 SRV |
457 | if (!tracing_is_on()) |
458 | return; | |
459 | ||
a9ce7c36 SRRH |
460 | } while (new_count != old_count); |
461 | } | |
462 | ||
6e444319 SRV |
463 | static int update_count(struct ftrace_probe_ops *ops, unsigned long ip, |
464 | void *data) | |
a9ce7c36 | 465 | { |
6e444319 | 466 | struct ftrace_func_mapper *mapper = data; |
fe014e24 | 467 | long *count = NULL; |
a9ce7c36 | 468 | |
fe014e24 SRV |
469 | if (mapper) |
470 | count = (long *)ftrace_func_mapper_find_ip(mapper, ip); | |
a9ce7c36 | 471 | |
fe014e24 SRV |
472 | if (count) { |
473 | if (*count <= 0) | |
474 | return 0; | |
a9ce7c36 | 475 | (*count)--; |
fe014e24 | 476 | } |
a9ce7c36 SRRH |
477 | |
478 | return 1; | |
dd42cd3e SRRH |
479 | } |
480 | ||
ad71d889 | 481 | static void |
bca6c8d0 | 482 | ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 483 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 484 | void *data) |
ad71d889 | 485 | { |
6e444319 | 486 | if (update_count(ops, ip, data)) |
ad71d889 SRRH |
487 | ftrace_dump(DUMP_ALL); |
488 | } | |
489 | ||
90e3c03c SRRH |
490 | /* Only dump the current CPU buffer. */ |
491 | static void | |
bca6c8d0 | 492 | ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 493 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 494 | void *data) |
90e3c03c | 495 | { |
6e444319 | 496 | if (update_count(ops, ip, data)) |
90e3c03c SRRH |
497 | ftrace_dump(DUMP_ORIG); |
498 | } | |
499 | ||
e110e3d1 | 500 | static int |
dd42cd3e | 501 | ftrace_probe_print(const char *name, struct seq_file *m, |
6e444319 SRV |
502 | unsigned long ip, struct ftrace_probe_ops *ops, |
503 | void *data) | |
dd42cd3e | 504 | { |
6e444319 | 505 | struct ftrace_func_mapper *mapper = data; |
fe014e24 | 506 | long *count = NULL; |
dd42cd3e SRRH |
507 | |
508 | seq_printf(m, "%ps:%s", (void *)ip, name); | |
509 | ||
fe014e24 SRV |
510 | if (mapper) |
511 | count = (long *)ftrace_func_mapper_find_ip(mapper, ip); | |
512 | ||
513 | if (count) | |
514 | seq_printf(m, ":count=%ld\n", *count); | |
dd42cd3e | 515 | else |
fe014e24 | 516 | seq_puts(m, ":unlimited\n"); |
dd42cd3e SRRH |
517 | |
518 | return 0; | |
519 | } | |
520 | ||
521 | static int | |
522 | ftrace_traceon_print(struct seq_file *m, unsigned long ip, | |
b5f081b5 SRV |
523 | struct ftrace_probe_ops *ops, |
524 | void *data) | |
dd42cd3e | 525 | { |
6e444319 | 526 | return ftrace_probe_print("traceon", m, ip, ops, data); |
dd42cd3e SRRH |
527 | } |
528 | ||
529 | static int | |
530 | ftrace_traceoff_print(struct seq_file *m, unsigned long ip, | |
531 | struct ftrace_probe_ops *ops, void *data) | |
532 | { | |
6e444319 | 533 | return ftrace_probe_print("traceoff", m, ip, ops, data); |
dd42cd3e SRRH |
534 | } |
535 | ||
536 | static int | |
537 | ftrace_stacktrace_print(struct seq_file *m, unsigned long ip, | |
538 | struct ftrace_probe_ops *ops, void *data) | |
539 | { | |
6e444319 | 540 | return ftrace_probe_print("stacktrace", m, ip, ops, data); |
dd42cd3e | 541 | } |
e110e3d1 | 542 | |
ad71d889 SRRH |
543 | static int |
544 | ftrace_dump_print(struct seq_file *m, unsigned long ip, | |
545 | struct ftrace_probe_ops *ops, void *data) | |
546 | { | |
6e444319 | 547 | return ftrace_probe_print("dump", m, ip, ops, data); |
ad71d889 SRRH |
548 | } |
549 | ||
90e3c03c SRRH |
550 | static int |
551 | ftrace_cpudump_print(struct seq_file *m, unsigned long ip, | |
552 | struct ftrace_probe_ops *ops, void *data) | |
553 | { | |
6e444319 | 554 | return ftrace_probe_print("cpudump", m, ip, ops, data); |
fe014e24 SRV |
555 | } |
556 | ||
557 | ||
558 | static int | |
b5f081b5 | 559 | ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr, |
6e444319 | 560 | unsigned long ip, void *init_data, void **data) |
fe014e24 | 561 | { |
6e444319 SRV |
562 | struct ftrace_func_mapper *mapper = *data; |
563 | ||
564 | if (!mapper) { | |
565 | mapper = allocate_ftrace_func_mapper(); | |
566 | if (!mapper) | |
567 | return -ENOMEM; | |
568 | *data = mapper; | |
569 | } | |
fe014e24 | 570 | |
6e444319 | 571 | return ftrace_func_mapper_add_ip(mapper, ip, init_data); |
fe014e24 SRV |
572 | } |
573 | ||
574 | static void | |
b5f081b5 | 575 | ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr, |
6e444319 | 576 | unsigned long ip, void *data) |
fe014e24 | 577 | { |
6e444319 SRV |
578 | struct ftrace_func_mapper *mapper = data; |
579 | ||
580 | if (!ip) { | |
581 | free_ftrace_func_mapper(mapper, NULL); | |
582 | return; | |
583 | } | |
fe014e24 SRV |
584 | |
585 | ftrace_func_mapper_remove_ip(mapper, ip); | |
90e3c03c SRRH |
586 | } |
587 | ||
8380d248 SRRH |
588 | static struct ftrace_probe_ops traceon_count_probe_ops = { |
589 | .func = ftrace_traceon_count, | |
dd42cd3e | 590 | .print = ftrace_traceon_print, |
fe014e24 SRV |
591 | .init = ftrace_count_init, |
592 | .free = ftrace_count_free, | |
8380d248 SRRH |
593 | }; |
594 | ||
595 | static struct ftrace_probe_ops traceoff_count_probe_ops = { | |
596 | .func = ftrace_traceoff_count, | |
dd42cd3e | 597 | .print = ftrace_traceoff_print, |
fe014e24 SRV |
598 | .init = ftrace_count_init, |
599 | .free = ftrace_count_free, | |
dd42cd3e SRRH |
600 | }; |
601 | ||
602 | static struct ftrace_probe_ops stacktrace_count_probe_ops = { | |
603 | .func = ftrace_stacktrace_count, | |
604 | .print = ftrace_stacktrace_print, | |
fe014e24 SRV |
605 | .init = ftrace_count_init, |
606 | .free = ftrace_count_free, | |
8380d248 SRRH |
607 | }; |
608 | ||
ad71d889 SRRH |
609 | static struct ftrace_probe_ops dump_probe_ops = { |
610 | .func = ftrace_dump_probe, | |
611 | .print = ftrace_dump_print, | |
fe014e24 SRV |
612 | .init = ftrace_count_init, |
613 | .free = ftrace_count_free, | |
ad71d889 SRRH |
614 | }; |
615 | ||
90e3c03c SRRH |
616 | static struct ftrace_probe_ops cpudump_probe_ops = { |
617 | .func = ftrace_cpudump_probe, | |
618 | .print = ftrace_cpudump_print, | |
619 | }; | |
620 | ||
b6887d79 | 621 | static struct ftrace_probe_ops traceon_probe_ops = { |
23b4ff3a | 622 | .func = ftrace_traceon, |
dd42cd3e | 623 | .print = ftrace_traceon_print, |
23b4ff3a SR |
624 | }; |
625 | ||
b6887d79 | 626 | static struct ftrace_probe_ops traceoff_probe_ops = { |
23b4ff3a | 627 | .func = ftrace_traceoff, |
dd42cd3e | 628 | .print = ftrace_traceoff_print, |
23b4ff3a SR |
629 | }; |
630 | ||
dd42cd3e SRRH |
631 | static struct ftrace_probe_ops stacktrace_probe_ops = { |
632 | .func = ftrace_stacktrace, | |
633 | .print = ftrace_stacktrace_print, | |
634 | }; | |
e110e3d1 | 635 | |
23b4ff3a | 636 | static int |
04ec7bb6 SRV |
637 | ftrace_trace_probe_callback(struct trace_array *tr, |
638 | struct ftrace_probe_ops *ops, | |
dd42cd3e SRRH |
639 | struct ftrace_hash *hash, char *glob, |
640 | char *cmd, char *param, int enable) | |
23b4ff3a | 641 | { |
23b4ff3a SR |
642 | void *count = (void *)-1; |
643 | char *number; | |
644 | int ret; | |
645 | ||
646 | /* hash funcs only work with set_ftrace_filter */ | |
647 | if (!enable) | |
648 | return -EINVAL; | |
649 | ||
d3d532d7 | 650 | if (glob[0] == '!') |
7b60f3d8 | 651 | return unregister_ftrace_function_probe_func(glob+1, tr, ops); |
8b8fa62c | 652 | |
23b4ff3a SR |
653 | if (!param) |
654 | goto out_reg; | |
655 | ||
656 | number = strsep(¶m, ":"); | |
657 | ||
658 | if (!strlen(number)) | |
659 | goto out_reg; | |
660 | ||
661 | /* | |
662 | * We use the callback data field (which is a pointer) | |
663 | * as our counter. | |
664 | */ | |
bcd83ea6 | 665 | ret = kstrtoul(number, 0, (unsigned long *)&count); |
23b4ff3a SR |
666 | if (ret) |
667 | return ret; | |
668 | ||
669 | out_reg: | |
04ec7bb6 | 670 | ret = register_ftrace_function_probe(glob, tr, ops, count); |
23b4ff3a | 671 | |
04aef32d | 672 | return ret < 0 ? ret : 0; |
23b4ff3a SR |
673 | } |
674 | ||
dd42cd3e | 675 | static int |
04ec7bb6 | 676 | ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash, |
dd42cd3e SRRH |
677 | char *glob, char *cmd, char *param, int enable) |
678 | { | |
679 | struct ftrace_probe_ops *ops; | |
680 | ||
0f179765 SRV |
681 | if (!tr) |
682 | return -ENODEV; | |
683 | ||
dd42cd3e SRRH |
684 | /* we register both traceon and traceoff to this callback */ |
685 | if (strcmp(cmd, "traceon") == 0) | |
686 | ops = param ? &traceon_count_probe_ops : &traceon_probe_ops; | |
687 | else | |
688 | ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops; | |
689 | ||
04ec7bb6 | 690 | return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, |
dd42cd3e SRRH |
691 | param, enable); |
692 | } | |
693 | ||
694 | static int | |
04ec7bb6 | 695 | ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash, |
dd42cd3e SRRH |
696 | char *glob, char *cmd, char *param, int enable) |
697 | { | |
698 | struct ftrace_probe_ops *ops; | |
699 | ||
0f179765 SRV |
700 | if (!tr) |
701 | return -ENODEV; | |
702 | ||
dd42cd3e SRRH |
703 | ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops; |
704 | ||
04ec7bb6 | 705 | return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, |
dd42cd3e SRRH |
706 | param, enable); |
707 | } | |
708 | ||
ad71d889 | 709 | static int |
04ec7bb6 | 710 | ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash, |
ad71d889 SRRH |
711 | char *glob, char *cmd, char *param, int enable) |
712 | { | |
713 | struct ftrace_probe_ops *ops; | |
714 | ||
0f179765 SRV |
715 | if (!tr) |
716 | return -ENODEV; | |
717 | ||
ad71d889 SRRH |
718 | ops = &dump_probe_ops; |
719 | ||
720 | /* Only dump once. */ | |
04ec7bb6 | 721 | return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, |
ad71d889 SRRH |
722 | "1", enable); |
723 | } | |
724 | ||
90e3c03c | 725 | static int |
04ec7bb6 | 726 | ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash, |
90e3c03c SRRH |
727 | char *glob, char *cmd, char *param, int enable) |
728 | { | |
729 | struct ftrace_probe_ops *ops; | |
730 | ||
0f179765 SRV |
731 | if (!tr) |
732 | return -ENODEV; | |
733 | ||
90e3c03c SRRH |
734 | ops = &cpudump_probe_ops; |
735 | ||
736 | /* Only dump once. */ | |
04ec7bb6 | 737 | return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, |
90e3c03c SRRH |
738 | "1", enable); |
739 | } | |
740 | ||
23b4ff3a SR |
741 | static struct ftrace_func_command ftrace_traceon_cmd = { |
742 | .name = "traceon", | |
743 | .func = ftrace_trace_onoff_callback, | |
744 | }; | |
745 | ||
746 | static struct ftrace_func_command ftrace_traceoff_cmd = { | |
747 | .name = "traceoff", | |
748 | .func = ftrace_trace_onoff_callback, | |
749 | }; | |
750 | ||
dd42cd3e SRRH |
751 | static struct ftrace_func_command ftrace_stacktrace_cmd = { |
752 | .name = "stacktrace", | |
753 | .func = ftrace_stacktrace_callback, | |
754 | }; | |
755 | ||
ad71d889 SRRH |
756 | static struct ftrace_func_command ftrace_dump_cmd = { |
757 | .name = "dump", | |
758 | .func = ftrace_dump_callback, | |
759 | }; | |
760 | ||
90e3c03c SRRH |
761 | static struct ftrace_func_command ftrace_cpudump_cmd = { |
762 | .name = "cpudump", | |
763 | .func = ftrace_cpudump_callback, | |
764 | }; | |
765 | ||
23b4ff3a SR |
766 | static int __init init_func_cmd_traceon(void) |
767 | { | |
768 | int ret; | |
769 | ||
770 | ret = register_ftrace_command(&ftrace_traceoff_cmd); | |
771 | if (ret) | |
772 | return ret; | |
773 | ||
774 | ret = register_ftrace_command(&ftrace_traceon_cmd); | |
775 | if (ret) | |
ad71d889 | 776 | goto out_free_traceoff; |
dd42cd3e SRRH |
777 | |
778 | ret = register_ftrace_command(&ftrace_stacktrace_cmd); | |
ad71d889 SRRH |
779 | if (ret) |
780 | goto out_free_traceon; | |
781 | ||
782 | ret = register_ftrace_command(&ftrace_dump_cmd); | |
783 | if (ret) | |
784 | goto out_free_stacktrace; | |
785 | ||
90e3c03c SRRH |
786 | ret = register_ftrace_command(&ftrace_cpudump_cmd); |
787 | if (ret) | |
788 | goto out_free_dump; | |
789 | ||
ad71d889 SRRH |
790 | return 0; |
791 | ||
90e3c03c SRRH |
792 | out_free_dump: |
793 | unregister_ftrace_command(&ftrace_dump_cmd); | |
ad71d889 SRRH |
794 | out_free_stacktrace: |
795 | unregister_ftrace_command(&ftrace_stacktrace_cmd); | |
796 | out_free_traceon: | |
797 | unregister_ftrace_command(&ftrace_traceon_cmd); | |
798 | out_free_traceoff: | |
799 | unregister_ftrace_command(&ftrace_traceoff_cmd); | |
800 | ||
23b4ff3a SR |
801 | return ret; |
802 | } | |
803 | #else | |
804 | static inline int init_func_cmd_traceon(void) | |
805 | { | |
806 | return 0; | |
807 | } | |
808 | #endif /* CONFIG_DYNAMIC_FTRACE */ | |
809 | ||
dbeafd0d | 810 | __init int init_function_trace(void) |
1b29b018 | 811 | { |
23b4ff3a | 812 | init_func_cmd_traceon(); |
1b29b018 SR |
813 | return register_tracer(&function_trace); |
814 | } |