Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1b29b018 SR |
2 | /* |
3 | * ring buffer based function tracer | |
4 | * | |
5 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
6 | * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> | |
7 | * | |
8 | * Based on code from the latency_tracer, that is: | |
9 | * | |
10 | * Copyright (C) 2004-2006 Ingo Molnar | |
6d49e352 | 11 | * Copyright (C) 2004 Nadia Yvette Chambers |
1b29b018 | 12 | */ |
23b4ff3a | 13 | #include <linux/ring_buffer.h> |
1b29b018 SR |
14 | #include <linux/debugfs.h> |
15 | #include <linux/uaccess.h> | |
16 | #include <linux/ftrace.h> | |
f20a5806 | 17 | #include <linux/slab.h> |
2e0f5761 | 18 | #include <linux/fs.h> |
1b29b018 SR |
19 | |
20 | #include "trace.h" | |
21 | ||
f20a5806 SRRH |
22 | static void tracing_start_function_trace(struct trace_array *tr); |
23 | static void tracing_stop_function_trace(struct trace_array *tr); | |
24 | static void | |
25 | function_trace_call(unsigned long ip, unsigned long parent_ip, | |
d19ad077 | 26 | struct ftrace_ops *op, struct ftrace_regs *fregs); |
f20a5806 SRRH |
27 | static void |
28 | function_stack_trace_call(unsigned long ip, unsigned long parent_ip, | |
d19ad077 | 29 | struct ftrace_ops *op, struct ftrace_regs *fregs); |
22db095d YKV |
30 | static void |
31 | function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, | |
32 | struct ftrace_ops *op, struct ftrace_regs *fregs); | |
33 | static void | |
34 | function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, | |
35 | struct ftrace_ops *op, | |
36 | struct ftrace_regs *fregs); | |
f20a5806 SRRH |
37 | static struct tracer_flags func_flags; |
38 | ||
39 | /* Our option */ | |
40 | enum { | |
22db095d YKV |
41 | |
42 | TRACE_FUNC_NO_OPTS = 0x0, /* No flags set. */ | |
43 | TRACE_FUNC_OPT_STACK = 0x1, | |
44 | TRACE_FUNC_OPT_NO_REPEATS = 0x2, | |
45 | ||
46 | /* Update this to next highest bit. */ | |
47 | TRACE_FUNC_OPT_HIGHEST_BIT = 0x4 | |
f20a5806 SRRH |
48 | }; |
49 | ||
22db095d | 50 | #define TRACE_FUNC_OPT_MASK (TRACE_FUNC_OPT_HIGHEST_BIT - 1) |
4994891e | 51 | |
4114fbfd | 52 | int ftrace_allocate_ftrace_ops(struct trace_array *tr) |
f20a5806 SRRH |
53 | { |
54 | struct ftrace_ops *ops; | |
a225cdd2 | 55 | |
4114fbfd MH |
56 | /* The top level array uses the "global_ops" */ |
57 | if (tr->flags & TRACE_ARRAY_FL_GLOBAL) | |
58 | return 0; | |
59 | ||
f20a5806 SRRH |
60 | ops = kzalloc(sizeof(*ops), GFP_KERNEL); |
61 | if (!ops) | |
62 | return -ENOMEM; | |
53614991 | 63 | |
48a42f5d | 64 | /* Currently only the non stack version is supported */ |
f20a5806 | 65 | ops->func = function_trace_call; |
a25d036d | 66 | ops->flags = FTRACE_OPS_FL_PID; |
f20a5806 SRRH |
67 | |
68 | tr->ops = ops; | |
69 | ops->private = tr; | |
4114fbfd | 70 | |
f20a5806 SRRH |
71 | return 0; |
72 | } | |
a225cdd2 | 73 | |
4114fbfd MH |
74 | void ftrace_free_ftrace_ops(struct trace_array *tr) |
75 | { | |
76 | kfree(tr->ops); | |
77 | tr->ops = NULL; | |
78 | } | |
591dffda SRRH |
79 | |
80 | int ftrace_create_function_files(struct trace_array *tr, | |
81 | struct dentry *parent) | |
82 | { | |
5d6c97c5 SRRH |
83 | /* |
84 | * The top level array uses the "global_ops", and the files are | |
85 | * created on boot up. | |
86 | */ | |
87 | if (tr->flags & TRACE_ARRAY_FL_GLOBAL) | |
88 | return 0; | |
89 | ||
4114fbfd MH |
90 | if (!tr->ops) |
91 | return -EINVAL; | |
591dffda SRRH |
92 | |
93 | ftrace_create_filter_files(tr->ops, parent); | |
94 | ||
95 | return 0; | |
96 | } | |
97 | ||
98 | void ftrace_destroy_function_files(struct trace_array *tr) | |
99 | { | |
100 | ftrace_destroy_filter_files(tr->ops); | |
4114fbfd | 101 | ftrace_free_ftrace_ops(tr); |
591dffda SRRH |
102 | } |
103 | ||
4994891e YKV |
104 | static ftrace_func_t select_trace_function(u32 flags_val) |
105 | { | |
106 | switch (flags_val & TRACE_FUNC_OPT_MASK) { | |
107 | case TRACE_FUNC_NO_OPTS: | |
108 | return function_trace_call; | |
109 | case TRACE_FUNC_OPT_STACK: | |
110 | return function_stack_trace_call; | |
22db095d YKV |
111 | case TRACE_FUNC_OPT_NO_REPEATS: |
112 | return function_no_repeats_trace_call; | |
113 | case TRACE_FUNC_OPT_STACK | TRACE_FUNC_OPT_NO_REPEATS: | |
114 | return function_stack_no_repeats_trace_call; | |
4994891e YKV |
115 | default: |
116 | return NULL; | |
117 | } | |
118 | } | |
119 | ||
22db095d YKV |
120 | static bool handle_func_repeats(struct trace_array *tr, u32 flags_val) |
121 | { | |
122 | if (!tr->last_func_repeats && | |
123 | (flags_val & TRACE_FUNC_OPT_NO_REPEATS)) { | |
124 | tr->last_func_repeats = alloc_percpu(struct trace_func_repeats); | |
125 | if (!tr->last_func_repeats) | |
126 | return false; | |
127 | } | |
128 | ||
129 | return true; | |
130 | } | |
131 | ||
b6f11df2 | 132 | static int function_trace_init(struct trace_array *tr) |
1b29b018 | 133 | { |
4104d326 | 134 | ftrace_func_t func; |
4104d326 SRRH |
135 | /* |
136 | * Instance trace_arrays get their ops allocated | |
137 | * at instance creation. Unless it failed | |
138 | * the allocation. | |
139 | */ | |
140 | if (!tr->ops) | |
591dffda | 141 | return -ENOMEM; |
4104d326 | 142 | |
4994891e YKV |
143 | func = select_trace_function(func_flags.val); |
144 | if (!func) | |
145 | return -EINVAL; | |
4104d326 | 146 | |
22db095d YKV |
147 | if (!handle_func_repeats(tr, func_flags.val)) |
148 | return -ENOMEM; | |
149 | ||
4104d326 | 150 | ftrace_init_array_ops(tr, func); |
f20a5806 | 151 | |
18d14ebd | 152 | tr->array_buffer.cpu = raw_smp_processor_id(); |
26bc83f4 | 153 | |
41bc8144 | 154 | tracing_start_cmdline_record(); |
f20a5806 | 155 | tracing_start_function_trace(tr); |
1c80025a | 156 | return 0; |
1b29b018 SR |
157 | } |
158 | ||
e309b41d | 159 | static void function_trace_reset(struct trace_array *tr) |
1b29b018 | 160 | { |
f20a5806 | 161 | tracing_stop_function_trace(tr); |
b6f11df2 | 162 | tracing_stop_cmdline_record(); |
4104d326 | 163 | ftrace_reset_array_ops(tr); |
1b29b018 SR |
164 | } |
165 | ||
9036990d SR |
166 | static void function_trace_start(struct trace_array *tr) |
167 | { | |
1c5eb448 | 168 | tracing_reset_online_cpus(&tr->array_buffer); |
9036990d SR |
169 | } |
170 | ||
bb3c3c95 | 171 | static void |
2f5f6ad9 | 172 | function_trace_call(unsigned long ip, unsigned long parent_ip, |
d19ad077 | 173 | struct ftrace_ops *op, struct ftrace_regs *fregs) |
bb3c3c95 | 174 | { |
f20a5806 | 175 | struct trace_array *tr = op->private; |
bb3c3c95 | 176 | struct trace_array_cpu *data; |
36590c50 | 177 | unsigned int trace_ctx; |
d41032a8 | 178 | int bit; |
bb3c3c95 | 179 | int cpu; |
bb3c3c95 | 180 | |
f20a5806 | 181 | if (unlikely(!tr->function_enabled)) |
bb3c3c95 SR |
182 | return; |
183 | ||
773c1670 | 184 | bit = ftrace_test_recursion_trylock(ip, parent_ip); |
6e4eb9cb SRV |
185 | if (bit < 0) |
186 | return; | |
187 | ||
36590c50 | 188 | trace_ctx = tracing_gen_ctx(); |
bb3c3c95 | 189 | |
897f68a4 | 190 | cpu = smp_processor_id(); |
1c5eb448 | 191 | data = per_cpu_ptr(tr->array_buffer.data, cpu); |
36590c50 SAS |
192 | if (!atomic_read(&data->disabled)) |
193 | trace_function(tr, ip, parent_ip, trace_ctx); | |
194 | ||
6e4eb9cb | 195 | ftrace_test_recursion_unlock(bit); |
bb3c3c95 SR |
196 | } |
197 | ||
2ee5b92a SRV |
198 | #ifdef CONFIG_UNWINDER_ORC |
199 | /* | |
200 | * Skip 2: | |
201 | * | |
202 | * function_stack_trace_call() | |
203 | * ftrace_call() | |
204 | */ | |
205 | #define STACK_SKIP 2 | |
206 | #else | |
207 | /* | |
208 | * Skip 3: | |
209 | * __trace_stack() | |
210 | * function_stack_trace_call() | |
211 | * ftrace_call() | |
212 | */ | |
213 | #define STACK_SKIP 3 | |
214 | #endif | |
215 | ||
53614991 | 216 | static void |
2f5f6ad9 | 217 | function_stack_trace_call(unsigned long ip, unsigned long parent_ip, |
d19ad077 | 218 | struct ftrace_ops *op, struct ftrace_regs *fregs) |
53614991 | 219 | { |
f20a5806 | 220 | struct trace_array *tr = op->private; |
53614991 SR |
221 | struct trace_array_cpu *data; |
222 | unsigned long flags; | |
223 | long disabled; | |
224 | int cpu; | |
36590c50 | 225 | unsigned int trace_ctx; |
53614991 | 226 | |
f20a5806 | 227 | if (unlikely(!tr->function_enabled)) |
53614991 SR |
228 | return; |
229 | ||
230 | /* | |
231 | * Need to use raw, since this must be called before the | |
232 | * recursive protection is performed. | |
233 | */ | |
234 | local_irq_save(flags); | |
235 | cpu = raw_smp_processor_id(); | |
1c5eb448 | 236 | data = per_cpu_ptr(tr->array_buffer.data, cpu); |
53614991 SR |
237 | disabled = atomic_inc_return(&data->disabled); |
238 | ||
239 | if (likely(disabled == 1)) { | |
36590c50 SAS |
240 | trace_ctx = tracing_gen_ctx_flags(flags); |
241 | trace_function(tr, ip, parent_ip, trace_ctx); | |
242 | __trace_stack(tr, trace_ctx, STACK_SKIP); | |
53614991 SR |
243 | } |
244 | ||
245 | atomic_dec(&data->disabled); | |
246 | local_irq_restore(flags); | |
247 | } | |
248 | ||
22db095d YKV |
249 | static inline bool is_repeat_check(struct trace_array *tr, |
250 | struct trace_func_repeats *last_info, | |
251 | unsigned long ip, unsigned long parent_ip) | |
252 | { | |
253 | if (last_info->ip == ip && | |
254 | last_info->parent_ip == parent_ip && | |
255 | last_info->count < U16_MAX) { | |
256 | last_info->ts_last_call = | |
257 | ring_buffer_time_stamp(tr->array_buffer.buffer); | |
258 | last_info->count++; | |
259 | return true; | |
260 | } | |
261 | ||
262 | return false; | |
263 | } | |
264 | ||
265 | static inline void process_repeats(struct trace_array *tr, | |
266 | unsigned long ip, unsigned long parent_ip, | |
267 | struct trace_func_repeats *last_info, | |
268 | unsigned int trace_ctx) | |
269 | { | |
270 | if (last_info->count) { | |
271 | trace_last_func_repeats(tr, last_info, trace_ctx); | |
272 | last_info->count = 0; | |
273 | } | |
274 | ||
275 | last_info->ip = ip; | |
276 | last_info->parent_ip = parent_ip; | |
277 | } | |
278 | ||
279 | static void | |
280 | function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, | |
281 | struct ftrace_ops *op, | |
282 | struct ftrace_regs *fregs) | |
283 | { | |
284 | struct trace_func_repeats *last_info; | |
285 | struct trace_array *tr = op->private; | |
286 | struct trace_array_cpu *data; | |
287 | unsigned int trace_ctx; | |
288 | unsigned long flags; | |
289 | int bit; | |
290 | int cpu; | |
291 | ||
292 | if (unlikely(!tr->function_enabled)) | |
293 | return; | |
294 | ||
295 | bit = ftrace_test_recursion_trylock(ip, parent_ip); | |
296 | if (bit < 0) | |
297 | return; | |
298 | ||
22db095d YKV |
299 | cpu = smp_processor_id(); |
300 | data = per_cpu_ptr(tr->array_buffer.data, cpu); | |
301 | if (atomic_read(&data->disabled)) | |
302 | goto out; | |
303 | ||
304 | /* | |
305 | * An interrupt may happen at any place here. But as far as I can see, | |
306 | * the only damage that this can cause is to mess up the repetition | |
307 | * counter without valuable data being lost. | |
308 | * TODO: think about a solution that is better than just hoping to be | |
309 | * lucky. | |
310 | */ | |
311 | last_info = per_cpu_ptr(tr->last_func_repeats, cpu); | |
312 | if (is_repeat_check(tr, last_info, ip, parent_ip)) | |
313 | goto out; | |
314 | ||
315 | local_save_flags(flags); | |
316 | trace_ctx = tracing_gen_ctx_flags(flags); | |
317 | process_repeats(tr, ip, parent_ip, last_info, trace_ctx); | |
318 | ||
319 | trace_function(tr, ip, parent_ip, trace_ctx); | |
320 | ||
321 | out: | |
322 | ftrace_test_recursion_unlock(bit); | |
22db095d YKV |
323 | } |
324 | ||
325 | static void | |
326 | function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip, | |
327 | struct ftrace_ops *op, | |
328 | struct ftrace_regs *fregs) | |
329 | { | |
330 | struct trace_func_repeats *last_info; | |
331 | struct trace_array *tr = op->private; | |
332 | struct trace_array_cpu *data; | |
333 | unsigned long flags; | |
334 | long disabled; | |
335 | int cpu; | |
336 | unsigned int trace_ctx; | |
337 | ||
338 | if (unlikely(!tr->function_enabled)) | |
339 | return; | |
340 | ||
341 | /* | |
342 | * Need to use raw, since this must be called before the | |
343 | * recursive protection is performed. | |
344 | */ | |
345 | local_irq_save(flags); | |
346 | cpu = raw_smp_processor_id(); | |
347 | data = per_cpu_ptr(tr->array_buffer.data, cpu); | |
348 | disabled = atomic_inc_return(&data->disabled); | |
349 | ||
350 | if (likely(disabled == 1)) { | |
351 | last_info = per_cpu_ptr(tr->last_func_repeats, cpu); | |
352 | if (is_repeat_check(tr, last_info, ip, parent_ip)) | |
353 | goto out; | |
354 | ||
355 | trace_ctx = tracing_gen_ctx_flags(flags); | |
356 | process_repeats(tr, ip, parent_ip, last_info, trace_ctx); | |
357 | ||
358 | trace_function(tr, ip, parent_ip, trace_ctx); | |
359 | __trace_stack(tr, trace_ctx, STACK_SKIP); | |
360 | } | |
361 | ||
362 | out: | |
363 | atomic_dec(&data->disabled); | |
364 | local_irq_restore(flags); | |
365 | } | |
366 | ||
53614991 SR |
367 | static struct tracer_opt func_opts[] = { |
368 | #ifdef CONFIG_STACKTRACE | |
369 | { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) }, | |
370 | #endif | |
22db095d | 371 | { TRACER_OPT(func-no-repeats, TRACE_FUNC_OPT_NO_REPEATS) }, |
53614991 SR |
372 | { } /* Always set a last empty entry */ |
373 | }; | |
374 | ||
375 | static struct tracer_flags func_flags = { | |
4994891e | 376 | .val = TRACE_FUNC_NO_OPTS, /* By default: all flags disabled */ |
53614991 SR |
377 | .opts = func_opts |
378 | }; | |
379 | ||
f20a5806 | 380 | static void tracing_start_function_trace(struct trace_array *tr) |
3eb36aa0 | 381 | { |
f20a5806 SRRH |
382 | tr->function_enabled = 0; |
383 | register_ftrace_function(tr->ops); | |
384 | tr->function_enabled = 1; | |
3eb36aa0 SR |
385 | } |
386 | ||
f20a5806 | 387 | static void tracing_stop_function_trace(struct trace_array *tr) |
3eb36aa0 | 388 | { |
f20a5806 SRRH |
389 | tr->function_enabled = 0; |
390 | unregister_ftrace_function(tr->ops); | |
3eb36aa0 SR |
391 | } |
392 | ||
d39cdd20 CH |
393 | static struct tracer function_trace; |
394 | ||
8c1a49ae SRRH |
395 | static int |
396 | func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) | |
53614991 | 397 | { |
4994891e YKV |
398 | ftrace_func_t func; |
399 | u32 new_flags; | |
d39cdd20 | 400 | |
4994891e YKV |
401 | /* Do nothing if already set. */ |
402 | if (!!set == !!(func_flags.val & bit)) | |
403 | return 0; | |
f20a5806 | 404 | |
4994891e YKV |
405 | /* We can change this flag only when not running. */ |
406 | if (tr->current_trace != &function_trace) | |
407 | return 0; | |
53614991 | 408 | |
4994891e YKV |
409 | new_flags = (func_flags.val & ~bit) | (set ? bit : 0); |
410 | func = select_trace_function(new_flags); | |
411 | if (!func) | |
f555f123 | 412 | return -EINVAL; |
4994891e YKV |
413 | |
414 | /* Check if there's anything to change. */ | |
415 | if (tr->ops->func == func) | |
416 | return 0; | |
417 | ||
22db095d YKV |
418 | if (!handle_func_repeats(tr, new_flags)) |
419 | return -ENOMEM; | |
420 | ||
4994891e YKV |
421 | unregister_ftrace_function(tr->ops); |
422 | tr->ops->func = func; | |
423 | register_ftrace_function(tr->ops); | |
53614991 | 424 | |
f555f123 | 425 | return 0; |
53614991 SR |
426 | } |
427 | ||
8f768993 | 428 | static struct tracer function_trace __tracer_data = |
1b29b018 | 429 | { |
3eb36aa0 SR |
430 | .name = "function", |
431 | .init = function_trace_init, | |
432 | .reset = function_trace_reset, | |
433 | .start = function_trace_start, | |
53614991 SR |
434 | .flags = &func_flags, |
435 | .set_flag = func_set_flag, | |
f20a5806 | 436 | .allow_instances = true, |
60a11774 | 437 | #ifdef CONFIG_FTRACE_SELFTEST |
3eb36aa0 | 438 | .selftest = trace_selftest_startup_function, |
60a11774 | 439 | #endif |
1b29b018 SR |
440 | }; |
441 | ||
23b4ff3a | 442 | #ifdef CONFIG_DYNAMIC_FTRACE |
fe014e24 | 443 | static void update_traceon_count(struct ftrace_probe_ops *ops, |
2290f2c5 SRV |
444 | unsigned long ip, |
445 | struct trace_array *tr, bool on, | |
6e444319 | 446 | void *data) |
23b4ff3a | 447 | { |
6e444319 | 448 | struct ftrace_func_mapper *mapper = data; |
fe014e24 SRV |
449 | long *count; |
450 | long old_count; | |
23b4ff3a | 451 | |
a9ce7c36 SRRH |
452 | /* |
453 | * Tracing gets disabled (or enabled) once per count. | |
0af26492 | 454 | * This function can be called at the same time on multiple CPUs. |
a9ce7c36 SRRH |
455 | * It is fine if both disable (or enable) tracing, as disabling |
456 | * (or enabling) the second time doesn't do anything as the | |
457 | * state of the tracer is already disabled (or enabled). | |
458 | * What needs to be synchronized in this case is that the count | |
459 | * only gets decremented once, even if the tracer is disabled | |
460 | * (or enabled) twice, as the second one is really a nop. | |
461 | * | |
462 | * The memory barriers guarantee that we only decrement the | |
463 | * counter once. First the count is read to a local variable | |
464 | * and a read barrier is used to make sure that it is loaded | |
465 | * before checking if the tracer is in the state we want. | |
466 | * If the tracer is not in the state we want, then the count | |
467 | * is guaranteed to be the old count. | |
468 | * | |
469 | * Next the tracer is set to the state we want (disabled or enabled) | |
470 | * then a write memory barrier is used to make sure that | |
471 | * the new state is visible before changing the counter by | |
472 | * one minus the old counter. This guarantees that another CPU | |
473 | * executing this code will see the new state before seeing | |
0af26492 | 474 | * the new counter value, and would not do anything if the new |
a9ce7c36 SRRH |
475 | * counter is seen. |
476 | * | |
477 | * Note, there is no synchronization between this and a user | |
478 | * setting the tracing_on file. But we currently don't care | |
479 | * about that. | |
480 | */ | |
fe014e24 SRV |
481 | count = (long *)ftrace_func_mapper_find_ip(mapper, ip); |
482 | old_count = *count; | |
483 | ||
484 | if (old_count <= 0) | |
a9ce7c36 | 485 | return; |
23b4ff3a | 486 | |
a9ce7c36 SRRH |
487 | /* Make sure we see count before checking tracing state */ |
488 | smp_rmb(); | |
23b4ff3a | 489 | |
2290f2c5 | 490 | if (on == !!tracer_tracing_is_on(tr)) |
a9ce7c36 SRRH |
491 | return; |
492 | ||
493 | if (on) | |
2290f2c5 | 494 | tracer_tracing_on(tr); |
a9ce7c36 | 495 | else |
2290f2c5 | 496 | tracer_tracing_off(tr); |
a9ce7c36 | 497 | |
a9ce7c36 SRRH |
498 | /* Make sure tracing state is visible before updating count */ |
499 | smp_wmb(); | |
500 | ||
501 | *count = old_count - 1; | |
23b4ff3a SR |
502 | } |
503 | ||
504 | static void | |
bca6c8d0 | 505 | ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 506 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 507 | void *data) |
23b4ff3a | 508 | { |
2290f2c5 | 509 | update_traceon_count(ops, ip, tr, 1, data); |
1c317143 | 510 | } |
23b4ff3a | 511 | |
1c317143 | 512 | static void |
bca6c8d0 | 513 | ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 514 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 515 | void *data) |
1c317143 | 516 | { |
2290f2c5 | 517 | update_traceon_count(ops, ip, tr, 0, data); |
23b4ff3a SR |
518 | } |
519 | ||
8380d248 | 520 | static void |
bca6c8d0 | 521 | ftrace_traceon(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 522 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 523 | void *data) |
8380d248 | 524 | { |
2290f2c5 | 525 | if (tracer_tracing_is_on(tr)) |
8380d248 SRRH |
526 | return; |
527 | ||
2290f2c5 | 528 | tracer_tracing_on(tr); |
8380d248 SRRH |
529 | } |
530 | ||
531 | static void | |
bca6c8d0 | 532 | ftrace_traceoff(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 533 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 534 | void *data) |
8380d248 | 535 | { |
2290f2c5 | 536 | if (!tracer_tracing_is_on(tr)) |
8380d248 SRRH |
537 | return; |
538 | ||
2290f2c5 | 539 | tracer_tracing_off(tr); |
8380d248 SRRH |
540 | } |
541 | ||
2ee5b92a SRV |
542 | #ifdef CONFIG_UNWINDER_ORC |
543 | /* | |
544 | * Skip 3: | |
545 | * | |
546 | * function_trace_probe_call() | |
547 | * ftrace_ops_assist_func() | |
548 | * ftrace_call() | |
549 | */ | |
550 | #define FTRACE_STACK_SKIP 3 | |
551 | #else | |
dd42cd3e | 552 | /* |
2ee5b92a SRV |
553 | * Skip 5: |
554 | * | |
555 | * __trace_stack() | |
dd42cd3e SRRH |
556 | * ftrace_stacktrace() |
557 | * function_trace_probe_call() | |
2ee5b92a | 558 | * ftrace_ops_assist_func() |
dd42cd3e SRRH |
559 | * ftrace_call() |
560 | */ | |
2ee5b92a SRV |
561 | #define FTRACE_STACK_SKIP 5 |
562 | #endif | |
dd42cd3e | 563 | |
dcc19d28 SRV |
564 | static __always_inline void trace_stack(struct trace_array *tr) |
565 | { | |
36590c50 | 566 | unsigned int trace_ctx; |
dcc19d28 | 567 | |
36590c50 | 568 | trace_ctx = tracing_gen_ctx(); |
dcc19d28 | 569 | |
36590c50 | 570 | __trace_stack(tr, trace_ctx, FTRACE_STACK_SKIP); |
dcc19d28 SRV |
571 | } |
572 | ||
dd42cd3e | 573 | static void |
bca6c8d0 | 574 | ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 575 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 576 | void *data) |
dd42cd3e | 577 | { |
dcc19d28 | 578 | trace_stack(tr); |
dd42cd3e SRRH |
579 | } |
580 | ||
581 | static void | |
bca6c8d0 | 582 | ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 583 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 584 | void *data) |
dd42cd3e | 585 | { |
6e444319 | 586 | struct ftrace_func_mapper *mapper = data; |
fe014e24 | 587 | long *count; |
a9ce7c36 SRRH |
588 | long old_count; |
589 | long new_count; | |
590 | ||
fe014e24 SRV |
591 | if (!tracing_is_on()) |
592 | return; | |
593 | ||
594 | /* unlimited? */ | |
595 | if (!mapper) { | |
dcc19d28 | 596 | trace_stack(tr); |
fe014e24 SRV |
597 | return; |
598 | } | |
599 | ||
600 | count = (long *)ftrace_func_mapper_find_ip(mapper, ip); | |
601 | ||
a9ce7c36 SRRH |
602 | /* |
603 | * Stack traces should only execute the number of times the | |
604 | * user specified in the counter. | |
605 | */ | |
606 | do { | |
a9ce7c36 SRRH |
607 | old_count = *count; |
608 | ||
609 | if (!old_count) | |
610 | return; | |
611 | ||
a9ce7c36 SRRH |
612 | new_count = old_count - 1; |
613 | new_count = cmpxchg(count, old_count, new_count); | |
614 | if (new_count == old_count) | |
dcc19d28 | 615 | trace_stack(tr); |
a9ce7c36 | 616 | |
fe014e24 SRV |
617 | if (!tracing_is_on()) |
618 | return; | |
619 | ||
a9ce7c36 SRRH |
620 | } while (new_count != old_count); |
621 | } | |
622 | ||
6e444319 SRV |
623 | static int update_count(struct ftrace_probe_ops *ops, unsigned long ip, |
624 | void *data) | |
a9ce7c36 | 625 | { |
6e444319 | 626 | struct ftrace_func_mapper *mapper = data; |
fe014e24 | 627 | long *count = NULL; |
a9ce7c36 | 628 | |
fe014e24 SRV |
629 | if (mapper) |
630 | count = (long *)ftrace_func_mapper_find_ip(mapper, ip); | |
a9ce7c36 | 631 | |
fe014e24 SRV |
632 | if (count) { |
633 | if (*count <= 0) | |
634 | return 0; | |
a9ce7c36 | 635 | (*count)--; |
fe014e24 | 636 | } |
a9ce7c36 SRRH |
637 | |
638 | return 1; | |
dd42cd3e SRRH |
639 | } |
640 | ||
ad71d889 | 641 | static void |
bca6c8d0 | 642 | ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 643 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 644 | void *data) |
ad71d889 | 645 | { |
6e444319 | 646 | if (update_count(ops, ip, data)) |
ad71d889 SRRH |
647 | ftrace_dump(DUMP_ALL); |
648 | } | |
649 | ||
90e3c03c SRRH |
650 | /* Only dump the current CPU buffer. */ |
651 | static void | |
bca6c8d0 | 652 | ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, |
b5f081b5 | 653 | struct trace_array *tr, struct ftrace_probe_ops *ops, |
6e444319 | 654 | void *data) |
90e3c03c | 655 | { |
6e444319 | 656 | if (update_count(ops, ip, data)) |
90e3c03c SRRH |
657 | ftrace_dump(DUMP_ORIG); |
658 | } | |
659 | ||
e110e3d1 | 660 | static int |
dd42cd3e | 661 | ftrace_probe_print(const char *name, struct seq_file *m, |
6e444319 SRV |
662 | unsigned long ip, struct ftrace_probe_ops *ops, |
663 | void *data) | |
dd42cd3e | 664 | { |
6e444319 | 665 | struct ftrace_func_mapper *mapper = data; |
fe014e24 | 666 | long *count = NULL; |
dd42cd3e SRRH |
667 | |
668 | seq_printf(m, "%ps:%s", (void *)ip, name); | |
669 | ||
fe014e24 SRV |
670 | if (mapper) |
671 | count = (long *)ftrace_func_mapper_find_ip(mapper, ip); | |
672 | ||
673 | if (count) | |
674 | seq_printf(m, ":count=%ld\n", *count); | |
dd42cd3e | 675 | else |
fe014e24 | 676 | seq_puts(m, ":unlimited\n"); |
dd42cd3e SRRH |
677 | |
678 | return 0; | |
679 | } | |
680 | ||
681 | static int | |
682 | ftrace_traceon_print(struct seq_file *m, unsigned long ip, | |
b5f081b5 SRV |
683 | struct ftrace_probe_ops *ops, |
684 | void *data) | |
dd42cd3e | 685 | { |
6e444319 | 686 | return ftrace_probe_print("traceon", m, ip, ops, data); |
dd42cd3e SRRH |
687 | } |
688 | ||
689 | static int | |
690 | ftrace_traceoff_print(struct seq_file *m, unsigned long ip, | |
691 | struct ftrace_probe_ops *ops, void *data) | |
692 | { | |
6e444319 | 693 | return ftrace_probe_print("traceoff", m, ip, ops, data); |
dd42cd3e SRRH |
694 | } |
695 | ||
696 | static int | |
697 | ftrace_stacktrace_print(struct seq_file *m, unsigned long ip, | |
698 | struct ftrace_probe_ops *ops, void *data) | |
699 | { | |
6e444319 | 700 | return ftrace_probe_print("stacktrace", m, ip, ops, data); |
dd42cd3e | 701 | } |
e110e3d1 | 702 | |
ad71d889 SRRH |
703 | static int |
704 | ftrace_dump_print(struct seq_file *m, unsigned long ip, | |
705 | struct ftrace_probe_ops *ops, void *data) | |
706 | { | |
6e444319 | 707 | return ftrace_probe_print("dump", m, ip, ops, data); |
ad71d889 SRRH |
708 | } |
709 | ||
90e3c03c SRRH |
710 | static int |
711 | ftrace_cpudump_print(struct seq_file *m, unsigned long ip, | |
712 | struct ftrace_probe_ops *ops, void *data) | |
713 | { | |
6e444319 | 714 | return ftrace_probe_print("cpudump", m, ip, ops, data); |
fe014e24 SRV |
715 | } |
716 | ||
717 | ||
718 | static int | |
b5f081b5 | 719 | ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr, |
6e444319 | 720 | unsigned long ip, void *init_data, void **data) |
fe014e24 | 721 | { |
6e444319 SRV |
722 | struct ftrace_func_mapper *mapper = *data; |
723 | ||
724 | if (!mapper) { | |
725 | mapper = allocate_ftrace_func_mapper(); | |
726 | if (!mapper) | |
727 | return -ENOMEM; | |
728 | *data = mapper; | |
729 | } | |
fe014e24 | 730 | |
6e444319 | 731 | return ftrace_func_mapper_add_ip(mapper, ip, init_data); |
fe014e24 SRV |
732 | } |
733 | ||
734 | static void | |
b5f081b5 | 735 | ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr, |
6e444319 | 736 | unsigned long ip, void *data) |
fe014e24 | 737 | { |
6e444319 SRV |
738 | struct ftrace_func_mapper *mapper = data; |
739 | ||
740 | if (!ip) { | |
741 | free_ftrace_func_mapper(mapper, NULL); | |
742 | return; | |
743 | } | |
fe014e24 SRV |
744 | |
745 | ftrace_func_mapper_remove_ip(mapper, ip); | |
90e3c03c SRRH |
746 | } |
747 | ||
8380d248 SRRH |
748 | static struct ftrace_probe_ops traceon_count_probe_ops = { |
749 | .func = ftrace_traceon_count, | |
dd42cd3e | 750 | .print = ftrace_traceon_print, |
fe014e24 SRV |
751 | .init = ftrace_count_init, |
752 | .free = ftrace_count_free, | |
8380d248 SRRH |
753 | }; |
754 | ||
755 | static struct ftrace_probe_ops traceoff_count_probe_ops = { | |
756 | .func = ftrace_traceoff_count, | |
dd42cd3e | 757 | .print = ftrace_traceoff_print, |
fe014e24 SRV |
758 | .init = ftrace_count_init, |
759 | .free = ftrace_count_free, | |
dd42cd3e SRRH |
760 | }; |
761 | ||
762 | static struct ftrace_probe_ops stacktrace_count_probe_ops = { | |
763 | .func = ftrace_stacktrace_count, | |
764 | .print = ftrace_stacktrace_print, | |
fe014e24 SRV |
765 | .init = ftrace_count_init, |
766 | .free = ftrace_count_free, | |
8380d248 SRRH |
767 | }; |
768 | ||
ad71d889 SRRH |
769 | static struct ftrace_probe_ops dump_probe_ops = { |
770 | .func = ftrace_dump_probe, | |
771 | .print = ftrace_dump_print, | |
fe014e24 SRV |
772 | .init = ftrace_count_init, |
773 | .free = ftrace_count_free, | |
ad71d889 SRRH |
774 | }; |
775 | ||
90e3c03c SRRH |
776 | static struct ftrace_probe_ops cpudump_probe_ops = { |
777 | .func = ftrace_cpudump_probe, | |
778 | .print = ftrace_cpudump_print, | |
779 | }; | |
780 | ||
b6887d79 | 781 | static struct ftrace_probe_ops traceon_probe_ops = { |
23b4ff3a | 782 | .func = ftrace_traceon, |
dd42cd3e | 783 | .print = ftrace_traceon_print, |
23b4ff3a SR |
784 | }; |
785 | ||
b6887d79 | 786 | static struct ftrace_probe_ops traceoff_probe_ops = { |
23b4ff3a | 787 | .func = ftrace_traceoff, |
dd42cd3e | 788 | .print = ftrace_traceoff_print, |
23b4ff3a SR |
789 | }; |
790 | ||
dd42cd3e SRRH |
791 | static struct ftrace_probe_ops stacktrace_probe_ops = { |
792 | .func = ftrace_stacktrace, | |
793 | .print = ftrace_stacktrace_print, | |
794 | }; | |
e110e3d1 | 795 | |
23b4ff3a | 796 | static int |
04ec7bb6 SRV |
797 | ftrace_trace_probe_callback(struct trace_array *tr, |
798 | struct ftrace_probe_ops *ops, | |
dd42cd3e SRRH |
799 | struct ftrace_hash *hash, char *glob, |
800 | char *cmd, char *param, int enable) | |
23b4ff3a | 801 | { |
23b4ff3a SR |
802 | void *count = (void *)-1; |
803 | char *number; | |
804 | int ret; | |
805 | ||
806 | /* hash funcs only work with set_ftrace_filter */ | |
807 | if (!enable) | |
808 | return -EINVAL; | |
809 | ||
d3d532d7 | 810 | if (glob[0] == '!') |
7b60f3d8 | 811 | return unregister_ftrace_function_probe_func(glob+1, tr, ops); |
8b8fa62c | 812 | |
23b4ff3a SR |
813 | if (!param) |
814 | goto out_reg; | |
815 | ||
816 | number = strsep(¶m, ":"); | |
817 | ||
818 | if (!strlen(number)) | |
819 | goto out_reg; | |
820 | ||
821 | /* | |
822 | * We use the callback data field (which is a pointer) | |
823 | * as our counter. | |
824 | */ | |
bcd83ea6 | 825 | ret = kstrtoul(number, 0, (unsigned long *)&count); |
23b4ff3a SR |
826 | if (ret) |
827 | return ret; | |
828 | ||
829 | out_reg: | |
04ec7bb6 | 830 | ret = register_ftrace_function_probe(glob, tr, ops, count); |
23b4ff3a | 831 | |
04aef32d | 832 | return ret < 0 ? ret : 0; |
23b4ff3a SR |
833 | } |
834 | ||
dd42cd3e | 835 | static int |
04ec7bb6 | 836 | ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash, |
dd42cd3e SRRH |
837 | char *glob, char *cmd, char *param, int enable) |
838 | { | |
839 | struct ftrace_probe_ops *ops; | |
840 | ||
0f179765 SRV |
841 | if (!tr) |
842 | return -ENODEV; | |
843 | ||
dd42cd3e SRRH |
844 | /* we register both traceon and traceoff to this callback */ |
845 | if (strcmp(cmd, "traceon") == 0) | |
846 | ops = param ? &traceon_count_probe_ops : &traceon_probe_ops; | |
847 | else | |
848 | ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops; | |
849 | ||
04ec7bb6 | 850 | return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, |
dd42cd3e SRRH |
851 | param, enable); |
852 | } | |
853 | ||
854 | static int | |
04ec7bb6 | 855 | ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash, |
dd42cd3e SRRH |
856 | char *glob, char *cmd, char *param, int enable) |
857 | { | |
858 | struct ftrace_probe_ops *ops; | |
859 | ||
0f179765 SRV |
860 | if (!tr) |
861 | return -ENODEV; | |
862 | ||
dd42cd3e SRRH |
863 | ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops; |
864 | ||
04ec7bb6 | 865 | return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, |
dd42cd3e SRRH |
866 | param, enable); |
867 | } | |
868 | ||
ad71d889 | 869 | static int |
04ec7bb6 | 870 | ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash, |
ad71d889 SRRH |
871 | char *glob, char *cmd, char *param, int enable) |
872 | { | |
873 | struct ftrace_probe_ops *ops; | |
874 | ||
0f179765 SRV |
875 | if (!tr) |
876 | return -ENODEV; | |
877 | ||
ad71d889 SRRH |
878 | ops = &dump_probe_ops; |
879 | ||
880 | /* Only dump once. */ | |
04ec7bb6 | 881 | return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, |
ad71d889 SRRH |
882 | "1", enable); |
883 | } | |
884 | ||
90e3c03c | 885 | static int |
04ec7bb6 | 886 | ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash, |
90e3c03c SRRH |
887 | char *glob, char *cmd, char *param, int enable) |
888 | { | |
889 | struct ftrace_probe_ops *ops; | |
890 | ||
0f179765 SRV |
891 | if (!tr) |
892 | return -ENODEV; | |
893 | ||
90e3c03c SRRH |
894 | ops = &cpudump_probe_ops; |
895 | ||
896 | /* Only dump once. */ | |
04ec7bb6 | 897 | return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd, |
90e3c03c SRRH |
898 | "1", enable); |
899 | } | |
900 | ||
23b4ff3a SR |
901 | static struct ftrace_func_command ftrace_traceon_cmd = { |
902 | .name = "traceon", | |
903 | .func = ftrace_trace_onoff_callback, | |
904 | }; | |
905 | ||
906 | static struct ftrace_func_command ftrace_traceoff_cmd = { | |
907 | .name = "traceoff", | |
908 | .func = ftrace_trace_onoff_callback, | |
909 | }; | |
910 | ||
dd42cd3e SRRH |
911 | static struct ftrace_func_command ftrace_stacktrace_cmd = { |
912 | .name = "stacktrace", | |
913 | .func = ftrace_stacktrace_callback, | |
914 | }; | |
915 | ||
ad71d889 SRRH |
916 | static struct ftrace_func_command ftrace_dump_cmd = { |
917 | .name = "dump", | |
918 | .func = ftrace_dump_callback, | |
919 | }; | |
920 | ||
90e3c03c SRRH |
921 | static struct ftrace_func_command ftrace_cpudump_cmd = { |
922 | .name = "cpudump", | |
923 | .func = ftrace_cpudump_callback, | |
924 | }; | |
925 | ||
23b4ff3a SR |
926 | static int __init init_func_cmd_traceon(void) |
927 | { | |
928 | int ret; | |
929 | ||
930 | ret = register_ftrace_command(&ftrace_traceoff_cmd); | |
931 | if (ret) | |
932 | return ret; | |
933 | ||
934 | ret = register_ftrace_command(&ftrace_traceon_cmd); | |
935 | if (ret) | |
ad71d889 | 936 | goto out_free_traceoff; |
dd42cd3e SRRH |
937 | |
938 | ret = register_ftrace_command(&ftrace_stacktrace_cmd); | |
ad71d889 SRRH |
939 | if (ret) |
940 | goto out_free_traceon; | |
941 | ||
942 | ret = register_ftrace_command(&ftrace_dump_cmd); | |
943 | if (ret) | |
944 | goto out_free_stacktrace; | |
945 | ||
90e3c03c SRRH |
946 | ret = register_ftrace_command(&ftrace_cpudump_cmd); |
947 | if (ret) | |
948 | goto out_free_dump; | |
949 | ||
ad71d889 SRRH |
950 | return 0; |
951 | ||
90e3c03c SRRH |
952 | out_free_dump: |
953 | unregister_ftrace_command(&ftrace_dump_cmd); | |
ad71d889 SRRH |
954 | out_free_stacktrace: |
955 | unregister_ftrace_command(&ftrace_stacktrace_cmd); | |
956 | out_free_traceon: | |
957 | unregister_ftrace_command(&ftrace_traceon_cmd); | |
958 | out_free_traceoff: | |
959 | unregister_ftrace_command(&ftrace_traceoff_cmd); | |
960 | ||
23b4ff3a SR |
961 | return ret; |
962 | } | |
963 | #else | |
964 | static inline int init_func_cmd_traceon(void) | |
965 | { | |
966 | return 0; | |
967 | } | |
968 | #endif /* CONFIG_DYNAMIC_FTRACE */ | |
969 | ||
dbeafd0d | 970 | __init int init_function_trace(void) |
1b29b018 | 971 | { |
23b4ff3a | 972 | init_func_cmd_traceon(); |
1b29b018 SR |
973 | return register_tracer(&function_trace); |
974 | } |