Commit | Line | Data |
---|---|---|
bcea3f96 | 1 | // SPDX-License-Identifier: GPL-2.0 |
81d68a96 | 2 | /* |
73d8b8bc | 3 | * trace irqs off critical timings |
81d68a96 SR |
4 | * |
5 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
6 | * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> | |
7 | * | |
8 | * From code in the latency_tracer, that is: | |
9 | * | |
10 | * Copyright (C) 2004-2006 Ingo Molnar | |
6d49e352 | 11 | * Copyright (C) 2004 Nadia Yvette Chambers |
81d68a96 SR |
12 | */ |
13 | #include <linux/kallsyms.h> | |
81d68a96 SR |
14 | #include <linux/uaccess.h> |
15 | #include <linux/module.h> | |
16 | #include <linux/ftrace.h> | |
eeeb080b | 17 | #include <linux/kprobes.h> |
81d68a96 SR |
18 | |
19 | #include "trace.h" | |
20 | ||
d5915816 JF |
21 | #include <trace/events/preemptirq.h> |
22 | ||
aaecaa0b | 23 | #if defined(CONFIG_IRQSOFF_TRACER) || defined(CONFIG_PREEMPT_TRACER) |
81d68a96 SR |
24 | static struct trace_array *irqsoff_trace __read_mostly; |
25 | static int tracer_enabled __read_mostly; | |
26 | ||
6cd8a4bb SR |
27 | static DEFINE_PER_CPU(int, tracing_cpu); |
28 | ||
5389f6fa | 29 | static DEFINE_RAW_SPINLOCK(max_trace_lock); |
89b2f978 | 30 | |
6cd8a4bb SR |
31 | enum { |
32 | TRACER_IRQS_OFF = (1 << 1), | |
33 | TRACER_PREEMPT_OFF = (1 << 2), | |
34 | }; | |
35 | ||
36 | static int trace_type __read_mostly; | |
37 | ||
613f04a0 | 38 | static int save_flags; |
e9d25fe6 | 39 | |
62b915f1 JO |
40 | static void stop_irqsoff_tracer(struct trace_array *tr, int graph); |
41 | static int start_irqsoff_tracer(struct trace_array *tr, int graph); | |
42 | ||
6cd8a4bb | 43 | #ifdef CONFIG_PREEMPT_TRACER |
e309b41d | 44 | static inline int |
f27107fa | 45 | preempt_trace(int pc) |
6cd8a4bb | 46 | { |
f27107fa | 47 | return ((trace_type & TRACER_PREEMPT_OFF) && pc); |
6cd8a4bb SR |
48 | } |
49 | #else | |
f27107fa | 50 | # define preempt_trace(pc) (0) |
6cd8a4bb SR |
51 | #endif |
52 | ||
53 | #ifdef CONFIG_IRQSOFF_TRACER | |
e309b41d | 54 | static inline int |
6cd8a4bb SR |
55 | irq_trace(void) |
56 | { | |
57 | return ((trace_type & TRACER_IRQS_OFF) && | |
58 | irqs_disabled()); | |
59 | } | |
60 | #else | |
61 | # define irq_trace() (0) | |
62 | #endif | |
63 | ||
62b915f1 | 64 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
03905582 | 65 | static int irqsoff_display_graph(struct trace_array *tr, int set); |
983f938a | 66 | # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH) |
03905582 SRRH |
67 | #else |
68 | static inline int irqsoff_display_graph(struct trace_array *tr, int set) | |
69 | { | |
70 | return -EINVAL; | |
71 | } | |
983f938a | 72 | # define is_graph(tr) false |
62b915f1 | 73 | #endif |
62b915f1 | 74 | |
81d68a96 SR |
75 | /* |
76 | * Sequence count - we record it when starting a measurement and | |
77 | * skip the latency if the sequence has changed - some other section | |
78 | * did a maximum and could disturb our measurement with serial console | |
79 | * printouts, etc. Truly coinciding maximum latencies should be rare | |
25985edc | 80 | * and what happens together happens separately as well, so this doesn't |
81d68a96 SR |
81 | * decrease the validity of the maximum found: |
82 | */ | |
83 | static __cacheline_aligned_in_smp unsigned long max_sequence; | |
84 | ||
606576ce | 85 | #ifdef CONFIG_FUNCTION_TRACER |
81d68a96 | 86 | /* |
5e6d2b9c SR |
87 | * Prologue for the preempt and irqs off function tracers. |
88 | * | |
89 | * Returns 1 if it is OK to continue, and data->disabled is | |
90 | * incremented. | |
91 | * 0 if the trace is to be ignored, and data->disabled | |
92 | * is kept the same. | |
93 | * | |
94 | * Note, this function is also used outside this ifdef but | |
95 | * inside the #ifdef of the function graph tracer below. | |
96 | * This is OK, since the function graph tracer is | |
97 | * dependent on the function tracer. | |
81d68a96 | 98 | */ |
5e6d2b9c SR |
99 | static int func_prolog_dec(struct trace_array *tr, |
100 | struct trace_array_cpu **data, | |
101 | unsigned long *flags) | |
81d68a96 | 102 | { |
81d68a96 SR |
103 | long disabled; |
104 | int cpu; | |
105 | ||
361943ad SR |
106 | /* |
107 | * Does not matter if we preempt. We test the flags | |
108 | * afterward, to see if irqs are disabled or not. | |
109 | * If we preempt and get a false positive, the flags | |
110 | * test will fail. | |
111 | */ | |
112 | cpu = raw_smp_processor_id(); | |
113 | if (likely(!per_cpu(tracing_cpu, cpu))) | |
5e6d2b9c | 114 | return 0; |
81d68a96 | 115 | |
5e6d2b9c | 116 | local_save_flags(*flags); |
cb86e053 SRRH |
117 | /* |
118 | * Slight chance to get a false positive on tracing_cpu, | |
119 | * although I'm starting to think there isn't a chance. | |
120 | * Leave this for now just to be paranoid. | |
121 | */ | |
122 | if (!irqs_disabled_flags(*flags) && !preempt_count()) | |
5e6d2b9c | 123 | return 0; |
81d68a96 | 124 | |
1c5eb448 | 125 | *data = per_cpu_ptr(tr->array_buffer.data, cpu); |
5e6d2b9c | 126 | disabled = atomic_inc_return(&(*data)->disabled); |
81d68a96 SR |
127 | |
128 | if (likely(disabled == 1)) | |
5e6d2b9c SR |
129 | return 1; |
130 | ||
131 | atomic_dec(&(*data)->disabled); | |
132 | ||
133 | return 0; | |
134 | } | |
135 | ||
136 | /* | |
137 | * irqsoff uses its own tracer function to keep the overhead down: | |
138 | */ | |
139 | static void | |
2f5f6ad9 | 140 | irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip, |
d19ad077 | 141 | struct ftrace_ops *op, struct ftrace_regs *fregs) |
5e6d2b9c SR |
142 | { |
143 | struct trace_array *tr = irqsoff_trace; | |
144 | struct trace_array_cpu *data; | |
145 | unsigned long flags; | |
36590c50 | 146 | unsigned int trace_ctx; |
5e6d2b9c SR |
147 | |
148 | if (!func_prolog_dec(tr, &data, &flags)) | |
149 | return; | |
150 | ||
36590c50 SAS |
151 | trace_ctx = tracing_gen_ctx_flags(flags); |
152 | ||
153 | trace_function(tr, ip, parent_ip, trace_ctx); | |
81d68a96 SR |
154 | |
155 | atomic_dec(&data->disabled); | |
156 | } | |
606576ce | 157 | #endif /* CONFIG_FUNCTION_TRACER */ |
81d68a96 | 158 | |
62b915f1 | 159 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
03905582 | 160 | static int irqsoff_display_graph(struct trace_array *tr, int set) |
62b915f1 JO |
161 | { |
162 | int cpu; | |
163 | ||
983f938a | 164 | if (!(is_graph(tr) ^ set)) |
62b915f1 JO |
165 | return 0; |
166 | ||
167 | stop_irqsoff_tracer(irqsoff_trace, !set); | |
168 | ||
169 | for_each_possible_cpu(cpu) | |
170 | per_cpu(tracing_cpu, cpu) = 0; | |
171 | ||
6d9b3fa5 | 172 | tr->max_latency = 0; |
1c5eb448 | 173 | tracing_reset_online_cpus(&irqsoff_trace->array_buffer); |
62b915f1 JO |
174 | |
175 | return start_irqsoff_tracer(irqsoff_trace, set); | |
176 | } | |
177 | ||
178 | static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) | |
179 | { | |
180 | struct trace_array *tr = irqsoff_trace; | |
181 | struct trace_array_cpu *data; | |
182 | unsigned long flags; | |
36590c50 | 183 | unsigned int trace_ctx; |
62b915f1 | 184 | int ret; |
62b915f1 | 185 | |
1a414428 SRRH |
186 | if (ftrace_graph_ignore_func(trace)) |
187 | return 0; | |
188 | /* | |
189 | * Do not trace a function if it's filtered by set_graph_notrace. | |
190 | * Make the index of ret stack negative to indicate that it should | |
191 | * ignore further functions. But it needs its own ret stack entry | |
192 | * to recover the original index in order to continue tracing after | |
193 | * returning from the function. | |
194 | */ | |
195 | if (ftrace_graph_notrace_addr(trace->func)) | |
196 | return 1; | |
197 | ||
5e6d2b9c | 198 | if (!func_prolog_dec(tr, &data, &flags)) |
62b915f1 JO |
199 | return 0; |
200 | ||
36590c50 SAS |
201 | trace_ctx = tracing_gen_ctx_flags(flags); |
202 | ret = __trace_graph_entry(tr, trace, trace_ctx); | |
62b915f1 | 203 | atomic_dec(&data->disabled); |
5e6d2b9c | 204 | |
62b915f1 JO |
205 | return ret; |
206 | } | |
207 | ||
208 | static void irqsoff_graph_return(struct ftrace_graph_ret *trace) | |
209 | { | |
210 | struct trace_array *tr = irqsoff_trace; | |
211 | struct trace_array_cpu *data; | |
212 | unsigned long flags; | |
36590c50 | 213 | unsigned int trace_ctx; |
62b915f1 | 214 | |
5cf99a0f SRV |
215 | ftrace_graph_addr_finish(trace); |
216 | ||
5e6d2b9c | 217 | if (!func_prolog_dec(tr, &data, &flags)) |
62b915f1 JO |
218 | return; |
219 | ||
36590c50 SAS |
220 | trace_ctx = tracing_gen_ctx_flags(flags); |
221 | __trace_graph_return(tr, trace, trace_ctx); | |
62b915f1 JO |
222 | atomic_dec(&data->disabled); |
223 | } | |
224 | ||
688f7089 SRV |
225 | static struct fgraph_ops fgraph_ops = { |
226 | .entryfunc = &irqsoff_graph_entry, | |
227 | .retfunc = &irqsoff_graph_return, | |
228 | }; | |
229 | ||
62b915f1 JO |
230 | static void irqsoff_trace_open(struct trace_iterator *iter) |
231 | { | |
983f938a | 232 | if (is_graph(iter->tr)) |
62b915f1 | 233 | graph_trace_open(iter); |
eecb91b9 ZY |
234 | else |
235 | iter->private = NULL; | |
62b915f1 JO |
236 | } |
237 | ||
238 | static void irqsoff_trace_close(struct trace_iterator *iter) | |
239 | { | |
240 | if (iter->private) | |
241 | graph_trace_close(iter); | |
242 | } | |
243 | ||
244 | #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_CPU | \ | |
321e68b0 | 245 | TRACE_GRAPH_PRINT_PROC | \ |
9acd8de6 | 246 | TRACE_GRAPH_PRINT_REL_TIME | \ |
321e68b0 | 247 | TRACE_GRAPH_PRINT_DURATION) |
62b915f1 JO |
248 | |
249 | static enum print_line_t irqsoff_print_line(struct trace_iterator *iter) | |
250 | { | |
62b915f1 JO |
251 | /* |
252 | * In graph mode call the graph tracer output function, | |
253 | * otherwise go with the TRACE_FN event handler | |
254 | */ | |
983f938a | 255 | if (is_graph(iter->tr)) |
0a772620 | 256 | return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS); |
62b915f1 JO |
257 | |
258 | return TRACE_TYPE_UNHANDLED; | |
259 | } | |
260 | ||
261 | static void irqsoff_print_header(struct seq_file *s) | |
262 | { | |
983f938a SRRH |
263 | struct trace_array *tr = irqsoff_trace; |
264 | ||
265 | if (is_graph(tr)) | |
0a772620 JO |
266 | print_graph_headers_flags(s, GRAPH_TRACER_FLAGS); |
267 | else | |
62b915f1 JO |
268 | trace_default_header(s); |
269 | } | |
270 | ||
62b915f1 JO |
271 | static void |
272 | __trace_function(struct trace_array *tr, | |
273 | unsigned long ip, unsigned long parent_ip, | |
36590c50 | 274 | unsigned int trace_ctx) |
62b915f1 | 275 | { |
983f938a | 276 | if (is_graph(tr)) |
36590c50 | 277 | trace_graph_function(tr, ip, parent_ip, trace_ctx); |
0a772620 | 278 | else |
36590c50 | 279 | trace_function(tr, ip, parent_ip, trace_ctx); |
62b915f1 JO |
280 | } |
281 | ||
282 | #else | |
283 | #define __trace_function trace_function | |
284 | ||
62b915f1 JO |
285 | static enum print_line_t irqsoff_print_line(struct trace_iterator *iter) |
286 | { | |
287 | return TRACE_TYPE_UNHANDLED; | |
288 | } | |
289 | ||
62b915f1 JO |
290 | static void irqsoff_trace_open(struct trace_iterator *iter) { } |
291 | static void irqsoff_trace_close(struct trace_iterator *iter) { } | |
7e9a49ef JO |
292 | |
293 | #ifdef CONFIG_FUNCTION_TRACER | |
294 | static void irqsoff_print_header(struct seq_file *s) | |
295 | { | |
296 | trace_default_header(s); | |
297 | } | |
298 | #else | |
299 | static void irqsoff_print_header(struct seq_file *s) | |
300 | { | |
301 | trace_latency_header(s); | |
302 | } | |
303 | #endif /* CONFIG_FUNCTION_TRACER */ | |
62b915f1 JO |
304 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
305 | ||
81d68a96 SR |
306 | /* |
307 | * Should this new latency be reported/recorded? | |
308 | */ | |
a5a1d1c2 | 309 | static bool report_latency(struct trace_array *tr, u64 delta) |
81d68a96 SR |
310 | { |
311 | if (tracing_thresh) { | |
312 | if (delta < tracing_thresh) | |
79851821 | 313 | return false; |
81d68a96 | 314 | } else { |
6d9b3fa5 | 315 | if (delta <= tr->max_latency) |
79851821 | 316 | return false; |
81d68a96 | 317 | } |
79851821 | 318 | return true; |
81d68a96 SR |
319 | } |
320 | ||
e309b41d | 321 | static void |
81d68a96 SR |
322 | check_critical_timing(struct trace_array *tr, |
323 | struct trace_array_cpu *data, | |
324 | unsigned long parent_ip, | |
325 | int cpu) | |
326 | { | |
a5a1d1c2 | 327 | u64 T0, T1, delta; |
81d68a96 | 328 | unsigned long flags; |
36590c50 | 329 | unsigned int trace_ctx; |
81d68a96 | 330 | |
81d68a96 | 331 | T0 = data->preempt_timestamp; |
750ed1a4 | 332 | T1 = ftrace_now(cpu); |
81d68a96 SR |
333 | delta = T1-T0; |
334 | ||
36590c50 | 335 | trace_ctx = tracing_gen_ctx(); |
6450c1d3 | 336 | |
6d9b3fa5 | 337 | if (!report_latency(tr, delta)) |
81d68a96 SR |
338 | goto out; |
339 | ||
5389f6fa | 340 | raw_spin_lock_irqsave(&max_trace_lock, flags); |
81d68a96 | 341 | |
89b2f978 | 342 | /* check if we are still the max latency */ |
6d9b3fa5 | 343 | if (!report_latency(tr, delta)) |
89b2f978 SR |
344 | goto out_unlock; |
345 | ||
36590c50 | 346 | __trace_function(tr, CALLER_ADDR0, parent_ip, trace_ctx); |
cc51a0fc | 347 | /* Skip 5 functions to get to the irq/preempt enable function */ |
36590c50 | 348 | __trace_stack(tr, trace_ctx, 5); |
81d68a96 | 349 | |
81d68a96 | 350 | if (data->critical_sequence != max_sequence) |
89b2f978 | 351 | goto out_unlock; |
81d68a96 | 352 | |
81d68a96 SR |
353 | data->critical_end = parent_ip; |
354 | ||
b5130b1e | 355 | if (likely(!is_tracing_stopped())) { |
6d9b3fa5 | 356 | tr->max_latency = delta; |
b5130b1e CE |
357 | update_max_tr_single(tr, current, cpu); |
358 | } | |
81d68a96 | 359 | |
81d68a96 SR |
360 | max_sequence++; |
361 | ||
89b2f978 | 362 | out_unlock: |
5389f6fa | 363 | raw_spin_unlock_irqrestore(&max_trace_lock, flags); |
89b2f978 | 364 | |
81d68a96 SR |
365 | out: |
366 | data->critical_sequence = max_sequence; | |
750ed1a4 | 367 | data->preempt_timestamp = ftrace_now(cpu); |
36590c50 | 368 | __trace_function(tr, CALLER_ADDR0, parent_ip, trace_ctx); |
81d68a96 SR |
369 | } |
370 | ||
eeeb080b | 371 | static nokprobe_inline void |
36590c50 | 372 | start_critical_timing(unsigned long ip, unsigned long parent_ip) |
81d68a96 SR |
373 | { |
374 | int cpu; | |
375 | struct trace_array *tr = irqsoff_trace; | |
376 | struct trace_array_cpu *data; | |
81d68a96 | 377 | |
10246fa3 | 378 | if (!tracer_enabled || !tracing_is_enabled()) |
81d68a96 SR |
379 | return; |
380 | ||
c5f888ca SR |
381 | cpu = raw_smp_processor_id(); |
382 | ||
383 | if (per_cpu(tracing_cpu, cpu)) | |
6cd8a4bb SR |
384 | return; |
385 | ||
1c5eb448 | 386 | data = per_cpu_ptr(tr->array_buffer.data, cpu); |
81d68a96 | 387 | |
c5f888ca | 388 | if (unlikely(!data) || atomic_read(&data->disabled)) |
81d68a96 SR |
389 | return; |
390 | ||
391 | atomic_inc(&data->disabled); | |
392 | ||
393 | data->critical_sequence = max_sequence; | |
750ed1a4 | 394 | data->preempt_timestamp = ftrace_now(cpu); |
6cd8a4bb | 395 | data->critical_start = parent_ip ? : ip; |
81d68a96 | 396 | |
36590c50 | 397 | __trace_function(tr, ip, parent_ip, tracing_gen_ctx()); |
81d68a96 | 398 | |
c5f888ca | 399 | per_cpu(tracing_cpu, cpu) = 1; |
6cd8a4bb | 400 | |
81d68a96 SR |
401 | atomic_dec(&data->disabled); |
402 | } | |
403 | ||
eeeb080b | 404 | static nokprobe_inline void |
36590c50 | 405 | stop_critical_timing(unsigned long ip, unsigned long parent_ip) |
81d68a96 SR |
406 | { |
407 | int cpu; | |
408 | struct trace_array *tr = irqsoff_trace; | |
409 | struct trace_array_cpu *data; | |
36590c50 | 410 | unsigned int trace_ctx; |
81d68a96 | 411 | |
c5f888ca | 412 | cpu = raw_smp_processor_id(); |
6cd8a4bb | 413 | /* Always clear the tracing cpu on stopping the trace */ |
c5f888ca SR |
414 | if (unlikely(per_cpu(tracing_cpu, cpu))) |
415 | per_cpu(tracing_cpu, cpu) = 0; | |
6cd8a4bb SR |
416 | else |
417 | return; | |
418 | ||
10246fa3 | 419 | if (!tracer_enabled || !tracing_is_enabled()) |
81d68a96 SR |
420 | return; |
421 | ||
1c5eb448 | 422 | data = per_cpu_ptr(tr->array_buffer.data, cpu); |
81d68a96 | 423 | |
3928a8a2 | 424 | if (unlikely(!data) || |
81d68a96 SR |
425 | !data->critical_start || atomic_read(&data->disabled)) |
426 | return; | |
427 | ||
428 | atomic_inc(&data->disabled); | |
c5f888ca | 429 | |
36590c50 SAS |
430 | trace_ctx = tracing_gen_ctx(); |
431 | __trace_function(tr, ip, parent_ip, trace_ctx); | |
6cd8a4bb | 432 | check_critical_timing(tr, data, parent_ip ? : ip, cpu); |
81d68a96 SR |
433 | data->critical_start = 0; |
434 | atomic_dec(&data->disabled); | |
435 | } | |
436 | ||
6cd8a4bb | 437 | /* start and stop critical timings used to for stoppage (in idle) */ |
e309b41d | 438 | void start_critical_timings(void) |
81d68a96 | 439 | { |
36590c50 SAS |
440 | if (preempt_trace(preempt_count()) || irq_trace()) |
441 | start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); | |
81d68a96 | 442 | } |
1fe37104 | 443 | EXPORT_SYMBOL_GPL(start_critical_timings); |
eeeb080b | 444 | NOKPROBE_SYMBOL(start_critical_timings); |
81d68a96 | 445 | |
e309b41d | 446 | void stop_critical_timings(void) |
81d68a96 | 447 | { |
36590c50 SAS |
448 | if (preempt_trace(preempt_count()) || irq_trace()) |
449 | stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); | |
81d68a96 | 450 | } |
1fe37104 | 451 | EXPORT_SYMBOL_GPL(stop_critical_timings); |
eeeb080b | 452 | NOKPROBE_SYMBOL(stop_critical_timings); |
81d68a96 | 453 | |
8179e8a1 SRRH |
454 | #ifdef CONFIG_FUNCTION_TRACER |
455 | static bool function_enabled; | |
456 | ||
4104d326 | 457 | static int register_irqsoff_function(struct trace_array *tr, int graph, int set) |
81d68a96 | 458 | { |
328df475 | 459 | int ret; |
62b915f1 | 460 | |
328df475 | 461 | /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */ |
983f938a | 462 | if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION))) |
328df475 SRRH |
463 | return 0; |
464 | ||
465 | if (graph) | |
688f7089 | 466 | ret = register_ftrace_graph(&fgraph_ops); |
328df475 | 467 | else |
4104d326 | 468 | ret = register_ftrace_function(tr->ops); |
328df475 SRRH |
469 | |
470 | if (!ret) | |
471 | function_enabled = true; | |
472 | ||
473 | return ret; | |
474 | } | |
475 | ||
4104d326 | 476 | static void unregister_irqsoff_function(struct trace_array *tr, int graph) |
328df475 SRRH |
477 | { |
478 | if (!function_enabled) | |
479 | return; | |
480 | ||
481 | if (graph) | |
688f7089 | 482 | unregister_ftrace_graph(&fgraph_ops); |
328df475 | 483 | else |
4104d326 | 484 | unregister_ftrace_function(tr->ops); |
328df475 SRRH |
485 | |
486 | function_enabled = false; | |
487 | } | |
488 | ||
8179e8a1 | 489 | static int irqsoff_function_set(struct trace_array *tr, u32 mask, int set) |
328df475 | 490 | { |
8179e8a1 SRRH |
491 | if (!(mask & TRACE_ITER_FUNCTION)) |
492 | return 0; | |
493 | ||
328df475 | 494 | if (set) |
983f938a | 495 | register_irqsoff_function(tr, is_graph(tr), 1); |
328df475 | 496 | else |
983f938a | 497 | unregister_irqsoff_function(tr, is_graph(tr)); |
8179e8a1 SRRH |
498 | return 1; |
499 | } | |
500 | #else | |
501 | static int register_irqsoff_function(struct trace_array *tr, int graph, int set) | |
502 | { | |
03905582 | 503 | return 0; |
328df475 | 504 | } |
8179e8a1 SRRH |
505 | static void unregister_irqsoff_function(struct trace_array *tr, int graph) { } |
506 | static inline int irqsoff_function_set(struct trace_array *tr, u32 mask, int set) | |
507 | { | |
508 | return 0; | |
509 | } | |
510 | #endif /* CONFIG_FUNCTION_TRACER */ | |
328df475 | 511 | |
bf6065b5 | 512 | static int irqsoff_flag_changed(struct trace_array *tr, u32 mask, int set) |
328df475 | 513 | { |
bf6065b5 SRRH |
514 | struct tracer *tracer = tr->current_trace; |
515 | ||
8179e8a1 SRRH |
516 | if (irqsoff_function_set(tr, mask, set)) |
517 | return 0; | |
03905582 | 518 | |
729358da | 519 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
03905582 SRRH |
520 | if (mask & TRACE_ITER_DISPLAY_GRAPH) |
521 | return irqsoff_display_graph(tr, set); | |
729358da | 522 | #endif |
328df475 SRRH |
523 | |
524 | return trace_keep_overwrite(tracer, mask, set); | |
525 | } | |
526 | ||
527 | static int start_irqsoff_tracer(struct trace_array *tr, int graph) | |
528 | { | |
529 | int ret; | |
530 | ||
4104d326 | 531 | ret = register_irqsoff_function(tr, graph, 0); |
62b915f1 JO |
532 | |
533 | if (!ret && tracing_is_enabled()) | |
9036990d | 534 | tracer_enabled = 1; |
94523e81 | 535 | else |
9036990d | 536 | tracer_enabled = 0; |
62b915f1 JO |
537 | |
538 | return ret; | |
81d68a96 SR |
539 | } |
540 | ||
62b915f1 | 541 | static void stop_irqsoff_tracer(struct trace_array *tr, int graph) |
81d68a96 | 542 | { |
81d68a96 | 543 | tracer_enabled = 0; |
62b915f1 | 544 | |
4104d326 | 545 | unregister_irqsoff_function(tr, graph); |
81d68a96 SR |
546 | } |
547 | ||
02f2f764 SRRH |
548 | static bool irqsoff_busy; |
549 | ||
550 | static int __irqsoff_tracer_init(struct trace_array *tr) | |
81d68a96 | 551 | { |
02f2f764 SRRH |
552 | if (irqsoff_busy) |
553 | return -EBUSY; | |
554 | ||
983f938a | 555 | save_flags = tr->trace_flags; |
613f04a0 SRRH |
556 | |
557 | /* non overwrite screws up the latency tracers */ | |
2b6080f2 SR |
558 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1); |
559 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1); | |
da7f84cd VR |
560 | /* without pause, we will produce garbage if another latency occurs */ |
561 | set_tracer_flag(tr, TRACE_ITER_PAUSE_ON_TRACE, 1); | |
e9d25fe6 | 562 | |
6d9b3fa5 | 563 | tr->max_latency = 0; |
81d68a96 | 564 | irqsoff_trace = tr; |
c5f888ca | 565 | /* make sure that the tracer is visible */ |
81d68a96 | 566 | smp_wmb(); |
62b915f1 | 567 | |
4104d326 SRRH |
568 | ftrace_init_array_ops(tr, irqsoff_tracer_call); |
569 | ||
570 | /* Only toplevel instance supports graph tracing */ | |
571 | if (start_irqsoff_tracer(tr, (tr->flags & TRACE_ARRAY_FL_GLOBAL && | |
983f938a | 572 | is_graph(tr)))) |
62b915f1 | 573 | printk(KERN_ERR "failed to start irqsoff tracer\n"); |
02f2f764 SRRH |
574 | |
575 | irqsoff_busy = true; | |
576 | return 0; | |
81d68a96 SR |
577 | } |
578 | ||
2b27ece6 | 579 | static void __irqsoff_tracer_reset(struct trace_array *tr) |
81d68a96 | 580 | { |
613f04a0 SRRH |
581 | int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT; |
582 | int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE; | |
da7f84cd | 583 | int pause_flag = save_flags & TRACE_ITER_PAUSE_ON_TRACE; |
613f04a0 | 584 | |
983f938a | 585 | stop_irqsoff_tracer(tr, is_graph(tr)); |
e9d25fe6 | 586 | |
2b6080f2 SR |
587 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag); |
588 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag); | |
da7f84cd | 589 | set_tracer_flag(tr, TRACE_ITER_PAUSE_ON_TRACE, pause_flag); |
4104d326 | 590 | ftrace_reset_array_ops(tr); |
02f2f764 SRRH |
591 | |
592 | irqsoff_busy = false; | |
81d68a96 SR |
593 | } |
594 | ||
9036990d SR |
595 | static void irqsoff_tracer_start(struct trace_array *tr) |
596 | { | |
9036990d | 597 | tracer_enabled = 1; |
9036990d SR |
598 | } |
599 | ||
600 | static void irqsoff_tracer_stop(struct trace_array *tr) | |
601 | { | |
602 | tracer_enabled = 0; | |
81d68a96 SR |
603 | } |
604 | ||
6cd8a4bb | 605 | #ifdef CONFIG_IRQSOFF_TRACER |
c3bc8fd6 JFG |
606 | /* |
607 | * We are only interested in hardirq on/off events: | |
608 | */ | |
3f1756dc | 609 | void tracer_hardirqs_on(unsigned long a0, unsigned long a1) |
c3bc8fd6 | 610 | { |
36590c50 SAS |
611 | if (!preempt_trace(preempt_count()) && irq_trace()) |
612 | stop_critical_timing(a0, a1); | |
c3bc8fd6 | 613 | } |
eeeb080b | 614 | NOKPROBE_SYMBOL(tracer_hardirqs_on); |
c3bc8fd6 | 615 | |
3f1756dc | 616 | void tracer_hardirqs_off(unsigned long a0, unsigned long a1) |
c3bc8fd6 | 617 | { |
36590c50 SAS |
618 | if (!preempt_trace(preempt_count()) && irq_trace()) |
619 | start_critical_timing(a0, a1); | |
c3bc8fd6 | 620 | } |
eeeb080b | 621 | NOKPROBE_SYMBOL(tracer_hardirqs_off); |
c3bc8fd6 | 622 | |
1c80025a | 623 | static int irqsoff_tracer_init(struct trace_array *tr) |
6cd8a4bb SR |
624 | { |
625 | trace_type = TRACER_IRQS_OFF; | |
626 | ||
02f2f764 | 627 | return __irqsoff_tracer_init(tr); |
6cd8a4bb | 628 | } |
2b27ece6 JFG |
629 | |
630 | static void irqsoff_tracer_reset(struct trace_array *tr) | |
631 | { | |
632 | __irqsoff_tracer_reset(tr); | |
633 | } | |
634 | ||
81d68a96 SR |
635 | static struct tracer irqsoff_tracer __read_mostly = |
636 | { | |
637 | .name = "irqsoff", | |
638 | .init = irqsoff_tracer_init, | |
639 | .reset = irqsoff_tracer_reset, | |
9036990d SR |
640 | .start = irqsoff_tracer_start, |
641 | .stop = irqsoff_tracer_stop, | |
f43c738b | 642 | .print_max = true, |
62b915f1 JO |
643 | .print_header = irqsoff_print_header, |
644 | .print_line = irqsoff_print_line, | |
328df475 | 645 | .flag_changed = irqsoff_flag_changed, |
60a11774 SR |
646 | #ifdef CONFIG_FTRACE_SELFTEST |
647 | .selftest = trace_selftest_startup_irqsoff, | |
648 | #endif | |
62b915f1 JO |
649 | .open = irqsoff_trace_open, |
650 | .close = irqsoff_trace_close, | |
02f2f764 | 651 | .allow_instances = true, |
f43c738b | 652 | .use_max_tr = true, |
81d68a96 | 653 | }; |
c3bc8fd6 | 654 | #endif /* CONFIG_IRQSOFF_TRACER */ |
6cd8a4bb SR |
655 | |
656 | #ifdef CONFIG_PREEMPT_TRACER | |
3f1756dc | 657 | void tracer_preempt_on(unsigned long a0, unsigned long a1) |
c3bc8fd6 | 658 | { |
36590c50 SAS |
659 | if (preempt_trace(preempt_count()) && !irq_trace()) |
660 | stop_critical_timing(a0, a1); | |
c3bc8fd6 JFG |
661 | } |
662 | ||
3f1756dc | 663 | void tracer_preempt_off(unsigned long a0, unsigned long a1) |
c3bc8fd6 | 664 | { |
36590c50 SAS |
665 | if (preempt_trace(preempt_count()) && !irq_trace()) |
666 | start_critical_timing(a0, a1); | |
c3bc8fd6 JFG |
667 | } |
668 | ||
1c80025a | 669 | static int preemptoff_tracer_init(struct trace_array *tr) |
6cd8a4bb SR |
670 | { |
671 | trace_type = TRACER_PREEMPT_OFF; | |
672 | ||
02f2f764 | 673 | return __irqsoff_tracer_init(tr); |
6cd8a4bb SR |
674 | } |
675 | ||
2b27ece6 JFG |
676 | static void preemptoff_tracer_reset(struct trace_array *tr) |
677 | { | |
678 | __irqsoff_tracer_reset(tr); | |
679 | } | |
680 | ||
6cd8a4bb SR |
681 | static struct tracer preemptoff_tracer __read_mostly = |
682 | { | |
683 | .name = "preemptoff", | |
684 | .init = preemptoff_tracer_init, | |
2b27ece6 | 685 | .reset = preemptoff_tracer_reset, |
9036990d SR |
686 | .start = irqsoff_tracer_start, |
687 | .stop = irqsoff_tracer_stop, | |
f43c738b | 688 | .print_max = true, |
62b915f1 JO |
689 | .print_header = irqsoff_print_header, |
690 | .print_line = irqsoff_print_line, | |
328df475 | 691 | .flag_changed = irqsoff_flag_changed, |
60a11774 SR |
692 | #ifdef CONFIG_FTRACE_SELFTEST |
693 | .selftest = trace_selftest_startup_preemptoff, | |
694 | #endif | |
62b915f1 JO |
695 | .open = irqsoff_trace_open, |
696 | .close = irqsoff_trace_close, | |
02f2f764 | 697 | .allow_instances = true, |
f43c738b | 698 | .use_max_tr = true, |
6cd8a4bb | 699 | }; |
c3bc8fd6 | 700 | #endif /* CONFIG_PREEMPT_TRACER */ |
6cd8a4bb | 701 | |
c3bc8fd6 | 702 | #if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER) |
6cd8a4bb | 703 | |
1c80025a | 704 | static int preemptirqsoff_tracer_init(struct trace_array *tr) |
6cd8a4bb SR |
705 | { |
706 | trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF; | |
707 | ||
02f2f764 | 708 | return __irqsoff_tracer_init(tr); |
6cd8a4bb SR |
709 | } |
710 | ||
2b27ece6 JFG |
711 | static void preemptirqsoff_tracer_reset(struct trace_array *tr) |
712 | { | |
713 | __irqsoff_tracer_reset(tr); | |
714 | } | |
715 | ||
6cd8a4bb SR |
716 | static struct tracer preemptirqsoff_tracer __read_mostly = |
717 | { | |
718 | .name = "preemptirqsoff", | |
719 | .init = preemptirqsoff_tracer_init, | |
2b27ece6 | 720 | .reset = preemptirqsoff_tracer_reset, |
9036990d SR |
721 | .start = irqsoff_tracer_start, |
722 | .stop = irqsoff_tracer_stop, | |
f43c738b | 723 | .print_max = true, |
62b915f1 JO |
724 | .print_header = irqsoff_print_header, |
725 | .print_line = irqsoff_print_line, | |
328df475 | 726 | .flag_changed = irqsoff_flag_changed, |
60a11774 SR |
727 | #ifdef CONFIG_FTRACE_SELFTEST |
728 | .selftest = trace_selftest_startup_preemptirqsoff, | |
729 | #endif | |
62b915f1 JO |
730 | .open = irqsoff_trace_open, |
731 | .close = irqsoff_trace_close, | |
02f2f764 | 732 | .allow_instances = true, |
f43c738b | 733 | .use_max_tr = true, |
6cd8a4bb | 734 | }; |
6cd8a4bb | 735 | #endif |
81d68a96 SR |
736 | |
737 | __init static int init_irqsoff_tracer(void) | |
738 | { | |
c3bc8fd6 JFG |
739 | #ifdef CONFIG_IRQSOFF_TRACER |
740 | register_tracer(&irqsoff_tracer); | |
aaecaa0b | 741 | #endif |
c3bc8fd6 JFG |
742 | #ifdef CONFIG_PREEMPT_TRACER |
743 | register_tracer(&preemptoff_tracer); | |
aaecaa0b | 744 | #endif |
c3bc8fd6 JFG |
745 | #if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER) |
746 | register_tracer(&preemptirqsoff_tracer); | |
aaecaa0b JF |
747 | #endif |
748 | ||
c3bc8fd6 | 749 | return 0; |
aaecaa0b | 750 | } |
c3bc8fd6 JFG |
751 | core_initcall(init_irqsoff_tracer); |
752 | #endif /* IRQSOFF_TRACER || PREEMPTOFF_TRACER */ |