Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
352ad25a SR |
2 | /* |
3 | * trace task wakeup timings | |
4 | * | |
5 | * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> | |
6 | * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> | |
7 | * | |
8 | * Based on code from the latency_tracer, that is: | |
9 | * | |
10 | * Copyright (C) 2004-2006 Ingo Molnar | |
6d49e352 | 11 | * Copyright (C) 2004 Nadia Yvette Chambers |
352ad25a SR |
12 | */ |
13 | #include <linux/module.h> | |
352ad25a SR |
14 | #include <linux/kallsyms.h> |
15 | #include <linux/uaccess.h> | |
16 | #include <linux/ftrace.h> | |
8bd75c77 | 17 | #include <linux/sched/rt.h> |
2d3d891d | 18 | #include <linux/sched/deadline.h> |
ad8d75ff | 19 | #include <trace/events/sched.h> |
352ad25a SR |
20 | #include "trace.h" |
21 | ||
22 | static struct trace_array *wakeup_trace; | |
23 | static int __read_mostly tracer_enabled; | |
24 | ||
25 | static struct task_struct *wakeup_task; | |
26 | static int wakeup_cpu; | |
478142c3 | 27 | static int wakeup_current_cpu; |
352ad25a | 28 | static unsigned wakeup_prio = -1; |
bfbf8d15 AK |
29 | static bool wakeup_rt; |
30 | static bool wakeup_dl; | |
31 | static bool tracing_dl; | |
352ad25a | 32 | |
445c8951 | 33 | static arch_spinlock_t wakeup_lock = |
edc35bd7 | 34 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
352ad25a | 35 | |
7495a5be | 36 | static void wakeup_reset(struct trace_array *tr); |
e309b41d | 37 | static void __wakeup_reset(struct trace_array *tr); |
317e04ca SRV |
38 | static int start_func_tracer(struct trace_array *tr, int graph); |
39 | static void stop_func_tracer(struct trace_array *tr, int graph); | |
352ad25a | 40 | |
613f04a0 | 41 | static int save_flags; |
7495a5be | 42 | |
7495a5be | 43 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
983f938a | 44 | # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH) |
03905582 | 45 | #else |
983f938a | 46 | # define is_graph(tr) false |
7495a5be | 47 | #endif |
7495a5be | 48 | |
606576ce | 49 | #ifdef CONFIG_FUNCTION_TRACER |
542181d3 | 50 | |
8179e8a1 SRRH |
51 | static bool function_enabled; |
52 | ||
7e18d8e7 | 53 | /* |
542181d3 SR |
54 | * Prologue for the wakeup function tracers. |
55 | * | |
56 | * Returns 1 if it is OK to continue, and preemption | |
57 | * is disabled and data->disabled is incremented. | |
58 | * 0 if the trace is to be ignored, and preemption | |
59 | * is not disabled and data->disabled is | |
60 | * kept the same. | |
61 | * | |
62 | * Note, this function is also used outside this ifdef but | |
63 | * inside the #ifdef of the function graph tracer below. | |
64 | * This is OK, since the function graph tracer is | |
65 | * dependent on the function tracer. | |
7e18d8e7 | 66 | */ |
542181d3 SR |
67 | static int |
68 | func_prolog_preempt_disable(struct trace_array *tr, | |
69 | struct trace_array_cpu **data, | |
36590c50 | 70 | unsigned int *trace_ctx) |
7e18d8e7 | 71 | { |
7e18d8e7 | 72 | long disabled; |
7e18d8e7 SR |
73 | int cpu; |
74 | ||
75 | if (likely(!wakeup_task)) | |
542181d3 | 76 | return 0; |
7e18d8e7 | 77 | |
36590c50 | 78 | *trace_ctx = tracing_gen_ctx(); |
5168ae50 | 79 | preempt_disable_notrace(); |
7e18d8e7 SR |
80 | |
81 | cpu = raw_smp_processor_id(); | |
478142c3 SR |
82 | if (cpu != wakeup_current_cpu) |
83 | goto out_enable; | |
84 | ||
1c5eb448 | 85 | *data = per_cpu_ptr(tr->array_buffer.data, cpu); |
542181d3 | 86 | disabled = atomic_inc_return(&(*data)->disabled); |
7e18d8e7 SR |
87 | if (unlikely(disabled != 1)) |
88 | goto out; | |
89 | ||
542181d3 | 90 | return 1; |
7e18d8e7 | 91 | |
542181d3 SR |
92 | out: |
93 | atomic_dec(&(*data)->disabled); | |
7e18d8e7 | 94 | |
542181d3 SR |
95 | out_enable: |
96 | preempt_enable_notrace(); | |
97 | return 0; | |
98 | } | |
99 | ||
729358da | 100 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
328df475 | 101 | |
03905582 | 102 | static int wakeup_display_graph(struct trace_array *tr, int set) |
7495a5be | 103 | { |
983f938a | 104 | if (!(is_graph(tr) ^ set)) |
7495a5be JO |
105 | return 0; |
106 | ||
4104d326 | 107 | stop_func_tracer(tr, !set); |
7495a5be JO |
108 | |
109 | wakeup_reset(wakeup_trace); | |
6d9b3fa5 | 110 | tr->max_latency = 0; |
7495a5be | 111 | |
4104d326 | 112 | return start_func_tracer(tr, set); |
7495a5be JO |
113 | } |
114 | ||
115 | static int wakeup_graph_entry(struct ftrace_graph_ent *trace) | |
116 | { | |
117 | struct trace_array *tr = wakeup_trace; | |
118 | struct trace_array_cpu *data; | |
36590c50 SAS |
119 | unsigned int trace_ctx; |
120 | int ret = 0; | |
7495a5be | 121 | |
1a414428 SRRH |
122 | if (ftrace_graph_ignore_func(trace)) |
123 | return 0; | |
124 | /* | |
125 | * Do not trace a function if it's filtered by set_graph_notrace. | |
126 | * Make the index of ret stack negative to indicate that it should | |
127 | * ignore further functions. But it needs its own ret stack entry | |
128 | * to recover the original index in order to continue tracing after | |
129 | * returning from the function. | |
130 | */ | |
131 | if (ftrace_graph_notrace_addr(trace->func)) | |
132 | return 1; | |
133 | ||
36590c50 | 134 | if (!func_prolog_preempt_disable(tr, &data, &trace_ctx)) |
7495a5be JO |
135 | return 0; |
136 | ||
36590c50 | 137 | ret = __trace_graph_entry(tr, trace, trace_ctx); |
7495a5be | 138 | atomic_dec(&data->disabled); |
7495a5be | 139 | preempt_enable_notrace(); |
542181d3 | 140 | |
7495a5be JO |
141 | return ret; |
142 | } | |
143 | ||
144 | static void wakeup_graph_return(struct ftrace_graph_ret *trace) | |
145 | { | |
146 | struct trace_array *tr = wakeup_trace; | |
147 | struct trace_array_cpu *data; | |
36590c50 | 148 | unsigned int trace_ctx; |
7495a5be | 149 | |
5cf99a0f SRV |
150 | ftrace_graph_addr_finish(trace); |
151 | ||
36590c50 | 152 | if (!func_prolog_preempt_disable(tr, &data, &trace_ctx)) |
7495a5be JO |
153 | return; |
154 | ||
36590c50 | 155 | __trace_graph_return(tr, trace, trace_ctx); |
7495a5be JO |
156 | atomic_dec(&data->disabled); |
157 | ||
7495a5be JO |
158 | preempt_enable_notrace(); |
159 | return; | |
160 | } | |
161 | ||
688f7089 SRV |
162 | static struct fgraph_ops fgraph_wakeup_ops = { |
163 | .entryfunc = &wakeup_graph_entry, | |
164 | .retfunc = &wakeup_graph_return, | |
165 | }; | |
166 | ||
7495a5be JO |
167 | static void wakeup_trace_open(struct trace_iterator *iter) |
168 | { | |
983f938a | 169 | if (is_graph(iter->tr)) |
7495a5be | 170 | graph_trace_open(iter); |
eecb91b9 ZY |
171 | else |
172 | iter->private = NULL; | |
7495a5be JO |
173 | } |
174 | ||
175 | static void wakeup_trace_close(struct trace_iterator *iter) | |
176 | { | |
177 | if (iter->private) | |
178 | graph_trace_close(iter); | |
179 | } | |
180 | ||
321e68b0 | 181 | #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \ |
97f0a3bc | 182 | TRACE_GRAPH_PRINT_CPU | \ |
9acd8de6 | 183 | TRACE_GRAPH_PRINT_REL_TIME | \ |
97f0a3bc CD |
184 | TRACE_GRAPH_PRINT_DURATION | \ |
185 | TRACE_GRAPH_PRINT_OVERHEAD | \ | |
186 | TRACE_GRAPH_PRINT_IRQS) | |
7495a5be JO |
187 | |
188 | static enum print_line_t wakeup_print_line(struct trace_iterator *iter) | |
189 | { | |
190 | /* | |
191 | * In graph mode call the graph tracer output function, | |
192 | * otherwise go with the TRACE_FN event handler | |
193 | */ | |
983f938a | 194 | if (is_graph(iter->tr)) |
7495a5be JO |
195 | return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS); |
196 | ||
197 | return TRACE_TYPE_UNHANDLED; | |
198 | } | |
199 | ||
200 | static void wakeup_print_header(struct seq_file *s) | |
201 | { | |
983f938a | 202 | if (is_graph(wakeup_trace)) |
7495a5be JO |
203 | print_graph_headers_flags(s, GRAPH_TRACER_FLAGS); |
204 | else | |
205 | trace_default_header(s); | |
206 | } | |
317e04ca | 207 | #endif /* else CONFIG_FUNCTION_GRAPH_TRACER */ |
7495a5be | 208 | |
317e04ca SRV |
209 | /* |
210 | * wakeup uses its own tracer function to keep the overhead down: | |
211 | */ | |
7495a5be | 212 | static void |
317e04ca | 213 | wakeup_tracer_call(unsigned long ip, unsigned long parent_ip, |
d19ad077 | 214 | struct ftrace_ops *op, struct ftrace_regs *fregs) |
7495a5be | 215 | { |
317e04ca SRV |
216 | struct trace_array *tr = wakeup_trace; |
217 | struct trace_array_cpu *data; | |
218 | unsigned long flags; | |
36590c50 | 219 | unsigned int trace_ctx; |
317e04ca | 220 | |
36590c50 | 221 | if (!func_prolog_preempt_disable(tr, &data, &trace_ctx)) |
317e04ca SRV |
222 | return; |
223 | ||
224 | local_irq_save(flags); | |
36590c50 | 225 | trace_function(tr, ip, parent_ip, trace_ctx); |
317e04ca SRV |
226 | local_irq_restore(flags); |
227 | ||
228 | atomic_dec(&data->disabled); | |
229 | preempt_enable_notrace(); | |
230 | } | |
231 | ||
232 | static int register_wakeup_function(struct trace_array *tr, int graph, int set) | |
233 | { | |
234 | int ret; | |
235 | ||
236 | /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */ | |
237 | if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION))) | |
238 | return 0; | |
239 | ||
240 | if (graph) | |
688f7089 | 241 | ret = register_ftrace_graph(&fgraph_wakeup_ops); |
7495a5be | 242 | else |
317e04ca SRV |
243 | ret = register_ftrace_function(tr->ops); |
244 | ||
245 | if (!ret) | |
246 | function_enabled = true; | |
247 | ||
248 | return ret; | |
7495a5be | 249 | } |
7495a5be | 250 | |
317e04ca SRV |
251 | static void unregister_wakeup_function(struct trace_array *tr, int graph) |
252 | { | |
253 | if (!function_enabled) | |
254 | return; | |
255 | ||
256 | if (graph) | |
688f7089 | 257 | unregister_ftrace_graph(&fgraph_wakeup_ops); |
317e04ca SRV |
258 | else |
259 | unregister_ftrace_function(tr->ops); | |
260 | ||
261 | function_enabled = false; | |
262 | } | |
263 | ||
264 | static int wakeup_function_set(struct trace_array *tr, u32 mask, int set) | |
265 | { | |
266 | if (!(mask & TRACE_ITER_FUNCTION)) | |
267 | return 0; | |
268 | ||
269 | if (set) | |
270 | register_wakeup_function(tr, is_graph(tr), 1); | |
271 | else | |
272 | unregister_wakeup_function(tr, is_graph(tr)); | |
273 | return 1; | |
274 | } | |
275 | #else /* CONFIG_FUNCTION_TRACER */ | |
276 | static int register_wakeup_function(struct trace_array *tr, int graph, int set) | |
277 | { | |
278 | return 0; | |
279 | } | |
280 | static void unregister_wakeup_function(struct trace_array *tr, int graph) { } | |
281 | static int wakeup_function_set(struct trace_array *tr, u32 mask, int set) | |
282 | { | |
283 | return 0; | |
284 | } | |
285 | #endif /* else CONFIG_FUNCTION_TRACER */ | |
286 | ||
287 | #ifndef CONFIG_FUNCTION_GRAPH_TRACER | |
7495a5be JO |
288 | static enum print_line_t wakeup_print_line(struct trace_iterator *iter) |
289 | { | |
290 | return TRACE_TYPE_UNHANDLED; | |
291 | } | |
292 | ||
7495a5be JO |
293 | static void wakeup_trace_open(struct trace_iterator *iter) { } |
294 | static void wakeup_trace_close(struct trace_iterator *iter) { } | |
7e9a49ef | 295 | |
7e9a49ef JO |
296 | static void wakeup_print_header(struct seq_file *s) |
297 | { | |
298 | trace_default_header(s); | |
299 | } | |
317e04ca SRV |
300 | #endif /* !CONFIG_FUNCTION_GRAPH_TRACER */ |
301 | ||
302 | static void | |
303 | __trace_function(struct trace_array *tr, | |
304 | unsigned long ip, unsigned long parent_ip, | |
36590c50 | 305 | unsigned int trace_ctx) |
317e04ca SRV |
306 | { |
307 | if (is_graph(tr)) | |
36590c50 | 308 | trace_graph_function(tr, ip, parent_ip, trace_ctx); |
317e04ca | 309 | else |
36590c50 | 310 | trace_function(tr, ip, parent_ip, trace_ctx); |
317e04ca SRV |
311 | } |
312 | ||
313 | static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set) | |
7e9a49ef | 314 | { |
317e04ca SRV |
315 | struct tracer *tracer = tr->current_trace; |
316 | ||
317 | if (wakeup_function_set(tr, mask, set)) | |
318 | return 0; | |
319 | ||
320 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | |
321 | if (mask & TRACE_ITER_DISPLAY_GRAPH) | |
322 | return wakeup_display_graph(tr, set); | |
323 | #endif | |
324 | ||
325 | return trace_keep_overwrite(tracer, mask, set); | |
326 | } | |
327 | ||
328 | static int start_func_tracer(struct trace_array *tr, int graph) | |
329 | { | |
330 | int ret; | |
331 | ||
332 | ret = register_wakeup_function(tr, graph, 0); | |
333 | ||
334 | if (!ret && tracing_is_enabled()) | |
335 | tracer_enabled = 1; | |
336 | else | |
337 | tracer_enabled = 0; | |
338 | ||
339 | return ret; | |
340 | } | |
341 | ||
342 | static void stop_func_tracer(struct trace_array *tr, int graph) | |
343 | { | |
344 | tracer_enabled = 0; | |
345 | ||
346 | unregister_wakeup_function(tr, graph); | |
7e9a49ef | 347 | } |
7495a5be | 348 | |
352ad25a SR |
349 | /* |
350 | * Should this new latency be reported/recorded? | |
351 | */ | |
a5a1d1c2 | 352 | static bool report_latency(struct trace_array *tr, u64 delta) |
352ad25a SR |
353 | { |
354 | if (tracing_thresh) { | |
355 | if (delta < tracing_thresh) | |
26ab2ef4 | 356 | return false; |
352ad25a | 357 | } else { |
6d9b3fa5 | 358 | if (delta <= tr->max_latency) |
26ab2ef4 | 359 | return false; |
352ad25a | 360 | } |
26ab2ef4 | 361 | return true; |
352ad25a SR |
362 | } |
363 | ||
38516ab5 SR |
364 | static void |
365 | probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu) | |
478142c3 SR |
366 | { |
367 | if (task != wakeup_task) | |
368 | return; | |
369 | ||
370 | wakeup_current_cpu = cpu; | |
371 | } | |
372 | ||
243f7610 SRRH |
373 | static void |
374 | tracing_sched_switch_trace(struct trace_array *tr, | |
375 | struct task_struct *prev, | |
376 | struct task_struct *next, | |
36590c50 | 377 | unsigned int trace_ctx) |
243f7610 | 378 | { |
2425bcb9 | 379 | struct trace_event_call *call = &event_context_switch; |
13292494 | 380 | struct trace_buffer *buffer = tr->array_buffer.buffer; |
243f7610 SRRH |
381 | struct ring_buffer_event *event; |
382 | struct ctx_switch_entry *entry; | |
383 | ||
384 | event = trace_buffer_lock_reserve(buffer, TRACE_CTX, | |
36590c50 | 385 | sizeof(*entry), trace_ctx); |
243f7610 SRRH |
386 | if (!event) |
387 | return; | |
388 | entry = ring_buffer_event_data(event); | |
389 | entry->prev_pid = prev->pid; | |
390 | entry->prev_prio = prev->prio; | |
1d48b080 | 391 | entry->prev_state = task_state_index(prev); |
243f7610 SRRH |
392 | entry->next_pid = next->pid; |
393 | entry->next_prio = next->prio; | |
1d48b080 | 394 | entry->next_state = task_state_index(next); |
243f7610 SRRH |
395 | entry->next_cpu = task_cpu(next); |
396 | ||
397 | if (!call_filter_check_discard(call, entry, buffer, event)) | |
36590c50 | 398 | trace_buffer_unlock_commit(tr, buffer, event, trace_ctx); |
243f7610 SRRH |
399 | } |
400 | ||
401 | static void | |
402 | tracing_sched_wakeup_trace(struct trace_array *tr, | |
403 | struct task_struct *wakee, | |
404 | struct task_struct *curr, | |
36590c50 | 405 | unsigned int trace_ctx) |
243f7610 | 406 | { |
2425bcb9 | 407 | struct trace_event_call *call = &event_wakeup; |
243f7610 SRRH |
408 | struct ring_buffer_event *event; |
409 | struct ctx_switch_entry *entry; | |
13292494 | 410 | struct trace_buffer *buffer = tr->array_buffer.buffer; |
243f7610 SRRH |
411 | |
412 | event = trace_buffer_lock_reserve(buffer, TRACE_WAKE, | |
36590c50 | 413 | sizeof(*entry), trace_ctx); |
243f7610 SRRH |
414 | if (!event) |
415 | return; | |
416 | entry = ring_buffer_event_data(event); | |
417 | entry->prev_pid = curr->pid; | |
418 | entry->prev_prio = curr->prio; | |
1d48b080 | 419 | entry->prev_state = task_state_index(curr); |
243f7610 SRRH |
420 | entry->next_pid = wakee->pid; |
421 | entry->next_prio = wakee->prio; | |
1d48b080 | 422 | entry->next_state = task_state_index(wakee); |
243f7610 SRRH |
423 | entry->next_cpu = task_cpu(wakee); |
424 | ||
425 | if (!call_filter_check_discard(call, entry, buffer, event)) | |
36590c50 | 426 | trace_buffer_unlock_commit(tr, buffer, event, trace_ctx); |
243f7610 SRRH |
427 | } |
428 | ||
5b82a1b0 | 429 | static void notrace |
c73464b1 | 430 | probe_wakeup_sched_switch(void *ignore, bool preempt, |
9c2136be DK |
431 | struct task_struct *prev, struct task_struct *next, |
432 | unsigned int prev_state) | |
352ad25a | 433 | { |
352ad25a | 434 | struct trace_array_cpu *data; |
a5a1d1c2 | 435 | u64 T0, T1, delta; |
352ad25a SR |
436 | unsigned long flags; |
437 | long disabled; | |
438 | int cpu; | |
36590c50 | 439 | unsigned int trace_ctx; |
352ad25a | 440 | |
b07c3f19 MD |
441 | tracing_record_cmdline(prev); |
442 | ||
352ad25a SR |
443 | if (unlikely(!tracer_enabled)) |
444 | return; | |
445 | ||
446 | /* | |
447 | * When we start a new trace, we set wakeup_task to NULL | |
448 | * and then set tracer_enabled = 1. We want to make sure | |
449 | * that another CPU does not see the tracer_enabled = 1 | |
450 | * and the wakeup_task with an older task, that might | |
451 | * actually be the same as next. | |
452 | */ | |
453 | smp_rmb(); | |
454 | ||
455 | if (next != wakeup_task) | |
456 | return; | |
457 | ||
352ad25a SR |
458 | /* disable local data, not wakeup_cpu data */ |
459 | cpu = raw_smp_processor_id(); | |
1c5eb448 | 460 | disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); |
352ad25a SR |
461 | if (likely(disabled != 1)) |
462 | goto out; | |
463 | ||
e59494f4 | 464 | local_irq_save(flags); |
36590c50 SAS |
465 | trace_ctx = tracing_gen_ctx_flags(flags); |
466 | ||
0199c4e6 | 467 | arch_spin_lock(&wakeup_lock); |
352ad25a SR |
468 | |
469 | /* We could race with grabbing wakeup_lock */ | |
470 | if (unlikely(!tracer_enabled || next != wakeup_task)) | |
471 | goto out_unlock; | |
472 | ||
9be24414 | 473 | /* The task we are waiting for is waking up */ |
1c5eb448 | 474 | data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu); |
9be24414 | 475 | |
36590c50 SAS |
476 | __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, trace_ctx); |
477 | tracing_sched_switch_trace(wakeup_trace, prev, next, trace_ctx); | |
478 | __trace_stack(wakeup_trace, trace_ctx, 0); | |
352ad25a | 479 | |
352ad25a | 480 | T0 = data->preempt_timestamp; |
750ed1a4 | 481 | T1 = ftrace_now(cpu); |
352ad25a SR |
482 | delta = T1-T0; |
483 | ||
6d9b3fa5 | 484 | if (!report_latency(wakeup_trace, delta)) |
352ad25a SR |
485 | goto out_unlock; |
486 | ||
b5130b1e | 487 | if (likely(!is_tracing_stopped())) { |
6d9b3fa5 | 488 | wakeup_trace->max_latency = delta; |
a35873a0 | 489 | update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu, NULL); |
b5130b1e | 490 | } |
352ad25a | 491 | |
352ad25a | 492 | out_unlock: |
b07c3f19 | 493 | __wakeup_reset(wakeup_trace); |
0199c4e6 | 494 | arch_spin_unlock(&wakeup_lock); |
e59494f4 | 495 | local_irq_restore(flags); |
352ad25a | 496 | out: |
1c5eb448 | 497 | atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); |
5b82a1b0 MD |
498 | } |
499 | ||
e309b41d | 500 | static void __wakeup_reset(struct trace_array *tr) |
352ad25a | 501 | { |
352ad25a SR |
502 | wakeup_cpu = -1; |
503 | wakeup_prio = -1; | |
bfbf8d15 | 504 | tracing_dl = false; |
352ad25a SR |
505 | |
506 | if (wakeup_task) | |
507 | put_task_struct(wakeup_task); | |
508 | ||
509 | wakeup_task = NULL; | |
510 | } | |
511 | ||
e309b41d | 512 | static void wakeup_reset(struct trace_array *tr) |
352ad25a SR |
513 | { |
514 | unsigned long flags; | |
515 | ||
1c5eb448 | 516 | tracing_reset_online_cpus(&tr->array_buffer); |
2f26ebd5 | 517 | |
e59494f4 | 518 | local_irq_save(flags); |
0199c4e6 | 519 | arch_spin_lock(&wakeup_lock); |
352ad25a | 520 | __wakeup_reset(tr); |
0199c4e6 | 521 | arch_spin_unlock(&wakeup_lock); |
e59494f4 | 522 | local_irq_restore(flags); |
352ad25a SR |
523 | } |
524 | ||
e309b41d | 525 | static void |
fbd705a0 | 526 | probe_wakeup(void *ignore, struct task_struct *p) |
352ad25a | 527 | { |
f8ec1062 | 528 | struct trace_array_cpu *data; |
352ad25a | 529 | int cpu = smp_processor_id(); |
352ad25a | 530 | long disabled; |
36590c50 | 531 | unsigned int trace_ctx; |
352ad25a | 532 | |
b07c3f19 MD |
533 | if (likely(!tracer_enabled)) |
534 | return; | |
535 | ||
536 | tracing_record_cmdline(p); | |
537 | tracing_record_cmdline(current); | |
538 | ||
af6ace76 DF |
539 | /* |
540 | * Semantic is like this: | |
541 | * - wakeup tracer handles all tasks in the system, independently | |
542 | * from their scheduling class; | |
543 | * - wakeup_rt tracer handles tasks belonging to sched_dl and | |
544 | * sched_rt class; | |
545 | * - wakeup_dl handles tasks belonging to sched_dl class only. | |
546 | */ | |
547 | if (tracing_dl || (wakeup_dl && !dl_task(p)) || | |
548 | (wakeup_rt && !dl_task(p) && !rt_task(p)) || | |
549 | (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio))) | |
352ad25a SR |
550 | return; |
551 | ||
1c5eb448 | 552 | disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); |
352ad25a SR |
553 | if (unlikely(disabled != 1)) |
554 | goto out; | |
555 | ||
36590c50 SAS |
556 | trace_ctx = tracing_gen_ctx(); |
557 | ||
352ad25a | 558 | /* interrupts should be off from try_to_wake_up */ |
0199c4e6 | 559 | arch_spin_lock(&wakeup_lock); |
352ad25a SR |
560 | |
561 | /* check for races. */ | |
af6ace76 DF |
562 | if (!tracer_enabled || tracing_dl || |
563 | (!dl_task(p) && p->prio >= wakeup_prio)) | |
352ad25a SR |
564 | goto out_locked; |
565 | ||
566 | /* reset the trace */ | |
b07c3f19 | 567 | __wakeup_reset(wakeup_trace); |
352ad25a SR |
568 | |
569 | wakeup_cpu = task_cpu(p); | |
478142c3 | 570 | wakeup_current_cpu = wakeup_cpu; |
352ad25a SR |
571 | wakeup_prio = p->prio; |
572 | ||
af6ace76 DF |
573 | /* |
574 | * Once you start tracing a -deadline task, don't bother tracing | |
575 | * another task until the first one wakes up. | |
576 | */ | |
577 | if (dl_task(p)) | |
bfbf8d15 | 578 | tracing_dl = true; |
af6ace76 | 579 | else |
bfbf8d15 | 580 | tracing_dl = false; |
af6ace76 | 581 | |
7b3c92b8 | 582 | wakeup_task = get_task_struct(p); |
352ad25a | 583 | |
1c5eb448 | 584 | data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu); |
f8ec1062 | 585 | data->preempt_timestamp = ftrace_now(cpu); |
36590c50 SAS |
586 | tracing_sched_wakeup_trace(wakeup_trace, p, current, trace_ctx); |
587 | __trace_stack(wakeup_trace, trace_ctx, 0); | |
301fd748 SR |
588 | |
589 | /* | |
590 | * We must be careful in using CALLER_ADDR2. But since wake_up | |
591 | * is not called by an assembly function (where as schedule is) | |
592 | * it should be safe to use it here. | |
593 | */ | |
36590c50 | 594 | __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, trace_ctx); |
352ad25a SR |
595 | |
596 | out_locked: | |
0199c4e6 | 597 | arch_spin_unlock(&wakeup_lock); |
352ad25a | 598 | out: |
1c5eb448 | 599 | atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); |
352ad25a SR |
600 | } |
601 | ||
e309b41d | 602 | static void start_wakeup_tracer(struct trace_array *tr) |
352ad25a | 603 | { |
5b82a1b0 MD |
604 | int ret; |
605 | ||
38516ab5 | 606 | ret = register_trace_sched_wakeup(probe_wakeup, NULL); |
5b82a1b0 | 607 | if (ret) { |
b07c3f19 | 608 | pr_info("wakeup trace: Couldn't activate tracepoint" |
5b82a1b0 MD |
609 | " probe to kernel_sched_wakeup\n"); |
610 | return; | |
611 | } | |
612 | ||
38516ab5 | 613 | ret = register_trace_sched_wakeup_new(probe_wakeup, NULL); |
5b82a1b0 | 614 | if (ret) { |
b07c3f19 | 615 | pr_info("wakeup trace: Couldn't activate tracepoint" |
5b82a1b0 MD |
616 | " probe to kernel_sched_wakeup_new\n"); |
617 | goto fail_deprobe; | |
618 | } | |
619 | ||
38516ab5 | 620 | ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL); |
5b82a1b0 | 621 | if (ret) { |
b07c3f19 | 622 | pr_info("sched trace: Couldn't activate tracepoint" |
73d8b8bc | 623 | " probe to kernel_sched_switch\n"); |
5b82a1b0 MD |
624 | goto fail_deprobe_wake_new; |
625 | } | |
626 | ||
38516ab5 | 627 | ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL); |
478142c3 SR |
628 | if (ret) { |
629 | pr_info("wakeup trace: Couldn't activate tracepoint" | |
630 | " probe to kernel_sched_migrate_task\n"); | |
50f9ad60 | 631 | goto fail_deprobe_sched_switch; |
478142c3 SR |
632 | } |
633 | ||
352ad25a SR |
634 | wakeup_reset(tr); |
635 | ||
636 | /* | |
637 | * Don't let the tracer_enabled = 1 show up before | |
638 | * the wakeup_task is reset. This may be overkill since | |
639 | * wakeup_reset does a spin_unlock after setting the | |
640 | * wakeup_task to NULL, but I want to be safe. | |
641 | * This is a slow path anyway. | |
642 | */ | |
643 | smp_wmb(); | |
644 | ||
983f938a | 645 | if (start_func_tracer(tr, is_graph(tr))) |
7495a5be | 646 | printk(KERN_ERR "failed to start wakeup tracer\n"); |
ad591240 | 647 | |
352ad25a | 648 | return; |
50f9ad60 KC |
649 | fail_deprobe_sched_switch: |
650 | unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL); | |
5b82a1b0 | 651 | fail_deprobe_wake_new: |
38516ab5 | 652 | unregister_trace_sched_wakeup_new(probe_wakeup, NULL); |
5b82a1b0 | 653 | fail_deprobe: |
38516ab5 | 654 | unregister_trace_sched_wakeup(probe_wakeup, NULL); |
352ad25a SR |
655 | } |
656 | ||
e309b41d | 657 | static void stop_wakeup_tracer(struct trace_array *tr) |
352ad25a SR |
658 | { |
659 | tracer_enabled = 0; | |
983f938a | 660 | stop_func_tracer(tr, is_graph(tr)); |
38516ab5 SR |
661 | unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL); |
662 | unregister_trace_sched_wakeup_new(probe_wakeup, NULL); | |
663 | unregister_trace_sched_wakeup(probe_wakeup, NULL); | |
664 | unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL); | |
352ad25a SR |
665 | } |
666 | ||
65daaca7 SRRH |
667 | static bool wakeup_busy; |
668 | ||
3244351c | 669 | static int __wakeup_tracer_init(struct trace_array *tr) |
352ad25a | 670 | { |
983f938a | 671 | save_flags = tr->trace_flags; |
613f04a0 SRRH |
672 | |
673 | /* non overwrite screws up the latency tracers */ | |
2b6080f2 SR |
674 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1); |
675 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1); | |
e9d25fe6 | 676 | |
6d9b3fa5 | 677 | tr->max_latency = 0; |
352ad25a | 678 | wakeup_trace = tr; |
4104d326 | 679 | ftrace_init_array_ops(tr, wakeup_tracer_call); |
c76f0694 | 680 | start_wakeup_tracer(tr); |
65daaca7 SRRH |
681 | |
682 | wakeup_busy = true; | |
1c80025a | 683 | return 0; |
352ad25a SR |
684 | } |
685 | ||
3244351c SR |
686 | static int wakeup_tracer_init(struct trace_array *tr) |
687 | { | |
65daaca7 SRRH |
688 | if (wakeup_busy) |
689 | return -EBUSY; | |
690 | ||
bfbf8d15 AK |
691 | wakeup_dl = false; |
692 | wakeup_rt = false; | |
3244351c SR |
693 | return __wakeup_tracer_init(tr); |
694 | } | |
695 | ||
696 | static int wakeup_rt_tracer_init(struct trace_array *tr) | |
697 | { | |
65daaca7 SRRH |
698 | if (wakeup_busy) |
699 | return -EBUSY; | |
700 | ||
bfbf8d15 AK |
701 | wakeup_dl = false; |
702 | wakeup_rt = true; | |
3244351c SR |
703 | return __wakeup_tracer_init(tr); |
704 | } | |
705 | ||
af6ace76 DF |
706 | static int wakeup_dl_tracer_init(struct trace_array *tr) |
707 | { | |
65daaca7 SRRH |
708 | if (wakeup_busy) |
709 | return -EBUSY; | |
710 | ||
bfbf8d15 AK |
711 | wakeup_dl = true; |
712 | wakeup_rt = false; | |
af6ace76 DF |
713 | return __wakeup_tracer_init(tr); |
714 | } | |
715 | ||
e309b41d | 716 | static void wakeup_tracer_reset(struct trace_array *tr) |
352ad25a | 717 | { |
613f04a0 SRRH |
718 | int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT; |
719 | int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE; | |
720 | ||
c76f0694 SR |
721 | stop_wakeup_tracer(tr); |
722 | /* make sure we put back any tasks we are tracing */ | |
723 | wakeup_reset(tr); | |
e9d25fe6 | 724 | |
2b6080f2 SR |
725 | set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag); |
726 | set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag); | |
4104d326 | 727 | ftrace_reset_array_ops(tr); |
65daaca7 | 728 | wakeup_busy = false; |
352ad25a SR |
729 | } |
730 | ||
9036990d SR |
731 | static void wakeup_tracer_start(struct trace_array *tr) |
732 | { | |
733 | wakeup_reset(tr); | |
734 | tracer_enabled = 1; | |
9036990d SR |
735 | } |
736 | ||
737 | static void wakeup_tracer_stop(struct trace_array *tr) | |
738 | { | |
739 | tracer_enabled = 0; | |
352ad25a SR |
740 | } |
741 | ||
742 | static struct tracer wakeup_tracer __read_mostly = | |
743 | { | |
744 | .name = "wakeup", | |
745 | .init = wakeup_tracer_init, | |
746 | .reset = wakeup_tracer_reset, | |
9036990d SR |
747 | .start = wakeup_tracer_start, |
748 | .stop = wakeup_tracer_stop, | |
f43c738b | 749 | .print_max = true, |
7495a5be JO |
750 | .print_header = wakeup_print_header, |
751 | .print_line = wakeup_print_line, | |
328df475 | 752 | .flag_changed = wakeup_flag_changed, |
60a11774 SR |
753 | #ifdef CONFIG_FTRACE_SELFTEST |
754 | .selftest = trace_selftest_startup_wakeup, | |
755 | #endif | |
7495a5be JO |
756 | .open = wakeup_trace_open, |
757 | .close = wakeup_trace_close, | |
65daaca7 | 758 | .allow_instances = true, |
f43c738b | 759 | .use_max_tr = true, |
352ad25a SR |
760 | }; |
761 | ||
3244351c SR |
762 | static struct tracer wakeup_rt_tracer __read_mostly = |
763 | { | |
764 | .name = "wakeup_rt", | |
765 | .init = wakeup_rt_tracer_init, | |
766 | .reset = wakeup_tracer_reset, | |
767 | .start = wakeup_tracer_start, | |
768 | .stop = wakeup_tracer_stop, | |
f43c738b | 769 | .print_max = true, |
7495a5be JO |
770 | .print_header = wakeup_print_header, |
771 | .print_line = wakeup_print_line, | |
328df475 | 772 | .flag_changed = wakeup_flag_changed, |
3244351c SR |
773 | #ifdef CONFIG_FTRACE_SELFTEST |
774 | .selftest = trace_selftest_startup_wakeup, | |
775 | #endif | |
7495a5be JO |
776 | .open = wakeup_trace_open, |
777 | .close = wakeup_trace_close, | |
65daaca7 | 778 | .allow_instances = true, |
f43c738b | 779 | .use_max_tr = true, |
3244351c SR |
780 | }; |
781 | ||
af6ace76 DF |
782 | static struct tracer wakeup_dl_tracer __read_mostly = |
783 | { | |
784 | .name = "wakeup_dl", | |
785 | .init = wakeup_dl_tracer_init, | |
786 | .reset = wakeup_tracer_reset, | |
787 | .start = wakeup_tracer_start, | |
788 | .stop = wakeup_tracer_stop, | |
af6ace76 DF |
789 | .print_max = true, |
790 | .print_header = wakeup_print_header, | |
791 | .print_line = wakeup_print_line, | |
af6ace76 DF |
792 | .flag_changed = wakeup_flag_changed, |
793 | #ifdef CONFIG_FTRACE_SELFTEST | |
794 | .selftest = trace_selftest_startup_wakeup, | |
795 | #endif | |
796 | .open = wakeup_trace_open, | |
797 | .close = wakeup_trace_close, | |
8d414bd2 | 798 | .allow_instances = true, |
af6ace76 DF |
799 | .use_max_tr = true, |
800 | }; | |
801 | ||
352ad25a SR |
802 | __init static int init_wakeup_tracer(void) |
803 | { | |
804 | int ret; | |
805 | ||
806 | ret = register_tracer(&wakeup_tracer); | |
807 | if (ret) | |
808 | return ret; | |
809 | ||
3244351c SR |
810 | ret = register_tracer(&wakeup_rt_tracer); |
811 | if (ret) | |
812 | return ret; | |
813 | ||
af6ace76 DF |
814 | ret = register_tracer(&wakeup_dl_tracer); |
815 | if (ret) | |
816 | return ret; | |
817 | ||
352ad25a SR |
818 | return 0; |
819 | } | |
6f415672 | 820 | core_initcall(init_wakeup_tracer); |