Commit | Line | Data |
---|---|---|
1f0d69a9 SR |
1 | /* |
2 | * unlikely profiler | |
3 | * | |
4 | * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> | |
5 | */ | |
6 | #include <linux/kallsyms.h> | |
7 | #include <linux/seq_file.h> | |
8 | #include <linux/spinlock.h> | |
65c6dc6a | 9 | #include <linux/irqflags.h> |
1f0d69a9 SR |
10 | #include <linux/debugfs.h> |
11 | #include <linux/uaccess.h> | |
12 | #include <linux/module.h> | |
13 | #include <linux/ftrace.h> | |
14 | #include <linux/hash.h> | |
15 | #include <linux/fs.h> | |
16 | #include <asm/local.h> | |
f633cef0 | 17 | |
1f0d69a9 | 18 | #include "trace.h" |
002bb86d | 19 | #include "trace_stat.h" |
f633cef0 | 20 | #include "trace_output.h" |
1f0d69a9 | 21 | |
2ed84eeb | 22 | #ifdef CONFIG_BRANCH_TRACER |
52f232cb | 23 | |
002bb86d | 24 | static struct tracer branch_trace; |
9f029e83 SR |
25 | static int branch_tracing_enabled __read_mostly; |
26 | static DEFINE_MUTEX(branch_tracing_mutex); | |
e302cf3f | 27 | |
9f029e83 | 28 | static struct trace_array *branch_tracer; |
52f232cb SR |
29 | |
30 | static void | |
9f029e83 | 31 | probe_likely_condition(struct ftrace_branch_data *f, int val, int expect) |
52f232cb | 32 | { |
9f029e83 | 33 | struct trace_array *tr = branch_tracer; |
52f232cb | 34 | struct ring_buffer_event *event; |
9f029e83 | 35 | struct trace_branch *entry; |
0a987751 | 36 | unsigned long flags; |
52f232cb SR |
37 | int cpu, pc; |
38 | const char *p; | |
39 | ||
40 | /* | |
41 | * I would love to save just the ftrace_likely_data pointer, but | |
42 | * this code can also be used by modules. Ugly things can happen | |
43 | * if the module is unloaded, and then we go and read the | |
44 | * pointer. This is slower, but much safer. | |
45 | */ | |
46 | ||
47 | if (unlikely(!tr)) | |
48 | return; | |
49 | ||
a5e25883 | 50 | local_irq_save(flags); |
52f232cb SR |
51 | cpu = raw_smp_processor_id(); |
52 | if (atomic_inc_return(&tr->data[cpu]->disabled) != 1) | |
53 | goto out; | |
54 | ||
51a763dd ACM |
55 | pc = preempt_count(); |
56 | event = trace_buffer_lock_reserve(tr, TRACE_BRANCH, | |
57 | sizeof(*entry), flags, pc); | |
52f232cb SR |
58 | if (!event) |
59 | goto out; | |
60 | ||
52f232cb | 61 | entry = ring_buffer_event_data(event); |
52f232cb SR |
62 | |
63 | /* Strip off the path, only save the file */ | |
64 | p = f->file + strlen(f->file); | |
65 | while (p >= f->file && *p != '/') | |
66 | p--; | |
67 | p++; | |
68 | ||
69 | strncpy(entry->func, f->func, TRACE_FUNC_SIZE); | |
70 | strncpy(entry->file, p, TRACE_FILE_SIZE); | |
71 | entry->func[TRACE_FUNC_SIZE] = 0; | |
72 | entry->file[TRACE_FILE_SIZE] = 0; | |
73 | entry->line = f->line; | |
74 | entry->correct = val == expect; | |
75 | ||
0a987751 | 76 | ring_buffer_unlock_commit(tr->buffer, event); |
52f232cb SR |
77 | |
78 | out: | |
79 | atomic_dec(&tr->data[cpu]->disabled); | |
a5e25883 | 80 | local_irq_restore(flags); |
52f232cb SR |
81 | } |
82 | ||
83 | static inline | |
9f029e83 | 84 | void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) |
52f232cb | 85 | { |
9f029e83 | 86 | if (!branch_tracing_enabled) |
52f232cb SR |
87 | return; |
88 | ||
89 | probe_likely_condition(f, val, expect); | |
90 | } | |
91 | ||
9f029e83 | 92 | int enable_branch_tracing(struct trace_array *tr) |
52f232cb SR |
93 | { |
94 | int ret = 0; | |
95 | ||
9f029e83 SR |
96 | mutex_lock(&branch_tracing_mutex); |
97 | branch_tracer = tr; | |
52f232cb SR |
98 | /* |
99 | * Must be seen before enabling. The reader is a condition | |
100 | * where we do not need a matching rmb() | |
101 | */ | |
102 | smp_wmb(); | |
9f029e83 SR |
103 | branch_tracing_enabled++; |
104 | mutex_unlock(&branch_tracing_mutex); | |
52f232cb SR |
105 | |
106 | return ret; | |
107 | } | |
108 | ||
9f029e83 | 109 | void disable_branch_tracing(void) |
52f232cb | 110 | { |
9f029e83 | 111 | mutex_lock(&branch_tracing_mutex); |
52f232cb | 112 | |
9f029e83 | 113 | if (!branch_tracing_enabled) |
52f232cb SR |
114 | goto out_unlock; |
115 | ||
9f029e83 | 116 | branch_tracing_enabled--; |
52f232cb SR |
117 | |
118 | out_unlock: | |
9f029e83 | 119 | mutex_unlock(&branch_tracing_mutex); |
52f232cb | 120 | } |
80e5ea45 SR |
121 | |
122 | static void start_branch_trace(struct trace_array *tr) | |
123 | { | |
124 | enable_branch_tracing(tr); | |
125 | } | |
126 | ||
127 | static void stop_branch_trace(struct trace_array *tr) | |
128 | { | |
129 | disable_branch_tracing(); | |
130 | } | |
131 | ||
1c80025a | 132 | static int branch_trace_init(struct trace_array *tr) |
80e5ea45 | 133 | { |
80e5ea45 | 134 | start_branch_trace(tr); |
1c80025a | 135 | return 0; |
80e5ea45 SR |
136 | } |
137 | ||
138 | static void branch_trace_reset(struct trace_array *tr) | |
139 | { | |
140 | stop_branch_trace(tr); | |
141 | } | |
142 | ||
ae7462b4 ACM |
143 | static enum print_line_t trace_branch_print(struct trace_iterator *iter, |
144 | int flags) | |
f633cef0 SR |
145 | { |
146 | struct trace_branch *field; | |
147 | ||
2c9b238e | 148 | trace_assign_type(field, iter->ent); |
f633cef0 | 149 | |
2c9b238e | 150 | if (trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n", |
f633cef0 SR |
151 | field->correct ? " ok " : " MISS ", |
152 | field->func, | |
153 | field->file, | |
154 | field->line)) | |
155 | return TRACE_TYPE_PARTIAL_LINE; | |
156 | ||
d9793bd8 | 157 | return TRACE_TYPE_HANDLED; |
f633cef0 SR |
158 | } |
159 | ||
e302cf3f | 160 | |
f633cef0 SR |
161 | static struct trace_event trace_branch_event = { |
162 | .type = TRACE_BRANCH, | |
163 | .trace = trace_branch_print, | |
164 | .latency_trace = trace_branch_print, | |
f633cef0 SR |
165 | }; |
166 | ||
002bb86d FW |
167 | static struct tracer branch_trace __read_mostly = |
168 | { | |
169 | .name = "branch", | |
170 | .init = branch_trace_init, | |
171 | .reset = branch_trace_reset, | |
172 | #ifdef CONFIG_FTRACE_SELFTEST | |
173 | .selftest = trace_selftest_startup_branch, | |
174 | #endif /* CONFIG_FTRACE_SELFTEST */ | |
175 | }; | |
176 | ||
177 | __init static int init_branch_tracer(void) | |
178 | { | |
179 | int ret; | |
180 | ||
181 | ret = register_ftrace_event(&trace_branch_event); | |
182 | if (!ret) { | |
183 | printk(KERN_WARNING "Warning: could not register " | |
184 | "branch events\n"); | |
185 | return 1; | |
186 | } | |
187 | return register_tracer(&branch_trace); | |
188 | } | |
189 | device_initcall(init_branch_tracer); | |
190 | ||
52f232cb SR |
191 | #else |
192 | static inline | |
9f029e83 | 193 | void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect) |
52f232cb SR |
194 | { |
195 | } | |
2ed84eeb | 196 | #endif /* CONFIG_BRANCH_TRACER */ |
52f232cb | 197 | |
9f029e83 | 198 | void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect) |
1f0d69a9 | 199 | { |
52f232cb SR |
200 | /* |
201 | * I would love to have a trace point here instead, but the | |
202 | * trace point code is so inundated with unlikely and likely | |
203 | * conditions that the recursive nightmare that exists is too | |
204 | * much to try to get working. At least for now. | |
205 | */ | |
206 | trace_likely_condition(f, val, expect); | |
207 | ||
1f0d69a9 SR |
208 | /* FIXME: Make this atomic! */ |
209 | if (val == expect) | |
210 | f->correct++; | |
211 | else | |
212 | f->incorrect++; | |
213 | } | |
214 | EXPORT_SYMBOL(ftrace_likely_update); | |
215 | ||
e302cf3f FW |
216 | extern unsigned long __start_annotated_branch_profile[]; |
217 | extern unsigned long __stop_annotated_branch_profile[]; | |
1f0d69a9 | 218 | |
e302cf3f | 219 | static int annotated_branch_stat_headers(struct seq_file *m) |
1f0d69a9 | 220 | { |
e302cf3f FW |
221 | seq_printf(m, " correct incorrect %% "); |
222 | seq_printf(m, " Function " | |
223 | " File Line\n" | |
224 | " ------- --------- - " | |
225 | " -------- " | |
226 | " ---- ----\n"); | |
227 | return 0; | |
1f0d69a9 SR |
228 | } |
229 | ||
e302cf3f | 230 | static inline long get_incorrect_percent(struct ftrace_branch_data *p) |
1f0d69a9 | 231 | { |
e302cf3f | 232 | long percent; |
1f0d69a9 | 233 | |
e302cf3f FW |
234 | if (p->correct) { |
235 | percent = p->incorrect * 100; | |
236 | percent /= p->correct + p->incorrect; | |
237 | } else | |
238 | percent = p->incorrect ? 100 : -1; | |
1f0d69a9 | 239 | |
e302cf3f | 240 | return percent; |
1f0d69a9 SR |
241 | } |
242 | ||
e302cf3f | 243 | static int branch_stat_show(struct seq_file *m, void *v) |
1f0d69a9 | 244 | { |
9f029e83 | 245 | struct ftrace_branch_data *p = v; |
1f0d69a9 | 246 | const char *f; |
bac28bfe | 247 | long percent; |
1f0d69a9 | 248 | |
1f0d69a9 SR |
249 | /* Only print the file, not the path */ |
250 | f = p->file + strlen(p->file); | |
251 | while (f >= p->file && *f != '/') | |
252 | f--; | |
253 | f++; | |
254 | ||
2bcd521a SR |
255 | /* |
256 | * The miss is overlayed on correct, and hit on incorrect. | |
257 | */ | |
e302cf3f | 258 | percent = get_incorrect_percent(p); |
1f0d69a9 | 259 | |
bac28bfe SR |
260 | seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect); |
261 | if (percent < 0) | |
262 | seq_printf(m, " X "); | |
263 | else | |
264 | seq_printf(m, "%3ld ", percent); | |
1f0d69a9 SR |
265 | seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line); |
266 | return 0; | |
267 | } | |
268 | ||
e302cf3f FW |
269 | static void *annotated_branch_stat_start(void) |
270 | { | |
271 | return __start_annotated_branch_profile; | |
272 | } | |
1f0d69a9 | 273 | |
e302cf3f FW |
274 | static void * |
275 | annotated_branch_stat_next(void *v, int idx) | |
1f0d69a9 | 276 | { |
e302cf3f | 277 | struct ftrace_branch_data *p = v; |
1f0d69a9 | 278 | |
e302cf3f | 279 | ++p; |
1f0d69a9 | 280 | |
e302cf3f FW |
281 | if ((void *)p >= (void *)__stop_annotated_branch_profile) |
282 | return NULL; | |
283 | ||
284 | return p; | |
1f0d69a9 SR |
285 | } |
286 | ||
e302cf3f FW |
287 | static int annotated_branch_stat_cmp(void *p1, void *p2) |
288 | { | |
289 | struct ftrace_branch_data *a = p1; | |
290 | struct ftrace_branch_data *b = p2; | |
291 | ||
292 | long percent_a, percent_b; | |
293 | ||
294 | percent_a = get_incorrect_percent(a); | |
295 | percent_b = get_incorrect_percent(b); | |
296 | ||
297 | if (percent_a < percent_b) | |
298 | return -1; | |
299 | if (percent_a > percent_b) | |
300 | return 1; | |
301 | else | |
302 | return 0; | |
303 | } | |
1f0d69a9 | 304 | |
002bb86d FW |
305 | static struct tracer_stat annotated_branch_stats = { |
306 | .name = "branch_annotated", | |
307 | .stat_start = annotated_branch_stat_start, | |
308 | .stat_next = annotated_branch_stat_next, | |
309 | .stat_cmp = annotated_branch_stat_cmp, | |
310 | .stat_headers = annotated_branch_stat_headers, | |
311 | .stat_show = branch_stat_show | |
312 | }; | |
313 | ||
314 | __init static int init_annotated_branch_stats(void) | |
315 | { | |
316 | int ret; | |
317 | ||
318 | ret = register_stat_tracer(&annotated_branch_stats); | |
319 | if (!ret) { | |
320 | printk(KERN_WARNING "Warning: could not register " | |
321 | "annotated branches stats\n"); | |
322 | return 1; | |
323 | } | |
324 | return 0; | |
325 | } | |
326 | fs_initcall(init_annotated_branch_stats); | |
327 | ||
2bcd521a | 328 | #ifdef CONFIG_PROFILE_ALL_BRANCHES |
2bcd521a | 329 | |
e302cf3f FW |
330 | extern unsigned long __start_branch_profile[]; |
331 | extern unsigned long __stop_branch_profile[]; | |
1f0d69a9 | 332 | |
e302cf3f FW |
333 | static int all_branch_stat_headers(struct seq_file *m) |
334 | { | |
335 | seq_printf(m, " miss hit %% "); | |
336 | seq_printf(m, " Function " | |
337 | " File Line\n" | |
338 | " ------- --------- - " | |
339 | " -------- " | |
340 | " ---- ----\n"); | |
341 | return 0; | |
342 | } | |
1f0d69a9 | 343 | |
e302cf3f | 344 | static void *all_branch_stat_start(void) |
1f0d69a9 | 345 | { |
e302cf3f FW |
346 | return __start_branch_profile; |
347 | } | |
348 | ||
349 | static void * | |
350 | all_branch_stat_next(void *v, int idx) | |
351 | { | |
352 | struct ftrace_branch_data *p = v; | |
1f0d69a9 | 353 | |
e302cf3f | 354 | ++p; |
1f0d69a9 | 355 | |
e302cf3f FW |
356 | if ((void *)p >= (void *)__stop_branch_profile) |
357 | return NULL; | |
1f0d69a9 | 358 | |
e302cf3f FW |
359 | return p; |
360 | } | |
2bcd521a | 361 | |
002bb86d FW |
362 | static struct tracer_stat all_branch_stats = { |
363 | .name = "branch_all", | |
034939b6 FW |
364 | .stat_start = all_branch_stat_start, |
365 | .stat_next = all_branch_stat_next, | |
366 | .stat_headers = all_branch_stat_headers, | |
002bb86d | 367 | .stat_show = branch_stat_show |
034939b6 | 368 | }; |
e302cf3f | 369 | |
002bb86d | 370 | __init static int all_annotated_branch_stats(void) |
e302cf3f | 371 | { |
e302cf3f | 372 | int ret; |
002bb86d FW |
373 | |
374 | ret = register_stat_tracer(&all_branch_stats); | |
e302cf3f | 375 | if (!ret) { |
002bb86d FW |
376 | printk(KERN_WARNING "Warning: could not register " |
377 | "all branches stats\n"); | |
e302cf3f FW |
378 | return 1; |
379 | } | |
002bb86d | 380 | return 0; |
e302cf3f | 381 | } |
002bb86d FW |
382 | fs_initcall(all_annotated_branch_stats); |
383 | #endif /* CONFIG_PROFILE_ALL_BRANCHES */ |