Commit | Line | Data |
---|---|---|
1e9b51c2 | 1 | /* |
e9a22d1f | 2 | * h/w branch tracer for x86 based on BTS |
1e9b51c2 | 3 | * |
5c5317de MM |
4 | * Copyright (C) 2008-2009 Intel Corporation. |
5 | * Markus Metzger <markus.t.metzger@gmail.com>, 2008-2009 | |
1e9b51c2 | 6 | */ |
2d542cf3 | 7 | #include <linux/kallsyms.h> |
1e9b51c2 MM |
8 | #include <linux/debugfs.h> |
9 | #include <linux/ftrace.h> | |
2d542cf3 | 10 | #include <linux/module.h> |
5c5317de MM |
11 | #include <linux/cpu.h> |
12 | #include <linux/smp.h> | |
2d542cf3 | 13 | #include <linux/fs.h> |
1e9b51c2 MM |
14 | |
15 | #include <asm/ds.h> | |
16 | ||
f0868d1e | 17 | #include "trace_output.h" |
e9a22d1f | 18 | #include "trace.h" |
1e9b51c2 MM |
19 | |
20 | ||
ba9372a8 | 21 | #define BTS_BUFFER_SIZE (1 << 13) |
1e9b51c2 | 22 | |
9705f69e TH |
23 | static DEFINE_PER_CPU(struct bts_tracer *, hwb_tracer); |
24 | static DEFINE_PER_CPU(unsigned char[BTS_BUFFER_SIZE], hwb_buffer); | |
1e9b51c2 | 25 | |
9705f69e | 26 | #define this_tracer per_cpu(hwb_tracer, smp_processor_id()) |
1e9b51c2 | 27 | |
ba9372a8 MM |
28 | static int trace_hw_branches_enabled __read_mostly; |
29 | static int trace_hw_branches_suspended __read_mostly; | |
b1818748 | 30 | static struct trace_array *hw_branch_trace __read_mostly; |
1e9b51c2 | 31 | |
5c5317de | 32 | |
de79f54f | 33 | static void bts_trace_init_cpu(int cpu) |
1e9b51c2 | 34 | { |
9705f69e TH |
35 | per_cpu(hwb_tracer, cpu) = |
36 | ds_request_bts_cpu(cpu, per_cpu(hwb_buffer, cpu), | |
37 | BTS_BUFFER_SIZE, NULL, (size_t)-1, | |
38 | BTS_KERNEL); | |
a93751ca | 39 | |
9705f69e TH |
40 | if (IS_ERR(per_cpu(hwb_tracer, cpu))) |
41 | per_cpu(hwb_tracer, cpu) = NULL; | |
1e9b51c2 MM |
42 | } |
43 | ||
ba9372a8 | 44 | static int bts_trace_init(struct trace_array *tr) |
1e9b51c2 | 45 | { |
de79f54f | 46 | int cpu; |
1e9b51c2 | 47 | |
ba9372a8 | 48 | hw_branch_trace = tr; |
de79f54f | 49 | trace_hw_branches_enabled = 0; |
1e9b51c2 | 50 | |
de79f54f MM |
51 | get_online_cpus(); |
52 | for_each_online_cpu(cpu) { | |
53 | bts_trace_init_cpu(cpu); | |
1e9b51c2 | 54 | |
9705f69e | 55 | if (likely(per_cpu(hwb_tracer, cpu))) |
de79f54f MM |
56 | trace_hw_branches_enabled = 1; |
57 | } | |
ba9372a8 | 58 | trace_hw_branches_suspended = 0; |
de79f54f | 59 | put_online_cpus(); |
ba9372a8 MM |
60 | |
61 | /* If we could not enable tracing on a single cpu, we fail. */ | |
de79f54f | 62 | return trace_hw_branches_enabled ? 0 : -EOPNOTSUPP; |
1e9b51c2 MM |
63 | } |
64 | ||
ba9372a8 | 65 | static void bts_trace_reset(struct trace_array *tr) |
1e9b51c2 | 66 | { |
de79f54f | 67 | int cpu; |
5c5317de | 68 | |
de79f54f MM |
69 | get_online_cpus(); |
70 | for_each_online_cpu(cpu) { | |
9705f69e TH |
71 | if (likely(per_cpu(hwb_tracer, cpu))) { |
72 | ds_release_bts(per_cpu(hwb_tracer, cpu)); | |
73 | per_cpu(hwb_tracer, cpu) = NULL; | |
de79f54f | 74 | } |
1e9b51c2 | 75 | } |
5c5317de | 76 | trace_hw_branches_enabled = 0; |
ba9372a8 | 77 | trace_hw_branches_suspended = 0; |
de79f54f | 78 | put_online_cpus(); |
1e9b51c2 MM |
79 | } |
80 | ||
ba9372a8 | 81 | static void bts_trace_start(struct trace_array *tr) |
1e9b51c2 | 82 | { |
de79f54f | 83 | int cpu; |
5c5317de | 84 | |
de79f54f MM |
85 | get_online_cpus(); |
86 | for_each_online_cpu(cpu) | |
9705f69e TH |
87 | if (likely(per_cpu(hwb_tracer, cpu))) |
88 | ds_resume_bts(per_cpu(hwb_tracer, cpu)); | |
ba9372a8 | 89 | trace_hw_branches_suspended = 0; |
de79f54f | 90 | put_online_cpus(); |
ba9372a8 | 91 | } |
1e9b51c2 | 92 | |
ba9372a8 MM |
93 | static void bts_trace_stop(struct trace_array *tr) |
94 | { | |
de79f54f | 95 | int cpu; |
ba9372a8 | 96 | |
de79f54f MM |
97 | get_online_cpus(); |
98 | for_each_online_cpu(cpu) | |
9705f69e TH |
99 | if (likely(per_cpu(hwb_tracer, cpu))) |
100 | ds_suspend_bts(per_cpu(hwb_tracer, cpu)); | |
ba9372a8 | 101 | trace_hw_branches_suspended = 1; |
de79f54f | 102 | put_online_cpus(); |
5c5317de MM |
103 | } |
104 | ||
105 | static int __cpuinit bts_hotcpu_handler(struct notifier_block *nfb, | |
106 | unsigned long action, void *hcpu) | |
107 | { | |
de79f54f | 108 | int cpu = (long)hcpu; |
5c5317de MM |
109 | |
110 | switch (action) { | |
111 | case CPU_ONLINE: | |
112 | case CPU_DOWN_FAILED: | |
de79f54f MM |
113 | /* The notification is sent with interrupts enabled. */ |
114 | if (trace_hw_branches_enabled) { | |
115 | bts_trace_init_cpu(cpu); | |
116 | ||
117 | if (trace_hw_branches_suspended && | |
9705f69e TH |
118 | likely(per_cpu(hwb_tracer, cpu))) |
119 | ds_suspend_bts(per_cpu(hwb_tracer, cpu)); | |
de79f54f | 120 | } |
5c5317de | 121 | break; |
de79f54f | 122 | |
5c5317de | 123 | case CPU_DOWN_PREPARE: |
de79f54f | 124 | /* The notification is sent with interrupts enabled. */ |
9705f69e TH |
125 | if (likely(per_cpu(hwb_tracer, cpu))) { |
126 | ds_release_bts(per_cpu(hwb_tracer, cpu)); | |
127 | per_cpu(hwb_tracer, cpu) = NULL; | |
de79f54f | 128 | } |
5c5317de MM |
129 | } |
130 | ||
5c5317de | 131 | return NOTIFY_DONE; |
1e9b51c2 MM |
132 | } |
133 | ||
5c5317de MM |
134 | static struct notifier_block bts_hotcpu_notifier __cpuinitdata = { |
135 | .notifier_call = bts_hotcpu_handler | |
136 | }; | |
137 | ||
1e9b51c2 MM |
138 | static void bts_trace_print_header(struct seq_file *m) |
139 | { | |
11edda06 | 140 | seq_puts(m, "# CPU# TO <- FROM\n"); |
1e9b51c2 MM |
141 | } |
142 | ||
143 | static enum print_line_t bts_trace_print_line(struct trace_iterator *iter) | |
144 | { | |
e9a22d1f | 145 | unsigned long symflags = TRACE_ITER_SYM_OFFSET; |
1e9b51c2 MM |
146 | struct trace_entry *entry = iter->ent; |
147 | struct trace_seq *seq = &iter->seq; | |
a93751ca | 148 | struct hw_branch_entry *it; |
1e9b51c2 MM |
149 | |
150 | trace_assign_type(it, entry); | |
151 | ||
a93751ca | 152 | if (entry->type == TRACE_HW_BRANCHES) { |
1830b52d | 153 | if (trace_seq_printf(seq, "%4d ", iter->cpu) && |
11edda06 MM |
154 | seq_print_ip_sym(seq, it->to, symflags) && |
155 | trace_seq_printf(seq, "\t <- ") && | |
156 | seq_print_ip_sym(seq, it->from, symflags) && | |
a93751ca MM |
157 | trace_seq_printf(seq, "\n")) |
158 | return TRACE_TYPE_HANDLED; | |
a419aef8 | 159 | return TRACE_TYPE_PARTIAL_LINE; |
1e9b51c2 MM |
160 | } |
161 | return TRACE_TYPE_UNHANDLED; | |
162 | } | |
163 | ||
b1818748 | 164 | void trace_hw_branch(u64 from, u64 to) |
1e9b51c2 | 165 | { |
e1112b4d | 166 | struct ftrace_event_call *call = &event_hw_branch; |
b1818748 | 167 | struct trace_array *tr = hw_branch_trace; |
1e9b51c2 | 168 | struct ring_buffer_event *event; |
c8647b28 | 169 | struct ring_buffer *buf; |
a93751ca | 170 | struct hw_branch_entry *entry; |
0a987751 | 171 | unsigned long irq1; |
5c5317de | 172 | int cpu; |
1e9b51c2 | 173 | |
5c5317de MM |
174 | if (unlikely(!tr)) |
175 | return; | |
176 | ||
177 | if (unlikely(!trace_hw_branches_enabled)) | |
1e9b51c2 | 178 | return; |
5c5317de MM |
179 | |
180 | local_irq_save(irq1); | |
181 | cpu = raw_smp_processor_id(); | |
182 | if (atomic_inc_return(&tr->data[cpu]->disabled) != 1) | |
183 | goto out; | |
184 | ||
c8647b28 ZX |
185 | buf = tr->buffer; |
186 | event = trace_buffer_lock_reserve(buf, TRACE_HW_BRANCHES, | |
51a763dd | 187 | sizeof(*entry), 0, 0); |
5c5317de MM |
188 | if (!event) |
189 | goto out; | |
1e9b51c2 MM |
190 | entry = ring_buffer_event_data(event); |
191 | tracing_generic_entry_update(&entry->ent, 0, from); | |
a93751ca | 192 | entry->ent.type = TRACE_HW_BRANCHES; |
1e9b51c2 MM |
193 | entry->from = from; |
194 | entry->to = to; | |
c8647b28 ZX |
195 | if (!filter_check_discard(call, entry, buf, event)) |
196 | trace_buffer_unlock_commit(buf, event, 0, 0); | |
5c5317de MM |
197 | |
198 | out: | |
199 | atomic_dec(&tr->data[cpu]->disabled); | |
200 | local_irq_restore(irq1); | |
1e9b51c2 MM |
201 | } |
202 | ||
b1818748 | 203 | static void trace_bts_at(const struct bts_trace *trace, void *at) |
1e9b51c2 | 204 | { |
a93751ca MM |
205 | struct bts_struct bts; |
206 | int err = 0; | |
1e9b51c2 | 207 | |
a93751ca MM |
208 | WARN_ON_ONCE(!trace->read); |
209 | if (!trace->read) | |
1e9b51c2 MM |
210 | return; |
211 | ||
a93751ca MM |
212 | err = trace->read(this_tracer, at, &bts); |
213 | if (err < 0) | |
214 | return; | |
1e9b51c2 | 215 | |
a93751ca MM |
216 | switch (bts.qualifier) { |
217 | case BTS_BRANCH: | |
b1818748 | 218 | trace_hw_branch(bts.variant.lbr.from, bts.variant.lbr.to); |
a93751ca MM |
219 | break; |
220 | } | |
1e9b51c2 MM |
221 | } |
222 | ||
5c5317de MM |
223 | /* |
224 | * Collect the trace on the current cpu and write it into the ftrace buffer. | |
225 | * | |
de79f54f | 226 | * pre: tracing must be suspended on the current cpu |
5c5317de | 227 | */ |
1e9b51c2 MM |
228 | static void trace_bts_cpu(void *arg) |
229 | { | |
ba9372a8 | 230 | struct trace_array *tr = (struct trace_array *)arg; |
a93751ca MM |
231 | const struct bts_trace *trace; |
232 | unsigned char *at; | |
1e9b51c2 | 233 | |
b1818748 | 234 | if (unlikely(!tr)) |
1e9b51c2 MM |
235 | return; |
236 | ||
5c5317de MM |
237 | if (unlikely(atomic_read(&tr->data[raw_smp_processor_id()]->disabled))) |
238 | return; | |
239 | ||
b1818748 MM |
240 | if (unlikely(!this_tracer)) |
241 | return; | |
242 | ||
a93751ca MM |
243 | trace = ds_read_bts(this_tracer); |
244 | if (!trace) | |
de79f54f | 245 | return; |
1e9b51c2 | 246 | |
a93751ca MM |
247 | for (at = trace->ds.top; (void *)at < trace->ds.end; |
248 | at += trace->ds.size) | |
b1818748 | 249 | trace_bts_at(trace, at); |
1e9b51c2 | 250 | |
a93751ca MM |
251 | for (at = trace->ds.begin; (void *)at < trace->ds.top; |
252 | at += trace->ds.size) | |
b1818748 | 253 | trace_bts_at(trace, at); |
1e9b51c2 MM |
254 | } |
255 | ||
256 | static void trace_bts_prepare(struct trace_iterator *iter) | |
257 | { | |
de79f54f | 258 | int cpu; |
5c5317de | 259 | |
de79f54f MM |
260 | get_online_cpus(); |
261 | for_each_online_cpu(cpu) | |
9705f69e TH |
262 | if (likely(per_cpu(hwb_tracer, cpu))) |
263 | ds_suspend_bts(per_cpu(hwb_tracer, cpu)); | |
de79f54f MM |
264 | /* |
265 | * We need to collect the trace on the respective cpu since ftrace | |
266 | * implicitly adds the record for the current cpu. | |
267 | * Once that is more flexible, we could collect the data from any cpu. | |
268 | */ | |
5c5317de | 269 | on_each_cpu(trace_bts_cpu, iter->tr, 1); |
1e9b51c2 | 270 | |
de79f54f | 271 | for_each_online_cpu(cpu) |
9705f69e TH |
272 | if (likely(per_cpu(hwb_tracer, cpu))) |
273 | ds_resume_bts(per_cpu(hwb_tracer, cpu)); | |
de79f54f | 274 | put_online_cpus(); |
1e9b51c2 MM |
275 | } |
276 | ||
e23b8ad8 MM |
277 | static void trace_bts_close(struct trace_iterator *iter) |
278 | { | |
279 | tracing_reset_online_cpus(iter->tr); | |
280 | } | |
281 | ||
b1818748 MM |
282 | void trace_hw_branch_oops(void) |
283 | { | |
de79f54f MM |
284 | if (this_tracer) { |
285 | ds_suspend_bts_noirq(this_tracer); | |
ba9372a8 | 286 | trace_bts_cpu(hw_branch_trace); |
de79f54f MM |
287 | ds_resume_bts_noirq(this_tracer); |
288 | } | |
b1818748 MM |
289 | } |
290 | ||
1e9b51c2 MM |
291 | struct tracer bts_tracer __read_mostly = |
292 | { | |
a93751ca | 293 | .name = "hw-branch-tracer", |
1e9b51c2 | 294 | .init = bts_trace_init, |
5c5317de | 295 | .reset = bts_trace_reset, |
1e9b51c2 MM |
296 | .print_header = bts_trace_print_header, |
297 | .print_line = bts_trace_print_line, | |
298 | .start = bts_trace_start, | |
299 | .stop = bts_trace_stop, | |
e23b8ad8 | 300 | .open = trace_bts_prepare, |
321bb5e1 MM |
301 | .close = trace_bts_close, |
302 | #ifdef CONFIG_FTRACE_SELFTEST | |
303 | .selftest = trace_selftest_startup_hw_branches, | |
304 | #endif /* CONFIG_FTRACE_SELFTEST */ | |
1e9b51c2 MM |
305 | }; |
306 | ||
307 | __init static int init_bts_trace(void) | |
308 | { | |
5e01cb69 | 309 | register_hotcpu_notifier(&bts_hotcpu_notifier); |
1e9b51c2 MM |
310 | return register_tracer(&bts_tracer); |
311 | } | |
312 | device_initcall(init_bts_trace); |