Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
de0428a7 KW |
2 | #include <linux/perf_event.h> |
3 | #include <linux/types.h> | |
4 | ||
5 | #include <asm/perf_event.h> | |
6 | #include <asm/msr.h> | |
3e702ff6 | 7 | #include <asm/insn.h> |
de0428a7 | 8 | |
27f6d22b | 9 | #include "../perf_event.h" |
caff2bef PZ |
10 | |
11 | enum { | |
12 | LBR_FORMAT_32 = 0x00, | |
13 | LBR_FORMAT_LIP = 0x01, | |
14 | LBR_FORMAT_EIP = 0x02, | |
15 | LBR_FORMAT_EIP_FLAGS = 0x03, | |
135c5612 | 16 | LBR_FORMAT_EIP_FLAGS2 = 0x04, |
50eab8f6 | 17 | LBR_FORMAT_INFO = 0x05, |
8b92c3a7 KL |
18 | LBR_FORMAT_TIME = 0x06, |
19 | LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_TIME, | |
135c5612 AK |
20 | }; |
21 | ||
e91c8d97 | 22 | static const enum { |
135c5612 AK |
23 | LBR_EIP_FLAGS = 1, |
24 | LBR_TSX = 2, | |
25 | } lbr_desc[LBR_FORMAT_MAX_KNOWN + 1] = { | |
26 | [LBR_FORMAT_EIP_FLAGS] = LBR_EIP_FLAGS, | |
27 | [LBR_FORMAT_EIP_FLAGS2] = LBR_EIP_FLAGS | LBR_TSX, | |
caff2bef PZ |
28 | }; |
29 | ||
c5cc2cd9 SE |
30 | /* |
31 | * Intel LBR_SELECT bits | |
32 | * Intel Vol3a, April 2011, Section 16.7 Table 16-10 | |
33 | * | |
34 | * Hardware branch filter (not available on all CPUs) | |
35 | */ | |
36 | #define LBR_KERNEL_BIT 0 /* do not capture at ring0 */ | |
37 | #define LBR_USER_BIT 1 /* do not capture at ring > 0 */ | |
38 | #define LBR_JCC_BIT 2 /* do not capture conditional branches */ | |
39 | #define LBR_REL_CALL_BIT 3 /* do not capture relative calls */ | |
40 | #define LBR_IND_CALL_BIT 4 /* do not capture indirect calls */ | |
41 | #define LBR_RETURN_BIT 5 /* do not capture near returns */ | |
42 | #define LBR_IND_JMP_BIT 6 /* do not capture indirect jumps */ | |
43 | #define LBR_REL_JMP_BIT 7 /* do not capture relative jumps */ | |
44 | #define LBR_FAR_BIT 8 /* do not capture far branches */ | |
e9d7f7cd | 45 | #define LBR_CALL_STACK_BIT 9 /* enable call stack */ |
c5cc2cd9 | 46 | |
b16a5b52 AK |
47 | /* |
48 | * Following bit only exists in Linux; we mask it out before writing it to | |
49 | * the actual MSR. But it helps the constraint perf code to understand | |
50 | * that this is a separate configuration. | |
51 | */ | |
52 | #define LBR_NO_INFO_BIT 63 /* don't read LBR_INFO. */ | |
53 | ||
c5cc2cd9 SE |
54 | #define LBR_KERNEL (1 << LBR_KERNEL_BIT) |
55 | #define LBR_USER (1 << LBR_USER_BIT) | |
56 | #define LBR_JCC (1 << LBR_JCC_BIT) | |
57 | #define LBR_REL_CALL (1 << LBR_REL_CALL_BIT) | |
58 | #define LBR_IND_CALL (1 << LBR_IND_CALL_BIT) | |
59 | #define LBR_RETURN (1 << LBR_RETURN_BIT) | |
60 | #define LBR_REL_JMP (1 << LBR_REL_JMP_BIT) | |
61 | #define LBR_IND_JMP (1 << LBR_IND_JMP_BIT) | |
62 | #define LBR_FAR (1 << LBR_FAR_BIT) | |
e9d7f7cd | 63 | #define LBR_CALL_STACK (1 << LBR_CALL_STACK_BIT) |
b16a5b52 | 64 | #define LBR_NO_INFO (1ULL << LBR_NO_INFO_BIT) |
c5cc2cd9 SE |
65 | |
66 | #define LBR_PLM (LBR_KERNEL | LBR_USER) | |
67 | ||
cf3beb7c | 68 | #define LBR_SEL_MASK 0x3ff /* valid bits in LBR_SELECT */ |
c5cc2cd9 SE |
69 | #define LBR_NOT_SUPP -1 /* LBR filter not supported */ |
70 | #define LBR_IGN 0 /* ignored */ | |
71 | ||
72 | #define LBR_ANY \ | |
73 | (LBR_JCC |\ | |
74 | LBR_REL_CALL |\ | |
75 | LBR_IND_CALL |\ | |
76 | LBR_RETURN |\ | |
77 | LBR_REL_JMP |\ | |
78 | LBR_IND_JMP |\ | |
79 | LBR_FAR) | |
80 | ||
3812bba8 DCC |
81 | #define LBR_FROM_FLAG_MISPRED BIT_ULL(63) |
82 | #define LBR_FROM_FLAG_IN_TX BIT_ULL(62) | |
83 | #define LBR_FROM_FLAG_ABORT BIT_ULL(61) | |
c5cc2cd9 | 84 | |
19fc9ddd DCC |
85 | #define LBR_FROM_SIGNEXT_2MSB (BIT_ULL(60) | BIT_ULL(59)) |
86 | ||
3e702ff6 SE |
87 | /* |
88 | * x86control flow change classification | |
89 | * x86control flow changes include branches, interrupts, traps, faults | |
90 | */ | |
91 | enum { | |
e9d7f7cd YZ |
92 | X86_BR_NONE = 0, /* unknown */ |
93 | ||
94 | X86_BR_USER = 1 << 0, /* branch target is user */ | |
95 | X86_BR_KERNEL = 1 << 1, /* branch target is kernel */ | |
96 | ||
97 | X86_BR_CALL = 1 << 2, /* call */ | |
98 | X86_BR_RET = 1 << 3, /* return */ | |
99 | X86_BR_SYSCALL = 1 << 4, /* syscall */ | |
100 | X86_BR_SYSRET = 1 << 5, /* syscall return */ | |
101 | X86_BR_INT = 1 << 6, /* sw interrupt */ | |
102 | X86_BR_IRET = 1 << 7, /* return from interrupt */ | |
103 | X86_BR_JCC = 1 << 8, /* conditional */ | |
104 | X86_BR_JMP = 1 << 9, /* jump */ | |
105 | X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */ | |
106 | X86_BR_IND_CALL = 1 << 11,/* indirect calls */ | |
107 | X86_BR_ABORT = 1 << 12,/* transaction abort */ | |
108 | X86_BR_IN_TX = 1 << 13,/* in transaction */ | |
109 | X86_BR_NO_TX = 1 << 14,/* not in transaction */ | |
aa54ae9b YZ |
110 | X86_BR_ZERO_CALL = 1 << 15,/* zero length call */ |
111 | X86_BR_CALL_STACK = 1 << 16,/* call stack */ | |
7b74cfb2 | 112 | X86_BR_IND_JMP = 1 << 17,/* indirect jump */ |
d5c7f9dc JY |
113 | |
114 | X86_BR_TYPE_SAVE = 1 << 18,/* indicate to save branch type */ | |
115 | ||
3e702ff6 SE |
116 | }; |
117 | ||
118 | #define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL) | |
135c5612 | 119 | #define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX) |
3e702ff6 SE |
120 | |
121 | #define X86_BR_ANY \ | |
122 | (X86_BR_CALL |\ | |
123 | X86_BR_RET |\ | |
124 | X86_BR_SYSCALL |\ | |
125 | X86_BR_SYSRET |\ | |
126 | X86_BR_INT |\ | |
127 | X86_BR_IRET |\ | |
128 | X86_BR_JCC |\ | |
129 | X86_BR_JMP |\ | |
130 | X86_BR_IRQ |\ | |
135c5612 | 131 | X86_BR_ABORT |\ |
aa54ae9b | 132 | X86_BR_IND_CALL |\ |
7b74cfb2 | 133 | X86_BR_IND_JMP |\ |
aa54ae9b | 134 | X86_BR_ZERO_CALL) |
3e702ff6 SE |
135 | |
136 | #define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY) | |
137 | ||
138 | #define X86_BR_ANY_CALL \ | |
139 | (X86_BR_CALL |\ | |
140 | X86_BR_IND_CALL |\ | |
aa54ae9b | 141 | X86_BR_ZERO_CALL |\ |
3e702ff6 SE |
142 | X86_BR_SYSCALL |\ |
143 | X86_BR_IRQ |\ | |
144 | X86_BR_INT) | |
145 | ||
146 | static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc); | |
147 | ||
caff2bef PZ |
148 | /* |
149 | * We only support LBR implementations that have FREEZE_LBRS_ON_PMI | |
150 | * otherwise it becomes near impossible to get a reliable stack. | |
151 | */ | |
152 | ||
1a78d937 | 153 | static void __intel_pmu_lbr_enable(bool pmi) |
caff2bef | 154 | { |
89cbc767 | 155 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
cd1f11de | 156 | u64 debugctl, lbr_select = 0, orig_debugctl; |
60ce0fbd | 157 | |
425507fa AK |
158 | /* |
159 | * No need to unfreeze manually, as v4 can do that as part | |
160 | * of the GLOBAL_STATUS ack. | |
161 | */ | |
162 | if (pmi && x86_pmu.version >= 4) | |
163 | return; | |
164 | ||
1a78d937 AK |
165 | /* |
166 | * No need to reprogram LBR_SELECT in a PMI, as it | |
167 | * did not change. | |
168 | */ | |
96f3eda6 | 169 | if (cpuc->lbr_sel) |
b16a5b52 | 170 | lbr_select = cpuc->lbr_sel->config & x86_pmu.lbr_sel_mask; |
6fc2e830 | 171 | if (!pmi && cpuc->lbr_sel) |
2c70d008 | 172 | wrmsrl(MSR_LBR_SELECT, lbr_select); |
caff2bef PZ |
173 | |
174 | rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); | |
cd1f11de | 175 | orig_debugctl = debugctl; |
2c70d008 YZ |
176 | debugctl |= DEBUGCTLMSR_LBR; |
177 | /* | |
178 | * LBR callstack does not work well with FREEZE_LBRS_ON_PMI. | |
179 | * If FREEZE_LBRS_ON_PMI is set, PMI near call/return instructions | |
180 | * may cause superfluous increase/decrease of LBR_TOS. | |
181 | */ | |
182 | if (!(lbr_select & LBR_CALL_STACK)) | |
183 | debugctl |= DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; | |
cd1f11de AK |
184 | if (orig_debugctl != debugctl) |
185 | wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); | |
caff2bef PZ |
186 | } |
187 | ||
188 | static void __intel_pmu_lbr_disable(void) | |
189 | { | |
190 | u64 debugctl; | |
191 | ||
192 | rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); | |
7c5ecaf7 | 193 | debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); |
caff2bef PZ |
194 | wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); |
195 | } | |
196 | ||
197 | static void intel_pmu_lbr_reset_32(void) | |
198 | { | |
199 | int i; | |
200 | ||
201 | for (i = 0; i < x86_pmu.lbr_nr; i++) | |
202 | wrmsrl(x86_pmu.lbr_from + i, 0); | |
203 | } | |
204 | ||
205 | static void intel_pmu_lbr_reset_64(void) | |
206 | { | |
207 | int i; | |
208 | ||
209 | for (i = 0; i < x86_pmu.lbr_nr; i++) { | |
210 | wrmsrl(x86_pmu.lbr_from + i, 0); | |
211 | wrmsrl(x86_pmu.lbr_to + i, 0); | |
50eab8f6 AK |
212 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) |
213 | wrmsrl(MSR_LBR_INFO_0 + i, 0); | |
caff2bef PZ |
214 | } |
215 | } | |
216 | ||
de0428a7 | 217 | void intel_pmu_lbr_reset(void) |
caff2bef | 218 | { |
8b077e4a KL |
219 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
220 | ||
74846d35 PZ |
221 | if (!x86_pmu.lbr_nr) |
222 | return; | |
223 | ||
8db909a7 | 224 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) |
caff2bef PZ |
225 | intel_pmu_lbr_reset_32(); |
226 | else | |
227 | intel_pmu_lbr_reset_64(); | |
8b077e4a KL |
228 | |
229 | cpuc->last_task_ctx = NULL; | |
230 | cpuc->last_log_id = 0; | |
caff2bef PZ |
231 | } |
232 | ||
76cb2c61 YZ |
233 | /* |
234 | * TOS = most recently recorded branch | |
235 | */ | |
236 | static inline u64 intel_pmu_lbr_tos(void) | |
237 | { | |
238 | u64 tos; | |
239 | ||
240 | rdmsrl(x86_pmu.lbr_tos, tos); | |
241 | return tos; | |
242 | } | |
243 | ||
244 | enum { | |
245 | LBR_NONE, | |
246 | LBR_VALID, | |
247 | }; | |
248 | ||
19fc9ddd DCC |
249 | /* |
250 | * For formats with LBR_TSX flags (e.g. LBR_FORMAT_EIP_FLAGS2), bits 61:62 in | |
251 | * MSR_LAST_BRANCH_FROM_x are the TSX flags when TSX is supported, but when | |
252 | * TSX is not supported they have no consistent behavior: | |
253 | * | |
254 | * - For wrmsr(), bits 61:62 are considered part of the sign extension. | |
255 | * - For HW updates (branch captures) bits 61:62 are always OFF and are not | |
256 | * part of the sign extension. | |
257 | * | |
258 | * Therefore, if: | |
259 | * | |
260 | * 1) LBR has TSX format | |
261 | * 2) CPU has no TSX support enabled | |
262 | * | |
263 | * ... then any value passed to wrmsr() must be sign extended to 63 bits and any | |
264 | * value from rdmsr() must be converted to have a 61 bits sign extension, | |
265 | * ignoring the TSX flags. | |
266 | */ | |
267 | static inline bool lbr_from_signext_quirk_needed(void) | |
268 | { | |
269 | int lbr_format = x86_pmu.intel_cap.lbr_format; | |
270 | bool tsx_support = boot_cpu_has(X86_FEATURE_HLE) || | |
271 | boot_cpu_has(X86_FEATURE_RTM); | |
272 | ||
273 | return !tsx_support && (lbr_desc[lbr_format] & LBR_TSX); | |
274 | } | |
275 | ||
276 | DEFINE_STATIC_KEY_FALSE(lbr_from_quirk_key); | |
277 | ||
278 | /* If quirk is enabled, ensure sign extension is 63 bits: */ | |
279 | inline u64 lbr_from_signext_quirk_wr(u64 val) | |
280 | { | |
281 | if (static_branch_unlikely(&lbr_from_quirk_key)) { | |
282 | /* | |
283 | * Sign extend into bits 61:62 while preserving bit 63. | |
284 | * | |
285 | * Quirk is enabled when TSX is disabled. Therefore TSX bits | |
286 | * in val are always OFF and must be changed to be sign | |
287 | * extension bits. Since bits 59:60 are guaranteed to be | |
288 | * part of the sign extension bits, we can just copy them | |
289 | * to 61:62. | |
290 | */ | |
291 | val |= (LBR_FROM_SIGNEXT_2MSB & val) << 2; | |
292 | } | |
293 | return val; | |
294 | } | |
295 | ||
71adae99 DCC |
296 | /* |
297 | * If quirk is needed, ensure sign extension is 61 bits: | |
298 | */ | |
e91c8d97 | 299 | static u64 lbr_from_signext_quirk_rd(u64 val) |
71adae99 | 300 | { |
d4cf1949 | 301 | if (static_branch_unlikely(&lbr_from_quirk_key)) { |
71adae99 DCC |
302 | /* |
303 | * Quirk is on when TSX is not enabled. Therefore TSX | |
304 | * flags must be read as OFF. | |
305 | */ | |
306 | val &= ~(LBR_FROM_FLAG_IN_TX | LBR_FROM_FLAG_ABORT); | |
d4cf1949 PZ |
307 | } |
308 | return val; | |
309 | } | |
310 | ||
311 | static inline void wrlbr_from(unsigned int idx, u64 val) | |
312 | { | |
313 | val = lbr_from_signext_quirk_wr(val); | |
314 | wrmsrl(x86_pmu.lbr_from + idx, val); | |
315 | } | |
316 | ||
317 | static inline void wrlbr_to(unsigned int idx, u64 val) | |
318 | { | |
319 | wrmsrl(x86_pmu.lbr_to + idx, val); | |
320 | } | |
321 | ||
322 | static inline u64 rdlbr_from(unsigned int idx) | |
323 | { | |
324 | u64 val; | |
325 | ||
326 | rdmsrl(x86_pmu.lbr_from + idx, val); | |
327 | ||
328 | return lbr_from_signext_quirk_rd(val); | |
329 | } | |
330 | ||
331 | static inline u64 rdlbr_to(unsigned int idx) | |
332 | { | |
333 | u64 val; | |
334 | ||
aefbc4d0 | 335 | rdmsrl(x86_pmu.lbr_to + idx, val); |
d4cf1949 | 336 | |
71adae99 DCC |
337 | return val; |
338 | } | |
339 | ||
76cb2c61 YZ |
340 | static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx) |
341 | { | |
8b077e4a | 342 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
76cb2c61 YZ |
343 | int i; |
344 | unsigned lbr_idx, mask; | |
345 | u64 tos; | |
346 | ||
347 | if (task_ctx->lbr_callstack_users == 0 || | |
348 | task_ctx->lbr_stack_state == LBR_NONE) { | |
349 | intel_pmu_lbr_reset(); | |
350 | return; | |
351 | } | |
352 | ||
b28ae956 | 353 | tos = task_ctx->tos; |
8b077e4a KL |
354 | /* |
355 | * Does not restore the LBR registers, if | |
356 | * - No one else touched them, and | |
357 | * - Did not enter C6 | |
358 | */ | |
359 | if ((task_ctx == cpuc->last_task_ctx) && | |
360 | (task_ctx->log_id == cpuc->last_log_id) && | |
361 | rdlbr_from(tos)) { | |
362 | task_ctx->lbr_stack_state = LBR_NONE; | |
363 | return; | |
364 | } | |
365 | ||
366 | mask = x86_pmu.lbr_nr - 1; | |
0592e57b | 367 | for (i = 0; i < task_ctx->valid_lbrs; i++) { |
76cb2c61 | 368 | lbr_idx = (tos - i) & mask; |
d4cf1949 PZ |
369 | wrlbr_from(lbr_idx, task_ctx->lbr_from[i]); |
370 | wrlbr_to (lbr_idx, task_ctx->lbr_to[i]); | |
371 | ||
50eab8f6 | 372 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) |
e0573364 | 373 | wrmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); |
76cb2c61 | 374 | } |
0592e57b KL |
375 | |
376 | for (; i < x86_pmu.lbr_nr; i++) { | |
377 | lbr_idx = (tos - i) & mask; | |
378 | wrlbr_from(lbr_idx, 0); | |
379 | wrlbr_to(lbr_idx, 0); | |
380 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) | |
381 | wrmsrl(MSR_LBR_INFO_0 + lbr_idx, 0); | |
382 | } | |
383 | ||
b28ae956 | 384 | wrmsrl(x86_pmu.lbr_tos, tos); |
76cb2c61 YZ |
385 | task_ctx->lbr_stack_state = LBR_NONE; |
386 | } | |
387 | ||
388 | static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx) | |
389 | { | |
8b077e4a | 390 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
76cb2c61 | 391 | unsigned lbr_idx, mask; |
0592e57b | 392 | u64 tos, from; |
d4cf1949 | 393 | int i; |
76cb2c61 YZ |
394 | |
395 | if (task_ctx->lbr_callstack_users == 0) { | |
396 | task_ctx->lbr_stack_state = LBR_NONE; | |
397 | return; | |
398 | } | |
399 | ||
400 | mask = x86_pmu.lbr_nr - 1; | |
401 | tos = intel_pmu_lbr_tos(); | |
0592e57b | 402 | for (i = 0; i < x86_pmu.lbr_nr; i++) { |
76cb2c61 | 403 | lbr_idx = (tos - i) & mask; |
0592e57b KL |
404 | from = rdlbr_from(lbr_idx); |
405 | if (!from) | |
406 | break; | |
407 | task_ctx->lbr_from[i] = from; | |
d4cf1949 | 408 | task_ctx->lbr_to[i] = rdlbr_to(lbr_idx); |
50eab8f6 | 409 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO) |
e0573364 | 410 | rdmsrl(MSR_LBR_INFO_0 + lbr_idx, task_ctx->lbr_info[i]); |
76cb2c61 | 411 | } |
0592e57b | 412 | task_ctx->valid_lbrs = i; |
b28ae956 | 413 | task_ctx->tos = tos; |
76cb2c61 | 414 | task_ctx->lbr_stack_state = LBR_VALID; |
8b077e4a KL |
415 | |
416 | cpuc->last_task_ctx = task_ctx; | |
417 | cpuc->last_log_id = ++task_ctx->log_id; | |
76cb2c61 YZ |
418 | } |
419 | ||
2a0ad3b3 YZ |
420 | void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in) |
421 | { | |
df6c3db8 | 422 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
76cb2c61 | 423 | struct x86_perf_task_context *task_ctx; |
2a0ad3b3 | 424 | |
df6c3db8 JO |
425 | if (!cpuc->lbr_users) |
426 | return; | |
427 | ||
76cb2c61 YZ |
428 | /* |
429 | * If LBR callstack feature is enabled and the stack was saved when | |
430 | * the task was scheduled out, restore the stack. Otherwise flush | |
431 | * the LBR stack. | |
432 | */ | |
433 | task_ctx = ctx ? ctx->task_ctx_data : NULL; | |
434 | if (task_ctx) { | |
3e2c1a67 | 435 | if (sched_in) |
76cb2c61 | 436 | __intel_pmu_lbr_restore(task_ctx); |
3e2c1a67 | 437 | else |
76cb2c61 | 438 | __intel_pmu_lbr_save(task_ctx); |
76cb2c61 YZ |
439 | return; |
440 | } | |
441 | ||
2a0ad3b3 | 442 | /* |
3e2c1a67 PZ |
443 | * Since a context switch can flip the address space and LBR entries |
444 | * are not tagged with an identifier, we need to wipe the LBR, even for | |
445 | * per-cpu events. You simply cannot resolve the branches from the old | |
446 | * address space. | |
447 | */ | |
448 | if (sched_in) | |
2a0ad3b3 | 449 | intel_pmu_lbr_reset(); |
2a0ad3b3 YZ |
450 | } |
451 | ||
63f0c1d8 YZ |
452 | static inline bool branch_user_callstack(unsigned br_sel) |
453 | { | |
454 | return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK); | |
455 | } | |
456 | ||
68f7082f | 457 | void intel_pmu_lbr_add(struct perf_event *event) |
caff2bef | 458 | { |
89cbc767 | 459 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
63f0c1d8 | 460 | struct x86_perf_task_context *task_ctx; |
caff2bef PZ |
461 | |
462 | if (!x86_pmu.lbr_nr) | |
463 | return; | |
464 | ||
3e702ff6 | 465 | cpuc->br_sel = event->hw.branch_reg.reg; |
caff2bef | 466 | |
a5dcff62 | 467 | if (branch_user_callstack(cpuc->br_sel) && event->ctx->task_ctx_data) { |
63f0c1d8 YZ |
468 | task_ctx = event->ctx->task_ctx_data; |
469 | task_ctx->lbr_callstack_users++; | |
470 | } | |
471 | ||
3e2c1a67 PZ |
472 | /* |
473 | * Request pmu::sched_task() callback, which will fire inside the | |
474 | * regular perf event scheduling, so that call will: | |
475 | * | |
476 | * - restore or wipe; when LBR-callstack, | |
477 | * - wipe; otherwise, | |
478 | * | |
479 | * when this is from __perf_event_task_sched_in(). | |
480 | * | |
481 | * However, if this is from perf_install_in_context(), no such callback | |
482 | * will follow and we'll need to reset the LBR here if this is the | |
483 | * first LBR event. | |
484 | * | |
485 | * The problem is, we cannot tell these cases apart... but we can | |
486 | * exclude the biggest chunk of cases by looking at | |
487 | * event->total_time_running. An event that has accrued runtime cannot | |
488 | * be 'new'. Conversely, a new event can get installed through the | |
489 | * context switch path for the first time. | |
490 | */ | |
2a0ad3b3 | 491 | perf_sched_cb_inc(event->ctx->pmu); |
3e2c1a67 PZ |
492 | if (!cpuc->lbr_users++ && !event->total_time_running) |
493 | intel_pmu_lbr_reset(); | |
caff2bef PZ |
494 | } |
495 | ||
68f7082f | 496 | void intel_pmu_lbr_del(struct perf_event *event) |
caff2bef | 497 | { |
89cbc767 | 498 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
63f0c1d8 | 499 | struct x86_perf_task_context *task_ctx; |
caff2bef PZ |
500 | |
501 | if (!x86_pmu.lbr_nr) | |
502 | return; | |
503 | ||
5c38181c DC |
504 | if (branch_user_callstack(cpuc->br_sel) && |
505 | event->ctx->task_ctx_data) { | |
63f0c1d8 YZ |
506 | task_ctx = event->ctx->task_ctx_data; |
507 | task_ctx->lbr_callstack_users--; | |
508 | } | |
509 | ||
caff2bef | 510 | cpuc->lbr_users--; |
b83a46e7 | 511 | WARN_ON_ONCE(cpuc->lbr_users < 0); |
2a0ad3b3 | 512 | perf_sched_cb_dec(event->ctx->pmu); |
caff2bef PZ |
513 | } |
514 | ||
1a78d937 | 515 | void intel_pmu_lbr_enable_all(bool pmi) |
caff2bef | 516 | { |
89cbc767 | 517 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
caff2bef PZ |
518 | |
519 | if (cpuc->lbr_users) | |
1a78d937 | 520 | __intel_pmu_lbr_enable(pmi); |
caff2bef PZ |
521 | } |
522 | ||
de0428a7 | 523 | void intel_pmu_lbr_disable_all(void) |
caff2bef | 524 | { |
89cbc767 | 525 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
caff2bef PZ |
526 | |
527 | if (cpuc->lbr_users) | |
528 | __intel_pmu_lbr_disable(); | |
529 | } | |
530 | ||
caff2bef PZ |
531 | static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) |
532 | { | |
533 | unsigned long mask = x86_pmu.lbr_nr - 1; | |
534 | u64 tos = intel_pmu_lbr_tos(); | |
535 | int i; | |
536 | ||
63fb3f9b | 537 | for (i = 0; i < x86_pmu.lbr_nr; i++) { |
caff2bef PZ |
538 | unsigned long lbr_idx = (tos - i) & mask; |
539 | union { | |
540 | struct { | |
541 | u32 from; | |
542 | u32 to; | |
543 | }; | |
544 | u64 lbr; | |
545 | } msr_lastbranch; | |
546 | ||
547 | rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); | |
548 | ||
bce38cd5 SE |
549 | cpuc->lbr_entries[i].from = msr_lastbranch.from; |
550 | cpuc->lbr_entries[i].to = msr_lastbranch.to; | |
551 | cpuc->lbr_entries[i].mispred = 0; | |
552 | cpuc->lbr_entries[i].predicted = 0; | |
f2200ac3 PZ |
553 | cpuc->lbr_entries[i].in_tx = 0; |
554 | cpuc->lbr_entries[i].abort = 0; | |
555 | cpuc->lbr_entries[i].cycles = 0; | |
d5c7f9dc | 556 | cpuc->lbr_entries[i].type = 0; |
bce38cd5 | 557 | cpuc->lbr_entries[i].reserved = 0; |
caff2bef PZ |
558 | } |
559 | cpuc->lbr_stack.nr = i; | |
560 | } | |
561 | ||
caff2bef PZ |
562 | /* |
563 | * Due to lack of segmentation in Linux the effective address (offset) | |
564 | * is the same as the linear address, allowing us to merge the LIP and EIP | |
565 | * LBR formats. | |
566 | */ | |
567 | static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) | |
568 | { | |
0592e57b | 569 | bool need_info = false, call_stack = false; |
caff2bef | 570 | unsigned long mask = x86_pmu.lbr_nr - 1; |
8db909a7 | 571 | int lbr_format = x86_pmu.intel_cap.lbr_format; |
caff2bef PZ |
572 | u64 tos = intel_pmu_lbr_tos(); |
573 | int i; | |
b7af41a1 | 574 | int out = 0; |
90405aa0 | 575 | int num = x86_pmu.lbr_nr; |
caff2bef | 576 | |
6fc2e830 SE |
577 | if (cpuc->lbr_sel) { |
578 | need_info = !(cpuc->lbr_sel->config & LBR_NO_INFO); | |
579 | if (cpuc->lbr_sel->config & LBR_CALL_STACK) | |
0592e57b | 580 | call_stack = true; |
6fc2e830 | 581 | } |
90405aa0 AK |
582 | |
583 | for (i = 0; i < num; i++) { | |
caff2bef | 584 | unsigned long lbr_idx = (tos - i) & mask; |
135c5612 AK |
585 | u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0; |
586 | int skip = 0; | |
50eab8f6 | 587 | u16 cycles = 0; |
135c5612 | 588 | int lbr_flags = lbr_desc[lbr_format]; |
caff2bef | 589 | |
d4cf1949 PZ |
590 | from = rdlbr_from(lbr_idx); |
591 | to = rdlbr_to(lbr_idx); | |
caff2bef | 592 | |
0592e57b KL |
593 | /* |
594 | * Read LBR call stack entries | |
595 | * until invalid entry (0s) is detected. | |
596 | */ | |
597 | if (call_stack && !from) | |
598 | break; | |
599 | ||
b16a5b52 | 600 | if (lbr_format == LBR_FORMAT_INFO && need_info) { |
50eab8f6 AK |
601 | u64 info; |
602 | ||
603 | rdmsrl(MSR_LBR_INFO_0 + lbr_idx, info); | |
604 | mis = !!(info & LBR_INFO_MISPRED); | |
605 | pred = !mis; | |
606 | in_tx = !!(info & LBR_INFO_IN_TX); | |
607 | abort = !!(info & LBR_INFO_ABORT); | |
608 | cycles = (info & LBR_INFO_CYCLES); | |
609 | } | |
8b92c3a7 KL |
610 | |
611 | if (lbr_format == LBR_FORMAT_TIME) { | |
612 | mis = !!(from & LBR_FROM_FLAG_MISPRED); | |
613 | pred = !mis; | |
614 | skip = 1; | |
615 | cycles = ((to >> 48) & LBR_INFO_CYCLES); | |
616 | ||
617 | to = (u64)((((s64)to) << 16) >> 16); | |
618 | } | |
619 | ||
135c5612 | 620 | if (lbr_flags & LBR_EIP_FLAGS) { |
bce38cd5 SE |
621 | mis = !!(from & LBR_FROM_FLAG_MISPRED); |
622 | pred = !mis; | |
135c5612 AK |
623 | skip = 1; |
624 | } | |
625 | if (lbr_flags & LBR_TSX) { | |
626 | in_tx = !!(from & LBR_FROM_FLAG_IN_TX); | |
627 | abort = !!(from & LBR_FROM_FLAG_ABORT); | |
628 | skip = 3; | |
caff2bef | 629 | } |
135c5612 | 630 | from = (u64)((((s64)from) << skip) >> skip); |
caff2bef | 631 | |
b7af41a1 AK |
632 | /* |
633 | * Some CPUs report duplicated abort records, | |
634 | * with the second entry not having an abort bit set. | |
635 | * Skip them here. This loop runs backwards, | |
636 | * so we need to undo the previous record. | |
637 | * If the abort just happened outside the window | |
638 | * the extra entry cannot be removed. | |
639 | */ | |
640 | if (abort && x86_pmu.lbr_double_abort && out > 0) | |
641 | out--; | |
642 | ||
643 | cpuc->lbr_entries[out].from = from; | |
644 | cpuc->lbr_entries[out].to = to; | |
645 | cpuc->lbr_entries[out].mispred = mis; | |
646 | cpuc->lbr_entries[out].predicted = pred; | |
647 | cpuc->lbr_entries[out].in_tx = in_tx; | |
648 | cpuc->lbr_entries[out].abort = abort; | |
50eab8f6 | 649 | cpuc->lbr_entries[out].cycles = cycles; |
d5c7f9dc | 650 | cpuc->lbr_entries[out].type = 0; |
b7af41a1 AK |
651 | cpuc->lbr_entries[out].reserved = 0; |
652 | out++; | |
caff2bef | 653 | } |
b7af41a1 | 654 | cpuc->lbr_stack.nr = out; |
caff2bef PZ |
655 | } |
656 | ||
de0428a7 | 657 | void intel_pmu_lbr_read(void) |
caff2bef | 658 | { |
89cbc767 | 659 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
caff2bef PZ |
660 | |
661 | if (!cpuc->lbr_users) | |
662 | return; | |
663 | ||
8db909a7 | 664 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) |
caff2bef PZ |
665 | intel_pmu_lbr_read_32(cpuc); |
666 | else | |
667 | intel_pmu_lbr_read_64(cpuc); | |
3e702ff6 SE |
668 | |
669 | intel_pmu_lbr_filter(cpuc); | |
670 | } | |
671 | ||
672 | /* | |
673 | * SW filter is used: | |
674 | * - in case there is no HW filter | |
675 | * - in case the HW filter has errata or limitations | |
676 | */ | |
e9d7f7cd | 677 | static int intel_pmu_setup_sw_lbr_filter(struct perf_event *event) |
3e702ff6 SE |
678 | { |
679 | u64 br_type = event->attr.branch_sample_type; | |
680 | int mask = 0; | |
681 | ||
682 | if (br_type & PERF_SAMPLE_BRANCH_USER) | |
683 | mask |= X86_BR_USER; | |
684 | ||
2b923c8f | 685 | if (br_type & PERF_SAMPLE_BRANCH_KERNEL) |
3e702ff6 SE |
686 | mask |= X86_BR_KERNEL; |
687 | ||
688 | /* we ignore BRANCH_HV here */ | |
689 | ||
690 | if (br_type & PERF_SAMPLE_BRANCH_ANY) | |
691 | mask |= X86_BR_ANY; | |
692 | ||
693 | if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL) | |
694 | mask |= X86_BR_ANY_CALL; | |
695 | ||
696 | if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN) | |
697 | mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET; | |
698 | ||
699 | if (br_type & PERF_SAMPLE_BRANCH_IND_CALL) | |
700 | mask |= X86_BR_IND_CALL; | |
135c5612 AK |
701 | |
702 | if (br_type & PERF_SAMPLE_BRANCH_ABORT_TX) | |
703 | mask |= X86_BR_ABORT; | |
704 | ||
705 | if (br_type & PERF_SAMPLE_BRANCH_IN_TX) | |
706 | mask |= X86_BR_IN_TX; | |
707 | ||
708 | if (br_type & PERF_SAMPLE_BRANCH_NO_TX) | |
709 | mask |= X86_BR_NO_TX; | |
710 | ||
37548914 AK |
711 | if (br_type & PERF_SAMPLE_BRANCH_COND) |
712 | mask |= X86_BR_JCC; | |
713 | ||
e9d7f7cd YZ |
714 | if (br_type & PERF_SAMPLE_BRANCH_CALL_STACK) { |
715 | if (!x86_pmu_has_lbr_callstack()) | |
716 | return -EOPNOTSUPP; | |
717 | if (mask & ~(X86_BR_USER | X86_BR_KERNEL)) | |
718 | return -EINVAL; | |
719 | mask |= X86_BR_CALL | X86_BR_IND_CALL | X86_BR_RET | | |
720 | X86_BR_CALL_STACK; | |
721 | } | |
722 | ||
7b74cfb2 SE |
723 | if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP) |
724 | mask |= X86_BR_IND_JMP; | |
725 | ||
d892819f SE |
726 | if (br_type & PERF_SAMPLE_BRANCH_CALL) |
727 | mask |= X86_BR_CALL | X86_BR_ZERO_CALL; | |
d5c7f9dc JY |
728 | |
729 | if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE) | |
730 | mask |= X86_BR_TYPE_SAVE; | |
731 | ||
3e702ff6 SE |
732 | /* |
733 | * stash actual user request into reg, it may | |
734 | * be used by fixup code for some CPU | |
735 | */ | |
736 | event->hw.branch_reg.reg = mask; | |
e9d7f7cd | 737 | return 0; |
caff2bef PZ |
738 | } |
739 | ||
60ce0fbd SE |
740 | /* |
741 | * setup the HW LBR filter | |
742 | * Used only when available, may not be enough to disambiguate | |
743 | * all branches, may need the help of the SW filter | |
744 | */ | |
745 | static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event) | |
746 | { | |
747 | struct hw_perf_event_extra *reg; | |
748 | u64 br_type = event->attr.branch_sample_type; | |
27ac905b YZ |
749 | u64 mask = 0, v; |
750 | int i; | |
60ce0fbd | 751 | |
2c44b193 | 752 | for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) { |
27ac905b | 753 | if (!(br_type & (1ULL << i))) |
60ce0fbd SE |
754 | continue; |
755 | ||
27ac905b | 756 | v = x86_pmu.lbr_sel_map[i]; |
60ce0fbd SE |
757 | if (v == LBR_NOT_SUPP) |
758 | return -EOPNOTSUPP; | |
60ce0fbd | 759 | |
3e702ff6 SE |
760 | if (v != LBR_IGN) |
761 | mask |= v; | |
60ce0fbd | 762 | } |
b16a5b52 | 763 | |
60ce0fbd SE |
764 | reg = &event->hw.branch_reg; |
765 | reg->idx = EXTRA_REG_LBR; | |
766 | ||
e9d7f7cd YZ |
767 | /* |
768 | * The first 9 bits (LBR_SEL_MASK) in LBR_SELECT operate | |
769 | * in suppress mode. So LBR_SELECT should be set to | |
770 | * (~mask & LBR_SEL_MASK) | (mask & ~LBR_SEL_MASK) | |
cf3beb7c KL |
771 | * But the 10th bit LBR_CALL_STACK does not operate |
772 | * in suppress mode. | |
e9d7f7cd | 773 | */ |
cf3beb7c | 774 | reg->config = mask ^ (x86_pmu.lbr_sel_mask & ~LBR_CALL_STACK); |
60ce0fbd | 775 | |
b16a5b52 AK |
776 | if ((br_type & PERF_SAMPLE_BRANCH_NO_CYCLES) && |
777 | (br_type & PERF_SAMPLE_BRANCH_NO_FLAGS) && | |
778 | (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_INFO)) | |
779 | reg->config |= LBR_NO_INFO; | |
780 | ||
60ce0fbd SE |
781 | return 0; |
782 | } | |
783 | ||
60ce0fbd SE |
784 | int intel_pmu_setup_lbr_filter(struct perf_event *event) |
785 | { | |
3e702ff6 | 786 | int ret = 0; |
60ce0fbd SE |
787 | |
788 | /* | |
789 | * no LBR on this PMU | |
790 | */ | |
791 | if (!x86_pmu.lbr_nr) | |
792 | return -EOPNOTSUPP; | |
793 | ||
794 | /* | |
3e702ff6 | 795 | * setup SW LBR filter |
60ce0fbd | 796 | */ |
e9d7f7cd YZ |
797 | ret = intel_pmu_setup_sw_lbr_filter(event); |
798 | if (ret) | |
799 | return ret; | |
3e702ff6 SE |
800 | |
801 | /* | |
802 | * setup HW LBR filter, if any | |
803 | */ | |
804 | if (x86_pmu.lbr_sel_map) | |
805 | ret = intel_pmu_setup_hw_lbr_filter(event); | |
806 | ||
807 | return ret; | |
808 | } | |
809 | ||
810 | /* | |
811 | * return the type of control flow change at address "from" | |
6a6256f9 | 812 | * instruction is not necessarily a branch (in case of interrupt). |
3e702ff6 SE |
813 | * |
814 | * The branch type returned also includes the priv level of the | |
815 | * target of the control flow change (X86_BR_USER, X86_BR_KERNEL). | |
816 | * | |
817 | * If a branch type is unknown OR the instruction cannot be | |
818 | * decoded (e.g., text page not present), then X86_BR_NONE is | |
819 | * returned. | |
820 | */ | |
135c5612 | 821 | static int branch_type(unsigned long from, unsigned long to, int abort) |
3e702ff6 SE |
822 | { |
823 | struct insn insn; | |
824 | void *addr; | |
6ba48ff4 | 825 | int bytes_read, bytes_left; |
3e702ff6 SE |
826 | int ret = X86_BR_NONE; |
827 | int ext, to_plm, from_plm; | |
828 | u8 buf[MAX_INSN_SIZE]; | |
829 | int is64 = 0; | |
830 | ||
831 | to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; | |
832 | from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; | |
833 | ||
834 | /* | |
835 | * maybe zero if lbr did not fill up after a reset by the time | |
836 | * we get a PMU interrupt | |
837 | */ | |
838 | if (from == 0 || to == 0) | |
839 | return X86_BR_NONE; | |
840 | ||
135c5612 AK |
841 | if (abort) |
842 | return X86_BR_ABORT | to_plm; | |
843 | ||
3e702ff6 SE |
844 | if (from_plm == X86_BR_USER) { |
845 | /* | |
846 | * can happen if measuring at the user level only | |
847 | * and we interrupt in a kernel thread, e.g., idle. | |
848 | */ | |
849 | if (!current->mm) | |
850 | return X86_BR_NONE; | |
851 | ||
852 | /* may fail if text not present */ | |
6ba48ff4 DH |
853 | bytes_left = copy_from_user_nmi(buf, (void __user *)from, |
854 | MAX_INSN_SIZE); | |
855 | bytes_read = MAX_INSN_SIZE - bytes_left; | |
856 | if (!bytes_read) | |
3e702ff6 SE |
857 | return X86_BR_NONE; |
858 | ||
859 | addr = buf; | |
6e15eb3b PZ |
860 | } else { |
861 | /* | |
862 | * The LBR logs any address in the IP, even if the IP just | |
863 | * faulted. This means userspace can control the from address. | |
864 | * Ensure we don't blindy read any address by validating it is | |
865 | * a known text address. | |
866 | */ | |
6ba48ff4 | 867 | if (kernel_text_address(from)) { |
6e15eb3b | 868 | addr = (void *)from; |
6ba48ff4 DH |
869 | /* |
870 | * Assume we can get the maximum possible size | |
871 | * when grabbing kernel data. This is not | |
872 | * _strictly_ true since we could possibly be | |
873 | * executing up next to a memory hole, but | |
874 | * it is very unlikely to be a problem. | |
875 | */ | |
876 | bytes_read = MAX_INSN_SIZE; | |
877 | } else { | |
6e15eb3b | 878 | return X86_BR_NONE; |
6ba48ff4 | 879 | } |
6e15eb3b | 880 | } |
3e702ff6 SE |
881 | |
882 | /* | |
883 | * decoder needs to know the ABI especially | |
884 | * on 64-bit systems running 32-bit apps | |
885 | */ | |
886 | #ifdef CONFIG_X86_64 | |
887 | is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32); | |
888 | #endif | |
6ba48ff4 | 889 | insn_init(&insn, addr, bytes_read, is64); |
3e702ff6 | 890 | insn_get_opcode(&insn); |
6ba48ff4 DH |
891 | if (!insn.opcode.got) |
892 | return X86_BR_ABORT; | |
3e702ff6 SE |
893 | |
894 | switch (insn.opcode.bytes[0]) { | |
895 | case 0xf: | |
896 | switch (insn.opcode.bytes[1]) { | |
897 | case 0x05: /* syscall */ | |
898 | case 0x34: /* sysenter */ | |
899 | ret = X86_BR_SYSCALL; | |
900 | break; | |
901 | case 0x07: /* sysret */ | |
902 | case 0x35: /* sysexit */ | |
903 | ret = X86_BR_SYSRET; | |
904 | break; | |
905 | case 0x80 ... 0x8f: /* conditional */ | |
906 | ret = X86_BR_JCC; | |
907 | break; | |
908 | default: | |
909 | ret = X86_BR_NONE; | |
910 | } | |
911 | break; | |
912 | case 0x70 ... 0x7f: /* conditional */ | |
913 | ret = X86_BR_JCC; | |
914 | break; | |
915 | case 0xc2: /* near ret */ | |
916 | case 0xc3: /* near ret */ | |
917 | case 0xca: /* far ret */ | |
918 | case 0xcb: /* far ret */ | |
919 | ret = X86_BR_RET; | |
920 | break; | |
921 | case 0xcf: /* iret */ | |
922 | ret = X86_BR_IRET; | |
923 | break; | |
924 | case 0xcc ... 0xce: /* int */ | |
925 | ret = X86_BR_INT; | |
926 | break; | |
927 | case 0xe8: /* call near rel */ | |
aa54ae9b YZ |
928 | insn_get_immediate(&insn); |
929 | if (insn.immediate1.value == 0) { | |
930 | /* zero length call */ | |
931 | ret = X86_BR_ZERO_CALL; | |
932 | break; | |
933 | } | |
2b0fc374 | 934 | /* fall through */ |
3e702ff6 SE |
935 | case 0x9a: /* call far absolute */ |
936 | ret = X86_BR_CALL; | |
937 | break; | |
938 | case 0xe0 ... 0xe3: /* loop jmp */ | |
939 | ret = X86_BR_JCC; | |
940 | break; | |
941 | case 0xe9 ... 0xeb: /* jmp */ | |
942 | ret = X86_BR_JMP; | |
943 | break; | |
944 | case 0xff: /* call near absolute, call far absolute ind */ | |
945 | insn_get_modrm(&insn); | |
946 | ext = (insn.modrm.bytes[0] >> 3) & 0x7; | |
947 | switch (ext) { | |
948 | case 2: /* near ind call */ | |
949 | case 3: /* far ind call */ | |
950 | ret = X86_BR_IND_CALL; | |
951 | break; | |
952 | case 4: | |
953 | case 5: | |
7b74cfb2 | 954 | ret = X86_BR_IND_JMP; |
3e702ff6 SE |
955 | break; |
956 | } | |
957 | break; | |
958 | default: | |
959 | ret = X86_BR_NONE; | |
60ce0fbd SE |
960 | } |
961 | /* | |
3e702ff6 SE |
962 | * interrupts, traps, faults (and thus ring transition) may |
963 | * occur on any instructions. Thus, to classify them correctly, | |
964 | * we need to first look at the from and to priv levels. If they | |
965 | * are different and to is in the kernel, then it indicates | |
966 | * a ring transition. If the from instruction is not a ring | |
967 | * transition instr (syscall, systenter, int), then it means | |
968 | * it was a irq, trap or fault. | |
969 | * | |
970 | * we have no way of detecting kernel to kernel faults. | |
971 | */ | |
972 | if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL | |
973 | && ret != X86_BR_SYSCALL && ret != X86_BR_INT) | |
974 | ret = X86_BR_IRQ; | |
975 | ||
976 | /* | |
977 | * branch priv level determined by target as | |
978 | * is done by HW when LBR_SELECT is implemented | |
60ce0fbd | 979 | */ |
3e702ff6 SE |
980 | if (ret != X86_BR_NONE) |
981 | ret |= to_plm; | |
60ce0fbd | 982 | |
3e702ff6 SE |
983 | return ret; |
984 | } | |
985 | ||
d5c7f9dc JY |
986 | #define X86_BR_TYPE_MAP_MAX 16 |
987 | ||
988 | static int branch_map[X86_BR_TYPE_MAP_MAX] = { | |
989 | PERF_BR_CALL, /* X86_BR_CALL */ | |
990 | PERF_BR_RET, /* X86_BR_RET */ | |
991 | PERF_BR_SYSCALL, /* X86_BR_SYSCALL */ | |
992 | PERF_BR_SYSRET, /* X86_BR_SYSRET */ | |
993 | PERF_BR_UNKNOWN, /* X86_BR_INT */ | |
994 | PERF_BR_UNKNOWN, /* X86_BR_IRET */ | |
995 | PERF_BR_COND, /* X86_BR_JCC */ | |
996 | PERF_BR_UNCOND, /* X86_BR_JMP */ | |
997 | PERF_BR_UNKNOWN, /* X86_BR_IRQ */ | |
998 | PERF_BR_IND_CALL, /* X86_BR_IND_CALL */ | |
999 | PERF_BR_UNKNOWN, /* X86_BR_ABORT */ | |
1000 | PERF_BR_UNKNOWN, /* X86_BR_IN_TX */ | |
1001 | PERF_BR_UNKNOWN, /* X86_BR_NO_TX */ | |
1002 | PERF_BR_CALL, /* X86_BR_ZERO_CALL */ | |
1003 | PERF_BR_UNKNOWN, /* X86_BR_CALL_STACK */ | |
1004 | PERF_BR_IND, /* X86_BR_IND_JMP */ | |
1005 | }; | |
1006 | ||
1007 | static int | |
1008 | common_branch_type(int type) | |
1009 | { | |
1010 | int i; | |
1011 | ||
1012 | type >>= 2; /* skip X86_BR_USER and X86_BR_KERNEL */ | |
1013 | ||
1014 | if (type) { | |
1015 | i = __ffs(type); | |
1016 | if (i < X86_BR_TYPE_MAP_MAX) | |
1017 | return branch_map[i]; | |
1018 | } | |
1019 | ||
1020 | return PERF_BR_UNKNOWN; | |
1021 | } | |
1022 | ||
3e702ff6 SE |
1023 | /* |
1024 | * implement actual branch filter based on user demand. | |
1025 | * Hardware may not exactly satisfy that request, thus | |
1026 | * we need to inspect opcodes. Mismatched branches are | |
1027 | * discarded. Therefore, the number of branches returned | |
1028 | * in PERF_SAMPLE_BRANCH_STACK sample may vary. | |
1029 | */ | |
1030 | static void | |
1031 | intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) | |
1032 | { | |
1033 | u64 from, to; | |
1034 | int br_sel = cpuc->br_sel; | |
1035 | int i, j, type; | |
1036 | bool compress = false; | |
1037 | ||
1038 | /* if sampling all branches, then nothing to filter */ | |
d5c7f9dc JY |
1039 | if (((br_sel & X86_BR_ALL) == X86_BR_ALL) && |
1040 | ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE)) | |
3e702ff6 SE |
1041 | return; |
1042 | ||
1043 | for (i = 0; i < cpuc->lbr_stack.nr; i++) { | |
1044 | ||
1045 | from = cpuc->lbr_entries[i].from; | |
1046 | to = cpuc->lbr_entries[i].to; | |
1047 | ||
135c5612 AK |
1048 | type = branch_type(from, to, cpuc->lbr_entries[i].abort); |
1049 | if (type != X86_BR_NONE && (br_sel & X86_BR_ANYTX)) { | |
1050 | if (cpuc->lbr_entries[i].in_tx) | |
1051 | type |= X86_BR_IN_TX; | |
1052 | else | |
1053 | type |= X86_BR_NO_TX; | |
1054 | } | |
3e702ff6 SE |
1055 | |
1056 | /* if type does not correspond, then discard */ | |
1057 | if (type == X86_BR_NONE || (br_sel & type) != type) { | |
1058 | cpuc->lbr_entries[i].from = 0; | |
1059 | compress = true; | |
1060 | } | |
d5c7f9dc JY |
1061 | |
1062 | if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE) | |
1063 | cpuc->lbr_entries[i].type = common_branch_type(type); | |
3e702ff6 SE |
1064 | } |
1065 | ||
1066 | if (!compress) | |
1067 | return; | |
1068 | ||
1069 | /* remove all entries with from=0 */ | |
1070 | for (i = 0; i < cpuc->lbr_stack.nr; ) { | |
1071 | if (!cpuc->lbr_entries[i].from) { | |
1072 | j = i; | |
1073 | while (++j < cpuc->lbr_stack.nr) | |
1074 | cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j]; | |
1075 | cpuc->lbr_stack.nr--; | |
1076 | if (!cpuc->lbr_entries[i].from) | |
1077 | continue; | |
1078 | } | |
1079 | i++; | |
1080 | } | |
60ce0fbd SE |
1081 | } |
1082 | ||
c22497f5 KL |
1083 | void intel_pmu_store_pebs_lbrs(struct pebs_lbr *lbr) |
1084 | { | |
1085 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); | |
1086 | int i; | |
1087 | ||
1088 | cpuc->lbr_stack.nr = x86_pmu.lbr_nr; | |
1089 | for (i = 0; i < x86_pmu.lbr_nr; i++) { | |
1090 | u64 info = lbr->lbr[i].info; | |
1091 | struct perf_branch_entry *e = &cpuc->lbr_entries[i]; | |
1092 | ||
1093 | e->from = lbr->lbr[i].from; | |
1094 | e->to = lbr->lbr[i].to; | |
1095 | e->mispred = !!(info & LBR_INFO_MISPRED); | |
1096 | e->predicted = !(info & LBR_INFO_MISPRED); | |
1097 | e->in_tx = !!(info & LBR_INFO_IN_TX); | |
1098 | e->abort = !!(info & LBR_INFO_ABORT); | |
1099 | e->cycles = info & LBR_INFO_CYCLES; | |
1100 | e->reserved = 0; | |
1101 | } | |
1102 | intel_pmu_lbr_filter(cpuc); | |
1103 | } | |
1104 | ||
c5cc2cd9 SE |
1105 | /* |
1106 | * Map interface branch filters onto LBR filters | |
1107 | */ | |
2c44b193 | 1108 | static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { |
27ac905b YZ |
1109 | [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, |
1110 | [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, | |
1111 | [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, | |
1112 | [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, | |
1113 | [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_REL_JMP | |
1114 | | LBR_IND_JMP | LBR_FAR, | |
c5cc2cd9 SE |
1115 | /* |
1116 | * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches | |
1117 | */ | |
27ac905b | 1118 | [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = |
c5cc2cd9 SE |
1119 | LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR, |
1120 | /* | |
1121 | * NHM/WSM erratum: must include IND_JMP to capture IND_CALL | |
1122 | */ | |
27ac905b YZ |
1123 | [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL | LBR_IND_JMP, |
1124 | [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, | |
7b74cfb2 | 1125 | [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, |
c5cc2cd9 SE |
1126 | }; |
1127 | ||
2c44b193 | 1128 | static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { |
27ac905b YZ |
1129 | [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, |
1130 | [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, | |
1131 | [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, | |
1132 | [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, | |
1133 | [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, | |
1134 | [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL | |
1135 | | LBR_FAR, | |
1136 | [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, | |
1137 | [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, | |
7b74cfb2 | 1138 | [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, |
d892819f | 1139 | [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, |
c5cc2cd9 SE |
1140 | }; |
1141 | ||
2c44b193 | 1142 | static const int hsw_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { |
e9d7f7cd YZ |
1143 | [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, |
1144 | [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, | |
1145 | [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, | |
1146 | [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGN, | |
1147 | [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, | |
1148 | [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL | |
1149 | | LBR_FAR, | |
1150 | [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, | |
1151 | [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, | |
1152 | [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_REL_CALL | LBR_IND_CALL | |
1153 | | LBR_RETURN | LBR_CALL_STACK, | |
7b74cfb2 | 1154 | [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, |
d892819f | 1155 | [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, |
e9d7f7cd YZ |
1156 | }; |
1157 | ||
c5cc2cd9 | 1158 | /* core */ |
066ce64c | 1159 | void __init intel_pmu_lbr_init_core(void) |
caff2bef | 1160 | { |
caff2bef | 1161 | x86_pmu.lbr_nr = 4; |
225ce539 SE |
1162 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
1163 | x86_pmu.lbr_from = MSR_LBR_CORE_FROM; | |
1164 | x86_pmu.lbr_to = MSR_LBR_CORE_TO; | |
c5cc2cd9 | 1165 | |
3e702ff6 SE |
1166 | /* |
1167 | * SW branch filter usage: | |
1168 | * - compensate for lack of HW filter | |
1169 | */ | |
caff2bef PZ |
1170 | } |
1171 | ||
c5cc2cd9 | 1172 | /* nehalem/westmere */ |
066ce64c | 1173 | void __init intel_pmu_lbr_init_nhm(void) |
caff2bef | 1174 | { |
caff2bef | 1175 | x86_pmu.lbr_nr = 16; |
225ce539 SE |
1176 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
1177 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; | |
1178 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; | |
c5cc2cd9 SE |
1179 | |
1180 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; | |
1181 | x86_pmu.lbr_sel_map = nhm_lbr_sel_map; | |
1182 | ||
3e702ff6 SE |
1183 | /* |
1184 | * SW branch filter usage: | |
1185 | * - workaround LBR_SEL errata (see above) | |
1186 | * - support syscall, sysret capture. | |
1187 | * That requires LBR_FAR but that means far | |
1188 | * jmp need to be filtered out | |
1189 | */ | |
caff2bef PZ |
1190 | } |
1191 | ||
c5cc2cd9 | 1192 | /* sandy bridge */ |
066ce64c | 1193 | void __init intel_pmu_lbr_init_snb(void) |
c5cc2cd9 SE |
1194 | { |
1195 | x86_pmu.lbr_nr = 16; | |
1196 | x86_pmu.lbr_tos = MSR_LBR_TOS; | |
1197 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; | |
1198 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; | |
1199 | ||
1200 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; | |
1201 | x86_pmu.lbr_sel_map = snb_lbr_sel_map; | |
1202 | ||
3e702ff6 SE |
1203 | /* |
1204 | * SW branch filter usage: | |
1205 | * - support syscall, sysret capture. | |
1206 | * That requires LBR_FAR but that means far | |
1207 | * jmp need to be filtered out | |
1208 | */ | |
c5cc2cd9 SE |
1209 | } |
1210 | ||
e9d7f7cd YZ |
1211 | /* haswell */ |
1212 | void intel_pmu_lbr_init_hsw(void) | |
1213 | { | |
1214 | x86_pmu.lbr_nr = 16; | |
1215 | x86_pmu.lbr_tos = MSR_LBR_TOS; | |
1216 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; | |
1217 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; | |
1218 | ||
1219 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; | |
1220 | x86_pmu.lbr_sel_map = hsw_lbr_sel_map; | |
19fc9ddd DCC |
1221 | |
1222 | if (lbr_from_signext_quirk_needed()) | |
1223 | static_branch_enable(&lbr_from_quirk_key); | |
e9d7f7cd YZ |
1224 | } |
1225 | ||
9a92e16f AK |
1226 | /* skylake */ |
1227 | __init void intel_pmu_lbr_init_skl(void) | |
1228 | { | |
1229 | x86_pmu.lbr_nr = 32; | |
1230 | x86_pmu.lbr_tos = MSR_LBR_TOS; | |
1231 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; | |
1232 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; | |
1233 | ||
1234 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; | |
1235 | x86_pmu.lbr_sel_map = hsw_lbr_sel_map; | |
1236 | ||
1237 | /* | |
1238 | * SW branch filter usage: | |
1239 | * - support syscall, sysret capture. | |
1240 | * That requires LBR_FAR but that means far | |
1241 | * jmp need to be filtered out | |
1242 | */ | |
9a92e16f AK |
1243 | } |
1244 | ||
c5cc2cd9 | 1245 | /* atom */ |
066ce64c | 1246 | void __init intel_pmu_lbr_init_atom(void) |
caff2bef | 1247 | { |
88c9a65e SE |
1248 | /* |
1249 | * only models starting at stepping 10 seems | |
1250 | * to have an operational LBR which can freeze | |
1251 | * on PMU interrupt | |
1252 | */ | |
3ec18cd8 | 1253 | if (boot_cpu_data.x86_model == 28 |
b399151c | 1254 | && boot_cpu_data.x86_stepping < 10) { |
88c9a65e SE |
1255 | pr_cont("LBR disabled due to erratum"); |
1256 | return; | |
1257 | } | |
1258 | ||
caff2bef | 1259 | x86_pmu.lbr_nr = 8; |
225ce539 SE |
1260 | x86_pmu.lbr_tos = MSR_LBR_TOS; |
1261 | x86_pmu.lbr_from = MSR_LBR_CORE_FROM; | |
1262 | x86_pmu.lbr_to = MSR_LBR_CORE_TO; | |
c5cc2cd9 | 1263 | |
3e702ff6 SE |
1264 | /* |
1265 | * SW branch filter usage: | |
1266 | * - compensate for lack of HW filter | |
1267 | */ | |
caff2bef | 1268 | } |
1e7b9390 | 1269 | |
f21d5adc KL |
1270 | /* slm */ |
1271 | void __init intel_pmu_lbr_init_slm(void) | |
1272 | { | |
1273 | x86_pmu.lbr_nr = 8; | |
1274 | x86_pmu.lbr_tos = MSR_LBR_TOS; | |
1275 | x86_pmu.lbr_from = MSR_LBR_CORE_FROM; | |
1276 | x86_pmu.lbr_to = MSR_LBR_CORE_TO; | |
1277 | ||
1278 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; | |
1279 | x86_pmu.lbr_sel_map = nhm_lbr_sel_map; | |
1280 | ||
1281 | /* | |
1282 | * SW branch filter usage: | |
1283 | * - compensate for lack of HW filter | |
1284 | */ | |
1285 | pr_cont("8-deep LBR, "); | |
1286 | } | |
1287 | ||
1e7b9390 HC |
1288 | /* Knights Landing */ |
1289 | void intel_pmu_lbr_init_knl(void) | |
1290 | { | |
1291 | x86_pmu.lbr_nr = 8; | |
1292 | x86_pmu.lbr_tos = MSR_LBR_TOS; | |
1293 | x86_pmu.lbr_from = MSR_LBR_NHM_FROM; | |
1294 | x86_pmu.lbr_to = MSR_LBR_NHM_TO; | |
1295 | ||
1296 | x86_pmu.lbr_sel_mask = LBR_SEL_MASK; | |
1297 | x86_pmu.lbr_sel_map = snb_lbr_sel_map; | |
16160c19 JT |
1298 | |
1299 | /* Knights Landing does have MISPREDICT bit */ | |
1300 | if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_LIP) | |
1301 | x86_pmu.intel_cap.lbr_format = LBR_FORMAT_EIP_FLAGS; | |
1e7b9390 | 1302 | } |