1 #include <linux/perf_event.h>
2 #include <linux/types.h>
4 #include <asm/perf_event.h>
7 #include "perf_event.h"
11 LBR_FORMAT_LIP = 0x01,
12 LBR_FORMAT_EIP = 0x02,
13 LBR_FORMAT_EIP_FLAGS = 0x03,
17 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
18 * otherwise it becomes near impossible to get a reliable stack.
21 static void __intel_pmu_lbr_enable(void)
25 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
26 debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
27 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
30 static void __intel_pmu_lbr_disable(void)
34 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
35 debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
36 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
39 static void intel_pmu_lbr_reset_32(void)
43 for (i = 0; i < x86_pmu.lbr_nr; i++)
44 wrmsrl(x86_pmu.lbr_from + i, 0);
47 static void intel_pmu_lbr_reset_64(void)
51 for (i = 0; i < x86_pmu.lbr_nr; i++) {
52 wrmsrl(x86_pmu.lbr_from + i, 0);
53 wrmsrl(x86_pmu.lbr_to + i, 0);
57 void intel_pmu_lbr_reset(void)
62 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
63 intel_pmu_lbr_reset_32();
65 intel_pmu_lbr_reset_64();
68 void intel_pmu_lbr_enable(struct perf_event *event)
70 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
76 * Reset the LBR stack if we changed task context to
80 if (event->ctx->task && cpuc->lbr_context != event->ctx) {
81 intel_pmu_lbr_reset();
82 cpuc->lbr_context = event->ctx;
88 void intel_pmu_lbr_disable(struct perf_event *event)
90 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
96 WARN_ON_ONCE(cpuc->lbr_users < 0);
98 if (cpuc->enabled && !cpuc->lbr_users)
99 __intel_pmu_lbr_disable();
102 void intel_pmu_lbr_enable_all(void)
104 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
107 __intel_pmu_lbr_enable();
110 void intel_pmu_lbr_disable_all(void)
112 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
115 __intel_pmu_lbr_disable();
118 static inline u64 intel_pmu_lbr_tos(void)
122 rdmsrl(x86_pmu.lbr_tos, tos);
127 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
129 unsigned long mask = x86_pmu.lbr_nr - 1;
130 u64 tos = intel_pmu_lbr_tos();
133 for (i = 0; i < x86_pmu.lbr_nr; i++) {
134 unsigned long lbr_idx = (tos - i) & mask;
143 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
145 cpuc->lbr_entries[i].from = msr_lastbranch.from;
146 cpuc->lbr_entries[i].to = msr_lastbranch.to;
147 cpuc->lbr_entries[i].flags = 0;
149 cpuc->lbr_stack.nr = i;
152 #define LBR_FROM_FLAG_MISPRED (1ULL << 63)
155 * Due to lack of segmentation in Linux the effective address (offset)
156 * is the same as the linear address, allowing us to merge the LIP and EIP
159 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
161 unsigned long mask = x86_pmu.lbr_nr - 1;
162 int lbr_format = x86_pmu.intel_cap.lbr_format;
163 u64 tos = intel_pmu_lbr_tos();
166 for (i = 0; i < x86_pmu.lbr_nr; i++) {
167 unsigned long lbr_idx = (tos - i) & mask;
168 u64 from, to, flags = 0;
170 rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
171 rdmsrl(x86_pmu.lbr_to + lbr_idx, to);
173 if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
174 flags = !!(from & LBR_FROM_FLAG_MISPRED);
175 from = (u64)((((s64)from) << 1) >> 1);
178 cpuc->lbr_entries[i].from = from;
179 cpuc->lbr_entries[i].to = to;
180 cpuc->lbr_entries[i].flags = flags;
182 cpuc->lbr_stack.nr = i;
185 void intel_pmu_lbr_read(void)
187 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
189 if (!cpuc->lbr_users)
192 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
193 intel_pmu_lbr_read_32(cpuc);
195 intel_pmu_lbr_read_64(cpuc);
198 void intel_pmu_lbr_init_core(void)
201 x86_pmu.lbr_tos = 0x01c9;
202 x86_pmu.lbr_from = 0x40;
203 x86_pmu.lbr_to = 0x60;
206 void intel_pmu_lbr_init_nhm(void)
209 x86_pmu.lbr_tos = 0x01c9;
210 x86_pmu.lbr_from = 0x680;
211 x86_pmu.lbr_to = 0x6c0;
214 void intel_pmu_lbr_init_atom(void)
217 x86_pmu.lbr_tos = 0x01c9;
218 x86_pmu.lbr_from = 0x40;
219 x86_pmu.lbr_to = 0x60;