1 #include <linux/perf_event.h>
2 #include <linux/types.h>
4 #include <asm/perf_event.h>
7 #include "perf_event.h"
11 LBR_FORMAT_LIP = 0x01,
12 LBR_FORMAT_EIP = 0x02,
13 LBR_FORMAT_EIP_FLAGS = 0x03,
17 * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
18 * otherwise it becomes near impossible to get a reliable stack.
21 static void __intel_pmu_lbr_enable(void)
25 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
26 debugctl |= (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
27 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
30 static void __intel_pmu_lbr_disable(void)
34 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
35 debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
36 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
39 static void intel_pmu_lbr_reset_32(void)
43 for (i = 0; i < x86_pmu.lbr_nr; i++)
44 wrmsrl(x86_pmu.lbr_from + i, 0);
47 static void intel_pmu_lbr_reset_64(void)
51 for (i = 0; i < x86_pmu.lbr_nr; i++) {
52 wrmsrl(x86_pmu.lbr_from + i, 0);
53 wrmsrl(x86_pmu.lbr_to + i, 0);
57 void intel_pmu_lbr_reset(void)
62 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
63 intel_pmu_lbr_reset_32();
65 intel_pmu_lbr_reset_64();
68 void intel_pmu_lbr_enable(struct perf_event *event)
70 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
76 * Reset the LBR stack if we changed task context to
80 if (event->ctx->task && cpuc->lbr_context != event->ctx) {
81 intel_pmu_lbr_reset();
82 cpuc->lbr_context = event->ctx;
88 void intel_pmu_lbr_disable(struct perf_event *event)
90 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
96 WARN_ON_ONCE(cpuc->lbr_users < 0);
98 if (cpuc->enabled && !cpuc->lbr_users)
99 __intel_pmu_lbr_disable();
102 void intel_pmu_lbr_enable_all(void)
104 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
107 __intel_pmu_lbr_enable();
110 void intel_pmu_lbr_disable_all(void)
112 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
115 __intel_pmu_lbr_disable();
118 static inline u64 intel_pmu_lbr_tos(void)
122 rdmsrl(x86_pmu.lbr_tos, tos);
127 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
129 unsigned long mask = x86_pmu.lbr_nr - 1;
130 u64 tos = intel_pmu_lbr_tos();
133 for (i = 0; i < x86_pmu.lbr_nr; i++) {
134 unsigned long lbr_idx = (tos - i) & mask;
143 rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
145 cpuc->lbr_entries[i].from = msr_lastbranch.from;
146 cpuc->lbr_entries[i].to = msr_lastbranch.to;
147 cpuc->lbr_entries[i].mispred = 0;
148 cpuc->lbr_entries[i].predicted = 0;
149 cpuc->lbr_entries[i].reserved = 0;
151 cpuc->lbr_stack.nr = i;
154 #define LBR_FROM_FLAG_MISPRED (1ULL << 63)
157 * Due to lack of segmentation in Linux the effective address (offset)
158 * is the same as the linear address, allowing us to merge the LIP and EIP
161 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
163 unsigned long mask = x86_pmu.lbr_nr - 1;
164 int lbr_format = x86_pmu.intel_cap.lbr_format;
165 u64 tos = intel_pmu_lbr_tos();
168 for (i = 0; i < x86_pmu.lbr_nr; i++) {
169 unsigned long lbr_idx = (tos - i) & mask;
170 u64 from, to, mis = 0, pred = 0;
172 rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
173 rdmsrl(x86_pmu.lbr_to + lbr_idx, to);
175 if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
176 mis = !!(from & LBR_FROM_FLAG_MISPRED);
178 from = (u64)((((s64)from) << 1) >> 1);
181 cpuc->lbr_entries[i].from = from;
182 cpuc->lbr_entries[i].to = to;
183 cpuc->lbr_entries[i].mispred = mis;
184 cpuc->lbr_entries[i].predicted = pred;
185 cpuc->lbr_entries[i].reserved = 0;
187 cpuc->lbr_stack.nr = i;
190 void intel_pmu_lbr_read(void)
192 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
194 if (!cpuc->lbr_users)
197 if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32)
198 intel_pmu_lbr_read_32(cpuc);
200 intel_pmu_lbr_read_64(cpuc);
203 void intel_pmu_lbr_init_core(void)
206 x86_pmu.lbr_tos = MSR_LBR_TOS;
207 x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
208 x86_pmu.lbr_to = MSR_LBR_CORE_TO;
211 void intel_pmu_lbr_init_nhm(void)
214 x86_pmu.lbr_tos = MSR_LBR_TOS;
215 x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
216 x86_pmu.lbr_to = MSR_LBR_NHM_TO;
219 void intel_pmu_lbr_init_atom(void)
222 x86_pmu.lbr_tos = MSR_LBR_TOS;
223 x86_pmu.lbr_from = MSR_LBR_CORE_FROM;
224 x86_pmu.lbr_to = MSR_LBR_CORE_TO;