Commit | Line | Data |
---|---|---|
a7e3ed1e | 1 | /* |
efc9f05d SE |
2 | * Per core/cpu state |
3 | * | |
4 | * Used to coordinate shared registers between HT threads or | |
5 | * among events on a single PMU. | |
a7e3ed1e | 6 | */ |
de0428a7 | 7 | |
c767a54b JP |
8 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
9 | ||
de0428a7 KW |
10 | #include <linux/stddef.h> |
11 | #include <linux/types.h> | |
12 | #include <linux/init.h> | |
13 | #include <linux/slab.h> | |
69c60c88 | 14 | #include <linux/export.h> |
aacfbe6a | 15 | #include <linux/nmi.h> |
de0428a7 | 16 | |
3a632cb2 | 17 | #include <asm/cpufeature.h> |
de0428a7 | 18 | #include <asm/hardirq.h> |
ef5f9f47 | 19 | #include <asm/intel-family.h> |
de0428a7 | 20 | #include <asm/apic.h> |
9b545c04 | 21 | #include <asm/cpu_device_id.h> |
de0428a7 | 22 | |
27f6d22b | 23 | #include "../perf_event.h" |
a7e3ed1e | 24 | |
f22f54f4 | 25 | /* |
b622d644 | 26 | * Intel PerfMon, used on Core and later. |
f22f54f4 | 27 | */ |
ec75a716 | 28 | static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly = |
f22f54f4 | 29 | { |
c3b7cdf1 PE |
30 | [PERF_COUNT_HW_CPU_CYCLES] = 0x003c, |
31 | [PERF_COUNT_HW_INSTRUCTIONS] = 0x00c0, | |
32 | [PERF_COUNT_HW_CACHE_REFERENCES] = 0x4f2e, | |
33 | [PERF_COUNT_HW_CACHE_MISSES] = 0x412e, | |
34 | [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c4, | |
35 | [PERF_COUNT_HW_BRANCH_MISSES] = 0x00c5, | |
36 | [PERF_COUNT_HW_BUS_CYCLES] = 0x013c, | |
37 | [PERF_COUNT_HW_REF_CPU_CYCLES] = 0x0300, /* pseudo-encoding */ | |
f22f54f4 PZ |
38 | }; |
39 | ||
5c543e3c | 40 | static struct event_constraint intel_core_event_constraints[] __read_mostly = |
f22f54f4 PZ |
41 | { |
42 | INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */ | |
43 | INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */ | |
44 | INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */ | |
45 | INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */ | |
46 | INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */ | |
47 | INTEL_EVENT_CONSTRAINT(0xc1, 0x1), /* FP_COMP_INSTR_RET */ | |
48 | EVENT_CONSTRAINT_END | |
49 | }; | |
50 | ||
5c543e3c | 51 | static struct event_constraint intel_core2_event_constraints[] __read_mostly = |
f22f54f4 | 52 | { |
b622d644 PZ |
53 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
54 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
cd09c0c4 | 55 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ |
f22f54f4 PZ |
56 | INTEL_EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */ |
57 | INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */ | |
58 | INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */ | |
59 | INTEL_EVENT_CONSTRAINT(0x13, 0x2), /* DIV */ | |
60 | INTEL_EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */ | |
61 | INTEL_EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */ | |
62 | INTEL_EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */ | |
63 | INTEL_EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */ | |
b622d644 | 64 | INTEL_EVENT_CONSTRAINT(0xc9, 0x1), /* ITLB_MISS_RETIRED (T30-9) */ |
f22f54f4 PZ |
65 | INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */ |
66 | EVENT_CONSTRAINT_END | |
67 | }; | |
68 | ||
5c543e3c | 69 | static struct event_constraint intel_nehalem_event_constraints[] __read_mostly = |
f22f54f4 | 70 | { |
b622d644 PZ |
71 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
72 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
cd09c0c4 | 73 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ |
f22f54f4 PZ |
74 | INTEL_EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */ |
75 | INTEL_EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */ | |
76 | INTEL_EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */ | |
77 | INTEL_EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */ | |
78 | INTEL_EVENT_CONSTRAINT(0x48, 0x3), /* L1D_PEND_MISS */ | |
79 | INTEL_EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */ | |
80 | INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */ | |
81 | INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */ | |
82 | EVENT_CONSTRAINT_END | |
83 | }; | |
84 | ||
5c543e3c | 85 | static struct extra_reg intel_nehalem_extra_regs[] __read_mostly = |
a7e3ed1e | 86 | { |
53ad0447 YZ |
87 | /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ |
88 | INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), | |
f20093ee | 89 | INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b), |
a7e3ed1e AK |
90 | EVENT_EXTRA_END |
91 | }; | |
92 | ||
5c543e3c | 93 | static struct event_constraint intel_westmere_event_constraints[] __read_mostly = |
f22f54f4 | 94 | { |
b622d644 PZ |
95 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
96 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
cd09c0c4 | 97 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ |
f22f54f4 PZ |
98 | INTEL_EVENT_CONSTRAINT(0x51, 0x3), /* L1D */ |
99 | INTEL_EVENT_CONSTRAINT(0x60, 0x1), /* OFFCORE_REQUESTS_OUTSTANDING */ | |
100 | INTEL_EVENT_CONSTRAINT(0x63, 0x3), /* CACHE_LOCK_CYCLES */ | |
d1100770 | 101 | INTEL_EVENT_CONSTRAINT(0xb3, 0x1), /* SNOOPQ_REQUEST_OUTSTANDING */ |
f22f54f4 PZ |
102 | EVENT_CONSTRAINT_END |
103 | }; | |
104 | ||
5c543e3c | 105 | static struct event_constraint intel_snb_event_constraints[] __read_mostly = |
b06b3d49 LM |
106 | { |
107 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ | |
108 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
cd09c0c4 | 109 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ |
fd4a5aef SE |
110 | INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */ |
111 | INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */ | |
112 | INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */ | |
113 | INTEL_UEVENT_CONSTRAINT(0x06a3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */ | |
b06b3d49 | 114 | INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */ |
b06b3d49 LM |
115 | INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */ |
116 | INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */ | |
f8378f52 AK |
117 | INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */ |
118 | INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */ | |
93fcf72c | 119 | |
9010ae4a SE |
120 | /* |
121 | * When HT is off these events can only run on the bottom 4 counters | |
122 | * When HT is on, they are impacted by the HT bug and require EXCL access | |
123 | */ | |
93fcf72c MD |
124 | INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */ |
125 | INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */ | |
126 | INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ | |
127 | INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */ | |
128 | ||
b06b3d49 LM |
129 | EVENT_CONSTRAINT_END |
130 | }; | |
131 | ||
69943182 SE |
132 | static struct event_constraint intel_ivb_event_constraints[] __read_mostly = |
133 | { | |
134 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ | |
135 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
136 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ | |
137 | INTEL_UEVENT_CONSTRAINT(0x0148, 0x4), /* L1D_PEND_MISS.PENDING */ | |
138 | INTEL_UEVENT_CONSTRAINT(0x0279, 0xf), /* IDQ.EMTPY */ | |
139 | INTEL_UEVENT_CONSTRAINT(0x019c, 0xf), /* IDQ_UOPS_NOT_DELIVERED.CORE */ | |
6113af14 | 140 | INTEL_UEVENT_CONSTRAINT(0x02a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_LDM_PENDING */ |
69943182 SE |
141 | INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */ |
142 | INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf), /* CYCLE_ACTIVITY.STALLS_L2_PENDING */ | |
143 | INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf), /* CYCLE_ACTIVITY.STALLS_LDM_PENDING */ | |
144 | INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */ | |
145 | INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */ | |
146 | INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */ | |
93fcf72c | 147 | |
9010ae4a SE |
148 | /* |
149 | * When HT is off these events can only run on the bottom 4 counters | |
150 | * When HT is on, they are impacted by the HT bug and require EXCL access | |
151 | */ | |
93fcf72c MD |
152 | INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */ |
153 | INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */ | |
154 | INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ | |
155 | INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */ | |
156 | ||
69943182 SE |
157 | EVENT_CONSTRAINT_END |
158 | }; | |
159 | ||
5c543e3c | 160 | static struct extra_reg intel_westmere_extra_regs[] __read_mostly = |
a7e3ed1e | 161 | { |
53ad0447 YZ |
162 | /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ |
163 | INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), | |
164 | INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1), | |
f20093ee | 165 | INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b), |
a7e3ed1e AK |
166 | EVENT_EXTRA_END |
167 | }; | |
168 | ||
0af3ac1f AK |
169 | static struct event_constraint intel_v1_event_constraints[] __read_mostly = |
170 | { | |
171 | EVENT_CONSTRAINT_END | |
172 | }; | |
173 | ||
5c543e3c | 174 | static struct event_constraint intel_gen_event_constraints[] __read_mostly = |
f22f54f4 | 175 | { |
b622d644 PZ |
176 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
177 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
cd09c0c4 | 178 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ |
f22f54f4 PZ |
179 | EVENT_CONSTRAINT_END |
180 | }; | |
181 | ||
1fa64180 YZ |
182 | static struct event_constraint intel_slm_event_constraints[] __read_mostly = |
183 | { | |
184 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ | |
185 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
1fa64180 YZ |
186 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */ |
187 | EVENT_CONSTRAINT_END | |
188 | }; | |
189 | ||
20f36278 | 190 | static struct event_constraint intel_skl_event_constraints[] = { |
9a92e16f AK |
191 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
192 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
193 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ | |
194 | INTEL_UEVENT_CONSTRAINT(0x1c0, 0x2), /* INST_RETIRED.PREC_DIST */ | |
9010ae4a SE |
195 | |
196 | /* | |
197 | * when HT is off, these can only run on the bottom 4 counters | |
198 | */ | |
199 | INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_INST_RETIRED.* */ | |
200 | INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_RETIRED.* */ | |
201 | INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_L3_HIT_RETIRED.* */ | |
202 | INTEL_EVENT_CONSTRAINT(0xcd, 0xf), /* MEM_TRANS_RETIRED.* */ | |
203 | INTEL_EVENT_CONSTRAINT(0xc6, 0xf), /* FRONTEND_RETIRED.* */ | |
204 | ||
9a92e16f AK |
205 | EVENT_CONSTRAINT_END |
206 | }; | |
207 | ||
1e7b9390 | 208 | static struct extra_reg intel_knl_extra_regs[] __read_mostly = { |
9c489fce LO |
209 | INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x799ffbb6e7ull, RSP_0), |
210 | INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x399ffbffe7ull, RSP_1), | |
1e7b9390 HC |
211 | EVENT_EXTRA_END |
212 | }; | |
213 | ||
ee89cbc2 | 214 | static struct extra_reg intel_snb_extra_regs[] __read_mostly = { |
53ad0447 YZ |
215 | /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ |
216 | INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0), | |
217 | INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1), | |
f20093ee | 218 | INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd), |
f1923820 SE |
219 | EVENT_EXTRA_END |
220 | }; | |
221 | ||
222 | static struct extra_reg intel_snbep_extra_regs[] __read_mostly = { | |
53ad0447 YZ |
223 | /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ |
224 | INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0), | |
225 | INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1), | |
f1a52789 | 226 | INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd), |
ee89cbc2 SE |
227 | EVENT_EXTRA_END |
228 | }; | |
229 | ||
9a92e16f AK |
230 | static struct extra_reg intel_skl_extra_regs[] __read_mostly = { |
231 | INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffff8fffull, RSP_0), | |
232 | INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffff8fffull, RSP_1), | |
233 | INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd), | |
dfe1f3cb AK |
234 | /* |
235 | * Note the low 8 bits eventsel code is not a continuous field, containing | |
236 | * some #GPing bits. These are masked out. | |
237 | */ | |
238 | INTEL_UEVENT_EXTRA_REG(0x01c6, MSR_PEBS_FRONTEND, 0x7fff17, FE), | |
9a92e16f AK |
239 | EVENT_EXTRA_END |
240 | }; | |
241 | ||
7f2ee91f IM |
242 | EVENT_ATTR_STR(mem-loads, mem_ld_nhm, "event=0x0b,umask=0x10,ldlat=3"); |
243 | EVENT_ATTR_STR(mem-loads, mem_ld_snb, "event=0xcd,umask=0x1,ldlat=3"); | |
244 | EVENT_ATTR_STR(mem-stores, mem_st_snb, "event=0xcd,umask=0x2"); | |
f20093ee | 245 | |
d4ae5529 | 246 | static struct attribute *nhm_mem_events_attrs[] = { |
f20093ee SE |
247 | EVENT_PTR(mem_ld_nhm), |
248 | NULL, | |
249 | }; | |
250 | ||
a39fcae7 AK |
251 | /* |
252 | * topdown events for Intel Core CPUs. | |
253 | * | |
254 | * The events are all in slots, which is a free slot in a 4 wide | |
255 | * pipeline. Some events are already reported in slots, for cycle | |
256 | * events we multiply by the pipeline width (4). | |
257 | * | |
258 | * With Hyper Threading on, topdown metrics are either summed or averaged | |
259 | * between the threads of a core: (count_t0 + count_t1). | |
260 | * | |
261 | * For the average case the metric is always scaled to pipeline width, | |
262 | * so we use factor 2 ((count_t0 + count_t1) / 2 * 4) | |
263 | */ | |
264 | ||
265 | EVENT_ATTR_STR_HT(topdown-total-slots, td_total_slots, | |
266 | "event=0x3c,umask=0x0", /* cpu_clk_unhalted.thread */ | |
267 | "event=0x3c,umask=0x0,any=1"); /* cpu_clk_unhalted.thread_any */ | |
268 | EVENT_ATTR_STR_HT(topdown-total-slots.scale, td_total_slots_scale, "4", "2"); | |
269 | EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued, | |
270 | "event=0xe,umask=0x1"); /* uops_issued.any */ | |
271 | EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired, | |
272 | "event=0xc2,umask=0x2"); /* uops_retired.retire_slots */ | |
273 | EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles, | |
274 | "event=0x9c,umask=0x1"); /* idq_uops_not_delivered_core */ | |
275 | EVENT_ATTR_STR_HT(topdown-recovery-bubbles, td_recovery_bubbles, | |
276 | "event=0xd,umask=0x3,cmask=1", /* int_misc.recovery_cycles */ | |
277 | "event=0xd,umask=0x3,cmask=1,any=1"); /* int_misc.recovery_cycles_any */ | |
278 | EVENT_ATTR_STR_HT(topdown-recovery-bubbles.scale, td_recovery_bubbles_scale, | |
279 | "4", "2"); | |
280 | ||
20f36278 | 281 | static struct attribute *snb_events_attrs[] = { |
a39fcae7 AK |
282 | EVENT_PTR(td_slots_issued), |
283 | EVENT_PTR(td_slots_retired), | |
284 | EVENT_PTR(td_fetch_bubbles), | |
285 | EVENT_PTR(td_total_slots), | |
286 | EVENT_PTR(td_total_slots_scale), | |
287 | EVENT_PTR(td_recovery_bubbles), | |
288 | EVENT_PTR(td_recovery_bubbles_scale), | |
f20093ee SE |
289 | NULL, |
290 | }; | |
291 | ||
d4ae5529 JO |
292 | static struct attribute *snb_mem_events_attrs[] = { |
293 | EVENT_PTR(mem_ld_snb), | |
294 | EVENT_PTR(mem_st_snb), | |
295 | NULL, | |
296 | }; | |
297 | ||
3a632cb2 AK |
298 | static struct event_constraint intel_hsw_event_constraints[] = { |
299 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ | |
300 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
301 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ | |
e0fbac1c | 302 | INTEL_UEVENT_CONSTRAINT(0x148, 0x4), /* L1D_PEND_MISS.PENDING */ |
3a632cb2 AK |
303 | INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */ |
304 | INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */ | |
305 | /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */ | |
c420f19b | 306 | INTEL_UEVENT_CONSTRAINT(0x08a3, 0x4), |
3a632cb2 | 307 | /* CYCLE_ACTIVITY.STALLS_L1D_PENDING */ |
c420f19b | 308 | INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4), |
3a632cb2 | 309 | /* CYCLE_ACTIVITY.CYCLES_NO_EXECUTE */ |
c420f19b | 310 | INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), |
93fcf72c | 311 | |
9010ae4a SE |
312 | /* |
313 | * When HT is off these events can only run on the bottom 4 counters | |
314 | * When HT is on, they are impacted by the HT bug and require EXCL access | |
315 | */ | |
93fcf72c MD |
316 | INTEL_EXCLEVT_CONSTRAINT(0xd0, 0xf), /* MEM_UOPS_RETIRED.* */ |
317 | INTEL_EXCLEVT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */ | |
318 | INTEL_EXCLEVT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ | |
319 | INTEL_EXCLEVT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */ | |
320 | ||
3a632cb2 AK |
321 | EVENT_CONSTRAINT_END |
322 | }; | |
323 | ||
20f36278 | 324 | static struct event_constraint intel_bdw_event_constraints[] = { |
91f1b705 AK |
325 | FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ |
326 | FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ | |
327 | FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ | |
328 | INTEL_UEVENT_CONSTRAINT(0x148, 0x4), /* L1D_PEND_MISS.PENDING */ | |
b7883a1c | 329 | INTEL_UBIT_EVENT_CONSTRAINT(0x8a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_MISS */ |
9010ae4a SE |
330 | /* |
331 | * when HT is off, these can only run on the bottom 4 counters | |
332 | */ | |
333 | INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_INST_RETIRED.* */ | |
334 | INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_RETIRED.* */ | |
335 | INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_L3_HIT_RETIRED.* */ | |
336 | INTEL_EVENT_CONSTRAINT(0xcd, 0xf), /* MEM_TRANS_RETIRED.* */ | |
91f1b705 AK |
337 | EVENT_CONSTRAINT_END |
338 | }; | |
339 | ||
f22f54f4 PZ |
340 | static u64 intel_pmu_event_map(int hw_event) |
341 | { | |
342 | return intel_perfmon_event_map[hw_event]; | |
343 | } | |
344 | ||
9a92e16f AK |
345 | /* |
346 | * Notes on the events: | |
347 | * - data reads do not include code reads (comparable to earlier tables) | |
348 | * - data counts include speculative execution (except L1 write, dtlb, bpu) | |
349 | * - remote node access includes remote memory, remote cache, remote mmio. | |
350 | * - prefetches are not included in the counts. | |
351 | * - icache miss does not include decoded icache | |
352 | */ | |
353 | ||
354 | #define SKL_DEMAND_DATA_RD BIT_ULL(0) | |
355 | #define SKL_DEMAND_RFO BIT_ULL(1) | |
356 | #define SKL_ANY_RESPONSE BIT_ULL(16) | |
357 | #define SKL_SUPPLIER_NONE BIT_ULL(17) | |
358 | #define SKL_L3_MISS_LOCAL_DRAM BIT_ULL(26) | |
359 | #define SKL_L3_MISS_REMOTE_HOP0_DRAM BIT_ULL(27) | |
360 | #define SKL_L3_MISS_REMOTE_HOP1_DRAM BIT_ULL(28) | |
361 | #define SKL_L3_MISS_REMOTE_HOP2P_DRAM BIT_ULL(29) | |
362 | #define SKL_L3_MISS (SKL_L3_MISS_LOCAL_DRAM| \ | |
363 | SKL_L3_MISS_REMOTE_HOP0_DRAM| \ | |
364 | SKL_L3_MISS_REMOTE_HOP1_DRAM| \ | |
365 | SKL_L3_MISS_REMOTE_HOP2P_DRAM) | |
366 | #define SKL_SPL_HIT BIT_ULL(30) | |
367 | #define SKL_SNOOP_NONE BIT_ULL(31) | |
368 | #define SKL_SNOOP_NOT_NEEDED BIT_ULL(32) | |
369 | #define SKL_SNOOP_MISS BIT_ULL(33) | |
370 | #define SKL_SNOOP_HIT_NO_FWD BIT_ULL(34) | |
371 | #define SKL_SNOOP_HIT_WITH_FWD BIT_ULL(35) | |
372 | #define SKL_SNOOP_HITM BIT_ULL(36) | |
373 | #define SKL_SNOOP_NON_DRAM BIT_ULL(37) | |
374 | #define SKL_ANY_SNOOP (SKL_SPL_HIT|SKL_SNOOP_NONE| \ | |
375 | SKL_SNOOP_NOT_NEEDED|SKL_SNOOP_MISS| \ | |
376 | SKL_SNOOP_HIT_NO_FWD|SKL_SNOOP_HIT_WITH_FWD| \ | |
377 | SKL_SNOOP_HITM|SKL_SNOOP_NON_DRAM) | |
378 | #define SKL_DEMAND_READ SKL_DEMAND_DATA_RD | |
379 | #define SKL_SNOOP_DRAM (SKL_SNOOP_NONE| \ | |
380 | SKL_SNOOP_NOT_NEEDED|SKL_SNOOP_MISS| \ | |
381 | SKL_SNOOP_HIT_NO_FWD|SKL_SNOOP_HIT_WITH_FWD| \ | |
382 | SKL_SNOOP_HITM|SKL_SPL_HIT) | |
383 | #define SKL_DEMAND_WRITE SKL_DEMAND_RFO | |
384 | #define SKL_LLC_ACCESS SKL_ANY_RESPONSE | |
385 | #define SKL_L3_MISS_REMOTE (SKL_L3_MISS_REMOTE_HOP0_DRAM| \ | |
386 | SKL_L3_MISS_REMOTE_HOP1_DRAM| \ | |
387 | SKL_L3_MISS_REMOTE_HOP2P_DRAM) | |
388 | ||
389 | static __initconst const u64 skl_hw_cache_event_ids | |
390 | [PERF_COUNT_HW_CACHE_MAX] | |
391 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
392 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
393 | { | |
394 | [ C(L1D ) ] = { | |
395 | [ C(OP_READ) ] = { | |
396 | [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_INST_RETIRED.ALL_LOADS */ | |
397 | [ C(RESULT_MISS) ] = 0x151, /* L1D.REPLACEMENT */ | |
398 | }, | |
399 | [ C(OP_WRITE) ] = { | |
400 | [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_INST_RETIRED.ALL_STORES */ | |
401 | [ C(RESULT_MISS) ] = 0x0, | |
402 | }, | |
403 | [ C(OP_PREFETCH) ] = { | |
404 | [ C(RESULT_ACCESS) ] = 0x0, | |
405 | [ C(RESULT_MISS) ] = 0x0, | |
406 | }, | |
407 | }, | |
408 | [ C(L1I ) ] = { | |
409 | [ C(OP_READ) ] = { | |
410 | [ C(RESULT_ACCESS) ] = 0x0, | |
411 | [ C(RESULT_MISS) ] = 0x283, /* ICACHE_64B.MISS */ | |
412 | }, | |
413 | [ C(OP_WRITE) ] = { | |
414 | [ C(RESULT_ACCESS) ] = -1, | |
415 | [ C(RESULT_MISS) ] = -1, | |
416 | }, | |
417 | [ C(OP_PREFETCH) ] = { | |
418 | [ C(RESULT_ACCESS) ] = 0x0, | |
419 | [ C(RESULT_MISS) ] = 0x0, | |
420 | }, | |
421 | }, | |
422 | [ C(LL ) ] = { | |
423 | [ C(OP_READ) ] = { | |
424 | [ C(RESULT_ACCESS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
425 | [ C(RESULT_MISS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
426 | }, | |
427 | [ C(OP_WRITE) ] = { | |
428 | [ C(RESULT_ACCESS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
429 | [ C(RESULT_MISS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
430 | }, | |
431 | [ C(OP_PREFETCH) ] = { | |
432 | [ C(RESULT_ACCESS) ] = 0x0, | |
433 | [ C(RESULT_MISS) ] = 0x0, | |
434 | }, | |
435 | }, | |
436 | [ C(DTLB) ] = { | |
437 | [ C(OP_READ) ] = { | |
438 | [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_INST_RETIRED.ALL_LOADS */ | |
fb3a5055 | 439 | [ C(RESULT_MISS) ] = 0xe08, /* DTLB_LOAD_MISSES.WALK_COMPLETED */ |
9a92e16f AK |
440 | }, |
441 | [ C(OP_WRITE) ] = { | |
442 | [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_INST_RETIRED.ALL_STORES */ | |
fb3a5055 | 443 | [ C(RESULT_MISS) ] = 0xe49, /* DTLB_STORE_MISSES.WALK_COMPLETED */ |
9a92e16f AK |
444 | }, |
445 | [ C(OP_PREFETCH) ] = { | |
446 | [ C(RESULT_ACCESS) ] = 0x0, | |
447 | [ C(RESULT_MISS) ] = 0x0, | |
448 | }, | |
449 | }, | |
450 | [ C(ITLB) ] = { | |
451 | [ C(OP_READ) ] = { | |
452 | [ C(RESULT_ACCESS) ] = 0x2085, /* ITLB_MISSES.STLB_HIT */ | |
453 | [ C(RESULT_MISS) ] = 0xe85, /* ITLB_MISSES.WALK_COMPLETED */ | |
454 | }, | |
455 | [ C(OP_WRITE) ] = { | |
456 | [ C(RESULT_ACCESS) ] = -1, | |
457 | [ C(RESULT_MISS) ] = -1, | |
458 | }, | |
459 | [ C(OP_PREFETCH) ] = { | |
460 | [ C(RESULT_ACCESS) ] = -1, | |
461 | [ C(RESULT_MISS) ] = -1, | |
462 | }, | |
463 | }, | |
464 | [ C(BPU ) ] = { | |
465 | [ C(OP_READ) ] = { | |
466 | [ C(RESULT_ACCESS) ] = 0xc4, /* BR_INST_RETIRED.ALL_BRANCHES */ | |
467 | [ C(RESULT_MISS) ] = 0xc5, /* BR_MISP_RETIRED.ALL_BRANCHES */ | |
468 | }, | |
469 | [ C(OP_WRITE) ] = { | |
470 | [ C(RESULT_ACCESS) ] = -1, | |
471 | [ C(RESULT_MISS) ] = -1, | |
472 | }, | |
473 | [ C(OP_PREFETCH) ] = { | |
474 | [ C(RESULT_ACCESS) ] = -1, | |
475 | [ C(RESULT_MISS) ] = -1, | |
476 | }, | |
477 | }, | |
478 | [ C(NODE) ] = { | |
479 | [ C(OP_READ) ] = { | |
480 | [ C(RESULT_ACCESS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
481 | [ C(RESULT_MISS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
482 | }, | |
483 | [ C(OP_WRITE) ] = { | |
484 | [ C(RESULT_ACCESS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
485 | [ C(RESULT_MISS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
486 | }, | |
487 | [ C(OP_PREFETCH) ] = { | |
488 | [ C(RESULT_ACCESS) ] = 0x0, | |
489 | [ C(RESULT_MISS) ] = 0x0, | |
490 | }, | |
491 | }, | |
492 | }; | |
493 | ||
494 | static __initconst const u64 skl_hw_cache_extra_regs | |
495 | [PERF_COUNT_HW_CACHE_MAX] | |
496 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
497 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
498 | { | |
499 | [ C(LL ) ] = { | |
500 | [ C(OP_READ) ] = { | |
501 | [ C(RESULT_ACCESS) ] = SKL_DEMAND_READ| | |
502 | SKL_LLC_ACCESS|SKL_ANY_SNOOP, | |
503 | [ C(RESULT_MISS) ] = SKL_DEMAND_READ| | |
504 | SKL_L3_MISS|SKL_ANY_SNOOP| | |
505 | SKL_SUPPLIER_NONE, | |
506 | }, | |
507 | [ C(OP_WRITE) ] = { | |
508 | [ C(RESULT_ACCESS) ] = SKL_DEMAND_WRITE| | |
509 | SKL_LLC_ACCESS|SKL_ANY_SNOOP, | |
510 | [ C(RESULT_MISS) ] = SKL_DEMAND_WRITE| | |
511 | SKL_L3_MISS|SKL_ANY_SNOOP| | |
512 | SKL_SUPPLIER_NONE, | |
513 | }, | |
514 | [ C(OP_PREFETCH) ] = { | |
515 | [ C(RESULT_ACCESS) ] = 0x0, | |
516 | [ C(RESULT_MISS) ] = 0x0, | |
517 | }, | |
518 | }, | |
519 | [ C(NODE) ] = { | |
520 | [ C(OP_READ) ] = { | |
521 | [ C(RESULT_ACCESS) ] = SKL_DEMAND_READ| | |
522 | SKL_L3_MISS_LOCAL_DRAM|SKL_SNOOP_DRAM, | |
523 | [ C(RESULT_MISS) ] = SKL_DEMAND_READ| | |
524 | SKL_L3_MISS_REMOTE|SKL_SNOOP_DRAM, | |
525 | }, | |
526 | [ C(OP_WRITE) ] = { | |
527 | [ C(RESULT_ACCESS) ] = SKL_DEMAND_WRITE| | |
528 | SKL_L3_MISS_LOCAL_DRAM|SKL_SNOOP_DRAM, | |
529 | [ C(RESULT_MISS) ] = SKL_DEMAND_WRITE| | |
530 | SKL_L3_MISS_REMOTE|SKL_SNOOP_DRAM, | |
531 | }, | |
532 | [ C(OP_PREFETCH) ] = { | |
533 | [ C(RESULT_ACCESS) ] = 0x0, | |
534 | [ C(RESULT_MISS) ] = 0x0, | |
535 | }, | |
536 | }, | |
537 | }; | |
538 | ||
74e6543f YZ |
539 | #define SNB_DMND_DATA_RD (1ULL << 0) |
540 | #define SNB_DMND_RFO (1ULL << 1) | |
541 | #define SNB_DMND_IFETCH (1ULL << 2) | |
542 | #define SNB_DMND_WB (1ULL << 3) | |
543 | #define SNB_PF_DATA_RD (1ULL << 4) | |
544 | #define SNB_PF_RFO (1ULL << 5) | |
545 | #define SNB_PF_IFETCH (1ULL << 6) | |
546 | #define SNB_LLC_DATA_RD (1ULL << 7) | |
547 | #define SNB_LLC_RFO (1ULL << 8) | |
548 | #define SNB_LLC_IFETCH (1ULL << 9) | |
549 | #define SNB_BUS_LOCKS (1ULL << 10) | |
550 | #define SNB_STRM_ST (1ULL << 11) | |
551 | #define SNB_OTHER (1ULL << 15) | |
552 | #define SNB_RESP_ANY (1ULL << 16) | |
553 | #define SNB_NO_SUPP (1ULL << 17) | |
554 | #define SNB_LLC_HITM (1ULL << 18) | |
555 | #define SNB_LLC_HITE (1ULL << 19) | |
556 | #define SNB_LLC_HITS (1ULL << 20) | |
557 | #define SNB_LLC_HITF (1ULL << 21) | |
558 | #define SNB_LOCAL (1ULL << 22) | |
559 | #define SNB_REMOTE (0xffULL << 23) | |
560 | #define SNB_SNP_NONE (1ULL << 31) | |
561 | #define SNB_SNP_NOT_NEEDED (1ULL << 32) | |
562 | #define SNB_SNP_MISS (1ULL << 33) | |
563 | #define SNB_NO_FWD (1ULL << 34) | |
564 | #define SNB_SNP_FWD (1ULL << 35) | |
565 | #define SNB_HITM (1ULL << 36) | |
566 | #define SNB_NON_DRAM (1ULL << 37) | |
567 | ||
568 | #define SNB_DMND_READ (SNB_DMND_DATA_RD|SNB_LLC_DATA_RD) | |
569 | #define SNB_DMND_WRITE (SNB_DMND_RFO|SNB_LLC_RFO) | |
570 | #define SNB_DMND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO) | |
571 | ||
572 | #define SNB_SNP_ANY (SNB_SNP_NONE|SNB_SNP_NOT_NEEDED| \ | |
573 | SNB_SNP_MISS|SNB_NO_FWD|SNB_SNP_FWD| \ | |
574 | SNB_HITM) | |
575 | ||
576 | #define SNB_DRAM_ANY (SNB_LOCAL|SNB_REMOTE|SNB_SNP_ANY) | |
577 | #define SNB_DRAM_REMOTE (SNB_REMOTE|SNB_SNP_ANY) | |
578 | ||
579 | #define SNB_L3_ACCESS SNB_RESP_ANY | |
580 | #define SNB_L3_MISS (SNB_DRAM_ANY|SNB_NON_DRAM) | |
581 | ||
582 | static __initconst const u64 snb_hw_cache_extra_regs | |
583 | [PERF_COUNT_HW_CACHE_MAX] | |
584 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
585 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
586 | { | |
587 | [ C(LL ) ] = { | |
588 | [ C(OP_READ) ] = { | |
589 | [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_L3_ACCESS, | |
590 | [ C(RESULT_MISS) ] = SNB_DMND_READ|SNB_L3_MISS, | |
591 | }, | |
592 | [ C(OP_WRITE) ] = { | |
593 | [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_L3_ACCESS, | |
594 | [ C(RESULT_MISS) ] = SNB_DMND_WRITE|SNB_L3_MISS, | |
595 | }, | |
596 | [ C(OP_PREFETCH) ] = { | |
597 | [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_L3_ACCESS, | |
598 | [ C(RESULT_MISS) ] = SNB_DMND_PREFETCH|SNB_L3_MISS, | |
599 | }, | |
600 | }, | |
601 | [ C(NODE) ] = { | |
602 | [ C(OP_READ) ] = { | |
603 | [ C(RESULT_ACCESS) ] = SNB_DMND_READ|SNB_DRAM_ANY, | |
604 | [ C(RESULT_MISS) ] = SNB_DMND_READ|SNB_DRAM_REMOTE, | |
605 | }, | |
606 | [ C(OP_WRITE) ] = { | |
607 | [ C(RESULT_ACCESS) ] = SNB_DMND_WRITE|SNB_DRAM_ANY, | |
608 | [ C(RESULT_MISS) ] = SNB_DMND_WRITE|SNB_DRAM_REMOTE, | |
609 | }, | |
610 | [ C(OP_PREFETCH) ] = { | |
611 | [ C(RESULT_ACCESS) ] = SNB_DMND_PREFETCH|SNB_DRAM_ANY, | |
612 | [ C(RESULT_MISS) ] = SNB_DMND_PREFETCH|SNB_DRAM_REMOTE, | |
613 | }, | |
614 | }, | |
615 | }; | |
616 | ||
b06b3d49 LM |
617 | static __initconst const u64 snb_hw_cache_event_ids |
618 | [PERF_COUNT_HW_CACHE_MAX] | |
619 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
620 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
621 | { | |
622 | [ C(L1D) ] = { | |
623 | [ C(OP_READ) ] = { | |
624 | [ C(RESULT_ACCESS) ] = 0xf1d0, /* MEM_UOP_RETIRED.LOADS */ | |
625 | [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPLACEMENT */ | |
626 | }, | |
627 | [ C(OP_WRITE) ] = { | |
628 | [ C(RESULT_ACCESS) ] = 0xf2d0, /* MEM_UOP_RETIRED.STORES */ | |
629 | [ C(RESULT_MISS) ] = 0x0851, /* L1D.ALL_M_REPLACEMENT */ | |
630 | }, | |
631 | [ C(OP_PREFETCH) ] = { | |
632 | [ C(RESULT_ACCESS) ] = 0x0, | |
633 | [ C(RESULT_MISS) ] = 0x024e, /* HW_PRE_REQ.DL1_MISS */ | |
634 | }, | |
635 | }, | |
636 | [ C(L1I ) ] = { | |
637 | [ C(OP_READ) ] = { | |
638 | [ C(RESULT_ACCESS) ] = 0x0, | |
639 | [ C(RESULT_MISS) ] = 0x0280, /* ICACHE.MISSES */ | |
640 | }, | |
641 | [ C(OP_WRITE) ] = { | |
642 | [ C(RESULT_ACCESS) ] = -1, | |
643 | [ C(RESULT_MISS) ] = -1, | |
644 | }, | |
645 | [ C(OP_PREFETCH) ] = { | |
646 | [ C(RESULT_ACCESS) ] = 0x0, | |
647 | [ C(RESULT_MISS) ] = 0x0, | |
648 | }, | |
649 | }, | |
650 | [ C(LL ) ] = { | |
b06b3d49 | 651 | [ C(OP_READ) ] = { |
63b6a675 | 652 | /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */ |
b06b3d49 | 653 | [ C(RESULT_ACCESS) ] = 0x01b7, |
63b6a675 PZ |
654 | /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */ |
655 | [ C(RESULT_MISS) ] = 0x01b7, | |
b06b3d49 LM |
656 | }, |
657 | [ C(OP_WRITE) ] = { | |
63b6a675 | 658 | /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */ |
b06b3d49 | 659 | [ C(RESULT_ACCESS) ] = 0x01b7, |
63b6a675 PZ |
660 | /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */ |
661 | [ C(RESULT_MISS) ] = 0x01b7, | |
b06b3d49 LM |
662 | }, |
663 | [ C(OP_PREFETCH) ] = { | |
63b6a675 | 664 | /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */ |
b06b3d49 | 665 | [ C(RESULT_ACCESS) ] = 0x01b7, |
63b6a675 PZ |
666 | /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */ |
667 | [ C(RESULT_MISS) ] = 0x01b7, | |
b06b3d49 LM |
668 | }, |
669 | }, | |
670 | [ C(DTLB) ] = { | |
671 | [ C(OP_READ) ] = { | |
672 | [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOP_RETIRED.ALL_LOADS */ | |
673 | [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.CAUSES_A_WALK */ | |
674 | }, | |
675 | [ C(OP_WRITE) ] = { | |
676 | [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOP_RETIRED.ALL_STORES */ | |
677 | [ C(RESULT_MISS) ] = 0x0149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */ | |
678 | }, | |
679 | [ C(OP_PREFETCH) ] = { | |
680 | [ C(RESULT_ACCESS) ] = 0x0, | |
681 | [ C(RESULT_MISS) ] = 0x0, | |
682 | }, | |
683 | }, | |
684 | [ C(ITLB) ] = { | |
685 | [ C(OP_READ) ] = { | |
686 | [ C(RESULT_ACCESS) ] = 0x1085, /* ITLB_MISSES.STLB_HIT */ | |
687 | [ C(RESULT_MISS) ] = 0x0185, /* ITLB_MISSES.CAUSES_A_WALK */ | |
688 | }, | |
689 | [ C(OP_WRITE) ] = { | |
690 | [ C(RESULT_ACCESS) ] = -1, | |
691 | [ C(RESULT_MISS) ] = -1, | |
692 | }, | |
693 | [ C(OP_PREFETCH) ] = { | |
694 | [ C(RESULT_ACCESS) ] = -1, | |
695 | [ C(RESULT_MISS) ] = -1, | |
696 | }, | |
697 | }, | |
698 | [ C(BPU ) ] = { | |
699 | [ C(OP_READ) ] = { | |
700 | [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ | |
701 | [ C(RESULT_MISS) ] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */ | |
702 | }, | |
703 | [ C(OP_WRITE) ] = { | |
704 | [ C(RESULT_ACCESS) ] = -1, | |
705 | [ C(RESULT_MISS) ] = -1, | |
706 | }, | |
707 | [ C(OP_PREFETCH) ] = { | |
708 | [ C(RESULT_ACCESS) ] = -1, | |
709 | [ C(RESULT_MISS) ] = -1, | |
710 | }, | |
711 | }, | |
89d6c0b5 PZ |
712 | [ C(NODE) ] = { |
713 | [ C(OP_READ) ] = { | |
74e6543f YZ |
714 | [ C(RESULT_ACCESS) ] = 0x01b7, |
715 | [ C(RESULT_MISS) ] = 0x01b7, | |
89d6c0b5 PZ |
716 | }, |
717 | [ C(OP_WRITE) ] = { | |
74e6543f YZ |
718 | [ C(RESULT_ACCESS) ] = 0x01b7, |
719 | [ C(RESULT_MISS) ] = 0x01b7, | |
89d6c0b5 PZ |
720 | }, |
721 | [ C(OP_PREFETCH) ] = { | |
74e6543f YZ |
722 | [ C(RESULT_ACCESS) ] = 0x01b7, |
723 | [ C(RESULT_MISS) ] = 0x01b7, | |
89d6c0b5 PZ |
724 | }, |
725 | }, | |
726 | ||
b06b3d49 LM |
727 | }; |
728 | ||
0f1b5ca2 AK |
729 | /* |
730 | * Notes on the events: | |
731 | * - data reads do not include code reads (comparable to earlier tables) | |
732 | * - data counts include speculative execution (except L1 write, dtlb, bpu) | |
733 | * - remote node access includes remote memory, remote cache, remote mmio. | |
734 | * - prefetches are not included in the counts because they are not | |
735 | * reliably counted. | |
736 | */ | |
737 | ||
738 | #define HSW_DEMAND_DATA_RD BIT_ULL(0) | |
739 | #define HSW_DEMAND_RFO BIT_ULL(1) | |
740 | #define HSW_ANY_RESPONSE BIT_ULL(16) | |
741 | #define HSW_SUPPLIER_NONE BIT_ULL(17) | |
742 | #define HSW_L3_MISS_LOCAL_DRAM BIT_ULL(22) | |
743 | #define HSW_L3_MISS_REMOTE_HOP0 BIT_ULL(27) | |
744 | #define HSW_L3_MISS_REMOTE_HOP1 BIT_ULL(28) | |
745 | #define HSW_L3_MISS_REMOTE_HOP2P BIT_ULL(29) | |
746 | #define HSW_L3_MISS (HSW_L3_MISS_LOCAL_DRAM| \ | |
747 | HSW_L3_MISS_REMOTE_HOP0|HSW_L3_MISS_REMOTE_HOP1| \ | |
748 | HSW_L3_MISS_REMOTE_HOP2P) | |
749 | #define HSW_SNOOP_NONE BIT_ULL(31) | |
750 | #define HSW_SNOOP_NOT_NEEDED BIT_ULL(32) | |
751 | #define HSW_SNOOP_MISS BIT_ULL(33) | |
752 | #define HSW_SNOOP_HIT_NO_FWD BIT_ULL(34) | |
753 | #define HSW_SNOOP_HIT_WITH_FWD BIT_ULL(35) | |
754 | #define HSW_SNOOP_HITM BIT_ULL(36) | |
755 | #define HSW_SNOOP_NON_DRAM BIT_ULL(37) | |
756 | #define HSW_ANY_SNOOP (HSW_SNOOP_NONE| \ | |
757 | HSW_SNOOP_NOT_NEEDED|HSW_SNOOP_MISS| \ | |
758 | HSW_SNOOP_HIT_NO_FWD|HSW_SNOOP_HIT_WITH_FWD| \ | |
759 | HSW_SNOOP_HITM|HSW_SNOOP_NON_DRAM) | |
760 | #define HSW_SNOOP_DRAM (HSW_ANY_SNOOP & ~HSW_SNOOP_NON_DRAM) | |
761 | #define HSW_DEMAND_READ HSW_DEMAND_DATA_RD | |
762 | #define HSW_DEMAND_WRITE HSW_DEMAND_RFO | |
763 | #define HSW_L3_MISS_REMOTE (HSW_L3_MISS_REMOTE_HOP0|\ | |
764 | HSW_L3_MISS_REMOTE_HOP1|HSW_L3_MISS_REMOTE_HOP2P) | |
765 | #define HSW_LLC_ACCESS HSW_ANY_RESPONSE | |
766 | ||
91f1b705 AK |
767 | #define BDW_L3_MISS_LOCAL BIT(26) |
768 | #define BDW_L3_MISS (BDW_L3_MISS_LOCAL| \ | |
769 | HSW_L3_MISS_REMOTE_HOP0|HSW_L3_MISS_REMOTE_HOP1| \ | |
770 | HSW_L3_MISS_REMOTE_HOP2P) | |
771 | ||
772 | ||
0f1b5ca2 AK |
773 | static __initconst const u64 hsw_hw_cache_event_ids |
774 | [PERF_COUNT_HW_CACHE_MAX] | |
775 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
776 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
777 | { | |
778 | [ C(L1D ) ] = { | |
779 | [ C(OP_READ) ] = { | |
780 | [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ | |
781 | [ C(RESULT_MISS) ] = 0x151, /* L1D.REPLACEMENT */ | |
782 | }, | |
783 | [ C(OP_WRITE) ] = { | |
784 | [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ | |
785 | [ C(RESULT_MISS) ] = 0x0, | |
786 | }, | |
787 | [ C(OP_PREFETCH) ] = { | |
788 | [ C(RESULT_ACCESS) ] = 0x0, | |
789 | [ C(RESULT_MISS) ] = 0x0, | |
790 | }, | |
791 | }, | |
792 | [ C(L1I ) ] = { | |
793 | [ C(OP_READ) ] = { | |
794 | [ C(RESULT_ACCESS) ] = 0x0, | |
795 | [ C(RESULT_MISS) ] = 0x280, /* ICACHE.MISSES */ | |
796 | }, | |
797 | [ C(OP_WRITE) ] = { | |
798 | [ C(RESULT_ACCESS) ] = -1, | |
799 | [ C(RESULT_MISS) ] = -1, | |
800 | }, | |
801 | [ C(OP_PREFETCH) ] = { | |
802 | [ C(RESULT_ACCESS) ] = 0x0, | |
803 | [ C(RESULT_MISS) ] = 0x0, | |
804 | }, | |
805 | }, | |
806 | [ C(LL ) ] = { | |
807 | [ C(OP_READ) ] = { | |
808 | [ C(RESULT_ACCESS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
809 | [ C(RESULT_MISS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
810 | }, | |
811 | [ C(OP_WRITE) ] = { | |
812 | [ C(RESULT_ACCESS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
813 | [ C(RESULT_MISS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
814 | }, | |
815 | [ C(OP_PREFETCH) ] = { | |
816 | [ C(RESULT_ACCESS) ] = 0x0, | |
817 | [ C(RESULT_MISS) ] = 0x0, | |
818 | }, | |
819 | }, | |
820 | [ C(DTLB) ] = { | |
821 | [ C(OP_READ) ] = { | |
822 | [ C(RESULT_ACCESS) ] = 0x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ | |
823 | [ C(RESULT_MISS) ] = 0x108, /* DTLB_LOAD_MISSES.MISS_CAUSES_A_WALK */ | |
824 | }, | |
825 | [ C(OP_WRITE) ] = { | |
826 | [ C(RESULT_ACCESS) ] = 0x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ | |
827 | [ C(RESULT_MISS) ] = 0x149, /* DTLB_STORE_MISSES.MISS_CAUSES_A_WALK */ | |
828 | }, | |
829 | [ C(OP_PREFETCH) ] = { | |
830 | [ C(RESULT_ACCESS) ] = 0x0, | |
831 | [ C(RESULT_MISS) ] = 0x0, | |
832 | }, | |
833 | }, | |
834 | [ C(ITLB) ] = { | |
835 | [ C(OP_READ) ] = { | |
836 | [ C(RESULT_ACCESS) ] = 0x6085, /* ITLB_MISSES.STLB_HIT */ | |
837 | [ C(RESULT_MISS) ] = 0x185, /* ITLB_MISSES.MISS_CAUSES_A_WALK */ | |
838 | }, | |
839 | [ C(OP_WRITE) ] = { | |
840 | [ C(RESULT_ACCESS) ] = -1, | |
841 | [ C(RESULT_MISS) ] = -1, | |
842 | }, | |
843 | [ C(OP_PREFETCH) ] = { | |
844 | [ C(RESULT_ACCESS) ] = -1, | |
845 | [ C(RESULT_MISS) ] = -1, | |
846 | }, | |
847 | }, | |
848 | [ C(BPU ) ] = { | |
849 | [ C(OP_READ) ] = { | |
850 | [ C(RESULT_ACCESS) ] = 0xc4, /* BR_INST_RETIRED.ALL_BRANCHES */ | |
851 | [ C(RESULT_MISS) ] = 0xc5, /* BR_MISP_RETIRED.ALL_BRANCHES */ | |
852 | }, | |
853 | [ C(OP_WRITE) ] = { | |
854 | [ C(RESULT_ACCESS) ] = -1, | |
855 | [ C(RESULT_MISS) ] = -1, | |
856 | }, | |
857 | [ C(OP_PREFETCH) ] = { | |
858 | [ C(RESULT_ACCESS) ] = -1, | |
859 | [ C(RESULT_MISS) ] = -1, | |
860 | }, | |
861 | }, | |
862 | [ C(NODE) ] = { | |
863 | [ C(OP_READ) ] = { | |
864 | [ C(RESULT_ACCESS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
865 | [ C(RESULT_MISS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
866 | }, | |
867 | [ C(OP_WRITE) ] = { | |
868 | [ C(RESULT_ACCESS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
869 | [ C(RESULT_MISS) ] = 0x1b7, /* OFFCORE_RESPONSE */ | |
870 | }, | |
871 | [ C(OP_PREFETCH) ] = { | |
872 | [ C(RESULT_ACCESS) ] = 0x0, | |
873 | [ C(RESULT_MISS) ] = 0x0, | |
874 | }, | |
875 | }, | |
876 | }; | |
877 | ||
878 | static __initconst const u64 hsw_hw_cache_extra_regs | |
879 | [PERF_COUNT_HW_CACHE_MAX] | |
880 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
881 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
882 | { | |
883 | [ C(LL ) ] = { | |
884 | [ C(OP_READ) ] = { | |
885 | [ C(RESULT_ACCESS) ] = HSW_DEMAND_READ| | |
886 | HSW_LLC_ACCESS, | |
887 | [ C(RESULT_MISS) ] = HSW_DEMAND_READ| | |
888 | HSW_L3_MISS|HSW_ANY_SNOOP, | |
889 | }, | |
890 | [ C(OP_WRITE) ] = { | |
891 | [ C(RESULT_ACCESS) ] = HSW_DEMAND_WRITE| | |
892 | HSW_LLC_ACCESS, | |
893 | [ C(RESULT_MISS) ] = HSW_DEMAND_WRITE| | |
894 | HSW_L3_MISS|HSW_ANY_SNOOP, | |
895 | }, | |
896 | [ C(OP_PREFETCH) ] = { | |
897 | [ C(RESULT_ACCESS) ] = 0x0, | |
898 | [ C(RESULT_MISS) ] = 0x0, | |
899 | }, | |
900 | }, | |
901 | [ C(NODE) ] = { | |
902 | [ C(OP_READ) ] = { | |
903 | [ C(RESULT_ACCESS) ] = HSW_DEMAND_READ| | |
904 | HSW_L3_MISS_LOCAL_DRAM| | |
905 | HSW_SNOOP_DRAM, | |
906 | [ C(RESULT_MISS) ] = HSW_DEMAND_READ| | |
907 | HSW_L3_MISS_REMOTE| | |
908 | HSW_SNOOP_DRAM, | |
909 | }, | |
910 | [ C(OP_WRITE) ] = { | |
911 | [ C(RESULT_ACCESS) ] = HSW_DEMAND_WRITE| | |
912 | HSW_L3_MISS_LOCAL_DRAM| | |
913 | HSW_SNOOP_DRAM, | |
914 | [ C(RESULT_MISS) ] = HSW_DEMAND_WRITE| | |
915 | HSW_L3_MISS_REMOTE| | |
916 | HSW_SNOOP_DRAM, | |
917 | }, | |
918 | [ C(OP_PREFETCH) ] = { | |
919 | [ C(RESULT_ACCESS) ] = 0x0, | |
920 | [ C(RESULT_MISS) ] = 0x0, | |
921 | }, | |
922 | }, | |
923 | }; | |
924 | ||
caaa8be3 | 925 | static __initconst const u64 westmere_hw_cache_event_ids |
f22f54f4 PZ |
926 | [PERF_COUNT_HW_CACHE_MAX] |
927 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
928 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
929 | { | |
930 | [ C(L1D) ] = { | |
931 | [ C(OP_READ) ] = { | |
932 | [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */ | |
933 | [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPL */ | |
934 | }, | |
935 | [ C(OP_WRITE) ] = { | |
936 | [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */ | |
937 | [ C(RESULT_MISS) ] = 0x0251, /* L1D.M_REPL */ | |
938 | }, | |
939 | [ C(OP_PREFETCH) ] = { | |
940 | [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */ | |
941 | [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */ | |
942 | }, | |
943 | }, | |
944 | [ C(L1I ) ] = { | |
945 | [ C(OP_READ) ] = { | |
946 | [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */ | |
947 | [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */ | |
948 | }, | |
949 | [ C(OP_WRITE) ] = { | |
950 | [ C(RESULT_ACCESS) ] = -1, | |
951 | [ C(RESULT_MISS) ] = -1, | |
952 | }, | |
953 | [ C(OP_PREFETCH) ] = { | |
954 | [ C(RESULT_ACCESS) ] = 0x0, | |
955 | [ C(RESULT_MISS) ] = 0x0, | |
956 | }, | |
957 | }, | |
958 | [ C(LL ) ] = { | |
959 | [ C(OP_READ) ] = { | |
63b6a675 | 960 | /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */ |
e994d7d2 | 961 | [ C(RESULT_ACCESS) ] = 0x01b7, |
63b6a675 PZ |
962 | /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */ |
963 | [ C(RESULT_MISS) ] = 0x01b7, | |
f22f54f4 | 964 | }, |
e994d7d2 AK |
965 | /* |
966 | * Use RFO, not WRITEBACK, because a write miss would typically occur | |
967 | * on RFO. | |
968 | */ | |
f22f54f4 | 969 | [ C(OP_WRITE) ] = { |
63b6a675 PZ |
970 | /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */ |
971 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
972 | /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */ | |
e994d7d2 | 973 | [ C(RESULT_MISS) ] = 0x01b7, |
f22f54f4 PZ |
974 | }, |
975 | [ C(OP_PREFETCH) ] = { | |
63b6a675 | 976 | /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */ |
e994d7d2 | 977 | [ C(RESULT_ACCESS) ] = 0x01b7, |
63b6a675 PZ |
978 | /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */ |
979 | [ C(RESULT_MISS) ] = 0x01b7, | |
f22f54f4 PZ |
980 | }, |
981 | }, | |
982 | [ C(DTLB) ] = { | |
983 | [ C(OP_READ) ] = { | |
984 | [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */ | |
985 | [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */ | |
986 | }, | |
987 | [ C(OP_WRITE) ] = { | |
988 | [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */ | |
989 | [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */ | |
990 | }, | |
991 | [ C(OP_PREFETCH) ] = { | |
992 | [ C(RESULT_ACCESS) ] = 0x0, | |
993 | [ C(RESULT_MISS) ] = 0x0, | |
994 | }, | |
995 | }, | |
996 | [ C(ITLB) ] = { | |
997 | [ C(OP_READ) ] = { | |
998 | [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */ | |
999 | [ C(RESULT_MISS) ] = 0x0185, /* ITLB_MISSES.ANY */ | |
1000 | }, | |
1001 | [ C(OP_WRITE) ] = { | |
1002 | [ C(RESULT_ACCESS) ] = -1, | |
1003 | [ C(RESULT_MISS) ] = -1, | |
1004 | }, | |
1005 | [ C(OP_PREFETCH) ] = { | |
1006 | [ C(RESULT_ACCESS) ] = -1, | |
1007 | [ C(RESULT_MISS) ] = -1, | |
1008 | }, | |
1009 | }, | |
1010 | [ C(BPU ) ] = { | |
1011 | [ C(OP_READ) ] = { | |
1012 | [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ | |
1013 | [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */ | |
1014 | }, | |
1015 | [ C(OP_WRITE) ] = { | |
1016 | [ C(RESULT_ACCESS) ] = -1, | |
1017 | [ C(RESULT_MISS) ] = -1, | |
1018 | }, | |
1019 | [ C(OP_PREFETCH) ] = { | |
1020 | [ C(RESULT_ACCESS) ] = -1, | |
1021 | [ C(RESULT_MISS) ] = -1, | |
1022 | }, | |
1023 | }, | |
89d6c0b5 PZ |
1024 | [ C(NODE) ] = { |
1025 | [ C(OP_READ) ] = { | |
1026 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1027 | [ C(RESULT_MISS) ] = 0x01b7, | |
1028 | }, | |
1029 | [ C(OP_WRITE) ] = { | |
1030 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1031 | [ C(RESULT_MISS) ] = 0x01b7, | |
1032 | }, | |
1033 | [ C(OP_PREFETCH) ] = { | |
1034 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1035 | [ C(RESULT_MISS) ] = 0x01b7, | |
1036 | }, | |
1037 | }, | |
f22f54f4 PZ |
1038 | }; |
1039 | ||
e994d7d2 | 1040 | /* |
63b6a675 PZ |
1041 | * Nehalem/Westmere MSR_OFFCORE_RESPONSE bits; |
1042 | * See IA32 SDM Vol 3B 30.6.1.3 | |
e994d7d2 AK |
1043 | */ |
1044 | ||
63b6a675 PZ |
1045 | #define NHM_DMND_DATA_RD (1 << 0) |
1046 | #define NHM_DMND_RFO (1 << 1) | |
1047 | #define NHM_DMND_IFETCH (1 << 2) | |
1048 | #define NHM_DMND_WB (1 << 3) | |
1049 | #define NHM_PF_DATA_RD (1 << 4) | |
1050 | #define NHM_PF_DATA_RFO (1 << 5) | |
1051 | #define NHM_PF_IFETCH (1 << 6) | |
1052 | #define NHM_OFFCORE_OTHER (1 << 7) | |
1053 | #define NHM_UNCORE_HIT (1 << 8) | |
1054 | #define NHM_OTHER_CORE_HIT_SNP (1 << 9) | |
1055 | #define NHM_OTHER_CORE_HITM (1 << 10) | |
1056 | /* reserved */ | |
1057 | #define NHM_REMOTE_CACHE_FWD (1 << 12) | |
1058 | #define NHM_REMOTE_DRAM (1 << 13) | |
1059 | #define NHM_LOCAL_DRAM (1 << 14) | |
1060 | #define NHM_NON_DRAM (1 << 15) | |
1061 | ||
87e24f4b PZ |
1062 | #define NHM_LOCAL (NHM_LOCAL_DRAM|NHM_REMOTE_CACHE_FWD) |
1063 | #define NHM_REMOTE (NHM_REMOTE_DRAM) | |
63b6a675 PZ |
1064 | |
1065 | #define NHM_DMND_READ (NHM_DMND_DATA_RD) | |
1066 | #define NHM_DMND_WRITE (NHM_DMND_RFO|NHM_DMND_WB) | |
1067 | #define NHM_DMND_PREFETCH (NHM_PF_DATA_RD|NHM_PF_DATA_RFO) | |
1068 | ||
1069 | #define NHM_L3_HIT (NHM_UNCORE_HIT|NHM_OTHER_CORE_HIT_SNP|NHM_OTHER_CORE_HITM) | |
87e24f4b | 1070 | #define NHM_L3_MISS (NHM_NON_DRAM|NHM_LOCAL_DRAM|NHM_REMOTE_DRAM|NHM_REMOTE_CACHE_FWD) |
63b6a675 | 1071 | #define NHM_L3_ACCESS (NHM_L3_HIT|NHM_L3_MISS) |
e994d7d2 AK |
1072 | |
1073 | static __initconst const u64 nehalem_hw_cache_extra_regs | |
1074 | [PERF_COUNT_HW_CACHE_MAX] | |
1075 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1076 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
1077 | { | |
1078 | [ C(LL ) ] = { | |
1079 | [ C(OP_READ) ] = { | |
63b6a675 PZ |
1080 | [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_L3_ACCESS, |
1081 | [ C(RESULT_MISS) ] = NHM_DMND_READ|NHM_L3_MISS, | |
e994d7d2 AK |
1082 | }, |
1083 | [ C(OP_WRITE) ] = { | |
63b6a675 PZ |
1084 | [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_L3_ACCESS, |
1085 | [ C(RESULT_MISS) ] = NHM_DMND_WRITE|NHM_L3_MISS, | |
e994d7d2 AK |
1086 | }, |
1087 | [ C(OP_PREFETCH) ] = { | |
63b6a675 PZ |
1088 | [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_L3_ACCESS, |
1089 | [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_L3_MISS, | |
e994d7d2 | 1090 | }, |
89d6c0b5 PZ |
1091 | }, |
1092 | [ C(NODE) ] = { | |
1093 | [ C(OP_READ) ] = { | |
87e24f4b PZ |
1094 | [ C(RESULT_ACCESS) ] = NHM_DMND_READ|NHM_LOCAL|NHM_REMOTE, |
1095 | [ C(RESULT_MISS) ] = NHM_DMND_READ|NHM_REMOTE, | |
89d6c0b5 PZ |
1096 | }, |
1097 | [ C(OP_WRITE) ] = { | |
87e24f4b PZ |
1098 | [ C(RESULT_ACCESS) ] = NHM_DMND_WRITE|NHM_LOCAL|NHM_REMOTE, |
1099 | [ C(RESULT_MISS) ] = NHM_DMND_WRITE|NHM_REMOTE, | |
89d6c0b5 PZ |
1100 | }, |
1101 | [ C(OP_PREFETCH) ] = { | |
87e24f4b PZ |
1102 | [ C(RESULT_ACCESS) ] = NHM_DMND_PREFETCH|NHM_LOCAL|NHM_REMOTE, |
1103 | [ C(RESULT_MISS) ] = NHM_DMND_PREFETCH|NHM_REMOTE, | |
89d6c0b5 PZ |
1104 | }, |
1105 | }, | |
e994d7d2 AK |
1106 | }; |
1107 | ||
caaa8be3 | 1108 | static __initconst const u64 nehalem_hw_cache_event_ids |
f22f54f4 PZ |
1109 | [PERF_COUNT_HW_CACHE_MAX] |
1110 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1111 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
1112 | { | |
1113 | [ C(L1D) ] = { | |
1114 | [ C(OP_READ) ] = { | |
f4929bd3 PZ |
1115 | [ C(RESULT_ACCESS) ] = 0x010b, /* MEM_INST_RETIRED.LOADS */ |
1116 | [ C(RESULT_MISS) ] = 0x0151, /* L1D.REPL */ | |
f22f54f4 PZ |
1117 | }, |
1118 | [ C(OP_WRITE) ] = { | |
f4929bd3 PZ |
1119 | [ C(RESULT_ACCESS) ] = 0x020b, /* MEM_INST_RETURED.STORES */ |
1120 | [ C(RESULT_MISS) ] = 0x0251, /* L1D.M_REPL */ | |
f22f54f4 PZ |
1121 | }, |
1122 | [ C(OP_PREFETCH) ] = { | |
1123 | [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS */ | |
1124 | [ C(RESULT_MISS) ] = 0x024e, /* L1D_PREFETCH.MISS */ | |
1125 | }, | |
1126 | }, | |
1127 | [ C(L1I ) ] = { | |
1128 | [ C(OP_READ) ] = { | |
1129 | [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */ | |
1130 | [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */ | |
1131 | }, | |
1132 | [ C(OP_WRITE) ] = { | |
1133 | [ C(RESULT_ACCESS) ] = -1, | |
1134 | [ C(RESULT_MISS) ] = -1, | |
1135 | }, | |
1136 | [ C(OP_PREFETCH) ] = { | |
1137 | [ C(RESULT_ACCESS) ] = 0x0, | |
1138 | [ C(RESULT_MISS) ] = 0x0, | |
1139 | }, | |
1140 | }, | |
1141 | [ C(LL ) ] = { | |
1142 | [ C(OP_READ) ] = { | |
e994d7d2 AK |
1143 | /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */ |
1144 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1145 | /* OFFCORE_RESPONSE.ANY_DATA.ANY_LLC_MISS */ | |
1146 | [ C(RESULT_MISS) ] = 0x01b7, | |
f22f54f4 | 1147 | }, |
e994d7d2 AK |
1148 | /* |
1149 | * Use RFO, not WRITEBACK, because a write miss would typically occur | |
1150 | * on RFO. | |
1151 | */ | |
f22f54f4 | 1152 | [ C(OP_WRITE) ] = { |
e994d7d2 AK |
1153 | /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */ |
1154 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1155 | /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */ | |
1156 | [ C(RESULT_MISS) ] = 0x01b7, | |
f22f54f4 PZ |
1157 | }, |
1158 | [ C(OP_PREFETCH) ] = { | |
e994d7d2 AK |
1159 | /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */ |
1160 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1161 | /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */ | |
1162 | [ C(RESULT_MISS) ] = 0x01b7, | |
f22f54f4 PZ |
1163 | }, |
1164 | }, | |
1165 | [ C(DTLB) ] = { | |
1166 | [ C(OP_READ) ] = { | |
1167 | [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */ | |
1168 | [ C(RESULT_MISS) ] = 0x0108, /* DTLB_LOAD_MISSES.ANY */ | |
1169 | }, | |
1170 | [ C(OP_WRITE) ] = { | |
1171 | [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */ | |
1172 | [ C(RESULT_MISS) ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS */ | |
1173 | }, | |
1174 | [ C(OP_PREFETCH) ] = { | |
1175 | [ C(RESULT_ACCESS) ] = 0x0, | |
1176 | [ C(RESULT_MISS) ] = 0x0, | |
1177 | }, | |
1178 | }, | |
1179 | [ C(ITLB) ] = { | |
1180 | [ C(OP_READ) ] = { | |
1181 | [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P */ | |
1182 | [ C(RESULT_MISS) ] = 0x20c8, /* ITLB_MISS_RETIRED */ | |
1183 | }, | |
1184 | [ C(OP_WRITE) ] = { | |
1185 | [ C(RESULT_ACCESS) ] = -1, | |
1186 | [ C(RESULT_MISS) ] = -1, | |
1187 | }, | |
1188 | [ C(OP_PREFETCH) ] = { | |
1189 | [ C(RESULT_ACCESS) ] = -1, | |
1190 | [ C(RESULT_MISS) ] = -1, | |
1191 | }, | |
1192 | }, | |
1193 | [ C(BPU ) ] = { | |
1194 | [ C(OP_READ) ] = { | |
1195 | [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ | |
1196 | [ C(RESULT_MISS) ] = 0x03e8, /* BPU_CLEARS.ANY */ | |
1197 | }, | |
1198 | [ C(OP_WRITE) ] = { | |
1199 | [ C(RESULT_ACCESS) ] = -1, | |
1200 | [ C(RESULT_MISS) ] = -1, | |
1201 | }, | |
1202 | [ C(OP_PREFETCH) ] = { | |
1203 | [ C(RESULT_ACCESS) ] = -1, | |
1204 | [ C(RESULT_MISS) ] = -1, | |
1205 | }, | |
1206 | }, | |
89d6c0b5 PZ |
1207 | [ C(NODE) ] = { |
1208 | [ C(OP_READ) ] = { | |
1209 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1210 | [ C(RESULT_MISS) ] = 0x01b7, | |
1211 | }, | |
1212 | [ C(OP_WRITE) ] = { | |
1213 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1214 | [ C(RESULT_MISS) ] = 0x01b7, | |
1215 | }, | |
1216 | [ C(OP_PREFETCH) ] = { | |
1217 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1218 | [ C(RESULT_MISS) ] = 0x01b7, | |
1219 | }, | |
1220 | }, | |
f22f54f4 PZ |
1221 | }; |
1222 | ||
caaa8be3 | 1223 | static __initconst const u64 core2_hw_cache_event_ids |
f22f54f4 PZ |
1224 | [PERF_COUNT_HW_CACHE_MAX] |
1225 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1226 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
1227 | { | |
1228 | [ C(L1D) ] = { | |
1229 | [ C(OP_READ) ] = { | |
1230 | [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI */ | |
1231 | [ C(RESULT_MISS) ] = 0x0140, /* L1D_CACHE_LD.I_STATE */ | |
1232 | }, | |
1233 | [ C(OP_WRITE) ] = { | |
1234 | [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI */ | |
1235 | [ C(RESULT_MISS) ] = 0x0141, /* L1D_CACHE_ST.I_STATE */ | |
1236 | }, | |
1237 | [ C(OP_PREFETCH) ] = { | |
1238 | [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS */ | |
1239 | [ C(RESULT_MISS) ] = 0, | |
1240 | }, | |
1241 | }, | |
1242 | [ C(L1I ) ] = { | |
1243 | [ C(OP_READ) ] = { | |
1244 | [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS */ | |
1245 | [ C(RESULT_MISS) ] = 0x0081, /* L1I.MISSES */ | |
1246 | }, | |
1247 | [ C(OP_WRITE) ] = { | |
1248 | [ C(RESULT_ACCESS) ] = -1, | |
1249 | [ C(RESULT_MISS) ] = -1, | |
1250 | }, | |
1251 | [ C(OP_PREFETCH) ] = { | |
1252 | [ C(RESULT_ACCESS) ] = 0, | |
1253 | [ C(RESULT_MISS) ] = 0, | |
1254 | }, | |
1255 | }, | |
1256 | [ C(LL ) ] = { | |
1257 | [ C(OP_READ) ] = { | |
1258 | [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */ | |
1259 | [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */ | |
1260 | }, | |
1261 | [ C(OP_WRITE) ] = { | |
1262 | [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */ | |
1263 | [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */ | |
1264 | }, | |
1265 | [ C(OP_PREFETCH) ] = { | |
1266 | [ C(RESULT_ACCESS) ] = 0, | |
1267 | [ C(RESULT_MISS) ] = 0, | |
1268 | }, | |
1269 | }, | |
1270 | [ C(DTLB) ] = { | |
1271 | [ C(OP_READ) ] = { | |
1272 | [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI (alias) */ | |
1273 | [ C(RESULT_MISS) ] = 0x0208, /* DTLB_MISSES.MISS_LD */ | |
1274 | }, | |
1275 | [ C(OP_WRITE) ] = { | |
1276 | [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI (alias) */ | |
1277 | [ C(RESULT_MISS) ] = 0x0808, /* DTLB_MISSES.MISS_ST */ | |
1278 | }, | |
1279 | [ C(OP_PREFETCH) ] = { | |
1280 | [ C(RESULT_ACCESS) ] = 0, | |
1281 | [ C(RESULT_MISS) ] = 0, | |
1282 | }, | |
1283 | }, | |
1284 | [ C(ITLB) ] = { | |
1285 | [ C(OP_READ) ] = { | |
1286 | [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */ | |
1287 | [ C(RESULT_MISS) ] = 0x1282, /* ITLBMISSES */ | |
1288 | }, | |
1289 | [ C(OP_WRITE) ] = { | |
1290 | [ C(RESULT_ACCESS) ] = -1, | |
1291 | [ C(RESULT_MISS) ] = -1, | |
1292 | }, | |
1293 | [ C(OP_PREFETCH) ] = { | |
1294 | [ C(RESULT_ACCESS) ] = -1, | |
1295 | [ C(RESULT_MISS) ] = -1, | |
1296 | }, | |
1297 | }, | |
1298 | [ C(BPU ) ] = { | |
1299 | [ C(OP_READ) ] = { | |
1300 | [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */ | |
1301 | [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */ | |
1302 | }, | |
1303 | [ C(OP_WRITE) ] = { | |
1304 | [ C(RESULT_ACCESS) ] = -1, | |
1305 | [ C(RESULT_MISS) ] = -1, | |
1306 | }, | |
1307 | [ C(OP_PREFETCH) ] = { | |
1308 | [ C(RESULT_ACCESS) ] = -1, | |
1309 | [ C(RESULT_MISS) ] = -1, | |
1310 | }, | |
1311 | }, | |
1312 | }; | |
1313 | ||
caaa8be3 | 1314 | static __initconst const u64 atom_hw_cache_event_ids |
f22f54f4 PZ |
1315 | [PERF_COUNT_HW_CACHE_MAX] |
1316 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1317 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
1318 | { | |
1319 | [ C(L1D) ] = { | |
1320 | [ C(OP_READ) ] = { | |
1321 | [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD */ | |
1322 | [ C(RESULT_MISS) ] = 0, | |
1323 | }, | |
1324 | [ C(OP_WRITE) ] = { | |
1325 | [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST */ | |
1326 | [ C(RESULT_MISS) ] = 0, | |
1327 | }, | |
1328 | [ C(OP_PREFETCH) ] = { | |
1329 | [ C(RESULT_ACCESS) ] = 0x0, | |
1330 | [ C(RESULT_MISS) ] = 0, | |
1331 | }, | |
1332 | }, | |
1333 | [ C(L1I ) ] = { | |
1334 | [ C(OP_READ) ] = { | |
1335 | [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS */ | |
1336 | [ C(RESULT_MISS) ] = 0x0280, /* L1I.MISSES */ | |
1337 | }, | |
1338 | [ C(OP_WRITE) ] = { | |
1339 | [ C(RESULT_ACCESS) ] = -1, | |
1340 | [ C(RESULT_MISS) ] = -1, | |
1341 | }, | |
1342 | [ C(OP_PREFETCH) ] = { | |
1343 | [ C(RESULT_ACCESS) ] = 0, | |
1344 | [ C(RESULT_MISS) ] = 0, | |
1345 | }, | |
1346 | }, | |
1347 | [ C(LL ) ] = { | |
1348 | [ C(OP_READ) ] = { | |
1349 | [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI */ | |
1350 | [ C(RESULT_MISS) ] = 0x4129, /* L2_LD.ISTATE */ | |
1351 | }, | |
1352 | [ C(OP_WRITE) ] = { | |
1353 | [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI */ | |
1354 | [ C(RESULT_MISS) ] = 0x412A, /* L2_ST.ISTATE */ | |
1355 | }, | |
1356 | [ C(OP_PREFETCH) ] = { | |
1357 | [ C(RESULT_ACCESS) ] = 0, | |
1358 | [ C(RESULT_MISS) ] = 0, | |
1359 | }, | |
1360 | }, | |
1361 | [ C(DTLB) ] = { | |
1362 | [ C(OP_READ) ] = { | |
1363 | [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI (alias) */ | |
1364 | [ C(RESULT_MISS) ] = 0x0508, /* DTLB_MISSES.MISS_LD */ | |
1365 | }, | |
1366 | [ C(OP_WRITE) ] = { | |
1367 | [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI (alias) */ | |
1368 | [ C(RESULT_MISS) ] = 0x0608, /* DTLB_MISSES.MISS_ST */ | |
1369 | }, | |
1370 | [ C(OP_PREFETCH) ] = { | |
1371 | [ C(RESULT_ACCESS) ] = 0, | |
1372 | [ C(RESULT_MISS) ] = 0, | |
1373 | }, | |
1374 | }, | |
1375 | [ C(ITLB) ] = { | |
1376 | [ C(OP_READ) ] = { | |
1377 | [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */ | |
1378 | [ C(RESULT_MISS) ] = 0x0282, /* ITLB.MISSES */ | |
1379 | }, | |
1380 | [ C(OP_WRITE) ] = { | |
1381 | [ C(RESULT_ACCESS) ] = -1, | |
1382 | [ C(RESULT_MISS) ] = -1, | |
1383 | }, | |
1384 | [ C(OP_PREFETCH) ] = { | |
1385 | [ C(RESULT_ACCESS) ] = -1, | |
1386 | [ C(RESULT_MISS) ] = -1, | |
1387 | }, | |
1388 | }, | |
1389 | [ C(BPU ) ] = { | |
1390 | [ C(OP_READ) ] = { | |
1391 | [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */ | |
1392 | [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */ | |
1393 | }, | |
1394 | [ C(OP_WRITE) ] = { | |
1395 | [ C(RESULT_ACCESS) ] = -1, | |
1396 | [ C(RESULT_MISS) ] = -1, | |
1397 | }, | |
1398 | [ C(OP_PREFETCH) ] = { | |
1399 | [ C(RESULT_ACCESS) ] = -1, | |
1400 | [ C(RESULT_MISS) ] = -1, | |
1401 | }, | |
1402 | }, | |
1403 | }; | |
1404 | ||
eb12b8ec AK |
1405 | EVENT_ATTR_STR(topdown-total-slots, td_total_slots_slm, "event=0x3c"); |
1406 | EVENT_ATTR_STR(topdown-total-slots.scale, td_total_slots_scale_slm, "2"); | |
1407 | /* no_alloc_cycles.not_delivered */ | |
1408 | EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles_slm, | |
1409 | "event=0xca,umask=0x50"); | |
1410 | EVENT_ATTR_STR(topdown-fetch-bubbles.scale, td_fetch_bubbles_scale_slm, "2"); | |
1411 | /* uops_retired.all */ | |
1412 | EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued_slm, | |
1413 | "event=0xc2,umask=0x10"); | |
1414 | /* uops_retired.all */ | |
1415 | EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired_slm, | |
1416 | "event=0xc2,umask=0x10"); | |
1417 | ||
1418 | static struct attribute *slm_events_attrs[] = { | |
1419 | EVENT_PTR(td_total_slots_slm), | |
1420 | EVENT_PTR(td_total_slots_scale_slm), | |
1421 | EVENT_PTR(td_fetch_bubbles_slm), | |
1422 | EVENT_PTR(td_fetch_bubbles_scale_slm), | |
1423 | EVENT_PTR(td_slots_issued_slm), | |
1424 | EVENT_PTR(td_slots_retired_slm), | |
1425 | NULL | |
1426 | }; | |
1427 | ||
1fa64180 YZ |
1428 | static struct extra_reg intel_slm_extra_regs[] __read_mostly = |
1429 | { | |
1430 | /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ | |
06c939c1 | 1431 | INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x768005ffffull, RSP_0), |
ae3f011f | 1432 | INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x368005ffffull, RSP_1), |
1fa64180 YZ |
1433 | EVENT_EXTRA_END |
1434 | }; | |
1435 | ||
1436 | #define SLM_DMND_READ SNB_DMND_DATA_RD | |
1437 | #define SLM_DMND_WRITE SNB_DMND_RFO | |
1438 | #define SLM_DMND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO) | |
1439 | ||
1440 | #define SLM_SNP_ANY (SNB_SNP_NONE|SNB_SNP_MISS|SNB_NO_FWD|SNB_HITM) | |
1441 | #define SLM_LLC_ACCESS SNB_RESP_ANY | |
1442 | #define SLM_LLC_MISS (SLM_SNP_ANY|SNB_NON_DRAM) | |
1443 | ||
1444 | static __initconst const u64 slm_hw_cache_extra_regs | |
1445 | [PERF_COUNT_HW_CACHE_MAX] | |
1446 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1447 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
1448 | { | |
1449 | [ C(LL ) ] = { | |
1450 | [ C(OP_READ) ] = { | |
1451 | [ C(RESULT_ACCESS) ] = SLM_DMND_READ|SLM_LLC_ACCESS, | |
6d374056 | 1452 | [ C(RESULT_MISS) ] = 0, |
1fa64180 YZ |
1453 | }, |
1454 | [ C(OP_WRITE) ] = { | |
1455 | [ C(RESULT_ACCESS) ] = SLM_DMND_WRITE|SLM_LLC_ACCESS, | |
1456 | [ C(RESULT_MISS) ] = SLM_DMND_WRITE|SLM_LLC_MISS, | |
1457 | }, | |
1458 | [ C(OP_PREFETCH) ] = { | |
1459 | [ C(RESULT_ACCESS) ] = SLM_DMND_PREFETCH|SLM_LLC_ACCESS, | |
1460 | [ C(RESULT_MISS) ] = SLM_DMND_PREFETCH|SLM_LLC_MISS, | |
1461 | }, | |
1462 | }, | |
1463 | }; | |
1464 | ||
1465 | static __initconst const u64 slm_hw_cache_event_ids | |
1466 | [PERF_COUNT_HW_CACHE_MAX] | |
1467 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1468 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = | |
1469 | { | |
1470 | [ C(L1D) ] = { | |
1471 | [ C(OP_READ) ] = { | |
1472 | [ C(RESULT_ACCESS) ] = 0, | |
1473 | [ C(RESULT_MISS) ] = 0x0104, /* LD_DCU_MISS */ | |
1474 | }, | |
1475 | [ C(OP_WRITE) ] = { | |
1476 | [ C(RESULT_ACCESS) ] = 0, | |
1477 | [ C(RESULT_MISS) ] = 0, | |
1478 | }, | |
1479 | [ C(OP_PREFETCH) ] = { | |
1480 | [ C(RESULT_ACCESS) ] = 0, | |
1481 | [ C(RESULT_MISS) ] = 0, | |
1482 | }, | |
1483 | }, | |
1484 | [ C(L1I ) ] = { | |
1485 | [ C(OP_READ) ] = { | |
1486 | [ C(RESULT_ACCESS) ] = 0x0380, /* ICACHE.ACCESSES */ | |
1487 | [ C(RESULT_MISS) ] = 0x0280, /* ICACGE.MISSES */ | |
1488 | }, | |
1489 | [ C(OP_WRITE) ] = { | |
1490 | [ C(RESULT_ACCESS) ] = -1, | |
1491 | [ C(RESULT_MISS) ] = -1, | |
1492 | }, | |
1493 | [ C(OP_PREFETCH) ] = { | |
1494 | [ C(RESULT_ACCESS) ] = 0, | |
1495 | [ C(RESULT_MISS) ] = 0, | |
1496 | }, | |
1497 | }, | |
1498 | [ C(LL ) ] = { | |
1499 | [ C(OP_READ) ] = { | |
1500 | /* OFFCORE_RESPONSE.ANY_DATA.LOCAL_CACHE */ | |
1501 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
6d374056 | 1502 | [ C(RESULT_MISS) ] = 0, |
1fa64180 YZ |
1503 | }, |
1504 | [ C(OP_WRITE) ] = { | |
1505 | /* OFFCORE_RESPONSE.ANY_RFO.LOCAL_CACHE */ | |
1506 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1507 | /* OFFCORE_RESPONSE.ANY_RFO.ANY_LLC_MISS */ | |
1508 | [ C(RESULT_MISS) ] = 0x01b7, | |
1509 | }, | |
1510 | [ C(OP_PREFETCH) ] = { | |
1511 | /* OFFCORE_RESPONSE.PREFETCH.LOCAL_CACHE */ | |
1512 | [ C(RESULT_ACCESS) ] = 0x01b7, | |
1513 | /* OFFCORE_RESPONSE.PREFETCH.ANY_LLC_MISS */ | |
1514 | [ C(RESULT_MISS) ] = 0x01b7, | |
1515 | }, | |
1516 | }, | |
1517 | [ C(DTLB) ] = { | |
1518 | [ C(OP_READ) ] = { | |
1519 | [ C(RESULT_ACCESS) ] = 0, | |
1520 | [ C(RESULT_MISS) ] = 0x0804, /* LD_DTLB_MISS */ | |
1521 | }, | |
1522 | [ C(OP_WRITE) ] = { | |
1523 | [ C(RESULT_ACCESS) ] = 0, | |
1524 | [ C(RESULT_MISS) ] = 0, | |
1525 | }, | |
1526 | [ C(OP_PREFETCH) ] = { | |
1527 | [ C(RESULT_ACCESS) ] = 0, | |
1528 | [ C(RESULT_MISS) ] = 0, | |
1529 | }, | |
1530 | }, | |
1531 | [ C(ITLB) ] = { | |
1532 | [ C(OP_READ) ] = { | |
1533 | [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P */ | |
6d374056 | 1534 | [ C(RESULT_MISS) ] = 0x40205, /* PAGE_WALKS.I_SIDE_WALKS */ |
1fa64180 YZ |
1535 | }, |
1536 | [ C(OP_WRITE) ] = { | |
1537 | [ C(RESULT_ACCESS) ] = -1, | |
1538 | [ C(RESULT_MISS) ] = -1, | |
1539 | }, | |
1540 | [ C(OP_PREFETCH) ] = { | |
1541 | [ C(RESULT_ACCESS) ] = -1, | |
1542 | [ C(RESULT_MISS) ] = -1, | |
1543 | }, | |
1544 | }, | |
1545 | [ C(BPU ) ] = { | |
1546 | [ C(OP_READ) ] = { | |
1547 | [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY */ | |
1548 | [ C(RESULT_MISS) ] = 0x00c5, /* BP_INST_RETIRED.MISPRED */ | |
1549 | }, | |
1550 | [ C(OP_WRITE) ] = { | |
1551 | [ C(RESULT_ACCESS) ] = -1, | |
1552 | [ C(RESULT_MISS) ] = -1, | |
1553 | }, | |
1554 | [ C(OP_PREFETCH) ] = { | |
1555 | [ C(RESULT_ACCESS) ] = -1, | |
1556 | [ C(RESULT_MISS) ] = -1, | |
1557 | }, | |
1558 | }, | |
1559 | }; | |
1560 | ||
ed827adb KL |
1561 | EVENT_ATTR_STR(topdown-total-slots, td_total_slots_glm, "event=0x3c"); |
1562 | EVENT_ATTR_STR(topdown-total-slots.scale, td_total_slots_scale_glm, "3"); | |
1563 | /* UOPS_NOT_DELIVERED.ANY */ | |
1564 | EVENT_ATTR_STR(topdown-fetch-bubbles, td_fetch_bubbles_glm, "event=0x9c"); | |
1565 | /* ISSUE_SLOTS_NOT_CONSUMED.RECOVERY */ | |
1566 | EVENT_ATTR_STR(topdown-recovery-bubbles, td_recovery_bubbles_glm, "event=0xca,umask=0x02"); | |
1567 | /* UOPS_RETIRED.ANY */ | |
1568 | EVENT_ATTR_STR(topdown-slots-retired, td_slots_retired_glm, "event=0xc2"); | |
1569 | /* UOPS_ISSUED.ANY */ | |
1570 | EVENT_ATTR_STR(topdown-slots-issued, td_slots_issued_glm, "event=0x0e"); | |
1571 | ||
1572 | static struct attribute *glm_events_attrs[] = { | |
1573 | EVENT_PTR(td_total_slots_glm), | |
1574 | EVENT_PTR(td_total_slots_scale_glm), | |
1575 | EVENT_PTR(td_fetch_bubbles_glm), | |
1576 | EVENT_PTR(td_recovery_bubbles_glm), | |
1577 | EVENT_PTR(td_slots_issued_glm), | |
1578 | EVENT_PTR(td_slots_retired_glm), | |
1579 | NULL | |
1580 | }; | |
1581 | ||
8b92c3a7 KL |
1582 | static struct extra_reg intel_glm_extra_regs[] __read_mostly = { |
1583 | /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ | |
1584 | INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x760005ffbfull, RSP_0), | |
1585 | INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x360005ffbfull, RSP_1), | |
1586 | EVENT_EXTRA_END | |
1587 | }; | |
1588 | ||
1589 | #define GLM_DEMAND_DATA_RD BIT_ULL(0) | |
1590 | #define GLM_DEMAND_RFO BIT_ULL(1) | |
1591 | #define GLM_ANY_RESPONSE BIT_ULL(16) | |
1592 | #define GLM_SNP_NONE_OR_MISS BIT_ULL(33) | |
1593 | #define GLM_DEMAND_READ GLM_DEMAND_DATA_RD | |
1594 | #define GLM_DEMAND_WRITE GLM_DEMAND_RFO | |
1595 | #define GLM_DEMAND_PREFETCH (SNB_PF_DATA_RD|SNB_PF_RFO) | |
1596 | #define GLM_LLC_ACCESS GLM_ANY_RESPONSE | |
1597 | #define GLM_SNP_ANY (GLM_SNP_NONE_OR_MISS|SNB_NO_FWD|SNB_HITM) | |
1598 | #define GLM_LLC_MISS (GLM_SNP_ANY|SNB_NON_DRAM) | |
1599 | ||
1600 | static __initconst const u64 glm_hw_cache_event_ids | |
1601 | [PERF_COUNT_HW_CACHE_MAX] | |
1602 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1603 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | |
1604 | [C(L1D)] = { | |
1605 | [C(OP_READ)] = { | |
1606 | [C(RESULT_ACCESS)] = 0x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ | |
1607 | [C(RESULT_MISS)] = 0x0, | |
1608 | }, | |
1609 | [C(OP_WRITE)] = { | |
1610 | [C(RESULT_ACCESS)] = 0x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ | |
1611 | [C(RESULT_MISS)] = 0x0, | |
1612 | }, | |
1613 | [C(OP_PREFETCH)] = { | |
1614 | [C(RESULT_ACCESS)] = 0x0, | |
1615 | [C(RESULT_MISS)] = 0x0, | |
1616 | }, | |
1617 | }, | |
1618 | [C(L1I)] = { | |
1619 | [C(OP_READ)] = { | |
1620 | [C(RESULT_ACCESS)] = 0x0380, /* ICACHE.ACCESSES */ | |
1621 | [C(RESULT_MISS)] = 0x0280, /* ICACHE.MISSES */ | |
1622 | }, | |
1623 | [C(OP_WRITE)] = { | |
1624 | [C(RESULT_ACCESS)] = -1, | |
1625 | [C(RESULT_MISS)] = -1, | |
1626 | }, | |
1627 | [C(OP_PREFETCH)] = { | |
1628 | [C(RESULT_ACCESS)] = 0x0, | |
1629 | [C(RESULT_MISS)] = 0x0, | |
1630 | }, | |
1631 | }, | |
1632 | [C(LL)] = { | |
1633 | [C(OP_READ)] = { | |
1634 | [C(RESULT_ACCESS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1635 | [C(RESULT_MISS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1636 | }, | |
1637 | [C(OP_WRITE)] = { | |
1638 | [C(RESULT_ACCESS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1639 | [C(RESULT_MISS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1640 | }, | |
1641 | [C(OP_PREFETCH)] = { | |
1642 | [C(RESULT_ACCESS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1643 | [C(RESULT_MISS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1644 | }, | |
1645 | }, | |
1646 | [C(DTLB)] = { | |
1647 | [C(OP_READ)] = { | |
1648 | [C(RESULT_ACCESS)] = 0x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ | |
1649 | [C(RESULT_MISS)] = 0x0, | |
1650 | }, | |
1651 | [C(OP_WRITE)] = { | |
1652 | [C(RESULT_ACCESS)] = 0x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ | |
1653 | [C(RESULT_MISS)] = 0x0, | |
1654 | }, | |
1655 | [C(OP_PREFETCH)] = { | |
1656 | [C(RESULT_ACCESS)] = 0x0, | |
1657 | [C(RESULT_MISS)] = 0x0, | |
1658 | }, | |
1659 | }, | |
1660 | [C(ITLB)] = { | |
1661 | [C(OP_READ)] = { | |
1662 | [C(RESULT_ACCESS)] = 0x00c0, /* INST_RETIRED.ANY_P */ | |
1663 | [C(RESULT_MISS)] = 0x0481, /* ITLB.MISS */ | |
1664 | }, | |
1665 | [C(OP_WRITE)] = { | |
1666 | [C(RESULT_ACCESS)] = -1, | |
1667 | [C(RESULT_MISS)] = -1, | |
1668 | }, | |
1669 | [C(OP_PREFETCH)] = { | |
1670 | [C(RESULT_ACCESS)] = -1, | |
1671 | [C(RESULT_MISS)] = -1, | |
1672 | }, | |
1673 | }, | |
1674 | [C(BPU)] = { | |
1675 | [C(OP_READ)] = { | |
1676 | [C(RESULT_ACCESS)] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ | |
1677 | [C(RESULT_MISS)] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */ | |
1678 | }, | |
1679 | [C(OP_WRITE)] = { | |
1680 | [C(RESULT_ACCESS)] = -1, | |
1681 | [C(RESULT_MISS)] = -1, | |
1682 | }, | |
1683 | [C(OP_PREFETCH)] = { | |
1684 | [C(RESULT_ACCESS)] = -1, | |
1685 | [C(RESULT_MISS)] = -1, | |
1686 | }, | |
1687 | }, | |
1688 | }; | |
1689 | ||
1690 | static __initconst const u64 glm_hw_cache_extra_regs | |
1691 | [PERF_COUNT_HW_CACHE_MAX] | |
1692 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1693 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | |
1694 | [C(LL)] = { | |
1695 | [C(OP_READ)] = { | |
1696 | [C(RESULT_ACCESS)] = GLM_DEMAND_READ| | |
1697 | GLM_LLC_ACCESS, | |
1698 | [C(RESULT_MISS)] = GLM_DEMAND_READ| | |
1699 | GLM_LLC_MISS, | |
1700 | }, | |
1701 | [C(OP_WRITE)] = { | |
1702 | [C(RESULT_ACCESS)] = GLM_DEMAND_WRITE| | |
1703 | GLM_LLC_ACCESS, | |
1704 | [C(RESULT_MISS)] = GLM_DEMAND_WRITE| | |
1705 | GLM_LLC_MISS, | |
1706 | }, | |
1707 | [C(OP_PREFETCH)] = { | |
1708 | [C(RESULT_ACCESS)] = GLM_DEMAND_PREFETCH| | |
1709 | GLM_LLC_ACCESS, | |
1710 | [C(RESULT_MISS)] = GLM_DEMAND_PREFETCH| | |
1711 | GLM_LLC_MISS, | |
1712 | }, | |
1713 | }, | |
1714 | }; | |
1715 | ||
dd0b06b5 KL |
1716 | static __initconst const u64 glp_hw_cache_event_ids |
1717 | [PERF_COUNT_HW_CACHE_MAX] | |
1718 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1719 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | |
1720 | [C(L1D)] = { | |
1721 | [C(OP_READ)] = { | |
1722 | [C(RESULT_ACCESS)] = 0x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ | |
1723 | [C(RESULT_MISS)] = 0x0, | |
1724 | }, | |
1725 | [C(OP_WRITE)] = { | |
1726 | [C(RESULT_ACCESS)] = 0x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ | |
1727 | [C(RESULT_MISS)] = 0x0, | |
1728 | }, | |
1729 | [C(OP_PREFETCH)] = { | |
1730 | [C(RESULT_ACCESS)] = 0x0, | |
1731 | [C(RESULT_MISS)] = 0x0, | |
1732 | }, | |
1733 | }, | |
1734 | [C(L1I)] = { | |
1735 | [C(OP_READ)] = { | |
1736 | [C(RESULT_ACCESS)] = 0x0380, /* ICACHE.ACCESSES */ | |
1737 | [C(RESULT_MISS)] = 0x0280, /* ICACHE.MISSES */ | |
1738 | }, | |
1739 | [C(OP_WRITE)] = { | |
1740 | [C(RESULT_ACCESS)] = -1, | |
1741 | [C(RESULT_MISS)] = -1, | |
1742 | }, | |
1743 | [C(OP_PREFETCH)] = { | |
1744 | [C(RESULT_ACCESS)] = 0x0, | |
1745 | [C(RESULT_MISS)] = 0x0, | |
1746 | }, | |
1747 | }, | |
1748 | [C(LL)] = { | |
1749 | [C(OP_READ)] = { | |
1750 | [C(RESULT_ACCESS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1751 | [C(RESULT_MISS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1752 | }, | |
1753 | [C(OP_WRITE)] = { | |
1754 | [C(RESULT_ACCESS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1755 | [C(RESULT_MISS)] = 0x1b7, /* OFFCORE_RESPONSE */ | |
1756 | }, | |
1757 | [C(OP_PREFETCH)] = { | |
1758 | [C(RESULT_ACCESS)] = 0x0, | |
1759 | [C(RESULT_MISS)] = 0x0, | |
1760 | }, | |
1761 | }, | |
1762 | [C(DTLB)] = { | |
1763 | [C(OP_READ)] = { | |
1764 | [C(RESULT_ACCESS)] = 0x81d0, /* MEM_UOPS_RETIRED.ALL_LOADS */ | |
1765 | [C(RESULT_MISS)] = 0xe08, /* DTLB_LOAD_MISSES.WALK_COMPLETED */ | |
1766 | }, | |
1767 | [C(OP_WRITE)] = { | |
1768 | [C(RESULT_ACCESS)] = 0x82d0, /* MEM_UOPS_RETIRED.ALL_STORES */ | |
1769 | [C(RESULT_MISS)] = 0xe49, /* DTLB_STORE_MISSES.WALK_COMPLETED */ | |
1770 | }, | |
1771 | [C(OP_PREFETCH)] = { | |
1772 | [C(RESULT_ACCESS)] = 0x0, | |
1773 | [C(RESULT_MISS)] = 0x0, | |
1774 | }, | |
1775 | }, | |
1776 | [C(ITLB)] = { | |
1777 | [C(OP_READ)] = { | |
1778 | [C(RESULT_ACCESS)] = 0x00c0, /* INST_RETIRED.ANY_P */ | |
1779 | [C(RESULT_MISS)] = 0x0481, /* ITLB.MISS */ | |
1780 | }, | |
1781 | [C(OP_WRITE)] = { | |
1782 | [C(RESULT_ACCESS)] = -1, | |
1783 | [C(RESULT_MISS)] = -1, | |
1784 | }, | |
1785 | [C(OP_PREFETCH)] = { | |
1786 | [C(RESULT_ACCESS)] = -1, | |
1787 | [C(RESULT_MISS)] = -1, | |
1788 | }, | |
1789 | }, | |
1790 | [C(BPU)] = { | |
1791 | [C(OP_READ)] = { | |
1792 | [C(RESULT_ACCESS)] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */ | |
1793 | [C(RESULT_MISS)] = 0x00c5, /* BR_MISP_RETIRED.ALL_BRANCHES */ | |
1794 | }, | |
1795 | [C(OP_WRITE)] = { | |
1796 | [C(RESULT_ACCESS)] = -1, | |
1797 | [C(RESULT_MISS)] = -1, | |
1798 | }, | |
1799 | [C(OP_PREFETCH)] = { | |
1800 | [C(RESULT_ACCESS)] = -1, | |
1801 | [C(RESULT_MISS)] = -1, | |
1802 | }, | |
1803 | }, | |
1804 | }; | |
1805 | ||
1806 | static __initconst const u64 glp_hw_cache_extra_regs | |
1807 | [PERF_COUNT_HW_CACHE_MAX] | |
1808 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1809 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | |
1810 | [C(LL)] = { | |
1811 | [C(OP_READ)] = { | |
1812 | [C(RESULT_ACCESS)] = GLM_DEMAND_READ| | |
1813 | GLM_LLC_ACCESS, | |
1814 | [C(RESULT_MISS)] = GLM_DEMAND_READ| | |
1815 | GLM_LLC_MISS, | |
1816 | }, | |
1817 | [C(OP_WRITE)] = { | |
1818 | [C(RESULT_ACCESS)] = GLM_DEMAND_WRITE| | |
1819 | GLM_LLC_ACCESS, | |
1820 | [C(RESULT_MISS)] = GLM_DEMAND_WRITE| | |
1821 | GLM_LLC_MISS, | |
1822 | }, | |
1823 | [C(OP_PREFETCH)] = { | |
1824 | [C(RESULT_ACCESS)] = 0x0, | |
1825 | [C(RESULT_MISS)] = 0x0, | |
1826 | }, | |
1827 | }, | |
1828 | }; | |
1829 | ||
1e7b9390 HC |
1830 | #define KNL_OT_L2_HITE BIT_ULL(19) /* Other Tile L2 Hit */ |
1831 | #define KNL_OT_L2_HITF BIT_ULL(20) /* Other Tile L2 Hit */ | |
1832 | #define KNL_MCDRAM_LOCAL BIT_ULL(21) | |
1833 | #define KNL_MCDRAM_FAR BIT_ULL(22) | |
1834 | #define KNL_DDR_LOCAL BIT_ULL(23) | |
1835 | #define KNL_DDR_FAR BIT_ULL(24) | |
1836 | #define KNL_DRAM_ANY (KNL_MCDRAM_LOCAL | KNL_MCDRAM_FAR | \ | |
1837 | KNL_DDR_LOCAL | KNL_DDR_FAR) | |
1838 | #define KNL_L2_READ SLM_DMND_READ | |
1839 | #define KNL_L2_WRITE SLM_DMND_WRITE | |
1840 | #define KNL_L2_PREFETCH SLM_DMND_PREFETCH | |
1841 | #define KNL_L2_ACCESS SLM_LLC_ACCESS | |
1842 | #define KNL_L2_MISS (KNL_OT_L2_HITE | KNL_OT_L2_HITF | \ | |
1843 | KNL_DRAM_ANY | SNB_SNP_ANY | \ | |
1844 | SNB_NON_DRAM) | |
1845 | ||
1846 | static __initconst const u64 knl_hw_cache_extra_regs | |
1847 | [PERF_COUNT_HW_CACHE_MAX] | |
1848 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
1849 | [PERF_COUNT_HW_CACHE_RESULT_MAX] = { | |
1850 | [C(LL)] = { | |
1851 | [C(OP_READ)] = { | |
1852 | [C(RESULT_ACCESS)] = KNL_L2_READ | KNL_L2_ACCESS, | |
1853 | [C(RESULT_MISS)] = 0, | |
1854 | }, | |
1855 | [C(OP_WRITE)] = { | |
1856 | [C(RESULT_ACCESS)] = KNL_L2_WRITE | KNL_L2_ACCESS, | |
1857 | [C(RESULT_MISS)] = KNL_L2_WRITE | KNL_L2_MISS, | |
1858 | }, | |
1859 | [C(OP_PREFETCH)] = { | |
1860 | [C(RESULT_ACCESS)] = KNL_L2_PREFETCH | KNL_L2_ACCESS, | |
1861 | [C(RESULT_MISS)] = KNL_L2_PREFETCH | KNL_L2_MISS, | |
1862 | }, | |
1863 | }, | |
1864 | }; | |
1865 | ||
1a78d937 | 1866 | /* |
c3d266c8 KL |
1867 | * Used from PMIs where the LBRs are already disabled. |
1868 | * | |
1869 | * This function could be called consecutively. It is required to remain in | |
1870 | * disabled state if called consecutively. | |
1871 | * | |
1872 | * During consecutive calls, the same disable value will be written to related | |
cecf6235 AS |
1873 | * registers, so the PMU state remains unchanged. |
1874 | * | |
1875 | * intel_bts events don't coexist with intel PMU's BTS events because of | |
1876 | * x86_add_exclusive(x86_lbr_exclusive_lbr); there's no need to keep them | |
1877 | * disabled around intel PMU's event batching etc, only inside the PMI handler. | |
1a78d937 AK |
1878 | */ |
1879 | static void __intel_pmu_disable_all(void) | |
f22f54f4 | 1880 | { |
89cbc767 | 1881 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
f22f54f4 PZ |
1882 | |
1883 | wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0); | |
1884 | ||
15c7ad51 | 1885 | if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) |
f22f54f4 | 1886 | intel_pmu_disable_bts(); |
ca037701 PZ |
1887 | |
1888 | intel_pmu_pebs_disable_all(); | |
1a78d937 AK |
1889 | } |
1890 | ||
1891 | static void intel_pmu_disable_all(void) | |
1892 | { | |
1893 | __intel_pmu_disable_all(); | |
caff2bef | 1894 | intel_pmu_lbr_disable_all(); |
f22f54f4 PZ |
1895 | } |
1896 | ||
1a78d937 | 1897 | static void __intel_pmu_enable_all(int added, bool pmi) |
f22f54f4 | 1898 | { |
89cbc767 | 1899 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
f22f54f4 | 1900 | |
d329527e | 1901 | intel_pmu_pebs_enable_all(); |
1a78d937 | 1902 | intel_pmu_lbr_enable_all(pmi); |
144d31e6 GN |
1903 | wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, |
1904 | x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask); | |
f22f54f4 | 1905 | |
15c7ad51 | 1906 | if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) { |
f22f54f4 | 1907 | struct perf_event *event = |
15c7ad51 | 1908 | cpuc->events[INTEL_PMC_IDX_FIXED_BTS]; |
f22f54f4 PZ |
1909 | |
1910 | if (WARN_ON_ONCE(!event)) | |
1911 | return; | |
1912 | ||
1913 | intel_pmu_enable_bts(event->hw.config); | |
cecf6235 | 1914 | } |
f22f54f4 PZ |
1915 | } |
1916 | ||
1a78d937 AK |
1917 | static void intel_pmu_enable_all(int added) |
1918 | { | |
1919 | __intel_pmu_enable_all(added, false); | |
1920 | } | |
1921 | ||
11164cd4 PZ |
1922 | /* |
1923 | * Workaround for: | |
1924 | * Intel Errata AAK100 (model 26) | |
1925 | * Intel Errata AAP53 (model 30) | |
40b91cd1 | 1926 | * Intel Errata BD53 (model 44) |
11164cd4 | 1927 | * |
351af072 ZY |
1928 | * The official story: |
1929 | * These chips need to be 'reset' when adding counters by programming the | |
1930 | * magic three (non-counting) events 0x4300B5, 0x4300D2, and 0x4300B1 either | |
1931 | * in sequence on the same PMC or on different PMCs. | |
1932 | * | |
1933 | * In practise it appears some of these events do in fact count, and | |
a97673a1 | 1934 | * we need to program all 4 events. |
11164cd4 | 1935 | */ |
351af072 | 1936 | static void intel_pmu_nhm_workaround(void) |
11164cd4 | 1937 | { |
89cbc767 | 1938 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
351af072 ZY |
1939 | static const unsigned long nhm_magic[4] = { |
1940 | 0x4300B5, | |
1941 | 0x4300D2, | |
1942 | 0x4300B1, | |
1943 | 0x4300B1 | |
1944 | }; | |
1945 | struct perf_event *event; | |
1946 | int i; | |
11164cd4 | 1947 | |
351af072 ZY |
1948 | /* |
1949 | * The Errata requires below steps: | |
1950 | * 1) Clear MSR_IA32_PEBS_ENABLE and MSR_CORE_PERF_GLOBAL_CTRL; | |
1951 | * 2) Configure 4 PERFEVTSELx with the magic events and clear | |
1952 | * the corresponding PMCx; | |
1953 | * 3) set bit0~bit3 of MSR_CORE_PERF_GLOBAL_CTRL; | |
1954 | * 4) Clear MSR_CORE_PERF_GLOBAL_CTRL; | |
1955 | * 5) Clear 4 pairs of ERFEVTSELx and PMCx; | |
1956 | */ | |
11164cd4 | 1957 | |
351af072 ZY |
1958 | /* |
1959 | * The real steps we choose are a little different from above. | |
1960 | * A) To reduce MSR operations, we don't run step 1) as they | |
1961 | * are already cleared before this function is called; | |
1962 | * B) Call x86_perf_event_update to save PMCx before configuring | |
1963 | * PERFEVTSELx with magic number; | |
1964 | * C) With step 5), we do clear only when the PERFEVTSELx is | |
1965 | * not used currently. | |
1966 | * D) Call x86_perf_event_set_period to restore PMCx; | |
1967 | */ | |
11164cd4 | 1968 | |
351af072 ZY |
1969 | /* We always operate 4 pairs of PERF Counters */ |
1970 | for (i = 0; i < 4; i++) { | |
1971 | event = cpuc->events[i]; | |
1972 | if (event) | |
1973 | x86_perf_event_update(event); | |
1974 | } | |
11164cd4 | 1975 | |
351af072 ZY |
1976 | for (i = 0; i < 4; i++) { |
1977 | wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, nhm_magic[i]); | |
1978 | wrmsrl(MSR_ARCH_PERFMON_PERFCTR0 + i, 0x0); | |
1979 | } | |
1980 | ||
1981 | wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0xf); | |
1982 | wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0x0); | |
11164cd4 | 1983 | |
351af072 ZY |
1984 | for (i = 0; i < 4; i++) { |
1985 | event = cpuc->events[i]; | |
1986 | ||
1987 | if (event) { | |
1988 | x86_perf_event_set_period(event); | |
31fa58af | 1989 | __x86_pmu_enable_event(&event->hw, |
351af072 ZY |
1990 | ARCH_PERFMON_EVENTSEL_ENABLE); |
1991 | } else | |
1992 | wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + i, 0x0); | |
11164cd4 | 1993 | } |
351af072 ZY |
1994 | } |
1995 | ||
1996 | static void intel_pmu_nhm_enable_all(int added) | |
1997 | { | |
1998 | if (added) | |
1999 | intel_pmu_nhm_workaround(); | |
11164cd4 PZ |
2000 | intel_pmu_enable_all(added); |
2001 | } | |
2002 | ||
400816f6 PZI |
2003 | static void intel_set_tfa(struct cpu_hw_events *cpuc, bool on) |
2004 | { | |
2005 | u64 val = on ? MSR_TFA_RTM_FORCE_ABORT : 0; | |
2006 | ||
2007 | if (cpuc->tfa_shadow != val) { | |
2008 | cpuc->tfa_shadow = val; | |
2009 | wrmsrl(MSR_TSX_FORCE_ABORT, val); | |
2010 | } | |
2011 | } | |
2012 | ||
2013 | static void intel_tfa_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr) | |
2014 | { | |
2015 | /* | |
2016 | * We're going to use PMC3, make sure TFA is set before we touch it. | |
2017 | */ | |
2018 | if (cntr == 3 && !cpuc->is_fake) | |
2019 | intel_set_tfa(cpuc, true); | |
2020 | } | |
2021 | ||
2022 | static void intel_tfa_pmu_enable_all(int added) | |
2023 | { | |
2024 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); | |
2025 | ||
2026 | /* | |
2027 | * If we find PMC3 is no longer used when we enable the PMU, we can | |
2028 | * clear TFA. | |
2029 | */ | |
2030 | if (!test_bit(3, cpuc->active_mask)) | |
2031 | intel_set_tfa(cpuc, false); | |
2032 | ||
2033 | intel_pmu_enable_all(added); | |
2034 | } | |
2035 | ||
af3bdb99 AK |
2036 | static void enable_counter_freeze(void) |
2037 | { | |
2038 | update_debugctlmsr(get_debugctlmsr() | | |
2039 | DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI); | |
2040 | } | |
2041 | ||
2042 | static void disable_counter_freeze(void) | |
2043 | { | |
2044 | update_debugctlmsr(get_debugctlmsr() & | |
2045 | ~DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI); | |
2046 | } | |
2047 | ||
f22f54f4 PZ |
2048 | static inline u64 intel_pmu_get_status(void) |
2049 | { | |
2050 | u64 status; | |
2051 | ||
2052 | rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); | |
2053 | ||
2054 | return status; | |
2055 | } | |
2056 | ||
2057 | static inline void intel_pmu_ack_status(u64 ack) | |
2058 | { | |
2059 | wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack); | |
2060 | } | |
2061 | ||
ca037701 | 2062 | static void intel_pmu_disable_fixed(struct hw_perf_event *hwc) |
f22f54f4 | 2063 | { |
15c7ad51 | 2064 | int idx = hwc->idx - INTEL_PMC_IDX_FIXED; |
f22f54f4 PZ |
2065 | u64 ctrl_val, mask; |
2066 | ||
2067 | mask = 0xfULL << (idx * 4); | |
2068 | ||
2069 | rdmsrl(hwc->config_base, ctrl_val); | |
2070 | ctrl_val &= ~mask; | |
7645a24c | 2071 | wrmsrl(hwc->config_base, ctrl_val); |
f22f54f4 PZ |
2072 | } |
2073 | ||
2b9e344d PZ |
2074 | static inline bool event_is_checkpointed(struct perf_event *event) |
2075 | { | |
2076 | return (event->hw.config & HSW_IN_TX_CHECKPOINTED) != 0; | |
2077 | } | |
2078 | ||
ca037701 | 2079 | static void intel_pmu_disable_event(struct perf_event *event) |
f22f54f4 | 2080 | { |
aff3d91a | 2081 | struct hw_perf_event *hwc = &event->hw; |
89cbc767 | 2082 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
aff3d91a | 2083 | |
15c7ad51 | 2084 | if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) { |
f22f54f4 PZ |
2085 | intel_pmu_disable_bts(); |
2086 | intel_pmu_drain_bts_buffer(); | |
2087 | return; | |
2088 | } | |
2089 | ||
144d31e6 GN |
2090 | cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx); |
2091 | cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx); | |
2b9e344d | 2092 | cpuc->intel_cp_status &= ~(1ull << hwc->idx); |
144d31e6 | 2093 | |
4f08b625 KL |
2094 | if (unlikely(event->attr.precise_ip)) |
2095 | intel_pmu_pebs_disable(event); | |
2096 | ||
f22f54f4 | 2097 | if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) { |
aff3d91a | 2098 | intel_pmu_disable_fixed(hwc); |
f22f54f4 PZ |
2099 | return; |
2100 | } | |
2101 | ||
aff3d91a | 2102 | x86_pmu_disable_event(event); |
f22f54f4 PZ |
2103 | } |
2104 | ||
68f7082f PZ |
2105 | static void intel_pmu_del_event(struct perf_event *event) |
2106 | { | |
2107 | if (needs_branch_stack(event)) | |
2108 | intel_pmu_lbr_del(event); | |
2109 | if (event->attr.precise_ip) | |
2110 | intel_pmu_pebs_del(event); | |
2111 | } | |
2112 | ||
ceb90d9e KL |
2113 | static void intel_pmu_read_event(struct perf_event *event) |
2114 | { | |
2115 | if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD) | |
2116 | intel_pmu_auto_reload_read(event); | |
2117 | else | |
2118 | x86_perf_event_update(event); | |
2119 | } | |
2120 | ||
4f08b625 | 2121 | static void intel_pmu_enable_fixed(struct perf_event *event) |
f22f54f4 | 2122 | { |
4f08b625 | 2123 | struct hw_perf_event *hwc = &event->hw; |
15c7ad51 | 2124 | int idx = hwc->idx - INTEL_PMC_IDX_FIXED; |
4f08b625 | 2125 | u64 ctrl_val, mask, bits = 0; |
f22f54f4 PZ |
2126 | |
2127 | /* | |
4f08b625 | 2128 | * Enable IRQ generation (0x8), if not PEBS, |
f22f54f4 PZ |
2129 | * and enable ring-3 counting (0x2) and ring-0 counting (0x1) |
2130 | * if requested: | |
2131 | */ | |
4f08b625 KL |
2132 | if (!event->attr.precise_ip) |
2133 | bits |= 0x8; | |
f22f54f4 PZ |
2134 | if (hwc->config & ARCH_PERFMON_EVENTSEL_USR) |
2135 | bits |= 0x2; | |
2136 | if (hwc->config & ARCH_PERFMON_EVENTSEL_OS) | |
2137 | bits |= 0x1; | |
2138 | ||
2139 | /* | |
2140 | * ANY bit is supported in v3 and up | |
2141 | */ | |
2142 | if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY) | |
2143 | bits |= 0x4; | |
2144 | ||
2145 | bits <<= (idx * 4); | |
2146 | mask = 0xfULL << (idx * 4); | |
2147 | ||
2148 | rdmsrl(hwc->config_base, ctrl_val); | |
2149 | ctrl_val &= ~mask; | |
2150 | ctrl_val |= bits; | |
7645a24c | 2151 | wrmsrl(hwc->config_base, ctrl_val); |
f22f54f4 PZ |
2152 | } |
2153 | ||
aff3d91a | 2154 | static void intel_pmu_enable_event(struct perf_event *event) |
f22f54f4 | 2155 | { |
aff3d91a | 2156 | struct hw_perf_event *hwc = &event->hw; |
89cbc767 | 2157 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
aff3d91a | 2158 | |
15c7ad51 | 2159 | if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) { |
0a3aee0d | 2160 | if (!__this_cpu_read(cpu_hw_events.enabled)) |
f22f54f4 PZ |
2161 | return; |
2162 | ||
2163 | intel_pmu_enable_bts(hwc->config); | |
2164 | return; | |
2165 | } | |
2166 | ||
144d31e6 GN |
2167 | if (event->attr.exclude_host) |
2168 | cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx); | |
2169 | if (event->attr.exclude_guest) | |
2170 | cpuc->intel_ctrl_host_mask |= (1ull << hwc->idx); | |
2171 | ||
2b9e344d PZ |
2172 | if (unlikely(event_is_checkpointed(event))) |
2173 | cpuc->intel_cp_status |= (1ull << hwc->idx); | |
2174 | ||
4f08b625 KL |
2175 | if (unlikely(event->attr.precise_ip)) |
2176 | intel_pmu_pebs_enable(event); | |
2177 | ||
f22f54f4 | 2178 | if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) { |
4f08b625 | 2179 | intel_pmu_enable_fixed(event); |
f22f54f4 PZ |
2180 | return; |
2181 | } | |
2182 | ||
31fa58af | 2183 | __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); |
f22f54f4 PZ |
2184 | } |
2185 | ||
68f7082f PZ |
2186 | static void intel_pmu_add_event(struct perf_event *event) |
2187 | { | |
2188 | if (event->attr.precise_ip) | |
2189 | intel_pmu_pebs_add(event); | |
2190 | if (needs_branch_stack(event)) | |
2191 | intel_pmu_lbr_add(event); | |
2192 | } | |
2193 | ||
f22f54f4 PZ |
2194 | /* |
2195 | * Save and restart an expired event. Called by NMI contexts, | |
2196 | * so it has to be careful about preempting normal event ops: | |
2197 | */ | |
de0428a7 | 2198 | int intel_pmu_save_and_restart(struct perf_event *event) |
f22f54f4 | 2199 | { |
cc2ad4ba | 2200 | x86_perf_event_update(event); |
2dbf0116 AK |
2201 | /* |
2202 | * For a checkpointed counter always reset back to 0. This | |
2203 | * avoids a situation where the counter overflows, aborts the | |
2204 | * transaction and is then set back to shortly before the | |
2205 | * overflow, and overflows and aborts again. | |
2206 | */ | |
2207 | if (unlikely(event_is_checkpointed(event))) { | |
2208 | /* No race with NMIs because the counter should not be armed */ | |
2209 | wrmsrl(event->hw.event_base, 0); | |
2210 | local64_set(&event->hw.prev_count, 0); | |
2211 | } | |
cc2ad4ba | 2212 | return x86_perf_event_set_period(event); |
f22f54f4 PZ |
2213 | } |
2214 | ||
2215 | static void intel_pmu_reset(void) | |
2216 | { | |
0a3aee0d | 2217 | struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds); |
f22f54f4 PZ |
2218 | unsigned long flags; |
2219 | int idx; | |
2220 | ||
948b1bb8 | 2221 | if (!x86_pmu.num_counters) |
f22f54f4 PZ |
2222 | return; |
2223 | ||
2224 | local_irq_save(flags); | |
2225 | ||
c767a54b | 2226 | pr_info("clearing PMU state on CPU#%d\n", smp_processor_id()); |
f22f54f4 | 2227 | |
948b1bb8 | 2228 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { |
715c85b1 PA |
2229 | wrmsrl_safe(x86_pmu_config_addr(idx), 0ull); |
2230 | wrmsrl_safe(x86_pmu_event_addr(idx), 0ull); | |
f22f54f4 | 2231 | } |
948b1bb8 | 2232 | for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) |
715c85b1 | 2233 | wrmsrl_safe(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull); |
948b1bb8 | 2234 | |
f22f54f4 PZ |
2235 | if (ds) |
2236 | ds->bts_index = ds->bts_buffer_base; | |
2237 | ||
8882edf7 AK |
2238 | /* Ack all overflows and disable fixed counters */ |
2239 | if (x86_pmu.version >= 2) { | |
2240 | intel_pmu_ack_status(intel_pmu_get_status()); | |
2241 | wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0); | |
2242 | } | |
2243 | ||
2244 | /* Reset LBRs and LBR freezing */ | |
2245 | if (x86_pmu.lbr_nr) { | |
2246 | update_debugctlmsr(get_debugctlmsr() & | |
2247 | ~(DEBUGCTLMSR_FREEZE_LBRS_ON_PMI|DEBUGCTLMSR_LBR)); | |
2248 | } | |
2249 | ||
f22f54f4 PZ |
2250 | local_irq_restore(flags); |
2251 | } | |
2252 | ||
ba12d20e | 2253 | static int handle_pmi_common(struct pt_regs *regs, u64 status) |
f22f54f4 PZ |
2254 | { |
2255 | struct perf_sample_data data; | |
ba12d20e KL |
2256 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
2257 | int bit; | |
2258 | int handled = 0; | |
f22f54f4 PZ |
2259 | |
2260 | inc_irq_stat(apic_perf_irqs); | |
ca037701 | 2261 | |
b292d7a1 | 2262 | /* |
d8020bee AK |
2263 | * Ignore a range of extra bits in status that do not indicate |
2264 | * overflow by themselves. | |
b292d7a1 | 2265 | */ |
d8020bee AK |
2266 | status &= ~(GLOBAL_STATUS_COND_CHG | |
2267 | GLOBAL_STATUS_ASIF | | |
2268 | GLOBAL_STATUS_LBRS_FROZEN); | |
2269 | if (!status) | |
ba12d20e | 2270 | return 0; |
daa864b8 SE |
2271 | /* |
2272 | * In case multiple PEBS events are sampled at the same time, | |
2273 | * it is possible to have GLOBAL_STATUS bit 62 set indicating | |
2274 | * PEBS buffer overflow and also seeing at most 3 PEBS counters | |
2275 | * having their bits set in the status register. This is a sign | |
2276 | * that there was at least one PEBS record pending at the time | |
2277 | * of the PMU interrupt. PEBS counters must only be processed | |
2278 | * via the drain_pebs() calls and not via the regular sample | |
2279 | * processing loop coming after that the function, otherwise | |
2280 | * phony regular samples may be generated in the sampling buffer | |
2281 | * not marked with the EXACT tag. Another possibility is to have | |
2282 | * one PEBS event and at least one non-PEBS event whic hoverflows | |
2283 | * while PEBS has armed. In this case, bit 62 of GLOBAL_STATUS will | |
2284 | * not be set, yet the overflow status bit for the PEBS counter will | |
2285 | * be on Skylake. | |
2286 | * | |
2287 | * To avoid this problem, we systematically ignore the PEBS-enabled | |
2288 | * counters from the GLOBAL_STATUS mask and we always process PEBS | |
2289 | * events via drain_pebs(). | |
2290 | */ | |
ec71a398 KL |
2291 | if (x86_pmu.flags & PMU_FL_PEBS_ALL) |
2292 | status &= ~cpuc->pebs_enabled; | |
2293 | else | |
2294 | status &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK); | |
b292d7a1 | 2295 | |
ca037701 PZ |
2296 | /* |
2297 | * PEBS overflow sets bit 62 in the global status register | |
2298 | */ | |
de725dec PZ |
2299 | if (__test_and_clear_bit(62, (unsigned long *)&status)) { |
2300 | handled++; | |
ca037701 | 2301 | x86_pmu.drain_pebs(regs); |
8077eca0 | 2302 | status &= x86_pmu.intel_ctrl | GLOBAL_STATUS_TRACE_TOPAPMI; |
de725dec | 2303 | } |
ca037701 | 2304 | |
52ca9ced AS |
2305 | /* |
2306 | * Intel PT | |
2307 | */ | |
2308 | if (__test_and_clear_bit(55, (unsigned long *)&status)) { | |
2309 | handled++; | |
2310 | intel_pt_interrupt(); | |
2311 | } | |
2312 | ||
2dbf0116 | 2313 | /* |
2b9e344d PZ |
2314 | * Checkpointed counters can lead to 'spurious' PMIs because the |
2315 | * rollback caused by the PMI will have cleared the overflow status | |
2316 | * bit. Therefore always force probe these counters. | |
2dbf0116 | 2317 | */ |
2b9e344d | 2318 | status |= cpuc->intel_cp_status; |
2dbf0116 | 2319 | |
984b3f57 | 2320 | for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) { |
f22f54f4 PZ |
2321 | struct perf_event *event = cpuc->events[bit]; |
2322 | ||
de725dec PZ |
2323 | handled++; |
2324 | ||
f22f54f4 PZ |
2325 | if (!test_bit(bit, cpuc->active_mask)) |
2326 | continue; | |
2327 | ||
2328 | if (!intel_pmu_save_and_restart(event)) | |
2329 | continue; | |
2330 | ||
fd0d000b | 2331 | perf_sample_data_init(&data, 0, event->hw.last_period); |
f22f54f4 | 2332 | |
60ce0fbd SE |
2333 | if (has_branch_stack(event)) |
2334 | data.br_stack = &cpuc->lbr_stack; | |
2335 | ||
a8b0ca17 | 2336 | if (perf_event_overflow(event, &data, regs)) |
a4eaf7f1 | 2337 | x86_pmu_stop(event, 0); |
f22f54f4 PZ |
2338 | } |
2339 | ||
ba12d20e KL |
2340 | return handled; |
2341 | } | |
2342 | ||
2a5bf23d | 2343 | static bool disable_counter_freezing = true; |
af3bdb99 AK |
2344 | static int __init intel_perf_counter_freezing_setup(char *s) |
2345 | { | |
2a5bf23d PZ |
2346 | bool res; |
2347 | ||
2348 | if (kstrtobool(s, &res)) | |
2349 | return -EINVAL; | |
2350 | ||
2351 | disable_counter_freezing = !res; | |
af3bdb99 AK |
2352 | return 1; |
2353 | } | |
2a5bf23d | 2354 | __setup("perf_v4_pmi=", intel_perf_counter_freezing_setup); |
af3bdb99 AK |
2355 | |
2356 | /* | |
2357 | * Simplified handler for Arch Perfmon v4: | |
2358 | * - We rely on counter freezing/unfreezing to enable/disable the PMU. | |
2359 | * This is done automatically on PMU ack. | |
2360 | * - Ack the PMU only after the APIC. | |
2361 | */ | |
2362 | ||
2363 | static int intel_pmu_handle_irq_v4(struct pt_regs *regs) | |
2364 | { | |
2365 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); | |
2366 | int handled = 0; | |
2367 | bool bts = false; | |
2368 | u64 status; | |
2369 | int pmu_enabled = cpuc->enabled; | |
2370 | int loops = 0; | |
2371 | ||
2372 | /* PMU has been disabled because of counter freezing */ | |
2373 | cpuc->enabled = 0; | |
2374 | if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) { | |
2375 | bts = true; | |
2376 | intel_bts_disable_local(); | |
2377 | handled = intel_pmu_drain_bts_buffer(); | |
2378 | handled += intel_bts_interrupt(); | |
2379 | } | |
2380 | status = intel_pmu_get_status(); | |
2381 | if (!status) | |
2382 | goto done; | |
2383 | again: | |
2384 | intel_pmu_lbr_read(); | |
2385 | if (++loops > 100) { | |
2386 | static bool warned; | |
2387 | ||
2388 | if (!warned) { | |
2389 | WARN(1, "perfevents: irq loop stuck!\n"); | |
2390 | perf_event_print_debug(); | |
2391 | warned = true; | |
2392 | } | |
2393 | intel_pmu_reset(); | |
2394 | goto done; | |
2395 | } | |
2396 | ||
2397 | ||
2398 | handled += handle_pmi_common(regs, status); | |
2399 | done: | |
2400 | /* Ack the PMI in the APIC */ | |
2401 | apic_write(APIC_LVTPC, APIC_DM_NMI); | |
2402 | ||
2403 | /* | |
2404 | * The counters start counting immediately while ack the status. | |
2405 | * Make it as close as possible to IRET. This avoids bogus | |
2406 | * freezing on Skylake CPUs. | |
2407 | */ | |
2408 | if (status) { | |
2409 | intel_pmu_ack_status(status); | |
2410 | } else { | |
2411 | /* | |
2412 | * CPU may issues two PMIs very close to each other. | |
2413 | * When the PMI handler services the first one, the | |
2414 | * GLOBAL_STATUS is already updated to reflect both. | |
2415 | * When it IRETs, the second PMI is immediately | |
2416 | * handled and it sees clear status. At the meantime, | |
2417 | * there may be a third PMI, because the freezing bit | |
2418 | * isn't set since the ack in first PMI handlers. | |
2419 | * Double check if there is more work to be done. | |
2420 | */ | |
2421 | status = intel_pmu_get_status(); | |
2422 | if (status) | |
2423 | goto again; | |
2424 | } | |
2425 | ||
2426 | if (bts) | |
2427 | intel_bts_enable_local(); | |
2428 | cpuc->enabled = pmu_enabled; | |
2429 | return handled; | |
2430 | } | |
2431 | ||
ba12d20e KL |
2432 | /* |
2433 | * This handler is triggered by the local APIC, so the APIC IRQ handling | |
2434 | * rules apply: | |
2435 | */ | |
2436 | static int intel_pmu_handle_irq(struct pt_regs *regs) | |
2437 | { | |
2438 | struct cpu_hw_events *cpuc; | |
2439 | int loops; | |
2440 | u64 status; | |
2441 | int handled; | |
2442 | int pmu_enabled; | |
2443 | ||
2444 | cpuc = this_cpu_ptr(&cpu_hw_events); | |
2445 | ||
2446 | /* | |
2447 | * Save the PMU state. | |
2448 | * It needs to be restored when leaving the handler. | |
2449 | */ | |
2450 | pmu_enabled = cpuc->enabled; | |
2451 | /* | |
2452 | * No known reason to not always do late ACK, | |
2453 | * but just in case do it opt-in. | |
2454 | */ | |
2455 | if (!x86_pmu.late_ack) | |
2456 | apic_write(APIC_LVTPC, APIC_DM_NMI); | |
2457 | intel_bts_disable_local(); | |
2458 | cpuc->enabled = 0; | |
2459 | __intel_pmu_disable_all(); | |
2460 | handled = intel_pmu_drain_bts_buffer(); | |
2461 | handled += intel_bts_interrupt(); | |
2462 | status = intel_pmu_get_status(); | |
2463 | if (!status) | |
2464 | goto done; | |
2465 | ||
2466 | loops = 0; | |
2467 | again: | |
2468 | intel_pmu_lbr_read(); | |
2469 | intel_pmu_ack_status(status); | |
2470 | if (++loops > 100) { | |
2471 | static bool warned; | |
2472 | ||
2473 | if (!warned) { | |
2474 | WARN(1, "perfevents: irq loop stuck!\n"); | |
2475 | perf_event_print_debug(); | |
2476 | warned = true; | |
2477 | } | |
2478 | intel_pmu_reset(); | |
2479 | goto done; | |
2480 | } | |
2481 | ||
2482 | handled += handle_pmi_common(regs, status); | |
2483 | ||
f22f54f4 PZ |
2484 | /* |
2485 | * Repeat if there is more work to be done: | |
2486 | */ | |
2487 | status = intel_pmu_get_status(); | |
2488 | if (status) | |
2489 | goto again; | |
2490 | ||
3fb2b8dd | 2491 | done: |
c3d266c8 | 2492 | /* Only restore PMU state when it's active. See x86_pmu_disable(). */ |
82d71ed0 KL |
2493 | cpuc->enabled = pmu_enabled; |
2494 | if (pmu_enabled) | |
c3d266c8 | 2495 | __intel_pmu_enable_all(0, true); |
cecf6235 | 2496 | intel_bts_enable_local(); |
c3d266c8 | 2497 | |
72db5596 AK |
2498 | /* |
2499 | * Only unmask the NMI after the overflow counters | |
2500 | * have been reset. This avoids spurious NMIs on | |
2501 | * Haswell CPUs. | |
2502 | */ | |
2503 | if (x86_pmu.late_ack) | |
2504 | apic_write(APIC_LVTPC, APIC_DM_NMI); | |
de725dec | 2505 | return handled; |
f22f54f4 PZ |
2506 | } |
2507 | ||
f22f54f4 | 2508 | static struct event_constraint * |
ca037701 | 2509 | intel_bts_constraints(struct perf_event *event) |
f22f54f4 | 2510 | { |
67266c10 | 2511 | if (unlikely(intel_pmu_has_bts(event))) |
f22f54f4 | 2512 | return &bts_constraint; |
ca037701 | 2513 | |
f22f54f4 PZ |
2514 | return NULL; |
2515 | } | |
2516 | ||
ae3f011f | 2517 | static int intel_alt_er(int idx, u64 config) |
b79e8941 | 2518 | { |
e01d8718 PZ |
2519 | int alt_idx = idx; |
2520 | ||
9a5e3fb5 | 2521 | if (!(x86_pmu.flags & PMU_FL_HAS_RSP_1)) |
5a425294 | 2522 | return idx; |
b79e8941 | 2523 | |
5a425294 | 2524 | if (idx == EXTRA_REG_RSP_0) |
ae3f011f | 2525 | alt_idx = EXTRA_REG_RSP_1; |
5a425294 PZ |
2526 | |
2527 | if (idx == EXTRA_REG_RSP_1) | |
ae3f011f | 2528 | alt_idx = EXTRA_REG_RSP_0; |
5a425294 | 2529 | |
ae3f011f KL |
2530 | if (config & ~x86_pmu.extra_regs[alt_idx].valid_mask) |
2531 | return idx; | |
2532 | ||
2533 | return alt_idx; | |
5a425294 PZ |
2534 | } |
2535 | ||
2536 | static void intel_fixup_er(struct perf_event *event, int idx) | |
2537 | { | |
2538 | event->hw.extra_reg.idx = idx; | |
2539 | ||
2540 | if (idx == EXTRA_REG_RSP_0) { | |
b79e8941 | 2541 | event->hw.config &= ~INTEL_ARCH_EVENT_MASK; |
53ad0447 | 2542 | event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_0].event; |
b79e8941 | 2543 | event->hw.extra_reg.reg = MSR_OFFCORE_RSP_0; |
5a425294 PZ |
2544 | } else if (idx == EXTRA_REG_RSP_1) { |
2545 | event->hw.config &= ~INTEL_ARCH_EVENT_MASK; | |
53ad0447 | 2546 | event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_1].event; |
5a425294 | 2547 | event->hw.extra_reg.reg = MSR_OFFCORE_RSP_1; |
b79e8941 | 2548 | } |
b79e8941 PZ |
2549 | } |
2550 | ||
efc9f05d SE |
2551 | /* |
2552 | * manage allocation of shared extra msr for certain events | |
2553 | * | |
2554 | * sharing can be: | |
2555 | * per-cpu: to be shared between the various events on a single PMU | |
2556 | * per-core: per-cpu + shared by HT threads | |
2557 | */ | |
a7e3ed1e | 2558 | static struct event_constraint * |
efc9f05d | 2559 | __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc, |
b36817e8 SE |
2560 | struct perf_event *event, |
2561 | struct hw_perf_event_extra *reg) | |
a7e3ed1e | 2562 | { |
efc9f05d | 2563 | struct event_constraint *c = &emptyconstraint; |
a7e3ed1e | 2564 | struct er_account *era; |
cd8a38d3 | 2565 | unsigned long flags; |
5a425294 | 2566 | int idx = reg->idx; |
a7e3ed1e | 2567 | |
5a425294 PZ |
2568 | /* |
2569 | * reg->alloc can be set due to existing state, so for fake cpuc we | |
2570 | * need to ignore this, otherwise we might fail to allocate proper fake | |
2571 | * state for this extra reg constraint. Also see the comment below. | |
2572 | */ | |
2573 | if (reg->alloc && !cpuc->is_fake) | |
b36817e8 | 2574 | return NULL; /* call x86_get_event_constraint() */ |
a7e3ed1e | 2575 | |
b79e8941 | 2576 | again: |
5a425294 | 2577 | era = &cpuc->shared_regs->regs[idx]; |
cd8a38d3 SE |
2578 | /* |
2579 | * we use spin_lock_irqsave() to avoid lockdep issues when | |
2580 | * passing a fake cpuc | |
2581 | */ | |
2582 | raw_spin_lock_irqsave(&era->lock, flags); | |
efc9f05d SE |
2583 | |
2584 | if (!atomic_read(&era->ref) || era->config == reg->config) { | |
2585 | ||
5a425294 PZ |
2586 | /* |
2587 | * If its a fake cpuc -- as per validate_{group,event}() we | |
2588 | * shouldn't touch event state and we can avoid doing so | |
2589 | * since both will only call get_event_constraints() once | |
2590 | * on each event, this avoids the need for reg->alloc. | |
2591 | * | |
2592 | * Not doing the ER fixup will only result in era->reg being | |
2593 | * wrong, but since we won't actually try and program hardware | |
2594 | * this isn't a problem either. | |
2595 | */ | |
2596 | if (!cpuc->is_fake) { | |
2597 | if (idx != reg->idx) | |
2598 | intel_fixup_er(event, idx); | |
2599 | ||
2600 | /* | |
2601 | * x86_schedule_events() can call get_event_constraints() | |
2602 | * multiple times on events in the case of incremental | |
2603 | * scheduling(). reg->alloc ensures we only do the ER | |
2604 | * allocation once. | |
2605 | */ | |
2606 | reg->alloc = 1; | |
2607 | } | |
2608 | ||
efc9f05d SE |
2609 | /* lock in msr value */ |
2610 | era->config = reg->config; | |
2611 | era->reg = reg->reg; | |
2612 | ||
2613 | /* one more user */ | |
2614 | atomic_inc(&era->ref); | |
2615 | ||
a7e3ed1e | 2616 | /* |
b36817e8 SE |
2617 | * need to call x86_get_event_constraint() |
2618 | * to check if associated event has constraints | |
a7e3ed1e | 2619 | */ |
b36817e8 | 2620 | c = NULL; |
5a425294 | 2621 | } else { |
ae3f011f | 2622 | idx = intel_alt_er(idx, reg->config); |
5a425294 PZ |
2623 | if (idx != reg->idx) { |
2624 | raw_spin_unlock_irqrestore(&era->lock, flags); | |
2625 | goto again; | |
2626 | } | |
a7e3ed1e | 2627 | } |
cd8a38d3 | 2628 | raw_spin_unlock_irqrestore(&era->lock, flags); |
a7e3ed1e | 2629 | |
efc9f05d SE |
2630 | return c; |
2631 | } | |
2632 | ||
2633 | static void | |
2634 | __intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc, | |
2635 | struct hw_perf_event_extra *reg) | |
2636 | { | |
2637 | struct er_account *era; | |
2638 | ||
2639 | /* | |
5a425294 PZ |
2640 | * Only put constraint if extra reg was actually allocated. Also takes |
2641 | * care of event which do not use an extra shared reg. | |
2642 | * | |
2643 | * Also, if this is a fake cpuc we shouldn't touch any event state | |
2644 | * (reg->alloc) and we don't care about leaving inconsistent cpuc state | |
2645 | * either since it'll be thrown out. | |
efc9f05d | 2646 | */ |
5a425294 | 2647 | if (!reg->alloc || cpuc->is_fake) |
efc9f05d SE |
2648 | return; |
2649 | ||
2650 | era = &cpuc->shared_regs->regs[reg->idx]; | |
2651 | ||
2652 | /* one fewer user */ | |
2653 | atomic_dec(&era->ref); | |
2654 | ||
2655 | /* allocate again next time */ | |
2656 | reg->alloc = 0; | |
2657 | } | |
2658 | ||
2659 | static struct event_constraint * | |
2660 | intel_shared_regs_constraints(struct cpu_hw_events *cpuc, | |
2661 | struct perf_event *event) | |
2662 | { | |
b36817e8 SE |
2663 | struct event_constraint *c = NULL, *d; |
2664 | struct hw_perf_event_extra *xreg, *breg; | |
2665 | ||
2666 | xreg = &event->hw.extra_reg; | |
2667 | if (xreg->idx != EXTRA_REG_NONE) { | |
2668 | c = __intel_shared_reg_get_constraints(cpuc, event, xreg); | |
2669 | if (c == &emptyconstraint) | |
2670 | return c; | |
2671 | } | |
2672 | breg = &event->hw.branch_reg; | |
2673 | if (breg->idx != EXTRA_REG_NONE) { | |
2674 | d = __intel_shared_reg_get_constraints(cpuc, event, breg); | |
2675 | if (d == &emptyconstraint) { | |
2676 | __intel_shared_reg_put_constraints(cpuc, xreg); | |
2677 | c = d; | |
2678 | } | |
2679 | } | |
efc9f05d | 2680 | return c; |
a7e3ed1e AK |
2681 | } |
2682 | ||
de0428a7 | 2683 | struct event_constraint * |
79cba822 SE |
2684 | x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx, |
2685 | struct perf_event *event) | |
de0428a7 KW |
2686 | { |
2687 | struct event_constraint *c; | |
2688 | ||
2689 | if (x86_pmu.event_constraints) { | |
2690 | for_each_event_constraint(c, x86_pmu.event_constraints) { | |
9fac2cf3 | 2691 | if ((event->hw.config & c->cmask) == c->code) { |
9fac2cf3 | 2692 | event->hw.flags |= c->flags; |
de0428a7 | 2693 | return c; |
9fac2cf3 | 2694 | } |
de0428a7 KW |
2695 | } |
2696 | } | |
2697 | ||
2698 | return &unconstrained; | |
2699 | } | |
2700 | ||
f22f54f4 | 2701 | static struct event_constraint * |
e979121b | 2702 | __intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx, |
79cba822 | 2703 | struct perf_event *event) |
f22f54f4 PZ |
2704 | { |
2705 | struct event_constraint *c; | |
2706 | ||
ca037701 PZ |
2707 | c = intel_bts_constraints(event); |
2708 | if (c) | |
2709 | return c; | |
2710 | ||
687805e4 | 2711 | c = intel_shared_regs_constraints(cpuc, event); |
f22f54f4 PZ |
2712 | if (c) |
2713 | return c; | |
2714 | ||
687805e4 | 2715 | c = intel_pebs_constraints(event); |
a7e3ed1e AK |
2716 | if (c) |
2717 | return c; | |
2718 | ||
79cba822 | 2719 | return x86_get_event_constraints(cpuc, idx, event); |
f22f54f4 PZ |
2720 | } |
2721 | ||
e979121b MD |
2722 | static void |
2723 | intel_start_scheduling(struct cpu_hw_events *cpuc) | |
2724 | { | |
2725 | struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; | |
1c565833 | 2726 | struct intel_excl_states *xl; |
e979121b | 2727 | int tid = cpuc->excl_thread_id; |
e979121b MD |
2728 | |
2729 | /* | |
2730 | * nothing needed if in group validation mode | |
2731 | */ | |
b37609c3 | 2732 | if (cpuc->is_fake || !is_ht_workaround_enabled()) |
e979121b | 2733 | return; |
b37609c3 | 2734 | |
e979121b MD |
2735 | /* |
2736 | * no exclusion needed | |
2737 | */ | |
17186ccd | 2738 | if (WARN_ON_ONCE(!excl_cntrs)) |
e979121b MD |
2739 | return; |
2740 | ||
e979121b MD |
2741 | xl = &excl_cntrs->states[tid]; |
2742 | ||
2743 | xl->sched_started = true; | |
e979121b MD |
2744 | /* |
2745 | * lock shared state until we are done scheduling | |
2746 | * in stop_event_scheduling() | |
2747 | * makes scheduling appear as a transaction | |
2748 | */ | |
e979121b | 2749 | raw_spin_lock(&excl_cntrs->lock); |
e979121b MD |
2750 | } |
2751 | ||
0c41e756 PZ |
2752 | static void intel_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr) |
2753 | { | |
2754 | struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; | |
2755 | struct event_constraint *c = cpuc->event_constraint[idx]; | |
2756 | struct intel_excl_states *xl; | |
2757 | int tid = cpuc->excl_thread_id; | |
2758 | ||
2759 | if (cpuc->is_fake || !is_ht_workaround_enabled()) | |
2760 | return; | |
2761 | ||
2762 | if (WARN_ON_ONCE(!excl_cntrs)) | |
2763 | return; | |
2764 | ||
2765 | if (!(c->flags & PERF_X86_EVENT_DYNAMIC)) | |
2766 | return; | |
2767 | ||
2768 | xl = &excl_cntrs->states[tid]; | |
2769 | ||
2770 | lockdep_assert_held(&excl_cntrs->lock); | |
2771 | ||
1fe684e3 | 2772 | if (c->flags & PERF_X86_EVENT_EXCL) |
43ef205b | 2773 | xl->state[cntr] = INTEL_EXCL_EXCLUSIVE; |
1fe684e3 | 2774 | else |
43ef205b | 2775 | xl->state[cntr] = INTEL_EXCL_SHARED; |
0c41e756 PZ |
2776 | } |
2777 | ||
e979121b MD |
2778 | static void |
2779 | intel_stop_scheduling(struct cpu_hw_events *cpuc) | |
2780 | { | |
2781 | struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; | |
1c565833 | 2782 | struct intel_excl_states *xl; |
e979121b | 2783 | int tid = cpuc->excl_thread_id; |
e979121b MD |
2784 | |
2785 | /* | |
2786 | * nothing needed if in group validation mode | |
2787 | */ | |
b37609c3 | 2788 | if (cpuc->is_fake || !is_ht_workaround_enabled()) |
e979121b MD |
2789 | return; |
2790 | /* | |
2791 | * no exclusion needed | |
2792 | */ | |
17186ccd | 2793 | if (WARN_ON_ONCE(!excl_cntrs)) |
e979121b MD |
2794 | return; |
2795 | ||
e979121b MD |
2796 | xl = &excl_cntrs->states[tid]; |
2797 | ||
e979121b MD |
2798 | xl->sched_started = false; |
2799 | /* | |
2800 | * release shared state lock (acquired in intel_start_scheduling()) | |
2801 | */ | |
2802 | raw_spin_unlock(&excl_cntrs->lock); | |
2803 | } | |
2804 | ||
11f8b2d6 PZI |
2805 | static struct event_constraint * |
2806 | dyn_constraint(struct cpu_hw_events *cpuc, struct event_constraint *c, int idx) | |
2807 | { | |
2808 | WARN_ON_ONCE(!cpuc->constraint_list); | |
2809 | ||
2810 | if (!(c->flags & PERF_X86_EVENT_DYNAMIC)) { | |
2811 | struct event_constraint *cx; | |
2812 | ||
2813 | /* | |
2814 | * grab pre-allocated constraint entry | |
2815 | */ | |
2816 | cx = &cpuc->constraint_list[idx]; | |
2817 | ||
2818 | /* | |
2819 | * initialize dynamic constraint | |
2820 | * with static constraint | |
2821 | */ | |
2822 | *cx = *c; | |
2823 | ||
2824 | /* | |
2825 | * mark constraint as dynamic | |
2826 | */ | |
2827 | cx->flags |= PERF_X86_EVENT_DYNAMIC; | |
2828 | c = cx; | |
2829 | } | |
2830 | ||
2831 | return c; | |
2832 | } | |
2833 | ||
e979121b MD |
2834 | static struct event_constraint * |
2835 | intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event, | |
2836 | int idx, struct event_constraint *c) | |
2837 | { | |
e979121b | 2838 | struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; |
1c565833 | 2839 | struct intel_excl_states *xlo; |
e979121b | 2840 | int tid = cpuc->excl_thread_id; |
1c565833 | 2841 | int is_excl, i; |
e979121b MD |
2842 | |
2843 | /* | |
2844 | * validating a group does not require | |
2845 | * enforcing cross-thread exclusion | |
2846 | */ | |
b37609c3 SE |
2847 | if (cpuc->is_fake || !is_ht_workaround_enabled()) |
2848 | return c; | |
2849 | ||
2850 | /* | |
2851 | * no exclusion needed | |
2852 | */ | |
17186ccd | 2853 | if (WARN_ON_ONCE(!excl_cntrs)) |
e979121b | 2854 | return c; |
e979121b | 2855 | |
e979121b MD |
2856 | /* |
2857 | * because we modify the constraint, we need | |
2858 | * to make a copy. Static constraints come | |
2859 | * from static const tables. | |
2860 | * | |
2861 | * only needed when constraint has not yet | |
2862 | * been cloned (marked dynamic) | |
2863 | */ | |
11f8b2d6 | 2864 | c = dyn_constraint(cpuc, c, idx); |
e979121b MD |
2865 | |
2866 | /* | |
2867 | * From here on, the constraint is dynamic. | |
2868 | * Either it was just allocated above, or it | |
2869 | * was allocated during a earlier invocation | |
2870 | * of this function | |
2871 | */ | |
2872 | ||
1c565833 PZ |
2873 | /* |
2874 | * state of sibling HT | |
2875 | */ | |
2876 | xlo = &excl_cntrs->states[tid ^ 1]; | |
2877 | ||
2878 | /* | |
2879 | * event requires exclusive counter access | |
2880 | * across HT threads | |
2881 | */ | |
2882 | is_excl = c->flags & PERF_X86_EVENT_EXCL; | |
2883 | if (is_excl && !(event->hw.flags & PERF_X86_EVENT_EXCL_ACCT)) { | |
2884 | event->hw.flags |= PERF_X86_EVENT_EXCL_ACCT; | |
2885 | if (!cpuc->n_excl++) | |
2886 | WRITE_ONCE(excl_cntrs->has_exclusive[tid], 1); | |
2887 | } | |
2888 | ||
e979121b MD |
2889 | /* |
2890 | * Modify static constraint with current dynamic | |
2891 | * state of thread | |
2892 | * | |
2893 | * EXCLUSIVE: sibling counter measuring exclusive event | |
2894 | * SHARED : sibling counter measuring non-exclusive event | |
2895 | * UNUSED : sibling counter unused | |
2896 | */ | |
aaf932e8 | 2897 | for_each_set_bit(i, c->idxmsk, X86_PMC_IDX_MAX) { |
e979121b MD |
2898 | /* |
2899 | * exclusive event in sibling counter | |
2900 | * our corresponding counter cannot be used | |
2901 | * regardless of our event | |
2902 | */ | |
1c565833 | 2903 | if (xlo->state[i] == INTEL_EXCL_EXCLUSIVE) |
aaf932e8 | 2904 | __clear_bit(i, c->idxmsk); |
e979121b MD |
2905 | /* |
2906 | * if measuring an exclusive event, sibling | |
2907 | * measuring non-exclusive, then counter cannot | |
2908 | * be used | |
2909 | */ | |
1c565833 | 2910 | if (is_excl && xlo->state[i] == INTEL_EXCL_SHARED) |
aaf932e8 | 2911 | __clear_bit(i, c->idxmsk); |
e979121b MD |
2912 | } |
2913 | ||
2914 | /* | |
2915 | * recompute actual bit weight for scheduling algorithm | |
2916 | */ | |
aaf932e8 | 2917 | c->weight = hweight64(c->idxmsk64); |
e979121b MD |
2918 | |
2919 | /* | |
2920 | * if we return an empty mask, then switch | |
2921 | * back to static empty constraint to avoid | |
2922 | * the cost of freeing later on | |
2923 | */ | |
aaf932e8 PZ |
2924 | if (c->weight == 0) |
2925 | c = &emptyconstraint; | |
e979121b | 2926 | |
aaf932e8 | 2927 | return c; |
e979121b MD |
2928 | } |
2929 | ||
2930 | static struct event_constraint * | |
2931 | intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx, | |
2932 | struct perf_event *event) | |
2933 | { | |
ebfb4988 | 2934 | struct event_constraint *c1 = NULL; |
a90738c2 | 2935 | struct event_constraint *c2; |
e979121b | 2936 | |
ebfb4988 PZ |
2937 | if (idx >= 0) /* fake does < 0 */ |
2938 | c1 = cpuc->event_constraint[idx]; | |
2939 | ||
e979121b MD |
2940 | /* |
2941 | * first time only | |
2942 | * - static constraint: no change across incremental scheduling calls | |
2943 | * - dynamic constraint: handled by intel_get_excl_constraints() | |
2944 | */ | |
a90738c2 SE |
2945 | c2 = __intel_get_event_constraints(cpuc, idx, event); |
2946 | if (c1 && (c1->flags & PERF_X86_EVENT_DYNAMIC)) { | |
2947 | bitmap_copy(c1->idxmsk, c2->idxmsk, X86_PMC_IDX_MAX); | |
2948 | c1->weight = c2->weight; | |
2949 | c2 = c1; | |
2950 | } | |
e979121b MD |
2951 | |
2952 | if (cpuc->excl_cntrs) | |
a90738c2 | 2953 | return intel_get_excl_constraints(cpuc, event, idx, c2); |
e979121b | 2954 | |
a90738c2 | 2955 | return c2; |
e979121b MD |
2956 | } |
2957 | ||
2958 | static void intel_put_excl_constraints(struct cpu_hw_events *cpuc, | |
2959 | struct perf_event *event) | |
2960 | { | |
2961 | struct hw_perf_event *hwc = &event->hw; | |
2962 | struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; | |
e979121b | 2963 | int tid = cpuc->excl_thread_id; |
1c565833 | 2964 | struct intel_excl_states *xl; |
e979121b MD |
2965 | |
2966 | /* | |
2967 | * nothing needed if in group validation mode | |
2968 | */ | |
2969 | if (cpuc->is_fake) | |
2970 | return; | |
2971 | ||
17186ccd | 2972 | if (WARN_ON_ONCE(!excl_cntrs)) |
e979121b MD |
2973 | return; |
2974 | ||
cc1790cf PZ |
2975 | if (hwc->flags & PERF_X86_EVENT_EXCL_ACCT) { |
2976 | hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT; | |
2977 | if (!--cpuc->n_excl) | |
2978 | WRITE_ONCE(excl_cntrs->has_exclusive[tid], 0); | |
2979 | } | |
e979121b MD |
2980 | |
2981 | /* | |
ba040653 PZ |
2982 | * If event was actually assigned, then mark the counter state as |
2983 | * unused now. | |
e979121b | 2984 | */ |
ba040653 PZ |
2985 | if (hwc->idx >= 0) { |
2986 | xl = &excl_cntrs->states[tid]; | |
2987 | ||
2988 | /* | |
2989 | * put_constraint may be called from x86_schedule_events() | |
2990 | * which already has the lock held so here make locking | |
2991 | * conditional. | |
2992 | */ | |
2993 | if (!xl->sched_started) | |
2994 | raw_spin_lock(&excl_cntrs->lock); | |
e979121b | 2995 | |
1c565833 | 2996 | xl->state[hwc->idx] = INTEL_EXCL_UNUSED; |
e979121b | 2997 | |
ba040653 PZ |
2998 | if (!xl->sched_started) |
2999 | raw_spin_unlock(&excl_cntrs->lock); | |
3000 | } | |
e979121b MD |
3001 | } |
3002 | ||
efc9f05d SE |
3003 | static void |
3004 | intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc, | |
a7e3ed1e AK |
3005 | struct perf_event *event) |
3006 | { | |
efc9f05d | 3007 | struct hw_perf_event_extra *reg; |
a7e3ed1e | 3008 | |
efc9f05d SE |
3009 | reg = &event->hw.extra_reg; |
3010 | if (reg->idx != EXTRA_REG_NONE) | |
3011 | __intel_shared_reg_put_constraints(cpuc, reg); | |
b36817e8 SE |
3012 | |
3013 | reg = &event->hw.branch_reg; | |
3014 | if (reg->idx != EXTRA_REG_NONE) | |
3015 | __intel_shared_reg_put_constraints(cpuc, reg); | |
efc9f05d | 3016 | } |
a7e3ed1e | 3017 | |
efc9f05d SE |
3018 | static void intel_put_event_constraints(struct cpu_hw_events *cpuc, |
3019 | struct perf_event *event) | |
3020 | { | |
3021 | intel_put_shared_regs_event_constraints(cpuc, event); | |
e979121b MD |
3022 | |
3023 | /* | |
3024 | * is PMU has exclusive counter restrictions, then | |
3025 | * all events are subject to and must call the | |
3026 | * put_excl_constraints() routine | |
3027 | */ | |
b371b594 | 3028 | if (cpuc->excl_cntrs) |
e979121b | 3029 | intel_put_excl_constraints(cpuc, event); |
e979121b MD |
3030 | } |
3031 | ||
0780c927 | 3032 | static void intel_pebs_aliases_core2(struct perf_event *event) |
b4cdc5c2 | 3033 | { |
0780c927 | 3034 | if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) { |
7639dae0 PZ |
3035 | /* |
3036 | * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P | |
3037 | * (0x003c) so that we can use it with PEBS. | |
3038 | * | |
3039 | * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't | |
3040 | * PEBS capable. However we can use INST_RETIRED.ANY_P | |
3041 | * (0x00c0), which is a PEBS capable event, to get the same | |
3042 | * count. | |
3043 | * | |
3044 | * INST_RETIRED.ANY_P counts the number of cycles that retires | |
3045 | * CNTMASK instructions. By setting CNTMASK to a value (16) | |
3046 | * larger than the maximum number of instructions that can be | |
3047 | * retired per cycle (4) and then inverting the condition, we | |
3048 | * count all cycles that retire 16 or less instructions, which | |
3049 | * is every cycle. | |
3050 | * | |
3051 | * Thereby we gain a PEBS capable cycle counter. | |
3052 | */ | |
f9b4eeb8 PZ |
3053 | u64 alt_config = X86_CONFIG(.event=0xc0, .inv=1, .cmask=16); |
3054 | ||
0780c927 PZ |
3055 | alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK); |
3056 | event->hw.config = alt_config; | |
3057 | } | |
3058 | } | |
3059 | ||
3060 | static void intel_pebs_aliases_snb(struct perf_event *event) | |
3061 | { | |
3062 | if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) { | |
3063 | /* | |
3064 | * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P | |
3065 | * (0x003c) so that we can use it with PEBS. | |
3066 | * | |
3067 | * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't | |
3068 | * PEBS capable. However we can use UOPS_RETIRED.ALL | |
3069 | * (0x01c2), which is a PEBS capable event, to get the same | |
3070 | * count. | |
3071 | * | |
3072 | * UOPS_RETIRED.ALL counts the number of cycles that retires | |
3073 | * CNTMASK micro-ops. By setting CNTMASK to a value (16) | |
3074 | * larger than the maximum number of micro-ops that can be | |
3075 | * retired per cycle (4) and then inverting the condition, we | |
3076 | * count all cycles that retire 16 or less micro-ops, which | |
3077 | * is every cycle. | |
3078 | * | |
3079 | * Thereby we gain a PEBS capable cycle counter. | |
3080 | */ | |
3081 | u64 alt_config = X86_CONFIG(.event=0xc2, .umask=0x01, .inv=1, .cmask=16); | |
7639dae0 PZ |
3082 | |
3083 | alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK); | |
3084 | event->hw.config = alt_config; | |
3085 | } | |
0780c927 PZ |
3086 | } |
3087 | ||
72469764 AK |
3088 | static void intel_pebs_aliases_precdist(struct perf_event *event) |
3089 | { | |
3090 | if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) { | |
3091 | /* | |
3092 | * Use an alternative encoding for CPU_CLK_UNHALTED.THREAD_P | |
3093 | * (0x003c) so that we can use it with PEBS. | |
3094 | * | |
3095 | * The regular CPU_CLK_UNHALTED.THREAD_P event (0x003c) isn't | |
3096 | * PEBS capable. However we can use INST_RETIRED.PREC_DIST | |
3097 | * (0x01c0), which is a PEBS capable event, to get the same | |
3098 | * count. | |
3099 | * | |
3100 | * The PREC_DIST event has special support to minimize sample | |
3101 | * shadowing effects. One drawback is that it can be | |
3102 | * only programmed on counter 1, but that seems like an | |
3103 | * acceptable trade off. | |
3104 | */ | |
3105 | u64 alt_config = X86_CONFIG(.event=0xc0, .umask=0x01, .inv=1, .cmask=16); | |
3106 | ||
3107 | alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK); | |
3108 | event->hw.config = alt_config; | |
3109 | } | |
3110 | } | |
3111 | ||
3112 | static void intel_pebs_aliases_ivb(struct perf_event *event) | |
3113 | { | |
3114 | if (event->attr.precise_ip < 3) | |
3115 | return intel_pebs_aliases_snb(event); | |
3116 | return intel_pebs_aliases_precdist(event); | |
3117 | } | |
3118 | ||
3119 | static void intel_pebs_aliases_skl(struct perf_event *event) | |
3120 | { | |
3121 | if (event->attr.precise_ip < 3) | |
3122 | return intel_pebs_aliases_core2(event); | |
3123 | return intel_pebs_aliases_precdist(event); | |
3124 | } | |
3125 | ||
174afc3e | 3126 | static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event) |
a7b58d21 | 3127 | { |
174afc3e | 3128 | unsigned long flags = x86_pmu.large_pebs_flags; |
a7b58d21 AK |
3129 | |
3130 | if (event->attr.use_clockid) | |
3131 | flags &= ~PERF_SAMPLE_TIME; | |
a47ba4d7 AK |
3132 | if (!event->attr.exclude_kernel) |
3133 | flags &= ~PERF_SAMPLE_REGS_USER; | |
3134 | if (event->attr.sample_regs_user & ~PEBS_REGS) | |
3135 | flags &= ~(PERF_SAMPLE_REGS_USER | PERF_SAMPLE_REGS_INTR); | |
a7b58d21 AK |
3136 | return flags; |
3137 | } | |
3138 | ||
ed6101bb JO |
3139 | static int intel_pmu_bts_config(struct perf_event *event) |
3140 | { | |
3141 | struct perf_event_attr *attr = &event->attr; | |
ed6101bb | 3142 | |
67266c10 | 3143 | if (unlikely(intel_pmu_has_bts(event))) { |
ed6101bb JO |
3144 | /* BTS is not supported by this architecture. */ |
3145 | if (!x86_pmu.bts_active) | |
3146 | return -EOPNOTSUPP; | |
3147 | ||
3148 | /* BTS is currently only allowed for user-mode. */ | |
3149 | if (!attr->exclude_kernel) | |
3150 | return -EOPNOTSUPP; | |
3151 | ||
472de49f JO |
3152 | /* BTS is not allowed for precise events. */ |
3153 | if (attr->precise_ip) | |
3154 | return -EOPNOTSUPP; | |
3155 | ||
ed6101bb JO |
3156 | /* disallow bts if conflicting events are present */ |
3157 | if (x86_add_exclusive(x86_lbr_exclusive_lbr)) | |
3158 | return -EBUSY; | |
3159 | ||
3160 | event->destroy = hw_perf_lbr_event_destroy; | |
3161 | } | |
3162 | ||
3163 | return 0; | |
3164 | } | |
3165 | ||
3166 | static int core_pmu_hw_config(struct perf_event *event) | |
3167 | { | |
3168 | int ret = x86_pmu_hw_config(event); | |
3169 | ||
3170 | if (ret) | |
3171 | return ret; | |
3172 | ||
3173 | return intel_pmu_bts_config(event); | |
3174 | } | |
3175 | ||
0780c927 PZ |
3176 | static int intel_pmu_hw_config(struct perf_event *event) |
3177 | { | |
3178 | int ret = x86_pmu_hw_config(event); | |
3179 | ||
ed6101bb JO |
3180 | if (ret) |
3181 | return ret; | |
3182 | ||
3183 | ret = intel_pmu_bts_config(event); | |
0780c927 PZ |
3184 | if (ret) |
3185 | return ret; | |
3186 | ||
851559e3 | 3187 | if (event->attr.precise_ip) { |
3569c0d7 | 3188 | if (!event->attr.freq) { |
851559e3 | 3189 | event->hw.flags |= PERF_X86_EVENT_AUTO_RELOAD; |
a7b58d21 | 3190 | if (!(event->attr.sample_type & |
174afc3e KL |
3191 | ~intel_pmu_large_pebs_flags(event))) |
3192 | event->hw.flags |= PERF_X86_EVENT_LARGE_PEBS; | |
3569c0d7 | 3193 | } |
851559e3 YZ |
3194 | if (x86_pmu.pebs_aliases) |
3195 | x86_pmu.pebs_aliases(event); | |
6cbc304f PZ |
3196 | |
3197 | if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) | |
3198 | event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY; | |
851559e3 | 3199 | } |
7639dae0 | 3200 | |
a46a2300 | 3201 | if (needs_branch_stack(event)) { |
60ce0fbd SE |
3202 | ret = intel_pmu_setup_lbr_filter(event); |
3203 | if (ret) | |
3204 | return ret; | |
48070342 AS |
3205 | |
3206 | /* | |
3207 | * BTS is set up earlier in this path, so don't account twice | |
3208 | */ | |
67266c10 | 3209 | if (!unlikely(intel_pmu_has_bts(event))) { |
48070342 AS |
3210 | /* disallow lbr if conflicting events are present */ |
3211 | if (x86_add_exclusive(x86_lbr_exclusive_lbr)) | |
3212 | return -EBUSY; | |
3213 | ||
3214 | event->destroy = hw_perf_lbr_event_destroy; | |
3215 | } | |
60ce0fbd SE |
3216 | } |
3217 | ||
b4cdc5c2 PZ |
3218 | if (event->attr.type != PERF_TYPE_RAW) |
3219 | return 0; | |
3220 | ||
3221 | if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY)) | |
3222 | return 0; | |
3223 | ||
3224 | if (x86_pmu.version < 3) | |
3225 | return -EINVAL; | |
3226 | ||
3227 | if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) | |
3228 | return -EACCES; | |
3229 | ||
3230 | event->hw.config |= ARCH_PERFMON_EVENTSEL_ANY; | |
3231 | ||
3232 | return 0; | |
3233 | } | |
3234 | ||
144d31e6 GN |
3235 | struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr) |
3236 | { | |
3237 | if (x86_pmu.guest_get_msrs) | |
3238 | return x86_pmu.guest_get_msrs(nr); | |
3239 | *nr = 0; | |
3240 | return NULL; | |
3241 | } | |
3242 | EXPORT_SYMBOL_GPL(perf_guest_get_msrs); | |
3243 | ||
3244 | static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr) | |
3245 | { | |
89cbc767 | 3246 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
144d31e6 GN |
3247 | struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs; |
3248 | ||
3249 | arr[0].msr = MSR_CORE_PERF_GLOBAL_CTRL; | |
3250 | arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask; | |
3251 | arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask; | |
9b545c04 AK |
3252 | if (x86_pmu.flags & PMU_FL_PEBS_ALL) |
3253 | arr[0].guest &= ~cpuc->pebs_enabled; | |
3254 | else | |
3255 | arr[0].guest &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK); | |
3256 | *nr = 1; | |
3257 | ||
3258 | if (x86_pmu.pebs && x86_pmu.pebs_no_isolation) { | |
3259 | /* | |
3260 | * If PMU counter has PEBS enabled it is not enough to | |
3261 | * disable counter on a guest entry since PEBS memory | |
3262 | * write can overshoot guest entry and corrupt guest | |
3263 | * memory. Disabling PEBS solves the problem. | |
3264 | * | |
3265 | * Don't do this if the CPU already enforces it. | |
3266 | */ | |
3267 | arr[1].msr = MSR_IA32_PEBS_ENABLE; | |
3268 | arr[1].host = cpuc->pebs_enabled; | |
3269 | arr[1].guest = 0; | |
3270 | *nr = 2; | |
3271 | } | |
144d31e6 | 3272 | |
144d31e6 GN |
3273 | return arr; |
3274 | } | |
3275 | ||
3276 | static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr) | |
3277 | { | |
89cbc767 | 3278 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
144d31e6 GN |
3279 | struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs; |
3280 | int idx; | |
3281 | ||
3282 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { | |
3283 | struct perf_event *event = cpuc->events[idx]; | |
3284 | ||
3285 | arr[idx].msr = x86_pmu_config_addr(idx); | |
3286 | arr[idx].host = arr[idx].guest = 0; | |
3287 | ||
3288 | if (!test_bit(idx, cpuc->active_mask)) | |
3289 | continue; | |
3290 | ||
3291 | arr[idx].host = arr[idx].guest = | |
3292 | event->hw.config | ARCH_PERFMON_EVENTSEL_ENABLE; | |
3293 | ||
3294 | if (event->attr.exclude_host) | |
3295 | arr[idx].host &= ~ARCH_PERFMON_EVENTSEL_ENABLE; | |
3296 | else if (event->attr.exclude_guest) | |
3297 | arr[idx].guest &= ~ARCH_PERFMON_EVENTSEL_ENABLE; | |
3298 | } | |
3299 | ||
3300 | *nr = x86_pmu.num_counters; | |
3301 | return arr; | |
3302 | } | |
3303 | ||
3304 | static void core_pmu_enable_event(struct perf_event *event) | |
3305 | { | |
3306 | if (!event->attr.exclude_host) | |
3307 | x86_pmu_enable_event(event); | |
3308 | } | |
3309 | ||
3310 | static void core_pmu_enable_all(int added) | |
3311 | { | |
89cbc767 | 3312 | struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); |
144d31e6 GN |
3313 | int idx; |
3314 | ||
3315 | for (idx = 0; idx < x86_pmu.num_counters; idx++) { | |
3316 | struct hw_perf_event *hwc = &cpuc->events[idx]->hw; | |
3317 | ||
3318 | if (!test_bit(idx, cpuc->active_mask) || | |
3319 | cpuc->events[idx]->attr.exclude_host) | |
3320 | continue; | |
3321 | ||
3322 | __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); | |
3323 | } | |
3324 | } | |
3325 | ||
3a632cb2 AK |
3326 | static int hsw_hw_config(struct perf_event *event) |
3327 | { | |
3328 | int ret = intel_pmu_hw_config(event); | |
3329 | ||
3330 | if (ret) | |
3331 | return ret; | |
3332 | if (!boot_cpu_has(X86_FEATURE_RTM) && !boot_cpu_has(X86_FEATURE_HLE)) | |
3333 | return 0; | |
3334 | event->hw.config |= event->attr.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED); | |
3335 | ||
3336 | /* | |
3337 | * IN_TX/IN_TX-CP filters are not supported by the Haswell PMU with | |
3338 | * PEBS or in ANY thread mode. Since the results are non-sensical forbid | |
3339 | * this combination. | |
3340 | */ | |
3341 | if ((event->hw.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)) && | |
3342 | ((event->hw.config & ARCH_PERFMON_EVENTSEL_ANY) || | |
3343 | event->attr.precise_ip > 0)) | |
3344 | return -EOPNOTSUPP; | |
3345 | ||
2dbf0116 AK |
3346 | if (event_is_checkpointed(event)) { |
3347 | /* | |
3348 | * Sampling of checkpointed events can cause situations where | |
3349 | * the CPU constantly aborts because of a overflow, which is | |
3350 | * then checkpointed back and ignored. Forbid checkpointing | |
3351 | * for sampling. | |
3352 | * | |
3353 | * But still allow a long sampling period, so that perf stat | |
3354 | * from KVM works. | |
3355 | */ | |
3356 | if (event->attr.sample_period > 0 && | |
3357 | event->attr.sample_period < 0x7fffffff) | |
3358 | return -EOPNOTSUPP; | |
3359 | } | |
3a632cb2 AK |
3360 | return 0; |
3361 | } | |
3362 | ||
dd0b06b5 KL |
3363 | static struct event_constraint counter0_constraint = |
3364 | INTEL_ALL_EVENT_CONSTRAINT(0, 0x1); | |
3365 | ||
3a632cb2 AK |
3366 | static struct event_constraint counter2_constraint = |
3367 | EVENT_CONSTRAINT(0, 0x4, 0); | |
3368 | ||
3369 | static struct event_constraint * | |
79cba822 SE |
3370 | hsw_get_event_constraints(struct cpu_hw_events *cpuc, int idx, |
3371 | struct perf_event *event) | |
3a632cb2 | 3372 | { |
79cba822 SE |
3373 | struct event_constraint *c; |
3374 | ||
3375 | c = intel_get_event_constraints(cpuc, idx, event); | |
3a632cb2 AK |
3376 | |
3377 | /* Handle special quirk on in_tx_checkpointed only in counter 2 */ | |
3378 | if (event->hw.config & HSW_IN_TX_CHECKPOINTED) { | |
3379 | if (c->idxmsk64 & (1U << 2)) | |
3380 | return &counter2_constraint; | |
3381 | return &emptyconstraint; | |
3382 | } | |
3383 | ||
3384 | return c; | |
3385 | } | |
3386 | ||
dd0b06b5 KL |
3387 | static struct event_constraint * |
3388 | glp_get_event_constraints(struct cpu_hw_events *cpuc, int idx, | |
3389 | struct perf_event *event) | |
3390 | { | |
3391 | struct event_constraint *c; | |
3392 | ||
3393 | /* :ppp means to do reduced skid PEBS which is PMC0 only. */ | |
3394 | if (event->attr.precise_ip == 3) | |
3395 | return &counter0_constraint; | |
3396 | ||
3397 | c = intel_get_event_constraints(cpuc, idx, event); | |
3398 | ||
3399 | return c; | |
3400 | } | |
3401 | ||
400816f6 PZI |
3402 | static bool allow_tsx_force_abort = true; |
3403 | ||
3404 | static struct event_constraint * | |
3405 | tfa_get_event_constraints(struct cpu_hw_events *cpuc, int idx, | |
3406 | struct perf_event *event) | |
3407 | { | |
3408 | struct event_constraint *c = hsw_get_event_constraints(cpuc, idx, event); | |
3409 | ||
3410 | /* | |
3411 | * Without TFA we must not use PMC3. | |
3412 | */ | |
ede271b0 | 3413 | if (!allow_tsx_force_abort && test_bit(3, c->idxmsk) && idx >= 0) { |
400816f6 PZI |
3414 | c = dyn_constraint(cpuc, c, idx); |
3415 | c->idxmsk64 &= ~(1ULL << 3); | |
3416 | c->weight--; | |
3417 | } | |
3418 | ||
3419 | return c; | |
3420 | } | |
3421 | ||
294fe0f5 AK |
3422 | /* |
3423 | * Broadwell: | |
3424 | * | |
3425 | * The INST_RETIRED.ALL period always needs to have lowest 6 bits cleared | |
3426 | * (BDM55) and it must not use a period smaller than 100 (BDM11). We combine | |
3427 | * the two to enforce a minimum period of 128 (the smallest value that has bits | |
3428 | * 0-5 cleared and >= 100). | |
3429 | * | |
3430 | * Because of how the code in x86_perf_event_set_period() works, the truncation | |
3431 | * of the lower 6 bits is 'harmless' as we'll occasionally add a longer period | |
3432 | * to make up for the 'lost' events due to carrying the 'error' in period_left. | |
3433 | * | |
3434 | * Therefore the effective (average) period matches the requested period, | |
3435 | * despite coarser hardware granularity. | |
3436 | */ | |
f605cfca | 3437 | static u64 bdw_limit_period(struct perf_event *event, u64 left) |
294fe0f5 AK |
3438 | { |
3439 | if ((event->hw.config & INTEL_ARCH_EVENT_MASK) == | |
3440 | X86_CONFIG(.event=0xc0, .umask=0x01)) { | |
3441 | if (left < 128) | |
3442 | left = 128; | |
e5ea9b54 | 3443 | left &= ~0x3fULL; |
294fe0f5 AK |
3444 | } |
3445 | return left; | |
3446 | } | |
3447 | ||
641cc938 JO |
3448 | PMU_FORMAT_ATTR(event, "config:0-7" ); |
3449 | PMU_FORMAT_ATTR(umask, "config:8-15" ); | |
3450 | PMU_FORMAT_ATTR(edge, "config:18" ); | |
3451 | PMU_FORMAT_ATTR(pc, "config:19" ); | |
3452 | PMU_FORMAT_ATTR(any, "config:21" ); /* v3 + */ | |
3453 | PMU_FORMAT_ATTR(inv, "config:23" ); | |
3454 | PMU_FORMAT_ATTR(cmask, "config:24-31" ); | |
3a632cb2 AK |
3455 | PMU_FORMAT_ATTR(in_tx, "config:32"); |
3456 | PMU_FORMAT_ATTR(in_tx_cp, "config:33"); | |
641cc938 JO |
3457 | |
3458 | static struct attribute *intel_arch_formats_attr[] = { | |
3459 | &format_attr_event.attr, | |
3460 | &format_attr_umask.attr, | |
3461 | &format_attr_edge.attr, | |
3462 | &format_attr_pc.attr, | |
3463 | &format_attr_inv.attr, | |
3464 | &format_attr_cmask.attr, | |
3465 | NULL, | |
3466 | }; | |
3467 | ||
0bf79d44 JO |
3468 | ssize_t intel_event_sysfs_show(char *page, u64 config) |
3469 | { | |
3470 | u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT); | |
3471 | ||
3472 | return x86_event_sysfs_show(page, config, event); | |
3473 | } | |
3474 | ||
d01b1f96 | 3475 | static struct intel_shared_regs *allocate_shared_regs(int cpu) |
efc9f05d SE |
3476 | { |
3477 | struct intel_shared_regs *regs; | |
3478 | int i; | |
3479 | ||
3480 | regs = kzalloc_node(sizeof(struct intel_shared_regs), | |
3481 | GFP_KERNEL, cpu_to_node(cpu)); | |
3482 | if (regs) { | |
3483 | /* | |
3484 | * initialize the locks to keep lockdep happy | |
3485 | */ | |
3486 | for (i = 0; i < EXTRA_REG_MAX; i++) | |
3487 | raw_spin_lock_init(®s->regs[i].lock); | |
3488 | ||
3489 | regs->core_id = -1; | |
3490 | } | |
3491 | return regs; | |
3492 | } | |
3493 | ||
6f6539ca MD |
3494 | static struct intel_excl_cntrs *allocate_excl_cntrs(int cpu) |
3495 | { | |
3496 | struct intel_excl_cntrs *c; | |
6f6539ca MD |
3497 | |
3498 | c = kzalloc_node(sizeof(struct intel_excl_cntrs), | |
3499 | GFP_KERNEL, cpu_to_node(cpu)); | |
3500 | if (c) { | |
3501 | raw_spin_lock_init(&c->lock); | |
6f6539ca MD |
3502 | c->core_id = -1; |
3503 | } | |
3504 | return c; | |
3505 | } | |
3506 | ||
a7e3ed1e | 3507 | |
d01b1f96 PZI |
3508 | int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu) |
3509 | { | |
6f6539ca MD |
3510 | if (x86_pmu.extra_regs || x86_pmu.lbr_sel_map) { |
3511 | cpuc->shared_regs = allocate_shared_regs(cpu); | |
3512 | if (!cpuc->shared_regs) | |
dbc72b7a | 3513 | goto err; |
6f6539ca | 3514 | } |
69092624 | 3515 | |
400816f6 | 3516 | if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA)) { |
6f6539ca MD |
3517 | size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint); |
3518 | ||
d01b1f96 | 3519 | cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu)); |
6f6539ca | 3520 | if (!cpuc->constraint_list) |
dbc72b7a | 3521 | goto err_shared_regs; |
400816f6 | 3522 | } |
6f6539ca | 3523 | |
400816f6 | 3524 | if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) { |
6f6539ca | 3525 | cpuc->excl_cntrs = allocate_excl_cntrs(cpu); |
dbc72b7a PZ |
3526 | if (!cpuc->excl_cntrs) |
3527 | goto err_constraint_list; | |
3528 | ||
6f6539ca MD |
3529 | cpuc->excl_thread_id = 0; |
3530 | } | |
a7e3ed1e | 3531 | |
95ca792c | 3532 | return 0; |
dbc72b7a PZ |
3533 | |
3534 | err_constraint_list: | |
3535 | kfree(cpuc->constraint_list); | |
3536 | cpuc->constraint_list = NULL; | |
3537 | ||
3538 | err_shared_regs: | |
3539 | kfree(cpuc->shared_regs); | |
3540 | cpuc->shared_regs = NULL; | |
3541 | ||
3542 | err: | |
95ca792c | 3543 | return -ENOMEM; |
a7e3ed1e AK |
3544 | } |
3545 | ||
d01b1f96 PZI |
3546 | static int intel_pmu_cpu_prepare(int cpu) |
3547 | { | |
3548 | return intel_cpuc_prepare(&per_cpu(cpu_hw_events, cpu), cpu); | |
3549 | } | |
3550 | ||
6089327f KL |
3551 | static void flip_smm_bit(void *data) |
3552 | { | |
3553 | unsigned long set = *(unsigned long *)data; | |
3554 | ||
3555 | if (set > 0) { | |
3556 | msr_set_bit(MSR_IA32_DEBUGCTLMSR, | |
3557 | DEBUGCTLMSR_FREEZE_IN_SMM_BIT); | |
3558 | } else { | |
3559 | msr_clear_bit(MSR_IA32_DEBUGCTLMSR, | |
3560 | DEBUGCTLMSR_FREEZE_IN_SMM_BIT); | |
3561 | } | |
3562 | } | |
3563 | ||
74846d35 PZ |
3564 | static void intel_pmu_cpu_starting(int cpu) |
3565 | { | |
a7e3ed1e AK |
3566 | struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); |
3567 | int core_id = topology_core_id(cpu); | |
3568 | int i; | |
3569 | ||
69092624 LM |
3570 | init_debug_store_on_cpu(cpu); |
3571 | /* | |
3572 | * Deal with CPUs that don't clear their LBRs on power-up. | |
3573 | */ | |
3574 | intel_pmu_lbr_reset(); | |
3575 | ||
b36817e8 SE |
3576 | cpuc->lbr_sel = NULL; |
3577 | ||
4e949e9b KL |
3578 | if (x86_pmu.version > 1) |
3579 | flip_smm_bit(&x86_pmu.attr_freeze_on_smi); | |
6089327f | 3580 | |
af3bdb99 AK |
3581 | if (x86_pmu.counter_freezing) |
3582 | enable_counter_freeze(); | |
3583 | ||
b36817e8 | 3584 | if (!cpuc->shared_regs) |
69092624 LM |
3585 | return; |
3586 | ||
9a5e3fb5 | 3587 | if (!(x86_pmu.flags & PMU_FL_NO_HT_SHARING)) { |
06931e62 | 3588 | for_each_cpu(i, topology_sibling_cpumask(cpu)) { |
b36817e8 | 3589 | struct intel_shared_regs *pc; |
a7e3ed1e | 3590 | |
b36817e8 SE |
3591 | pc = per_cpu(cpu_hw_events, i).shared_regs; |
3592 | if (pc && pc->core_id == core_id) { | |
8f04b853 | 3593 | cpuc->kfree_on_online[0] = cpuc->shared_regs; |
b36817e8 SE |
3594 | cpuc->shared_regs = pc; |
3595 | break; | |
3596 | } | |
a7e3ed1e | 3597 | } |
b36817e8 SE |
3598 | cpuc->shared_regs->core_id = core_id; |
3599 | cpuc->shared_regs->refcnt++; | |
a7e3ed1e AK |
3600 | } |
3601 | ||
b36817e8 SE |
3602 | if (x86_pmu.lbr_sel_map) |
3603 | cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR]; | |
6f6539ca MD |
3604 | |
3605 | if (x86_pmu.flags & PMU_FL_EXCL_CNTRS) { | |
06931e62 | 3606 | for_each_cpu(i, topology_sibling_cpumask(cpu)) { |
4e71de79 | 3607 | struct cpu_hw_events *sibling; |
6f6539ca MD |
3608 | struct intel_excl_cntrs *c; |
3609 | ||
4e71de79 ZC |
3610 | sibling = &per_cpu(cpu_hw_events, i); |
3611 | c = sibling->excl_cntrs; | |
6f6539ca MD |
3612 | if (c && c->core_id == core_id) { |
3613 | cpuc->kfree_on_online[1] = cpuc->excl_cntrs; | |
3614 | cpuc->excl_cntrs = c; | |
4e71de79 ZC |
3615 | if (!sibling->excl_thread_id) |
3616 | cpuc->excl_thread_id = 1; | |
6f6539ca MD |
3617 | break; |
3618 | } | |
3619 | } | |
3620 | cpuc->excl_cntrs->core_id = core_id; | |
3621 | cpuc->excl_cntrs->refcnt++; | |
3622 | } | |
74846d35 PZ |
3623 | } |
3624 | ||
d01b1f96 | 3625 | static void free_excl_cntrs(struct cpu_hw_events *cpuc) |
74846d35 | 3626 | { |
6f6539ca | 3627 | struct intel_excl_cntrs *c; |
a7e3ed1e | 3628 | |
6f6539ca MD |
3629 | c = cpuc->excl_cntrs; |
3630 | if (c) { | |
3631 | if (c->core_id == -1 || --c->refcnt == 0) | |
3632 | kfree(c); | |
3633 | cpuc->excl_cntrs = NULL; | |
6f6539ca | 3634 | } |
400816f6 PZI |
3635 | |
3636 | kfree(cpuc->constraint_list); | |
3637 | cpuc->constraint_list = NULL; | |
b37609c3 | 3638 | } |
a7e3ed1e | 3639 | |
b37609c3 | 3640 | static void intel_pmu_cpu_dying(int cpu) |
602cae04 PZ |
3641 | { |
3642 | fini_debug_store_on_cpu(cpu); | |
3643 | ||
3644 | if (x86_pmu.counter_freezing) | |
3645 | disable_counter_freeze(); | |
3646 | } | |
3647 | ||
d01b1f96 | 3648 | void intel_cpuc_finish(struct cpu_hw_events *cpuc) |
b37609c3 | 3649 | { |
b37609c3 SE |
3650 | struct intel_shared_regs *pc; |
3651 | ||
3652 | pc = cpuc->shared_regs; | |
3653 | if (pc) { | |
3654 | if (pc->core_id == -1 || --pc->refcnt == 0) | |
3655 | kfree(pc); | |
3656 | cpuc->shared_regs = NULL; | |
e979121b MD |
3657 | } |
3658 | ||
d01b1f96 PZI |
3659 | free_excl_cntrs(cpuc); |
3660 | } | |
3661 | ||
3662 | static void intel_pmu_cpu_dead(int cpu) | |
3663 | { | |
3664 | intel_cpuc_finish(&per_cpu(cpu_hw_events, cpu)); | |
74846d35 PZ |
3665 | } |
3666 | ||
9c964efa YZ |
3667 | static void intel_pmu_sched_task(struct perf_event_context *ctx, |
3668 | bool sched_in) | |
3669 | { | |
df6c3db8 JO |
3670 | intel_pmu_pebs_sched_task(ctx, sched_in); |
3671 | intel_pmu_lbr_sched_task(ctx, sched_in); | |
9c964efa YZ |
3672 | } |
3673 | ||
81ec3f3c JO |
3674 | static int intel_pmu_check_period(struct perf_event *event, u64 value) |
3675 | { | |
3676 | return intel_pmu_has_bts_period(event, value) ? -EINVAL : 0; | |
3677 | } | |
3678 | ||
641cc938 JO |
3679 | PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63"); |
3680 | ||
a63fcab4 SE |
3681 | PMU_FORMAT_ATTR(ldlat, "config1:0-15"); |
3682 | ||
d0dc8494 AK |
3683 | PMU_FORMAT_ATTR(frontend, "config1:0-23"); |
3684 | ||
641cc938 JO |
3685 | static struct attribute *intel_arch3_formats_attr[] = { |
3686 | &format_attr_event.attr, | |
3687 | &format_attr_umask.attr, | |
3688 | &format_attr_edge.attr, | |
3689 | &format_attr_pc.attr, | |
3690 | &format_attr_any.attr, | |
3691 | &format_attr_inv.attr, | |
3692 | &format_attr_cmask.attr, | |
a5df70c3 AK |
3693 | NULL, |
3694 | }; | |
3695 | ||
3696 | static struct attribute *hsw_format_attr[] = { | |
3a632cb2 AK |
3697 | &format_attr_in_tx.attr, |
3698 | &format_attr_in_tx_cp.attr, | |
a5df70c3 AK |
3699 | &format_attr_offcore_rsp.attr, |
3700 | &format_attr_ldlat.attr, | |
3701 | NULL | |
3702 | }; | |
641cc938 | 3703 | |
a5df70c3 AK |
3704 | static struct attribute *nhm_format_attr[] = { |
3705 | &format_attr_offcore_rsp.attr, | |
3706 | &format_attr_ldlat.attr, | |
3707 | NULL | |
3708 | }; | |
3709 | ||
3710 | static struct attribute *slm_format_attr[] = { | |
3711 | &format_attr_offcore_rsp.attr, | |
3712 | NULL | |
641cc938 JO |
3713 | }; |
3714 | ||
d0dc8494 AK |
3715 | static struct attribute *skl_format_attr[] = { |
3716 | &format_attr_frontend.attr, | |
3717 | NULL, | |
3718 | }; | |
3719 | ||
3b6e0421 JO |
3720 | static __initconst const struct x86_pmu core_pmu = { |
3721 | .name = "core", | |
3722 | .handle_irq = x86_pmu_handle_irq, | |
3723 | .disable_all = x86_pmu_disable_all, | |
3724 | .enable_all = core_pmu_enable_all, | |
3725 | .enable = core_pmu_enable_event, | |
3726 | .disable = x86_pmu_disable_event, | |
ed6101bb | 3727 | .hw_config = core_pmu_hw_config, |
3b6e0421 JO |
3728 | .schedule_events = x86_schedule_events, |
3729 | .eventsel = MSR_ARCH_PERFMON_EVENTSEL0, | |
3730 | .perfctr = MSR_ARCH_PERFMON_PERFCTR0, | |
3731 | .event_map = intel_pmu_event_map, | |
3732 | .max_events = ARRAY_SIZE(intel_perfmon_event_map), | |
3733 | .apic = 1, | |
174afc3e | 3734 | .large_pebs_flags = LARGE_PEBS_FLAGS, |
a7b58d21 | 3735 | |
3b6e0421 JO |
3736 | /* |
3737 | * Intel PMCs cannot be accessed sanely above 32-bit width, | |
3738 | * so we install an artificial 1<<31 period regardless of | |
3739 | * the generic event period: | |
3740 | */ | |
3741 | .max_period = (1ULL<<31) - 1, | |
3742 | .get_event_constraints = intel_get_event_constraints, | |
3743 | .put_event_constraints = intel_put_event_constraints, | |
3744 | .event_constraints = intel_core_event_constraints, | |
3745 | .guest_get_msrs = core_guest_get_msrs, | |
3746 | .format_attrs = intel_arch_formats_attr, | |
3747 | .events_sysfs_show = intel_event_sysfs_show, | |
3748 | ||
3749 | /* | |
3750 | * Virtual (or funny metal) CPU can define x86_pmu.extra_regs | |
3751 | * together with PMU version 1 and thus be using core_pmu with | |
3752 | * shared_regs. We need following callbacks here to allocate | |
3753 | * it properly. | |
3754 | */ | |
3755 | .cpu_prepare = intel_pmu_cpu_prepare, | |
3756 | .cpu_starting = intel_pmu_cpu_starting, | |
3757 | .cpu_dying = intel_pmu_cpu_dying, | |
602cae04 | 3758 | .cpu_dead = intel_pmu_cpu_dead, |
81ec3f3c JO |
3759 | |
3760 | .check_period = intel_pmu_check_period, | |
3b6e0421 JO |
3761 | }; |
3762 | ||
4e949e9b KL |
3763 | static struct attribute *intel_pmu_attrs[]; |
3764 | ||
caaa8be3 | 3765 | static __initconst const struct x86_pmu intel_pmu = { |
f22f54f4 PZ |
3766 | .name = "Intel", |
3767 | .handle_irq = intel_pmu_handle_irq, | |
3768 | .disable_all = intel_pmu_disable_all, | |
3769 | .enable_all = intel_pmu_enable_all, | |
3770 | .enable = intel_pmu_enable_event, | |
3771 | .disable = intel_pmu_disable_event, | |
68f7082f PZ |
3772 | .add = intel_pmu_add_event, |
3773 | .del = intel_pmu_del_event, | |
ceb90d9e | 3774 | .read = intel_pmu_read_event, |
b4cdc5c2 | 3775 | .hw_config = intel_pmu_hw_config, |
a072738e | 3776 | .schedule_events = x86_schedule_events, |
f22f54f4 PZ |
3777 | .eventsel = MSR_ARCH_PERFMON_EVENTSEL0, |
3778 | .perfctr = MSR_ARCH_PERFMON_PERFCTR0, | |
3779 | .event_map = intel_pmu_event_map, | |
f22f54f4 PZ |
3780 | .max_events = ARRAY_SIZE(intel_perfmon_event_map), |
3781 | .apic = 1, | |
174afc3e | 3782 | .large_pebs_flags = LARGE_PEBS_FLAGS, |
f22f54f4 PZ |
3783 | /* |
3784 | * Intel PMCs cannot be accessed sanely above 32 bit width, | |
3785 | * so we install an artificial 1<<31 period regardless of | |
3786 | * the generic event period: | |
3787 | */ | |
3788 | .max_period = (1ULL << 31) - 1, | |
3f6da390 | 3789 | .get_event_constraints = intel_get_event_constraints, |
a7e3ed1e | 3790 | .put_event_constraints = intel_put_event_constraints, |
0780c927 | 3791 | .pebs_aliases = intel_pebs_aliases_core2, |
3f6da390 | 3792 | |
641cc938 | 3793 | .format_attrs = intel_arch3_formats_attr, |
0bf79d44 | 3794 | .events_sysfs_show = intel_event_sysfs_show, |
641cc938 | 3795 | |
4e949e9b KL |
3796 | .attrs = intel_pmu_attrs, |
3797 | ||
a7e3ed1e | 3798 | .cpu_prepare = intel_pmu_cpu_prepare, |
74846d35 PZ |
3799 | .cpu_starting = intel_pmu_cpu_starting, |
3800 | .cpu_dying = intel_pmu_cpu_dying, | |
602cae04 PZ |
3801 | .cpu_dead = intel_pmu_cpu_dead, |
3802 | ||
144d31e6 | 3803 | .guest_get_msrs = intel_guest_get_msrs, |
9c964efa | 3804 | .sched_task = intel_pmu_sched_task, |
81ec3f3c JO |
3805 | |
3806 | .check_period = intel_pmu_check_period, | |
f22f54f4 PZ |
3807 | }; |
3808 | ||
c1d6f42f | 3809 | static __init void intel_clovertown_quirk(void) |
3c44780b PZ |
3810 | { |
3811 | /* | |
3812 | * PEBS is unreliable due to: | |
3813 | * | |
3814 | * AJ67 - PEBS may experience CPL leaks | |
3815 | * AJ68 - PEBS PMI may be delayed by one event | |
3816 | * AJ69 - GLOBAL_STATUS[62] will only be set when DEBUGCTL[12] | |
3817 | * AJ106 - FREEZE_LBRS_ON_PMI doesn't work in combination with PEBS | |
3818 | * | |
3819 | * AJ67 could be worked around by restricting the OS/USR flags. | |
3820 | * AJ69 could be worked around by setting PMU_FREEZE_ON_PMI. | |
3821 | * | |
3822 | * AJ106 could possibly be worked around by not allowing LBR | |
3823 | * usage from PEBS, including the fixup. | |
3824 | * AJ68 could possibly be worked around by always programming | |
ec75a716 | 3825 | * a pebs_event_reset[0] value and coping with the lost events. |
3c44780b PZ |
3826 | * |
3827 | * But taken together it might just make sense to not enable PEBS on | |
3828 | * these chips. | |
3829 | */ | |
c767a54b | 3830 | pr_warn("PEBS disabled due to CPU errata\n"); |
3c44780b PZ |
3831 | x86_pmu.pebs = 0; |
3832 | x86_pmu.pebs_constraints = NULL; | |
3833 | } | |
3834 | ||
9b545c04 AK |
3835 | static const struct x86_cpu_desc isolation_ucodes[] = { |
3836 | INTEL_CPU_DESC(INTEL_FAM6_HASWELL_CORE, 3, 0x0000001f), | |
3837 | INTEL_CPU_DESC(INTEL_FAM6_HASWELL_ULT, 1, 0x0000001e), | |
3838 | INTEL_CPU_DESC(INTEL_FAM6_HASWELL_GT3E, 1, 0x00000015), | |
3839 | INTEL_CPU_DESC(INTEL_FAM6_HASWELL_X, 2, 0x00000037), | |
3840 | INTEL_CPU_DESC(INTEL_FAM6_HASWELL_X, 4, 0x0000000a), | |
3841 | INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_CORE, 4, 0x00000023), | |
3842 | INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_GT3E, 1, 0x00000014), | |
3843 | INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_XEON_D, 2, 0x00000010), | |
3844 | INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_XEON_D, 3, 0x07000009), | |
3845 | INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_XEON_D, 4, 0x0f000009), | |
3846 | INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_XEON_D, 5, 0x0e000002), | |
3847 | INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_X, 2, 0x0b000014), | |
3848 | INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 3, 0x00000021), | |
3849 | INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 4, 0x00000000), | |
3850 | INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_MOBILE, 3, 0x0000007c), | |
3851 | INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_DESKTOP, 3, 0x0000007c), | |
3852 | INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_DESKTOP, 9, 0x0000004e), | |
3853 | INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_MOBILE, 9, 0x0000004e), | |
3854 | INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_MOBILE, 10, 0x0000004e), | |
3855 | INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_MOBILE, 11, 0x0000004e), | |
3856 | INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_MOBILE, 12, 0x0000004e), | |
3857 | INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_DESKTOP, 10, 0x0000004e), | |
3858 | INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_DESKTOP, 11, 0x0000004e), | |
3859 | INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_DESKTOP, 12, 0x0000004e), | |
3860 | INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_DESKTOP, 13, 0x0000004e), | |
3861 | {} | |
3862 | }; | |
3863 | ||
3864 | static void intel_check_pebs_isolation(void) | |
3865 | { | |
3866 | x86_pmu.pebs_no_isolation = !x86_cpu_has_min_microcode_rev(isolation_ucodes); | |
3867 | } | |
3868 | ||
3869 | static __init void intel_pebs_isolation_quirk(void) | |
3870 | { | |
3871 | WARN_ON_ONCE(x86_pmu.check_microcode); | |
3872 | x86_pmu.check_microcode = intel_check_pebs_isolation; | |
3873 | intel_check_pebs_isolation(); | |
3874 | } | |
3875 | ||
a96fff8d KL |
3876 | static const struct x86_cpu_desc pebs_ucodes[] = { |
3877 | INTEL_CPU_DESC(INTEL_FAM6_SANDYBRIDGE, 7, 0x00000028), | |
3878 | INTEL_CPU_DESC(INTEL_FAM6_SANDYBRIDGE_X, 6, 0x00000618), | |
3879 | INTEL_CPU_DESC(INTEL_FAM6_SANDYBRIDGE_X, 7, 0x0000070c), | |
3880 | {} | |
3881 | }; | |
c93dc84c | 3882 | |
a96fff8d KL |
3883 | static bool intel_snb_pebs_broken(void) |
3884 | { | |
3885 | return !x86_cpu_has_min_microcode_rev(pebs_ucodes); | |
c93dc84c PZ |
3886 | } |
3887 | ||
3888 | static void intel_snb_check_microcode(void) | |
3889 | { | |
a96fff8d | 3890 | if (intel_snb_pebs_broken() == x86_pmu.pebs_broken) |
c93dc84c PZ |
3891 | return; |
3892 | ||
3893 | /* | |
3894 | * Serialized by the microcode lock.. | |
3895 | */ | |
3896 | if (x86_pmu.pebs_broken) { | |
3897 | pr_info("PEBS enabled due to microcode update\n"); | |
3898 | x86_pmu.pebs_broken = 0; | |
3899 | } else { | |
3900 | pr_info("PEBS disabled due to CPU errata, please upgrade microcode\n"); | |
3901 | x86_pmu.pebs_broken = 1; | |
3902 | } | |
3903 | } | |
3904 | ||
19fc9ddd DCC |
3905 | static bool is_lbr_from(unsigned long msr) |
3906 | { | |
3907 | unsigned long lbr_from_nr = x86_pmu.lbr_from + x86_pmu.lbr_nr; | |
3908 | ||
3909 | return x86_pmu.lbr_from <= msr && msr < lbr_from_nr; | |
3910 | } | |
3911 | ||
338b522c KL |
3912 | /* |
3913 | * Under certain circumstances, access certain MSR may cause #GP. | |
3914 | * The function tests if the input MSR can be safely accessed. | |
3915 | */ | |
3916 | static bool check_msr(unsigned long msr, u64 mask) | |
3917 | { | |
3918 | u64 val_old, val_new, val_tmp; | |
3919 | ||
3920 | /* | |
3921 | * Read the current value, change it and read it back to see if it | |
3922 | * matches, this is needed to detect certain hardware emulators | |
3923 | * (qemu/kvm) that don't trap on the MSR access and always return 0s. | |
3924 | */ | |
3925 | if (rdmsrl_safe(msr, &val_old)) | |
3926 | return false; | |
3927 | ||
3928 | /* | |
3929 | * Only change the bits which can be updated by wrmsrl. | |
3930 | */ | |
3931 | val_tmp = val_old ^ mask; | |
19fc9ddd DCC |
3932 | |
3933 | if (is_lbr_from(msr)) | |
3934 | val_tmp = lbr_from_signext_quirk_wr(val_tmp); | |
3935 | ||
338b522c KL |
3936 | if (wrmsrl_safe(msr, val_tmp) || |
3937 | rdmsrl_safe(msr, &val_new)) | |
3938 | return false; | |
3939 | ||
19fc9ddd DCC |
3940 | /* |
3941 | * Quirk only affects validation in wrmsr(), so wrmsrl()'s value | |
3942 | * should equal rdmsrl()'s even with the quirk. | |
3943 | */ | |
338b522c KL |
3944 | if (val_new != val_tmp) |
3945 | return false; | |
3946 | ||
19fc9ddd DCC |
3947 | if (is_lbr_from(msr)) |
3948 | val_old = lbr_from_signext_quirk_wr(val_old); | |
3949 | ||
338b522c KL |
3950 | /* Here it's sure that the MSR can be safely accessed. |
3951 | * Restore the old value and return. | |
3952 | */ | |
3953 | wrmsrl(msr, val_old); | |
3954 | ||
3955 | return true; | |
3956 | } | |
3957 | ||
c1d6f42f | 3958 | static __init void intel_sandybridge_quirk(void) |
6a600a8b | 3959 | { |
c93dc84c | 3960 | x86_pmu.check_microcode = intel_snb_check_microcode; |
1ba143a5 | 3961 | cpus_read_lock(); |
c93dc84c | 3962 | intel_snb_check_microcode(); |
1ba143a5 | 3963 | cpus_read_unlock(); |
6a600a8b PZ |
3964 | } |
3965 | ||
c1d6f42f PZ |
3966 | static const struct { int id; char *name; } intel_arch_events_map[] __initconst = { |
3967 | { PERF_COUNT_HW_CPU_CYCLES, "cpu cycles" }, | |
3968 | { PERF_COUNT_HW_INSTRUCTIONS, "instructions" }, | |
3969 | { PERF_COUNT_HW_BUS_CYCLES, "bus cycles" }, | |
3970 | { PERF_COUNT_HW_CACHE_REFERENCES, "cache references" }, | |
3971 | { PERF_COUNT_HW_CACHE_MISSES, "cache misses" }, | |
3972 | { PERF_COUNT_HW_BRANCH_INSTRUCTIONS, "branch instructions" }, | |
3973 | { PERF_COUNT_HW_BRANCH_MISSES, "branch misses" }, | |
ffb871bc GN |
3974 | }; |
3975 | ||
c1d6f42f PZ |
3976 | static __init void intel_arch_events_quirk(void) |
3977 | { | |
3978 | int bit; | |
3979 | ||
3980 | /* disable event that reported as not presend by cpuid */ | |
3981 | for_each_set_bit(bit, x86_pmu.events_mask, ARRAY_SIZE(intel_arch_events_map)) { | |
3982 | intel_perfmon_event_map[intel_arch_events_map[bit].id] = 0; | |
c767a54b JP |
3983 | pr_warn("CPUID marked event: \'%s\' unavailable\n", |
3984 | intel_arch_events_map[bit].name); | |
c1d6f42f PZ |
3985 | } |
3986 | } | |
3987 | ||
3988 | static __init void intel_nehalem_quirk(void) | |
3989 | { | |
3990 | union cpuid10_ebx ebx; | |
3991 | ||
3992 | ebx.full = x86_pmu.events_maskl; | |
3993 | if (ebx.split.no_branch_misses_retired) { | |
3994 | /* | |
3995 | * Erratum AAJ80 detected, we work it around by using | |
3996 | * the BR_MISP_EXEC.ANY event. This will over-count | |
3997 | * branch-misses, but it's still much better than the | |
3998 | * architectural event which is often completely bogus: | |
3999 | */ | |
4000 | intel_perfmon_event_map[PERF_COUNT_HW_BRANCH_MISSES] = 0x7f89; | |
4001 | ebx.split.no_branch_misses_retired = 0; | |
4002 | x86_pmu.events_maskl = ebx.full; | |
c767a54b | 4003 | pr_info("CPU erratum AAJ80 worked around\n"); |
c1d6f42f PZ |
4004 | } |
4005 | } | |
4006 | ||
bef9f271 | 4007 | static const struct x86_cpu_desc counter_freezing_ucodes[] = { |
af63147c KL |
4008 | INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT, 2, 0x0000000e), |
4009 | INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT, 9, 0x0000002e), | |
4010 | INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT, 10, 0x00000008), | |
4011 | INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT_X, 1, 0x00000028), | |
4012 | INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT_PLUS, 1, 0x00000028), | |
4013 | INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT_PLUS, 8, 0x00000006), | |
bef9f271 KL |
4014 | {} |
4015 | }; | |
7c5314b8 | 4016 | |
bef9f271 KL |
4017 | static bool intel_counter_freezing_broken(void) |
4018 | { | |
4019 | return !x86_cpu_has_min_microcode_rev(counter_freezing_ucodes); | |
7c5314b8 KL |
4020 | } |
4021 | ||
bef9f271 | 4022 | static __init void intel_counter_freezing_quirk(void) |
7c5314b8 KL |
4023 | { |
4024 | /* Check if it's already disabled */ | |
4025 | if (disable_counter_freezing) | |
4026 | return; | |
4027 | ||
4028 | /* | |
4029 | * If the system starts with the wrong ucode, leave the | |
4030 | * counter-freezing feature permanently disabled. | |
4031 | */ | |
bef9f271 | 4032 | if (intel_counter_freezing_broken()) { |
7c5314b8 KL |
4033 | pr_info("PMU counter freezing disabled due to CPU errata," |
4034 | "please upgrade microcode\n"); | |
4035 | x86_pmu.counter_freezing = false; | |
4036 | x86_pmu.handle_irq = intel_pmu_handle_irq; | |
4037 | } | |
4038 | } | |
4039 | ||
93fcf72c MD |
4040 | /* |
4041 | * enable software workaround for errata: | |
4042 | * SNB: BJ122 | |
4043 | * IVB: BV98 | |
4044 | * HSW: HSD29 | |
4045 | * | |
4046 | * Only needed when HT is enabled. However detecting | |
b37609c3 SE |
4047 | * if HT is enabled is difficult (model specific). So instead, |
4048 | * we enable the workaround in the early boot, and verify if | |
4049 | * it is needed in a later initcall phase once we have valid | |
4050 | * topology information to check if HT is actually enabled | |
93fcf72c MD |
4051 | */ |
4052 | static __init void intel_ht_bug(void) | |
4053 | { | |
b37609c3 | 4054 | x86_pmu.flags |= PMU_FL_EXCL_CNTRS | PMU_FL_EXCL_ENABLED; |
93fcf72c | 4055 | |
93fcf72c | 4056 | x86_pmu.start_scheduling = intel_start_scheduling; |
0c41e756 | 4057 | x86_pmu.commit_scheduling = intel_commit_scheduling; |
93fcf72c | 4058 | x86_pmu.stop_scheduling = intel_stop_scheduling; |
93fcf72c MD |
4059 | } |
4060 | ||
7f2ee91f IM |
4061 | EVENT_ATTR_STR(mem-loads, mem_ld_hsw, "event=0xcd,umask=0x1,ldlat=3"); |
4062 | EVENT_ATTR_STR(mem-stores, mem_st_hsw, "event=0xd0,umask=0x82") | |
f9134f36 | 4063 | |
4b2c4f1f | 4064 | /* Haswell special events */ |
7f2ee91f IM |
4065 | EVENT_ATTR_STR(tx-start, tx_start, "event=0xc9,umask=0x1"); |
4066 | EVENT_ATTR_STR(tx-commit, tx_commit, "event=0xc9,umask=0x2"); | |
4067 | EVENT_ATTR_STR(tx-abort, tx_abort, "event=0xc9,umask=0x4"); | |
4068 | EVENT_ATTR_STR(tx-capacity, tx_capacity, "event=0x54,umask=0x2"); | |
4069 | EVENT_ATTR_STR(tx-conflict, tx_conflict, "event=0x54,umask=0x1"); | |
4070 | EVENT_ATTR_STR(el-start, el_start, "event=0xc8,umask=0x1"); | |
4071 | EVENT_ATTR_STR(el-commit, el_commit, "event=0xc8,umask=0x2"); | |
4072 | EVENT_ATTR_STR(el-abort, el_abort, "event=0xc8,umask=0x4"); | |
4073 | EVENT_ATTR_STR(el-capacity, el_capacity, "event=0x54,umask=0x2"); | |
4074 | EVENT_ATTR_STR(el-conflict, el_conflict, "event=0x54,umask=0x1"); | |
4075 | EVENT_ATTR_STR(cycles-t, cycles_t, "event=0x3c,in_tx=1"); | |
4076 | EVENT_ATTR_STR(cycles-ct, cycles_ct, "event=0x3c,in_tx=1,in_tx_cp=1"); | |
4b2c4f1f | 4077 | |
f9134f36 | 4078 | static struct attribute *hsw_events_attrs[] = { |
58ba4d5a AK |
4079 | EVENT_PTR(td_slots_issued), |
4080 | EVENT_PTR(td_slots_retired), | |
4081 | EVENT_PTR(td_fetch_bubbles), | |
4082 | EVENT_PTR(td_total_slots), | |
4083 | EVENT_PTR(td_total_slots_scale), | |
4084 | EVENT_PTR(td_recovery_bubbles), | |
4085 | EVENT_PTR(td_recovery_bubbles_scale), | |
4086 | NULL | |
4087 | }; | |
4088 | ||
d4ae5529 JO |
4089 | static struct attribute *hsw_mem_events_attrs[] = { |
4090 | EVENT_PTR(mem_ld_hsw), | |
4091 | EVENT_PTR(mem_st_hsw), | |
4092 | NULL, | |
4093 | }; | |
4094 | ||
58ba4d5a | 4095 | static struct attribute *hsw_tsx_events_attrs[] = { |
4b2c4f1f AK |
4096 | EVENT_PTR(tx_start), |
4097 | EVENT_PTR(tx_commit), | |
4098 | EVENT_PTR(tx_abort), | |
4099 | EVENT_PTR(tx_capacity), | |
4100 | EVENT_PTR(tx_conflict), | |
4101 | EVENT_PTR(el_start), | |
4102 | EVENT_PTR(el_commit), | |
4103 | EVENT_PTR(el_abort), | |
4104 | EVENT_PTR(el_capacity), | |
4105 | EVENT_PTR(el_conflict), | |
4106 | EVENT_PTR(cycles_t), | |
4107 | EVENT_PTR(cycles_ct), | |
f9134f36 AK |
4108 | NULL |
4109 | }; | |
4110 | ||
6089327f KL |
4111 | static ssize_t freeze_on_smi_show(struct device *cdev, |
4112 | struct device_attribute *attr, | |
4113 | char *buf) | |
4114 | { | |
4115 | return sprintf(buf, "%lu\n", x86_pmu.attr_freeze_on_smi); | |
4116 | } | |
4117 | ||
4118 | static DEFINE_MUTEX(freeze_on_smi_mutex); | |
4119 | ||
4120 | static ssize_t freeze_on_smi_store(struct device *cdev, | |
4121 | struct device_attribute *attr, | |
4122 | const char *buf, size_t count) | |
4123 | { | |
4124 | unsigned long val; | |
4125 | ssize_t ret; | |
4126 | ||
4127 | ret = kstrtoul(buf, 0, &val); | |
4128 | if (ret) | |
4129 | return ret; | |
4130 | ||
4131 | if (val > 1) | |
4132 | return -EINVAL; | |
4133 | ||
4134 | mutex_lock(&freeze_on_smi_mutex); | |
4135 | ||
4136 | if (x86_pmu.attr_freeze_on_smi == val) | |
4137 | goto done; | |
4138 | ||
4139 | x86_pmu.attr_freeze_on_smi = val; | |
4140 | ||
4141 | get_online_cpus(); | |
4142 | on_each_cpu(flip_smm_bit, &val, 1); | |
4143 | put_online_cpus(); | |
4144 | done: | |
4145 | mutex_unlock(&freeze_on_smi_mutex); | |
4146 | ||
4147 | return count; | |
4148 | } | |
4149 | ||
4150 | static DEVICE_ATTR_RW(freeze_on_smi); | |
4151 | ||
b00233b5 AK |
4152 | static ssize_t branches_show(struct device *cdev, |
4153 | struct device_attribute *attr, | |
4154 | char *buf) | |
4155 | { | |
4156 | return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu.lbr_nr); | |
4157 | } | |
4158 | ||
4159 | static DEVICE_ATTR_RO(branches); | |
4160 | ||
4161 | static struct attribute *lbr_attrs[] = { | |
4162 | &dev_attr_branches.attr, | |
4163 | NULL | |
4164 | }; | |
4165 | ||
4166 | static char pmu_name_str[30]; | |
4167 | ||
4168 | static ssize_t pmu_name_show(struct device *cdev, | |
4169 | struct device_attribute *attr, | |
4170 | char *buf) | |
4171 | { | |
4172 | return snprintf(buf, PAGE_SIZE, "%s\n", pmu_name_str); | |
4173 | } | |
4174 | ||
4175 | static DEVICE_ATTR_RO(pmu_name); | |
4176 | ||
b00233b5 | 4177 | static struct attribute *intel_pmu_caps_attrs[] = { |
5da382eb PZ |
4178 | &dev_attr_pmu_name.attr, |
4179 | NULL | |
b00233b5 AK |
4180 | }; |
4181 | ||
c634dc6b | 4182 | static DEVICE_BOOL_ATTR(allow_tsx_force_abort, 0644, allow_tsx_force_abort); |
400816f6 | 4183 | |
6089327f KL |
4184 | static struct attribute *intel_pmu_attrs[] = { |
4185 | &dev_attr_freeze_on_smi.attr, | |
400816f6 | 4186 | NULL, /* &dev_attr_allow_tsx_force_abort.attr.attr */ |
6089327f KL |
4187 | NULL, |
4188 | }; | |
4189 | ||
d4ae5529 JO |
4190 | static __init struct attribute ** |
4191 | get_events_attrs(struct attribute **base, | |
4192 | struct attribute **mem, | |
4193 | struct attribute **tsx) | |
4194 | { | |
4195 | struct attribute **attrs = base; | |
4196 | struct attribute **old; | |
4197 | ||
4198 | if (mem && x86_pmu.pebs) | |
4199 | attrs = merge_attr(attrs, mem); | |
4200 | ||
4201 | if (tsx && boot_cpu_has(X86_FEATURE_RTM)) { | |
4202 | old = attrs; | |
4203 | attrs = merge_attr(attrs, tsx); | |
4204 | if (old != base) | |
4205 | kfree(old); | |
4206 | } | |
4207 | ||
4208 | return attrs; | |
4209 | } | |
4210 | ||
de0428a7 | 4211 | __init int intel_pmu_init(void) |
f22f54f4 | 4212 | { |
7ad1437d | 4213 | struct attribute **extra_attr = NULL; |
d4ae5529 JO |
4214 | struct attribute **mem_attr = NULL; |
4215 | struct attribute **tsx_attr = NULL; | |
7ad1437d | 4216 | struct attribute **to_free = NULL; |
f22f54f4 PZ |
4217 | union cpuid10_edx edx; |
4218 | union cpuid10_eax eax; | |
ffb871bc | 4219 | union cpuid10_ebx ebx; |
a1eac7ac | 4220 | struct event_constraint *c; |
f22f54f4 | 4221 | unsigned int unused; |
338b522c KL |
4222 | struct extra_reg *er; |
4223 | int version, i; | |
b00233b5 | 4224 | char *name; |
f22f54f4 PZ |
4225 | |
4226 | if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) { | |
a072738e CG |
4227 | switch (boot_cpu_data.x86) { |
4228 | case 0x6: | |
4229 | return p6_pmu_init(); | |
e717bf4e VW |
4230 | case 0xb: |
4231 | return knc_pmu_init(); | |
a072738e CG |
4232 | case 0xf: |
4233 | return p4_pmu_init(); | |
4234 | } | |
f22f54f4 | 4235 | return -ENODEV; |
f22f54f4 PZ |
4236 | } |
4237 | ||
4238 | /* | |
4239 | * Check whether the Architectural PerfMon supports | |
4240 | * Branch Misses Retired hw_event or not. | |
4241 | */ | |
ffb871bc GN |
4242 | cpuid(10, &eax.full, &ebx.full, &unused, &edx.full); |
4243 | if (eax.split.mask_length < ARCH_PERFMON_EVENTS_COUNT) | |
f22f54f4 PZ |
4244 | return -ENODEV; |
4245 | ||
4246 | version = eax.split.version_id; | |
4247 | if (version < 2) | |
4248 | x86_pmu = core_pmu; | |
4249 | else | |
4250 | x86_pmu = intel_pmu; | |
4251 | ||
4252 | x86_pmu.version = version; | |
948b1bb8 RR |
4253 | x86_pmu.num_counters = eax.split.num_counters; |
4254 | x86_pmu.cntval_bits = eax.split.bit_width; | |
4255 | x86_pmu.cntval_mask = (1ULL << eax.split.bit_width) - 1; | |
f22f54f4 | 4256 | |
c1d6f42f PZ |
4257 | x86_pmu.events_maskl = ebx.full; |
4258 | x86_pmu.events_mask_len = eax.split.mask_length; | |
4259 | ||
70ab7003 AK |
4260 | x86_pmu.max_pebs_events = min_t(unsigned, MAX_PEBS_EVENTS, x86_pmu.num_counters); |
4261 | ||
f22f54f4 PZ |
4262 | /* |
4263 | * Quirk: v2 perfmon does not report fixed-purpose events, so | |
f92b7604 | 4264 | * assume at least 3 events, when not running in a hypervisor: |
f22f54f4 | 4265 | */ |
f92b7604 IP |
4266 | if (version > 1) { |
4267 | int assume = 3 * !boot_cpu_has(X86_FEATURE_HYPERVISOR); | |
4268 | ||
4269 | x86_pmu.num_counters_fixed = | |
4270 | max((int)edx.split.num_counters_fixed, assume); | |
4271 | } | |
f22f54f4 | 4272 | |
af3bdb99 AK |
4273 | if (version >= 4) |
4274 | x86_pmu.counter_freezing = !disable_counter_freezing; | |
4275 | ||
c9b08884 | 4276 | if (boot_cpu_has(X86_FEATURE_PDCM)) { |
8db909a7 PZ |
4277 | u64 capabilities; |
4278 | ||
4279 | rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities); | |
4280 | x86_pmu.intel_cap.capabilities = capabilities; | |
4281 | } | |
4282 | ||
ca037701 PZ |
4283 | intel_ds_init(); |
4284 | ||
c1d6f42f PZ |
4285 | x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */ |
4286 | ||
f22f54f4 PZ |
4287 | /* |
4288 | * Install the hw-cache-events table: | |
4289 | */ | |
4290 | switch (boot_cpu_data.x86_model) { | |
ef5f9f47 | 4291 | case INTEL_FAM6_CORE_YONAH: |
f22f54f4 | 4292 | pr_cont("Core events, "); |
b00233b5 | 4293 | name = "core"; |
f22f54f4 PZ |
4294 | break; |
4295 | ||
ef5f9f47 | 4296 | case INTEL_FAM6_CORE2_MEROM: |
c1d6f42f | 4297 | x86_add_quirk(intel_clovertown_quirk); |
2b0fc374 GS |
4298 | /* fall through */ |
4299 | ||
ef5f9f47 DH |
4300 | case INTEL_FAM6_CORE2_MEROM_L: |
4301 | case INTEL_FAM6_CORE2_PENRYN: | |
4302 | case INTEL_FAM6_CORE2_DUNNINGTON: | |
f22f54f4 PZ |
4303 | memcpy(hw_cache_event_ids, core2_hw_cache_event_ids, |
4304 | sizeof(hw_cache_event_ids)); | |
4305 | ||
caff2bef PZ |
4306 | intel_pmu_lbr_init_core(); |
4307 | ||
f22f54f4 | 4308 | x86_pmu.event_constraints = intel_core2_event_constraints; |
17e31629 | 4309 | x86_pmu.pebs_constraints = intel_core2_pebs_event_constraints; |
f22f54f4 | 4310 | pr_cont("Core2 events, "); |
b00233b5 | 4311 | name = "core2"; |
f22f54f4 PZ |
4312 | break; |
4313 | ||
ef5f9f47 DH |
4314 | case INTEL_FAM6_NEHALEM: |
4315 | case INTEL_FAM6_NEHALEM_EP: | |
4316 | case INTEL_FAM6_NEHALEM_EX: | |
f22f54f4 PZ |
4317 | memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids, |
4318 | sizeof(hw_cache_event_ids)); | |
e994d7d2 AK |
4319 | memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs, |
4320 | sizeof(hw_cache_extra_regs)); | |
f22f54f4 | 4321 | |
caff2bef PZ |
4322 | intel_pmu_lbr_init_nhm(); |
4323 | ||
f22f54f4 | 4324 | x86_pmu.event_constraints = intel_nehalem_event_constraints; |
17e31629 | 4325 | x86_pmu.pebs_constraints = intel_nehalem_pebs_event_constraints; |
11164cd4 | 4326 | x86_pmu.enable_all = intel_pmu_nhm_enable_all; |
a7e3ed1e | 4327 | x86_pmu.extra_regs = intel_nehalem_extra_regs; |
ec75a716 | 4328 | |
d4ae5529 | 4329 | mem_attr = nhm_mem_events_attrs; |
f20093ee | 4330 | |
91fc4cc0 | 4331 | /* UOPS_ISSUED.STALLED_CYCLES */ |
f9b4eeb8 PZ |
4332 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = |
4333 | X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); | |
91fc4cc0 | 4334 | /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */ |
f9b4eeb8 PZ |
4335 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = |
4336 | X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1); | |
94403f88 | 4337 | |
e17dc653 | 4338 | intel_pmu_pebs_data_source_nhm(); |
c1d6f42f | 4339 | x86_add_quirk(intel_nehalem_quirk); |
95298355 | 4340 | x86_pmu.pebs_no_tlb = 1; |
a5df70c3 | 4341 | extra_attr = nhm_format_attr; |
ec75a716 | 4342 | |
11164cd4 | 4343 | pr_cont("Nehalem events, "); |
b00233b5 | 4344 | name = "nehalem"; |
f22f54f4 | 4345 | break; |
caff2bef | 4346 | |
f2c4db1b PZ |
4347 | case INTEL_FAM6_ATOM_BONNELL: |
4348 | case INTEL_FAM6_ATOM_BONNELL_MID: | |
4349 | case INTEL_FAM6_ATOM_SALTWELL: | |
4350 | case INTEL_FAM6_ATOM_SALTWELL_MID: | |
4351 | case INTEL_FAM6_ATOM_SALTWELL_TABLET: | |
f22f54f4 PZ |
4352 | memcpy(hw_cache_event_ids, atom_hw_cache_event_ids, |
4353 | sizeof(hw_cache_event_ids)); | |
4354 | ||
caff2bef PZ |
4355 | intel_pmu_lbr_init_atom(); |
4356 | ||
f22f54f4 | 4357 | x86_pmu.event_constraints = intel_gen_event_constraints; |
17e31629 | 4358 | x86_pmu.pebs_constraints = intel_atom_pebs_event_constraints; |
673d188b | 4359 | x86_pmu.pebs_aliases = intel_pebs_aliases_core2; |
f22f54f4 | 4360 | pr_cont("Atom events, "); |
b00233b5 | 4361 | name = "bonnell"; |
f22f54f4 PZ |
4362 | break; |
4363 | ||
f2c4db1b PZ |
4364 | case INTEL_FAM6_ATOM_SILVERMONT: |
4365 | case INTEL_FAM6_ATOM_SILVERMONT_X: | |
4366 | case INTEL_FAM6_ATOM_SILVERMONT_MID: | |
ef5f9f47 | 4367 | case INTEL_FAM6_ATOM_AIRMONT: |
f2c4db1b | 4368 | case INTEL_FAM6_ATOM_AIRMONT_MID: |
1fa64180 YZ |
4369 | memcpy(hw_cache_event_ids, slm_hw_cache_event_ids, |
4370 | sizeof(hw_cache_event_ids)); | |
4371 | memcpy(hw_cache_extra_regs, slm_hw_cache_extra_regs, | |
4372 | sizeof(hw_cache_extra_regs)); | |
4373 | ||
f21d5adc | 4374 | intel_pmu_lbr_init_slm(); |
1fa64180 YZ |
4375 | |
4376 | x86_pmu.event_constraints = intel_slm_event_constraints; | |
4377 | x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints; | |
4378 | x86_pmu.extra_regs = intel_slm_extra_regs; | |
9a5e3fb5 | 4379 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; |
eb12b8ec | 4380 | x86_pmu.cpu_events = slm_events_attrs; |
a5df70c3 | 4381 | extra_attr = slm_format_attr; |
1fa64180 | 4382 | pr_cont("Silvermont events, "); |
b00233b5 | 4383 | name = "silvermont"; |
1fa64180 YZ |
4384 | break; |
4385 | ||
ef5f9f47 | 4386 | case INTEL_FAM6_ATOM_GOLDMONT: |
f2c4db1b | 4387 | case INTEL_FAM6_ATOM_GOLDMONT_X: |
af63147c | 4388 | x86_add_quirk(intel_counter_freezing_quirk); |
8b92c3a7 KL |
4389 | memcpy(hw_cache_event_ids, glm_hw_cache_event_ids, |
4390 | sizeof(hw_cache_event_ids)); | |
4391 | memcpy(hw_cache_extra_regs, glm_hw_cache_extra_regs, | |
4392 | sizeof(hw_cache_extra_regs)); | |
4393 | ||
4394 | intel_pmu_lbr_init_skl(); | |
4395 | ||
4396 | x86_pmu.event_constraints = intel_slm_event_constraints; | |
4397 | x86_pmu.pebs_constraints = intel_glm_pebs_event_constraints; | |
4398 | x86_pmu.extra_regs = intel_glm_extra_regs; | |
4399 | /* | |
4400 | * It's recommended to use CPU_CLK_UNHALTED.CORE_P + NPEBS | |
4401 | * for precise cycles. | |
4402 | * :pp is identical to :ppp | |
4403 | */ | |
4404 | x86_pmu.pebs_aliases = NULL; | |
4405 | x86_pmu.pebs_prec_dist = true; | |
ccbebba4 | 4406 | x86_pmu.lbr_pt_coexist = true; |
8b92c3a7 | 4407 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; |
ed827adb | 4408 | x86_pmu.cpu_events = glm_events_attrs; |
a5df70c3 | 4409 | extra_attr = slm_format_attr; |
8b92c3a7 | 4410 | pr_cont("Goldmont events, "); |
b00233b5 | 4411 | name = "goldmont"; |
8b92c3a7 KL |
4412 | break; |
4413 | ||
f2c4db1b | 4414 | case INTEL_FAM6_ATOM_GOLDMONT_PLUS: |
bef9f271 | 4415 | x86_add_quirk(intel_counter_freezing_quirk); |
dd0b06b5 KL |
4416 | memcpy(hw_cache_event_ids, glp_hw_cache_event_ids, |
4417 | sizeof(hw_cache_event_ids)); | |
4418 | memcpy(hw_cache_extra_regs, glp_hw_cache_extra_regs, | |
4419 | sizeof(hw_cache_extra_regs)); | |
4420 | ||
4421 | intel_pmu_lbr_init_skl(); | |
4422 | ||
4423 | x86_pmu.event_constraints = intel_slm_event_constraints; | |
dd0b06b5 KL |
4424 | x86_pmu.extra_regs = intel_glm_extra_regs; |
4425 | /* | |
4426 | * It's recommended to use CPU_CLK_UNHALTED.CORE_P + NPEBS | |
4427 | * for precise cycles. | |
4428 | */ | |
4429 | x86_pmu.pebs_aliases = NULL; | |
4430 | x86_pmu.pebs_prec_dist = true; | |
4431 | x86_pmu.lbr_pt_coexist = true; | |
4432 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; | |
a38b0ba1 | 4433 | x86_pmu.flags |= PMU_FL_PEBS_ALL; |
dd0b06b5 KL |
4434 | x86_pmu.get_event_constraints = glp_get_event_constraints; |
4435 | x86_pmu.cpu_events = glm_events_attrs; | |
4436 | /* Goldmont Plus has 4-wide pipeline */ | |
4437 | event_attr_td_total_slots_scale_glm.event_str = "4"; | |
a5df70c3 | 4438 | extra_attr = slm_format_attr; |
dd0b06b5 | 4439 | pr_cont("Goldmont plus events, "); |
b00233b5 | 4440 | name = "goldmont_plus"; |
dd0b06b5 KL |
4441 | break; |
4442 | ||
ef5f9f47 DH |
4443 | case INTEL_FAM6_WESTMERE: |
4444 | case INTEL_FAM6_WESTMERE_EP: | |
4445 | case INTEL_FAM6_WESTMERE_EX: | |
f22f54f4 PZ |
4446 | memcpy(hw_cache_event_ids, westmere_hw_cache_event_ids, |
4447 | sizeof(hw_cache_event_ids)); | |
e994d7d2 AK |
4448 | memcpy(hw_cache_extra_regs, nehalem_hw_cache_extra_regs, |
4449 | sizeof(hw_cache_extra_regs)); | |
f22f54f4 | 4450 | |
caff2bef PZ |
4451 | intel_pmu_lbr_init_nhm(); |
4452 | ||
f22f54f4 | 4453 | x86_pmu.event_constraints = intel_westmere_event_constraints; |
40b91cd1 | 4454 | x86_pmu.enable_all = intel_pmu_nhm_enable_all; |
17e31629 | 4455 | x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints; |
a7e3ed1e | 4456 | x86_pmu.extra_regs = intel_westmere_extra_regs; |
9a5e3fb5 | 4457 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; |
30112039 | 4458 | |
d4ae5529 | 4459 | mem_attr = nhm_mem_events_attrs; |
f20093ee | 4460 | |
30112039 | 4461 | /* UOPS_ISSUED.STALLED_CYCLES */ |
f9b4eeb8 PZ |
4462 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = |
4463 | X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); | |
30112039 | 4464 | /* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */ |
f9b4eeb8 PZ |
4465 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = |
4466 | X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1); | |
30112039 | 4467 | |
e17dc653 | 4468 | intel_pmu_pebs_data_source_nhm(); |
a5df70c3 | 4469 | extra_attr = nhm_format_attr; |
f22f54f4 | 4470 | pr_cont("Westmere events, "); |
b00233b5 | 4471 | name = "westmere"; |
f22f54f4 | 4472 | break; |
b622d644 | 4473 | |
ef5f9f47 DH |
4474 | case INTEL_FAM6_SANDYBRIDGE: |
4475 | case INTEL_FAM6_SANDYBRIDGE_X: | |
47a8863d | 4476 | x86_add_quirk(intel_sandybridge_quirk); |
93fcf72c | 4477 | x86_add_quirk(intel_ht_bug); |
b06b3d49 LM |
4478 | memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, |
4479 | sizeof(hw_cache_event_ids)); | |
74e6543f YZ |
4480 | memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs, |
4481 | sizeof(hw_cache_extra_regs)); | |
b06b3d49 | 4482 | |
c5cc2cd9 | 4483 | intel_pmu_lbr_init_snb(); |
b06b3d49 LM |
4484 | |
4485 | x86_pmu.event_constraints = intel_snb_event_constraints; | |
de0428a7 | 4486 | x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints; |
0780c927 | 4487 | x86_pmu.pebs_aliases = intel_pebs_aliases_snb; |
ef5f9f47 | 4488 | if (boot_cpu_data.x86_model == INTEL_FAM6_SANDYBRIDGE_X) |
f1923820 SE |
4489 | x86_pmu.extra_regs = intel_snbep_extra_regs; |
4490 | else | |
4491 | x86_pmu.extra_regs = intel_snb_extra_regs; | |
93fcf72c MD |
4492 | |
4493 | ||
ee89cbc2 | 4494 | /* all extra regs are per-cpu when HT is on */ |
9a5e3fb5 SE |
4495 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; |
4496 | x86_pmu.flags |= PMU_FL_NO_HT_SHARING; | |
e04d1b23 | 4497 | |
f20093ee | 4498 | x86_pmu.cpu_events = snb_events_attrs; |
d4ae5529 | 4499 | mem_attr = snb_mem_events_attrs; |
f20093ee | 4500 | |
e04d1b23 | 4501 | /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ |
f9b4eeb8 PZ |
4502 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = |
4503 | X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); | |
e04d1b23 | 4504 | /* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/ |
f9b4eeb8 PZ |
4505 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = |
4506 | X86_CONFIG(.event=0xb1, .umask=0x01, .inv=1, .cmask=1); | |
e04d1b23 | 4507 | |
a5df70c3 AK |
4508 | extra_attr = nhm_format_attr; |
4509 | ||
b06b3d49 | 4510 | pr_cont("SandyBridge events, "); |
b00233b5 | 4511 | name = "sandybridge"; |
b06b3d49 | 4512 | break; |
0f7c29ce | 4513 | |
ef5f9f47 DH |
4514 | case INTEL_FAM6_IVYBRIDGE: |
4515 | case INTEL_FAM6_IVYBRIDGE_X: | |
93fcf72c | 4516 | x86_add_quirk(intel_ht_bug); |
20a36e39 SE |
4517 | memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, |
4518 | sizeof(hw_cache_event_ids)); | |
1996388e VW |
4519 | /* dTLB-load-misses on IVB is different than SNB */ |
4520 | hw_cache_event_ids[C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = 0x8108; /* DTLB_LOAD_MISSES.DEMAND_LD_MISS_CAUSES_A_WALK */ | |
4521 | ||
20a36e39 SE |
4522 | memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs, |
4523 | sizeof(hw_cache_extra_regs)); | |
4524 | ||
4525 | intel_pmu_lbr_init_snb(); | |
4526 | ||
69943182 | 4527 | x86_pmu.event_constraints = intel_ivb_event_constraints; |
20a36e39 | 4528 | x86_pmu.pebs_constraints = intel_ivb_pebs_event_constraints; |
72469764 AK |
4529 | x86_pmu.pebs_aliases = intel_pebs_aliases_ivb; |
4530 | x86_pmu.pebs_prec_dist = true; | |
ef5f9f47 | 4531 | if (boot_cpu_data.x86_model == INTEL_FAM6_IVYBRIDGE_X) |
f1923820 SE |
4532 | x86_pmu.extra_regs = intel_snbep_extra_regs; |
4533 | else | |
4534 | x86_pmu.extra_regs = intel_snb_extra_regs; | |
20a36e39 | 4535 | /* all extra regs are per-cpu when HT is on */ |
9a5e3fb5 SE |
4536 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; |
4537 | x86_pmu.flags |= PMU_FL_NO_HT_SHARING; | |
20a36e39 | 4538 | |
f20093ee | 4539 | x86_pmu.cpu_events = snb_events_attrs; |
d4ae5529 | 4540 | mem_attr = snb_mem_events_attrs; |
f20093ee | 4541 | |
20a36e39 SE |
4542 | /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ |
4543 | intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = | |
4544 | X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); | |
4545 | ||
a5df70c3 AK |
4546 | extra_attr = nhm_format_attr; |
4547 | ||
20a36e39 | 4548 | pr_cont("IvyBridge events, "); |
b00233b5 | 4549 | name = "ivybridge"; |
20a36e39 SE |
4550 | break; |
4551 | ||
b06b3d49 | 4552 | |
ef5f9f47 DH |
4553 | case INTEL_FAM6_HASWELL_CORE: |
4554 | case INTEL_FAM6_HASWELL_X: | |
4555 | case INTEL_FAM6_HASWELL_ULT: | |
4556 | case INTEL_FAM6_HASWELL_GT3E: | |
93fcf72c | 4557 | x86_add_quirk(intel_ht_bug); |
9b545c04 | 4558 | x86_add_quirk(intel_pebs_isolation_quirk); |
72db5596 | 4559 | x86_pmu.late_ack = true; |
0f1b5ca2 AK |
4560 | memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids)); |
4561 | memcpy(hw_cache_extra_regs, hsw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); | |
3a632cb2 | 4562 | |
e9d7f7cd | 4563 | intel_pmu_lbr_init_hsw(); |
3a632cb2 AK |
4564 | |
4565 | x86_pmu.event_constraints = intel_hsw_event_constraints; | |
3044318f | 4566 | x86_pmu.pebs_constraints = intel_hsw_pebs_event_constraints; |
36bbb2f2 | 4567 | x86_pmu.extra_regs = intel_snbep_extra_regs; |
72469764 AK |
4568 | x86_pmu.pebs_aliases = intel_pebs_aliases_ivb; |
4569 | x86_pmu.pebs_prec_dist = true; | |
3a632cb2 | 4570 | /* all extra regs are per-cpu when HT is on */ |
9a5e3fb5 SE |
4571 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; |
4572 | x86_pmu.flags |= PMU_FL_NO_HT_SHARING; | |
3a632cb2 AK |
4573 | |
4574 | x86_pmu.hw_config = hsw_hw_config; | |
4575 | x86_pmu.get_event_constraints = hsw_get_event_constraints; | |
d4ae5529 | 4576 | x86_pmu.cpu_events = hsw_events_attrs; |
b7af41a1 | 4577 | x86_pmu.lbr_double_abort = true; |
a5df70c3 AK |
4578 | extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? |
4579 | hsw_format_attr : nhm_format_attr; | |
d4ae5529 JO |
4580 | mem_attr = hsw_mem_events_attrs; |
4581 | tsx_attr = hsw_tsx_events_attrs; | |
3a632cb2 | 4582 | pr_cont("Haswell events, "); |
b00233b5 | 4583 | name = "haswell"; |
3a632cb2 AK |
4584 | break; |
4585 | ||
ef5f9f47 DH |
4586 | case INTEL_FAM6_BROADWELL_CORE: |
4587 | case INTEL_FAM6_BROADWELL_XEON_D: | |
4588 | case INTEL_FAM6_BROADWELL_GT3E: | |
4589 | case INTEL_FAM6_BROADWELL_X: | |
9b545c04 | 4590 | x86_add_quirk(intel_pebs_isolation_quirk); |
91f1b705 AK |
4591 | x86_pmu.late_ack = true; |
4592 | memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids)); | |
4593 | memcpy(hw_cache_extra_regs, hsw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); | |
4594 | ||
4595 | /* L3_MISS_LOCAL_DRAM is BIT(26) in Broadwell */ | |
4596 | hw_cache_extra_regs[C(LL)][C(OP_READ)][C(RESULT_MISS)] = HSW_DEMAND_READ | | |
4597 | BDW_L3_MISS|HSW_SNOOP_DRAM; | |
4598 | hw_cache_extra_regs[C(LL)][C(OP_WRITE)][C(RESULT_MISS)] = HSW_DEMAND_WRITE|BDW_L3_MISS| | |
4599 | HSW_SNOOP_DRAM; | |
4600 | hw_cache_extra_regs[C(NODE)][C(OP_READ)][C(RESULT_ACCESS)] = HSW_DEMAND_READ| | |
4601 | BDW_L3_MISS_LOCAL|HSW_SNOOP_DRAM; | |
4602 | hw_cache_extra_regs[C(NODE)][C(OP_WRITE)][C(RESULT_ACCESS)] = HSW_DEMAND_WRITE| | |
4603 | BDW_L3_MISS_LOCAL|HSW_SNOOP_DRAM; | |
4604 | ||
78d504bc | 4605 | intel_pmu_lbr_init_hsw(); |
91f1b705 AK |
4606 | |
4607 | x86_pmu.event_constraints = intel_bdw_event_constraints; | |
b3e62463 | 4608 | x86_pmu.pebs_constraints = intel_bdw_pebs_event_constraints; |
91f1b705 | 4609 | x86_pmu.extra_regs = intel_snbep_extra_regs; |
72469764 AK |
4610 | x86_pmu.pebs_aliases = intel_pebs_aliases_ivb; |
4611 | x86_pmu.pebs_prec_dist = true; | |
91f1b705 | 4612 | /* all extra regs are per-cpu when HT is on */ |
9a5e3fb5 SE |
4613 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; |
4614 | x86_pmu.flags |= PMU_FL_NO_HT_SHARING; | |
91f1b705 AK |
4615 | |
4616 | x86_pmu.hw_config = hsw_hw_config; | |
4617 | x86_pmu.get_event_constraints = hsw_get_event_constraints; | |
d4ae5529 | 4618 | x86_pmu.cpu_events = hsw_events_attrs; |
294fe0f5 | 4619 | x86_pmu.limit_period = bdw_limit_period; |
a5df70c3 AK |
4620 | extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? |
4621 | hsw_format_attr : nhm_format_attr; | |
d4ae5529 JO |
4622 | mem_attr = hsw_mem_events_attrs; |
4623 | tsx_attr = hsw_tsx_events_attrs; | |
91f1b705 | 4624 | pr_cont("Broadwell events, "); |
b00233b5 | 4625 | name = "broadwell"; |
91f1b705 AK |
4626 | break; |
4627 | ||
ef5f9f47 | 4628 | case INTEL_FAM6_XEON_PHI_KNL: |
608284bf | 4629 | case INTEL_FAM6_XEON_PHI_KNM: |
1e7b9390 HC |
4630 | memcpy(hw_cache_event_ids, |
4631 | slm_hw_cache_event_ids, sizeof(hw_cache_event_ids)); | |
4632 | memcpy(hw_cache_extra_regs, | |
4633 | knl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); | |
4634 | intel_pmu_lbr_init_knl(); | |
4635 | ||
4636 | x86_pmu.event_constraints = intel_slm_event_constraints; | |
4637 | x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints; | |
4638 | x86_pmu.extra_regs = intel_knl_extra_regs; | |
4639 | ||
4640 | /* all extra regs are per-cpu when HT is on */ | |
4641 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; | |
4642 | x86_pmu.flags |= PMU_FL_NO_HT_SHARING; | |
a5df70c3 | 4643 | extra_attr = slm_format_attr; |
608284bf | 4644 | pr_cont("Knights Landing/Mill events, "); |
b00233b5 | 4645 | name = "knights-landing"; |
1e7b9390 HC |
4646 | break; |
4647 | ||
ef5f9f47 DH |
4648 | case INTEL_FAM6_SKYLAKE_MOBILE: |
4649 | case INTEL_FAM6_SKYLAKE_DESKTOP: | |
4650 | case INTEL_FAM6_SKYLAKE_X: | |
4651 | case INTEL_FAM6_KABYLAKE_MOBILE: | |
4652 | case INTEL_FAM6_KABYLAKE_DESKTOP: | |
9b545c04 | 4653 | x86_add_quirk(intel_pebs_isolation_quirk); |
9a92e16f AK |
4654 | x86_pmu.late_ack = true; |
4655 | memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids)); | |
4656 | memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs)); | |
4657 | intel_pmu_lbr_init_skl(); | |
4658 | ||
a39fcae7 AK |
4659 | /* INT_MISC.RECOVERY_CYCLES has umask 1 in Skylake */ |
4660 | event_attr_td_recovery_bubbles.event_str_noht = | |
4661 | "event=0xd,umask=0x1,cmask=1"; | |
4662 | event_attr_td_recovery_bubbles.event_str_ht = | |
4663 | "event=0xd,umask=0x1,cmask=1,any=1"; | |
4664 | ||
9a92e16f AK |
4665 | x86_pmu.event_constraints = intel_skl_event_constraints; |
4666 | x86_pmu.pebs_constraints = intel_skl_pebs_event_constraints; | |
4667 | x86_pmu.extra_regs = intel_skl_extra_regs; | |
72469764 AK |
4668 | x86_pmu.pebs_aliases = intel_pebs_aliases_skl; |
4669 | x86_pmu.pebs_prec_dist = true; | |
9a92e16f AK |
4670 | /* all extra regs are per-cpu when HT is on */ |
4671 | x86_pmu.flags |= PMU_FL_HAS_RSP_1; | |
4672 | x86_pmu.flags |= PMU_FL_NO_HT_SHARING; | |
4673 | ||
4674 | x86_pmu.hw_config = hsw_hw_config; | |
4675 | x86_pmu.get_event_constraints = hsw_get_event_constraints; | |
a5df70c3 AK |
4676 | extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? |
4677 | hsw_format_attr : nhm_format_attr; | |
4678 | extra_attr = merge_attr(extra_attr, skl_format_attr); | |
7ad1437d | 4679 | to_free = extra_attr; |
d4ae5529 JO |
4680 | x86_pmu.cpu_events = hsw_events_attrs; |
4681 | mem_attr = hsw_mem_events_attrs; | |
4682 | tsx_attr = hsw_tsx_events_attrs; | |
6ae5fa61 AK |
4683 | intel_pmu_pebs_data_source_skl( |
4684 | boot_cpu_data.x86_model == INTEL_FAM6_SKYLAKE_X); | |
400816f6 PZI |
4685 | |
4686 | if (boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) { | |
4687 | x86_pmu.flags |= PMU_FL_TFA; | |
4688 | x86_pmu.get_event_constraints = tfa_get_event_constraints; | |
4689 | x86_pmu.enable_all = intel_tfa_pmu_enable_all; | |
4690 | x86_pmu.commit_scheduling = intel_tfa_commit_scheduling; | |
4691 | intel_pmu_attrs[1] = &dev_attr_allow_tsx_force_abort.attr.attr; | |
4692 | } | |
4693 | ||
9a92e16f | 4694 | pr_cont("Skylake events, "); |
b00233b5 | 4695 | name = "skylake"; |
9a92e16f AK |
4696 | break; |
4697 | ||
f22f54f4 | 4698 | default: |
0af3ac1f AK |
4699 | switch (x86_pmu.version) { |
4700 | case 1: | |
4701 | x86_pmu.event_constraints = intel_v1_event_constraints; | |
4702 | pr_cont("generic architected perfmon v1, "); | |
b00233b5 | 4703 | name = "generic_arch_v1"; |
0af3ac1f AK |
4704 | break; |
4705 | default: | |
4706 | /* | |
4707 | * default constraints for v2 and up | |
4708 | */ | |
4709 | x86_pmu.event_constraints = intel_gen_event_constraints; | |
4710 | pr_cont("generic architected perfmon, "); | |
b00233b5 | 4711 | name = "generic_arch_v2+"; |
0af3ac1f AK |
4712 | break; |
4713 | } | |
f22f54f4 | 4714 | } |
ffb871bc | 4715 | |
0e96f31e | 4716 | snprintf(pmu_name_str, sizeof(pmu_name_str), "%s", name); |
b00233b5 | 4717 | |
a5df70c3 AK |
4718 | if (version >= 2 && extra_attr) { |
4719 | x86_pmu.format_attrs = merge_attr(intel_arch3_formats_attr, | |
4720 | extra_attr); | |
4721 | WARN_ON(!x86_pmu.format_attrs); | |
4722 | } | |
4723 | ||
d4ae5529 JO |
4724 | x86_pmu.cpu_events = get_events_attrs(x86_pmu.cpu_events, |
4725 | mem_attr, tsx_attr); | |
4726 | ||
a1eac7ac RR |
4727 | if (x86_pmu.num_counters > INTEL_PMC_MAX_GENERIC) { |
4728 | WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!", | |
4729 | x86_pmu.num_counters, INTEL_PMC_MAX_GENERIC); | |
4730 | x86_pmu.num_counters = INTEL_PMC_MAX_GENERIC; | |
4731 | } | |
ad5013d5 | 4732 | x86_pmu.intel_ctrl = (1ULL << x86_pmu.num_counters) - 1; |
a1eac7ac RR |
4733 | |
4734 | if (x86_pmu.num_counters_fixed > INTEL_PMC_MAX_FIXED) { | |
4735 | WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!", | |
4736 | x86_pmu.num_counters_fixed, INTEL_PMC_MAX_FIXED); | |
4737 | x86_pmu.num_counters_fixed = INTEL_PMC_MAX_FIXED; | |
4738 | } | |
4739 | ||
4740 | x86_pmu.intel_ctrl |= | |
4741 | ((1LL << x86_pmu.num_counters_fixed)-1) << INTEL_PMC_IDX_FIXED; | |
4742 | ||
4743 | if (x86_pmu.event_constraints) { | |
4744 | /* | |
4745 | * event on fixed counter2 (REF_CYCLES) only works on this | |
4746 | * counter, so do not extend mask to generic counters | |
4747 | */ | |
4748 | for_each_event_constraint(c, x86_pmu.event_constraints) { | |
2c33645d PI |
4749 | if (c->cmask == FIXED_EVENT_FLAGS |
4750 | && c->idxmsk64 != INTEL_PMC_MSK_FIXED_REF_CYCLES) { | |
4751 | c->idxmsk64 |= (1ULL << x86_pmu.num_counters) - 1; | |
a1eac7ac | 4752 | } |
2c33645d | 4753 | c->idxmsk64 &= |
6d6f2833 | 4754 | ~(~0ULL << (INTEL_PMC_IDX_FIXED + x86_pmu.num_counters_fixed)); |
2c33645d | 4755 | c->weight = hweight64(c->idxmsk64); |
a1eac7ac RR |
4756 | } |
4757 | } | |
4758 | ||
338b522c KL |
4759 | /* |
4760 | * Access LBR MSR may cause #GP under certain circumstances. | |
4761 | * E.g. KVM doesn't support LBR MSR | |
4762 | * Check all LBT MSR here. | |
4763 | * Disable LBR access if any LBR MSRs can not be accessed. | |
4764 | */ | |
4765 | if (x86_pmu.lbr_nr && !check_msr(x86_pmu.lbr_tos, 0x3UL)) | |
4766 | x86_pmu.lbr_nr = 0; | |
4767 | for (i = 0; i < x86_pmu.lbr_nr; i++) { | |
4768 | if (!(check_msr(x86_pmu.lbr_from + i, 0xffffUL) && | |
4769 | check_msr(x86_pmu.lbr_to + i, 0xffffUL))) | |
4770 | x86_pmu.lbr_nr = 0; | |
4771 | } | |
4772 | ||
b00233b5 AK |
4773 | x86_pmu.caps_attrs = intel_pmu_caps_attrs; |
4774 | ||
4775 | if (x86_pmu.lbr_nr) { | |
4776 | x86_pmu.caps_attrs = merge_attr(x86_pmu.caps_attrs, lbr_attrs); | |
f09509b9 | 4777 | pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr); |
b00233b5 AK |
4778 | } |
4779 | ||
338b522c KL |
4780 | /* |
4781 | * Access extra MSR may cause #GP under certain circumstances. | |
4782 | * E.g. KVM doesn't support offcore event | |
4783 | * Check all extra_regs here. | |
4784 | */ | |
4785 | if (x86_pmu.extra_regs) { | |
4786 | for (er = x86_pmu.extra_regs; er->msr; er++) { | |
8c4fe709 | 4787 | er->extra_msr_access = check_msr(er->msr, 0x11UL); |
338b522c KL |
4788 | /* Disable LBR select mapping */ |
4789 | if ((er->idx == EXTRA_REG_LBR) && !er->extra_msr_access) | |
4790 | x86_pmu.lbr_sel_map = NULL; | |
4791 | } | |
4792 | } | |
4793 | ||
069e0c3c AK |
4794 | /* Support full width counters using alternative MSR range */ |
4795 | if (x86_pmu.intel_cap.full_width_write) { | |
7f612a7f | 4796 | x86_pmu.max_period = x86_pmu.cntval_mask >> 1; |
069e0c3c AK |
4797 | x86_pmu.perfctr = MSR_IA32_PMC0; |
4798 | pr_cont("full-width counters, "); | |
4799 | } | |
4800 | ||
af3bdb99 AK |
4801 | /* |
4802 | * For arch perfmon 4 use counter freezing to avoid | |
4803 | * several MSR accesses in the PMI. | |
4804 | */ | |
4805 | if (x86_pmu.counter_freezing) | |
4806 | x86_pmu.handle_irq = intel_pmu_handle_irq_v4; | |
4807 | ||
7ad1437d | 4808 | kfree(to_free); |
f22f54f4 PZ |
4809 | return 0; |
4810 | } | |
b37609c3 SE |
4811 | |
4812 | /* | |
4813 | * HT bug: phase 2 init | |
4814 | * Called once we have valid topology information to check | |
4815 | * whether or not HT is enabled | |
4816 | * If HT is off, then we disable the workaround | |
4817 | */ | |
4818 | static __init int fixup_ht_bug(void) | |
4819 | { | |
030ba6cd | 4820 | int c; |
b37609c3 SE |
4821 | /* |
4822 | * problem not present on this CPU model, nothing to do | |
4823 | */ | |
4824 | if (!(x86_pmu.flags & PMU_FL_EXCL_ENABLED)) | |
4825 | return 0; | |
4826 | ||
030ba6cd | 4827 | if (topology_max_smt_threads() > 1) { |
b37609c3 SE |
4828 | pr_info("PMU erratum BJ122, BV98, HSD29 worked around, HT is on\n"); |
4829 | return 0; | |
4830 | } | |
4831 | ||
2406e3b1 PZ |
4832 | cpus_read_lock(); |
4833 | ||
4834 | hardlockup_detector_perf_stop(); | |
b37609c3 SE |
4835 | |
4836 | x86_pmu.flags &= ~(PMU_FL_EXCL_CNTRS | PMU_FL_EXCL_ENABLED); | |
4837 | ||
b37609c3 | 4838 | x86_pmu.start_scheduling = NULL; |
0c41e756 | 4839 | x86_pmu.commit_scheduling = NULL; |
b37609c3 SE |
4840 | x86_pmu.stop_scheduling = NULL; |
4841 | ||
2406e3b1 | 4842 | hardlockup_detector_perf_restart(); |
b37609c3 | 4843 | |
1ba143a5 | 4844 | for_each_online_cpu(c) |
d01b1f96 | 4845 | free_excl_cntrs(&per_cpu(cpu_hw_events, c)); |
b37609c3 | 4846 | |
1ba143a5 | 4847 | cpus_read_unlock(); |
b37609c3 SE |
4848 | pr_info("PMU erratum BJ122, BV98, HSD29 workaround disabled, HT off\n"); |
4849 | return 0; | |
4850 | } | |
4851 | subsys_initcall(fixup_ht_bug) |