Commit | Line | Data |
---|---|---|
7ce1346a | 1 | /* |
940b2f2f | 2 | * Support cstate residency counters |
7ce1346a KL |
3 | * |
4 | * Copyright (C) 2015, Intel Corp. | |
5 | * Author: Kan Liang (kan.liang@intel.com) | |
6 | * | |
7 | * This library is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU Library General Public | |
9 | * License as published by the Free Software Foundation; either | |
10 | * version 2 of the License, or (at your option) any later version. | |
11 | * | |
12 | * This library is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
15 | * Library General Public License for more details. | |
16 | * | |
17 | */ | |
18 | ||
19 | /* | |
20 | * This file export cstate related free running (read-only) counters | |
21 | * for perf. These counters may be use simultaneously by other tools, | |
22 | * such as turbostat. However, it still make sense to implement them | |
23 | * in perf. Because we can conveniently collect them together with | |
24 | * other events, and allow to use them from tools without special MSR | |
25 | * access code. | |
26 | * | |
27 | * The events only support system-wide mode counting. There is no | |
28 | * sampling support because it is not supported by the hardware. | |
29 | * | |
30 | * According to counters' scope and category, two PMUs are registered | |
31 | * with the perf_event core subsystem. | |
32 | * - 'cstate_core': The counter is available for each physical core. | |
33 | * The counters include CORE_C*_RESIDENCY. | |
34 | * - 'cstate_pkg': The counter is available for each physical package. | |
35 | * The counters include PKG_C*_RESIDENCY. | |
36 | * | |
37 | * All of these counters are specified in the IntelĀ® 64 and IA-32 | |
38 | * Architectures Software Developer.s Manual Vol3b. | |
39 | * | |
40 | * Model specific counters: | |
41 | * MSR_CORE_C1_RES: CORE C1 Residency Counter | |
42 | * perf code: 0x00 | |
1159e094 | 43 | * Available model: SLM,AMT,GLM,CNL |
7ce1346a KL |
44 | * Scope: Core (each processor core has a MSR) |
45 | * MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter | |
46 | * perf code: 0x01 | |
1159e094 | 47 | * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,GLM, |
1ffa6c04 | 48 | * CNL,KBL,CML |
7ce1346a KL |
49 | * Scope: Core |
50 | * MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter | |
51 | * perf code: 0x02 | |
1159e094 | 52 | * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW, |
1ffa6c04 | 53 | * SKL,KNL,GLM,CNL,KBL,CML |
7ce1346a KL |
54 | * Scope: Core |
55 | * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter | |
56 | * perf code: 0x03 | |
1ffa6c04 | 57 | * Available model: SNB,IVB,HSW,BDW,SKL,CNL,KBL,CML |
7ce1346a KL |
58 | * Scope: Core |
59 | * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter. | |
60 | * perf code: 0x00 | |
1ffa6c04 KL |
61 | * Available model: SNB,IVB,HSW,BDW,SKL,KNL,GLM,CNL, |
62 | * KBL,CML | |
7ce1346a KL |
63 | * Scope: Package (physical package) |
64 | * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter. | |
65 | * perf code: 0x01 | |
1159e094 | 66 | * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL, |
1ffa6c04 | 67 | * GLM,CNL,KBL,CML |
7ce1346a KL |
68 | * Scope: Package (physical package) |
69 | * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter. | |
70 | * perf code: 0x02 | |
889882bc | 71 | * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW |
1ffa6c04 | 72 | * SKL,KNL,GLM,CNL,KBL,CML |
7ce1346a KL |
73 | * Scope: Package (physical package) |
74 | * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter. | |
75 | * perf code: 0x03 | |
1ffa6c04 KL |
76 | * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,CNL, |
77 | * KBL,CML | |
7ce1346a KL |
78 | * Scope: Package (physical package) |
79 | * MSR_PKG_C8_RESIDENCY: Package C8 Residency Counter. | |
80 | * perf code: 0x04 | |
1ffa6c04 | 81 | * Available model: HSW ULT,KBL,CNL,CML |
7ce1346a KL |
82 | * Scope: Package (physical package) |
83 | * MSR_PKG_C9_RESIDENCY: Package C9 Residency Counter. | |
84 | * perf code: 0x05 | |
1ffa6c04 | 85 | * Available model: HSW ULT,KBL,CNL,CML |
7ce1346a KL |
86 | * Scope: Package (physical package) |
87 | * MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter. | |
88 | * perf code: 0x06 | |
1ffa6c04 | 89 | * Available model: HSW ULT,KBL,GLM,CNL,CML |
7ce1346a KL |
90 | * Scope: Package (physical package) |
91 | * | |
92 | */ | |
93 | ||
94 | #include <linux/module.h> | |
95 | #include <linux/slab.h> | |
96 | #include <linux/perf_event.h> | |
a5f81290 | 97 | #include <linux/nospec.h> |
7ce1346a | 98 | #include <asm/cpu_device_id.h> |
bf4ad541 | 99 | #include <asm/intel-family.h> |
27f6d22b | 100 | #include "../perf_event.h" |
8f2a28c5 | 101 | #include "../probe.h" |
7ce1346a | 102 | |
c7afba32 TG |
103 | MODULE_LICENSE("GPL"); |
104 | ||
7ce1346a KL |
105 | #define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format) \ |
106 | static ssize_t __cstate_##_var##_show(struct kobject *kobj, \ | |
107 | struct kobj_attribute *attr, \ | |
108 | char *page) \ | |
109 | { \ | |
110 | BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ | |
111 | return sprintf(page, _format "\n"); \ | |
112 | } \ | |
113 | static struct kobj_attribute format_attr_##_var = \ | |
114 | __ATTR(_name, 0444, __cstate_##_var##_show, NULL) | |
115 | ||
116 | static ssize_t cstate_get_attr_cpumask(struct device *dev, | |
117 | struct device_attribute *attr, | |
118 | char *buf); | |
119 | ||
424646ee TG |
120 | /* Model -> events mapping */ |
121 | struct cstate_model { | |
122 | unsigned long core_events; | |
123 | unsigned long pkg_events; | |
124 | unsigned long quirks; | |
125 | }; | |
126 | ||
127 | /* Quirk flags */ | |
128 | #define SLM_PKG_C6_USE_C7_MSR (1UL << 0) | |
889882bc | 129 | #define KNL_CORE_C6_MSR (1UL << 1) |
424646ee | 130 | |
7ce1346a KL |
131 | struct perf_cstate_msr { |
132 | u64 msr; | |
133 | struct perf_pmu_events_attr *attr; | |
7ce1346a KL |
134 | }; |
135 | ||
136 | ||
137 | /* cstate_core PMU */ | |
7ce1346a KL |
138 | static struct pmu cstate_core_pmu; |
139 | static bool has_cstate_core; | |
140 | ||
424646ee | 141 | enum perf_cstate_core_events { |
7ce1346a KL |
142 | PERF_CSTATE_CORE_C1_RES = 0, |
143 | PERF_CSTATE_CORE_C3_RES, | |
144 | PERF_CSTATE_CORE_C6_RES, | |
145 | PERF_CSTATE_CORE_C7_RES, | |
146 | ||
147 | PERF_CSTATE_CORE_EVENT_MAX, | |
148 | }; | |
149 | ||
8f2a28c5 JO |
150 | PMU_EVENT_ATTR_STRING(c1-residency, attr_cstate_core_c1, "event=0x00"); |
151 | PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_core_c3, "event=0x01"); | |
152 | PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_core_c6, "event=0x02"); | |
153 | PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_core_c7, "event=0x03"); | |
7ce1346a | 154 | |
8f2a28c5 JO |
155 | static unsigned long core_msr_mask; |
156 | ||
157 | PMU_EVENT_GROUP(events, cstate_core_c1); | |
158 | PMU_EVENT_GROUP(events, cstate_core_c3); | |
159 | PMU_EVENT_GROUP(events, cstate_core_c6); | |
160 | PMU_EVENT_GROUP(events, cstate_core_c7); | |
161 | ||
162 | static bool test_msr(int idx, void *data) | |
163 | { | |
164 | return test_bit(idx, (unsigned long *) data); | |
165 | } | |
166 | ||
167 | static struct perf_msr core_msr[] = { | |
168 | [PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES, &group_cstate_core_c1, test_msr }, | |
169 | [PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY, &group_cstate_core_c3, test_msr }, | |
170 | [PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY, &group_cstate_core_c6, test_msr }, | |
171 | [PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY, &group_cstate_core_c7, test_msr }, | |
7ce1346a KL |
172 | }; |
173 | ||
8f2a28c5 | 174 | static struct attribute *attrs_empty[] = { |
7ce1346a KL |
175 | NULL, |
176 | }; | |
177 | ||
8f2a28c5 JO |
178 | /* |
179 | * There are no default events, but we need to create | |
180 | * "events" group (with empty attrs) before updating | |
181 | * it with detected events. | |
182 | */ | |
7ce1346a KL |
183 | static struct attribute_group core_events_attr_group = { |
184 | .name = "events", | |
8f2a28c5 | 185 | .attrs = attrs_empty, |
7ce1346a KL |
186 | }; |
187 | ||
188 | DEFINE_CSTATE_FORMAT_ATTR(core_event, event, "config:0-63"); | |
189 | static struct attribute *core_format_attrs[] = { | |
190 | &format_attr_core_event.attr, | |
191 | NULL, | |
192 | }; | |
193 | ||
194 | static struct attribute_group core_format_attr_group = { | |
195 | .name = "format", | |
196 | .attrs = core_format_attrs, | |
197 | }; | |
198 | ||
199 | static cpumask_t cstate_core_cpu_mask; | |
200 | static DEVICE_ATTR(cpumask, S_IRUGO, cstate_get_attr_cpumask, NULL); | |
201 | ||
202 | static struct attribute *cstate_cpumask_attrs[] = { | |
203 | &dev_attr_cpumask.attr, | |
204 | NULL, | |
205 | }; | |
206 | ||
207 | static struct attribute_group cpumask_attr_group = { | |
208 | .attrs = cstate_cpumask_attrs, | |
209 | }; | |
210 | ||
211 | static const struct attribute_group *core_attr_groups[] = { | |
212 | &core_events_attr_group, | |
213 | &core_format_attr_group, | |
214 | &cpumask_attr_group, | |
215 | NULL, | |
216 | }; | |
217 | ||
7ce1346a | 218 | /* cstate_pkg PMU */ |
7ce1346a KL |
219 | static struct pmu cstate_pkg_pmu; |
220 | static bool has_cstate_pkg; | |
221 | ||
424646ee | 222 | enum perf_cstate_pkg_events { |
7ce1346a KL |
223 | PERF_CSTATE_PKG_C2_RES = 0, |
224 | PERF_CSTATE_PKG_C3_RES, | |
225 | PERF_CSTATE_PKG_C6_RES, | |
226 | PERF_CSTATE_PKG_C7_RES, | |
227 | PERF_CSTATE_PKG_C8_RES, | |
228 | PERF_CSTATE_PKG_C9_RES, | |
229 | PERF_CSTATE_PKG_C10_RES, | |
230 | ||
231 | PERF_CSTATE_PKG_EVENT_MAX, | |
232 | }; | |
233 | ||
8f2a28c5 JO |
234 | PMU_EVENT_ATTR_STRING(c2-residency, attr_cstate_pkg_c2, "event=0x00"); |
235 | PMU_EVENT_ATTR_STRING(c3-residency, attr_cstate_pkg_c3, "event=0x01"); | |
236 | PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_pkg_c6, "event=0x02"); | |
237 | PMU_EVENT_ATTR_STRING(c7-residency, attr_cstate_pkg_c7, "event=0x03"); | |
238 | PMU_EVENT_ATTR_STRING(c8-residency, attr_cstate_pkg_c8, "event=0x04"); | |
239 | PMU_EVENT_ATTR_STRING(c9-residency, attr_cstate_pkg_c9, "event=0x05"); | |
240 | PMU_EVENT_ATTR_STRING(c10-residency, attr_cstate_pkg_c10, "event=0x06"); | |
241 | ||
242 | static unsigned long pkg_msr_mask; | |
243 | ||
244 | PMU_EVENT_GROUP(events, cstate_pkg_c2); | |
245 | PMU_EVENT_GROUP(events, cstate_pkg_c3); | |
246 | PMU_EVENT_GROUP(events, cstate_pkg_c6); | |
247 | PMU_EVENT_GROUP(events, cstate_pkg_c7); | |
248 | PMU_EVENT_GROUP(events, cstate_pkg_c8); | |
249 | PMU_EVENT_GROUP(events, cstate_pkg_c9); | |
250 | PMU_EVENT_GROUP(events, cstate_pkg_c10); | |
251 | ||
252 | static struct perf_msr pkg_msr[] = { | |
253 | [PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY, &group_cstate_pkg_c2, test_msr }, | |
254 | [PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY, &group_cstate_pkg_c3, test_msr }, | |
255 | [PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY, &group_cstate_pkg_c6, test_msr }, | |
256 | [PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY, &group_cstate_pkg_c7, test_msr }, | |
257 | [PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY, &group_cstate_pkg_c8, test_msr }, | |
258 | [PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY, &group_cstate_pkg_c9, test_msr }, | |
259 | [PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY, &group_cstate_pkg_c10, test_msr }, | |
7ce1346a KL |
260 | }; |
261 | ||
262 | static struct attribute_group pkg_events_attr_group = { | |
263 | .name = "events", | |
8f2a28c5 | 264 | .attrs = attrs_empty, |
7ce1346a KL |
265 | }; |
266 | ||
267 | DEFINE_CSTATE_FORMAT_ATTR(pkg_event, event, "config:0-63"); | |
268 | static struct attribute *pkg_format_attrs[] = { | |
269 | &format_attr_pkg_event.attr, | |
270 | NULL, | |
271 | }; | |
272 | static struct attribute_group pkg_format_attr_group = { | |
273 | .name = "format", | |
274 | .attrs = pkg_format_attrs, | |
275 | }; | |
276 | ||
277 | static cpumask_t cstate_pkg_cpu_mask; | |
278 | ||
279 | static const struct attribute_group *pkg_attr_groups[] = { | |
280 | &pkg_events_attr_group, | |
281 | &pkg_format_attr_group, | |
282 | &cpumask_attr_group, | |
283 | NULL, | |
284 | }; | |
285 | ||
7ce1346a KL |
286 | static ssize_t cstate_get_attr_cpumask(struct device *dev, |
287 | struct device_attribute *attr, | |
288 | char *buf) | |
289 | { | |
290 | struct pmu *pmu = dev_get_drvdata(dev); | |
291 | ||
292 | if (pmu == &cstate_core_pmu) | |
293 | return cpumap_print_to_pagebuf(true, buf, &cstate_core_cpu_mask); | |
294 | else if (pmu == &cstate_pkg_pmu) | |
295 | return cpumap_print_to_pagebuf(true, buf, &cstate_pkg_cpu_mask); | |
296 | else | |
297 | return 0; | |
298 | } | |
299 | ||
300 | static int cstate_pmu_event_init(struct perf_event *event) | |
301 | { | |
302 | u64 cfg = event->attr.config; | |
49de0493 | 303 | int cpu; |
7ce1346a KL |
304 | |
305 | if (event->attr.type != event->pmu->type) | |
306 | return -ENOENT; | |
307 | ||
308 | /* unsupported modes and filters */ | |
2ff40250 | 309 | if (event->attr.sample_period) /* no sampling */ |
7ce1346a KL |
310 | return -EINVAL; |
311 | ||
49de0493 TG |
312 | if (event->cpu < 0) |
313 | return -EINVAL; | |
314 | ||
7ce1346a KL |
315 | if (event->pmu == &cstate_core_pmu) { |
316 | if (cfg >= PERF_CSTATE_CORE_EVENT_MAX) | |
317 | return -EINVAL; | |
8f2a28c5 JO |
318 | cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_CORE_EVENT_MAX); |
319 | if (!(core_msr_mask & (1 << cfg))) | |
7ce1346a KL |
320 | return -EINVAL; |
321 | event->hw.event_base = core_msr[cfg].msr; | |
49de0493 TG |
322 | cpu = cpumask_any_and(&cstate_core_cpu_mask, |
323 | topology_sibling_cpumask(event->cpu)); | |
7ce1346a KL |
324 | } else if (event->pmu == &cstate_pkg_pmu) { |
325 | if (cfg >= PERF_CSTATE_PKG_EVENT_MAX) | |
326 | return -EINVAL; | |
a5f81290 | 327 | cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX); |
8f2a28c5 | 328 | if (!(pkg_msr_mask & (1 << cfg))) |
7ce1346a KL |
329 | return -EINVAL; |
330 | event->hw.event_base = pkg_msr[cfg].msr; | |
49de0493 | 331 | cpu = cpumask_any_and(&cstate_pkg_cpu_mask, |
cb63ba0f | 332 | topology_die_cpumask(event->cpu)); |
49de0493 | 333 | } else { |
7ce1346a | 334 | return -ENOENT; |
49de0493 TG |
335 | } |
336 | ||
337 | if (cpu >= nr_cpu_ids) | |
338 | return -ENODEV; | |
7ce1346a | 339 | |
49de0493 | 340 | event->cpu = cpu; |
7ce1346a KL |
341 | event->hw.config = cfg; |
342 | event->hw.idx = -1; | |
49de0493 | 343 | return 0; |
7ce1346a KL |
344 | } |
345 | ||
346 | static inline u64 cstate_pmu_read_counter(struct perf_event *event) | |
347 | { | |
348 | u64 val; | |
349 | ||
350 | rdmsrl(event->hw.event_base, val); | |
351 | return val; | |
352 | } | |
353 | ||
354 | static void cstate_pmu_event_update(struct perf_event *event) | |
355 | { | |
356 | struct hw_perf_event *hwc = &event->hw; | |
357 | u64 prev_raw_count, new_raw_count; | |
358 | ||
359 | again: | |
360 | prev_raw_count = local64_read(&hwc->prev_count); | |
361 | new_raw_count = cstate_pmu_read_counter(event); | |
362 | ||
363 | if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, | |
364 | new_raw_count) != prev_raw_count) | |
365 | goto again; | |
366 | ||
367 | local64_add(new_raw_count - prev_raw_count, &event->count); | |
368 | } | |
369 | ||
370 | static void cstate_pmu_event_start(struct perf_event *event, int mode) | |
371 | { | |
372 | local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event)); | |
373 | } | |
374 | ||
375 | static void cstate_pmu_event_stop(struct perf_event *event, int mode) | |
376 | { | |
377 | cstate_pmu_event_update(event); | |
378 | } | |
379 | ||
380 | static void cstate_pmu_event_del(struct perf_event *event, int mode) | |
381 | { | |
382 | cstate_pmu_event_stop(event, PERF_EF_UPDATE); | |
383 | } | |
384 | ||
385 | static int cstate_pmu_event_add(struct perf_event *event, int mode) | |
386 | { | |
387 | if (mode & PERF_EF_START) | |
388 | cstate_pmu_event_start(event, mode); | |
389 | ||
390 | return 0; | |
391 | } | |
392 | ||
49de0493 TG |
393 | /* |
394 | * Check if exiting cpu is the designated reader. If so migrate the | |
395 | * events when there is a valid target available | |
396 | */ | |
77c34ef1 | 397 | static int cstate_cpu_exit(unsigned int cpu) |
7ce1346a | 398 | { |
49de0493 TG |
399 | unsigned int target; |
400 | ||
401 | if (has_cstate_core && | |
402 | cpumask_test_and_clear_cpu(cpu, &cstate_core_cpu_mask)) { | |
403 | ||
404 | target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); | |
405 | /* Migrate events if there is a valid target */ | |
406 | if (target < nr_cpu_ids) { | |
7ce1346a | 407 | cpumask_set_cpu(target, &cstate_core_cpu_mask); |
7ce1346a | 408 | perf_pmu_migrate_context(&cstate_core_pmu, cpu, target); |
49de0493 | 409 | } |
7ce1346a KL |
410 | } |
411 | ||
49de0493 TG |
412 | if (has_cstate_pkg && |
413 | cpumask_test_and_clear_cpu(cpu, &cstate_pkg_cpu_mask)) { | |
414 | ||
cb63ba0f | 415 | target = cpumask_any_but(topology_die_cpumask(cpu), cpu); |
49de0493 TG |
416 | /* Migrate events if there is a valid target */ |
417 | if (target < nr_cpu_ids) { | |
7ce1346a | 418 | cpumask_set_cpu(target, &cstate_pkg_cpu_mask); |
7ce1346a | 419 | perf_pmu_migrate_context(&cstate_pkg_pmu, cpu, target); |
49de0493 | 420 | } |
7ce1346a | 421 | } |
77c34ef1 | 422 | return 0; |
7ce1346a KL |
423 | } |
424 | ||
77c34ef1 | 425 | static int cstate_cpu_init(unsigned int cpu) |
7ce1346a | 426 | { |
49de0493 | 427 | unsigned int target; |
7ce1346a | 428 | |
49de0493 TG |
429 | /* |
430 | * If this is the first online thread of that core, set it in | |
431 | * the core cpu mask as the designated reader. | |
432 | */ | |
433 | target = cpumask_any_and(&cstate_core_cpu_mask, | |
434 | topology_sibling_cpumask(cpu)); | |
435 | ||
436 | if (has_cstate_core && target >= nr_cpu_ids) | |
437 | cpumask_set_cpu(cpu, &cstate_core_cpu_mask); | |
438 | ||
439 | /* | |
440 | * If this is the first online thread of that package, set it | |
441 | * in the package cpu mask as the designated reader. | |
442 | */ | |
443 | target = cpumask_any_and(&cstate_pkg_cpu_mask, | |
cb63ba0f | 444 | topology_die_cpumask(cpu)); |
49de0493 TG |
445 | if (has_cstate_pkg && target >= nr_cpu_ids) |
446 | cpumask_set_cpu(cpu, &cstate_pkg_cpu_mask); | |
7ce1346a | 447 | |
77c34ef1 | 448 | return 0; |
7ce1346a KL |
449 | } |
450 | ||
d9f3b450 | 451 | static const struct attribute_group *core_attr_update[] = { |
8f2a28c5 JO |
452 | &group_cstate_core_c1, |
453 | &group_cstate_core_c3, | |
454 | &group_cstate_core_c6, | |
455 | &group_cstate_core_c7, | |
456 | NULL, | |
457 | }; | |
458 | ||
d9f3b450 | 459 | static const struct attribute_group *pkg_attr_update[] = { |
8f2a28c5 JO |
460 | &group_cstate_pkg_c2, |
461 | &group_cstate_pkg_c3, | |
462 | &group_cstate_pkg_c6, | |
463 | &group_cstate_pkg_c7, | |
464 | &group_cstate_pkg_c8, | |
465 | &group_cstate_pkg_c9, | |
466 | &group_cstate_pkg_c10, | |
467 | NULL, | |
468 | }; | |
469 | ||
424646ee TG |
470 | static struct pmu cstate_core_pmu = { |
471 | .attr_groups = core_attr_groups, | |
8f2a28c5 | 472 | .attr_update = core_attr_update, |
424646ee TG |
473 | .name = "cstate_core", |
474 | .task_ctx_nr = perf_invalid_context, | |
475 | .event_init = cstate_pmu_event_init, | |
476 | .add = cstate_pmu_event_add, | |
477 | .del = cstate_pmu_event_del, | |
478 | .start = cstate_pmu_event_start, | |
479 | .stop = cstate_pmu_event_stop, | |
480 | .read = cstate_pmu_event_update, | |
2ff40250 | 481 | .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE, |
74545f63 | 482 | .module = THIS_MODULE, |
424646ee TG |
483 | }; |
484 | ||
485 | static struct pmu cstate_pkg_pmu = { | |
486 | .attr_groups = pkg_attr_groups, | |
8f2a28c5 | 487 | .attr_update = pkg_attr_update, |
424646ee TG |
488 | .name = "cstate_pkg", |
489 | .task_ctx_nr = perf_invalid_context, | |
490 | .event_init = cstate_pmu_event_init, | |
491 | .add = cstate_pmu_event_add, | |
492 | .del = cstate_pmu_event_del, | |
493 | .start = cstate_pmu_event_start, | |
494 | .stop = cstate_pmu_event_stop, | |
495 | .read = cstate_pmu_event_update, | |
2ff40250 | 496 | .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE, |
74545f63 | 497 | .module = THIS_MODULE, |
424646ee TG |
498 | }; |
499 | ||
500 | static const struct cstate_model nhm_cstates __initconst = { | |
501 | .core_events = BIT(PERF_CSTATE_CORE_C3_RES) | | |
502 | BIT(PERF_CSTATE_CORE_C6_RES), | |
503 | ||
504 | .pkg_events = BIT(PERF_CSTATE_PKG_C3_RES) | | |
505 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
506 | BIT(PERF_CSTATE_PKG_C7_RES), | |
507 | }; | |
508 | ||
509 | static const struct cstate_model snb_cstates __initconst = { | |
510 | .core_events = BIT(PERF_CSTATE_CORE_C3_RES) | | |
511 | BIT(PERF_CSTATE_CORE_C6_RES) | | |
512 | BIT(PERF_CSTATE_CORE_C7_RES), | |
513 | ||
514 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
515 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
516 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
517 | BIT(PERF_CSTATE_PKG_C7_RES), | |
518 | }; | |
519 | ||
520 | static const struct cstate_model hswult_cstates __initconst = { | |
521 | .core_events = BIT(PERF_CSTATE_CORE_C3_RES) | | |
522 | BIT(PERF_CSTATE_CORE_C6_RES) | | |
523 | BIT(PERF_CSTATE_CORE_C7_RES), | |
524 | ||
525 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
526 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
527 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
528 | BIT(PERF_CSTATE_PKG_C7_RES) | | |
529 | BIT(PERF_CSTATE_PKG_C8_RES) | | |
530 | BIT(PERF_CSTATE_PKG_C9_RES) | | |
531 | BIT(PERF_CSTATE_PKG_C10_RES), | |
532 | }; | |
533 | ||
1159e094 HP |
534 | static const struct cstate_model cnl_cstates __initconst = { |
535 | .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | | |
536 | BIT(PERF_CSTATE_CORE_C3_RES) | | |
537 | BIT(PERF_CSTATE_CORE_C6_RES) | | |
538 | BIT(PERF_CSTATE_CORE_C7_RES), | |
539 | ||
540 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
541 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
542 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
543 | BIT(PERF_CSTATE_PKG_C7_RES) | | |
544 | BIT(PERF_CSTATE_PKG_C8_RES) | | |
545 | BIT(PERF_CSTATE_PKG_C9_RES) | | |
546 | BIT(PERF_CSTATE_PKG_C10_RES), | |
547 | }; | |
548 | ||
424646ee TG |
549 | static const struct cstate_model slm_cstates __initconst = { |
550 | .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | | |
551 | BIT(PERF_CSTATE_CORE_C6_RES), | |
552 | ||
553 | .pkg_events = BIT(PERF_CSTATE_PKG_C6_RES), | |
554 | .quirks = SLM_PKG_C6_USE_C7_MSR, | |
555 | }; | |
556 | ||
889882bc LO |
557 | |
558 | static const struct cstate_model knl_cstates __initconst = { | |
559 | .core_events = BIT(PERF_CSTATE_CORE_C6_RES), | |
560 | ||
561 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
562 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
563 | BIT(PERF_CSTATE_PKG_C6_RES), | |
564 | .quirks = KNL_CORE_C6_MSR, | |
565 | }; | |
566 | ||
567 | ||
5c10b048 HP |
568 | static const struct cstate_model glm_cstates __initconst = { |
569 | .core_events = BIT(PERF_CSTATE_CORE_C1_RES) | | |
570 | BIT(PERF_CSTATE_CORE_C3_RES) | | |
571 | BIT(PERF_CSTATE_CORE_C6_RES), | |
572 | ||
573 | .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | | |
574 | BIT(PERF_CSTATE_PKG_C3_RES) | | |
575 | BIT(PERF_CSTATE_PKG_C6_RES) | | |
576 | BIT(PERF_CSTATE_PKG_C10_RES), | |
577 | }; | |
578 | ||
889882bc | 579 | |
424646ee TG |
580 | #define X86_CSTATES_MODEL(model, states) \ |
581 | { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long) &(states) } | |
582 | ||
583 | static const struct x86_cpu_id intel_cstates_match[] __initconst = { | |
bf4ad541 DH |
584 | X86_CSTATES_MODEL(INTEL_FAM6_NEHALEM, nhm_cstates), |
585 | X86_CSTATES_MODEL(INTEL_FAM6_NEHALEM_EP, nhm_cstates), | |
586 | X86_CSTATES_MODEL(INTEL_FAM6_NEHALEM_EX, nhm_cstates), | |
424646ee | 587 | |
bf4ad541 DH |
588 | X86_CSTATES_MODEL(INTEL_FAM6_WESTMERE, nhm_cstates), |
589 | X86_CSTATES_MODEL(INTEL_FAM6_WESTMERE_EP, nhm_cstates), | |
590 | X86_CSTATES_MODEL(INTEL_FAM6_WESTMERE_EX, nhm_cstates), | |
424646ee | 591 | |
bf4ad541 DH |
592 | X86_CSTATES_MODEL(INTEL_FAM6_SANDYBRIDGE, snb_cstates), |
593 | X86_CSTATES_MODEL(INTEL_FAM6_SANDYBRIDGE_X, snb_cstates), | |
424646ee | 594 | |
bf4ad541 DH |
595 | X86_CSTATES_MODEL(INTEL_FAM6_IVYBRIDGE, snb_cstates), |
596 | X86_CSTATES_MODEL(INTEL_FAM6_IVYBRIDGE_X, snb_cstates), | |
424646ee | 597 | |
5e741407 PZ |
598 | X86_CSTATES_MODEL(INTEL_FAM6_HASWELL, snb_cstates), |
599 | X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_X, snb_cstates), | |
600 | X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_G, snb_cstates), | |
424646ee | 601 | |
af239c44 | 602 | X86_CSTATES_MODEL(INTEL_FAM6_HASWELL_L, hswult_cstates), |
424646ee | 603 | |
c66f78a6 | 604 | X86_CSTATES_MODEL(INTEL_FAM6_ATOM_SILVERMONT, slm_cstates), |
5ebb34ed | 605 | X86_CSTATES_MODEL(INTEL_FAM6_ATOM_SILVERMONT_D, slm_cstates), |
c66f78a6 | 606 | X86_CSTATES_MODEL(INTEL_FAM6_ATOM_AIRMONT, slm_cstates), |
424646ee | 607 | |
5ebb34ed PZ |
608 | X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL, snb_cstates), |
609 | X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_D, snb_cstates), | |
610 | X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_G, snb_cstates), | |
611 | X86_CSTATES_MODEL(INTEL_FAM6_BROADWELL_X, snb_cstates), | |
424646ee | 612 | |
af239c44 PZ |
613 | X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE_L, snb_cstates), |
614 | X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE, snb_cstates), | |
b09c146f | 615 | X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE_X, snb_cstates), |
889882bc | 616 | |
af239c44 PZ |
617 | X86_CSTATES_MODEL(INTEL_FAM6_KABYLAKE_L, hswult_cstates), |
618 | X86_CSTATES_MODEL(INTEL_FAM6_KABYLAKE, hswult_cstates), | |
1ffa6c04 KL |
619 | X86_CSTATES_MODEL(INTEL_FAM6_COMETLAKE_L, hswult_cstates), |
620 | X86_CSTATES_MODEL(INTEL_FAM6_COMETLAKE, hswult_cstates), | |
f2029b1e | 621 | |
af239c44 | 622 | X86_CSTATES_MODEL(INTEL_FAM6_CANNONLAKE_L, cnl_cstates), |
1159e094 | 623 | |
889882bc | 624 | X86_CSTATES_MODEL(INTEL_FAM6_XEON_PHI_KNL, knl_cstates), |
1dba23b1 | 625 | X86_CSTATES_MODEL(INTEL_FAM6_XEON_PHI_KNM, knl_cstates), |
5c10b048 | 626 | |
c66f78a6 | 627 | X86_CSTATES_MODEL(INTEL_FAM6_ATOM_GOLDMONT, glm_cstates), |
5ebb34ed | 628 | X86_CSTATES_MODEL(INTEL_FAM6_ATOM_GOLDMONT_D, glm_cstates), |
b09c146f | 629 | |
f2c4db1b | 630 | X86_CSTATES_MODEL(INTEL_FAM6_ATOM_GOLDMONT_PLUS, glm_cstates), |
f08c47d1 | 631 | |
af239c44 PZ |
632 | X86_CSTATES_MODEL(INTEL_FAM6_ICELAKE_L, snb_cstates), |
633 | X86_CSTATES_MODEL(INTEL_FAM6_ICELAKE, snb_cstates), | |
424646ee TG |
634 | { }, |
635 | }; | |
636 | MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match); | |
637 | ||
424646ee | 638 | static int __init cstate_probe(const struct cstate_model *cm) |
7ce1346a KL |
639 | { |
640 | /* SLM has different MSR for PKG C6 */ | |
424646ee | 641 | if (cm->quirks & SLM_PKG_C6_USE_C7_MSR) |
7ce1346a | 642 | pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY; |
7ce1346a | 643 | |
889882bc LO |
644 | /* KNL has different MSR for CORE C6 */ |
645 | if (cm->quirks & KNL_CORE_C6_MSR) | |
646 | pkg_msr[PERF_CSTATE_CORE_C6_RES].msr = MSR_KNL_CORE_C6_RESIDENCY; | |
647 | ||
648 | ||
8f2a28c5 JO |
649 | core_msr_mask = perf_msr_probe(core_msr, PERF_CSTATE_CORE_EVENT_MAX, |
650 | true, (void *) &cm->core_events); | |
651 | ||
652 | pkg_msr_mask = perf_msr_probe(pkg_msr, PERF_CSTATE_PKG_EVENT_MAX, | |
653 | true, (void *) &cm->pkg_events); | |
7ce1346a | 654 | |
8f2a28c5 JO |
655 | has_cstate_core = !!core_msr_mask; |
656 | has_cstate_pkg = !!pkg_msr_mask; | |
7ce1346a KL |
657 | |
658 | return (has_cstate_core || has_cstate_pkg) ? 0 : -ENODEV; | |
659 | } | |
660 | ||
c7afba32 | 661 | static inline void cstate_cleanup(void) |
7ce1346a | 662 | { |
834fcd29 TG |
663 | cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_ONLINE); |
664 | cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_CSTATE_STARTING); | |
665 | ||
d29859e7 TG |
666 | if (has_cstate_core) |
667 | perf_pmu_unregister(&cstate_core_pmu); | |
7ce1346a | 668 | |
d29859e7 TG |
669 | if (has_cstate_pkg) |
670 | perf_pmu_unregister(&cstate_pkg_pmu); | |
7ce1346a KL |
671 | } |
672 | ||
d29859e7 | 673 | static int __init cstate_init(void) |
7ce1346a | 674 | { |
77c34ef1 | 675 | int err; |
d29859e7 | 676 | |
77c34ef1 | 677 | cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_STARTING, |
834fcd29 | 678 | "perf/x86/cstate:starting", cstate_cpu_init, NULL); |
77c34ef1 | 679 | cpuhp_setup_state(CPUHP_AP_PERF_X86_CSTATE_ONLINE, |
834fcd29 | 680 | "perf/x86/cstate:online", NULL, cstate_cpu_exit); |
7ce1346a KL |
681 | |
682 | if (has_cstate_core) { | |
683 | err = perf_pmu_register(&cstate_core_pmu, cstate_core_pmu.name, -1); | |
d29859e7 TG |
684 | if (err) { |
685 | has_cstate_core = false; | |
686 | pr_info("Failed to register cstate core pmu\n"); | |
834fcd29 | 687 | cstate_cleanup(); |
77c34ef1 | 688 | return err; |
d29859e7 | 689 | } |
7ce1346a KL |
690 | } |
691 | ||
692 | if (has_cstate_pkg) { | |
cb63ba0f KL |
693 | if (topology_max_die_per_package() > 1) { |
694 | err = perf_pmu_register(&cstate_pkg_pmu, | |
695 | "cstate_die", -1); | |
696 | } else { | |
697 | err = perf_pmu_register(&cstate_pkg_pmu, | |
698 | cstate_pkg_pmu.name, -1); | |
699 | } | |
d29859e7 TG |
700 | if (err) { |
701 | has_cstate_pkg = false; | |
702 | pr_info("Failed to register cstate pkg pmu\n"); | |
703 | cstate_cleanup(); | |
77c34ef1 | 704 | return err; |
d29859e7 | 705 | } |
7ce1346a | 706 | } |
834fcd29 | 707 | return 0; |
7ce1346a KL |
708 | } |
709 | ||
710 | static int __init cstate_pmu_init(void) | |
711 | { | |
424646ee | 712 | const struct x86_cpu_id *id; |
7ce1346a KL |
713 | int err; |
714 | ||
424646ee | 715 | if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) |
7ce1346a KL |
716 | return -ENODEV; |
717 | ||
424646ee TG |
718 | id = x86_match_cpu(intel_cstates_match); |
719 | if (!id) | |
720 | return -ENODEV; | |
721 | ||
722 | err = cstate_probe((const struct cstate_model *) id->driver_data); | |
7ce1346a KL |
723 | if (err) |
724 | return err; | |
725 | ||
d29859e7 | 726 | return cstate_init(); |
7ce1346a | 727 | } |
c7afba32 TG |
728 | module_init(cstate_pmu_init); |
729 | ||
730 | static void __exit cstate_pmu_exit(void) | |
731 | { | |
c7afba32 | 732 | cstate_cleanup(); |
c7afba32 TG |
733 | } |
734 | module_exit(cstate_pmu_exit); |