Commit | Line | Data |
---|---|---|
a61127c2 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
26717172 LB |
2 | /* |
3 | * intel_idle.c - native hardware idle loop for modern Intel processors | |
4 | * | |
317e5ec3 | 5 | * Copyright (c) 2013 - 2020, Intel Corporation. |
26717172 | 6 | * Len Brown <len.brown@intel.com> |
317e5ec3 | 7 | * Rafael J. Wysocki <rafael.j.wysocki@intel.com> |
26717172 LB |
8 | */ |
9 | ||
10 | /* | |
8bb2e2a8 | 11 | * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT |
26717172 LB |
12 | * in lieu of the legacy ACPI processor_idle driver. The intent is to |
13 | * make Linux more efficient on these processors, as intel_idle knows | |
14 | * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. | |
15 | */ | |
16 | ||
17 | /* | |
18 | * Design Assumptions | |
19 | * | |
20 | * All CPUs have same idle states as boot CPU | |
21 | * | |
22 | * Chipset BM_STS (bus master status) bit is a NOP | |
8bb2e2a8 AM |
23 | * for preventing entry into deep C-states |
24 | * | |
25 | * CPU will flush caches as needed when entering a C-state via MWAIT | |
26 | * (in contrast to entering ACPI C3, in which case the WBINVD | |
27 | * instruction needs to be executed to flush the caches) | |
26717172 LB |
28 | */ |
29 | ||
30 | /* | |
31 | * Known limitations | |
32 | * | |
26717172 LB |
33 | * ACPI has a .suspend hack to turn off deep c-statees during suspend |
34 | * to avoid complications with the lapic timer workaround. | |
35 | * Have not seen issues with suspend, but may need same workaround here. | |
36 | * | |
26717172 LB |
37 | */ |
38 | ||
39 | /* un-comment DEBUG to enable pr_debug() statements */ | |
651bc581 | 40 | /* #define DEBUG */ |
26717172 | 41 | |
654d08a4 JP |
42 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
43 | ||
18734958 | 44 | #include <linux/acpi.h> |
26717172 LB |
45 | #include <linux/kernel.h> |
46 | #include <linux/cpuidle.h> | |
76962caa | 47 | #include <linux/tick.h> |
26717172 LB |
48 | #include <trace/events/power.h> |
49 | #include <linux/sched.h> | |
bf5835bc | 50 | #include <linux/sched/smt.h> |
2a2d31c8 SL |
51 | #include <linux/notifier.h> |
52 | #include <linux/cpu.h> | |
02c4fae9 | 53 | #include <linux/moduleparam.h> |
b66b8b9a | 54 | #include <asm/cpu_device_id.h> |
db73c5a8 | 55 | #include <asm/intel-family.h> |
bf5835bc | 56 | #include <asm/nospec-branch.h> |
bc83cccc | 57 | #include <asm/mwait.h> |
14796fca | 58 | #include <asm/msr.h> |
26717172 | 59 | |
317e5ec3 | 60 | #define INTEL_IDLE_VERSION "0.5.1" |
26717172 | 61 | |
26717172 LB |
62 | static struct cpuidle_driver intel_idle_driver = { |
63 | .name = "intel_idle", | |
64 | .owner = THIS_MODULE, | |
65 | }; | |
66 | /* intel_idle.max_cstate=0 disables driver */ | |
137ecc77 | 67 | static int max_cstate = CPUIDLE_STATE_MAX - 1; |
4dcb78ee | 68 | static unsigned int disabled_states_mask; |
da0e58c0 | 69 | static unsigned int preferred_states_mask; |
26717172 | 70 | |
6eb0443a | 71 | static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; |
7f843dd7 RW |
72 | |
73 | static unsigned long auto_demotion_disable_flags; | |
39c184a6 AB |
74 | |
75 | static enum { | |
76 | C1E_PROMOTION_PRESERVE, | |
77 | C1E_PROMOTION_ENABLE, | |
78 | C1E_PROMOTION_DISABLE | |
79 | } c1e_promotion = C1E_PROMOTION_PRESERVE; | |
7f843dd7 | 80 | |
b66b8b9a AK |
81 | struct idle_cpu { |
82 | struct cpuidle_state *state_table; | |
83 | ||
84 | /* | |
85 | * Hardware C-state auto-demotion may not always be optimal. | |
86 | * Indicate which enable bits to clear here. | |
87 | */ | |
88 | unsigned long auto_demotion_disable_flags; | |
8c058d53 | 89 | bool byt_auto_demotion_disable_flag; |
32e95180 | 90 | bool disable_promotion_to_c1e; |
bff8e60a | 91 | bool use_acpi; |
b66b8b9a AK |
92 | }; |
93 | ||
7f843dd7 | 94 | static const struct idle_cpu *icpu __initdata; |
7f843dd7 | 95 | static struct cpuidle_state *cpuidle_state_table __initdata; |
26717172 | 96 | |
6eb0443a RW |
97 | static unsigned int mwait_substates __initdata; |
98 | ||
c227233a AB |
99 | /* |
100 | * Enable interrupts before entering the C-state. On some platforms and for | |
101 | * some C-states, this may measurably decrease interrupt latency. | |
102 | */ | |
103 | #define CPUIDLE_FLAG_IRQ_ENABLE BIT(14) | |
104 | ||
bff8e60a RW |
105 | /* |
106 | * Enable this state by default even if the ACPI _CST does not list it. | |
107 | */ | |
108 | #define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15) | |
109 | ||
bf5835bc PZ |
110 | /* |
111 | * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE | |
112 | * above. | |
113 | */ | |
114 | #define CPUIDLE_FLAG_IBRS BIT(16) | |
115 | ||
b1beab48 LB |
116 | /* |
117 | * MWAIT takes an 8-bit "hint" in EAX "suggesting" | |
118 | * the C-state (top nibble) and sub-state (bottom nibble) | |
119 | * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. | |
120 | * | |
121 | * We store the hint at the top of our "flags" for each state. | |
122 | */ | |
123 | #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) | |
124 | #define MWAIT2flg(eax) ((eax & 0xFF) << 24) | |
125 | ||
32d4fd57 PZ |
126 | static __always_inline int __intel_idle(struct cpuidle_device *dev, |
127 | struct cpuidle_driver *drv, int index) | |
128 | { | |
129 | struct cpuidle_state *state = &drv->states[index]; | |
130 | unsigned long eax = flg2MWAIT(state->flags); | |
131 | unsigned long ecx = 1; /* break on interrupt flag */ | |
132 | ||
133 | mwait_idle_with_hints(eax, ecx); | |
134 | ||
135 | return index; | |
136 | } | |
137 | ||
30a996fb RW |
138 | /** |
139 | * intel_idle - Ask the processor to enter the given idle state. | |
140 | * @dev: cpuidle device of the target CPU. | |
141 | * @drv: cpuidle driver (assumed to point to intel_idle_driver). | |
142 | * @index: Target idle state index. | |
143 | * | |
144 | * Use the MWAIT instruction to notify the processor that the CPU represented by | |
145 | * @dev is idle and it can try to enter the idle state corresponding to @index. | |
146 | * | |
147 | * If the local APIC timer is not known to be reliable in the target idle state, | |
148 | * enable one-shot tick broadcasting for the target CPU before executing MWAIT. | |
149 | * | |
30a996fb RW |
150 | * Must be called under local_irq_disable(). |
151 | */ | |
152 | static __cpuidle int intel_idle(struct cpuidle_device *dev, | |
153 | struct cpuidle_driver *drv, int index) | |
154 | { | |
32d4fd57 PZ |
155 | return __intel_idle(dev, drv, index); |
156 | } | |
30a996fb | 157 | |
32d4fd57 PZ |
158 | static __cpuidle int intel_idle_irq(struct cpuidle_device *dev, |
159 | struct cpuidle_driver *drv, int index) | |
160 | { | |
161 | int ret; | |
c227233a | 162 | |
32d4fd57 PZ |
163 | raw_local_irq_enable(); |
164 | ret = __intel_idle(dev, drv, index); | |
d295ad34 WL |
165 | |
166 | /* | |
167 | * The lockdep hardirqs state may be changed to 'on' with timer | |
168 | * tick interrupt followed by __do_softirq(). Use local_irq_disable() | |
169 | * to keep the hardirqs state correct. | |
170 | */ | |
171 | local_irq_disable(); | |
30a996fb | 172 | |
32d4fd57 | 173 | return ret; |
30a996fb RW |
174 | } |
175 | ||
bf5835bc PZ |
176 | static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev, |
177 | struct cpuidle_driver *drv, int index) | |
178 | { | |
179 | bool smt_active = sched_smt_active(); | |
180 | u64 spec_ctrl = spec_ctrl_current(); | |
181 | int ret; | |
182 | ||
183 | if (smt_active) | |
184 | wrmsrl(MSR_IA32_SPEC_CTRL, 0); | |
185 | ||
186 | ret = __intel_idle(dev, drv, index); | |
187 | ||
188 | if (smt_active) | |
189 | wrmsrl(MSR_IA32_SPEC_CTRL, spec_ctrl); | |
190 | ||
191 | return ret; | |
192 | } | |
193 | ||
30a996fb RW |
194 | /** |
195 | * intel_idle_s2idle - Ask the processor to enter the given idle state. | |
196 | * @dev: cpuidle device of the target CPU. | |
197 | * @drv: cpuidle driver (assumed to point to intel_idle_driver). | |
198 | * @index: Target idle state index. | |
199 | * | |
200 | * Use the MWAIT instruction to notify the processor that the CPU represented by | |
201 | * @dev is idle and it can try to enter the idle state corresponding to @index. | |
202 | * | |
203 | * Invoked as a suspend-to-idle callback routine with frozen user space, frozen | |
204 | * scheduler tick and suspended scheduler clock on the target CPU. | |
205 | */ | |
efe97112 NL |
206 | static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev, |
207 | struct cpuidle_driver *drv, int index) | |
30a996fb RW |
208 | { |
209 | unsigned long eax = flg2MWAIT(drv->states[index].flags); | |
210 | unsigned long ecx = 1; /* break on interrupt flag */ | |
211 | ||
212 | mwait_idle_with_hints(eax, ecx); | |
efe97112 NL |
213 | |
214 | return 0; | |
30a996fb RW |
215 | } |
216 | ||
26717172 LB |
217 | /* |
218 | * States are indexed by the cstate number, | |
219 | * which is also the index into the MWAIT hint array. | |
220 | * Thus C0 is a dummy. | |
221 | */ | |
ab1a8522 | 222 | static struct cpuidle_state nehalem_cstates[] __initdata = { |
e022e7eb | 223 | { |
de09cdd0 | 224 | .name = "C1", |
26717172 | 225 | .desc = "MWAIT 0x00", |
b82b6cca | 226 | .flags = MWAIT2flg(0x00), |
26717172 | 227 | .exit_latency = 3, |
26717172 | 228 | .target_residency = 6, |
5fe2e527 | 229 | .enter = &intel_idle, |
28ba086e | 230 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 231 | { |
de09cdd0 | 232 | .name = "C1E", |
32e95180 | 233 | .desc = "MWAIT 0x01", |
e6d4f08a | 234 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
32e95180 LB |
235 | .exit_latency = 10, |
236 | .target_residency = 20, | |
5fe2e527 | 237 | .enter = &intel_idle, |
28ba086e | 238 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 239 | { |
de09cdd0 | 240 | .name = "C3", |
26717172 | 241 | .desc = "MWAIT 0x10", |
b82b6cca | 242 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 243 | .exit_latency = 20, |
26717172 | 244 | .target_residency = 80, |
5fe2e527 | 245 | .enter = &intel_idle, |
28ba086e | 246 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 247 | { |
de09cdd0 | 248 | .name = "C6", |
26717172 | 249 | .desc = "MWAIT 0x20", |
b82b6cca | 250 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 251 | .exit_latency = 200, |
26717172 | 252 | .target_residency = 800, |
5fe2e527 | 253 | .enter = &intel_idle, |
28ba086e | 254 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
255 | { |
256 | .enter = NULL } | |
26717172 LB |
257 | }; |
258 | ||
ab1a8522 | 259 | static struct cpuidle_state snb_cstates[] __initdata = { |
e022e7eb | 260 | { |
de09cdd0 | 261 | .name = "C1", |
d13780d4 | 262 | .desc = "MWAIT 0x00", |
b82b6cca | 263 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
264 | .exit_latency = 2, |
265 | .target_residency = 2, | |
5fe2e527 | 266 | .enter = &intel_idle, |
28ba086e | 267 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 268 | { |
de09cdd0 | 269 | .name = "C1E", |
32e95180 | 270 | .desc = "MWAIT 0x01", |
e6d4f08a | 271 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
32e95180 LB |
272 | .exit_latency = 10, |
273 | .target_residency = 20, | |
5fe2e527 | 274 | .enter = &intel_idle, |
28ba086e | 275 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 276 | { |
de09cdd0 | 277 | .name = "C3", |
d13780d4 | 278 | .desc = "MWAIT 0x10", |
b82b6cca | 279 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 280 | .exit_latency = 80, |
ddbd550d | 281 | .target_residency = 211, |
5fe2e527 | 282 | .enter = &intel_idle, |
28ba086e | 283 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 284 | { |
de09cdd0 | 285 | .name = "C6", |
d13780d4 | 286 | .desc = "MWAIT 0x20", |
b82b6cca | 287 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 288 | .exit_latency = 104, |
ddbd550d | 289 | .target_residency = 345, |
5fe2e527 | 290 | .enter = &intel_idle, |
28ba086e | 291 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 292 | { |
de09cdd0 | 293 | .name = "C7", |
d13780d4 | 294 | .desc = "MWAIT 0x30", |
b82b6cca | 295 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 296 | .exit_latency = 109, |
ddbd550d | 297 | .target_residency = 345, |
5fe2e527 | 298 | .enter = &intel_idle, |
28ba086e | 299 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
300 | { |
301 | .enter = NULL } | |
d13780d4 LB |
302 | }; |
303 | ||
ab1a8522 | 304 | static struct cpuidle_state byt_cstates[] __initdata = { |
718987d6 | 305 | { |
de09cdd0 | 306 | .name = "C1", |
718987d6 | 307 | .desc = "MWAIT 0x00", |
b82b6cca | 308 | .flags = MWAIT2flg(0x00), |
718987d6 LB |
309 | .exit_latency = 1, |
310 | .target_residency = 1, | |
5fe2e527 | 311 | .enter = &intel_idle, |
28ba086e | 312 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 313 | { |
de09cdd0 | 314 | .name = "C6N", |
718987d6 | 315 | .desc = "MWAIT 0x58", |
b82b6cca | 316 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 317 | .exit_latency = 300, |
718987d6 | 318 | .target_residency = 275, |
5fe2e527 | 319 | .enter = &intel_idle, |
28ba086e | 320 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 321 | { |
de09cdd0 | 322 | .name = "C6S", |
718987d6 | 323 | .desc = "MWAIT 0x52", |
b82b6cca | 324 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 325 | .exit_latency = 500, |
718987d6 | 326 | .target_residency = 560, |
5fe2e527 | 327 | .enter = &intel_idle, |
28ba086e | 328 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 329 | { |
de09cdd0 | 330 | .name = "C7", |
718987d6 | 331 | .desc = "MWAIT 0x60", |
b82b6cca | 332 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 | 333 | .exit_latency = 1200, |
d7ef7671 | 334 | .target_residency = 4000, |
5fe2e527 | 335 | .enter = &intel_idle, |
28ba086e | 336 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 337 | { |
de09cdd0 | 338 | .name = "C7S", |
718987d6 | 339 | .desc = "MWAIT 0x64", |
b82b6cca | 340 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 LB |
341 | .exit_latency = 10000, |
342 | .target_residency = 20000, | |
5fe2e527 | 343 | .enter = &intel_idle, |
28ba086e | 344 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 LB |
345 | { |
346 | .enter = NULL } | |
347 | }; | |
348 | ||
ab1a8522 | 349 | static struct cpuidle_state cht_cstates[] __initdata = { |
cab07a56 | 350 | { |
de09cdd0 | 351 | .name = "C1", |
cab07a56 LB |
352 | .desc = "MWAIT 0x00", |
353 | .flags = MWAIT2flg(0x00), | |
354 | .exit_latency = 1, | |
355 | .target_residency = 1, | |
356 | .enter = &intel_idle, | |
28ba086e | 357 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 358 | { |
de09cdd0 | 359 | .name = "C6N", |
cab07a56 LB |
360 | .desc = "MWAIT 0x58", |
361 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, | |
362 | .exit_latency = 80, | |
363 | .target_residency = 275, | |
364 | .enter = &intel_idle, | |
28ba086e | 365 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 366 | { |
de09cdd0 | 367 | .name = "C6S", |
cab07a56 LB |
368 | .desc = "MWAIT 0x52", |
369 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, | |
370 | .exit_latency = 200, | |
371 | .target_residency = 560, | |
372 | .enter = &intel_idle, | |
28ba086e | 373 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 374 | { |
de09cdd0 | 375 | .name = "C7", |
cab07a56 LB |
376 | .desc = "MWAIT 0x60", |
377 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
378 | .exit_latency = 1200, | |
379 | .target_residency = 4000, | |
380 | .enter = &intel_idle, | |
28ba086e | 381 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 382 | { |
de09cdd0 | 383 | .name = "C7S", |
cab07a56 LB |
384 | .desc = "MWAIT 0x64", |
385 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, | |
386 | .exit_latency = 10000, | |
387 | .target_residency = 20000, | |
388 | .enter = &intel_idle, | |
28ba086e | 389 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 LB |
390 | { |
391 | .enter = NULL } | |
392 | }; | |
393 | ||
ab1a8522 | 394 | static struct cpuidle_state ivb_cstates[] __initdata = { |
e022e7eb | 395 | { |
de09cdd0 | 396 | .name = "C1", |
6edab08c | 397 | .desc = "MWAIT 0x00", |
b82b6cca | 398 | .flags = MWAIT2flg(0x00), |
6edab08c LB |
399 | .exit_latency = 1, |
400 | .target_residency = 1, | |
5fe2e527 | 401 | .enter = &intel_idle, |
28ba086e | 402 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 403 | { |
de09cdd0 | 404 | .name = "C1E", |
32e95180 | 405 | .desc = "MWAIT 0x01", |
e6d4f08a | 406 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
32e95180 LB |
407 | .exit_latency = 10, |
408 | .target_residency = 20, | |
5fe2e527 | 409 | .enter = &intel_idle, |
28ba086e | 410 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 411 | { |
de09cdd0 | 412 | .name = "C3", |
6edab08c | 413 | .desc = "MWAIT 0x10", |
b82b6cca | 414 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
415 | .exit_latency = 59, |
416 | .target_residency = 156, | |
5fe2e527 | 417 | .enter = &intel_idle, |
28ba086e | 418 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 419 | { |
de09cdd0 | 420 | .name = "C6", |
6edab08c | 421 | .desc = "MWAIT 0x20", |
b82b6cca | 422 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
423 | .exit_latency = 80, |
424 | .target_residency = 300, | |
5fe2e527 | 425 | .enter = &intel_idle, |
28ba086e | 426 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 427 | { |
de09cdd0 | 428 | .name = "C7", |
6edab08c | 429 | .desc = "MWAIT 0x30", |
b82b6cca | 430 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
431 | .exit_latency = 87, |
432 | .target_residency = 300, | |
5fe2e527 | 433 | .enter = &intel_idle, |
28ba086e | 434 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
435 | { |
436 | .enter = NULL } | |
6edab08c LB |
437 | }; |
438 | ||
ab1a8522 | 439 | static struct cpuidle_state ivt_cstates[] __initdata = { |
0138d8f0 | 440 | { |
de09cdd0 | 441 | .name = "C1", |
0138d8f0 | 442 | .desc = "MWAIT 0x00", |
b82b6cca | 443 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
444 | .exit_latency = 1, |
445 | .target_residency = 1, | |
5fe2e527 | 446 | .enter = &intel_idle, |
28ba086e | 447 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 448 | { |
de09cdd0 | 449 | .name = "C1E", |
0138d8f0 | 450 | .desc = "MWAIT 0x01", |
e6d4f08a | 451 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
0138d8f0 LB |
452 | .exit_latency = 10, |
453 | .target_residency = 80, | |
5fe2e527 | 454 | .enter = &intel_idle, |
28ba086e | 455 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 456 | { |
de09cdd0 | 457 | .name = "C3", |
0138d8f0 | 458 | .desc = "MWAIT 0x10", |
b82b6cca | 459 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
460 | .exit_latency = 59, |
461 | .target_residency = 156, | |
5fe2e527 | 462 | .enter = &intel_idle, |
28ba086e | 463 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 464 | { |
de09cdd0 | 465 | .name = "C6", |
0138d8f0 | 466 | .desc = "MWAIT 0x20", |
b82b6cca | 467 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
468 | .exit_latency = 82, |
469 | .target_residency = 300, | |
5fe2e527 | 470 | .enter = &intel_idle, |
28ba086e | 471 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 LB |
472 | { |
473 | .enter = NULL } | |
474 | }; | |
475 | ||
ab1a8522 | 476 | static struct cpuidle_state ivt_cstates_4s[] __initdata = { |
0138d8f0 | 477 | { |
de09cdd0 | 478 | .name = "C1", |
0138d8f0 | 479 | .desc = "MWAIT 0x00", |
b82b6cca | 480 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
481 | .exit_latency = 1, |
482 | .target_residency = 1, | |
5fe2e527 | 483 | .enter = &intel_idle, |
28ba086e | 484 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 485 | { |
de09cdd0 | 486 | .name = "C1E", |
0138d8f0 | 487 | .desc = "MWAIT 0x01", |
e6d4f08a | 488 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
0138d8f0 LB |
489 | .exit_latency = 10, |
490 | .target_residency = 250, | |
5fe2e527 | 491 | .enter = &intel_idle, |
28ba086e | 492 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 493 | { |
de09cdd0 | 494 | .name = "C3", |
0138d8f0 | 495 | .desc = "MWAIT 0x10", |
b82b6cca | 496 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
497 | .exit_latency = 59, |
498 | .target_residency = 300, | |
5fe2e527 | 499 | .enter = &intel_idle, |
28ba086e | 500 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 501 | { |
de09cdd0 | 502 | .name = "C6", |
0138d8f0 | 503 | .desc = "MWAIT 0x20", |
b82b6cca | 504 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
505 | .exit_latency = 84, |
506 | .target_residency = 400, | |
5fe2e527 | 507 | .enter = &intel_idle, |
28ba086e | 508 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 LB |
509 | { |
510 | .enter = NULL } | |
511 | }; | |
512 | ||
ab1a8522 | 513 | static struct cpuidle_state ivt_cstates_8s[] __initdata = { |
0138d8f0 | 514 | { |
de09cdd0 | 515 | .name = "C1", |
0138d8f0 | 516 | .desc = "MWAIT 0x00", |
b82b6cca | 517 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
518 | .exit_latency = 1, |
519 | .target_residency = 1, | |
5fe2e527 | 520 | .enter = &intel_idle, |
28ba086e | 521 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 522 | { |
de09cdd0 | 523 | .name = "C1E", |
0138d8f0 | 524 | .desc = "MWAIT 0x01", |
e6d4f08a | 525 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
0138d8f0 LB |
526 | .exit_latency = 10, |
527 | .target_residency = 500, | |
5fe2e527 | 528 | .enter = &intel_idle, |
28ba086e | 529 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 530 | { |
de09cdd0 | 531 | .name = "C3", |
0138d8f0 | 532 | .desc = "MWAIT 0x10", |
b82b6cca | 533 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
534 | .exit_latency = 59, |
535 | .target_residency = 600, | |
5fe2e527 | 536 | .enter = &intel_idle, |
28ba086e | 537 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 538 | { |
de09cdd0 | 539 | .name = "C6", |
0138d8f0 | 540 | .desc = "MWAIT 0x20", |
b82b6cca | 541 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
542 | .exit_latency = 88, |
543 | .target_residency = 700, | |
5fe2e527 | 544 | .enter = &intel_idle, |
28ba086e | 545 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 LB |
546 | { |
547 | .enter = NULL } | |
548 | }; | |
549 | ||
ab1a8522 | 550 | static struct cpuidle_state hsw_cstates[] __initdata = { |
e022e7eb | 551 | { |
de09cdd0 | 552 | .name = "C1", |
85a4d2d4 | 553 | .desc = "MWAIT 0x00", |
b82b6cca | 554 | .flags = MWAIT2flg(0x00), |
85a4d2d4 LB |
555 | .exit_latency = 2, |
556 | .target_residency = 2, | |
5fe2e527 | 557 | .enter = &intel_idle, |
28ba086e | 558 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 559 | { |
de09cdd0 | 560 | .name = "C1E", |
32e95180 | 561 | .desc = "MWAIT 0x01", |
e6d4f08a | 562 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
32e95180 LB |
563 | .exit_latency = 10, |
564 | .target_residency = 20, | |
5fe2e527 | 565 | .enter = &intel_idle, |
28ba086e | 566 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 567 | { |
de09cdd0 | 568 | .name = "C3", |
85a4d2d4 | 569 | .desc = "MWAIT 0x10", |
b82b6cca | 570 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
571 | .exit_latency = 33, |
572 | .target_residency = 100, | |
5fe2e527 | 573 | .enter = &intel_idle, |
28ba086e | 574 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 575 | { |
de09cdd0 | 576 | .name = "C6", |
85a4d2d4 | 577 | .desc = "MWAIT 0x20", |
b82b6cca | 578 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
579 | .exit_latency = 133, |
580 | .target_residency = 400, | |
5fe2e527 | 581 | .enter = &intel_idle, |
28ba086e | 582 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 583 | { |
de09cdd0 | 584 | .name = "C7s", |
85a4d2d4 | 585 | .desc = "MWAIT 0x32", |
b82b6cca | 586 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
587 | .exit_latency = 166, |
588 | .target_residency = 500, | |
5fe2e527 | 589 | .enter = &intel_idle, |
28ba086e | 590 | .enter_s2idle = intel_idle_s2idle, }, |
86239ceb | 591 | { |
de09cdd0 | 592 | .name = "C8", |
86239ceb | 593 | .desc = "MWAIT 0x40", |
b82b6cca | 594 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
595 | .exit_latency = 300, |
596 | .target_residency = 900, | |
5fe2e527 | 597 | .enter = &intel_idle, |
28ba086e | 598 | .enter_s2idle = intel_idle_s2idle, }, |
86239ceb | 599 | { |
de09cdd0 | 600 | .name = "C9", |
86239ceb | 601 | .desc = "MWAIT 0x50", |
b82b6cca | 602 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
603 | .exit_latency = 600, |
604 | .target_residency = 1800, | |
5fe2e527 | 605 | .enter = &intel_idle, |
28ba086e | 606 | .enter_s2idle = intel_idle_s2idle, }, |
86239ceb | 607 | { |
de09cdd0 | 608 | .name = "C10", |
86239ceb | 609 | .desc = "MWAIT 0x60", |
b82b6cca | 610 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
611 | .exit_latency = 2600, |
612 | .target_residency = 7700, | |
5fe2e527 | 613 | .enter = &intel_idle, |
28ba086e | 614 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
615 | { |
616 | .enter = NULL } | |
85a4d2d4 | 617 | }; |
ab1a8522 | 618 | static struct cpuidle_state bdw_cstates[] __initdata = { |
a138b568 | 619 | { |
de09cdd0 | 620 | .name = "C1", |
a138b568 | 621 | .desc = "MWAIT 0x00", |
b82b6cca | 622 | .flags = MWAIT2flg(0x00), |
a138b568 LB |
623 | .exit_latency = 2, |
624 | .target_residency = 2, | |
5fe2e527 | 625 | .enter = &intel_idle, |
28ba086e | 626 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 627 | { |
de09cdd0 | 628 | .name = "C1E", |
a138b568 | 629 | .desc = "MWAIT 0x01", |
e6d4f08a | 630 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
a138b568 LB |
631 | .exit_latency = 10, |
632 | .target_residency = 20, | |
5fe2e527 | 633 | .enter = &intel_idle, |
28ba086e | 634 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 635 | { |
de09cdd0 | 636 | .name = "C3", |
a138b568 | 637 | .desc = "MWAIT 0x10", |
b82b6cca | 638 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
639 | .exit_latency = 40, |
640 | .target_residency = 100, | |
5fe2e527 | 641 | .enter = &intel_idle, |
28ba086e | 642 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 643 | { |
de09cdd0 | 644 | .name = "C6", |
a138b568 | 645 | .desc = "MWAIT 0x20", |
b82b6cca | 646 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
647 | .exit_latency = 133, |
648 | .target_residency = 400, | |
5fe2e527 | 649 | .enter = &intel_idle, |
28ba086e | 650 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 651 | { |
de09cdd0 | 652 | .name = "C7s", |
a138b568 | 653 | .desc = "MWAIT 0x32", |
b82b6cca | 654 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
655 | .exit_latency = 166, |
656 | .target_residency = 500, | |
5fe2e527 | 657 | .enter = &intel_idle, |
28ba086e | 658 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 659 | { |
de09cdd0 | 660 | .name = "C8", |
a138b568 | 661 | .desc = "MWAIT 0x40", |
b82b6cca | 662 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
663 | .exit_latency = 300, |
664 | .target_residency = 900, | |
5fe2e527 | 665 | .enter = &intel_idle, |
28ba086e | 666 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 667 | { |
de09cdd0 | 668 | .name = "C9", |
a138b568 | 669 | .desc = "MWAIT 0x50", |
b82b6cca | 670 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
671 | .exit_latency = 600, |
672 | .target_residency = 1800, | |
5fe2e527 | 673 | .enter = &intel_idle, |
28ba086e | 674 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 675 | { |
de09cdd0 | 676 | .name = "C10", |
a138b568 | 677 | .desc = "MWAIT 0x60", |
b82b6cca | 678 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
679 | .exit_latency = 2600, |
680 | .target_residency = 7700, | |
5fe2e527 | 681 | .enter = &intel_idle, |
28ba086e | 682 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 LB |
683 | { |
684 | .enter = NULL } | |
685 | }; | |
85a4d2d4 | 686 | |
ab1a8522 | 687 | static struct cpuidle_state skl_cstates[] __initdata = { |
493f133f | 688 | { |
de09cdd0 | 689 | .name = "C1", |
493f133f LB |
690 | .desc = "MWAIT 0x00", |
691 | .flags = MWAIT2flg(0x00), | |
692 | .exit_latency = 2, | |
693 | .target_residency = 2, | |
694 | .enter = &intel_idle, | |
28ba086e | 695 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 696 | { |
de09cdd0 | 697 | .name = "C1E", |
493f133f | 698 | .desc = "MWAIT 0x01", |
e6d4f08a | 699 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
493f133f LB |
700 | .exit_latency = 10, |
701 | .target_residency = 20, | |
702 | .enter = &intel_idle, | |
28ba086e | 703 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 704 | { |
de09cdd0 | 705 | .name = "C3", |
493f133f LB |
706 | .desc = "MWAIT 0x10", |
707 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
708 | .exit_latency = 70, | |
709 | .target_residency = 100, | |
710 | .enter = &intel_idle, | |
28ba086e | 711 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 712 | { |
de09cdd0 | 713 | .name = "C6", |
493f133f | 714 | .desc = "MWAIT 0x20", |
bf5835bc | 715 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, |
135919a3 | 716 | .exit_latency = 85, |
493f133f LB |
717 | .target_residency = 200, |
718 | .enter = &intel_idle, | |
28ba086e | 719 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 720 | { |
de09cdd0 | 721 | .name = "C7s", |
493f133f | 722 | .desc = "MWAIT 0x33", |
bf5835bc | 723 | .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, |
493f133f LB |
724 | .exit_latency = 124, |
725 | .target_residency = 800, | |
726 | .enter = &intel_idle, | |
28ba086e | 727 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 728 | { |
de09cdd0 | 729 | .name = "C8", |
493f133f | 730 | .desc = "MWAIT 0x40", |
bf5835bc | 731 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, |
135919a3 | 732 | .exit_latency = 200, |
493f133f LB |
733 | .target_residency = 800, |
734 | .enter = &intel_idle, | |
28ba086e | 735 | .enter_s2idle = intel_idle_s2idle, }, |
135919a3 | 736 | { |
de09cdd0 | 737 | .name = "C9", |
135919a3 | 738 | .desc = "MWAIT 0x50", |
bf5835bc | 739 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, |
135919a3 LB |
740 | .exit_latency = 480, |
741 | .target_residency = 5000, | |
742 | .enter = &intel_idle, | |
28ba086e | 743 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 744 | { |
de09cdd0 | 745 | .name = "C10", |
493f133f | 746 | .desc = "MWAIT 0x60", |
bf5835bc | 747 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, |
493f133f LB |
748 | .exit_latency = 890, |
749 | .target_residency = 5000, | |
750 | .enter = &intel_idle, | |
28ba086e | 751 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f LB |
752 | { |
753 | .enter = NULL } | |
754 | }; | |
755 | ||
ab1a8522 | 756 | static struct cpuidle_state skx_cstates[] __initdata = { |
f9e71657 | 757 | { |
de09cdd0 | 758 | .name = "C1", |
f9e71657 | 759 | .desc = "MWAIT 0x00", |
c227233a | 760 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, |
f9e71657 LB |
761 | .exit_latency = 2, |
762 | .target_residency = 2, | |
763 | .enter = &intel_idle, | |
28ba086e | 764 | .enter_s2idle = intel_idle_s2idle, }, |
f9e71657 | 765 | { |
de09cdd0 | 766 | .name = "C1E", |
f9e71657 | 767 | .desc = "MWAIT 0x01", |
e6d4f08a | 768 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
f9e71657 LB |
769 | .exit_latency = 10, |
770 | .target_residency = 20, | |
771 | .enter = &intel_idle, | |
28ba086e | 772 | .enter_s2idle = intel_idle_s2idle, }, |
f9e71657 | 773 | { |
de09cdd0 | 774 | .name = "C6", |
f9e71657 | 775 | .desc = "MWAIT 0x20", |
bf5835bc | 776 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS, |
f9e71657 LB |
777 | .exit_latency = 133, |
778 | .target_residency = 600, | |
779 | .enter = &intel_idle, | |
28ba086e | 780 | .enter_s2idle = intel_idle_s2idle, }, |
f9e71657 LB |
781 | { |
782 | .enter = NULL } | |
783 | }; | |
784 | ||
a472ad2b CY |
785 | static struct cpuidle_state icx_cstates[] __initdata = { |
786 | { | |
787 | .name = "C1", | |
788 | .desc = "MWAIT 0x00", | |
c227233a | 789 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE, |
a472ad2b CY |
790 | .exit_latency = 1, |
791 | .target_residency = 1, | |
792 | .enter = &intel_idle, | |
793 | .enter_s2idle = intel_idle_s2idle, }, | |
794 | { | |
795 | .name = "C1E", | |
796 | .desc = "MWAIT 0x01", | |
797 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, | |
798 | .exit_latency = 4, | |
799 | .target_residency = 4, | |
800 | .enter = &intel_idle, | |
801 | .enter_s2idle = intel_idle_s2idle, }, | |
802 | { | |
803 | .name = "C6", | |
804 | .desc = "MWAIT 0x20", | |
805 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
d484b8bf AB |
806 | .exit_latency = 170, |
807 | .target_residency = 600, | |
a472ad2b CY |
808 | .enter = &intel_idle, |
809 | .enter_s2idle = intel_idle_s2idle, }, | |
810 | { | |
811 | .enter = NULL } | |
812 | }; | |
813 | ||
d1cf8bbf ZR |
814 | /* |
815 | * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa. | |
816 | * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL. | |
817 | * But in this case there is effectively no C1, because C1 requests are | |
818 | * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1 | |
819 | * and C1E requests end up with C1, so there is effectively no C1E. | |
820 | * | |
821 | * By default we enable C1E and disable C1 by marking it with | |
822 | * 'CPUIDLE_FLAG_UNUSABLE'. | |
823 | */ | |
824 | static struct cpuidle_state adl_cstates[] __initdata = { | |
825 | { | |
826 | .name = "C1", | |
827 | .desc = "MWAIT 0x00", | |
828 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, | |
829 | .exit_latency = 1, | |
830 | .target_residency = 1, | |
831 | .enter = &intel_idle, | |
832 | .enter_s2idle = intel_idle_s2idle, }, | |
833 | { | |
834 | .name = "C1E", | |
835 | .desc = "MWAIT 0x01", | |
836 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, | |
837 | .exit_latency = 2, | |
838 | .target_residency = 4, | |
839 | .enter = &intel_idle, | |
840 | .enter_s2idle = intel_idle_s2idle, }, | |
841 | { | |
842 | .name = "C6", | |
843 | .desc = "MWAIT 0x20", | |
844 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
845 | .exit_latency = 220, | |
846 | .target_residency = 600, | |
847 | .enter = &intel_idle, | |
848 | .enter_s2idle = intel_idle_s2idle, }, | |
849 | { | |
850 | .name = "C8", | |
851 | .desc = "MWAIT 0x40", | |
852 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
853 | .exit_latency = 280, | |
854 | .target_residency = 800, | |
855 | .enter = &intel_idle, | |
856 | .enter_s2idle = intel_idle_s2idle, }, | |
857 | { | |
858 | .name = "C10", | |
859 | .desc = "MWAIT 0x60", | |
860 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
861 | .exit_latency = 680, | |
862 | .target_residency = 2000, | |
863 | .enter = &intel_idle, | |
864 | .enter_s2idle = intel_idle_s2idle, }, | |
865 | { | |
866 | .enter = NULL } | |
867 | }; | |
868 | ||
869 | static struct cpuidle_state adl_l_cstates[] __initdata = { | |
870 | { | |
871 | .name = "C1", | |
872 | .desc = "MWAIT 0x00", | |
873 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, | |
874 | .exit_latency = 1, | |
875 | .target_residency = 1, | |
876 | .enter = &intel_idle, | |
877 | .enter_s2idle = intel_idle_s2idle, }, | |
878 | { | |
879 | .name = "C1E", | |
880 | .desc = "MWAIT 0x01", | |
881 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, | |
882 | .exit_latency = 2, | |
883 | .target_residency = 4, | |
884 | .enter = &intel_idle, | |
885 | .enter_s2idle = intel_idle_s2idle, }, | |
886 | { | |
887 | .name = "C6", | |
888 | .desc = "MWAIT 0x20", | |
889 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
890 | .exit_latency = 170, | |
891 | .target_residency = 500, | |
892 | .enter = &intel_idle, | |
893 | .enter_s2idle = intel_idle_s2idle, }, | |
894 | { | |
895 | .name = "C8", | |
896 | .desc = "MWAIT 0x40", | |
897 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
898 | .exit_latency = 200, | |
899 | .target_residency = 600, | |
900 | .enter = &intel_idle, | |
901 | .enter_s2idle = intel_idle_s2idle, }, | |
902 | { | |
903 | .name = "C10", | |
904 | .desc = "MWAIT 0x60", | |
905 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
906 | .exit_latency = 230, | |
907 | .target_residency = 700, | |
908 | .enter = &intel_idle, | |
909 | .enter_s2idle = intel_idle_s2idle, }, | |
910 | { | |
911 | .enter = NULL } | |
912 | }; | |
913 | ||
9edf3c0f AB |
914 | static struct cpuidle_state spr_cstates[] __initdata = { |
915 | { | |
916 | .name = "C1", | |
917 | .desc = "MWAIT 0x00", | |
918 | .flags = MWAIT2flg(0x00), | |
919 | .exit_latency = 1, | |
920 | .target_residency = 1, | |
921 | .enter = &intel_idle, | |
922 | .enter_s2idle = intel_idle_s2idle, }, | |
923 | { | |
924 | .name = "C1E", | |
925 | .desc = "MWAIT 0x01", | |
1548fac4 | 926 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
9edf3c0f AB |
927 | .exit_latency = 2, |
928 | .target_residency = 4, | |
929 | .enter = &intel_idle, | |
930 | .enter_s2idle = intel_idle_s2idle, }, | |
931 | { | |
932 | .name = "C6", | |
933 | .desc = "MWAIT 0x20", | |
934 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
935 | .exit_latency = 290, | |
936 | .target_residency = 800, | |
937 | .enter = &intel_idle, | |
938 | .enter_s2idle = intel_idle_s2idle, }, | |
939 | { | |
940 | .enter = NULL } | |
941 | }; | |
942 | ||
ab1a8522 | 943 | static struct cpuidle_state atom_cstates[] __initdata = { |
e022e7eb | 944 | { |
de09cdd0 | 945 | .name = "C1E", |
26717172 | 946 | .desc = "MWAIT 0x00", |
b82b6cca | 947 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
948 | .exit_latency = 10, |
949 | .target_residency = 20, | |
5fe2e527 | 950 | .enter = &intel_idle, |
28ba086e | 951 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 952 | { |
de09cdd0 | 953 | .name = "C2", |
26717172 | 954 | .desc = "MWAIT 0x10", |
b82b6cca | 955 | .flags = MWAIT2flg(0x10), |
26717172 | 956 | .exit_latency = 20, |
26717172 | 957 | .target_residency = 80, |
5fe2e527 | 958 | .enter = &intel_idle, |
28ba086e | 959 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 960 | { |
de09cdd0 | 961 | .name = "C4", |
26717172 | 962 | .desc = "MWAIT 0x30", |
b82b6cca | 963 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 964 | .exit_latency = 100, |
26717172 | 965 | .target_residency = 400, |
5fe2e527 | 966 | .enter = &intel_idle, |
28ba086e | 967 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 968 | { |
de09cdd0 | 969 | .name = "C6", |
7fcca7d9 | 970 | .desc = "MWAIT 0x52", |
b82b6cca | 971 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
7fcca7d9 | 972 | .exit_latency = 140, |
7fcca7d9 | 973 | .target_residency = 560, |
5fe2e527 | 974 | .enter = &intel_idle, |
28ba086e | 975 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
976 | { |
977 | .enter = NULL } | |
26717172 | 978 | }; |
ab1a8522 | 979 | static struct cpuidle_state tangier_cstates[] __initdata = { |
5e7ec268 | 980 | { |
de09cdd0 | 981 | .name = "C1", |
5e7ec268 AS |
982 | .desc = "MWAIT 0x00", |
983 | .flags = MWAIT2flg(0x00), | |
984 | .exit_latency = 1, | |
985 | .target_residency = 4, | |
986 | .enter = &intel_idle, | |
28ba086e | 987 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 988 | { |
de09cdd0 | 989 | .name = "C4", |
5e7ec268 AS |
990 | .desc = "MWAIT 0x30", |
991 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, | |
992 | .exit_latency = 100, | |
993 | .target_residency = 400, | |
994 | .enter = &intel_idle, | |
28ba086e | 995 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 996 | { |
de09cdd0 | 997 | .name = "C6", |
5e7ec268 AS |
998 | .desc = "MWAIT 0x52", |
999 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1000 | .exit_latency = 140, | |
1001 | .target_residency = 560, | |
1002 | .enter = &intel_idle, | |
28ba086e | 1003 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 1004 | { |
de09cdd0 | 1005 | .name = "C7", |
5e7ec268 AS |
1006 | .desc = "MWAIT 0x60", |
1007 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1008 | .exit_latency = 1200, | |
1009 | .target_residency = 4000, | |
1010 | .enter = &intel_idle, | |
28ba086e | 1011 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 1012 | { |
de09cdd0 | 1013 | .name = "C9", |
5e7ec268 AS |
1014 | .desc = "MWAIT 0x64", |
1015 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1016 | .exit_latency = 10000, | |
1017 | .target_residency = 20000, | |
1018 | .enter = &intel_idle, | |
28ba086e | 1019 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 AS |
1020 | { |
1021 | .enter = NULL } | |
1022 | }; | |
ab1a8522 | 1023 | static struct cpuidle_state avn_cstates[] __initdata = { |
fab04b22 | 1024 | { |
de09cdd0 | 1025 | .name = "C1", |
fab04b22 | 1026 | .desc = "MWAIT 0x00", |
b82b6cca | 1027 | .flags = MWAIT2flg(0x00), |
fab04b22 LB |
1028 | .exit_latency = 2, |
1029 | .target_residency = 2, | |
5fe2e527 | 1030 | .enter = &intel_idle, |
28ba086e | 1031 | .enter_s2idle = intel_idle_s2idle, }, |
fab04b22 | 1032 | { |
de09cdd0 | 1033 | .name = "C6", |
fab04b22 | 1034 | .desc = "MWAIT 0x51", |
b82b6cca | 1035 | .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, |
fab04b22 LB |
1036 | .exit_latency = 15, |
1037 | .target_residency = 45, | |
5fe2e527 | 1038 | .enter = &intel_idle, |
28ba086e | 1039 | .enter_s2idle = intel_idle_s2idle, }, |
88390996 JL |
1040 | { |
1041 | .enter = NULL } | |
fab04b22 | 1042 | }; |
ab1a8522 | 1043 | static struct cpuidle_state knl_cstates[] __initdata = { |
281baf7a | 1044 | { |
de09cdd0 | 1045 | .name = "C1", |
281baf7a DC |
1046 | .desc = "MWAIT 0x00", |
1047 | .flags = MWAIT2flg(0x00), | |
1048 | .exit_latency = 1, | |
1049 | .target_residency = 2, | |
1050 | .enter = &intel_idle, | |
28ba086e | 1051 | .enter_s2idle = intel_idle_s2idle }, |
281baf7a | 1052 | { |
de09cdd0 | 1053 | .name = "C6", |
281baf7a DC |
1054 | .desc = "MWAIT 0x10", |
1055 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1056 | .exit_latency = 120, | |
1057 | .target_residency = 500, | |
1058 | .enter = &intel_idle, | |
28ba086e | 1059 | .enter_s2idle = intel_idle_s2idle }, |
281baf7a DC |
1060 | { |
1061 | .enter = NULL } | |
1062 | }; | |
26717172 | 1063 | |
ab1a8522 | 1064 | static struct cpuidle_state bxt_cstates[] __initdata = { |
5dcef694 | 1065 | { |
de09cdd0 | 1066 | .name = "C1", |
5dcef694 LB |
1067 | .desc = "MWAIT 0x00", |
1068 | .flags = MWAIT2flg(0x00), | |
1069 | .exit_latency = 2, | |
1070 | .target_residency = 2, | |
1071 | .enter = &intel_idle, | |
28ba086e | 1072 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 1073 | { |
de09cdd0 | 1074 | .name = "C1E", |
5dcef694 | 1075 | .desc = "MWAIT 0x01", |
e6d4f08a | 1076 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
5dcef694 LB |
1077 | .exit_latency = 10, |
1078 | .target_residency = 20, | |
1079 | .enter = &intel_idle, | |
28ba086e | 1080 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 1081 | { |
de09cdd0 | 1082 | .name = "C6", |
5dcef694 LB |
1083 | .desc = "MWAIT 0x20", |
1084 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1085 | .exit_latency = 133, | |
1086 | .target_residency = 133, | |
1087 | .enter = &intel_idle, | |
28ba086e | 1088 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 1089 | { |
de09cdd0 | 1090 | .name = "C7s", |
5dcef694 LB |
1091 | .desc = "MWAIT 0x31", |
1092 | .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1093 | .exit_latency = 155, | |
1094 | .target_residency = 155, | |
1095 | .enter = &intel_idle, | |
28ba086e | 1096 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 1097 | { |
de09cdd0 | 1098 | .name = "C8", |
5dcef694 LB |
1099 | .desc = "MWAIT 0x40", |
1100 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1101 | .exit_latency = 1000, | |
1102 | .target_residency = 1000, | |
1103 | .enter = &intel_idle, | |
28ba086e | 1104 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 1105 | { |
de09cdd0 | 1106 | .name = "C9", |
5dcef694 LB |
1107 | .desc = "MWAIT 0x50", |
1108 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1109 | .exit_latency = 2000, | |
1110 | .target_residency = 2000, | |
1111 | .enter = &intel_idle, | |
28ba086e | 1112 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 1113 | { |
de09cdd0 | 1114 | .name = "C10", |
5dcef694 LB |
1115 | .desc = "MWAIT 0x60", |
1116 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1117 | .exit_latency = 10000, | |
1118 | .target_residency = 10000, | |
1119 | .enter = &intel_idle, | |
28ba086e | 1120 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 LB |
1121 | { |
1122 | .enter = NULL } | |
1123 | }; | |
1124 | ||
ab1a8522 | 1125 | static struct cpuidle_state dnv_cstates[] __initdata = { |
0080d65b | 1126 | { |
de09cdd0 | 1127 | .name = "C1", |
0080d65b JP |
1128 | .desc = "MWAIT 0x00", |
1129 | .flags = MWAIT2flg(0x00), | |
1130 | .exit_latency = 2, | |
1131 | .target_residency = 2, | |
1132 | .enter = &intel_idle, | |
28ba086e | 1133 | .enter_s2idle = intel_idle_s2idle, }, |
0080d65b | 1134 | { |
de09cdd0 | 1135 | .name = "C1E", |
0080d65b | 1136 | .desc = "MWAIT 0x01", |
e6d4f08a | 1137 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, |
0080d65b JP |
1138 | .exit_latency = 10, |
1139 | .target_residency = 20, | |
1140 | .enter = &intel_idle, | |
28ba086e | 1141 | .enter_s2idle = intel_idle_s2idle, }, |
0080d65b | 1142 | { |
de09cdd0 | 1143 | .name = "C6", |
0080d65b JP |
1144 | .desc = "MWAIT 0x20", |
1145 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1146 | .exit_latency = 50, | |
1147 | .target_residency = 500, | |
1148 | .enter = &intel_idle, | |
28ba086e | 1149 | .enter_s2idle = intel_idle_s2idle, }, |
0080d65b JP |
1150 | { |
1151 | .enter = NULL } | |
1152 | }; | |
1153 | ||
9cf93f05 AB |
1154 | /* |
1155 | * Note, depending on HW and FW revision, SnowRidge SoC may or may not support | |
1156 | * C6, and this is indicated in the CPUID mwait leaf. | |
1157 | */ | |
1158 | static struct cpuidle_state snr_cstates[] __initdata = { | |
1159 | { | |
1160 | .name = "C1", | |
1161 | .desc = "MWAIT 0x00", | |
1162 | .flags = MWAIT2flg(0x00), | |
1163 | .exit_latency = 2, | |
1164 | .target_residency = 2, | |
1165 | .enter = &intel_idle, | |
1166 | .enter_s2idle = intel_idle_s2idle, }, | |
1167 | { | |
1168 | .name = "C1E", | |
1169 | .desc = "MWAIT 0x01", | |
1170 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, | |
1171 | .exit_latency = 15, | |
1172 | .target_residency = 25, | |
1173 | .enter = &intel_idle, | |
1174 | .enter_s2idle = intel_idle_s2idle, }, | |
1175 | { | |
1176 | .name = "C6", | |
1177 | .desc = "MWAIT 0x20", | |
1178 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
1179 | .exit_latency = 130, | |
1180 | .target_residency = 500, | |
1181 | .enter = &intel_idle, | |
1182 | .enter_s2idle = intel_idle_s2idle, }, | |
1183 | { | |
1184 | .enter = NULL } | |
1185 | }; | |
1186 | ||
ab1a8522 | 1187 | static const struct idle_cpu idle_cpu_nehalem __initconst = { |
b66b8b9a | 1188 | .state_table = nehalem_cstates, |
b66b8b9a | 1189 | .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, |
32e95180 | 1190 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
1191 | }; |
1192 | ||
ab1a8522 | 1193 | static const struct idle_cpu idle_cpu_nhx __initconst = { |
e6d4f08a RW |
1194 | .state_table = nehalem_cstates, |
1195 | .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, | |
1196 | .disable_promotion_to_c1e = true, | |
1197 | .use_acpi = true, | |
1198 | }; | |
1199 | ||
ab1a8522 | 1200 | static const struct idle_cpu idle_cpu_atom __initconst = { |
b66b8b9a AK |
1201 | .state_table = atom_cstates, |
1202 | }; | |
1203 | ||
ab1a8522 | 1204 | static const struct idle_cpu idle_cpu_tangier __initconst = { |
5e7ec268 AS |
1205 | .state_table = tangier_cstates, |
1206 | }; | |
1207 | ||
ab1a8522 | 1208 | static const struct idle_cpu idle_cpu_lincroft __initconst = { |
b66b8b9a AK |
1209 | .state_table = atom_cstates, |
1210 | .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, | |
1211 | }; | |
1212 | ||
ab1a8522 | 1213 | static const struct idle_cpu idle_cpu_snb __initconst = { |
b66b8b9a | 1214 | .state_table = snb_cstates, |
32e95180 | 1215 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
1216 | }; |
1217 | ||
ab1a8522 | 1218 | static const struct idle_cpu idle_cpu_snx __initconst = { |
e6d4f08a RW |
1219 | .state_table = snb_cstates, |
1220 | .disable_promotion_to_c1e = true, | |
1221 | .use_acpi = true, | |
1222 | }; | |
1223 | ||
ab1a8522 | 1224 | static const struct idle_cpu idle_cpu_byt __initconst = { |
718987d6 LB |
1225 | .state_table = byt_cstates, |
1226 | .disable_promotion_to_c1e = true, | |
8c058d53 | 1227 | .byt_auto_demotion_disable_flag = true, |
718987d6 LB |
1228 | }; |
1229 | ||
ab1a8522 | 1230 | static const struct idle_cpu idle_cpu_cht __initconst = { |
cab07a56 LB |
1231 | .state_table = cht_cstates, |
1232 | .disable_promotion_to_c1e = true, | |
1233 | .byt_auto_demotion_disable_flag = true, | |
1234 | }; | |
1235 | ||
ab1a8522 | 1236 | static const struct idle_cpu idle_cpu_ivb __initconst = { |
6edab08c | 1237 | .state_table = ivb_cstates, |
32e95180 | 1238 | .disable_promotion_to_c1e = true, |
6edab08c LB |
1239 | }; |
1240 | ||
ab1a8522 | 1241 | static const struct idle_cpu idle_cpu_ivt __initconst = { |
0138d8f0 LB |
1242 | .state_table = ivt_cstates, |
1243 | .disable_promotion_to_c1e = true, | |
e6d4f08a | 1244 | .use_acpi = true, |
0138d8f0 LB |
1245 | }; |
1246 | ||
ab1a8522 | 1247 | static const struct idle_cpu idle_cpu_hsw __initconst = { |
85a4d2d4 | 1248 | .state_table = hsw_cstates, |
32e95180 | 1249 | .disable_promotion_to_c1e = true, |
85a4d2d4 LB |
1250 | }; |
1251 | ||
ab1a8522 | 1252 | static const struct idle_cpu idle_cpu_hsx __initconst = { |
e6d4f08a RW |
1253 | .state_table = hsw_cstates, |
1254 | .disable_promotion_to_c1e = true, | |
1255 | .use_acpi = true, | |
1256 | }; | |
1257 | ||
ab1a8522 | 1258 | static const struct idle_cpu idle_cpu_bdw __initconst = { |
a138b568 LB |
1259 | .state_table = bdw_cstates, |
1260 | .disable_promotion_to_c1e = true, | |
1261 | }; | |
1262 | ||
ab1a8522 | 1263 | static const struct idle_cpu idle_cpu_bdx __initconst = { |
e6d4f08a RW |
1264 | .state_table = bdw_cstates, |
1265 | .disable_promotion_to_c1e = true, | |
1266 | .use_acpi = true, | |
1267 | }; | |
1268 | ||
ab1a8522 | 1269 | static const struct idle_cpu idle_cpu_skl __initconst = { |
493f133f LB |
1270 | .state_table = skl_cstates, |
1271 | .disable_promotion_to_c1e = true, | |
1272 | }; | |
1273 | ||
ab1a8522 | 1274 | static const struct idle_cpu idle_cpu_skx __initconst = { |
f9e71657 LB |
1275 | .state_table = skx_cstates, |
1276 | .disable_promotion_to_c1e = true, | |
e6d4f08a | 1277 | .use_acpi = true, |
f9e71657 | 1278 | }; |
493f133f | 1279 | |
a472ad2b CY |
1280 | static const struct idle_cpu idle_cpu_icx __initconst = { |
1281 | .state_table = icx_cstates, | |
1282 | .disable_promotion_to_c1e = true, | |
1283 | .use_acpi = true, | |
1284 | }; | |
1285 | ||
d1cf8bbf ZR |
1286 | static const struct idle_cpu idle_cpu_adl __initconst = { |
1287 | .state_table = adl_cstates, | |
1288 | }; | |
1289 | ||
1290 | static const struct idle_cpu idle_cpu_adl_l __initconst = { | |
1291 | .state_table = adl_l_cstates, | |
1292 | }; | |
1293 | ||
9edf3c0f AB |
1294 | static const struct idle_cpu idle_cpu_spr __initconst = { |
1295 | .state_table = spr_cstates, | |
1296 | .disable_promotion_to_c1e = true, | |
1297 | .use_acpi = true, | |
1298 | }; | |
1299 | ||
ab1a8522 | 1300 | static const struct idle_cpu idle_cpu_avn __initconst = { |
fab04b22 LB |
1301 | .state_table = avn_cstates, |
1302 | .disable_promotion_to_c1e = true, | |
e6d4f08a | 1303 | .use_acpi = true, |
fab04b22 LB |
1304 | }; |
1305 | ||
ab1a8522 | 1306 | static const struct idle_cpu idle_cpu_knl __initconst = { |
281baf7a | 1307 | .state_table = knl_cstates, |
e6d4f08a | 1308 | .use_acpi = true, |
281baf7a DC |
1309 | }; |
1310 | ||
ab1a8522 | 1311 | static const struct idle_cpu idle_cpu_bxt __initconst = { |
5dcef694 LB |
1312 | .state_table = bxt_cstates, |
1313 | .disable_promotion_to_c1e = true, | |
1314 | }; | |
1315 | ||
ab1a8522 | 1316 | static const struct idle_cpu idle_cpu_dnv __initconst = { |
0080d65b JP |
1317 | .state_table = dnv_cstates, |
1318 | .disable_promotion_to_c1e = true, | |
e6d4f08a | 1319 | .use_acpi = true, |
0080d65b JP |
1320 | }; |
1321 | ||
9cf93f05 AB |
1322 | static const struct idle_cpu idle_cpu_snr __initconst = { |
1323 | .state_table = snr_cstates, | |
1324 | .disable_promotion_to_c1e = true, | |
1325 | .use_acpi = true, | |
1326 | }; | |
1327 | ||
d5cdc3c4 | 1328 | static const struct x86_cpu_id intel_idle_ids[] __initconst = { |
4a9f45a0 TG |
1329 | X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &idle_cpu_nhx), |
1330 | X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &idle_cpu_nehalem), | |
1331 | X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G, &idle_cpu_nehalem), | |
1332 | X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &idle_cpu_nehalem), | |
1333 | X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &idle_cpu_nhx), | |
1334 | X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &idle_cpu_nhx), | |
1335 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL, &idle_cpu_atom), | |
1336 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID, &idle_cpu_lincroft), | |
1337 | X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &idle_cpu_nhx), | |
1338 | X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &idle_cpu_snb), | |
1339 | X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &idle_cpu_snx), | |
1340 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL, &idle_cpu_atom), | |
1341 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &idle_cpu_byt), | |
1342 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier), | |
1343 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &idle_cpu_cht), | |
1344 | X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &idle_cpu_ivb), | |
1345 | X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &idle_cpu_ivt), | |
1346 | X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &idle_cpu_hsw), | |
1347 | X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &idle_cpu_hsx), | |
1348 | X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &idle_cpu_hsw), | |
1349 | X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &idle_cpu_hsw), | |
1350 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &idle_cpu_avn), | |
1351 | X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &idle_cpu_bdw), | |
1352 | X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &idle_cpu_bdw), | |
1353 | X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &idle_cpu_bdx), | |
1354 | X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &idle_cpu_bdx), | |
1355 | X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &idle_cpu_skl), | |
1356 | X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &idle_cpu_skl), | |
1357 | X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &idle_cpu_skl), | |
1358 | X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &idle_cpu_skl), | |
1359 | X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &idle_cpu_skx), | |
a472ad2b | 1360 | X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &idle_cpu_icx), |
22141d5f | 1361 | X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &idle_cpu_icx), |
d1cf8bbf ZR |
1362 | X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &idle_cpu_adl), |
1363 | X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &idle_cpu_adl_l), | |
9edf3c0f | 1364 | X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr), |
4a9f45a0 TG |
1365 | X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl), |
1366 | X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl), | |
1367 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt), | |
1368 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &idle_cpu_bxt), | |
1369 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &idle_cpu_dnv), | |
9cf93f05 | 1370 | X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &idle_cpu_snr), |
b66b8b9a AK |
1371 | {} |
1372 | }; | |
b66b8b9a | 1373 | |
18734958 | 1374 | static const struct x86_cpu_id intel_mwait_ids[] __initconst = { |
4a9f45a0 | 1375 | X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL), |
18734958 RW |
1376 | {} |
1377 | }; | |
1378 | ||
095928ae | 1379 | static bool __init intel_idle_max_cstate_reached(int cstate) |
18734958 RW |
1380 | { |
1381 | if (cstate + 1 > max_cstate) { | |
1382 | pr_info("max_cstate %d reached\n", max_cstate); | |
1383 | return true; | |
1384 | } | |
1385 | return false; | |
1386 | } | |
1387 | ||
4d916140 PZ |
1388 | static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state) |
1389 | { | |
1390 | unsigned long eax = flg2MWAIT(state->flags); | |
1391 | ||
1392 | if (boot_cpu_has(X86_FEATURE_ARAT)) | |
1393 | return false; | |
1394 | ||
1395 | /* | |
1396 | * Switch over to one-shot tick broadcast if the target C-state | |
1397 | * is deeper than C1. | |
1398 | */ | |
1399 | return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK); | |
1400 | } | |
1401 | ||
18734958 RW |
1402 | #ifdef CONFIG_ACPI_PROCESSOR_CSTATE |
1403 | #include <acpi/processor.h> | |
1404 | ||
4ec32d9e RW |
1405 | static bool no_acpi __read_mostly; |
1406 | module_param(no_acpi, bool, 0444); | |
1407 | MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list"); | |
1408 | ||
3a5be9b8 RW |
1409 | static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */ |
1410 | module_param_named(use_acpi, force_use_acpi, bool, 0444); | |
1411 | MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list"); | |
1412 | ||
095928ae | 1413 | static struct acpi_processor_power acpi_state_table __initdata; |
18734958 RW |
1414 | |
1415 | /** | |
1416 | * intel_idle_cst_usable - Check if the _CST information can be used. | |
1417 | * | |
1418 | * Check if all of the C-states listed by _CST in the max_cstate range are | |
1419 | * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT. | |
1420 | */ | |
095928ae | 1421 | static bool __init intel_idle_cst_usable(void) |
18734958 RW |
1422 | { |
1423 | int cstate, limit; | |
1424 | ||
1425 | limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1), | |
1426 | acpi_state_table.count); | |
1427 | ||
1428 | for (cstate = 1; cstate < limit; cstate++) { | |
1429 | struct acpi_processor_cx *cx = &acpi_state_table.states[cstate]; | |
1430 | ||
1431 | if (cx->entry_method != ACPI_CSTATE_FFH) | |
1432 | return false; | |
1433 | } | |
1434 | ||
1435 | return true; | |
1436 | } | |
1437 | ||
095928ae | 1438 | static bool __init intel_idle_acpi_cst_extract(void) |
18734958 RW |
1439 | { |
1440 | unsigned int cpu; | |
1441 | ||
4ec32d9e RW |
1442 | if (no_acpi) { |
1443 | pr_debug("Not allowed to use ACPI _CST\n"); | |
1444 | return false; | |
1445 | } | |
1446 | ||
18734958 RW |
1447 | for_each_possible_cpu(cpu) { |
1448 | struct acpi_processor *pr = per_cpu(processors, cpu); | |
1449 | ||
1450 | if (!pr) | |
1451 | continue; | |
1452 | ||
1453 | if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table)) | |
1454 | continue; | |
1455 | ||
1456 | acpi_state_table.count++; | |
1457 | ||
1458 | if (!intel_idle_cst_usable()) | |
1459 | continue; | |
1460 | ||
75af76d0 MG |
1461 | if (!acpi_processor_claim_cst_control()) |
1462 | break; | |
18734958 RW |
1463 | |
1464 | return true; | |
1465 | } | |
1466 | ||
75af76d0 | 1467 | acpi_state_table.count = 0; |
18734958 RW |
1468 | pr_debug("ACPI _CST not found or not usable\n"); |
1469 | return false; | |
1470 | } | |
1471 | ||
095928ae | 1472 | static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) |
18734958 RW |
1473 | { |
1474 | int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); | |
1475 | ||
1476 | /* | |
1477 | * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of | |
1478 | * the interesting states are ACPI_CSTATE_FFH. | |
1479 | */ | |
1480 | for (cstate = 1; cstate < limit; cstate++) { | |
1481 | struct acpi_processor_cx *cx; | |
1482 | struct cpuidle_state *state; | |
1483 | ||
4e0ba557 | 1484 | if (intel_idle_max_cstate_reached(cstate - 1)) |
18734958 RW |
1485 | break; |
1486 | ||
1487 | cx = &acpi_state_table.states[cstate]; | |
1488 | ||
1489 | state = &drv->states[drv->state_count++]; | |
1490 | ||
1491 | snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); | |
1492 | strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); | |
1493 | state->exit_latency = cx->latency; | |
1494 | /* | |
1495 | * For C1-type C-states use the same number for both the exit | |
1496 | * latency and target residency, because that is the case for | |
1497 | * C1 in the majority of the static C-states tables above. | |
1498 | * For the other types of C-states, however, set the target | |
1499 | * residency to 3 times the exit latency which should lead to | |
1500 | * a reasonable balance between energy-efficiency and | |
1501 | * performance in the majority of interesting cases. | |
1502 | */ | |
1503 | state->target_residency = cx->latency; | |
1504 | if (cx->type > ACPI_STATE_C1) | |
1505 | state->target_residency *= 3; | |
1506 | ||
1507 | state->flags = MWAIT2flg(cx->address); | |
1508 | if (cx->type > ACPI_STATE_C2) | |
1509 | state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; | |
1510 | ||
4dcb78ee RW |
1511 | if (disabled_states_mask & BIT(cstate)) |
1512 | state->flags |= CPUIDLE_FLAG_OFF; | |
1513 | ||
6e1d2bc6 PZ |
1514 | if (intel_idle_state_needs_timer_stop(state)) |
1515 | state->flags |= CPUIDLE_FLAG_TIMER_STOP; | |
1516 | ||
18734958 RW |
1517 | state->enter = intel_idle; |
1518 | state->enter_s2idle = intel_idle_s2idle; | |
1519 | } | |
1520 | } | |
bff8e60a | 1521 | |
095928ae | 1522 | static bool __init intel_idle_off_by_default(u32 mwait_hint) |
bff8e60a RW |
1523 | { |
1524 | int cstate, limit; | |
1525 | ||
1526 | /* | |
1527 | * If there are no _CST C-states, do not disable any C-states by | |
1528 | * default. | |
1529 | */ | |
1530 | if (!acpi_state_table.count) | |
1531 | return false; | |
1532 | ||
1533 | limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count); | |
1534 | /* | |
1535 | * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of | |
1536 | * the interesting states are ACPI_CSTATE_FFH. | |
1537 | */ | |
1538 | for (cstate = 1; cstate < limit; cstate++) { | |
1539 | if (acpi_state_table.states[cstate].address == mwait_hint) | |
1540 | return false; | |
1541 | } | |
1542 | return true; | |
1543 | } | |
18734958 | 1544 | #else /* !CONFIG_ACPI_PROCESSOR_CSTATE */ |
3a5be9b8 RW |
1545 | #define force_use_acpi (false) |
1546 | ||
18734958 RW |
1547 | static inline bool intel_idle_acpi_cst_extract(void) { return false; } |
1548 | static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { } | |
bff8e60a | 1549 | static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; } |
18734958 RW |
1550 | #endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */ |
1551 | ||
6eacb15f RW |
1552 | /** |
1553 | * ivt_idle_state_table_update - Tune the idle states table for Ivy Town. | |
0138d8f0 | 1554 | * |
6eacb15f RW |
1555 | * Tune IVT multi-socket targets. |
1556 | * Assumption: num_sockets == (max_package_num + 1). | |
0138d8f0 | 1557 | */ |
095928ae | 1558 | static void __init ivt_idle_state_table_update(void) |
0138d8f0 LB |
1559 | { |
1560 | /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ | |
d70e28f5 LB |
1561 | int cpu, package_num, num_sockets = 1; |
1562 | ||
1563 | for_each_online_cpu(cpu) { | |
1564 | package_num = topology_physical_package_id(cpu); | |
1565 | if (package_num + 1 > num_sockets) { | |
1566 | num_sockets = package_num + 1; | |
1567 | ||
1568 | if (num_sockets > 4) { | |
1569 | cpuidle_state_table = ivt_cstates_8s; | |
1570 | return; | |
0138d8f0 LB |
1571 | } |
1572 | } | |
d70e28f5 LB |
1573 | } |
1574 | ||
1575 | if (num_sockets > 2) | |
1576 | cpuidle_state_table = ivt_cstates_4s; | |
1577 | ||
1578 | /* else, 1 and 2 socket systems use default ivt_cstates */ | |
1579 | } | |
5dcef694 | 1580 | |
86e9466a RW |
1581 | /** |
1582 | * irtl_2_usec - IRTL to microseconds conversion. | |
1583 | * @irtl: IRTL MSR value. | |
1584 | * | |
1585 | * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds. | |
5dcef694 | 1586 | */ |
095928ae | 1587 | static unsigned long long __init irtl_2_usec(unsigned long long irtl) |
5dcef694 | 1588 | { |
86e9466a RW |
1589 | static const unsigned int irtl_ns_units[] __initconst = { |
1590 | 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 | |
1591 | }; | |
5dcef694 LB |
1592 | unsigned long long ns; |
1593 | ||
3451ab3e JB |
1594 | if (!irtl) |
1595 | return 0; | |
1596 | ||
bef45096 | 1597 | ns = irtl_ns_units[(irtl >> 10) & 0x7]; |
5dcef694 | 1598 | |
86e9466a | 1599 | return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC); |
5dcef694 | 1600 | } |
86e9466a | 1601 | |
6eacb15f RW |
1602 | /** |
1603 | * bxt_idle_state_table_update - Fix up the Broxton idle states table. | |
5dcef694 | 1604 | * |
6eacb15f RW |
1605 | * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the |
1606 | * definitive maximum latency and use the same value for target_residency. | |
5dcef694 | 1607 | */ |
095928ae | 1608 | static void __init bxt_idle_state_table_update(void) |
5dcef694 LB |
1609 | { |
1610 | unsigned long long msr; | |
3451ab3e | 1611 | unsigned int usec; |
5dcef694 LB |
1612 | |
1613 | rdmsrl(MSR_PKGC6_IRTL, msr); | |
3451ab3e JB |
1614 | usec = irtl_2_usec(msr); |
1615 | if (usec) { | |
5dcef694 LB |
1616 | bxt_cstates[2].exit_latency = usec; |
1617 | bxt_cstates[2].target_residency = usec; | |
1618 | } | |
1619 | ||
1620 | rdmsrl(MSR_PKGC7_IRTL, msr); | |
3451ab3e JB |
1621 | usec = irtl_2_usec(msr); |
1622 | if (usec) { | |
5dcef694 LB |
1623 | bxt_cstates[3].exit_latency = usec; |
1624 | bxt_cstates[3].target_residency = usec; | |
1625 | } | |
1626 | ||
1627 | rdmsrl(MSR_PKGC8_IRTL, msr); | |
3451ab3e JB |
1628 | usec = irtl_2_usec(msr); |
1629 | if (usec) { | |
5dcef694 LB |
1630 | bxt_cstates[4].exit_latency = usec; |
1631 | bxt_cstates[4].target_residency = usec; | |
1632 | } | |
1633 | ||
1634 | rdmsrl(MSR_PKGC9_IRTL, msr); | |
3451ab3e JB |
1635 | usec = irtl_2_usec(msr); |
1636 | if (usec) { | |
5dcef694 LB |
1637 | bxt_cstates[5].exit_latency = usec; |
1638 | bxt_cstates[5].target_residency = usec; | |
1639 | } | |
1640 | ||
1641 | rdmsrl(MSR_PKGC10_IRTL, msr); | |
3451ab3e JB |
1642 | usec = irtl_2_usec(msr); |
1643 | if (usec) { | |
5dcef694 LB |
1644 | bxt_cstates[6].exit_latency = usec; |
1645 | bxt_cstates[6].target_residency = usec; | |
1646 | } | |
1647 | ||
1648 | } | |
6eacb15f RW |
1649 | |
1650 | /** | |
1651 | * sklh_idle_state_table_update - Fix up the Sky Lake idle states table. | |
d70e28f5 | 1652 | * |
6eacb15f | 1653 | * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled. |
d70e28f5 | 1654 | */ |
095928ae | 1655 | static void __init sklh_idle_state_table_update(void) |
d70e28f5 LB |
1656 | { |
1657 | unsigned long long msr; | |
1658 | unsigned int eax, ebx, ecx, edx; | |
1659 | ||
1660 | ||
1661 | /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ | |
1662 | if (max_cstate <= 7) | |
1663 | return; | |
1664 | ||
1665 | /* if PC10 not present in CPUID.MWAIT.EDX */ | |
1666 | if ((mwait_substates & (0xF << 28)) == 0) | |
1667 | return; | |
1668 | ||
6cfb2374 | 1669 | rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); |
d70e28f5 LB |
1670 | |
1671 | /* PC10 is not enabled in PKG C-state limit */ | |
1672 | if ((msr & 0xF) != 8) | |
1673 | return; | |
1674 | ||
1675 | ecx = 0; | |
1676 | cpuid(7, &eax, &ebx, &ecx, &edx); | |
1677 | ||
1678 | /* if SGX is present */ | |
1679 | if (ebx & (1 << 2)) { | |
0138d8f0 | 1680 | |
32ad73db | 1681 | rdmsrl(MSR_IA32_FEAT_CTL, msr); |
d70e28f5 LB |
1682 | |
1683 | /* if SGX is enabled */ | |
1684 | if (msr & (1 << 18)) | |
1685 | return; | |
1686 | } | |
1687 | ||
ba1e78a1 RW |
1688 | skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */ |
1689 | skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */ | |
d70e28f5 | 1690 | } |
d70e28f5 | 1691 | |
64233338 CY |
1692 | /** |
1693 | * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake | |
1694 | * idle states table. | |
1695 | */ | |
1696 | static void __init skx_idle_state_table_update(void) | |
1697 | { | |
1698 | unsigned long long msr; | |
1699 | ||
1700 | rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); | |
1701 | ||
1702 | /* | |
1703 | * 000b: C0/C1 (no package C-state support) | |
1704 | * 001b: C2 | |
1705 | * 010b: C6 (non-retention) | |
1706 | * 011b: C6 (retention) | |
1707 | * 111b: No Package C state limits. | |
1708 | */ | |
1709 | if ((msr & 0x7) < 2) { | |
1710 | /* | |
1711 | * Uses the CC6 + PC0 latency and 3 times of | |
1712 | * latency for target_residency if the PC6 | |
1713 | * is disabled in BIOS. This is consistent | |
1714 | * with how intel_idle driver uses _CST | |
1715 | * to set the target_residency. | |
1716 | */ | |
1717 | skx_cstates[2].exit_latency = 92; | |
1718 | skx_cstates[2].target_residency = 276; | |
1719 | } | |
1720 | } | |
1721 | ||
d1cf8bbf ZR |
1722 | /** |
1723 | * adl_idle_state_table_update - Adjust AlderLake idle states table. | |
1724 | */ | |
1725 | static void __init adl_idle_state_table_update(void) | |
1726 | { | |
1727 | /* Check if user prefers C1 over C1E. */ | |
1728 | if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) { | |
1729 | cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE; | |
1730 | cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE; | |
1731 | ||
1732 | /* Disable C1E by clearing the "C1E promotion" bit. */ | |
1733 | c1e_promotion = C1E_PROMOTION_DISABLE; | |
1734 | return; | |
1735 | } | |
1736 | ||
1737 | /* Make sure C1E is enabled by default */ | |
1738 | c1e_promotion = C1E_PROMOTION_ENABLE; | |
1739 | } | |
1740 | ||
da0e58c0 AB |
1741 | /** |
1742 | * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table. | |
1743 | */ | |
1744 | static void __init spr_idle_state_table_update(void) | |
1745 | { | |
3a9cf77b AB |
1746 | unsigned long long msr; |
1747 | ||
3a9cf77b AB |
1748 | /* |
1749 | * By default, the C6 state assumes the worst-case scenario of package | |
1750 | * C6. However, if PC6 is disabled, we update the numbers to match | |
1751 | * core C6. | |
1752 | */ | |
1753 | rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); | |
1754 | ||
1755 | /* Limit value 2 and above allow for PC6. */ | |
1756 | if ((msr & 0x7) < 2) { | |
1757 | spr_cstates[2].exit_latency = 190; | |
1758 | spr_cstates[2].target_residency = 600; | |
1759 | } | |
da0e58c0 AB |
1760 | } |
1761 | ||
1aefbd7a RW |
1762 | static bool __init intel_idle_verify_cstate(unsigned int mwait_hint) |
1763 | { | |
1764 | unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1; | |
1765 | unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) & | |
1766 | MWAIT_SUBSTATE_MASK; | |
1767 | ||
1768 | /* Ignore the C-state if there are NO sub-states in CPUID for it. */ | |
1769 | if (num_substates == 0) | |
1770 | return false; | |
1771 | ||
1772 | if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) | |
1773 | mark_tsc_unstable("TSC halts in idle states deeper than C2"); | |
1774 | ||
1775 | return true; | |
1776 | } | |
1777 | ||
095928ae | 1778 | static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) |
d70e28f5 | 1779 | { |
3d3a1ae9 | 1780 | int cstate; |
d70e28f5 | 1781 | |
3d3a1ae9 | 1782 | switch (boot_cpu_data.x86_model) { |
db73c5a8 | 1783 | case INTEL_FAM6_IVYBRIDGE_X: |
d70e28f5 LB |
1784 | ivt_idle_state_table_update(); |
1785 | break; | |
db73c5a8 | 1786 | case INTEL_FAM6_ATOM_GOLDMONT: |
f2c4db1b | 1787 | case INTEL_FAM6_ATOM_GOLDMONT_PLUS: |
5dcef694 LB |
1788 | bxt_idle_state_table_update(); |
1789 | break; | |
c66f78a6 | 1790 | case INTEL_FAM6_SKYLAKE: |
d70e28f5 LB |
1791 | sklh_idle_state_table_update(); |
1792 | break; | |
64233338 CY |
1793 | case INTEL_FAM6_SKYLAKE_X: |
1794 | skx_idle_state_table_update(); | |
1795 | break; | |
da0e58c0 AB |
1796 | case INTEL_FAM6_SAPPHIRERAPIDS_X: |
1797 | spr_idle_state_table_update(); | |
1798 | break; | |
d1cf8bbf ZR |
1799 | case INTEL_FAM6_ALDERLAKE: |
1800 | case INTEL_FAM6_ALDERLAKE_L: | |
1801 | adl_idle_state_table_update(); | |
1802 | break; | |
0138d8f0 | 1803 | } |
46bcfad7 | 1804 | |
e022e7eb | 1805 | for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { |
9f3d6daf | 1806 | unsigned int mwait_hint; |
46bcfad7 | 1807 | |
18734958 | 1808 | if (intel_idle_max_cstate_reached(cstate)) |
e022e7eb LB |
1809 | break; |
1810 | ||
18734958 RW |
1811 | if (!cpuidle_state_table[cstate].enter && |
1812 | !cpuidle_state_table[cstate].enter_s2idle) | |
46bcfad7 | 1813 | break; |
46bcfad7 | 1814 | |
9f3d6daf | 1815 | /* If marked as unusable, skip this state. */ |
ba1e78a1 | 1816 | if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) { |
654d08a4 JP |
1817 | pr_debug("state %s is disabled\n", |
1818 | cpuidle_state_table[cstate].name); | |
d70e28f5 LB |
1819 | continue; |
1820 | } | |
1821 | ||
9f3d6daf RW |
1822 | mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); |
1823 | if (!intel_idle_verify_cstate(mwait_hint)) | |
1824 | continue; | |
d70e28f5 | 1825 | |
9f3d6daf | 1826 | /* Structure copy. */ |
bff8e60a RW |
1827 | drv->states[drv->state_count] = cpuidle_state_table[cstate]; |
1828 | ||
32d4fd57 PZ |
1829 | if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_IRQ_ENABLE) |
1830 | drv->states[drv->state_count].enter = intel_idle_irq; | |
1831 | ||
bf5835bc PZ |
1832 | if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) && |
1833 | cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_IBRS) { | |
1834 | WARN_ON_ONCE(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_IRQ_ENABLE); | |
1835 | drv->states[drv->state_count].enter = intel_idle_ibrs; | |
1836 | } | |
1837 | ||
4dcb78ee RW |
1838 | if ((disabled_states_mask & BIT(drv->state_count)) || |
1839 | ((icpu->use_acpi || force_use_acpi) && | |
1840 | intel_idle_off_by_default(mwait_hint) && | |
1841 | !(cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_ALWAYS_ENABLE))) | |
bff8e60a RW |
1842 | drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF; |
1843 | ||
6e1d2bc6 PZ |
1844 | if (intel_idle_state_needs_timer_stop(&drv->states[drv->state_count])) |
1845 | drv->states[drv->state_count].flags |= CPUIDLE_FLAG_TIMER_STOP; | |
1846 | ||
bff8e60a | 1847 | drv->state_count++; |
46bcfad7 DD |
1848 | } |
1849 | ||
8c058d53 LB |
1850 | if (icpu->byt_auto_demotion_disable_flag) { |
1851 | wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); | |
1852 | wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); | |
1853 | } | |
46bcfad7 DD |
1854 | } |
1855 | ||
6eacb15f RW |
1856 | /** |
1857 | * intel_idle_cpuidle_driver_init - Create the list of available idle states. | |
1858 | * @drv: cpuidle driver structure to initialize. | |
18734958 | 1859 | */ |
3d3a1ae9 | 1860 | static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv) |
18734958 | 1861 | { |
18734958 | 1862 | cpuidle_poll_state_init(drv); |
4dcb78ee RW |
1863 | |
1864 | if (disabled_states_mask & BIT(0)) | |
1865 | drv->states[0].flags |= CPUIDLE_FLAG_OFF; | |
1866 | ||
18734958 RW |
1867 | drv->state_count = 1; |
1868 | ||
1869 | if (icpu) | |
1870 | intel_idle_init_cstates_icpu(drv); | |
1871 | else | |
1872 | intel_idle_init_cstates_acpi(drv); | |
1873 | } | |
46bcfad7 | 1874 | |
1aefbd7a RW |
1875 | static void auto_demotion_disable(void) |
1876 | { | |
1877 | unsigned long long msr_bits; | |
1878 | ||
1879 | rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); | |
7f843dd7 | 1880 | msr_bits &= ~auto_demotion_disable_flags; |
1aefbd7a RW |
1881 | wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); |
1882 | } | |
1883 | ||
da0e58c0 AB |
1884 | static void c1e_promotion_enable(void) |
1885 | { | |
1886 | unsigned long long msr_bits; | |
1887 | ||
1888 | rdmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
1889 | msr_bits |= 0x2; | |
1890 | wrmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
1891 | } | |
1892 | ||
1aefbd7a RW |
1893 | static void c1e_promotion_disable(void) |
1894 | { | |
1895 | unsigned long long msr_bits; | |
1896 | ||
1897 | rdmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
1898 | msr_bits &= ~0x2; | |
1899 | wrmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
1900 | } | |
1901 | ||
6eacb15f RW |
1902 | /** |
1903 | * intel_idle_cpu_init - Register the target CPU with the cpuidle core. | |
1904 | * @cpu: CPU to initialize. | |
1905 | * | |
1906 | * Register a cpuidle device object for @cpu and update its MSRs in accordance | |
1907 | * with the processor model flags. | |
26717172 | 1908 | */ |
fb1013a0 | 1909 | static int intel_idle_cpu_init(unsigned int cpu) |
26717172 | 1910 | { |
26717172 LB |
1911 | struct cpuidle_device *dev; |
1912 | ||
65b7f839 | 1913 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); |
65b7f839 | 1914 | dev->cpu = cpu; |
26717172 | 1915 | |
65b7f839 | 1916 | if (cpuidle_register_device(dev)) { |
654d08a4 | 1917 | pr_debug("cpuidle_register_device %d failed!\n", cpu); |
65b7f839 | 1918 | return -EIO; |
26717172 LB |
1919 | } |
1920 | ||
7f843dd7 | 1921 | if (auto_demotion_disable_flags) |
fb1013a0 | 1922 | auto_demotion_disable(); |
65b7f839 | 1923 | |
39c184a6 AB |
1924 | if (c1e_promotion == C1E_PROMOTION_ENABLE) |
1925 | c1e_promotion_enable(); | |
1926 | else if (c1e_promotion == C1E_PROMOTION_DISABLE) | |
fb1013a0 SAS |
1927 | c1e_promotion_disable(); |
1928 | ||
1929 | return 0; | |
1930 | } | |
1931 | ||
1932 | static int intel_idle_cpu_online(unsigned int cpu) | |
1933 | { | |
1934 | struct cpuidle_device *dev; | |
1935 | ||
dab20177 | 1936 | if (!boot_cpu_has(X86_FEATURE_ARAT)) |
cbd2c4c2 | 1937 | tick_broadcast_enable(); |
fb1013a0 SAS |
1938 | |
1939 | /* | |
1940 | * Some systems can hotplug a cpu at runtime after | |
1941 | * the kernel has booted, we have to initialize the | |
1942 | * driver in this case | |
1943 | */ | |
1944 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); | |
1945 | if (!dev->registered) | |
1946 | return intel_idle_cpu_init(cpu); | |
dbf87ab8 | 1947 | |
26717172 LB |
1948 | return 0; |
1949 | } | |
26717172 | 1950 | |
0755a9bd RW |
1951 | /** |
1952 | * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices. | |
1953 | */ | |
1954 | static void __init intel_idle_cpuidle_devices_uninit(void) | |
1955 | { | |
1956 | int i; | |
1957 | ||
1958 | for_each_online_cpu(i) | |
1959 | cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i)); | |
1960 | } | |
1961 | ||
26717172 LB |
1962 | static int __init intel_idle_init(void) |
1963 | { | |
a6c86e33 RW |
1964 | const struct x86_cpu_id *id; |
1965 | unsigned int eax, ebx, ecx; | |
fb1013a0 | 1966 | int retval; |
26717172 | 1967 | |
d1896049 TR |
1968 | /* Do not load intel_idle at all for now if idle= is passed */ |
1969 | if (boot_option_idle_override != IDLE_NO_OVERRIDE) | |
1970 | return -ENODEV; | |
1971 | ||
a6c86e33 RW |
1972 | if (max_cstate == 0) { |
1973 | pr_debug("disabled\n"); | |
1974 | return -EPERM; | |
1975 | } | |
1976 | ||
1977 | id = x86_match_cpu(intel_idle_ids); | |
1978 | if (id) { | |
1979 | if (!boot_cpu_has(X86_FEATURE_MWAIT)) { | |
1980 | pr_debug("Please enable MWAIT in BIOS SETUP\n"); | |
1981 | return -ENODEV; | |
1982 | } | |
1983 | } else { | |
1984 | id = x86_match_cpu(intel_mwait_ids); | |
1985 | if (!id) | |
1986 | return -ENODEV; | |
1987 | } | |
1988 | ||
1989 | if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) | |
1990 | return -ENODEV; | |
1991 | ||
1992 | cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); | |
1993 | ||
1994 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || | |
1995 | !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || | |
1996 | !mwait_substates) | |
1997 | return -ENODEV; | |
1998 | ||
1999 | pr_debug("MWAIT substates: 0x%x\n", mwait_substates); | |
2000 | ||
2001 | icpu = (const struct idle_cpu *)id->driver_data; | |
2002 | if (icpu) { | |
2003 | cpuidle_state_table = icpu->state_table; | |
7f843dd7 | 2004 | auto_demotion_disable_flags = icpu->auto_demotion_disable_flags; |
39c184a6 AB |
2005 | if (icpu->disable_promotion_to_c1e) |
2006 | c1e_promotion = C1E_PROMOTION_DISABLE; | |
3a5be9b8 | 2007 | if (icpu->use_acpi || force_use_acpi) |
a6c86e33 RW |
2008 | intel_idle_acpi_cst_extract(); |
2009 | } else if (!intel_idle_acpi_cst_extract()) { | |
2010 | return -ENODEV; | |
2011 | } | |
2012 | ||
2013 | pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", | |
2014 | boot_cpu_data.x86_model); | |
26717172 | 2015 | |
e9df69cc | 2016 | intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); |
533da74a | 2017 | if (!intel_idle_cpuidle_devices) |
e9df69cc RC |
2018 | return -ENOMEM; |
2019 | ||
3d3a1ae9 RW |
2020 | intel_idle_cpuidle_driver_init(&intel_idle_driver); |
2021 | ||
26717172 LB |
2022 | retval = cpuidle_register_driver(&intel_idle_driver); |
2023 | if (retval) { | |
3735d524 | 2024 | struct cpuidle_driver *drv = cpuidle_get_driver(); |
654d08a4 JP |
2025 | printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), |
2026 | drv ? drv->name : "none"); | |
fb1013a0 | 2027 | goto init_driver_fail; |
26717172 LB |
2028 | } |
2029 | ||
fb1013a0 SAS |
2030 | retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", |
2031 | intel_idle_cpu_online, NULL); | |
2032 | if (retval < 0) | |
2033 | goto hp_setup_fail; | |
26717172 | 2034 | |
40ab82e0 | 2035 | pr_debug("Local APIC timer is reliable in %s\n", |
dab20177 | 2036 | boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1"); |
2259a819 | 2037 | |
26717172 | 2038 | return 0; |
fb1013a0 SAS |
2039 | |
2040 | hp_setup_fail: | |
2041 | intel_idle_cpuidle_devices_uninit(); | |
2042 | cpuidle_unregister_driver(&intel_idle_driver); | |
2043 | init_driver_fail: | |
2044 | free_percpu(intel_idle_cpuidle_devices); | |
2045 | return retval; | |
2046 | ||
26717172 | 2047 | } |
02c4fae9 | 2048 | device_initcall(intel_idle_init); |
26717172 | 2049 | |
02c4fae9 PG |
2050 | /* |
2051 | * We are not really modular, but we used to support that. Meaning we also | |
2052 | * support "intel_idle.max_cstate=..." at boot and also a read-only export of | |
2053 | * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param | |
2054 | * is the easiest way (currently) to continue doing that. | |
2055 | */ | |
26717172 | 2056 | module_param(max_cstate, int, 0444); |
4dcb78ee RW |
2057 | /* |
2058 | * The positions of the bits that are set in this number are the indices of the | |
2059 | * idle states to be disabled by default (as reflected by the names of the | |
2060 | * corresponding idle state directories in sysfs, "state0", "state1" ... | |
2061 | * "state<i>" ..., where <i> is the index of the given state). | |
2062 | */ | |
2063 | module_param_named(states_off, disabled_states_mask, uint, 0444); | |
2064 | MODULE_PARM_DESC(states_off, "Mask of disabled idle states"); | |
da0e58c0 AB |
2065 | /* |
2066 | * Some platforms come with mutually exclusive C-states, so that if one is | |
2067 | * enabled, the other C-states must not be used. Example: C1 and C1E on | |
2068 | * Sapphire Rapids platform. This parameter allows for selecting the | |
2069 | * preferred C-states among the groups of mutually exclusive C-states - the | |
2070 | * selected C-states will be registered, the other C-states from the mutually | |
2071 | * exclusive group won't be registered. If the platform has no mutually | |
2072 | * exclusive C-states, this parameter has no effect. | |
2073 | */ | |
2074 | module_param_named(preferred_cstates, preferred_states_mask, uint, 0444); | |
2075 | MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states"); |