projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Linux 6.16-rc6
[linux-2.6-block.git] / drivers / idle / intel_idle.c
CommitLineData
a61127c2 1// SPDX-License-Identifier: GPL-2.0-only
26717172
LB		 2/*
3 * intel_idle.c - native hardware idle loop for modern Intel processors
4 *
317e5ec3 5 * Copyright (c) 2013 - 2020, Intel Corporation.
26717172 6 * Len Brown <len.brown@intel.com>
317e5ec3 7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com>
26717172
LB		 8 */
9
10/*
8bb2e2a8 11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
26717172
LB		 12 * in lieu of the legacy ACPI processor_idle driver. The intent is to
13 * make Linux more efficient on these processors, as intel_idle knows
14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
15 */
16
17/*
18 * Design Assumptions
19 *
20 * All CPUs have same idle states as boot CPU
21 *
22 * Chipset BM_STS (bus master status) bit is a NOP
8bb2e2a8
AM		 23 * for preventing entry into deep C-states
24 *
25 * CPU will flush caches as needed when entering a C-state via MWAIT
26 * (in contrast to entering ACPI C3, in which case the WBINVD
27 * instruction needs to be executed to flush the caches)
26717172
LB		 28 */
29
30/*
31 * Known limitations
32 *
26717172
LB		 33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
34 * to avoid complications with the lapic timer workaround.
35 * Have not seen issues with suspend, but may need same workaround here.
36 *
26717172
LB		 37 */
38
39/* un-comment DEBUG to enable pr_debug() statements */
651bc581 40/* #define DEBUG */
26717172 41
654d08a4
JP		 42#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43
18734958 44#include <linux/acpi.h>
26717172
LB		 45#include <linux/kernel.h>
46#include <linux/cpuidle.h>
76962caa 47#include <linux/tick.h>
26717172
LB		 48#include <trace/events/power.h>
49#include <linux/sched.h>
bf5835bc 50#include <linux/sched/smt.h>
6138f345 51#include <linux/mutex.h>
2a2d31c8
SL		 52#include <linux/notifier.h>
53#include <linux/cpu.h>
02c4fae9 54#include <linux/moduleparam.h>
6138f345 55#include <linux/sysfs.h>
968e3000 56#include <asm/cpuid/api.h>
b66b8b9a 57#include <asm/cpu_device_id.h>
db73c5a8 58#include <asm/intel-family.h>
bc83cccc 59#include <asm/mwait.h>
75062030 60#include <asm/spec-ctrl.h>
efef7f18 61#include <asm/msr.h>
c157d351 62#include <asm/tsc.h>
9f011293 63#include <asm/fpu/api.h>
fc4ca953 64#include <asm/smp.h>
26717172 65
317e5ec3 66#define INTEL_IDLE_VERSION "0.5.1"
26717172 67
26717172
LB		 68static struct cpuidle_driver intel_idle_driver = {
69 .name = "intel_idle",
70 .owner = THIS_MODULE,
71};
72/* intel_idle.max_cstate=0 disables driver */
137ecc77 73static int max_cstate = CPUIDLE_STATE_MAX - 1;
bd446829
AB		 74static unsigned int disabled_states_mask __read_mostly;
75static unsigned int preferred_states_mask __read_mostly;
4152379a 76static bool force_irq_on __read_mostly;
aa1567a7 77static bool ibrs_off __read_mostly;
26717172 78
6eb0443a 79static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
7f843dd7
RW		 80
81static unsigned long auto_demotion_disable_flags;
39c184a6
AB		 82
83static enum {
84 C1E_PROMOTION_PRESERVE,
85 C1E_PROMOTION_ENABLE,
86 C1E_PROMOTION_DISABLE
87} c1e_promotion = C1E_PROMOTION_PRESERVE;
7f843dd7 88
b66b8b9a
AK		 89struct idle_cpu {
90 struct cpuidle_state *state_table;
91
92 /*
93 * Hardware C-state auto-demotion may not always be optimal.
94 * Indicate which enable bits to clear here.
95 */
96 unsigned long auto_demotion_disable_flags;
32e95180 97 bool disable_promotion_to_c1e;
6138f345 98 bool c1_demotion_supported;
bff8e60a 99 bool use_acpi;
b66b8b9a
AK		 100};
101
6138f345
AB		 102static bool c1_demotion_supported;
103static DEFINE_MUTEX(c1_demotion_mutex);
104
105static struct device *sysfs_root __initdata;
106
7f843dd7 107static const struct idle_cpu *icpu __initdata;
7f843dd7 108static struct cpuidle_state *cpuidle_state_table __initdata;
26717172 109
6eb0443a
RW		 110static unsigned int mwait_substates __initdata;
111
c227233a
AB		 112/*
113 * Enable interrupts before entering the C-state. On some platforms and for
114 * some C-states, this may measurably decrease interrupt latency.
115 */
116#define CPUIDLE_FLAG_IRQ_ENABLE BIT(14)
117
bff8e60a
RW		 118/*
119 * Enable this state by default even if the ACPI _CST does not list it.
120 */
121#define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)
122
bf5835bc
PZ		 123/*
124 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
125 * above.
126 */
127#define CPUIDLE_FLAG_IBRS BIT(16)
128
9f011293
CB		 129/*
130 * Initialize large xstate for the C6-state entrance.
131 */
132#define CPUIDLE_FLAG_INIT_XSTATE BIT(17)
133
4c411cca
AB		 134/*
135 * Ignore the sub-state when matching mwait hints between the ACPI _CST and
136 * custom tables.
137 */
138#define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18)
139
b1beab48
LB		 140/*
141 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
142 * the C-state (top nibble) and sub-state (bottom nibble)
143 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
144 *
145 * We store the hint at the top of our "flags" for each state.
146 */
147#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
148#define MWAIT2flg(eax) ((eax & 0xFF) << 24)
149
32d4fd57 150static __always_inline int __intel_idle(struct cpuidle_device *dev,
edc8fc01
PZ		 151 struct cpuidle_driver *drv,
152 int index, bool irqoff)
32d4fd57
PZ		 153{
154 struct cpuidle_state *state = &drv->states[index];
72840238
UB		 155 unsigned int eax = flg2MWAIT(state->flags);
156 unsigned int ecx = 1*irqoff; /* break on interrupt flag */
32d4fd57
PZ		 157
158 mwait_idle_with_hints(eax, ecx);
159
160 return index;
161}
162
30a996fb
RW		 163/**
164 * intel_idle - Ask the processor to enter the given idle state.
165 * @dev: cpuidle device of the target CPU.
166 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
167 * @index: Target idle state index.
168 *
169 * Use the MWAIT instruction to notify the processor that the CPU represented by
170 * @dev is idle and it can try to enter the idle state corresponding to @index.
171 *
172 * If the local APIC timer is not known to be reliable in the target idle state,
173 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
174 *
30a996fb
RW		 175 * Must be called under local_irq_disable().
176 */
177static __cpuidle int intel_idle(struct cpuidle_device *dev,
178 struct cpuidle_driver *drv, int index)
179{
edc8fc01 180 return __intel_idle(dev, drv, index, true);
32d4fd57 181}
30a996fb 182
32d4fd57
PZ		 183static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
184 struct cpuidle_driver *drv, int index)
185{
edc8fc01 186 return __intel_idle(dev, drv, index, false);
30a996fb
RW		 187}
188
bf5835bc
PZ		 189static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
190 struct cpuidle_driver *drv, int index)
191{
192 bool smt_active = sched_smt_active();
193 u64 spec_ctrl = spec_ctrl_current();
194 int ret;
195
196 if (smt_active)
75062030 197 __update_spec_ctrl(0);
bf5835bc 198
edc8fc01 199 ret = __intel_idle(dev, drv, index, true);
bf5835bc
PZ		 200
201 if (smt_active)
75062030 202 __update_spec_ctrl(spec_ctrl);
bf5835bc
PZ		 203
204 return ret;
205}
206
9f011293
CB		 207static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
208 struct cpuidle_driver *drv, int index)
209{
210 fpu_idle_fpregs();
edc8fc01 211 return __intel_idle(dev, drv, index, true);
9f011293
CB		 212}
213
30a996fb
RW		 214/**
215 * intel_idle_s2idle - Ask the processor to enter the given idle state.
216 * @dev: cpuidle device of the target CPU.
217 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
218 * @index: Target idle state index.
219 *
220 * Use the MWAIT instruction to notify the processor that the CPU represented by
221 * @dev is idle and it can try to enter the idle state corresponding to @index.
222 *
223 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
224 * scheduler tick and suspended scheduler clock on the target CPU.
225 */
efe97112
NL		 226static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
227 struct cpuidle_driver *drv, int index)
30a996fb 228{
9f011293 229 struct cpuidle_state *state = &drv->states[index];
72840238
UB		 230 unsigned int eax = flg2MWAIT(state->flags);
231 unsigned int ecx = 1; /* break on interrupt flag */
9f011293
CB		 232
233 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
234 fpu_idle_fpregs();
30a996fb
RW		 235
236 mwait_idle_with_hints(eax, ecx);
efe97112
NL		 237
238 return 0;
30a996fb
RW		 239}
240
fc4ca953
PW		 241static void intel_idle_enter_dead(struct cpuidle_device *dev, int index)
242{
243 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
244 struct cpuidle_state *state = &drv->states[index];
245 unsigned long eax = flg2MWAIT(state->flags);
246
247 mwait_play_dead(eax);
248}
249
26717172
LB		 250/*
251 * States are indexed by the cstate number,
252 * which is also the index into the MWAIT hint array.
253 * Thus C0 is a dummy.
254 */
ab1a8522 255static struct cpuidle_state nehalem_cstates[] __initdata = {
e022e7eb 256 {
de09cdd0 257 .name = "C1",
26717172 258 .desc = "MWAIT 0x00",
b82b6cca 259 .flags = MWAIT2flg(0x00),
26717172 260 .exit_latency = 3,
26717172 261 .target_residency = 6,
5fe2e527 262 .enter = &intel_idle,
28ba086e 263 .enter_s2idle = intel_idle_s2idle, },
32e95180 264 {
de09cdd0 265 .name = "C1E",
32e95180 266 .desc = "MWAIT 0x01",
e6d4f08a 267 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
32e95180
LB		 268 .exit_latency = 10,
269 .target_residency = 20,
5fe2e527 270 .enter = &intel_idle,
28ba086e 271 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 272 {
de09cdd0 273 .name = "C3",
26717172 274 .desc = "MWAIT 0x10",
b82b6cca 275 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
26717172 276 .exit_latency = 20,
26717172 277 .target_residency = 80,
5fe2e527 278 .enter = &intel_idle,
28ba086e 279 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 280 {
de09cdd0 281 .name = "C6",
26717172 282 .desc = "MWAIT 0x20",
b82b6cca 283 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
26717172 284 .exit_latency = 200,
26717172 285 .target_residency = 800,
5fe2e527 286 .enter = &intel_idle,
28ba086e 287 .enter_s2idle = intel_idle_s2idle, },
e022e7eb
LB		 288 {
289 .enter = NULL }
26717172
LB		 290};
291
ab1a8522 292static struct cpuidle_state snb_cstates[] __initdata = {
e022e7eb 293 {
de09cdd0 294 .name = "C1",
d13780d4 295 .desc = "MWAIT 0x00",
b82b6cca 296 .flags = MWAIT2flg(0x00),
32e95180
LB		 297 .exit_latency = 2,
298 .target_residency = 2,
5fe2e527 299 .enter = &intel_idle,
28ba086e 300 .enter_s2idle = intel_idle_s2idle, },
32e95180 301 {
de09cdd0 302 .name = "C1E",
32e95180 303 .desc = "MWAIT 0x01",
e6d4f08a 304 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
32e95180
LB		 305 .exit_latency = 10,
306 .target_residency = 20,
5fe2e527 307 .enter = &intel_idle,
28ba086e 308 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 309 {
de09cdd0 310 .name = "C3",
d13780d4 311 .desc = "MWAIT 0x10",
b82b6cca 312 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
d13780d4 313 .exit_latency = 80,
ddbd550d 314 .target_residency = 211,
5fe2e527 315 .enter = &intel_idle,
28ba086e 316 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 317 {
de09cdd0 318 .name = "C6",
d13780d4 319 .desc = "MWAIT 0x20",
b82b6cca 320 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
d13780d4 321 .exit_latency = 104,
ddbd550d 322 .target_residency = 345,
5fe2e527 323 .enter = &intel_idle,
28ba086e 324 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 325 {
de09cdd0 326 .name = "C7",
d13780d4 327 .desc = "MWAIT 0x30",
b82b6cca 328 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
d13780d4 329 .exit_latency = 109,
ddbd550d 330 .target_residency = 345,
5fe2e527 331 .enter = &intel_idle,
28ba086e 332 .enter_s2idle = intel_idle_s2idle, },
e022e7eb
LB		 333 {
334 .enter = NULL }
d13780d4
LB		 335};
336
ab1a8522 337static struct cpuidle_state byt_cstates[] __initdata = {
718987d6 338 {
de09cdd0 339 .name = "C1",
718987d6 340 .desc = "MWAIT 0x00",
b82b6cca 341 .flags = MWAIT2flg(0x00),
718987d6
LB		 342 .exit_latency = 1,
343 .target_residency = 1,
5fe2e527 344 .enter = &intel_idle,
28ba086e 345 .enter_s2idle = intel_idle_s2idle, },
718987d6 346 {
de09cdd0 347 .name = "C6N",
718987d6 348 .desc = "MWAIT 0x58",
b82b6cca 349 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
d7ef7671 350 .exit_latency = 300,
718987d6 351 .target_residency = 275,
5fe2e527 352 .enter = &intel_idle,
28ba086e 353 .enter_s2idle = intel_idle_s2idle, },
718987d6 354 {
de09cdd0 355 .name = "C6S",
718987d6 356 .desc = "MWAIT 0x52",
b82b6cca 357 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
d7ef7671 358 .exit_latency = 500,
718987d6 359 .target_residency = 560,
5fe2e527 360 .enter = &intel_idle,
28ba086e 361 .enter_s2idle = intel_idle_s2idle, },
718987d6 362 {
de09cdd0 363 .name = "C7",
718987d6 364 .desc = "MWAIT 0x60",
b82b6cca 365 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
718987d6 366 .exit_latency = 1200,
d7ef7671 367 .target_residency = 4000,
5fe2e527 368 .enter = &intel_idle,
28ba086e 369 .enter_s2idle = intel_idle_s2idle, },
718987d6 370 {
de09cdd0 371 .name = "C7S",
718987d6 372 .desc = "MWAIT 0x64",
b82b6cca 373 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
718987d6
LB		 374 .exit_latency = 10000,
375 .target_residency = 20000,
5fe2e527 376 .enter = &intel_idle,
28ba086e 377 .enter_s2idle = intel_idle_s2idle, },
718987d6
LB		 378 {
379 .enter = NULL }
380};
381
ab1a8522 382static struct cpuidle_state cht_cstates[] __initdata = {
cab07a56 383 {
de09cdd0 384 .name = "C1",
cab07a56
LB		 385 .desc = "MWAIT 0x00",
386 .flags = MWAIT2flg(0x00),
387 .exit_latency = 1,
388 .target_residency = 1,
389 .enter = &intel_idle,
28ba086e 390 .enter_s2idle = intel_idle_s2idle, },
cab07a56 391 {
de09cdd0 392 .name = "C6N",
cab07a56
LB		 393 .desc = "MWAIT 0x58",
394 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
395 .exit_latency = 80,
396 .target_residency = 275,
397 .enter = &intel_idle,
28ba086e 398 .enter_s2idle = intel_idle_s2idle, },
cab07a56 399 {
de09cdd0 400 .name = "C6S",
cab07a56
LB		 401 .desc = "MWAIT 0x52",
402 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
403 .exit_latency = 200,
404 .target_residency = 560,
405 .enter = &intel_idle,
28ba086e 406 .enter_s2idle = intel_idle_s2idle, },
cab07a56 407 {
de09cdd0 408 .name = "C7",
cab07a56
LB		 409 .desc = "MWAIT 0x60",
410 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
411 .exit_latency = 1200,
412 .target_residency = 4000,
413 .enter = &intel_idle,
28ba086e 414 .enter_s2idle = intel_idle_s2idle, },
cab07a56 415 {
de09cdd0 416 .name = "C7S",
cab07a56
LB		 417 .desc = "MWAIT 0x64",
418 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
419 .exit_latency = 10000,
420 .target_residency = 20000,
421 .enter = &intel_idle,
28ba086e 422 .enter_s2idle = intel_idle_s2idle, },
cab07a56
LB		 423 {
424 .enter = NULL }
425};
426
ab1a8522 427static struct cpuidle_state ivb_cstates[] __initdata = {
e022e7eb 428 {
de09cdd0 429 .name = "C1",
6edab08c 430 .desc = "MWAIT 0x00",
b82b6cca 431 .flags = MWAIT2flg(0x00),
6edab08c
LB		 432 .exit_latency = 1,
433 .target_residency = 1,
5fe2e527 434 .enter = &intel_idle,
28ba086e 435 .enter_s2idle = intel_idle_s2idle, },
32e95180 436 {
de09cdd0 437 .name = "C1E",
32e95180 438 .desc = "MWAIT 0x01",
e6d4f08a 439 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
32e95180
LB		 440 .exit_latency = 10,
441 .target_residency = 20,
5fe2e527 442 .enter = &intel_idle,
28ba086e 443 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 444 {
de09cdd0 445 .name = "C3",
6edab08c 446 .desc = "MWAIT 0x10",
b82b6cca 447 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
6edab08c
LB		 448 .exit_latency = 59,
449 .target_residency = 156,
5fe2e527 450 .enter = &intel_idle,
28ba086e 451 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 452 {
de09cdd0 453 .name = "C6",
6edab08c 454 .desc = "MWAIT 0x20",
b82b6cca 455 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
6edab08c
LB		 456 .exit_latency = 80,
457 .target_residency = 300,
5fe2e527 458 .enter = &intel_idle,
28ba086e 459 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 460 {
de09cdd0 461 .name = "C7",
6edab08c 462 .desc = "MWAIT 0x30",
b82b6cca 463 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
6edab08c
LB		 464 .exit_latency = 87,
465 .target_residency = 300,
5fe2e527 466 .enter = &intel_idle,
28ba086e 467 .enter_s2idle = intel_idle_s2idle, },
e022e7eb
LB		 468 {
469 .enter = NULL }
6edab08c
LB		 470};
471
ab1a8522 472static struct cpuidle_state ivt_cstates[] __initdata = {
0138d8f0 473 {
de09cdd0 474 .name = "C1",
0138d8f0 475 .desc = "MWAIT 0x00",
b82b6cca 476 .flags = MWAIT2flg(0x00),
0138d8f0
LB		 477 .exit_latency = 1,
478 .target_residency = 1,
5fe2e527 479 .enter = &intel_idle,
28ba086e 480 .enter_s2idle = intel_idle_s2idle, },
0138d8f0 481 {
de09cdd0 482 .name = "C1E",
0138d8f0 483 .desc = "MWAIT 0x01",
e6d4f08a 484 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
0138d8f0
LB		 485 .exit_latency = 10,
486 .target_residency = 80,
5fe2e527 487 .enter = &intel_idle,
28ba086e 488 .enter_s2idle = intel_idle_s2idle, },
0138d8f0 489 {
de09cdd0 490 .name = "C3",
0138d8f0 491 .desc = "MWAIT 0x10",
b82b6cca 492 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
0138d8f0
LB		 493 .exit_latency = 59,
494 .target_residency = 156,
5fe2e527 495 .enter = &intel_idle,
28ba086e 496 .enter_s2idle = intel_idle_s2idle, },
0138d8f0 497 {
de09cdd0 498 .name = "C6",
0138d8f0 499 .desc = "MWAIT 0x20",
b82b6cca 500 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
0138d8f0
LB		 501 .exit_latency = 82,
502 .target_residency = 300,
5fe2e527 503 .enter = &intel_idle,
28ba086e 504 .enter_s2idle = intel_idle_s2idle, },
0138d8f0
LB		 505 {
506 .enter = NULL }
507};
508
ab1a8522 509static struct cpuidle_state ivt_cstates_4s[] __initdata = {
0138d8f0 510 {
de09cdd0 511 .name = "C1",
0138d8f0 512 .desc = "MWAIT 0x00",
b82b6cca 513 .flags = MWAIT2flg(0x00),
0138d8f0
LB		 514 .exit_latency = 1,
515 .target_residency = 1,
5fe2e527 516 .enter = &intel_idle,
28ba086e 517 .enter_s2idle = intel_idle_s2idle, },
0138d8f0 518 {
de09cdd0 519 .name = "C1E",
0138d8f0 520 .desc = "MWAIT 0x01",
e6d4f08a 521 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
0138d8f0
LB		 522 .exit_latency = 10,
523 .target_residency = 250,
5fe2e527 524 .enter = &intel_idle,
28ba086e 525 .enter_s2idle = intel_idle_s2idle, },
0138d8f0 526 {
de09cdd0 527 .name = "C3",
0138d8f0 528 .desc = "MWAIT 0x10",
b82b6cca 529 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
0138d8f0
LB		 530 .exit_latency = 59,
531 .target_residency = 300,
5fe2e527 532 .enter = &intel_idle,
28ba086e 533 .enter_s2idle = intel_idle_s2idle, },
0138d8f0 534 {
de09cdd0 535 .name = "C6",
0138d8f0 536 .desc = "MWAIT 0x20",
b82b6cca 537 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
0138d8f0
LB		 538 .exit_latency = 84,
539 .target_residency = 400,
5fe2e527 540 .enter = &intel_idle,
28ba086e 541 .enter_s2idle = intel_idle_s2idle, },
0138d8f0
LB		 542 {
543 .enter = NULL }
544};
545
ab1a8522 546static struct cpuidle_state ivt_cstates_8s[] __initdata = {
0138d8f0 547 {
de09cdd0 548 .name = "C1",
0138d8f0 549 .desc = "MWAIT 0x00",
b82b6cca 550 .flags = MWAIT2flg(0x00),
0138d8f0
LB		 551 .exit_latency = 1,
552 .target_residency = 1,
5fe2e527 553 .enter = &intel_idle,
28ba086e 554 .enter_s2idle = intel_idle_s2idle, },
0138d8f0 555 {
de09cdd0 556 .name = "C1E",
0138d8f0 557 .desc = "MWAIT 0x01",
e6d4f08a 558 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
0138d8f0
LB		 559 .exit_latency = 10,
560 .target_residency = 500,
5fe2e527 561 .enter = &intel_idle,
28ba086e 562 .enter_s2idle = intel_idle_s2idle, },
0138d8f0 563 {
de09cdd0 564 .name = "C3",
0138d8f0 565 .desc = "MWAIT 0x10",
b82b6cca 566 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
0138d8f0
LB		 567 .exit_latency = 59,
568 .target_residency = 600,
5fe2e527 569 .enter = &intel_idle,
28ba086e 570 .enter_s2idle = intel_idle_s2idle, },
0138d8f0 571 {
de09cdd0 572 .name = "C6",
0138d8f0 573 .desc = "MWAIT 0x20",
b82b6cca 574 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
0138d8f0
LB		 575 .exit_latency = 88,
576 .target_residency = 700,
5fe2e527 577 .enter = &intel_idle,
28ba086e 578 .enter_s2idle = intel_idle_s2idle, },
0138d8f0
LB		 579 {
580 .enter = NULL }
581};
582
ab1a8522 583static struct cpuidle_state hsw_cstates[] __initdata = {
e022e7eb 584 {
de09cdd0 585 .name = "C1",
85a4d2d4 586 .desc = "MWAIT 0x00",
b82b6cca 587 .flags = MWAIT2flg(0x00),
85a4d2d4
LB		 588 .exit_latency = 2,
589 .target_residency = 2,
5fe2e527 590 .enter = &intel_idle,
28ba086e 591 .enter_s2idle = intel_idle_s2idle, },
32e95180 592 {
de09cdd0 593 .name = "C1E",
32e95180 594 .desc = "MWAIT 0x01",
e6d4f08a 595 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
32e95180
LB		 596 .exit_latency = 10,
597 .target_residency = 20,
5fe2e527 598 .enter = &intel_idle,
28ba086e 599 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 600 {
de09cdd0 601 .name = "C3",
85a4d2d4 602 .desc = "MWAIT 0x10",
b82b6cca 603 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
85a4d2d4
LB		 604 .exit_latency = 33,
605 .target_residency = 100,
5fe2e527 606 .enter = &intel_idle,
28ba086e 607 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 608 {
de09cdd0 609 .name = "C6",
85a4d2d4 610 .desc = "MWAIT 0x20",
b82b6cca 611 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
85a4d2d4
LB		 612 .exit_latency = 133,
613 .target_residency = 400,
5fe2e527 614 .enter = &intel_idle,
28ba086e 615 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 616 {
de09cdd0 617 .name = "C7s",
85a4d2d4 618 .desc = "MWAIT 0x32",
b82b6cca 619 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
85a4d2d4
LB		 620 .exit_latency = 166,
621 .target_residency = 500,
5fe2e527 622 .enter = &intel_idle,
28ba086e 623 .enter_s2idle = intel_idle_s2idle, },
86239ceb 624 {
de09cdd0 625 .name = "C8",
86239ceb 626 .desc = "MWAIT 0x40",
b82b6cca 627 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
86239ceb
LB		 628 .exit_latency = 300,
629 .target_residency = 900,
5fe2e527 630 .enter = &intel_idle,
28ba086e 631 .enter_s2idle = intel_idle_s2idle, },
86239ceb 632 {
de09cdd0 633 .name = "C9",
86239ceb 634 .desc = "MWAIT 0x50",
b82b6cca 635 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
86239ceb
LB		 636 .exit_latency = 600,
637 .target_residency = 1800,
5fe2e527 638 .enter = &intel_idle,
28ba086e 639 .enter_s2idle = intel_idle_s2idle, },
86239ceb 640 {
de09cdd0 641 .name = "C10",
86239ceb 642 .desc = "MWAIT 0x60",
b82b6cca 643 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
86239ceb
LB		 644 .exit_latency = 2600,
645 .target_residency = 7700,
5fe2e527 646 .enter = &intel_idle,
28ba086e 647 .enter_s2idle = intel_idle_s2idle, },
e022e7eb
LB		 648 {
649 .enter = NULL }
85a4d2d4 650};
ab1a8522 651static struct cpuidle_state bdw_cstates[] __initdata = {
a138b568 652 {
de09cdd0 653 .name = "C1",
a138b568 654 .desc = "MWAIT 0x00",
b82b6cca 655 .flags = MWAIT2flg(0x00),
a138b568
LB		 656 .exit_latency = 2,
657 .target_residency = 2,
5fe2e527 658 .enter = &intel_idle,
28ba086e 659 .enter_s2idle = intel_idle_s2idle, },
a138b568 660 {
de09cdd0 661 .name = "C1E",
a138b568 662 .desc = "MWAIT 0x01",
e6d4f08a 663 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
a138b568
LB		 664 .exit_latency = 10,
665 .target_residency = 20,
5fe2e527 666 .enter = &intel_idle,
28ba086e 667 .enter_s2idle = intel_idle_s2idle, },
a138b568 668 {
de09cdd0 669 .name = "C3",
a138b568 670 .desc = "MWAIT 0x10",
b82b6cca 671 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
a138b568
LB		 672 .exit_latency = 40,
673 .target_residency = 100,
5fe2e527 674 .enter = &intel_idle,
28ba086e 675 .enter_s2idle = intel_idle_s2idle, },
a138b568 676 {
de09cdd0 677 .name = "C6",
a138b568 678 .desc = "MWAIT 0x20",
b82b6cca 679 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
a138b568
LB		 680 .exit_latency = 133,
681 .target_residency = 400,
5fe2e527 682 .enter = &intel_idle,
28ba086e 683 .enter_s2idle = intel_idle_s2idle, },
a138b568 684 {
de09cdd0 685 .name = "C7s",
a138b568 686 .desc = "MWAIT 0x32",
b82b6cca 687 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
a138b568
LB		 688 .exit_latency = 166,
689 .target_residency = 500,
5fe2e527 690 .enter = &intel_idle,
28ba086e 691 .enter_s2idle = intel_idle_s2idle, },
a138b568 692 {
de09cdd0 693 .name = "C8",
a138b568 694 .desc = "MWAIT 0x40",
b82b6cca 695 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
a138b568
LB		 696 .exit_latency = 300,
697 .target_residency = 900,
5fe2e527 698 .enter = &intel_idle,
28ba086e 699 .enter_s2idle = intel_idle_s2idle, },
a138b568 700 {
de09cdd0 701 .name = "C9",
a138b568 702 .desc = "MWAIT 0x50",
b82b6cca 703 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
a138b568
LB		 704 .exit_latency = 600,
705 .target_residency = 1800,
5fe2e527 706 .enter = &intel_idle,
28ba086e 707 .enter_s2idle = intel_idle_s2idle, },
a138b568 708 {
de09cdd0 709 .name = "C10",
a138b568 710 .desc = "MWAIT 0x60",
b82b6cca 711 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
a138b568
LB		 712 .exit_latency = 2600,
713 .target_residency = 7700,
5fe2e527 714 .enter = &intel_idle,
28ba086e 715 .enter_s2idle = intel_idle_s2idle, },
a138b568
LB		 716 {
717 .enter = NULL }
718};
85a4d2d4 719
ab1a8522 720static struct cpuidle_state skl_cstates[] __initdata = {
493f133f 721 {
de09cdd0 722 .name = "C1",
493f133f
LB		 723 .desc = "MWAIT 0x00",
724 .flags = MWAIT2flg(0x00),
725 .exit_latency = 2,
726 .target_residency = 2,
727 .enter = &intel_idle,
28ba086e 728 .enter_s2idle = intel_idle_s2idle, },
493f133f 729 {
de09cdd0 730 .name = "C1E",
493f133f 731 .desc = "MWAIT 0x01",
e6d4f08a 732 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
493f133f
LB		 733 .exit_latency = 10,
734 .target_residency = 20,
735 .enter = &intel_idle,
28ba086e 736 .enter_s2idle = intel_idle_s2idle, },
493f133f 737 {
de09cdd0 738 .name = "C3",
493f133f
LB		 739 .desc = "MWAIT 0x10",
740 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
741 .exit_latency = 70,
742 .target_residency = 100,
743 .enter = &intel_idle,
28ba086e 744 .enter_s2idle = intel_idle_s2idle, },
493f133f 745 {
de09cdd0 746 .name = "C6",
493f133f 747 .desc = "MWAIT 0x20",
bf5835bc 748 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
135919a3 749 .exit_latency = 85,
493f133f
LB		 750 .target_residency = 200,
751 .enter = &intel_idle,
28ba086e 752 .enter_s2idle = intel_idle_s2idle, },
493f133f 753 {
de09cdd0 754 .name = "C7s",
493f133f 755 .desc = "MWAIT 0x33",
bf5835bc 756 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
493f133f
LB		 757 .exit_latency = 124,
758 .target_residency = 800,
759 .enter = &intel_idle,
28ba086e 760 .enter_s2idle = intel_idle_s2idle, },
493f133f 761 {
de09cdd0 762 .name = "C8",
493f133f 763 .desc = "MWAIT 0x40",
bf5835bc 764 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
135919a3 765 .exit_latency = 200,
493f133f
LB		 766 .target_residency = 800,
767 .enter = &intel_idle,
28ba086e 768 .enter_s2idle = intel_idle_s2idle, },
135919a3 769 {
de09cdd0 770 .name = "C9",
135919a3 771 .desc = "MWAIT 0x50",
bf5835bc 772 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
135919a3
LB		 773 .exit_latency = 480,
774 .target_residency = 5000,
775 .enter = &intel_idle,
28ba086e 776 .enter_s2idle = intel_idle_s2idle, },
493f133f 777 {
de09cdd0 778 .name = "C10",
493f133f 779 .desc = "MWAIT 0x60",
bf5835bc 780 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
493f133f
LB		 781 .exit_latency = 890,
782 .target_residency = 5000,
783 .enter = &intel_idle,
28ba086e 784 .enter_s2idle = intel_idle_s2idle, },
493f133f
LB		 785 {
786 .enter = NULL }
787};
788
ab1a8522 789static struct cpuidle_state skx_cstates[] __initdata = {
f9e71657 790 {
de09cdd0 791 .name = "C1",
f9e71657 792 .desc = "MWAIT 0x00",
c227233a 793 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
f9e71657
LB		 794 .exit_latency = 2,
795 .target_residency = 2,
796 .enter = &intel_idle,
28ba086e 797 .enter_s2idle = intel_idle_s2idle, },
f9e71657 798 {
de09cdd0 799 .name = "C1E",
f9e71657 800 .desc = "MWAIT 0x01",
e6d4f08a 801 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
f9e71657
LB		 802 .exit_latency = 10,
803 .target_residency = 20,
804 .enter = &intel_idle,
28ba086e 805 .enter_s2idle = intel_idle_s2idle, },
f9e71657 806 {
de09cdd0 807 .name = "C6",
f9e71657 808 .desc = "MWAIT 0x20",
bf5835bc 809 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
f9e71657
LB		 810 .exit_latency = 133,
811 .target_residency = 600,
812 .enter = &intel_idle,
28ba086e 813 .enter_s2idle = intel_idle_s2idle, },
f9e71657
LB		 814 {
815 .enter = NULL }
816};
817
a472ad2b
CY		 818static struct cpuidle_state icx_cstates[] __initdata = {
819 {
820 .name = "C1",
821 .desc = "MWAIT 0x00",
c227233a 822 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
a472ad2b
CY		 823 .exit_latency = 1,
824 .target_residency = 1,
825 .enter = &intel_idle,
826 .enter_s2idle = intel_idle_s2idle, },
827 {
828 .name = "C1E",
829 .desc = "MWAIT 0x01",
830 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
831 .exit_latency = 4,
832 .target_residency = 4,
833 .enter = &intel_idle,
834 .enter_s2idle = intel_idle_s2idle, },
835 {
836 .name = "C6",
837 .desc = "MWAIT 0x20",
838 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
d484b8bf
AB		 839 .exit_latency = 170,
840 .target_residency = 600,
a472ad2b
CY		 841 .enter = &intel_idle,
842 .enter_s2idle = intel_idle_s2idle, },
843 {
844 .enter = NULL }
845};
846
d1cf8bbf
ZR		 847/*
848 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
849 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
850 * But in this case there is effectively no C1, because C1 requests are
851 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
852 * and C1E requests end up with C1, so there is effectively no C1E.
853 *
854 * By default we enable C1E and disable C1 by marking it with
855 * 'CPUIDLE_FLAG_UNUSABLE'.
856 */
857static struct cpuidle_state adl_cstates[] __initdata = {
858 {
859 .name = "C1",
860 .desc = "MWAIT 0x00",
861 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
862 .exit_latency = 1,
863 .target_residency = 1,
864 .enter = &intel_idle,
865 .enter_s2idle = intel_idle_s2idle, },
866 {
867 .name = "C1E",
868 .desc = "MWAIT 0x01",
869 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
870 .exit_latency = 2,
871 .target_residency = 4,
872 .enter = &intel_idle,
873 .enter_s2idle = intel_idle_s2idle, },
874 {
875 .name = "C6",
876 .desc = "MWAIT 0x20",
877 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
878 .exit_latency = 220,
879 .target_residency = 600,
880 .enter = &intel_idle,
881 .enter_s2idle = intel_idle_s2idle, },
882 {
883 .name = "C8",
884 .desc = "MWAIT 0x40",
885 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
886 .exit_latency = 280,
887 .target_residency = 800,
888 .enter = &intel_idle,
889 .enter_s2idle = intel_idle_s2idle, },
890 {
891 .name = "C10",
892 .desc = "MWAIT 0x60",
893 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
894 .exit_latency = 680,
895 .target_residency = 2000,
896 .enter = &intel_idle,
897 .enter_s2idle = intel_idle_s2idle, },
898 {
899 .enter = NULL }
900};
901
902static struct cpuidle_state adl_l_cstates[] __initdata = {
903 {
904 .name = "C1",
905 .desc = "MWAIT 0x00",
906 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
907 .exit_latency = 1,
908 .target_residency = 1,
909 .enter = &intel_idle,
910 .enter_s2idle = intel_idle_s2idle, },
911 {
912 .name = "C1E",
913 .desc = "MWAIT 0x01",
914 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
915 .exit_latency = 2,
916 .target_residency = 4,
917 .enter = &intel_idle,
918 .enter_s2idle = intel_idle_s2idle, },
919 {
920 .name = "C6",
921 .desc = "MWAIT 0x20",
922 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
923 .exit_latency = 170,
924 .target_residency = 500,
925 .enter = &intel_idle,
926 .enter_s2idle = intel_idle_s2idle, },
927 {
928 .name = "C8",
929 .desc = "MWAIT 0x40",
930 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
931 .exit_latency = 200,
932 .target_residency = 600,
933 .enter = &intel_idle,
934 .enter_s2idle = intel_idle_s2idle, },
935 {
936 .name = "C10",
937 .desc = "MWAIT 0x60",
938 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
939 .exit_latency = 230,
940 .target_residency = 700,
941 .enter = &intel_idle,
942 .enter_s2idle = intel_idle_s2idle, },
943 {
944 .enter = NULL }
945};
946
eeae55ed
ZR		 947static struct cpuidle_state mtl_l_cstates[] __initdata = {
948 {
949 .name = "C1E",
950 .desc = "MWAIT 0x01",
951 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
952 .exit_latency = 1,
953 .target_residency = 1,
954 .enter = &intel_idle,
955 .enter_s2idle = intel_idle_s2idle, },
956 {
957 .name = "C6",
958 .desc = "MWAIT 0x20",
959 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
960 .exit_latency = 140,
961 .target_residency = 420,
962 .enter = &intel_idle,
963 .enter_s2idle = intel_idle_s2idle, },
964 {
965 .name = "C10",
966 .desc = "MWAIT 0x60",
967 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
968 .exit_latency = 310,
969 .target_residency = 930,
970 .enter = &intel_idle,
971 .enter_s2idle = intel_idle_s2idle, },
972 {
973 .enter = NULL }
974};
975
882cdb06 976static struct cpuidle_state gmt_cstates[] __initdata = {
65c0c236
ZR		 977 {
978 .name = "C1",
979 .desc = "MWAIT 0x00",
980 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
981 .exit_latency = 1,
982 .target_residency = 1,
983 .enter = &intel_idle,
984 .enter_s2idle = intel_idle_s2idle, },
985 {
986 .name = "C1E",
987 .desc = "MWAIT 0x01",
988 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
989 .exit_latency = 2,
990 .target_residency = 4,
991 .enter = &intel_idle,
992 .enter_s2idle = intel_idle_s2idle, },
993 {
994 .name = "C6",
995 .desc = "MWAIT 0x20",
996 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
997 .exit_latency = 195,
998 .target_residency = 585,
999 .enter = &intel_idle,
1000 .enter_s2idle = intel_idle_s2idle, },
1001 {
1002 .name = "C8",
1003 .desc = "MWAIT 0x40",
1004 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1005 .exit_latency = 260,
1006 .target_residency = 1040,
1007 .enter = &intel_idle,
1008 .enter_s2idle = intel_idle_s2idle, },
1009 {
1010 .name = "C10",
1011 .desc = "MWAIT 0x60",
1012 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1013 .exit_latency = 660,
1014 .target_residency = 1980,
1015 .enter = &intel_idle,
1016 .enter_s2idle = intel_idle_s2idle, },
1017 {
1018 .enter = NULL }
1019};
1020
9edf3c0f
AB		 1021static struct cpuidle_state spr_cstates[] __initdata = {
1022 {
1023 .name = "C1",
1024 .desc = "MWAIT 0x00",
1025 .flags = MWAIT2flg(0x00),
1026 .exit_latency = 1,
1027 .target_residency = 1,
1028 .enter = &intel_idle,
1029 .enter_s2idle = intel_idle_s2idle, },
1030 {
1031 .name = "C1E",
1032 .desc = "MWAIT 0x01",
1548fac4 1033 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
9edf3c0f
AB		 1034 .exit_latency = 2,
1035 .target_residency = 4,
1036 .enter = &intel_idle,
1037 .enter_s2idle = intel_idle_s2idle, },
1038 {
1039 .name = "C6",
1040 .desc = "MWAIT 0x20",
9f011293
CB		 1041 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1042 CPUIDLE_FLAG_INIT_XSTATE,
9edf3c0f
AB		 1043 .exit_latency = 290,
1044 .target_residency = 800,
1045 .enter = &intel_idle,
1046 .enter_s2idle = intel_idle_s2idle, },
1047 {
1048 .enter = NULL }
1049};
1050
370406bf
AB		 1051static struct cpuidle_state gnr_cstates[] __initdata = {
1052 {
1053 .name = "C1",
1054 .desc = "MWAIT 0x00",
1055 .flags = MWAIT2flg(0x00),
1056 .exit_latency = 1,
1057 .target_residency = 1,
1058 .enter = &intel_idle,
1059 .enter_s2idle = intel_idle_s2idle, },
1060 {
1061 .name = "C1E",
1062 .desc = "MWAIT 0x01",
1063 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1064 .exit_latency = 4,
1065 .target_residency = 4,
1066 .enter = &intel_idle,
1067 .enter_s2idle = intel_idle_s2idle, },
1068 {
1069 .name = "C6",
1070 .desc = "MWAIT 0x20",
1071 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
4c411cca
AB		 1072 CPUIDLE_FLAG_INIT_XSTATE |
1073 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
370406bf
AB		 1074 .exit_latency = 170,
1075 .target_residency = 650,
1076 .enter = &intel_idle,
1077 .enter_s2idle = intel_idle_s2idle, },
1078 {
1079 .name = "C6P",
1080 .desc = "MWAIT 0x21",
1081 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
4c411cca
AB		 1082 CPUIDLE_FLAG_INIT_XSTATE |
1083 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
370406bf
AB		 1084 .exit_latency = 210,
1085 .target_residency = 1000,
1086 .enter = &intel_idle,
1087 .enter_s2idle = intel_idle_s2idle, },
1088 {
1089 .enter = NULL }
1090};
1091
f557e0d1
AB		 1092static struct cpuidle_state gnrd_cstates[] __initdata = {
1093 {
1094 .name = "C1",
1095 .desc = "MWAIT 0x00",
1096 .flags = MWAIT2flg(0x00),
1097 .exit_latency = 1,
1098 .target_residency = 1,
1099 .enter = &intel_idle,
1100 .enter_s2idle = intel_idle_s2idle, },
1101 {
1102 .name = "C1E",
1103 .desc = "MWAIT 0x01",
1104 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1105 .exit_latency = 4,
1106 .target_residency = 4,
1107 .enter = &intel_idle,
1108 .enter_s2idle = intel_idle_s2idle, },
1109 {
1110 .name = "C6",
1111 .desc = "MWAIT 0x20",
1112 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1113 CPUIDLE_FLAG_INIT_XSTATE |
1114 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1115 .exit_latency = 220,
1116 .target_residency = 650,
1117 .enter = &intel_idle,
1118 .enter_s2idle = intel_idle_s2idle, },
1119 {
1120 .name = "C6P",
1121 .desc = "MWAIT 0x21",
1122 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1123 CPUIDLE_FLAG_INIT_XSTATE |
1124 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1125 .exit_latency = 240,
1126 .target_residency = 750,
1127 .enter = &intel_idle,
1128 .enter_s2idle = intel_idle_s2idle, },
1129 {
1130 .enter = NULL }
1131};
1132
ab1a8522 1133static struct cpuidle_state atom_cstates[] __initdata = {
e022e7eb 1134 {
de09cdd0 1135 .name = "C1E",
26717172 1136 .desc = "MWAIT 0x00",
b82b6cca 1137 .flags = MWAIT2flg(0x00),
32e95180
LB		 1138 .exit_latency = 10,
1139 .target_residency = 20,
5fe2e527 1140 .enter = &intel_idle,
28ba086e 1141 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 1142 {
de09cdd0 1143 .name = "C2",
26717172 1144 .desc = "MWAIT 0x10",
b82b6cca 1145 .flags = MWAIT2flg(0x10),
26717172 1146 .exit_latency = 20,
26717172 1147 .target_residency = 80,
5fe2e527 1148 .enter = &intel_idle,
28ba086e 1149 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 1150 {
de09cdd0 1151 .name = "C4",
26717172 1152 .desc = "MWAIT 0x30",
b82b6cca 1153 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
26717172 1154 .exit_latency = 100,
26717172 1155 .target_residency = 400,
5fe2e527 1156 .enter = &intel_idle,
28ba086e 1157 .enter_s2idle = intel_idle_s2idle, },
e022e7eb 1158 {
de09cdd0 1159 .name = "C6",
7fcca7d9 1160 .desc = "MWAIT 0x52",
b82b6cca 1161 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
7fcca7d9 1162 .exit_latency = 140,
7fcca7d9 1163 .target_residency = 560,
5fe2e527 1164 .enter = &intel_idle,
28ba086e 1165 .enter_s2idle = intel_idle_s2idle, },
e022e7eb
LB		 1166 {
1167 .enter = NULL }
26717172 1168};
ab1a8522 1169static struct cpuidle_state tangier_cstates[] __initdata = {
5e7ec268 1170 {
de09cdd0 1171 .name = "C1",
5e7ec268
AS		 1172 .desc = "MWAIT 0x00",
1173 .flags = MWAIT2flg(0x00),
1174 .exit_latency = 1,
1175 .target_residency = 4,
1176 .enter = &intel_idle,
28ba086e 1177 .enter_s2idle = intel_idle_s2idle, },
5e7ec268 1178 {
de09cdd0 1179 .name = "C4",
5e7ec268
AS		 1180 .desc = "MWAIT 0x30",
1181 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1182 .exit_latency = 100,
1183 .target_residency = 400,
1184 .enter = &intel_idle,
28ba086e 1185 .enter_s2idle = intel_idle_s2idle, },
5e7ec268 1186 {
de09cdd0 1187 .name = "C6",
5e7ec268
AS		 1188 .desc = "MWAIT 0x52",
1189 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1190 .exit_latency = 140,
1191 .target_residency = 560,
1192 .enter = &intel_idle,
28ba086e 1193 .enter_s2idle = intel_idle_s2idle, },
5e7ec268 1194 {
de09cdd0 1195 .name = "C7",
5e7ec268
AS		 1196 .desc = "MWAIT 0x60",
1197 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1198 .exit_latency = 1200,
1199 .target_residency = 4000,
1200 .enter = &intel_idle,
28ba086e 1201 .enter_s2idle = intel_idle_s2idle, },
5e7ec268 1202 {
de09cdd0 1203 .name = "C9",
5e7ec268
AS		 1204 .desc = "MWAIT 0x64",
1205 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1206 .exit_latency = 10000,
1207 .target_residency = 20000,
1208 .enter = &intel_idle,
28ba086e 1209 .enter_s2idle = intel_idle_s2idle, },
5e7ec268
AS		 1210 {
1211 .enter = NULL }
1212};
ab1a8522 1213static struct cpuidle_state avn_cstates[] __initdata = {
fab04b22 1214 {
de09cdd0 1215 .name = "C1",
fab04b22 1216 .desc = "MWAIT 0x00",
b82b6cca 1217 .flags = MWAIT2flg(0x00),
fab04b22
LB		 1218 .exit_latency = 2,
1219 .target_residency = 2,
5fe2e527 1220 .enter = &intel_idle,
28ba086e 1221 .enter_s2idle = intel_idle_s2idle, },
fab04b22 1222 {
de09cdd0 1223 .name = "C6",
fab04b22 1224 .desc = "MWAIT 0x51",
b82b6cca 1225 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
fab04b22
LB		 1226 .exit_latency = 15,
1227 .target_residency = 45,
5fe2e527 1228 .enter = &intel_idle,
28ba086e 1229 .enter_s2idle = intel_idle_s2idle, },
88390996
JL		 1230 {
1231 .enter = NULL }
fab04b22 1232};
ab1a8522 1233static struct cpuidle_state knl_cstates[] __initdata = {
281baf7a 1234 {
de09cdd0 1235 .name = "C1",
281baf7a
DC		 1236 .desc = "MWAIT 0x00",
1237 .flags = MWAIT2flg(0x00),
1238 .exit_latency = 1,
1239 .target_residency = 2,
1240 .enter = &intel_idle,
28ba086e 1241 .enter_s2idle = intel_idle_s2idle },
281baf7a 1242 {
de09cdd0 1243 .name = "C6",
281baf7a
DC		 1244 .desc = "MWAIT 0x10",
1245 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1246 .exit_latency = 120,
1247 .target_residency = 500,
1248 .enter = &intel_idle,
28ba086e 1249 .enter_s2idle = intel_idle_s2idle },
281baf7a
DC		 1250 {
1251 .enter = NULL }
1252};
26717172 1253
ab1a8522 1254static struct cpuidle_state bxt_cstates[] __initdata = {
5dcef694 1255 {
de09cdd0 1256 .name = "C1",
5dcef694
LB		 1257 .desc = "MWAIT 0x00",
1258 .flags = MWAIT2flg(0x00),
1259 .exit_latency = 2,
1260 .target_residency = 2,
1261 .enter = &intel_idle,
28ba086e 1262 .enter_s2idle = intel_idle_s2idle, },
5dcef694 1263 {
de09cdd0 1264 .name = "C1E",
5dcef694 1265 .desc = "MWAIT 0x01",
e6d4f08a 1266 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
5dcef694
LB		 1267 .exit_latency = 10,
1268 .target_residency = 20,
1269 .enter = &intel_idle,
28ba086e 1270 .enter_s2idle = intel_idle_s2idle, },
5dcef694 1271 {
de09cdd0 1272 .name = "C6",
5dcef694
LB		 1273 .desc = "MWAIT 0x20",
1274 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1275 .exit_latency = 133,
1276 .target_residency = 133,
1277 .enter = &intel_idle,
28ba086e 1278 .enter_s2idle = intel_idle_s2idle, },
5dcef694 1279 {
de09cdd0 1280 .name = "C7s",
5dcef694
LB		 1281 .desc = "MWAIT 0x31",
1282 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1283 .exit_latency = 155,
1284 .target_residency = 155,
1285 .enter = &intel_idle,
28ba086e 1286 .enter_s2idle = intel_idle_s2idle, },
5dcef694 1287 {
de09cdd0 1288 .name = "C8",
5dcef694
LB		 1289 .desc = "MWAIT 0x40",
1290 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1291 .exit_latency = 1000,
1292 .target_residency = 1000,
1293 .enter = &intel_idle,
28ba086e 1294 .enter_s2idle = intel_idle_s2idle, },
5dcef694 1295 {
de09cdd0 1296 .name = "C9",
5dcef694
LB		 1297 .desc = "MWAIT 0x50",
1298 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1299 .exit_latency = 2000,
1300 .target_residency = 2000,
1301 .enter = &intel_idle,
28ba086e 1302 .enter_s2idle = intel_idle_s2idle, },
5dcef694 1303 {
de09cdd0 1304 .name = "C10",
5dcef694
LB		 1305 .desc = "MWAIT 0x60",
1306 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1307 .exit_latency = 10000,
1308 .target_residency = 10000,
1309 .enter = &intel_idle,
28ba086e 1310 .enter_s2idle = intel_idle_s2idle, },
5dcef694
LB		 1311 {
1312 .enter = NULL }
1313};
1314
ab1a8522 1315static struct cpuidle_state dnv_cstates[] __initdata = {
0080d65b 1316 {
de09cdd0 1317 .name = "C1",
0080d65b
JP		 1318 .desc = "MWAIT 0x00",
1319 .flags = MWAIT2flg(0x00),
1320 .exit_latency = 2,
1321 .target_residency = 2,
1322 .enter = &intel_idle,
28ba086e 1323 .enter_s2idle = intel_idle_s2idle, },
0080d65b 1324 {
de09cdd0 1325 .name = "C1E",
0080d65b 1326 .desc = "MWAIT 0x01",
e6d4f08a 1327 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
0080d65b
JP		 1328 .exit_latency = 10,
1329 .target_residency = 20,
1330 .enter = &intel_idle,
28ba086e 1331 .enter_s2idle = intel_idle_s2idle, },
0080d65b 1332 {
de09cdd0 1333 .name = "C6",
0080d65b
JP		 1334 .desc = "MWAIT 0x20",
1335 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1336 .exit_latency = 50,
1337 .target_residency = 500,
1338 .enter = &intel_idle,
28ba086e 1339 .enter_s2idle = intel_idle_s2idle, },
0080d65b
JP		 1340 {
1341 .enter = NULL }
1342};
1343
9cf93f05
AB		 1344/*
1345 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1346 * C6, and this is indicated in the CPUID mwait leaf.
1347 */
1348static struct cpuidle_state snr_cstates[] __initdata = {
1349 {
1350 .name = "C1",
1351 .desc = "MWAIT 0x00",
1352 .flags = MWAIT2flg(0x00),
1353 .exit_latency = 2,
1354 .target_residency = 2,
1355 .enter = &intel_idle,
1356 .enter_s2idle = intel_idle_s2idle, },
1357 {
1358 .name = "C1E",
1359 .desc = "MWAIT 0x01",
1360 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1361 .exit_latency = 15,
1362 .target_residency = 25,
1363 .enter = &intel_idle,
1364 .enter_s2idle = intel_idle_s2idle, },
1365 {
1366 .name = "C6",
1367 .desc = "MWAIT 0x20",
1368 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1369 .exit_latency = 130,
1370 .target_residency = 500,
1371 .enter = &intel_idle,
1372 .enter_s2idle = intel_idle_s2idle, },
1373 {
1374 .enter = NULL }
1375};
1376
ac89d11b
AB		 1377static struct cpuidle_state grr_cstates[] __initdata = {
1378 {
1379 .name = "C1",
1380 .desc = "MWAIT 0x00",
1381 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1382 .exit_latency = 1,
1383 .target_residency = 1,
1384 .enter = &intel_idle,
1385 .enter_s2idle = intel_idle_s2idle, },
1386 {
1387 .name = "C1E",
1388 .desc = "MWAIT 0x01",
1389 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1390 .exit_latency = 2,
1391 .target_residency = 10,
1392 .enter = &intel_idle,
1393 .enter_s2idle = intel_idle_s2idle, },
1394 {
1395 .name = "C6S",
1396 .desc = "MWAIT 0x22",
1397 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1398 .exit_latency = 140,
1399 .target_residency = 500,
1400 .enter = &intel_idle,
1401 .enter_s2idle = intel_idle_s2idle, },
1402 {
1403 .enter = NULL }
1404};
1405
92813fd5
AB		 1406static struct cpuidle_state srf_cstates[] __initdata = {
1407 {
1408 .name = "C1",
1409 .desc = "MWAIT 0x00",
1410 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1411 .exit_latency = 1,
1412 .target_residency = 1,
1413 .enter = &intel_idle,
1414 .enter_s2idle = intel_idle_s2idle, },
1415 {
1416 .name = "C1E",
1417 .desc = "MWAIT 0x01",
1418 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1419 .exit_latency = 2,
1420 .target_residency = 10,
1421 .enter = &intel_idle,
1422 .enter_s2idle = intel_idle_s2idle, },
1423 {
1424 .name = "C6S",
1425 .desc = "MWAIT 0x22",
4c411cca
AB		 1426 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED |
1427 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
92813fd5
AB		 1428 .exit_latency = 270,
1429 .target_residency = 700,
1430 .enter = &intel_idle,
1431 .enter_s2idle = intel_idle_s2idle, },
1432 {
1433 .name = "C6SP",
1434 .desc = "MWAIT 0x23",
4c411cca
AB		 1435 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED |
1436 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
92813fd5
AB		 1437 .exit_latency = 310,
1438 .target_residency = 900,
1439 .enter = &intel_idle,
1440 .enter_s2idle = intel_idle_s2idle, },
1441 {
1442 .enter = NULL }
1443};
1444
ab1a8522 1445static const struct idle_cpu idle_cpu_nehalem __initconst = {
b66b8b9a 1446 .state_table = nehalem_cstates,
b66b8b9a 1447 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
32e95180 1448 .disable_promotion_to_c1e = true,
b66b8b9a
AK		 1449};
1450
ab1a8522 1451static const struct idle_cpu idle_cpu_nhx __initconst = {
e6d4f08a
RW		 1452 .state_table = nehalem_cstates,
1453 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1454 .disable_promotion_to_c1e = true,
1455 .use_acpi = true,
1456};
1457
ab1a8522 1458static const struct idle_cpu idle_cpu_atom __initconst = {
b66b8b9a
AK		 1459 .state_table = atom_cstates,
1460};
1461
ab1a8522 1462static const struct idle_cpu idle_cpu_tangier __initconst = {
5e7ec268
AS		 1463 .state_table = tangier_cstates,
1464};
1465
ab1a8522 1466static const struct idle_cpu idle_cpu_lincroft __initconst = {
b66b8b9a
AK		 1467 .state_table = atom_cstates,
1468 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1469};
1470
ab1a8522 1471static const struct idle_cpu idle_cpu_snb __initconst = {
b66b8b9a 1472 .state_table = snb_cstates,
32e95180 1473 .disable_promotion_to_c1e = true,
b66b8b9a
AK		 1474};
1475
ab1a8522 1476static const struct idle_cpu idle_cpu_snx __initconst = {
e6d4f08a
RW		 1477 .state_table = snb_cstates,
1478 .disable_promotion_to_c1e = true,
1479 .use_acpi = true,
1480};
1481
ab1a8522 1482static const struct idle_cpu idle_cpu_byt __initconst = {
718987d6
LB		 1483 .state_table = byt_cstates,
1484 .disable_promotion_to_c1e = true,
1485};
1486
ab1a8522 1487static const struct idle_cpu idle_cpu_cht __initconst = {
cab07a56
LB		 1488 .state_table = cht_cstates,
1489 .disable_promotion_to_c1e = true,
cab07a56
LB		 1490};
1491
ab1a8522 1492static const struct idle_cpu idle_cpu_ivb __initconst = {
6edab08c 1493 .state_table = ivb_cstates,
32e95180 1494 .disable_promotion_to_c1e = true,
6edab08c
LB		 1495};
1496
ab1a8522 1497static const struct idle_cpu idle_cpu_ivt __initconst = {
0138d8f0
LB		 1498 .state_table = ivt_cstates,
1499 .disable_promotion_to_c1e = true,
e6d4f08a 1500 .use_acpi = true,
0138d8f0
LB		 1501};
1502
ab1a8522 1503static const struct idle_cpu idle_cpu_hsw __initconst = {
85a4d2d4 1504 .state_table = hsw_cstates,
32e95180 1505 .disable_promotion_to_c1e = true,
85a4d2d4
LB		 1506};
1507
ab1a8522 1508static const struct idle_cpu idle_cpu_hsx __initconst = {
e6d4f08a
RW		 1509 .state_table = hsw_cstates,
1510 .disable_promotion_to_c1e = true,
1511 .use_acpi = true,
1512};
1513
ab1a8522 1514static const struct idle_cpu idle_cpu_bdw __initconst = {
a138b568
LB		 1515 .state_table = bdw_cstates,
1516 .disable_promotion_to_c1e = true,
1517};
1518
ab1a8522 1519static const struct idle_cpu idle_cpu_bdx __initconst = {
e6d4f08a
RW		 1520 .state_table = bdw_cstates,
1521 .disable_promotion_to_c1e = true,
1522 .use_acpi = true,
1523};
1524
ab1a8522 1525static const struct idle_cpu idle_cpu_skl __initconst = {
493f133f
LB		 1526 .state_table = skl_cstates,
1527 .disable_promotion_to_c1e = true,
1528};
1529
ab1a8522 1530static const struct idle_cpu idle_cpu_skx __initconst = {
f9e71657
LB		 1531 .state_table = skx_cstates,
1532 .disable_promotion_to_c1e = true,
e6d4f08a 1533 .use_acpi = true,
f9e71657 1534};
493f133f 1535
a472ad2b
CY		 1536static const struct idle_cpu idle_cpu_icx __initconst = {
1537 .state_table = icx_cstates,
1538 .disable_promotion_to_c1e = true,
1539 .use_acpi = true,
1540};
1541
d1cf8bbf
ZR		 1542static const struct idle_cpu idle_cpu_adl __initconst = {
1543 .state_table = adl_cstates,
1544};
1545
1546static const struct idle_cpu idle_cpu_adl_l __initconst = {
1547 .state_table = adl_l_cstates,
1548};
1549
eeae55ed
ZR		 1550static const struct idle_cpu idle_cpu_mtl_l __initconst = {
1551 .state_table = mtl_l_cstates,
1552};
1553
882cdb06
PZ		 1554static const struct idle_cpu idle_cpu_gmt __initconst = {
1555 .state_table = gmt_cstates,
65c0c236
ZR		 1556};
1557
9edf3c0f
AB		 1558static const struct idle_cpu idle_cpu_spr __initconst = {
1559 .state_table = spr_cstates,
1560 .disable_promotion_to_c1e = true,
6138f345 1561 .c1_demotion_supported = true,
9edf3c0f
AB		 1562 .use_acpi = true,
1563};
1564
370406bf
AB		 1565static const struct idle_cpu idle_cpu_gnr __initconst = {
1566 .state_table = gnr_cstates,
1567 .disable_promotion_to_c1e = true,
6138f345 1568 .c1_demotion_supported = true,
370406bf
AB		 1569 .use_acpi = true,
1570};
1571
f557e0d1
AB		 1572static const struct idle_cpu idle_cpu_gnrd __initconst = {
1573 .state_table = gnrd_cstates,
1574 .disable_promotion_to_c1e = true,
6138f345 1575 .c1_demotion_supported = true,
f557e0d1
AB		 1576 .use_acpi = true,
1577};
1578
ab1a8522 1579static const struct idle_cpu idle_cpu_avn __initconst = {
fab04b22
LB		 1580 .state_table = avn_cstates,
1581 .disable_promotion_to_c1e = true,
e6d4f08a 1582 .use_acpi = true,
fab04b22
LB		 1583};
1584
ab1a8522 1585static const struct idle_cpu idle_cpu_knl __initconst = {
281baf7a 1586 .state_table = knl_cstates,
e6d4f08a 1587 .use_acpi = true,
281baf7a
DC		 1588};
1589
ab1a8522 1590static const struct idle_cpu idle_cpu_bxt __initconst = {
5dcef694
LB		 1591 .state_table = bxt_cstates,
1592 .disable_promotion_to_c1e = true,
1593};
1594
ab1a8522 1595static const struct idle_cpu idle_cpu_dnv __initconst = {
0080d65b
JP		 1596 .state_table = dnv_cstates,
1597 .disable_promotion_to_c1e = true,
e6d4f08a 1598 .use_acpi = true,
0080d65b
JP		 1599};
1600
5bb33212
KHF		 1601static const struct idle_cpu idle_cpu_tmt __initconst = {
1602 .disable_promotion_to_c1e = true,
1603};
1604
9cf93f05
AB		 1605static const struct idle_cpu idle_cpu_snr __initconst = {
1606 .state_table = snr_cstates,
1607 .disable_promotion_to_c1e = true,
1608 .use_acpi = true,
1609};
1610
ac89d11b
AB		 1611static const struct idle_cpu idle_cpu_grr __initconst = {
1612 .state_table = grr_cstates,
1613 .disable_promotion_to_c1e = true,
6138f345 1614 .c1_demotion_supported = true,
ac89d11b
AB		 1615 .use_acpi = true,
1616};
1617
92813fd5
AB		 1618static const struct idle_cpu idle_cpu_srf __initconst = {
1619 .state_table = srf_cstates,
1620 .disable_promotion_to_c1e = true,
6138f345 1621 .c1_demotion_supported = true,
92813fd5
AB		 1622 .use_acpi = true,
1623};
1624
d5cdc3c4 1625static const struct x86_cpu_id intel_idle_ids[] __initconst = {
17c4fc38
TL		 1626 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx),
1627 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem),
1628 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem),
1629 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem),
1630 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx),
1631 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx),
1632 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom),
1633 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft),
1634 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx),
1635 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb),
1636 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx),
1637 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom),
1638 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt),
1639 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1640 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht),
1641 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb),
1642 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt),
1643 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw),
1644 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx),
1645 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw),
1646 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw),
1647 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn),
1648 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw),
1649 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw),
1650 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx),
1651 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx),
1652 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl),
1653 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl),
1654 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl),
1655 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl),
1656 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx),
1657 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx),
1658 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx),
1659 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl),
1660 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l),
1661 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l),
1662 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt),
1663 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr),
1664 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr),
370406bf 1665 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr),
f557e0d1 1666 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &idle_cpu_gnrd),
17c4fc38
TL		 1667 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl),
1668 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl),
1669 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt),
1670 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt),
1671 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv),
5bb33212
KHF		 1672 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt),
1673 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt),
17c4fc38
TL		 1674 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr),
1675 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr),
1676 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf),
eeed4bfb 1677 X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, &idle_cpu_srf),
b66b8b9a
AK		 1678 {}
1679};
b66b8b9a 1680
18734958 1681static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
4a9f45a0 1682 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),
18734958
RW		 1683 {}
1684};
1685
095928ae 1686static bool __init intel_idle_max_cstate_reached(int cstate)
18734958
RW		 1687{
1688 if (cstate + 1 > max_cstate) {
1689 pr_info("max_cstate %d reached\n", max_cstate);
1690 return true;
1691 }
1692 return false;
1693}
1694
4d916140
PZ		 1695static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1696{
1697 unsigned long eax = flg2MWAIT(state->flags);
1698
1699 if (boot_cpu_has(X86_FEATURE_ARAT))
1700 return false;
1701
1702 /*
1703 * Switch over to one-shot tick broadcast if the target C-state
1704 * is deeper than C1.
1705 */
1706 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1707}
1708
18734958
RW		 1709#ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1710#include <acpi/processor.h>
1711
4ec32d9e
RW		 1712static bool no_acpi __read_mostly;
1713module_param(no_acpi, bool, 0444);
1714MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1715
3a5be9b8
RW		 1716static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1717module_param_named(use_acpi, force_use_acpi, bool, 0444);
1718MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1719
5e7e39ae
DA		 1720static bool no_native __read_mostly; /* No effect if no_acpi is set. */
1721module_param_named(no_native, no_native, bool, 0444);
1722MODULE_PARM_DESC(no_native, "Ignore cpu specific (native) idle states in lieu of ACPI idle states");
1723
095928ae 1724static struct acpi_processor_power acpi_state_table __initdata;
18734958
RW		 1725
1726/**
1727 * intel_idle_cst_usable - Check if the _CST information can be used.
1728 *
1729 * Check if all of the C-states listed by _CST in the max_cstate range are
1730 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1731 */
095928ae 1732static bool __init intel_idle_cst_usable(void)
18734958
RW		 1733{
1734 int cstate, limit;
1735
1736 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1737 acpi_state_table.count);
1738
1739 for (cstate = 1; cstate < limit; cstate++) {
1740 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1741
1742 if (cx->entry_method != ACPI_CSTATE_FFH)
1743 return false;
1744 }
1745
1746 return true;
1747}
1748
095928ae 1749static bool __init intel_idle_acpi_cst_extract(void)
18734958
RW		 1750{
1751 unsigned int cpu;
1752
4ec32d9e
RW		 1753 if (no_acpi) {
1754 pr_debug("Not allowed to use ACPI _CST\n");
1755 return false;
1756 }
1757
18734958
RW		 1758 for_each_possible_cpu(cpu) {
1759 struct acpi_processor *pr = per_cpu(processors, cpu);
1760
1761 if (!pr)
1762 continue;
1763
1764 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1765 continue;
1766
1767 acpi_state_table.count++;
1768
1769 if (!intel_idle_cst_usable())
1770 continue;
1771
75af76d0
MG		 1772 if (!acpi_processor_claim_cst_control())
1773 break;
18734958
RW		 1774
1775 return true;
1776 }
1777
75af76d0 1778 acpi_state_table.count = 0;
18734958
RW		 1779 pr_debug("ACPI _CST not found or not usable\n");
1780 return false;
1781}
1782
095928ae 1783static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
18734958
RW		 1784{
1785 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1786
1787 /*
1788 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1789 * the interesting states are ACPI_CSTATE_FFH.
1790 */
1791 for (cstate = 1; cstate < limit; cstate++) {
1792 struct acpi_processor_cx *cx;
1793 struct cpuidle_state *state;
1794
4e0ba557 1795 if (intel_idle_max_cstate_reached(cstate - 1))
18734958
RW		 1796 break;
1797
1798 cx = &acpi_state_table.states[cstate];
1799
1800 state = &drv->states[drv->state_count++];
1801
1802 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
0dbc0f49 1803 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
18734958
RW		 1804 state->exit_latency = cx->latency;
1805 /*
1806 * For C1-type C-states use the same number for both the exit
1807 * latency and target residency, because that is the case for
1808 * C1 in the majority of the static C-states tables above.
1809 * For the other types of C-states, however, set the target
1810 * residency to 3 times the exit latency which should lead to
1811 * a reasonable balance between energy-efficiency and
1812 * performance in the majority of interesting cases.
1813 */
1814 state->target_residency = cx->latency;
1815 if (cx->type > ACPI_STATE_C1)
1816 state->target_residency *= 3;
1817
1818 state->flags = MWAIT2flg(cx->address);
1819 if (cx->type > ACPI_STATE_C2)
1820 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1821
4dcb78ee
RW		 1822 if (disabled_states_mask & BIT(cstate))
1823 state->flags |= CPUIDLE_FLAG_OFF;
1824
6e1d2bc6
PZ		 1825 if (intel_idle_state_needs_timer_stop(state))
1826 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1827
c157d351
TG		 1828 if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1829 mark_tsc_unstable("TSC halts in idle");
1830
18734958 1831 state->enter = intel_idle;
fc4ca953 1832 state->enter_dead = intel_idle_enter_dead;
18734958
RW		 1833 state->enter_s2idle = intel_idle_s2idle;
1834 }
1835}
bff8e60a 1836
4c411cca 1837static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
bff8e60a
RW		 1838{
1839 int cstate, limit;
1840
1841 /*
1842 * If there are no _CST C-states, do not disable any C-states by
1843 * default.
1844 */
1845 if (!acpi_state_table.count)
1846 return false;
1847
1848 limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1849 /*
1850 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1851 * the interesting states are ACPI_CSTATE_FFH.
1852 */
1853 for (cstate = 1; cstate < limit; cstate++) {
4c411cca
AB		 1854 u32 acpi_hint = acpi_state_table.states[cstate].address;
1855 u32 table_hint = mwait_hint;
1856
1857 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) {
1858 acpi_hint &= ~MWAIT_SUBSTATE_MASK;
1859 table_hint &= ~MWAIT_SUBSTATE_MASK;
1860 }
1861
1862 if (acpi_hint == table_hint)
bff8e60a
RW		 1863 return false;
1864 }
1865 return true;
1866}
5e7e39ae
DA		 1867
1868static inline bool ignore_native(void)
1869{
1870 return no_native && !no_acpi;
1871}
18734958 1872#else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
3a5be9b8
RW		 1873#define force_use_acpi (false)
1874
18734958
RW		 1875static inline bool intel_idle_acpi_cst_extract(void) { return false; }
1876static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
4c411cca
AB		 1877static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
1878{
1879 return false;
1880}
5e7e39ae 1881static inline bool ignore_native(void) { return false; }
18734958
RW		 1882#endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1883
6eacb15f
RW		 1884/**
1885 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
0138d8f0 1886 *
6eacb15f
RW		 1887 * Tune IVT multi-socket targets.
1888 * Assumption: num_sockets == (max_package_num + 1).
0138d8f0 1889 */
095928ae 1890static void __init ivt_idle_state_table_update(void)
0138d8f0
LB		 1891{
1892 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
d70e28f5
LB		 1893 int cpu, package_num, num_sockets = 1;
1894
1895 for_each_online_cpu(cpu) {
1896 package_num = topology_physical_package_id(cpu);
1897 if (package_num + 1 > num_sockets) {
1898 num_sockets = package_num + 1;
1899
1900 if (num_sockets > 4) {
1901 cpuidle_state_table = ivt_cstates_8s;
1902 return;
0138d8f0
LB		 1903 }
1904 }
d70e28f5
LB		 1905 }
1906
1907 if (num_sockets > 2)
1908 cpuidle_state_table = ivt_cstates_4s;
1909
1910 /* else, 1 and 2 socket systems use default ivt_cstates */
1911}
5dcef694 1912
86e9466a
RW		 1913/**
1914 * irtl_2_usec - IRTL to microseconds conversion.
1915 * @irtl: IRTL MSR value.
1916 *
1917 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
5dcef694 1918 */
095928ae 1919static unsigned long long __init irtl_2_usec(unsigned long long irtl)
5dcef694 1920{
86e9466a
RW		 1921 static const unsigned int irtl_ns_units[] __initconst = {
1922 1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1923 };
5dcef694
LB		 1924 unsigned long long ns;
1925
3451ab3e
JB		 1926 if (!irtl)
1927 return 0;
1928
bef45096 1929 ns = irtl_ns_units[(irtl >> 10) & 0x7];
5dcef694 1930
86e9466a 1931 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
5dcef694 1932}
86e9466a 1933
6eacb15f
RW		 1934/**
1935 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
5dcef694 1936 *
6eacb15f
RW		 1937 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
1938 * definitive maximum latency and use the same value for target_residency.
5dcef694 1939 */
095928ae 1940static void __init bxt_idle_state_table_update(void)
5dcef694
LB		 1941{
1942 unsigned long long msr;
3451ab3e 1943 unsigned int usec;
5dcef694 1944
c435e608 1945 rdmsrq(MSR_PKGC6_IRTL, msr);
3451ab3e
JB		 1946 usec = irtl_2_usec(msr);
1947 if (usec) {
5dcef694
LB		 1948 bxt_cstates[2].exit_latency = usec;
1949 bxt_cstates[2].target_residency = usec;
1950 }
1951
c435e608 1952 rdmsrq(MSR_PKGC7_IRTL, msr);
3451ab3e
JB		 1953 usec = irtl_2_usec(msr);
1954 if (usec) {
5dcef694
LB		 1955 bxt_cstates[3].exit_latency = usec;
1956 bxt_cstates[3].target_residency = usec;
1957 }
1958
c435e608 1959 rdmsrq(MSR_PKGC8_IRTL, msr);
3451ab3e
JB		 1960 usec = irtl_2_usec(msr);
1961 if (usec) {
5dcef694
LB		 1962 bxt_cstates[4].exit_latency = usec;
1963 bxt_cstates[4].target_residency = usec;
1964 }
1965
c435e608 1966 rdmsrq(MSR_PKGC9_IRTL, msr);
3451ab3e
JB		 1967 usec = irtl_2_usec(msr);
1968 if (usec) {
5dcef694
LB		 1969 bxt_cstates[5].exit_latency = usec;
1970 bxt_cstates[5].target_residency = usec;
1971 }
1972
c435e608 1973 rdmsrq(MSR_PKGC10_IRTL, msr);
3451ab3e
JB		 1974 usec = irtl_2_usec(msr);
1975 if (usec) {
5dcef694
LB		 1976 bxt_cstates[6].exit_latency = usec;
1977 bxt_cstates[6].target_residency = usec;
1978 }
1979
1980}
6eacb15f
RW		 1981
1982/**
1983 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
d70e28f5 1984 *
6eacb15f 1985 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
d70e28f5 1986 */
095928ae 1987static void __init sklh_idle_state_table_update(void)
d70e28f5
LB		 1988{
1989 unsigned long long msr;
1990 unsigned int eax, ebx, ecx, edx;
1991
1992
1993 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1994 if (max_cstate <= 7)
1995 return;
1996
1997 /* if PC10 not present in CPUID.MWAIT.EDX */
1998 if ((mwait_substates & (0xF << 28)) == 0)
1999 return;
2000
c435e608 2001 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr);
d70e28f5
LB		 2002
2003 /* PC10 is not enabled in PKG C-state limit */
2004 if ((msr & 0xF) != 8)
2005 return;
2006
2007 ecx = 0;
2008 cpuid(7, &eax, &ebx, &ecx, &edx);
2009
2010 /* if SGX is present */
2011 if (ebx & (1 << 2)) {
0138d8f0 2012
c435e608 2013 rdmsrq(MSR_IA32_FEAT_CTL, msr);
d70e28f5
LB		 2014
2015 /* if SGX is enabled */
2016 if (msr & (1 << 18))
2017 return;
2018 }
2019
ba1e78a1
RW		 2020 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */
2021 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */
d70e28f5 2022}
d70e28f5 2023
64233338
CY		 2024/**
2025 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
2026 * idle states table.
2027 */
2028static void __init skx_idle_state_table_update(void)
2029{
2030 unsigned long long msr;
2031
c435e608 2032 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr);
64233338
CY		 2033
2034 /*
2035 * 000b: C0/C1 (no package C-state support)
2036 * 001b: C2
2037 * 010b: C6 (non-retention)
2038 * 011b: C6 (retention)
2039 * 111b: No Package C state limits.
2040 */
2041 if ((msr & 0x7) < 2) {
2042 /*
2043 * Uses the CC6 + PC0 latency and 3 times of
2044 * latency for target_residency if the PC6
2045 * is disabled in BIOS. This is consistent
2046 * with how intel_idle driver uses _CST
2047 * to set the target_residency.
2048 */
2049 skx_cstates[2].exit_latency = 92;
2050 skx_cstates[2].target_residency = 276;
2051 }
2052}
2053
d1cf8bbf
ZR		 2054/**
2055 * adl_idle_state_table_update - Adjust AlderLake idle states table.
2056 */
2057static void __init adl_idle_state_table_update(void)
2058{
2059 /* Check if user prefers C1 over C1E. */
2060 if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {
2061 cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;
2062 cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;
2063
2064 /* Disable C1E by clearing the "C1E promotion" bit. */
2065 c1e_promotion = C1E_PROMOTION_DISABLE;
2066 return;
2067 }
2068
2069 /* Make sure C1E is enabled by default */
2070 c1e_promotion = C1E_PROMOTION_ENABLE;
2071}
2072
da0e58c0
AB		 2073/**
2074 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
2075 */
2076static void __init spr_idle_state_table_update(void)
2077{
3a9cf77b
AB		 2078 unsigned long long msr;
2079
3a9cf77b
AB		 2080 /*
2081 * By default, the C6 state assumes the worst-case scenario of package
2082 * C6. However, if PC6 is disabled, we update the numbers to match
2083 * core C6.
2084 */
c435e608 2085 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr);
3a9cf77b
AB		 2086
2087 /* Limit value 2 and above allow for PC6. */
2088 if ((msr & 0x7) < 2) {
2089 spr_cstates[2].exit_latency = 190;
2090 spr_cstates[2].target_residency = 600;
2091 }
da0e58c0
AB		 2092}
2093
c93d13b6
AB		 2094/**
2095 * byt_cht_auto_demotion_disable - Disable Bay/Cherry Trail auto-demotion.
2096 */
2097static void __init byt_cht_auto_demotion_disable(void)
2098{
78255eb2
IM		 2099 wrmsrq(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
2100 wrmsrq(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
c93d13b6
AB		 2101}
2102
1aefbd7a
RW		 2103static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
2104{
6b8e288f
HR		 2105 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) &
2106 MWAIT_CSTATE_MASK;
1aefbd7a
RW		 2107 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
2108 MWAIT_SUBSTATE_MASK;
2109
2110 /* Ignore the C-state if there are NO sub-states in CPUID for it. */
2111 if (num_substates == 0)
2112 return false;
2113
2114 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
2115 mark_tsc_unstable("TSC halts in idle states deeper than C2");
2116
2117 return true;
2118}
2119
4622ba92
AV		 2120static void state_update_enter_method(struct cpuidle_state *state, int cstate)
2121{
2122 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
2123 /*
2124 * Combining with XSTATE with IBRS or IRQ_ENABLE flags
2125 * is not currently supported but this driver.
2126 */
2127 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
2128 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2129 state->enter = intel_idle_xstate;
7826c069
AV		 2130 return;
2131 }
2132
2133 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
aa1567a7 2134 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {
4622ba92
AV		 2135 /*
2136 * IBRS mitigation requires that C-states are entered
2137 * with interrupts disabled.
2138 */
aa1567a7
WL		 2139 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))
2140 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;
4622ba92
AV		 2141 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2142 state->enter = intel_idle_ibrs;
7826c069
AV		 2143 return;
2144 }
2145
2146 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
4622ba92 2147 state->enter = intel_idle_irq;
7826c069
AV		 2148 return;
2149 }
2150
2151 if (force_irq_on) {
4622ba92
AV		 2152 pr_info("forced intel_idle_irq for state %d\n", cstate);
2153 state->enter = intel_idle_irq;
2154 }
2155}
2156
095928ae 2157static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
d70e28f5 2158{
3d3a1ae9 2159 int cstate;
d70e28f5 2160
17c4fc38
TL		 2161 switch (boot_cpu_data.x86_vfm) {
2162 case INTEL_IVYBRIDGE_X:
d70e28f5
LB		 2163 ivt_idle_state_table_update();
2164 break;
17c4fc38
TL		 2165 case INTEL_ATOM_GOLDMONT:
2166 case INTEL_ATOM_GOLDMONT_PLUS:
5dcef694
LB		 2167 bxt_idle_state_table_update();
2168 break;
17c4fc38 2169 case INTEL_SKYLAKE:
d70e28f5
LB		 2170 sklh_idle_state_table_update();
2171 break;
17c4fc38 2172 case INTEL_SKYLAKE_X:
64233338
CY		 2173 skx_idle_state_table_update();
2174 break;
17c4fc38
TL		 2175 case INTEL_SAPPHIRERAPIDS_X:
2176 case INTEL_EMERALDRAPIDS_X:
da0e58c0
AB		 2177 spr_idle_state_table_update();
2178 break;
17c4fc38
TL		 2179 case INTEL_ALDERLAKE:
2180 case INTEL_ALDERLAKE_L:
2181 case INTEL_ATOM_GRACEMONT:
d1cf8bbf
ZR		 2182 adl_idle_state_table_update();
2183 break;
c93d13b6
AB		 2184 case INTEL_ATOM_SILVERMONT:
2185 case INTEL_ATOM_AIRMONT:
2186 byt_cht_auto_demotion_disable();
2187 break;
0138d8f0 2188 }
46bcfad7 2189
e022e7eb 2190 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
1abffbd8 2191 struct cpuidle_state *state;
9f3d6daf 2192 unsigned int mwait_hint;
46bcfad7 2193
18734958 2194 if (intel_idle_max_cstate_reached(cstate))
e022e7eb
LB		 2195 break;
2196
18734958
RW		 2197 if (!cpuidle_state_table[cstate].enter &&
2198 !cpuidle_state_table[cstate].enter_s2idle)
46bcfad7 2199 break;
46bcfad7 2200
fc4ca953
PW		 2201 if (!cpuidle_state_table[cstate].enter_dead)
2202 cpuidle_state_table[cstate].enter_dead = intel_idle_enter_dead;
2203
9f3d6daf 2204 /* If marked as unusable, skip this state. */
ba1e78a1 2205 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
654d08a4
JP		 2206 pr_debug("state %s is disabled\n",
2207 cpuidle_state_table[cstate].name);
d70e28f5
LB		 2208 continue;
2209 }
2210
9f3d6daf 2211 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
5534f446 2212 if (!intel_idle_verify_cstate(mwait_hint))
9f3d6daf 2213 continue;
d70e28f5 2214
9f3d6daf 2215 /* Structure copy. */
bff8e60a 2216 drv->states[drv->state_count] = cpuidle_state_table[cstate];
1abffbd8 2217 state = &drv->states[drv->state_count];
bff8e60a 2218
4622ba92
AV		 2219 state_update_enter_method(state, cstate);
2220
bf5835bc 2221
4dcb78ee
RW		 2222 if ((disabled_states_mask & BIT(drv->state_count)) ||
2223 ((icpu->use_acpi || force_use_acpi) &&
4c411cca 2224 intel_idle_off_by_default(state->flags, mwait_hint) &&
1abffbd8
AB		 2225 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2226 state->flags |= CPUIDLE_FLAG_OFF;
bff8e60a 2227
1abffbd8
AB		 2228 if (intel_idle_state_needs_timer_stop(state))
2229 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
6e1d2bc6 2230
bff8e60a 2231 drv->state_count++;
46bcfad7 2232 }
46bcfad7
DD		 2233}
2234
6eacb15f
RW		 2235/**
2236 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2237 * @drv: cpuidle driver structure to initialize.
18734958 2238 */
3d3a1ae9 2239static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
18734958 2240{
18734958 2241 cpuidle_poll_state_init(drv);
4dcb78ee
RW		 2242
2243 if (disabled_states_mask & BIT(0))
2244 drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2245
18734958
RW		 2246 drv->state_count = 1;
2247
5bb33212 2248 if (icpu && icpu->state_table)
18734958
RW		 2249 intel_idle_init_cstates_icpu(drv);
2250 else
2251 intel_idle_init_cstates_acpi(drv);
2252}
46bcfad7 2253
1aefbd7a
RW		 2254static void auto_demotion_disable(void)
2255{
2256 unsigned long long msr_bits;
2257
c435e608 2258 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
7f843dd7 2259 msr_bits &= ~auto_demotion_disable_flags;
78255eb2 2260 wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
1aefbd7a
RW		 2261}
2262
da0e58c0
AB		 2263static void c1e_promotion_enable(void)
2264{
2265 unsigned long long msr_bits;
2266
c435e608 2267 rdmsrq(MSR_IA32_POWER_CTL, msr_bits);
da0e58c0 2268 msr_bits |= 0x2;
78255eb2 2269 wrmsrq(MSR_IA32_POWER_CTL, msr_bits);
da0e58c0
AB		 2270}
2271
1aefbd7a
RW		 2272static void c1e_promotion_disable(void)
2273{
2274 unsigned long long msr_bits;
2275
c435e608 2276 rdmsrq(MSR_IA32_POWER_CTL, msr_bits);
1aefbd7a 2277 msr_bits &= ~0x2;
78255eb2 2278 wrmsrq(MSR_IA32_POWER_CTL, msr_bits);
1aefbd7a
RW		 2279}
2280
6eacb15f
RW		 2281/**
2282 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2283 * @cpu: CPU to initialize.
2284 *
2285 * Register a cpuidle device object for @cpu and update its MSRs in accordance
2286 * with the processor model flags.
26717172 2287 */
fb1013a0 2288static int intel_idle_cpu_init(unsigned int cpu)
26717172 2289{
26717172
LB		 2290 struct cpuidle_device *dev;
2291
65b7f839 2292 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
65b7f839 2293 dev->cpu = cpu;
26717172 2294
65b7f839 2295 if (cpuidle_register_device(dev)) {
654d08a4 2296 pr_debug("cpuidle_register_device %d failed!\n", cpu);
65b7f839 2297 return -EIO;
26717172
LB		 2298 }
2299
7f843dd7 2300 if (auto_demotion_disable_flags)
fb1013a0 2301 auto_demotion_disable();
65b7f839 2302
39c184a6
AB		 2303 if (c1e_promotion == C1E_PROMOTION_ENABLE)
2304 c1e_promotion_enable();
2305 else if (c1e_promotion == C1E_PROMOTION_DISABLE)
fb1013a0
SAS		 2306 c1e_promotion_disable();
2307
2308 return 0;
2309}
2310
2311static int intel_idle_cpu_online(unsigned int cpu)
2312{
2313 struct cpuidle_device *dev;
2314
dab20177 2315 if (!boot_cpu_has(X86_FEATURE_ARAT))
cbd2c4c2 2316 tick_broadcast_enable();
fb1013a0
SAS		 2317
2318 /*
2319 * Some systems can hotplug a cpu at runtime after
2320 * the kernel has booted, we have to initialize the
2321 * driver in this case
2322 */
2323 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2324 if (!dev->registered)
2325 return intel_idle_cpu_init(cpu);
dbf87ab8 2326
26717172
LB		 2327 return 0;
2328}
26717172 2329
0755a9bd
RW		 2330/**
2331 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2332 */
2333static void __init intel_idle_cpuidle_devices_uninit(void)
2334{
2335 int i;
2336
2337 for_each_online_cpu(i)
2338 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2339}
2340
6138f345
AB		 2341static void intel_c1_demotion_toggle(void *enable)
2342{
2343 unsigned long long msr_val;
2344
2345 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_val);
2346 /*
2347 * Enable/disable C1 undemotion along with C1 demotion, as this is the
2348 * most sensible configuration in general.
2349 */
2350 if (enable)
2351 msr_val |= NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE;
2352 else
2353 msr_val &= ~(NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE);
2354 wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_val);
2355}
2356
2357static ssize_t intel_c1_demotion_store(struct device *dev,
2358 struct device_attribute *attr,
2359 const char *buf, size_t count)
2360{
2361 bool enable;
2362 int err;
2363
2364 err = kstrtobool(buf, &enable);
2365 if (err)
2366 return err;
2367
2368 mutex_lock(&c1_demotion_mutex);
2369 /* Enable/disable C1 demotion on all CPUs */
2370 on_each_cpu(intel_c1_demotion_toggle, (void *)enable, 1);
2371 mutex_unlock(&c1_demotion_mutex);
2372
2373 return count;
2374}
2375
2376static ssize_t intel_c1_demotion_show(struct device *dev,
2377 struct device_attribute *attr, char *buf)
2378{
2379 unsigned long long msr_val;
2380
2381 /*
2382 * Read the MSR value for a CPU and assume it is the same for all CPUs. Any other
2383 * configuration would be a BIOS bug.
2384 */
2385 rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_val);
2386 return sysfs_emit(buf, "%d\n", !!(msr_val & NHM_C1_AUTO_DEMOTE));
2387}
2388static DEVICE_ATTR_RW(intel_c1_demotion);
2389
2390static int __init intel_idle_sysfs_init(void)
2391{
2392 int err;
2393
2394 if (!c1_demotion_supported)
2395 return 0;
2396
2397 sysfs_root = bus_get_dev_root(&cpu_subsys);
2398 if (!sysfs_root)
2399 return 0;
2400
2401 err = sysfs_add_file_to_group(&sysfs_root->kobj,
2402 &dev_attr_intel_c1_demotion.attr,
2403 "cpuidle");
2404 if (err) {
2405 put_device(sysfs_root);
2406 return err;
2407 }
2408
2409 return 0;
2410}
2411
2412static void __init intel_idle_sysfs_uninit(void)
2413{
2414 if (!sysfs_root)
2415 return;
2416
2417 sysfs_remove_file_from_group(&sysfs_root->kobj,
2418 &dev_attr_intel_c1_demotion.attr,
2419 "cpuidle");
2420 put_device(sysfs_root);
2421}
2422
26717172
LB		 2423static int __init intel_idle_init(void)
2424{
a6c86e33
RW		 2425 const struct x86_cpu_id *id;
2426 unsigned int eax, ebx, ecx;
fb1013a0 2427 int retval;
26717172 2428
d1896049
TR		 2429 /* Do not load intel_idle at all for now if idle= is passed */
2430 if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2431 return -ENODEV;
2432
a6c86e33
RW		 2433 if (max_cstate == 0) {
2434 pr_debug("disabled\n");
2435 return -EPERM;
2436 }
2437
2438 id = x86_match_cpu(intel_idle_ids);
2439 if (id) {
2440 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2441 pr_debug("Please enable MWAIT in BIOS SETUP\n");
2442 return -ENODEV;
2443 }
2444 } else {
2445 id = x86_match_cpu(intel_mwait_ids);
2446 if (!id)
2447 return -ENODEV;
2448 }
2449
e5d3a578 2450 cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates);
a6c86e33
RW		 2451
2452 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2453 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2454 !mwait_substates)
2455 return -ENODEV;
2456
2457 pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2458
2459 icpu = (const struct idle_cpu *)id->driver_data;
5e7e39ae
DA		 2460 if (icpu && ignore_native()) {
2461 pr_debug("ignoring native CPU idle states\n");
2462 icpu = NULL;
2463 }
a6c86e33 2464 if (icpu) {
5bb33212
KHF		 2465 if (icpu->state_table)
2466 cpuidle_state_table = icpu->state_table;
2467 else if (!intel_idle_acpi_cst_extract())
2468 return -ENODEV;
2469
7f843dd7 2470 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
39c184a6
AB		 2471 if (icpu->disable_promotion_to_c1e)
2472 c1e_promotion = C1E_PROMOTION_DISABLE;
6138f345
AB		 2473 if (icpu->c1_demotion_supported)
2474 c1_demotion_supported = true;
3a5be9b8 2475 if (icpu->use_acpi || force_use_acpi)
a6c86e33
RW		 2476 intel_idle_acpi_cst_extract();
2477 } else if (!intel_idle_acpi_cst_extract()) {
2478 return -ENODEV;
2479 }
2480
2481 pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2482 boot_cpu_data.x86_model);
26717172 2483
e9df69cc 2484 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
533da74a 2485 if (!intel_idle_cpuidle_devices)
e9df69cc
RC		 2486 return -ENOMEM;
2487
6138f345
AB		 2488 retval = intel_idle_sysfs_init();
2489 if (retval)
2490 pr_warn("failed to initialized sysfs");
2491
3d3a1ae9
RW		 2492 intel_idle_cpuidle_driver_init(&intel_idle_driver);
2493
26717172
LB		 2494 retval = cpuidle_register_driver(&intel_idle_driver);
2495 if (retval) {
3735d524 2496 struct cpuidle_driver *drv = cpuidle_get_driver();
654d08a4
JP		 2497 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2498 drv ? drv->name : "none");
fb1013a0 2499 goto init_driver_fail;
26717172
LB		 2500 }
2501
fb1013a0
SAS		 2502 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2503 intel_idle_cpu_online, NULL);
2504 if (retval < 0)
2505 goto hp_setup_fail;
26717172 2506
40ab82e0 2507 pr_debug("Local APIC timer is reliable in %s\n",
dab20177 2508 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2259a819 2509
a430c11f
RW		 2510 arch_cpu_rescan_dead_smt_siblings();
2511
26717172 2512 return 0;
fb1013a0
SAS		 2513
2514hp_setup_fail:
2515 intel_idle_cpuidle_devices_uninit();
2516 cpuidle_unregister_driver(&intel_idle_driver);
2517init_driver_fail:
6138f345 2518 intel_idle_sysfs_uninit();
fb1013a0
SAS		 2519 free_percpu(intel_idle_cpuidle_devices);
2520 return retval;
2521
26717172 2522}
c0f69138 2523subsys_initcall_sync(intel_idle_init);
26717172 2524
02c4fae9
PG		 2525/*
2526 * We are not really modular, but we used to support that. Meaning we also
2527 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2528 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2529 * is the easiest way (currently) to continue doing that.
2530 */
26717172 2531module_param(max_cstate, int, 0444);
4dcb78ee
RW		 2532/*
2533 * The positions of the bits that are set in this number are the indices of the
2534 * idle states to be disabled by default (as reflected by the names of the
2535 * corresponding idle state directories in sysfs, "state0", "state1" ...
2536 * "state<i>" ..., where <i> is the index of the given state).
2537 */
2538module_param_named(states_off, disabled_states_mask, uint, 0444);
2539MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
da0e58c0
AB		 2540/*
2541 * Some platforms come with mutually exclusive C-states, so that if one is
2542 * enabled, the other C-states must not be used. Example: C1 and C1E on
2543 * Sapphire Rapids platform. This parameter allows for selecting the
2544 * preferred C-states among the groups of mutually exclusive C-states - the
2545 * selected C-states will be registered, the other C-states from the mutually
2546 * exclusive group won't be registered. If the platform has no mutually
2547 * exclusive C-states, this parameter has no effect.
2548 */
2549module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);
2550MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
4152379a
AB		 2551/*
2552 * Debugging option that forces the driver to enter all C-states with
2553 * interrupts enabled. Does not apply to C-states with
2554 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2555 */
2556module_param(force_irq_on, bool, 0444);
aa1567a7
WL		 2557/*
2558 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and
2559 * CPUIDLE_FLAG_IRQ_ENABLE isn't set.
2560 */
2561module_param(ibrs_off, bool, 0444);
2562MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");
Linux 6.x block layer and io_uring tree(s)