Commit | Line | Data |
---|---|---|
26717172 LB |
1 | /* |
2 | * intel_idle.c - native hardware idle loop for modern Intel processors | |
3 | * | |
fab04b22 | 4 | * Copyright (c) 2013, Intel Corporation. |
26717172 LB |
5 | * Len Brown <len.brown@intel.com> |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify it | |
8 | * under the terms and conditions of the GNU General Public License, | |
9 | * version 2, as published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope it will be useful, but WITHOUT | |
12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
14 | * more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License along with | |
17 | * this program; if not, write to the Free Software Foundation, Inc., | |
18 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
19 | */ | |
20 | ||
21 | /* | |
22 | * intel_idle is a cpuidle driver that loads on specific Intel processors | |
23 | * in lieu of the legacy ACPI processor_idle driver. The intent is to | |
24 | * make Linux more efficient on these processors, as intel_idle knows | |
25 | * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. | |
26 | */ | |
27 | ||
28 | /* | |
29 | * Design Assumptions | |
30 | * | |
31 | * All CPUs have same idle states as boot CPU | |
32 | * | |
33 | * Chipset BM_STS (bus master status) bit is a NOP | |
34 | * for preventing entry into deep C-stats | |
35 | */ | |
36 | ||
37 | /* | |
38 | * Known limitations | |
39 | * | |
40 | * The driver currently initializes for_each_online_cpu() upon modprobe. | |
41 | * It it unaware of subsequent processors hot-added to the system. | |
42 | * This means that if you boot with maxcpus=n and later online | |
43 | * processors above n, those processors will use C1 only. | |
44 | * | |
45 | * ACPI has a .suspend hack to turn off deep c-statees during suspend | |
46 | * to avoid complications with the lapic timer workaround. | |
47 | * Have not seen issues with suspend, but may need same workaround here. | |
48 | * | |
26717172 LB |
49 | */ |
50 | ||
51 | /* un-comment DEBUG to enable pr_debug() statements */ | |
52 | #define DEBUG | |
53 | ||
654d08a4 JP |
54 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
55 | ||
26717172 LB |
56 | #include <linux/kernel.h> |
57 | #include <linux/cpuidle.h> | |
76962caa | 58 | #include <linux/tick.h> |
26717172 LB |
59 | #include <trace/events/power.h> |
60 | #include <linux/sched.h> | |
2a2d31c8 SL |
61 | #include <linux/notifier.h> |
62 | #include <linux/cpu.h> | |
02c4fae9 | 63 | #include <linux/moduleparam.h> |
b66b8b9a | 64 | #include <asm/cpu_device_id.h> |
db73c5a8 | 65 | #include <asm/intel-family.h> |
bc83cccc | 66 | #include <asm/mwait.h> |
14796fca | 67 | #include <asm/msr.h> |
26717172 | 68 | |
d70e28f5 | 69 | #define INTEL_IDLE_VERSION "0.4.1" |
26717172 | 70 | |
26717172 LB |
71 | static struct cpuidle_driver intel_idle_driver = { |
72 | .name = "intel_idle", | |
73 | .owner = THIS_MODULE, | |
74 | }; | |
75 | /* intel_idle.max_cstate=0 disables driver */ | |
137ecc77 | 76 | static int max_cstate = CPUIDLE_STATE_MAX - 1; |
26717172 | 77 | |
c4236282 | 78 | static unsigned int mwait_substates; |
26717172 | 79 | |
2a2d31c8 | 80 | #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF |
26717172 | 81 | /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ |
d13780d4 | 82 | static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ |
26717172 | 83 | |
b66b8b9a AK |
84 | struct idle_cpu { |
85 | struct cpuidle_state *state_table; | |
86 | ||
87 | /* | |
88 | * Hardware C-state auto-demotion may not always be optimal. | |
89 | * Indicate which enable bits to clear here. | |
90 | */ | |
91 | unsigned long auto_demotion_disable_flags; | |
8c058d53 | 92 | bool byt_auto_demotion_disable_flag; |
32e95180 | 93 | bool disable_promotion_to_c1e; |
b66b8b9a AK |
94 | }; |
95 | ||
96 | static const struct idle_cpu *icpu; | |
3265eba0 | 97 | static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; |
46bcfad7 DD |
98 | static int intel_idle(struct cpuidle_device *dev, |
99 | struct cpuidle_driver *drv, int index); | |
28ba086e | 100 | static void intel_idle_s2idle(struct cpuidle_device *dev, |
5fe2e527 | 101 | struct cpuidle_driver *drv, int index); |
26717172 LB |
102 | static struct cpuidle_state *cpuidle_state_table; |
103 | ||
956d033f LB |
104 | /* |
105 | * Set this flag for states where the HW flushes the TLB for us | |
106 | * and so we don't need cross-calls to keep it consistent. | |
107 | * If this flag is set, SW flushes the TLB, so even if the | |
108 | * HW doesn't do the flushing, this flag is safe to use. | |
109 | */ | |
110 | #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 | |
111 | ||
b1beab48 LB |
112 | /* |
113 | * MWAIT takes an 8-bit "hint" in EAX "suggesting" | |
114 | * the C-state (top nibble) and sub-state (bottom nibble) | |
115 | * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. | |
116 | * | |
117 | * We store the hint at the top of our "flags" for each state. | |
118 | */ | |
119 | #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) | |
120 | #define MWAIT2flg(eax) ((eax & 0xFF) << 24) | |
121 | ||
26717172 LB |
122 | /* |
123 | * States are indexed by the cstate number, | |
124 | * which is also the index into the MWAIT hint array. | |
125 | * Thus C0 is a dummy. | |
126 | */ | |
ba0dc81e | 127 | static struct cpuidle_state nehalem_cstates[] = { |
e022e7eb | 128 | { |
de09cdd0 | 129 | .name = "C1", |
26717172 | 130 | .desc = "MWAIT 0x00", |
b82b6cca | 131 | .flags = MWAIT2flg(0x00), |
26717172 | 132 | .exit_latency = 3, |
26717172 | 133 | .target_residency = 6, |
5fe2e527 | 134 | .enter = &intel_idle, |
28ba086e | 135 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 136 | { |
de09cdd0 | 137 | .name = "C1E", |
32e95180 | 138 | .desc = "MWAIT 0x01", |
b82b6cca | 139 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
140 | .exit_latency = 10, |
141 | .target_residency = 20, | |
5fe2e527 | 142 | .enter = &intel_idle, |
28ba086e | 143 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 144 | { |
de09cdd0 | 145 | .name = "C3", |
26717172 | 146 | .desc = "MWAIT 0x10", |
b82b6cca | 147 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 148 | .exit_latency = 20, |
26717172 | 149 | .target_residency = 80, |
5fe2e527 | 150 | .enter = &intel_idle, |
28ba086e | 151 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 152 | { |
de09cdd0 | 153 | .name = "C6", |
26717172 | 154 | .desc = "MWAIT 0x20", |
b82b6cca | 155 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 156 | .exit_latency = 200, |
26717172 | 157 | .target_residency = 800, |
5fe2e527 | 158 | .enter = &intel_idle, |
28ba086e | 159 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
160 | { |
161 | .enter = NULL } | |
26717172 LB |
162 | }; |
163 | ||
ba0dc81e | 164 | static struct cpuidle_state snb_cstates[] = { |
e022e7eb | 165 | { |
de09cdd0 | 166 | .name = "C1", |
d13780d4 | 167 | .desc = "MWAIT 0x00", |
b82b6cca | 168 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
169 | .exit_latency = 2, |
170 | .target_residency = 2, | |
5fe2e527 | 171 | .enter = &intel_idle, |
28ba086e | 172 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 173 | { |
de09cdd0 | 174 | .name = "C1E", |
32e95180 | 175 | .desc = "MWAIT 0x01", |
b82b6cca | 176 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
177 | .exit_latency = 10, |
178 | .target_residency = 20, | |
5fe2e527 | 179 | .enter = &intel_idle, |
28ba086e | 180 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 181 | { |
de09cdd0 | 182 | .name = "C3", |
d13780d4 | 183 | .desc = "MWAIT 0x10", |
b82b6cca | 184 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 185 | .exit_latency = 80, |
ddbd550d | 186 | .target_residency = 211, |
5fe2e527 | 187 | .enter = &intel_idle, |
28ba086e | 188 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 189 | { |
de09cdd0 | 190 | .name = "C6", |
d13780d4 | 191 | .desc = "MWAIT 0x20", |
b82b6cca | 192 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 193 | .exit_latency = 104, |
ddbd550d | 194 | .target_residency = 345, |
5fe2e527 | 195 | .enter = &intel_idle, |
28ba086e | 196 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 197 | { |
de09cdd0 | 198 | .name = "C7", |
d13780d4 | 199 | .desc = "MWAIT 0x30", |
b82b6cca | 200 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 201 | .exit_latency = 109, |
ddbd550d | 202 | .target_residency = 345, |
5fe2e527 | 203 | .enter = &intel_idle, |
28ba086e | 204 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
205 | { |
206 | .enter = NULL } | |
d13780d4 LB |
207 | }; |
208 | ||
718987d6 LB |
209 | static struct cpuidle_state byt_cstates[] = { |
210 | { | |
de09cdd0 | 211 | .name = "C1", |
718987d6 | 212 | .desc = "MWAIT 0x00", |
b82b6cca | 213 | .flags = MWAIT2flg(0x00), |
718987d6 LB |
214 | .exit_latency = 1, |
215 | .target_residency = 1, | |
5fe2e527 | 216 | .enter = &intel_idle, |
28ba086e | 217 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 218 | { |
de09cdd0 | 219 | .name = "C6N", |
718987d6 | 220 | .desc = "MWAIT 0x58", |
b82b6cca | 221 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 222 | .exit_latency = 300, |
718987d6 | 223 | .target_residency = 275, |
5fe2e527 | 224 | .enter = &intel_idle, |
28ba086e | 225 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 226 | { |
de09cdd0 | 227 | .name = "C6S", |
718987d6 | 228 | .desc = "MWAIT 0x52", |
b82b6cca | 229 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 230 | .exit_latency = 500, |
718987d6 | 231 | .target_residency = 560, |
5fe2e527 | 232 | .enter = &intel_idle, |
28ba086e | 233 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 234 | { |
de09cdd0 | 235 | .name = "C7", |
718987d6 | 236 | .desc = "MWAIT 0x60", |
b82b6cca | 237 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 | 238 | .exit_latency = 1200, |
d7ef7671 | 239 | .target_residency = 4000, |
5fe2e527 | 240 | .enter = &intel_idle, |
28ba086e | 241 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 | 242 | { |
de09cdd0 | 243 | .name = "C7S", |
718987d6 | 244 | .desc = "MWAIT 0x64", |
b82b6cca | 245 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 LB |
246 | .exit_latency = 10000, |
247 | .target_residency = 20000, | |
5fe2e527 | 248 | .enter = &intel_idle, |
28ba086e | 249 | .enter_s2idle = intel_idle_s2idle, }, |
718987d6 LB |
250 | { |
251 | .enter = NULL } | |
252 | }; | |
253 | ||
cab07a56 LB |
254 | static struct cpuidle_state cht_cstates[] = { |
255 | { | |
de09cdd0 | 256 | .name = "C1", |
cab07a56 LB |
257 | .desc = "MWAIT 0x00", |
258 | .flags = MWAIT2flg(0x00), | |
259 | .exit_latency = 1, | |
260 | .target_residency = 1, | |
261 | .enter = &intel_idle, | |
28ba086e | 262 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 263 | { |
de09cdd0 | 264 | .name = "C6N", |
cab07a56 LB |
265 | .desc = "MWAIT 0x58", |
266 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, | |
267 | .exit_latency = 80, | |
268 | .target_residency = 275, | |
269 | .enter = &intel_idle, | |
28ba086e | 270 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 271 | { |
de09cdd0 | 272 | .name = "C6S", |
cab07a56 LB |
273 | .desc = "MWAIT 0x52", |
274 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, | |
275 | .exit_latency = 200, | |
276 | .target_residency = 560, | |
277 | .enter = &intel_idle, | |
28ba086e | 278 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 279 | { |
de09cdd0 | 280 | .name = "C7", |
cab07a56 LB |
281 | .desc = "MWAIT 0x60", |
282 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
283 | .exit_latency = 1200, | |
284 | .target_residency = 4000, | |
285 | .enter = &intel_idle, | |
28ba086e | 286 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 | 287 | { |
de09cdd0 | 288 | .name = "C7S", |
cab07a56 LB |
289 | .desc = "MWAIT 0x64", |
290 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, | |
291 | .exit_latency = 10000, | |
292 | .target_residency = 20000, | |
293 | .enter = &intel_idle, | |
28ba086e | 294 | .enter_s2idle = intel_idle_s2idle, }, |
cab07a56 LB |
295 | { |
296 | .enter = NULL } | |
297 | }; | |
298 | ||
ba0dc81e | 299 | static struct cpuidle_state ivb_cstates[] = { |
e022e7eb | 300 | { |
de09cdd0 | 301 | .name = "C1", |
6edab08c | 302 | .desc = "MWAIT 0x00", |
b82b6cca | 303 | .flags = MWAIT2flg(0x00), |
6edab08c LB |
304 | .exit_latency = 1, |
305 | .target_residency = 1, | |
5fe2e527 | 306 | .enter = &intel_idle, |
28ba086e | 307 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 308 | { |
de09cdd0 | 309 | .name = "C1E", |
32e95180 | 310 | .desc = "MWAIT 0x01", |
b82b6cca | 311 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
312 | .exit_latency = 10, |
313 | .target_residency = 20, | |
5fe2e527 | 314 | .enter = &intel_idle, |
28ba086e | 315 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 316 | { |
de09cdd0 | 317 | .name = "C3", |
6edab08c | 318 | .desc = "MWAIT 0x10", |
b82b6cca | 319 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
320 | .exit_latency = 59, |
321 | .target_residency = 156, | |
5fe2e527 | 322 | .enter = &intel_idle, |
28ba086e | 323 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 324 | { |
de09cdd0 | 325 | .name = "C6", |
6edab08c | 326 | .desc = "MWAIT 0x20", |
b82b6cca | 327 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
328 | .exit_latency = 80, |
329 | .target_residency = 300, | |
5fe2e527 | 330 | .enter = &intel_idle, |
28ba086e | 331 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 332 | { |
de09cdd0 | 333 | .name = "C7", |
6edab08c | 334 | .desc = "MWAIT 0x30", |
b82b6cca | 335 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
336 | .exit_latency = 87, |
337 | .target_residency = 300, | |
5fe2e527 | 338 | .enter = &intel_idle, |
28ba086e | 339 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
340 | { |
341 | .enter = NULL } | |
6edab08c LB |
342 | }; |
343 | ||
0138d8f0 LB |
344 | static struct cpuidle_state ivt_cstates[] = { |
345 | { | |
de09cdd0 | 346 | .name = "C1", |
0138d8f0 | 347 | .desc = "MWAIT 0x00", |
b82b6cca | 348 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
349 | .exit_latency = 1, |
350 | .target_residency = 1, | |
5fe2e527 | 351 | .enter = &intel_idle, |
28ba086e | 352 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 353 | { |
de09cdd0 | 354 | .name = "C1E", |
0138d8f0 | 355 | .desc = "MWAIT 0x01", |
b82b6cca | 356 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
357 | .exit_latency = 10, |
358 | .target_residency = 80, | |
5fe2e527 | 359 | .enter = &intel_idle, |
28ba086e | 360 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 361 | { |
de09cdd0 | 362 | .name = "C3", |
0138d8f0 | 363 | .desc = "MWAIT 0x10", |
b82b6cca | 364 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
365 | .exit_latency = 59, |
366 | .target_residency = 156, | |
5fe2e527 | 367 | .enter = &intel_idle, |
28ba086e | 368 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 369 | { |
de09cdd0 | 370 | .name = "C6", |
0138d8f0 | 371 | .desc = "MWAIT 0x20", |
b82b6cca | 372 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
373 | .exit_latency = 82, |
374 | .target_residency = 300, | |
5fe2e527 | 375 | .enter = &intel_idle, |
28ba086e | 376 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 LB |
377 | { |
378 | .enter = NULL } | |
379 | }; | |
380 | ||
381 | static struct cpuidle_state ivt_cstates_4s[] = { | |
382 | { | |
de09cdd0 | 383 | .name = "C1", |
0138d8f0 | 384 | .desc = "MWAIT 0x00", |
b82b6cca | 385 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
386 | .exit_latency = 1, |
387 | .target_residency = 1, | |
5fe2e527 | 388 | .enter = &intel_idle, |
28ba086e | 389 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 390 | { |
de09cdd0 | 391 | .name = "C1E", |
0138d8f0 | 392 | .desc = "MWAIT 0x01", |
b82b6cca | 393 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
394 | .exit_latency = 10, |
395 | .target_residency = 250, | |
5fe2e527 | 396 | .enter = &intel_idle, |
28ba086e | 397 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 398 | { |
de09cdd0 | 399 | .name = "C3", |
0138d8f0 | 400 | .desc = "MWAIT 0x10", |
b82b6cca | 401 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
402 | .exit_latency = 59, |
403 | .target_residency = 300, | |
5fe2e527 | 404 | .enter = &intel_idle, |
28ba086e | 405 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 406 | { |
de09cdd0 | 407 | .name = "C6", |
0138d8f0 | 408 | .desc = "MWAIT 0x20", |
b82b6cca | 409 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
410 | .exit_latency = 84, |
411 | .target_residency = 400, | |
5fe2e527 | 412 | .enter = &intel_idle, |
28ba086e | 413 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 LB |
414 | { |
415 | .enter = NULL } | |
416 | }; | |
417 | ||
418 | static struct cpuidle_state ivt_cstates_8s[] = { | |
419 | { | |
de09cdd0 | 420 | .name = "C1", |
0138d8f0 | 421 | .desc = "MWAIT 0x00", |
b82b6cca | 422 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
423 | .exit_latency = 1, |
424 | .target_residency = 1, | |
5fe2e527 | 425 | .enter = &intel_idle, |
28ba086e | 426 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 427 | { |
de09cdd0 | 428 | .name = "C1E", |
0138d8f0 | 429 | .desc = "MWAIT 0x01", |
b82b6cca | 430 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
431 | .exit_latency = 10, |
432 | .target_residency = 500, | |
5fe2e527 | 433 | .enter = &intel_idle, |
28ba086e | 434 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 435 | { |
de09cdd0 | 436 | .name = "C3", |
0138d8f0 | 437 | .desc = "MWAIT 0x10", |
b82b6cca | 438 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
439 | .exit_latency = 59, |
440 | .target_residency = 600, | |
5fe2e527 | 441 | .enter = &intel_idle, |
28ba086e | 442 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 | 443 | { |
de09cdd0 | 444 | .name = "C6", |
0138d8f0 | 445 | .desc = "MWAIT 0x20", |
b82b6cca | 446 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
447 | .exit_latency = 88, |
448 | .target_residency = 700, | |
5fe2e527 | 449 | .enter = &intel_idle, |
28ba086e | 450 | .enter_s2idle = intel_idle_s2idle, }, |
0138d8f0 LB |
451 | { |
452 | .enter = NULL } | |
453 | }; | |
454 | ||
ba0dc81e | 455 | static struct cpuidle_state hsw_cstates[] = { |
e022e7eb | 456 | { |
de09cdd0 | 457 | .name = "C1", |
85a4d2d4 | 458 | .desc = "MWAIT 0x00", |
b82b6cca | 459 | .flags = MWAIT2flg(0x00), |
85a4d2d4 LB |
460 | .exit_latency = 2, |
461 | .target_residency = 2, | |
5fe2e527 | 462 | .enter = &intel_idle, |
28ba086e | 463 | .enter_s2idle = intel_idle_s2idle, }, |
32e95180 | 464 | { |
de09cdd0 | 465 | .name = "C1E", |
32e95180 | 466 | .desc = "MWAIT 0x01", |
b82b6cca | 467 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
468 | .exit_latency = 10, |
469 | .target_residency = 20, | |
5fe2e527 | 470 | .enter = &intel_idle, |
28ba086e | 471 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 472 | { |
de09cdd0 | 473 | .name = "C3", |
85a4d2d4 | 474 | .desc = "MWAIT 0x10", |
b82b6cca | 475 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
476 | .exit_latency = 33, |
477 | .target_residency = 100, | |
5fe2e527 | 478 | .enter = &intel_idle, |
28ba086e | 479 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 480 | { |
de09cdd0 | 481 | .name = "C6", |
85a4d2d4 | 482 | .desc = "MWAIT 0x20", |
b82b6cca | 483 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
484 | .exit_latency = 133, |
485 | .target_residency = 400, | |
5fe2e527 | 486 | .enter = &intel_idle, |
28ba086e | 487 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 488 | { |
de09cdd0 | 489 | .name = "C7s", |
85a4d2d4 | 490 | .desc = "MWAIT 0x32", |
b82b6cca | 491 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
492 | .exit_latency = 166, |
493 | .target_residency = 500, | |
5fe2e527 | 494 | .enter = &intel_idle, |
28ba086e | 495 | .enter_s2idle = intel_idle_s2idle, }, |
86239ceb | 496 | { |
de09cdd0 | 497 | .name = "C8", |
86239ceb | 498 | .desc = "MWAIT 0x40", |
b82b6cca | 499 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
500 | .exit_latency = 300, |
501 | .target_residency = 900, | |
5fe2e527 | 502 | .enter = &intel_idle, |
28ba086e | 503 | .enter_s2idle = intel_idle_s2idle, }, |
86239ceb | 504 | { |
de09cdd0 | 505 | .name = "C9", |
86239ceb | 506 | .desc = "MWAIT 0x50", |
b82b6cca | 507 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
508 | .exit_latency = 600, |
509 | .target_residency = 1800, | |
5fe2e527 | 510 | .enter = &intel_idle, |
28ba086e | 511 | .enter_s2idle = intel_idle_s2idle, }, |
86239ceb | 512 | { |
de09cdd0 | 513 | .name = "C10", |
86239ceb | 514 | .desc = "MWAIT 0x60", |
b82b6cca | 515 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
516 | .exit_latency = 2600, |
517 | .target_residency = 7700, | |
5fe2e527 | 518 | .enter = &intel_idle, |
28ba086e | 519 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
520 | { |
521 | .enter = NULL } | |
85a4d2d4 | 522 | }; |
a138b568 LB |
523 | static struct cpuidle_state bdw_cstates[] = { |
524 | { | |
de09cdd0 | 525 | .name = "C1", |
a138b568 | 526 | .desc = "MWAIT 0x00", |
b82b6cca | 527 | .flags = MWAIT2flg(0x00), |
a138b568 LB |
528 | .exit_latency = 2, |
529 | .target_residency = 2, | |
5fe2e527 | 530 | .enter = &intel_idle, |
28ba086e | 531 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 532 | { |
de09cdd0 | 533 | .name = "C1E", |
a138b568 | 534 | .desc = "MWAIT 0x01", |
b82b6cca | 535 | .flags = MWAIT2flg(0x01), |
a138b568 LB |
536 | .exit_latency = 10, |
537 | .target_residency = 20, | |
5fe2e527 | 538 | .enter = &intel_idle, |
28ba086e | 539 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 540 | { |
de09cdd0 | 541 | .name = "C3", |
a138b568 | 542 | .desc = "MWAIT 0x10", |
b82b6cca | 543 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
544 | .exit_latency = 40, |
545 | .target_residency = 100, | |
5fe2e527 | 546 | .enter = &intel_idle, |
28ba086e | 547 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 548 | { |
de09cdd0 | 549 | .name = "C6", |
a138b568 | 550 | .desc = "MWAIT 0x20", |
b82b6cca | 551 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
552 | .exit_latency = 133, |
553 | .target_residency = 400, | |
5fe2e527 | 554 | .enter = &intel_idle, |
28ba086e | 555 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 556 | { |
de09cdd0 | 557 | .name = "C7s", |
a138b568 | 558 | .desc = "MWAIT 0x32", |
b82b6cca | 559 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
560 | .exit_latency = 166, |
561 | .target_residency = 500, | |
5fe2e527 | 562 | .enter = &intel_idle, |
28ba086e | 563 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 564 | { |
de09cdd0 | 565 | .name = "C8", |
a138b568 | 566 | .desc = "MWAIT 0x40", |
b82b6cca | 567 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
568 | .exit_latency = 300, |
569 | .target_residency = 900, | |
5fe2e527 | 570 | .enter = &intel_idle, |
28ba086e | 571 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 572 | { |
de09cdd0 | 573 | .name = "C9", |
a138b568 | 574 | .desc = "MWAIT 0x50", |
b82b6cca | 575 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
576 | .exit_latency = 600, |
577 | .target_residency = 1800, | |
5fe2e527 | 578 | .enter = &intel_idle, |
28ba086e | 579 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 | 580 | { |
de09cdd0 | 581 | .name = "C10", |
a138b568 | 582 | .desc = "MWAIT 0x60", |
b82b6cca | 583 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
584 | .exit_latency = 2600, |
585 | .target_residency = 7700, | |
5fe2e527 | 586 | .enter = &intel_idle, |
28ba086e | 587 | .enter_s2idle = intel_idle_s2idle, }, |
a138b568 LB |
588 | { |
589 | .enter = NULL } | |
590 | }; | |
85a4d2d4 | 591 | |
493f133f LB |
592 | static struct cpuidle_state skl_cstates[] = { |
593 | { | |
de09cdd0 | 594 | .name = "C1", |
493f133f LB |
595 | .desc = "MWAIT 0x00", |
596 | .flags = MWAIT2flg(0x00), | |
597 | .exit_latency = 2, | |
598 | .target_residency = 2, | |
599 | .enter = &intel_idle, | |
28ba086e | 600 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 601 | { |
de09cdd0 | 602 | .name = "C1E", |
493f133f LB |
603 | .desc = "MWAIT 0x01", |
604 | .flags = MWAIT2flg(0x01), | |
605 | .exit_latency = 10, | |
606 | .target_residency = 20, | |
607 | .enter = &intel_idle, | |
28ba086e | 608 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 609 | { |
de09cdd0 | 610 | .name = "C3", |
493f133f LB |
611 | .desc = "MWAIT 0x10", |
612 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
613 | .exit_latency = 70, | |
614 | .target_residency = 100, | |
615 | .enter = &intel_idle, | |
28ba086e | 616 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 617 | { |
de09cdd0 | 618 | .name = "C6", |
493f133f LB |
619 | .desc = "MWAIT 0x20", |
620 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
135919a3 | 621 | .exit_latency = 85, |
493f133f LB |
622 | .target_residency = 200, |
623 | .enter = &intel_idle, | |
28ba086e | 624 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 625 | { |
de09cdd0 | 626 | .name = "C7s", |
493f133f LB |
627 | .desc = "MWAIT 0x33", |
628 | .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, | |
629 | .exit_latency = 124, | |
630 | .target_residency = 800, | |
631 | .enter = &intel_idle, | |
28ba086e | 632 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 633 | { |
de09cdd0 | 634 | .name = "C8", |
493f133f LB |
635 | .desc = "MWAIT 0x40", |
636 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
135919a3 | 637 | .exit_latency = 200, |
493f133f LB |
638 | .target_residency = 800, |
639 | .enter = &intel_idle, | |
28ba086e | 640 | .enter_s2idle = intel_idle_s2idle, }, |
135919a3 | 641 | { |
de09cdd0 | 642 | .name = "C9", |
135919a3 LB |
643 | .desc = "MWAIT 0x50", |
644 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, | |
645 | .exit_latency = 480, | |
646 | .target_residency = 5000, | |
647 | .enter = &intel_idle, | |
28ba086e | 648 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f | 649 | { |
de09cdd0 | 650 | .name = "C10", |
493f133f LB |
651 | .desc = "MWAIT 0x60", |
652 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
653 | .exit_latency = 890, | |
654 | .target_residency = 5000, | |
655 | .enter = &intel_idle, | |
28ba086e | 656 | .enter_s2idle = intel_idle_s2idle, }, |
493f133f LB |
657 | { |
658 | .enter = NULL } | |
659 | }; | |
660 | ||
f9e71657 LB |
661 | static struct cpuidle_state skx_cstates[] = { |
662 | { | |
de09cdd0 | 663 | .name = "C1", |
f9e71657 LB |
664 | .desc = "MWAIT 0x00", |
665 | .flags = MWAIT2flg(0x00), | |
666 | .exit_latency = 2, | |
667 | .target_residency = 2, | |
668 | .enter = &intel_idle, | |
28ba086e | 669 | .enter_s2idle = intel_idle_s2idle, }, |
f9e71657 | 670 | { |
de09cdd0 | 671 | .name = "C1E", |
f9e71657 LB |
672 | .desc = "MWAIT 0x01", |
673 | .flags = MWAIT2flg(0x01), | |
674 | .exit_latency = 10, | |
675 | .target_residency = 20, | |
676 | .enter = &intel_idle, | |
28ba086e | 677 | .enter_s2idle = intel_idle_s2idle, }, |
f9e71657 | 678 | { |
de09cdd0 | 679 | .name = "C6", |
f9e71657 LB |
680 | .desc = "MWAIT 0x20", |
681 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
682 | .exit_latency = 133, | |
683 | .target_residency = 600, | |
684 | .enter = &intel_idle, | |
28ba086e | 685 | .enter_s2idle = intel_idle_s2idle, }, |
f9e71657 LB |
686 | { |
687 | .enter = NULL } | |
688 | }; | |
689 | ||
ba0dc81e | 690 | static struct cpuidle_state atom_cstates[] = { |
e022e7eb | 691 | { |
de09cdd0 | 692 | .name = "C1E", |
26717172 | 693 | .desc = "MWAIT 0x00", |
b82b6cca | 694 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
695 | .exit_latency = 10, |
696 | .target_residency = 20, | |
5fe2e527 | 697 | .enter = &intel_idle, |
28ba086e | 698 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 699 | { |
de09cdd0 | 700 | .name = "C2", |
26717172 | 701 | .desc = "MWAIT 0x10", |
b82b6cca | 702 | .flags = MWAIT2flg(0x10), |
26717172 | 703 | .exit_latency = 20, |
26717172 | 704 | .target_residency = 80, |
5fe2e527 | 705 | .enter = &intel_idle, |
28ba086e | 706 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 707 | { |
de09cdd0 | 708 | .name = "C4", |
26717172 | 709 | .desc = "MWAIT 0x30", |
b82b6cca | 710 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 711 | .exit_latency = 100, |
26717172 | 712 | .target_residency = 400, |
5fe2e527 | 713 | .enter = &intel_idle, |
28ba086e | 714 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb | 715 | { |
de09cdd0 | 716 | .name = "C6", |
7fcca7d9 | 717 | .desc = "MWAIT 0x52", |
b82b6cca | 718 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
7fcca7d9 | 719 | .exit_latency = 140, |
7fcca7d9 | 720 | .target_residency = 560, |
5fe2e527 | 721 | .enter = &intel_idle, |
28ba086e | 722 | .enter_s2idle = intel_idle_s2idle, }, |
e022e7eb LB |
723 | { |
724 | .enter = NULL } | |
26717172 | 725 | }; |
5e7ec268 AS |
726 | static struct cpuidle_state tangier_cstates[] = { |
727 | { | |
de09cdd0 | 728 | .name = "C1", |
5e7ec268 AS |
729 | .desc = "MWAIT 0x00", |
730 | .flags = MWAIT2flg(0x00), | |
731 | .exit_latency = 1, | |
732 | .target_residency = 4, | |
733 | .enter = &intel_idle, | |
28ba086e | 734 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 735 | { |
de09cdd0 | 736 | .name = "C4", |
5e7ec268 AS |
737 | .desc = "MWAIT 0x30", |
738 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, | |
739 | .exit_latency = 100, | |
740 | .target_residency = 400, | |
741 | .enter = &intel_idle, | |
28ba086e | 742 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 743 | { |
de09cdd0 | 744 | .name = "C6", |
5e7ec268 AS |
745 | .desc = "MWAIT 0x52", |
746 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, | |
747 | .exit_latency = 140, | |
748 | .target_residency = 560, | |
749 | .enter = &intel_idle, | |
28ba086e | 750 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 751 | { |
de09cdd0 | 752 | .name = "C7", |
5e7ec268 AS |
753 | .desc = "MWAIT 0x60", |
754 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
755 | .exit_latency = 1200, | |
756 | .target_residency = 4000, | |
757 | .enter = &intel_idle, | |
28ba086e | 758 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 | 759 | { |
de09cdd0 | 760 | .name = "C9", |
5e7ec268 AS |
761 | .desc = "MWAIT 0x64", |
762 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, | |
763 | .exit_latency = 10000, | |
764 | .target_residency = 20000, | |
765 | .enter = &intel_idle, | |
28ba086e | 766 | .enter_s2idle = intel_idle_s2idle, }, |
5e7ec268 AS |
767 | { |
768 | .enter = NULL } | |
769 | }; | |
88390996 | 770 | static struct cpuidle_state avn_cstates[] = { |
fab04b22 | 771 | { |
de09cdd0 | 772 | .name = "C1", |
fab04b22 | 773 | .desc = "MWAIT 0x00", |
b82b6cca | 774 | .flags = MWAIT2flg(0x00), |
fab04b22 LB |
775 | .exit_latency = 2, |
776 | .target_residency = 2, | |
5fe2e527 | 777 | .enter = &intel_idle, |
28ba086e | 778 | .enter_s2idle = intel_idle_s2idle, }, |
fab04b22 | 779 | { |
de09cdd0 | 780 | .name = "C6", |
fab04b22 | 781 | .desc = "MWAIT 0x51", |
b82b6cca | 782 | .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, |
fab04b22 LB |
783 | .exit_latency = 15, |
784 | .target_residency = 45, | |
5fe2e527 | 785 | .enter = &intel_idle, |
28ba086e | 786 | .enter_s2idle = intel_idle_s2idle, }, |
88390996 JL |
787 | { |
788 | .enter = NULL } | |
fab04b22 | 789 | }; |
281baf7a DC |
790 | static struct cpuidle_state knl_cstates[] = { |
791 | { | |
de09cdd0 | 792 | .name = "C1", |
281baf7a DC |
793 | .desc = "MWAIT 0x00", |
794 | .flags = MWAIT2flg(0x00), | |
795 | .exit_latency = 1, | |
796 | .target_residency = 2, | |
797 | .enter = &intel_idle, | |
28ba086e | 798 | .enter_s2idle = intel_idle_s2idle }, |
281baf7a | 799 | { |
de09cdd0 | 800 | .name = "C6", |
281baf7a DC |
801 | .desc = "MWAIT 0x10", |
802 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
803 | .exit_latency = 120, | |
804 | .target_residency = 500, | |
805 | .enter = &intel_idle, | |
28ba086e | 806 | .enter_s2idle = intel_idle_s2idle }, |
281baf7a DC |
807 | { |
808 | .enter = NULL } | |
809 | }; | |
26717172 | 810 | |
5dcef694 LB |
811 | static struct cpuidle_state bxt_cstates[] = { |
812 | { | |
de09cdd0 | 813 | .name = "C1", |
5dcef694 LB |
814 | .desc = "MWAIT 0x00", |
815 | .flags = MWAIT2flg(0x00), | |
816 | .exit_latency = 2, | |
817 | .target_residency = 2, | |
818 | .enter = &intel_idle, | |
28ba086e | 819 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 820 | { |
de09cdd0 | 821 | .name = "C1E", |
5dcef694 LB |
822 | .desc = "MWAIT 0x01", |
823 | .flags = MWAIT2flg(0x01), | |
824 | .exit_latency = 10, | |
825 | .target_residency = 20, | |
826 | .enter = &intel_idle, | |
28ba086e | 827 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 828 | { |
de09cdd0 | 829 | .name = "C6", |
5dcef694 LB |
830 | .desc = "MWAIT 0x20", |
831 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
832 | .exit_latency = 133, | |
833 | .target_residency = 133, | |
834 | .enter = &intel_idle, | |
28ba086e | 835 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 836 | { |
de09cdd0 | 837 | .name = "C7s", |
5dcef694 LB |
838 | .desc = "MWAIT 0x31", |
839 | .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, | |
840 | .exit_latency = 155, | |
841 | .target_residency = 155, | |
842 | .enter = &intel_idle, | |
28ba086e | 843 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 844 | { |
de09cdd0 | 845 | .name = "C8", |
5dcef694 LB |
846 | .desc = "MWAIT 0x40", |
847 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
848 | .exit_latency = 1000, | |
849 | .target_residency = 1000, | |
850 | .enter = &intel_idle, | |
28ba086e | 851 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 852 | { |
de09cdd0 | 853 | .name = "C9", |
5dcef694 LB |
854 | .desc = "MWAIT 0x50", |
855 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, | |
856 | .exit_latency = 2000, | |
857 | .target_residency = 2000, | |
858 | .enter = &intel_idle, | |
28ba086e | 859 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 | 860 | { |
de09cdd0 | 861 | .name = "C10", |
5dcef694 LB |
862 | .desc = "MWAIT 0x60", |
863 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
864 | .exit_latency = 10000, | |
865 | .target_residency = 10000, | |
866 | .enter = &intel_idle, | |
28ba086e | 867 | .enter_s2idle = intel_idle_s2idle, }, |
5dcef694 LB |
868 | { |
869 | .enter = NULL } | |
870 | }; | |
871 | ||
0080d65b JP |
872 | static struct cpuidle_state dnv_cstates[] = { |
873 | { | |
de09cdd0 | 874 | .name = "C1", |
0080d65b JP |
875 | .desc = "MWAIT 0x00", |
876 | .flags = MWAIT2flg(0x00), | |
877 | .exit_latency = 2, | |
878 | .target_residency = 2, | |
879 | .enter = &intel_idle, | |
28ba086e | 880 | .enter_s2idle = intel_idle_s2idle, }, |
0080d65b | 881 | { |
de09cdd0 | 882 | .name = "C1E", |
0080d65b JP |
883 | .desc = "MWAIT 0x01", |
884 | .flags = MWAIT2flg(0x01), | |
885 | .exit_latency = 10, | |
886 | .target_residency = 20, | |
887 | .enter = &intel_idle, | |
28ba086e | 888 | .enter_s2idle = intel_idle_s2idle, }, |
0080d65b | 889 | { |
de09cdd0 | 890 | .name = "C6", |
0080d65b JP |
891 | .desc = "MWAIT 0x20", |
892 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
893 | .exit_latency = 50, | |
894 | .target_residency = 500, | |
895 | .enter = &intel_idle, | |
28ba086e | 896 | .enter_s2idle = intel_idle_s2idle, }, |
0080d65b JP |
897 | { |
898 | .enter = NULL } | |
899 | }; | |
900 | ||
26717172 LB |
901 | /** |
902 | * intel_idle | |
903 | * @dev: cpuidle_device | |
46bcfad7 | 904 | * @drv: cpuidle driver |
e978aa7d | 905 | * @index: index of cpuidle state |
26717172 | 906 | * |
63ff07be | 907 | * Must be called under local_irq_disable(). |
26717172 | 908 | */ |
6727ad9e CM |
909 | static __cpuidle int intel_idle(struct cpuidle_device *dev, |
910 | struct cpuidle_driver *drv, int index) | |
26717172 LB |
911 | { |
912 | unsigned long ecx = 1; /* break on interrupt flag */ | |
46bcfad7 | 913 | struct cpuidle_state *state = &drv->states[index]; |
b1beab48 | 914 | unsigned long eax = flg2MWAIT(state->flags); |
26717172 | 915 | unsigned int cstate; |
0563bb7b | 916 | bool uninitialized_var(tick); |
67535736 | 917 | int cpu = smp_processor_id(); |
26717172 | 918 | |
6110a1f4 | 919 | /* |
67535736 AL |
920 | * leave_mm() to avoid costly and often unnecessary wakeups |
921 | * for flushing the user TLB's associated with the active mm. | |
6110a1f4 | 922 | */ |
67535736 AL |
923 | if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) |
924 | leave_mm(cpu); | |
6110a1f4 | 925 | |
0563bb7b JB |
926 | if (!static_cpu_has(X86_FEATURE_ARAT)) { |
927 | cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & | |
928 | MWAIT_CSTATE_MASK) + 1; | |
929 | tick = false; | |
930 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) { | |
931 | tick = true; | |
932 | tick_broadcast_enter(); | |
933 | } | |
934 | } | |
26717172 | 935 | |
16824255 | 936 | mwait_idle_with_hints(eax, ecx); |
26717172 | 937 | |
0563bb7b | 938 | if (!static_cpu_has(X86_FEATURE_ARAT) && tick) |
f6cee191 | 939 | tick_broadcast_exit(); |
26717172 | 940 | |
e978aa7d | 941 | return index; |
26717172 LB |
942 | } |
943 | ||
5fe2e527 | 944 | /** |
28ba086e | 945 | * intel_idle_s2idle - simplified "enter" callback routine for suspend-to-idle |
5fe2e527 RW |
946 | * @dev: cpuidle_device |
947 | * @drv: cpuidle driver | |
948 | * @index: state index | |
949 | */ | |
28ba086e | 950 | static void intel_idle_s2idle(struct cpuidle_device *dev, |
5fe2e527 RW |
951 | struct cpuidle_driver *drv, int index) |
952 | { | |
953 | unsigned long ecx = 1; /* break on interrupt flag */ | |
954 | unsigned long eax = flg2MWAIT(drv->states[index].flags); | |
955 | ||
956 | mwait_idle_with_hints(eax, ecx); | |
957 | } | |
958 | ||
fb1013a0 | 959 | static void __setup_broadcast_timer(bool on) |
2a2d31c8 | 960 | { |
76962caa TG |
961 | if (on) |
962 | tick_broadcast_enable(); | |
963 | else | |
964 | tick_broadcast_disable(); | |
2a2d31c8 SL |
965 | } |
966 | ||
fb1013a0 | 967 | static void auto_demotion_disable(void) |
14796fca LB |
968 | { |
969 | unsigned long long msr_bits; | |
970 | ||
6cfb2374 | 971 | rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); |
b66b8b9a | 972 | msr_bits &= ~(icpu->auto_demotion_disable_flags); |
6cfb2374 | 973 | wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits); |
14796fca | 974 | } |
fb1013a0 | 975 | static void c1e_promotion_disable(void) |
32e95180 LB |
976 | { |
977 | unsigned long long msr_bits; | |
978 | ||
979 | rdmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
980 | msr_bits &= ~0x2; | |
981 | wrmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
982 | } | |
14796fca | 983 | |
b66b8b9a AK |
984 | static const struct idle_cpu idle_cpu_nehalem = { |
985 | .state_table = nehalem_cstates, | |
b66b8b9a | 986 | .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, |
32e95180 | 987 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
988 | }; |
989 | ||
990 | static const struct idle_cpu idle_cpu_atom = { | |
991 | .state_table = atom_cstates, | |
992 | }; | |
993 | ||
5e7ec268 AS |
994 | static const struct idle_cpu idle_cpu_tangier = { |
995 | .state_table = tangier_cstates, | |
996 | }; | |
997 | ||
b66b8b9a AK |
998 | static const struct idle_cpu idle_cpu_lincroft = { |
999 | .state_table = atom_cstates, | |
1000 | .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, | |
1001 | }; | |
1002 | ||
1003 | static const struct idle_cpu idle_cpu_snb = { | |
1004 | .state_table = snb_cstates, | |
32e95180 | 1005 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
1006 | }; |
1007 | ||
718987d6 LB |
1008 | static const struct idle_cpu idle_cpu_byt = { |
1009 | .state_table = byt_cstates, | |
1010 | .disable_promotion_to_c1e = true, | |
8c058d53 | 1011 | .byt_auto_demotion_disable_flag = true, |
718987d6 LB |
1012 | }; |
1013 | ||
cab07a56 LB |
1014 | static const struct idle_cpu idle_cpu_cht = { |
1015 | .state_table = cht_cstates, | |
1016 | .disable_promotion_to_c1e = true, | |
1017 | .byt_auto_demotion_disable_flag = true, | |
1018 | }; | |
1019 | ||
6edab08c LB |
1020 | static const struct idle_cpu idle_cpu_ivb = { |
1021 | .state_table = ivb_cstates, | |
32e95180 | 1022 | .disable_promotion_to_c1e = true, |
6edab08c LB |
1023 | }; |
1024 | ||
0138d8f0 LB |
1025 | static const struct idle_cpu idle_cpu_ivt = { |
1026 | .state_table = ivt_cstates, | |
1027 | .disable_promotion_to_c1e = true, | |
1028 | }; | |
1029 | ||
85a4d2d4 LB |
1030 | static const struct idle_cpu idle_cpu_hsw = { |
1031 | .state_table = hsw_cstates, | |
32e95180 | 1032 | .disable_promotion_to_c1e = true, |
85a4d2d4 LB |
1033 | }; |
1034 | ||
a138b568 LB |
1035 | static const struct idle_cpu idle_cpu_bdw = { |
1036 | .state_table = bdw_cstates, | |
1037 | .disable_promotion_to_c1e = true, | |
1038 | }; | |
1039 | ||
493f133f LB |
1040 | static const struct idle_cpu idle_cpu_skl = { |
1041 | .state_table = skl_cstates, | |
1042 | .disable_promotion_to_c1e = true, | |
1043 | }; | |
1044 | ||
f9e71657 LB |
1045 | static const struct idle_cpu idle_cpu_skx = { |
1046 | .state_table = skx_cstates, | |
1047 | .disable_promotion_to_c1e = true, | |
1048 | }; | |
493f133f | 1049 | |
fab04b22 LB |
1050 | static const struct idle_cpu idle_cpu_avn = { |
1051 | .state_table = avn_cstates, | |
1052 | .disable_promotion_to_c1e = true, | |
1053 | }; | |
1054 | ||
281baf7a DC |
1055 | static const struct idle_cpu idle_cpu_knl = { |
1056 | .state_table = knl_cstates, | |
1057 | }; | |
1058 | ||
5dcef694 LB |
1059 | static const struct idle_cpu idle_cpu_bxt = { |
1060 | .state_table = bxt_cstates, | |
1061 | .disable_promotion_to_c1e = true, | |
1062 | }; | |
1063 | ||
0080d65b JP |
1064 | static const struct idle_cpu idle_cpu_dnv = { |
1065 | .state_table = dnv_cstates, | |
1066 | .disable_promotion_to_c1e = true, | |
1067 | }; | |
1068 | ||
d5cdc3c4 | 1069 | static const struct x86_cpu_id intel_idle_ids[] __initconst = { |
a4a008e5 AS |
1070 | INTEL_CPU_FAM6(NEHALEM_EP, idle_cpu_nehalem), |
1071 | INTEL_CPU_FAM6(NEHALEM, idle_cpu_nehalem), | |
1072 | INTEL_CPU_FAM6(NEHALEM_G, idle_cpu_nehalem), | |
1073 | INTEL_CPU_FAM6(WESTMERE, idle_cpu_nehalem), | |
1074 | INTEL_CPU_FAM6(WESTMERE_EP, idle_cpu_nehalem), | |
1075 | INTEL_CPU_FAM6(NEHALEM_EX, idle_cpu_nehalem), | |
c05f3642 LT |
1076 | INTEL_CPU_FAM6(ATOM_BONNELL, idle_cpu_atom), |
1077 | INTEL_CPU_FAM6(ATOM_BONNELL_MID, idle_cpu_lincroft), | |
a4a008e5 AS |
1078 | INTEL_CPU_FAM6(WESTMERE_EX, idle_cpu_nehalem), |
1079 | INTEL_CPU_FAM6(SANDYBRIDGE, idle_cpu_snb), | |
1080 | INTEL_CPU_FAM6(SANDYBRIDGE_X, idle_cpu_snb), | |
c05f3642 LT |
1081 | INTEL_CPU_FAM6(ATOM_SALTWELL, idle_cpu_atom), |
1082 | INTEL_CPU_FAM6(ATOM_SILVERMONT, idle_cpu_byt), | |
1083 | INTEL_CPU_FAM6(ATOM_SILVERMONT_MID, idle_cpu_tangier), | |
a4a008e5 AS |
1084 | INTEL_CPU_FAM6(ATOM_AIRMONT, idle_cpu_cht), |
1085 | INTEL_CPU_FAM6(IVYBRIDGE, idle_cpu_ivb), | |
1086 | INTEL_CPU_FAM6(IVYBRIDGE_X, idle_cpu_ivt), | |
1087 | INTEL_CPU_FAM6(HASWELL_CORE, idle_cpu_hsw), | |
1088 | INTEL_CPU_FAM6(HASWELL_X, idle_cpu_hsw), | |
1089 | INTEL_CPU_FAM6(HASWELL_ULT, idle_cpu_hsw), | |
1090 | INTEL_CPU_FAM6(HASWELL_GT3E, idle_cpu_hsw), | |
c05f3642 | 1091 | INTEL_CPU_FAM6(ATOM_SILVERMONT_X, idle_cpu_avn), |
a4a008e5 AS |
1092 | INTEL_CPU_FAM6(BROADWELL_CORE, idle_cpu_bdw), |
1093 | INTEL_CPU_FAM6(BROADWELL_GT3E, idle_cpu_bdw), | |
1094 | INTEL_CPU_FAM6(BROADWELL_X, idle_cpu_bdw), | |
1095 | INTEL_CPU_FAM6(BROADWELL_XEON_D, idle_cpu_bdw), | |
1096 | INTEL_CPU_FAM6(SKYLAKE_MOBILE, idle_cpu_skl), | |
1097 | INTEL_CPU_FAM6(SKYLAKE_DESKTOP, idle_cpu_skl), | |
1098 | INTEL_CPU_FAM6(KABYLAKE_MOBILE, idle_cpu_skl), | |
1099 | INTEL_CPU_FAM6(KABYLAKE_DESKTOP, idle_cpu_skl), | |
1100 | INTEL_CPU_FAM6(SKYLAKE_X, idle_cpu_skx), | |
1101 | INTEL_CPU_FAM6(XEON_PHI_KNL, idle_cpu_knl), | |
1102 | INTEL_CPU_FAM6(XEON_PHI_KNM, idle_cpu_knl), | |
1103 | INTEL_CPU_FAM6(ATOM_GOLDMONT, idle_cpu_bxt), | |
c05f3642 LT |
1104 | INTEL_CPU_FAM6(ATOM_GOLDMONT_PLUS, idle_cpu_bxt), |
1105 | INTEL_CPU_FAM6(ATOM_GOLDMONT_X, idle_cpu_dnv), | |
04b1d5d0 | 1106 | INTEL_CPU_FAM6(ATOM_TREMONT_X, idle_cpu_dnv), |
b66b8b9a AK |
1107 | {} |
1108 | }; | |
b66b8b9a | 1109 | |
26717172 LB |
1110 | /* |
1111 | * intel_idle_probe() | |
1112 | */ | |
00f3e755 | 1113 | static int __init intel_idle_probe(void) |
26717172 | 1114 | { |
c4236282 | 1115 | unsigned int eax, ebx, ecx; |
b66b8b9a | 1116 | const struct x86_cpu_id *id; |
26717172 LB |
1117 | |
1118 | if (max_cstate == 0) { | |
654d08a4 | 1119 | pr_debug("disabled\n"); |
26717172 LB |
1120 | return -EPERM; |
1121 | } | |
1122 | ||
b66b8b9a AK |
1123 | id = x86_match_cpu(intel_idle_ids); |
1124 | if (!id) { | |
1125 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && | |
1126 | boot_cpu_data.x86 == 6) | |
654d08a4 JP |
1127 | pr_debug("does not run on family %d model %d\n", |
1128 | boot_cpu_data.x86, boot_cpu_data.x86_model); | |
26717172 | 1129 | return -ENODEV; |
b66b8b9a | 1130 | } |
26717172 | 1131 | |
a4c44753 LB |
1132 | if (!boot_cpu_has(X86_FEATURE_MWAIT)) { |
1133 | pr_debug("Please enable MWAIT in BIOS SETUP\n"); | |
1134 | return -ENODEV; | |
1135 | } | |
1136 | ||
26717172 LB |
1137 | if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) |
1138 | return -ENODEV; | |
1139 | ||
c4236282 | 1140 | cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); |
26717172 LB |
1141 | |
1142 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || | |
5c2a9f06 TR |
1143 | !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || |
1144 | !mwait_substates) | |
26717172 | 1145 | return -ENODEV; |
26717172 | 1146 | |
654d08a4 | 1147 | pr_debug("MWAIT substates: 0x%x\n", mwait_substates); |
26717172 | 1148 | |
b66b8b9a AK |
1149 | icpu = (const struct idle_cpu *)id->driver_data; |
1150 | cpuidle_state_table = icpu->state_table; | |
26717172 | 1151 | |
654d08a4 JP |
1152 | pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n", |
1153 | boot_cpu_data.x86_model); | |
26717172 | 1154 | |
26717172 LB |
1155 | return 0; |
1156 | } | |
1157 | ||
1158 | /* | |
1159 | * intel_idle_cpuidle_devices_uninit() | |
ca42489d | 1160 | * Unregisters the cpuidle devices. |
26717172 LB |
1161 | */ |
1162 | static void intel_idle_cpuidle_devices_uninit(void) | |
1163 | { | |
1164 | int i; | |
1165 | struct cpuidle_device *dev; | |
1166 | ||
1167 | for_each_online_cpu(i) { | |
1168 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); | |
1169 | cpuidle_unregister_device(dev); | |
1170 | } | |
26717172 | 1171 | } |
0138d8f0 LB |
1172 | |
1173 | /* | |
d70e28f5 | 1174 | * ivt_idle_state_table_update(void) |
0138d8f0 | 1175 | * |
d70e28f5 | 1176 | * Tune IVT multi-socket targets |
0138d8f0 LB |
1177 | * Assumption: num_sockets == (max_package_num + 1) |
1178 | */ | |
d70e28f5 | 1179 | static void ivt_idle_state_table_update(void) |
0138d8f0 LB |
1180 | { |
1181 | /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ | |
d70e28f5 LB |
1182 | int cpu, package_num, num_sockets = 1; |
1183 | ||
1184 | for_each_online_cpu(cpu) { | |
1185 | package_num = topology_physical_package_id(cpu); | |
1186 | if (package_num + 1 > num_sockets) { | |
1187 | num_sockets = package_num + 1; | |
1188 | ||
1189 | if (num_sockets > 4) { | |
1190 | cpuidle_state_table = ivt_cstates_8s; | |
1191 | return; | |
0138d8f0 LB |
1192 | } |
1193 | } | |
d70e28f5 LB |
1194 | } |
1195 | ||
1196 | if (num_sockets > 2) | |
1197 | cpuidle_state_table = ivt_cstates_4s; | |
1198 | ||
1199 | /* else, 1 and 2 socket systems use default ivt_cstates */ | |
1200 | } | |
5dcef694 LB |
1201 | |
1202 | /* | |
1203 | * Translate IRTL (Interrupt Response Time Limit) MSR to usec | |
1204 | */ | |
1205 | ||
1206 | static unsigned int irtl_ns_units[] = { | |
1207 | 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 }; | |
1208 | ||
1209 | static unsigned long long irtl_2_usec(unsigned long long irtl) | |
1210 | { | |
1211 | unsigned long long ns; | |
1212 | ||
3451ab3e JB |
1213 | if (!irtl) |
1214 | return 0; | |
1215 | ||
bef45096 | 1216 | ns = irtl_ns_units[(irtl >> 10) & 0x7]; |
5dcef694 LB |
1217 | |
1218 | return div64_u64((irtl & 0x3FF) * ns, 1000); | |
1219 | } | |
1220 | /* | |
1221 | * bxt_idle_state_table_update(void) | |
1222 | * | |
1223 | * On BXT, we trust the IRTL to show the definitive maximum latency | |
1224 | * We use the same value for target_residency. | |
1225 | */ | |
1226 | static void bxt_idle_state_table_update(void) | |
1227 | { | |
1228 | unsigned long long msr; | |
3451ab3e | 1229 | unsigned int usec; |
5dcef694 LB |
1230 | |
1231 | rdmsrl(MSR_PKGC6_IRTL, msr); | |
3451ab3e JB |
1232 | usec = irtl_2_usec(msr); |
1233 | if (usec) { | |
5dcef694 LB |
1234 | bxt_cstates[2].exit_latency = usec; |
1235 | bxt_cstates[2].target_residency = usec; | |
1236 | } | |
1237 | ||
1238 | rdmsrl(MSR_PKGC7_IRTL, msr); | |
3451ab3e JB |
1239 | usec = irtl_2_usec(msr); |
1240 | if (usec) { | |
5dcef694 LB |
1241 | bxt_cstates[3].exit_latency = usec; |
1242 | bxt_cstates[3].target_residency = usec; | |
1243 | } | |
1244 | ||
1245 | rdmsrl(MSR_PKGC8_IRTL, msr); | |
3451ab3e JB |
1246 | usec = irtl_2_usec(msr); |
1247 | if (usec) { | |
5dcef694 LB |
1248 | bxt_cstates[4].exit_latency = usec; |
1249 | bxt_cstates[4].target_residency = usec; | |
1250 | } | |
1251 | ||
1252 | rdmsrl(MSR_PKGC9_IRTL, msr); | |
3451ab3e JB |
1253 | usec = irtl_2_usec(msr); |
1254 | if (usec) { | |
5dcef694 LB |
1255 | bxt_cstates[5].exit_latency = usec; |
1256 | bxt_cstates[5].target_residency = usec; | |
1257 | } | |
1258 | ||
1259 | rdmsrl(MSR_PKGC10_IRTL, msr); | |
3451ab3e JB |
1260 | usec = irtl_2_usec(msr); |
1261 | if (usec) { | |
5dcef694 LB |
1262 | bxt_cstates[6].exit_latency = usec; |
1263 | bxt_cstates[6].target_residency = usec; | |
1264 | } | |
1265 | ||
1266 | } | |
d70e28f5 LB |
1267 | /* |
1268 | * sklh_idle_state_table_update(void) | |
1269 | * | |
1270 | * On SKL-H (model 0x5e) disable C8 and C9 if: | |
1271 | * C10 is enabled and SGX disabled | |
1272 | */ | |
1273 | static void sklh_idle_state_table_update(void) | |
1274 | { | |
1275 | unsigned long long msr; | |
1276 | unsigned int eax, ebx, ecx, edx; | |
1277 | ||
1278 | ||
1279 | /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ | |
1280 | if (max_cstate <= 7) | |
1281 | return; | |
1282 | ||
1283 | /* if PC10 not present in CPUID.MWAIT.EDX */ | |
1284 | if ((mwait_substates & (0xF << 28)) == 0) | |
1285 | return; | |
1286 | ||
6cfb2374 | 1287 | rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr); |
d70e28f5 LB |
1288 | |
1289 | /* PC10 is not enabled in PKG C-state limit */ | |
1290 | if ((msr & 0xF) != 8) | |
1291 | return; | |
1292 | ||
1293 | ecx = 0; | |
1294 | cpuid(7, &eax, &ebx, &ecx, &edx); | |
1295 | ||
1296 | /* if SGX is present */ | |
1297 | if (ebx & (1 << 2)) { | |
0138d8f0 | 1298 | |
d70e28f5 LB |
1299 | rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); |
1300 | ||
1301 | /* if SGX is enabled */ | |
1302 | if (msr & (1 << 18)) | |
1303 | return; | |
1304 | } | |
1305 | ||
1306 | skl_cstates[5].disabled = 1; /* C8-SKL */ | |
1307 | skl_cstates[6].disabled = 1; /* C9-SKL */ | |
1308 | } | |
1309 | /* | |
1310 | * intel_idle_state_table_update() | |
1311 | * | |
1312 | * Update the default state_table for this CPU-id | |
1313 | */ | |
1314 | ||
1315 | static void intel_idle_state_table_update(void) | |
1316 | { | |
1317 | switch (boot_cpu_data.x86_model) { | |
1318 | ||
db73c5a8 | 1319 | case INTEL_FAM6_IVYBRIDGE_X: |
d70e28f5 LB |
1320 | ivt_idle_state_table_update(); |
1321 | break; | |
db73c5a8 | 1322 | case INTEL_FAM6_ATOM_GOLDMONT: |
f2c4db1b | 1323 | case INTEL_FAM6_ATOM_GOLDMONT_PLUS: |
5dcef694 LB |
1324 | bxt_idle_state_table_update(); |
1325 | break; | |
db73c5a8 | 1326 | case INTEL_FAM6_SKYLAKE_DESKTOP: |
d70e28f5 LB |
1327 | sklh_idle_state_table_update(); |
1328 | break; | |
0138d8f0 | 1329 | } |
0138d8f0 LB |
1330 | } |
1331 | ||
46bcfad7 DD |
1332 | /* |
1333 | * intel_idle_cpuidle_driver_init() | |
1334 | * allocate, initialize cpuidle_states | |
1335 | */ | |
5469c827 | 1336 | static void __init intel_idle_cpuidle_driver_init(void) |
46bcfad7 DD |
1337 | { |
1338 | int cstate; | |
1339 | struct cpuidle_driver *drv = &intel_idle_driver; | |
1340 | ||
0138d8f0 LB |
1341 | intel_idle_state_table_update(); |
1342 | ||
1b39e3f8 | 1343 | cpuidle_poll_state_init(drv); |
46bcfad7 DD |
1344 | drv->state_count = 1; |
1345 | ||
e022e7eb | 1346 | for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { |
24bfa950 | 1347 | int num_substates, mwait_hint, mwait_cstate; |
46bcfad7 | 1348 | |
7dd0e0af | 1349 | if ((cpuidle_state_table[cstate].enter == NULL) && |
28ba086e | 1350 | (cpuidle_state_table[cstate].enter_s2idle == NULL)) |
e022e7eb LB |
1351 | break; |
1352 | ||
1353 | if (cstate + 1 > max_cstate) { | |
654d08a4 | 1354 | pr_info("max_cstate %d reached\n", max_cstate); |
46bcfad7 DD |
1355 | break; |
1356 | } | |
1357 | ||
e022e7eb LB |
1358 | mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); |
1359 | mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); | |
e022e7eb | 1360 | |
24bfa950 | 1361 | /* number of sub-states for this state in CPUID.MWAIT */ |
e022e7eb | 1362 | num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) |
46bcfad7 | 1363 | & MWAIT_SUBSTATE_MASK; |
e022e7eb | 1364 | |
24bfa950 LB |
1365 | /* if NO sub-states for this state in CPUID, skip it */ |
1366 | if (num_substates == 0) | |
46bcfad7 | 1367 | continue; |
46bcfad7 | 1368 | |
d70e28f5 LB |
1369 | /* if state marked as disabled, skip it */ |
1370 | if (cpuidle_state_table[cstate].disabled != 0) { | |
654d08a4 JP |
1371 | pr_debug("state %s is disabled\n", |
1372 | cpuidle_state_table[cstate].name); | |
d70e28f5 LB |
1373 | continue; |
1374 | } | |
1375 | ||
1376 | ||
e022e7eb | 1377 | if (((mwait_cstate + 1) > 2) && |
46bcfad7 DD |
1378 | !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) |
1379 | mark_tsc_unstable("TSC halts in idle" | |
1380 | " states deeper than C2"); | |
1381 | ||
1382 | drv->states[drv->state_count] = /* structure copy */ | |
1383 | cpuidle_state_table[cstate]; | |
1384 | ||
1385 | drv->state_count += 1; | |
1386 | } | |
1387 | ||
8c058d53 LB |
1388 | if (icpu->byt_auto_demotion_disable_flag) { |
1389 | wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); | |
1390 | wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); | |
1391 | } | |
46bcfad7 DD |
1392 | } |
1393 | ||
1394 | ||
26717172 | 1395 | /* |
65b7f839 | 1396 | * intel_idle_cpu_init() |
26717172 | 1397 | * allocate, initialize, register cpuidle_devices |
65b7f839 | 1398 | * @cpu: cpu/core to initialize |
26717172 | 1399 | */ |
fb1013a0 | 1400 | static int intel_idle_cpu_init(unsigned int cpu) |
26717172 | 1401 | { |
26717172 LB |
1402 | struct cpuidle_device *dev; |
1403 | ||
65b7f839 | 1404 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); |
65b7f839 | 1405 | dev->cpu = cpu; |
26717172 | 1406 | |
65b7f839 | 1407 | if (cpuidle_register_device(dev)) { |
654d08a4 | 1408 | pr_debug("cpuidle_register_device %d failed!\n", cpu); |
65b7f839 | 1409 | return -EIO; |
26717172 LB |
1410 | } |
1411 | ||
b66b8b9a | 1412 | if (icpu->auto_demotion_disable_flags) |
fb1013a0 | 1413 | auto_demotion_disable(); |
65b7f839 | 1414 | |
dbf87ab8 | 1415 | if (icpu->disable_promotion_to_c1e) |
fb1013a0 SAS |
1416 | c1e_promotion_disable(); |
1417 | ||
1418 | return 0; | |
1419 | } | |
1420 | ||
1421 | static int intel_idle_cpu_online(unsigned int cpu) | |
1422 | { | |
1423 | struct cpuidle_device *dev; | |
1424 | ||
1425 | if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) | |
1426 | __setup_broadcast_timer(true); | |
1427 | ||
1428 | /* | |
1429 | * Some systems can hotplug a cpu at runtime after | |
1430 | * the kernel has booted, we have to initialize the | |
1431 | * driver in this case | |
1432 | */ | |
1433 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); | |
1434 | if (!dev->registered) | |
1435 | return intel_idle_cpu_init(cpu); | |
dbf87ab8 | 1436 | |
26717172 LB |
1437 | return 0; |
1438 | } | |
26717172 LB |
1439 | |
1440 | static int __init intel_idle_init(void) | |
1441 | { | |
fb1013a0 | 1442 | int retval; |
26717172 | 1443 | |
d1896049 TR |
1444 | /* Do not load intel_idle at all for now if idle= is passed */ |
1445 | if (boot_option_idle_override != IDLE_NO_OVERRIDE) | |
1446 | return -ENODEV; | |
1447 | ||
26717172 LB |
1448 | retval = intel_idle_probe(); |
1449 | if (retval) | |
1450 | return retval; | |
1451 | ||
e9df69cc RC |
1452 | intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); |
1453 | if (intel_idle_cpuidle_devices == NULL) | |
1454 | return -ENOMEM; | |
1455 | ||
46bcfad7 | 1456 | intel_idle_cpuidle_driver_init(); |
26717172 LB |
1457 | retval = cpuidle_register_driver(&intel_idle_driver); |
1458 | if (retval) { | |
3735d524 | 1459 | struct cpuidle_driver *drv = cpuidle_get_driver(); |
654d08a4 JP |
1460 | printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"), |
1461 | drv ? drv->name : "none"); | |
fb1013a0 | 1462 | goto init_driver_fail; |
26717172 LB |
1463 | } |
1464 | ||
2259a819 RC |
1465 | if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ |
1466 | lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; | |
2259a819 | 1467 | |
fb1013a0 SAS |
1468 | retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", |
1469 | intel_idle_cpu_online, NULL); | |
1470 | if (retval < 0) | |
1471 | goto hp_setup_fail; | |
26717172 | 1472 | |
654d08a4 JP |
1473 | pr_debug("lapic_timer_reliable_states 0x%x\n", |
1474 | lapic_timer_reliable_states); | |
2259a819 | 1475 | |
26717172 | 1476 | return 0; |
fb1013a0 SAS |
1477 | |
1478 | hp_setup_fail: | |
1479 | intel_idle_cpuidle_devices_uninit(); | |
1480 | cpuidle_unregister_driver(&intel_idle_driver); | |
1481 | init_driver_fail: | |
1482 | free_percpu(intel_idle_cpuidle_devices); | |
1483 | return retval; | |
1484 | ||
26717172 | 1485 | } |
02c4fae9 | 1486 | device_initcall(intel_idle_init); |
26717172 | 1487 | |
02c4fae9 PG |
1488 | /* |
1489 | * We are not really modular, but we used to support that. Meaning we also | |
1490 | * support "intel_idle.max_cstate=..." at boot and also a read-only export of | |
1491 | * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param | |
1492 | * is the easiest way (currently) to continue doing that. | |
1493 | */ | |
26717172 | 1494 | module_param(max_cstate, int, 0444); |