Commit | Line | Data |
---|---|---|
26717172 LB |
1 | /* |
2 | * intel_idle.c - native hardware idle loop for modern Intel processors | |
3 | * | |
fab04b22 | 4 | * Copyright (c) 2013, Intel Corporation. |
26717172 LB |
5 | * Len Brown <len.brown@intel.com> |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify it | |
8 | * under the terms and conditions of the GNU General Public License, | |
9 | * version 2, as published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope it will be useful, but WITHOUT | |
12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
14 | * more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License along with | |
17 | * this program; if not, write to the Free Software Foundation, Inc., | |
18 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
19 | */ | |
20 | ||
21 | /* | |
22 | * intel_idle is a cpuidle driver that loads on specific Intel processors | |
23 | * in lieu of the legacy ACPI processor_idle driver. The intent is to | |
24 | * make Linux more efficient on these processors, as intel_idle knows | |
25 | * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. | |
26 | */ | |
27 | ||
28 | /* | |
29 | * Design Assumptions | |
30 | * | |
31 | * All CPUs have same idle states as boot CPU | |
32 | * | |
33 | * Chipset BM_STS (bus master status) bit is a NOP | |
34 | * for preventing entry into deep C-stats | |
35 | */ | |
36 | ||
37 | /* | |
38 | * Known limitations | |
39 | * | |
40 | * The driver currently initializes for_each_online_cpu() upon modprobe. | |
41 | * It it unaware of subsequent processors hot-added to the system. | |
42 | * This means that if you boot with maxcpus=n and later online | |
43 | * processors above n, those processors will use C1 only. | |
44 | * | |
45 | * ACPI has a .suspend hack to turn off deep c-statees during suspend | |
46 | * to avoid complications with the lapic timer workaround. | |
47 | * Have not seen issues with suspend, but may need same workaround here. | |
48 | * | |
26717172 LB |
49 | */ |
50 | ||
51 | /* un-comment DEBUG to enable pr_debug() statements */ | |
52 | #define DEBUG | |
53 | ||
54 | #include <linux/kernel.h> | |
55 | #include <linux/cpuidle.h> | |
76962caa | 56 | #include <linux/tick.h> |
26717172 LB |
57 | #include <trace/events/power.h> |
58 | #include <linux/sched.h> | |
2a2d31c8 SL |
59 | #include <linux/notifier.h> |
60 | #include <linux/cpu.h> | |
02c4fae9 | 61 | #include <linux/moduleparam.h> |
b66b8b9a | 62 | #include <asm/cpu_device_id.h> |
db73c5a8 | 63 | #include <asm/intel-family.h> |
bc83cccc | 64 | #include <asm/mwait.h> |
14796fca | 65 | #include <asm/msr.h> |
26717172 | 66 | |
d70e28f5 | 67 | #define INTEL_IDLE_VERSION "0.4.1" |
26717172 LB |
68 | #define PREFIX "intel_idle: " |
69 | ||
26717172 LB |
70 | static struct cpuidle_driver intel_idle_driver = { |
71 | .name = "intel_idle", | |
72 | .owner = THIS_MODULE, | |
73 | }; | |
74 | /* intel_idle.max_cstate=0 disables driver */ | |
137ecc77 | 75 | static int max_cstate = CPUIDLE_STATE_MAX - 1; |
26717172 | 76 | |
c4236282 | 77 | static unsigned int mwait_substates; |
26717172 | 78 | |
2a2d31c8 | 79 | #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF |
26717172 | 80 | /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ |
d13780d4 | 81 | static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ |
26717172 | 82 | |
b66b8b9a AK |
83 | struct idle_cpu { |
84 | struct cpuidle_state *state_table; | |
85 | ||
86 | /* | |
87 | * Hardware C-state auto-demotion may not always be optimal. | |
88 | * Indicate which enable bits to clear here. | |
89 | */ | |
90 | unsigned long auto_demotion_disable_flags; | |
8c058d53 | 91 | bool byt_auto_demotion_disable_flag; |
32e95180 | 92 | bool disable_promotion_to_c1e; |
b66b8b9a AK |
93 | }; |
94 | ||
95 | static const struct idle_cpu *icpu; | |
3265eba0 | 96 | static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; |
46bcfad7 DD |
97 | static int intel_idle(struct cpuidle_device *dev, |
98 | struct cpuidle_driver *drv, int index); | |
5fe2e527 RW |
99 | static void intel_idle_freeze(struct cpuidle_device *dev, |
100 | struct cpuidle_driver *drv, int index); | |
25ac7761 | 101 | static int intel_idle_cpu_init(int cpu); |
26717172 LB |
102 | |
103 | static struct cpuidle_state *cpuidle_state_table; | |
104 | ||
956d033f LB |
105 | /* |
106 | * Set this flag for states where the HW flushes the TLB for us | |
107 | * and so we don't need cross-calls to keep it consistent. | |
108 | * If this flag is set, SW flushes the TLB, so even if the | |
109 | * HW doesn't do the flushing, this flag is safe to use. | |
110 | */ | |
111 | #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 | |
112 | ||
b1beab48 LB |
113 | /* |
114 | * MWAIT takes an 8-bit "hint" in EAX "suggesting" | |
115 | * the C-state (top nibble) and sub-state (bottom nibble) | |
116 | * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. | |
117 | * | |
118 | * We store the hint at the top of our "flags" for each state. | |
119 | */ | |
120 | #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) | |
121 | #define MWAIT2flg(eax) ((eax & 0xFF) << 24) | |
122 | ||
26717172 LB |
123 | /* |
124 | * States are indexed by the cstate number, | |
125 | * which is also the index into the MWAIT hint array. | |
126 | * Thus C0 is a dummy. | |
127 | */ | |
ba0dc81e | 128 | static struct cpuidle_state nehalem_cstates[] = { |
e022e7eb | 129 | { |
15e123e5 | 130 | .name = "C1-NHM", |
26717172 | 131 | .desc = "MWAIT 0x00", |
b82b6cca | 132 | .flags = MWAIT2flg(0x00), |
26717172 | 133 | .exit_latency = 3, |
26717172 | 134 | .target_residency = 6, |
5fe2e527 RW |
135 | .enter = &intel_idle, |
136 | .enter_freeze = intel_idle_freeze, }, | |
32e95180 LB |
137 | { |
138 | .name = "C1E-NHM", | |
139 | .desc = "MWAIT 0x01", | |
b82b6cca | 140 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
141 | .exit_latency = 10, |
142 | .target_residency = 20, | |
5fe2e527 RW |
143 | .enter = &intel_idle, |
144 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 145 | { |
15e123e5 | 146 | .name = "C3-NHM", |
26717172 | 147 | .desc = "MWAIT 0x10", |
b82b6cca | 148 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 149 | .exit_latency = 20, |
26717172 | 150 | .target_residency = 80, |
5fe2e527 RW |
151 | .enter = &intel_idle, |
152 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 153 | { |
15e123e5 | 154 | .name = "C6-NHM", |
26717172 | 155 | .desc = "MWAIT 0x20", |
b82b6cca | 156 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 157 | .exit_latency = 200, |
26717172 | 158 | .target_residency = 800, |
5fe2e527 RW |
159 | .enter = &intel_idle, |
160 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
161 | { |
162 | .enter = NULL } | |
26717172 LB |
163 | }; |
164 | ||
ba0dc81e | 165 | static struct cpuidle_state snb_cstates[] = { |
e022e7eb | 166 | { |
15e123e5 | 167 | .name = "C1-SNB", |
d13780d4 | 168 | .desc = "MWAIT 0x00", |
b82b6cca | 169 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
170 | .exit_latency = 2, |
171 | .target_residency = 2, | |
5fe2e527 RW |
172 | .enter = &intel_idle, |
173 | .enter_freeze = intel_idle_freeze, }, | |
32e95180 LB |
174 | { |
175 | .name = "C1E-SNB", | |
176 | .desc = "MWAIT 0x01", | |
b82b6cca | 177 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
178 | .exit_latency = 10, |
179 | .target_residency = 20, | |
5fe2e527 RW |
180 | .enter = &intel_idle, |
181 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 182 | { |
15e123e5 | 183 | .name = "C3-SNB", |
d13780d4 | 184 | .desc = "MWAIT 0x10", |
b82b6cca | 185 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 186 | .exit_latency = 80, |
ddbd550d | 187 | .target_residency = 211, |
5fe2e527 RW |
188 | .enter = &intel_idle, |
189 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 190 | { |
15e123e5 | 191 | .name = "C6-SNB", |
d13780d4 | 192 | .desc = "MWAIT 0x20", |
b82b6cca | 193 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 194 | .exit_latency = 104, |
ddbd550d | 195 | .target_residency = 345, |
5fe2e527 RW |
196 | .enter = &intel_idle, |
197 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 198 | { |
15e123e5 | 199 | .name = "C7-SNB", |
d13780d4 | 200 | .desc = "MWAIT 0x30", |
b82b6cca | 201 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 202 | .exit_latency = 109, |
ddbd550d | 203 | .target_residency = 345, |
5fe2e527 RW |
204 | .enter = &intel_idle, |
205 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
206 | { |
207 | .enter = NULL } | |
d13780d4 LB |
208 | }; |
209 | ||
718987d6 LB |
210 | static struct cpuidle_state byt_cstates[] = { |
211 | { | |
212 | .name = "C1-BYT", | |
213 | .desc = "MWAIT 0x00", | |
b82b6cca | 214 | .flags = MWAIT2flg(0x00), |
718987d6 LB |
215 | .exit_latency = 1, |
216 | .target_residency = 1, | |
5fe2e527 RW |
217 | .enter = &intel_idle, |
218 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
219 | { |
220 | .name = "C6N-BYT", | |
221 | .desc = "MWAIT 0x58", | |
b82b6cca | 222 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 223 | .exit_latency = 300, |
718987d6 | 224 | .target_residency = 275, |
5fe2e527 RW |
225 | .enter = &intel_idle, |
226 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
227 | { |
228 | .name = "C6S-BYT", | |
229 | .desc = "MWAIT 0x52", | |
b82b6cca | 230 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
d7ef7671 | 231 | .exit_latency = 500, |
718987d6 | 232 | .target_residency = 560, |
5fe2e527 RW |
233 | .enter = &intel_idle, |
234 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
235 | { |
236 | .name = "C7-BYT", | |
237 | .desc = "MWAIT 0x60", | |
b82b6cca | 238 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 | 239 | .exit_latency = 1200, |
d7ef7671 | 240 | .target_residency = 4000, |
5fe2e527 RW |
241 | .enter = &intel_idle, |
242 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
243 | { |
244 | .name = "C7S-BYT", | |
245 | .desc = "MWAIT 0x64", | |
b82b6cca | 246 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, |
718987d6 LB |
247 | .exit_latency = 10000, |
248 | .target_residency = 20000, | |
5fe2e527 RW |
249 | .enter = &intel_idle, |
250 | .enter_freeze = intel_idle_freeze, }, | |
718987d6 LB |
251 | { |
252 | .enter = NULL } | |
253 | }; | |
254 | ||
cab07a56 LB |
255 | static struct cpuidle_state cht_cstates[] = { |
256 | { | |
257 | .name = "C1-CHT", | |
258 | .desc = "MWAIT 0x00", | |
259 | .flags = MWAIT2flg(0x00), | |
260 | .exit_latency = 1, | |
261 | .target_residency = 1, | |
262 | .enter = &intel_idle, | |
263 | .enter_freeze = intel_idle_freeze, }, | |
264 | { | |
265 | .name = "C6N-CHT", | |
266 | .desc = "MWAIT 0x58", | |
267 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED, | |
268 | .exit_latency = 80, | |
269 | .target_residency = 275, | |
270 | .enter = &intel_idle, | |
271 | .enter_freeze = intel_idle_freeze, }, | |
272 | { | |
273 | .name = "C6S-CHT", | |
274 | .desc = "MWAIT 0x52", | |
275 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, | |
276 | .exit_latency = 200, | |
277 | .target_residency = 560, | |
278 | .enter = &intel_idle, | |
279 | .enter_freeze = intel_idle_freeze, }, | |
280 | { | |
281 | .name = "C7-CHT", | |
282 | .desc = "MWAIT 0x60", | |
283 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
284 | .exit_latency = 1200, | |
285 | .target_residency = 4000, | |
286 | .enter = &intel_idle, | |
287 | .enter_freeze = intel_idle_freeze, }, | |
288 | { | |
289 | .name = "C7S-CHT", | |
290 | .desc = "MWAIT 0x64", | |
291 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, | |
292 | .exit_latency = 10000, | |
293 | .target_residency = 20000, | |
294 | .enter = &intel_idle, | |
295 | .enter_freeze = intel_idle_freeze, }, | |
296 | { | |
297 | .enter = NULL } | |
298 | }; | |
299 | ||
ba0dc81e | 300 | static struct cpuidle_state ivb_cstates[] = { |
e022e7eb | 301 | { |
6edab08c LB |
302 | .name = "C1-IVB", |
303 | .desc = "MWAIT 0x00", | |
b82b6cca | 304 | .flags = MWAIT2flg(0x00), |
6edab08c LB |
305 | .exit_latency = 1, |
306 | .target_residency = 1, | |
5fe2e527 RW |
307 | .enter = &intel_idle, |
308 | .enter_freeze = intel_idle_freeze, }, | |
32e95180 LB |
309 | { |
310 | .name = "C1E-IVB", | |
311 | .desc = "MWAIT 0x01", | |
b82b6cca | 312 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
313 | .exit_latency = 10, |
314 | .target_residency = 20, | |
5fe2e527 RW |
315 | .enter = &intel_idle, |
316 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 317 | { |
6edab08c LB |
318 | .name = "C3-IVB", |
319 | .desc = "MWAIT 0x10", | |
b82b6cca | 320 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
321 | .exit_latency = 59, |
322 | .target_residency = 156, | |
5fe2e527 RW |
323 | .enter = &intel_idle, |
324 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 325 | { |
6edab08c LB |
326 | .name = "C6-IVB", |
327 | .desc = "MWAIT 0x20", | |
b82b6cca | 328 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
329 | .exit_latency = 80, |
330 | .target_residency = 300, | |
5fe2e527 RW |
331 | .enter = &intel_idle, |
332 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 333 | { |
6edab08c LB |
334 | .name = "C7-IVB", |
335 | .desc = "MWAIT 0x30", | |
b82b6cca | 336 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
337 | .exit_latency = 87, |
338 | .target_residency = 300, | |
5fe2e527 RW |
339 | .enter = &intel_idle, |
340 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
341 | { |
342 | .enter = NULL } | |
6edab08c LB |
343 | }; |
344 | ||
0138d8f0 LB |
345 | static struct cpuidle_state ivt_cstates[] = { |
346 | { | |
347 | .name = "C1-IVT", | |
348 | .desc = "MWAIT 0x00", | |
b82b6cca | 349 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
350 | .exit_latency = 1, |
351 | .target_residency = 1, | |
5fe2e527 RW |
352 | .enter = &intel_idle, |
353 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
354 | { |
355 | .name = "C1E-IVT", | |
356 | .desc = "MWAIT 0x01", | |
b82b6cca | 357 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
358 | .exit_latency = 10, |
359 | .target_residency = 80, | |
5fe2e527 RW |
360 | .enter = &intel_idle, |
361 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
362 | { |
363 | .name = "C3-IVT", | |
364 | .desc = "MWAIT 0x10", | |
b82b6cca | 365 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
366 | .exit_latency = 59, |
367 | .target_residency = 156, | |
5fe2e527 RW |
368 | .enter = &intel_idle, |
369 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
370 | { |
371 | .name = "C6-IVT", | |
372 | .desc = "MWAIT 0x20", | |
b82b6cca | 373 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
374 | .exit_latency = 82, |
375 | .target_residency = 300, | |
5fe2e527 RW |
376 | .enter = &intel_idle, |
377 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
378 | { |
379 | .enter = NULL } | |
380 | }; | |
381 | ||
382 | static struct cpuidle_state ivt_cstates_4s[] = { | |
383 | { | |
384 | .name = "C1-IVT-4S", | |
385 | .desc = "MWAIT 0x00", | |
b82b6cca | 386 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
387 | .exit_latency = 1, |
388 | .target_residency = 1, | |
5fe2e527 RW |
389 | .enter = &intel_idle, |
390 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
391 | { |
392 | .name = "C1E-IVT-4S", | |
393 | .desc = "MWAIT 0x01", | |
b82b6cca | 394 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
395 | .exit_latency = 10, |
396 | .target_residency = 250, | |
5fe2e527 RW |
397 | .enter = &intel_idle, |
398 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
399 | { |
400 | .name = "C3-IVT-4S", | |
401 | .desc = "MWAIT 0x10", | |
b82b6cca | 402 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
403 | .exit_latency = 59, |
404 | .target_residency = 300, | |
5fe2e527 RW |
405 | .enter = &intel_idle, |
406 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
407 | { |
408 | .name = "C6-IVT-4S", | |
409 | .desc = "MWAIT 0x20", | |
b82b6cca | 410 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
411 | .exit_latency = 84, |
412 | .target_residency = 400, | |
5fe2e527 RW |
413 | .enter = &intel_idle, |
414 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
415 | { |
416 | .enter = NULL } | |
417 | }; | |
418 | ||
419 | static struct cpuidle_state ivt_cstates_8s[] = { | |
420 | { | |
421 | .name = "C1-IVT-8S", | |
422 | .desc = "MWAIT 0x00", | |
b82b6cca | 423 | .flags = MWAIT2flg(0x00), |
0138d8f0 LB |
424 | .exit_latency = 1, |
425 | .target_residency = 1, | |
5fe2e527 RW |
426 | .enter = &intel_idle, |
427 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
428 | { |
429 | .name = "C1E-IVT-8S", | |
430 | .desc = "MWAIT 0x01", | |
b82b6cca | 431 | .flags = MWAIT2flg(0x01), |
0138d8f0 LB |
432 | .exit_latency = 10, |
433 | .target_residency = 500, | |
5fe2e527 RW |
434 | .enter = &intel_idle, |
435 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
436 | { |
437 | .name = "C3-IVT-8S", | |
438 | .desc = "MWAIT 0x10", | |
b82b6cca | 439 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
440 | .exit_latency = 59, |
441 | .target_residency = 600, | |
5fe2e527 RW |
442 | .enter = &intel_idle, |
443 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
444 | { |
445 | .name = "C6-IVT-8S", | |
446 | .desc = "MWAIT 0x20", | |
b82b6cca | 447 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
0138d8f0 LB |
448 | .exit_latency = 88, |
449 | .target_residency = 700, | |
5fe2e527 RW |
450 | .enter = &intel_idle, |
451 | .enter_freeze = intel_idle_freeze, }, | |
0138d8f0 LB |
452 | { |
453 | .enter = NULL } | |
454 | }; | |
455 | ||
ba0dc81e | 456 | static struct cpuidle_state hsw_cstates[] = { |
e022e7eb | 457 | { |
85a4d2d4 LB |
458 | .name = "C1-HSW", |
459 | .desc = "MWAIT 0x00", | |
b82b6cca | 460 | .flags = MWAIT2flg(0x00), |
85a4d2d4 LB |
461 | .exit_latency = 2, |
462 | .target_residency = 2, | |
5fe2e527 RW |
463 | .enter = &intel_idle, |
464 | .enter_freeze = intel_idle_freeze, }, | |
32e95180 LB |
465 | { |
466 | .name = "C1E-HSW", | |
467 | .desc = "MWAIT 0x01", | |
b82b6cca | 468 | .flags = MWAIT2flg(0x01), |
32e95180 LB |
469 | .exit_latency = 10, |
470 | .target_residency = 20, | |
5fe2e527 RW |
471 | .enter = &intel_idle, |
472 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 473 | { |
85a4d2d4 LB |
474 | .name = "C3-HSW", |
475 | .desc = "MWAIT 0x10", | |
b82b6cca | 476 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
477 | .exit_latency = 33, |
478 | .target_residency = 100, | |
5fe2e527 RW |
479 | .enter = &intel_idle, |
480 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 481 | { |
85a4d2d4 LB |
482 | .name = "C6-HSW", |
483 | .desc = "MWAIT 0x20", | |
b82b6cca | 484 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
485 | .exit_latency = 133, |
486 | .target_residency = 400, | |
5fe2e527 RW |
487 | .enter = &intel_idle, |
488 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 489 | { |
85a4d2d4 LB |
490 | .name = "C7s-HSW", |
491 | .desc = "MWAIT 0x32", | |
b82b6cca | 492 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
85a4d2d4 LB |
493 | .exit_latency = 166, |
494 | .target_residency = 500, | |
5fe2e527 RW |
495 | .enter = &intel_idle, |
496 | .enter_freeze = intel_idle_freeze, }, | |
86239ceb LB |
497 | { |
498 | .name = "C8-HSW", | |
499 | .desc = "MWAIT 0x40", | |
b82b6cca | 500 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
501 | .exit_latency = 300, |
502 | .target_residency = 900, | |
5fe2e527 RW |
503 | .enter = &intel_idle, |
504 | .enter_freeze = intel_idle_freeze, }, | |
86239ceb LB |
505 | { |
506 | .name = "C9-HSW", | |
507 | .desc = "MWAIT 0x50", | |
b82b6cca | 508 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
509 | .exit_latency = 600, |
510 | .target_residency = 1800, | |
5fe2e527 RW |
511 | .enter = &intel_idle, |
512 | .enter_freeze = intel_idle_freeze, }, | |
86239ceb LB |
513 | { |
514 | .name = "C10-HSW", | |
515 | .desc = "MWAIT 0x60", | |
b82b6cca | 516 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
86239ceb LB |
517 | .exit_latency = 2600, |
518 | .target_residency = 7700, | |
5fe2e527 RW |
519 | .enter = &intel_idle, |
520 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
521 | { |
522 | .enter = NULL } | |
85a4d2d4 | 523 | }; |
a138b568 LB |
524 | static struct cpuidle_state bdw_cstates[] = { |
525 | { | |
526 | .name = "C1-BDW", | |
527 | .desc = "MWAIT 0x00", | |
b82b6cca | 528 | .flags = MWAIT2flg(0x00), |
a138b568 LB |
529 | .exit_latency = 2, |
530 | .target_residency = 2, | |
5fe2e527 RW |
531 | .enter = &intel_idle, |
532 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
533 | { |
534 | .name = "C1E-BDW", | |
535 | .desc = "MWAIT 0x01", | |
b82b6cca | 536 | .flags = MWAIT2flg(0x01), |
a138b568 LB |
537 | .exit_latency = 10, |
538 | .target_residency = 20, | |
5fe2e527 RW |
539 | .enter = &intel_idle, |
540 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
541 | { |
542 | .name = "C3-BDW", | |
543 | .desc = "MWAIT 0x10", | |
b82b6cca | 544 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
545 | .exit_latency = 40, |
546 | .target_residency = 100, | |
5fe2e527 RW |
547 | .enter = &intel_idle, |
548 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
549 | { |
550 | .name = "C6-BDW", | |
551 | .desc = "MWAIT 0x20", | |
b82b6cca | 552 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
553 | .exit_latency = 133, |
554 | .target_residency = 400, | |
5fe2e527 RW |
555 | .enter = &intel_idle, |
556 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
557 | { |
558 | .name = "C7s-BDW", | |
559 | .desc = "MWAIT 0x32", | |
b82b6cca | 560 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
561 | .exit_latency = 166, |
562 | .target_residency = 500, | |
5fe2e527 RW |
563 | .enter = &intel_idle, |
564 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
565 | { |
566 | .name = "C8-BDW", | |
567 | .desc = "MWAIT 0x40", | |
b82b6cca | 568 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
569 | .exit_latency = 300, |
570 | .target_residency = 900, | |
5fe2e527 RW |
571 | .enter = &intel_idle, |
572 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
573 | { |
574 | .name = "C9-BDW", | |
575 | .desc = "MWAIT 0x50", | |
b82b6cca | 576 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
577 | .exit_latency = 600, |
578 | .target_residency = 1800, | |
5fe2e527 RW |
579 | .enter = &intel_idle, |
580 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
581 | { |
582 | .name = "C10-BDW", | |
583 | .desc = "MWAIT 0x60", | |
b82b6cca | 584 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, |
a138b568 LB |
585 | .exit_latency = 2600, |
586 | .target_residency = 7700, | |
5fe2e527 RW |
587 | .enter = &intel_idle, |
588 | .enter_freeze = intel_idle_freeze, }, | |
a138b568 LB |
589 | { |
590 | .enter = NULL } | |
591 | }; | |
85a4d2d4 | 592 | |
493f133f LB |
593 | static struct cpuidle_state skl_cstates[] = { |
594 | { | |
595 | .name = "C1-SKL", | |
596 | .desc = "MWAIT 0x00", | |
597 | .flags = MWAIT2flg(0x00), | |
598 | .exit_latency = 2, | |
599 | .target_residency = 2, | |
600 | .enter = &intel_idle, | |
601 | .enter_freeze = intel_idle_freeze, }, | |
602 | { | |
603 | .name = "C1E-SKL", | |
604 | .desc = "MWAIT 0x01", | |
605 | .flags = MWAIT2flg(0x01), | |
606 | .exit_latency = 10, | |
607 | .target_residency = 20, | |
608 | .enter = &intel_idle, | |
609 | .enter_freeze = intel_idle_freeze, }, | |
610 | { | |
611 | .name = "C3-SKL", | |
612 | .desc = "MWAIT 0x10", | |
613 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
614 | .exit_latency = 70, | |
615 | .target_residency = 100, | |
616 | .enter = &intel_idle, | |
617 | .enter_freeze = intel_idle_freeze, }, | |
618 | { | |
619 | .name = "C6-SKL", | |
620 | .desc = "MWAIT 0x20", | |
621 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
135919a3 | 622 | .exit_latency = 85, |
493f133f LB |
623 | .target_residency = 200, |
624 | .enter = &intel_idle, | |
625 | .enter_freeze = intel_idle_freeze, }, | |
626 | { | |
627 | .name = "C7s-SKL", | |
628 | .desc = "MWAIT 0x33", | |
629 | .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED, | |
630 | .exit_latency = 124, | |
631 | .target_residency = 800, | |
632 | .enter = &intel_idle, | |
633 | .enter_freeze = intel_idle_freeze, }, | |
634 | { | |
635 | .name = "C8-SKL", | |
636 | .desc = "MWAIT 0x40", | |
637 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
135919a3 | 638 | .exit_latency = 200, |
493f133f LB |
639 | .target_residency = 800, |
640 | .enter = &intel_idle, | |
641 | .enter_freeze = intel_idle_freeze, }, | |
135919a3 LB |
642 | { |
643 | .name = "C9-SKL", | |
644 | .desc = "MWAIT 0x50", | |
645 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, | |
646 | .exit_latency = 480, | |
647 | .target_residency = 5000, | |
648 | .enter = &intel_idle, | |
649 | .enter_freeze = intel_idle_freeze, }, | |
493f133f LB |
650 | { |
651 | .name = "C10-SKL", | |
652 | .desc = "MWAIT 0x60", | |
653 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
654 | .exit_latency = 890, | |
655 | .target_residency = 5000, | |
656 | .enter = &intel_idle, | |
657 | .enter_freeze = intel_idle_freeze, }, | |
658 | { | |
659 | .enter = NULL } | |
660 | }; | |
661 | ||
f9e71657 LB |
662 | static struct cpuidle_state skx_cstates[] = { |
663 | { | |
664 | .name = "C1-SKX", | |
665 | .desc = "MWAIT 0x00", | |
666 | .flags = MWAIT2flg(0x00), | |
667 | .exit_latency = 2, | |
668 | .target_residency = 2, | |
669 | .enter = &intel_idle, | |
670 | .enter_freeze = intel_idle_freeze, }, | |
671 | { | |
672 | .name = "C1E-SKX", | |
673 | .desc = "MWAIT 0x01", | |
674 | .flags = MWAIT2flg(0x01), | |
675 | .exit_latency = 10, | |
676 | .target_residency = 20, | |
677 | .enter = &intel_idle, | |
678 | .enter_freeze = intel_idle_freeze, }, | |
679 | { | |
680 | .name = "C6-SKX", | |
681 | .desc = "MWAIT 0x20", | |
682 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
683 | .exit_latency = 133, | |
684 | .target_residency = 600, | |
685 | .enter = &intel_idle, | |
686 | .enter_freeze = intel_idle_freeze, }, | |
687 | { | |
688 | .enter = NULL } | |
689 | }; | |
690 | ||
ba0dc81e | 691 | static struct cpuidle_state atom_cstates[] = { |
e022e7eb | 692 | { |
32e95180 | 693 | .name = "C1E-ATM", |
26717172 | 694 | .desc = "MWAIT 0x00", |
b82b6cca | 695 | .flags = MWAIT2flg(0x00), |
32e95180 LB |
696 | .exit_latency = 10, |
697 | .target_residency = 20, | |
5fe2e527 RW |
698 | .enter = &intel_idle, |
699 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 700 | { |
15e123e5 | 701 | .name = "C2-ATM", |
26717172 | 702 | .desc = "MWAIT 0x10", |
b82b6cca | 703 | .flags = MWAIT2flg(0x10), |
26717172 | 704 | .exit_latency = 20, |
26717172 | 705 | .target_residency = 80, |
5fe2e527 RW |
706 | .enter = &intel_idle, |
707 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 708 | { |
15e123e5 | 709 | .name = "C4-ATM", |
26717172 | 710 | .desc = "MWAIT 0x30", |
b82b6cca | 711 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 712 | .exit_latency = 100, |
26717172 | 713 | .target_residency = 400, |
5fe2e527 RW |
714 | .enter = &intel_idle, |
715 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb | 716 | { |
15e123e5 | 717 | .name = "C6-ATM", |
7fcca7d9 | 718 | .desc = "MWAIT 0x52", |
b82b6cca | 719 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, |
7fcca7d9 | 720 | .exit_latency = 140, |
7fcca7d9 | 721 | .target_residency = 560, |
5fe2e527 RW |
722 | .enter = &intel_idle, |
723 | .enter_freeze = intel_idle_freeze, }, | |
e022e7eb LB |
724 | { |
725 | .enter = NULL } | |
26717172 | 726 | }; |
88390996 | 727 | static struct cpuidle_state avn_cstates[] = { |
fab04b22 LB |
728 | { |
729 | .name = "C1-AVN", | |
730 | .desc = "MWAIT 0x00", | |
b82b6cca | 731 | .flags = MWAIT2flg(0x00), |
fab04b22 LB |
732 | .exit_latency = 2, |
733 | .target_residency = 2, | |
5fe2e527 RW |
734 | .enter = &intel_idle, |
735 | .enter_freeze = intel_idle_freeze, }, | |
fab04b22 LB |
736 | { |
737 | .name = "C6-AVN", | |
738 | .desc = "MWAIT 0x51", | |
b82b6cca | 739 | .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED, |
fab04b22 LB |
740 | .exit_latency = 15, |
741 | .target_residency = 45, | |
5fe2e527 RW |
742 | .enter = &intel_idle, |
743 | .enter_freeze = intel_idle_freeze, }, | |
88390996 JL |
744 | { |
745 | .enter = NULL } | |
fab04b22 | 746 | }; |
281baf7a DC |
747 | static struct cpuidle_state knl_cstates[] = { |
748 | { | |
749 | .name = "C1-KNL", | |
750 | .desc = "MWAIT 0x00", | |
751 | .flags = MWAIT2flg(0x00), | |
752 | .exit_latency = 1, | |
753 | .target_residency = 2, | |
754 | .enter = &intel_idle, | |
755 | .enter_freeze = intel_idle_freeze }, | |
756 | { | |
757 | .name = "C6-KNL", | |
758 | .desc = "MWAIT 0x10", | |
759 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED, | |
760 | .exit_latency = 120, | |
761 | .target_residency = 500, | |
762 | .enter = &intel_idle, | |
763 | .enter_freeze = intel_idle_freeze }, | |
764 | { | |
765 | .enter = NULL } | |
766 | }; | |
26717172 | 767 | |
5dcef694 LB |
768 | static struct cpuidle_state bxt_cstates[] = { |
769 | { | |
770 | .name = "C1-BXT", | |
771 | .desc = "MWAIT 0x00", | |
772 | .flags = MWAIT2flg(0x00), | |
773 | .exit_latency = 2, | |
774 | .target_residency = 2, | |
775 | .enter = &intel_idle, | |
776 | .enter_freeze = intel_idle_freeze, }, | |
777 | { | |
778 | .name = "C1E-BXT", | |
779 | .desc = "MWAIT 0x01", | |
780 | .flags = MWAIT2flg(0x01), | |
781 | .exit_latency = 10, | |
782 | .target_residency = 20, | |
783 | .enter = &intel_idle, | |
784 | .enter_freeze = intel_idle_freeze, }, | |
785 | { | |
786 | .name = "C6-BXT", | |
787 | .desc = "MWAIT 0x20", | |
788 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
789 | .exit_latency = 133, | |
790 | .target_residency = 133, | |
791 | .enter = &intel_idle, | |
792 | .enter_freeze = intel_idle_freeze, }, | |
793 | { | |
794 | .name = "C7s-BXT", | |
795 | .desc = "MWAIT 0x31", | |
796 | .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED, | |
797 | .exit_latency = 155, | |
798 | .target_residency = 155, | |
799 | .enter = &intel_idle, | |
800 | .enter_freeze = intel_idle_freeze, }, | |
801 | { | |
802 | .name = "C8-BXT", | |
803 | .desc = "MWAIT 0x40", | |
804 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, | |
805 | .exit_latency = 1000, | |
806 | .target_residency = 1000, | |
807 | .enter = &intel_idle, | |
808 | .enter_freeze = intel_idle_freeze, }, | |
809 | { | |
810 | .name = "C9-BXT", | |
811 | .desc = "MWAIT 0x50", | |
812 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED, | |
813 | .exit_latency = 2000, | |
814 | .target_residency = 2000, | |
815 | .enter = &intel_idle, | |
816 | .enter_freeze = intel_idle_freeze, }, | |
817 | { | |
818 | .name = "C10-BXT", | |
819 | .desc = "MWAIT 0x60", | |
820 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, | |
821 | .exit_latency = 10000, | |
822 | .target_residency = 10000, | |
823 | .enter = &intel_idle, | |
824 | .enter_freeze = intel_idle_freeze, }, | |
825 | { | |
826 | .enter = NULL } | |
827 | }; | |
828 | ||
0080d65b JP |
829 | static struct cpuidle_state dnv_cstates[] = { |
830 | { | |
831 | .name = "C1-DNV", | |
832 | .desc = "MWAIT 0x00", | |
833 | .flags = MWAIT2flg(0x00), | |
834 | .exit_latency = 2, | |
835 | .target_residency = 2, | |
836 | .enter = &intel_idle, | |
837 | .enter_freeze = intel_idle_freeze, }, | |
838 | { | |
839 | .name = "C1E-DNV", | |
840 | .desc = "MWAIT 0x01", | |
841 | .flags = MWAIT2flg(0x01), | |
842 | .exit_latency = 10, | |
843 | .target_residency = 20, | |
844 | .enter = &intel_idle, | |
845 | .enter_freeze = intel_idle_freeze, }, | |
846 | { | |
847 | .name = "C6-DNV", | |
848 | .desc = "MWAIT 0x20", | |
849 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, | |
850 | .exit_latency = 50, | |
851 | .target_residency = 500, | |
852 | .enter = &intel_idle, | |
853 | .enter_freeze = intel_idle_freeze, }, | |
854 | { | |
855 | .enter = NULL } | |
856 | }; | |
857 | ||
26717172 LB |
858 | /** |
859 | * intel_idle | |
860 | * @dev: cpuidle_device | |
46bcfad7 | 861 | * @drv: cpuidle driver |
e978aa7d | 862 | * @index: index of cpuidle state |
26717172 | 863 | * |
63ff07be | 864 | * Must be called under local_irq_disable(). |
26717172 | 865 | */ |
46bcfad7 DD |
866 | static int intel_idle(struct cpuidle_device *dev, |
867 | struct cpuidle_driver *drv, int index) | |
26717172 LB |
868 | { |
869 | unsigned long ecx = 1; /* break on interrupt flag */ | |
46bcfad7 | 870 | struct cpuidle_state *state = &drv->states[index]; |
b1beab48 | 871 | unsigned long eax = flg2MWAIT(state->flags); |
26717172 | 872 | unsigned int cstate; |
26717172 LB |
873 | int cpu = smp_processor_id(); |
874 | ||
875 | cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1; | |
876 | ||
6110a1f4 | 877 | /* |
c8381cc3 LB |
878 | * leave_mm() to avoid costly and often unnecessary wakeups |
879 | * for flushing the user TLB's associated with the active mm. | |
6110a1f4 | 880 | */ |
c8381cc3 | 881 | if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) |
6110a1f4 SS |
882 | leave_mm(cpu); |
883 | ||
26717172 | 884 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) |
f6cee191 | 885 | tick_broadcast_enter(); |
26717172 | 886 | |
16824255 | 887 | mwait_idle_with_hints(eax, ecx); |
26717172 | 888 | |
26717172 | 889 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) |
f6cee191 | 890 | tick_broadcast_exit(); |
26717172 | 891 | |
e978aa7d | 892 | return index; |
26717172 LB |
893 | } |
894 | ||
5fe2e527 RW |
895 | /** |
896 | * intel_idle_freeze - simplified "enter" callback routine for suspend-to-idle | |
897 | * @dev: cpuidle_device | |
898 | * @drv: cpuidle driver | |
899 | * @index: state index | |
900 | */ | |
901 | static void intel_idle_freeze(struct cpuidle_device *dev, | |
902 | struct cpuidle_driver *drv, int index) | |
903 | { | |
904 | unsigned long ecx = 1; /* break on interrupt flag */ | |
905 | unsigned long eax = flg2MWAIT(drv->states[index].flags); | |
906 | ||
907 | mwait_idle_with_hints(eax, ecx); | |
908 | } | |
909 | ||
2a2d31c8 SL |
910 | static void __setup_broadcast_timer(void *arg) |
911 | { | |
76962caa | 912 | unsigned long on = (unsigned long)arg; |
2a2d31c8 | 913 | |
76962caa TG |
914 | if (on) |
915 | tick_broadcast_enable(); | |
916 | else | |
917 | tick_broadcast_disable(); | |
2a2d31c8 SL |
918 | } |
919 | ||
25ac7761 DL |
920 | static int cpu_hotplug_notify(struct notifier_block *n, |
921 | unsigned long action, void *hcpu) | |
2a2d31c8 SL |
922 | { |
923 | int hotcpu = (unsigned long)hcpu; | |
25ac7761 | 924 | struct cpuidle_device *dev; |
2a2d31c8 | 925 | |
e2401453 | 926 | switch (action & ~CPU_TASKS_FROZEN) { |
2a2d31c8 | 927 | case CPU_ONLINE: |
25ac7761 DL |
928 | |
929 | if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) | |
930 | smp_call_function_single(hotcpu, __setup_broadcast_timer, | |
931 | (void *)true, 1); | |
932 | ||
933 | /* | |
934 | * Some systems can hotplug a cpu at runtime after | |
935 | * the kernel has booted, we have to initialize the | |
936 | * driver in this case | |
937 | */ | |
938 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu); | |
08820546 RC |
939 | if (dev->registered) |
940 | break; | |
941 | ||
942 | if (intel_idle_cpu_init(hotcpu)) | |
943 | return NOTIFY_BAD; | |
25ac7761 | 944 | |
2a2d31c8 | 945 | break; |
2a2d31c8 SL |
946 | } |
947 | return NOTIFY_OK; | |
948 | } | |
949 | ||
25ac7761 DL |
950 | static struct notifier_block cpu_hotplug_notifier = { |
951 | .notifier_call = cpu_hotplug_notify, | |
2a2d31c8 SL |
952 | }; |
953 | ||
14796fca LB |
954 | static void auto_demotion_disable(void *dummy) |
955 | { | |
956 | unsigned long long msr_bits; | |
957 | ||
958 | rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); | |
b66b8b9a | 959 | msr_bits &= ~(icpu->auto_demotion_disable_flags); |
14796fca LB |
960 | wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); |
961 | } | |
32e95180 LB |
962 | static void c1e_promotion_disable(void *dummy) |
963 | { | |
964 | unsigned long long msr_bits; | |
965 | ||
966 | rdmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
967 | msr_bits &= ~0x2; | |
968 | wrmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
969 | } | |
14796fca | 970 | |
b66b8b9a AK |
971 | static const struct idle_cpu idle_cpu_nehalem = { |
972 | .state_table = nehalem_cstates, | |
b66b8b9a | 973 | .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, |
32e95180 | 974 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
975 | }; |
976 | ||
977 | static const struct idle_cpu idle_cpu_atom = { | |
978 | .state_table = atom_cstates, | |
979 | }; | |
980 | ||
981 | static const struct idle_cpu idle_cpu_lincroft = { | |
982 | .state_table = atom_cstates, | |
983 | .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, | |
984 | }; | |
985 | ||
986 | static const struct idle_cpu idle_cpu_snb = { | |
987 | .state_table = snb_cstates, | |
32e95180 | 988 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
989 | }; |
990 | ||
718987d6 LB |
991 | static const struct idle_cpu idle_cpu_byt = { |
992 | .state_table = byt_cstates, | |
993 | .disable_promotion_to_c1e = true, | |
8c058d53 | 994 | .byt_auto_demotion_disable_flag = true, |
718987d6 LB |
995 | }; |
996 | ||
cab07a56 LB |
997 | static const struct idle_cpu idle_cpu_cht = { |
998 | .state_table = cht_cstates, | |
999 | .disable_promotion_to_c1e = true, | |
1000 | .byt_auto_demotion_disable_flag = true, | |
1001 | }; | |
1002 | ||
6edab08c LB |
1003 | static const struct idle_cpu idle_cpu_ivb = { |
1004 | .state_table = ivb_cstates, | |
32e95180 | 1005 | .disable_promotion_to_c1e = true, |
6edab08c LB |
1006 | }; |
1007 | ||
0138d8f0 LB |
1008 | static const struct idle_cpu idle_cpu_ivt = { |
1009 | .state_table = ivt_cstates, | |
1010 | .disable_promotion_to_c1e = true, | |
1011 | }; | |
1012 | ||
85a4d2d4 LB |
1013 | static const struct idle_cpu idle_cpu_hsw = { |
1014 | .state_table = hsw_cstates, | |
32e95180 | 1015 | .disable_promotion_to_c1e = true, |
85a4d2d4 LB |
1016 | }; |
1017 | ||
a138b568 LB |
1018 | static const struct idle_cpu idle_cpu_bdw = { |
1019 | .state_table = bdw_cstates, | |
1020 | .disable_promotion_to_c1e = true, | |
1021 | }; | |
1022 | ||
493f133f LB |
1023 | static const struct idle_cpu idle_cpu_skl = { |
1024 | .state_table = skl_cstates, | |
1025 | .disable_promotion_to_c1e = true, | |
1026 | }; | |
1027 | ||
f9e71657 LB |
1028 | static const struct idle_cpu idle_cpu_skx = { |
1029 | .state_table = skx_cstates, | |
1030 | .disable_promotion_to_c1e = true, | |
1031 | }; | |
493f133f | 1032 | |
fab04b22 LB |
1033 | static const struct idle_cpu idle_cpu_avn = { |
1034 | .state_table = avn_cstates, | |
1035 | .disable_promotion_to_c1e = true, | |
1036 | }; | |
1037 | ||
281baf7a DC |
1038 | static const struct idle_cpu idle_cpu_knl = { |
1039 | .state_table = knl_cstates, | |
1040 | }; | |
1041 | ||
5dcef694 LB |
1042 | static const struct idle_cpu idle_cpu_bxt = { |
1043 | .state_table = bxt_cstates, | |
1044 | .disable_promotion_to_c1e = true, | |
1045 | }; | |
1046 | ||
0080d65b JP |
1047 | static const struct idle_cpu idle_cpu_dnv = { |
1048 | .state_table = dnv_cstates, | |
1049 | .disable_promotion_to_c1e = true, | |
1050 | }; | |
1051 | ||
b66b8b9a AK |
1052 | #define ICPU(model, cpu) \ |
1053 | { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } | |
1054 | ||
d5cdc3c4 | 1055 | static const struct x86_cpu_id intel_idle_ids[] __initconst = { |
db73c5a8 DH |
1056 | ICPU(INTEL_FAM6_NEHALEM_EP, idle_cpu_nehalem), |
1057 | ICPU(INTEL_FAM6_NEHALEM, idle_cpu_nehalem), | |
4b3b234f | 1058 | ICPU(INTEL_FAM6_NEHALEM_G, idle_cpu_nehalem), |
db73c5a8 DH |
1059 | ICPU(INTEL_FAM6_WESTMERE, idle_cpu_nehalem), |
1060 | ICPU(INTEL_FAM6_WESTMERE_EP, idle_cpu_nehalem), | |
1061 | ICPU(INTEL_FAM6_NEHALEM_EX, idle_cpu_nehalem), | |
1062 | ICPU(INTEL_FAM6_ATOM_PINEVIEW, idle_cpu_atom), | |
1063 | ICPU(INTEL_FAM6_ATOM_LINCROFT, idle_cpu_lincroft), | |
1064 | ICPU(INTEL_FAM6_WESTMERE_EX, idle_cpu_nehalem), | |
1065 | ICPU(INTEL_FAM6_SANDYBRIDGE, idle_cpu_snb), | |
1066 | ICPU(INTEL_FAM6_SANDYBRIDGE_X, idle_cpu_snb), | |
1067 | ICPU(INTEL_FAM6_ATOM_CEDARVIEW, idle_cpu_atom), | |
1068 | ICPU(INTEL_FAM6_ATOM_SILVERMONT1, idle_cpu_byt), | |
1069 | ICPU(INTEL_FAM6_ATOM_AIRMONT, idle_cpu_cht), | |
1070 | ICPU(INTEL_FAM6_IVYBRIDGE, idle_cpu_ivb), | |
1071 | ICPU(INTEL_FAM6_IVYBRIDGE_X, idle_cpu_ivt), | |
1072 | ICPU(INTEL_FAM6_HASWELL_CORE, idle_cpu_hsw), | |
1073 | ICPU(INTEL_FAM6_HASWELL_X, idle_cpu_hsw), | |
1074 | ICPU(INTEL_FAM6_HASWELL_ULT, idle_cpu_hsw), | |
1075 | ICPU(INTEL_FAM6_HASWELL_GT3E, idle_cpu_hsw), | |
1076 | ICPU(INTEL_FAM6_ATOM_SILVERMONT2, idle_cpu_avn), | |
1077 | ICPU(INTEL_FAM6_BROADWELL_CORE, idle_cpu_bdw), | |
1078 | ICPU(INTEL_FAM6_BROADWELL_GT3E, idle_cpu_bdw), | |
1079 | ICPU(INTEL_FAM6_BROADWELL_X, idle_cpu_bdw), | |
1080 | ICPU(INTEL_FAM6_BROADWELL_XEON_D, idle_cpu_bdw), | |
1081 | ICPU(INTEL_FAM6_SKYLAKE_MOBILE, idle_cpu_skl), | |
1082 | ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, idle_cpu_skl), | |
1083 | ICPU(INTEL_FAM6_KABYLAKE_MOBILE, idle_cpu_skl), | |
1084 | ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, idle_cpu_skl), | |
1085 | ICPU(INTEL_FAM6_SKYLAKE_X, idle_cpu_skx), | |
1086 | ICPU(INTEL_FAM6_XEON_PHI_KNL, idle_cpu_knl), | |
1087 | ICPU(INTEL_FAM6_ATOM_GOLDMONT, idle_cpu_bxt), | |
0080d65b | 1088 | ICPU(INTEL_FAM6_ATOM_DENVERTON, idle_cpu_dnv), |
b66b8b9a AK |
1089 | {} |
1090 | }; | |
b66b8b9a | 1091 | |
26717172 LB |
1092 | /* |
1093 | * intel_idle_probe() | |
1094 | */ | |
00f3e755 | 1095 | static int __init intel_idle_probe(void) |
26717172 | 1096 | { |
c4236282 | 1097 | unsigned int eax, ebx, ecx; |
b66b8b9a | 1098 | const struct x86_cpu_id *id; |
26717172 LB |
1099 | |
1100 | if (max_cstate == 0) { | |
1101 | pr_debug(PREFIX "disabled\n"); | |
1102 | return -EPERM; | |
1103 | } | |
1104 | ||
b66b8b9a AK |
1105 | id = x86_match_cpu(intel_idle_ids); |
1106 | if (!id) { | |
1107 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && | |
1108 | boot_cpu_data.x86 == 6) | |
1109 | pr_debug(PREFIX "does not run on family %d model %d\n", | |
1110 | boot_cpu_data.x86, boot_cpu_data.x86_model); | |
26717172 | 1111 | return -ENODEV; |
b66b8b9a | 1112 | } |
26717172 LB |
1113 | |
1114 | if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) | |
1115 | return -ENODEV; | |
1116 | ||
c4236282 | 1117 | cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); |
26717172 LB |
1118 | |
1119 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || | |
5c2a9f06 TR |
1120 | !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || |
1121 | !mwait_substates) | |
26717172 | 1122 | return -ENODEV; |
26717172 | 1123 | |
c4236282 | 1124 | pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates); |
26717172 | 1125 | |
b66b8b9a AK |
1126 | icpu = (const struct idle_cpu *)id->driver_data; |
1127 | cpuidle_state_table = icpu->state_table; | |
26717172 LB |
1128 | |
1129 | pr_debug(PREFIX "v" INTEL_IDLE_VERSION | |
1130 | " model 0x%X\n", boot_cpu_data.x86_model); | |
1131 | ||
26717172 LB |
1132 | return 0; |
1133 | } | |
1134 | ||
1135 | /* | |
1136 | * intel_idle_cpuidle_devices_uninit() | |
ca42489d | 1137 | * Unregisters the cpuidle devices. |
26717172 LB |
1138 | */ |
1139 | static void intel_idle_cpuidle_devices_uninit(void) | |
1140 | { | |
1141 | int i; | |
1142 | struct cpuidle_device *dev; | |
1143 | ||
1144 | for_each_online_cpu(i) { | |
1145 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); | |
1146 | cpuidle_unregister_device(dev); | |
1147 | } | |
26717172 | 1148 | } |
0138d8f0 LB |
1149 | |
1150 | /* | |
d70e28f5 | 1151 | * ivt_idle_state_table_update(void) |
0138d8f0 | 1152 | * |
d70e28f5 | 1153 | * Tune IVT multi-socket targets |
0138d8f0 LB |
1154 | * Assumption: num_sockets == (max_package_num + 1) |
1155 | */ | |
d70e28f5 | 1156 | static void ivt_idle_state_table_update(void) |
0138d8f0 LB |
1157 | { |
1158 | /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ | |
d70e28f5 LB |
1159 | int cpu, package_num, num_sockets = 1; |
1160 | ||
1161 | for_each_online_cpu(cpu) { | |
1162 | package_num = topology_physical_package_id(cpu); | |
1163 | if (package_num + 1 > num_sockets) { | |
1164 | num_sockets = package_num + 1; | |
1165 | ||
1166 | if (num_sockets > 4) { | |
1167 | cpuidle_state_table = ivt_cstates_8s; | |
1168 | return; | |
0138d8f0 LB |
1169 | } |
1170 | } | |
d70e28f5 LB |
1171 | } |
1172 | ||
1173 | if (num_sockets > 2) | |
1174 | cpuidle_state_table = ivt_cstates_4s; | |
1175 | ||
1176 | /* else, 1 and 2 socket systems use default ivt_cstates */ | |
1177 | } | |
5dcef694 LB |
1178 | |
1179 | /* | |
1180 | * Translate IRTL (Interrupt Response Time Limit) MSR to usec | |
1181 | */ | |
1182 | ||
1183 | static unsigned int irtl_ns_units[] = { | |
1184 | 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 }; | |
1185 | ||
1186 | static unsigned long long irtl_2_usec(unsigned long long irtl) | |
1187 | { | |
1188 | unsigned long long ns; | |
1189 | ||
3451ab3e JB |
1190 | if (!irtl) |
1191 | return 0; | |
1192 | ||
bef45096 | 1193 | ns = irtl_ns_units[(irtl >> 10) & 0x7]; |
5dcef694 LB |
1194 | |
1195 | return div64_u64((irtl & 0x3FF) * ns, 1000); | |
1196 | } | |
1197 | /* | |
1198 | * bxt_idle_state_table_update(void) | |
1199 | * | |
1200 | * On BXT, we trust the IRTL to show the definitive maximum latency | |
1201 | * We use the same value for target_residency. | |
1202 | */ | |
1203 | static void bxt_idle_state_table_update(void) | |
1204 | { | |
1205 | unsigned long long msr; | |
3451ab3e | 1206 | unsigned int usec; |
5dcef694 LB |
1207 | |
1208 | rdmsrl(MSR_PKGC6_IRTL, msr); | |
3451ab3e JB |
1209 | usec = irtl_2_usec(msr); |
1210 | if (usec) { | |
5dcef694 LB |
1211 | bxt_cstates[2].exit_latency = usec; |
1212 | bxt_cstates[2].target_residency = usec; | |
1213 | } | |
1214 | ||
1215 | rdmsrl(MSR_PKGC7_IRTL, msr); | |
3451ab3e JB |
1216 | usec = irtl_2_usec(msr); |
1217 | if (usec) { | |
5dcef694 LB |
1218 | bxt_cstates[3].exit_latency = usec; |
1219 | bxt_cstates[3].target_residency = usec; | |
1220 | } | |
1221 | ||
1222 | rdmsrl(MSR_PKGC8_IRTL, msr); | |
3451ab3e JB |
1223 | usec = irtl_2_usec(msr); |
1224 | if (usec) { | |
5dcef694 LB |
1225 | bxt_cstates[4].exit_latency = usec; |
1226 | bxt_cstates[4].target_residency = usec; | |
1227 | } | |
1228 | ||
1229 | rdmsrl(MSR_PKGC9_IRTL, msr); | |
3451ab3e JB |
1230 | usec = irtl_2_usec(msr); |
1231 | if (usec) { | |
5dcef694 LB |
1232 | bxt_cstates[5].exit_latency = usec; |
1233 | bxt_cstates[5].target_residency = usec; | |
1234 | } | |
1235 | ||
1236 | rdmsrl(MSR_PKGC10_IRTL, msr); | |
3451ab3e JB |
1237 | usec = irtl_2_usec(msr); |
1238 | if (usec) { | |
5dcef694 LB |
1239 | bxt_cstates[6].exit_latency = usec; |
1240 | bxt_cstates[6].target_residency = usec; | |
1241 | } | |
1242 | ||
1243 | } | |
d70e28f5 LB |
1244 | /* |
1245 | * sklh_idle_state_table_update(void) | |
1246 | * | |
1247 | * On SKL-H (model 0x5e) disable C8 and C9 if: | |
1248 | * C10 is enabled and SGX disabled | |
1249 | */ | |
1250 | static void sklh_idle_state_table_update(void) | |
1251 | { | |
1252 | unsigned long long msr; | |
1253 | unsigned int eax, ebx, ecx, edx; | |
1254 | ||
1255 | ||
1256 | /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */ | |
1257 | if (max_cstate <= 7) | |
1258 | return; | |
1259 | ||
1260 | /* if PC10 not present in CPUID.MWAIT.EDX */ | |
1261 | if ((mwait_substates & (0xF << 28)) == 0) | |
1262 | return; | |
1263 | ||
1264 | rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr); | |
1265 | ||
1266 | /* PC10 is not enabled in PKG C-state limit */ | |
1267 | if ((msr & 0xF) != 8) | |
1268 | return; | |
1269 | ||
1270 | ecx = 0; | |
1271 | cpuid(7, &eax, &ebx, &ecx, &edx); | |
1272 | ||
1273 | /* if SGX is present */ | |
1274 | if (ebx & (1 << 2)) { | |
0138d8f0 | 1275 | |
d70e28f5 LB |
1276 | rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); |
1277 | ||
1278 | /* if SGX is enabled */ | |
1279 | if (msr & (1 << 18)) | |
1280 | return; | |
1281 | } | |
1282 | ||
1283 | skl_cstates[5].disabled = 1; /* C8-SKL */ | |
1284 | skl_cstates[6].disabled = 1; /* C9-SKL */ | |
1285 | } | |
1286 | /* | |
1287 | * intel_idle_state_table_update() | |
1288 | * | |
1289 | * Update the default state_table for this CPU-id | |
1290 | */ | |
1291 | ||
1292 | static void intel_idle_state_table_update(void) | |
1293 | { | |
1294 | switch (boot_cpu_data.x86_model) { | |
1295 | ||
db73c5a8 | 1296 | case INTEL_FAM6_IVYBRIDGE_X: |
d70e28f5 LB |
1297 | ivt_idle_state_table_update(); |
1298 | break; | |
db73c5a8 | 1299 | case INTEL_FAM6_ATOM_GOLDMONT: |
5dcef694 LB |
1300 | bxt_idle_state_table_update(); |
1301 | break; | |
db73c5a8 | 1302 | case INTEL_FAM6_SKYLAKE_DESKTOP: |
d70e28f5 LB |
1303 | sklh_idle_state_table_update(); |
1304 | break; | |
0138d8f0 | 1305 | } |
0138d8f0 LB |
1306 | } |
1307 | ||
46bcfad7 DD |
1308 | /* |
1309 | * intel_idle_cpuidle_driver_init() | |
1310 | * allocate, initialize cpuidle_states | |
1311 | */ | |
5469c827 | 1312 | static void __init intel_idle_cpuidle_driver_init(void) |
46bcfad7 DD |
1313 | { |
1314 | int cstate; | |
1315 | struct cpuidle_driver *drv = &intel_idle_driver; | |
1316 | ||
0138d8f0 LB |
1317 | intel_idle_state_table_update(); |
1318 | ||
46bcfad7 DD |
1319 | drv->state_count = 1; |
1320 | ||
e022e7eb | 1321 | for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { |
24bfa950 | 1322 | int num_substates, mwait_hint, mwait_cstate; |
46bcfad7 | 1323 | |
7dd0e0af LB |
1324 | if ((cpuidle_state_table[cstate].enter == NULL) && |
1325 | (cpuidle_state_table[cstate].enter_freeze == NULL)) | |
e022e7eb LB |
1326 | break; |
1327 | ||
1328 | if (cstate + 1 > max_cstate) { | |
46bcfad7 DD |
1329 | printk(PREFIX "max_cstate %d reached\n", |
1330 | max_cstate); | |
1331 | break; | |
1332 | } | |
1333 | ||
e022e7eb LB |
1334 | mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); |
1335 | mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); | |
e022e7eb | 1336 | |
24bfa950 | 1337 | /* number of sub-states for this state in CPUID.MWAIT */ |
e022e7eb | 1338 | num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) |
46bcfad7 | 1339 | & MWAIT_SUBSTATE_MASK; |
e022e7eb | 1340 | |
24bfa950 LB |
1341 | /* if NO sub-states for this state in CPUID, skip it */ |
1342 | if (num_substates == 0) | |
46bcfad7 | 1343 | continue; |
46bcfad7 | 1344 | |
d70e28f5 LB |
1345 | /* if state marked as disabled, skip it */ |
1346 | if (cpuidle_state_table[cstate].disabled != 0) { | |
1347 | pr_debug(PREFIX "state %s is disabled", | |
1348 | cpuidle_state_table[cstate].name); | |
1349 | continue; | |
1350 | } | |
1351 | ||
1352 | ||
e022e7eb | 1353 | if (((mwait_cstate + 1) > 2) && |
46bcfad7 DD |
1354 | !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) |
1355 | mark_tsc_unstable("TSC halts in idle" | |
1356 | " states deeper than C2"); | |
1357 | ||
1358 | drv->states[drv->state_count] = /* structure copy */ | |
1359 | cpuidle_state_table[cstate]; | |
1360 | ||
1361 | drv->state_count += 1; | |
1362 | } | |
1363 | ||
8c058d53 LB |
1364 | if (icpu->byt_auto_demotion_disable_flag) { |
1365 | wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0); | |
1366 | wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0); | |
1367 | } | |
46bcfad7 DD |
1368 | } |
1369 | ||
1370 | ||
26717172 | 1371 | /* |
65b7f839 | 1372 | * intel_idle_cpu_init() |
26717172 | 1373 | * allocate, initialize, register cpuidle_devices |
65b7f839 | 1374 | * @cpu: cpu/core to initialize |
26717172 | 1375 | */ |
25ac7761 | 1376 | static int intel_idle_cpu_init(int cpu) |
26717172 | 1377 | { |
26717172 LB |
1378 | struct cpuidle_device *dev; |
1379 | ||
65b7f839 | 1380 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); |
26717172 | 1381 | |
65b7f839 | 1382 | dev->cpu = cpu; |
26717172 | 1383 | |
65b7f839 TR |
1384 | if (cpuidle_register_device(dev)) { |
1385 | pr_debug(PREFIX "cpuidle_register_device %d failed!\n", cpu); | |
65b7f839 | 1386 | return -EIO; |
26717172 LB |
1387 | } |
1388 | ||
b66b8b9a | 1389 | if (icpu->auto_demotion_disable_flags) |
65b7f839 TR |
1390 | smp_call_function_single(cpu, auto_demotion_disable, NULL, 1); |
1391 | ||
dbf87ab8 BZ |
1392 | if (icpu->disable_promotion_to_c1e) |
1393 | smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1); | |
1394 | ||
26717172 LB |
1395 | return 0; |
1396 | } | |
26717172 LB |
1397 | |
1398 | static int __init intel_idle_init(void) | |
1399 | { | |
65b7f839 | 1400 | int retval, i; |
26717172 | 1401 | |
d1896049 TR |
1402 | /* Do not load intel_idle at all for now if idle= is passed */ |
1403 | if (boot_option_idle_override != IDLE_NO_OVERRIDE) | |
1404 | return -ENODEV; | |
1405 | ||
26717172 LB |
1406 | retval = intel_idle_probe(); |
1407 | if (retval) | |
1408 | return retval; | |
1409 | ||
e9df69cc RC |
1410 | intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); |
1411 | if (intel_idle_cpuidle_devices == NULL) | |
1412 | return -ENOMEM; | |
1413 | ||
46bcfad7 | 1414 | intel_idle_cpuidle_driver_init(); |
26717172 LB |
1415 | retval = cpuidle_register_driver(&intel_idle_driver); |
1416 | if (retval) { | |
3735d524 | 1417 | struct cpuidle_driver *drv = cpuidle_get_driver(); |
26717172 | 1418 | printk(KERN_DEBUG PREFIX "intel_idle yielding to %s", |
3735d524 | 1419 | drv ? drv->name : "none"); |
e9df69cc | 1420 | free_percpu(intel_idle_cpuidle_devices); |
26717172 LB |
1421 | return retval; |
1422 | } | |
1423 | ||
07494d54 SB |
1424 | cpu_notifier_register_begin(); |
1425 | ||
65b7f839 TR |
1426 | for_each_online_cpu(i) { |
1427 | retval = intel_idle_cpu_init(i); | |
1428 | if (retval) { | |
b69ef2c0 | 1429 | intel_idle_cpuidle_devices_uninit(); |
07494d54 | 1430 | cpu_notifier_register_done(); |
65b7f839 | 1431 | cpuidle_unregister_driver(&intel_idle_driver); |
ca42489d | 1432 | free_percpu(intel_idle_cpuidle_devices); |
65b7f839 TR |
1433 | return retval; |
1434 | } | |
26717172 | 1435 | } |
07494d54 SB |
1436 | __register_cpu_notifier(&cpu_hotplug_notifier); |
1437 | ||
2259a819 RC |
1438 | if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ |
1439 | lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; | |
1440 | else | |
1441 | on_each_cpu(__setup_broadcast_timer, (void *)true, 1); | |
1442 | ||
07494d54 | 1443 | cpu_notifier_register_done(); |
26717172 | 1444 | |
2259a819 RC |
1445 | pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n", |
1446 | lapic_timer_reliable_states); | |
1447 | ||
26717172 LB |
1448 | return 0; |
1449 | } | |
02c4fae9 | 1450 | device_initcall(intel_idle_init); |
26717172 | 1451 | |
02c4fae9 PG |
1452 | /* |
1453 | * We are not really modular, but we used to support that. Meaning we also | |
1454 | * support "intel_idle.max_cstate=..." at boot and also a read-only export of | |
1455 | * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param | |
1456 | * is the easiest way (currently) to continue doing that. | |
1457 | */ | |
26717172 | 1458 | module_param(max_cstate, int, 0444); |