Commit | Line | Data |
---|---|---|
26717172 LB |
1 | /* |
2 | * intel_idle.c - native hardware idle loop for modern Intel processors | |
3 | * | |
fab04b22 | 4 | * Copyright (c) 2013, Intel Corporation. |
26717172 LB |
5 | * Len Brown <len.brown@intel.com> |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify it | |
8 | * under the terms and conditions of the GNU General Public License, | |
9 | * version 2, as published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope it will be useful, but WITHOUT | |
12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
13 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
14 | * more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License along with | |
17 | * this program; if not, write to the Free Software Foundation, Inc., | |
18 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. | |
19 | */ | |
20 | ||
21 | /* | |
22 | * intel_idle is a cpuidle driver that loads on specific Intel processors | |
23 | * in lieu of the legacy ACPI processor_idle driver. The intent is to | |
24 | * make Linux more efficient on these processors, as intel_idle knows | |
25 | * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs. | |
26 | */ | |
27 | ||
28 | /* | |
29 | * Design Assumptions | |
30 | * | |
31 | * All CPUs have same idle states as boot CPU | |
32 | * | |
33 | * Chipset BM_STS (bus master status) bit is a NOP | |
34 | * for preventing entry into deep C-stats | |
35 | */ | |
36 | ||
37 | /* | |
38 | * Known limitations | |
39 | * | |
40 | * The driver currently initializes for_each_online_cpu() upon modprobe. | |
41 | * It it unaware of subsequent processors hot-added to the system. | |
42 | * This means that if you boot with maxcpus=n and later online | |
43 | * processors above n, those processors will use C1 only. | |
44 | * | |
45 | * ACPI has a .suspend hack to turn off deep c-statees during suspend | |
46 | * to avoid complications with the lapic timer workaround. | |
47 | * Have not seen issues with suspend, but may need same workaround here. | |
48 | * | |
49 | * There is currently no kernel-based automatic probing/loading mechanism | |
50 | * if the driver is built as a module. | |
51 | */ | |
52 | ||
53 | /* un-comment DEBUG to enable pr_debug() statements */ | |
54 | #define DEBUG | |
55 | ||
56 | #include <linux/kernel.h> | |
57 | #include <linux/cpuidle.h> | |
58 | #include <linux/clockchips.h> | |
26717172 LB |
59 | #include <trace/events/power.h> |
60 | #include <linux/sched.h> | |
2a2d31c8 SL |
61 | #include <linux/notifier.h> |
62 | #include <linux/cpu.h> | |
7c52d551 | 63 | #include <linux/module.h> |
b66b8b9a | 64 | #include <asm/cpu_device_id.h> |
bc83cccc | 65 | #include <asm/mwait.h> |
14796fca | 66 | #include <asm/msr.h> |
26717172 LB |
67 | |
68 | #define INTEL_IDLE_VERSION "0.4" | |
69 | #define PREFIX "intel_idle: " | |
70 | ||
26717172 LB |
71 | static struct cpuidle_driver intel_idle_driver = { |
72 | .name = "intel_idle", | |
73 | .owner = THIS_MODULE, | |
74 | }; | |
75 | /* intel_idle.max_cstate=0 disables driver */ | |
137ecc77 | 76 | static int max_cstate = CPUIDLE_STATE_MAX - 1; |
26717172 | 77 | |
c4236282 | 78 | static unsigned int mwait_substates; |
26717172 | 79 | |
2a2d31c8 | 80 | #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF |
26717172 | 81 | /* Reliable LAPIC Timer States, bit 1 for C1 etc. */ |
d13780d4 | 82 | static unsigned int lapic_timer_reliable_states = (1 << 1); /* Default to only C1 */ |
26717172 | 83 | |
b66b8b9a AK |
84 | struct idle_cpu { |
85 | struct cpuidle_state *state_table; | |
86 | ||
87 | /* | |
88 | * Hardware C-state auto-demotion may not always be optimal. | |
89 | * Indicate which enable bits to clear here. | |
90 | */ | |
91 | unsigned long auto_demotion_disable_flags; | |
32e95180 | 92 | bool disable_promotion_to_c1e; |
b66b8b9a AK |
93 | }; |
94 | ||
95 | static const struct idle_cpu *icpu; | |
3265eba0 | 96 | static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; |
46bcfad7 DD |
97 | static int intel_idle(struct cpuidle_device *dev, |
98 | struct cpuidle_driver *drv, int index); | |
25ac7761 | 99 | static int intel_idle_cpu_init(int cpu); |
26717172 LB |
100 | |
101 | static struct cpuidle_state *cpuidle_state_table; | |
102 | ||
956d033f LB |
103 | /* |
104 | * Set this flag for states where the HW flushes the TLB for us | |
105 | * and so we don't need cross-calls to keep it consistent. | |
106 | * If this flag is set, SW flushes the TLB, so even if the | |
107 | * HW doesn't do the flushing, this flag is safe to use. | |
108 | */ | |
109 | #define CPUIDLE_FLAG_TLB_FLUSHED 0x10000 | |
110 | ||
b1beab48 LB |
111 | /* |
112 | * MWAIT takes an 8-bit "hint" in EAX "suggesting" | |
113 | * the C-state (top nibble) and sub-state (bottom nibble) | |
114 | * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc. | |
115 | * | |
116 | * We store the hint at the top of our "flags" for each state. | |
117 | */ | |
118 | #define flg2MWAIT(flags) (((flags) >> 24) & 0xFF) | |
119 | #define MWAIT2flg(eax) ((eax & 0xFF) << 24) | |
120 | ||
26717172 LB |
121 | /* |
122 | * States are indexed by the cstate number, | |
123 | * which is also the index into the MWAIT hint array. | |
124 | * Thus C0 is a dummy. | |
125 | */ | |
ba0dc81e | 126 | static struct cpuidle_state nehalem_cstates[] = { |
e022e7eb | 127 | { |
15e123e5 | 128 | .name = "C1-NHM", |
26717172 | 129 | .desc = "MWAIT 0x00", |
b1beab48 | 130 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, |
26717172 | 131 | .exit_latency = 3, |
26717172 LB |
132 | .target_residency = 6, |
133 | .enter = &intel_idle }, | |
32e95180 LB |
134 | { |
135 | .name = "C1E-NHM", | |
136 | .desc = "MWAIT 0x01", | |
137 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, | |
138 | .exit_latency = 10, | |
139 | .target_residency = 20, | |
140 | .enter = &intel_idle }, | |
e022e7eb | 141 | { |
15e123e5 | 142 | .name = "C3-NHM", |
26717172 | 143 | .desc = "MWAIT 0x10", |
b1beab48 | 144 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 145 | .exit_latency = 20, |
26717172 LB |
146 | .target_residency = 80, |
147 | .enter = &intel_idle }, | |
e022e7eb | 148 | { |
15e123e5 | 149 | .name = "C6-NHM", |
26717172 | 150 | .desc = "MWAIT 0x20", |
b1beab48 | 151 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 152 | .exit_latency = 200, |
26717172 LB |
153 | .target_residency = 800, |
154 | .enter = &intel_idle }, | |
e022e7eb LB |
155 | { |
156 | .enter = NULL } | |
26717172 LB |
157 | }; |
158 | ||
ba0dc81e | 159 | static struct cpuidle_state snb_cstates[] = { |
e022e7eb | 160 | { |
15e123e5 | 161 | .name = "C1-SNB", |
d13780d4 | 162 | .desc = "MWAIT 0x00", |
b1beab48 | 163 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, |
32e95180 LB |
164 | .exit_latency = 2, |
165 | .target_residency = 2, | |
166 | .enter = &intel_idle }, | |
167 | { | |
168 | .name = "C1E-SNB", | |
169 | .desc = "MWAIT 0x01", | |
170 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, | |
171 | .exit_latency = 10, | |
172 | .target_residency = 20, | |
d13780d4 | 173 | .enter = &intel_idle }, |
e022e7eb | 174 | { |
15e123e5 | 175 | .name = "C3-SNB", |
d13780d4 | 176 | .desc = "MWAIT 0x10", |
b1beab48 | 177 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 178 | .exit_latency = 80, |
ddbd550d | 179 | .target_residency = 211, |
d13780d4 | 180 | .enter = &intel_idle }, |
e022e7eb | 181 | { |
15e123e5 | 182 | .name = "C6-SNB", |
d13780d4 | 183 | .desc = "MWAIT 0x20", |
b1beab48 | 184 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 185 | .exit_latency = 104, |
ddbd550d | 186 | .target_residency = 345, |
d13780d4 | 187 | .enter = &intel_idle }, |
e022e7eb | 188 | { |
15e123e5 | 189 | .name = "C7-SNB", |
d13780d4 | 190 | .desc = "MWAIT 0x30", |
b1beab48 | 191 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
d13780d4 | 192 | .exit_latency = 109, |
ddbd550d | 193 | .target_residency = 345, |
d13780d4 | 194 | .enter = &intel_idle }, |
e022e7eb LB |
195 | { |
196 | .enter = NULL } | |
d13780d4 LB |
197 | }; |
198 | ||
718987d6 LB |
199 | static struct cpuidle_state byt_cstates[] = { |
200 | { | |
201 | .name = "C1-BYT", | |
202 | .desc = "MWAIT 0x00", | |
203 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, | |
204 | .exit_latency = 1, | |
205 | .target_residency = 1, | |
206 | .enter = &intel_idle }, | |
207 | { | |
208 | .name = "C1E-BYT", | |
209 | .desc = "MWAIT 0x01", | |
210 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, | |
211 | .exit_latency = 15, | |
212 | .target_residency = 30, | |
213 | .enter = &intel_idle }, | |
214 | { | |
215 | .name = "C6N-BYT", | |
216 | .desc = "MWAIT 0x58", | |
217 | .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
218 | .exit_latency = 40, | |
219 | .target_residency = 275, | |
220 | .enter = &intel_idle }, | |
221 | { | |
222 | .name = "C6S-BYT", | |
223 | .desc = "MWAIT 0x52", | |
224 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
225 | .exit_latency = 140, | |
226 | .target_residency = 560, | |
227 | .enter = &intel_idle }, | |
228 | { | |
229 | .name = "C7-BYT", | |
230 | .desc = "MWAIT 0x60", | |
231 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
232 | .exit_latency = 1200, | |
233 | .target_residency = 1500, | |
234 | .enter = &intel_idle }, | |
235 | { | |
236 | .name = "C7S-BYT", | |
237 | .desc = "MWAIT 0x64", | |
238 | .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
239 | .exit_latency = 10000, | |
240 | .target_residency = 20000, | |
241 | .enter = &intel_idle }, | |
242 | { | |
243 | .enter = NULL } | |
244 | }; | |
245 | ||
ba0dc81e | 246 | static struct cpuidle_state ivb_cstates[] = { |
e022e7eb | 247 | { |
6edab08c LB |
248 | .name = "C1-IVB", |
249 | .desc = "MWAIT 0x00", | |
b1beab48 | 250 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, |
6edab08c LB |
251 | .exit_latency = 1, |
252 | .target_residency = 1, | |
253 | .enter = &intel_idle }, | |
32e95180 LB |
254 | { |
255 | .name = "C1E-IVB", | |
256 | .desc = "MWAIT 0x01", | |
257 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, | |
258 | .exit_latency = 10, | |
259 | .target_residency = 20, | |
260 | .enter = &intel_idle }, | |
e022e7eb | 261 | { |
6edab08c LB |
262 | .name = "C3-IVB", |
263 | .desc = "MWAIT 0x10", | |
b1beab48 | 264 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
265 | .exit_latency = 59, |
266 | .target_residency = 156, | |
267 | .enter = &intel_idle }, | |
e022e7eb | 268 | { |
6edab08c LB |
269 | .name = "C6-IVB", |
270 | .desc = "MWAIT 0x20", | |
b1beab48 | 271 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
272 | .exit_latency = 80, |
273 | .target_residency = 300, | |
274 | .enter = &intel_idle }, | |
e022e7eb | 275 | { |
6edab08c LB |
276 | .name = "C7-IVB", |
277 | .desc = "MWAIT 0x30", | |
b1beab48 | 278 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
6edab08c LB |
279 | .exit_latency = 87, |
280 | .target_residency = 300, | |
281 | .enter = &intel_idle }, | |
e022e7eb LB |
282 | { |
283 | .enter = NULL } | |
6edab08c LB |
284 | }; |
285 | ||
0138d8f0 LB |
286 | static struct cpuidle_state ivt_cstates[] = { |
287 | { | |
288 | .name = "C1-IVT", | |
289 | .desc = "MWAIT 0x00", | |
290 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, | |
291 | .exit_latency = 1, | |
292 | .target_residency = 1, | |
293 | .enter = &intel_idle }, | |
294 | { | |
295 | .name = "C1E-IVT", | |
296 | .desc = "MWAIT 0x01", | |
297 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, | |
298 | .exit_latency = 10, | |
299 | .target_residency = 80, | |
300 | .enter = &intel_idle }, | |
301 | { | |
302 | .name = "C3-IVT", | |
303 | .desc = "MWAIT 0x10", | |
304 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
305 | .exit_latency = 59, | |
306 | .target_residency = 156, | |
307 | .enter = &intel_idle }, | |
308 | { | |
309 | .name = "C6-IVT", | |
310 | .desc = "MWAIT 0x20", | |
311 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
312 | .exit_latency = 82, | |
313 | .target_residency = 300, | |
314 | .enter = &intel_idle }, | |
315 | { | |
316 | .enter = NULL } | |
317 | }; | |
318 | ||
319 | static struct cpuidle_state ivt_cstates_4s[] = { | |
320 | { | |
321 | .name = "C1-IVT-4S", | |
322 | .desc = "MWAIT 0x00", | |
323 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, | |
324 | .exit_latency = 1, | |
325 | .target_residency = 1, | |
326 | .enter = &intel_idle }, | |
327 | { | |
328 | .name = "C1E-IVT-4S", | |
329 | .desc = "MWAIT 0x01", | |
330 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, | |
331 | .exit_latency = 10, | |
332 | .target_residency = 250, | |
333 | .enter = &intel_idle }, | |
334 | { | |
335 | .name = "C3-IVT-4S", | |
336 | .desc = "MWAIT 0x10", | |
337 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
338 | .exit_latency = 59, | |
339 | .target_residency = 300, | |
340 | .enter = &intel_idle }, | |
341 | { | |
342 | .name = "C6-IVT-4S", | |
343 | .desc = "MWAIT 0x20", | |
344 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
345 | .exit_latency = 84, | |
346 | .target_residency = 400, | |
347 | .enter = &intel_idle }, | |
348 | { | |
349 | .enter = NULL } | |
350 | }; | |
351 | ||
352 | static struct cpuidle_state ivt_cstates_8s[] = { | |
353 | { | |
354 | .name = "C1-IVT-8S", | |
355 | .desc = "MWAIT 0x00", | |
356 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, | |
357 | .exit_latency = 1, | |
358 | .target_residency = 1, | |
359 | .enter = &intel_idle }, | |
360 | { | |
361 | .name = "C1E-IVT-8S", | |
362 | .desc = "MWAIT 0x01", | |
363 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, | |
364 | .exit_latency = 10, | |
365 | .target_residency = 500, | |
366 | .enter = &intel_idle }, | |
367 | { | |
368 | .name = "C3-IVT-8S", | |
369 | .desc = "MWAIT 0x10", | |
370 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
371 | .exit_latency = 59, | |
372 | .target_residency = 600, | |
373 | .enter = &intel_idle }, | |
374 | { | |
375 | .name = "C6-IVT-8S", | |
376 | .desc = "MWAIT 0x20", | |
377 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
378 | .exit_latency = 88, | |
379 | .target_residency = 700, | |
380 | .enter = &intel_idle }, | |
381 | { | |
382 | .enter = NULL } | |
383 | }; | |
384 | ||
ba0dc81e | 385 | static struct cpuidle_state hsw_cstates[] = { |
e022e7eb | 386 | { |
85a4d2d4 LB |
387 | .name = "C1-HSW", |
388 | .desc = "MWAIT 0x00", | |
389 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, | |
390 | .exit_latency = 2, | |
391 | .target_residency = 2, | |
392 | .enter = &intel_idle }, | |
32e95180 LB |
393 | { |
394 | .name = "C1E-HSW", | |
395 | .desc = "MWAIT 0x01", | |
396 | .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_TIME_VALID, | |
397 | .exit_latency = 10, | |
398 | .target_residency = 20, | |
399 | .enter = &intel_idle }, | |
e022e7eb | 400 | { |
85a4d2d4 LB |
401 | .name = "C3-HSW", |
402 | .desc = "MWAIT 0x10", | |
403 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
404 | .exit_latency = 33, | |
405 | .target_residency = 100, | |
406 | .enter = &intel_idle }, | |
e022e7eb | 407 | { |
85a4d2d4 LB |
408 | .name = "C6-HSW", |
409 | .desc = "MWAIT 0x20", | |
410 | .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
411 | .exit_latency = 133, | |
412 | .target_residency = 400, | |
413 | .enter = &intel_idle }, | |
e022e7eb | 414 | { |
85a4d2d4 LB |
415 | .name = "C7s-HSW", |
416 | .desc = "MWAIT 0x32", | |
417 | .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
418 | .exit_latency = 166, | |
419 | .target_residency = 500, | |
420 | .enter = &intel_idle }, | |
86239ceb LB |
421 | { |
422 | .name = "C8-HSW", | |
423 | .desc = "MWAIT 0x40", | |
424 | .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
425 | .exit_latency = 300, | |
426 | .target_residency = 900, | |
427 | .enter = &intel_idle }, | |
428 | { | |
429 | .name = "C9-HSW", | |
430 | .desc = "MWAIT 0x50", | |
431 | .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
432 | .exit_latency = 600, | |
433 | .target_residency = 1800, | |
434 | .enter = &intel_idle }, | |
435 | { | |
436 | .name = "C10-HSW", | |
437 | .desc = "MWAIT 0x60", | |
438 | .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, | |
439 | .exit_latency = 2600, | |
440 | .target_residency = 7700, | |
441 | .enter = &intel_idle }, | |
e022e7eb LB |
442 | { |
443 | .enter = NULL } | |
85a4d2d4 LB |
444 | }; |
445 | ||
ba0dc81e | 446 | static struct cpuidle_state atom_cstates[] = { |
e022e7eb | 447 | { |
32e95180 | 448 | .name = "C1E-ATM", |
26717172 | 449 | .desc = "MWAIT 0x00", |
b1beab48 | 450 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, |
32e95180 LB |
451 | .exit_latency = 10, |
452 | .target_residency = 20, | |
26717172 | 453 | .enter = &intel_idle }, |
e022e7eb | 454 | { |
15e123e5 | 455 | .name = "C2-ATM", |
26717172 | 456 | .desc = "MWAIT 0x10", |
b1beab48 | 457 | .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TIME_VALID, |
26717172 | 458 | .exit_latency = 20, |
26717172 LB |
459 | .target_residency = 80, |
460 | .enter = &intel_idle }, | |
e022e7eb | 461 | { |
15e123e5 | 462 | .name = "C4-ATM", |
26717172 | 463 | .desc = "MWAIT 0x30", |
b1beab48 | 464 | .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
26717172 | 465 | .exit_latency = 100, |
26717172 LB |
466 | .target_residency = 400, |
467 | .enter = &intel_idle }, | |
e022e7eb | 468 | { |
15e123e5 | 469 | .name = "C6-ATM", |
7fcca7d9 | 470 | .desc = "MWAIT 0x52", |
b1beab48 | 471 | .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
7fcca7d9 | 472 | .exit_latency = 140, |
7fcca7d9 LB |
473 | .target_residency = 560, |
474 | .enter = &intel_idle }, | |
e022e7eb LB |
475 | { |
476 | .enter = NULL } | |
26717172 | 477 | }; |
88390996 | 478 | static struct cpuidle_state avn_cstates[] = { |
fab04b22 LB |
479 | { |
480 | .name = "C1-AVN", | |
481 | .desc = "MWAIT 0x00", | |
482 | .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_TIME_VALID, | |
483 | .exit_latency = 2, | |
484 | .target_residency = 2, | |
485 | .enter = &intel_idle }, | |
486 | { | |
487 | .name = "C6-AVN", | |
488 | .desc = "MWAIT 0x51", | |
22e580d0 | 489 | .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED, |
fab04b22 LB |
490 | .exit_latency = 15, |
491 | .target_residency = 45, | |
492 | .enter = &intel_idle }, | |
88390996 JL |
493 | { |
494 | .enter = NULL } | |
fab04b22 | 495 | }; |
26717172 | 496 | |
26717172 LB |
497 | /** |
498 | * intel_idle | |
499 | * @dev: cpuidle_device | |
46bcfad7 | 500 | * @drv: cpuidle driver |
e978aa7d | 501 | * @index: index of cpuidle state |
26717172 | 502 | * |
63ff07be | 503 | * Must be called under local_irq_disable(). |
26717172 | 504 | */ |
46bcfad7 DD |
505 | static int intel_idle(struct cpuidle_device *dev, |
506 | struct cpuidle_driver *drv, int index) | |
26717172 LB |
507 | { |
508 | unsigned long ecx = 1; /* break on interrupt flag */ | |
46bcfad7 | 509 | struct cpuidle_state *state = &drv->states[index]; |
b1beab48 | 510 | unsigned long eax = flg2MWAIT(state->flags); |
26717172 | 511 | unsigned int cstate; |
26717172 LB |
512 | int cpu = smp_processor_id(); |
513 | ||
514 | cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1; | |
515 | ||
6110a1f4 | 516 | /* |
c8381cc3 LB |
517 | * leave_mm() to avoid costly and often unnecessary wakeups |
518 | * for flushing the user TLB's associated with the active mm. | |
6110a1f4 | 519 | */ |
c8381cc3 | 520 | if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED) |
6110a1f4 SS |
521 | leave_mm(cpu); |
522 | ||
26717172 LB |
523 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) |
524 | clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu); | |
525 | ||
16824255 | 526 | mwait_idle_with_hints(eax, ecx); |
26717172 | 527 | |
26717172 LB |
528 | if (!(lapic_timer_reliable_states & (1 << (cstate)))) |
529 | clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu); | |
530 | ||
e978aa7d | 531 | return index; |
26717172 LB |
532 | } |
533 | ||
2a2d31c8 SL |
534 | static void __setup_broadcast_timer(void *arg) |
535 | { | |
536 | unsigned long reason = (unsigned long)arg; | |
537 | int cpu = smp_processor_id(); | |
538 | ||
539 | reason = reason ? | |
540 | CLOCK_EVT_NOTIFY_BROADCAST_ON : CLOCK_EVT_NOTIFY_BROADCAST_OFF; | |
541 | ||
542 | clockevents_notify(reason, &cpu); | |
543 | } | |
544 | ||
25ac7761 DL |
545 | static int cpu_hotplug_notify(struct notifier_block *n, |
546 | unsigned long action, void *hcpu) | |
2a2d31c8 SL |
547 | { |
548 | int hotcpu = (unsigned long)hcpu; | |
25ac7761 | 549 | struct cpuidle_device *dev; |
2a2d31c8 | 550 | |
e2401453 | 551 | switch (action & ~CPU_TASKS_FROZEN) { |
2a2d31c8 | 552 | case CPU_ONLINE: |
25ac7761 DL |
553 | |
554 | if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) | |
555 | smp_call_function_single(hotcpu, __setup_broadcast_timer, | |
556 | (void *)true, 1); | |
557 | ||
558 | /* | |
559 | * Some systems can hotplug a cpu at runtime after | |
560 | * the kernel has booted, we have to initialize the | |
561 | * driver in this case | |
562 | */ | |
563 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu); | |
564 | if (!dev->registered) | |
565 | intel_idle_cpu_init(hotcpu); | |
566 | ||
2a2d31c8 | 567 | break; |
2a2d31c8 SL |
568 | } |
569 | return NOTIFY_OK; | |
570 | } | |
571 | ||
25ac7761 DL |
572 | static struct notifier_block cpu_hotplug_notifier = { |
573 | .notifier_call = cpu_hotplug_notify, | |
2a2d31c8 SL |
574 | }; |
575 | ||
14796fca LB |
576 | static void auto_demotion_disable(void *dummy) |
577 | { | |
578 | unsigned long long msr_bits; | |
579 | ||
580 | rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); | |
b66b8b9a | 581 | msr_bits &= ~(icpu->auto_demotion_disable_flags); |
14796fca LB |
582 | wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); |
583 | } | |
32e95180 LB |
584 | static void c1e_promotion_disable(void *dummy) |
585 | { | |
586 | unsigned long long msr_bits; | |
587 | ||
588 | rdmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
589 | msr_bits &= ~0x2; | |
590 | wrmsrl(MSR_IA32_POWER_CTL, msr_bits); | |
591 | } | |
14796fca | 592 | |
b66b8b9a AK |
593 | static const struct idle_cpu idle_cpu_nehalem = { |
594 | .state_table = nehalem_cstates, | |
b66b8b9a | 595 | .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE, |
32e95180 | 596 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
597 | }; |
598 | ||
599 | static const struct idle_cpu idle_cpu_atom = { | |
600 | .state_table = atom_cstates, | |
601 | }; | |
602 | ||
603 | static const struct idle_cpu idle_cpu_lincroft = { | |
604 | .state_table = atom_cstates, | |
605 | .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, | |
606 | }; | |
607 | ||
608 | static const struct idle_cpu idle_cpu_snb = { | |
609 | .state_table = snb_cstates, | |
32e95180 | 610 | .disable_promotion_to_c1e = true, |
b66b8b9a AK |
611 | }; |
612 | ||
718987d6 LB |
613 | static const struct idle_cpu idle_cpu_byt = { |
614 | .state_table = byt_cstates, | |
615 | .disable_promotion_to_c1e = true, | |
616 | }; | |
617 | ||
6edab08c LB |
618 | static const struct idle_cpu idle_cpu_ivb = { |
619 | .state_table = ivb_cstates, | |
32e95180 | 620 | .disable_promotion_to_c1e = true, |
6edab08c LB |
621 | }; |
622 | ||
0138d8f0 LB |
623 | static const struct idle_cpu idle_cpu_ivt = { |
624 | .state_table = ivt_cstates, | |
625 | .disable_promotion_to_c1e = true, | |
626 | }; | |
627 | ||
85a4d2d4 LB |
628 | static const struct idle_cpu idle_cpu_hsw = { |
629 | .state_table = hsw_cstates, | |
32e95180 | 630 | .disable_promotion_to_c1e = true, |
85a4d2d4 LB |
631 | }; |
632 | ||
fab04b22 LB |
633 | static const struct idle_cpu idle_cpu_avn = { |
634 | .state_table = avn_cstates, | |
635 | .disable_promotion_to_c1e = true, | |
636 | }; | |
637 | ||
b66b8b9a AK |
638 | #define ICPU(model, cpu) \ |
639 | { X86_VENDOR_INTEL, 6, model, X86_FEATURE_MWAIT, (unsigned long)&cpu } | |
640 | ||
641 | static const struct x86_cpu_id intel_idle_ids[] = { | |
642 | ICPU(0x1a, idle_cpu_nehalem), | |
643 | ICPU(0x1e, idle_cpu_nehalem), | |
644 | ICPU(0x1f, idle_cpu_nehalem), | |
8bf11938 BH |
645 | ICPU(0x25, idle_cpu_nehalem), |
646 | ICPU(0x2c, idle_cpu_nehalem), | |
647 | ICPU(0x2e, idle_cpu_nehalem), | |
b66b8b9a AK |
648 | ICPU(0x1c, idle_cpu_atom), |
649 | ICPU(0x26, idle_cpu_lincroft), | |
8bf11938 | 650 | ICPU(0x2f, idle_cpu_nehalem), |
b66b8b9a AK |
651 | ICPU(0x2a, idle_cpu_snb), |
652 | ICPU(0x2d, idle_cpu_snb), | |
acead1b0 | 653 | ICPU(0x36, idle_cpu_atom), |
718987d6 | 654 | ICPU(0x37, idle_cpu_byt), |
6edab08c | 655 | ICPU(0x3a, idle_cpu_ivb), |
0138d8f0 | 656 | ICPU(0x3e, idle_cpu_ivt), |
85a4d2d4 LB |
657 | ICPU(0x3c, idle_cpu_hsw), |
658 | ICPU(0x3f, idle_cpu_hsw), | |
659 | ICPU(0x45, idle_cpu_hsw), | |
0b15841b | 660 | ICPU(0x46, idle_cpu_hsw), |
fab04b22 | 661 | ICPU(0x4D, idle_cpu_avn), |
b66b8b9a AK |
662 | {} |
663 | }; | |
664 | MODULE_DEVICE_TABLE(x86cpu, intel_idle_ids); | |
665 | ||
26717172 LB |
666 | /* |
667 | * intel_idle_probe() | |
668 | */ | |
00f3e755 | 669 | static int __init intel_idle_probe(void) |
26717172 | 670 | { |
c4236282 | 671 | unsigned int eax, ebx, ecx; |
b66b8b9a | 672 | const struct x86_cpu_id *id; |
26717172 LB |
673 | |
674 | if (max_cstate == 0) { | |
675 | pr_debug(PREFIX "disabled\n"); | |
676 | return -EPERM; | |
677 | } | |
678 | ||
b66b8b9a AK |
679 | id = x86_match_cpu(intel_idle_ids); |
680 | if (!id) { | |
681 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && | |
682 | boot_cpu_data.x86 == 6) | |
683 | pr_debug(PREFIX "does not run on family %d model %d\n", | |
684 | boot_cpu_data.x86, boot_cpu_data.x86_model); | |
26717172 | 685 | return -ENODEV; |
b66b8b9a | 686 | } |
26717172 LB |
687 | |
688 | if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF) | |
689 | return -ENODEV; | |
690 | ||
c4236282 | 691 | cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates); |
26717172 LB |
692 | |
693 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) || | |
5c2a9f06 TR |
694 | !(ecx & CPUID5_ECX_INTERRUPT_BREAK) || |
695 | !mwait_substates) | |
26717172 | 696 | return -ENODEV; |
26717172 | 697 | |
c4236282 | 698 | pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates); |
26717172 | 699 | |
b66b8b9a AK |
700 | icpu = (const struct idle_cpu *)id->driver_data; |
701 | cpuidle_state_table = icpu->state_table; | |
26717172 | 702 | |
56b9aea3 | 703 | if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ |
2a2d31c8 | 704 | lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; |
25ac7761 | 705 | else |
39a74fde | 706 | on_each_cpu(__setup_broadcast_timer, (void *)true, 1); |
25ac7761 | 707 | |
26717172 LB |
708 | pr_debug(PREFIX "v" INTEL_IDLE_VERSION |
709 | " model 0x%X\n", boot_cpu_data.x86_model); | |
710 | ||
711 | pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n", | |
712 | lapic_timer_reliable_states); | |
713 | return 0; | |
714 | } | |
715 | ||
716 | /* | |
717 | * intel_idle_cpuidle_devices_uninit() | |
718 | * unregister, free cpuidle_devices | |
719 | */ | |
720 | static void intel_idle_cpuidle_devices_uninit(void) | |
721 | { | |
722 | int i; | |
723 | struct cpuidle_device *dev; | |
724 | ||
725 | for_each_online_cpu(i) { | |
726 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, i); | |
727 | cpuidle_unregister_device(dev); | |
728 | } | |
729 | ||
730 | free_percpu(intel_idle_cpuidle_devices); | |
731 | return; | |
732 | } | |
0138d8f0 LB |
733 | |
734 | /* | |
735 | * intel_idle_state_table_update() | |
736 | * | |
737 | * Update the default state_table for this CPU-id | |
738 | * | |
739 | * Currently used to access tuned IVT multi-socket targets | |
740 | * Assumption: num_sockets == (max_package_num + 1) | |
741 | */ | |
742 | void intel_idle_state_table_update(void) | |
743 | { | |
744 | /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */ | |
745 | if (boot_cpu_data.x86_model == 0x3e) { /* IVT */ | |
746 | int cpu, package_num, num_sockets = 1; | |
747 | ||
748 | for_each_online_cpu(cpu) { | |
749 | package_num = topology_physical_package_id(cpu); | |
750 | if (package_num + 1 > num_sockets) { | |
751 | num_sockets = package_num + 1; | |
752 | ||
753 | if (num_sockets > 4) | |
754 | cpuidle_state_table = ivt_cstates_8s; | |
755 | return; | |
756 | } | |
757 | } | |
758 | ||
759 | if (num_sockets > 2) | |
760 | cpuidle_state_table = ivt_cstates_4s; | |
761 | /* else, 1 and 2 socket systems use default ivt_cstates */ | |
762 | } | |
763 | return; | |
764 | } | |
765 | ||
46bcfad7 DD |
766 | /* |
767 | * intel_idle_cpuidle_driver_init() | |
768 | * allocate, initialize cpuidle_states | |
769 | */ | |
00f3e755 | 770 | static int __init intel_idle_cpuidle_driver_init(void) |
46bcfad7 DD |
771 | { |
772 | int cstate; | |
773 | struct cpuidle_driver *drv = &intel_idle_driver; | |
774 | ||
0138d8f0 LB |
775 | intel_idle_state_table_update(); |
776 | ||
46bcfad7 DD |
777 | drv->state_count = 1; |
778 | ||
e022e7eb | 779 | for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) { |
24bfa950 | 780 | int num_substates, mwait_hint, mwait_cstate; |
46bcfad7 | 781 | |
e022e7eb LB |
782 | if (cpuidle_state_table[cstate].enter == NULL) |
783 | break; | |
784 | ||
785 | if (cstate + 1 > max_cstate) { | |
46bcfad7 DD |
786 | printk(PREFIX "max_cstate %d reached\n", |
787 | max_cstate); | |
788 | break; | |
789 | } | |
790 | ||
e022e7eb LB |
791 | mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags); |
792 | mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint); | |
e022e7eb | 793 | |
24bfa950 | 794 | /* number of sub-states for this state in CPUID.MWAIT */ |
e022e7eb | 795 | num_substates = (mwait_substates >> ((mwait_cstate + 1) * 4)) |
46bcfad7 | 796 | & MWAIT_SUBSTATE_MASK; |
e022e7eb | 797 | |
24bfa950 LB |
798 | /* if NO sub-states for this state in CPUID, skip it */ |
799 | if (num_substates == 0) | |
46bcfad7 | 800 | continue; |
46bcfad7 | 801 | |
e022e7eb | 802 | if (((mwait_cstate + 1) > 2) && |
46bcfad7 DD |
803 | !boot_cpu_has(X86_FEATURE_NONSTOP_TSC)) |
804 | mark_tsc_unstable("TSC halts in idle" | |
805 | " states deeper than C2"); | |
806 | ||
807 | drv->states[drv->state_count] = /* structure copy */ | |
808 | cpuidle_state_table[cstate]; | |
809 | ||
810 | drv->state_count += 1; | |
811 | } | |
812 | ||
b66b8b9a | 813 | if (icpu->auto_demotion_disable_flags) |
39a74fde | 814 | on_each_cpu(auto_demotion_disable, NULL, 1); |
46bcfad7 | 815 | |
32e95180 LB |
816 | if (icpu->disable_promotion_to_c1e) /* each-cpu is redundant */ |
817 | on_each_cpu(c1e_promotion_disable, NULL, 1); | |
818 | ||
46bcfad7 DD |
819 | return 0; |
820 | } | |
821 | ||
822 | ||
26717172 | 823 | /* |
65b7f839 | 824 | * intel_idle_cpu_init() |
26717172 | 825 | * allocate, initialize, register cpuidle_devices |
65b7f839 | 826 | * @cpu: cpu/core to initialize |
26717172 | 827 | */ |
25ac7761 | 828 | static int intel_idle_cpu_init(int cpu) |
26717172 | 829 | { |
26717172 LB |
830 | struct cpuidle_device *dev; |
831 | ||
65b7f839 | 832 | dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); |
26717172 | 833 | |
65b7f839 | 834 | dev->cpu = cpu; |
26717172 | 835 | |
65b7f839 TR |
836 | if (cpuidle_register_device(dev)) { |
837 | pr_debug(PREFIX "cpuidle_register_device %d failed!\n", cpu); | |
838 | intel_idle_cpuidle_devices_uninit(); | |
839 | return -EIO; | |
26717172 LB |
840 | } |
841 | ||
b66b8b9a | 842 | if (icpu->auto_demotion_disable_flags) |
65b7f839 TR |
843 | smp_call_function_single(cpu, auto_demotion_disable, NULL, 1); |
844 | ||
dbf87ab8 BZ |
845 | if (icpu->disable_promotion_to_c1e) |
846 | smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1); | |
847 | ||
26717172 LB |
848 | return 0; |
849 | } | |
26717172 LB |
850 | |
851 | static int __init intel_idle_init(void) | |
852 | { | |
65b7f839 | 853 | int retval, i; |
26717172 | 854 | |
d1896049 TR |
855 | /* Do not load intel_idle at all for now if idle= is passed */ |
856 | if (boot_option_idle_override != IDLE_NO_OVERRIDE) | |
857 | return -ENODEV; | |
858 | ||
26717172 LB |
859 | retval = intel_idle_probe(); |
860 | if (retval) | |
861 | return retval; | |
862 | ||
46bcfad7 | 863 | intel_idle_cpuidle_driver_init(); |
26717172 LB |
864 | retval = cpuidle_register_driver(&intel_idle_driver); |
865 | if (retval) { | |
3735d524 | 866 | struct cpuidle_driver *drv = cpuidle_get_driver(); |
26717172 | 867 | printk(KERN_DEBUG PREFIX "intel_idle yielding to %s", |
3735d524 | 868 | drv ? drv->name : "none"); |
26717172 LB |
869 | return retval; |
870 | } | |
871 | ||
65b7f839 TR |
872 | intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device); |
873 | if (intel_idle_cpuidle_devices == NULL) | |
874 | return -ENOMEM; | |
875 | ||
07494d54 SB |
876 | cpu_notifier_register_begin(); |
877 | ||
65b7f839 TR |
878 | for_each_online_cpu(i) { |
879 | retval = intel_idle_cpu_init(i); | |
880 | if (retval) { | |
07494d54 | 881 | cpu_notifier_register_done(); |
65b7f839 TR |
882 | cpuidle_unregister_driver(&intel_idle_driver); |
883 | return retval; | |
884 | } | |
26717172 | 885 | } |
07494d54 SB |
886 | __register_cpu_notifier(&cpu_hotplug_notifier); |
887 | ||
888 | cpu_notifier_register_done(); | |
26717172 LB |
889 | |
890 | return 0; | |
891 | } | |
892 | ||
893 | static void __exit intel_idle_exit(void) | |
894 | { | |
895 | intel_idle_cpuidle_devices_uninit(); | |
896 | cpuidle_unregister_driver(&intel_idle_driver); | |
897 | ||
07494d54 | 898 | cpu_notifier_register_begin(); |
25ac7761 DL |
899 | |
900 | if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) | |
39a74fde | 901 | on_each_cpu(__setup_broadcast_timer, (void *)false, 1); |
07494d54 SB |
902 | __unregister_cpu_notifier(&cpu_hotplug_notifier); |
903 | ||
904 | cpu_notifier_register_done(); | |
2a2d31c8 | 905 | |
26717172 LB |
906 | return; |
907 | } | |
908 | ||
909 | module_init(intel_idle_init); | |
910 | module_exit(intel_idle_exit); | |
911 | ||
26717172 | 912 | module_param(max_cstate, int, 0444); |
26717172 LB |
913 | |
914 | MODULE_AUTHOR("Len Brown <len.brown@intel.com>"); | |
915 | MODULE_DESCRIPTION("Cpuidle driver for Intel Hardware v" INTEL_IDLE_VERSION); | |
916 | MODULE_LICENSE("GPL"); |