Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
bc83cccc PA |
2 | #ifndef _ASM_X86_MWAIT_H |
3 | #define _ASM_X86_MWAIT_H | |
4 | ||
16824255 | 5 | #include <linux/sched.h> |
4c822698 | 6 | #include <linux/sched/idle.h> |
16824255 | 7 | |
cd4d09ec | 8 | #include <asm/cpufeature.h> |
07f07f55 | 9 | #include <asm/nospec-branch.h> |
cd4d09ec | 10 | |
bc83cccc PA |
11 | #define MWAIT_SUBSTATE_MASK 0xf |
12 | #define MWAIT_CSTATE_MASK 0xf | |
13 | #define MWAIT_SUBSTATE_SIZE 4 | |
e022e7eb LB |
14 | #define MWAIT_HINT2CSTATE(hint) (((hint) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) |
15 | #define MWAIT_HINT2SUBSTATE(hint) ((hint) & MWAIT_CSTATE_MASK) | |
bc83cccc PA |
16 | |
17 | #define CPUID_MWAIT_LEAF 5 | |
18 | #define CPUID5_ECX_EXTENSIONS_SUPPORTED 0x1 | |
19 | #define CPUID5_ECX_INTERRUPT_BREAK 0x2 | |
20 | ||
21 | #define MWAIT_ECX_INTERRUPT_BREAK 0x1 | |
f9675674 | 22 | #define MWAITX_ECX_TIMER_ENABLE BIT(1) |
e8824890 | 23 | #define MWAITX_MAX_WAIT_CYCLES UINT_MAX |
454de1e7 | 24 | #define MWAITX_DISABLE_CSTATES 0xf0 |
cec5f268 KMP |
25 | #define TPAUSE_C01_STATE 1 |
26 | #define TPAUSE_C02_STATE 0 | |
bc83cccc | 27 | |
16824255 PZ |
28 | static inline void __monitor(const void *eax, unsigned long ecx, |
29 | unsigned long edx) | |
30 | { | |
31 | /* "monitor %eax, %ecx, %edx;" */ | |
32 | asm volatile(".byte 0x0f, 0x01, 0xc8;" | |
33 | :: "a" (eax), "c" (ecx), "d"(edx)); | |
34 | } | |
35 | ||
f9675674 HR |
36 | static inline void __monitorx(const void *eax, unsigned long ecx, |
37 | unsigned long edx) | |
38 | { | |
39 | /* "monitorx %eax, %ecx, %edx;" */ | |
40 | asm volatile(".byte 0x0f, 0x01, 0xfa;" | |
41 | :: "a" (eax), "c" (ecx), "d"(edx)); | |
42 | } | |
43 | ||
16824255 PZ |
44 | static inline void __mwait(unsigned long eax, unsigned long ecx) |
45 | { | |
07f07f55 TG |
46 | mds_idle_clear_cpu_buffers(); |
47 | ||
16824255 PZ |
48 | /* "mwait %eax, %ecx;" */ |
49 | asm volatile(".byte 0x0f, 0x01, 0xc9;" | |
50 | :: "a" (eax), "c" (ecx)); | |
51 | } | |
52 | ||
f9675674 HR |
53 | /* |
54 | * MWAITX allows for a timer expiration to get the core out a wait state in | |
55 | * addition to the default MWAIT exit condition of a store appearing at a | |
56 | * monitored virtual address. | |
57 | * | |
58 | * Registers: | |
59 | * | |
60 | * MWAITX ECX[1]: enable timer if set | |
61 | * MWAITX EBX[31:0]: max wait time expressed in SW P0 clocks. The software P0 | |
62 | * frequency is the same as the TSC frequency. | |
63 | * | |
64 | * Below is a comparison between MWAIT and MWAITX on AMD processors: | |
65 | * | |
66 | * MWAIT MWAITX | |
67 | * opcode 0f 01 c9 | 0f 01 fb | |
68 | * ECX[0] value of RFLAGS.IF seen by instruction | |
69 | * ECX[1] unused/#GP if set | enable timer if set | |
70 | * ECX[31:2] unused/#GP if set | |
71 | * EAX unused (reserve for hint) | |
72 | * EBX[31:0] unused | max wait time (P0 clocks) | |
73 | * | |
74 | * MONITOR MONITORX | |
75 | * opcode 0f 01 c8 | 0f 01 fa | |
76 | * EAX (logical) address to monitor | |
77 | * ECX #GP if not zero | |
78 | */ | |
79 | static inline void __mwaitx(unsigned long eax, unsigned long ebx, | |
80 | unsigned long ecx) | |
81 | { | |
07f07f55 TG |
82 | /* No MDS buffer clear as this is AMD/HYGON only */ |
83 | ||
f9675674 HR |
84 | /* "mwaitx %eax, %ebx, %ecx;" */ |
85 | asm volatile(".byte 0x0f, 0x01, 0xfb;" | |
86 | :: "a" (eax), "b" (ebx), "c" (ecx)); | |
87 | } | |
88 | ||
b253149b LB |
89 | static inline void __sti_mwait(unsigned long eax, unsigned long ecx) |
90 | { | |
91 | trace_hardirqs_on(); | |
ab3765a0 ZD |
92 | |
93 | mds_idle_clear_cpu_buffers(); | |
b253149b LB |
94 | /* "mwait %eax, %ecx;" */ |
95 | asm volatile("sti; .byte 0x0f, 0x01, 0xc9;" | |
96 | :: "a" (eax), "c" (ecx)); | |
97 | } | |
98 | ||
16824255 PZ |
99 | /* |
100 | * This uses new MONITOR/MWAIT instructions on P4 processors with PNI, | |
101 | * which can obviate IPI to trigger checking of need_resched. | |
102 | * We execute MONITOR against need_resched and enter optimized wait state | |
103 | * through MWAIT. Whenever someone changes need_resched, we would be woken | |
104 | * up from MWAIT (without an IPI). | |
105 | * | |
106 | * New with Core Duo processors, MWAIT can take some hints based on CPU | |
107 | * capability. | |
108 | */ | |
109 | static inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx) | |
110 | { | |
08e237fa | 111 | if (static_cpu_has_bug(X86_BUG_MONITOR) || !current_set_polling_and_test()) { |
9b13a93d | 112 | if (static_cpu_has_bug(X86_BUG_CLFLUSH_MONITOR)) { |
7e98b719 | 113 | mb(); |
16824255 | 114 | clflush((void *)¤t_thread_info()->flags); |
7e98b719 PA |
115 | mb(); |
116 | } | |
16824255 PZ |
117 | |
118 | __monitor((void *)¤t_thread_info()->flags, 0, 0); | |
119 | if (!need_resched()) | |
120 | __mwait(eax, ecx); | |
121 | } | |
8cb75e0c | 122 | current_clr_polling(); |
16824255 PZ |
123 | } |
124 | ||
cec5f268 KMP |
125 | /* |
126 | * Caller can specify whether to enter C0.1 (low latency, less | |
127 | * power saving) or C0.2 state (saves more power, but longer wakeup | |
128 | * latency). This may be overridden by the IA32_UMWAIT_CONTROL MSR | |
129 | * which can force requests for C0.2 to be downgraded to C0.1. | |
130 | */ | |
131 | static inline void __tpause(u32 ecx, u32 edx, u32 eax) | |
132 | { | |
133 | /* "tpause %ecx, %edx, %eax;" */ | |
134 | #ifdef CONFIG_AS_TPAUSE | |
135 | asm volatile("tpause %%ecx\n" | |
136 | : | |
137 | : "c"(ecx), "d"(edx), "a"(eax)); | |
138 | #else | |
139 | asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1\t\n" | |
140 | : | |
141 | : "c"(ecx), "d"(edx), "a"(eax)); | |
142 | #endif | |
143 | } | |
144 | ||
bc83cccc | 145 | #endif /* _ASM_X86_MWAIT_H */ |