Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
f05e798a DH |
2 | #ifndef _ASM_X86_SPECIAL_INSNS_H |
3 | #define _ASM_X86_SPECIAL_INSNS_H | |
4 | ||
5 | ||
6 | #ifdef __KERNEL__ | |
7 | ||
719d359d | 8 | #include <asm/nops.h> |
873d50d5 KC |
9 | #include <asm/processor-flags.h> |
10 | #include <linux/jump_label.h> | |
719d359d | 11 | |
f05e798a DH |
12 | /* |
13 | * Volatile isn't enough to prevent the compiler from reordering the | |
14 | * read/write functions for the control registers and messing everything up. | |
15 | * A memory clobber would solve the problem, but would prevent reordering of | |
16 | * all loads stores around it, which can hurt performance. Solution is to | |
17 | * use a variable and mimic reads and writes to it to enforce serialization | |
18 | */ | |
1d10f6ee | 19 | extern unsigned long __force_order; |
f05e798a | 20 | |
7652ac92 | 21 | void native_write_cr0(unsigned long val); |
873d50d5 | 22 | |
f05e798a DH |
23 | static inline unsigned long native_read_cr0(void) |
24 | { | |
25 | unsigned long val; | |
26 | asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); | |
27 | return val; | |
28 | } | |
29 | ||
f05e798a DH |
30 | static inline unsigned long native_read_cr2(void) |
31 | { | |
32 | unsigned long val; | |
33 | asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); | |
34 | return val; | |
35 | } | |
36 | ||
37 | static inline void native_write_cr2(unsigned long val) | |
38 | { | |
39 | asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); | |
40 | } | |
41 | ||
6c690ee1 | 42 | static inline unsigned long __native_read_cr3(void) |
f05e798a DH |
43 | { |
44 | unsigned long val; | |
45 | asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); | |
46 | return val; | |
47 | } | |
48 | ||
49 | static inline void native_write_cr3(unsigned long val) | |
50 | { | |
51 | asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); | |
52 | } | |
53 | ||
54 | static inline unsigned long native_read_cr4(void) | |
55 | { | |
56 | unsigned long val; | |
f05e798a | 57 | #ifdef CONFIG_X86_32 |
1ef55be1 AL |
58 | /* |
59 | * This could fault if CR4 does not exist. Non-existent CR4 | |
60 | * is functionally equivalent to CR4 == 0. Keep it simple and pretend | |
61 | * that CR4 == 0 on CPUs that don't have CR4. | |
62 | */ | |
f05e798a DH |
63 | asm volatile("1: mov %%cr4, %0\n" |
64 | "2:\n" | |
65 | _ASM_EXTABLE(1b, 2b) | |
66 | : "=r" (val), "=m" (__force_order) : "0" (0)); | |
67 | #else | |
1ef55be1 AL |
68 | /* CR4 always exists on x86_64. */ |
69 | asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); | |
f05e798a DH |
70 | #endif |
71 | return val; | |
72 | } | |
73 | ||
7652ac92 | 74 | void native_write_cr4(unsigned long val); |
f05e798a DH |
75 | |
76 | #ifdef CONFIG_X86_64 | |
77 | static inline unsigned long native_read_cr8(void) | |
78 | { | |
79 | unsigned long cr8; | |
80 | asm volatile("movq %%cr8,%0" : "=r" (cr8)); | |
81 | return cr8; | |
82 | } | |
83 | ||
84 | static inline void native_write_cr8(unsigned long val) | |
85 | { | |
86 | asm volatile("movq %0,%%cr8" :: "r" (val) : "memory"); | |
87 | } | |
88 | #endif | |
89 | ||
a927cb83 | 90 | #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS |
c806e887 | 91 | static inline u32 rdpkru(void) |
a927cb83 DH |
92 | { |
93 | u32 ecx = 0; | |
94 | u32 edx, pkru; | |
95 | ||
96 | /* | |
97 | * "rdpkru" instruction. Places PKRU contents in to EAX, | |
98 | * clears EDX and requires that ecx=0. | |
99 | */ | |
100 | asm volatile(".byte 0x0f,0x01,0xee\n\t" | |
101 | : "=a" (pkru), "=d" (edx) | |
102 | : "c" (ecx)); | |
103 | return pkru; | |
104 | } | |
9e90199c | 105 | |
c806e887 | 106 | static inline void wrpkru(u32 pkru) |
9e90199c XG |
107 | { |
108 | u32 ecx = 0, edx = 0; | |
109 | ||
110 | /* | |
111 | * "wrpkru" instruction. Loads contents in EAX to PKRU, | |
112 | * requires that ecx = edx = 0. | |
113 | */ | |
114 | asm volatile(".byte 0x0f,0x01,0xef\n\t" | |
115 | : : "a" (pkru), "c"(ecx), "d"(edx)); | |
116 | } | |
c806e887 SAS |
117 | |
118 | static inline void __write_pkru(u32 pkru) | |
119 | { | |
577ff465 SAS |
120 | /* |
121 | * WRPKRU is relatively expensive compared to RDPKRU. | |
122 | * Avoid WRPKRU when it would not change the value. | |
123 | */ | |
124 | if (pkru == rdpkru()) | |
125 | return; | |
126 | ||
c806e887 SAS |
127 | wrpkru(pkru); |
128 | } | |
129 | ||
a927cb83 | 130 | #else |
c806e887 | 131 | static inline u32 rdpkru(void) |
a927cb83 DH |
132 | { |
133 | return 0; | |
134 | } | |
9e90199c XG |
135 | |
136 | static inline void __write_pkru(u32 pkru) | |
137 | { | |
138 | } | |
a927cb83 DH |
139 | #endif |
140 | ||
f05e798a DH |
141 | static inline void native_wbinvd(void) |
142 | { | |
143 | asm volatile("wbinvd": : :"memory"); | |
144 | } | |
145 | ||
277d5b40 | 146 | extern asmlinkage void native_load_gs_index(unsigned); |
f05e798a | 147 | |
87930019 JG |
148 | static inline unsigned long __read_cr4(void) |
149 | { | |
150 | return native_read_cr4(); | |
151 | } | |
152 | ||
fdc0269e | 153 | #ifdef CONFIG_PARAVIRT_XXL |
f05e798a | 154 | #include <asm/paravirt.h> |
fdc0269e | 155 | #else |
f05e798a DH |
156 | |
157 | static inline unsigned long read_cr0(void) | |
158 | { | |
159 | return native_read_cr0(); | |
160 | } | |
161 | ||
162 | static inline void write_cr0(unsigned long x) | |
163 | { | |
164 | native_write_cr0(x); | |
165 | } | |
166 | ||
167 | static inline unsigned long read_cr2(void) | |
168 | { | |
169 | return native_read_cr2(); | |
170 | } | |
171 | ||
172 | static inline void write_cr2(unsigned long x) | |
173 | { | |
174 | native_write_cr2(x); | |
175 | } | |
176 | ||
6c690ee1 AL |
177 | /* |
178 | * Careful! CR3 contains more than just an address. You probably want | |
179 | * read_cr3_pa() instead. | |
180 | */ | |
181 | static inline unsigned long __read_cr3(void) | |
f05e798a | 182 | { |
6c690ee1 | 183 | return __native_read_cr3(); |
f05e798a DH |
184 | } |
185 | ||
186 | static inline void write_cr3(unsigned long x) | |
187 | { | |
188 | native_write_cr3(x); | |
189 | } | |
190 | ||
1e02ce4c | 191 | static inline void __write_cr4(unsigned long x) |
f05e798a DH |
192 | { |
193 | native_write_cr4(x); | |
194 | } | |
195 | ||
196 | static inline void wbinvd(void) | |
197 | { | |
198 | native_wbinvd(); | |
199 | } | |
200 | ||
201 | #ifdef CONFIG_X86_64 | |
202 | ||
203 | static inline unsigned long read_cr8(void) | |
204 | { | |
205 | return native_read_cr8(); | |
206 | } | |
207 | ||
208 | static inline void write_cr8(unsigned long x) | |
209 | { | |
210 | native_write_cr8(x); | |
211 | } | |
212 | ||
213 | static inline void load_gs_index(unsigned selector) | |
214 | { | |
215 | native_load_gs_index(selector); | |
216 | } | |
217 | ||
218 | #endif | |
219 | ||
fdc0269e | 220 | #endif /* CONFIG_PARAVIRT_XXL */ |
f05e798a | 221 | |
f05e798a DH |
222 | static inline void clflush(volatile void *__p) |
223 | { | |
224 | asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); | |
225 | } | |
226 | ||
171699f7 RZ |
227 | static inline void clflushopt(volatile void *__p) |
228 | { | |
229 | alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0", | |
230 | ".byte 0x66; clflush %P0", | |
231 | X86_FEATURE_CLFLUSHOPT, | |
232 | "+m" (*(volatile char __force *)__p)); | |
233 | } | |
234 | ||
d9dc64f3 RZ |
235 | static inline void clwb(volatile void *__p) |
236 | { | |
237 | volatile struct { char x[64]; } *p = __p; | |
238 | ||
239 | asm volatile(ALTERNATIVE_2( | |
240 | ".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])", | |
241 | ".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */ | |
242 | X86_FEATURE_CLFLUSHOPT, | |
243 | ".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */ | |
244 | X86_FEATURE_CLWB) | |
245 | : [p] "+m" (*p) | |
246 | : [pax] "a" (p)); | |
247 | } | |
248 | ||
f05e798a DH |
249 | #define nop() asm volatile ("nop") |
250 | ||
251 | ||
252 | #endif /* __KERNEL__ */ | |
253 | ||
254 | #endif /* _ASM_X86_SPECIAL_INSNS_H */ |