Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
f05e798a DH |
2 | #ifndef _ASM_X86_SPECIAL_INSNS_H |
3 | #define _ASM_X86_SPECIAL_INSNS_H | |
4 | ||
f05e798a | 5 | #ifdef __KERNEL__ |
719d359d | 6 | #include <asm/nops.h> |
873d50d5 | 7 | #include <asm/processor-flags.h> |
154fcf3a TG |
8 | |
9 | #include <linux/errno.h> | |
410367e3 | 10 | #include <linux/irqflags.h> |
873d50d5 | 11 | #include <linux/jump_label.h> |
719d359d | 12 | |
f05e798a | 13 | /* |
aa5cacdc AS |
14 | * The compiler should not reorder volatile asm statements with respect to each |
15 | * other: they should execute in program order. However GCC 4.9.x and 5.x have | |
16 | * a bug (which was fixed in 8.1, 7.3 and 6.5) where they might reorder | |
17 | * volatile asm. The write functions are not affected since they have memory | |
18 | * clobbers preventing reordering. To prevent reads from being reordered with | |
19 | * respect to writes, use a dummy memory operand. | |
f05e798a | 20 | */ |
aa5cacdc AS |
21 | |
22 | #define __FORCE_ORDER "m"(*(unsigned int *)0x1000UL) | |
f05e798a | 23 | |
7652ac92 | 24 | void native_write_cr0(unsigned long val); |
873d50d5 | 25 | |
f05e798a DH |
26 | static inline unsigned long native_read_cr0(void) |
27 | { | |
28 | unsigned long val; | |
aa5cacdc | 29 | asm volatile("mov %%cr0,%0\n\t" : "=r" (val) : __FORCE_ORDER); |
f05e798a DH |
30 | return val; |
31 | } | |
32 | ||
2823e83a | 33 | static __always_inline unsigned long native_read_cr2(void) |
f05e798a DH |
34 | { |
35 | unsigned long val; | |
aa5cacdc | 36 | asm volatile("mov %%cr2,%0\n\t" : "=r" (val) : __FORCE_ORDER); |
f05e798a DH |
37 | return val; |
38 | } | |
39 | ||
2823e83a | 40 | static __always_inline void native_write_cr2(unsigned long val) |
f05e798a | 41 | { |
aa5cacdc | 42 | asm volatile("mov %0,%%cr2": : "r" (val) : "memory"); |
f05e798a DH |
43 | } |
44 | ||
6c690ee1 | 45 | static inline unsigned long __native_read_cr3(void) |
f05e798a DH |
46 | { |
47 | unsigned long val; | |
aa5cacdc | 48 | asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER); |
f05e798a DH |
49 | return val; |
50 | } | |
51 | ||
52 | static inline void native_write_cr3(unsigned long val) | |
53 | { | |
aa5cacdc | 54 | asm volatile("mov %0,%%cr3": : "r" (val) : "memory"); |
f05e798a DH |
55 | } |
56 | ||
57 | static inline unsigned long native_read_cr4(void) | |
58 | { | |
59 | unsigned long val; | |
f05e798a | 60 | #ifdef CONFIG_X86_32 |
1ef55be1 AL |
61 | /* |
62 | * This could fault if CR4 does not exist. Non-existent CR4 | |
63 | * is functionally equivalent to CR4 == 0. Keep it simple and pretend | |
64 | * that CR4 == 0 on CPUs that don't have CR4. | |
65 | */ | |
f05e798a DH |
66 | asm volatile("1: mov %%cr4, %0\n" |
67 | "2:\n" | |
68 | _ASM_EXTABLE(1b, 2b) | |
aa5cacdc | 69 | : "=r" (val) : "0" (0), __FORCE_ORDER); |
f05e798a | 70 | #else |
1ef55be1 | 71 | /* CR4 always exists on x86_64. */ |
aa5cacdc | 72 | asm volatile("mov %%cr4,%0\n\t" : "=r" (val) : __FORCE_ORDER); |
f05e798a DH |
73 | #endif |
74 | return val; | |
75 | } | |
76 | ||
7652ac92 | 77 | void native_write_cr4(unsigned long val); |
f05e798a | 78 | |
a927cb83 | 79 | #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS |
c806e887 | 80 | static inline u32 rdpkru(void) |
a927cb83 DH |
81 | { |
82 | u32 ecx = 0; | |
83 | u32 edx, pkru; | |
84 | ||
85 | /* | |
86 | * "rdpkru" instruction. Places PKRU contents in to EAX, | |
87 | * clears EDX and requires that ecx=0. | |
88 | */ | |
89 | asm volatile(".byte 0x0f,0x01,0xee\n\t" | |
90 | : "=a" (pkru), "=d" (edx) | |
91 | : "c" (ecx)); | |
92 | return pkru; | |
93 | } | |
9e90199c | 94 | |
c806e887 | 95 | static inline void wrpkru(u32 pkru) |
9e90199c XG |
96 | { |
97 | u32 ecx = 0, edx = 0; | |
98 | ||
99 | /* | |
100 | * "wrpkru" instruction. Loads contents in EAX to PKRU, | |
101 | * requires that ecx = edx = 0. | |
102 | */ | |
103 | asm volatile(".byte 0x0f,0x01,0xef\n\t" | |
104 | : : "a" (pkru), "c"(ecx), "d"(edx)); | |
105 | } | |
c806e887 | 106 | |
a927cb83 | 107 | #else |
c806e887 | 108 | static inline u32 rdpkru(void) |
a927cb83 DH |
109 | { |
110 | return 0; | |
111 | } | |
9e90199c | 112 | |
72a6c08c | 113 | static inline void wrpkru(u32 pkru) |
9e90199c XG |
114 | { |
115 | } | |
a927cb83 DH |
116 | #endif |
117 | ||
10a09940 | 118 | static __always_inline void native_wbinvd(void) |
f05e798a DH |
119 | { |
120 | asm volatile("wbinvd": : :"memory"); | |
121 | } | |
122 | ||
87930019 JG |
123 | static inline unsigned long __read_cr4(void) |
124 | { | |
125 | return native_read_cr4(); | |
126 | } | |
127 | ||
fdc0269e | 128 | #ifdef CONFIG_PARAVIRT_XXL |
f05e798a | 129 | #include <asm/paravirt.h> |
fdc0269e | 130 | #else |
f05e798a DH |
131 | |
132 | static inline unsigned long read_cr0(void) | |
133 | { | |
134 | return native_read_cr0(); | |
135 | } | |
136 | ||
137 | static inline void write_cr0(unsigned long x) | |
138 | { | |
139 | native_write_cr0(x); | |
140 | } | |
141 | ||
2823e83a | 142 | static __always_inline unsigned long read_cr2(void) |
f05e798a DH |
143 | { |
144 | return native_read_cr2(); | |
145 | } | |
146 | ||
2823e83a | 147 | static __always_inline void write_cr2(unsigned long x) |
f05e798a DH |
148 | { |
149 | native_write_cr2(x); | |
150 | } | |
151 | ||
6c690ee1 AL |
152 | /* |
153 | * Careful! CR3 contains more than just an address. You probably want | |
154 | * read_cr3_pa() instead. | |
155 | */ | |
156 | static inline unsigned long __read_cr3(void) | |
f05e798a | 157 | { |
6c690ee1 | 158 | return __native_read_cr3(); |
f05e798a DH |
159 | } |
160 | ||
161 | static inline void write_cr3(unsigned long x) | |
162 | { | |
163 | native_write_cr3(x); | |
164 | } | |
165 | ||
1e02ce4c | 166 | static inline void __write_cr4(unsigned long x) |
f05e798a DH |
167 | { |
168 | native_write_cr4(x); | |
169 | } | |
170 | ||
10a09940 | 171 | static __always_inline void wbinvd(void) |
f05e798a DH |
172 | { |
173 | native_wbinvd(); | |
174 | } | |
175 | ||
fdc0269e | 176 | #endif /* CONFIG_PARAVIRT_XXL */ |
f05e798a | 177 | |
2ec8efe6 | 178 | static __always_inline void clflush(volatile void *__p) |
f05e798a DH |
179 | { |
180 | asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); | |
181 | } | |
182 | ||
171699f7 RZ |
183 | static inline void clflushopt(volatile void *__p) |
184 | { | |
a89dfde3 | 185 | alternative_io(".byte 0x3e; clflush %P0", |
171699f7 RZ |
186 | ".byte 0x66; clflush %P0", |
187 | X86_FEATURE_CLFLUSHOPT, | |
188 | "+m" (*(volatile char __force *)__p)); | |
189 | } | |
190 | ||
d9dc64f3 RZ |
191 | static inline void clwb(volatile void *__p) |
192 | { | |
193 | volatile struct { char x[64]; } *p = __p; | |
194 | ||
195 | asm volatile(ALTERNATIVE_2( | |
a89dfde3 | 196 | ".byte 0x3e; clflush (%[pax])", |
d9dc64f3 RZ |
197 | ".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */ |
198 | X86_FEATURE_CLFLUSHOPT, | |
199 | ".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */ | |
200 | X86_FEATURE_CLWB) | |
201 | : [p] "+m" (*p) | |
202 | : [pax] "a" (p)); | |
203 | } | |
204 | ||
92805476 RE |
205 | #ifdef CONFIG_X86_USER_SHADOW_STACK |
206 | static inline int write_user_shstk_64(u64 __user *addr, u64 val) | |
207 | { | |
4356e9f8 | 208 | asm goto("1: wrussq %[val], (%[addr])\n" |
92805476 RE |
209 | _ASM_EXTABLE(1b, %l[fail]) |
210 | :: [addr] "r" (addr), [val] "r" (val) | |
211 | :: fail); | |
212 | return 0; | |
213 | fail: | |
214 | return -EFAULT; | |
215 | } | |
216 | #endif /* CONFIG_X86_USER_SHADOW_STACK */ | |
217 | ||
f05e798a DH |
218 | #define nop() asm volatile ("nop") |
219 | ||
bf9c912f RN |
220 | static inline void serialize(void) |
221 | { | |
222 | /* Instruction opcode for SERIALIZE; supported in binutils >= 2.35. */ | |
223 | asm volatile(".byte 0xf, 0x1, 0xe8" ::: "memory"); | |
224 | } | |
225 | ||
0888e103 | 226 | /* The dst parameter must be 64-bytes aligned */ |
5bdd1818 | 227 | static inline void movdir64b(void *dst, const void *src) |
0888e103 DJ |
228 | { |
229 | const struct { char _[64]; } *__src = src; | |
5bdd1818 | 230 | struct { char _[64]; } *__dst = dst; |
0888e103 DJ |
231 | |
232 | /* | |
233 | * MOVDIR64B %(rdx), rax. | |
234 | * | |
235 | * Both __src and __dst must be memory constraints in order to tell the | |
236 | * compiler that no other memory accesses should be reordered around | |
237 | * this one. | |
238 | * | |
239 | * Also, both must be supplied as lvalues because this tells | |
240 | * the compiler what the object is (its size) the instruction accesses. | |
241 | * I.e., not the pointers but what they point to, thus the deref'ing '*'. | |
242 | */ | |
243 | asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02" | |
244 | : "+m" (*__dst) | |
245 | : "m" (*__src), "a" (__dst), "d" (__src)); | |
246 | } | |
247 | ||
5bdd1818 KH |
248 | static inline void movdir64b_io(void __iomem *dst, const void *src) |
249 | { | |
250 | movdir64b((void __force *)dst, src); | |
251 | } | |
252 | ||
7f5933f8 DJ |
253 | /** |
254 | * enqcmds - Enqueue a command in supervisor (CPL0) mode | |
255 | * @dst: destination, in MMIO space (must be 512-bit aligned) | |
256 | * @src: 512 bits memory operand | |
257 | * | |
258 | * The ENQCMDS instruction allows software to write a 512-bit command to | |
259 | * a 512-bit-aligned special MMIO region that supports the instruction. | |
260 | * A return status is loaded into the ZF flag in the RFLAGS register. | |
261 | * ZF = 0 equates to success, and ZF = 1 indicates retry or error. | |
262 | * | |
263 | * This function issues the ENQCMDS instruction to submit data from | |
264 | * kernel space to MMIO space, in a unit of 512 bits. Order of data access | |
265 | * is not guaranteed, nor is a memory barrier performed afterwards. It | |
266 | * returns 0 on success and -EAGAIN on failure. | |
267 | * | |
268 | * Warning: Do not use this helper unless your driver has checked that the | |
269 | * ENQCMDS instruction is supported on the platform and the device accepts | |
270 | * ENQCMDS. | |
271 | */ | |
272 | static inline int enqcmds(void __iomem *dst, const void *src) | |
273 | { | |
274 | const struct { char _[64]; } *__src = src; | |
5c99720b | 275 | struct { char _[64]; } __iomem *__dst = dst; |
d81ff5fe | 276 | bool zf; |
7f5933f8 DJ |
277 | |
278 | /* | |
279 | * ENQCMDS %(rdx), rax | |
280 | * | |
281 | * See movdir64b()'s comment on operand specification. | |
282 | */ | |
283 | asm volatile(".byte 0xf3, 0x0f, 0x38, 0xf8, 0x02, 0x66, 0x90" | |
284 | CC_SET(z) | |
285 | : CC_OUT(z) (zf), "+m" (*__dst) | |
286 | : "m" (*__src), "a" (__dst), "d" (__src)); | |
287 | ||
288 | /* Submission failure is indicated via EFLAGS.ZF=1 */ | |
289 | if (zf) | |
290 | return -EAGAIN; | |
291 | ||
292 | return 0; | |
293 | } | |
294 | ||
821ad23d | 295 | static __always_inline void tile_release(void) |
f17b1687 CB |
296 | { |
297 | /* | |
298 | * Instruction opcode for TILERELEASE; supported in binutils | |
299 | * version >= 2.36. | |
300 | */ | |
301 | asm volatile(".byte 0xc4, 0xe2, 0x78, 0x49, 0xc0"); | |
302 | } | |
303 | ||
f05e798a DH |
304 | #endif /* __KERNEL__ */ |
305 | ||
306 | #endif /* _ASM_X86_SPECIAL_INSNS_H */ |