Commit | Line | Data |
---|---|---|
f05e798a DH |
1 | #ifndef _ASM_X86_SPECIAL_INSNS_H |
2 | #define _ASM_X86_SPECIAL_INSNS_H | |
3 | ||
4 | ||
5 | #ifdef __KERNEL__ | |
6 | ||
719d359d RZ |
7 | #include <asm/nops.h> |
8 | ||
f05e798a DH |
9 | static inline void native_clts(void) |
10 | { | |
11 | asm volatile("clts"); | |
12 | } | |
13 | ||
14 | /* | |
15 | * Volatile isn't enough to prevent the compiler from reordering the | |
16 | * read/write functions for the control registers and messing everything up. | |
17 | * A memory clobber would solve the problem, but would prevent reordering of | |
18 | * all loads stores around it, which can hurt performance. Solution is to | |
19 | * use a variable and mimic reads and writes to it to enforce serialization | |
20 | */ | |
1d10f6ee | 21 | extern unsigned long __force_order; |
f05e798a DH |
22 | |
23 | static inline unsigned long native_read_cr0(void) | |
24 | { | |
25 | unsigned long val; | |
26 | asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); | |
27 | return val; | |
28 | } | |
29 | ||
30 | static inline void native_write_cr0(unsigned long val) | |
31 | { | |
32 | asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order)); | |
33 | } | |
34 | ||
35 | static inline unsigned long native_read_cr2(void) | |
36 | { | |
37 | unsigned long val; | |
38 | asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); | |
39 | return val; | |
40 | } | |
41 | ||
42 | static inline void native_write_cr2(unsigned long val) | |
43 | { | |
44 | asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); | |
45 | } | |
46 | ||
47 | static inline unsigned long native_read_cr3(void) | |
48 | { | |
49 | unsigned long val; | |
50 | asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); | |
51 | return val; | |
52 | } | |
53 | ||
54 | static inline void native_write_cr3(unsigned long val) | |
55 | { | |
56 | asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); | |
57 | } | |
58 | ||
59 | static inline unsigned long native_read_cr4(void) | |
60 | { | |
61 | unsigned long val; | |
62 | asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); | |
63 | return val; | |
64 | } | |
65 | ||
66 | static inline unsigned long native_read_cr4_safe(void) | |
67 | { | |
68 | unsigned long val; | |
69 | /* This could fault if %cr4 does not exist. In x86_64, a cr4 always | |
70 | * exists, so it will never fail. */ | |
71 | #ifdef CONFIG_X86_32 | |
72 | asm volatile("1: mov %%cr4, %0\n" | |
73 | "2:\n" | |
74 | _ASM_EXTABLE(1b, 2b) | |
75 | : "=r" (val), "=m" (__force_order) : "0" (0)); | |
76 | #else | |
77 | val = native_read_cr4(); | |
78 | #endif | |
79 | return val; | |
80 | } | |
81 | ||
82 | static inline void native_write_cr4(unsigned long val) | |
83 | { | |
84 | asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order)); | |
85 | } | |
86 | ||
87 | #ifdef CONFIG_X86_64 | |
88 | static inline unsigned long native_read_cr8(void) | |
89 | { | |
90 | unsigned long cr8; | |
91 | asm volatile("movq %%cr8,%0" : "=r" (cr8)); | |
92 | return cr8; | |
93 | } | |
94 | ||
95 | static inline void native_write_cr8(unsigned long val) | |
96 | { | |
97 | asm volatile("movq %0,%%cr8" :: "r" (val) : "memory"); | |
98 | } | |
99 | #endif | |
100 | ||
101 | static inline void native_wbinvd(void) | |
102 | { | |
103 | asm volatile("wbinvd": : :"memory"); | |
104 | } | |
105 | ||
277d5b40 | 106 | extern asmlinkage void native_load_gs_index(unsigned); |
f05e798a DH |
107 | |
108 | #ifdef CONFIG_PARAVIRT | |
109 | #include <asm/paravirt.h> | |
110 | #else | |
111 | ||
112 | static inline unsigned long read_cr0(void) | |
113 | { | |
114 | return native_read_cr0(); | |
115 | } | |
116 | ||
117 | static inline void write_cr0(unsigned long x) | |
118 | { | |
119 | native_write_cr0(x); | |
120 | } | |
121 | ||
122 | static inline unsigned long read_cr2(void) | |
123 | { | |
124 | return native_read_cr2(); | |
125 | } | |
126 | ||
127 | static inline void write_cr2(unsigned long x) | |
128 | { | |
129 | native_write_cr2(x); | |
130 | } | |
131 | ||
132 | static inline unsigned long read_cr3(void) | |
133 | { | |
134 | return native_read_cr3(); | |
135 | } | |
136 | ||
137 | static inline void write_cr3(unsigned long x) | |
138 | { | |
139 | native_write_cr3(x); | |
140 | } | |
141 | ||
1e02ce4c | 142 | static inline unsigned long __read_cr4(void) |
f05e798a DH |
143 | { |
144 | return native_read_cr4(); | |
145 | } | |
146 | ||
1e02ce4c | 147 | static inline unsigned long __read_cr4_safe(void) |
f05e798a DH |
148 | { |
149 | return native_read_cr4_safe(); | |
150 | } | |
151 | ||
1e02ce4c | 152 | static inline void __write_cr4(unsigned long x) |
f05e798a DH |
153 | { |
154 | native_write_cr4(x); | |
155 | } | |
156 | ||
157 | static inline void wbinvd(void) | |
158 | { | |
159 | native_wbinvd(); | |
160 | } | |
161 | ||
162 | #ifdef CONFIG_X86_64 | |
163 | ||
164 | static inline unsigned long read_cr8(void) | |
165 | { | |
166 | return native_read_cr8(); | |
167 | } | |
168 | ||
169 | static inline void write_cr8(unsigned long x) | |
170 | { | |
171 | native_write_cr8(x); | |
172 | } | |
173 | ||
174 | static inline void load_gs_index(unsigned selector) | |
175 | { | |
176 | native_load_gs_index(selector); | |
177 | } | |
178 | ||
179 | #endif | |
180 | ||
181 | /* Clear the 'TS' bit */ | |
182 | static inline void clts(void) | |
183 | { | |
184 | native_clts(); | |
185 | } | |
186 | ||
187 | #endif/* CONFIG_PARAVIRT */ | |
188 | ||
189 | #define stts() write_cr0(read_cr0() | X86_CR0_TS) | |
190 | ||
191 | static inline void clflush(volatile void *__p) | |
192 | { | |
193 | asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); | |
194 | } | |
195 | ||
171699f7 RZ |
196 | static inline void clflushopt(volatile void *__p) |
197 | { | |
198 | alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0", | |
199 | ".byte 0x66; clflush %P0", | |
200 | X86_FEATURE_CLFLUSHOPT, | |
201 | "+m" (*(volatile char __force *)__p)); | |
202 | } | |
203 | ||
d9dc64f3 RZ |
204 | static inline void clwb(volatile void *__p) |
205 | { | |
206 | volatile struct { char x[64]; } *p = __p; | |
207 | ||
208 | asm volatile(ALTERNATIVE_2( | |
209 | ".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])", | |
210 | ".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */ | |
211 | X86_FEATURE_CLFLUSHOPT, | |
212 | ".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */ | |
213 | X86_FEATURE_CLWB) | |
214 | : [p] "+m" (*p) | |
215 | : [pax] "a" (p)); | |
216 | } | |
217 | ||
ca7d9b79 RZ |
218 | /** |
219 | * pcommit_sfence() - persistent commit and fence | |
220 | * | |
221 | * The PCOMMIT instruction ensures that data that has been flushed from the | |
222 | * processor's cache hierarchy with CLWB, CLFLUSHOPT or CLFLUSH is accepted to | |
223 | * memory and is durable on the DIMM. The primary use case for this is | |
224 | * persistent memory. | |
225 | * | |
226 | * This function shows how to properly use CLWB/CLFLUSHOPT/CLFLUSH and PCOMMIT | |
227 | * with appropriate fencing. | |
228 | * | |
229 | * Example: | |
230 | * void flush_and_commit_buffer(void *vaddr, unsigned int size) | |
231 | * { | |
232 | * unsigned long clflush_mask = boot_cpu_data.x86_clflush_size - 1; | |
233 | * void *vend = vaddr + size; | |
234 | * void *p; | |
235 | * | |
236 | * for (p = (void *)((unsigned long)vaddr & ~clflush_mask); | |
237 | * p < vend; p += boot_cpu_data.x86_clflush_size) | |
238 | * clwb(p); | |
239 | * | |
240 | * // SFENCE to order CLWB/CLFLUSHOPT/CLFLUSH cache flushes | |
241 | * // MFENCE via mb() also works | |
242 | * wmb(); | |
243 | * | |
244 | * // PCOMMIT and the required SFENCE for ordering | |
245 | * pcommit_sfence(); | |
246 | * } | |
247 | * | |
248 | * After this function completes the data pointed to by 'vaddr' has been | |
249 | * accepted to memory and will be durable if the 'vaddr' points to persistent | |
250 | * memory. | |
251 | * | |
252 | * PCOMMIT must always be ordered by an MFENCE or SFENCE, so to help simplify | |
253 | * things we include both the PCOMMIT and the required SFENCE in the | |
254 | * alternatives generated by pcommit_sfence(). | |
255 | */ | |
719d359d RZ |
256 | static inline void pcommit_sfence(void) |
257 | { | |
258 | alternative(ASM_NOP7, | |
259 | ".byte 0x66, 0x0f, 0xae, 0xf8\n\t" /* pcommit */ | |
260 | "sfence", | |
261 | X86_FEATURE_PCOMMIT); | |
262 | } | |
263 | ||
f05e798a DH |
264 | #define nop() asm volatile ("nop") |
265 | ||
266 | ||
267 | #endif /* __KERNEL__ */ | |
268 | ||
269 | #endif /* _ASM_X86_SPECIAL_INSNS_H */ |