Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
26eef70c AK |
2 | #ifndef ARCH_X86_KVM_X86_H |
3 | #define ARCH_X86_KVM_X86_H | |
4 | ||
5 | #include <linux/kvm_host.h> | |
8d93c874 | 6 | #include <asm/pvclock.h> |
3eeb3288 | 7 | #include "kvm_cache_regs.h" |
26eef70c | 8 | |
c8e88717 BM |
9 | #define KVM_DEFAULT_PLE_GAP 128 |
10 | #define KVM_VMX_DEFAULT_PLE_WINDOW 4096 | |
11 | #define KVM_DEFAULT_PLE_WINDOW_GROW 2 | |
12 | #define KVM_DEFAULT_PLE_WINDOW_SHRINK 0 | |
13 | #define KVM_VMX_DEFAULT_PLE_WINDOW_MAX UINT_MAX | |
8566ac8b BM |
14 | #define KVM_SVM_DEFAULT_PLE_WINDOW_MAX USHRT_MAX |
15 | #define KVM_SVM_DEFAULT_PLE_WINDOW 3000 | |
c8e88717 BM |
16 | |
17 | static inline unsigned int __grow_ple_window(unsigned int val, | |
18 | unsigned int base, unsigned int modifier, unsigned int max) | |
19 | { | |
20 | u64 ret = val; | |
21 | ||
22 | if (modifier < 1) | |
23 | return base; | |
24 | ||
25 | if (modifier < base) | |
26 | ret *= modifier; | |
27 | else | |
28 | ret += modifier; | |
29 | ||
30 | return min(ret, (u64)max); | |
31 | } | |
32 | ||
33 | static inline unsigned int __shrink_ple_window(unsigned int val, | |
34 | unsigned int base, unsigned int modifier, unsigned int min) | |
35 | { | |
36 | if (modifier < 1) | |
37 | return base; | |
38 | ||
39 | if (modifier < base) | |
40 | val /= modifier; | |
41 | else | |
42 | val -= modifier; | |
43 | ||
44 | return max(val, min); | |
45 | } | |
46 | ||
74545705 RK |
47 | #define MSR_IA32_CR_PAT_DEFAULT 0x0007040600070406ULL |
48 | ||
26eef70c AK |
49 | static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu) |
50 | { | |
5c7d4f9a | 51 | vcpu->arch.exception.pending = false; |
664f8e26 | 52 | vcpu->arch.exception.injected = false; |
26eef70c AK |
53 | } |
54 | ||
66fd3f7f GN |
55 | static inline void kvm_queue_interrupt(struct kvm_vcpu *vcpu, u8 vector, |
56 | bool soft) | |
937a7eae | 57 | { |
04140b41 | 58 | vcpu->arch.interrupt.injected = true; |
66fd3f7f | 59 | vcpu->arch.interrupt.soft = soft; |
937a7eae AK |
60 | vcpu->arch.interrupt.nr = vector; |
61 | } | |
62 | ||
63 | static inline void kvm_clear_interrupt_queue(struct kvm_vcpu *vcpu) | |
64 | { | |
04140b41 | 65 | vcpu->arch.interrupt.injected = false; |
937a7eae AK |
66 | } |
67 | ||
3298b75c GN |
68 | static inline bool kvm_event_needs_reinjection(struct kvm_vcpu *vcpu) |
69 | { | |
04140b41 | 70 | return vcpu->arch.exception.injected || vcpu->arch.interrupt.injected || |
3298b75c GN |
71 | vcpu->arch.nmi_injected; |
72 | } | |
66fd3f7f GN |
73 | |
74 | static inline bool kvm_exception_is_soft(unsigned int nr) | |
75 | { | |
76 | return (nr == BP_VECTOR) || (nr == OF_VECTOR); | |
77 | } | |
fc61b800 | 78 | |
3eeb3288 AK |
79 | static inline bool is_protmode(struct kvm_vcpu *vcpu) |
80 | { | |
81 | return kvm_read_cr0_bits(vcpu, X86_CR0_PE); | |
82 | } | |
83 | ||
836a1b3c AK |
84 | static inline int is_long_mode(struct kvm_vcpu *vcpu) |
85 | { | |
86 | #ifdef CONFIG_X86_64 | |
f6801dff | 87 | return vcpu->arch.efer & EFER_LMA; |
836a1b3c AK |
88 | #else |
89 | return 0; | |
90 | #endif | |
91 | } | |
92 | ||
5777392e NA |
93 | static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu) |
94 | { | |
95 | int cs_db, cs_l; | |
96 | ||
97 | if (!is_long_mode(vcpu)) | |
98 | return false; | |
99 | kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); | |
100 | return cs_l; | |
101 | } | |
102 | ||
855feb67 YZ |
103 | static inline bool is_la57_mode(struct kvm_vcpu *vcpu) |
104 | { | |
105 | #ifdef CONFIG_X86_64 | |
106 | return (vcpu->arch.efer & EFER_LMA) && | |
107 | kvm_read_cr4_bits(vcpu, X86_CR4_LA57); | |
108 | #else | |
109 | return 0; | |
110 | #endif | |
111 | } | |
112 | ||
0447378a MO |
113 | static inline bool x86_exception_has_error_code(unsigned int vector) |
114 | { | |
115 | static u32 exception_has_error_code = BIT(DF_VECTOR) | BIT(TS_VECTOR) | | |
116 | BIT(NP_VECTOR) | BIT(SS_VECTOR) | BIT(GP_VECTOR) | | |
117 | BIT(PF_VECTOR) | BIT(AC_VECTOR); | |
118 | ||
119 | return (1U << vector) & exception_has_error_code; | |
120 | } | |
121 | ||
6539e738 JR |
122 | static inline bool mmu_is_nested(struct kvm_vcpu *vcpu) |
123 | { | |
124 | return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu; | |
125 | } | |
126 | ||
836a1b3c AK |
127 | static inline int is_pae(struct kvm_vcpu *vcpu) |
128 | { | |
129 | return kvm_read_cr4_bits(vcpu, X86_CR4_PAE); | |
130 | } | |
131 | ||
132 | static inline int is_pse(struct kvm_vcpu *vcpu) | |
133 | { | |
134 | return kvm_read_cr4_bits(vcpu, X86_CR4_PSE); | |
135 | } | |
136 | ||
137 | static inline int is_paging(struct kvm_vcpu *vcpu) | |
138 | { | |
c36fc04e | 139 | return likely(kvm_read_cr0_bits(vcpu, X86_CR0_PG)); |
836a1b3c AK |
140 | } |
141 | ||
24d1b15f JR |
142 | static inline u32 bit(int bitno) |
143 | { | |
144 | return 1 << (bitno & 31); | |
145 | } | |
146 | ||
fd8cb433 YZ |
147 | static inline u8 vcpu_virt_addr_bits(struct kvm_vcpu *vcpu) |
148 | { | |
149 | return kvm_read_cr4_bits(vcpu, X86_CR4_LA57) ? 57 : 48; | |
150 | } | |
151 | ||
152 | static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt) | |
153 | { | |
154 | return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48; | |
155 | } | |
156 | ||
157 | static inline u64 get_canonical(u64 la, u8 vaddr_bits) | |
158 | { | |
159 | return ((int64_t)la << (64 - vaddr_bits)) >> (64 - vaddr_bits); | |
160 | } | |
161 | ||
162 | static inline bool is_noncanonical_address(u64 la, struct kvm_vcpu *vcpu) | |
163 | { | |
164 | #ifdef CONFIG_X86_64 | |
165 | return get_canonical(la, vcpu_virt_addr_bits(vcpu)) != la; | |
166 | #else | |
167 | return false; | |
168 | #endif | |
169 | } | |
170 | ||
171 | static inline bool emul_is_noncanonical_address(u64 la, | |
172 | struct x86_emulate_ctxt *ctxt) | |
173 | { | |
174 | #ifdef CONFIG_X86_64 | |
175 | return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la; | |
176 | #else | |
177 | return false; | |
178 | #endif | |
179 | } | |
180 | ||
bebb106a XG |
181 | static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu, |
182 | gva_t gva, gfn_t gfn, unsigned access) | |
183 | { | |
ddfd1730 SC |
184 | u64 gen = kvm_memslots(vcpu->kvm)->generation; |
185 | ||
361209e0 | 186 | if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS)) |
ddfd1730 SC |
187 | return; |
188 | ||
9034e6e8 PB |
189 | /* |
190 | * If this is a shadow nested page table, the "GVA" is | |
191 | * actually a nGPA. | |
192 | */ | |
193 | vcpu->arch.mmio_gva = mmu_is_nested(vcpu) ? 0 : gva & PAGE_MASK; | |
bebb106a XG |
194 | vcpu->arch.access = access; |
195 | vcpu->arch.mmio_gfn = gfn; | |
ddfd1730 | 196 | vcpu->arch.mmio_gen = gen; |
56f17dd3 DM |
197 | } |
198 | ||
199 | static inline bool vcpu_match_mmio_gen(struct kvm_vcpu *vcpu) | |
200 | { | |
201 | return vcpu->arch.mmio_gen == kvm_memslots(vcpu->kvm)->generation; | |
bebb106a XG |
202 | } |
203 | ||
204 | /* | |
56f17dd3 DM |
205 | * Clear the mmio cache info for the given gva. If gva is MMIO_GVA_ANY, we |
206 | * clear all mmio cache info. | |
bebb106a | 207 | */ |
56f17dd3 DM |
208 | #define MMIO_GVA_ANY (~(gva_t)0) |
209 | ||
bebb106a XG |
210 | static inline void vcpu_clear_mmio_info(struct kvm_vcpu *vcpu, gva_t gva) |
211 | { | |
56f17dd3 | 212 | if (gva != MMIO_GVA_ANY && vcpu->arch.mmio_gva != (gva & PAGE_MASK)) |
bebb106a XG |
213 | return; |
214 | ||
215 | vcpu->arch.mmio_gva = 0; | |
216 | } | |
217 | ||
218 | static inline bool vcpu_match_mmio_gva(struct kvm_vcpu *vcpu, unsigned long gva) | |
219 | { | |
56f17dd3 DM |
220 | if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gva && |
221 | vcpu->arch.mmio_gva == (gva & PAGE_MASK)) | |
bebb106a XG |
222 | return true; |
223 | ||
224 | return false; | |
225 | } | |
226 | ||
227 | static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) | |
228 | { | |
56f17dd3 DM |
229 | if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gfn && |
230 | vcpu->arch.mmio_gfn == gpa >> PAGE_SHIFT) | |
bebb106a XG |
231 | return true; |
232 | ||
233 | return false; | |
234 | } | |
235 | ||
5777392e NA |
236 | static inline unsigned long kvm_register_readl(struct kvm_vcpu *vcpu, |
237 | enum kvm_reg reg) | |
238 | { | |
239 | unsigned long val = kvm_register_read(vcpu, reg); | |
240 | ||
241 | return is_64_bit_mode(vcpu) ? val : (u32)val; | |
242 | } | |
243 | ||
27e6fb5d NA |
244 | static inline void kvm_register_writel(struct kvm_vcpu *vcpu, |
245 | enum kvm_reg reg, | |
246 | unsigned long val) | |
247 | { | |
248 | if (!is_64_bit_mode(vcpu)) | |
249 | val = (u32)val; | |
250 | return kvm_register_write(vcpu, reg, val); | |
251 | } | |
252 | ||
41dbc6bc PB |
253 | static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) |
254 | { | |
255 | return !(kvm->arch.disabled_quirks & quirk); | |
256 | } | |
257 | ||
bab5bb39 | 258 | void kvm_set_pending_timer(struct kvm_vcpu *vcpu); |
71f9833b | 259 | int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip); |
ff9d07a0 | 260 | |
8fe8ab46 | 261 | void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr); |
108b249c | 262 | u64 get_kvmclock_ns(struct kvm *kvm); |
99e3e30a | 263 | |
ce14e868 | 264 | int kvm_read_guest_virt(struct kvm_vcpu *vcpu, |
064aea77 NHE |
265 | gva_t addr, void *val, unsigned int bytes, |
266 | struct x86_exception *exception); | |
267 | ||
ce14e868 | 268 | int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, |
6a4d7550 NHE |
269 | gva_t addr, void *val, unsigned int bytes, |
270 | struct x86_exception *exception); | |
271 | ||
082d06ed WL |
272 | int handle_ud(struct kvm_vcpu *vcpu); |
273 | ||
da998b46 JM |
274 | void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu); |
275 | ||
19efffa2 | 276 | void kvm_vcpu_mtrr_init(struct kvm_vcpu *vcpu); |
ff53604b | 277 | u8 kvm_mtrr_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn); |
4566654b | 278 | bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data); |
ff53604b XG |
279 | int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data); |
280 | int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); | |
6a39bbc5 XG |
281 | bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, |
282 | int page_num); | |
52004014 | 283 | bool kvm_vector_hashing_enabled(void); |
c60658d1 SC |
284 | int x86_emulate_instruction(struct kvm_vcpu *vcpu, unsigned long cr2, |
285 | int emulation_type, void *insn, int insn_len); | |
4566654b | 286 | |
d91cab78 DH |
287 | #define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ |
288 | | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ | |
17a511f8 HH |
289 | | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ |
290 | | XFEATURE_MASK_PKRU) | |
00b27a3e AK |
291 | extern u64 host_xcr0; |
292 | ||
4ff41732 PB |
293 | extern u64 kvm_supported_xcr0(void); |
294 | ||
9ed96e87 MT |
295 | extern unsigned int min_timer_period_us; |
296 | ||
c4ae60e4 LA |
297 | extern bool enable_vmware_backdoor; |
298 | ||
54e9818f | 299 | extern struct static_key kvm_no_apic_vcpu; |
b51012de | 300 | |
8d93c874 MT |
301 | static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec) |
302 | { | |
303 | return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult, | |
304 | vcpu->arch.virtual_tsc_shift); | |
305 | } | |
306 | ||
b51012de PB |
307 | /* Same "calling convention" as do_div: |
308 | * - divide (n << 32) by base | |
309 | * - put result in n | |
310 | * - return remainder | |
311 | */ | |
312 | #define do_shl32_div32(n, base) \ | |
313 | ({ \ | |
314 | u32 __quot, __rem; \ | |
315 | asm("divl %2" : "=a" (__quot), "=d" (__rem) \ | |
316 | : "rm" (base), "0" (0), "1" ((u32) n)); \ | |
317 | n = __quot; \ | |
318 | __rem; \ | |
319 | }) | |
320 | ||
4d5422ce | 321 | static inline bool kvm_mwait_in_guest(struct kvm *kvm) |
668fffa3 | 322 | { |
4d5422ce | 323 | return kvm->arch.mwait_in_guest; |
668fffa3 MT |
324 | } |
325 | ||
caa057a2 WL |
326 | static inline bool kvm_hlt_in_guest(struct kvm *kvm) |
327 | { | |
328 | return kvm->arch.hlt_in_guest; | |
329 | } | |
330 | ||
b31c114b WL |
331 | static inline bool kvm_pause_in_guest(struct kvm *kvm) |
332 | { | |
333 | return kvm->arch.pause_in_guest; | |
334 | } | |
335 | ||
dd60d217 AK |
336 | DECLARE_PER_CPU(struct kvm_vcpu *, current_vcpu); |
337 | ||
338 | static inline void kvm_before_interrupt(struct kvm_vcpu *vcpu) | |
339 | { | |
340 | __this_cpu_write(current_vcpu, vcpu); | |
341 | } | |
342 | ||
343 | static inline void kvm_after_interrupt(struct kvm_vcpu *vcpu) | |
344 | { | |
345 | __this_cpu_write(current_vcpu, NULL); | |
346 | } | |
347 | ||
674ea351 PB |
348 | |
349 | static inline bool kvm_pat_valid(u64 data) | |
350 | { | |
351 | if (data & 0xF8F8F8F8F8F8F8F8ull) | |
352 | return false; | |
353 | /* 0, 1, 4, 5, 6, 7 are valid values. */ | |
354 | return (data | ((data & 0x0202020202020202ull) << 1)) == data; | |
355 | } | |
356 | ||
1811d979 WC |
357 | void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu); |
358 | void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu); | |
674ea351 | 359 | |
26eef70c | 360 | #endif |