[linux-block.git] / arch / x86 / kvm / x86.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef ARCH_X86_KVM_X86_H
#define ARCH_X86_KVM_X86_H

#include <asm/processor.h>
#include <asm/mwait.h>
#include <linux/kvm_host.h>
#include <asm/pvclock.h>
#include "kvm_cache_regs.h"

#define MSR_IA32_CR_PAT_DEFAULT  0x0007040600070406ULL

static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu)
{
	vcpu->arch.exception.injected = false;
}

static inline void kvm_queue_interrupt(struct kvm_vcpu *vcpu, u8 vector,
	bool soft)
{
	vcpu->arch.interrupt.pending = true;
	vcpu->arch.interrupt.soft = soft;
	vcpu->arch.interrupt.nr = vector;
}

static inline void kvm_clear_interrupt_queue(struct kvm_vcpu *vcpu)
{
	vcpu->arch.interrupt.pending = false;
}

static inline bool kvm_event_needs_reinjection(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.exception.injected || vcpu->arch.interrupt.pending ||
		vcpu->arch.nmi_injected;
}

static inline bool kvm_exception_is_soft(unsigned int nr)
{
	return (nr == BP_VECTOR) || (nr == OF_VECTOR);
}

static inline bool is_protmode(struct kvm_vcpu *vcpu)
{
	return kvm_read_cr0_bits(vcpu, X86_CR0_PE);
}

static inline int is_long_mode(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
	return vcpu->arch.efer & EFER_LMA;
#else
	return 0;
#endif
}

static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)
{
	int cs_db, cs_l;

	if (!is_long_mode(vcpu))
		return false;
	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
	return cs_l;
}

static inline bool is_la57_mode(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
	return (vcpu->arch.efer & EFER_LMA) &&
		 kvm_read_cr4_bits(vcpu, X86_CR4_LA57);
#else
	return 0;
#endif
}

static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu;
}

static inline int is_pae(struct kvm_vcpu *vcpu)
{
	return kvm_read_cr4_bits(vcpu, X86_CR4_PAE);
}

static inline int is_pse(struct kvm_vcpu *vcpu)
{
	return kvm_read_cr4_bits(vcpu, X86_CR4_PSE);
}

static inline int is_paging(struct kvm_vcpu *vcpu)
{
	return likely(kvm_read_cr0_bits(vcpu, X86_CR0_PG));
}

static inline u32 bit(int bitno)
{
	return 1 << (bitno & 31);
}

static inline u8 vcpu_virt_addr_bits(struct kvm_vcpu *vcpu)
{
	return kvm_read_cr4_bits(vcpu, X86_CR4_LA57) ? 57 : 48;
}

static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
{
	return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
}

static inline u64 get_canonical(u64 la, u8 vaddr_bits)
{
	return ((int64_t)la << (64 - vaddr_bits)) >> (64 - vaddr_bits);
}

static inline bool is_noncanonical_address(u64 la, struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
	return get_canonical(la, vcpu_virt_addr_bits(vcpu)) != la;
#else
	return false;
#endif
}

static inline bool emul_is_noncanonical_address(u64 la,
						struct x86_emulate_ctxt *ctxt)
{
#ifdef CONFIG_X86_64
	return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la;
#else
	return false;
#endif
}

static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu,
					gva_t gva, gfn_t gfn, unsigned access)
{
	/*
	 * If this is a shadow nested page table, the "GVA" is
	 * actually a nGPA.
	 */
	vcpu->arch.mmio_gva = mmu_is_nested(vcpu) ? 0 : gva & PAGE_MASK;
	vcpu->arch.access = access;
	vcpu->arch.mmio_gfn = gfn;
	vcpu->arch.mmio_gen = kvm_memslots(vcpu->kvm)->generation;
}

static inline bool vcpu_match_mmio_gen(struct kvm_vcpu *vcpu)
{
	return vcpu->arch.mmio_gen == kvm_memslots(vcpu->kvm)->generation;
}

/*
 * Clear the mmio cache info for the given gva. If gva is MMIO_GVA_ANY, we
 * clear all mmio cache info.
 */
#define MMIO_GVA_ANY (~(gva_t)0)

static inline void vcpu_clear_mmio_info(struct kvm_vcpu *vcpu, gva_t gva)
{
	if (gva != MMIO_GVA_ANY && vcpu->arch.mmio_gva != (gva & PAGE_MASK))
		return;

	vcpu->arch.mmio_gva = 0;
}

static inline bool vcpu_match_mmio_gva(struct kvm_vcpu *vcpu, unsigned long gva)
{
	if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gva &&
	      vcpu->arch.mmio_gva == (gva & PAGE_MASK))
		return true;

	return false;
}

static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
{
	if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gfn &&
	      vcpu->arch.mmio_gfn == gpa >> PAGE_SHIFT)
		return true;

	return false;
}

static inline unsigned long kvm_register_readl(struct kvm_vcpu *vcpu,
					       enum kvm_reg reg)
{
	unsigned long val = kvm_register_read(vcpu, reg);

	return is_64_bit_mode(vcpu) ? val : (u32)val;
}

static inline void kvm_register_writel(struct kvm_vcpu *vcpu,
				       enum kvm_reg reg,
				       unsigned long val)
{
	if (!is_64_bit_mode(vcpu))
		val = (u32)val;
	return kvm_register_write(vcpu, reg, val);
}

static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
{
	return !(kvm->arch.disabled_quirks & quirk);
}

void kvm_before_handle_nmi(struct kvm_vcpu *vcpu);
void kvm_after_handle_nmi(struct kvm_vcpu *vcpu);
void kvm_set_pending_timer(struct kvm_vcpu *vcpu);
int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);

void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr);
u64 get_kvmclock_ns(struct kvm *kvm);

int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
	gva_t addr, void *val, unsigned int bytes,
	struct x86_exception *exception);

int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
	gva_t addr, void *val, unsigned int bytes,
	struct x86_exception *exception);

void kvm_vcpu_mtrr_init(struct kvm_vcpu *vcpu);
u8 kvm_mtrr_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn);
bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data);
int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data);
int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata);
bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn,
					  int page_num);
bool kvm_vector_hashing_enabled(void);

#define KVM_SUPPORTED_XCR0     (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \
				| XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \
				| XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \
				| XFEATURE_MASK_PKRU)
extern u64 host_xcr0;

extern u64 kvm_supported_xcr0(void);

extern unsigned int min_timer_period_us;

extern unsigned int lapic_timer_advance_ns;

extern struct static_key kvm_no_apic_vcpu;

static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
{
	return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
				   vcpu->arch.virtual_tsc_shift);
}

/* Same "calling convention" as do_div:
 * - divide (n << 32) by base
 * - put result in n
 * - return remainder
 */
#define do_shl32_div32(n, base)					\
	({							\
	    u32 __quot, __rem;					\
	    asm("divl %2" : "=a" (__quot), "=d" (__rem)		\
			: "rm" (base), "0" (0), "1" ((u32) n));	\
	    n = __quot;						\
	    __rem;						\
	 })

static inline bool kvm_mwait_in_guest(void)
{
	unsigned int eax, ebx, ecx, edx;

	if (!cpu_has(&boot_cpu_data, X86_FEATURE_MWAIT))
		return false;

	switch (boot_cpu_data.x86_vendor) {
	case X86_VENDOR_AMD:
		/* All AMD CPUs have a working MWAIT implementation */
		return true;
	case X86_VENDOR_INTEL:
		/* Handle Intel below */
		break;
	default:
		return false;
	}

	/*
	 * Intel CPUs without CPUID5_ECX_INTERRUPT_BREAK are problematic as
	 * they would allow guest to stop the CPU completely by disabling
	 * interrupts then invoking MWAIT.
	 */
	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
		return false;

	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);

	if (!(ecx & CPUID5_ECX_INTERRUPT_BREAK))
		return false;

	return true;
}

#endif
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
26eef70c AK	2	#ifndef ARCH_X86_KVM_X86_H
	3	#define ARCH_X86_KVM_X86_H
	4
668fffa3 MT	5	#include <asm/processor.h>
668fffa3 MT	6	#include <asm/mwait.h>
26eef70c	7	#include <linux/kvm_host.h>
8d93c874	8	#include <asm/pvclock.h>
3eeb3288	9	#include "kvm_cache_regs.h"
26eef70c	10
74545705 RK	11	#define MSR_IA32_CR_PAT_DEFAULT 0x0007040600070406ULL
74545705 RK	12
26eef70c AK	13	static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu)
26eef70c AK	14	{
664f8e26	15	vcpu->arch.exception.injected = false;
26eef70c AK	16	}
26eef70c AK	17
66fd3f7f GN	18	static inline void kvm_queue_interrupt(struct kvm_vcpu *vcpu, u8 vector,
66fd3f7f GN	19	bool soft)
937a7eae AK	20	{
937a7eae AK	21	vcpu->arch.interrupt.pending = true;
66fd3f7f	22	vcpu->arch.interrupt.soft = soft;
937a7eae AK	23	vcpu->arch.interrupt.nr = vector;
	24	}
	25
	26	static inline void kvm_clear_interrupt_queue(struct kvm_vcpu *vcpu)
	27	{
	28	vcpu->arch.interrupt.pending = false;
	29	}
	30
3298b75c GN	31	static inline bool kvm_event_needs_reinjection(struct kvm_vcpu *vcpu)
3298b75c GN	32	{
664f8e26	33	return vcpu->arch.exception.injected \|\| vcpu->arch.interrupt.pending \|\|
3298b75c GN	34	vcpu->arch.nmi_injected;
3298b75c GN	35	}
66fd3f7f GN	36
	37	static inline bool kvm_exception_is_soft(unsigned int nr)
	38	{
	39	return (nr == BP_VECTOR) \|\| (nr == OF_VECTOR);
	40	}
fc61b800	41
3eeb3288 AK	42	static inline bool is_protmode(struct kvm_vcpu *vcpu)
	43	{
	44	return kvm_read_cr0_bits(vcpu, X86_CR0_PE);
	45	}
	46
836a1b3c AK	47	static inline int is_long_mode(struct kvm_vcpu *vcpu)
	48	{
	49	#ifdef CONFIG_X86_64
f6801dff	50	return vcpu->arch.efer & EFER_LMA;
836a1b3c AK	51	#else
	52	return 0;
	53	#endif
	54	}
	55
5777392e NA	56	static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu)
	57	{
	58	int cs_db, cs_l;
	59
	60	if (!is_long_mode(vcpu))
	61	return false;
	62	kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
	63	return cs_l;
	64	}
	65
855feb67 YZ	66	static inline bool is_la57_mode(struct kvm_vcpu *vcpu)
	67	{
	68	#ifdef CONFIG_X86_64
	69	return (vcpu->arch.efer & EFER_LMA) &&
	70	kvm_read_cr4_bits(vcpu, X86_CR4_LA57);
	71	#else
	72	return 0;
	73	#endif
	74	}
	75
6539e738 JR	76	static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
	77	{
	78	return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu;
	79	}
	80
836a1b3c AK	81	static inline int is_pae(struct kvm_vcpu *vcpu)
	82	{
	83	return kvm_read_cr4_bits(vcpu, X86_CR4_PAE);
	84	}
	85
	86	static inline int is_pse(struct kvm_vcpu *vcpu)
	87	{
	88	return kvm_read_cr4_bits(vcpu, X86_CR4_PSE);
	89	}
	90
	91	static inline int is_paging(struct kvm_vcpu *vcpu)
	92	{
c36fc04e	93	return likely(kvm_read_cr0_bits(vcpu, X86_CR0_PG));
836a1b3c AK	94	}
836a1b3c AK	95
24d1b15f JR	96	static inline u32 bit(int bitno)
	97	{
	98	return 1 << (bitno & 31);
	99	}
	100
fd8cb433 YZ	101	static inline u8 vcpu_virt_addr_bits(struct kvm_vcpu *vcpu)
	102	{
	103	return kvm_read_cr4_bits(vcpu, X86_CR4_LA57) ? 57 : 48;
	104	}
	105
	106	static inline u8 ctxt_virt_addr_bits(struct x86_emulate_ctxt *ctxt)
	107	{
	108	return (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_LA57) ? 57 : 48;
	109	}
	110
	111	static inline u64 get_canonical(u64 la, u8 vaddr_bits)
	112	{
	113	return ((int64_t)la << (64 - vaddr_bits)) >> (64 - vaddr_bits);
	114	}
	115
	116	static inline bool is_noncanonical_address(u64 la, struct kvm_vcpu *vcpu)
	117	{
	118	#ifdef CONFIG_X86_64
	119	return get_canonical(la, vcpu_virt_addr_bits(vcpu)) != la;
	120	#else
	121	return false;
	122	#endif
	123	}
	124
	125	static inline bool emul_is_noncanonical_address(u64 la,
	126	struct x86_emulate_ctxt *ctxt)
	127	{
	128	#ifdef CONFIG_X86_64
	129	return get_canonical(la, ctxt_virt_addr_bits(ctxt)) != la;
	130	#else
	131	return false;
	132	#endif
	133	}
	134
bebb106a XG	135	static inline void vcpu_cache_mmio_info(struct kvm_vcpu *vcpu,
	136	gva_t gva, gfn_t gfn, unsigned access)
	137	{
9034e6e8 PB	138	/*
	139	* If this is a shadow nested page table, the "GVA" is
	140	* actually a nGPA.
	141	*/
	142	vcpu->arch.mmio_gva = mmu_is_nested(vcpu) ? 0 : gva & PAGE_MASK;
bebb106a XG	143	vcpu->arch.access = access;
bebb106a XG	144	vcpu->arch.mmio_gfn = gfn;
56f17dd3 DM	145	vcpu->arch.mmio_gen = kvm_memslots(vcpu->kvm)->generation;
	146	}
	147
	148	static inline bool vcpu_match_mmio_gen(struct kvm_vcpu *vcpu)
	149	{
	150	return vcpu->arch.mmio_gen == kvm_memslots(vcpu->kvm)->generation;
bebb106a XG	151	}
	152
	153	/*
56f17dd3 DM	154	* Clear the mmio cache info for the given gva. If gva is MMIO_GVA_ANY, we
56f17dd3 DM	155	* clear all mmio cache info.
bebb106a	156	*/
56f17dd3 DM	157	#define MMIO_GVA_ANY (~(gva_t)0)
56f17dd3 DM	158
bebb106a XG	159	static inline void vcpu_clear_mmio_info(struct kvm_vcpu *vcpu, gva_t gva)
bebb106a XG	160	{
56f17dd3	161	if (gva != MMIO_GVA_ANY && vcpu->arch.mmio_gva != (gva & PAGE_MASK))
bebb106a XG	162	return;
	163
	164	vcpu->arch.mmio_gva = 0;
	165	}
	166
	167	static inline bool vcpu_match_mmio_gva(struct kvm_vcpu *vcpu, unsigned long gva)
	168	{
56f17dd3 DM	169	if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gva &&
56f17dd3 DM	170	vcpu->arch.mmio_gva == (gva & PAGE_MASK))
bebb106a XG	171	return true;
	172
	173	return false;
	174	}
	175
	176	static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
	177	{
56f17dd3 DM	178	if (vcpu_match_mmio_gen(vcpu) && vcpu->arch.mmio_gfn &&
56f17dd3 DM	179	vcpu->arch.mmio_gfn == gpa >> PAGE_SHIFT)
bebb106a XG	180	return true;
	181
	182	return false;
	183	}
	184
5777392e NA	185	static inline unsigned long kvm_register_readl(struct kvm_vcpu *vcpu,
	186	enum kvm_reg reg)
	187	{
	188	unsigned long val = kvm_register_read(vcpu, reg);
	189
	190	return is_64_bit_mode(vcpu) ? val : (u32)val;
	191	}
	192
27e6fb5d NA	193	static inline void kvm_register_writel(struct kvm_vcpu *vcpu,
	194	enum kvm_reg reg,
	195	unsigned long val)
	196	{
	197	if (!is_64_bit_mode(vcpu))
	198	val = (u32)val;
	199	return kvm_register_write(vcpu, reg, val);
	200	}
	201
41dbc6bc PB	202	static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
	203	{
	204	return !(kvm->arch.disabled_quirks & quirk);
	205	}
	206
ff9d07a0 ZY	207	void kvm_before_handle_nmi(struct kvm_vcpu *vcpu);
ff9d07a0 ZY	208	void kvm_after_handle_nmi(struct kvm_vcpu *vcpu);
bab5bb39	209	void kvm_set_pending_timer(struct kvm_vcpu *vcpu);
71f9833b	210	int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);
ff9d07a0	211
8fe8ab46	212	void kvm_write_tsc(struct kvm_vcpu vcpu, struct msr_data msr);
108b249c	213	u64 get_kvmclock_ns(struct kvm *kvm);
99e3e30a	214
064aea77 NHE	215	int kvm_read_guest_virt(struct x86_emulate_ctxt *ctxt,
	216	gva_t addr, void *val, unsigned int bytes,
	217	struct x86_exception *exception);
	218
6a4d7550 NHE	219	int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
	220	gva_t addr, void *val, unsigned int bytes,
	221	struct x86_exception *exception);
	222
19efffa2	223	void kvm_vcpu_mtrr_init(struct kvm_vcpu *vcpu);
ff53604b	224	u8 kvm_mtrr_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn);
4566654b	225	bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data);
ff53604b XG	226	int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data);
ff53604b XG	227	int kvm_mtrr_get_msr(struct kvm_vcpu vcpu, u32 msr, u64 pdata);
6a39bbc5 XG	228	bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn,
6a39bbc5 XG	229	int page_num);
52004014	230	bool kvm_vector_hashing_enabled(void);
4566654b	231
d91cab78 DH	232	#define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP \| XFEATURE_MASK_SSE \
d91cab78 DH	233	\| XFEATURE_MASK_YMM \| XFEATURE_MASK_BNDREGS \
17a511f8 HH	234	\| XFEATURE_MASK_BNDCSR \| XFEATURE_MASK_AVX512 \
17a511f8 HH	235	\| XFEATURE_MASK_PKRU)
00b27a3e AK	236	extern u64 host_xcr0;
00b27a3e AK	237
4ff41732 PB	238	extern u64 kvm_supported_xcr0(void);
4ff41732 PB	239
9ed96e87 MT	240	extern unsigned int min_timer_period_us;
9ed96e87 MT	241
d0659d94 MT	242	extern unsigned int lapic_timer_advance_ns;
d0659d94 MT	243
54e9818f	244	extern struct static_key kvm_no_apic_vcpu;
b51012de	245
8d93c874 MT	246	static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
	247	{
	248	return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
	249	vcpu->arch.virtual_tsc_shift);
	250	}
	251
b51012de PB	252	/* Same "calling convention" as do_div:
	253	* - divide (n << 32) by base
	254	* - put result in n
	255	* - return remainder
	256	*/
	257	#define do_shl32_div32(n, base) \
	258	({ \
	259	u32 __quot, __rem; \
	260	asm("divl %2" : "=a" (__quot), "=d" (__rem) \
	261	: "rm" (base), "0" (0), "1" ((u32) n)); \
	262	n = __quot; \
	263	__rem; \
	264	})
	265
668fffa3 MT	266	static inline bool kvm_mwait_in_guest(void)
	267	{
	268	unsigned int eax, ebx, ecx, edx;
	269
	270	if (!cpu_has(&boot_cpu_data, X86_FEATURE_MWAIT))
	271	return false;
	272
	273	switch (boot_cpu_data.x86_vendor) {
	274	case X86_VENDOR_AMD:
	275	/* All AMD CPUs have a working MWAIT implementation */
	276	return true;
	277	case X86_VENDOR_INTEL:
	278	/* Handle Intel below */
	279	break;
	280	default:
	281	return false;
	282	}
	283
	284	/*
	285	* Intel CPUs without CPUID5_ECX_INTERRUPT_BREAK are problematic as
	286	* they would allow guest to stop the CPU completely by disabling
	287	* interrupts then invoking MWAIT.
	288	*/
	289	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
	290	return false;
	291
	292	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
	293
	294	if (!(ecx & CPUID5_ECX_INTERRUPT_BREAK))
	295	return false;
	296
	297	return true;
	298	}
	299
26eef70c	300	#endif