tools/testing/selftests/kvm/include/x86_64/processor.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * tools/testing/selftests/kvm/include/x86_64/processor.h
   4  *
   5  * Copyright (C) 2018, Google LLC.
   6  */
   7
   8 #ifndef SELFTEST_KVM_PROCESSOR_H
   9 #define SELFTEST_KVM_PROCESSOR_H
  10
  11 #include <assert.h>
  12 #include <stdint.h>
  13 #include <syscall.h>
  14
  15 #include <asm/msr-index.h>
  16 #include <asm/prctl.h>
  17
  18 #include <linux/kvm_para.h>
  19 #include <linux/stringify.h>
  20
  21 #include "../kvm_util.h"
  22
  23 extern bool host_cpu_is_intel;
  24 extern bool host_cpu_is_amd;
  25
  26 enum vm_guest_x86_subtype {
  27         VM_SUBTYPE_NONE = 0,
  28         VM_SUBTYPE_SEV,
  29         VM_SUBTYPE_SEV_ES,
  30 };
  31
  32 /* Forced emulation prefix, used to invoke the emulator unconditionally. */
  33 #define KVM_FEP "ud2; .byte 'k', 'v', 'm';"
  34
  35 #define NMI_VECTOR              0x02
  36
  37 #define X86_EFLAGS_FIXED         (1u << 1)
  38
  39 #define X86_CR4_VME             (1ul << 0)
  40 #define X86_CR4_PVI             (1ul << 1)
  41 #define X86_CR4_TSD             (1ul << 2)
  42 #define X86_CR4_DE              (1ul << 3)
  43 #define X86_CR4_PSE             (1ul << 4)
  44 #define X86_CR4_PAE             (1ul << 5)
  45 #define X86_CR4_MCE             (1ul << 6)
  46 #define X86_CR4_PGE             (1ul << 7)
  47 #define X86_CR4_PCE             (1ul << 8)
  48 #define X86_CR4_OSFXSR          (1ul << 9)
  49 #define X86_CR4_OSXMMEXCPT      (1ul << 10)
  50 #define X86_CR4_UMIP            (1ul << 11)
  51 #define X86_CR4_LA57            (1ul << 12)
  52 #define X86_CR4_VMXE            (1ul << 13)
  53 #define X86_CR4_SMXE            (1ul << 14)
  54 #define X86_CR4_FSGSBASE        (1ul << 16)
  55 #define X86_CR4_PCIDE           (1ul << 17)
  56 #define X86_CR4_OSXSAVE         (1ul << 18)
  57 #define X86_CR4_SMEP            (1ul << 20)
  58 #define X86_CR4_SMAP            (1ul << 21)
  59 #define X86_CR4_PKE             (1ul << 22)
  60
  61 struct xstate_header {
  62         u64                             xstate_bv;
  63         u64                             xcomp_bv;
  64         u64                             reserved[6];
  65 } __attribute__((packed));
  66
  67 struct xstate {
  68         u8                              i387[512];
  69         struct xstate_header            header;
  70         u8                              extended_state_area[0];
  71 } __attribute__ ((packed, aligned (64)));
  72
  73 #define XFEATURE_MASK_FP                BIT_ULL(0)
  74 #define XFEATURE_MASK_SSE               BIT_ULL(1)
  75 #define XFEATURE_MASK_YMM               BIT_ULL(2)
  76 #define XFEATURE_MASK_BNDREGS           BIT_ULL(3)
  77 #define XFEATURE_MASK_BNDCSR            BIT_ULL(4)
  78 #define XFEATURE_MASK_OPMASK            BIT_ULL(5)
  79 #define XFEATURE_MASK_ZMM_Hi256         BIT_ULL(6)
  80 #define XFEATURE_MASK_Hi16_ZMM          BIT_ULL(7)
  81 #define XFEATURE_MASK_PT                BIT_ULL(8)
  82 #define XFEATURE_MASK_PKRU              BIT_ULL(9)
  83 #define XFEATURE_MASK_PASID             BIT_ULL(10)
  84 #define XFEATURE_MASK_CET_USER          BIT_ULL(11)
  85 #define XFEATURE_MASK_CET_KERNEL        BIT_ULL(12)
  86 #define XFEATURE_MASK_LBR               BIT_ULL(15)
  87 #define XFEATURE_MASK_XTILE_CFG         BIT_ULL(17)
  88 #define XFEATURE_MASK_XTILE_DATA        BIT_ULL(18)
  89
  90 #define XFEATURE_MASK_AVX512            (XFEATURE_MASK_OPMASK | \
  91                                          XFEATURE_MASK_ZMM_Hi256 | \
  92                                          XFEATURE_MASK_Hi16_ZMM)
  93 #define XFEATURE_MASK_XTILE             (XFEATURE_MASK_XTILE_DATA | \
  94                                          XFEATURE_MASK_XTILE_CFG)
  95
  96 /* Note, these are ordered alphabetically to match kvm_cpuid_entry2.  Eww. */
  97 enum cpuid_output_regs {
  98         KVM_CPUID_EAX,
  99         KVM_CPUID_EBX,
 100         KVM_CPUID_ECX,
 101         KVM_CPUID_EDX
 102 };
 103
 104 /*
 105  * Pack the information into a 64-bit value so that each X86_FEATURE_XXX can be
 106  * passed by value with no overhead.
 107  */
 108 struct kvm_x86_cpu_feature {
 109         u32     function;
 110         u16     index;
 111         u8      reg;
 112         u8      bit;
 113 };
 114 #define KVM_X86_CPU_FEATURE(fn, idx, gpr, __bit)                                \
 115 ({                                                                              \
 116         struct kvm_x86_cpu_feature feature = {                                  \
 117                 .function = fn,                                                 \
 118                 .index = idx,                                                   \
 119                 .reg = KVM_CPUID_##gpr,                                         \
 120                 .bit = __bit,                                                   \
 121         };                                                                      \
 122                                                                                 \
 123         kvm_static_assert((fn & 0xc0000000) == 0 ||                             \
 124                           (fn & 0xc0000000) == 0x40000000 ||                    \
 125                           (fn & 0xc0000000) == 0x80000000 ||                    \
 126                           (fn & 0xc0000000) == 0xc0000000);                     \
 127         kvm_static_assert(idx < BIT(sizeof(feature.index) * BITS_PER_BYTE));    \
 128         feature;                                                                \
 129 })
 130
 131 /*
 132  * Basic Leafs, a.k.a. Intel defined
 133  */
 134 #define X86_FEATURE_MWAIT               KVM_X86_CPU_FEATURE(0x1, 0, ECX, 3)
 135 #define X86_FEATURE_VMX                 KVM_X86_CPU_FEATURE(0x1, 0, ECX, 5)
 136 #define X86_FEATURE_SMX                 KVM_X86_CPU_FEATURE(0x1, 0, ECX, 6)
 137 #define X86_FEATURE_PDCM                KVM_X86_CPU_FEATURE(0x1, 0, ECX, 15)
 138 #define X86_FEATURE_PCID                KVM_X86_CPU_FEATURE(0x1, 0, ECX, 17)
 139 #define X86_FEATURE_X2APIC              KVM_X86_CPU_FEATURE(0x1, 0, ECX, 21)
 140 #define X86_FEATURE_MOVBE               KVM_X86_CPU_FEATURE(0x1, 0, ECX, 22)
 141 #define X86_FEATURE_TSC_DEADLINE_TIMER  KVM_X86_CPU_FEATURE(0x1, 0, ECX, 24)
 142 #define X86_FEATURE_XSAVE               KVM_X86_CPU_FEATURE(0x1, 0, ECX, 26)
 143 #define X86_FEATURE_OSXSAVE             KVM_X86_CPU_FEATURE(0x1, 0, ECX, 27)
 144 #define X86_FEATURE_RDRAND              KVM_X86_CPU_FEATURE(0x1, 0, ECX, 30)
 145 #define X86_FEATURE_HYPERVISOR          KVM_X86_CPU_FEATURE(0x1, 0, ECX, 31)
 146 #define X86_FEATURE_PAE                 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 6)
 147 #define X86_FEATURE_MCE                 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 7)
 148 #define X86_FEATURE_APIC                KVM_X86_CPU_FEATURE(0x1, 0, EDX, 9)
 149 #define X86_FEATURE_CLFLUSH             KVM_X86_CPU_FEATURE(0x1, 0, EDX, 19)
 150 #define X86_FEATURE_XMM                 KVM_X86_CPU_FEATURE(0x1, 0, EDX, 25)
 151 #define X86_FEATURE_XMM2                KVM_X86_CPU_FEATURE(0x1, 0, EDX, 26)
 152 #define X86_FEATURE_FSGSBASE            KVM_X86_CPU_FEATURE(0x7, 0, EBX, 0)
 153 #define X86_FEATURE_TSC_ADJUST          KVM_X86_CPU_FEATURE(0x7, 0, EBX, 1)
 154 #define X86_FEATURE_SGX                 KVM_X86_CPU_FEATURE(0x7, 0, EBX, 2)
 155 #define X86_FEATURE_HLE                 KVM_X86_CPU_FEATURE(0x7, 0, EBX, 4)
 156 #define X86_FEATURE_SMEP                KVM_X86_CPU_FEATURE(0x7, 0, EBX, 7)
 157 #define X86_FEATURE_INVPCID             KVM_X86_CPU_FEATURE(0x7, 0, EBX, 10)
 158 #define X86_FEATURE_RTM                 KVM_X86_CPU_FEATURE(0x7, 0, EBX, 11)
 159 #define X86_FEATURE_MPX                 KVM_X86_CPU_FEATURE(0x7, 0, EBX, 14)
 160 #define X86_FEATURE_SMAP                KVM_X86_CPU_FEATURE(0x7, 0, EBX, 20)
 161 #define X86_FEATURE_PCOMMIT             KVM_X86_CPU_FEATURE(0x7, 0, EBX, 22)
 162 #define X86_FEATURE_CLFLUSHOPT          KVM_X86_CPU_FEATURE(0x7, 0, EBX, 23)
 163 #define X86_FEATURE_CLWB                KVM_X86_CPU_FEATURE(0x7, 0, EBX, 24)
 164 #define X86_FEATURE_UMIP                KVM_X86_CPU_FEATURE(0x7, 0, ECX, 2)
 165 #define X86_FEATURE_PKU                 KVM_X86_CPU_FEATURE(0x7, 0, ECX, 3)
 166 #define X86_FEATURE_OSPKE               KVM_X86_CPU_FEATURE(0x7, 0, ECX, 4)
 167 #define X86_FEATURE_LA57                KVM_X86_CPU_FEATURE(0x7, 0, ECX, 16)
 168 #define X86_FEATURE_RDPID               KVM_X86_CPU_FEATURE(0x7, 0, ECX, 22)
 169 #define X86_FEATURE_SGX_LC              KVM_X86_CPU_FEATURE(0x7, 0, ECX, 30)
 170 #define X86_FEATURE_SHSTK               KVM_X86_CPU_FEATURE(0x7, 0, ECX, 7)
 171 #define X86_FEATURE_IBT                 KVM_X86_CPU_FEATURE(0x7, 0, EDX, 20)
 172 #define X86_FEATURE_AMX_TILE            KVM_X86_CPU_FEATURE(0x7, 0, EDX, 24)
 173 #define X86_FEATURE_SPEC_CTRL           KVM_X86_CPU_FEATURE(0x7, 0, EDX, 26)
 174 #define X86_FEATURE_ARCH_CAPABILITIES   KVM_X86_CPU_FEATURE(0x7, 0, EDX, 29)
 175 #define X86_FEATURE_PKS                 KVM_X86_CPU_FEATURE(0x7, 0, ECX, 31)
 176 #define X86_FEATURE_XTILECFG            KVM_X86_CPU_FEATURE(0xD, 0, EAX, 17)
 177 #define X86_FEATURE_XTILEDATA           KVM_X86_CPU_FEATURE(0xD, 0, EAX, 18)
 178 #define X86_FEATURE_XSAVES              KVM_X86_CPU_FEATURE(0xD, 1, EAX, 3)
 179 #define X86_FEATURE_XFD                 KVM_X86_CPU_FEATURE(0xD, 1, EAX, 4)
 180 #define X86_FEATURE_XTILEDATA_XFD       KVM_X86_CPU_FEATURE(0xD, 18, ECX, 2)
 181
 182 /*
 183  * Extended Leafs, a.k.a. AMD defined
 184  */
 185 #define X86_FEATURE_SVM                 KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 2)
 186 #define X86_FEATURE_NX                  KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20)
 187 #define X86_FEATURE_GBPAGES             KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26)
 188 #define X86_FEATURE_RDTSCP              KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27)
 189 #define X86_FEATURE_LM                  KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29)
 190 #define X86_FEATURE_INVTSC              KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8)
 191 #define X86_FEATURE_RDPRU               KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4)
 192 #define X86_FEATURE_AMD_IBPB            KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12)
 193 #define X86_FEATURE_NPT                 KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0)
 194 #define X86_FEATURE_LBRV                KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1)
 195 #define X86_FEATURE_NRIPS               KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3)
 196 #define X86_FEATURE_TSCRATEMSR          KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 4)
 197 #define X86_FEATURE_PAUSEFILTER         KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 10)
 198 #define X86_FEATURE_PFTHRESHOLD         KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 12)
 199 #define X86_FEATURE_VGIF                KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 16)
 200 #define X86_FEATURE_SEV                 KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1)
 201 #define X86_FEATURE_SEV_ES              KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3)
 202
 203 /*
 204  * KVM defined paravirt features.
 205  */
 206 #define X86_FEATURE_KVM_CLOCKSOURCE     KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 0)
 207 #define X86_FEATURE_KVM_NOP_IO_DELAY    KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 1)
 208 #define X86_FEATURE_KVM_MMU_OP          KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 2)
 209 #define X86_FEATURE_KVM_CLOCKSOURCE2    KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 3)
 210 #define X86_FEATURE_KVM_ASYNC_PF        KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 4)
 211 #define X86_FEATURE_KVM_STEAL_TIME      KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 5)
 212 #define X86_FEATURE_KVM_PV_EOI          KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 6)
 213 #define X86_FEATURE_KVM_PV_UNHALT       KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 7)
 214 /* Bit 8 apparently isn't used?!?! */
 215 #define X86_FEATURE_KVM_PV_TLB_FLUSH    KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 9)
 216 #define X86_FEATURE_KVM_ASYNC_PF_VMEXIT KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 10)
 217 #define X86_FEATURE_KVM_PV_SEND_IPI     KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 11)
 218 #define X86_FEATURE_KVM_POLL_CONTROL    KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 12)
 219 #define X86_FEATURE_KVM_PV_SCHED_YIELD  KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 13)
 220 #define X86_FEATURE_KVM_ASYNC_PF_INT    KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 14)
 221 #define X86_FEATURE_KVM_MSI_EXT_DEST_ID KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 15)
 222 #define X86_FEATURE_KVM_HC_MAP_GPA_RANGE        KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 16)
 223 #define X86_FEATURE_KVM_MIGRATION_CONTROL       KVM_X86_CPU_FEATURE(0x40000001, 0, EAX, 17)
 224
 225 /*
 226  * Same idea as X86_FEATURE_XXX, but X86_PROPERTY_XXX retrieves a multi-bit
 227  * value/property as opposed to a single-bit feature.  Again, pack the info
 228  * into a 64-bit value to pass by value with no overhead.
 229  */
 230 struct kvm_x86_cpu_property {
 231         u32     function;
 232         u8      index;
 233         u8      reg;
 234         u8      lo_bit;
 235         u8      hi_bit;
 236 };
 237 #define KVM_X86_CPU_PROPERTY(fn, idx, gpr, low_bit, high_bit)                   \
 238 ({                                                                              \
 239         struct kvm_x86_cpu_property property = {                                \
 240                 .function = fn,                                                 \
 241                 .index = idx,                                                   \
 242                 .reg = KVM_CPUID_##gpr,                                         \
 243                 .lo_bit = low_bit,                                              \
 244                 .hi_bit = high_bit,                                             \
 245         };                                                                      \
 246                                                                                 \
 247         kvm_static_assert(low_bit < high_bit);                                  \
 248         kvm_static_assert((fn & 0xc0000000) == 0 ||                             \
 249                           (fn & 0xc0000000) == 0x40000000 ||                    \
 250                           (fn & 0xc0000000) == 0x80000000 ||                    \
 251                           (fn & 0xc0000000) == 0xc0000000);                     \
 252         kvm_static_assert(idx < BIT(sizeof(property.index) * BITS_PER_BYTE));   \
 253         property;                                                               \
 254 })
 255
 256 #define X86_PROPERTY_MAX_BASIC_LEAF             KVM_X86_CPU_PROPERTY(0, 0, EAX, 0, 31)
 257 #define X86_PROPERTY_PMU_VERSION                KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 0, 7)
 258 #define X86_PROPERTY_PMU_NR_GP_COUNTERS         KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 8, 15)
 259 #define X86_PROPERTY_PMU_GP_COUNTERS_BIT_WIDTH  KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 16, 23)
 260 #define X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH  KVM_X86_CPU_PROPERTY(0xa, 0, EAX, 24, 31)
 261 #define X86_PROPERTY_PMU_EVENTS_MASK            KVM_X86_CPU_PROPERTY(0xa, 0, EBX, 0, 7)
 262 #define X86_PROPERTY_PMU_FIXED_COUNTERS_BITMASK KVM_X86_CPU_PROPERTY(0xa, 0, ECX, 0, 31)
 263 #define X86_PROPERTY_PMU_NR_FIXED_COUNTERS      KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 0, 4)
 264 #define X86_PROPERTY_PMU_FIXED_COUNTERS_BIT_WIDTH       KVM_X86_CPU_PROPERTY(0xa, 0, EDX, 5, 12)
 265
 266 #define X86_PROPERTY_SUPPORTED_XCR0_LO          KVM_X86_CPU_PROPERTY(0xd,  0, EAX,  0, 31)
 267 #define X86_PROPERTY_XSTATE_MAX_SIZE_XCR0       KVM_X86_CPU_PROPERTY(0xd,  0, EBX,  0, 31)
 268 #define X86_PROPERTY_XSTATE_MAX_SIZE            KVM_X86_CPU_PROPERTY(0xd,  0, ECX,  0, 31)
 269 #define X86_PROPERTY_SUPPORTED_XCR0_HI          KVM_X86_CPU_PROPERTY(0xd,  0, EDX,  0, 31)
 270
 271 #define X86_PROPERTY_XSTATE_TILE_SIZE           KVM_X86_CPU_PROPERTY(0xd, 18, EAX,  0, 31)
 272 #define X86_PROPERTY_XSTATE_TILE_OFFSET         KVM_X86_CPU_PROPERTY(0xd, 18, EBX,  0, 31)
 273 #define X86_PROPERTY_AMX_MAX_PALETTE_TABLES     KVM_X86_CPU_PROPERTY(0x1d, 0, EAX,  0, 31)
 274 #define X86_PROPERTY_AMX_TOTAL_TILE_BYTES       KVM_X86_CPU_PROPERTY(0x1d, 1, EAX,  0, 15)
 275 #define X86_PROPERTY_AMX_BYTES_PER_TILE         KVM_X86_CPU_PROPERTY(0x1d, 1, EAX, 16, 31)
 276 #define X86_PROPERTY_AMX_BYTES_PER_ROW          KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 0,  15)
 277 #define X86_PROPERTY_AMX_NR_TILE_REGS           KVM_X86_CPU_PROPERTY(0x1d, 1, EBX, 16, 31)
 278 #define X86_PROPERTY_AMX_MAX_ROWS               KVM_X86_CPU_PROPERTY(0x1d, 1, ECX, 0,  15)
 279
 280 #define X86_PROPERTY_MAX_KVM_LEAF               KVM_X86_CPU_PROPERTY(0x40000000, 0, EAX, 0, 31)
 281
 282 #define X86_PROPERTY_MAX_EXT_LEAF               KVM_X86_CPU_PROPERTY(0x80000000, 0, EAX, 0, 31)
 283 #define X86_PROPERTY_MAX_PHY_ADDR               KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 0, 7)
 284 #define X86_PROPERTY_MAX_VIRT_ADDR              KVM_X86_CPU_PROPERTY(0x80000008, 0, EAX, 8, 15)
 285 #define X86_PROPERTY_SEV_C_BIT                  KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 0, 5)
 286 #define X86_PROPERTY_PHYS_ADDR_REDUCTION        KVM_X86_CPU_PROPERTY(0x8000001F, 0, EBX, 6, 11)
 287
 288 #define X86_PROPERTY_MAX_CENTAUR_LEAF           KVM_X86_CPU_PROPERTY(0xC0000000, 0, EAX, 0, 31)
 289
 290 /*
 291  * Intel's architectural PMU events are bizarre.  They have a "feature" bit
 292  * that indicates the feature is _not_ supported, and a property that states
 293  * the length of the bit mask of unsupported features.  A feature is supported
 294  * if the size of the bit mask is larger than the "unavailable" bit, and said
 295  * bit is not set.  Fixed counters also bizarre enumeration, but inverted from
 296  * arch events for general purpose counters.  Fixed counters are supported if a
 297  * feature flag is set **OR** the total number of fixed counters is greater
 298  * than index of the counter.
 299  *
 300  * Wrap the events for general purpose and fixed counters to simplify checking
 301  * whether or not a given architectural event is supported.
 302  */
 303 struct kvm_x86_pmu_feature {
 304         struct kvm_x86_cpu_feature f;
 305 };
 306 #define KVM_X86_PMU_FEATURE(__reg, __bit)                               \
 307 ({                                                                      \
 308         struct kvm_x86_pmu_feature feature = {                          \
 309                 .f = KVM_X86_CPU_FEATURE(0xa, 0, __reg, __bit),         \
 310         };                                                              \
 311                                                                         \
 312         kvm_static_assert(KVM_CPUID_##__reg == KVM_CPUID_EBX ||         \
 313                           KVM_CPUID_##__reg == KVM_CPUID_ECX);          \
 314         feature;                                                        \
 315 })
 316
 317 #define X86_PMU_FEATURE_CPU_CYCLES                      KVM_X86_PMU_FEATURE(EBX, 0)
 318 #define X86_PMU_FEATURE_INSNS_RETIRED                   KVM_X86_PMU_FEATURE(EBX, 1)
 319 #define X86_PMU_FEATURE_REFERENCE_CYCLES                KVM_X86_PMU_FEATURE(EBX, 2)
 320 #define X86_PMU_FEATURE_LLC_REFERENCES                  KVM_X86_PMU_FEATURE(EBX, 3)
 321 #define X86_PMU_FEATURE_LLC_MISSES                      KVM_X86_PMU_FEATURE(EBX, 4)
 322 #define X86_PMU_FEATURE_BRANCH_INSNS_RETIRED            KVM_X86_PMU_FEATURE(EBX, 5)
 323 #define X86_PMU_FEATURE_BRANCHES_MISPREDICTED           KVM_X86_PMU_FEATURE(EBX, 6)
 324 #define X86_PMU_FEATURE_TOPDOWN_SLOTS                   KVM_X86_PMU_FEATURE(EBX, 7)
 325
 326 #define X86_PMU_FEATURE_INSNS_RETIRED_FIXED             KVM_X86_PMU_FEATURE(ECX, 0)
 327 #define X86_PMU_FEATURE_CPU_CYCLES_FIXED                KVM_X86_PMU_FEATURE(ECX, 1)
 328 #define X86_PMU_FEATURE_REFERENCE_TSC_CYCLES_FIXED      KVM_X86_PMU_FEATURE(ECX, 2)
 329 #define X86_PMU_FEATURE_TOPDOWN_SLOTS_FIXED             KVM_X86_PMU_FEATURE(ECX, 3)
 330
 331 static inline unsigned int x86_family(unsigned int eax)
 332 {
 333         unsigned int x86;
 334
 335         x86 = (eax >> 8) & 0xf;
 336
 337         if (x86 == 0xf)
 338                 x86 += (eax >> 20) & 0xff;
 339
 340         return x86;
 341 }
 342
 343 static inline unsigned int x86_model(unsigned int eax)
 344 {
 345         return ((eax >> 12) & 0xf0) | ((eax >> 4) & 0x0f);
 346 }
 347
 348 /* Page table bitfield declarations */
 349 #define PTE_PRESENT_MASK        BIT_ULL(0)
 350 #define PTE_WRITABLE_MASK       BIT_ULL(1)
 351 #define PTE_USER_MASK           BIT_ULL(2)
 352 #define PTE_ACCESSED_MASK       BIT_ULL(5)
 353 #define PTE_DIRTY_MASK          BIT_ULL(6)
 354 #define PTE_LARGE_MASK          BIT_ULL(7)
 355 #define PTE_GLOBAL_MASK         BIT_ULL(8)
 356 #define PTE_NX_MASK             BIT_ULL(63)
 357
 358 #define PHYSICAL_PAGE_MASK      GENMASK_ULL(51, 12)
 359
 360 #define PAGE_SHIFT              12
 361 #define PAGE_SIZE               (1ULL << PAGE_SHIFT)
 362 #define PAGE_MASK               (~(PAGE_SIZE-1) & PHYSICAL_PAGE_MASK)
 363
 364 #define HUGEPAGE_SHIFT(x)       (PAGE_SHIFT + (((x) - 1) * 9))
 365 #define HUGEPAGE_SIZE(x)        (1UL << HUGEPAGE_SHIFT(x))
 366 #define HUGEPAGE_MASK(x)        (~(HUGEPAGE_SIZE(x) - 1) & PHYSICAL_PAGE_MASK)
 367
 368 #define PTE_GET_PA(pte)         ((pte) & PHYSICAL_PAGE_MASK)
 369 #define PTE_GET_PFN(pte)        (PTE_GET_PA(pte) >> PAGE_SHIFT)
 370
 371 /* General Registers in 64-Bit Mode */
 372 struct gpr64_regs {
 373         u64 rax;
 374         u64 rcx;
 375         u64 rdx;
 376         u64 rbx;
 377         u64 rsp;
 378         u64 rbp;
 379         u64 rsi;
 380         u64 rdi;
 381         u64 r8;
 382         u64 r9;
 383         u64 r10;
 384         u64 r11;
 385         u64 r12;
 386         u64 r13;
 387         u64 r14;
 388         u64 r15;
 389 };
 390
 391 struct desc64 {
 392         uint16_t limit0;
 393         uint16_t base0;
 394         unsigned base1:8, type:4, s:1, dpl:2, p:1;
 395         unsigned limit1:4, avl:1, l:1, db:1, g:1, base2:8;
 396         uint32_t base3;
 397         uint32_t zero1;
 398 } __attribute__((packed));
 399
 400 struct desc_ptr {
 401         uint16_t size;
 402         uint64_t address;
 403 } __attribute__((packed));
 404
 405 struct kvm_x86_state {
 406         struct kvm_xsave *xsave;
 407         struct kvm_vcpu_events events;
 408         struct kvm_mp_state mp_state;
 409         struct kvm_regs regs;
 410         struct kvm_xcrs xcrs;
 411         struct kvm_sregs sregs;
 412         struct kvm_debugregs debugregs;
 413         union {
 414                 struct kvm_nested_state nested;
 415                 char nested_[16384];
 416         };
 417         struct kvm_msrs msrs;
 418 };
 419
 420 static inline uint64_t get_desc64_base(const struct desc64 *desc)
 421 {
 422         return ((uint64_t)desc->base3 << 32) |
 423                 (desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24));
 424 }
 425
 426 static inline uint64_t rdtsc(void)
 427 {
 428         uint32_t eax, edx;
 429         uint64_t tsc_val;
 430         /*
 431          * The lfence is to wait (on Intel CPUs) until all previous
 432          * instructions have been executed. If software requires RDTSC to be
 433          * executed prior to execution of any subsequent instruction, it can
 434          * execute LFENCE immediately after RDTSC
 435          */
 436         __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx));
 437         tsc_val = ((uint64_t)edx) << 32 | eax;
 438         return tsc_val;
 439 }
 440
 441 static inline uint64_t rdtscp(uint32_t *aux)
 442 {
 443         uint32_t eax, edx;
 444
 445         __asm__ __volatile__("rdtscp" : "=a"(eax), "=d"(edx), "=c"(*aux));
 446         return ((uint64_t)edx) << 32 | eax;
 447 }
 448
 449 static inline uint64_t rdmsr(uint32_t msr)
 450 {
 451         uint32_t a, d;
 452
 453         __asm__ __volatile__("rdmsr" : "=a"(a), "=d"(d) : "c"(msr) : "memory");
 454
 455         return a | ((uint64_t) d << 32);
 456 }
 457
 458 static inline void wrmsr(uint32_t msr, uint64_t value)
 459 {
 460         uint32_t a = value;
 461         uint32_t d = value >> 32;
 462
 463         __asm__ __volatile__("wrmsr" :: "a"(a), "d"(d), "c"(msr) : "memory");
 464 }
 465
 466
 467 static inline uint16_t inw(uint16_t port)
 468 {
 469         uint16_t tmp;
 470
 471         __asm__ __volatile__("in %%dx, %%ax"
 472                 : /* output */ "=a" (tmp)
 473                 : /* input */ "d" (port));
 474
 475         return tmp;
 476 }
 477
 478 static inline uint16_t get_es(void)
 479 {
 480         uint16_t es;
 481
 482         __asm__ __volatile__("mov %%es, %[es]"
 483                              : /* output */ [es]"=rm"(es));
 484         return es;
 485 }
 486
 487 static inline uint16_t get_cs(void)
 488 {
 489         uint16_t cs;
 490
 491         __asm__ __volatile__("mov %%cs, %[cs]"
 492                              : /* output */ [cs]"=rm"(cs));
 493         return cs;
 494 }
 495
 496 static inline uint16_t get_ss(void)
 497 {
 498         uint16_t ss;
 499
 500         __asm__ __volatile__("mov %%ss, %[ss]"
 501                              : /* output */ [ss]"=rm"(ss));
 502         return ss;
 503 }
 504
 505 static inline uint16_t get_ds(void)
 506 {
 507         uint16_t ds;
 508
 509         __asm__ __volatile__("mov %%ds, %[ds]"
 510                              : /* output */ [ds]"=rm"(ds));
 511         return ds;
 512 }
 513
 514 static inline uint16_t get_fs(void)
 515 {
 516         uint16_t fs;
 517
 518         __asm__ __volatile__("mov %%fs, %[fs]"
 519                              : /* output */ [fs]"=rm"(fs));
 520         return fs;
 521 }
 522
 523 static inline uint16_t get_gs(void)
 524 {
 525         uint16_t gs;
 526
 527         __asm__ __volatile__("mov %%gs, %[gs]"
 528                              : /* output */ [gs]"=rm"(gs));
 529         return gs;
 530 }
 531
 532 static inline uint16_t get_tr(void)
 533 {
 534         uint16_t tr;
 535
 536         __asm__ __volatile__("str %[tr]"
 537                              : /* output */ [tr]"=rm"(tr));
 538         return tr;
 539 }
 540
 541 static inline uint64_t get_cr0(void)
 542 {
 543         uint64_t cr0;
 544
 545         __asm__ __volatile__("mov %%cr0, %[cr0]"
 546                              : /* output */ [cr0]"=r"(cr0));
 547         return cr0;
 548 }
 549
 550 static inline uint64_t get_cr3(void)
 551 {
 552         uint64_t cr3;
 553
 554         __asm__ __volatile__("mov %%cr3, %[cr3]"
 555                              : /* output */ [cr3]"=r"(cr3));
 556         return cr3;
 557 }
 558
 559 static inline uint64_t get_cr4(void)
 560 {
 561         uint64_t cr4;
 562
 563         __asm__ __volatile__("mov %%cr4, %[cr4]"
 564                              : /* output */ [cr4]"=r"(cr4));
 565         return cr4;
 566 }
 567
 568 static inline void set_cr4(uint64_t val)
 569 {
 570         __asm__ __volatile__("mov %0, %%cr4" : : "r" (val) : "memory");
 571 }
 572
 573 static inline u64 xgetbv(u32 index)
 574 {
 575         u32 eax, edx;
 576
 577         __asm__ __volatile__("xgetbv;"
 578                      : "=a" (eax), "=d" (edx)
 579                      : "c" (index));
 580         return eax | ((u64)edx << 32);
 581 }
 582
 583 static inline void xsetbv(u32 index, u64 value)
 584 {
 585         u32 eax = value;
 586         u32 edx = value >> 32;
 587
 588         __asm__ __volatile__("xsetbv" :: "a" (eax), "d" (edx), "c" (index));
 589 }
 590
 591 static inline void wrpkru(u32 pkru)
 592 {
 593         /* Note, ECX and EDX are architecturally required to be '0'. */
 594         asm volatile(".byte 0x0f,0x01,0xef\n\t"
 595                      : : "a" (pkru), "c"(0), "d"(0));
 596 }
 597
 598 static inline struct desc_ptr get_gdt(void)
 599 {
 600         struct desc_ptr gdt;
 601         __asm__ __volatile__("sgdt %[gdt]"
 602                              : /* output */ [gdt]"=m"(gdt));
 603         return gdt;
 604 }
 605
 606 static inline struct desc_ptr get_idt(void)
 607 {
 608         struct desc_ptr idt;
 609         __asm__ __volatile__("sidt %[idt]"
 610                              : /* output */ [idt]"=m"(idt));
 611         return idt;
 612 }
 613
 614 static inline void outl(uint16_t port, uint32_t value)
 615 {
 616         __asm__ __volatile__("outl %%eax, %%dx" : : "d"(port), "a"(value));
 617 }
 618
 619 static inline void __cpuid(uint32_t function, uint32_t index,
 620                            uint32_t *eax, uint32_t *ebx,
 621                            uint32_t *ecx, uint32_t *edx)
 622 {
 623         *eax = function;
 624         *ecx = index;
 625
 626         asm volatile("cpuid"
 627             : "=a" (*eax),
 628               "=b" (*ebx),
 629               "=c" (*ecx),
 630               "=d" (*edx)
 631             : "0" (*eax), "2" (*ecx)
 632             : "memory");
 633 }
 634
 635 static inline void cpuid(uint32_t function,
 636                          uint32_t *eax, uint32_t *ebx,
 637                          uint32_t *ecx, uint32_t *edx)
 638 {
 639         return __cpuid(function, 0, eax, ebx, ecx, edx);
 640 }
 641
 642 static inline uint32_t this_cpu_fms(void)
 643 {
 644         uint32_t eax, ebx, ecx, edx;
 645
 646         cpuid(1, &eax, &ebx, &ecx, &edx);
 647         return eax;
 648 }
 649
 650 static inline uint32_t this_cpu_family(void)
 651 {
 652         return x86_family(this_cpu_fms());
 653 }
 654
 655 static inline uint32_t this_cpu_model(void)
 656 {
 657         return x86_model(this_cpu_fms());
 658 }
 659
 660 static inline bool this_cpu_vendor_string_is(const char *vendor)
 661 {
 662         const uint32_t *chunk = (const uint32_t *)vendor;
 663         uint32_t eax, ebx, ecx, edx;
 664
 665         cpuid(0, &eax, &ebx, &ecx, &edx);
 666         return (ebx == chunk[0] && edx == chunk[1] && ecx == chunk[2]);
 667 }
 668
 669 static inline bool this_cpu_is_intel(void)
 670 {
 671         return this_cpu_vendor_string_is("GenuineIntel");
 672 }
 673
 674 /*
 675  * Exclude early K5 samples with a vendor string of "AMDisbetter!"
 676  */
 677 static inline bool this_cpu_is_amd(void)
 678 {
 679         return this_cpu_vendor_string_is("AuthenticAMD");
 680 }
 681
 682 static inline uint32_t __this_cpu_has(uint32_t function, uint32_t index,
 683                                       uint8_t reg, uint8_t lo, uint8_t hi)
 684 {
 685         uint32_t gprs[4];
 686
 687         __cpuid(function, index,
 688                 &gprs[KVM_CPUID_EAX], &gprs[KVM_CPUID_EBX],
 689                 &gprs[KVM_CPUID_ECX], &gprs[KVM_CPUID_EDX]);
 690
 691         return (gprs[reg] & GENMASK(hi, lo)) >> lo;
 692 }
 693
 694 static inline bool this_cpu_has(struct kvm_x86_cpu_feature feature)
 695 {
 696         return __this_cpu_has(feature.function, feature.index,
 697                               feature.reg, feature.bit, feature.bit);
 698 }
 699
 700 static inline uint32_t this_cpu_property(struct kvm_x86_cpu_property property)
 701 {
 702         return __this_cpu_has(property.function, property.index,
 703                               property.reg, property.lo_bit, property.hi_bit);
 704 }
 705
 706 static __always_inline bool this_cpu_has_p(struct kvm_x86_cpu_property property)
 707 {
 708         uint32_t max_leaf;
 709
 710         switch (property.function & 0xc0000000) {
 711         case 0:
 712                 max_leaf = this_cpu_property(X86_PROPERTY_MAX_BASIC_LEAF);
 713                 break;
 714         case 0x40000000:
 715                 max_leaf = this_cpu_property(X86_PROPERTY_MAX_KVM_LEAF);
 716                 break;
 717         case 0x80000000:
 718                 max_leaf = this_cpu_property(X86_PROPERTY_MAX_EXT_LEAF);
 719                 break;
 720         case 0xc0000000:
 721                 max_leaf = this_cpu_property(X86_PROPERTY_MAX_CENTAUR_LEAF);
 722         }
 723         return max_leaf >= property.function;
 724 }
 725
 726 static inline bool this_pmu_has(struct kvm_x86_pmu_feature feature)
 727 {
 728         uint32_t nr_bits;
 729
 730         if (feature.f.reg == KVM_CPUID_EBX) {
 731                 nr_bits = this_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH);
 732                 return nr_bits > feature.f.bit && !this_cpu_has(feature.f);
 733         }
 734
 735         GUEST_ASSERT(feature.f.reg == KVM_CPUID_ECX);
 736         nr_bits = this_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
 737         return nr_bits > feature.f.bit || this_cpu_has(feature.f);
 738 }
 739
 740 static __always_inline uint64_t this_cpu_supported_xcr0(void)
 741 {
 742         if (!this_cpu_has_p(X86_PROPERTY_SUPPORTED_XCR0_LO))
 743                 return 0;
 744
 745         return this_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_LO) |
 746                ((uint64_t)this_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32);
 747 }
 748
 749 typedef u32             __attribute__((vector_size(16))) sse128_t;
 750 #define __sse128_u      union { sse128_t vec; u64 as_u64[2]; u32 as_u32[4]; }
 751 #define sse128_lo(x)    ({ __sse128_u t; t.vec = x; t.as_u64[0]; })
 752 #define sse128_hi(x)    ({ __sse128_u t; t.vec = x; t.as_u64[1]; })
 753
 754 static inline void read_sse_reg(int reg, sse128_t *data)
 755 {
 756         switch (reg) {
 757         case 0:
 758                 asm("movdqa %%xmm0, %0" : "=m"(*data));
 759                 break;
 760         case 1:
 761                 asm("movdqa %%xmm1, %0" : "=m"(*data));
 762                 break;
 763         case 2:
 764                 asm("movdqa %%xmm2, %0" : "=m"(*data));
 765                 break;
 766         case 3:
 767                 asm("movdqa %%xmm3, %0" : "=m"(*data));
 768                 break;
 769         case 4:
 770                 asm("movdqa %%xmm4, %0" : "=m"(*data));
 771                 break;
 772         case 5:
 773                 asm("movdqa %%xmm5, %0" : "=m"(*data));
 774                 break;
 775         case 6:
 776                 asm("movdqa %%xmm6, %0" : "=m"(*data));
 777                 break;
 778         case 7:
 779                 asm("movdqa %%xmm7, %0" : "=m"(*data));
 780                 break;
 781         default:
 782                 BUG();
 783         }
 784 }
 785
 786 static inline void write_sse_reg(int reg, const sse128_t *data)
 787 {
 788         switch (reg) {
 789         case 0:
 790                 asm("movdqa %0, %%xmm0" : : "m"(*data));
 791                 break;
 792         case 1:
 793                 asm("movdqa %0, %%xmm1" : : "m"(*data));
 794                 break;
 795         case 2:
 796                 asm("movdqa %0, %%xmm2" : : "m"(*data));
 797                 break;
 798         case 3:
 799                 asm("movdqa %0, %%xmm3" : : "m"(*data));
 800                 break;
 801         case 4:
 802                 asm("movdqa %0, %%xmm4" : : "m"(*data));
 803                 break;
 804         case 5:
 805                 asm("movdqa %0, %%xmm5" : : "m"(*data));
 806                 break;
 807         case 6:
 808                 asm("movdqa %0, %%xmm6" : : "m"(*data));
 809                 break;
 810         case 7:
 811                 asm("movdqa %0, %%xmm7" : : "m"(*data));
 812                 break;
 813         default:
 814                 BUG();
 815         }
 816 }
 817
 818 static inline void cpu_relax(void)
 819 {
 820         asm volatile("rep; nop" ::: "memory");
 821 }
 822
 823 #define ud2()                   \
 824         __asm__ __volatile__(   \
 825                 "ud2\n" \
 826                 )
 827
 828 #define hlt()                   \
 829         __asm__ __volatile__(   \
 830                 "hlt\n" \
 831                 )
 832
 833 struct kvm_x86_state *vcpu_save_state(struct kvm_vcpu *vcpu);
 834 void vcpu_load_state(struct kvm_vcpu *vcpu, struct kvm_x86_state *state);
 835 void kvm_x86_state_cleanup(struct kvm_x86_state *state);
 836
 837 const struct kvm_msr_list *kvm_get_msr_index_list(void);
 838 const struct kvm_msr_list *kvm_get_feature_msr_index_list(void);
 839 bool kvm_msr_is_in_save_restore_list(uint32_t msr_index);
 840 uint64_t kvm_get_feature_msr(uint64_t msr_index);
 841
 842 static inline void vcpu_msrs_get(struct kvm_vcpu *vcpu,
 843                                  struct kvm_msrs *msrs)
 844 {
 845         int r = __vcpu_ioctl(vcpu, KVM_GET_MSRS, msrs);
 846
 847         TEST_ASSERT(r == msrs->nmsrs,
 848                     "KVM_GET_MSRS failed, r: %i (failed on MSR %x)",
 849                     r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index);
 850 }
 851 static inline void vcpu_msrs_set(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs)
 852 {
 853         int r = __vcpu_ioctl(vcpu, KVM_SET_MSRS, msrs);
 854
 855         TEST_ASSERT(r == msrs->nmsrs,
 856                     "KVM_SET_MSRS failed, r: %i (failed on MSR %x)",
 857                     r, r < 0 || r >= msrs->nmsrs ? -1 : msrs->entries[r].index);
 858 }
 859 static inline void vcpu_debugregs_get(struct kvm_vcpu *vcpu,
 860                                       struct kvm_debugregs *debugregs)
 861 {
 862         vcpu_ioctl(vcpu, KVM_GET_DEBUGREGS, debugregs);
 863 }
 864 static inline void vcpu_debugregs_set(struct kvm_vcpu *vcpu,
 865                                       struct kvm_debugregs *debugregs)
 866 {
 867         vcpu_ioctl(vcpu, KVM_SET_DEBUGREGS, debugregs);
 868 }
 869 static inline void vcpu_xsave_get(struct kvm_vcpu *vcpu,
 870                                   struct kvm_xsave *xsave)
 871 {
 872         vcpu_ioctl(vcpu, KVM_GET_XSAVE, xsave);
 873 }
 874 static inline void vcpu_xsave2_get(struct kvm_vcpu *vcpu,
 875                                    struct kvm_xsave *xsave)
 876 {
 877         vcpu_ioctl(vcpu, KVM_GET_XSAVE2, xsave);
 878 }
 879 static inline void vcpu_xsave_set(struct kvm_vcpu *vcpu,
 880                                   struct kvm_xsave *xsave)
 881 {
 882         vcpu_ioctl(vcpu, KVM_SET_XSAVE, xsave);
 883 }
 884 static inline void vcpu_xcrs_get(struct kvm_vcpu *vcpu,
 885                                  struct kvm_xcrs *xcrs)
 886 {
 887         vcpu_ioctl(vcpu, KVM_GET_XCRS, xcrs);
 888 }
 889 static inline void vcpu_xcrs_set(struct kvm_vcpu *vcpu, struct kvm_xcrs *xcrs)
 890 {
 891         vcpu_ioctl(vcpu, KVM_SET_XCRS, xcrs);
 892 }
 893
 894 const struct kvm_cpuid_entry2 *get_cpuid_entry(const struct kvm_cpuid2 *cpuid,
 895                                                uint32_t function, uint32_t index);
 896 const struct kvm_cpuid2 *kvm_get_supported_cpuid(void);
 897 const struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void);
 898 const struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vcpu *vcpu);
 899
 900 static inline uint32_t kvm_cpu_fms(void)
 901 {
 902         return get_cpuid_entry(kvm_get_supported_cpuid(), 0x1, 0)->eax;
 903 }
 904
 905 static inline uint32_t kvm_cpu_family(void)
 906 {
 907         return x86_family(kvm_cpu_fms());
 908 }
 909
 910 static inline uint32_t kvm_cpu_model(void)
 911 {
 912         return x86_model(kvm_cpu_fms());
 913 }
 914
 915 bool kvm_cpuid_has(const struct kvm_cpuid2 *cpuid,
 916                    struct kvm_x86_cpu_feature feature);
 917
 918 static inline bool kvm_cpu_has(struct kvm_x86_cpu_feature feature)
 919 {
 920         return kvm_cpuid_has(kvm_get_supported_cpuid(), feature);
 921 }
 922
 923 uint32_t kvm_cpuid_property(const struct kvm_cpuid2 *cpuid,
 924                             struct kvm_x86_cpu_property property);
 925
 926 static inline uint32_t kvm_cpu_property(struct kvm_x86_cpu_property property)
 927 {
 928         return kvm_cpuid_property(kvm_get_supported_cpuid(), property);
 929 }
 930
 931 static __always_inline bool kvm_cpu_has_p(struct kvm_x86_cpu_property property)
 932 {
 933         uint32_t max_leaf;
 934
 935         switch (property.function & 0xc0000000) {
 936         case 0:
 937                 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_BASIC_LEAF);
 938                 break;
 939         case 0x40000000:
 940                 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_KVM_LEAF);
 941                 break;
 942         case 0x80000000:
 943                 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_EXT_LEAF);
 944                 break;
 945         case 0xc0000000:
 946                 max_leaf = kvm_cpu_property(X86_PROPERTY_MAX_CENTAUR_LEAF);
 947         }
 948         return max_leaf >= property.function;
 949 }
 950
 951 static inline bool kvm_pmu_has(struct kvm_x86_pmu_feature feature)
 952 {
 953         uint32_t nr_bits;
 954
 955         if (feature.f.reg == KVM_CPUID_EBX) {
 956                 nr_bits = kvm_cpu_property(X86_PROPERTY_PMU_EBX_BIT_VECTOR_LENGTH);
 957                 return nr_bits > feature.f.bit && !kvm_cpu_has(feature.f);
 958         }
 959
 960         TEST_ASSERT_EQ(feature.f.reg, KVM_CPUID_ECX);
 961         nr_bits = kvm_cpu_property(X86_PROPERTY_PMU_NR_FIXED_COUNTERS);
 962         return nr_bits > feature.f.bit || kvm_cpu_has(feature.f);
 963 }
 964
 965 static __always_inline uint64_t kvm_cpu_supported_xcr0(void)
 966 {
 967         if (!kvm_cpu_has_p(X86_PROPERTY_SUPPORTED_XCR0_LO))
 968                 return 0;
 969
 970         return kvm_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_LO) |
 971                ((uint64_t)kvm_cpu_property(X86_PROPERTY_SUPPORTED_XCR0_HI) << 32);
 972 }
 973
 974 static inline size_t kvm_cpuid2_size(int nr_entries)
 975 {
 976         return sizeof(struct kvm_cpuid2) +
 977                sizeof(struct kvm_cpuid_entry2) * nr_entries;
 978 }
 979
 980 /*
 981  * Allocate a "struct kvm_cpuid2* instance, with the 0-length arrary of
 982  * entries sized to hold @nr_entries.  The caller is responsible for freeing
 983  * the struct.
 984  */
 985 static inline struct kvm_cpuid2 *allocate_kvm_cpuid2(int nr_entries)
 986 {
 987         struct kvm_cpuid2 *cpuid;
 988
 989         cpuid = malloc(kvm_cpuid2_size(nr_entries));
 990         TEST_ASSERT(cpuid, "-ENOMEM when allocating kvm_cpuid2");
 991
 992         cpuid->nent = nr_entries;
 993
 994         return cpuid;
 995 }
 996
 997 void vcpu_init_cpuid(struct kvm_vcpu *vcpu, const struct kvm_cpuid2 *cpuid);
 998 void vcpu_set_hv_cpuid(struct kvm_vcpu *vcpu);
 999
1000 static inline struct kvm_cpuid_entry2 *__vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,
1001                                                               uint32_t function,
1002                                                               uint32_t index)
1003 {
1004         return (struct kvm_cpuid_entry2 *)get_cpuid_entry(vcpu->cpuid,
1005                                                           function, index);
1006 }
1007
1008 static inline struct kvm_cpuid_entry2 *vcpu_get_cpuid_entry(struct kvm_vcpu *vcpu,
1009                                                             uint32_t function)
1010 {
1011         return __vcpu_get_cpuid_entry(vcpu, function, 0);
1012 }
1013
1014 static inline int __vcpu_set_cpuid(struct kvm_vcpu *vcpu)
1015 {
1016         int r;
1017
1018         TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first");
1019         r = __vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid);
1020         if (r)
1021                 return r;
1022
1023         /* On success, refresh the cache to pick up adjustments made by KVM. */
1024         vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid);
1025         return 0;
1026 }
1027
1028 static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu)
1029 {
1030         TEST_ASSERT(vcpu->cpuid, "Must do vcpu_init_cpuid() first");
1031         vcpu_ioctl(vcpu, KVM_SET_CPUID2, vcpu->cpuid);
1032
1033         /* Refresh the cache to pick up adjustments made by KVM. */
1034         vcpu_ioctl(vcpu, KVM_GET_CPUID2, vcpu->cpuid);
1035 }
1036
1037 void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu,
1038                              struct kvm_x86_cpu_property property,
1039                              uint32_t value);
1040 void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, uint8_t maxphyaddr);
1041
1042 void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, uint32_t function);
1043
1044 static inline bool vcpu_cpuid_has(struct kvm_vcpu *vcpu,
1045                                   struct kvm_x86_cpu_feature feature)
1046 {
1047         struct kvm_cpuid_entry2 *entry;
1048
1049         entry = __vcpu_get_cpuid_entry(vcpu, feature.function, feature.index);
1050         return *((&entry->eax) + feature.reg) & BIT(feature.bit);
1051 }
1052
1053 void vcpu_set_or_clear_cpuid_feature(struct kvm_vcpu *vcpu,
1054                                      struct kvm_x86_cpu_feature feature,
1055                                      bool set);
1056
1057 static inline void vcpu_set_cpuid_feature(struct kvm_vcpu *vcpu,
1058                                           struct kvm_x86_cpu_feature feature)
1059 {
1060         vcpu_set_or_clear_cpuid_feature(vcpu, feature, true);
1061
1062 }
1063
1064 static inline void vcpu_clear_cpuid_feature(struct kvm_vcpu *vcpu,
1065                                             struct kvm_x86_cpu_feature feature)
1066 {
1067         vcpu_set_or_clear_cpuid_feature(vcpu, feature, false);
1068 }
1069
1070 uint64_t vcpu_get_msr(struct kvm_vcpu *vcpu, uint64_t msr_index);
1071 int _vcpu_set_msr(struct kvm_vcpu *vcpu, uint64_t msr_index, uint64_t msr_value);
1072
1073 /*
1074  * Assert on an MSR access(es) and pretty print the MSR name when possible.
1075  * Note, the caller provides the stringified name so that the name of macro is
1076  * printed, not the value the macro resolves to (due to macro expansion).
1077  */
1078 #define TEST_ASSERT_MSR(cond, fmt, msr, str, args...)                           \
1079 do {                                                                            \
1080         if (__builtin_constant_p(msr)) {                                        \
1081                 TEST_ASSERT(cond, fmt, str, args);                              \
1082         } else if (!(cond)) {                                                   \
1083                 char buf[16];                                                   \
1084                                                                                 \
1085                 snprintf(buf, sizeof(buf), "MSR 0x%x", msr);                    \
1086                 TEST_ASSERT(cond, fmt, buf, args);                              \
1087         }                                                                       \
1088 } while (0)
1089
1090 /*
1091  * Returns true if KVM should return the last written value when reading an MSR
1092  * from userspace, e.g. the MSR isn't a command MSR, doesn't emulate state that
1093  * is changing, etc.  This is NOT an exhaustive list!  The intent is to filter
1094  * out MSRs that are not durable _and_ that a selftest wants to write.
1095  */
1096 static inline bool is_durable_msr(uint32_t msr)
1097 {
1098         return msr != MSR_IA32_TSC;
1099 }
1100
1101 #define vcpu_set_msr(vcpu, msr, val)                                                    \
1102 do {                                                                                    \
1103         uint64_t r, v = val;                                                            \
1104                                                                                         \
1105         TEST_ASSERT_MSR(_vcpu_set_msr(vcpu, msr, v) == 1,                               \
1106                         "KVM_SET_MSRS failed on %s, value = 0x%lx", msr, #msr, v);      \
1107         if (!is_durable_msr(msr))                                                       \
1108                 break;                                                                  \
1109         r = vcpu_get_msr(vcpu, msr);                                                    \
1110         TEST_ASSERT_MSR(r == v, "Set %s to '0x%lx', got back '0x%lx'", msr, #msr, v, r);\
1111 } while (0)
1112
1113 void kvm_get_cpu_address_width(unsigned int *pa_bits, unsigned int *va_bits);
1114 void kvm_init_vm_address_properties(struct kvm_vm *vm);
1115 bool vm_is_unrestricted_guest(struct kvm_vm *vm);
1116
1117 struct ex_regs {
1118         uint64_t rax, rcx, rdx, rbx;
1119         uint64_t rbp, rsi, rdi;
1120         uint64_t r8, r9, r10, r11;
1121         uint64_t r12, r13, r14, r15;
1122         uint64_t vector;
1123         uint64_t error_code;
1124         uint64_t rip;
1125         uint64_t cs;
1126         uint64_t rflags;
1127 };
1128
1129 struct idt_entry {
1130         uint16_t offset0;
1131         uint16_t selector;
1132         uint16_t ist : 3;
1133         uint16_t : 5;
1134         uint16_t type : 4;
1135         uint16_t : 1;
1136         uint16_t dpl : 2;
1137         uint16_t p : 1;
1138         uint16_t offset1;
1139         uint32_t offset2; uint32_t reserved;
1140 };
1141
1142 void vm_init_descriptor_tables(struct kvm_vm *vm);
1143 void vcpu_init_descriptor_tables(struct kvm_vcpu *vcpu);
1144 void vm_install_exception_handler(struct kvm_vm *vm, int vector,
1145                         void (*handler)(struct ex_regs *));
1146
1147 /* If a toddler were to say "abracadabra". */
1148 #define KVM_EXCEPTION_MAGIC 0xabacadabaULL
1149
1150 /*
1151  * KVM selftest exception fixup uses registers to coordinate with the exception
1152  * handler, versus the kernel's in-memory tables and KVM-Unit-Tests's in-memory
1153  * per-CPU data.  Using only registers avoids having to map memory into the
1154  * guest, doesn't require a valid, stable GS.base, and reduces the risk of
1155  * for recursive faults when accessing memory in the handler.  The downside to
1156  * using registers is that it restricts what registers can be used by the actual
1157  * instruction.  But, selftests are 64-bit only, making register* pressure a
1158  * minor concern.  Use r9-r11 as they are volatile, i.e. don't need to be saved
1159  * by the callee, and except for r11 are not implicit parameters to any
1160  * instructions.  Ideally, fixup would use r8-r10 and thus avoid implicit
1161  * parameters entirely, but Hyper-V's hypercall ABI uses r8 and testing Hyper-V
1162  * is higher priority than testing non-faulting SYSCALL/SYSRET.
1163  *
1164  * Note, the fixup handler deliberately does not handle #DE, i.e. the vector
1165  * is guaranteed to be non-zero on fault.
1166  *
1167  * REGISTER INPUTS:
1168  * r9  = MAGIC
1169  * r10 = RIP
1170  * r11 = new RIP on fault
1171  *
1172  * REGISTER OUTPUTS:
1173  * r9  = exception vector (non-zero)
1174  * r10 = error code
1175  */
1176 #define __KVM_ASM_SAFE(insn, fep)                               \
1177         "mov $" __stringify(KVM_EXCEPTION_MAGIC) ", %%r9\n\t"   \
1178         "lea 1f(%%rip), %%r10\n\t"                              \
1179         "lea 2f(%%rip), %%r11\n\t"                              \
1180         fep "1: " insn "\n\t"                                   \
1181         "xor %%r9, %%r9\n\t"                                    \
1182         "2:\n\t"                                                \
1183         "mov  %%r9b, %[vector]\n\t"                             \
1184         "mov  %%r10, %[error_code]\n\t"
1185
1186 #define KVM_ASM_SAFE(insn) __KVM_ASM_SAFE(insn, "")
1187 #define KVM_ASM_SAFE_FEP(insn) __KVM_ASM_SAFE(insn, KVM_FEP)
1188
1189 #define KVM_ASM_SAFE_OUTPUTS(v, ec)     [vector] "=qm"(v), [error_code] "=rm"(ec)
1190 #define KVM_ASM_SAFE_CLOBBERS   "r9", "r10", "r11"
1191
1192 #define kvm_asm_safe(insn, inputs...)                                   \
1193 ({                                                                      \
1194         uint64_t ign_error_code;                                        \
1195         uint8_t vector;                                                 \
1196                                                                         \
1197         asm volatile(KVM_ASM_SAFE(insn)                                 \
1198                      : KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code)     \
1199                      : inputs                                           \
1200                      : KVM_ASM_SAFE_CLOBBERS);                          \
1201         vector;                                                         \
1202 })
1203
1204 #define kvm_asm_safe_ec(insn, error_code, inputs...)                    \
1205 ({                                                                      \
1206         uint8_t vector;                                                 \
1207                                                                         \
1208         asm volatile(KVM_ASM_SAFE(insn)                                 \
1209                      : KVM_ASM_SAFE_OUTPUTS(vector, error_code)         \
1210                      : inputs                                           \
1211                      : KVM_ASM_SAFE_CLOBBERS);                          \
1212         vector;                                                         \
1213 })
1214
1215 #define kvm_asm_safe_fep(insn, inputs...)                               \
1216 ({                                                                      \
1217         uint64_t ign_error_code;                                        \
1218         uint8_t vector;                                                 \
1219                                                                         \
1220         asm volatile(KVM_ASM_SAFE(insn)                                 \
1221                      : KVM_ASM_SAFE_OUTPUTS(vector, ign_error_code)     \
1222                      : inputs                                           \
1223                      : KVM_ASM_SAFE_CLOBBERS);                          \
1224         vector;                                                         \
1225 })
1226
1227 #define kvm_asm_safe_ec_fep(insn, error_code, inputs...)                \
1228 ({                                                                      \
1229         uint8_t vector;                                                 \
1230                                                                         \
1231         asm volatile(KVM_ASM_SAFE_FEP(insn)                             \
1232                      : KVM_ASM_SAFE_OUTPUTS(vector, error_code)         \
1233                      : inputs                                           \
1234                      : KVM_ASM_SAFE_CLOBBERS);                          \
1235         vector;                                                         \
1236 })
1237
1238 #define BUILD_READ_U64_SAFE_HELPER(insn, _fep, _FEP)                    \
1239 static inline uint8_t insn##_safe ##_fep(uint32_t idx, uint64_t *val)   \
1240 {                                                                       \
1241         uint64_t error_code;                                            \
1242         uint8_t vector;                                                 \
1243         uint32_t a, d;                                                  \
1244                                                                         \
1245         asm volatile(KVM_ASM_SAFE##_FEP(#insn)                          \
1246                      : "=a"(a), "=d"(d),                                \
1247                        KVM_ASM_SAFE_OUTPUTS(vector, error_code)         \
1248                      : "c"(idx)                                         \
1249                      : KVM_ASM_SAFE_CLOBBERS);                          \
1250                                                                         \
1251         *val = (uint64_t)a | ((uint64_t)d << 32);                       \
1252         return vector;                                                  \
1253 }
1254
1255 /*
1256  * Generate {insn}_safe() and {insn}_safe_fep() helpers for instructions that
1257  * use ECX as in input index, and EDX:EAX as a 64-bit output.
1258  */
1259 #define BUILD_READ_U64_SAFE_HELPERS(insn)                               \
1260         BUILD_READ_U64_SAFE_HELPER(insn, , )                            \
1261         BUILD_READ_U64_SAFE_HELPER(insn, _fep, _FEP)                    \
1262
1263 BUILD_READ_U64_SAFE_HELPERS(rdmsr)
1264 BUILD_READ_U64_SAFE_HELPERS(rdpmc)
1265 BUILD_READ_U64_SAFE_HELPERS(xgetbv)
1266
1267 static inline uint8_t wrmsr_safe(uint32_t msr, uint64_t val)
1268 {
1269         return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr));
1270 }
1271
1272 static inline uint8_t xsetbv_safe(uint32_t index, uint64_t value)
1273 {
1274         u32 eax = value;
1275         u32 edx = value >> 32;
1276
1277         return kvm_asm_safe("xsetbv", "a" (eax), "d" (edx), "c" (index));
1278 }
1279
1280 bool kvm_is_tdp_enabled(void);
1281
1282 static inline bool kvm_is_pmu_enabled(void)
1283 {
1284         return get_kvm_param_bool("enable_pmu");
1285 }
1286
1287 static inline bool kvm_is_forced_emulation_enabled(void)
1288 {
1289         return !!get_kvm_param_integer("force_emulation_prefix");
1290 }
1291
1292 uint64_t *__vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr,
1293                                     int *level);
1294 uint64_t *vm_get_page_table_entry(struct kvm_vm *vm, uint64_t vaddr);
1295
1296 uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
1297                        uint64_t a3);
1298 uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1);
1299 void xen_hypercall(uint64_t nr, uint64_t a0, void *a1);
1300
1301 static inline uint64_t __kvm_hypercall_map_gpa_range(uint64_t gpa,
1302                                                      uint64_t size, uint64_t flags)
1303 {
1304         return kvm_hypercall(KVM_HC_MAP_GPA_RANGE, gpa, size >> PAGE_SHIFT, flags, 0);
1305 }
1306
1307 static inline void kvm_hypercall_map_gpa_range(uint64_t gpa, uint64_t size,
1308                                                uint64_t flags)
1309 {
1310         uint64_t ret = __kvm_hypercall_map_gpa_range(gpa, size, flags);
1311
1312         GUEST_ASSERT(!ret);
1313 }
1314
1315 void __vm_xsave_require_permission(uint64_t xfeature, const char *name);
1316
1317 #define vm_xsave_require_permission(xfeature)   \
1318         __vm_xsave_require_permission(xfeature, #xfeature)
1319
1320 enum pg_level {
1321         PG_LEVEL_NONE,
1322         PG_LEVEL_4K,
1323         PG_LEVEL_2M,
1324         PG_LEVEL_1G,
1325         PG_LEVEL_512G,
1326         PG_LEVEL_NUM
1327 };
1328
1329 #define PG_LEVEL_SHIFT(_level) ((_level - 1) * 9 + 12)
1330 #define PG_LEVEL_SIZE(_level) (1ull << PG_LEVEL_SHIFT(_level))
1331
1332 #define PG_SIZE_4K PG_LEVEL_SIZE(PG_LEVEL_4K)
1333 #define PG_SIZE_2M PG_LEVEL_SIZE(PG_LEVEL_2M)
1334 #define PG_SIZE_1G PG_LEVEL_SIZE(PG_LEVEL_1G)
1335
1336 void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level);
1337 void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
1338                     uint64_t nr_bytes, int level);
1339
1340 /*
1341  * Basic CPU control in CR0
1342  */
1343 #define X86_CR0_PE          (1UL<<0) /* Protection Enable */
1344 #define X86_CR0_MP          (1UL<<1) /* Monitor Coprocessor */
1345 #define X86_CR0_EM          (1UL<<2) /* Emulation */
1346 #define X86_CR0_TS          (1UL<<3) /* Task Switched */
1347 #define X86_CR0_ET          (1UL<<4) /* Extension Type */
1348 #define X86_CR0_NE          (1UL<<5) /* Numeric Error */
1349 #define X86_CR0_WP          (1UL<<16) /* Write Protect */
1350 #define X86_CR0_AM          (1UL<<18) /* Alignment Mask */
1351 #define X86_CR0_NW          (1UL<<29) /* Not Write-through */
1352 #define X86_CR0_CD          (1UL<<30) /* Cache Disable */
1353 #define X86_CR0_PG          (1UL<<31) /* Paging */
1354
1355 #define PFERR_PRESENT_BIT 0
1356 #define PFERR_WRITE_BIT 1
1357 #define PFERR_USER_BIT 2
1358 #define PFERR_RSVD_BIT 3
1359 #define PFERR_FETCH_BIT 4
1360 #define PFERR_PK_BIT 5
1361 #define PFERR_SGX_BIT 15
1362 #define PFERR_GUEST_FINAL_BIT 32
1363 #define PFERR_GUEST_PAGE_BIT 33
1364 #define PFERR_IMPLICIT_ACCESS_BIT 48
1365
1366 #define PFERR_PRESENT_MASK      BIT(PFERR_PRESENT_BIT)
1367 #define PFERR_WRITE_MASK        BIT(PFERR_WRITE_BIT)
1368 #define PFERR_USER_MASK         BIT(PFERR_USER_BIT)
1369 #define PFERR_RSVD_MASK         BIT(PFERR_RSVD_BIT)
1370 #define PFERR_FETCH_MASK        BIT(PFERR_FETCH_BIT)
1371 #define PFERR_PK_MASK           BIT(PFERR_PK_BIT)
1372 #define PFERR_SGX_MASK          BIT(PFERR_SGX_BIT)
1373 #define PFERR_GUEST_FINAL_MASK  BIT_ULL(PFERR_GUEST_FINAL_BIT)
1374 #define PFERR_GUEST_PAGE_MASK   BIT_ULL(PFERR_GUEST_PAGE_BIT)
1375 #define PFERR_IMPLICIT_ACCESS   BIT_ULL(PFERR_IMPLICIT_ACCESS_BIT)
1376
1377 bool sys_clocksource_is_based_on_tsc(void);
1378
1379 #endif /* SELFTEST_KVM_PROCESSOR_H */