arch/x86/kvm/vmx/hyperv.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   3
   4 #include <linux/errno.h>
   5 #include <linux/smp.h>
   6
   7 #include "../cpuid.h"
   8 #include "hyperv.h"
   9 #include "nested.h"
  10 #include "vmcs.h"
  11 #include "vmx.h"
  12 #include "trace.h"
  13
  14 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
  15
  16 /*
  17  * Enlightened VMCSv1 doesn't support these:
  18  *
  19  *      POSTED_INTR_NV                  = 0x00000002,
  20  *      GUEST_INTR_STATUS               = 0x00000810,
  21  *      APIC_ACCESS_ADDR                = 0x00002014,
  22  *      POSTED_INTR_DESC_ADDR           = 0x00002016,
  23  *      EOI_EXIT_BITMAP0                = 0x0000201c,
  24  *      EOI_EXIT_BITMAP1                = 0x0000201e,
  25  *      EOI_EXIT_BITMAP2                = 0x00002020,
  26  *      EOI_EXIT_BITMAP3                = 0x00002022,
  27  *      GUEST_PML_INDEX                 = 0x00000812,
  28  *      PML_ADDRESS                     = 0x0000200e,
  29  *      VM_FUNCTION_CONTROL             = 0x00002018,
  30  *      EPTP_LIST_ADDRESS               = 0x00002024,
  31  *      VMREAD_BITMAP                   = 0x00002026,
  32  *      VMWRITE_BITMAP                  = 0x00002028,
  33  *
  34  *      TSC_MULTIPLIER                  = 0x00002032,
  35  *      PLE_GAP                         = 0x00004020,
  36  *      PLE_WINDOW                      = 0x00004022,
  37  *      VMX_PREEMPTION_TIMER_VALUE      = 0x0000482E,
  38  *
  39  * Currently unsupported in KVM:
  40  *      GUEST_IA32_RTIT_CTL             = 0x00002814,
  41  */
  42 #define EVMCS1_SUPPORTED_PINCTRL                                        \
  43         (PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |                          \
  44          PIN_BASED_EXT_INTR_MASK |                                      \
  45          PIN_BASED_NMI_EXITING |                                        \
  46          PIN_BASED_VIRTUAL_NMIS)
  47
  48 #define EVMCS1_SUPPORTED_EXEC_CTRL                                      \
  49         (CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |                          \
  50          CPU_BASED_HLT_EXITING |                                        \
  51          CPU_BASED_CR3_LOAD_EXITING |                                   \
  52          CPU_BASED_CR3_STORE_EXITING |                                  \
  53          CPU_BASED_UNCOND_IO_EXITING |                                  \
  54          CPU_BASED_MOV_DR_EXITING |                                     \
  55          CPU_BASED_USE_TSC_OFFSETTING |                                 \
  56          CPU_BASED_MWAIT_EXITING |                                      \
  57          CPU_BASED_MONITOR_EXITING |                                    \
  58          CPU_BASED_INVLPG_EXITING |                                     \
  59          CPU_BASED_RDPMC_EXITING |                                      \
  60          CPU_BASED_INTR_WINDOW_EXITING |                                \
  61          CPU_BASED_CR8_LOAD_EXITING |                                   \
  62          CPU_BASED_CR8_STORE_EXITING |                                  \
  63          CPU_BASED_RDTSC_EXITING |                                      \
  64          CPU_BASED_TPR_SHADOW |                                         \
  65          CPU_BASED_USE_IO_BITMAPS |                                     \
  66          CPU_BASED_MONITOR_TRAP_FLAG |                                  \
  67          CPU_BASED_USE_MSR_BITMAPS |                                    \
  68          CPU_BASED_NMI_WINDOW_EXITING |                                 \
  69          CPU_BASED_PAUSE_EXITING |                                      \
  70          CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
  71
  72 #define EVMCS1_SUPPORTED_2NDEXEC                                        \
  73         (SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |                        \
  74          SECONDARY_EXEC_WBINVD_EXITING |                                \
  75          SECONDARY_EXEC_ENABLE_VPID |                                   \
  76          SECONDARY_EXEC_ENABLE_EPT |                                    \
  77          SECONDARY_EXEC_UNRESTRICTED_GUEST |                            \
  78          SECONDARY_EXEC_DESC |                                          \
  79          SECONDARY_EXEC_ENABLE_RDTSCP |                                 \
  80          SECONDARY_EXEC_ENABLE_INVPCID |                                \
  81          SECONDARY_EXEC_XSAVES |                                        \
  82          SECONDARY_EXEC_RDSEED_EXITING |                                \
  83          SECONDARY_EXEC_RDRAND_EXITING |                                \
  84          SECONDARY_EXEC_TSC_SCALING |                                   \
  85          SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |                         \
  86          SECONDARY_EXEC_PT_USE_GPA |                                    \
  87          SECONDARY_EXEC_PT_CONCEAL_VMX |                                \
  88          SECONDARY_EXEC_BUS_LOCK_DETECTION |                            \
  89          SECONDARY_EXEC_NOTIFY_VM_EXITING |                             \
  90          SECONDARY_EXEC_ENCLS_EXITING)
  91
  92 #define EVMCS1_SUPPORTED_3RDEXEC (0ULL)
  93
  94 #define EVMCS1_SUPPORTED_VMEXIT_CTRL                                    \
  95         (VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |                            \
  96          VM_EXIT_SAVE_DEBUG_CONTROLS |                                  \
  97          VM_EXIT_ACK_INTR_ON_EXIT |                                     \
  98          VM_EXIT_HOST_ADDR_SPACE_SIZE |                                 \
  99          VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |                           \
 100          VM_EXIT_SAVE_IA32_PAT |                                        \
 101          VM_EXIT_LOAD_IA32_PAT |                                        \
 102          VM_EXIT_SAVE_IA32_EFER |                                       \
 103          VM_EXIT_LOAD_IA32_EFER |                                       \
 104          VM_EXIT_CLEAR_BNDCFGS |                                        \
 105          VM_EXIT_PT_CONCEAL_PIP |                                       \
 106          VM_EXIT_CLEAR_IA32_RTIT_CTL)
 107
 108 #define EVMCS1_SUPPORTED_VMENTRY_CTRL                                   \
 109         (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR |                           \
 110          VM_ENTRY_LOAD_DEBUG_CONTROLS |                                 \
 111          VM_ENTRY_IA32E_MODE |                                          \
 112          VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL |                          \
 113          VM_ENTRY_LOAD_IA32_PAT |                                       \
 114          VM_ENTRY_LOAD_IA32_EFER |                                      \
 115          VM_ENTRY_LOAD_BNDCFGS |                                        \
 116          VM_ENTRY_PT_CONCEAL_PIP |                                      \
 117          VM_ENTRY_LOAD_IA32_RTIT_CTL)
 118
 119 #define EVMCS1_SUPPORTED_VMFUNC (0)
 120
 121 DEFINE_STATIC_KEY_FALSE(enable_evmcs);
 122
 123 #define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x)
 124 #define EVMCS1_FIELD(number, name, clean_field)[ROL16(number, 6)] = \
 125                 {EVMCS1_OFFSET(name), clean_field}
 126
 127 const struct evmcs_field vmcs_field_to_evmcs_1[] = {
 128         /* 64 bit rw */
 129         EVMCS1_FIELD(GUEST_RIP, guest_rip,
 130                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 131         EVMCS1_FIELD(GUEST_RSP, guest_rsp,
 132                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC),
 133         EVMCS1_FIELD(GUEST_RFLAGS, guest_rflags,
 134                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC),
 135         EVMCS1_FIELD(HOST_IA32_PAT, host_ia32_pat,
 136                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 137         EVMCS1_FIELD(HOST_IA32_EFER, host_ia32_efer,
 138                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 139         EVMCS1_FIELD(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl,
 140                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 141         EVMCS1_FIELD(HOST_CR0, host_cr0,
 142                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 143         EVMCS1_FIELD(HOST_CR3, host_cr3,
 144                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 145         EVMCS1_FIELD(HOST_CR4, host_cr4,
 146                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 147         EVMCS1_FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp,
 148                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 149         EVMCS1_FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip,
 150                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 151         EVMCS1_FIELD(HOST_RIP, host_rip,
 152                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 153         EVMCS1_FIELD(IO_BITMAP_A, io_bitmap_a,
 154                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP),
 155         EVMCS1_FIELD(IO_BITMAP_B, io_bitmap_b,
 156                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP),
 157         EVMCS1_FIELD(MSR_BITMAP, msr_bitmap,
 158                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP),
 159         EVMCS1_FIELD(GUEST_ES_BASE, guest_es_base,
 160                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 161         EVMCS1_FIELD(GUEST_CS_BASE, guest_cs_base,
 162                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 163         EVMCS1_FIELD(GUEST_SS_BASE, guest_ss_base,
 164                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 165         EVMCS1_FIELD(GUEST_DS_BASE, guest_ds_base,
 166                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 167         EVMCS1_FIELD(GUEST_FS_BASE, guest_fs_base,
 168                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 169         EVMCS1_FIELD(GUEST_GS_BASE, guest_gs_base,
 170                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 171         EVMCS1_FIELD(GUEST_LDTR_BASE, guest_ldtr_base,
 172                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 173         EVMCS1_FIELD(GUEST_TR_BASE, guest_tr_base,
 174                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 175         EVMCS1_FIELD(GUEST_GDTR_BASE, guest_gdtr_base,
 176                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 177         EVMCS1_FIELD(GUEST_IDTR_BASE, guest_idtr_base,
 178                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 179         EVMCS1_FIELD(TSC_OFFSET, tsc_offset,
 180                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
 181         EVMCS1_FIELD(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr,
 182                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
 183         EVMCS1_FIELD(VMCS_LINK_POINTER, vmcs_link_pointer,
 184                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 185         EVMCS1_FIELD(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl,
 186                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 187         EVMCS1_FIELD(GUEST_IA32_PAT, guest_ia32_pat,
 188                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 189         EVMCS1_FIELD(GUEST_IA32_EFER, guest_ia32_efer,
 190                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 191         EVMCS1_FIELD(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl,
 192                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 193         EVMCS1_FIELD(GUEST_PDPTR0, guest_pdptr0,
 194                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 195         EVMCS1_FIELD(GUEST_PDPTR1, guest_pdptr1,
 196                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 197         EVMCS1_FIELD(GUEST_PDPTR2, guest_pdptr2,
 198                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 199         EVMCS1_FIELD(GUEST_PDPTR3, guest_pdptr3,
 200                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 201         EVMCS1_FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions,
 202                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 203         EVMCS1_FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp,
 204                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 205         EVMCS1_FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip,
 206                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 207         EVMCS1_FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask,
 208                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 209         EVMCS1_FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask,
 210                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 211         EVMCS1_FIELD(CR0_READ_SHADOW, cr0_read_shadow,
 212                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 213         EVMCS1_FIELD(CR4_READ_SHADOW, cr4_read_shadow,
 214                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 215         EVMCS1_FIELD(GUEST_CR0, guest_cr0,
 216                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 217         EVMCS1_FIELD(GUEST_CR3, guest_cr3,
 218                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 219         EVMCS1_FIELD(GUEST_CR4, guest_cr4,
 220                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 221         EVMCS1_FIELD(GUEST_DR7, guest_dr7,
 222                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR),
 223         EVMCS1_FIELD(HOST_FS_BASE, host_fs_base,
 224                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 225         EVMCS1_FIELD(HOST_GS_BASE, host_gs_base,
 226                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 227         EVMCS1_FIELD(HOST_TR_BASE, host_tr_base,
 228                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 229         EVMCS1_FIELD(HOST_GDTR_BASE, host_gdtr_base,
 230                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 231         EVMCS1_FIELD(HOST_IDTR_BASE, host_idtr_base,
 232                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 233         EVMCS1_FIELD(HOST_RSP, host_rsp,
 234                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER),
 235         EVMCS1_FIELD(EPT_POINTER, ept_pointer,
 236                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT),
 237         EVMCS1_FIELD(GUEST_BNDCFGS, guest_bndcfgs,
 238                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 239         EVMCS1_FIELD(XSS_EXIT_BITMAP, xss_exit_bitmap,
 240                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
 241         EVMCS1_FIELD(ENCLS_EXITING_BITMAP, encls_exiting_bitmap,
 242                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
 243         EVMCS1_FIELD(TSC_MULTIPLIER, tsc_multiplier,
 244                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2),
 245         /*
 246          * Not used by KVM:
 247          *
 248          * EVMCS1_FIELD(0x00006828, guest_ia32_s_cet,
 249          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 250          * EVMCS1_FIELD(0x0000682A, guest_ssp,
 251          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC),
 252          * EVMCS1_FIELD(0x0000682C, guest_ia32_int_ssp_table_addr,
 253          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 254          * EVMCS1_FIELD(0x00002816, guest_ia32_lbr_ctl,
 255          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 256          * EVMCS1_FIELD(0x00006C18, host_ia32_s_cet,
 257          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 258          * EVMCS1_FIELD(0x00006C1A, host_ssp,
 259          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 260          * EVMCS1_FIELD(0x00006C1C, host_ia32_int_ssp_table_addr,
 261          *           HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 262          */
 263
 264         /* 64 bit read only */
 265         EVMCS1_FIELD(GUEST_PHYSICAL_ADDRESS, guest_physical_address,
 266                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 267         EVMCS1_FIELD(EXIT_QUALIFICATION, exit_qualification,
 268                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 269         /*
 270          * Not defined in KVM:
 271          *
 272          * EVMCS1_FIELD(0x00006402, exit_io_instruction_ecx,
 273          *              HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE);
 274          * EVMCS1_FIELD(0x00006404, exit_io_instruction_esi,
 275          *              HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE);
 276          * EVMCS1_FIELD(0x00006406, exit_io_instruction_esi,
 277          *              HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE);
 278          * EVMCS1_FIELD(0x00006408, exit_io_instruction_eip,
 279          *              HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE);
 280          */
 281         EVMCS1_FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address,
 282                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 283
 284         /*
 285          * No mask defined in the spec as Hyper-V doesn't currently support
 286          * these. Future proof by resetting the whole clean field mask on
 287          * access.
 288          */
 289         EVMCS1_FIELD(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr,
 290                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 291         EVMCS1_FIELD(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr,
 292                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 293         EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr,
 294                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 295
 296         /* 32 bit rw */
 297         EVMCS1_FIELD(TPR_THRESHOLD, tpr_threshold,
 298                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 299         EVMCS1_FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info,
 300                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC),
 301         EVMCS1_FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control,
 302                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC),
 303         EVMCS1_FIELD(EXCEPTION_BITMAP, exception_bitmap,
 304                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN),
 305         EVMCS1_FIELD(VM_ENTRY_CONTROLS, vm_entry_controls,
 306                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY),
 307         EVMCS1_FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field,
 308                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT),
 309         EVMCS1_FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE,
 310                      vm_entry_exception_error_code,
 311                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT),
 312         EVMCS1_FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len,
 313                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT),
 314         EVMCS1_FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs,
 315                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 316         EVMCS1_FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control,
 317                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1),
 318         EVMCS1_FIELD(VM_EXIT_CONTROLS, vm_exit_controls,
 319                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1),
 320         EVMCS1_FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control,
 321                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1),
 322         EVMCS1_FIELD(GUEST_ES_LIMIT, guest_es_limit,
 323                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 324         EVMCS1_FIELD(GUEST_CS_LIMIT, guest_cs_limit,
 325                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 326         EVMCS1_FIELD(GUEST_SS_LIMIT, guest_ss_limit,
 327                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 328         EVMCS1_FIELD(GUEST_DS_LIMIT, guest_ds_limit,
 329                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 330         EVMCS1_FIELD(GUEST_FS_LIMIT, guest_fs_limit,
 331                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 332         EVMCS1_FIELD(GUEST_GS_LIMIT, guest_gs_limit,
 333                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 334         EVMCS1_FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit,
 335                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 336         EVMCS1_FIELD(GUEST_TR_LIMIT, guest_tr_limit,
 337                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 338         EVMCS1_FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit,
 339                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 340         EVMCS1_FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit,
 341                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 342         EVMCS1_FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes,
 343                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 344         EVMCS1_FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes,
 345                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 346         EVMCS1_FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes,
 347                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 348         EVMCS1_FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes,
 349                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 350         EVMCS1_FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes,
 351                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 352         EVMCS1_FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes,
 353                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 354         EVMCS1_FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes,
 355                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 356         EVMCS1_FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes,
 357                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 358         EVMCS1_FIELD(GUEST_ACTIVITY_STATE, guest_activity_state,
 359                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 360         EVMCS1_FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs,
 361                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1),
 362
 363         /* 32 bit read only */
 364         EVMCS1_FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error,
 365                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 366         EVMCS1_FIELD(VM_EXIT_REASON, vm_exit_reason,
 367                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 368         EVMCS1_FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info,
 369                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 370         EVMCS1_FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code,
 371                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 372         EVMCS1_FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field,
 373                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 374         EVMCS1_FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code,
 375                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 376         EVMCS1_FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len,
 377                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 378         EVMCS1_FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info,
 379                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE),
 380
 381         /* No mask defined in the spec (not used) */
 382         EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask,
 383                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 384         EVMCS1_FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match,
 385                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 386         EVMCS1_FIELD(CR3_TARGET_COUNT, cr3_target_count,
 387                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 388         EVMCS1_FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count,
 389                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 390         EVMCS1_FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count,
 391                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 392         EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count,
 393                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL),
 394
 395         /* 16 bit rw */
 396         EVMCS1_FIELD(HOST_ES_SELECTOR, host_es_selector,
 397                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 398         EVMCS1_FIELD(HOST_CS_SELECTOR, host_cs_selector,
 399                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 400         EVMCS1_FIELD(HOST_SS_SELECTOR, host_ss_selector,
 401                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 402         EVMCS1_FIELD(HOST_DS_SELECTOR, host_ds_selector,
 403                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 404         EVMCS1_FIELD(HOST_FS_SELECTOR, host_fs_selector,
 405                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 406         EVMCS1_FIELD(HOST_GS_SELECTOR, host_gs_selector,
 407                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 408         EVMCS1_FIELD(HOST_TR_SELECTOR, host_tr_selector,
 409                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1),
 410         EVMCS1_FIELD(GUEST_ES_SELECTOR, guest_es_selector,
 411                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 412         EVMCS1_FIELD(GUEST_CS_SELECTOR, guest_cs_selector,
 413                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 414         EVMCS1_FIELD(GUEST_SS_SELECTOR, guest_ss_selector,
 415                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 416         EVMCS1_FIELD(GUEST_DS_SELECTOR, guest_ds_selector,
 417                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 418         EVMCS1_FIELD(GUEST_FS_SELECTOR, guest_fs_selector,
 419                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 420         EVMCS1_FIELD(GUEST_GS_SELECTOR, guest_gs_selector,
 421                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 422         EVMCS1_FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector,
 423                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 424         EVMCS1_FIELD(GUEST_TR_SELECTOR, guest_tr_selector,
 425                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2),
 426         EVMCS1_FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id,
 427                      HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT),
 428 };
 429 const unsigned int nr_evmcs_1_fields = ARRAY_SIZE(vmcs_field_to_evmcs_1);
 430
 431 u64 nested_get_evmptr(struct kvm_vcpu *vcpu)
 432 {
 433         struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
 434
 435         if (unlikely(kvm_hv_get_assist_page(vcpu)))
 436                 return EVMPTR_INVALID;
 437
 438         if (unlikely(!hv_vcpu->vp_assist_page.enlighten_vmentry))
 439                 return EVMPTR_INVALID;
 440
 441         return hv_vcpu->vp_assist_page.current_nested_vmcs;
 442 }
 443
 444 uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu)
 445 {
 446         /*
 447          * vmcs_version represents the range of supported Enlightened VMCS
 448          * versions: lower 8 bits is the minimal version, higher 8 bits is the
 449          * maximum supported version. KVM supports versions from 1 to
 450          * KVM_EVMCS_VERSION.
 451          *
 452          * Note, do not check the Hyper-V is fully enabled in guest CPUID, this
 453          * helper is used to _get_ the vCPU's supported CPUID.
 454          */
 455         if (kvm_cpu_cap_get(X86_FEATURE_VMX) &&
 456             (!vcpu || to_vmx(vcpu)->nested.enlightened_vmcs_enabled))
 457                 return (KVM_EVMCS_VERSION << 8) | 1;
 458
 459         return 0;
 460 }
 461
 462 enum evmcs_revision {
 463         EVMCSv1_LEGACY,
 464         NR_EVMCS_REVISIONS,
 465 };
 466
 467 enum evmcs_ctrl_type {
 468         EVMCS_EXIT_CTRLS,
 469         EVMCS_ENTRY_CTRLS,
 470         EVMCS_EXEC_CTRL,
 471         EVMCS_2NDEXEC,
 472         EVMCS_3RDEXEC,
 473         EVMCS_PINCTRL,
 474         EVMCS_VMFUNC,
 475         NR_EVMCS_CTRLS,
 476 };
 477
 478 static const u32 evmcs_supported_ctrls[NR_EVMCS_CTRLS][NR_EVMCS_REVISIONS] = {
 479         [EVMCS_EXIT_CTRLS] = {
 480                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMEXIT_CTRL,
 481         },
 482         [EVMCS_ENTRY_CTRLS] = {
 483                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMENTRY_CTRL,
 484         },
 485         [EVMCS_EXEC_CTRL] = {
 486                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_EXEC_CTRL,
 487         },
 488         [EVMCS_2NDEXEC] = {
 489                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_2NDEXEC & ~SECONDARY_EXEC_TSC_SCALING,
 490         },
 491         [EVMCS_3RDEXEC] = {
 492                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_3RDEXEC,
 493         },
 494         [EVMCS_PINCTRL] = {
 495                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_PINCTRL,
 496         },
 497         [EVMCS_VMFUNC] = {
 498                 [EVMCSv1_LEGACY] = EVMCS1_SUPPORTED_VMFUNC,
 499         },
 500 };
 501
 502 static u32 evmcs_get_supported_ctls(enum evmcs_ctrl_type ctrl_type)
 503 {
 504         enum evmcs_revision evmcs_rev = EVMCSv1_LEGACY;
 505
 506         return evmcs_supported_ctrls[ctrl_type][evmcs_rev];
 507 }
 508
 509 static bool evmcs_has_perf_global_ctrl(struct kvm_vcpu *vcpu)
 510 {
 511         struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
 512
 513         /*
 514          * PERF_GLOBAL_CTRL has a quirk where some Windows guests may fail to
 515          * boot if a PV CPUID feature flag is not also set.  Treat the fields
 516          * as unsupported if the flag is not set in guest CPUID.  This should
 517          * be called only for guest accesses, and all guest accesses should be
 518          * gated on Hyper-V being enabled and initialized.
 519          */
 520         if (WARN_ON_ONCE(!hv_vcpu))
 521                 return false;
 522
 523         return hv_vcpu->cpuid_cache.nested_ebx & HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL;
 524 }
 525
 526 void nested_evmcs_filter_control_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
 527 {
 528         u32 ctl_low = (u32)*pdata;
 529         u32 ctl_high = (u32)(*pdata >> 32);
 530         u32 supported_ctrls;
 531
 532         /*
 533          * Hyper-V 2016 and 2019 try using these features even when eVMCS
 534          * is enabled but there are no corresponding fields.
 535          */
 536         switch (msr_index) {
 537         case MSR_IA32_VMX_EXIT_CTLS:
 538         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
 539                 supported_ctrls = evmcs_get_supported_ctls(EVMCS_EXIT_CTRLS);
 540                 if (!evmcs_has_perf_global_ctrl(vcpu))
 541                         supported_ctrls &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
 542                 ctl_high &= supported_ctrls;
 543                 break;
 544         case MSR_IA32_VMX_ENTRY_CTLS:
 545         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
 546                 supported_ctrls = evmcs_get_supported_ctls(EVMCS_ENTRY_CTRLS);
 547                 if (!evmcs_has_perf_global_ctrl(vcpu))
 548                         supported_ctrls &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
 549                 ctl_high &= supported_ctrls;
 550                 break;
 551         case MSR_IA32_VMX_PROCBASED_CTLS:
 552         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
 553                 ctl_high &= evmcs_get_supported_ctls(EVMCS_EXEC_CTRL);
 554                 break;
 555         case MSR_IA32_VMX_PROCBASED_CTLS2:
 556                 ctl_high &= evmcs_get_supported_ctls(EVMCS_2NDEXEC);
 557                 break;
 558         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
 559         case MSR_IA32_VMX_PINBASED_CTLS:
 560                 ctl_high &= evmcs_get_supported_ctls(EVMCS_PINCTRL);
 561                 break;
 562         case MSR_IA32_VMX_VMFUNC:
 563                 ctl_low &= evmcs_get_supported_ctls(EVMCS_VMFUNC);
 564                 break;
 565         }
 566
 567         *pdata = ctl_low | ((u64)ctl_high << 32);
 568 }
 569
 570 static bool nested_evmcs_is_valid_controls(enum evmcs_ctrl_type ctrl_type,
 571                                            u32 val)
 572 {
 573         return !(val & ~evmcs_get_supported_ctls(ctrl_type));
 574 }
 575
 576 int nested_evmcs_check_controls(struct vmcs12 *vmcs12)
 577 {
 578         if (CC(!nested_evmcs_is_valid_controls(EVMCS_PINCTRL,
 579                                                vmcs12->pin_based_vm_exec_control)))
 580                 return -EINVAL;
 581
 582         if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXEC_CTRL,
 583                                                vmcs12->cpu_based_vm_exec_control)))
 584                 return -EINVAL;
 585
 586         if (CC(!nested_evmcs_is_valid_controls(EVMCS_2NDEXEC,
 587                                                vmcs12->secondary_vm_exec_control)))
 588                 return -EINVAL;
 589
 590         if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXIT_CTRLS,
 591                                                vmcs12->vm_exit_controls)))
 592                 return -EINVAL;
 593
 594         if (CC(!nested_evmcs_is_valid_controls(EVMCS_ENTRY_CTRLS,
 595                                                vmcs12->vm_entry_controls)))
 596                 return -EINVAL;
 597
 598         /*
 599          * VM-Func controls are 64-bit, but KVM currently doesn't support any
 600          * controls in bits 63:32, i.e. dropping those bits on the consistency
 601          * check is intentional.
 602          */
 603         if (WARN_ON_ONCE(vmcs12->vm_function_control >> 32))
 604                 return -EINVAL;
 605
 606         if (CC(!nested_evmcs_is_valid_controls(EVMCS_VMFUNC,
 607                                                vmcs12->vm_function_control)))
 608                 return -EINVAL;
 609
 610         return 0;
 611 }
 612
 613 #if IS_ENABLED(CONFIG_HYPERV)
 614 /*
 615  * KVM on Hyper-V always uses the latest known eVMCSv1 revision, the assumption
 616  * is: in case a feature has corresponding fields in eVMCS described and it was
 617  * exposed in VMX feature MSRs, KVM is free to use it. Warn if KVM meets a
 618  * feature which has no corresponding eVMCS field, this likely means that KVM
 619  * needs to be updated.
 620  */
 621 #define evmcs_check_vmcs_conf(field, ctrl)                                      \
 622         do {                                                                    \
 623                 typeof(vmcs_conf->field) unsupported;                           \
 624                                                                                 \
 625                 unsupported = vmcs_conf->field & ~EVMCS1_SUPPORTED_ ## ctrl;    \
 626                 if (unsupported) {                                              \
 627                         pr_warn_once(#field " unsupported with eVMCS: 0x%llx\n",\
 628                                      (u64)unsupported);                         \
 629                         vmcs_conf->field &= EVMCS1_SUPPORTED_ ## ctrl;          \
 630                 }                                                               \
 631         }                                                                       \
 632         while (0)
 633
 634 void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf)
 635 {
 636         evmcs_check_vmcs_conf(cpu_based_exec_ctrl, EXEC_CTRL);
 637         evmcs_check_vmcs_conf(pin_based_exec_ctrl, PINCTRL);
 638         evmcs_check_vmcs_conf(cpu_based_2nd_exec_ctrl, 2NDEXEC);
 639         evmcs_check_vmcs_conf(cpu_based_3rd_exec_ctrl, 3RDEXEC);
 640         evmcs_check_vmcs_conf(vmentry_ctrl, VMENTRY_CTRL);
 641         evmcs_check_vmcs_conf(vmexit_ctrl, VMEXIT_CTRL);
 642 }
 643 #endif
 644
 645 int nested_enable_evmcs(struct kvm_vcpu *vcpu,
 646                         uint16_t *vmcs_version)
 647 {
 648         struct vcpu_vmx *vmx = to_vmx(vcpu);
 649
 650         vmx->nested.enlightened_vmcs_enabled = true;
 651
 652         if (vmcs_version)
 653                 *vmcs_version = nested_get_evmcs_version(vcpu);
 654
 655         return 0;
 656 }
 657
 658 bool nested_evmcs_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu)
 659 {
 660         struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu);
 661         struct vcpu_vmx *vmx = to_vmx(vcpu);
 662         struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
 663
 664         if (!hv_vcpu || !evmcs)
 665                 return false;
 666
 667         if (!evmcs->hv_enlightenments_control.nested_flush_hypercall)
 668                 return false;
 669
 670         return hv_vcpu->vp_assist_page.nested_control.features.directhypercall;
 671 }
 672
 673 void vmx_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu)
 674 {
 675         nested_vmx_vmexit(vcpu, HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH, 0, 0);
 676 }