KVM: nVMX: Don't "put" vCPU or host state when switching VMCS

When switching between vmcs01 and vmcs02, KVM isn't actually switching
between guest and host.  If guest state is already loaded (the likely,
if not guaranteed, case), keep the guest state loaded and manually swap
the loaded_cpu_state pointer after propagating saved host state to the
new vmcs0{1,2}.

Avoiding the switch between guest and host reduces the latency of
switching between vmcs01 and vmcs02 by several hundred cycles, and
reduces the roundtrip time of a nested VM by upwards of 1000 cycles.

Signed-off-by: Sean Christopherson <sean.j.christopherson@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index a090bfb10796e5959c6bc6809418e3e35b392fe5..856ac90ce10abb093679e882e1fbecc3fdbcbfb9 100644 (file)
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -248,18 +248,39 @@ static void free_nested(struct kvm_vcpu *vcpu)
        free_loaded_vmcs(&vmx->nested.vmcs02);
 }
 
+static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx,
+                                    struct loaded_vmcs *prev)
+{
+       struct vmcs_host_state *dest, *src;
+
+       if (unlikely(!vmx->guest_state_loaded))
+               return;
+
+       src = &prev->host_state;
+       dest = &vmx->loaded_vmcs->host_state;
+
+       vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base);
+       dest->ldt_sel = src->ldt_sel;
+#ifdef CONFIG_X86_64
+       dest->ds_sel = src->ds_sel;
+       dest->es_sel = src->es_sel;
+#endif
+}
+
 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
 {
        struct vcpu_vmx *vmx = to_vmx(vcpu);
+       struct loaded_vmcs *prev;
        int cpu;
 
        if (vmx->loaded_vmcs == vmcs)
                return;
 
        cpu = get_cpu();
-       vmx_vcpu_put(vcpu);
+       prev = vmx->loaded_vmcs;
        vmx->loaded_vmcs = vmcs;
        vmx_vcpu_load(vcpu, cpu);
+       vmx_sync_vmcs_host_state(vmx, prev);
        put_cpu();
 
        vm_entry_controls_reset_shadow(vmx);

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 02c54f684ce50022eab77f0024121a8b0cd0659d..26b3d96ef5a50a266ff086e0bfb1ca597b54b0ca 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -1039,6 +1039,33 @@ static void pt_guest_exit(struct vcpu_vmx *vmx)
        wrmsrl(MSR_IA32_RTIT_CTL, vmx->pt_desc.host.ctl);
 }
 
+void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel,
+                       unsigned long fs_base, unsigned long gs_base)
+{
+       if (unlikely(fs_sel != host->fs_sel)) {
+               if (!(fs_sel & 7))
+                       vmcs_write16(HOST_FS_SELECTOR, fs_sel);
+               else
+                       vmcs_write16(HOST_FS_SELECTOR, 0);
+               host->fs_sel = fs_sel;
+       }
+       if (unlikely(gs_sel != host->gs_sel)) {
+               if (!(gs_sel & 7))
+                       vmcs_write16(HOST_GS_SELECTOR, gs_sel);
+               else
+                       vmcs_write16(HOST_GS_SELECTOR, 0);
+               host->gs_sel = gs_sel;
+       }
+       if (unlikely(fs_base != host->fs_base)) {
+               vmcs_writel(HOST_FS_BASE, fs_base);
+               host->fs_base = fs_base;
+       }
+       if (unlikely(gs_base != host->gs_base)) {
+               vmcs_writel(HOST_GS_BASE, gs_base);
+               host->gs_base = gs_base;
+       }
+}
+
 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
 {
        struct vcpu_vmx *vmx = to_vmx(vcpu);
@@ -1102,29 +1129,7 @@ void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
        gs_base = segment_base(gs_sel);
 #endif
 
-       if (unlikely(fs_sel != host_state->fs_sel)) {
-               if (!(fs_sel & 7))
-                       vmcs_write16(HOST_FS_SELECTOR, fs_sel);
-               else
-                       vmcs_write16(HOST_FS_SELECTOR, 0);
-               host_state->fs_sel = fs_sel;
-       }
-       if (unlikely(gs_sel != host_state->gs_sel)) {
-               if (!(gs_sel & 7))
-                       vmcs_write16(HOST_GS_SELECTOR, gs_sel);
-               else
-                       vmcs_write16(HOST_GS_SELECTOR, 0);
-               host_state->gs_sel = gs_sel;
-       }
-       if (unlikely(fs_base != host_state->fs_base)) {
-               vmcs_writel(HOST_FS_BASE, fs_base);
-               host_state->fs_base = fs_base;
-       }
-       if (unlikely(gs_base != host_state->gs_base)) {
-               vmcs_writel(HOST_GS_BASE, gs_base);
-               host_state->gs_base = gs_base;
-       }
-
+       vmx_set_host_fs_gs(host_state, fs_sel, gs_sel, fs_base, gs_base);
        vmx->guest_state_loaded = true;
 }
 
@@ -1314,7 +1319,7 @@ static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
                pi_set_sn(pi_desc);
 }
 
-void vmx_vcpu_put(struct kvm_vcpu *vcpu)
+static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
 {
        vmx_vcpu_pi_put(vcpu);
 

diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index 4c5c24f37c5f98f32028cc83cfcbba2f5bde7558..0bd598c47aece09628f1ee281eed72844a4d485c 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -303,11 +303,12 @@ struct kvm_vmx {
 
 bool nested_vmx_allowed(struct kvm_vcpu *vcpu);
 void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
-void vmx_vcpu_put(struct kvm_vcpu *vcpu);
 int allocate_vpid(void);
 void free_vpid(int vpid);
 void vmx_set_constant_host_state(struct vcpu_vmx *vmx);
 void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu);
+void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel,
+                       unsigned long fs_base, unsigned long gs_base);
 int vmx_get_cpl(struct kvm_vcpu *vcpu);
 unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu);
 void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);