KVM: SVM: Save/restore non-volatile GPRs in SEV-ES VMRUN via host save area

Use the host save area to save/restore non-volatile (callee-saved)
registers in __svm_sev_es_vcpu_run() to take advantage of hardware loading
all registers from the save area on #VMEXIT.  KVM still needs to save the
registers it wants restored, but the loads are handled automatically by
hardware.

Aside from less assembly code, letting hardware do the restoration means
stack frames are preserved for the entirety of __svm_sev_es_vcpu_run().

Opportunistically add a comment to call out why @svm needs to be saved
across VMRUN->#VMEXIT, as it's not easy to decipher that from the macro
hell.

Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Michael Roth <michael.roth@amd.com>
Cc: Alexey Kardashevskiy <aik@amd.com>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
Link: https://lore.kernel.org/r/20240223204233.3337324-6-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index d1a9f9951635819c7a585882f06bcf6415ac2cdc..9aaf83c8d57df7d3877484a87060769956dbb9cb 100644 (file)
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -1503,6 +1503,11 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu)
        __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE));
 }
 
+static struct sev_es_save_area *sev_es_host_save_area(struct svm_cpu_data *sd)
+{
+       return page_address(sd->save_area) + 0x400;
+}
+
 static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
 {
        struct vcpu_svm *svm = to_svm(vcpu);
@@ -1519,12 +1524,8 @@ static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
         * or subsequent vmload of host save area.
         */
        vmsave(sd->save_area_pa);
-       if (sev_es_guest(vcpu->kvm)) {
-               struct sev_es_save_area *hostsa;
-               hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400);
-
-               sev_es_prepare_switch_to_guest(hostsa);
-       }
+       if (sev_es_guest(vcpu->kvm))
+               sev_es_prepare_switch_to_guest(sev_es_host_save_area(sd));
 
        if (tsc_scaling)
                __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
@@ -4101,6 +4102,7 @@ static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
 
 static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
 {
+       struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
        struct vcpu_svm *svm = to_svm(vcpu);
 
        guest_state_enter_irqoff();
@@ -4108,7 +4110,8 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_in
        amd_clear_divider();
 
        if (sev_es_guest(vcpu->kvm))
-               __svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted);
+               __svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted,
+                                     sev_es_host_save_area(sd));
        else
                __svm_vcpu_run(svm, spec_ctrl_intercepted);
 

diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 7f1fbd874c4582b0b6d3735b62c3c85de2021074..33878efdebc82987efb574ce1e49e2f3df5637b8 100644 (file)
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -698,7 +698,8 @@ struct page *snp_safe_alloc_page(struct kvm_vcpu *vcpu);
 
 /* vmenter.S */
 
-void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);
+void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted,
+                          struct sev_es_save_area *hostsa);
 void __svm_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);
 
 #define DEFINE_KVM_GHCB_ACCESSORS(field)                                               \

diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S
index db94fb6f610aebd79edb31a8ac40f3a3ea522fdb..7f627d535afc5a55faf6692b80b0be349e942320 100644 (file)
--- a/arch/x86/kvm/svm/vmenter.S
+++ b/arch/x86/kvm/svm/vmenter.S
@@ -292,23 +292,35 @@ SYM_FUNC_START(__svm_vcpu_run)
 SYM_FUNC_END(__svm_vcpu_run)
 
 #ifdef CONFIG_KVM_AMD_SEV
+
+
+#ifdef CONFIG_X86_64
+#define SEV_ES_GPRS_BASE 0x300
+#define SEV_ES_RBX     (SEV_ES_GPRS_BASE + __VCPU_REGS_RBX * WORD_SIZE)
+#define SEV_ES_RBP     (SEV_ES_GPRS_BASE + __VCPU_REGS_RBP * WORD_SIZE)
+#define SEV_ES_R12     (SEV_ES_GPRS_BASE + __VCPU_REGS_R12 * WORD_SIZE)
+#define SEV_ES_R13     (SEV_ES_GPRS_BASE + __VCPU_REGS_R13 * WORD_SIZE)
+#define SEV_ES_R14     (SEV_ES_GPRS_BASE + __VCPU_REGS_R14 * WORD_SIZE)
+#define SEV_ES_R15     (SEV_ES_GPRS_BASE + __VCPU_REGS_R15 * WORD_SIZE)
+#endif
+
 /**
  * __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode
  * @svm:       struct vcpu_svm *
  * @spec_ctrl_intercepted: bool
  */
 SYM_FUNC_START(__svm_sev_es_vcpu_run)
-       push %rbp
-       push %r15
-       push %r14
-       push %r13
-       push %r12
-       push %rbx
-
        /*
-        * Save variables needed after vmexit on the stack, in inverse
-        * order compared to when they are needed.
+        * Save non-volatile (callee-saved) registers to the host save area.
+        * Except for RAX and RSP, all GPRs are restored on #VMEXIT, but not
+        * saved on VMRUN.
         */
+       mov %rbp, SEV_ES_RBP (%rdx)
+       mov %r15, SEV_ES_R15 (%rdx)
+       mov %r14, SEV_ES_R14 (%rdx)
+       mov %r13, SEV_ES_R13 (%rdx)
+       mov %r12, SEV_ES_R12 (%rdx)
+       mov %rbx, SEV_ES_RBX (%rdx)
 
        /* Accessed directly from the stack in RESTORE_HOST_SPEC_CTRL.  */
        push %rsi
@@ -316,7 +328,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
        /* Save @svm. */
        push %rdi
 
-       /* Clobbers RAX, RCX, RDX.  */
+       /* Clobbers RAX, RCX, RDX (@hostsa). */
        RESTORE_GUEST_SPEC_CTRL
 
        /* Get svm->current_vmcb->pa into RAX. */
@@ -338,7 +350,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
        FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
 #endif
 
-       /* Clobbers RAX, RCX, RDX.  */
+       /* Clobbers RAX, RCX, RDX, consumes RDI (@svm). */
        RESTORE_HOST_SPEC_CTRL
 
        /*
@@ -353,13 +365,6 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
        /* "Pop" and discard @spec_ctrl_intercepted. */
        pop %rax
 
-       pop %rbx
-
-       pop %r12
-       pop %r13
-       pop %r14
-       pop %r15
-       pop %rbp
        RET
 
        RESTORE_GUEST_SPEC_CTRL_BODY