KVM: x86/mmu: Zap only obsolete roots if a root shadow page is zapped

Zap only obsolete roots when responding to zapping a single root shadow
page.  Because KVM keeps root_count elevated when stuffing a previous
root into its PGD cache, shadowing a 64-bit guest means that zapping any
root causes all vCPUs to reload all roots, even if their current root is
not affected by the zap.

For many kernels, zapping a single root is a frequent operation, e.g. in
Linux it happens whenever an mm is dropped, e.g. process exits, etc...

Signed-off-by: Sean Christopherson <seanjc@google.com>
Reviewed-by: Ben Gardon <bgardon@google.com>
Message-Id: <20220225182248.3812651-5-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index ccec837e520d8007d3bb4dee7c8b30984dd9d93a..c45ab8b5c37f03d1bb29aa234f7501871f8f0467 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -102,6 +102,8 @@
 #define KVM_REQ_MSR_FILTER_CHANGED     KVM_ARCH_REQ(29)
 #define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \
        KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
+#define KVM_REQ_MMU_FREE_OBSOLETE_ROOTS \
+       KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
 
 #define CR0_RESERVED_BITS                                               \
        (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \

diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 1d0c1904d69a3d744e9c2a4edae8a99820b12b03..bf8dbc4bb12aa2ae438ca7c7f766e26a6a09908e 100644 (file)
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -80,6 +80,7 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 
 int kvm_mmu_load(struct kvm_vcpu *vcpu);
 void kvm_mmu_unload(struct kvm_vcpu *vcpu);
+void kvm_mmu_free_obsolete_roots(struct kvm_vcpu *vcpu);
 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
 void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu);
 

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 32c6d4b33d032704d38060c14ffddba9d801c3f2..825996408465b3fb049959e462c0847f9cd509a1 100644 (file)
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -2310,7 +2310,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
                                       struct list_head *invalid_list,
                                       int *nr_zapped)
 {
-       bool list_unstable;
+       bool list_unstable, zapped_root = false;
 
        trace_kvm_mmu_prepare_zap_page(sp);
        ++kvm->stat.mmu_shadow_zapped;
@@ -2352,14 +2352,20 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
                 * in kvm_mmu_zap_all_fast().  Note, is_obsolete_sp() also
                 * treats invalid shadow pages as being obsolete.
                 */
-               if (!is_obsolete_sp(kvm, sp))
-                       kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
+               zapped_root = !is_obsolete_sp(kvm, sp);
        }
 
        if (sp->lpage_disallowed)
                unaccount_huge_nx_page(kvm, sp);
 
        sp->role.invalid = 1;
+
+       /*
+        * Make the request to free obsolete roots after marking the root
+        * invalid, otherwise other vCPUs may not see it as invalid.
+        */
+       if (zapped_root)
+               kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
        return list_unstable;
 }
 
@@ -3947,7 +3953,7 @@ static bool is_page_fault_stale(struct kvm_vcpu *vcpu,
         * previous root, then __kvm_mmu_prepare_zap_page() signals all vCPUs
         * to reload even if no vCPU is actively using the root.
         */
-       if (!sp && kvm_test_request(KVM_REQ_MMU_RELOAD, vcpu))
+       if (!sp && kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
                return true;
 
        return fault->slot &&
@@ -4180,8 +4186,8 @@ void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd)
        /*
         * It's possible that the cached previous root page is obsolete because
         * of a change in the MMU generation number. However, changing the
-        * generation number is accompanied by KVM_REQ_MMU_RELOAD, which will
-        * free the root set here and allocate a new one.
+        * generation number is accompanied by KVM_REQ_MMU_FREE_OBSOLETE_ROOTS,
+        * which will free the root set here and allocate a new one.
         */
        kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
 
@@ -5085,6 +5091,51 @@ void kvm_mmu_unload(struct kvm_vcpu *vcpu)
        vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
 }
 
+static bool is_obsolete_root(struct kvm *kvm, hpa_t root_hpa)
+{
+       struct kvm_mmu_page *sp;
+
+       if (!VALID_PAGE(root_hpa))
+               return false;
+
+       /*
+        * When freeing obsolete roots, treat roots as obsolete if they don't
+        * have an associated shadow page.  This does mean KVM will get false
+        * positives and free roots that don't strictly need to be freed, but
+        * such false positives are relatively rare:
+        *
+        *  (a) only PAE paging and nested NPT has roots without shadow pages
+        *  (b) remote reloads due to a memslot update obsoletes _all_ roots
+        *  (c) KVM doesn't track previous roots for PAE paging, and the guest
+        *      is unlikely to zap an in-use PGD.
+        */
+       sp = to_shadow_page(root_hpa);
+       return !sp || is_obsolete_sp(kvm, sp);
+}
+
+static void __kvm_mmu_free_obsolete_roots(struct kvm *kvm, struct kvm_mmu *mmu)
+{
+       unsigned long roots_to_free = 0;
+       int i;
+
+       if (is_obsolete_root(kvm, mmu->root.hpa))
+               roots_to_free |= KVM_MMU_ROOT_CURRENT;
+
+       for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
+               if (is_obsolete_root(kvm, mmu->root.hpa))
+                       roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
+       }
+
+       if (roots_to_free)
+               kvm_mmu_free_roots(kvm, mmu, roots_to_free);
+}
+
+void kvm_mmu_free_obsolete_roots(struct kvm_vcpu *vcpu)
+{
+       __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.root_mmu);
+       __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu);
+}
+
 static bool need_remote_flush(u64 old, u64 new)
 {
        if (!is_shadow_present_pte(old))
@@ -5656,7 +5707,7 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm)
         * Note: we need to do this under the protection of mmu_lock,
         * otherwise, vcpu would purge shadow page but miss tlb flush.
         */
-       kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
+       kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
 
        kvm_zap_obsolete_pages(kvm);
 

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 18977693517a42015cafdffe9bd0eb0cc516a117..cf17af4d6904583a653a544eb6d89628319b2323 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -9866,8 +9866,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
                                goto out;
                        }
                }
-               if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu))
-                       kvm_mmu_unload(vcpu);
+               if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
+                       kvm_mmu_free_obsolete_roots(vcpu);
                if (kvm_check_request(KVM_REQ_MIGRATE_TIMER, vcpu))
                        __kvm_migrate_timers(vcpu);
                if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu))