KVM: x86/mmu: Skip invalid TDP MMU roots when write-protecting SPTEs

When write-protecting SPTEs, don't process invalid roots as invalid roots
are unreachable, i.e. can't be used to access guest memory and thus don't
need to be write-protected.

Note, this is *almost* a nop for kvm_tdp_mmu_clear_dirty_pt_masked(),
which is called under slots_lock, i.e. is mutually exclusive with
kvm_mmu_zap_all_fast().  But it's possible for something other than the
"fast zap" thread to grab a reference to an invalid root and thus keep a
root alive (but completely empty) after kvm_mmu_zap_all_fast() completes.

The kvm_tdp_mmu_write_protect_gfn() case is more interesting as KVM write-
protects SPTEs for reasons other than dirty logging, e.g. if a KVM creates
a SPTE for a nested VM while a fast zap is in-progress.

Add another TDP MMU iterator to visit only valid roots, and
opportunistically convert kvm_tdp_mmu_get_vcpu_root_hpa() to said iterator.

Link: https://lore.kernel.org/r/20240111020048.844847-6-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 1a9c16e5c287a0c32dec8e46eb2eabcb86155549..e0a8343f66dc6f347dcd7759320b4ea8731d41e5 100644 (file)
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -171,12 +171,19 @@ static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm,
  * Holding mmu_lock for write obviates the need for RCU protection as the list
  * is guaranteed to be stable.
  */
-#define for_each_tdp_mmu_root(_kvm, _root, _as_id)                             \
+#define __for_each_tdp_mmu_root(_kvm, _root, _as_id, _only_valid)              \
        list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link)             \
                if (kvm_lockdep_assert_mmu_lock_held(_kvm, false) &&            \
-                   _as_id >= 0 && kvm_mmu_page_as_id(_root) != _as_id) {       \
+                   ((_as_id >= 0 && kvm_mmu_page_as_id(_root) != _as_id) ||    \
+                    ((_only_valid) && (_root)->role.invalid))) {               \
                } else
 
+#define for_each_tdp_mmu_root(_kvm, _root, _as_id)                     \
+       __for_each_tdp_mmu_root(_kvm, _root, _as_id, false)
+
+#define for_each_valid_tdp_mmu_root(_kvm, _root, _as_id)               \
+       __for_each_tdp_mmu_root(_kvm, _root, _as_id, true)
+
 static struct kvm_mmu_page *tdp_mmu_alloc_sp(struct kvm_vcpu *vcpu)
 {
        struct kvm_mmu_page *sp;
@@ -224,11 +231,8 @@ hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu)
 
        lockdep_assert_held_write(&kvm->mmu_lock);
 
-       /*
-        * Check for an existing root before allocating a new one.  Note, the
-        * role check prevents consuming an invalid root.
-        */
-       for_each_tdp_mmu_root(kvm, root, kvm_mmu_role_as_id(role)) {
+       /* Check for an existing root before allocating a new one. */
+       for_each_valid_tdp_mmu_root(kvm, root, kvm_mmu_role_as_id(role)) {
                if (root->role.word == role.word &&
                    kvm_tdp_mmu_get_root(root))
                        goto out;
@@ -1639,7 +1643,7 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm,
 {
        struct kvm_mmu_page *root;
 
-       for_each_tdp_mmu_root(kvm, root, slot->as_id)
+       for_each_valid_tdp_mmu_root(kvm, root, slot->as_id)
                clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot);
 }
 
@@ -1757,7 +1761,7 @@ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
        bool spte_set = false;
 
        lockdep_assert_held_write(&kvm->mmu_lock);
-       for_each_tdp_mmu_root(kvm, root, slot->as_id)
+       for_each_valid_tdp_mmu_root(kvm, root, slot->as_id)
                spte_set |= write_protect_gfn(kvm, root, gfn, min_level);
 
        return spte_set;