Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm

Pull more KVM updates from Paolo Bonzini:
 "x86:

   - take into account HVA before retrying on MMU notifier race

   - fixes for nested AMD guests without NPT

   - allow INVPCID in guest without PCID

   - disable PML in hardware when not in use

   - MMU code cleanups:

* tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (28 commits)
  KVM: SVM: Fix nested VM-Exit on #GP interception handling
  KVM: vmx/pmu: Fix dummy check if lbr_desc->event is created
  KVM: x86/mmu: Consider the hva in mmu_notifier retry
  KVM: x86/mmu: Skip mmu_notifier check when handling MMIO page fault
  KVM: Documentation: rectify rst markup in KVM_GET_SUPPORTED_HV_CPUID
  KVM: nSVM: prepare guest save area while is_guest_mode is true
  KVM: x86/mmu: Remove a variety of unnecessary exports
  KVM: x86: Fold "write-protect large" use case into generic write-protect
  KVM: x86/mmu: Don't set dirty bits when disabling dirty logging w/ PML
  KVM: VMX: Dynamically enable/disable PML based on memslot dirty logging
  KVM: x86: Further clarify the logic and comments for toggling log dirty
  KVM: x86: Move MMU's PML logic to common code
  KVM: x86/mmu: Make dirty log size hook (PML) a value, not a function
  KVM: x86/mmu: Expand on the comment in kvm_vcpu_ad_need_write_protect()
  KVM: nVMX: Disable PML in hardware when running L2
  KVM: x86/mmu: Consult max mapping level when zapping collapsible SPTEs
  KVM: x86/mmu: Pass the memslot to the rmap callbacks
  KVM: x86/mmu: Split out max mapping level calculation to helper
  KVM: x86/mmu: Expand collapsible SPTE zap for TDP MMU to ZONE_DEVICE and HugeTLB pages
  KVM: nVMX: no need to undo inject_page_fault change on nested vmexit
  ...

diff --combined arch/powerpc/kvm/book3s_64_mmu_hv.c
index c77f2d4f44ca6d68abdd0c546f6b1dc89293b988,8e06cd3f759cb8c4570ea1864e62a0e8697dd29d..bb6773594cf8280fc275a6269d58f4da0070e689
--- 1/arch/powerpc/kvm/book3s_64_mmu_hv.c
--- 2/arch/powerpc/kvm/book3s_64_mmu_hv.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c
@@@ -27,7 -27,6 +27,7 @@@
  #include <asm/cputable.h>
  #include <asm/pte-walk.h>
  
 +#include "book3s.h"
  #include "trace_hv.h"
  
  //#define DEBUG_RESIZE_HPT    1
@@@ -591,7 -590,7 +591,7 @@@ int kvmppc_book3s_hv_page_fault(struct 
        } else {
                /* Call KVM generic code to do the slow-path check */
                pfn = __gfn_to_pfn_memslot(memslot, gfn, false, NULL,
-                                          writing, &write_ok);
+                                          writing, &write_ok, NULL);
                if (is_error_noslot_pfn(pfn))
                        return -EFAULT;
                page = NULL;

diff --combined arch/x86/kvm/x86.c
index 884e5b3838c735db2e772c80638667fcdb390866,1d2bc89431a222f48246e0a2b388aaf428d14cfb..3712bb5245eb9a206b872bb69e476d172ef343ed
--- 1/arch/x86/kvm/x86.c
--- 2/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@@ -1785,7 -1785,6 +1785,7 @@@ EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr)
  
  static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu)
  {
 +      xfer_to_guest_mode_prepare();
        return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) ||
                xfer_to_guest_mode_work_pending();
  }
@@@ -5215,10 -5214,18 +5215,18 @@@ static int kvm_vm_ioctl_reinject(struc
  
  void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
  {
+ 
        /*
-        * Flush potentially hardware-cached dirty pages to dirty_bitmap.
+        * Flush all CPUs' dirty log buffers to the  dirty_bitmap.  Called
+        * before reporting dirty_bitmap to userspace.  KVM flushes the buffers
+        * on all VM-Exits, thus we only need to kick running vCPUs to force a
+        * VM-Exit.
         */
-       static_call_cond(kvm_x86_flush_log_dirty)(kvm);
+       struct kvm_vcpu *vcpu;
+       int i;
+ 
+       kvm_for_each_vcpu(i, vcpu, kvm)
+               kvm_vcpu_kick(vcpu);
  }
  
  int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
@@@ -8980,6 -8987,9 +8988,9 @@@ static int vcpu_enter_guest(struct kvm_
                        kvm_check_async_pf_completion(vcpu);
                if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu))
                        static_call(kvm_x86_msr_filter_changed)(vcpu);
+ 
+               if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu))
+                       static_call(kvm_x86_update_cpu_dirty_logging)(vcpu);
        }
  
        if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
@@@ -10748,75 -10758,96 +10759,96 @@@ int kvm_arch_prepare_memory_region(stru
        return 0;
  }
  
+ 
+ static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable)
+ {
+       struct kvm_arch *ka = &kvm->arch;
+ 
+       if (!kvm_x86_ops.cpu_dirty_log_size)
+               return;
+ 
+       if ((enable && ++ka->cpu_dirty_logging_count == 1) ||
+           (!enable && --ka->cpu_dirty_logging_count == 0))
+               kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING);
+ 
+       WARN_ON_ONCE(ka->cpu_dirty_logging_count < 0);
+ }
+ 
  static void kvm_mmu_slot_apply_flags(struct kvm *kvm,
                                     struct kvm_memory_slot *old,
                                     struct kvm_memory_slot *new,
                                     enum kvm_mr_change change)
  {
+       bool log_dirty_pages = new->flags & KVM_MEM_LOG_DIRTY_PAGES;
+ 
        /*
-        * Nothing to do for RO slots or CREATE/MOVE/DELETE of a slot.
-        * See comments below.
+        * Update CPU dirty logging if dirty logging is being toggled.  This
+        * applies to all operations.
         */
-       if ((change != KVM_MR_FLAGS_ONLY) || (new->flags & KVM_MEM_READONLY))
-               return;
+       if ((old->flags ^ new->flags) & KVM_MEM_LOG_DIRTY_PAGES)
+               kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages);
  
        /*
-        * Dirty logging tracks sptes in 4k granularity, meaning that large
-        * sptes have to be split.  If live migration is successful, the guest
-        * in the source machine will be destroyed and large sptes will be
-        * created in the destination. However, if the guest continues to run
-        * in the source machine (for example if live migration fails), small
-        * sptes will remain around and cause bad performance.
-        *
-        * Scan sptes if dirty logging has been stopped, dropping those
-        * which can be collapsed into a single large-page spte.  Later
-        * page faults will create the large-page sptes.
+        * Nothing more to do for RO slots (which can't be dirtied and can't be
+        * made writable) or CREATE/MOVE/DELETE of a slot.
         *
-        * There is no need to do this in any of the following cases:
+        * For a memslot with dirty logging disabled:
         * CREATE:      No dirty mappings will already exist.
         * MOVE/DELETE: The old mappings will already have been cleaned up by
         *              kvm_arch_flush_shadow_memslot()
+        *
+        * For a memslot with dirty logging enabled:
+        * CREATE:      No shadow pages exist, thus nothing to write-protect
+        *              and no dirty bits to clear.
+        * MOVE/DELETE: The old mappings will already have been cleaned up by
+        *              kvm_arch_flush_shadow_memslot().
         */
-       if ((old->flags & KVM_MEM_LOG_DIRTY_PAGES) &&
-           !(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
-               kvm_mmu_zap_collapsible_sptes(kvm, new);
+       if ((change != KVM_MR_FLAGS_ONLY) || (new->flags & KVM_MEM_READONLY))
+               return;
  
        /*
-        * Enable or disable dirty logging for the slot.
-        *
-        * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of the old
-        * slot have been zapped so no dirty logging updates are needed for
-        * the old slot.
-        * For KVM_MR_CREATE and KVM_MR_MOVE, once the new slot is visible
-        * any mappings that might be created in it will consume the
-        * properties of the new slot and do not need to be updated here.
-        *
-        * When PML is enabled, the kvm_x86_ops dirty logging hooks are
-        * called to enable/disable dirty logging.
-        *
-        * When disabling dirty logging with PML enabled, the D-bit is set
-        * for sptes in the slot in order to prevent unnecessary GPA
-        * logging in the PML buffer (and potential PML buffer full VMEXIT).
-        * This guarantees leaving PML enabled for the guest's lifetime
-        * won't have any additional overhead from PML when the guest is
-        * running with dirty logging disabled.
-        *
-        * When enabling dirty logging, large sptes are write-protected
-        * so they can be split on first write.  New large sptes cannot
-        * be created for this slot until the end of the logging.
-        * See the comments in fast_page_fault().
-        * For small sptes, nothing is done if the dirty log is in the
-        * initial-all-set state.  Otherwise, depending on whether pml
-        * is enabled the D-bit or the W-bit will be cleared.
+        * READONLY and non-flags changes were filtered out above, and the only
+        * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty
+        * logging isn't being toggled on or off.
         */
-       if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
-               if (kvm_x86_ops.slot_enable_log_dirty) {
-                       static_call(kvm_x86_slot_enable_log_dirty)(kvm, new);
-               } else {
-                       int level =
-                               kvm_dirty_log_manual_protect_and_init_set(kvm) ?
-                               PG_LEVEL_2M : PG_LEVEL_4K;
+       if (WARN_ON_ONCE(!((old->flags ^ new->flags) & KVM_MEM_LOG_DIRTY_PAGES)))
+               return;
+ 
+       if (!log_dirty_pages) {
+               /*
+                * Dirty logging tracks sptes in 4k granularity, meaning that
+                * large sptes have to be split.  If live migration succeeds,
+                * the guest in the source machine will be destroyed and large
+                * sptes will be created in the destination.  However, if the
+                * guest continues to run in the source machine (for example if
+                * live migration fails), small sptes will remain around and
+                * cause bad performance.
+                *
+                * Scan sptes if dirty logging has been stopped, dropping those
+                * which can be collapsed into a single large-page spte.  Later
+                * page faults will create the large-page sptes.
+                */
+               kvm_mmu_zap_collapsible_sptes(kvm, new);
+       } else {
+               /* By default, write-protect everything to log writes. */
+               int level = PG_LEVEL_4K;
+ 
+               if (kvm_x86_ops.cpu_dirty_log_size) {
+                       /*
+                        * Clear all dirty bits, unless pages are treated as
+                        * dirty from the get-go.
+                        */
+                       if (!kvm_dirty_log_manual_protect_and_init_set(kvm))
+                               kvm_mmu_slot_leaf_clear_dirty(kvm, new);
  
+                       /*
+                        * Write-protect large pages on write so that dirty
+                        * logging happens at 4k granularity.  No need to
+                        * write-protect small SPTEs since write accesses are
+                        * logged by the CPU via dirty bits.
+                        */
+                       level = PG_LEVEL_2M;
+               } else if (kvm_dirty_log_manual_protect_and_init_set(kvm)) {
                        /*
                         * If we're with initial-all-set, we don't need
                         * to write protect any small page because
@@@ -10825,10 -10856,9 +10857,9 @@@
                         * so that the page split can happen lazily on
                         * the first write to the huge page.
                         */
-                       kvm_mmu_slot_remove_write_access(kvm, new, level);
+                       level = PG_LEVEL_2M;
                }
-       } else {
-               static_call_cond(kvm_x86_slot_disable_log_dirty)(kvm, new);
+               kvm_mmu_slot_remove_write_access(kvm, new, level);
        }
  }