powerpc/64s/radix: keep kernel ERAT over local process/guest invalidates

ISA v3.0 radix modes provide SLBIA variants which can invalidate ERAT
for effPID!=0 or for effLPID!=0, which allows user and guest
invalidations to retain kernel/host ERAT entries.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
index 14f43b6cf7ec642840171a73673291b560125869..971bdf84f6fcb3731ad88729c4ef091070b81b42 100644 (file)
--- a/arch/powerpc/include/asm/ppc-opcode.h
+++ b/arch/powerpc/include/asm/ppc-opcode.h
@@ -591,7 +591,16 @@
 
 #define PPC_SLBIA(IH)  stringify_in_c(.long PPC_INST_SLBIA | \
                                       ((IH & 0x7) << 21))
+
+/*
+ * These may only be used on ISA v3.0 or later (aka. CPU_FTR_ARCH_300, radix
+ * implies CPU_FTR_ARCH_300). USER/GUEST invalidates may only be used by radix
+ * mode (on HPT these would also invalidate various SLBEs which may not be
+ * desired).
+ */
 #define PPC_ISA_3_0_INVALIDATE_ERAT    PPC_SLBIA(7)
+#define PPC_RADIX_INVALIDATE_ERAT_USER PPC_SLBIA(3)
+#define PPC_RADIX_INVALIDATE_ERAT_GUEST        PPC_SLBIA(6)
 
 #define VCMPEQUD_RC(vrt, vra, vrb)     stringify_in_c(.long PPC_INST_VCMPEQUD | \
                              ___PPC_RT(vrt) | ___PPC_RA(vra) | \

diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c
index ca4f006abd771cd7cd535179ba584ea0fa32d117..8fa93114d6301e03081a2434c6cca8095118b3f7 100644 (file)
--- a/arch/powerpc/kvm/book3s_hv_builtin.c
+++ b/arch/powerpc/kvm/book3s_hv_builtin.c
@@ -823,6 +823,8 @@ static void flush_guest_tlb(struct kvm *kvm)
                                     : : "r" (rb), "i" (1), "i" (1), "i" (0),
                                       "r" (0) : "memory");
                }
+               asm volatile("ptesync": : :"memory");
+               asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory");
        } else {
                for (set = 0; set < kvm->arch.tlb_sets; ++set) {
                        /* R=0 PRS=0 RIC=0 */
@@ -831,9 +833,9 @@ static void flush_guest_tlb(struct kvm *kvm)
                                       "r" (0) : "memory");
                        rb += PPC_BIT(51);      /* increment set number */
                }
+               asm volatile("ptesync": : :"memory");
+               asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
        }
-       asm volatile("ptesync": : :"memory");
-       asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
 }
 
 void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu,

diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
index 7539c714438e579db5f46e4444e1e28448f753f7..edf39e9a4ec4181825f6938c3ca2c726c33ff7e8 100644 (file)
--- a/arch/powerpc/mm/book3s64/radix_tlb.c
+++ b/arch/powerpc/mm/book3s64/radix_tlb.c
@@ -262,7 +262,7 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
                __tlbiel_pid(pid, set, RIC_FLUSH_TLB);
 
        asm volatile("ptesync": : :"memory");
-       asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory");
+       asm volatile(PPC_RADIX_INVALIDATE_ERAT_USER "; isync" : : :"memory");
 }
 
 static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
@@ -314,7 +314,7 @@ static inline void _tlbiel_lpid(unsigned long lpid, unsigned long ric)
                __tlbiel_lpid(lpid, set, RIC_FLUSH_TLB);
 
        asm volatile("ptesync": : :"memory");
-       asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory");
+       asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST "; isync" : : :"memory");
 }
 
 static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric)
@@ -366,7 +366,7 @@ static inline void _tlbiel_lpid_guest(unsigned long lpid, unsigned long ric)
                __tlbiel_lpid_guest(lpid, set, RIC_FLUSH_TLB);
 
        asm volatile("ptesync": : :"memory");
-       asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory");
+       asm volatile(PPC_RADIX_INVALIDATE_ERAT_GUEST : : :"memory");
 }