x86/sev: Evict cache lines during SNP memory validation

An SNP cache coherency vulnerability requires a cache line eviction
mitigation when validating memory after a page state change to private.
The specific mitigation is to touch the first and last byte of each 4K
page that is being validated. There is no need to perform the mitigation
when performing a page state change to shared and rescinding validation.

CPUID bit Fn8000001F_EBX[31] defines the COHERENCY_SFW_NO CPUID bit
that, when set, indicates that the software mitigation for this
vulnerability is not needed.

Implement the mitigation and invoke it when validating memory (making it
private) and the COHERENCY_SFW_NO bit is not set, indicating the SNP
guest is vulnerable.

Co-developed-by: Michael Roth <michael.roth@amd.com>
Signed-off-by: Michael Roth <michael.roth@amd.com>
Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Acked-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/arch/x86/boot/cpuflags.c b/arch/x86/boot/cpuflags.c
index 916bac09b464da7132cd2cb81f713a2aca63ab1e..63e037e94e4c03cefb1628927009cc5c1fabec9d 100644 (file)
--- a/arch/x86/boot/cpuflags.c
+++ b/arch/x86/boot/cpuflags.c
@@ -106,5 +106,18 @@ void get_cpuflags(void)
                        cpuid(0x80000001, &ignored, &ignored, &cpu.flags[6],
                              &cpu.flags[1]);
                }
+
+               if (max_amd_level >= 0x8000001f) {
+                       u32 ebx;
+
+                       /*
+                        * The X86_FEATURE_COHERENCY_SFW_NO feature bit is in
+                        * the virtualization flags entry (word 8) and set by
+                        * scattered.c, so the bit needs to be explicitly set.
+                        */
+                       cpuid(0x8000001f, &ignored, &ebx, &ignored, &ignored);
+                       if (ebx & BIT(31))
+                               set_bit(X86_FEATURE_COHERENCY_SFW_NO, cpu.flags);
+               }
        }
 }

diff --git a/arch/x86/boot/startup/sev-shared.c b/arch/x86/boot/startup/sev-shared.c
index 7a706db87b932ffdc13b809fbc427216f0bfb295..ac7dfd21ddd4d5c307a7d91d5a39aa748dba2c22 100644 (file)
--- a/arch/x86/boot/startup/sev-shared.c
+++ b/arch/x86/boot/startup/sev-shared.c
@@ -810,6 +810,13 @@ static void __head pvalidate_4k_page(unsigned long vaddr, unsigned long paddr,
                if (ret)
                        sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PVALIDATE);
        }
+
+       /*
+        * If validating memory (making it private) and affected by the
+        * cache-coherency vulnerability, perform the cache eviction mitigation.
+        */
+       if (validate && !has_cpuflag(X86_FEATURE_COHERENCY_SFW_NO))
+               sev_evict_cache((void *)vaddr, 1);
 }
 
 /*

diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index fc59ce78c477d11ec7f3edf450816b89e57a5fc5..400a6ab75d45a466e9b5a8cd20044858094b312a 100644 (file)
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -358,10 +358,31 @@ static void svsm_pval_pages(struct snp_psc_desc *desc)
 
 static void pvalidate_pages(struct snp_psc_desc *desc)
 {
+       struct psc_entry *e;
+       unsigned int i;
+
        if (snp_vmpl)
                svsm_pval_pages(desc);
        else
                pval_pages(desc);
+
+       /*
+        * If not affected by the cache-coherency vulnerability there is no need
+        * to perform the cache eviction mitigation.
+        */
+       if (cpu_feature_enabled(X86_FEATURE_COHERENCY_SFW_NO))
+               return;
+
+       for (i = 0; i <= desc->hdr.end_entry; i++) {
+               e = &desc->entries[i];
+
+               /*
+                * If validating memory (making it private) perform the cache
+                * eviction mitigation.
+                */
+               if (e->operation == SNP_PAGE_STATE_PRIVATE)
+                       sev_evict_cache(pfn_to_kaddr(e->gfn), e->pagesize ? 512 : 1);
+       }
 }
 
 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 602957dd2609ce4f533bf78e431662138b393aec..06fc0479a23f01e5a65526fc185713294013f793 100644 (file)
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -218,6 +218,7 @@
 #define X86_FEATURE_FLEXPRIORITY       ( 8*32+ 1) /* "flexpriority" Intel FlexPriority */
 #define X86_FEATURE_EPT                        ( 8*32+ 2) /* "ept" Intel Extended Page Table */
 #define X86_FEATURE_VPID               ( 8*32+ 3) /* "vpid" Intel Virtual Processor ID */
+#define X86_FEATURE_COHERENCY_SFW_NO   ( 8*32+ 4) /* SNP cache coherency software work around not needed */
 
 #define X86_FEATURE_VMMCALL            ( 8*32+15) /* "vmmcall" Prefer VMMCALL to VMCALL */
 #define X86_FEATURE_XENPV              ( 8*32+16) /* Xen paravirtual guest */

diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index 89075ff19afa32acb4713e09fd50c170876b3b69..02236962fdb108009e419e010d3862688e1f694d 100644 (file)
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -619,6 +619,24 @@ int rmp_make_shared(u64 pfn, enum pg_level level);
 void snp_leak_pages(u64 pfn, unsigned int npages);
 void kdump_sev_callback(void);
 void snp_fixup_e820_tables(void);
+
+static inline void sev_evict_cache(void *va, int npages)
+{
+       volatile u8 val __always_unused;
+       u8 *bytes = va;
+       int page_idx;
+
+       /*
+        * For SEV guests, a read from the first/last cache-lines of a 4K page
+        * using the guest key is sufficient to cause a flush of all cache-lines
+        * associated with that 4K page without incurring all the overhead of a
+        * full CLFLUSH sequence.
+        */
+       for (page_idx = 0; page_idx < npages; page_idx++) {
+               val = bytes[page_idx * PAGE_SIZE];
+               val = bytes[page_idx * PAGE_SIZE + PAGE_SIZE - 1];
+       }
+}
 #else
 static inline bool snp_probe_rmptable_info(void) { return false; }
 static inline int snp_rmptable_init(void) { return -ENOSYS; }
@@ -634,6 +652,7 @@ static inline int rmp_make_shared(u64 pfn, enum pg_level level) { return -ENODEV
 static inline void snp_leak_pages(u64 pfn, unsigned int npages) {}
 static inline void kdump_sev_callback(void) { }
 static inline void snp_fixup_e820_tables(void) {}
+static inline void sev_evict_cache(void *va, int npages) {}
 #endif
 
 #endif

diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index b4a1f6732a3aad4d2dc760bb35f30cde27c9a759..6b868afb26c3198f0591690906141e9d46caf8b7 100644 (file)
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -48,6 +48,7 @@ static const struct cpuid_bit cpuid_bits[] = {
        { X86_FEATURE_PROC_FEEDBACK,            CPUID_EDX, 11, 0x80000007, 0 },
        { X86_FEATURE_AMD_FAST_CPPC,            CPUID_EDX, 15, 0x80000007, 0 },
        { X86_FEATURE_MBA,                      CPUID_EBX,  6, 0x80000008, 0 },
+       { X86_FEATURE_COHERENCY_SFW_NO,         CPUID_EBX, 31, 0x8000001f, 0 },
        { X86_FEATURE_SMBA,                     CPUID_EBX,  2, 0x80000020, 0 },
        { X86_FEATURE_BMEC,                     CPUID_EBX,  3, 0x80000020, 0 },
        { X86_FEATURE_TSA_SQ_NO,                CPUID_ECX,  1, 0x80000021, 0 },