x86/pv: Switch SWAPGS to ALTERNATIVE

[ Upstream commit 53c9d9240944088274aadbbbafc6138ca462db4f ]

SWAPGS is used only for interrupts coming from user mode or for
returning to user mode. So there is no reason to use the PARAVIRT
framework, as it can easily be replaced by an ALTERNATIVE depending
on X86_FEATURE_XENPV.

There are several instances using the PV-aware SWAPGS macro in paths
which are never executed in a Xen PV guest. Replace those with the
plain swapgs instruction. For SWAPGS_UNSAFE_STACK the same applies.

Signed-off-by: Juergen Gross <jgross@suse.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Andy Lutomirski <luto@kernel.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20210120135555.32594-5-jgross@suse.com
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index de541ea2788ebfde1cf605880ace7f572fa8771a..166554a109aeb3f883fd6d1361e14927f2cd9943 100644 (file)
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -673,7 +673,7 @@ native_irq_return_ldt:
         */
 
        pushq   %rdi                            /* Stash user RDI */
-       SWAPGS                                  /* to kernel GS */
+       swapgs                                  /* to kernel GS */
        SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi   /* to kernel CR3 */
 
        movq    PER_CPU_VAR(espfix_waddr), %rdi
@@ -703,7 +703,7 @@ native_irq_return_ldt:
        orq     PER_CPU_VAR(espfix_stack), %rax
 
        SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
-       SWAPGS                                  /* to user GS */
+       swapgs                                  /* to user GS */
        popq    %rdi                            /* Restore user RDI */
 
        movq    %rax, %rsp
@@ -947,7 +947,7 @@ SYM_CODE_START_LOCAL(paranoid_entry)
        ret
 
 .Lparanoid_entry_swapgs:
-       SWAPGS
+       swapgs
 
        /*
         * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
@@ -1005,7 +1005,7 @@ SYM_CODE_START_LOCAL(paranoid_exit)
        jnz             restore_regs_and_return_to_kernel
 
        /* We are returning to a context with user GSBASE */
-       SWAPGS_UNSAFE_STACK
+       swapgs
        jmp             restore_regs_and_return_to_kernel
 SYM_CODE_END(paranoid_exit)
 
@@ -1431,7 +1431,7 @@ nmi_no_fsgsbase:
        jnz     nmi_restore
 
 nmi_swapgs:
-       SWAPGS_UNSAFE_STACK
+       swapgs
 
 nmi_restore:
        POP_REGS

diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
index 2dfc8d380dab1143c29c215390ba37b1335e4bd7..8c86edefa11508ab3401ead3bcfd757b3834e015 100644 (file)
--- a/arch/x86/include/asm/irqflags.h
+++ b/arch/x86/include/asm/irqflags.h
@@ -131,18 +131,6 @@ static __always_inline unsigned long arch_local_irq_save(void)
 #define SAVE_FLAGS(x)          pushfq; popq %rax
 #endif
 
-#define SWAPGS swapgs
-/*
- * Currently paravirt can't handle swapgs nicely when we
- * don't have a stack we can rely on (such as a user space
- * stack).  So we either find a way around these or just fault
- * and emulate if a guest tries to call swapgs directly.
- *
- * Either way, this is a good way to document that we don't
- * have a reliable stack. x86_64 only.
- */
-#define SWAPGS_UNSAFE_STACK    swapgs
-
 #define INTERRUPT_RETURN       jmp native_iret
 #define USERGS_SYSRET64                                \
        swapgs;                                 \
@@ -170,6 +158,14 @@ static __always_inline int arch_irqs_disabled(void)
 
        return arch_irqs_disabled_flags(flags);
 }
+#else
+#ifdef CONFIG_X86_64
+#ifdef CONFIG_XEN_PV
+#define SWAPGS ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV
+#else
+#define SWAPGS swapgs
+#endif
+#endif
 #endif /* !__ASSEMBLY__ */
 
 #endif

diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index d25cc6830e895b3142f69684e5758fce2ef84a90..5647bcdba776e49f6c3744255a06d8c22c194a22 100644 (file)
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -776,26 +776,6 @@ extern void default_banner(void);
 
 #ifdef CONFIG_X86_64
 #ifdef CONFIG_PARAVIRT_XXL
-/*
- * If swapgs is used while the userspace stack is still current,
- * there's no way to call a pvop.  The PV replacement *must* be
- * inlined, or the swapgs instruction must be trapped and emulated.
- */
-#define SWAPGS_UNSAFE_STACK                                            \
-       PARA_SITE(PARA_PATCH(PV_CPU_swapgs), swapgs)
-
-/*
- * Note: swapgs is very special, and in practise is either going to be
- * implemented with a single "swapgs" instruction or something very
- * special.  Either way, we don't need to save any registers for
- * it.
- */
-#define SWAPGS                                                         \
-       PARA_SITE(PARA_PATCH(PV_CPU_swapgs),                            \
-                 ANNOTATE_RETPOLINE_SAFE;                              \
-                 call PARA_INDIRECT(pv_ops+PV_CPU_swapgs);             \
-                )
-
 #define USERGS_SYSRET64                                                        \
        PARA_SITE(PARA_PATCH(PV_CPU_usergs_sysret64),                   \
                  ANNOTATE_RETPOLINE_SAFE;                              \

diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 0fad9f61c76ab8dfb78a022cc119867df40eb254..903d71884fa25dca7bf3fe4ef70d3be278c34abb 100644 (file)
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -169,8 +169,6 @@ struct pv_cpu_ops {
           frame set up. */
        void (*iret)(void);
 
-       void (*swapgs)(void);
-
        void (*start_context_switch)(struct task_struct *prev);
        void (*end_context_switch)(struct task_struct *next);
 #endif

diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index 828be792231e9d935796b4178289303ce80f9e68..1354bc30614d7e9411167093c6df09d784cdd090 100644 (file)
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -15,7 +15,6 @@ int main(void)
 #ifdef CONFIG_PARAVIRT_XXL
        OFFSET(PV_CPU_usergs_sysret64, paravirt_patch_template,
               cpu.usergs_sysret64);
-       OFFSET(PV_CPU_swapgs, paravirt_patch_template, cpu.swapgs);
 #ifdef CONFIG_DEBUG_ENTRY
        OFFSET(PV_IRQ_save_fl, paravirt_patch_template, irq.save_fl);
 #endif

diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 6c3407ba6ee9851e5f0913a10d5a53926f3afdcb..5e5fcf5c376de76bd29a45bd5c452909351bfde2 100644 (file)
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -312,7 +312,6 @@ struct paravirt_patch_template pv_ops = {
 
        .cpu.usergs_sysret64    = native_usergs_sysret64,
        .cpu.iret               = native_iret,
-       .cpu.swapgs             = native_swapgs,
 
 #ifdef CONFIG_X86_IOPL_IOPERM
        .cpu.invalidate_io_bitmap       = native_tss_invalidate_io_bitmap,

diff --git a/arch/x86/kernel/paravirt_patch.c b/arch/x86/kernel/paravirt_patch.c
index ace6e334cb39384dc8f76384d0b10724fd30e924..7c518b08aa3c5c0fcd49fe89f1259e65c18cfd71 100644 (file)
--- a/arch/x86/kernel/paravirt_patch.c
+++ b/arch/x86/kernel/paravirt_patch.c
@@ -28,7 +28,6 @@ struct patch_xxl {
        const unsigned char     irq_restore_fl[2];
        const unsigned char     cpu_wbinvd[2];
        const unsigned char     cpu_usergs_sysret64[6];
-       const unsigned char     cpu_swapgs[3];
        const unsigned char     mov64[3];
 };
 
@@ -43,7 +42,6 @@ static const struct patch_xxl patch_data_xxl = {
        .cpu_wbinvd             = { 0x0f, 0x09 },       // wbinvd
        .cpu_usergs_sysret64    = { 0x0f, 0x01, 0xf8,
                                    0x48, 0x0f, 0x07 }, // swapgs; sysretq
-       .cpu_swapgs             = { 0x0f, 0x01, 0xf8 }, // swapgs
        .mov64                  = { 0x48, 0x89, 0xf8 }, // mov %rdi, %rax
 };
 
@@ -86,7 +84,6 @@ unsigned int native_patch(u8 type, void *insn_buff, unsigned long addr,
        PATCH_CASE(mmu, write_cr3, xxl, insn_buff, len);
 
        PATCH_CASE(cpu, usergs_sysret64, xxl, insn_buff, len);
-       PATCH_CASE(cpu, swapgs, xxl, insn_buff, len);
        PATCH_CASE(cpu, wbinvd, xxl, insn_buff, len);
 #endif
 

diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 5af0421ef74ba44df87f4845c288b7d5ba55064a..16ff25d6935e78d69a42c4a24174fd5702c280b9 100644 (file)
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -1083,9 +1083,6 @@ static const struct pv_cpu_ops xen_cpu_ops __initconst = {
 #endif
        .io_delay = xen_io_delay,
 
-       /* Xen takes care of %gs when switching to usermode for us */
-       .swapgs = paravirt_nop,
-
        .start_context_switch = paravirt_start_context_switch,
        .end_context_switch = xen_end_context_switch,
 };