Merge tag 'v6.8-rc4' into x86/percpu, to resolve conflicts and refresh the branch

[linux-block.git] / arch / x86 / kernel / head_64.S

diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index bfe5ec2f4f83fe17de6334d2ee80b8a0d9743200..bb8ee1ce696836667caa5700b4bcce6cb2ab5488 100644 (file)
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -9,7 +9,7 @@
  *  Copyright (C) 2005 Eric Biederman <ebiederm@xmission.com>
  */
 
-
+#include <linux/export.h>
 #include <linux/linkage.h>
 #include <linux/threads.h>
 #include <linux/init.h>
@@ -22,7 +22,6 @@
 #include <asm/percpu.h>
 #include <asm/nops.h>
 #include "../entry/calling.h"
-#include <asm/export.h>
 #include <asm/nospec-branch.h>
 #include <asm/apicdef.h>
 #include <asm/fixmap.h>
@@ -115,6 +114,28 @@ SYM_CODE_START_NOALIGN(startup_64)
 
        /* Form the CR3 value being sure to include the CR3 modifier */
        addq    $(early_top_pgt - __START_KERNEL_map), %rax
+
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+       mov     %rax, %rdi
+       mov     %rax, %r14
+
+       addq    phys_base(%rip), %rdi
+
+       /*
+        * For SEV guests: Verify that the C-bit is correct. A malicious
+        * hypervisor could lie about the C-bit position to perform a ROP
+        * attack on the guest by writing to the unencrypted stack and wait for
+        * the next RET instruction.
+        */
+       call    sev_verify_cbit
+
+       /*
+        * Restore CR3 value without the phys_base which will be added
+        * below, before writing %cr3.
+        */
+        mov    %r14, %rax
+#endif
+
        jmp 1f
 SYM_CODE_END(startup_64)
 
@@ -180,10 +201,10 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
        movl    $0, %ecx
 #endif
 
-       /* Enable PAE mode, PGE and LA57 */
-       orl     $(X86_CR4_PAE | X86_CR4_PGE), %ecx
+       /* Enable PAE mode, PSE, PGE and LA57 */
+       orl     $(X86_CR4_PAE | X86_CR4_PSE | X86_CR4_PGE), %ecx
 #ifdef CONFIG_X86_5LEVEL
-       testl   $1, __pgtable_l5_enabled(%rip)
+       testb   $1, __pgtable_l5_enabled(%rip)
        jz      1f
        orl     $X86_CR4_LA57, %ecx
 1:
@@ -193,22 +214,13 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
        /* Setup early boot stage 4-/5-level pagetables. */
        addq    phys_base(%rip), %rax
 
-       /*
-        * For SEV guests: Verify that the C-bit is correct. A malicious
-        * hypervisor could lie about the C-bit position to perform a ROP
-        * attack on the guest by writing to the unencrypted stack and wait for
-        * the next RET instruction.
-        */
-       movq    %rax, %rdi
-       call    sev_verify_cbit
-
        /*
         * Switch to new page-table
         *
         * For the boot CPU this switches to early_top_pgt which still has the
-        * indentity mappings present. The secondary CPUs will switch to the
+        * identity mappings present. The secondary CPUs will switch to the
         * init_top_pgt here, away from the trampoline_pgd and unmap the
-        * indentity mapped ranges.
+        * identity mapped ranges.
         */
        movq    %rax, %cr3
 
@@ -256,6 +268,22 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
        testl   $X2APIC_ENABLE, %eax
        jnz     .Lread_apicid_msr
 
+#ifdef CONFIG_X86_X2APIC
+       /*
+        * If system is in X2APIC mode then MMIO base might not be
+        * mapped causing the MMIO read below to fault. Faults can't
+        * be handled at that point.
+        */
+       cmpl    $0, x2apic_mode(%rip)
+       jz      .Lread_apicid_mmio
+
+       /* Force the AP into X2APIC mode. */
+       orl     $X2APIC_ENABLE, %eax
+       wrmsr
+       jmp     .Lread_apicid_msr
+#endif
+
+.Lread_apicid_mmio:
        /* Read the APIC ID from the fix-mapped MMIO space. */
        movq    apic_mmio_base(%rip), %rcx
        addq    $APIC_ID, %rcx