Merge patch series "RISC-V: Align the shadow stack"

Palmer Dabbelt <palmer@rivosinc.com> says:

This contains a pair of cleanups that depend on a fix that has already
landed upstream.

* b4-shazam-merge:
  RISC-V: Add some comments about the shadow and overflow stacks
  RISC-V: Align the shadow stack
  riscv: fix race when vmap stack overflow

Link: https://lore.kernel.org/r/20221130023515.20217-1-palmer@rivosinc.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h
index 1b471ff731788608a1715f6f25796483a5077da0..816e753de636d48f8bb6f08c8d321824b144ecaa 100644 (file)
--- a/arch/riscv/include/asm/asm.h
+++ b/arch/riscv/include/asm/asm.h
@@ -23,6 +23,7 @@
 #define REG_L          __REG_SEL(ld, lw)
 #define REG_S          __REG_SEL(sd, sw)
 #define REG_SC         __REG_SEL(sc.d, sc.w)
+#define REG_AMOSWAP_AQ __REG_SEL(amoswap.d.aq, amoswap.w.aq)
 #define REG_ASM                __REG_SEL(.dword, .word)
 #define SZREG          __REG_SEL(8, 4)
 #define LGREG          __REG_SEL(3, 2)

diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index 32b2ddafedb0cb1083492bbd68ab3e3c2df0bfb7..99d38fdf8b18f2f18fe3bae2dce2d7a3e5b11b5a 100644 (file)
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -397,6 +397,19 @@ handle_syscall_trace_exit:
 
 #ifdef CONFIG_VMAP_STACK
 handle_kernel_stack_overflow:
+       /*
+        * Takes the psuedo-spinlock for the shadow stack, in case multiple
+        * harts are concurrently overflowing their kernel stacks.  We could
+        * store any value here, but since we're overflowing the kernel stack
+        * already we only have SP to use as a scratch register.  So we just
+        * swap in the address of the spinlock, as that's definately non-zero.
+        *
+        * Pairs with a store_release in handle_bad_stack().
+        */
+1:     la sp, spin_shadow_stack
+       REG_AMOSWAP_AQ sp, sp, (sp)
+       bnez sp, 1b
+
        la sp, shadow_stack
        addi sp, sp, SHADOW_OVERFLOW_STACK_SIZE
 

diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
index f3e96d60a2ff384a4ff7b342ba8daa45e22aaff1..549bde5c970a13e9412148c7c45aca1ef2034825 100644 (file)
--- a/arch/riscv/kernel/traps.c
+++ b/arch/riscv/kernel/traps.c
@@ -208,13 +208,28 @@ int is_valid_bugaddr(unsigned long pc)
 #endif /* CONFIG_GENERIC_BUG */
 
 #ifdef CONFIG_VMAP_STACK
+/*
+ * Extra stack space that allows us to provide panic messages when the kernel
+ * has overflowed its stack.
+ */
 static DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)],
                overflow_stack)__aligned(16);
 /*
- * shadow stack, handled_ kernel_ stack_ overflow(in kernel/entry.S) is used
- * to get per-cpu overflow stack(get_overflow_stack).
+ * A temporary stack for use by handle_kernel_stack_overflow.  This is used so
+ * we can call into C code to get the per-hart overflow stack.  Usage of this
+ * stack must be protected by spin_shadow_stack.
  */
-long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)];
+long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE/sizeof(long)] __aligned(16);
+
+/*
+ * A pseudo spinlock to protect the shadow stack from being used by multiple
+ * harts concurrently.  This isn't a real spinlock because the lock side must
+ * be taken without a valid stack and only a single register, it's only taken
+ * while in the process of panicing anyway so the performance and error
+ * checking a proper spinlock gives us doesn't matter.
+ */
+unsigned long spin_shadow_stack;
+
 asmlinkage unsigned long get_overflow_stack(void)
 {
        return (unsigned long)this_cpu_ptr(overflow_stack) +
@@ -226,6 +241,15 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs)
        unsigned long tsk_stk = (unsigned long)current->stack;
        unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
 
+       /*
+        * We're done with the shadow stack by this point, as we're on the
+        * overflow stack.  Tell any other concurrent overflowing harts that
+        * they can proceed with panicing by releasing the pseudo-spinlock.
+        *
+        * This pairs with an amoswap.aq in handle_kernel_stack_overflow.
+        */
+       smp_store_release(&spin_shadow_stack, 0);
+
        console_verbose();
 
        pr_emerg("Insufficient stack space to handle exception!\n");