x86/shstk: Add user control-protection fault handler

A control-protection fault is triggered when a control-flow transfer
attempt violates Shadow Stack or Indirect Branch Tracking constraints.
For example, the return address for a RET instruction differs from the copy
on the shadow stack.

There already exists a control-protection fault handler for handling kernel
IBT faults. Refactor this fault handler into separate user and kernel
handlers, like the page fault handler. Add a control-protection handler
for usermode. To avoid ifdeffery, put them both in a new file cet.c, which
is compiled in the case of either of the two CET features supported in the
kernel: kernel IBT or user mode shadow stack. Move some static inline
functions from traps.c into a header so they can be used in cet.c.

Opportunistically fix a comment in the kernel IBT part of the fault
handler that is on the end of the line instead of preceding it.

Keep the same behavior for the kernel side of the fault handler, except for
converting a BUG to a WARN in the case of a #CP happening when the feature
is missing. This unifies the behavior with the new shadow stack code, and
also prevents the kernel from crashing under this situation which is
potentially recoverable.

The control-protection fault handler works in a similar way as the general
protection fault handler. It provides the si_code SEGV_CPERR to the signal
handler.

Co-developed-by: Yu-cheng Yu <yu-cheng.yu@intel.com>
Signed-off-by: Yu-cheng Yu <yu-cheng.yu@intel.com>
Signed-off-by: Rick Edgecombe <rick.p.edgecombe@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Kees Cook <keescook@chromium.org>
Acked-by: Mike Rapoport (IBM) <rppt@kernel.org>
Tested-by: Pengfei Xu <pengfei.xu@intel.com>
Tested-by: John Allen <john.allen@amd.com>
Tested-by: Kees Cook <keescook@chromium.org>
Link: https://lore.kernel.org/all/20230613001108.3040476-28-rick.p.edgecombe%40intel.com

diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c
index 8d0afa11bed5448dd0aefca20ec8cfbaed80783f..79a6730fa0eb7d6581343bdbbeb6834dcaabb9f8 100644 (file)
--- a/arch/arm/kernel/signal.c
+++ b/arch/arm/kernel/signal.c
@@ -682,7 +682,7 @@ asmlinkage void do_rseq_syscall(struct pt_regs *regs)
  */
 static_assert(NSIGILL  == 11);
 static_assert(NSIGFPE  == 15);
-static_assert(NSIGSEGV == 9);
+static_assert(NSIGSEGV == 10);
 static_assert(NSIGBUS  == 5);
 static_assert(NSIGTRAP == 6);
 static_assert(NSIGCHLD == 6);

diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index e304f7ebec2a91ff88c83afa20d8722a61f99cfa..0df8cc295ea511cd6fb3d910a64ecd3d934aeda8 100644 (file)
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -1344,7 +1344,7 @@ void __init minsigstksz_setup(void)
  */
 static_assert(NSIGILL  == 11);
 static_assert(NSIGFPE  == 15);
-static_assert(NSIGSEGV == 9);
+static_assert(NSIGSEGV == 10);
 static_assert(NSIGBUS  == 5);
 static_assert(NSIGTRAP == 6);
 static_assert(NSIGCHLD == 6);

diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index 4700f8522d27b191858bff1924587793151a6acd..bbd542704730aa1519c96e0ed5c5b6213a729ee2 100644 (file)
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -460,7 +460,7 @@ void compat_setup_restart_syscall(struct pt_regs *regs)
  */
 static_assert(NSIGILL  == 11);
 static_assert(NSIGFPE  == 15);
-static_assert(NSIGSEGV == 9);
+static_assert(NSIGSEGV == 10);
 static_assert(NSIGBUS  == 5);
 static_assert(NSIGTRAP == 6);
 static_assert(NSIGCHLD == 6);

diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c
index ca450c7bc53fd832577b056f7d193ca1fe47557a..a23cdd7459bbfd4d4280ec902ffe17216f526f70 100644 (file)
--- a/arch/sparc/kernel/signal32.c
+++ b/arch/sparc/kernel/signal32.c
@@ -753,7 +753,7 @@ out:
  */
 static_assert(NSIGILL  == 11);
 static_assert(NSIGFPE  == 15);
-static_assert(NSIGSEGV == 9);
+static_assert(NSIGSEGV == 10);
 static_assert(NSIGBUS  == 5);
 static_assert(NSIGTRAP == 6);
 static_assert(NSIGCHLD == 6);

diff --git a/arch/sparc/kernel/signal_64.c b/arch/sparc/kernel/signal_64.c
index 570e43e6fda5cabb72f225fb392dbf91e61945f2..b4e410976e0d5c1c8414ff3334a7ec84cc745f22 100644 (file)
--- a/arch/sparc/kernel/signal_64.c
+++ b/arch/sparc/kernel/signal_64.c
@@ -562,7 +562,7 @@ void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long
  */
 static_assert(NSIGILL  == 11);
 static_assert(NSIGFPE  == 15);
-static_assert(NSIGSEGV == 9);
+static_assert(NSIGSEGV == 10);
 static_assert(NSIGBUS  == 5);
 static_assert(NSIGTRAP == 6);
 static_assert(NSIGCHLD == 6);

diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h
index b9c7eae2e70f383e1f40fdd278ee27454ffb512d..702d93fdd10e8d44015cc687cb90106ae5bd422c 100644 (file)
--- a/arch/x86/include/asm/disabled-features.h
+++ b/arch/x86/include/asm/disabled-features.h
@@ -111,6 +111,12 @@
 #define DISABLE_USER_SHSTK     (1 << (X86_FEATURE_USER_SHSTK & 31))
 #endif
 
+#ifdef CONFIG_X86_KERNEL_IBT
+#define DISABLE_IBT    0
+#else
+#define DISABLE_IBT    (1 << (X86_FEATURE_IBT & 31))
+#endif
+
 /*
  * Make sure to add features to the correct mask
  */
@@ -134,7 +140,7 @@
 #define DISABLED_MASK16        (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \
                         DISABLE_ENQCMD)
 #define DISABLED_MASK17        0
-#define DISABLED_MASK18        0
+#define DISABLED_MASK18        (DISABLE_IBT)
 #define DISABLED_MASK19        0
 #define DISABLED_MASK20        0
 #define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 21)

diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index b241af4ce9b401268bd1b32e1841c146a40068dc..61e0e6301f0985b1f217a94b8f048faf11f5e20b 100644 (file)
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -614,7 +614,7 @@ DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault);
 #endif
 
 /* #CP */
-#ifdef CONFIG_X86_KERNEL_IBT
+#ifdef CONFIG_X86_CET
 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP,        exc_control_protection);
 #endif
 

diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 47ecfff2c83dade6d993461c1b9c7494a05ffb28..75e0dabf0c4517105e7162639b3cd4965df0976e 100644 (file)
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -47,4 +47,16 @@ void __noreturn handle_stack_overflow(struct pt_regs *regs,
                                      struct stack_info *info);
 #endif
 
+static inline void cond_local_irq_enable(struct pt_regs *regs)
+{
+       if (regs->flags & X86_EFLAGS_IF)
+               local_irq_enable();
+}
+
+static inline void cond_local_irq_disable(struct pt_regs *regs)
+{
+       if (regs->flags & X86_EFLAGS_IF)
+               local_irq_disable();
+}
+
 #endif /* _ASM_X86_TRAPS_H */

diff --git a/arch/x86/kernel/cet.c b/arch/x86/kernel/cet.c
index 7ad22b705b64ca57d62d7c5ca6a95ecbd05c5125..cc10d8be9d74cd91d09dc4a5376edc540c3fb736 100644 (file)
--- a/arch/x86/kernel/cet.c
+++ b/arch/x86/kernel/cet.c
@@ -4,10 +4,6 @@
 #include <asm/bugs.h>
 #include <asm/traps.h>
 
-static __ro_after_init bool ibt_fatal = true;
-
-extern void ibt_selftest_ip(void); /* code label defined in asm below */
-
 enum cp_error_code {
        CP_EC        = (1 << 15) - 1,
 
@@ -20,15 +16,80 @@ enum cp_error_code {
        CP_ENCL      = 1 << 15,
 };
 
-DEFINE_IDTENTRY_ERRORCODE(exc_control_protection)
+static const char cp_err[][10] = {
+       [0] = "unknown",
+       [1] = "near ret",
+       [2] = "far/iret",
+       [3] = "endbranch",
+       [4] = "rstorssp",
+       [5] = "setssbsy",
+};
+
+static const char *cp_err_string(unsigned long error_code)
+{
+       unsigned int cpec = error_code & CP_EC;
+
+       if (cpec >= ARRAY_SIZE(cp_err))
+               cpec = 0;
+       return cp_err[cpec];
+}
+
+static void do_unexpected_cp(struct pt_regs *regs, unsigned long error_code)
+{
+       WARN_ONCE(1, "Unexpected %s #CP, error_code: %s\n",
+                 user_mode(regs) ? "user mode" : "kernel mode",
+                 cp_err_string(error_code));
+}
+
+static DEFINE_RATELIMIT_STATE(cpf_rate, DEFAULT_RATELIMIT_INTERVAL,
+                             DEFAULT_RATELIMIT_BURST);
+
+static void do_user_cp_fault(struct pt_regs *regs, unsigned long error_code)
 {
-       if (!cpu_feature_enabled(X86_FEATURE_IBT)) {
-               pr_err("Unexpected #CP\n");
-               BUG();
+       struct task_struct *tsk;
+       unsigned long ssp;
+
+       /*
+        * An exception was just taken from userspace. Since interrupts are disabled
+        * here, no scheduling should have messed with the registers yet and they
+        * will be whatever is live in userspace. So read the SSP before enabling
+        * interrupts so locking the fpregs to do it later is not required.
+        */
+       rdmsrl(MSR_IA32_PL3_SSP, ssp);
+
+       cond_local_irq_enable(regs);
+
+       tsk = current;
+       tsk->thread.error_code = error_code;
+       tsk->thread.trap_nr = X86_TRAP_CP;
+
+       /* Ratelimit to prevent log spamming. */
+       if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
+           __ratelimit(&cpf_rate)) {
+               pr_emerg("%s[%d] control protection ip:%lx sp:%lx ssp:%lx error:%lx(%s)%s",
+                        tsk->comm, task_pid_nr(tsk),
+                        regs->ip, regs->sp, ssp, error_code,
+                        cp_err_string(error_code),
+                        error_code & CP_ENCL ? " in enclave" : "");
+               print_vma_addr(KERN_CONT " in ", regs->ip);
+               pr_cont("\n");
        }
 
-       if (WARN_ON_ONCE(user_mode(regs) || (error_code & CP_EC) != CP_ENDBR))
+       force_sig_fault(SIGSEGV, SEGV_CPERR, (void __user *)0);
+       cond_local_irq_disable(regs);
+}
+
+static __ro_after_init bool ibt_fatal = true;
+
+/* code label defined in asm below */
+extern void ibt_selftest_ip(void);
+
+static void do_kernel_cp_fault(struct pt_regs *regs, unsigned long error_code)
+{
+       if ((error_code & CP_EC) != CP_ENDBR) {
+               do_unexpected_cp(regs, error_code);
                return;
+       }
 
        if (unlikely(regs->ip == (unsigned long)&ibt_selftest_ip)) {
                regs->ax = 0;
@@ -74,3 +135,18 @@ static int __init ibt_setup(char *str)
 }
 
 __setup("ibt=", ibt_setup);
+
+DEFINE_IDTENTRY_ERRORCODE(exc_control_protection)
+{
+       if (user_mode(regs)) {
+               if (cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
+                       do_user_cp_fault(regs, error_code);
+               else
+                       do_unexpected_cp(regs, error_code);
+       } else {
+               if (cpu_feature_enabled(X86_FEATURE_IBT))
+                       do_kernel_cp_fault(regs, error_code);
+               else
+                       do_unexpected_cp(regs, error_code);
+       }
+}

diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c
index a58c6bc1cd68c2be54ebbe5bb149fa9c954b38d7..5074b84203595468a7233a10acea0421b06bac8e 100644 (file)
--- a/arch/x86/kernel/idt.c
+++ b/arch/x86/kernel/idt.c
@@ -107,7 +107,7 @@ static const __initconst struct idt_data def_idts[] = {
        ISTG(X86_TRAP_MC,               asm_exc_machine_check, IST_INDEX_MCE),
 #endif
 
-#ifdef CONFIG_X86_KERNEL_IBT
+#ifdef CONFIG_X86_CET
        INTG(X86_TRAP_CP,               asm_exc_control_protection),
 #endif
 

diff --git a/arch/x86/kernel/signal_32.c b/arch/x86/kernel/signal_32.c
index 9027fc088f97191694b87af63097455b99682fd7..c12624bc82a31b3100497836b1a89a6d8887e16f 100644 (file)
--- a/arch/x86/kernel/signal_32.c
+++ b/arch/x86/kernel/signal_32.c
@@ -402,7 +402,7 @@ Efault:
 */
 static_assert(NSIGILL  == 11);
 static_assert(NSIGFPE  == 15);
-static_assert(NSIGSEGV == 9);
+static_assert(NSIGSEGV == 10);
 static_assert(NSIGBUS  == 5);
 static_assert(NSIGTRAP == 6);
 static_assert(NSIGCHLD == 6);

diff --git a/arch/x86/kernel/signal_64.c b/arch/x86/kernel/signal_64.c
index 13a1e6083837a828f5a7932959e3fda39fae22eb..0e808c72bf7eaa330814e747f28177577e095663 100644 (file)
--- a/arch/x86/kernel/signal_64.c
+++ b/arch/x86/kernel/signal_64.c
@@ -403,7 +403,7 @@ void sigaction_compat_abi(struct k_sigaction *act, struct k_sigaction *oact)
 */
 static_assert(NSIGILL  == 11);
 static_assert(NSIGFPE  == 15);
-static_assert(NSIGSEGV == 9);
+static_assert(NSIGSEGV == 10);
 static_assert(NSIGBUS  == 5);
 static_assert(NSIGTRAP == 6);
 static_assert(NSIGCHLD == 6);

diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 6f666dfa97de2024e7bf2badfd32e1bfb2efab4b..f358350624b2e2c19f5390485028d470e5cb9f9e 100644 (file)
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -77,18 +77,6 @@
 
 DECLARE_BITMAP(system_vectors, NR_VECTORS);
 
-static inline void cond_local_irq_enable(struct pt_regs *regs)
-{
-       if (regs->flags & X86_EFLAGS_IF)
-               local_irq_enable();
-}
-
-static inline void cond_local_irq_disable(struct pt_regs *regs)
-{
-       if (regs->flags & X86_EFLAGS_IF)
-               local_irq_disable();
-}
-
 __always_inline int is_valid_bugaddr(unsigned long addr)
 {
        if (addr < TASK_SIZE_MAX)

diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 93b658248d013c5a64c441fbcd5110945501211e..aa797b892b5af2b170a165f27ecb8434b8ba0f3c 100644 (file)
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -694,7 +694,7 @@ static struct trap_array_entry trap_array[] = {
        TRAP_ENTRY(exc_coprocessor_error,               false ),
        TRAP_ENTRY(exc_alignment_check,                 false ),
        TRAP_ENTRY(exc_simd_coprocessor_error,          false ),
-#ifdef CONFIG_X86_KERNEL_IBT
+#ifdef CONFIG_X86_CET
        TRAP_ENTRY(exc_control_protection,              false ),
 #endif
 };

diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S
index 08f1ceb9eb815857fb15a9d83c23f88b153aa20f..9e5e680087853a34f8a2fc7a935e60e2c28f12d5 100644 (file)
--- a/arch/x86/xen/xen-asm.S
+++ b/arch/x86/xen/xen-asm.S
@@ -148,7 +148,7 @@ xen_pv_trap asm_exc_page_fault
 xen_pv_trap asm_exc_spurious_interrupt_bug
 xen_pv_trap asm_exc_coprocessor_error
 xen_pv_trap asm_exc_alignment_check
-#ifdef CONFIG_X86_KERNEL_IBT
+#ifdef CONFIG_X86_CET
 xen_pv_trap asm_exc_control_protection
 #endif
 #ifdef CONFIG_X86_MCE

diff --git a/include/uapi/asm-generic/siginfo.h b/include/uapi/asm-generic/siginfo.h
index ffbe4cec9f32de390af4887de2e0aa42c945f153..0f52d0ac47c56739f770e0ae78e53e9a5f8caa7e 100644 (file)
--- a/include/uapi/asm-generic/siginfo.h
+++ b/include/uapi/asm-generic/siginfo.h
@@ -242,7 +242,8 @@ typedef struct siginfo {
 #define SEGV_ADIPERR   7       /* Precise MCD exception */
 #define SEGV_MTEAERR   8       /* Asynchronous ARM MTE error */
 #define SEGV_MTESERR   9       /* Synchronous ARM MTE exception */
-#define NSIGSEGV       9
+#define SEGV_CPERR     10      /* Control protection fault */
+#define NSIGSEGV       10
 
 /*
  * SIGBUS si_codes