Merge patch series "riscv: KCFI support"
authorPalmer Dabbelt <palmer@rivosinc.com>
Wed, 23 Aug 2023 21:16:44 +0000 (14:16 -0700)
committerPalmer Dabbelt <palmer@rivosinc.com>
Thu, 31 Aug 2023 07:18:32 +0000 (00:18 -0700)
Sami Tolvanen <samitolvanen@google.com> says:

This series adds KCFI support for RISC-V. KCFI is a fine-grained
forward-edge control-flow integrity scheme supported in Clang >=16,
which ensures indirect calls in instrumented code can only branch to
functions whose type matches the function pointer type, thus making
code reuse attacks more difficult.

Patch 1 implements a pt_regs based syscall wrapper to address
function pointer type mismatches in syscall handling. Patches 2 and 3
annotate indirectly called assembly functions with CFI types. Patch 4
implements error handling for indirect call checks. Patch 5 disables
CFI for arch/riscv/purgatory. Patch 6 finally allows CONFIG_CFI_CLANG
to be enabled for RISC-V.

Note that Clang 16 has a generic architecture-agnostic KCFI
implementation, which does work with the kernel, but doesn't produce
a stable code sequence for indirect call checks, which means
potential failures just trap and won't result in informative error
messages. Clang 17 includes a RISC-V specific back-end implementation
for KCFI, which emits a predictable code sequence for the checks and a
.kcfi_traps section with locations of the traps, which patch 5 uses to
produce more useful errors.

The type mismatch fixes and annotations in the first three patches
also become necessary in future if the kernel decides to support
fine-grained CFI implemented using the hardware landing pad
feature proposed in the in-progress Zicfisslp extension. Once the
specification is ratified and hardware support emerges, implementing
runtime patching support that replaces KCFI instrumentation with
Zicfisslp landing pads might also be feasible (similarly to KCFI to
FineIBT patching on x86_64), allowing distributions to ship a unified
kernel binary for all devices.

* b4-shazam-merge:
  riscv: Allow CONFIG_CFI_CLANG to be selected
  riscv/purgatory: Disable CFI
  riscv: Add CFI error handling
  riscv: Add ftrace_stub_graph
  riscv: Add types to indirectly called assembly functions
  riscv: Implement syscall wrappers

Link: https://lore.kernel.org/r/20230710183544.999540-8-samitolvanen@google.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
14 files changed:
arch/riscv/Kconfig
arch/riscv/include/asm/cfi.h [new file with mode: 0644]
arch/riscv/include/asm/insn.h
arch/riscv/include/asm/syscall.h
arch/riscv/include/asm/syscall_wrapper.h [new file with mode: 0644]
arch/riscv/kernel/Makefile
arch/riscv/kernel/cfi.c [new file with mode: 0644]
arch/riscv/kernel/compat_syscall_table.c
arch/riscv/kernel/mcount.S
arch/riscv/kernel/suspend_entry.S
arch/riscv/kernel/sys_riscv.c
arch/riscv/kernel/syscall_table.c
arch/riscv/kernel/traps.c
arch/riscv/purgatory/Makefile

index afa7160b136c87a0daf3cf33f8fa01198858c1f3..1494bca1b53a1c9d99983a5dbd01136da8582490 100644 (file)
@@ -35,6 +35,7 @@ config RISCV
        select ARCH_HAS_SET_MEMORY if MMU
        select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL
        select ARCH_HAS_STRICT_MODULE_RWX if MMU && !XIP_KERNEL
+       select ARCH_HAS_SYSCALL_WRAPPER
        select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
        select ARCH_HAS_UBSAN_SANITIZE_ALL
        select ARCH_HAS_VDSO_DATA
@@ -42,12 +43,14 @@ config RISCV
        select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT
        select ARCH_STACKWALK
        select ARCH_SUPPORTS_ATOMIC_RMW
+       select ARCH_SUPPORTS_CFI_CLANG
        select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU
        select ARCH_SUPPORTS_HUGETLBFS if MMU
        select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU
        select ARCH_SUPPORTS_PER_VMA_LOCK if MMU
        select ARCH_USE_MEMTEST
        select ARCH_USE_QUEUED_RWLOCKS
+       select ARCH_USES_CFI_TRAPS if CFI_CLANG
        select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU
        select ARCH_WANT_FRAME_POINTERS
        select ARCH_WANT_GENERAL_HUGETLB if !RISCV_ISA_SVNAPOT
diff --git a/arch/riscv/include/asm/cfi.h b/arch/riscv/include/asm/cfi.h
new file mode 100644 (file)
index 0000000..56bf9d6
--- /dev/null
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_RISCV_CFI_H
+#define _ASM_RISCV_CFI_H
+
+/*
+ * Clang Control Flow Integrity (CFI) support.
+ *
+ * Copyright (C) 2023 Google LLC
+ */
+
+#include <linux/cfi.h>
+
+#ifdef CONFIG_CFI_CLANG
+enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
+#else
+static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
+{
+       return BUG_TRAP_TYPE_NONE;
+}
+#endif /* CONFIG_CFI_CLANG */
+
+#endif /* _ASM_RISCV_CFI_H */
index 4e1505cef8aa40fd543c4c2f00bc357a6cb5c4be..9c23f598434c2084fe464d28d756c47aea9c4cc5 100644 (file)
@@ -63,6 +63,7 @@
 #define RVG_RS1_OPOFF          15
 #define RVG_RS2_OPOFF          20
 #define RVG_RD_OPOFF           7
+#define RVG_RS1_MASK           GENMASK(4, 0)
 #define RVG_RD_MASK            GENMASK(4, 0)
 
 /* The bit field of immediate value in RVC J instruction */
 #define RVC_C2_RS1_OPOFF       7
 #define RVC_C2_RS2_OPOFF       2
 #define RVC_C2_RD_OPOFF                7
+#define RVC_C2_RS1_MASK                GENMASK(4, 0)
 
 /* parts of opcode for RVG*/
 #define RVG_OPCODE_FENCE       0x0f
@@ -278,6 +280,10 @@ static __always_inline bool riscv_insn_is_branch(u32 code)
 #define RV_X(X, s, mask)  (((X) >> (s)) & (mask))
 #define RVC_X(X, s, mask) RV_X(X, s, mask)
 
+#define RV_EXTRACT_RS1_REG(x) \
+       ({typeof(x) x_ = (x); \
+       (RV_X(x_, RVG_RS1_OPOFF, RVG_RS1_MASK)); })
+
 #define RV_EXTRACT_RD_REG(x) \
        ({typeof(x) x_ = (x); \
        (RV_X(x_, RVG_RD_OPOFF, RVG_RD_MASK)); })
@@ -305,6 +311,10 @@ static __always_inline bool riscv_insn_is_branch(u32 code)
        (RV_X(x_, RV_B_IMM_11_OPOFF, RV_B_IMM_11_MASK) << RV_B_IMM_11_OFF) | \
        (RV_IMM_SIGN(x_) << RV_B_IMM_SIGN_OFF); })
 
+#define RVC_EXTRACT_C2_RS1_REG(x) \
+       ({typeof(x) x_ = (x); \
+       (RV_X(x_, RVC_C2_RS1_OPOFF, RVC_C2_RS1_MASK)); })
+
 #define RVC_EXTRACT_JTYPE_IMM(x) \
        ({typeof(x) x_ = (x); \
        (RVC_X(x_, RVC_J_IMM_3_1_OPOFF, RVC_J_IMM_3_1_MASK) << RVC_J_IMM_3_1_OFF) | \
index 0148c6bd9675d01141454862a65fa221baf87958..121fff429dce66b31fe79b691b8edd816c8019e9 100644 (file)
@@ -75,7 +75,7 @@ static inline int syscall_get_arch(struct task_struct *task)
 #endif
 }
 
-typedef long (*syscall_t)(ulong, ulong, ulong, ulong, ulong, ulong, ulong);
+typedef long (*syscall_t)(const struct pt_regs *);
 static inline void syscall_handler(struct pt_regs *regs, ulong syscall)
 {
        syscall_t fn;
@@ -87,8 +87,7 @@ static inline void syscall_handler(struct pt_regs *regs, ulong syscall)
 #endif
                fn = sys_call_table[syscall];
 
-       regs->a0 = fn(regs->orig_a0, regs->a1, regs->a2,
-                     regs->a3, regs->a4, regs->a5, regs->a6);
+       regs->a0 = fn(regs);
 }
 
 static inline bool arch_syscall_is_vdso_sigreturn(struct pt_regs *regs)
diff --git a/arch/riscv/include/asm/syscall_wrapper.h b/arch/riscv/include/asm/syscall_wrapper.h
new file mode 100644 (file)
index 0000000..1d7942c
--- /dev/null
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * syscall_wrapper.h - riscv specific wrappers to syscall definitions
+ *
+ * Based on arch/arm64/include/syscall_wrapper.h
+ */
+
+#ifndef __ASM_SYSCALL_WRAPPER_H
+#define __ASM_SYSCALL_WRAPPER_H
+
+#include <asm/ptrace.h>
+
+asmlinkage long __riscv_sys_ni_syscall(const struct pt_regs *);
+
+#define SC_RISCV_REGS_TO_ARGS(x, ...)                          \
+       __MAP(x,__SC_ARGS                                       \
+             ,,regs->orig_a0,,regs->a1,,regs->a2               \
+             ,,regs->a3,,regs->a4,,regs->a5,,regs->a6)
+
+#ifdef CONFIG_COMPAT
+
+#define COMPAT_SYSCALL_DEFINEx(x, name, ...)                                           \
+       asmlinkage long __riscv_compat_sys##name(const struct pt_regs *regs);           \
+       ALLOW_ERROR_INJECTION(__riscv_compat_sys##name, ERRNO);                         \
+       static long __se_compat_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__));              \
+       static inline long __do_compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__));       \
+       asmlinkage long __riscv_compat_sys##name(const struct pt_regs *regs)            \
+       {                                                                               \
+               return __se_compat_sys##name(SC_RISCV_REGS_TO_ARGS(x,__VA_ARGS__));     \
+       }                                                                               \
+       static long __se_compat_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__))               \
+       {                                                                               \
+               return __do_compat_sys##name(__MAP(x,__SC_DELOUSE,__VA_ARGS__));        \
+       }                                                                               \
+       static inline long __do_compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))
+
+#define COMPAT_SYSCALL_DEFINE0(sname)                                                  \
+       asmlinkage long __riscv_compat_sys_##sname(const struct pt_regs *__unused);     \
+       ALLOW_ERROR_INJECTION(__riscv_compat_sys_##sname, ERRNO);                       \
+       asmlinkage long __riscv_compat_sys_##sname(const struct pt_regs *__unused)
+
+#define COND_SYSCALL_COMPAT(name)                                                      \
+       asmlinkage long __weak __riscv_compat_sys_##name(const struct pt_regs *regs);   \
+       asmlinkage long __weak __riscv_compat_sys_##name(const struct pt_regs *regs)    \
+       {                                                                               \
+               return sys_ni_syscall();                                                \
+       }
+
+#define COMPAT_SYS_NI(name) \
+       SYSCALL_ALIAS(__riscv_compat_sys_##name, sys_ni_posix_timers);
+
+#endif /* CONFIG_COMPAT */
+
+#define __SYSCALL_DEFINEx(x, name, ...)                                                \
+       asmlinkage long __riscv_sys##name(const struct pt_regs *regs);          \
+       ALLOW_ERROR_INJECTION(__riscv_sys##name, ERRNO);                        \
+       static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__));             \
+       static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__));      \
+       asmlinkage long __riscv_sys##name(const struct pt_regs *regs)           \
+       {                                                                       \
+               return __se_sys##name(SC_RISCV_REGS_TO_ARGS(x,__VA_ARGS__));    \
+       }                                                                       \
+       static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__))              \
+       {                                                                       \
+               long ret = __do_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__));      \
+               __MAP(x,__SC_TEST,__VA_ARGS__);                                 \
+               __PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__));               \
+               return ret;                                                     \
+       }                                                                       \
+       static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))
+
+#define SYSCALL_DEFINE0(sname)                                                 \
+       SYSCALL_METADATA(_##sname, 0);                                          \
+       asmlinkage long __riscv_sys_##sname(const struct pt_regs *__unused);    \
+       ALLOW_ERROR_INJECTION(__riscv_sys_##sname, ERRNO);                      \
+       asmlinkage long __riscv_sys_##sname(const struct pt_regs *__unused)
+
+#define COND_SYSCALL(name)                                                     \
+       asmlinkage long __weak __riscv_sys_##name(const struct pt_regs *regs);  \
+       asmlinkage long __weak __riscv_sys_##name(const struct pt_regs *regs)   \
+       {                                                                       \
+               return sys_ni_syscall();                                        \
+       }
+
+#define SYS_NI(name) SYSCALL_ALIAS(__riscv_sys_##name, sys_ni_posix_timers);
+
+#endif /* __ASM_SYSCALL_WRAPPER_H */
index 506cc4a9a45a555eb3032b23caed8def62c9aa71..6ac56af42f4a0c847bcc258e7cb9e6df0ffbb7bb 100644 (file)
@@ -91,6 +91,8 @@ obj-$(CONFIG_CRASH_CORE)      += crash_core.o
 
 obj-$(CONFIG_JUMP_LABEL)       += jump_label.o
 
+obj-$(CONFIG_CFI_CLANG)                += cfi.o
+
 obj-$(CONFIG_EFI)              += efi.o
 obj-$(CONFIG_COMPAT)           += compat_syscall_table.o
 obj-$(CONFIG_COMPAT)           += compat_signal.o
diff --git a/arch/riscv/kernel/cfi.c b/arch/riscv/kernel/cfi.c
new file mode 100644 (file)
index 0000000..820158d
--- /dev/null
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Clang Control Flow Integrity (CFI) support.
+ *
+ * Copyright (C) 2023 Google LLC
+ */
+#include <asm/cfi.h>
+#include <asm/insn.h>
+
+/*
+ * Returns the target address and the expected type when regs->epc points
+ * to a compiler-generated CFI trap.
+ */
+static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target,
+                           u32 *type)
+{
+       unsigned long *regs_ptr = (unsigned long *)regs;
+       int rs1_num;
+       u32 insn;
+
+       *target = *type = 0;
+
+       /*
+        * The compiler generates the following instruction sequence
+        * for indirect call checks:
+        *
+        *   lw      t1, -4(<reg>)
+        *   lui     t2, <hi20>
+        *   addiw   t2, t2, <lo12>
+        *   beq     t1, t2, .Ltmp1
+        *   ebreak  ; <- regs->epc
+        *   .Ltmp1:
+        *   jalr    <reg>
+        *
+        * We can read the expected type and the target address from the
+        * registers passed to the beq/jalr instructions.
+        */
+       if (get_kernel_nofault(insn, (void *)regs->epc - 4))
+               return false;
+       if (!riscv_insn_is_beq(insn))
+               return false;
+
+       *type = (u32)regs_ptr[RV_EXTRACT_RS1_REG(insn)];
+
+       if (get_kernel_nofault(insn, (void *)regs->epc) ||
+           get_kernel_nofault(insn, (void *)regs->epc + GET_INSN_LENGTH(insn)))
+               return false;
+
+       if (riscv_insn_is_jalr(insn))
+               rs1_num = RV_EXTRACT_RS1_REG(insn);
+       else if (riscv_insn_is_c_jalr(insn))
+               rs1_num = RVC_EXTRACT_C2_RS1_REG(insn);
+       else
+               return false;
+
+       *target = regs_ptr[rs1_num];
+
+       return true;
+}
+
+/*
+ * Checks if the ebreak trap is because of a CFI failure, and handles the trap
+ * if needed. Returns a bug_trap_type value similarly to report_bug.
+ */
+enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
+{
+       unsigned long target;
+       u32 type;
+
+       if (!is_cfi_trap(regs->epc))
+               return BUG_TRAP_TYPE_NONE;
+
+       if (!decode_cfi_insn(regs, &target, &type))
+               return report_cfi_failure_noaddr(regs, regs->epc);
+
+       return report_cfi_failure(regs, regs->epc, &target, type);
+}
index 651f2b009c2812c6af75d0acce89b45ceaefd835..ad7f2d712f5fd070b3e4d8687a8b8d147f17162c 100644 (file)
@@ -9,11 +9,15 @@
 #include <asm/syscall.h>
 
 #undef __SYSCALL
-#define __SYSCALL(nr, call)      [nr] = (call),
+#define __SYSCALL(nr, call)    asmlinkage long __riscv_##call(const struct pt_regs *);
+#include <asm/unistd.h>
+
+#undef __SYSCALL
+#define __SYSCALL(nr, call)      [nr] = __riscv_##call,
 
 asmlinkage long compat_sys_rt_sigreturn(void);
 
 void * const compat_sys_call_table[__NR_syscalls] = {
-       [0 ... __NR_syscalls - 1] = sys_ni_syscall,
+       [0 ... __NR_syscalls - 1] = __riscv_sys_ni_syscall,
 #include <asm/unistd.h>
 };
index 8a6e5a9e842a68bf056ed30738defd9b48f10921..8818a8fa9ff3af7b3427e2456f1650c5c6cb7168 100644 (file)
@@ -3,6 +3,7 @@
 
 #include <linux/init.h>
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/asm.h>
 #include <asm/csr.h>
 #include <asm/unistd.h>
        addi    sp, sp, 4*SZREG
        .endm
 
-ENTRY(ftrace_stub)
+SYM_TYPED_FUNC_START(ftrace_stub)
 #ifdef CONFIG_DYNAMIC_FTRACE
        .global MCOUNT_NAME
        .set    MCOUNT_NAME, ftrace_stub
 #endif
        ret
-ENDPROC(ftrace_stub)
+SYM_FUNC_END(ftrace_stub)
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
+SYM_TYPED_FUNC_START(ftrace_stub_graph)
+       ret
+SYM_FUNC_END(ftrace_stub_graph)
+
 ENTRY(return_to_handler)
 /*
  * On implementing the frame point test, the ideal way is to compare the
index 12b52afe09a421c2340065ff7612debe86a4c3a5..f7960c7c5f9e25081424f3edbe526896c1692303 100644 (file)
@@ -5,6 +5,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/asm.h>
 #include <asm/asm-offsets.h>
 #include <asm/assembler.h>
@@ -58,7 +59,7 @@ ENTRY(__cpu_suspend_enter)
        ret
 END(__cpu_suspend_enter)
 
-ENTRY(__cpu_resume_enter)
+SYM_TYPED_FUNC_START(__cpu_resume_enter)
        /* Load the global pointer */
        .option push
        .option norelax
@@ -94,4 +95,4 @@ ENTRY(__cpu_resume_enter)
 
        /* Return to C code */
        ret
-END(__cpu_resume_enter)
+SYM_FUNC_END(__cpu_resume_enter)
index 26ef5526bfb4cc45e57096b536b75e5bb3116600..473159b5f303ab9c7f784bb388814446b13b61f2 100644 (file)
@@ -335,3 +335,9 @@ SYSCALL_DEFINE5(riscv_hwprobe, struct riscv_hwprobe __user *, pairs,
        return do_riscv_hwprobe(pairs, pair_count, cpu_count,
                                cpus, flags);
 }
+
+/* Not defined using SYSCALL_DEFINE0 to avoid error injection */
+asmlinkage long __riscv_sys_ni_syscall(const struct pt_regs *__unused)
+{
+       return -ENOSYS;
+}
index 44b1420a22705a81eca174d748bfc2d2986302d7..dda91376490340c111e84f937495c6c35d5b63ee 100644 (file)
 #include <asm/syscall.h>
 
 #undef __SYSCALL
-#define __SYSCALL(nr, call)    [nr] = (call),
+#define __SYSCALL(nr, call)    asmlinkage long __riscv_##call(const struct pt_regs *);
+#include <asm/unistd.h>
+
+#undef __SYSCALL
+#define __SYSCALL(nr, call)    [nr] = __riscv_##call,
 
 void * const sys_call_table[__NR_syscalls] = {
-       [0 ... __NR_syscalls - 1] = sys_ni_syscall,
+       [0 ... __NR_syscalls - 1] = __riscv_sys_ni_syscall,
 #include <asm/unistd.h>
 };
index f910dfccbf5d2aaf5b838cdbfd809f8e937bdfd5..212dc20631fb92b442f57149dc44f1393cfa39b7 100644 (file)
@@ -21,6 +21,7 @@
 
 #include <asm/asm-prototypes.h>
 #include <asm/bug.h>
+#include <asm/cfi.h>
 #include <asm/csr.h>
 #include <asm/processor.h>
 #include <asm/ptrace.h>
@@ -271,7 +272,8 @@ void handle_break(struct pt_regs *regs)
                                                                == NOTIFY_STOP)
                return;
 #endif
-       else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN)
+       else if (report_bug(regs->epc, regs) == BUG_TRAP_TYPE_WARN ||
+                handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN)
                regs->epc += get_break_insn_length(regs->epc);
        else
                die(regs, "Kernel BUG");
index dc20e166983e3be9257b8d9a2b4d71423a8eff92..9e6476719abbbb4e9206cd89292be7f25261a5b6 100644 (file)
@@ -77,6 +77,10 @@ ifdef CONFIG_STACKPROTECTOR_STRONG
 PURGATORY_CFLAGS_REMOVE                += -fstack-protector-strong
 endif
 
+ifdef CONFIG_CFI_CLANG
+PURGATORY_CFLAGS_REMOVE                += $(CC_FLAGS_CFI)
+endif
+
 CFLAGS_REMOVE_purgatory.o      += $(PURGATORY_CFLAGS_REMOVE)
 CFLAGS_purgatory.o             += $(PURGATORY_CFLAGS)