x86/its: Add support for ITS-safe indirect thunk

Due to ITS, indirect branches in the lower half of a cacheline may be
vulnerable to branch target injection attack.

Introduce ITS-safe thunks to patch indirect branches in the lower half of
cacheline with the thunk. Also thunk any eBPF generated indirect branches
in emit_indirect_jump().

Below category of indirect branches are not mitigated:

- Indirect branches in the .init section are not mitigated because they are
  discarded after boot.
- Indirect branches that are explicitly marked retpoline-safe.

Note that retpoline also mitigates the indirect branches against ITS. This
is because the retpoline sequence fills an RSB entry before RET, and it
does not suffer from RSB-underflow part of the ITS.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 4b9f378e05f6be9f0c90a722c4fba66ff96f073f..a6bd0590c4371b785035d69ba81630abcc5f891d 100644 (file)
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2710,6 +2710,17 @@ config MITIGATION_SSB
          of speculative execution in a similar way to the Meltdown and Spectre
          security vulnerabilities.
 
+config MITIGATION_ITS
+       bool "Enable Indirect Target Selection mitigation"
+       depends on CPU_SUP_INTEL && X86_64
+       depends on MITIGATION_RETPOLINE && MITIGATION_RETHUNK
+       default y
+       help
+         Enable Indirect Target Selection (ITS) mitigation. ITS is a bug in
+         BPU on some Intel CPUs that may allow Spectre V2 style attacks. If
+         disabled, mitigation cannot be enabled via cmdline.
+         See <file:Documentation/admin-guide/hw-vuln/indirect-target-selection.rst>
+
 endif
 
 config ARCH_HAS_ADD_PAGES

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 2ff978fa414b5a7192f13489fb3a1ee5b4c24c2f..03af8191073a21a9acfd0c5914c32fcccbf0558f 100644 (file)
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -481,6 +481,7 @@
 #define X86_FEATURE_AMD_HETEROGENEOUS_CORES (21*32 + 6) /* Heterogeneous Core Topology */
 #define X86_FEATURE_AMD_WORKLOAD_CLASS (21*32 + 7) /* Workload Classification */
 #define X86_FEATURE_PREFER_YMM         (21*32 + 8) /* Avoid ZMM registers due to downclocking */
+#define X86_FEATURE_INDIRECT_THUNK_ITS (21*32 + 9) /* Use thunk for indirect branches in lower half of cacheline */
 
 /*
  * BUG word(s)

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 5c43f145454ddda781e21806752737b42789c86d..a5eb0cd93c9273ddfcf6d20c3583cb02e9f2a8fa 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -336,10 +336,14 @@
 
 #else /* __ASSEMBLER__ */
 
+#define ITS_THUNK_SIZE 64
+
 typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
+typedef u8 its_thunk_t[ITS_THUNK_SIZE];
 extern retpoline_thunk_t __x86_indirect_thunk_array[];
 extern retpoline_thunk_t __x86_indirect_call_thunk_array[];
 extern retpoline_thunk_t __x86_indirect_jump_thunk_array[];
+extern its_thunk_t      __x86_indirect_its_thunk_array[];
 
 #ifdef CONFIG_MITIGATION_RETHUNK
 extern void __x86_return_thunk(void);

diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index bf82c6f7d690557dd76efef6fab216f94005ed54..333c0295b0a522b70bf6945fed630d831d590bd3 100644 (file)
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -581,7 +581,8 @@ static int emit_indirect(int op, int reg, u8 *bytes)
        return i;
 }
 
-static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+static int __emit_trampoline(void *addr, struct insn *insn, u8 *bytes,
+                            void *call_dest, void *jmp_dest)
 {
        u8 op = insn->opcode.bytes[0];
        int i = 0;
@@ -602,7 +603,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8
        switch (op) {
        case CALL_INSN_OPCODE:
                __text_gen_insn(bytes+i, op, addr+i,
-                               __x86_indirect_call_thunk_array[reg],
+                               call_dest,
                                CALL_INSN_SIZE);
                i += CALL_INSN_SIZE;
                break;
@@ -610,7 +611,7 @@ static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8
        case JMP32_INSN_OPCODE:
 clang_jcc:
                __text_gen_insn(bytes+i, op, addr+i,
-                               __x86_indirect_jump_thunk_array[reg],
+                               jmp_dest,
                                JMP32_INSN_SIZE);
                i += JMP32_INSN_SIZE;
                break;
@@ -625,6 +626,35 @@ clang_jcc:
        return i;
 }
 
+static int emit_call_track_retpoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+{
+       return __emit_trampoline(addr, insn, bytes,
+                                __x86_indirect_call_thunk_array[reg],
+                                __x86_indirect_jump_thunk_array[reg]);
+}
+
+#ifdef CONFIG_MITIGATION_ITS
+static int emit_its_trampoline(void *addr, struct insn *insn, int reg, u8 *bytes)
+{
+       return __emit_trampoline(addr, insn, bytes,
+                                __x86_indirect_its_thunk_array[reg],
+                                __x86_indirect_its_thunk_array[reg]);
+}
+
+/* Check if an indirect branch is at ITS-unsafe address */
+static bool cpu_wants_indirect_its_thunk_at(unsigned long addr, int reg)
+{
+       if (!cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS))
+               return false;
+
+       /* Indirect branch opcode is 2 or 3 bytes depending on reg */
+       addr += 1 + reg / 8;
+
+       /* Lower-half of the cacheline? */
+       return !(addr & 0x20);
+}
+#endif
+
 /*
  * Rewrite the compiler generated retpoline thunk calls.
  *
@@ -699,6 +729,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
                bytes[i++] = 0xe8; /* LFENCE */
        }
 
+#ifdef CONFIG_MITIGATION_ITS
+       /*
+        * Check if the address of last byte of emitted-indirect is in
+        * lower-half of the cacheline. Such branches need ITS mitigation.
+        */
+       if (cpu_wants_indirect_its_thunk_at((unsigned long)addr + i, reg))
+               return emit_its_trampoline(addr, insn, reg, bytes);
+#endif
+
        ret = emit_indirect(op, reg, bytes + i);
        if (ret < 0)
                return ret;

diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index ccdc45e5b75961711580f62cb2a73f48c1c433c5..c3e048e7e161aa3e485b838898e3256aff3b166c 100644 (file)
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -497,6 +497,12 @@ PROVIDE(__ref_stack_chk_guard = __stack_chk_guard);
                "SRSO function pair won't alias");
 #endif
 
+#if defined(CONFIG_MITIGATION_ITS) && !defined(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)
+. = ASSERT(__x86_indirect_its_thunk_rax & 0x20, "__x86_indirect_thunk_rax not in second half of cacheline");
+. = ASSERT(((__x86_indirect_its_thunk_rcx - __x86_indirect_its_thunk_rax) % 64) == 0, "Indirect thunks are not cacheline apart");
+. = ASSERT(__x86_indirect_its_thunk_array == __x86_indirect_its_thunk_rax, "Gap in ITS thunk array");
+#endif
+
 #endif /* CONFIG_X86_64 */
 
 /*

diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S
index a26c43abd47d8dfc8ff1c4ac94da6e76e466beea..a06891892853f90ce5e2b570a114eac82c27e99d 100644 (file)
--- a/arch/x86/lib/retpoline.S
+++ b/arch/x86/lib/retpoline.S
@@ -367,6 +367,34 @@ SYM_FUNC_END(call_depth_return_thunk)
 
 #endif /* CONFIG_MITIGATION_CALL_DEPTH_TRACKING */
 
+#ifdef CONFIG_MITIGATION_ITS
+
+.macro ITS_THUNK reg
+
+SYM_INNER_LABEL(__x86_indirect_its_thunk_\reg, SYM_L_GLOBAL)
+       UNWIND_HINT_UNDEFINED
+       ANNOTATE_NOENDBR
+       ANNOTATE_RETPOLINE_SAFE
+       jmp *%\reg
+       int3
+       .align 32, 0xcc         /* fill to the end of the line */
+       .skip  32, 0xcc         /* skip to the next upper half */
+.endm
+
+/* ITS mitigation requires thunks be aligned to upper half of cacheline */
+.align 64, 0xcc
+.skip 32, 0xcc
+SYM_CODE_START(__x86_indirect_its_thunk_array)
+
+#define GEN(reg) ITS_THUNK reg
+#include <asm/GEN-for-each-reg.h>
+#undef GEN
+
+       .align 64, 0xcc
+SYM_CODE_END(__x86_indirect_its_thunk_array)
+
+#endif
+
 /*
  * This function name is magical and is used by -mfunction-return=thunk-extern
  * for the compiler to generate JMPs to it.

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 9e5fe2ba858f08408b427e395eb64be9ee15cf29..01e8119db5dbaffaa1aed33c0692327d5cecdc2c 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -661,7 +661,10 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)
 {
        u8 *prog = *pprog;
 
-       if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
+       if (cpu_feature_enabled(X86_FEATURE_INDIRECT_THUNK_ITS)) {
+               OPTIMIZER_HIDE_VAR(reg);
+               emit_jump(&prog, &__x86_indirect_its_thunk_array[reg], ip);
+       } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE_LFENCE)) {
                EMIT_LFENCE();
                EMIT2(0xFF, 0xE0 + reg);
        } else if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) {