arch/x86/include/asm/nospec-branch.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2
   3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
   4 #define _ASM_X86_NOSPEC_BRANCH_H_
   5
   6 #include <asm/alternative.h>
   7 #include <asm/alternative-asm.h>
   8 #include <asm/cpufeatures.h>
   9 #include <asm/msr-index.h>
  10
  11 #ifdef __ASSEMBLY__
  12
  13 /*
  14  * This should be used immediately before a retpoline alternative.  It tells
  15  * objtool where the retpolines are so that it can make sense of the control
  16  * flow by just reading the original instruction(s) and ignoring the
  17  * alternatives.
  18  */
  19 .macro ANNOTATE_NOSPEC_ALTERNATIVE
  20         .Lannotate_\@:
  21         .pushsection .discard.nospec
  22         .long .Lannotate_\@ - .
  23         .popsection
  24 .endm
  25
  26 /*
  27  * These are the bare retpoline primitives for indirect jmp and call.
  28  * Do not use these directly; they only exist to make the ALTERNATIVE
  29  * invocation below less ugly.
  30  */
  31 .macro RETPOLINE_JMP reg:req
  32         call    .Ldo_rop_\@
  33 .Lspec_trap_\@:
  34         pause
  35         lfence
  36         jmp     .Lspec_trap_\@
  37 .Ldo_rop_\@:
  38         mov     \reg, (%_ASM_SP)
  39         ret
  40 .endm
  41
  42 /*
  43  * This is a wrapper around RETPOLINE_JMP so the called function in reg
  44  * returns to the instruction after the macro.
  45  */
  46 .macro RETPOLINE_CALL reg:req
  47         jmp     .Ldo_call_\@
  48 .Ldo_retpoline_jmp_\@:
  49         RETPOLINE_JMP \reg
  50 .Ldo_call_\@:
  51         call    .Ldo_retpoline_jmp_\@
  52 .endm
  53
  54 /*
  55  * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
  56  * indirect jmp/call which may be susceptible to the Spectre variant 2
  57  * attack.
  58  */
  59 .macro JMP_NOSPEC reg:req
  60 #ifdef CONFIG_RETPOLINE
  61         ANNOTATE_NOSPEC_ALTERNATIVE
  62         ALTERNATIVE_2 __stringify(jmp *\reg),                           \
  63                 __stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE, \
  64                 __stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
  65 #else
  66         jmp     *\reg
  67 #endif
  68 .endm
  69
  70 .macro CALL_NOSPEC reg:req
  71 #ifdef CONFIG_RETPOLINE
  72         ANNOTATE_NOSPEC_ALTERNATIVE
  73         ALTERNATIVE_2 __stringify(call *\reg),                          \
  74                 __stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
  75                 __stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
  76 #else
  77         call    *\reg
  78 #endif
  79 .endm
  80
  81 /* This clobbers the BX register */
  82 .macro FILL_RETURN_BUFFER nr:req ftr:req
  83 #ifdef CONFIG_RETPOLINE
  84         ALTERNATIVE "", "call __clear_rsb", \ftr
  85 #endif
  86 .endm
  87
  88 #else /* __ASSEMBLY__ */
  89
  90 #define ANNOTATE_NOSPEC_ALTERNATIVE                             \
  91         "999:\n\t"                                              \
  92         ".pushsection .discard.nospec\n\t"                      \
  93         ".long 999b - .\n\t"                                    \
  94         ".popsection\n\t"
  95
  96 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
  97
  98 /*
  99  * Since the inline asm uses the %V modifier which is only in newer GCC,
 100  * the 64-bit one is dependent on RETPOLINE not CONFIG_RETPOLINE.
 101  */
 102 # define CALL_NOSPEC                                            \
 103         ANNOTATE_NOSPEC_ALTERNATIVE                             \
 104         ALTERNATIVE(                                            \
 105         "call *%[thunk_target]\n",                              \
 106         "call __x86_indirect_thunk_%V[thunk_target]\n",         \
 107         X86_FEATURE_RETPOLINE)
 108 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
 109
 110 #elif defined(CONFIG_X86_32) && defined(CONFIG_RETPOLINE)
 111 /*
 112  * For i386 we use the original ret-equivalent retpoline, because
 113  * otherwise we'll run out of registers. We don't care about CET
 114  * here, anyway.
 115  */
 116 # define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",     \
 117         "       jmp    904f;\n"                                 \
 118         "       .align 16\n"                                    \
 119         "901:   call   903f;\n"                                 \
 120         "902:   pause;\n"                                       \
 121         "       lfence;\n"                                      \
 122         "       jmp    902b;\n"                                 \
 123         "       .align 16\n"                                    \
 124         "903:   addl   $4, %%esp;\n"                            \
 125         "       pushl  %[thunk_target];\n"                      \
 126         "       ret;\n"                                         \
 127         "       .align 16\n"                                    \
 128         "904:   call   901b;\n",                                \
 129         X86_FEATURE_RETPOLINE)
 130
 131 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 132 #else /* No retpoline for C / inline asm */
 133 # define CALL_NOSPEC "call *%[thunk_target]\n"
 134 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
 135 #endif
 136
 137 /* The Spectre V2 mitigation variants */
 138 enum spectre_v2_mitigation {
 139         SPECTRE_V2_NONE,
 140         SPECTRE_V2_RETPOLINE_MINIMAL,
 141         SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
 142         SPECTRE_V2_RETPOLINE_GENERIC,
 143         SPECTRE_V2_RETPOLINE_AMD,
 144         SPECTRE_V2_IBRS,
 145 };
 146
 147 extern char __indirect_thunk_start[];
 148 extern char __indirect_thunk_end[];
 149
 150 /*
 151  * On VMEXIT we must ensure that no RSB predictions learned in the guest
 152  * can be followed in the host, by overwriting the RSB completely. Both
 153  * retpoline and IBRS mitigations for Spectre v2 need this; only on future
 154  * CPUs with IBRS_ALL *might* it be avoided.
 155  */
 156 static inline void vmexit_fill_RSB(void)
 157 {
 158 #ifdef CONFIG_RETPOLINE
 159         alternative_input("",
 160                           "call __fill_rsb",
 161                           X86_FEATURE_RETPOLINE,
 162                           ASM_NO_INPUT_CLOBBER(_ASM_BX, "memory"));
 163 #endif
 164 }
 165
 166 static inline void indirect_branch_prediction_barrier(void)
 167 {
 168         asm volatile(ALTERNATIVE("",
 169                                  "movl %[msr], %%ecx\n\t"
 170                                  "movl %[val], %%eax\n\t"
 171                                  "movl $0, %%edx\n\t"
 172                                  "wrmsr",
 173                                  X86_FEATURE_USE_IBPB)
 174                      : : [msr] "i" (MSR_IA32_PRED_CMD),
 175                          [val] "i" (PRED_CMD_IBPB)
 176                      : "eax", "ecx", "edx", "memory");
 177 }
 178
 179 #endif /* __ASSEMBLY__ */
 180 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */