Commit | Line | Data |
---|---|---|
76b04384 DW |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | ||
3 | #include <linux/stringify.h> | |
4 | #include <linux/linkage.h> | |
5 | #include <asm/dwarf2.h> | |
6 | #include <asm/cpufeatures.h> | |
5e21a3ec | 7 | #include <asm/alternative.h> |
5d821386 | 8 | #include <asm/asm-offsets.h> |
76b04384 DW |
9 | #include <asm/export.h> |
10 | #include <asm/nospec-branch.h> | |
cc1ac9c7 | 11 | #include <asm/unwind_hints.h> |
5d821386 | 12 | #include <asm/percpu.h> |
cc1ac9c7 | 13 | #include <asm/frame.h> |
76b04384 | 14 | |
9bc0bb50 PZ |
15 | .section .text.__x86.indirect_thunk |
16 | ||
3b6c1747 PZ |
17 | |
18 | .macro POLINE reg | |
cc1ac9c7 | 19 | ANNOTATE_INTRA_FUNCTION_CALL |
11925185 | 20 | call .Ldo_rop_\@ |
3b6c1747 | 21 | int3 |
cc1ac9c7 | 22 | .Ldo_rop_\@: |
11925185 | 23 | mov %\reg, (%_ASM_SP) |
b735bd3e | 24 | UNWIND_HINT_FUNC |
3b6c1747 PZ |
25 | .endm |
26 | ||
27 | .macro RETPOLINE reg | |
28 | POLINE \reg | |
f94909ce | 29 | RET |
11925185 PZ |
30 | .endm |
31 | ||
32 | .macro THUNK reg | |
11925185 | 33 | |
1a6f7442 PZ |
34 | .align RETPOLINE_THUNK_SIZE |
35 | SYM_INNER_LABEL(__x86_indirect_thunk_\reg, SYM_L_GLOBAL) | |
fb799447 | 36 | UNWIND_HINT_UNDEFINED |
1c0513de | 37 | ANNOTATE_NOENDBR |
11925185 | 38 | |
00e15333 PZ |
39 | ALTERNATIVE_2 __stringify(RETPOLINE \reg), \ |
40 | __stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *%\reg; int3), X86_FEATURE_RETPOLINE_LFENCE, \ | |
41 | __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *%\reg), ALT_NOT(X86_FEATURE_RETPOLINE) | |
11925185 | 42 | |
76b04384 DW |
43 | .endm |
44 | ||
45 | /* | |
46 | * Despite being an assembler file we can't just use .irp here | |
47 | * because __KSYM_DEPS__ only uses the C preprocessor and would | |
48 | * only see one instance of "__x86_indirect_thunk_\reg" rather | |
49 | * than one per register with the correct names. So we do it | |
50 | * the simple and nasty way... | |
ca3f0d80 PZ |
51 | * |
52 | * Worse, you can only have a single EXPORT_SYMBOL per line, | |
53 | * and CPP can't insert newlines, so we have to repeat everything | |
54 | * at least twice. | |
76b04384 | 55 | */ |
ca3f0d80 PZ |
56 | |
57 | #define __EXPORT_THUNK(sym) _ASM_NOKPROBE(sym); EXPORT_SYMBOL(sym) | |
ca3f0d80 | 58 | |
1a6f7442 PZ |
59 | .align RETPOLINE_THUNK_SIZE |
60 | SYM_CODE_START(__x86_indirect_thunk_array) | |
61 | ||
ca3f0d80 PZ |
62 | #define GEN(reg) THUNK reg |
63 | #include <asm/GEN-for-each-reg.h> | |
ca3f0d80 | 64 | #undef GEN |
b6d3d994 | 65 | |
1a6f7442 PZ |
66 | .align RETPOLINE_THUNK_SIZE |
67 | SYM_CODE_END(__x86_indirect_thunk_array) | |
68 | ||
3b6c1747 PZ |
69 | #define GEN(reg) __EXPORT_THUNK(__x86_indirect_thunk_ ## reg) |
70 | #include <asm/GEN-for-each-reg.h> | |
71 | #undef GEN | |
72 | ||
73 | #ifdef CONFIG_CALL_DEPTH_TRACKING | |
74 | .macro CALL_THUNK reg | |
75 | .align RETPOLINE_THUNK_SIZE | |
76 | ||
77 | SYM_INNER_LABEL(__x86_indirect_call_thunk_\reg, SYM_L_GLOBAL) | |
fb799447 | 78 | UNWIND_HINT_UNDEFINED |
3b6c1747 PZ |
79 | ANNOTATE_NOENDBR |
80 | ||
81 | CALL_DEPTH_ACCOUNT | |
82 | POLINE \reg | |
83 | ANNOTATE_UNRET_SAFE | |
84 | ret | |
85 | int3 | |
86 | .endm | |
87 | ||
88 | .align RETPOLINE_THUNK_SIZE | |
89 | SYM_CODE_START(__x86_indirect_call_thunk_array) | |
90 | ||
91 | #define GEN(reg) CALL_THUNK reg | |
92 | #include <asm/GEN-for-each-reg.h> | |
93 | #undef GEN | |
94 | ||
95 | .align RETPOLINE_THUNK_SIZE | |
96 | SYM_CODE_END(__x86_indirect_call_thunk_array) | |
97 | ||
98 | #define GEN(reg) __EXPORT_THUNK(__x86_indirect_call_thunk_ ## reg) | |
ca3f0d80 | 99 | #include <asm/GEN-for-each-reg.h> |
b6d3d994 | 100 | #undef GEN |
0b53c374 | 101 | |
3b6c1747 PZ |
102 | .macro JUMP_THUNK reg |
103 | .align RETPOLINE_THUNK_SIZE | |
104 | ||
105 | SYM_INNER_LABEL(__x86_indirect_jump_thunk_\reg, SYM_L_GLOBAL) | |
fb799447 | 106 | UNWIND_HINT_UNDEFINED |
3b6c1747 PZ |
107 | ANNOTATE_NOENDBR |
108 | POLINE \reg | |
109 | ANNOTATE_UNRET_SAFE | |
110 | ret | |
111 | int3 | |
112 | .endm | |
113 | ||
114 | .align RETPOLINE_THUNK_SIZE | |
115 | SYM_CODE_START(__x86_indirect_jump_thunk_array) | |
116 | ||
117 | #define GEN(reg) JUMP_THUNK reg | |
118 | #include <asm/GEN-for-each-reg.h> | |
119 | #undef GEN | |
120 | ||
121 | .align RETPOLINE_THUNK_SIZE | |
122 | SYM_CODE_END(__x86_indirect_jump_thunk_array) | |
123 | ||
124 | #define GEN(reg) __EXPORT_THUNK(__x86_indirect_jump_thunk_ ## reg) | |
125 | #include <asm/GEN-for-each-reg.h> | |
126 | #undef GEN | |
127 | #endif | |
0b53c374 PZ |
128 | /* |
129 | * This function name is magical and is used by -mfunction-return=thunk-extern | |
130 | * for the compiler to generate JMPs to it. | |
131 | */ | |
f43b9876 PZ |
132 | #ifdef CONFIG_RETHUNK |
133 | ||
a149180f PZ |
134 | .section .text.__x86.return_thunk |
135 | ||
136 | /* | |
137 | * Safety details here pertain to the AMD Zen{1,2} microarchitecture: | |
138 | * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for | |
139 | * alignment within the BTB. | |
140 | * 2) The instruction at zen_untrain_ret must contain, and not | |
141 | * end with, the 0xc3 byte of the RET. | |
142 | * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread | |
143 | * from re-poisioning the BTB prediction. | |
144 | */ | |
145 | .align 64 | |
f220125b | 146 | .skip 64 - (__x86_return_thunk - zen_untrain_ret), 0xcc |
9a48d604 BPA |
147 | SYM_START(zen_untrain_ret, SYM_L_GLOBAL, SYM_A_NONE) |
148 | ANNOTATE_NOENDBR | |
a149180f PZ |
149 | /* |
150 | * As executed from zen_untrain_ret, this is: | |
151 | * | |
152 | * TEST $0xcc, %bl | |
153 | * LFENCE | |
154 | * JMP __x86_return_thunk | |
155 | * | |
156 | * Executing the TEST instruction has a side effect of evicting any BTB | |
157 | * prediction (potentially attacker controlled) attached to the RET, as | |
158 | * __x86_return_thunk + 1 isn't an instruction boundary at the moment. | |
159 | */ | |
160 | .byte 0xf6 | |
161 | ||
162 | /* | |
163 | * As executed from __x86_return_thunk, this is a plain RET. | |
164 | * | |
165 | * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8. | |
166 | * | |
167 | * We subsequently jump backwards and architecturally execute the RET. | |
168 | * This creates a correct BTB prediction (type=ret), but in the | |
169 | * meantime we suffer Straight Line Speculation (because the type was | |
170 | * no branch) which is halted by the INT3. | |
171 | * | |
172 | * With SMT enabled and STIBP active, a sibling thread cannot poison | |
173 | * RET's prediction to a type of its choice, but can evict the | |
174 | * prediction due to competitive sharing. If the prediction is | |
175 | * evicted, __x86_return_thunk will suffer Straight Line Speculation | |
176 | * which will be contained safely by the INT3. | |
177 | */ | |
178 | SYM_INNER_LABEL(__x86_return_thunk, SYM_L_GLOBAL) | |
0b53c374 PZ |
179 | ret |
180 | int3 | |
181 | SYM_CODE_END(__x86_return_thunk) | |
182 | ||
a149180f PZ |
183 | /* |
184 | * Ensure the TEST decoding / BTB invalidation is complete. | |
185 | */ | |
186 | lfence | |
187 | ||
188 | /* | |
189 | * Jump back and execute the RET in the middle of the TEST instruction. | |
190 | * INT3 is for SLS protection. | |
191 | */ | |
192 | jmp __x86_return_thunk | |
193 | int3 | |
194 | SYM_FUNC_END(zen_untrain_ret) | |
195 | __EXPORT_THUNK(zen_untrain_ret) | |
196 | ||
197 | EXPORT_SYMBOL(__x86_return_thunk) | |
f43b9876 PZ |
198 | |
199 | #endif /* CONFIG_RETHUNK */ | |
5d821386 TG |
200 | |
201 | #ifdef CONFIG_CALL_DEPTH_TRACKING | |
202 | ||
203 | .align 64 | |
204 | SYM_FUNC_START(__x86_return_skl) | |
205 | ANNOTATE_NOENDBR | |
f5c1bb2a TG |
206 | /* |
207 | * Keep the hotpath in a 16byte I-fetch for the non-debug | |
208 | * case. | |
209 | */ | |
210 | CALL_THUNKS_DEBUG_INC_RETS | |
5d821386 TG |
211 | shlq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth) |
212 | jz 1f | |
213 | ANNOTATE_UNRET_SAFE | |
214 | ret | |
215 | int3 | |
216 | 1: | |
f5c1bb2a | 217 | CALL_THUNKS_DEBUG_INC_STUFFS |
5d821386 TG |
218 | .rept 16 |
219 | ANNOTATE_INTRA_FUNCTION_CALL | |
220 | call 2f | |
221 | int3 | |
222 | 2: | |
223 | .endr | |
224 | add $(8*16), %rsp | |
225 | ||
226 | CREDIT_CALL_DEPTH | |
227 | ||
228 | ANNOTATE_UNRET_SAFE | |
229 | ret | |
230 | int3 | |
231 | SYM_FUNC_END(__x86_return_skl) | |
232 | ||
233 | #endif /* CONFIG_CALL_DEPTH_TRACKING */ |