Commit | Line | Data |
---|---|---|
e6d6c071 JP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/static_call.h> | |
3 | #include <linux/memory.h> | |
4 | #include <linux/bug.h> | |
5 | #include <asm/text-patching.h> | |
6 | ||
452cddbf PZ |
7 | enum insn_type { |
8 | CALL = 0, /* site call */ | |
9 | NOP = 1, /* site cond-call */ | |
10 | JMP = 2, /* tramp / site tail-call */ | |
11 | RET = 3, /* tramp / site cond-tail-call */ | |
923510c8 | 12 | JCC = 4, |
452cddbf PZ |
13 | }; |
14 | ||
ee88d363 PZ |
15 | /* |
16 | * ud1 %esp, %ecx - a 3 byte #UD that is unique to trampolines, chosen such | |
17 | * that there is no false-positive trampoline identification while also being a | |
18 | * speculation stop. | |
19 | */ | |
20 | static const u8 tramp_ud[] = { 0x0f, 0xb9, 0xcc }; | |
21 | ||
3f2a8fc4 | 22 | /* |
1cd5f059 | 23 | * cs cs cs xorl %eax, %eax - a single 5 byte instruction that clears %[er]ax |
3f2a8fc4 | 24 | */ |
1cd5f059 | 25 | static const u8 xor5rax[] = { 0x2e, 0x2e, 0x2e, 0x31, 0xc0 }; |
3f2a8fc4 | 26 | |
e463a09a PZ |
27 | static const u8 retinsn[] = { RET_INSN_OPCODE, 0xcc, 0xcc, 0xcc, 0xcc }; |
28 | ||
923510c8 PZ |
29 | static u8 __is_Jcc(u8 *insn) /* Jcc.d32 */ |
30 | { | |
31 | u8 ret = 0; | |
32 | ||
33 | if (insn[0] == 0x0f) { | |
34 | u8 tmp = insn[1]; | |
35 | if ((tmp & 0xf0) == 0x80) | |
36 | ret = tmp; | |
37 | } | |
38 | ||
39 | return ret; | |
40 | } | |
41 | ||
42 | extern void __static_call_return(void); | |
43 | ||
44 | asm (".global __static_call_return\n\t" | |
45 | ".type __static_call_return, @function\n\t" | |
46 | ASM_FUNC_ALIGN "\n\t" | |
47 | "__static_call_return:\n\t" | |
48 | ANNOTATE_NOENDBR | |
49 | ANNOTATE_RETPOLINE_SAFE | |
50 | "ret; int3\n\t" | |
51 | ".size __static_call_return, . - __static_call_return \n\t"); | |
52 | ||
c27c753e TG |
53 | static void __ref __static_call_transform(void *insn, enum insn_type type, |
54 | void *func, bool modinit) | |
e6d6c071 | 55 | { |
3f2a8fc4 | 56 | const void *emulate = NULL; |
452cddbf PZ |
57 | int size = CALL_INSN_SIZE; |
58 | const void *code; | |
923510c8 PZ |
59 | u8 op, buf[6]; |
60 | ||
61 | if ((type == JMP || type == RET) && (op = __is_Jcc(insn))) | |
62 | type = JCC; | |
e6d6c071 | 63 | |
452cddbf PZ |
64 | switch (type) { |
65 | case CALL: | |
7825451f | 66 | func = callthunks_translate_call_dest(func); |
452cddbf | 67 | code = text_gen_insn(CALL_INSN_OPCODE, insn, func); |
3f2a8fc4 PZ |
68 | if (func == &__static_call_return0) { |
69 | emulate = code; | |
70 | code = &xor5rax; | |
71 | } | |
72 | ||
452cddbf PZ |
73 | break; |
74 | ||
75 | case NOP: | |
a89dfde3 | 76 | code = x86_nops[5]; |
452cddbf PZ |
77 | break; |
78 | ||
79 | case JMP: | |
80 | code = text_gen_insn(JMP32_INSN_OPCODE, insn, func); | |
81 | break; | |
82 | ||
83 | case RET: | |
ee88d363 | 84 | if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) |
770ae1b7 | 85 | code = text_gen_insn(JMP32_INSN_OPCODE, insn, x86_return_thunk); |
ee88d363 PZ |
86 | else |
87 | code = &retinsn; | |
452cddbf | 88 | break; |
923510c8 PZ |
89 | |
90 | case JCC: | |
91 | if (!func) { | |
92 | func = __static_call_return; | |
93 | if (cpu_feature_enabled(X86_FEATURE_RETHUNK)) | |
94 | func = x86_return_thunk; | |
95 | } | |
96 | ||
97 | buf[0] = 0x0f; | |
98 | __text_gen_insn(buf+1, op, insn+1, func, 5); | |
99 | code = buf; | |
100 | size = 6; | |
101 | ||
102 | break; | |
452cddbf | 103 | } |
e6d6c071 | 104 | |
452cddbf | 105 | if (memcmp(insn, code, size) == 0) |
e6d6c071 JP |
106 | return; |
107 | ||
c27c753e | 108 | if (system_state == SYSTEM_BOOTING || modinit) |
a945c834 PZ |
109 | return text_poke_early(insn, code, size); |
110 | ||
3f2a8fc4 | 111 | text_poke_bp(insn, code, size, emulate); |
e6d6c071 JP |
112 | } |
113 | ||
923510c8 | 114 | static void __static_call_validate(u8 *insn, bool tail, bool tramp) |
6c3fce79 | 115 | { |
923510c8 | 116 | u8 opcode = insn[0]; |
6c3fce79 | 117 | |
ee88d363 | 118 | if (tramp && memcmp(insn+5, tramp_ud, 3)) { |
2105a927 PZ |
119 | pr_err("trampoline signature fail"); |
120 | BUG(); | |
121 | } | |
122 | ||
6c3fce79 PZ |
123 | if (tail) { |
124 | if (opcode == JMP32_INSN_OPCODE || | |
923510c8 PZ |
125 | opcode == RET_INSN_OPCODE || |
126 | __is_Jcc(insn)) | |
6c3fce79 PZ |
127 | return; |
128 | } else { | |
129 | if (opcode == CALL_INSN_OPCODE || | |
a89dfde3 | 130 | !memcmp(insn, x86_nops[5], 5) || |
3f2a8fc4 | 131 | !memcmp(insn, xor5rax, 5)) |
6c3fce79 PZ |
132 | return; |
133 | } | |
134 | ||
135 | /* | |
136 | * If we ever trigger this, our text is corrupt, we'll probably not live long. | |
137 | */ | |
2105a927 PZ |
138 | pr_err("unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn); |
139 | BUG(); | |
6c3fce79 PZ |
140 | } |
141 | ||
5b06fd3b PZ |
142 | static inline enum insn_type __sc_insn(bool null, bool tail) |
143 | { | |
144 | /* | |
145 | * Encode the following table without branches: | |
146 | * | |
147 | * tail null insn | |
148 | * -----+-------+------ | |
149 | * 0 | 0 | CALL | |
150 | * 0 | 1 | NOP | |
151 | * 1 | 0 | JMP | |
152 | * 1 | 1 | RET | |
153 | */ | |
154 | return 2*tail + null; | |
155 | } | |
156 | ||
157 | void arch_static_call_transform(void *site, void *tramp, void *func, bool tail) | |
e6d6c071 JP |
158 | { |
159 | mutex_lock(&text_mutex); | |
160 | ||
6c3fce79 | 161 | if (tramp) { |
2105a927 | 162 | __static_call_validate(tramp, true, true); |
c27c753e | 163 | __static_call_transform(tramp, __sc_insn(!func, true), func, false); |
6c3fce79 | 164 | } |
e6d6c071 | 165 | |
6c3fce79 | 166 | if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site) { |
2105a927 | 167 | __static_call_validate(site, tail, false); |
c27c753e | 168 | __static_call_transform(site, __sc_insn(!func, tail), func, false); |
6c3fce79 | 169 | } |
1e7e4788 | 170 | |
e6d6c071 JP |
171 | mutex_unlock(&text_mutex); |
172 | } | |
173 | EXPORT_SYMBOL_GPL(arch_static_call_transform); | |
ee88d363 | 174 | |
f43b9876 | 175 | #ifdef CONFIG_RETHUNK |
ee88d363 PZ |
176 | /* |
177 | * This is called by apply_returns() to fix up static call trampolines, | |
178 | * specifically ARCH_DEFINE_STATIC_CALL_NULL_TRAMP which is recorded as | |
179 | * having a return trampoline. | |
180 | * | |
181 | * The problem is that static_call() is available before determining | |
182 | * X86_FEATURE_RETHUNK and, by implication, running alternatives. | |
183 | * | |
184 | * This means that __static_call_transform() above can have overwritten the | |
185 | * return trampoline and we now need to fix things up to be consistent. | |
186 | */ | |
187 | bool __static_call_fixup(void *tramp, u8 op, void *dest) | |
188 | { | |
189 | if (memcmp(tramp+5, tramp_ud, 3)) { | |
190 | /* Not a trampoline site, not our problem. */ | |
191 | return false; | |
192 | } | |
193 | ||
c27c753e | 194 | mutex_lock(&text_mutex); |
ee88d363 | 195 | if (op == RET_INSN_OPCODE || dest == &__x86_return_thunk) |
c27c753e TG |
196 | __static_call_transform(tramp, RET, NULL, true); |
197 | mutex_unlock(&text_mutex); | |
ee88d363 PZ |
198 | |
199 | return true; | |
200 | } | |
201 | #endif |