Commit | Line | Data |
---|---|---|
115284d8 JP |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _LINUX_STATIC_CALL_H | |
3 | #define _LINUX_STATIC_CALL_H | |
4 | ||
5 | /* | |
6 | * Static call support | |
7 | * | |
8 | * Static calls use code patching to hard-code function pointers into direct | |
9 | * branch instructions. They give the flexibility of function pointers, but | |
10 | * with improved performance. This is especially important for cases where | |
11 | * retpolines would otherwise be used, as retpolines can significantly impact | |
12 | * performance. | |
13 | * | |
14 | * | |
15 | * API overview: | |
16 | * | |
17 | * DECLARE_STATIC_CALL(name, func); | |
18 | * DEFINE_STATIC_CALL(name, func); | |
452cddbf | 19 | * DEFINE_STATIC_CALL_NULL(name, typename); |
9ae6ab27 PZ |
20 | * DEFINE_STATIC_CALL_RET0(name, typename); |
21 | * | |
22 | * __static_call_return0; | |
23 | * | |
115284d8 | 24 | * static_call(name)(args...); |
452cddbf | 25 | * static_call_cond(name)(args...); |
115284d8 | 26 | * static_call_update(name, func); |
6ea312d9 | 27 | * static_call_query(name); |
115284d8 | 28 | * |
9ae6ab27 PZ |
29 | * EXPORT_STATIC_CALL{,_TRAMP}{,_GPL}() |
30 | * | |
115284d8 JP |
31 | * Usage example: |
32 | * | |
33 | * # Start with the following functions (with identical prototypes): | |
34 | * int func_a(int arg1, int arg2); | |
35 | * int func_b(int arg1, int arg2); | |
36 | * | |
37 | * # Define a 'my_name' reference, associated with func_a() by default | |
38 | * DEFINE_STATIC_CALL(my_name, func_a); | |
39 | * | |
40 | * # Call func_a() | |
41 | * static_call(my_name)(arg1, arg2); | |
42 | * | |
43 | * # Update 'my_name' to point to func_b() | |
44 | * static_call_update(my_name, &func_b); | |
45 | * | |
46 | * # Call func_b() | |
47 | * static_call(my_name)(arg1, arg2); | |
48 | * | |
49 | * | |
50 | * Implementation details: | |
51 | * | |
52 | * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). | |
53 | * Otherwise basic indirect calls are used (with function pointers). | |
54 | * | |
55 | * Each static_call() site calls into a trampoline associated with the name. | |
56 | * The trampoline has a direct branch to the default function. Updates to a | |
57 | * name will modify the trampoline's branch destination. | |
58 | * | |
59 | * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites | |
60 | * themselves will be patched at runtime to call the functions directly, | |
61 | * rather than calling through the trampoline. This requires objtool or a | |
62 | * compiler plugin to detect all the static_call() sites and annotate them | |
63 | * in the .static_call_sites section. | |
452cddbf PZ |
64 | * |
65 | * | |
66 | * Notes on NULL function pointers: | |
67 | * | |
68 | * Static_call()s support NULL functions, with many of the caveats that | |
69 | * regular function pointers have. | |
70 | * | |
71 | * Clearly calling a NULL function pointer is 'BAD', so too for | |
72 | * static_call()s (although when HAVE_STATIC_CALL it might not be immediately | |
73 | * fatal). A NULL static_call can be the result of: | |
74 | * | |
75 | * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int)); | |
76 | * | |
77 | * which is equivalent to declaring a NULL function pointer with just a | |
78 | * typename: | |
79 | * | |
80 | * void (*my_func_ptr)(int arg1) = NULL; | |
81 | * | |
82 | * or using static_call_update() with a NULL function. In both cases the | |
83 | * HAVE_STATIC_CALL implementation will patch the trampoline with a RET | |
84 | * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE | |
85 | * architectures can patch the trampoline call to a NOP. | |
86 | * | |
87 | * In all cases, any argument evaluation is unconditional. Unlike a regular | |
88 | * conditional function pointer call: | |
89 | * | |
90 | * if (my_func_ptr) | |
91 | * my_func_ptr(arg1) | |
92 | * | |
93 | * where the argument evaludation also depends on the pointer value. | |
94 | * | |
95 | * When calling a static_call that can be NULL, use: | |
96 | * | |
97 | * static_call_cond(name)(arg1); | |
98 | * | |
99 | * which will include the required value tests to avoid NULL-pointer | |
100 | * dereferences. | |
6ea312d9 JG |
101 | * |
102 | * To query which function is currently set to be called, use: | |
103 | * | |
104 | * func = static_call_query(name); | |
9ae6ab27 PZ |
105 | * |
106 | * | |
107 | * DEFINE_STATIC_CALL_RET0 / __static_call_return0: | |
108 | * | |
109 | * Just like how DEFINE_STATIC_CALL_NULL() / static_call_cond() optimize the | |
110 | * conditional void function call, DEFINE_STATIC_CALL_RET0 / | |
111 | * __static_call_return0 optimize the do nothing return 0 function. | |
112 | * | |
113 | * This feature is strictly UB per the C standard (since it casts a function | |
114 | * pointer to a different signature) and relies on the architecture ABI to | |
115 | * make things work. In particular it relies on Caller Stack-cleanup and the | |
116 | * whole return register being clobbered for short return values. All normal | |
117 | * CDECL style ABIs conform. | |
118 | * | |
119 | * In particular the x86_64 implementation replaces the 5 byte CALL | |
120 | * instruction at the callsite with a 5 byte clear of the RAX register, | |
121 | * completely eliding any function call overhead. | |
122 | * | |
123 | * Notably argument setup is unconditional. | |
124 | * | |
125 | * | |
126 | * EXPORT_STATIC_CALL() vs EXPORT_STATIC_CALL_TRAMP(): | |
127 | * | |
128 | * The difference is that the _TRAMP variant tries to only export the | |
129 | * trampoline with the result that a module can use static_call{,_cond}() but | |
130 | * not static_call_update(). | |
131 | * | |
115284d8 JP |
132 | */ |
133 | ||
134 | #include <linux/types.h> | |
135 | #include <linux/cpu.h> | |
136 | #include <linux/static_call_types.h> | |
137 | ||
138 | #ifdef CONFIG_HAVE_STATIC_CALL | |
139 | #include <asm/static_call.h> | |
140 | ||
141 | /* | |
142 | * Either @site or @tramp can be NULL. | |
143 | */ | |
5b06fd3b | 144 | extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail); |
115284d8 JP |
145 | |
146 | #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) | |
147 | ||
115284d8 JP |
148 | #else |
149 | #define STATIC_CALL_TRAMP_ADDR(name) NULL | |
150 | #endif | |
151 | ||
115284d8 JP |
152 | #define static_call_update(name, func) \ |
153 | ({ \ | |
9432bbd9 | 154 | typeof(&STATIC_CALL_TRAMP(name)) __F = (func); \ |
115284d8 | 155 | __static_call_update(&STATIC_CALL_KEY(name), \ |
9432bbd9 | 156 | STATIC_CALL_TRAMP_ADDR(name), __F); \ |
115284d8 JP |
157 | }) |
158 | ||
6ea312d9 JG |
159 | #define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func)) |
160 | ||
9183c3f9 JP |
161 | #ifdef CONFIG_HAVE_STATIC_CALL_INLINE |
162 | ||
69e0ad37 | 163 | extern int __init static_call_init(void); |
a945c834 | 164 | |
9183c3f9 JP |
165 | struct static_call_mod { |
166 | struct static_call_mod *next; | |
167 | struct module *mod; /* for vmlinux, mod == NULL */ | |
168 | struct static_call_site *sites; | |
169 | }; | |
170 | ||
73f44fe1 JP |
171 | /* For finding the key associated with a trampoline */ |
172 | struct static_call_tramp_key { | |
173 | s32 tramp; | |
174 | s32 key; | |
175 | }; | |
176 | ||
9183c3f9 JP |
177 | extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); |
178 | extern int static_call_mod_init(struct module *mod); | |
6333e8f7 | 179 | extern int static_call_text_reserved(void *start, void *end); |
9183c3f9 | 180 | |
3f2a8fc4 PZ |
181 | extern long __static_call_return0(void); |
182 | ||
29fd0194 | 183 | #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ |
9183c3f9 JP |
184 | DECLARE_STATIC_CALL(name, _func); \ |
185 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
29fd0194 | 186 | .func = _func_init, \ |
a945c834 | 187 | .type = 1, \ |
9183c3f9 | 188 | }; \ |
29fd0194 | 189 | ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init) |
9183c3f9 | 190 | |
452cddbf PZ |
191 | #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
192 | DECLARE_STATIC_CALL(name, _func); \ | |
193 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
194 | .func = NULL, \ | |
195 | .type = 1, \ | |
196 | }; \ | |
197 | ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) | |
198 | ||
452cddbf | 199 | #define static_call_cond(name) (void)__static_call(name) |
9183c3f9 JP |
200 | |
201 | #define EXPORT_STATIC_CALL(name) \ | |
202 | EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ | |
203 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) | |
9183c3f9 JP |
204 | #define EXPORT_STATIC_CALL_GPL(name) \ |
205 | EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ | |
206 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) | |
207 | ||
73f44fe1 JP |
208 | /* Leave the key unexported, so modules can't change static call targets: */ |
209 | #define EXPORT_STATIC_CALL_TRAMP(name) \ | |
210 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \ | |
211 | ARCH_ADD_TRAMP_KEY(name) | |
212 | #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ | |
213 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \ | |
214 | ARCH_ADD_TRAMP_KEY(name) | |
215 | ||
9183c3f9 | 216 | #elif defined(CONFIG_HAVE_STATIC_CALL) |
115284d8 | 217 | |
69e0ad37 | 218 | static inline int static_call_init(void) { return 0; } |
a945c834 | 219 | |
29fd0194 | 220 | #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ |
115284d8 JP |
221 | DECLARE_STATIC_CALL(name, _func); \ |
222 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
29fd0194 | 223 | .func = _func_init, \ |
115284d8 | 224 | }; \ |
29fd0194 | 225 | ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init) |
115284d8 | 226 | |
452cddbf PZ |
227 | #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
228 | DECLARE_STATIC_CALL(name, _func); \ | |
229 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
230 | .func = NULL, \ | |
231 | }; \ | |
232 | ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) | |
233 | ||
6ea312d9 | 234 | |
452cddbf | 235 | #define static_call_cond(name) (void)__static_call(name) |
115284d8 JP |
236 | |
237 | static inline | |
238 | void __static_call_update(struct static_call_key *key, void *tramp, void *func) | |
239 | { | |
240 | cpus_read_lock(); | |
241 | WRITE_ONCE(key->func, func); | |
5b06fd3b | 242 | arch_static_call_transform(NULL, tramp, func, false); |
115284d8 JP |
243 | cpus_read_unlock(); |
244 | } | |
245 | ||
6333e8f7 PZ |
246 | static inline int static_call_text_reserved(void *start, void *end) |
247 | { | |
248 | return 0; | |
249 | } | |
250 | ||
3f2a8fc4 PZ |
251 | static inline long __static_call_return0(void) |
252 | { | |
253 | return 0; | |
254 | } | |
255 | ||
115284d8 JP |
256 | #define EXPORT_STATIC_CALL(name) \ |
257 | EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ | |
258 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) | |
115284d8 JP |
259 | #define EXPORT_STATIC_CALL_GPL(name) \ |
260 | EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ | |
261 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) | |
262 | ||
73f44fe1 JP |
263 | /* Leave the key unexported, so modules can't change static call targets: */ |
264 | #define EXPORT_STATIC_CALL_TRAMP(name) \ | |
265 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) | |
266 | #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ | |
267 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) | |
268 | ||
115284d8 JP |
269 | #else /* Generic implementation */ |
270 | ||
69e0ad37 | 271 | static inline int static_call_init(void) { return 0; } |
a945c834 | 272 | |
3f2a8fc4 PZ |
273 | static inline long __static_call_return0(void) |
274 | { | |
275 | return 0; | |
276 | } | |
277 | ||
29fd0194 | 278 | #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ |
115284d8 JP |
279 | DECLARE_STATIC_CALL(name, _func); \ |
280 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
29fd0194 | 281 | .func = _func_init, \ |
115284d8 JP |
282 | } |
283 | ||
452cddbf PZ |
284 | #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
285 | DECLARE_STATIC_CALL(name, _func); \ | |
286 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
287 | .func = NULL, \ | |
288 | } | |
289 | ||
452cddbf PZ |
290 | static inline void __static_call_nop(void) { } |
291 | ||
292 | /* | |
293 | * This horrific hack takes care of two things: | |
294 | * | |
295 | * - it ensures the compiler will only load the function pointer ONCE, | |
296 | * which avoids a reload race. | |
297 | * | |
298 | * - it ensures the argument evaluation is unconditional, similar | |
299 | * to the HAVE_STATIC_CALL variant. | |
300 | * | |
301 | * Sadly current GCC/Clang (10 for both) do not optimize this properly | |
302 | * and will emit an indirect call for the NULL case :-( | |
303 | */ | |
304 | #define __static_call_cond(name) \ | |
305 | ({ \ | |
306 | void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \ | |
307 | if (!func) \ | |
308 | func = &__static_call_nop; \ | |
309 | (typeof(STATIC_CALL_TRAMP(name))*)func; \ | |
310 | }) | |
311 | ||
312 | #define static_call_cond(name) (void)__static_call_cond(name) | |
313 | ||
115284d8 JP |
314 | static inline |
315 | void __static_call_update(struct static_call_key *key, void *tramp, void *func) | |
316 | { | |
317 | WRITE_ONCE(key->func, func); | |
318 | } | |
319 | ||
6333e8f7 PZ |
320 | static inline int static_call_text_reserved(void *start, void *end) |
321 | { | |
322 | return 0; | |
323 | } | |
324 | ||
115284d8 JP |
325 | #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name)) |
326 | #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)) | |
327 | ||
328 | #endif /* CONFIG_HAVE_STATIC_CALL */ | |
329 | ||
29fd0194 FW |
330 | #define DEFINE_STATIC_CALL(name, _func) \ |
331 | __DEFINE_STATIC_CALL(name, _func, _func) | |
332 | ||
333 | #define DEFINE_STATIC_CALL_RET0(name, _func) \ | |
334 | __DEFINE_STATIC_CALL(name, _func, __static_call_return0) | |
335 | ||
115284d8 | 336 | #endif /* _LINUX_STATIC_CALL_H */ |