Commit | Line | Data |
---|---|---|
115284d8 JP |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _LINUX_STATIC_CALL_H | |
3 | #define _LINUX_STATIC_CALL_H | |
4 | ||
5 | /* | |
6 | * Static call support | |
7 | * | |
8 | * Static calls use code patching to hard-code function pointers into direct | |
9 | * branch instructions. They give the flexibility of function pointers, but | |
10 | * with improved performance. This is especially important for cases where | |
11 | * retpolines would otherwise be used, as retpolines can significantly impact | |
12 | * performance. | |
13 | * | |
14 | * | |
15 | * API overview: | |
16 | * | |
17 | * DECLARE_STATIC_CALL(name, func); | |
18 | * DEFINE_STATIC_CALL(name, func); | |
452cddbf | 19 | * DEFINE_STATIC_CALL_NULL(name, typename); |
9ae6ab27 PZ |
20 | * DEFINE_STATIC_CALL_RET0(name, typename); |
21 | * | |
22 | * __static_call_return0; | |
23 | * | |
115284d8 | 24 | * static_call(name)(args...); |
452cddbf | 25 | * static_call_cond(name)(args...); |
115284d8 | 26 | * static_call_update(name, func); |
6ea312d9 | 27 | * static_call_query(name); |
115284d8 | 28 | * |
9ae6ab27 PZ |
29 | * EXPORT_STATIC_CALL{,_TRAMP}{,_GPL}() |
30 | * | |
115284d8 JP |
31 | * Usage example: |
32 | * | |
33 | * # Start with the following functions (with identical prototypes): | |
34 | * int func_a(int arg1, int arg2); | |
35 | * int func_b(int arg1, int arg2); | |
36 | * | |
37 | * # Define a 'my_name' reference, associated with func_a() by default | |
38 | * DEFINE_STATIC_CALL(my_name, func_a); | |
39 | * | |
40 | * # Call func_a() | |
41 | * static_call(my_name)(arg1, arg2); | |
42 | * | |
43 | * # Update 'my_name' to point to func_b() | |
44 | * static_call_update(my_name, &func_b); | |
45 | * | |
46 | * # Call func_b() | |
47 | * static_call(my_name)(arg1, arg2); | |
48 | * | |
49 | * | |
50 | * Implementation details: | |
51 | * | |
52 | * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). | |
53 | * Otherwise basic indirect calls are used (with function pointers). | |
54 | * | |
55 | * Each static_call() site calls into a trampoline associated with the name. | |
56 | * The trampoline has a direct branch to the default function. Updates to a | |
57 | * name will modify the trampoline's branch destination. | |
58 | * | |
59 | * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites | |
60 | * themselves will be patched at runtime to call the functions directly, | |
61 | * rather than calling through the trampoline. This requires objtool or a | |
62 | * compiler plugin to detect all the static_call() sites and annotate them | |
63 | * in the .static_call_sites section. | |
452cddbf PZ |
64 | * |
65 | * | |
66 | * Notes on NULL function pointers: | |
67 | * | |
68 | * Static_call()s support NULL functions, with many of the caveats that | |
69 | * regular function pointers have. | |
70 | * | |
71 | * Clearly calling a NULL function pointer is 'BAD', so too for | |
72 | * static_call()s (although when HAVE_STATIC_CALL it might not be immediately | |
73 | * fatal). A NULL static_call can be the result of: | |
74 | * | |
75 | * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int)); | |
76 | * | |
77 | * which is equivalent to declaring a NULL function pointer with just a | |
78 | * typename: | |
79 | * | |
80 | * void (*my_func_ptr)(int arg1) = NULL; | |
81 | * | |
82 | * or using static_call_update() with a NULL function. In both cases the | |
83 | * HAVE_STATIC_CALL implementation will patch the trampoline with a RET | |
84 | * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE | |
85 | * architectures can patch the trampoline call to a NOP. | |
86 | * | |
87 | * In all cases, any argument evaluation is unconditional. Unlike a regular | |
88 | * conditional function pointer call: | |
89 | * | |
90 | * if (my_func_ptr) | |
91 | * my_func_ptr(arg1) | |
92 | * | |
93 | * where the argument evaludation also depends on the pointer value. | |
94 | * | |
95 | * When calling a static_call that can be NULL, use: | |
96 | * | |
97 | * static_call_cond(name)(arg1); | |
98 | * | |
99 | * which will include the required value tests to avoid NULL-pointer | |
100 | * dereferences. | |
6ea312d9 JG |
101 | * |
102 | * To query which function is currently set to be called, use: | |
103 | * | |
104 | * func = static_call_query(name); | |
9ae6ab27 PZ |
105 | * |
106 | * | |
107 | * DEFINE_STATIC_CALL_RET0 / __static_call_return0: | |
108 | * | |
109 | * Just like how DEFINE_STATIC_CALL_NULL() / static_call_cond() optimize the | |
110 | * conditional void function call, DEFINE_STATIC_CALL_RET0 / | |
111 | * __static_call_return0 optimize the do nothing return 0 function. | |
112 | * | |
113 | * This feature is strictly UB per the C standard (since it casts a function | |
114 | * pointer to a different signature) and relies on the architecture ABI to | |
115 | * make things work. In particular it relies on Caller Stack-cleanup and the | |
116 | * whole return register being clobbered for short return values. All normal | |
117 | * CDECL style ABIs conform. | |
118 | * | |
119 | * In particular the x86_64 implementation replaces the 5 byte CALL | |
120 | * instruction at the callsite with a 5 byte clear of the RAX register, | |
121 | * completely eliding any function call overhead. | |
122 | * | |
123 | * Notably argument setup is unconditional. | |
124 | * | |
125 | * | |
126 | * EXPORT_STATIC_CALL() vs EXPORT_STATIC_CALL_TRAMP(): | |
127 | * | |
128 | * The difference is that the _TRAMP variant tries to only export the | |
129 | * trampoline with the result that a module can use static_call{,_cond}() but | |
130 | * not static_call_update(). | |
131 | * | |
115284d8 JP |
132 | */ |
133 | ||
134 | #include <linux/types.h> | |
135 | #include <linux/cpu.h> | |
136 | #include <linux/static_call_types.h> | |
137 | ||
138 | #ifdef CONFIG_HAVE_STATIC_CALL | |
139 | #include <asm/static_call.h> | |
140 | ||
141 | /* | |
142 | * Either @site or @tramp can be NULL. | |
143 | */ | |
5b06fd3b | 144 | extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail); |
115284d8 JP |
145 | |
146 | #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) | |
147 | ||
115284d8 JP |
148 | #else |
149 | #define STATIC_CALL_TRAMP_ADDR(name) NULL | |
150 | #endif | |
151 | ||
115284d8 JP |
152 | #define static_call_update(name, func) \ |
153 | ({ \ | |
9432bbd9 | 154 | typeof(&STATIC_CALL_TRAMP(name)) __F = (func); \ |
115284d8 | 155 | __static_call_update(&STATIC_CALL_KEY(name), \ |
9432bbd9 | 156 | STATIC_CALL_TRAMP_ADDR(name), __F); \ |
115284d8 JP |
157 | }) |
158 | ||
6ea312d9 JG |
159 | #define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func)) |
160 | ||
9183c3f9 JP |
161 | #ifdef CONFIG_HAVE_STATIC_CALL_INLINE |
162 | ||
69e0ad37 | 163 | extern int __init static_call_init(void); |
a945c834 | 164 | |
7825451f PZ |
165 | extern void static_call_force_reinit(void); |
166 | ||
9183c3f9 JP |
167 | struct static_call_mod { |
168 | struct static_call_mod *next; | |
169 | struct module *mod; /* for vmlinux, mod == NULL */ | |
170 | struct static_call_site *sites; | |
171 | }; | |
172 | ||
73f44fe1 JP |
173 | /* For finding the key associated with a trampoline */ |
174 | struct static_call_tramp_key { | |
175 | s32 tramp; | |
176 | s32 key; | |
177 | }; | |
178 | ||
9183c3f9 JP |
179 | extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); |
180 | extern int static_call_mod_init(struct module *mod); | |
6333e8f7 | 181 | extern int static_call_text_reserved(void *start, void *end); |
9183c3f9 | 182 | |
3f2a8fc4 PZ |
183 | extern long __static_call_return0(void); |
184 | ||
df21c0d7 | 185 | #define DEFINE_STATIC_CALL(name, _func) \ |
9183c3f9 JP |
186 | DECLARE_STATIC_CALL(name, _func); \ |
187 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
df21c0d7 | 188 | .func = _func, \ |
a945c834 | 189 | .type = 1, \ |
9183c3f9 | 190 | }; \ |
df21c0d7 | 191 | ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) |
9183c3f9 | 192 | |
452cddbf PZ |
193 | #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
194 | DECLARE_STATIC_CALL(name, _func); \ | |
195 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
196 | .func = NULL, \ | |
197 | .type = 1, \ | |
198 | }; \ | |
199 | ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) | |
200 | ||
5517d500 CL |
201 | #define DEFINE_STATIC_CALL_RET0(name, _func) \ |
202 | DECLARE_STATIC_CALL(name, _func); \ | |
203 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
204 | .func = __static_call_return0, \ | |
205 | .type = 1, \ | |
206 | }; \ | |
207 | ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) | |
208 | ||
452cddbf | 209 | #define static_call_cond(name) (void)__static_call(name) |
9183c3f9 JP |
210 | |
211 | #define EXPORT_STATIC_CALL(name) \ | |
212 | EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ | |
213 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) | |
9183c3f9 JP |
214 | #define EXPORT_STATIC_CALL_GPL(name) \ |
215 | EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ | |
216 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) | |
217 | ||
73f44fe1 JP |
218 | /* Leave the key unexported, so modules can't change static call targets: */ |
219 | #define EXPORT_STATIC_CALL_TRAMP(name) \ | |
220 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \ | |
221 | ARCH_ADD_TRAMP_KEY(name) | |
222 | #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ | |
223 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \ | |
224 | ARCH_ADD_TRAMP_KEY(name) | |
225 | ||
9183c3f9 | 226 | #elif defined(CONFIG_HAVE_STATIC_CALL) |
115284d8 | 227 | |
69e0ad37 | 228 | static inline int static_call_init(void) { return 0; } |
a945c834 | 229 | |
df21c0d7 | 230 | #define DEFINE_STATIC_CALL(name, _func) \ |
115284d8 JP |
231 | DECLARE_STATIC_CALL(name, _func); \ |
232 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
df21c0d7 | 233 | .func = _func, \ |
115284d8 | 234 | }; \ |
df21c0d7 | 235 | ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) |
115284d8 | 236 | |
452cddbf PZ |
237 | #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
238 | DECLARE_STATIC_CALL(name, _func); \ | |
239 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
240 | .func = NULL, \ | |
241 | }; \ | |
242 | ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) | |
243 | ||
5517d500 CL |
244 | #define DEFINE_STATIC_CALL_RET0(name, _func) \ |
245 | DECLARE_STATIC_CALL(name, _func); \ | |
246 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
247 | .func = __static_call_return0, \ | |
248 | }; \ | |
249 | ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) | |
6ea312d9 | 250 | |
452cddbf | 251 | #define static_call_cond(name) (void)__static_call(name) |
115284d8 JP |
252 | |
253 | static inline | |
254 | void __static_call_update(struct static_call_key *key, void *tramp, void *func) | |
255 | { | |
256 | cpus_read_lock(); | |
257 | WRITE_ONCE(key->func, func); | |
5b06fd3b | 258 | arch_static_call_transform(NULL, tramp, func, false); |
115284d8 JP |
259 | cpus_read_unlock(); |
260 | } | |
261 | ||
6333e8f7 PZ |
262 | static inline int static_call_text_reserved(void *start, void *end) |
263 | { | |
264 | return 0; | |
265 | } | |
266 | ||
8fd4ddda | 267 | extern long __static_call_return0(void); |
3f2a8fc4 | 268 | |
115284d8 JP |
269 | #define EXPORT_STATIC_CALL(name) \ |
270 | EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ | |
271 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) | |
115284d8 JP |
272 | #define EXPORT_STATIC_CALL_GPL(name) \ |
273 | EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ | |
274 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) | |
275 | ||
73f44fe1 JP |
276 | /* Leave the key unexported, so modules can't change static call targets: */ |
277 | #define EXPORT_STATIC_CALL_TRAMP(name) \ | |
278 | EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) | |
279 | #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ | |
280 | EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) | |
281 | ||
115284d8 JP |
282 | #else /* Generic implementation */ |
283 | ||
69e0ad37 | 284 | static inline int static_call_init(void) { return 0; } |
a945c834 | 285 | |
3f2a8fc4 PZ |
286 | static inline long __static_call_return0(void) |
287 | { | |
288 | return 0; | |
289 | } | |
290 | ||
29fd0194 | 291 | #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ |
115284d8 JP |
292 | DECLARE_STATIC_CALL(name, _func); \ |
293 | struct static_call_key STATIC_CALL_KEY(name) = { \ | |
29fd0194 | 294 | .func = _func_init, \ |
115284d8 JP |
295 | } |
296 | ||
df21c0d7 CL |
297 | #define DEFINE_STATIC_CALL(name, _func) \ |
298 | __DEFINE_STATIC_CALL(name, _func, _func) | |
299 | ||
452cddbf | 300 | #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
df21c0d7 | 301 | __DEFINE_STATIC_CALL(name, _func, NULL) |
452cddbf | 302 | |
5517d500 CL |
303 | #define DEFINE_STATIC_CALL_RET0(name, _func) \ |
304 | __DEFINE_STATIC_CALL(name, _func, __static_call_return0) | |
305 | ||
452cddbf PZ |
306 | static inline void __static_call_nop(void) { } |
307 | ||
308 | /* | |
309 | * This horrific hack takes care of two things: | |
310 | * | |
311 | * - it ensures the compiler will only load the function pointer ONCE, | |
312 | * which avoids a reload race. | |
313 | * | |
314 | * - it ensures the argument evaluation is unconditional, similar | |
315 | * to the HAVE_STATIC_CALL variant. | |
316 | * | |
317 | * Sadly current GCC/Clang (10 for both) do not optimize this properly | |
318 | * and will emit an indirect call for the NULL case :-( | |
319 | */ | |
320 | #define __static_call_cond(name) \ | |
321 | ({ \ | |
322 | void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \ | |
323 | if (!func) \ | |
324 | func = &__static_call_nop; \ | |
325 | (typeof(STATIC_CALL_TRAMP(name))*)func; \ | |
326 | }) | |
327 | ||
328 | #define static_call_cond(name) (void)__static_call_cond(name) | |
329 | ||
115284d8 JP |
330 | static inline |
331 | void __static_call_update(struct static_call_key *key, void *tramp, void *func) | |
332 | { | |
333 | WRITE_ONCE(key->func, func); | |
334 | } | |
335 | ||
6333e8f7 PZ |
336 | static inline int static_call_text_reserved(void *start, void *end) |
337 | { | |
338 | return 0; | |
339 | } | |
340 | ||
115284d8 JP |
341 | #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name)) |
342 | #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)) | |
343 | ||
344 | #endif /* CONFIG_HAVE_STATIC_CALL */ | |
345 | ||
346 | #endif /* _LINUX_STATIC_CALL_H */ |