[linux-block.git] / include / linux / static_call.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_STATIC_CALL_H
#define _LINUX_STATIC_CALL_H

/*
 * Static call support
 *
 * Static calls use code patching to hard-code function pointers into direct
 * branch instructions. They give the flexibility of function pointers, but
 * with improved performance. This is especially important for cases where
 * retpolines would otherwise be used, as retpolines can significantly impact
 * performance.
 *
 *
 * API overview:
 *
 *   DECLARE_STATIC_CALL(name, func);
 *   DEFINE_STATIC_CALL(name, func);
 *   DEFINE_STATIC_CALL_NULL(name, typename);
 *   DEFINE_STATIC_CALL_RET0(name, typename);
 *
 *   __static_call_return0;
 *
 *   static_call(name)(args...);
 *   static_call_cond(name)(args...);
 *   static_call_update(name, func);
 *   static_call_query(name);
 *
 *   EXPORT_STATIC_CALL{,_TRAMP}{,_GPL}()
 *
 * Usage example:
 *
 *   # Start with the following functions (with identical prototypes):
 *   int func_a(int arg1, int arg2);
 *   int func_b(int arg1, int arg2);
 *
 *   # Define a 'my_name' reference, associated with func_a() by default
 *   DEFINE_STATIC_CALL(my_name, func_a);
 *
 *   # Call func_a()
 *   static_call(my_name)(arg1, arg2);
 *
 *   # Update 'my_name' to point to func_b()
 *   static_call_update(my_name, &func_b);
 *
 *   # Call func_b()
 *   static_call(my_name)(arg1, arg2);
 *
 *
 * Implementation details:
 *
 *   This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL).
 *   Otherwise basic indirect calls are used (with function pointers).
 *
 *   Each static_call() site calls into a trampoline associated with the name.
 *   The trampoline has a direct branch to the default function.  Updates to a
 *   name will modify the trampoline's branch destination.
 *
 *   If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites
 *   themselves will be patched at runtime to call the functions directly,
 *   rather than calling through the trampoline.  This requires objtool or a
 *   compiler plugin to detect all the static_call() sites and annotate them
 *   in the .static_call_sites section.
 *
 *
 * Notes on NULL function pointers:
 *
 *   Static_call()s support NULL functions, with many of the caveats that
 *   regular function pointers have.
 *
 *   Clearly calling a NULL function pointer is 'BAD', so too for
 *   static_call()s (although when HAVE_STATIC_CALL it might not be immediately
 *   fatal). A NULL static_call can be the result of:
 *
 *     DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int));
 *
 *   which is equivalent to declaring a NULL function pointer with just a
 *   typename:
 *
 *     void (*my_func_ptr)(int arg1) = NULL;
 *
 *   or using static_call_update() with a NULL function. In both cases the
 *   HAVE_STATIC_CALL implementation will patch the trampoline with a RET
 *   instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE
 *   architectures can patch the trampoline call to a NOP.
 *
 *   In all cases, any argument evaluation is unconditional. Unlike a regular
 *   conditional function pointer call:
 *
 *     if (my_func_ptr)
 *         my_func_ptr(arg1)
 *
 *   where the argument evaludation also depends on the pointer value.
 *
 *   When calling a static_call that can be NULL, use:
 *
 *     static_call_cond(name)(arg1);
 *
 *   which will include the required value tests to avoid NULL-pointer
 *   dereferences.
 *
 *   To query which function is currently set to be called, use:
 *
 *   func = static_call_query(name);
 *
 *
 * DEFINE_STATIC_CALL_RET0 / __static_call_return0:
 *
 *   Just like how DEFINE_STATIC_CALL_NULL() / static_call_cond() optimize the
 *   conditional void function call, DEFINE_STATIC_CALL_RET0 /
 *   __static_call_return0 optimize the do nothing return 0 function.
 *
 *   This feature is strictly UB per the C standard (since it casts a function
 *   pointer to a different signature) and relies on the architecture ABI to
 *   make things work. In particular it relies on Caller Stack-cleanup and the
 *   whole return register being clobbered for short return values. All normal
 *   CDECL style ABIs conform.
 *
 *   In particular the x86_64 implementation replaces the 5 byte CALL
 *   instruction at the callsite with a 5 byte clear of the RAX register,
 *   completely eliding any function call overhead.
 *
 *   Notably argument setup is unconditional.
 *
 *
 * EXPORT_STATIC_CALL() vs EXPORT_STATIC_CALL_TRAMP():
 *
 *   The difference is that the _TRAMP variant tries to only export the
 *   trampoline with the result that a module can use static_call{,_cond}() but
 *   not static_call_update().
 *
 */

#include <linux/types.h>
#include <linux/cpu.h>
#include <linux/static_call_types.h>

#ifdef CONFIG_HAVE_STATIC_CALL
#include <asm/static_call.h>

/*
 * Either @site or @tramp can be NULL.
 */
extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);

#define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)

#else
#define STATIC_CALL_TRAMP_ADDR(name) NULL
#endif

#define static_call_update(name, func)					\
({									\
	typeof(&STATIC_CALL_TRAMP(name)) __F = (func);			\
	__static_call_update(&STATIC_CALL_KEY(name),			\
			     STATIC_CALL_TRAMP_ADDR(name), __F);	\
})

#define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func))

#ifdef CONFIG_HAVE_STATIC_CALL_INLINE

extern int __init static_call_init(void);

extern void static_call_force_reinit(void);

struct static_call_mod {
	struct static_call_mod *next;
	struct module *mod; /* for vmlinux, mod == NULL */
	struct static_call_site *sites;
};

/* For finding the key associated with a trampoline */
struct static_call_tramp_key {
	s32 tramp;
	s32 key;
};

extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
extern int static_call_mod_init(struct module *mod);
extern int static_call_text_reserved(void *start, void *end);

extern long __static_call_return0(void);

#define DEFINE_STATIC_CALL(name, _func)					\
	DECLARE_STATIC_CALL(name, _func);				\
	struct static_call_key STATIC_CALL_KEY(name) = {		\
		.func = _func,						\
		.type = 1,						\
	};								\
	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)

#define DEFINE_STATIC_CALL_NULL(name, _func)				\
	DECLARE_STATIC_CALL(name, _func);				\
	struct static_call_key STATIC_CALL_KEY(name) = {		\
		.func = NULL,						\
		.type = 1,						\
	};								\
	ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)

#define DEFINE_STATIC_CALL_RET0(name, _func)				\
	DECLARE_STATIC_CALL(name, _func);				\
	struct static_call_key STATIC_CALL_KEY(name) = {		\
		.func = __static_call_return0,				\
		.type = 1,						\
	};								\
	ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)

#define static_call_cond(name)	(void)__static_call(name)

#define EXPORT_STATIC_CALL(name)					\
	EXPORT_SYMBOL(STATIC_CALL_KEY(name));				\
	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
#define EXPORT_STATIC_CALL_GPL(name)					\
	EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name));			\
	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))

/* Leave the key unexported, so modules can't change static call targets: */
#define EXPORT_STATIC_CALL_TRAMP(name)					\
	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name));				\
	ARCH_ADD_TRAMP_KEY(name)
#define EXPORT_STATIC_CALL_TRAMP_GPL(name)				\
	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name));			\
	ARCH_ADD_TRAMP_KEY(name)

#elif defined(CONFIG_HAVE_STATIC_CALL)

static inline int static_call_init(void) { return 0; }

#define DEFINE_STATIC_CALL(name, _func)					\
	DECLARE_STATIC_CALL(name, _func);				\
	struct static_call_key STATIC_CALL_KEY(name) = {		\
		.func = _func,						\
	};								\
	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)

#define DEFINE_STATIC_CALL_NULL(name, _func)				\
	DECLARE_STATIC_CALL(name, _func);				\
	struct static_call_key STATIC_CALL_KEY(name) = {		\
		.func = NULL,						\
	};								\
	ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)

#define DEFINE_STATIC_CALL_RET0(name, _func)				\
	DECLARE_STATIC_CALL(name, _func);				\
	struct static_call_key STATIC_CALL_KEY(name) = {		\
		.func = __static_call_return0,				\
	};								\
	ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)

#define static_call_cond(name)	(void)__static_call(name)

static inline
void __static_call_update(struct static_call_key *key, void *tramp, void *func)
{
	cpus_read_lock();
	WRITE_ONCE(key->func, func);
	arch_static_call_transform(NULL, tramp, func, false);
	cpus_read_unlock();
}

static inline int static_call_text_reserved(void *start, void *end)
{
	return 0;
}

extern long __static_call_return0(void);

#define EXPORT_STATIC_CALL(name)					\
	EXPORT_SYMBOL(STATIC_CALL_KEY(name));				\
	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
#define EXPORT_STATIC_CALL_GPL(name)					\
	EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name));			\
	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))

/* Leave the key unexported, so modules can't change static call targets: */
#define EXPORT_STATIC_CALL_TRAMP(name)					\
	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
#define EXPORT_STATIC_CALL_TRAMP_GPL(name)				\
	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))

#else /* Generic implementation */

static inline int static_call_init(void) { return 0; }

static inline long __static_call_return0(void)
{
	return 0;
}

#define __DEFINE_STATIC_CALL(name, _func, _func_init)			\
	DECLARE_STATIC_CALL(name, _func);				\
	struct static_call_key STATIC_CALL_KEY(name) = {		\
		.func = _func_init,					\
	}

#define DEFINE_STATIC_CALL(name, _func)					\
	__DEFINE_STATIC_CALL(name, _func, _func)

#define DEFINE_STATIC_CALL_NULL(name, _func)				\
	__DEFINE_STATIC_CALL(name, _func, NULL)

#define DEFINE_STATIC_CALL_RET0(name, _func)				\
	__DEFINE_STATIC_CALL(name, _func, __static_call_return0)

static inline void __static_call_nop(void) { }

/*
 * This horrific hack takes care of two things:
 *
 *  - it ensures the compiler will only load the function pointer ONCE,
 *    which avoids a reload race.
 *
 *  - it ensures the argument evaluation is unconditional, similar
 *    to the HAVE_STATIC_CALL variant.
 *
 * Sadly current GCC/Clang (10 for both) do not optimize this properly
 * and will emit an indirect call for the NULL case :-(
 */
#define __static_call_cond(name)					\
({									\
	void *func = READ_ONCE(STATIC_CALL_KEY(name).func);		\
	if (!func)							\
		func = &__static_call_nop;				\
	(typeof(STATIC_CALL_TRAMP(name))*)func;				\
})

#define static_call_cond(name)	(void)__static_call_cond(name)

static inline
void __static_call_update(struct static_call_key *key, void *tramp, void *func)
{
	WRITE_ONCE(key->func, func);
}

static inline int static_call_text_reserved(void *start, void *end)
{
	return 0;
}

#define EXPORT_STATIC_CALL(name)	EXPORT_SYMBOL(STATIC_CALL_KEY(name))
#define EXPORT_STATIC_CALL_GPL(name)	EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))

#endif /* CONFIG_HAVE_STATIC_CALL */

#endif /* _LINUX_STATIC_CALL_H */
Commit	Line	Data
115284d8 JP	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _LINUX_STATIC_CALL_H
	3	#define _LINUX_STATIC_CALL_H
	4
	5	/*
	6	* Static call support
	7	*
	8	* Static calls use code patching to hard-code function pointers into direct
	9	* branch instructions. They give the flexibility of function pointers, but
	10	* with improved performance. This is especially important for cases where
	11	* retpolines would otherwise be used, as retpolines can significantly impact
	12	* performance.
	13	*
	14	*
	15	* API overview:
	16	*
	17	* DECLARE_STATIC_CALL(name, func);
	18	* DEFINE_STATIC_CALL(name, func);
452cddbf	19	* DEFINE_STATIC_CALL_NULL(name, typename);
9ae6ab27 PZ	20	* DEFINE_STATIC_CALL_RET0(name, typename);
	21	*
	22	* __static_call_return0;
	23	*
115284d8	24	* static_call(name)(args...);
452cddbf	25	* static_call_cond(name)(args...);
115284d8	26	* static_call_update(name, func);
6ea312d9	27	* static_call_query(name);
115284d8	28	*
9ae6ab27 PZ	29	* EXPORT_STATIC_CALL{,_TRAMP}{,_GPL}()
9ae6ab27 PZ	30	*
115284d8 JP	31	* Usage example:
	32	*
	33	* # Start with the following functions (with identical prototypes):
	34	* int func_a(int arg1, int arg2);
	35	* int func_b(int arg1, int arg2);
	36	*
	37	* # Define a 'my_name' reference, associated with func_a() by default
	38	* DEFINE_STATIC_CALL(my_name, func_a);
	39	*
	40	* # Call func_a()
	41	* static_call(my_name)(arg1, arg2);
	42	*
	43	* # Update 'my_name' to point to func_b()
	44	* static_call_update(my_name, &func_b);
	45	*
	46	* # Call func_b()
	47	* static_call(my_name)(arg1, arg2);
	48	*
	49	*
	50	* Implementation details:
	51	*
	52	* This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL).
	53	* Otherwise basic indirect calls are used (with function pointers).
	54	*
	55	* Each static_call() site calls into a trampoline associated with the name.
	56	* The trampoline has a direct branch to the default function. Updates to a
	57	* name will modify the trampoline's branch destination.
	58	*
	59	* If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites
	60	* themselves will be patched at runtime to call the functions directly,
	61	* rather than calling through the trampoline. This requires objtool or a
	62	* compiler plugin to detect all the static_call() sites and annotate them
	63	* in the .static_call_sites section.
452cddbf PZ	64	*
	65	*
	66	* Notes on NULL function pointers:
	67	*
	68	* Static_call()s support NULL functions, with many of the caveats that
	69	* regular function pointers have.
	70	*
	71	* Clearly calling a NULL function pointer is 'BAD', so too for
	72	* static_call()s (although when HAVE_STATIC_CALL it might not be immediately
	73	* fatal). A NULL static_call can be the result of:
	74	*
	75	* DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int));
	76	*
	77	* which is equivalent to declaring a NULL function pointer with just a
	78	* typename:
	79	*
	80	* void (*my_func_ptr)(int arg1) = NULL;
	81	*
	82	* or using static_call_update() with a NULL function. In both cases the
	83	* HAVE_STATIC_CALL implementation will patch the trampoline with a RET
	84	* instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE
	85	* architectures can patch the trampoline call to a NOP.
	86	*
	87	* In all cases, any argument evaluation is unconditional. Unlike a regular
	88	* conditional function pointer call:
	89	*
	90	* if (my_func_ptr)
	91	* my_func_ptr(arg1)
	92	*
	93	* where the argument evaludation also depends on the pointer value.
	94	*
	95	* When calling a static_call that can be NULL, use:
	96	*
	97	* static_call_cond(name)(arg1);
	98	*
	99	* which will include the required value tests to avoid NULL-pointer
	100	* dereferences.
6ea312d9 JG	101	*
	102	* To query which function is currently set to be called, use:
	103	*
	104	* func = static_call_query(name);
9ae6ab27 PZ	105	*
	106	*
	107	* DEFINE_STATIC_CALL_RET0 / __static_call_return0:
	108	*
	109	* Just like how DEFINE_STATIC_CALL_NULL() / static_call_cond() optimize the
	110	* conditional void function call, DEFINE_STATIC_CALL_RET0 /
	111	* __static_call_return0 optimize the do nothing return 0 function.
	112	*
	113	* This feature is strictly UB per the C standard (since it casts a function
	114	* pointer to a different signature) and relies on the architecture ABI to
	115	* make things work. In particular it relies on Caller Stack-cleanup and the
	116	* whole return register being clobbered for short return values. All normal
	117	* CDECL style ABIs conform.
	118	*
	119	* In particular the x86_64 implementation replaces the 5 byte CALL
	120	* instruction at the callsite with a 5 byte clear of the RAX register,
	121	* completely eliding any function call overhead.
	122	*
	123	* Notably argument setup is unconditional.
	124	*
	125	*
	126	* EXPORT_STATIC_CALL() vs EXPORT_STATIC_CALL_TRAMP():
	127	*
	128	* The difference is that the _TRAMP variant tries to only export the
	129	* trampoline with the result that a module can use static_call{,_cond}() but
	130	* not static_call_update().
	131	*
115284d8 JP	132	*/
	133
	134	#include <linux/types.h>
	135	#include <linux/cpu.h>
	136	#include <linux/static_call_types.h>
	137
	138	#ifdef CONFIG_HAVE_STATIC_CALL
	139	#include <asm/static_call.h>
	140
	141	/*
	142	* Either @site or @tramp can be NULL.
	143	*/
5b06fd3b	144	extern void arch_static_call_transform(void site, void tramp, void *func, bool tail);
115284d8 JP	145
	146	#define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
	147
115284d8 JP	148	#else
	149	#define STATIC_CALL_TRAMP_ADDR(name) NULL
	150	#endif
	151
115284d8 JP	152	#define static_call_update(name, func) \
115284d8 JP	153	({ \
9432bbd9	154	typeof(&STATIC_CALL_TRAMP(name)) __F = (func); \
115284d8	155	__static_call_update(&STATIC_CALL_KEY(name), \
9432bbd9	156	STATIC_CALL_TRAMP_ADDR(name), __F); \
115284d8 JP	157	})
115284d8 JP	158
6ea312d9 JG	159	#define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func))
6ea312d9 JG	160
9183c3f9 JP	161	#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
9183c3f9 JP	162
69e0ad37	163	extern int __init static_call_init(void);
a945c834	164
7825451f PZ	165	extern void static_call_force_reinit(void);
7825451f PZ	166
9183c3f9 JP	167	struct static_call_mod {
	168	struct static_call_mod *next;
	169	struct module mod; / for vmlinux, mod == NULL */
	170	struct static_call_site *sites;
	171	};
	172
73f44fe1 JP	173	/* For finding the key associated with a trampoline */
	174	struct static_call_tramp_key {
	175	s32 tramp;
	176	s32 key;
	177	};
	178
9183c3f9 JP	179	extern void __static_call_update(struct static_call_key key, void tramp, void *func);
9183c3f9 JP	180	extern int static_call_mod_init(struct module *mod);
6333e8f7	181	extern int static_call_text_reserved(void start, void end);
9183c3f9	182
3f2a8fc4 PZ	183	extern long __static_call_return0(void);
3f2a8fc4 PZ	184
df21c0d7	185	#define DEFINE_STATIC_CALL(name, _func) \
9183c3f9 JP	186	DECLARE_STATIC_CALL(name, _func); \
9183c3f9 JP	187	struct static_call_key STATIC_CALL_KEY(name) = { \
df21c0d7	188	.func = _func, \
a945c834	189	.type = 1, \
9183c3f9	190	}; \
df21c0d7	191	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
9183c3f9	192
452cddbf PZ	193	#define DEFINE_STATIC_CALL_NULL(name, _func) \
	194	DECLARE_STATIC_CALL(name, _func); \
	195	struct static_call_key STATIC_CALL_KEY(name) = { \
	196	.func = NULL, \
	197	.type = 1, \
	198	}; \
	199	ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
	200
5517d500 CL	201	#define DEFINE_STATIC_CALL_RET0(name, _func) \
	202	DECLARE_STATIC_CALL(name, _func); \
	203	struct static_call_key STATIC_CALL_KEY(name) = { \
	204	.func = __static_call_return0, \
	205	.type = 1, \
	206	}; \
	207	ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)
	208
452cddbf	209	#define static_call_cond(name) (void)__static_call(name)
9183c3f9 JP	210
	211	#define EXPORT_STATIC_CALL(name) \
	212	EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
	213	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
9183c3f9 JP	214	#define EXPORT_STATIC_CALL_GPL(name) \
	215	EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
	216	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
	217
73f44fe1 JP	218	/* Leave the key unexported, so modules can't change static call targets: */
	219	#define EXPORT_STATIC_CALL_TRAMP(name) \
	220	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \
	221	ARCH_ADD_TRAMP_KEY(name)
	222	#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
	223	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \
	224	ARCH_ADD_TRAMP_KEY(name)
	225
9183c3f9	226	#elif defined(CONFIG_HAVE_STATIC_CALL)
115284d8	227
69e0ad37	228	static inline int static_call_init(void) { return 0; }
a945c834	229
df21c0d7	230	#define DEFINE_STATIC_CALL(name, _func) \
115284d8 JP	231	DECLARE_STATIC_CALL(name, _func); \
115284d8 JP	232	struct static_call_key STATIC_CALL_KEY(name) = { \
df21c0d7	233	.func = _func, \
115284d8	234	}; \
df21c0d7	235	ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
115284d8	236
452cddbf PZ	237	#define DEFINE_STATIC_CALL_NULL(name, _func) \
	238	DECLARE_STATIC_CALL(name, _func); \
	239	struct static_call_key STATIC_CALL_KEY(name) = { \
	240	.func = NULL, \
	241	}; \
	242	ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
	243
5517d500 CL	244	#define DEFINE_STATIC_CALL_RET0(name, _func) \
	245	DECLARE_STATIC_CALL(name, _func); \
	246	struct static_call_key STATIC_CALL_KEY(name) = { \
	247	.func = __static_call_return0, \
	248	}; \
	249	ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)
6ea312d9	250
452cddbf	251	#define static_call_cond(name) (void)__static_call(name)
115284d8 JP	252
	253	static inline
	254	void __static_call_update(struct static_call_key key, void tramp, void *func)
	255	{
	256	cpus_read_lock();
	257	WRITE_ONCE(key->func, func);
5b06fd3b	258	arch_static_call_transform(NULL, tramp, func, false);
115284d8 JP	259	cpus_read_unlock();
	260	}
	261
6333e8f7 PZ	262	static inline int static_call_text_reserved(void start, void end)
	263	{
	264	return 0;
	265	}
	266
8fd4ddda	267	extern long __static_call_return0(void);
3f2a8fc4	268
115284d8 JP	269	#define EXPORT_STATIC_CALL(name) \
	270	EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
	271	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
115284d8 JP	272	#define EXPORT_STATIC_CALL_GPL(name) \
	273	EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
	274	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
	275
73f44fe1 JP	276	/* Leave the key unexported, so modules can't change static call targets: */
	277	#define EXPORT_STATIC_CALL_TRAMP(name) \
	278	EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
	279	#define EXPORT_STATIC_CALL_TRAMP_GPL(name) \
	280	EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
	281
115284d8 JP	282	#else /* Generic implementation */
115284d8 JP	283
69e0ad37	284	static inline int static_call_init(void) { return 0; }
a945c834	285
3f2a8fc4 PZ	286	static inline long __static_call_return0(void)
	287	{
	288	return 0;
	289	}
	290
29fd0194	291	#define __DEFINE_STATIC_CALL(name, _func, _func_init) \
115284d8 JP	292	DECLARE_STATIC_CALL(name, _func); \
115284d8 JP	293	struct static_call_key STATIC_CALL_KEY(name) = { \
29fd0194	294	.func = _func_init, \
115284d8 JP	295	}
115284d8 JP	296
df21c0d7 CL	297	#define DEFINE_STATIC_CALL(name, _func) \
	298	__DEFINE_STATIC_CALL(name, _func, _func)
	299
452cddbf	300	#define DEFINE_STATIC_CALL_NULL(name, _func) \
df21c0d7	301	__DEFINE_STATIC_CALL(name, _func, NULL)
452cddbf	302
5517d500 CL	303	#define DEFINE_STATIC_CALL_RET0(name, _func) \
	304	__DEFINE_STATIC_CALL(name, _func, __static_call_return0)
	305
452cddbf PZ	306	static inline void __static_call_nop(void) { }
	307
	308	/*
	309	* This horrific hack takes care of two things:
	310	*
	311	* - it ensures the compiler will only load the function pointer ONCE,
	312	* which avoids a reload race.
	313	*
	314	* - it ensures the argument evaluation is unconditional, similar
	315	* to the HAVE_STATIC_CALL variant.
	316	*
	317	* Sadly current GCC/Clang (10 for both) do not optimize this properly
	318	* and will emit an indirect call for the NULL case :-(
	319	*/
	320	#define __static_call_cond(name) \
	321	({ \
	322	void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \
	323	if (!func) \
	324	func = &__static_call_nop; \
	325	(typeof(STATIC_CALL_TRAMP(name))*)func; \
	326	})
	327
	328	#define static_call_cond(name) (void)__static_call_cond(name)
	329
115284d8 JP	330	static inline
	331	void __static_call_update(struct static_call_key key, void tramp, void *func)
	332	{
	333	WRITE_ONCE(key->func, func);
	334	}
	335
6333e8f7 PZ	336	static inline int static_call_text_reserved(void start, void end)
	337	{
	338	return 0;
	339	}
	340
115284d8 JP	341	#define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name))
	342	#define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
	343
	344	#endif /* CONFIG_HAVE_STATIC_CALL */
	345
	346	#endif /* _LINUX_STATIC_CALL_H */