x86/static_call: Add inline static call implementation for x86-64

[linux-2.6-block.git] / kernel / static_call.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/init.h>
#include <linux/static_call.h>
#include <linux/bug.h>
#include <linux/smp.h>
#include <linux/sort.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/cpu.h>
#include <linux/processor.h>
#include <asm/sections.h>

extern struct static_call_site __start_static_call_sites[],
			       __stop_static_call_sites[];

static bool static_call_initialized;

#define STATIC_CALL_INIT 1UL

/* mutex to protect key modules/sites */
static DEFINE_MUTEX(static_call_mutex);

static void static_call_lock(void)
{
	mutex_lock(&static_call_mutex);
}

static void static_call_unlock(void)
{
	mutex_unlock(&static_call_mutex);
}

static inline void *static_call_addr(struct static_call_site *site)
{
	return (void *)((long)site->addr + (long)&site->addr);
}


static inline struct static_call_key *static_call_key(const struct static_call_site *site)
{
	return (struct static_call_key *)
		(((long)site->key + (long)&site->key) & ~STATIC_CALL_INIT);
}

/* These assume the key is word-aligned. */
static inline bool static_call_is_init(struct static_call_site *site)
{
	return ((long)site->key + (long)&site->key) & STATIC_CALL_INIT;
}

static inline void static_call_set_init(struct static_call_site *site)
{
	site->key = ((long)static_call_key(site) | STATIC_CALL_INIT) -
		    (long)&site->key;
}

static int static_call_site_cmp(const void *_a, const void *_b)
{
	const struct static_call_site *a = _a;
	const struct static_call_site *b = _b;
	const struct static_call_key *key_a = static_call_key(a);
	const struct static_call_key *key_b = static_call_key(b);

	if (key_a < key_b)
		return -1;

	if (key_a > key_b)
		return 1;

	return 0;
}

static void static_call_site_swap(void *_a, void *_b, int size)
{
	long delta = (unsigned long)_a - (unsigned long)_b;
	struct static_call_site *a = _a;
	struct static_call_site *b = _b;
	struct static_call_site tmp = *a;

	a->addr = b->addr  - delta;
	a->key  = b->key   - delta;

	b->addr = tmp.addr + delta;
	b->key  = tmp.key  + delta;
}

static inline void static_call_sort_entries(struct static_call_site *start,
					    struct static_call_site *stop)
{
	sort(start, stop - start, sizeof(struct static_call_site),
	     static_call_site_cmp, static_call_site_swap);
}

void __static_call_update(struct static_call_key *key, void *tramp, void *func)
{
	struct static_call_site *site, *stop;
	struct static_call_mod *site_mod;

	cpus_read_lock();
	static_call_lock();

	if (key->func == func)
		goto done;

	key->func = func;

	arch_static_call_transform(NULL, tramp, func);

	/*
	 * If uninitialized, we'll not update the callsites, but they still
	 * point to the trampoline and we just patched that.
	 */
	if (WARN_ON_ONCE(!static_call_initialized))
		goto done;

	for (site_mod = key->mods; site_mod; site_mod = site_mod->next) {
		struct module *mod = site_mod->mod;

		if (!site_mod->sites) {
			/*
			 * This can happen if the static call key is defined in
			 * a module which doesn't use it.
			 */
			continue;
		}

		stop = __stop_static_call_sites;

#ifdef CONFIG_MODULES
		if (mod) {
			stop = mod->static_call_sites +
			       mod->num_static_call_sites;
		}
#endif

		for (site = site_mod->sites;
		     site < stop && static_call_key(site) == key; site++) {
			void *site_addr = static_call_addr(site);

			if (static_call_is_init(site)) {
				/*
				 * Don't write to call sites which were in
				 * initmem and have since been freed.
				 */
				if (!mod && system_state >= SYSTEM_RUNNING)
					continue;
				if (mod && !within_module_init((unsigned long)site_addr, mod))
					continue;
			}

			if (!kernel_text_address((unsigned long)site_addr)) {
				WARN_ONCE(1, "can't patch static call site at %pS",
					  site_addr);
				continue;
			}

			arch_static_call_transform(site_addr, NULL, func);
		}
	}

done:
	static_call_unlock();
	cpus_read_unlock();
}
EXPORT_SYMBOL_GPL(__static_call_update);

static int __static_call_init(struct module *mod,
			      struct static_call_site *start,
			      struct static_call_site *stop)
{
	struct static_call_site *site;
	struct static_call_key *key, *prev_key = NULL;
	struct static_call_mod *site_mod;

	if (start == stop)
		return 0;

	static_call_sort_entries(start, stop);

	for (site = start; site < stop; site++) {
		void *site_addr = static_call_addr(site);

		if ((mod && within_module_init((unsigned long)site_addr, mod)) ||
		    (!mod && init_section_contains(site_addr, 1)))
			static_call_set_init(site);

		key = static_call_key(site);
		if (key != prev_key) {
			prev_key = key;

			site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
			if (!site_mod)
				return -ENOMEM;

			site_mod->mod = mod;
			site_mod->sites = site;
			site_mod->next = key->mods;
			key->mods = site_mod;
		}

		arch_static_call_transform(site_addr, NULL, key->func);
	}

	return 0;
}

static int addr_conflict(struct static_call_site *site, void *start, void *end)
{
	unsigned long addr = (unsigned long)static_call_addr(site);

	if (addr <= (unsigned long)end &&
	    addr + CALL_INSN_SIZE > (unsigned long)start)
		return 1;

	return 0;
}

static int __static_call_text_reserved(struct static_call_site *iter_start,
				       struct static_call_site *iter_stop,
				       void *start, void *end)
{
	struct static_call_site *iter = iter_start;

	while (iter < iter_stop) {
		if (addr_conflict(iter, start, end))
			return 1;
		iter++;
	}

	return 0;
}

#ifdef CONFIG_MODULES

static int __static_call_mod_text_reserved(void *start, void *end)
{
	struct module *mod;
	int ret;

	preempt_disable();
	mod = __module_text_address((unsigned long)start);
	WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
	if (!try_module_get(mod))
		mod = NULL;
	preempt_enable();

	if (!mod)
		return 0;

	ret = __static_call_text_reserved(mod->static_call_sites,
			mod->static_call_sites + mod->num_static_call_sites,
			start, end);

	module_put(mod);

	return ret;
}

static int static_call_add_module(struct module *mod)
{
	return __static_call_init(mod, mod->static_call_sites,
				  mod->static_call_sites + mod->num_static_call_sites);
}

static void static_call_del_module(struct module *mod)
{
	struct static_call_site *start = mod->static_call_sites;
	struct static_call_site *stop = mod->static_call_sites +
					mod->num_static_call_sites;
	struct static_call_key *key, *prev_key = NULL;
	struct static_call_mod *site_mod, **prev;
	struct static_call_site *site;

	for (site = start; site < stop; site++) {
		key = static_call_key(site);
		if (key == prev_key)
			continue;

		prev_key = key;

		for (prev = &key->mods, site_mod = key->mods;
		     site_mod && site_mod->mod != mod;
		     prev = &site_mod->next, site_mod = site_mod->next)
			;

		if (!site_mod)
			continue;

		*prev = site_mod->next;
		kfree(site_mod);
	}
}

static int static_call_module_notify(struct notifier_block *nb,
				     unsigned long val, void *data)
{
	struct module *mod = data;
	int ret = 0;

	cpus_read_lock();
	static_call_lock();

	switch (val) {
	case MODULE_STATE_COMING:
		ret = static_call_add_module(mod);
		if (ret) {
			WARN(1, "Failed to allocate memory for static calls");
			static_call_del_module(mod);
		}
		break;
	case MODULE_STATE_GOING:
		static_call_del_module(mod);
		break;
	}

	static_call_unlock();
	cpus_read_unlock();

	return notifier_from_errno(ret);
}

static struct notifier_block static_call_module_nb = {
	.notifier_call = static_call_module_notify,
};

#else

static inline int __static_call_mod_text_reserved(void *start, void *end)
{
	return 0;
}

#endif /* CONFIG_MODULES */

int static_call_text_reserved(void *start, void *end)
{
	int ret = __static_call_text_reserved(__start_static_call_sites,
			__stop_static_call_sites, start, end);

	if (ret)
		return ret;

	return __static_call_mod_text_reserved(start, end);
}

static void __init static_call_init(void)
{
	int ret;

	if (static_call_initialized)
		return;

	cpus_read_lock();
	static_call_lock();
	ret = __static_call_init(NULL, __start_static_call_sites,
				 __stop_static_call_sites);
	static_call_unlock();
	cpus_read_unlock();

	if (ret) {
		pr_err("Failed to allocate memory for static_call!\n");
		BUG();
	}

	static_call_initialized = true;

#ifdef CONFIG_MODULES
	register_module_notifier(&static_call_module_nb);
#endif
}
early_initcall(static_call_init);