[linux-2.6-block.git] / lib / alloc_tag.c

// SPDX-License-Identifier: GPL-2.0-only
#include <linux/alloc_tag.h>
#include <linux/fs.h>
#include <linux/gfp.h>
#include <linux/module.h>
#include <linux/page_ext.h>
#include <linux/proc_fs.h>
#include <linux/seq_buf.h>
#include <linux/seq_file.h>

static struct codetag_type *alloc_tag_cttype;

DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
EXPORT_SYMBOL(_shared_alloc_tag);

DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
			mem_alloc_profiling_key);

struct allocinfo_private {
	struct codetag_iterator iter;
	bool print_header;
};

static void *allocinfo_start(struct seq_file *m, loff_t *pos)
{
	struct allocinfo_private *priv;
	struct codetag *ct;
	loff_t node = *pos;

	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
	m->private = priv;
	if (!priv)
		return NULL;

	priv->print_header = (node == 0);
	codetag_lock_module_list(alloc_tag_cttype, true);
	priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
	while ((ct = codetag_next_ct(&priv->iter)) != NULL && node)
		node--;

	return ct ? priv : NULL;
}

static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
{
	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
	struct codetag *ct = codetag_next_ct(&priv->iter);

	(*pos)++;
	if (!ct)
		return NULL;

	return priv;
}

static void allocinfo_stop(struct seq_file *m, void *arg)
{
	struct allocinfo_private *priv = (struct allocinfo_private *)m->private;

	if (priv) {
		codetag_lock_module_list(alloc_tag_cttype, false);
		kfree(priv);
	}
}

static void print_allocinfo_header(struct seq_buf *buf)
{
	/* Output format version, so we can change it. */
	seq_buf_printf(buf, "allocinfo - version: 1.0\n");
	seq_buf_printf(buf, "#     <size>  <calls> <tag info>\n");
}

static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
{
	struct alloc_tag *tag = ct_to_alloc_tag(ct);
	struct alloc_tag_counters counter = alloc_tag_read(tag);
	s64 bytes = counter.bytes;

	seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
	codetag_to_text(out, ct);
	seq_buf_putc(out, ' ');
	seq_buf_putc(out, '\n');
}

static int allocinfo_show(struct seq_file *m, void *arg)
{
	struct allocinfo_private *priv = (struct allocinfo_private *)arg;
	char *bufp;
	size_t n = seq_get_buf(m, &bufp);
	struct seq_buf buf;

	seq_buf_init(&buf, bufp, n);
	if (priv->print_header) {
		print_allocinfo_header(&buf);
		priv->print_header = false;
	}
	alloc_tag_to_text(&buf, priv->iter.ct);
	seq_commit(m, seq_buf_used(&buf));
	return 0;
}

static const struct seq_operations allocinfo_seq_op = {
	.start	= allocinfo_start,
	.next	= allocinfo_next,
	.stop	= allocinfo_stop,
	.show	= allocinfo_show,
};

size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
{
	struct codetag_iterator iter;
	struct codetag *ct;
	struct codetag_bytes n;
	unsigned int i, nr = 0;

	if (can_sleep)
		codetag_lock_module_list(alloc_tag_cttype, true);
	else if (!codetag_trylock_module_list(alloc_tag_cttype))
		return 0;

	iter = codetag_get_ct_iter(alloc_tag_cttype);
	while ((ct = codetag_next_ct(&iter))) {
		struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));

		n.ct	= ct;
		n.bytes = counter.bytes;

		for (i = 0; i < nr; i++)
			if (n.bytes > tags[i].bytes)
				break;

		if (i < count) {
			nr -= nr == count;
			memmove(&tags[i + 1],
				&tags[i],
				sizeof(tags[0]) * (nr - i));
			nr++;
			tags[i] = n;
		}
	}

	codetag_lock_module_list(alloc_tag_cttype, false);

	return nr;
}

static void __init procfs_init(void)
{
	proc_create_seq("allocinfo", 0400, NULL, &allocinfo_seq_op);
}

static bool alloc_tag_module_unload(struct codetag_type *cttype,
				    struct codetag_module *cmod)
{
	struct codetag_iterator iter = codetag_get_ct_iter(cttype);
	struct alloc_tag_counters counter;
	bool module_unused = true;
	struct alloc_tag *tag;
	struct codetag *ct;

	for (ct = codetag_next_ct(&iter); ct; ct = codetag_next_ct(&iter)) {
		if (iter.cmod != cmod)
			continue;

		tag = ct_to_alloc_tag(ct);
		counter = alloc_tag_read(tag);

		if (WARN(counter.bytes,
			 "%s:%u module %s func:%s has %llu allocated at module unload",
			 ct->filename, ct->lineno, ct->modname, ct->function, counter.bytes))
			module_unused = false;
	}

	return module_unused;
}

#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
static bool mem_profiling_support __meminitdata = true;
#else
static bool mem_profiling_support __meminitdata;
#endif

static int __init setup_early_mem_profiling(char *str)
{
	bool enable;

	if (!str || !str[0])
		return -EINVAL;

	if (!strncmp(str, "never", 5)) {
		enable = false;
		mem_profiling_support = false;
	} else {
		int res;

		res = kstrtobool(str, &enable);
		if (res)
			return res;

		mem_profiling_support = true;
	}

	if (enable != static_key_enabled(&mem_alloc_profiling_key)) {
		if (enable)
			static_branch_enable(&mem_alloc_profiling_key);
		else
			static_branch_disable(&mem_alloc_profiling_key);
	}

	return 0;
}
early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);

static __init bool need_page_alloc_tagging(void)
{
	return mem_profiling_support;
}

static __init void init_page_alloc_tagging(void)
{
}

struct page_ext_operations page_alloc_tagging_ops = {
	.size = sizeof(union codetag_ref),
	.need = need_page_alloc_tagging,
	.init = init_page_alloc_tagging,
};
EXPORT_SYMBOL(page_alloc_tagging_ops);

static struct ctl_table memory_allocation_profiling_sysctls[] = {
	{
		.procname	= "mem_profiling",
		.data		= &mem_alloc_profiling_key,
#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
		.mode		= 0444,
#else
		.mode		= 0644,
#endif
		.proc_handler	= proc_do_static_key,
	},
	{ }
};

static int __init alloc_tag_init(void)
{
	const struct codetag_type_desc desc = {
		.section	= "alloc_tags",
		.tag_size	= sizeof(struct alloc_tag),
		.module_unload	= alloc_tag_module_unload,
	};

	alloc_tag_cttype = codetag_register_type(&desc);
	if (IS_ERR(alloc_tag_cttype))
		return PTR_ERR(alloc_tag_cttype);

	if (!mem_profiling_support)
		memory_allocation_profiling_sysctls[0].mode = 0444;
	register_sysctl_init("vm", memory_allocation_profiling_sysctls);
	procfs_init();

	return 0;
}
module_init(alloc_tag_init);
Commit	Line	Data
22d407b1 SB	1	// SPDX-License-Identifier: GPL-2.0-only
	2	#include <linux/alloc_tag.h>
	3	#include <linux/fs.h>
	4	#include <linux/gfp.h>
	5	#include <linux/module.h>
dcfe378c	6	#include <linux/page_ext.h>
22d407b1 SB	7	#include <linux/proc_fs.h>
	8	#include <linux/seq_buf.h>
	9	#include <linux/seq_file.h>
	10
	11	static struct codetag_type *alloc_tag_cttype;
	12
	13	DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
	14	EXPORT_SYMBOL(_shared_alloc_tag);
	15
	16	DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
	17	mem_alloc_profiling_key);
	18
a38568a0 SB	19	struct allocinfo_private {
	20	struct codetag_iterator iter;
	21	bool print_header;
	22	};
	23
22d407b1 SB	24	static void allocinfo_start(struct seq_file m, loff_t *pos)
22d407b1 SB	25	{
a38568a0	26	struct allocinfo_private *priv;
22d407b1 SB	27	struct codetag *ct;
	28	loff_t node = *pos;
	29
a38568a0 SB	30	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
	31	m->private = priv;
	32	if (!priv)
22d407b1 SB	33	return NULL;
22d407b1 SB	34
a38568a0	35	priv->print_header = (node == 0);
22d407b1	36	codetag_lock_module_list(alloc_tag_cttype, true);
a38568a0 SB	37	priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
a38568a0 SB	38	while ((ct = codetag_next_ct(&priv->iter)) != NULL && node)
22d407b1 SB	39	node--;
22d407b1 SB	40
a38568a0	41	return ct ? priv : NULL;
22d407b1 SB	42	}
	43
	44	static void allocinfo_next(struct seq_file m, void arg, loff_t pos)
	45	{
a38568a0 SB	46	struct allocinfo_private priv = (struct allocinfo_private )arg;
a38568a0 SB	47	struct codetag *ct = codetag_next_ct(&priv->iter);
22d407b1 SB	48
	49	(*pos)++;
	50	if (!ct)
	51	return NULL;
	52
a38568a0	53	return priv;
22d407b1 SB	54	}
	55
	56	static void allocinfo_stop(struct seq_file m, void arg)
	57	{
a38568a0	58	struct allocinfo_private priv = (struct allocinfo_private )m->private;
22d407b1	59
a38568a0	60	if (priv) {
22d407b1	61	codetag_lock_module_list(alloc_tag_cttype, false);
a38568a0	62	kfree(priv);
22d407b1 SB	63	}
	64	}
	65
a38568a0 SB	66	static void print_allocinfo_header(struct seq_buf *buf)
	67	{
	68	/* Output format version, so we can change it. */
	69	seq_buf_printf(buf, "allocinfo - version: 1.0\n");
	70	seq_buf_printf(buf, "# <size> <calls> <tag info>\n");
	71	}
	72
22d407b1 SB	73	static void alloc_tag_to_text(struct seq_buf out, struct codetag ct)
	74	{
	75	struct alloc_tag *tag = ct_to_alloc_tag(ct);
	76	struct alloc_tag_counters counter = alloc_tag_read(tag);
	77	s64 bytes = counter.bytes;
	78
	79	seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
	80	codetag_to_text(out, ct);
	81	seq_buf_putc(out, ' ');
	82	seq_buf_putc(out, '\n');
	83	}
	84
	85	static int allocinfo_show(struct seq_file m, void arg)
	86	{
a38568a0	87	struct allocinfo_private priv = (struct allocinfo_private )arg;
22d407b1 SB	88	char *bufp;
	89	size_t n = seq_get_buf(m, &bufp);
	90	struct seq_buf buf;
	91
	92	seq_buf_init(&buf, bufp, n);
a38568a0 SB	93	if (priv->print_header) {
	94	print_allocinfo_header(&buf);
	95	priv->print_header = false;
	96	}
	97	alloc_tag_to_text(&buf, priv->iter.ct);
22d407b1 SB	98	seq_commit(m, seq_buf_used(&buf));
	99	return 0;
	100	}
	101
	102	static const struct seq_operations allocinfo_seq_op = {
	103	.start = allocinfo_start,
	104	.next = allocinfo_next,
	105	.stop = allocinfo_stop,
	106	.show = allocinfo_show,
	107	};
	108
1438d349 SB	109	size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
	110	{
	111	struct codetag_iterator iter;
	112	struct codetag *ct;
	113	struct codetag_bytes n;
	114	unsigned int i, nr = 0;
	115
	116	if (can_sleep)
	117	codetag_lock_module_list(alloc_tag_cttype, true);
	118	else if (!codetag_trylock_module_list(alloc_tag_cttype))
	119	return 0;
	120
	121	iter = codetag_get_ct_iter(alloc_tag_cttype);
	122	while ((ct = codetag_next_ct(&iter))) {
	123	struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
	124
	125	n.ct = ct;
	126	n.bytes = counter.bytes;
	127
	128	for (i = 0; i < nr; i++)
	129	if (n.bytes > tags[i].bytes)
	130	break;
	131
	132	if (i < count) {
	133	nr -= nr == count;
	134	memmove(&tags[i + 1],
	135	&tags[i],
	136	sizeof(tags[0]) * (nr - i));
	137	nr++;
	138	tags[i] = n;
	139	}
	140	}
	141
	142	codetag_lock_module_list(alloc_tag_cttype, false);
	143
	144	return nr;
	145	}
	146
22d407b1 SB	147	static void __init procfs_init(void)
22d407b1 SB	148	{
e1310695	149	proc_create_seq("allocinfo", 0400, NULL, &allocinfo_seq_op);
22d407b1 SB	150	}
	151
	152	static bool alloc_tag_module_unload(struct codetag_type *cttype,
	153	struct codetag_module *cmod)
	154	{
	155	struct codetag_iterator iter = codetag_get_ct_iter(cttype);
	156	struct alloc_tag_counters counter;
	157	bool module_unused = true;
	158	struct alloc_tag *tag;
	159	struct codetag *ct;
	160
	161	for (ct = codetag_next_ct(&iter); ct; ct = codetag_next_ct(&iter)) {
	162	if (iter.cmod != cmod)
	163	continue;
	164
	165	tag = ct_to_alloc_tag(ct);
	166	counter = alloc_tag_read(tag);
	167
	168	if (WARN(counter.bytes,
	169	"%s:%u module %s func:%s has %llu allocated at module unload",
	170	ct->filename, ct->lineno, ct->modname, ct->function, counter.bytes))
	171	module_unused = false;
	172	}
	173
	174	return module_unused;
	175	}
	176
8d469d0b SB	177	#ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
	178	static bool mem_profiling_support __meminitdata = true;
	179	#else
	180	static bool mem_profiling_support __meminitdata;
	181	#endif
	182
	183	static int __init setup_early_mem_profiling(char *str)
	184	{
	185	bool enable;
	186
	187	if (!str \|\| !str[0])
	188	return -EINVAL;
	189
	190	if (!strncmp(str, "never", 5)) {
	191	enable = false;
	192	mem_profiling_support = false;
	193	} else {
	194	int res;
	195
	196	res = kstrtobool(str, &enable);
	197	if (res)
	198	return res;
	199
	200	mem_profiling_support = true;
	201	}
	202
	203	if (enable != static_key_enabled(&mem_alloc_profiling_key)) {
	204	if (enable)
	205	static_branch_enable(&mem_alloc_profiling_key);
	206	else
	207	static_branch_disable(&mem_alloc_profiling_key);
	208	}
	209
	210	return 0;
	211	}
	212	early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
	213
dcfe378c SB	214	static __init bool need_page_alloc_tagging(void)
dcfe378c SB	215	{
8d469d0b	216	return mem_profiling_support;
dcfe378c SB	217	}
	218
	219	static __init void init_page_alloc_tagging(void)
	220	{
	221	}
	222
	223	struct page_ext_operations page_alloc_tagging_ops = {
	224	.size = sizeof(union codetag_ref),
	225	.need = need_page_alloc_tagging,
	226	.init = init_page_alloc_tagging,
	227	};
	228	EXPORT_SYMBOL(page_alloc_tagging_ops);
	229
22d407b1 SB	230	static struct ctl_table memory_allocation_profiling_sysctls[] = {
	231	{
	232	.procname = "mem_profiling",
	233	.data = &mem_alloc_profiling_key,
	234	#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
	235	.mode = 0444,
	236	#else
	237	.mode = 0644,
	238	#endif
	239	.proc_handler = proc_do_static_key,
	240	},
	241	{ }
	242	};
	243
	244	static int __init alloc_tag_init(void)
	245	{
	246	const struct codetag_type_desc desc = {
	247	.section = "alloc_tags",
	248	.tag_size = sizeof(struct alloc_tag),
	249	.module_unload = alloc_tag_module_unload,
	250	};
	251
	252	alloc_tag_cttype = codetag_register_type(&desc);
	253	if (IS_ERR(alloc_tag_cttype))
	254	return PTR_ERR(alloc_tag_cttype);
	255
8d469d0b SB	256	if (!mem_profiling_support)
8d469d0b SB	257	memory_allocation_profiling_sysctls[0].mode = 0444;
22d407b1 SB	258	register_sysctl_init("vm", memory_allocation_profiling_sysctls);
	259	procfs_init();
	260
	261	return 0;
	262	}
	263	module_init(alloc_tag_init);