[linux-2.6-block.git] / kernel / events / callchain.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Performance events callchain code, extracted from core.c:
 *
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
 *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
 */

#include <linux/perf_event.h>
#include <linux/slab.h>
#include <linux/sched/task_stack.h>

#include "internal.h"

struct callchain_cpus_entries {
	struct rcu_head			rcu_head;
	struct perf_callchain_entry	*cpu_entries[0];
};

int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;

static inline size_t perf_callchain_entry__sizeof(void)
{
	return (sizeof(struct perf_callchain_entry) +
		sizeof(__u64) * (sysctl_perf_event_max_stack +
				 sysctl_perf_event_max_contexts_per_stack));
}

static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
static atomic_t nr_callchain_events;
static DEFINE_MUTEX(callchain_mutex);
static struct callchain_cpus_entries *callchain_cpus_entries;


__weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
				  struct pt_regs *regs)
{
}

__weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
				struct pt_regs *regs)
{
}

static void release_callchain_buffers_rcu(struct rcu_head *head)
{
	struct callchain_cpus_entries *entries;
	int cpu;

	entries = container_of(head, struct callchain_cpus_entries, rcu_head);

	for_each_possible_cpu(cpu)
		kfree(entries->cpu_entries[cpu]);

	kfree(entries);
}

static void release_callchain_buffers(void)
{
	struct callchain_cpus_entries *entries;

	entries = callchain_cpus_entries;
	RCU_INIT_POINTER(callchain_cpus_entries, NULL);
	call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
}

static int alloc_callchain_buffers(void)
{
	int cpu;
	int size;
	struct callchain_cpus_entries *entries;

	/*
	 * We can't use the percpu allocation API for data that can be
	 * accessed from NMI. Use a temporary manual per cpu allocation
	 * until that gets sorted out.
	 */
	size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);

	entries = kzalloc(size, GFP_KERNEL);
	if (!entries)
		return -ENOMEM;

	size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;

	for_each_possible_cpu(cpu) {
		entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
							 cpu_to_node(cpu));
		if (!entries->cpu_entries[cpu])
			goto fail;
	}

	rcu_assign_pointer(callchain_cpus_entries, entries);

	return 0;

fail:
	for_each_possible_cpu(cpu)
		kfree(entries->cpu_entries[cpu]);
	kfree(entries);

	return -ENOMEM;
}

int get_callchain_buffers(int event_max_stack)
{
	int err = 0;
	int count;

	mutex_lock(&callchain_mutex);

	count = atomic_inc_return(&nr_callchain_events);
	if (WARN_ON_ONCE(count < 1)) {
		err = -EINVAL;
		goto exit;
	}

	/*
	 * If requesting per event more than the global cap,
	 * return a different error to help userspace figure
	 * this out.
	 *
	 * And also do it here so that we have &callchain_mutex held.
	 */
	if (event_max_stack > sysctl_perf_event_max_stack) {
		err = -EOVERFLOW;
		goto exit;
	}

	if (count == 1)
		err = alloc_callchain_buffers();
exit:
	if (err)
		atomic_dec(&nr_callchain_events);

	mutex_unlock(&callchain_mutex);

	return err;
}

void put_callchain_buffers(void)
{
	if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
		release_callchain_buffers();
		mutex_unlock(&callchain_mutex);
	}
}

static struct perf_callchain_entry *get_callchain_entry(int *rctx)
{
	int cpu;
	struct callchain_cpus_entries *entries;

	*rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
	if (*rctx == -1)
		return NULL;

	entries = rcu_dereference(callchain_cpus_entries);
	if (!entries)
		return NULL;

	cpu = smp_processor_id();

	return (((void *)entries->cpu_entries[cpu]) +
		(*rctx * perf_callchain_entry__sizeof()));
}

static void
put_callchain_entry(int rctx)
{
	put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
}

struct perf_callchain_entry *
get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
		   u32 max_stack, bool crosstask, bool add_mark)
{
	struct perf_callchain_entry *entry;
	struct perf_callchain_entry_ctx ctx;
	int rctx;

	entry = get_callchain_entry(&rctx);
	if (rctx == -1)
		return NULL;

	if (!entry)
		goto exit_put;

	ctx.entry     = entry;
	ctx.max_stack = max_stack;
	ctx.nr	      = entry->nr = init_nr;
	ctx.contexts       = 0;
	ctx.contexts_maxed = false;

	if (kernel && !user_mode(regs)) {
		if (add_mark)
			perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
		perf_callchain_kernel(&ctx, regs);
	}

	if (user) {
		if (!user_mode(regs)) {
			if  (current->mm)
				regs = task_pt_regs(current);
			else
				regs = NULL;
		}

		if (regs) {
			mm_segment_t fs;

			if (crosstask)
				goto exit_put;

			if (add_mark)
				perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);

			fs = get_fs();
			set_fs(USER_DS);
			perf_callchain_user(&ctx, regs);
			set_fs(fs);
		}
	}

exit_put:
	put_callchain_entry(rctx);

	return entry;
}

/*
 * Used for sysctl_perf_event_max_stack and
 * sysctl_perf_event_max_contexts_per_stack.
 */
int perf_event_max_stack_handler(struct ctl_table *table, int write,
				 void __user *buffer, size_t *lenp, loff_t *ppos)
{
	int *value = table->data;
	int new_value = *value, ret;
	struct ctl_table new_table = *table;

	new_table.data = &new_value;
	ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
	if (ret || !write)
		return ret;

	mutex_lock(&callchain_mutex);
	if (atomic_read(&nr_callchain_events))
		ret = -EBUSY;
	else
		*value = new_value;

	mutex_unlock(&callchain_mutex);

	return ret;
}
Commit	Line	Data
8e86e015	1	// SPDX-License-Identifier: GPL-2.0
9251f904 BP	2	/*
	3	* Performance events callchain code, extracted from core.c:
	4	*
	5	* Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
	6	* Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
90eec103	7	* Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
3723c632	8	* Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9251f904 BP	9	*/
	10
	11	#include <linux/perf_event.h>
	12	#include <linux/slab.h>
68db0cf1 IM	13	#include <linux/sched/task_stack.h>
68db0cf1 IM	14
9251f904 BP	15	#include "internal.h"
	16
	17	struct callchain_cpus_entries {
	18	struct rcu_head rcu_head;
	19	struct perf_callchain_entry *cpu_entries[0];
	20	};
	21
c5dfd78e	22	int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
c85b0334	23	int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;
c5dfd78e ACM	24
	25	static inline size_t perf_callchain_entry__sizeof(void)
	26	{
	27	return (sizeof(struct perf_callchain_entry) +
c85b0334 ACM	28	sizeof(__u64) * (sysctl_perf_event_max_stack +
c85b0334 ACM	29	sysctl_perf_event_max_contexts_per_stack));
c5dfd78e ACM	30	}
c5dfd78e ACM	31
9251f904 BP	32	static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
	33	static atomic_t nr_callchain_events;
	34	static DEFINE_MUTEX(callchain_mutex);
	35	static struct callchain_cpus_entries *callchain_cpus_entries;
	36
	37
cfbcf468	38	__weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
9251f904 BP	39	struct pt_regs *regs)
	40	{
	41	}
	42
cfbcf468	43	__weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
9251f904 BP	44	struct pt_regs *regs)
	45	{
	46	}
	47
	48	static void release_callchain_buffers_rcu(struct rcu_head *head)
	49	{
	50	struct callchain_cpus_entries *entries;
	51	int cpu;
	52
	53	entries = container_of(head, struct callchain_cpus_entries, rcu_head);
	54
	55	for_each_possible_cpu(cpu)
	56	kfree(entries->cpu_entries[cpu]);
	57
	58	kfree(entries);
	59	}
	60
	61	static void release_callchain_buffers(void)
	62	{
	63	struct callchain_cpus_entries *entries;
	64
	65	entries = callchain_cpus_entries;
e0455e19	66	RCU_INIT_POINTER(callchain_cpus_entries, NULL);
9251f904 BP	67	call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
	68	}
	69
	70	static int alloc_callchain_buffers(void)
	71	{
	72	int cpu;
	73	int size;
	74	struct callchain_cpus_entries *entries;
	75
	76	/*
	77	* We can't use the percpu allocation API for data that can be
	78	* accessed from NMI. Use a temporary manual per cpu allocation
	79	* until that gets sorted out.
	80	*/
	81	size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
	82
	83	entries = kzalloc(size, GFP_KERNEL);
	84	if (!entries)
	85	return -ENOMEM;
	86
c5dfd78e	87	size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;
9251f904 BP	88
	89	for_each_possible_cpu(cpu) {
	90	entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
	91	cpu_to_node(cpu));
	92	if (!entries->cpu_entries[cpu])
	93	goto fail;
	94	}
	95
	96	rcu_assign_pointer(callchain_cpus_entries, entries);
	97
	98	return 0;
	99
	100	fail:
	101	for_each_possible_cpu(cpu)
	102	kfree(entries->cpu_entries[cpu]);
	103	kfree(entries);
	104
	105	return -ENOMEM;
	106	}
	107
97c79a38	108	int get_callchain_buffers(int event_max_stack)
9251f904 BP	109	{
	110	int err = 0;
	111	int count;
	112
	113	mutex_lock(&callchain_mutex);
	114
	115	count = atomic_inc_return(&nr_callchain_events);
	116	if (WARN_ON_ONCE(count < 1)) {
	117	err = -EINVAL;
	118	goto exit;
	119	}
	120
5af44ca5 JO	121	/*
	122	* If requesting per event more than the global cap,
	123	* return a different error to help userspace figure
	124	* this out.
	125	*
	126	* And also do it here so that we have &callchain_mutex held.
	127	*/
	128	if (event_max_stack > sysctl_perf_event_max_stack) {
	129	err = -EOVERFLOW;
	130	goto exit;
	131	}
	132
bfb3d7b8 JO	133	if (count == 1)
bfb3d7b8 JO	134	err = alloc_callchain_buffers();
9251f904	135	exit:
90983b16 FW	136	if (err)
90983b16 FW	137	atomic_dec(&nr_callchain_events);
9251f904	138
fc3b86d6 FW	139	mutex_unlock(&callchain_mutex);
fc3b86d6 FW	140
9251f904 BP	141	return err;
	142	}
	143
	144	void put_callchain_buffers(void)
	145	{
	146	if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
	147	release_callchain_buffers();
	148	mutex_unlock(&callchain_mutex);
	149	}
	150	}
	151
	152	static struct perf_callchain_entry get_callchain_entry(int rctx)
	153	{
	154	int cpu;
	155	struct callchain_cpus_entries *entries;
	156
4a32fea9	157	*rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
9251f904 BP	158	if (*rctx == -1)
	159	return NULL;
	160
	161	entries = rcu_dereference(callchain_cpus_entries);
	162	if (!entries)
	163	return NULL;
	164
	165	cpu = smp_processor_id();
	166
c5dfd78e ACM	167	return (((void *)entries->cpu_entries[cpu]) +
c5dfd78e ACM	168	(rctx perf_callchain_entry__sizeof()));
9251f904 BP	169	}
	170
	171	static void
	172	put_callchain_entry(int rctx)
	173	{
4a32fea9	174	put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
9251f904 BP	175	}
9251f904 BP	176
568b329a AS	177	struct perf_callchain_entry *
568b329a AS	178	get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
cfbcf468	179	u32 max_stack, bool crosstask, bool add_mark)
568b329a AS	180	{
568b329a AS	181	struct perf_callchain_entry *entry;
cfbcf468	182	struct perf_callchain_entry_ctx ctx;
568b329a AS	183	int rctx;
568b329a AS	184
9251f904 BP	185	entry = get_callchain_entry(&rctx);
	186	if (rctx == -1)
	187	return NULL;
	188
	189	if (!entry)
	190	goto exit_put;
	191
cfbcf468 ACM	192	ctx.entry = entry;
cfbcf468 ACM	193	ctx.max_stack = max_stack;
3b1fff08	194	ctx.nr = entry->nr = init_nr;
c85b0334 ACM	195	ctx.contexts = 0;
c85b0334 ACM	196	ctx.contexts_maxed = false;
9251f904	197
d0775264	198	if (kernel && !user_mode(regs)) {
568b329a	199	if (add_mark)
3e4de4ec	200	perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
cfbcf468	201	perf_callchain_kernel(&ctx, regs);
9251f904 BP	202	}
9251f904 BP	203
d0775264 FW	204	if (user) {
	205	if (!user_mode(regs)) {
	206	if (current->mm)
	207	regs = task_pt_regs(current);
	208	else
	209	regs = NULL;
	210	}
	211
	212	if (regs) {
88b0193d WD	213	mm_segment_t fs;
88b0193d WD	214
568b329a	215	if (crosstask)
d0775264 FW	216	goto exit_put;
d0775264 FW	217
568b329a	218	if (add_mark)
3e4de4ec	219	perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
88b0193d WD	220
	221	fs = get_fs();
	222	set_fs(USER_DS);
cfbcf468	223	perf_callchain_user(&ctx, regs);
88b0193d	224	set_fs(fs);
d0775264	225	}
9251f904 BP	226	}
	227
	228	exit_put:
	229	put_callchain_entry(rctx);
	230
	231	return entry;
	232	}
c5dfd78e	233
c85b0334 ACM	234	/*
	235	* Used for sysctl_perf_event_max_stack and
	236	* sysctl_perf_event_max_contexts_per_stack.
	237	*/
c5dfd78e ACM	238	int perf_event_max_stack_handler(struct ctl_table *table, int write,
	239	void __user buffer, size_t lenp, loff_t *ppos)
	240	{
a831100a ACM	241	int *value = table->data;
a831100a ACM	242	int new_value = *value, ret;
c5dfd78e ACM	243	struct ctl_table new_table = *table;
	244
	245	new_table.data = &new_value;
	246	ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
	247	if (ret \|\| !write)
	248	return ret;
	249
	250	mutex_lock(&callchain_mutex);
	251	if (atomic_read(&nr_callchain_events))
	252	ret = -EBUSY;
	253	else
a831100a	254	*value = new_value;
c5dfd78e ACM	255
	256	mutex_unlock(&callchain_mutex);
	257
	258	return ret;
	259	}