[linux-2.6-block.git] / kernel / trace / trace_event_perf.c

// SPDX-License-Identifier: GPL-2.0
/*
 * trace event based perf event profiling/tracing
 *
 * Copyright (C) 2009 Red Hat Inc, Peter Zijlstra
 * Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com>
 */

#include <linux/module.h>
#include <linux/kprobes.h>
#include "trace.h"
#include "trace_probe.h"

static char __percpu *perf_trace_buf[PERF_NR_CONTEXTS];

/*
 * Force it to be aligned to unsigned long to avoid misaligned accesses
 * suprises
 */
typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)])
	perf_trace_t;

/* Count the events in use (per event id, not per instance) */
static int	total_ref_count;

static int perf_trace_event_perm(struct trace_event_call *tp_event,
				 struct perf_event *p_event)
{
	if (tp_event->perf_perm) {
		int ret = tp_event->perf_perm(tp_event, p_event);
		if (ret)
			return ret;
	}

	/*
	 * We checked and allowed to create parent,
	 * allow children without checking.
	 */
	if (p_event->parent)
		return 0;

	/*
	 * It's ok to check current process (owner) permissions in here,
	 * because code below is called only via perf_event_open syscall.
	 */

	/* The ftrace function trace is allowed only for root. */
	if (ftrace_event_is_function(tp_event)) {
		if (perf_paranoid_tracepoint_raw() && !capable(CAP_SYS_ADMIN))
			return -EPERM;

		if (!is_sampling_event(p_event))
			return 0;

		/*
		 * We don't allow user space callchains for  function trace
		 * event, due to issues with page faults while tracing page
		 * fault handler and its overall trickiness nature.
		 */
		if (!p_event->attr.exclude_callchain_user)
			return -EINVAL;

		/*
		 * Same reason to disable user stack dump as for user space
		 * callchains above.
		 */
		if (p_event->attr.sample_type & PERF_SAMPLE_STACK_USER)
			return -EINVAL;
	}

	/* No tracing, just counting, so no obvious leak */
	if (!(p_event->attr.sample_type & PERF_SAMPLE_RAW))
		return 0;

	/* Some events are ok to be traced by non-root users... */
	if (p_event->attach_state == PERF_ATTACH_TASK) {
		if (tp_event->flags & TRACE_EVENT_FL_CAP_ANY)
			return 0;
	}

	/*
	 * ...otherwise raw tracepoint data can be a severe data leak,
	 * only allow root to have these.
	 */
	if (perf_paranoid_tracepoint_raw() && !capable(CAP_SYS_ADMIN))
		return -EPERM;

	return 0;
}

static int perf_trace_event_reg(struct trace_event_call *tp_event,
				struct perf_event *p_event)
{
	struct hlist_head __percpu *list;
	int ret = -ENOMEM;
	int cpu;

	p_event->tp_event = tp_event;
	if (tp_event->perf_refcount++ > 0)
		return 0;

	list = alloc_percpu(struct hlist_head);
	if (!list)
		goto fail;

	for_each_possible_cpu(cpu)
		INIT_HLIST_HEAD(per_cpu_ptr(list, cpu));

	tp_event->perf_events = list;

	if (!total_ref_count) {
		char __percpu *buf;
		int i;

		for (i = 0; i < PERF_NR_CONTEXTS; i++) {
			buf = (char __percpu *)alloc_percpu(perf_trace_t);
			if (!buf)
				goto fail;

			perf_trace_buf[i] = buf;
		}
	}

	ret = tp_event->class->reg(tp_event, TRACE_REG_PERF_REGISTER, NULL);
	if (ret)
		goto fail;

	total_ref_count++;
	return 0;

fail:
	if (!total_ref_count) {
		int i;

		for (i = 0; i < PERF_NR_CONTEXTS; i++) {
			free_percpu(perf_trace_buf[i]);
			perf_trace_buf[i] = NULL;
		}
	}

	if (!--tp_event->perf_refcount) {
		free_percpu(tp_event->perf_events);
		tp_event->perf_events = NULL;
	}

	return ret;
}

static void perf_trace_event_unreg(struct perf_event *p_event)
{
	struct trace_event_call *tp_event = p_event->tp_event;
	int i;

	if (--tp_event->perf_refcount > 0)
		goto out;

	tp_event->class->reg(tp_event, TRACE_REG_PERF_UNREGISTER, NULL);

	/*
	 * Ensure our callback won't be called anymore. The buffers
	 * will be freed after that.
	 */
	tracepoint_synchronize_unregister();

	free_percpu(tp_event->perf_events);
	tp_event->perf_events = NULL;

	if (!--total_ref_count) {
		for (i = 0; i < PERF_NR_CONTEXTS; i++) {
			free_percpu(perf_trace_buf[i]);
			perf_trace_buf[i] = NULL;
		}
	}
out:
	module_put(tp_event->mod);
}

static int perf_trace_event_open(struct perf_event *p_event)
{
	struct trace_event_call *tp_event = p_event->tp_event;
	return tp_event->class->reg(tp_event, TRACE_REG_PERF_OPEN, p_event);
}

static void perf_trace_event_close(struct perf_event *p_event)
{
	struct trace_event_call *tp_event = p_event->tp_event;
	tp_event->class->reg(tp_event, TRACE_REG_PERF_CLOSE, p_event);
}

static int perf_trace_event_init(struct trace_event_call *tp_event,
				 struct perf_event *p_event)
{
	int ret;

	ret = perf_trace_event_perm(tp_event, p_event);
	if (ret)
		return ret;

	ret = perf_trace_event_reg(tp_event, p_event);
	if (ret)
		return ret;

	ret = perf_trace_event_open(p_event);
	if (ret) {
		perf_trace_event_unreg(p_event);
		return ret;
	}

	return 0;
}

int perf_trace_init(struct perf_event *p_event)
{
	struct trace_event_call *tp_event;
	u64 event_id = p_event->attr.config;
	int ret = -EINVAL;

	mutex_lock(&event_mutex);
	list_for_each_entry(tp_event, &ftrace_events, list) {
		if (tp_event->event.type == event_id &&
		    tp_event->class && tp_event->class->reg &&
		    try_module_get(tp_event->mod)) {
			ret = perf_trace_event_init(tp_event, p_event);
			if (ret)
				module_put(tp_event->mod);
			break;
		}
	}
	mutex_unlock(&event_mutex);

	return ret;
}

void perf_trace_destroy(struct perf_event *p_event)
{
	mutex_lock(&event_mutex);
	perf_trace_event_close(p_event);
	perf_trace_event_unreg(p_event);
	mutex_unlock(&event_mutex);
}

#ifdef CONFIG_KPROBE_EVENTS
int perf_kprobe_init(struct perf_event *p_event, bool is_retprobe)
{
	int ret;
	char *func = NULL;
	struct trace_event_call *tp_event;

	if (p_event->attr.kprobe_func) {
		func = kzalloc(KSYM_NAME_LEN, GFP_KERNEL);
		if (!func)
			return -ENOMEM;
		ret = strncpy_from_user(
			func, u64_to_user_ptr(p_event->attr.kprobe_func),
			KSYM_NAME_LEN);
		if (ret == KSYM_NAME_LEN)
			ret = -E2BIG;
		if (ret < 0)
			goto out;

		if (func[0] == '\0') {
			kfree(func);
			func = NULL;
		}
	}

	tp_event = create_local_trace_kprobe(
		func, (void *)(unsigned long)(p_event->attr.kprobe_addr),
		p_event->attr.probe_offset, is_retprobe);
	if (IS_ERR(tp_event)) {
		ret = PTR_ERR(tp_event);
		goto out;
	}

	ret = perf_trace_event_init(tp_event, p_event);
	if (ret)
		destroy_local_trace_kprobe(tp_event);
out:
	kfree(func);
	return ret;
}

void perf_kprobe_destroy(struct perf_event *p_event)
{
	perf_trace_event_close(p_event);
	perf_trace_event_unreg(p_event);

	destroy_local_trace_kprobe(p_event->tp_event);
}
#endif /* CONFIG_KPROBE_EVENTS */

#ifdef CONFIG_UPROBE_EVENTS
int perf_uprobe_init(struct perf_event *p_event,
		     unsigned long ref_ctr_offset, bool is_retprobe)
{
	int ret;
	char *path = NULL;
	struct trace_event_call *tp_event;

	if (!p_event->attr.uprobe_path)
		return -EINVAL;

	path = strndup_user(u64_to_user_ptr(p_event->attr.uprobe_path),
			    PATH_MAX);
	if (IS_ERR(path)) {
		ret = PTR_ERR(path);
		return (ret == -EINVAL) ? -E2BIG : ret;
	}
	if (path[0] == '\0') {
		ret = -EINVAL;
		goto out;
	}

	tp_event = create_local_trace_uprobe(path, p_event->attr.probe_offset,
					     ref_ctr_offset, is_retprobe);
	if (IS_ERR(tp_event)) {
		ret = PTR_ERR(tp_event);
		goto out;
	}

	/*
	 * local trace_uprobe need to hold event_mutex to call
	 * uprobe_buffer_enable() and uprobe_buffer_disable().
	 * event_mutex is not required for local trace_kprobes.
	 */
	mutex_lock(&event_mutex);
	ret = perf_trace_event_init(tp_event, p_event);
	if (ret)
		destroy_local_trace_uprobe(tp_event);
	mutex_unlock(&event_mutex);
out:
	kfree(path);
	return ret;
}

void perf_uprobe_destroy(struct perf_event *p_event)
{
	mutex_lock(&event_mutex);
	perf_trace_event_close(p_event);
	perf_trace_event_unreg(p_event);
	mutex_unlock(&event_mutex);
	destroy_local_trace_uprobe(p_event->tp_event);
}
#endif /* CONFIG_UPROBE_EVENTS */

int perf_trace_add(struct perf_event *p_event, int flags)
{
	struct trace_event_call *tp_event = p_event->tp_event;

	if (!(flags & PERF_EF_START))
		p_event->hw.state = PERF_HES_STOPPED;

	/*
	 * If TRACE_REG_PERF_ADD returns false; no custom action was performed
	 * and we need to take the default action of enqueueing our event on
	 * the right per-cpu hlist.
	 */
	if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_ADD, p_event)) {
		struct hlist_head __percpu *pcpu_list;
		struct hlist_head *list;

		pcpu_list = tp_event->perf_events;
		if (WARN_ON_ONCE(!pcpu_list))
			return -EINVAL;

		list = this_cpu_ptr(pcpu_list);
		hlist_add_head_rcu(&p_event->hlist_entry, list);
	}

	return 0;
}

void perf_trace_del(struct perf_event *p_event, int flags)
{
	struct trace_event_call *tp_event = p_event->tp_event;

	/*
	 * If TRACE_REG_PERF_DEL returns false; no custom action was performed
	 * and we need to take the default action of dequeueing our event from
	 * the right per-cpu hlist.
	 */
	if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event))
		hlist_del_rcu(&p_event->hlist_entry);
}

void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp)
{
	char *raw_data;
	int rctx;

	BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));

	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
		      "perf buffer not large enough"))
		return NULL;

	*rctxp = rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		return NULL;

	if (regs)
		*regs = this_cpu_ptr(&__perf_regs[rctx]);
	raw_data = this_cpu_ptr(perf_trace_buf[rctx]);

	/* zero the dead bytes from align to not leak stack to user */
	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
	return raw_data;
}
EXPORT_SYMBOL_GPL(perf_trace_buf_alloc);
NOKPROBE_SYMBOL(perf_trace_buf_alloc);

void perf_trace_buf_update(void *record, u16 type)
{
	struct trace_entry *entry = record;
	int pc = preempt_count();
	unsigned long flags;

	local_save_flags(flags);
	tracing_generic_entry_update(entry, flags, pc);
	entry->type = type;
}
NOKPROBE_SYMBOL(perf_trace_buf_update);

#ifdef CONFIG_FUNCTION_TRACER
static void
perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
			  struct ftrace_ops *ops, struct pt_regs *pt_regs)
{
	struct ftrace_entry *entry;
	struct perf_event *event;
	struct hlist_head head;
	struct pt_regs regs;
	int rctx;

	if ((unsigned long)ops->private != smp_processor_id())
		return;

	event = container_of(ops, struct perf_event, ftrace_ops);

	/*
	 * @event->hlist entry is NULL (per INIT_HLIST_NODE), and all
	 * the perf code does is hlist_for_each_entry_rcu(), so we can
	 * get away with simply setting the @head.first pointer in order
	 * to create a singular list.
	 */
	head.first = &event->hlist_entry;

#define ENTRY_SIZE (ALIGN(sizeof(struct ftrace_entry) + sizeof(u32), \
		    sizeof(u64)) - sizeof(u32))

	BUILD_BUG_ON(ENTRY_SIZE > PERF_MAX_TRACE_SIZE);

	memset(&regs, 0, sizeof(regs));
	perf_fetch_caller_regs(&regs);

	entry = perf_trace_buf_alloc(ENTRY_SIZE, NULL, &rctx);
	if (!entry)
		return;

	entry->ip = ip;
	entry->parent_ip = parent_ip;
	perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, TRACE_FN,
			      1, &regs, &head, NULL);

#undef ENTRY_SIZE
}

static int perf_ftrace_function_register(struct perf_event *event)
{
	struct ftrace_ops *ops = &event->ftrace_ops;

	ops->flags   = FTRACE_OPS_FL_RCU;
	ops->func    = perf_ftrace_function_call;
	ops->private = (void *)(unsigned long)nr_cpu_ids;

	return register_ftrace_function(ops);
}

static int perf_ftrace_function_unregister(struct perf_event *event)
{
	struct ftrace_ops *ops = &event->ftrace_ops;
	int ret = unregister_ftrace_function(ops);
	ftrace_free_filter(ops);
	return ret;
}

int perf_ftrace_event_register(struct trace_event_call *call,
			       enum trace_reg type, void *data)
{
	struct perf_event *event = data;

	switch (type) {
	case TRACE_REG_REGISTER:
	case TRACE_REG_UNREGISTER:
		break;
	case TRACE_REG_PERF_REGISTER:
	case TRACE_REG_PERF_UNREGISTER:
		return 0;
	case TRACE_REG_PERF_OPEN:
		return perf_ftrace_function_register(data);
	case TRACE_REG_PERF_CLOSE:
		return perf_ftrace_function_unregister(data);
	case TRACE_REG_PERF_ADD:
		event->ftrace_ops.private = (void *)(unsigned long)smp_processor_id();
		return 1;
	case TRACE_REG_PERF_DEL:
		event->ftrace_ops.private = (void *)(unsigned long)nr_cpu_ids;
		return 1;
	}

	return -EINVAL;
}
#endif /* CONFIG_FUNCTION_TRACER */
Commit	Line	Data
bcea3f96	1	// SPDX-License-Identifier: GPL-2.0
ac199db0	2	/*
97d5a220	3	* trace event based perf event profiling/tracing
ac199db0	4	*
90eec103	5	* Copyright (C) 2009 Red Hat Inc, Peter Zijlstra
c530665c	6	* Copyright (C) 2009-2010 Frederic Weisbecker <fweisbec@gmail.com>
ac199db0 PZ	7	*/
ac199db0 PZ	8
558e6547	9	#include <linux/module.h>
430ad5a6	10	#include <linux/kprobes.h>
ac199db0	11	#include "trace.h"
e12f03d7	12	#include "trace_probe.h"
ac199db0	13
6016ee13	14	static char __percpu *perf_trace_buf[PERF_NR_CONTEXTS];
20ab4425	15
eb1e7961 FW	16	/*
	17	* Force it to be aligned to unsigned long to avoid misaligned accesses
	18	* suprises
	19	*/
	20	typedef typeof(unsigned long [PERF_MAX_TRACE_SIZE / sizeof(unsigned long)])
	21	perf_trace_t;
ce71b9df	22
20ab4425	23	/* Count the events in use (per event id, not per instance) */
97d5a220	24	static int total_ref_count;
20ab4425	25
2425bcb9	26	static int perf_trace_event_perm(struct trace_event_call *tp_event,
61c32659 FW	27	struct perf_event *p_event)
61c32659 FW	28	{
d5b5f391 PZ	29	if (tp_event->perf_perm) {
	30	int ret = tp_event->perf_perm(tp_event, p_event);
	31	if (ret)
	32	return ret;
	33	}
	34
f4be073d JO	35	/*
	36	* We checked and allowed to create parent,
	37	* allow children without checking.
	38	*/
	39	if (p_event->parent)
	40	return 0;
	41
	42	/*
	43	* It's ok to check current process (owner) permissions in here,
	44	* because code below is called only via perf_event_open syscall.
	45	*/
	46
ced39002	47	/* The ftrace function trace is allowed only for root. */
cfa77bc4 JO	48	if (ftrace_event_is_function(tp_event)) {
	49	if (perf_paranoid_tracepoint_raw() && !capable(CAP_SYS_ADMIN))
	50	return -EPERM;
	51
0a74c5b3 JO	52	if (!is_sampling_event(p_event))
	53	return 0;
	54
cfa77bc4 JO	55	/*
	56	* We don't allow user space callchains for function trace
	57	* event, due to issues with page faults while tracing page
	58	* fault handler and its overall trickiness nature.
	59	*/
	60	if (!p_event->attr.exclude_callchain_user)
	61	return -EINVAL;
63c45f4b JO	62
	63	/*
	64	* Same reason to disable user stack dump as for user space
	65	* callchains above.
	66	*/
	67	if (p_event->attr.sample_type & PERF_SAMPLE_STACK_USER)
	68	return -EINVAL;
cfa77bc4	69	}
ced39002	70
61c32659 FW	71	/* No tracing, just counting, so no obvious leak */
	72	if (!(p_event->attr.sample_type & PERF_SAMPLE_RAW))
	73	return 0;
	74
	75	/* Some events are ok to be traced by non-root users... */
	76	if (p_event->attach_state == PERF_ATTACH_TASK) {
	77	if (tp_event->flags & TRACE_EVENT_FL_CAP_ANY)
	78	return 0;
	79	}
	80
	81	/*
	82	* ...otherwise raw tracepoint data can be a severe data leak,
	83	* only allow root to have these.
	84	*/
	85	if (perf_paranoid_tracepoint_raw() && !capable(CAP_SYS_ADMIN))
	86	return -EPERM;
	87
	88	return 0;
	89	}
	90
2425bcb9	91	static int perf_trace_event_reg(struct trace_event_call *tp_event,
ceec0b6f	92	struct perf_event *p_event)
e5e25cf4	93	{
6016ee13	94	struct hlist_head __percpu *list;
ceec0b6f	95	int ret = -ENOMEM;
1c024eca	96	int cpu;
20ab4425	97
1c024eca PZ	98	p_event->tp_event = tp_event;
1c024eca PZ	99	if (tp_event->perf_refcount++ > 0)
e5e25cf4 FW	100	return 0;
e5e25cf4 FW	101
1c024eca PZ	102	list = alloc_percpu(struct hlist_head);
	103	if (!list)
	104	goto fail;
	105
	106	for_each_possible_cpu(cpu)
	107	INIT_HLIST_HEAD(per_cpu_ptr(list, cpu));
20ab4425	108
1c024eca	109	tp_event->perf_events = list;
e5e25cf4	110
97d5a220	111	if (!total_ref_count) {
6016ee13	112	char __percpu *buf;
b7e2ecef	113	int i;
20ab4425	114
7ae07ea3	115	for (i = 0; i < PERF_NR_CONTEXTS; i++) {
6016ee13	116	buf = (char __percpu *)alloc_percpu(perf_trace_t);
b7e2ecef	117	if (!buf)
1c024eca	118	goto fail;
20ab4425	119
1c024eca	120	perf_trace_buf[i] = buf;
b7e2ecef	121	}
20ab4425 FW	122	}
20ab4425 FW	123
ceec0b6f	124	ret = tp_event->class->reg(tp_event, TRACE_REG_PERF_REGISTER, NULL);
1c024eca PZ	125	if (ret)
1c024eca PZ	126	goto fail;
20ab4425	127
1c024eca PZ	128	total_ref_count++;
	129	return 0;
	130
	131	fail:
97d5a220	132	if (!total_ref_count) {
b7e2ecef PZ	133	int i;
b7e2ecef PZ	134
7ae07ea3	135	for (i = 0; i < PERF_NR_CONTEXTS; i++) {
b7e2ecef PZ	136	free_percpu(perf_trace_buf[i]);
	137	perf_trace_buf[i] = NULL;
	138	}
fe8e5b5a	139	}
1c024eca PZ	140
	141	if (!--tp_event->perf_refcount) {
	142	free_percpu(tp_event->perf_events);
	143	tp_event->perf_events = NULL;
fe8e5b5a	144	}
20ab4425 FW	145
20ab4425 FW	146	return ret;
e5e25cf4 FW	147	}
e5e25cf4 FW	148
ceec0b6f JO	149	static void perf_trace_event_unreg(struct perf_event *p_event)
ceec0b6f JO	150	{
2425bcb9	151	struct trace_event_call *tp_event = p_event->tp_event;
ceec0b6f JO	152	int i;
	153
	154	if (--tp_event->perf_refcount > 0)
	155	goto out;
	156
	157	tp_event->class->reg(tp_event, TRACE_REG_PERF_UNREGISTER, NULL);
	158
	159	/*
	160	* Ensure our callback won't be called anymore. The buffers
	161	* will be freed after that.
	162	*/
	163	tracepoint_synchronize_unregister();
	164
	165	free_percpu(tp_event->perf_events);
	166	tp_event->perf_events = NULL;
	167
	168	if (!--total_ref_count) {
	169	for (i = 0; i < PERF_NR_CONTEXTS; i++) {
	170	free_percpu(perf_trace_buf[i]);
	171	perf_trace_buf[i] = NULL;
	172	}
	173	}
	174	out:
	175	module_put(tp_event->mod);
	176	}
	177
	178	static int perf_trace_event_open(struct perf_event *p_event)
	179	{
2425bcb9	180	struct trace_event_call *tp_event = p_event->tp_event;
ceec0b6f JO	181	return tp_event->class->reg(tp_event, TRACE_REG_PERF_OPEN, p_event);
	182	}
	183
	184	static void perf_trace_event_close(struct perf_event *p_event)
	185	{
2425bcb9	186	struct trace_event_call *tp_event = p_event->tp_event;
ceec0b6f JO	187	tp_event->class->reg(tp_event, TRACE_REG_PERF_CLOSE, p_event);
	188	}
	189
2425bcb9	190	static int perf_trace_event_init(struct trace_event_call *tp_event,
ceec0b6f JO	191	struct perf_event *p_event)
	192	{
	193	int ret;
	194
	195	ret = perf_trace_event_perm(tp_event, p_event);
	196	if (ret)
	197	return ret;
	198
	199	ret = perf_trace_event_reg(tp_event, p_event);
	200	if (ret)
	201	return ret;
	202
	203	ret = perf_trace_event_open(p_event);
	204	if (ret) {
	205	perf_trace_event_unreg(p_event);
	206	return ret;
	207	}
	208
	209	return 0;
	210	}
	211
1c024eca	212	int perf_trace_init(struct perf_event *p_event)
ac199db0	213	{
2425bcb9	214	struct trace_event_call *tp_event;
0022cedd	215	u64 event_id = p_event->attr.config;
20c8928a	216	int ret = -EINVAL;
ac199db0	217
20c8928a	218	mutex_lock(&event_mutex);
1c024eca	219	list_for_each_entry(tp_event, &ftrace_events, list) {
ff5f149b	220	if (tp_event->event.type == event_id &&
a1d0ce82	221	tp_event->class && tp_event->class->reg &&
1c024eca PZ	222	try_module_get(tp_event->mod)) {
1c024eca PZ	223	ret = perf_trace_event_init(tp_event, p_event);
9cb627d5 LZ	224	if (ret)
9cb627d5 LZ	225	module_put(tp_event->mod);
20c8928a LZ	226	break;
20c8928a LZ	227	}
ac199db0	228	}
20c8928a	229	mutex_unlock(&event_mutex);
ac199db0	230
20c8928a	231	return ret;
ac199db0 PZ	232	}
ac199db0 PZ	233
ceec0b6f JO	234	void perf_trace_destroy(struct perf_event *p_event)
	235	{
	236	mutex_lock(&event_mutex);
	237	perf_trace_event_close(p_event);
	238	perf_trace_event_unreg(p_event);
	239	mutex_unlock(&event_mutex);
	240	}
	241
e12f03d7 SL	242	#ifdef CONFIG_KPROBE_EVENTS
	243	int perf_kprobe_init(struct perf_event *p_event, bool is_retprobe)
	244	{
	245	int ret;
	246	char *func = NULL;
	247	struct trace_event_call *tp_event;
	248
	249	if (p_event->attr.kprobe_func) {
	250	func = kzalloc(KSYM_NAME_LEN, GFP_KERNEL);
	251	if (!func)
	252	return -ENOMEM;
	253	ret = strncpy_from_user(
	254	func, u64_to_user_ptr(p_event->attr.kprobe_func),
	255	KSYM_NAME_LEN);
5da13ab8 MH	256	if (ret == KSYM_NAME_LEN)
5da13ab8 MH	257	ret = -E2BIG;
e12f03d7 SL	258	if (ret < 0)
	259	goto out;
	260
	261	if (func[0] == '\0') {
	262	kfree(func);
	263	func = NULL;
	264	}
	265	}
	266
	267	tp_event = create_local_trace_kprobe(
	268	func, (void *)(unsigned long)(p_event->attr.kprobe_addr),
	269	p_event->attr.probe_offset, is_retprobe);
	270	if (IS_ERR(tp_event)) {
	271	ret = PTR_ERR(tp_event);
	272	goto out;
	273	}
	274
	275	ret = perf_trace_event_init(tp_event, p_event);
	276	if (ret)
	277	destroy_local_trace_kprobe(tp_event);
	278	out:
	279	kfree(func);
	280	return ret;
	281	}
	282
	283	void perf_kprobe_destroy(struct perf_event *p_event)
	284	{
	285	perf_trace_event_close(p_event);
	286	perf_trace_event_unreg(p_event);
	287
	288	destroy_local_trace_kprobe(p_event->tp_event);
	289	}
	290	#endif /* CONFIG_KPROBE_EVENTS */
	291
33ea4b24	292	#ifdef CONFIG_UPROBE_EVENTS
a6ca88b2 SL	293	int perf_uprobe_init(struct perf_event *p_event,
a6ca88b2 SL	294	unsigned long ref_ctr_offset, bool is_retprobe)
33ea4b24 SL	295	{
	296	int ret;
	297	char *path = NULL;
	298	struct trace_event_call *tp_event;
	299
	300	if (!p_event->attr.uprobe_path)
	301	return -EINVAL;
83540fbc JH	302
	303	path = strndup_user(u64_to_user_ptr(p_event->attr.uprobe_path),
	304	PATH_MAX);
	305	if (IS_ERR(path)) {
	306	ret = PTR_ERR(path);
	307	return (ret == -EINVAL) ? -E2BIG : ret;
	308	}
33ea4b24 SL	309	if (path[0] == '\0') {
	310	ret = -EINVAL;
	311	goto out;
	312	}
	313
a6ca88b2 SL	314	tp_event = create_local_trace_uprobe(path, p_event->attr.probe_offset,
a6ca88b2 SL	315	ref_ctr_offset, is_retprobe);
33ea4b24 SL	316	if (IS_ERR(tp_event)) {
	317	ret = PTR_ERR(tp_event);
	318	goto out;
	319	}
	320
	321	/*
	322	* local trace_uprobe need to hold event_mutex to call
	323	* uprobe_buffer_enable() and uprobe_buffer_disable().
	324	* event_mutex is not required for local trace_kprobes.
	325	*/
	326	mutex_lock(&event_mutex);
	327	ret = perf_trace_event_init(tp_event, p_event);
	328	if (ret)
	329	destroy_local_trace_uprobe(tp_event);
	330	mutex_unlock(&event_mutex);
	331	out:
	332	kfree(path);
	333	return ret;
	334	}
	335
	336	void perf_uprobe_destroy(struct perf_event *p_event)
	337	{
	338	mutex_lock(&event_mutex);
	339	perf_trace_event_close(p_event);
	340	perf_trace_event_unreg(p_event);
	341	mutex_unlock(&event_mutex);
	342	destroy_local_trace_uprobe(p_event->tp_event);
	343	}
	344	#endif /* CONFIG_UPROBE_EVENTS */
	345
a4eaf7f1	346	int perf_trace_add(struct perf_event *p_event, int flags)
e5e25cf4	347	{
2425bcb9	348	struct trace_event_call *tp_event = p_event->tp_event;
20ab4425	349
a4eaf7f1 PZ	350	if (!(flags & PERF_EF_START))
	351	p_event->hw.state = PERF_HES_STOPPED;
	352
466c81c4 PZ	353	/*
	354	* If TRACE_REG_PERF_ADD returns false; no custom action was performed
	355	* and we need to take the default action of enqueueing our event on
	356	* the right per-cpu hlist.
	357	*/
	358	if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_ADD, p_event)) {
	359	struct hlist_head __percpu *pcpu_list;
	360	struct hlist_head *list;
	361
	362	pcpu_list = tp_event->perf_events;
	363	if (WARN_ON_ONCE(!pcpu_list))
	364	return -EINVAL;
	365
	366	list = this_cpu_ptr(pcpu_list);
	367	hlist_add_head_rcu(&p_event->hlist_entry, list);
	368	}
20ab4425	369
466c81c4	370	return 0;
1c024eca	371	}
20ab4425	372
a4eaf7f1	373	void perf_trace_del(struct perf_event *p_event, int flags)
1c024eca	374	{
2425bcb9	375	struct trace_event_call *tp_event = p_event->tp_event;
466c81c4 PZ	376
	377	/*
	378	* If TRACE_REG_PERF_DEL returns false; no custom action was performed
	379	* and we need to take the default action of dequeueing our event from
	380	* the right per-cpu hlist.
	381	*/
	382	if (!tp_event->class->reg(tp_event, TRACE_REG_PERF_DEL, p_event))
	383	hlist_del_rcu(&p_event->hlist_entry);
e5e25cf4 FW	384	}
e5e25cf4 FW	385
1e1dcd93	386	void perf_trace_buf_alloc(int size, struct pt_regs regs, int rctxp)
430ad5a6	387	{
1c024eca	388	char *raw_data;
1e1dcd93	389	int rctx;
430ad5a6	390
eb1e7961 FW	391	BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(unsigned long));
eb1e7961 FW	392
cd92bf61	393	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE,
1e1dcd93	394	"perf buffer not large enough"))
cd92bf61 ON	395	return NULL;
cd92bf61 ON	396
1e1dcd93 AS	397	*rctxp = rctx = perf_swevent_get_recursion_context();
1e1dcd93 AS	398	if (rctx < 0)
1c024eca	399	return NULL;
430ad5a6	400
86038c5e	401	if (regs)
1e1dcd93 AS	402	*regs = this_cpu_ptr(&__perf_regs[rctx]);
1e1dcd93 AS	403	raw_data = this_cpu_ptr(perf_trace_buf[rctx]);
430ad5a6 XG	404
430ad5a6 XG	405	/* zero the dead bytes from align to not leak stack to user */
eb1e7961	406	memset(&raw_data[size - sizeof(u64)], 0, sizeof(u64));
1e1dcd93 AS	407	return raw_data;
	408	}
	409	EXPORT_SYMBOL_GPL(perf_trace_buf_alloc);
	410	NOKPROBE_SYMBOL(perf_trace_buf_alloc);
	411
	412	void perf_trace_buf_update(void *record, u16 type)
	413	{
	414	struct trace_entry *entry = record;
	415	int pc = preempt_count();
	416	unsigned long flags;
430ad5a6	417
87f44bbc PZ	418	local_save_flags(flags);
87f44bbc PZ	419	tracing_generic_entry_update(entry, flags, pc);
430ad5a6	420	entry->type = type;
430ad5a6	421	}
1e1dcd93	422	NOKPROBE_SYMBOL(perf_trace_buf_update);
ced39002 JO	423
	424	#ifdef CONFIG_FUNCTION_TRACER
	425	static void
2f5f6ad9	426	perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
a1e2e31d	427	struct ftrace_ops ops, struct pt_regs pt_regs)
ced39002 JO	428	{
ced39002 JO	429	struct ftrace_entry *entry;
466c81c4 PZ	430	struct perf_event *event;
466c81c4 PZ	431	struct hlist_head head;
ced39002 JO	432	struct pt_regs regs;
	433	int rctx;
	434
466c81c4	435	if ((unsigned long)ops->private != smp_processor_id())
b8ebfd3f ON	436	return;
b8ebfd3f ON	437
466c81c4 PZ	438	event = container_of(ops, struct perf_event, ftrace_ops);
	439
	440	/*
	441	* @event->hlist entry is NULL (per INIT_HLIST_NODE), and all
	442	* the perf code does is hlist_for_each_entry_rcu(), so we can
	443	* get away with simply setting the @head.first pointer in order
	444	* to create a singular list.
	445	*/
	446	head.first = &event->hlist_entry;
	447
ced39002 JO	448	#define ENTRY_SIZE (ALIGN(sizeof(struct ftrace_entry) + sizeof(u32), \
	449	sizeof(u64)) - sizeof(u32))
	450
	451	BUILD_BUG_ON(ENTRY_SIZE > PERF_MAX_TRACE_SIZE);
	452
ec5e099d	453	memset(&regs, 0, sizeof(regs));
ced39002 JO	454	perf_fetch_caller_regs(&regs);
ced39002 JO	455
1e1dcd93	456	entry = perf_trace_buf_alloc(ENTRY_SIZE, NULL, &rctx);
ced39002 JO	457	if (!entry)
	458	return;
	459
	460	entry->ip = ip;
	461	entry->parent_ip = parent_ip;
1e1dcd93	462	perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, TRACE_FN,
466c81c4	463	1, &regs, &head, NULL);
ced39002 JO	464
	465	#undef ENTRY_SIZE
	466	}
	467
	468	static int perf_ftrace_function_register(struct perf_event *event)
	469	{
	470	struct ftrace_ops *ops = &event->ftrace_ops;
	471
1dd311e6	472	ops->flags = FTRACE_OPS_FL_RCU;
466c81c4 PZ	473	ops->func = perf_ftrace_function_call;
	474	ops->private = (void *)(unsigned long)nr_cpu_ids;
	475
ced39002 JO	476	return register_ftrace_function(ops);
	477	}
	478
	479	static int perf_ftrace_function_unregister(struct perf_event *event)
	480	{
	481	struct ftrace_ops *ops = &event->ftrace_ops;
5500fa51 JO	482	int ret = unregister_ftrace_function(ops);
	483	ftrace_free_filter(ops);
	484	return ret;
ced39002 JO	485	}
ced39002 JO	486
2425bcb9	487	int perf_ftrace_event_register(struct trace_event_call *call,
ced39002 JO	488	enum trace_reg type, void *data)
ced39002 JO	489	{
466c81c4 PZ	490	struct perf_event *event = data;
466c81c4 PZ	491
ced39002 JO	492	switch (type) {
	493	case TRACE_REG_REGISTER:
	494	case TRACE_REG_UNREGISTER:
	495	break;
	496	case TRACE_REG_PERF_REGISTER:
	497	case TRACE_REG_PERF_UNREGISTER:
	498	return 0;
	499	case TRACE_REG_PERF_OPEN:
	500	return perf_ftrace_function_register(data);
	501	case TRACE_REG_PERF_CLOSE:
	502	return perf_ftrace_function_unregister(data);
	503	case TRACE_REG_PERF_ADD:
466c81c4 PZ	504	event->ftrace_ops.private = (void *)(unsigned long)smp_processor_id();
466c81c4 PZ	505	return 1;
ced39002	506	case TRACE_REG_PERF_DEL:
466c81c4 PZ	507	event->ftrace_ops.private = (void *)(unsigned long)nr_cpu_ids;
466c81c4 PZ	508	return 1;
ced39002 JO	509	}
	510
	511	return -EINVAL;
	512	}
	513	#endif /* CONFIG_FUNCTION_TRACER */