[linux-2.6-block.git] / kernel / trace / trace_syscalls.c

#include <trace/syscall.h>
#include <trace/events/syscalls.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/perf_event.h>
#include <asm/syscall.h>

#include "trace_output.h"
#include "trace.h"

static DEFINE_MUTEX(syscall_trace_lock);
static int sys_refcount_enter;
static int sys_refcount_exit;
static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);

extern unsigned long __start_syscalls_metadata[];
extern unsigned long __stop_syscalls_metadata[];

static struct syscall_metadata **syscalls_metadata;

static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
{
	struct syscall_metadata *start;
	struct syscall_metadata *stop;
	char str[KSYM_SYMBOL_LEN];


	start = (struct syscall_metadata *)__start_syscalls_metadata;
	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	kallsyms_lookup(syscall, NULL, NULL, NULL, str);

	for ( ; start < stop; start++) {
		/*
		 * Only compare after the "sys" prefix. Archs that use
		 * syscall wrappers may have syscalls symbols aliases prefixed
		 * with "SyS" instead of "sys", leading to an unwanted
		 * mismatch.
		 */
		if (start->name && !strcmp(start->name + 3, str + 3))
			return start;
	}
	return NULL;
}

static struct syscall_metadata *syscall_nr_to_meta(int nr)
{
	if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
		return NULL;

	return syscalls_metadata[nr];
}

enum print_line_t
print_syscall_enter(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_enter *trace;
	struct syscall_metadata *entry;
	int i, ret, syscall;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry)
		goto end;

	if (entry->enter_event->id != ent->type) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = trace_seq_printf(s, "%s(", entry->name);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	for (i = 0; i < entry->nb_args; i++) {
		/* parameter types */
		if (trace_flags & TRACE_ITER_VERBOSE) {
			ret = trace_seq_printf(s, "%s ", entry->types[i]);
			if (!ret)
				return TRACE_TYPE_PARTIAL_LINE;
		}
		/* parameter values */
		ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
				       trace->args[i],
				       i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return TRACE_TYPE_PARTIAL_LINE;
	}

	ret = trace_seq_putc(s, ')');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

end:
	ret =  trace_seq_putc(s, '\n');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

enum print_line_t
print_syscall_exit(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_exit *trace;
	int syscall;
	struct syscall_metadata *entry;
	int ret;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry) {
		trace_seq_printf(s, "\n");
		return TRACE_TYPE_HANDLED;
	}

	if (entry->exit_event->id != ent->type) {
		WARN_ON_ONCE(1);
		return TRACE_TYPE_UNHANDLED;
	}

	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
				trace->ret);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

extern char *__bad_type_size(void);

#define SYSCALL_FIELD(type, name)					\
	sizeof(type) != sizeof(trace.name) ?				\
		__bad_type_size() :					\
		#type, #name, offsetof(typeof(trace), name),		\
		sizeof(trace.name), is_signed_type(type)

int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s)
{
	int i;
	int ret;
	struct syscall_metadata *entry = call->data;
	struct syscall_trace_enter trace;
	int offset = offsetof(struct syscall_trace_enter, args);

	ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n",
			       SYSCALL_FIELD(int, nr));
	if (!ret)
		return 0;

	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
				        entry->args[i]);
		if (!ret)
			return 0;
		ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;"
				       "\tsigned:%u;\n", offset,
				       sizeof(unsigned long),
				       is_signed_type(unsigned long));
		if (!ret)
			return 0;
		offset += sizeof(unsigned long);
	}

	trace_seq_puts(s, "\nprint fmt: \"");
	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
				        sizeof(unsigned long),
					i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return 0;
	}
	trace_seq_putc(s, '"');

	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
				       entry->args[i]);
		if (!ret)
			return 0;
	}

	return trace_seq_putc(s, '\n');
}

int syscall_exit_format(struct ftrace_event_call *call, struct trace_seq *s)
{
	int ret;
	struct syscall_trace_exit trace;

	ret = trace_seq_printf(s,
			       "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n"
			       "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n",
			       SYSCALL_FIELD(int, nr),
			       SYSCALL_FIELD(long, ret));
	if (!ret)
		return 0;

	return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
}

int syscall_enter_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_enter trace;
	struct syscall_metadata *meta = call->data;
	int ret;
	int i;
	int offset = offsetof(typeof(trace), args);

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	for (i = 0; i < meta->nb_args; i++) {
		ret = trace_define_field(call, meta->types[i],
					 meta->args[i], offset,
					 sizeof(unsigned long), 0,
					 FILTER_OTHER);
		offset += sizeof(unsigned long);
	}

	return ret;
}

int syscall_exit_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_exit trace;
	int ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
				 FILTER_OTHER);

	return ret;
}

void ftrace_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_trace_enter *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int size;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->enter_event->id, size, 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);

	if (!filter_current_check_discard(buffer, sys_data->enter_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

void ftrace_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_trace_exit *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->exit_event->id, sizeof(*entry), 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	entry->ret = syscall_get_return_value(current, regs);

	if (!filter_current_check_discard(buffer, sys_data->exit_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

int reg_event_syscall_enter(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_enter)
		ret = register_trace_sys_enter(ftrace_syscall_enter);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_enter_syscalls);
		sys_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_enter(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_enter--;
	clear_bit(num, enabled_enter_syscalls);
	if (!sys_refcount_enter)
		unregister_trace_sys_enter(ftrace_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

int reg_event_syscall_exit(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_exit)
		ret = register_trace_sys_exit(ftrace_syscall_exit);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall exit trace point");
	} else {
		set_bit(num, enabled_exit_syscalls);
		sys_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_exit(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_exit--;
	clear_bit(num, enabled_exit_syscalls);
	if (!sys_refcount_exit)
		unregister_trace_sys_exit(ftrace_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

int init_syscall_trace(struct ftrace_event_call *call)
{
	int id;

	id = register_ftrace_event(call->event);
	if (!id)
		return -ENODEV;
	call->id = id;
	INIT_LIST_HEAD(&call->fields);
	return 0;
}

int __init init_ftrace_syscalls(void)
{
	struct syscall_metadata *meta;
	unsigned long addr;
	int i;

	syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) *
					NR_syscalls, GFP_KERNEL);
	if (!syscalls_metadata) {
		WARN_ON(1);
		return -ENOMEM;
	}

	for (i = 0; i < NR_syscalls; i++) {
		addr = arch_syscall_addr(i);
		meta = find_syscall_meta(addr);
		if (!meta)
			continue;

		meta->syscall_nr = i;
		syscalls_metadata[i] = meta;
	}

	return 0;
}
core_initcall(init_ftrace_syscalls);

#ifdef CONFIG_EVENT_PROFILE

static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
static int sys_prof_refcount_enter;
static int sys_prof_refcount_exit;

static void prof_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_enter *rec;
	unsigned long flags;
	char *trace_buf;
	char *raw_data;
	int syscall_nr;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* get the size after alignment with the u32 buffer size field */
	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	size = ALIGN(size + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		      "profile buffer not large enough"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_enter *) raw_data;
	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->enter_event->id;
	rec->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
			       (unsigned long *)&rec->args);
	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int prof_sysenter_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_enter)
		ret = register_trace_sys_enter(prof_syscall_enter);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_enter_syscalls);
		sys_prof_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void prof_sysenter_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_enter--;
	clear_bit(num, enabled_prof_enter_syscalls);
	if (!sys_prof_refcount_enter)
		unregister_trace_sys_enter(prof_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

static void prof_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_exit *rec;
	unsigned long flags;
	int syscall_nr;
	char *trace_buf;
	char *raw_data;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* We can probably do that at build time */
	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	/*
	 * Impossible, but be paranoid with the future
	 * How to put this check outside runtime?
	 */
	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		"exit event has grown above profile buffer size"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_exit *)raw_data;

	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->exit_event->id;
	rec->nr = syscall_nr;
	rec->ret = syscall_get_return_value(current, regs);

	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int prof_sysexit_enable(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_exit)
		ret = register_trace_sys_exit(prof_syscall_exit);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_exit_syscalls);
		sys_prof_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void prof_sysexit_disable(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_exit--;
	clear_bit(num, enabled_prof_exit_syscalls);
	if (!sys_prof_refcount_exit)
		unregister_trace_sys_exit(prof_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

#endif
Commit	Line	Data
47788c58	1	#include <trace/syscall.h>
1c569f02	2	#include <trace/events/syscalls.h>
ee08c6ec	3	#include <linux/kernel.h>
fb34a08c	4	#include <linux/ftrace.h>
cdd6c482	5	#include <linux/perf_event.h>
ee08c6ec FW	6	#include <asm/syscall.h>
	7
	8	#include "trace_output.h"
	9	#include "trace.h"
	10
5be71b61	11	static DEFINE_MUTEX(syscall_trace_lock);
fb34a08c JB	12	static int sys_refcount_enter;
fb34a08c JB	13	static int sys_refcount_exit;
57421dbb JB	14	static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
57421dbb JB	15	static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
ee08c6ec	16
c44fc770 FW	17	extern unsigned long __start_syscalls_metadata[];
	18	extern unsigned long __stop_syscalls_metadata[];
	19
	20	static struct syscall_metadata **syscalls_metadata;
	21
	22	static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
	23	{
	24	struct syscall_metadata *start;
	25	struct syscall_metadata *stop;
	26	char str[KSYM_SYMBOL_LEN];
	27
	28
	29	start = (struct syscall_metadata *)__start_syscalls_metadata;
	30	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	31	kallsyms_lookup(syscall, NULL, NULL, NULL, str);
	32
	33	for ( ; start < stop; start++) {
	34	/*
	35	* Only compare after the "sys" prefix. Archs that use
	36	* syscall wrappers may have syscalls symbols aliases prefixed
	37	* with "SyS" instead of "sys", leading to an unwanted
	38	* mismatch.
	39	*/
	40	if (start->name && !strcmp(start->name + 3, str + 3))
	41	return start;
	42	}
	43	return NULL;
	44	}
	45
	46	static struct syscall_metadata *syscall_nr_to_meta(int nr)
	47	{
	48	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)
	49	return NULL;
	50
	51	return syscalls_metadata[nr];
	52	}
	53
bed1ffca FW	54	enum print_line_t
	55	print_syscall_enter(struct trace_iterator *iter, int flags)
	56	{
	57	struct trace_seq *s = &iter->seq;
	58	struct trace_entry *ent = iter->ent;
	59	struct syscall_trace_enter *trace;
	60	struct syscall_metadata *entry;
	61	int i, ret, syscall;
	62
64c12e04	63	trace = (typeof(trace))ent;
bed1ffca	64	syscall = trace->nr;
bed1ffca	65	entry = syscall_nr_to_meta(syscall);
64c12e04	66
bed1ffca FW	67	if (!entry)
	68	goto end;
	69
fcc19438	70	if (entry->enter_event->id != ent->type) {
64c12e04 JB	71	WARN_ON_ONCE(1);
	72	goto end;
	73	}
	74
bed1ffca FW	75	ret = trace_seq_printf(s, "%s(", entry->name);
	76	if (!ret)
	77	return TRACE_TYPE_PARTIAL_LINE;
	78
	79	for (i = 0; i < entry->nb_args; i++) {
	80	/* parameter types */
ba8b3a40	81	if (trace_flags & TRACE_ITER_VERBOSE) {
bed1ffca FW	82	ret = trace_seq_printf(s, "%s ", entry->types[i]);
	83	if (!ret)
	84	return TRACE_TYPE_PARTIAL_LINE;
	85	}
	86	/* parameter values */
4539f077	87	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
bed1ffca	88	trace->args[i],
4539f077	89	i == entry->nb_args - 1 ? "" : ", ");
bed1ffca FW	90	if (!ret)
	91	return TRACE_TYPE_PARTIAL_LINE;
	92	}
	93
4539f077 LZ	94	ret = trace_seq_putc(s, ')');
	95	if (!ret)
	96	return TRACE_TYPE_PARTIAL_LINE;
	97
bed1ffca	98	end:
4539f077 LZ	99	ret = trace_seq_putc(s, '\n');
	100	if (!ret)
	101	return TRACE_TYPE_PARTIAL_LINE;
	102
bed1ffca FW	103	return TRACE_TYPE_HANDLED;
	104	}
	105
	106	enum print_line_t
	107	print_syscall_exit(struct trace_iterator *iter, int flags)
	108	{
	109	struct trace_seq *s = &iter->seq;
	110	struct trace_entry *ent = iter->ent;
	111	struct syscall_trace_exit *trace;
	112	int syscall;
	113	struct syscall_metadata *entry;
	114	int ret;
	115
64c12e04	116	trace = (typeof(trace))ent;
bed1ffca	117	syscall = trace->nr;
bed1ffca	118	entry = syscall_nr_to_meta(syscall);
64c12e04	119
bed1ffca FW	120	if (!entry) {
	121	trace_seq_printf(s, "\n");
	122	return TRACE_TYPE_HANDLED;
	123	}
	124
fcc19438	125	if (entry->exit_event->id != ent->type) {
64c12e04 JB	126	WARN_ON_ONCE(1);
	127	return TRACE_TYPE_UNHANDLED;
	128	}
	129
bed1ffca FW	130	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
	131	trace->ret);
	132	if (!ret)
	133	return TRACE_TYPE_PARTIAL_LINE;
	134
	135	return TRACE_TYPE_HANDLED;
	136	}
	137
e6971969 LZ	138	extern char *__bad_type_size(void);
	139
	140	#define SYSCALL_FIELD(type, name) \
	141	sizeof(type) != sizeof(trace.name) ? \
	142	__bad_type_size() : \
26a50744 TZ	143	#type, #name, offsetof(typeof(trace), name), \
26a50744 TZ	144	sizeof(trace.name), is_signed_type(type)
e6971969	145
10a5b66f	146	int syscall_enter_format(struct ftrace_event_call call, struct trace_seq s)
dc4ddb4c FW	147	{
dc4ddb4c FW	148	int i;
e6971969	149	int ret;
31c16b13	150	struct syscall_metadata *entry = call->data;
e6971969 LZ	151	struct syscall_trace_enter trace;
e6971969 LZ	152	int offset = offsetof(struct syscall_trace_enter, args);
dc4ddb4c	153
26a50744 TZ	154	ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
26a50744 TZ	155	"\tsigned:%u;\n",
e6971969 LZ	156	SYSCALL_FIELD(int, nr));
	157	if (!ret)
	158	return 0;
dc4ddb4c FW	159
	160	for (i = 0; i < entry->nb_args; i++) {
	161	ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
	162	entry->args[i]);
	163	if (!ret)
	164	return 0;
26a50744 TZ	165	ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;"
	166	"\tsigned:%u;\n", offset,
	167	sizeof(unsigned long),
	168	is_signed_type(unsigned long));
dc4ddb4c FW	169	if (!ret)
	170	return 0;
	171	offset += sizeof(unsigned long);
	172	}
	173
4539f077	174	trace_seq_puts(s, "\nprint fmt: \"");
dc4ddb4c	175	for (i = 0; i < entry->nb_args; i++) {
e6971969	176	ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
dc4ddb4c	177	sizeof(unsigned long),
4539f077	178	i == entry->nb_args - 1 ? "" : ", ");
dc4ddb4c FW	179	if (!ret)
	180	return 0;
	181	}
4539f077	182	trace_seq_putc(s, '"');
dc4ddb4c FW	183
dc4ddb4c FW	184	for (i = 0; i < entry->nb_args; i++) {
4539f077 LZ	185	ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
4539f077 LZ	186	entry->args[i]);
dc4ddb4c FW	187	if (!ret)
	188	return 0;
	189	}
	190
4539f077	191	return trace_seq_putc(s, '\n');
dc4ddb4c FW	192	}
dc4ddb4c FW	193
10a5b66f LZ	194	int syscall_exit_format(struct ftrace_event_call call, struct trace_seq s)
	195	{
	196	int ret;
	197	struct syscall_trace_exit trace;
	198
	199	ret = trace_seq_printf(s,
26a50744 TZ	200	"\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
	201	"\tsigned:%u;\n"
	202	"\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
	203	"\tsigned:%u;\n",
10a5b66f	204	SYSCALL_FIELD(int, nr),
ee949a86	205	SYSCALL_FIELD(long, ret));
10a5b66f LZ	206	if (!ret)
	207	return 0;
	208
	209	return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
	210	}
	211
540b7b8d LZ	212	int syscall_enter_define_fields(struct ftrace_event_call *call)
	213	{
	214	struct syscall_trace_enter trace;
31c16b13	215	struct syscall_metadata *meta = call->data;
540b7b8d	216	int ret;
540b7b8d LZ	217	int i;
	218	int offset = offsetof(typeof(trace), args);
	219
0f1ef51d LJ	220	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	221	if (ret)
	222	return ret;
	223
540b7b8d	224	for (i = 0; i < meta->nb_args; i++) {
aeaeae11 FW	225	ret = trace_define_field(call, meta->types[i],
aeaeae11 FW	226	meta->args[i], offset,
43b51ead LZ	227	sizeof(unsigned long), 0,
43b51ead LZ	228	FILTER_OTHER);
540b7b8d LZ	229	offset += sizeof(unsigned long);
	230	}
	231
	232	return ret;
	233	}
	234
	235	int syscall_exit_define_fields(struct ftrace_event_call *call)
	236	{
	237	struct syscall_trace_exit trace;
	238	int ret;
	239
0f1ef51d LJ	240	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	241	if (ret)
	242	return ret;
	243
26a50744	244	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
43b51ead	245	FILTER_OTHER);
540b7b8d LZ	246
	247	return ret;
	248	}
	249
fb34a08c	250	void ftrace_syscall_enter(struct pt_regs *regs, long id)
ee08c6ec	251	{
bed1ffca FW	252	struct syscall_trace_enter *entry;
	253	struct syscall_metadata *sys_data;
	254	struct ring_buffer_event *event;
e77405ad	255	struct ring_buffer *buffer;
bed1ffca	256	int size;
ee08c6ec FW	257	int syscall_nr;
	258
	259	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	260	if (syscall_nr < 0)
cd0980fc HB	261	return;
fb34a08c JB	262	if (!test_bit(syscall_nr, enabled_enter_syscalls))
fb34a08c JB	263	return;
ee08c6ec	264
bed1ffca FW	265	sys_data = syscall_nr_to_meta(syscall_nr);
	266	if (!sys_data)
	267	return;
	268
	269	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;
	270
fcc19438 LJ	271	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	272	sys_data->enter_event->id, size, 0, 0);
bed1ffca FW	273	if (!event)
	274	return;
	275
	276	entry = ring_buffer_event_data(event);
	277	entry->nr = syscall_nr;
	278	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
	279
e77405ad SR	280	if (!filter_current_check_discard(buffer, sys_data->enter_event,
	281	entry, event))
	282	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	283	}
ee08c6ec FW	284
fb34a08c	285	void ftrace_syscall_exit(struct pt_regs *regs, long ret)
ee08c6ec	286	{
bed1ffca FW	287	struct syscall_trace_exit *entry;
	288	struct syscall_metadata *sys_data;
	289	struct ring_buffer_event *event;
e77405ad	290	struct ring_buffer *buffer;
ee08c6ec FW	291	int syscall_nr;
	292
	293	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	294	if (syscall_nr < 0)
cd0980fc HB	295	return;
fb34a08c JB	296	if (!test_bit(syscall_nr, enabled_exit_syscalls))
fb34a08c JB	297	return;
ee08c6ec	298
bed1ffca FW	299	sys_data = syscall_nr_to_meta(syscall_nr);
	300	if (!sys_data)
	301	return;
	302
fcc19438 LJ	303	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	304	sys_data->exit_event->id, sizeof(*entry), 0, 0);
bed1ffca FW	305	if (!event)
	306	return;
	307
	308	entry = ring_buffer_event_data(event);
	309	entry->nr = syscall_nr;
	310	entry->ret = syscall_get_return_value(current, regs);
	311
e77405ad SR	312	if (!filter_current_check_discard(buffer, sys_data->exit_event,
	313	entry, event))
	314	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	315	}
ee08c6ec FW	316
bd1a5c84	317	int reg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	318	{
fb34a08c JB	319	int ret = 0;
fb34a08c JB	320	int num;
fb34a08c	321
c252f657	322	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	323	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	324	return -ENOSYS;
	325	mutex_lock(&syscall_trace_lock);
	326	if (!sys_refcount_enter)
1c569f02	327	ret = register_trace_sys_enter(ftrace_syscall_enter);
fb34a08c JB	328	if (ret) {
	329	pr_info("event trace: Could not activate"
	330	"syscall entry trace point");
	331	} else {
	332	set_bit(num, enabled_enter_syscalls);
	333	sys_refcount_enter++;
	334	}
	335	mutex_unlock(&syscall_trace_lock);
	336	return ret;
ee08c6ec FW	337	}
ee08c6ec FW	338
bd1a5c84	339	void unreg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	340	{
fb34a08c	341	int num;
ee08c6ec	342
c252f657	343	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	344	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	345	return;
	346	mutex_lock(&syscall_trace_lock);
	347	sys_refcount_enter--;
	348	clear_bit(num, enabled_enter_syscalls);
	349	if (!sys_refcount_enter)
1c569f02	350	unregister_trace_sys_enter(ftrace_syscall_enter);
fb34a08c JB	351	mutex_unlock(&syscall_trace_lock);
fb34a08c JB	352	}
ee08c6ec	353
bd1a5c84	354	int reg_event_syscall_exit(struct ftrace_event_call *call)
ee08c6ec	355	{
fb34a08c JB	356	int ret = 0;
fb34a08c JB	357	int num;
fb34a08c	358
c252f657	359	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	360	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	361	return -ENOSYS;
	362	mutex_lock(&syscall_trace_lock);
	363	if (!sys_refcount_exit)
1c569f02	364	ret = register_trace_sys_exit(ftrace_syscall_exit);
fb34a08c JB	365	if (ret) {
	366	pr_info("event trace: Could not activate"
	367	"syscall exit trace point");
	368	} else {
	369	set_bit(num, enabled_exit_syscalls);
	370	sys_refcount_exit++;
ee08c6ec	371	}
fb34a08c JB	372	mutex_unlock(&syscall_trace_lock);
	373	return ret;
	374	}
ee08c6ec	375
bd1a5c84	376	void unreg_event_syscall_exit(struct ftrace_event_call *call)
fb34a08c JB	377	{
fb34a08c JB	378	int num;
ee08c6ec	379
c252f657	380	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	381	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	382	return;
	383	mutex_lock(&syscall_trace_lock);
	384	sys_refcount_exit--;
	385	clear_bit(num, enabled_exit_syscalls);
	386	if (!sys_refcount_exit)
1c569f02	387	unregister_trace_sys_exit(ftrace_syscall_exit);
fb34a08c	388	mutex_unlock(&syscall_trace_lock);
ee08c6ec	389	}
fb34a08c	390
a1301da0 LJ	391	int init_syscall_trace(struct ftrace_event_call *call)
	392	{
	393	int id;
	394
	395	id = register_ftrace_event(call->event);
	396	if (!id)
	397	return -ENODEV;
	398	call->id = id;
	399	INIT_LIST_HEAD(&call->fields);
	400	return 0;
	401	}
	402
c44fc770 FW	403	int __init init_ftrace_syscalls(void)
	404	{
	405	struct syscall_metadata *meta;
	406	unsigned long addr;
	407	int i;
	408
	409	syscalls_metadata = kzalloc(sizeof(syscalls_metadata)
	410	NR_syscalls, GFP_KERNEL);
	411	if (!syscalls_metadata) {
	412	WARN_ON(1);
	413	return -ENOMEM;
	414	}
	415
	416	for (i = 0; i < NR_syscalls; i++) {
	417	addr = arch_syscall_addr(i);
	418	meta = find_syscall_meta(addr);
c252f657 LJ	419	if (!meta)
	420	continue;
	421
	422	meta->syscall_nr = i;
c44fc770 FW	423	syscalls_metadata[i] = meta;
	424	}
	425
	426	return 0;
	427	}
	428	core_initcall(init_ftrace_syscalls);
	429
f4b5ffcc	430	#ifdef CONFIG_EVENT_PROFILE
19007a67	431
57421dbb JB	432	static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
57421dbb JB	433	static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
f4b5ffcc JB	434	static int sys_prof_refcount_enter;
	435	static int sys_prof_refcount_exit;
	436
	437	static void prof_syscall_enter(struct pt_regs *regs, long id)
	438	{
	439	struct syscall_metadata *sys_data;
20ab4425 FW	440	struct syscall_trace_enter *rec;
20ab4425 FW	441	unsigned long flags;
ce71b9df	442	char *trace_buf;
20ab4425	443	char *raw_data;
f4b5ffcc	444	int syscall_nr;
4ed7c92d	445	int rctx;
19007a67	446	int size;
20ab4425	447	int cpu;
f4b5ffcc JB	448
	449	syscall_nr = syscall_get_nr(current, regs);
	450	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
	451	return;
	452
	453	sys_data = syscall_nr_to_meta(syscall_nr);
	454	if (!sys_data)
	455	return;
	456
19007a67 FW	457	/* get the size after alignment with the u32 buffer size field */
	458	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	459	size = ALIGN(size + sizeof(u32), sizeof(u64));
	460	size -= sizeof(u32);
	461
20ab4425 FW	462	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	463	"profile buffer not large enough"))
	464	return;
	465
	466	/* Protect the per cpu buffer, begin the rcu read side */
	467	local_irq_save(flags);
19007a67	468
4ed7c92d PZ	469	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	470	if (rctx < 0)
ce71b9df FW	471	goto end_recursion;
ce71b9df FW	472
20ab4425 FW	473	cpu = smp_processor_id();
20ab4425 FW	474
28889bf9	475	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	476
444a2a3b	477	if (!trace_buf)
20ab4425	478	goto end;
19007a67	479
ce71b9df	480	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	481
	482	/* zero the dead bytes from align to not leak stack to user */
	483	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	484
	485	rec = (struct syscall_trace_enter *) raw_data;
	486	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	487	rec->ent.type = sys_data->enter_event->id;
20ab4425 FW	488	rec->nr = syscall_nr;
	489	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
	490	(unsigned long *)&rec->args);
fcc19438	491	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);
20ab4425 FW	492
20ab4425 FW	493	end:
4ed7c92d	494	perf_swevent_put_recursion_context(rctx);
ce71b9df	495	end_recursion:
20ab4425	496	local_irq_restore(flags);
f4b5ffcc JB	497	}
f4b5ffcc JB	498
3bbe84e9	499	int prof_sysenter_enable(struct ftrace_event_call *call)
f4b5ffcc JB	500	{
	501	int ret = 0;
	502	int num;
	503
3bbe84e9	504	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	505
	506	mutex_lock(&syscall_trace_lock);
	507	if (!sys_prof_refcount_enter)
1c569f02	508	ret = register_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	509	if (ret) {
	510	pr_info("event trace: Could not activate"
	511	"syscall entry trace point");
	512	} else {
	513	set_bit(num, enabled_prof_enter_syscalls);
	514	sys_prof_refcount_enter++;
	515	}
	516	mutex_unlock(&syscall_trace_lock);
	517	return ret;
	518	}
	519
3bbe84e9	520	void prof_sysenter_disable(struct ftrace_event_call *call)
f4b5ffcc JB	521	{
	522	int num;
	523
3bbe84e9	524	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	525
	526	mutex_lock(&syscall_trace_lock);
	527	sys_prof_refcount_enter--;
	528	clear_bit(num, enabled_prof_enter_syscalls);
	529	if (!sys_prof_refcount_enter)
1c569f02	530	unregister_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	531	mutex_unlock(&syscall_trace_lock);
	532	}
	533
	534	static void prof_syscall_exit(struct pt_regs *regs, long ret)
	535	{
	536	struct syscall_metadata *sys_data;
20ab4425 FW	537	struct syscall_trace_exit *rec;
20ab4425 FW	538	unsigned long flags;
f4b5ffcc	539	int syscall_nr;
ce71b9df	540	char *trace_buf;
20ab4425	541	char *raw_data;
4ed7c92d	542	int rctx;
20ab4425 FW	543	int size;
20ab4425 FW	544	int cpu;
f4b5ffcc JB	545
	546	syscall_nr = syscall_get_nr(current, regs);
	547	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
	548	return;
	549
	550	sys_data = syscall_nr_to_meta(syscall_nr);
	551	if (!sys_data)
	552	return;
	553
20ab4425 FW	554	/* We can probably do that at build time */
	555	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	556	size -= sizeof(u32);
19007a67	557
20ab4425 FW	558	/*
	559	* Impossible, but be paranoid with the future
	560	* How to put this check outside runtime?
	561	*/
	562	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	563	"exit event has grown above profile buffer size"))
	564	return;
	565
	566	/* Protect the per cpu buffer, begin the rcu read side */
	567	local_irq_save(flags);
ce71b9df	568
4ed7c92d PZ	569	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	570	if (rctx < 0)
ce71b9df FW	571	goto end_recursion;
ce71b9df FW	572
20ab4425 FW	573	cpu = smp_processor_id();
20ab4425 FW	574
28889bf9	575	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	576
444a2a3b	577	if (!trace_buf)
20ab4425 FW	578	goto end;
20ab4425 FW	579
ce71b9df	580	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	581
	582	/* zero the dead bytes from align to not leak stack to user */
	583	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	584
	585	rec = (struct syscall_trace_exit *)raw_data;
	586
	587	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	588	rec->ent.type = sys_data->exit_event->id;
20ab4425 FW	589	rec->nr = syscall_nr;
	590	rec->ret = syscall_get_return_value(current, regs);
	591
fcc19438	592	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);
20ab4425 FW	593
20ab4425 FW	594	end:
4ed7c92d	595	perf_swevent_put_recursion_context(rctx);
ce71b9df	596	end_recursion:
20ab4425	597	local_irq_restore(flags);
f4b5ffcc JB	598	}
f4b5ffcc JB	599
3bbe84e9	600	int prof_sysexit_enable(struct ftrace_event_call *call)
f4b5ffcc JB	601	{
	602	int ret = 0;
	603	int num;
	604
3bbe84e9	605	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	606
	607	mutex_lock(&syscall_trace_lock);
	608	if (!sys_prof_refcount_exit)
1c569f02	609	ret = register_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	610	if (ret) {
	611	pr_info("event trace: Could not activate"
	612	"syscall entry trace point");
	613	} else {
	614	set_bit(num, enabled_prof_exit_syscalls);
	615	sys_prof_refcount_exit++;
	616	}
	617	mutex_unlock(&syscall_trace_lock);
	618	return ret;
	619	}
	620
3bbe84e9	621	void prof_sysexit_disable(struct ftrace_event_call *call)
f4b5ffcc JB	622	{
	623	int num;
	624
3bbe84e9	625	num = ((struct syscall_metadata *)call->data)->syscall_nr;
f4b5ffcc JB	626
	627	mutex_lock(&syscall_trace_lock);
	628	sys_prof_refcount_exit--;
	629	clear_bit(num, enabled_prof_exit_syscalls);
	630	if (!sys_prof_refcount_exit)
1c569f02	631	unregister_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	632	mutex_unlock(&syscall_trace_lock);
	633	}
	634
	635	#endif
	636
	637