[linux-2.6-block.git] / kernel / trace / trace_syscalls.c

#include <trace/syscall.h>
#include <trace/events/syscalls.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/perf_event.h>
#include <asm/syscall.h>

#include "trace_output.h"
#include "trace.h"

static DEFINE_MUTEX(syscall_trace_lock);
static int sys_refcount_enter;
static int sys_refcount_exit;
static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);

extern unsigned long __start_syscalls_metadata[];
extern unsigned long __stop_syscalls_metadata[];

static struct syscall_metadata **syscalls_metadata;

static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
{
	struct syscall_metadata *start;
	struct syscall_metadata *stop;
	char str[KSYM_SYMBOL_LEN];


	start = (struct syscall_metadata *)__start_syscalls_metadata;
	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	kallsyms_lookup(syscall, NULL, NULL, NULL, str);

	for ( ; start < stop; start++) {
		/*
		 * Only compare after the "sys" prefix. Archs that use
		 * syscall wrappers may have syscalls symbols aliases prefixed
		 * with "SyS" instead of "sys", leading to an unwanted
		 * mismatch.
		 */
		if (start->name && !strcmp(start->name + 3, str + 3))
			return start;
	}
	return NULL;
}

static struct syscall_metadata *syscall_nr_to_meta(int nr)
{
	if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
		return NULL;

	return syscalls_metadata[nr];
}

static int syscall_name_to_nr(const char *name)
{
	int i;

	if (!syscalls_metadata)
		return -1;

	for (i = 0; i < NR_syscalls; i++) {
		if (syscalls_metadata[i]) {
			if (!strcmp(syscalls_metadata[i]->name, name))
				return i;
		}
	}
	return -1;
}

enum print_line_t
print_syscall_enter(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_enter *trace;
	struct syscall_metadata *entry;
	int i, ret, syscall;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry)
		goto end;

	if (entry->enter_event->id != ent->type) {
		WARN_ON_ONCE(1);
		goto end;
	}

	ret = trace_seq_printf(s, "%s(", entry->name);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	for (i = 0; i < entry->nb_args; i++) {
		/* parameter types */
		if (trace_flags & TRACE_ITER_VERBOSE) {
			ret = trace_seq_printf(s, "%s ", entry->types[i]);
			if (!ret)
				return TRACE_TYPE_PARTIAL_LINE;
		}
		/* parameter values */
		ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
				       trace->args[i],
				       i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return TRACE_TYPE_PARTIAL_LINE;
	}

	ret = trace_seq_putc(s, ')');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

end:
	ret =  trace_seq_putc(s, '\n');
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

enum print_line_t
print_syscall_exit(struct trace_iterator *iter, int flags)
{
	struct trace_seq *s = &iter->seq;
	struct trace_entry *ent = iter->ent;
	struct syscall_trace_exit *trace;
	int syscall;
	struct syscall_metadata *entry;
	int ret;

	trace = (typeof(trace))ent;
	syscall = trace->nr;
	entry = syscall_nr_to_meta(syscall);

	if (!entry) {
		trace_seq_printf(s, "\n");
		return TRACE_TYPE_HANDLED;
	}

	if (entry->exit_event->id != ent->type) {
		WARN_ON_ONCE(1);
		return TRACE_TYPE_UNHANDLED;
	}

	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
				trace->ret);
	if (!ret)
		return TRACE_TYPE_PARTIAL_LINE;

	return TRACE_TYPE_HANDLED;
}

extern char *__bad_type_size(void);

#define SYSCALL_FIELD(type, name)					\
	sizeof(type) != sizeof(trace.name) ?				\
		__bad_type_size() :					\
		#type, #name, offsetof(typeof(trace), name),		\
		sizeof(trace.name), is_signed_type(type)

int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s)
{
	int i;
	int ret;
	struct syscall_metadata *entry = call->data;
	struct syscall_trace_enter trace;
	int offset = offsetof(struct syscall_trace_enter, args);

	ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n",
			       SYSCALL_FIELD(int, nr));
	if (!ret)
		return 0;

	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
				        entry->args[i]);
		if (!ret)
			return 0;
		ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;"
				       "\tsigned:%u;\n", offset,
				       sizeof(unsigned long),
				       is_signed_type(unsigned long));
		if (!ret)
			return 0;
		offset += sizeof(unsigned long);
	}

	trace_seq_puts(s, "\nprint fmt: \"");
	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
				        sizeof(unsigned long),
					i == entry->nb_args - 1 ? "" : ", ");
		if (!ret)
			return 0;
	}
	trace_seq_putc(s, '"');

	for (i = 0; i < entry->nb_args; i++) {
		ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
				       entry->args[i]);
		if (!ret)
			return 0;
	}

	return trace_seq_putc(s, '\n');
}

int syscall_exit_format(struct ftrace_event_call *call, struct trace_seq *s)
{
	int ret;
	struct syscall_trace_exit trace;

	ret = trace_seq_printf(s,
			       "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n"
			       "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
			       "\tsigned:%u;\n",
			       SYSCALL_FIELD(int, nr),
			       SYSCALL_FIELD(long, ret));
	if (!ret)
		return 0;

	return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
}

int syscall_enter_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_enter trace;
	struct syscall_metadata *meta = call->data;
	int ret;
	int i;
	int offset = offsetof(typeof(trace), args);

	ret = trace_define_common_fields(call);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	for (i = 0; i < meta->nb_args; i++) {
		ret = trace_define_field(call, meta->types[i],
					 meta->args[i], offset,
					 sizeof(unsigned long), 0,
					 FILTER_OTHER);
		offset += sizeof(unsigned long);
	}

	return ret;
}

int syscall_exit_define_fields(struct ftrace_event_call *call)
{
	struct syscall_trace_exit trace;
	int ret;

	ret = trace_define_common_fields(call);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	if (ret)
		return ret;

	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
				 FILTER_OTHER);

	return ret;
}

void ftrace_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_trace_enter *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int size;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->enter_event->id, size, 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);

	if (!filter_current_check_discard(buffer, sys_data->enter_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

void ftrace_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_trace_exit *entry;
	struct syscall_metadata *sys_data;
	struct ring_buffer_event *event;
	struct ring_buffer *buffer;
	int syscall_nr;

	syscall_nr = syscall_get_nr(current, regs);
	if (syscall_nr < 0)
		return;
	if (!test_bit(syscall_nr, enabled_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	event = trace_current_buffer_lock_reserve(&buffer,
			sys_data->exit_event->id, sizeof(*entry), 0, 0);
	if (!event)
		return;

	entry = ring_buffer_event_data(event);
	entry->nr = syscall_nr;
	entry->ret = syscall_get_return_value(current, regs);

	if (!filter_current_check_discard(buffer, sys_data->exit_event,
					  entry, event))
		trace_current_buffer_unlock_commit(buffer, event, 0, 0);
}

int reg_event_syscall_enter(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_enter)
		ret = register_trace_sys_enter(ftrace_syscall_enter);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_enter_syscalls);
		sys_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_enter(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_enter--;
	clear_bit(num, enabled_enter_syscalls);
	if (!sys_refcount_enter)
		unregister_trace_sys_enter(ftrace_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

int reg_event_syscall_exit(struct ftrace_event_call *call)
{
	int ret = 0;
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;
	mutex_lock(&syscall_trace_lock);
	if (!sys_refcount_exit)
		ret = register_trace_sys_exit(ftrace_syscall_exit);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall exit trace point");
	} else {
		set_bit(num, enabled_exit_syscalls);
		sys_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_event_syscall_exit(struct ftrace_event_call *call)
{
	int num;

	num = ((struct syscall_metadata *)call->data)->syscall_nr;
	if (num < 0 || num >= NR_syscalls)
		return;
	mutex_lock(&syscall_trace_lock);
	sys_refcount_exit--;
	clear_bit(num, enabled_exit_syscalls);
	if (!sys_refcount_exit)
		unregister_trace_sys_exit(ftrace_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

int __init init_ftrace_syscalls(void)
{
	struct syscall_metadata *meta;
	unsigned long addr;
	int i;

	syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) *
					NR_syscalls, GFP_KERNEL);
	if (!syscalls_metadata) {
		WARN_ON(1);
		return -ENOMEM;
	}

	for (i = 0; i < NR_syscalls; i++) {
		addr = arch_syscall_addr(i);
		meta = find_syscall_meta(addr);
		if (!meta)
			continue;

		meta->syscall_nr = i;
		syscalls_metadata[i] = meta;
	}

	return 0;
}
core_initcall(init_ftrace_syscalls);

#ifdef CONFIG_EVENT_PROFILE

static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
static int sys_prof_refcount_enter;
static int sys_prof_refcount_exit;

static void prof_syscall_enter(struct pt_regs *regs, long id)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_enter *rec;
	unsigned long flags;
	char *trace_buf;
	char *raw_data;
	int syscall_nr;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* get the size after alignment with the u32 buffer size field */
	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	size = ALIGN(size + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		      "profile buffer not large enough"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_enter *) raw_data;
	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->enter_event->id;
	rec->nr = syscall_nr;
	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
			       (unsigned long *)&rec->args);
	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int reg_prof_syscall_enter(char *name)
{
	int ret = 0;
	int num;

	num = syscall_name_to_nr(name);
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_enter)
		ret = register_trace_sys_enter(prof_syscall_enter);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_enter_syscalls);
		sys_prof_refcount_enter++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_prof_syscall_enter(char *name)
{
	int num;

	num = syscall_name_to_nr(name);
	if (num < 0 || num >= NR_syscalls)
		return;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_enter--;
	clear_bit(num, enabled_prof_enter_syscalls);
	if (!sys_prof_refcount_enter)
		unregister_trace_sys_enter(prof_syscall_enter);
	mutex_unlock(&syscall_trace_lock);
}

static void prof_syscall_exit(struct pt_regs *regs, long ret)
{
	struct syscall_metadata *sys_data;
	struct syscall_trace_exit *rec;
	unsigned long flags;
	int syscall_nr;
	char *trace_buf;
	char *raw_data;
	int rctx;
	int size;
	int cpu;

	syscall_nr = syscall_get_nr(current, regs);
	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
		return;

	sys_data = syscall_nr_to_meta(syscall_nr);
	if (!sys_data)
		return;

	/* We can probably do that at build time */
	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	size -= sizeof(u32);

	/*
	 * Impossible, but be paranoid with the future
	 * How to put this check outside runtime?
	 */
	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
		"exit event has grown above profile buffer size"))
		return;

	/* Protect the per cpu buffer, begin the rcu read side */
	local_irq_save(flags);

	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		goto end_recursion;

	cpu = smp_processor_id();

	trace_buf = rcu_dereference(perf_trace_buf);

	if (!trace_buf)
		goto end;

	raw_data = per_cpu_ptr(trace_buf, cpu);

	/* zero the dead bytes from align to not leak stack to user */
	*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;

	rec = (struct syscall_trace_exit *)raw_data;

	tracing_generic_entry_update(&rec->ent, 0, 0);
	rec->ent.type = sys_data->exit_event->id;
	rec->nr = syscall_nr;
	rec->ret = syscall_get_return_value(current, regs);

	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);

end:
	perf_swevent_put_recursion_context(rctx);
end_recursion:
	local_irq_restore(flags);
}

int reg_prof_syscall_exit(char *name)
{
	int ret = 0;
	int num;

	num = syscall_name_to_nr(name);
	if (num < 0 || num >= NR_syscalls)
		return -ENOSYS;

	mutex_lock(&syscall_trace_lock);
	if (!sys_prof_refcount_exit)
		ret = register_trace_sys_exit(prof_syscall_exit);
	if (ret) {
		pr_info("event trace: Could not activate"
				"syscall entry trace point");
	} else {
		set_bit(num, enabled_prof_exit_syscalls);
		sys_prof_refcount_exit++;
	}
	mutex_unlock(&syscall_trace_lock);
	return ret;
}

void unreg_prof_syscall_exit(char *name)
{
	int num;

	num = syscall_name_to_nr(name);
	if (num < 0 || num >= NR_syscalls)
		return;

	mutex_lock(&syscall_trace_lock);
	sys_prof_refcount_exit--;
	clear_bit(num, enabled_prof_exit_syscalls);
	if (!sys_prof_refcount_exit)
		unregister_trace_sys_exit(prof_syscall_exit);
	mutex_unlock(&syscall_trace_lock);
}

#endif
Commit	Line	Data
47788c58	1	#include <trace/syscall.h>
1c569f02	2	#include <trace/events/syscalls.h>
ee08c6ec	3	#include <linux/kernel.h>
fb34a08c	4	#include <linux/ftrace.h>
cdd6c482	5	#include <linux/perf_event.h>
ee08c6ec FW	6	#include <asm/syscall.h>
	7
	8	#include "trace_output.h"
	9	#include "trace.h"
	10
5be71b61	11	static DEFINE_MUTEX(syscall_trace_lock);
fb34a08c JB	12	static int sys_refcount_enter;
fb34a08c JB	13	static int sys_refcount_exit;
57421dbb JB	14	static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls);
57421dbb JB	15	static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls);
ee08c6ec	16
c44fc770 FW	17	extern unsigned long __start_syscalls_metadata[];
	18	extern unsigned long __stop_syscalls_metadata[];
	19
	20	static struct syscall_metadata **syscalls_metadata;
	21
	22	static struct syscall_metadata *find_syscall_meta(unsigned long syscall)
	23	{
	24	struct syscall_metadata *start;
	25	struct syscall_metadata *stop;
	26	char str[KSYM_SYMBOL_LEN];
	27
	28
	29	start = (struct syscall_metadata *)__start_syscalls_metadata;
	30	stop = (struct syscall_metadata *)__stop_syscalls_metadata;
	31	kallsyms_lookup(syscall, NULL, NULL, NULL, str);
	32
	33	for ( ; start < stop; start++) {
	34	/*
	35	* Only compare after the "sys" prefix. Archs that use
	36	* syscall wrappers may have syscalls symbols aliases prefixed
	37	* with "SyS" instead of "sys", leading to an unwanted
	38	* mismatch.
	39	*/
	40	if (start->name && !strcmp(start->name + 3, str + 3))
	41	return start;
	42	}
	43	return NULL;
	44	}
	45
	46	static struct syscall_metadata *syscall_nr_to_meta(int nr)
	47	{
	48	if (!syscalls_metadata \|\| nr >= NR_syscalls \|\| nr < 0)
	49	return NULL;
	50
	51	return syscalls_metadata[nr];
	52	}
	53
c252f657	54	static int syscall_name_to_nr(const char *name)
c44fc770 FW	55	{
	56	int i;
	57
	58	if (!syscalls_metadata)
	59	return -1;
	60
	61	for (i = 0; i < NR_syscalls; i++) {
	62	if (syscalls_metadata[i]) {
	63	if (!strcmp(syscalls_metadata[i]->name, name))
	64	return i;
	65	}
	66	}
	67	return -1;
	68	}
	69
bed1ffca FW	70	enum print_line_t
	71	print_syscall_enter(struct trace_iterator *iter, int flags)
	72	{
	73	struct trace_seq *s = &iter->seq;
	74	struct trace_entry *ent = iter->ent;
	75	struct syscall_trace_enter *trace;
	76	struct syscall_metadata *entry;
	77	int i, ret, syscall;
	78
64c12e04	79	trace = (typeof(trace))ent;
bed1ffca	80	syscall = trace->nr;
bed1ffca	81	entry = syscall_nr_to_meta(syscall);
64c12e04	82
bed1ffca FW	83	if (!entry)
	84	goto end;
	85
fcc19438	86	if (entry->enter_event->id != ent->type) {
64c12e04 JB	87	WARN_ON_ONCE(1);
	88	goto end;
	89	}
	90
bed1ffca FW	91	ret = trace_seq_printf(s, "%s(", entry->name);
	92	if (!ret)
	93	return TRACE_TYPE_PARTIAL_LINE;
	94
	95	for (i = 0; i < entry->nb_args; i++) {
	96	/* parameter types */
ba8b3a40	97	if (trace_flags & TRACE_ITER_VERBOSE) {
bed1ffca FW	98	ret = trace_seq_printf(s, "%s ", entry->types[i]);
	99	if (!ret)
	100	return TRACE_TYPE_PARTIAL_LINE;
	101	}
	102	/* parameter values */
4539f077	103	ret = trace_seq_printf(s, "%s: %lx%s", entry->args[i],
bed1ffca	104	trace->args[i],
4539f077	105	i == entry->nb_args - 1 ? "" : ", ");
bed1ffca FW	106	if (!ret)
	107	return TRACE_TYPE_PARTIAL_LINE;
	108	}
	109
4539f077 LZ	110	ret = trace_seq_putc(s, ')');
	111	if (!ret)
	112	return TRACE_TYPE_PARTIAL_LINE;
	113
bed1ffca	114	end:
4539f077 LZ	115	ret = trace_seq_putc(s, '\n');
	116	if (!ret)
	117	return TRACE_TYPE_PARTIAL_LINE;
	118
bed1ffca FW	119	return TRACE_TYPE_HANDLED;
	120	}
	121
	122	enum print_line_t
	123	print_syscall_exit(struct trace_iterator *iter, int flags)
	124	{
	125	struct trace_seq *s = &iter->seq;
	126	struct trace_entry *ent = iter->ent;
	127	struct syscall_trace_exit *trace;
	128	int syscall;
	129	struct syscall_metadata *entry;
	130	int ret;
	131
64c12e04	132	trace = (typeof(trace))ent;
bed1ffca	133	syscall = trace->nr;
bed1ffca	134	entry = syscall_nr_to_meta(syscall);
64c12e04	135
bed1ffca FW	136	if (!entry) {
	137	trace_seq_printf(s, "\n");
	138	return TRACE_TYPE_HANDLED;
	139	}
	140
fcc19438	141	if (entry->exit_event->id != ent->type) {
64c12e04 JB	142	WARN_ON_ONCE(1);
	143	return TRACE_TYPE_UNHANDLED;
	144	}
	145
bed1ffca FW	146	ret = trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
	147	trace->ret);
	148	if (!ret)
	149	return TRACE_TYPE_PARTIAL_LINE;
	150
	151	return TRACE_TYPE_HANDLED;
	152	}
	153
e6971969 LZ	154	extern char *__bad_type_size(void);
	155
	156	#define SYSCALL_FIELD(type, name) \
	157	sizeof(type) != sizeof(trace.name) ? \
	158	__bad_type_size() : \
26a50744 TZ	159	#type, #name, offsetof(typeof(trace), name), \
26a50744 TZ	160	sizeof(trace.name), is_signed_type(type)
e6971969	161
10a5b66f	162	int syscall_enter_format(struct ftrace_event_call call, struct trace_seq s)
dc4ddb4c FW	163	{
dc4ddb4c FW	164	int i;
e6971969	165	int ret;
31c16b13	166	struct syscall_metadata *entry = call->data;
e6971969 LZ	167	struct syscall_trace_enter trace;
e6971969 LZ	168	int offset = offsetof(struct syscall_trace_enter, args);
dc4ddb4c	169
26a50744 TZ	170	ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
26a50744 TZ	171	"\tsigned:%u;\n",
e6971969 LZ	172	SYSCALL_FIELD(int, nr));
	173	if (!ret)
	174	return 0;
dc4ddb4c FW	175
	176	for (i = 0; i < entry->nb_args; i++) {
	177	ret = trace_seq_printf(s, "\tfield:%s %s;", entry->types[i],
	178	entry->args[i]);
	179	if (!ret)
	180	return 0;
26a50744 TZ	181	ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;"
	182	"\tsigned:%u;\n", offset,
	183	sizeof(unsigned long),
	184	is_signed_type(unsigned long));
dc4ddb4c FW	185	if (!ret)
	186	return 0;
	187	offset += sizeof(unsigned long);
	188	}
	189
4539f077	190	trace_seq_puts(s, "\nprint fmt: \"");
dc4ddb4c	191	for (i = 0; i < entry->nb_args; i++) {
e6971969	192	ret = trace_seq_printf(s, "%s: 0x%%0%zulx%s", entry->args[i],
dc4ddb4c	193	sizeof(unsigned long),
4539f077	194	i == entry->nb_args - 1 ? "" : ", ");
dc4ddb4c FW	195	if (!ret)
	196	return 0;
	197	}
4539f077	198	trace_seq_putc(s, '"');
dc4ddb4c FW	199
dc4ddb4c FW	200	for (i = 0; i < entry->nb_args; i++) {
4539f077 LZ	201	ret = trace_seq_printf(s, ", ((unsigned long)(REC->%s))",
4539f077 LZ	202	entry->args[i]);
dc4ddb4c FW	203	if (!ret)
	204	return 0;
	205	}
	206
4539f077	207	return trace_seq_putc(s, '\n');
dc4ddb4c FW	208	}
dc4ddb4c FW	209
10a5b66f LZ	210	int syscall_exit_format(struct ftrace_event_call call, struct trace_seq s)
	211	{
	212	int ret;
	213	struct syscall_trace_exit trace;
	214
	215	ret = trace_seq_printf(s,
26a50744 TZ	216	"\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
	217	"\tsigned:%u;\n"
	218	"\tfield:%s %s;\toffset:%zu;\tsize:%zu;"
	219	"\tsigned:%u;\n",
10a5b66f	220	SYSCALL_FIELD(int, nr),
ee949a86	221	SYSCALL_FIELD(long, ret));
10a5b66f LZ	222	if (!ret)
	223	return 0;
	224
	225	return trace_seq_printf(s, "\nprint fmt: \"0x%%lx\", REC->ret\n");
	226	}
	227
540b7b8d LZ	228	int syscall_enter_define_fields(struct ftrace_event_call *call)
	229	{
	230	struct syscall_trace_enter trace;
31c16b13	231	struct syscall_metadata *meta = call->data;
540b7b8d	232	int ret;
540b7b8d LZ	233	int i;
	234	int offset = offsetof(typeof(trace), args);
	235
540b7b8d LZ	236	ret = trace_define_common_fields(call);
	237	if (ret)
	238	return ret;
	239
0f1ef51d LJ	240	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	241	if (ret)
	242	return ret;
	243
540b7b8d	244	for (i = 0; i < meta->nb_args; i++) {
aeaeae11 FW	245	ret = trace_define_field(call, meta->types[i],
aeaeae11 FW	246	meta->args[i], offset,
43b51ead LZ	247	sizeof(unsigned long), 0,
43b51ead LZ	248	FILTER_OTHER);
540b7b8d LZ	249	offset += sizeof(unsigned long);
	250	}
	251
	252	return ret;
	253	}
	254
	255	int syscall_exit_define_fields(struct ftrace_event_call *call)
	256	{
	257	struct syscall_trace_exit trace;
	258	int ret;
	259
	260	ret = trace_define_common_fields(call);
	261	if (ret)
	262	return ret;
	263
0f1ef51d LJ	264	ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER);
	265	if (ret)
	266	return ret;
	267
26a50744	268	ret = trace_define_field(call, SYSCALL_FIELD(long, ret),
43b51ead	269	FILTER_OTHER);
540b7b8d LZ	270
	271	return ret;
	272	}
	273
fb34a08c	274	void ftrace_syscall_enter(struct pt_regs *regs, long id)
ee08c6ec	275	{
bed1ffca FW	276	struct syscall_trace_enter *entry;
	277	struct syscall_metadata *sys_data;
	278	struct ring_buffer_event *event;
e77405ad	279	struct ring_buffer *buffer;
bed1ffca	280	int size;
ee08c6ec FW	281	int syscall_nr;
	282
	283	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	284	if (syscall_nr < 0)
cd0980fc HB	285	return;
fb34a08c JB	286	if (!test_bit(syscall_nr, enabled_enter_syscalls))
fb34a08c JB	287	return;
ee08c6ec	288
bed1ffca FW	289	sys_data = syscall_nr_to_meta(syscall_nr);
	290	if (!sys_data)
	291	return;
	292
	293	size = sizeof(entry) + sizeof(unsigned long) sys_data->nb_args;
	294
fcc19438 LJ	295	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	296	sys_data->enter_event->id, size, 0, 0);
bed1ffca FW	297	if (!event)
	298	return;
	299
	300	entry = ring_buffer_event_data(event);
	301	entry->nr = syscall_nr;
	302	syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
	303
e77405ad SR	304	if (!filter_current_check_discard(buffer, sys_data->enter_event,
	305	entry, event))
	306	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	307	}
ee08c6ec FW	308
fb34a08c	309	void ftrace_syscall_exit(struct pt_regs *regs, long ret)
ee08c6ec	310	{
bed1ffca FW	311	struct syscall_trace_exit *entry;
	312	struct syscall_metadata *sys_data;
	313	struct ring_buffer_event *event;
e77405ad	314	struct ring_buffer *buffer;
ee08c6ec FW	315	int syscall_nr;
	316
	317	syscall_nr = syscall_get_nr(current, regs);
cd0980fc HB	318	if (syscall_nr < 0)
cd0980fc HB	319	return;
fb34a08c JB	320	if (!test_bit(syscall_nr, enabled_exit_syscalls))
fb34a08c JB	321	return;
ee08c6ec	322
bed1ffca FW	323	sys_data = syscall_nr_to_meta(syscall_nr);
	324	if (!sys_data)
	325	return;
	326
fcc19438 LJ	327	event = trace_current_buffer_lock_reserve(&buffer,
fcc19438 LJ	328	sys_data->exit_event->id, sizeof(*entry), 0, 0);
bed1ffca FW	329	if (!event)
	330	return;
	331
	332	entry = ring_buffer_event_data(event);
	333	entry->nr = syscall_nr;
	334	entry->ret = syscall_get_return_value(current, regs);
	335
e77405ad SR	336	if (!filter_current_check_discard(buffer, sys_data->exit_event,
	337	entry, event))
	338	trace_current_buffer_unlock_commit(buffer, event, 0, 0);
ee08c6ec FW	339	}
ee08c6ec FW	340
bd1a5c84	341	int reg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	342	{
fb34a08c JB	343	int ret = 0;
fb34a08c JB	344	int num;
fb34a08c	345
c252f657	346	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	347	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	348	return -ENOSYS;
	349	mutex_lock(&syscall_trace_lock);
	350	if (!sys_refcount_enter)
1c569f02	351	ret = register_trace_sys_enter(ftrace_syscall_enter);
fb34a08c JB	352	if (ret) {
	353	pr_info("event trace: Could not activate"
	354	"syscall entry trace point");
	355	} else {
	356	set_bit(num, enabled_enter_syscalls);
	357	sys_refcount_enter++;
	358	}
	359	mutex_unlock(&syscall_trace_lock);
	360	return ret;
ee08c6ec FW	361	}
ee08c6ec FW	362
bd1a5c84	363	void unreg_event_syscall_enter(struct ftrace_event_call *call)
ee08c6ec	364	{
fb34a08c	365	int num;
ee08c6ec	366
c252f657	367	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	368	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	369	return;
	370	mutex_lock(&syscall_trace_lock);
	371	sys_refcount_enter--;
	372	clear_bit(num, enabled_enter_syscalls);
	373	if (!sys_refcount_enter)
1c569f02	374	unregister_trace_sys_enter(ftrace_syscall_enter);
fb34a08c JB	375	mutex_unlock(&syscall_trace_lock);
fb34a08c JB	376	}
ee08c6ec	377
bd1a5c84	378	int reg_event_syscall_exit(struct ftrace_event_call *call)
ee08c6ec	379	{
fb34a08c JB	380	int ret = 0;
fb34a08c JB	381	int num;
fb34a08c	382
c252f657	383	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	384	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	385	return -ENOSYS;
	386	mutex_lock(&syscall_trace_lock);
	387	if (!sys_refcount_exit)
1c569f02	388	ret = register_trace_sys_exit(ftrace_syscall_exit);
fb34a08c JB	389	if (ret) {
	390	pr_info("event trace: Could not activate"
	391	"syscall exit trace point");
	392	} else {
	393	set_bit(num, enabled_exit_syscalls);
	394	sys_refcount_exit++;
ee08c6ec	395	}
fb34a08c JB	396	mutex_unlock(&syscall_trace_lock);
	397	return ret;
	398	}
ee08c6ec	399
bd1a5c84	400	void unreg_event_syscall_exit(struct ftrace_event_call *call)
fb34a08c JB	401	{
fb34a08c JB	402	int num;
ee08c6ec	403
c252f657	404	num = ((struct syscall_metadata *)call->data)->syscall_nr;
57421dbb	405	if (num < 0 \|\| num >= NR_syscalls)
fb34a08c JB	406	return;
	407	mutex_lock(&syscall_trace_lock);
	408	sys_refcount_exit--;
	409	clear_bit(num, enabled_exit_syscalls);
	410	if (!sys_refcount_exit)
1c569f02	411	unregister_trace_sys_exit(ftrace_syscall_exit);
fb34a08c	412	mutex_unlock(&syscall_trace_lock);
ee08c6ec	413	}
fb34a08c	414
c44fc770 FW	415	int __init init_ftrace_syscalls(void)
	416	{
	417	struct syscall_metadata *meta;
	418	unsigned long addr;
	419	int i;
	420
	421	syscalls_metadata = kzalloc(sizeof(syscalls_metadata)
	422	NR_syscalls, GFP_KERNEL);
	423	if (!syscalls_metadata) {
	424	WARN_ON(1);
	425	return -ENOMEM;
	426	}
	427
	428	for (i = 0; i < NR_syscalls; i++) {
	429	addr = arch_syscall_addr(i);
	430	meta = find_syscall_meta(addr);
c252f657 LJ	431	if (!meta)
	432	continue;
	433
	434	meta->syscall_nr = i;
c44fc770 FW	435	syscalls_metadata[i] = meta;
	436	}
	437
	438	return 0;
	439	}
	440	core_initcall(init_ftrace_syscalls);
	441
f4b5ffcc	442	#ifdef CONFIG_EVENT_PROFILE
19007a67	443
57421dbb JB	444	static DECLARE_BITMAP(enabled_prof_enter_syscalls, NR_syscalls);
57421dbb JB	445	static DECLARE_BITMAP(enabled_prof_exit_syscalls, NR_syscalls);
f4b5ffcc JB	446	static int sys_prof_refcount_enter;
	447	static int sys_prof_refcount_exit;
	448
	449	static void prof_syscall_enter(struct pt_regs *regs, long id)
	450	{
	451	struct syscall_metadata *sys_data;
20ab4425 FW	452	struct syscall_trace_enter *rec;
20ab4425 FW	453	unsigned long flags;
ce71b9df	454	char *trace_buf;
20ab4425	455	char *raw_data;
f4b5ffcc	456	int syscall_nr;
4ed7c92d	457	int rctx;
19007a67	458	int size;
20ab4425	459	int cpu;
f4b5ffcc JB	460
	461	syscall_nr = syscall_get_nr(current, regs);
	462	if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
	463	return;
	464
	465	sys_data = syscall_nr_to_meta(syscall_nr);
	466	if (!sys_data)
	467	return;
	468
19007a67 FW	469	/* get the size after alignment with the u32 buffer size field */
	470	size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
	471	size = ALIGN(size + sizeof(u32), sizeof(u64));
	472	size -= sizeof(u32);
	473
20ab4425 FW	474	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	475	"profile buffer not large enough"))
	476	return;
	477
	478	/* Protect the per cpu buffer, begin the rcu read side */
	479	local_irq_save(flags);
19007a67	480
4ed7c92d PZ	481	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	482	if (rctx < 0)
ce71b9df FW	483	goto end_recursion;
ce71b9df FW	484
20ab4425 FW	485	cpu = smp_processor_id();
20ab4425 FW	486
28889bf9	487	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	488
444a2a3b	489	if (!trace_buf)
20ab4425	490	goto end;
19007a67	491
ce71b9df	492	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	493
	494	/* zero the dead bytes from align to not leak stack to user */
	495	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	496
	497	rec = (struct syscall_trace_enter *) raw_data;
	498	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	499	rec->ent.type = sys_data->enter_event->id;
20ab4425 FW	500	rec->nr = syscall_nr;
	501	syscall_get_arguments(current, regs, 0, sys_data->nb_args,
	502	(unsigned long *)&rec->args);
fcc19438	503	perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size);
20ab4425 FW	504
20ab4425 FW	505	end:
4ed7c92d	506	perf_swevent_put_recursion_context(rctx);
ce71b9df	507	end_recursion:
20ab4425	508	local_irq_restore(flags);
f4b5ffcc JB	509	}
	510
	511	int reg_prof_syscall_enter(char *name)
	512	{
	513	int ret = 0;
	514	int num;
	515
	516	num = syscall_name_to_nr(name);
57421dbb	517	if (num < 0 \|\| num >= NR_syscalls)
f4b5ffcc JB	518	return -ENOSYS;
	519
	520	mutex_lock(&syscall_trace_lock);
	521	if (!sys_prof_refcount_enter)
1c569f02	522	ret = register_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	523	if (ret) {
	524	pr_info("event trace: Could not activate"
	525	"syscall entry trace point");
	526	} else {
	527	set_bit(num, enabled_prof_enter_syscalls);
	528	sys_prof_refcount_enter++;
	529	}
	530	mutex_unlock(&syscall_trace_lock);
	531	return ret;
	532	}
	533
	534	void unreg_prof_syscall_enter(char *name)
	535	{
	536	int num;
	537
	538	num = syscall_name_to_nr(name);
57421dbb	539	if (num < 0 \|\| num >= NR_syscalls)
f4b5ffcc JB	540	return;
	541
	542	mutex_lock(&syscall_trace_lock);
	543	sys_prof_refcount_enter--;
	544	clear_bit(num, enabled_prof_enter_syscalls);
	545	if (!sys_prof_refcount_enter)
1c569f02	546	unregister_trace_sys_enter(prof_syscall_enter);
f4b5ffcc JB	547	mutex_unlock(&syscall_trace_lock);
	548	}
	549
	550	static void prof_syscall_exit(struct pt_regs *regs, long ret)
	551	{
	552	struct syscall_metadata *sys_data;
20ab4425 FW	553	struct syscall_trace_exit *rec;
20ab4425 FW	554	unsigned long flags;
f4b5ffcc	555	int syscall_nr;
ce71b9df	556	char *trace_buf;
20ab4425	557	char *raw_data;
4ed7c92d	558	int rctx;
20ab4425 FW	559	int size;
20ab4425 FW	560	int cpu;
f4b5ffcc JB	561
	562	syscall_nr = syscall_get_nr(current, regs);
	563	if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
	564	return;
	565
	566	sys_data = syscall_nr_to_meta(syscall_nr);
	567	if (!sys_data)
	568	return;
	569
20ab4425 FW	570	/* We can probably do that at build time */
	571	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
	572	size -= sizeof(u32);
19007a67	573
20ab4425 FW	574	/*
	575	* Impossible, but be paranoid with the future
	576	* How to put this check outside runtime?
	577	*/
	578	if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
	579	"exit event has grown above profile buffer size"))
	580	return;
	581
	582	/* Protect the per cpu buffer, begin the rcu read side */
	583	local_irq_save(flags);
ce71b9df	584
4ed7c92d PZ	585	rctx = perf_swevent_get_recursion_context();
4ed7c92d PZ	586	if (rctx < 0)
ce71b9df FW	587	goto end_recursion;
ce71b9df FW	588
20ab4425 FW	589	cpu = smp_processor_id();
20ab4425 FW	590
28889bf9	591	trace_buf = rcu_dereference(perf_trace_buf);
20ab4425	592
444a2a3b	593	if (!trace_buf)
20ab4425 FW	594	goto end;
20ab4425 FW	595
ce71b9df	596	raw_data = per_cpu_ptr(trace_buf, cpu);
20ab4425 FW	597
	598	/* zero the dead bytes from align to not leak stack to user */
	599	(u64 )(&raw_data[size - sizeof(u64)]) = 0ULL;
	600
	601	rec = (struct syscall_trace_exit *)raw_data;
	602
	603	tracing_generic_entry_update(&rec->ent, 0, 0);
fcc19438	604	rec->ent.type = sys_data->exit_event->id;
20ab4425 FW	605	rec->nr = syscall_nr;
	606	rec->ret = syscall_get_return_value(current, regs);
	607
fcc19438	608	perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size);
20ab4425 FW	609
20ab4425 FW	610	end:
4ed7c92d	611	perf_swevent_put_recursion_context(rctx);
ce71b9df	612	end_recursion:
20ab4425	613	local_irq_restore(flags);
f4b5ffcc JB	614	}
	615
	616	int reg_prof_syscall_exit(char *name)
	617	{
	618	int ret = 0;
	619	int num;
	620
	621	num = syscall_name_to_nr(name);
57421dbb	622	if (num < 0 \|\| num >= NR_syscalls)
f4b5ffcc JB	623	return -ENOSYS;
	624
	625	mutex_lock(&syscall_trace_lock);
	626	if (!sys_prof_refcount_exit)
1c569f02	627	ret = register_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	628	if (ret) {
	629	pr_info("event trace: Could not activate"
	630	"syscall entry trace point");
	631	} else {
	632	set_bit(num, enabled_prof_exit_syscalls);
	633	sys_prof_refcount_exit++;
	634	}
	635	mutex_unlock(&syscall_trace_lock);
	636	return ret;
	637	}
	638
	639	void unreg_prof_syscall_exit(char *name)
	640	{
	641	int num;
	642
	643	num = syscall_name_to_nr(name);
57421dbb	644	if (num < 0 \|\| num >= NR_syscalls)
f4b5ffcc JB	645	return;
	646
	647	mutex_lock(&syscall_trace_lock);
	648	sys_prof_refcount_exit--;
	649	clear_bit(num, enabled_prof_exit_syscalls);
	650	if (!sys_prof_refcount_exit)
1c569f02	651	unregister_trace_sys_exit(prof_syscall_exit);
f4b5ffcc JB	652	mutex_unlock(&syscall_trace_lock);
	653	}
	654
	655	#endif
	656
	657