[linux-block.git] / kernel / trace / trace_functions.c

/*
 * ring buffer based function tracer
 *
 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
 *
 * Based on code from the latency_tracer, that is:
 *
 *  Copyright (C) 2004-2006 Ingo Molnar
 *  Copyright (C) 2004 Nadia Yvette Chambers
 */
#include <linux/ring_buffer.h>
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#include <linux/ftrace.h>
#include <linux/slab.h>
#include <linux/fs.h>

#include "trace.h"

static void tracing_start_function_trace(struct trace_array *tr);
static void tracing_stop_function_trace(struct trace_array *tr);
static void
function_trace_call(unsigned long ip, unsigned long parent_ip,
		    struct ftrace_ops *op, struct pt_regs *pt_regs);
static void
function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
			  struct ftrace_ops *op, struct pt_regs *pt_regs);
static struct ftrace_ops trace_ops;
static struct ftrace_ops trace_stack_ops;
static struct tracer_flags func_flags;

/* Our option */
enum {
	TRACE_FUNC_OPT_STACK	= 0x1,
};

static int allocate_ftrace_ops(struct trace_array *tr)
{
	struct ftrace_ops *ops;

	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
	if (!ops)
		return -ENOMEM;

	/* Currently only the non stack verision is supported */
	ops->func = function_trace_call;
	ops->flags = FTRACE_OPS_FL_RECURSION_SAFE;

	tr->ops = ops;
	ops->private = tr;
	return 0;
}


int ftrace_create_function_files(struct trace_array *tr,
				 struct dentry *parent)
{
	int ret;

	/* The top level array uses the "global_ops". */
	if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL)) {
		ret = allocate_ftrace_ops(tr);
		if (ret)
			return ret;
	}

	ftrace_create_filter_files(tr->ops, parent);

	return 0;
}

void ftrace_destroy_function_files(struct trace_array *tr)
{
	ftrace_destroy_filter_files(tr->ops);
	kfree(tr->ops);
	tr->ops = NULL;
}

static int function_trace_init(struct trace_array *tr)
{
	struct ftrace_ops *ops;

	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
		/* There's only one global tr */
		if (!trace_ops.private) {
			trace_ops.private = tr;
			trace_stack_ops.private = tr;
		}

		if (func_flags.val & TRACE_FUNC_OPT_STACK)
			ops = &trace_stack_ops;
		else
			ops = &trace_ops;
		tr->ops = ops;
	} else if (!tr->ops) {
		/*
		 * Instance trace_arrays get their ops allocated
		 * at instance creation. Unless it failed
		 * the allocation.
		 */
		return -ENOMEM;
	}

	tr->trace_buffer.cpu = get_cpu();
	put_cpu();

	tracing_start_cmdline_record();
	tracing_start_function_trace(tr);
	return 0;
}

static void function_trace_reset(struct trace_array *tr)
{
	tracing_stop_function_trace(tr);
	tracing_stop_cmdline_record();
}

static void function_trace_start(struct trace_array *tr)
{
	tracing_reset_online_cpus(&tr->trace_buffer);
}

static void
function_trace_call(unsigned long ip, unsigned long parent_ip,
		    struct ftrace_ops *op, struct pt_regs *pt_regs)
{
	struct trace_array *tr = op->private;
	struct trace_array_cpu *data;
	unsigned long flags;
	int bit;
	int cpu;
	int pc;

	if (unlikely(!tr->function_enabled))
		return;

	pc = preempt_count();
	preempt_disable_notrace();

	bit = trace_test_and_set_recursion(TRACE_FTRACE_START, TRACE_FTRACE_MAX);
	if (bit < 0)
		goto out;

	cpu = smp_processor_id();
	data = per_cpu_ptr(tr->trace_buffer.data, cpu);
	if (!atomic_read(&data->disabled)) {
		local_save_flags(flags);
		trace_function(tr, ip, parent_ip, flags, pc);
	}
	trace_clear_recursion(bit);

 out:
	preempt_enable_notrace();
}

static void
function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
			  struct ftrace_ops *op, struct pt_regs *pt_regs)
{
	struct trace_array *tr = op->private;
	struct trace_array_cpu *data;
	unsigned long flags;
	long disabled;
	int cpu;
	int pc;

	if (unlikely(!tr->function_enabled))
		return;

	/*
	 * Need to use raw, since this must be called before the
	 * recursive protection is performed.
	 */
	local_irq_save(flags);
	cpu = raw_smp_processor_id();
	data = per_cpu_ptr(tr->trace_buffer.data, cpu);
	disabled = atomic_inc_return(&data->disabled);

	if (likely(disabled == 1)) {
		pc = preempt_count();
		trace_function(tr, ip, parent_ip, flags, pc);
		/*
		 * skip over 5 funcs:
		 *    __ftrace_trace_stack,
		 *    __trace_stack,
		 *    function_stack_trace_call
		 *    ftrace_list_func
		 *    ftrace_call
		 */
		__trace_stack(tr, flags, 5, pc);
	}

	atomic_dec(&data->disabled);
	local_irq_restore(flags);
}

static struct ftrace_ops trace_ops __read_mostly =
{
	.func = function_trace_call,
	.flags = FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_RECURSION_SAFE,
};

static struct ftrace_ops trace_stack_ops __read_mostly =
{
	.func = function_stack_trace_call,
	.flags = FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_RECURSION_SAFE,
};

static struct tracer_opt func_opts[] = {
#ifdef CONFIG_STACKTRACE
	{ TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
#endif
	{ } /* Always set a last empty entry */
};

static struct tracer_flags func_flags = {
	.val = 0, /* By default: all flags disabled */
	.opts = func_opts
};

static void tracing_start_function_trace(struct trace_array *tr)
{
	tr->function_enabled = 0;
	register_ftrace_function(tr->ops);
	tr->function_enabled = 1;
}

static void tracing_stop_function_trace(struct trace_array *tr)
{
	tr->function_enabled = 0;
	unregister_ftrace_function(tr->ops);
}

static int
func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
{
	switch (bit) {
	case TRACE_FUNC_OPT_STACK:
		/* do nothing if already set */
		if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
			break;

		unregister_ftrace_function(tr->ops);

		if (set) {
			tr->ops = &trace_stack_ops;
			register_ftrace_function(tr->ops);
		} else {
			tr->ops = &trace_ops;
			register_ftrace_function(tr->ops);
		}

		break;
	default:
		return -EINVAL;
	}

	return 0;
}

static struct tracer function_trace __tracer_data =
{
	.name		= "function",
	.init		= function_trace_init,
	.reset		= function_trace_reset,
	.start		= function_trace_start,
	.wait_pipe	= poll_wait_pipe,
	.flags		= &func_flags,
	.set_flag	= func_set_flag,
	.allow_instances = true,
#ifdef CONFIG_FTRACE_SELFTEST
	.selftest	= trace_selftest_startup_function,
#endif
};

#ifdef CONFIG_DYNAMIC_FTRACE
static int update_count(void **data)
{
	unsigned long *count = (long *)data;

	if (!*count)
		return 0;

	if (*count != -1)
		(*count)--;

	return 1;
}

static void
ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, void **data)
{
	if (tracing_is_on())
		return;

	if (update_count(data))
		tracing_on();
}

static void
ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, void **data)
{
	if (!tracing_is_on())
		return;

	if (update_count(data))
		tracing_off();
}

static void
ftrace_traceon(unsigned long ip, unsigned long parent_ip, void **data)
{
	if (tracing_is_on())
		return;

	tracing_on();
}

static void
ftrace_traceoff(unsigned long ip, unsigned long parent_ip, void **data)
{
	if (!tracing_is_on())
		return;

	tracing_off();
}

/*
 * Skip 4:
 *   ftrace_stacktrace()
 *   function_trace_probe_call()
 *   ftrace_ops_list_func()
 *   ftrace_call()
 */
#define STACK_SKIP 4

static void
ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, void **data)
{
	trace_dump_stack(STACK_SKIP);
}

static void
ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, void **data)
{
	if (!tracing_is_on())
		return;

	if (update_count(data))
		trace_dump_stack(STACK_SKIP);
}

static void
ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, void **data)
{
	if (update_count(data))
		ftrace_dump(DUMP_ALL);
}

/* Only dump the current CPU buffer. */
static void
ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, void **data)
{
	if (update_count(data))
		ftrace_dump(DUMP_ORIG);
}

static int
ftrace_probe_print(const char *name, struct seq_file *m,
		   unsigned long ip, void *data)
{
	long count = (long)data;

	seq_printf(m, "%ps:%s", (void *)ip, name);

	if (count == -1)
		seq_printf(m, ":unlimited\n");
	else
		seq_printf(m, ":count=%ld\n", count);

	return 0;
}

static int
ftrace_traceon_print(struct seq_file *m, unsigned long ip,
			 struct ftrace_probe_ops *ops, void *data)
{
	return ftrace_probe_print("traceon", m, ip, data);
}

static int
ftrace_traceoff_print(struct seq_file *m, unsigned long ip,
			 struct ftrace_probe_ops *ops, void *data)
{
	return ftrace_probe_print("traceoff", m, ip, data);
}

static int
ftrace_stacktrace_print(struct seq_file *m, unsigned long ip,
			struct ftrace_probe_ops *ops, void *data)
{
	return ftrace_probe_print("stacktrace", m, ip, data);
}

static int
ftrace_dump_print(struct seq_file *m, unsigned long ip,
			struct ftrace_probe_ops *ops, void *data)
{
	return ftrace_probe_print("dump", m, ip, data);
}

static int
ftrace_cpudump_print(struct seq_file *m, unsigned long ip,
			struct ftrace_probe_ops *ops, void *data)
{
	return ftrace_probe_print("cpudump", m, ip, data);
}

static struct ftrace_probe_ops traceon_count_probe_ops = {
	.func			= ftrace_traceon_count,
	.print			= ftrace_traceon_print,
};

static struct ftrace_probe_ops traceoff_count_probe_ops = {
	.func			= ftrace_traceoff_count,
	.print			= ftrace_traceoff_print,
};

static struct ftrace_probe_ops stacktrace_count_probe_ops = {
	.func			= ftrace_stacktrace_count,
	.print			= ftrace_stacktrace_print,
};

static struct ftrace_probe_ops dump_probe_ops = {
	.func			= ftrace_dump_probe,
	.print			= ftrace_dump_print,
};

static struct ftrace_probe_ops cpudump_probe_ops = {
	.func			= ftrace_cpudump_probe,
	.print			= ftrace_cpudump_print,
};

static struct ftrace_probe_ops traceon_probe_ops = {
	.func			= ftrace_traceon,
	.print			= ftrace_traceon_print,
};

static struct ftrace_probe_ops traceoff_probe_ops = {
	.func			= ftrace_traceoff,
	.print			= ftrace_traceoff_print,
};

static struct ftrace_probe_ops stacktrace_probe_ops = {
	.func			= ftrace_stacktrace,
	.print			= ftrace_stacktrace_print,
};

static int
ftrace_trace_probe_callback(struct ftrace_probe_ops *ops,
			    struct ftrace_hash *hash, char *glob,
			    char *cmd, char *param, int enable)
{
	void *count = (void *)-1;
	char *number;
	int ret;

	/* hash funcs only work with set_ftrace_filter */
	if (!enable)
		return -EINVAL;

	if (glob[0] == '!') {
		unregister_ftrace_function_probe_func(glob+1, ops);
		return 0;
	}

	if (!param)
		goto out_reg;

	number = strsep(&param, ":");

	if (!strlen(number))
		goto out_reg;

	/*
	 * We use the callback data field (which is a pointer)
	 * as our counter.
	 */
	ret = kstrtoul(number, 0, (unsigned long *)&count);
	if (ret)
		return ret;

 out_reg:
	ret = register_ftrace_function_probe(glob, ops, count);

	return ret < 0 ? ret : 0;
}

static int
ftrace_trace_onoff_callback(struct ftrace_hash *hash,
			    char *glob, char *cmd, char *param, int enable)
{
	struct ftrace_probe_ops *ops;

	/* we register both traceon and traceoff to this callback */
	if (strcmp(cmd, "traceon") == 0)
		ops = param ? &traceon_count_probe_ops : &traceon_probe_ops;
	else
		ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops;

	return ftrace_trace_probe_callback(ops, hash, glob, cmd,
					   param, enable);
}

static int
ftrace_stacktrace_callback(struct ftrace_hash *hash,
			   char *glob, char *cmd, char *param, int enable)
{
	struct ftrace_probe_ops *ops;

	ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops;

	return ftrace_trace_probe_callback(ops, hash, glob, cmd,
					   param, enable);
}

static int
ftrace_dump_callback(struct ftrace_hash *hash,
			   char *glob, char *cmd, char *param, int enable)
{
	struct ftrace_probe_ops *ops;

	ops = &dump_probe_ops;

	/* Only dump once. */
	return ftrace_trace_probe_callback(ops, hash, glob, cmd,
					   "1", enable);
}

static int
ftrace_cpudump_callback(struct ftrace_hash *hash,
			   char *glob, char *cmd, char *param, int enable)
{
	struct ftrace_probe_ops *ops;

	ops = &cpudump_probe_ops;

	/* Only dump once. */
	return ftrace_trace_probe_callback(ops, hash, glob, cmd,
					   "1", enable);
}

static struct ftrace_func_command ftrace_traceon_cmd = {
	.name			= "traceon",
	.func			= ftrace_trace_onoff_callback,
};

static struct ftrace_func_command ftrace_traceoff_cmd = {
	.name			= "traceoff",
	.func			= ftrace_trace_onoff_callback,
};

static struct ftrace_func_command ftrace_stacktrace_cmd = {
	.name			= "stacktrace",
	.func			= ftrace_stacktrace_callback,
};

static struct ftrace_func_command ftrace_dump_cmd = {
	.name			= "dump",
	.func			= ftrace_dump_callback,
};

static struct ftrace_func_command ftrace_cpudump_cmd = {
	.name			= "cpudump",
	.func			= ftrace_cpudump_callback,
};

static int __init init_func_cmd_traceon(void)
{
	int ret;

	ret = register_ftrace_command(&ftrace_traceoff_cmd);
	if (ret)
		return ret;

	ret = register_ftrace_command(&ftrace_traceon_cmd);
	if (ret)
		goto out_free_traceoff;

	ret = register_ftrace_command(&ftrace_stacktrace_cmd);
	if (ret)
		goto out_free_traceon;

	ret = register_ftrace_command(&ftrace_dump_cmd);
	if (ret)
		goto out_free_stacktrace;

	ret = register_ftrace_command(&ftrace_cpudump_cmd);
	if (ret)
		goto out_free_dump;

	return 0;

 out_free_dump:
	unregister_ftrace_command(&ftrace_dump_cmd);
 out_free_stacktrace:
	unregister_ftrace_command(&ftrace_stacktrace_cmd);
 out_free_traceon:
	unregister_ftrace_command(&ftrace_traceon_cmd);
 out_free_traceoff:
	unregister_ftrace_command(&ftrace_traceoff_cmd);

	return ret;
}
#else
static inline int init_func_cmd_traceon(void)
{
	return 0;
}
#endif /* CONFIG_DYNAMIC_FTRACE */

static __init int init_function_trace(void)
{
	init_func_cmd_traceon();
	return register_tracer(&function_trace);
}
core_initcall(init_function_trace);
Commit	Line	Data
1b29b018 SR	1	/*
	2	* ring buffer based function tracer
	3	*
	4	* Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
	5	* Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
	6	*
	7	* Based on code from the latency_tracer, that is:
	8	*
	9	* Copyright (C) 2004-2006 Ingo Molnar
6d49e352	10	* Copyright (C) 2004 Nadia Yvette Chambers
1b29b018	11	*/
23b4ff3a	12	#include <linux/ring_buffer.h>
1b29b018 SR	13	#include <linux/debugfs.h>
	14	#include <linux/uaccess.h>
	15	#include <linux/ftrace.h>
f20a5806	16	#include <linux/slab.h>
2e0f5761	17	#include <linux/fs.h>
1b29b018 SR	18
	19	#include "trace.h"
	20
f20a5806 SRRH	21	static void tracing_start_function_trace(struct trace_array *tr);
	22	static void tracing_stop_function_trace(struct trace_array *tr);
	23	static void
	24	function_trace_call(unsigned long ip, unsigned long parent_ip,
	25	struct ftrace_ops op, struct pt_regs pt_regs);
	26	static void
	27	function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
	28	struct ftrace_ops op, struct pt_regs pt_regs);
	29	static struct ftrace_ops trace_ops;
	30	static struct ftrace_ops trace_stack_ops;
	31	static struct tracer_flags func_flags;
	32
	33	/* Our option */
	34	enum {
	35	TRACE_FUNC_OPT_STACK = 0x1,
	36	};
	37
	38	static int allocate_ftrace_ops(struct trace_array *tr)
	39	{
	40	struct ftrace_ops *ops;
a225cdd2	41
f20a5806 SRRH	42	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
	43	if (!ops)
	44	return -ENOMEM;
53614991	45
f20a5806 SRRH	46	/* Currently only the non stack verision is supported */
	47	ops->func = function_trace_call;
	48	ops->flags = FTRACE_OPS_FL_RECURSION_SAFE;
	49
	50	tr->ops = ops;
	51	ops->private = tr;
	52	return 0;
	53	}
a225cdd2	54
591dffda SRRH	55
	56	int ftrace_create_function_files(struct trace_array *tr,
	57	struct dentry *parent)
	58	{
	59	int ret;
	60
	61	/* The top level array uses the "global_ops". */
	62	if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL)) {
	63	ret = allocate_ftrace_ops(tr);
	64	if (ret)
	65	return ret;
	66	}
	67
	68	ftrace_create_filter_files(tr->ops, parent);
	69
	70	return 0;
	71	}
	72
	73	void ftrace_destroy_function_files(struct trace_array *tr)
	74	{
	75	ftrace_destroy_filter_files(tr->ops);
	76	kfree(tr->ops);
	77	tr->ops = NULL;
	78	}
	79
b6f11df2	80	static int function_trace_init(struct trace_array *tr)
1b29b018	81	{
f20a5806	82	struct ftrace_ops *ops;
f20a5806 SRRH	83
	84	if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
	85	/* There's only one global tr */
	86	if (!trace_ops.private) {
	87	trace_ops.private = tr;
	88	trace_stack_ops.private = tr;
	89	}
	90
	91	if (func_flags.val & TRACE_FUNC_OPT_STACK)
	92	ops = &trace_stack_ops;
	93	else
	94	ops = &trace_ops;
	95	tr->ops = ops;
591dffda SRRH	96	} else if (!tr->ops) {
	97	/*
	98	* Instance trace_arrays get their ops allocated
	99	* at instance creation. Unless it failed
	100	* the allocation.
	101	*/
	102	return -ENOMEM;
f20a5806 SRRH	103	}
f20a5806 SRRH	104
12883efb	105	tr->trace_buffer.cpu = get_cpu();
26bc83f4 SR	106	put_cpu();
26bc83f4 SR	107
41bc8144	108	tracing_start_cmdline_record();
f20a5806	109	tracing_start_function_trace(tr);
1c80025a	110	return 0;
1b29b018 SR	111	}
1b29b018 SR	112
e309b41d	113	static void function_trace_reset(struct trace_array *tr)
1b29b018	114	{
f20a5806	115	tracing_stop_function_trace(tr);
b6f11df2	116	tracing_stop_cmdline_record();
1b29b018 SR	117	}
1b29b018 SR	118
9036990d SR	119	static void function_trace_start(struct trace_array *tr)
9036990d SR	120	{
12883efb	121	tracing_reset_online_cpus(&tr->trace_buffer);
9036990d SR	122	}
9036990d SR	123
bb3c3c95	124	static void
2f5f6ad9	125	function_trace_call(unsigned long ip, unsigned long parent_ip,
a1e2e31d	126	struct ftrace_ops op, struct pt_regs pt_regs)
bb3c3c95	127	{
f20a5806	128	struct trace_array *tr = op->private;
bb3c3c95 SR	129	struct trace_array_cpu *data;
bb3c3c95 SR	130	unsigned long flags;
d41032a8	131	int bit;
bb3c3c95 SR	132	int cpu;
	133	int pc;
	134
f20a5806	135	if (unlikely(!tr->function_enabled))
bb3c3c95 SR	136	return;
bb3c3c95 SR	137
897f68a4 SR	138	pc = preempt_count();
897f68a4 SR	139	preempt_disable_notrace();
bb3c3c95	140
897f68a4 SR	141	bit = trace_test_and_set_recursion(TRACE_FTRACE_START, TRACE_FTRACE_MAX);
	142	if (bit < 0)
	143	goto out;
	144
	145	cpu = smp_processor_id();
12883efb	146	data = per_cpu_ptr(tr->trace_buffer.data, cpu);
897f68a4 SR	147	if (!atomic_read(&data->disabled)) {
897f68a4 SR	148	local_save_flags(flags);
7be42151	149	trace_function(tr, ip, parent_ip, flags, pc);
bb3c3c95	150	}
897f68a4	151	trace_clear_recursion(bit);
bb3c3c95	152
897f68a4 SR	153	out:
897f68a4 SR	154	preempt_enable_notrace();
bb3c3c95 SR	155	}
bb3c3c95 SR	156
53614991	157	static void
2f5f6ad9	158	function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
a1e2e31d	159	struct ftrace_ops op, struct pt_regs pt_regs)
53614991	160	{
f20a5806	161	struct trace_array *tr = op->private;
53614991 SR	162	struct trace_array_cpu *data;
	163	unsigned long flags;
	164	long disabled;
	165	int cpu;
	166	int pc;
	167
f20a5806	168	if (unlikely(!tr->function_enabled))
53614991 SR	169	return;
	170
	171	/*
	172	* Need to use raw, since this must be called before the
	173	* recursive protection is performed.
	174	*/
	175	local_irq_save(flags);
	176	cpu = raw_smp_processor_id();
12883efb	177	data = per_cpu_ptr(tr->trace_buffer.data, cpu);
53614991 SR	178	disabled = atomic_inc_return(&data->disabled);
	179
	180	if (likely(disabled == 1)) {
	181	pc = preempt_count();
7be42151	182	trace_function(tr, ip, parent_ip, flags, pc);
53614991 SR	183	/*
	184	* skip over 5 funcs:
	185	* __ftrace_trace_stack,
	186	* __trace_stack,
	187	* function_stack_trace_call
	188	* ftrace_list_func
	189	* ftrace_call
	190	*/
7be42151	191	__trace_stack(tr, flags, 5, pc);
53614991 SR	192	}
	193
	194	atomic_dec(&data->disabled);
	195	local_irq_restore(flags);
	196	}
	197
bb3c3c95 SR	198	static struct ftrace_ops trace_ops __read_mostly =
	199	{
	200	.func = function_trace_call,
4740974a	201	.flags = FTRACE_OPS_FL_GLOBAL \| FTRACE_OPS_FL_RECURSION_SAFE,
bb3c3c95 SR	202	};
bb3c3c95 SR	203
53614991 SR	204	static struct ftrace_ops trace_stack_ops __read_mostly =
	205	{
	206	.func = function_stack_trace_call,
4740974a	207	.flags = FTRACE_OPS_FL_GLOBAL \| FTRACE_OPS_FL_RECURSION_SAFE,
53614991 SR	208	};
53614991 SR	209
53614991 SR	210	static struct tracer_opt func_opts[] = {
	211	#ifdef CONFIG_STACKTRACE
	212	{ TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
	213	#endif
	214	{ } /* Always set a last empty entry */
	215	};
	216
	217	static struct tracer_flags func_flags = {
	218	.val = 0, /* By default: all flags disabled */
	219	.opts = func_opts
	220	};
	221
f20a5806	222	static void tracing_start_function_trace(struct trace_array *tr)
3eb36aa0	223	{
f20a5806 SRRH	224	tr->function_enabled = 0;
	225	register_ftrace_function(tr->ops);
	226	tr->function_enabled = 1;
3eb36aa0 SR	227	}
3eb36aa0 SR	228
f20a5806	229	static void tracing_stop_function_trace(struct trace_array *tr)
3eb36aa0	230	{
f20a5806 SRRH	231	tr->function_enabled = 0;
f20a5806 SRRH	232	unregister_ftrace_function(tr->ops);
3eb36aa0 SR	233	}
3eb36aa0 SR	234
8c1a49ae SRRH	235	static int
8c1a49ae SRRH	236	func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
53614991	237	{
f555f123 AV	238	switch (bit) {
f555f123 AV	239	case TRACE_FUNC_OPT_STACK:
53614991 SR	240	/* do nothing if already set */
53614991 SR	241	if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
f555f123	242	break;
53614991	243
f20a5806 SRRH	244	unregister_ftrace_function(tr->ops);
f20a5806 SRRH	245
3eb36aa0	246	if (set) {
f20a5806 SRRH	247	tr->ops = &trace_stack_ops;
f20a5806 SRRH	248	register_ftrace_function(tr->ops);
3eb36aa0	249	} else {
f20a5806 SRRH	250	tr->ops = &trace_ops;
f20a5806 SRRH	251	register_ftrace_function(tr->ops);
3eb36aa0	252	}
53614991	253
f555f123 AV	254	break;
	255	default:
	256	return -EINVAL;
53614991 SR	257	}
53614991 SR	258
f555f123	259	return 0;
53614991 SR	260	}
53614991 SR	261
8f768993	262	static struct tracer function_trace __tracer_data =
1b29b018	263	{
3eb36aa0 SR	264	.name = "function",
	265	.init = function_trace_init,
	266	.reset = function_trace_reset,
	267	.start = function_trace_start,
6eaaa5d5	268	.wait_pipe = poll_wait_pipe,
53614991 SR	269	.flags = &func_flags,
53614991 SR	270	.set_flag = func_set_flag,
f20a5806	271	.allow_instances = true,
60a11774	272	#ifdef CONFIG_FTRACE_SELFTEST
3eb36aa0	273	.selftest = trace_selftest_startup_function,
60a11774	274	#endif
1b29b018 SR	275	};
1b29b018 SR	276
23b4ff3a	277	#ifdef CONFIG_DYNAMIC_FTRACE
1c317143	278	static int update_count(void **data)
23b4ff3a	279	{
1c317143	280	unsigned long count = (long )data;
23b4ff3a SR	281
23b4ff3a SR	282	if (!*count)
1c317143	283	return 0;
23b4ff3a SR	284
	285	if (*count != -1)
	286	(*count)--;
	287
1c317143	288	return 1;
23b4ff3a SR	289	}
	290
	291	static void
8380d248	292	ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, void **data)
23b4ff3a	293	{
1c317143	294	if (tracing_is_on())
23b4ff3a SR	295	return;
23b4ff3a SR	296
1c317143 SRRH	297	if (update_count(data))
	298	tracing_on();
	299	}
23b4ff3a	300
1c317143	301	static void
8380d248	302	ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, void **data)
1c317143 SRRH	303	{
	304	if (!tracing_is_on())
	305	return;
23b4ff3a	306
1c317143 SRRH	307	if (update_count(data))
1c317143 SRRH	308	tracing_off();
23b4ff3a SR	309	}
23b4ff3a SR	310
8380d248 SRRH	311	static void
	312	ftrace_traceon(unsigned long ip, unsigned long parent_ip, void **data)
	313	{
	314	if (tracing_is_on())
	315	return;
	316
	317	tracing_on();
	318	}
	319
	320	static void
	321	ftrace_traceoff(unsigned long ip, unsigned long parent_ip, void **data)
	322	{
	323	if (!tracing_is_on())
	324	return;
	325
	326	tracing_off();
	327	}
	328
dd42cd3e SRRH	329	/*
	330	* Skip 4:
	331	* ftrace_stacktrace()
	332	* function_trace_probe_call()
	333	* ftrace_ops_list_func()
	334	* ftrace_call()
	335	*/
	336	#define STACK_SKIP 4
	337
	338	static void
	339	ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, void **data)
	340	{
	341	trace_dump_stack(STACK_SKIP);
	342	}
	343
	344	static void
	345	ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, void **data)
	346	{
	347	if (!tracing_is_on())
	348	return;
	349
	350	if (update_count(data))
	351	trace_dump_stack(STACK_SKIP);
	352	}
	353
ad71d889 SRRH	354	static void
	355	ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, void **data)
	356	{
	357	if (update_count(data))
	358	ftrace_dump(DUMP_ALL);
	359	}
	360
90e3c03c SRRH	361	/* Only dump the current CPU buffer. */
	362	static void
	363	ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, void **data)
	364	{
	365	if (update_count(data))
	366	ftrace_dump(DUMP_ORIG);
	367	}
	368
e110e3d1	369	static int
dd42cd3e SRRH	370	ftrace_probe_print(const char name, struct seq_file m,
	371	unsigned long ip, void *data)
	372	{
	373	long count = (long)data;
	374
	375	seq_printf(m, "%ps:%s", (void *)ip, name);
	376
	377	if (count == -1)
	378	seq_printf(m, ":unlimited\n");
	379	else
	380	seq_printf(m, ":count=%ld\n", count);
	381
	382	return 0;
	383	}
	384
	385	static int
	386	ftrace_traceon_print(struct seq_file *m, unsigned long ip,
	387	struct ftrace_probe_ops ops, void data)
	388	{
	389	return ftrace_probe_print("traceon", m, ip, data);
	390	}
	391
	392	static int
	393	ftrace_traceoff_print(struct seq_file *m, unsigned long ip,
	394	struct ftrace_probe_ops ops, void data)
	395	{
	396	return ftrace_probe_print("traceoff", m, ip, data);
	397	}
	398
	399	static int
	400	ftrace_stacktrace_print(struct seq_file *m, unsigned long ip,
	401	struct ftrace_probe_ops ops, void data)
	402	{
	403	return ftrace_probe_print("stacktrace", m, ip, data);
	404	}
e110e3d1	405
ad71d889 SRRH	406	static int
	407	ftrace_dump_print(struct seq_file *m, unsigned long ip,
	408	struct ftrace_probe_ops ops, void data)
	409	{
	410	return ftrace_probe_print("dump", m, ip, data);
	411	}
	412
90e3c03c SRRH	413	static int
	414	ftrace_cpudump_print(struct seq_file *m, unsigned long ip,
	415	struct ftrace_probe_ops ops, void data)
	416	{
	417	return ftrace_probe_print("cpudump", m, ip, data);
	418	}
	419
8380d248 SRRH	420	static struct ftrace_probe_ops traceon_count_probe_ops = {
8380d248 SRRH	421	.func = ftrace_traceon_count,
dd42cd3e	422	.print = ftrace_traceon_print,
8380d248 SRRH	423	};
	424
	425	static struct ftrace_probe_ops traceoff_count_probe_ops = {
	426	.func = ftrace_traceoff_count,
dd42cd3e SRRH	427	.print = ftrace_traceoff_print,
	428	};
	429
	430	static struct ftrace_probe_ops stacktrace_count_probe_ops = {
	431	.func = ftrace_stacktrace_count,
	432	.print = ftrace_stacktrace_print,
8380d248 SRRH	433	};
8380d248 SRRH	434
ad71d889 SRRH	435	static struct ftrace_probe_ops dump_probe_ops = {
	436	.func = ftrace_dump_probe,
	437	.print = ftrace_dump_print,
	438	};
	439
90e3c03c SRRH	440	static struct ftrace_probe_ops cpudump_probe_ops = {
	441	.func = ftrace_cpudump_probe,
	442	.print = ftrace_cpudump_print,
	443	};
	444
b6887d79	445	static struct ftrace_probe_ops traceon_probe_ops = {
23b4ff3a	446	.func = ftrace_traceon,
dd42cd3e	447	.print = ftrace_traceon_print,
23b4ff3a SR	448	};
23b4ff3a SR	449
b6887d79	450	static struct ftrace_probe_ops traceoff_probe_ops = {
23b4ff3a	451	.func = ftrace_traceoff,
dd42cd3e	452	.print = ftrace_traceoff_print,
23b4ff3a SR	453	};
23b4ff3a SR	454
dd42cd3e SRRH	455	static struct ftrace_probe_ops stacktrace_probe_ops = {
	456	.func = ftrace_stacktrace,
	457	.print = ftrace_stacktrace_print,
	458	};
e110e3d1	459
23b4ff3a	460	static int
dd42cd3e SRRH	461	ftrace_trace_probe_callback(struct ftrace_probe_ops *ops,
	462	struct ftrace_hash hash, char glob,
	463	char cmd, char param, int enable)
23b4ff3a	464	{
23b4ff3a SR	465	void count = (void )-1;
	466	char *number;
	467	int ret;
	468
	469	/* hash funcs only work with set_ftrace_filter */
	470	if (!enable)
	471	return -EINVAL;
	472
8b8fa62c SRRH	473	if (glob[0] == '!') {
	474	unregister_ftrace_function_probe_func(glob+1, ops);
	475	return 0;
	476	}
	477
23b4ff3a SR	478	if (!param)
	479	goto out_reg;
	480
	481	number = strsep(&param, ":");
	482
	483	if (!strlen(number))
	484	goto out_reg;
	485
	486	/*
	487	* We use the callback data field (which is a pointer)
	488	* as our counter.
	489	*/
bcd83ea6	490	ret = kstrtoul(number, 0, (unsigned long *)&count);
23b4ff3a SR	491	if (ret)
	492	return ret;
	493
	494	out_reg:
b6887d79	495	ret = register_ftrace_function_probe(glob, ops, count);
23b4ff3a	496
04aef32d	497	return ret < 0 ? ret : 0;
23b4ff3a SR	498	}
23b4ff3a SR	499
dd42cd3e SRRH	500	static int
	501	ftrace_trace_onoff_callback(struct ftrace_hash *hash,
	502	char glob, char cmd, char *param, int enable)
	503	{
	504	struct ftrace_probe_ops *ops;
	505
	506	/* we register both traceon and traceoff to this callback */
	507	if (strcmp(cmd, "traceon") == 0)
	508	ops = param ? &traceon_count_probe_ops : &traceon_probe_ops;
	509	else
	510	ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops;
	511
	512	return ftrace_trace_probe_callback(ops, hash, glob, cmd,
	513	param, enable);
	514	}
	515
	516	static int
	517	ftrace_stacktrace_callback(struct ftrace_hash *hash,
	518	char glob, char cmd, char *param, int enable)
	519	{
	520	struct ftrace_probe_ops *ops;
	521
	522	ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops;
	523
	524	return ftrace_trace_probe_callback(ops, hash, glob, cmd,
	525	param, enable);
	526	}
	527
ad71d889 SRRH	528	static int
	529	ftrace_dump_callback(struct ftrace_hash *hash,
	530	char glob, char cmd, char *param, int enable)
	531	{
	532	struct ftrace_probe_ops *ops;
	533
	534	ops = &dump_probe_ops;
	535
	536	/* Only dump once. */
	537	return ftrace_trace_probe_callback(ops, hash, glob, cmd,
	538	"1", enable);
	539	}
	540
90e3c03c SRRH	541	static int
	542	ftrace_cpudump_callback(struct ftrace_hash *hash,
	543	char glob, char cmd, char *param, int enable)
	544	{
	545	struct ftrace_probe_ops *ops;
	546
	547	ops = &cpudump_probe_ops;
	548
	549	/* Only dump once. */
	550	return ftrace_trace_probe_callback(ops, hash, glob, cmd,
	551	"1", enable);
	552	}
	553
23b4ff3a SR	554	static struct ftrace_func_command ftrace_traceon_cmd = {
	555	.name = "traceon",
	556	.func = ftrace_trace_onoff_callback,
	557	};
	558
	559	static struct ftrace_func_command ftrace_traceoff_cmd = {
	560	.name = "traceoff",
	561	.func = ftrace_trace_onoff_callback,
	562	};
	563
dd42cd3e SRRH	564	static struct ftrace_func_command ftrace_stacktrace_cmd = {
	565	.name = "stacktrace",
	566	.func = ftrace_stacktrace_callback,
	567	};
	568
ad71d889 SRRH	569	static struct ftrace_func_command ftrace_dump_cmd = {
	570	.name = "dump",
	571	.func = ftrace_dump_callback,
	572	};
	573
90e3c03c SRRH	574	static struct ftrace_func_command ftrace_cpudump_cmd = {
	575	.name = "cpudump",
	576	.func = ftrace_cpudump_callback,
	577	};
	578
23b4ff3a SR	579	static int __init init_func_cmd_traceon(void)
	580	{
	581	int ret;
	582
	583	ret = register_ftrace_command(&ftrace_traceoff_cmd);
	584	if (ret)
	585	return ret;
	586
	587	ret = register_ftrace_command(&ftrace_traceon_cmd);
	588	if (ret)
ad71d889	589	goto out_free_traceoff;
dd42cd3e SRRH	590
dd42cd3e SRRH	591	ret = register_ftrace_command(&ftrace_stacktrace_cmd);
ad71d889 SRRH	592	if (ret)
	593	goto out_free_traceon;
	594
	595	ret = register_ftrace_command(&ftrace_dump_cmd);
	596	if (ret)
	597	goto out_free_stacktrace;
	598
90e3c03c SRRH	599	ret = register_ftrace_command(&ftrace_cpudump_cmd);
	600	if (ret)
	601	goto out_free_dump;
	602
ad71d889 SRRH	603	return 0;
ad71d889 SRRH	604
90e3c03c SRRH	605	out_free_dump:
90e3c03c SRRH	606	unregister_ftrace_command(&ftrace_dump_cmd);
ad71d889 SRRH	607	out_free_stacktrace:
	608	unregister_ftrace_command(&ftrace_stacktrace_cmd);
	609	out_free_traceon:
	610	unregister_ftrace_command(&ftrace_traceon_cmd);
	611	out_free_traceoff:
	612	unregister_ftrace_command(&ftrace_traceoff_cmd);
	613
23b4ff3a SR	614	return ret;
	615	}
	616	#else
	617	static inline int init_func_cmd_traceon(void)
	618	{
	619	return 0;
	620	}
	621	#endif /* CONFIG_DYNAMIC_FTRACE */
	622
1b29b018 SR	623	static __init int init_function_trace(void)
1b29b018 SR	624	{
23b4ff3a	625	init_func_cmd_traceon();
1b29b018 SR	626	return register_tracer(&function_trace);
1b29b018 SR	627	}
6f415672	628	core_initcall(init_function_trace);