2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator *tracepoint_print_iter;
70 int tracepoint_printk;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt[] = {
78 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
84 * To prevent the comm cache from being overwritten when no
85 * tracing is active, only save the comm when a trace event
88 static DEFINE_PER_CPU(bool, trace_cmdline_save);
91 * Kill all tracing for good (never come back).
92 * It is initialized to 1 but will turn to zero if the initialization
93 * of the tracer is successful. But that is the only place that sets
96 static int tracing_disabled = 1;
98 cpumask_var_t __read_mostly tracing_buffer_mask;
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
116 enum ftrace_dump_mode ftrace_dump_on_oops;
118 /* When set, tracing will stop when a WARN*() is hit */
119 int __disable_trace_on_warning;
121 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
122 /* Map of enums to their values, for "enum_map" file */
123 struct trace_enum_map_head {
125 unsigned long length;
128 union trace_enum_map_item;
130 struct trace_enum_map_tail {
132 * "end" is first and points to NULL as it must be different
133 * than "mod" or "enum_string"
135 union trace_enum_map_item *next;
136 const char *end; /* points to NULL */
139 static DEFINE_MUTEX(trace_enum_mutex);
142 * The trace_enum_maps are saved in an array with two extra elements,
143 * one at the beginning, and one at the end. The beginning item contains
144 * the count of the saved maps (head.length), and the module they
145 * belong to if not built in (head.mod). The ending item contains a
146 * pointer to the next array of saved enum_map items.
148 union trace_enum_map_item {
149 struct trace_enum_map map;
150 struct trace_enum_map_head head;
151 struct trace_enum_map_tail tail;
154 static union trace_enum_map_item *trace_enum_maps;
155 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
157 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
159 #define MAX_TRACER_SIZE 100
160 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
161 static char *default_bootup_tracer;
163 static bool allocate_snapshot;
165 static int __init set_cmdline_ftrace(char *str)
167 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
168 default_bootup_tracer = bootup_tracer_buf;
169 /* We are using ftrace early, expand it */
170 ring_buffer_expanded = true;
173 __setup("ftrace=", set_cmdline_ftrace);
175 static int __init set_ftrace_dump_on_oops(char *str)
177 if (*str++ != '=' || !*str) {
178 ftrace_dump_on_oops = DUMP_ALL;
182 if (!strcmp("orig_cpu", str)) {
183 ftrace_dump_on_oops = DUMP_ORIG;
189 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
191 static int __init stop_trace_on_warning(char *str)
193 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
194 __disable_trace_on_warning = 1;
197 __setup("traceoff_on_warning", stop_trace_on_warning);
199 static int __init boot_alloc_snapshot(char *str)
201 allocate_snapshot = true;
202 /* We also need the main ring buffer expanded */
203 ring_buffer_expanded = true;
206 __setup("alloc_snapshot", boot_alloc_snapshot);
209 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
211 static int __init set_trace_boot_options(char *str)
213 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
216 __setup("trace_options=", set_trace_boot_options);
218 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
219 static char *trace_boot_clock __initdata;
221 static int __init set_trace_boot_clock(char *str)
223 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
224 trace_boot_clock = trace_boot_clock_buf;
227 __setup("trace_clock=", set_trace_boot_clock);
229 static int __init set_tracepoint_printk(char *str)
231 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
232 tracepoint_printk = 1;
235 __setup("tp_printk", set_tracepoint_printk);
237 unsigned long long ns2usecs(cycle_t nsec)
244 /* trace_flags holds trace_options default values */
245 #define TRACE_DEFAULT_FLAGS \
246 (FUNCTION_DEFAULT_FLAGS | \
247 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
248 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
249 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
250 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
252 /* trace_options that are only supported by global_trace */
253 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
254 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
256 /* trace_flags that are default zero for instances */
257 #define ZEROED_TRACE_FLAGS \
258 TRACE_ITER_EVENT_FORK
261 * The global_trace is the descriptor that holds the tracing
262 * buffers for the live tracing. For each CPU, it contains
263 * a link list of pages that will store trace entries. The
264 * page descriptor of the pages in the memory is used to hold
265 * the link list by linking the lru item in the page descriptor
266 * to each of the pages in the buffer per CPU.
268 * For each active CPU there is a data field that holds the
269 * pages for the buffer for that CPU. Each CPU has the same number
270 * of pages allocated for its buffer.
272 static struct trace_array global_trace = {
273 .trace_flags = TRACE_DEFAULT_FLAGS,
276 LIST_HEAD(ftrace_trace_arrays);
278 int trace_array_get(struct trace_array *this_tr)
280 struct trace_array *tr;
283 mutex_lock(&trace_types_lock);
284 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
291 mutex_unlock(&trace_types_lock);
296 static void __trace_array_put(struct trace_array *this_tr)
298 WARN_ON(!this_tr->ref);
302 void trace_array_put(struct trace_array *this_tr)
304 mutex_lock(&trace_types_lock);
305 __trace_array_put(this_tr);
306 mutex_unlock(&trace_types_lock);
309 int call_filter_check_discard(struct trace_event_call *call, void *rec,
310 struct ring_buffer *buffer,
311 struct ring_buffer_event *event)
313 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
314 !filter_match_preds(call->filter, rec)) {
315 __trace_event_discard_commit(buffer, event);
322 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
326 /* Early boot up does not have a buffer yet */
328 return trace_clock_local();
330 ts = ring_buffer_time_stamp(buf->buffer, cpu);
331 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
336 cycle_t ftrace_now(int cpu)
338 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
342 * tracing_is_enabled - Show if global_trace has been disabled
344 * Shows if the global trace has been enabled or not. It uses the
345 * mirror flag "buffer_disabled" to be used in fast paths such as for
346 * the irqsoff tracer. But it may be inaccurate due to races. If you
347 * need to know the accurate state, use tracing_is_on() which is a little
348 * slower, but accurate.
350 int tracing_is_enabled(void)
353 * For quick access (irqsoff uses this in fast path), just
354 * return the mirror variable of the state of the ring buffer.
355 * It's a little racy, but we don't really care.
358 return !global_trace.buffer_disabled;
362 * trace_buf_size is the size in bytes that is allocated
363 * for a buffer. Note, the number of bytes is always rounded
366 * This number is purposely set to a low number of 16384.
367 * If the dump on oops happens, it will be much appreciated
368 * to not have to wait for all that output. Anyway this can be
369 * boot time and run time configurable.
371 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
373 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
375 /* trace_types holds a link list of available tracers. */
376 static struct tracer *trace_types __read_mostly;
379 * trace_types_lock is used to protect the trace_types list.
381 DEFINE_MUTEX(trace_types_lock);
384 * serialize the access of the ring buffer
386 * ring buffer serializes readers, but it is low level protection.
387 * The validity of the events (which returns by ring_buffer_peek() ..etc)
388 * are not protected by ring buffer.
390 * The content of events may become garbage if we allow other process consumes
391 * these events concurrently:
392 * A) the page of the consumed events may become a normal page
393 * (not reader page) in ring buffer, and this page will be rewrited
394 * by events producer.
395 * B) The page of the consumed events may become a page for splice_read,
396 * and this page will be returned to system.
398 * These primitives allow multi process access to different cpu ring buffer
401 * These primitives don't distinguish read-only and read-consume access.
402 * Multi read-only access are also serialized.
406 static DECLARE_RWSEM(all_cpu_access_lock);
407 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
409 static inline void trace_access_lock(int cpu)
411 if (cpu == RING_BUFFER_ALL_CPUS) {
412 /* gain it for accessing the whole ring buffer. */
413 down_write(&all_cpu_access_lock);
415 /* gain it for accessing a cpu ring buffer. */
417 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
418 down_read(&all_cpu_access_lock);
420 /* Secondly block other access to this @cpu ring buffer. */
421 mutex_lock(&per_cpu(cpu_access_lock, cpu));
425 static inline void trace_access_unlock(int cpu)
427 if (cpu == RING_BUFFER_ALL_CPUS) {
428 up_write(&all_cpu_access_lock);
430 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
431 up_read(&all_cpu_access_lock);
435 static inline void trace_access_lock_init(void)
439 for_each_possible_cpu(cpu)
440 mutex_init(&per_cpu(cpu_access_lock, cpu));
445 static DEFINE_MUTEX(access_lock);
447 static inline void trace_access_lock(int cpu)
450 mutex_lock(&access_lock);
453 static inline void trace_access_unlock(int cpu)
456 mutex_unlock(&access_lock);
459 static inline void trace_access_lock_init(void)
465 #ifdef CONFIG_STACKTRACE
466 static void __ftrace_trace_stack(struct ring_buffer *buffer,
468 int skip, int pc, struct pt_regs *regs);
469 static inline void ftrace_trace_stack(struct trace_array *tr,
470 struct ring_buffer *buffer,
472 int skip, int pc, struct pt_regs *regs);
475 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
477 int skip, int pc, struct pt_regs *regs)
480 static inline void ftrace_trace_stack(struct trace_array *tr,
481 struct ring_buffer *buffer,
483 int skip, int pc, struct pt_regs *regs)
489 static void tracer_tracing_on(struct trace_array *tr)
491 if (tr->trace_buffer.buffer)
492 ring_buffer_record_on(tr->trace_buffer.buffer);
494 * This flag is looked at when buffers haven't been allocated
495 * yet, or by some tracers (like irqsoff), that just want to
496 * know if the ring buffer has been disabled, but it can handle
497 * races of where it gets disabled but we still do a record.
498 * As the check is in the fast path of the tracers, it is more
499 * important to be fast than accurate.
501 tr->buffer_disabled = 0;
502 /* Make the flag seen by readers */
507 * tracing_on - enable tracing buffers
509 * This function enables tracing buffers that may have been
510 * disabled with tracing_off.
512 void tracing_on(void)
514 tracer_tracing_on(&global_trace);
516 EXPORT_SYMBOL_GPL(tracing_on);
519 * __trace_puts - write a constant string into the trace buffer.
520 * @ip: The address of the caller
521 * @str: The constant string to write
522 * @size: The size of the string.
524 int __trace_puts(unsigned long ip, const char *str, int size)
526 struct ring_buffer_event *event;
527 struct ring_buffer *buffer;
528 struct print_entry *entry;
529 unsigned long irq_flags;
533 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
536 pc = preempt_count();
538 if (unlikely(tracing_selftest_running || tracing_disabled))
541 alloc = sizeof(*entry) + size + 2; /* possible \n added */
543 local_save_flags(irq_flags);
544 buffer = global_trace.trace_buffer.buffer;
545 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
550 entry = ring_buffer_event_data(event);
553 memcpy(&entry->buf, str, size);
555 /* Add a newline if necessary */
556 if (entry->buf[size - 1] != '\n') {
557 entry->buf[size] = '\n';
558 entry->buf[size + 1] = '\0';
560 entry->buf[size] = '\0';
562 __buffer_unlock_commit(buffer, event);
563 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
567 EXPORT_SYMBOL_GPL(__trace_puts);
570 * __trace_bputs - write the pointer to a constant string into trace buffer
571 * @ip: The address of the caller
572 * @str: The constant string to write to the buffer to
574 int __trace_bputs(unsigned long ip, const char *str)
576 struct ring_buffer_event *event;
577 struct ring_buffer *buffer;
578 struct bputs_entry *entry;
579 unsigned long irq_flags;
580 int size = sizeof(struct bputs_entry);
583 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
586 pc = preempt_count();
588 if (unlikely(tracing_selftest_running || tracing_disabled))
591 local_save_flags(irq_flags);
592 buffer = global_trace.trace_buffer.buffer;
593 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
598 entry = ring_buffer_event_data(event);
602 __buffer_unlock_commit(buffer, event);
603 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
607 EXPORT_SYMBOL_GPL(__trace_bputs);
609 #ifdef CONFIG_TRACER_SNAPSHOT
611 * trace_snapshot - take a snapshot of the current buffer.
613 * This causes a swap between the snapshot buffer and the current live
614 * tracing buffer. You can use this to take snapshots of the live
615 * trace when some condition is triggered, but continue to trace.
617 * Note, make sure to allocate the snapshot with either
618 * a tracing_snapshot_alloc(), or by doing it manually
619 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
621 * If the snapshot buffer is not allocated, it will stop tracing.
622 * Basically making a permanent snapshot.
624 void tracing_snapshot(void)
626 struct trace_array *tr = &global_trace;
627 struct tracer *tracer = tr->current_trace;
631 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
632 internal_trace_puts("*** snapshot is being ignored ***\n");
636 if (!tr->allocated_snapshot) {
637 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
638 internal_trace_puts("*** stopping trace here! ***\n");
643 /* Note, snapshot can not be used when the tracer uses it */
644 if (tracer->use_max_tr) {
645 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
646 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
650 local_irq_save(flags);
651 update_max_tr(tr, current, smp_processor_id());
652 local_irq_restore(flags);
654 EXPORT_SYMBOL_GPL(tracing_snapshot);
656 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
657 struct trace_buffer *size_buf, int cpu_id);
658 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
660 static int alloc_snapshot(struct trace_array *tr)
664 if (!tr->allocated_snapshot) {
666 /* allocate spare buffer */
667 ret = resize_buffer_duplicate_size(&tr->max_buffer,
668 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
672 tr->allocated_snapshot = true;
678 static void free_snapshot(struct trace_array *tr)
681 * We don't free the ring buffer. instead, resize it because
682 * The max_tr ring buffer has some state (e.g. ring->clock) and
683 * we want preserve it.
685 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
686 set_buffer_entries(&tr->max_buffer, 1);
687 tracing_reset_online_cpus(&tr->max_buffer);
688 tr->allocated_snapshot = false;
692 * tracing_alloc_snapshot - allocate snapshot buffer.
694 * This only allocates the snapshot buffer if it isn't already
695 * allocated - it doesn't also take a snapshot.
697 * This is meant to be used in cases where the snapshot buffer needs
698 * to be set up for events that can't sleep but need to be able to
699 * trigger a snapshot.
701 int tracing_alloc_snapshot(void)
703 struct trace_array *tr = &global_trace;
706 ret = alloc_snapshot(tr);
711 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
714 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
716 * This is similar to trace_snapshot(), but it will allocate the
717 * snapshot buffer if it isn't already allocated. Use this only
718 * where it is safe to sleep, as the allocation may sleep.
720 * This causes a swap between the snapshot buffer and the current live
721 * tracing buffer. You can use this to take snapshots of the live
722 * trace when some condition is triggered, but continue to trace.
724 void tracing_snapshot_alloc(void)
728 ret = tracing_alloc_snapshot();
734 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
736 void tracing_snapshot(void)
738 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
740 EXPORT_SYMBOL_GPL(tracing_snapshot);
741 int tracing_alloc_snapshot(void)
743 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
746 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
747 void tracing_snapshot_alloc(void)
752 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
753 #endif /* CONFIG_TRACER_SNAPSHOT */
755 static void tracer_tracing_off(struct trace_array *tr)
757 if (tr->trace_buffer.buffer)
758 ring_buffer_record_off(tr->trace_buffer.buffer);
760 * This flag is looked at when buffers haven't been allocated
761 * yet, or by some tracers (like irqsoff), that just want to
762 * know if the ring buffer has been disabled, but it can handle
763 * races of where it gets disabled but we still do a record.
764 * As the check is in the fast path of the tracers, it is more
765 * important to be fast than accurate.
767 tr->buffer_disabled = 1;
768 /* Make the flag seen by readers */
773 * tracing_off - turn off tracing buffers
775 * This function stops the tracing buffers from recording data.
776 * It does not disable any overhead the tracers themselves may
777 * be causing. This function simply causes all recording to
778 * the ring buffers to fail.
780 void tracing_off(void)
782 tracer_tracing_off(&global_trace);
784 EXPORT_SYMBOL_GPL(tracing_off);
786 void disable_trace_on_warning(void)
788 if (__disable_trace_on_warning)
793 * tracer_tracing_is_on - show real state of ring buffer enabled
794 * @tr : the trace array to know if ring buffer is enabled
796 * Shows real state of the ring buffer if it is enabled or not.
798 static int tracer_tracing_is_on(struct trace_array *tr)
800 if (tr->trace_buffer.buffer)
801 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
802 return !tr->buffer_disabled;
806 * tracing_is_on - show state of ring buffers enabled
808 int tracing_is_on(void)
810 return tracer_tracing_is_on(&global_trace);
812 EXPORT_SYMBOL_GPL(tracing_is_on);
814 static int __init set_buf_size(char *str)
816 unsigned long buf_size;
820 buf_size = memparse(str, &str);
821 /* nr_entries can not be zero */
824 trace_buf_size = buf_size;
827 __setup("trace_buf_size=", set_buf_size);
829 static int __init set_tracing_thresh(char *str)
831 unsigned long threshold;
836 ret = kstrtoul(str, 0, &threshold);
839 tracing_thresh = threshold * 1000;
842 __setup("tracing_thresh=", set_tracing_thresh);
844 unsigned long nsecs_to_usecs(unsigned long nsecs)
850 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
851 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
852 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
853 * of strings in the order that the enums were defined.
858 /* These must match the bit postions in trace_iterator_flags */
859 static const char *trace_options[] = {
867 int in_ns; /* is this clock in nanoseconds? */
869 { trace_clock_local, "local", 1 },
870 { trace_clock_global, "global", 1 },
871 { trace_clock_counter, "counter", 0 },
872 { trace_clock_jiffies, "uptime", 0 },
873 { trace_clock, "perf", 1 },
874 { ktime_get_mono_fast_ns, "mono", 1 },
875 { ktime_get_raw_fast_ns, "mono_raw", 1 },
880 * trace_parser_get_init - gets the buffer for trace parser
882 int trace_parser_get_init(struct trace_parser *parser, int size)
884 memset(parser, 0, sizeof(*parser));
886 parser->buffer = kmalloc(size, GFP_KERNEL);
895 * trace_parser_put - frees the buffer for trace parser
897 void trace_parser_put(struct trace_parser *parser)
899 kfree(parser->buffer);
903 * trace_get_user - reads the user input string separated by space
904 * (matched by isspace(ch))
906 * For each string found the 'struct trace_parser' is updated,
907 * and the function returns.
909 * Returns number of bytes read.
911 * See kernel/trace/trace.h for 'struct trace_parser' details.
913 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
914 size_t cnt, loff_t *ppos)
921 trace_parser_clear(parser);
923 ret = get_user(ch, ubuf++);
931 * The parser is not finished with the last write,
932 * continue reading the user input without skipping spaces.
935 /* skip white space */
936 while (cnt && isspace(ch)) {
937 ret = get_user(ch, ubuf++);
944 /* only spaces were written */
954 /* read the non-space input */
955 while (cnt && !isspace(ch)) {
956 if (parser->idx < parser->size - 1)
957 parser->buffer[parser->idx++] = ch;
962 ret = get_user(ch, ubuf++);
969 /* We either got finished input or we have to wait for another call. */
971 parser->buffer[parser->idx] = 0;
972 parser->cont = false;
973 } else if (parser->idx < parser->size - 1) {
975 parser->buffer[parser->idx++] = ch;
988 /* TODO add a seq_buf_to_buffer() */
989 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
993 if (trace_seq_used(s) <= s->seq.readpos)
996 len = trace_seq_used(s) - s->seq.readpos;
999 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1001 s->seq.readpos += cnt;
1005 unsigned long __read_mostly tracing_thresh;
1007 #ifdef CONFIG_TRACER_MAX_TRACE
1009 * Copy the new maximum trace into the separate maximum-trace
1010 * structure. (this way the maximum trace is permanently saved,
1011 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1014 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1016 struct trace_buffer *trace_buf = &tr->trace_buffer;
1017 struct trace_buffer *max_buf = &tr->max_buffer;
1018 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1019 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1022 max_buf->time_start = data->preempt_timestamp;
1024 max_data->saved_latency = tr->max_latency;
1025 max_data->critical_start = data->critical_start;
1026 max_data->critical_end = data->critical_end;
1028 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1029 max_data->pid = tsk->pid;
1031 * If tsk == current, then use current_uid(), as that does not use
1032 * RCU. The irq tracer can be called out of RCU scope.
1035 max_data->uid = current_uid();
1037 max_data->uid = task_uid(tsk);
1039 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1040 max_data->policy = tsk->policy;
1041 max_data->rt_priority = tsk->rt_priority;
1043 /* record this tasks comm */
1044 tracing_record_cmdline(tsk);
1048 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1050 * @tsk: the task with the latency
1051 * @cpu: The cpu that initiated the trace.
1053 * Flip the buffers between the @tr and the max_tr and record information
1054 * about which task was the cause of this latency.
1057 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1059 struct ring_buffer *buf;
1064 WARN_ON_ONCE(!irqs_disabled());
1066 if (!tr->allocated_snapshot) {
1067 /* Only the nop tracer should hit this when disabling */
1068 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1072 arch_spin_lock(&tr->max_lock);
1074 buf = tr->trace_buffer.buffer;
1075 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1076 tr->max_buffer.buffer = buf;
1078 __update_max_tr(tr, tsk, cpu);
1079 arch_spin_unlock(&tr->max_lock);
1083 * update_max_tr_single - only copy one trace over, and reset the rest
1085 * @tsk - task with the latency
1086 * @cpu - the cpu of the buffer to copy.
1088 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1091 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1098 WARN_ON_ONCE(!irqs_disabled());
1099 if (!tr->allocated_snapshot) {
1100 /* Only the nop tracer should hit this when disabling */
1101 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1105 arch_spin_lock(&tr->max_lock);
1107 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1109 if (ret == -EBUSY) {
1111 * We failed to swap the buffer due to a commit taking
1112 * place on this CPU. We fail to record, but we reset
1113 * the max trace buffer (no one writes directly to it)
1114 * and flag that it failed.
1116 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1117 "Failed to swap buffers due to commit in progress\n");
1120 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1122 __update_max_tr(tr, tsk, cpu);
1123 arch_spin_unlock(&tr->max_lock);
1125 #endif /* CONFIG_TRACER_MAX_TRACE */
1127 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1129 /* Iterators are static, they should be filled or empty */
1130 if (trace_buffer_iter(iter, iter->cpu_file))
1133 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1137 #ifdef CONFIG_FTRACE_STARTUP_TEST
1138 static int run_tracer_selftest(struct tracer *type)
1140 struct trace_array *tr = &global_trace;
1141 struct tracer *saved_tracer = tr->current_trace;
1144 if (!type->selftest || tracing_selftest_disabled)
1148 * Run a selftest on this tracer.
1149 * Here we reset the trace buffer, and set the current
1150 * tracer to be this tracer. The tracer can then run some
1151 * internal tracing to verify that everything is in order.
1152 * If we fail, we do not register this tracer.
1154 tracing_reset_online_cpus(&tr->trace_buffer);
1156 tr->current_trace = type;
1158 #ifdef CONFIG_TRACER_MAX_TRACE
1159 if (type->use_max_tr) {
1160 /* If we expanded the buffers, make sure the max is expanded too */
1161 if (ring_buffer_expanded)
1162 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1163 RING_BUFFER_ALL_CPUS);
1164 tr->allocated_snapshot = true;
1168 /* the test is responsible for initializing and enabling */
1169 pr_info("Testing tracer %s: ", type->name);
1170 ret = type->selftest(type, tr);
1171 /* the test is responsible for resetting too */
1172 tr->current_trace = saved_tracer;
1174 printk(KERN_CONT "FAILED!\n");
1175 /* Add the warning after printing 'FAILED' */
1179 /* Only reset on passing, to avoid touching corrupted buffers */
1180 tracing_reset_online_cpus(&tr->trace_buffer);
1182 #ifdef CONFIG_TRACER_MAX_TRACE
1183 if (type->use_max_tr) {
1184 tr->allocated_snapshot = false;
1186 /* Shrink the max buffer again */
1187 if (ring_buffer_expanded)
1188 ring_buffer_resize(tr->max_buffer.buffer, 1,
1189 RING_BUFFER_ALL_CPUS);
1193 printk(KERN_CONT "PASSED\n");
1197 static inline int run_tracer_selftest(struct tracer *type)
1201 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1203 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1205 static void __init apply_trace_boot_options(void);
1208 * register_tracer - register a tracer with the ftrace system.
1209 * @type - the plugin for the tracer
1211 * Register a new plugin tracer.
1213 int __init register_tracer(struct tracer *type)
1219 pr_info("Tracer must have a name\n");
1223 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1224 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1228 mutex_lock(&trace_types_lock);
1230 tracing_selftest_running = true;
1232 for (t = trace_types; t; t = t->next) {
1233 if (strcmp(type->name, t->name) == 0) {
1235 pr_info("Tracer %s already registered\n",
1242 if (!type->set_flag)
1243 type->set_flag = &dummy_set_flag;
1245 /*allocate a dummy tracer_flags*/
1246 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1251 type->flags->val = 0;
1252 type->flags->opts = dummy_tracer_opt;
1254 if (!type->flags->opts)
1255 type->flags->opts = dummy_tracer_opt;
1257 /* store the tracer for __set_tracer_option */
1258 type->flags->trace = type;
1260 ret = run_tracer_selftest(type);
1264 type->next = trace_types;
1266 add_tracer_options(&global_trace, type);
1269 tracing_selftest_running = false;
1270 mutex_unlock(&trace_types_lock);
1272 if (ret || !default_bootup_tracer)
1275 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1278 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1279 /* Do we want this tracer to start on bootup? */
1280 tracing_set_tracer(&global_trace, type->name);
1281 default_bootup_tracer = NULL;
1283 apply_trace_boot_options();
1285 /* disable other selftests, since this will break it. */
1286 tracing_selftest_disabled = true;
1287 #ifdef CONFIG_FTRACE_STARTUP_TEST
1288 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1296 void tracing_reset(struct trace_buffer *buf, int cpu)
1298 struct ring_buffer *buffer = buf->buffer;
1303 ring_buffer_record_disable(buffer);
1305 /* Make sure all commits have finished */
1306 synchronize_sched();
1307 ring_buffer_reset_cpu(buffer, cpu);
1309 ring_buffer_record_enable(buffer);
1312 void tracing_reset_online_cpus(struct trace_buffer *buf)
1314 struct ring_buffer *buffer = buf->buffer;
1320 ring_buffer_record_disable(buffer);
1322 /* Make sure all commits have finished */
1323 synchronize_sched();
1325 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1327 for_each_online_cpu(cpu)
1328 ring_buffer_reset_cpu(buffer, cpu);
1330 ring_buffer_record_enable(buffer);
1333 /* Must have trace_types_lock held */
1334 void tracing_reset_all_online_cpus(void)
1336 struct trace_array *tr;
1338 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1339 tracing_reset_online_cpus(&tr->trace_buffer);
1340 #ifdef CONFIG_TRACER_MAX_TRACE
1341 tracing_reset_online_cpus(&tr->max_buffer);
1346 #define SAVED_CMDLINES_DEFAULT 128
1347 #define NO_CMDLINE_MAP UINT_MAX
1348 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1349 struct saved_cmdlines_buffer {
1350 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1351 unsigned *map_cmdline_to_pid;
1352 unsigned cmdline_num;
1354 char *saved_cmdlines;
1356 static struct saved_cmdlines_buffer *savedcmd;
1358 /* temporary disable recording */
1359 static atomic_t trace_record_cmdline_disabled __read_mostly;
1361 static inline char *get_saved_cmdlines(int idx)
1363 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1366 static inline void set_cmdline(int idx, const char *cmdline)
1368 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1371 static int allocate_cmdlines_buffer(unsigned int val,
1372 struct saved_cmdlines_buffer *s)
1374 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1376 if (!s->map_cmdline_to_pid)
1379 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1380 if (!s->saved_cmdlines) {
1381 kfree(s->map_cmdline_to_pid);
1386 s->cmdline_num = val;
1387 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1388 sizeof(s->map_pid_to_cmdline));
1389 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1390 val * sizeof(*s->map_cmdline_to_pid));
1395 static int trace_create_savedcmd(void)
1399 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1403 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1413 int is_tracing_stopped(void)
1415 return global_trace.stop_count;
1419 * tracing_start - quick start of the tracer
1421 * If tracing is enabled but was stopped by tracing_stop,
1422 * this will start the tracer back up.
1424 void tracing_start(void)
1426 struct ring_buffer *buffer;
1427 unsigned long flags;
1429 if (tracing_disabled)
1432 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1433 if (--global_trace.stop_count) {
1434 if (global_trace.stop_count < 0) {
1435 /* Someone screwed up their debugging */
1437 global_trace.stop_count = 0;
1442 /* Prevent the buffers from switching */
1443 arch_spin_lock(&global_trace.max_lock);
1445 buffer = global_trace.trace_buffer.buffer;
1447 ring_buffer_record_enable(buffer);
1449 #ifdef CONFIG_TRACER_MAX_TRACE
1450 buffer = global_trace.max_buffer.buffer;
1452 ring_buffer_record_enable(buffer);
1455 arch_spin_unlock(&global_trace.max_lock);
1458 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1461 static void tracing_start_tr(struct trace_array *tr)
1463 struct ring_buffer *buffer;
1464 unsigned long flags;
1466 if (tracing_disabled)
1469 /* If global, we need to also start the max tracer */
1470 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1471 return tracing_start();
1473 raw_spin_lock_irqsave(&tr->start_lock, flags);
1475 if (--tr->stop_count) {
1476 if (tr->stop_count < 0) {
1477 /* Someone screwed up their debugging */
1484 buffer = tr->trace_buffer.buffer;
1486 ring_buffer_record_enable(buffer);
1489 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1493 * tracing_stop - quick stop of the tracer
1495 * Light weight way to stop tracing. Use in conjunction with
1498 void tracing_stop(void)
1500 struct ring_buffer *buffer;
1501 unsigned long flags;
1503 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1504 if (global_trace.stop_count++)
1507 /* Prevent the buffers from switching */
1508 arch_spin_lock(&global_trace.max_lock);
1510 buffer = global_trace.trace_buffer.buffer;
1512 ring_buffer_record_disable(buffer);
1514 #ifdef CONFIG_TRACER_MAX_TRACE
1515 buffer = global_trace.max_buffer.buffer;
1517 ring_buffer_record_disable(buffer);
1520 arch_spin_unlock(&global_trace.max_lock);
1523 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1526 static void tracing_stop_tr(struct trace_array *tr)
1528 struct ring_buffer *buffer;
1529 unsigned long flags;
1531 /* If global, we need to also stop the max tracer */
1532 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1533 return tracing_stop();
1535 raw_spin_lock_irqsave(&tr->start_lock, flags);
1536 if (tr->stop_count++)
1539 buffer = tr->trace_buffer.buffer;
1541 ring_buffer_record_disable(buffer);
1544 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1547 void trace_stop_cmdline_recording(void);
1549 static int trace_save_cmdline(struct task_struct *tsk)
1553 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1557 * It's not the end of the world if we don't get
1558 * the lock, but we also don't want to spin
1559 * nor do we want to disable interrupts,
1560 * so if we miss here, then better luck next time.
1562 if (!arch_spin_trylock(&trace_cmdline_lock))
1565 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1566 if (idx == NO_CMDLINE_MAP) {
1567 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1570 * Check whether the cmdline buffer at idx has a pid
1571 * mapped. We are going to overwrite that entry so we
1572 * need to clear the map_pid_to_cmdline. Otherwise we
1573 * would read the new comm for the old pid.
1575 pid = savedcmd->map_cmdline_to_pid[idx];
1576 if (pid != NO_CMDLINE_MAP)
1577 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1579 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1580 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1582 savedcmd->cmdline_idx = idx;
1585 set_cmdline(idx, tsk->comm);
1587 arch_spin_unlock(&trace_cmdline_lock);
1592 static void __trace_find_cmdline(int pid, char comm[])
1597 strcpy(comm, "<idle>");
1601 if (WARN_ON_ONCE(pid < 0)) {
1602 strcpy(comm, "<XXX>");
1606 if (pid > PID_MAX_DEFAULT) {
1607 strcpy(comm, "<...>");
1611 map = savedcmd->map_pid_to_cmdline[pid];
1612 if (map != NO_CMDLINE_MAP)
1613 strcpy(comm, get_saved_cmdlines(map));
1615 strcpy(comm, "<...>");
1618 void trace_find_cmdline(int pid, char comm[])
1621 arch_spin_lock(&trace_cmdline_lock);
1623 __trace_find_cmdline(pid, comm);
1625 arch_spin_unlock(&trace_cmdline_lock);
1629 void tracing_record_cmdline(struct task_struct *tsk)
1631 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1634 if (!__this_cpu_read(trace_cmdline_save))
1637 if (trace_save_cmdline(tsk))
1638 __this_cpu_write(trace_cmdline_save, false);
1642 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1645 struct task_struct *tsk = current;
1647 entry->preempt_count = pc & 0xff;
1648 entry->pid = (tsk) ? tsk->pid : 0;
1650 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1651 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1653 TRACE_FLAG_IRQS_NOSUPPORT |
1655 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
1656 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1657 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1658 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1659 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1661 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1663 static __always_inline void
1664 trace_event_setup(struct ring_buffer_event *event,
1665 int type, unsigned long flags, int pc)
1667 struct trace_entry *ent = ring_buffer_event_data(event);
1669 tracing_generic_entry_update(ent, flags, pc);
1673 struct ring_buffer_event *
1674 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1677 unsigned long flags, int pc)
1679 struct ring_buffer_event *event;
1681 event = ring_buffer_lock_reserve(buffer, len);
1683 trace_event_setup(event, type, flags, pc);
1688 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1689 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
1690 static int trace_buffered_event_ref;
1693 * trace_buffered_event_enable - enable buffering events
1695 * When events are being filtered, it is quicker to use a temporary
1696 * buffer to write the event data into if there's a likely chance
1697 * that it will not be committed. The discard of the ring buffer
1698 * is not as fast as committing, and is much slower than copying
1701 * When an event is to be filtered, allocate per cpu buffers to
1702 * write the event data into, and if the event is filtered and discarded
1703 * it is simply dropped, otherwise, the entire data is to be committed
1706 void trace_buffered_event_enable(void)
1708 struct ring_buffer_event *event;
1712 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1714 if (trace_buffered_event_ref++)
1717 for_each_tracing_cpu(cpu) {
1718 page = alloc_pages_node(cpu_to_node(cpu),
1719 GFP_KERNEL | __GFP_NORETRY, 0);
1723 event = page_address(page);
1724 memset(event, 0, sizeof(*event));
1726 per_cpu(trace_buffered_event, cpu) = event;
1729 if (cpu == smp_processor_id() &&
1730 this_cpu_read(trace_buffered_event) !=
1731 per_cpu(trace_buffered_event, cpu))
1738 trace_buffered_event_disable();
1741 static void enable_trace_buffered_event(void *data)
1743 /* Probably not needed, but do it anyway */
1745 this_cpu_dec(trace_buffered_event_cnt);
1748 static void disable_trace_buffered_event(void *data)
1750 this_cpu_inc(trace_buffered_event_cnt);
1754 * trace_buffered_event_disable - disable buffering events
1756 * When a filter is removed, it is faster to not use the buffered
1757 * events, and to commit directly into the ring buffer. Free up
1758 * the temp buffers when there are no more users. This requires
1759 * special synchronization with current events.
1761 void trace_buffered_event_disable(void)
1765 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1767 if (WARN_ON_ONCE(!trace_buffered_event_ref))
1770 if (--trace_buffered_event_ref)
1774 /* For each CPU, set the buffer as used. */
1775 smp_call_function_many(tracing_buffer_mask,
1776 disable_trace_buffered_event, NULL, 1);
1779 /* Wait for all current users to finish */
1780 synchronize_sched();
1782 for_each_tracing_cpu(cpu) {
1783 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
1784 per_cpu(trace_buffered_event, cpu) = NULL;
1787 * Make sure trace_buffered_event is NULL before clearing
1788 * trace_buffered_event_cnt.
1793 /* Do the work on each cpu */
1794 smp_call_function_many(tracing_buffer_mask,
1795 enable_trace_buffered_event, NULL, 1);
1800 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1802 __this_cpu_write(trace_cmdline_save, true);
1804 /* If this is the temp buffer, we need to commit fully */
1805 if (this_cpu_read(trace_buffered_event) == event) {
1806 /* Length is in event->array[0] */
1807 ring_buffer_write(buffer, event->array[0], &event->array[1]);
1808 /* Release the temp buffer */
1809 this_cpu_dec(trace_buffered_event_cnt);
1811 ring_buffer_unlock_commit(buffer, event);
1814 static struct ring_buffer *temp_buffer;
1816 struct ring_buffer_event *
1817 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1818 struct trace_event_file *trace_file,
1819 int type, unsigned long len,
1820 unsigned long flags, int pc)
1822 struct ring_buffer_event *entry;
1825 *current_rb = trace_file->tr->trace_buffer.buffer;
1827 if ((trace_file->flags &
1828 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
1829 (entry = this_cpu_read(trace_buffered_event))) {
1830 /* Try to use the per cpu buffer first */
1831 val = this_cpu_inc_return(trace_buffered_event_cnt);
1833 trace_event_setup(entry, type, flags, pc);
1834 entry->array[0] = len;
1837 this_cpu_dec(trace_buffered_event_cnt);
1840 entry = trace_buffer_lock_reserve(*current_rb,
1841 type, len, flags, pc);
1843 * If tracing is off, but we have triggers enabled
1844 * we still need to look at the event data. Use the temp_buffer
1845 * to store the trace event for the tigger to use. It's recusive
1846 * safe and will not be recorded anywhere.
1848 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
1849 *current_rb = temp_buffer;
1850 entry = trace_buffer_lock_reserve(*current_rb,
1851 type, len, flags, pc);
1855 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1857 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1858 struct ring_buffer *buffer,
1859 struct ring_buffer_event *event,
1860 unsigned long flags, int pc,
1861 struct pt_regs *regs)
1863 __buffer_unlock_commit(buffer, event);
1865 ftrace_trace_stack(tr, buffer, flags, 0, pc, regs);
1866 ftrace_trace_userstack(buffer, flags, pc);
1870 trace_function(struct trace_array *tr,
1871 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1874 struct trace_event_call *call = &event_function;
1875 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1876 struct ring_buffer_event *event;
1877 struct ftrace_entry *entry;
1879 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1883 entry = ring_buffer_event_data(event);
1885 entry->parent_ip = parent_ip;
1887 if (!call_filter_check_discard(call, entry, buffer, event))
1888 __buffer_unlock_commit(buffer, event);
1891 #ifdef CONFIG_STACKTRACE
1893 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1894 struct ftrace_stack {
1895 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1898 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1899 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1901 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1902 unsigned long flags,
1903 int skip, int pc, struct pt_regs *regs)
1905 struct trace_event_call *call = &event_kernel_stack;
1906 struct ring_buffer_event *event;
1907 struct stack_entry *entry;
1908 struct stack_trace trace;
1910 int size = FTRACE_STACK_ENTRIES;
1912 trace.nr_entries = 0;
1916 * Since events can happen in NMIs there's no safe way to
1917 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1918 * or NMI comes in, it will just have to use the default
1919 * FTRACE_STACK_SIZE.
1921 preempt_disable_notrace();
1923 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1925 * We don't need any atomic variables, just a barrier.
1926 * If an interrupt comes in, we don't care, because it would
1927 * have exited and put the counter back to what we want.
1928 * We just need a barrier to keep gcc from moving things
1932 if (use_stack == 1) {
1933 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1934 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1937 save_stack_trace_regs(regs, &trace);
1939 save_stack_trace(&trace);
1941 if (trace.nr_entries > size)
1942 size = trace.nr_entries;
1944 /* From now on, use_stack is a boolean */
1947 size *= sizeof(unsigned long);
1949 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1950 sizeof(*entry) + size, flags, pc);
1953 entry = ring_buffer_event_data(event);
1955 memset(&entry->caller, 0, size);
1958 memcpy(&entry->caller, trace.entries,
1959 trace.nr_entries * sizeof(unsigned long));
1961 trace.max_entries = FTRACE_STACK_ENTRIES;
1962 trace.entries = entry->caller;
1964 save_stack_trace_regs(regs, &trace);
1966 save_stack_trace(&trace);
1969 entry->size = trace.nr_entries;
1971 if (!call_filter_check_discard(call, entry, buffer, event))
1972 __buffer_unlock_commit(buffer, event);
1975 /* Again, don't let gcc optimize things here */
1977 __this_cpu_dec(ftrace_stack_reserve);
1978 preempt_enable_notrace();
1982 static inline void ftrace_trace_stack(struct trace_array *tr,
1983 struct ring_buffer *buffer,
1984 unsigned long flags,
1985 int skip, int pc, struct pt_regs *regs)
1987 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
1990 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1993 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1996 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
2000 * trace_dump_stack - record a stack back trace in the trace buffer
2001 * @skip: Number of functions to skip (helper handlers)
2003 void trace_dump_stack(int skip)
2005 unsigned long flags;
2007 if (tracing_disabled || tracing_selftest_running)
2010 local_save_flags(flags);
2013 * Skip 3 more, seems to get us at the caller of
2017 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2018 flags, skip, preempt_count(), NULL);
2021 static DEFINE_PER_CPU(int, user_stack_count);
2024 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
2026 struct trace_event_call *call = &event_user_stack;
2027 struct ring_buffer_event *event;
2028 struct userstack_entry *entry;
2029 struct stack_trace trace;
2031 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
2035 * NMIs can not handle page faults, even with fix ups.
2036 * The save user stack can (and often does) fault.
2038 if (unlikely(in_nmi()))
2042 * prevent recursion, since the user stack tracing may
2043 * trigger other kernel events.
2046 if (__this_cpu_read(user_stack_count))
2049 __this_cpu_inc(user_stack_count);
2051 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2052 sizeof(*entry), flags, pc);
2054 goto out_drop_count;
2055 entry = ring_buffer_event_data(event);
2057 entry->tgid = current->tgid;
2058 memset(&entry->caller, 0, sizeof(entry->caller));
2060 trace.nr_entries = 0;
2061 trace.max_entries = FTRACE_STACK_ENTRIES;
2063 trace.entries = entry->caller;
2065 save_stack_trace_user(&trace);
2066 if (!call_filter_check_discard(call, entry, buffer, event))
2067 __buffer_unlock_commit(buffer, event);
2070 __this_cpu_dec(user_stack_count);
2076 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
2078 ftrace_trace_userstack(tr, flags, preempt_count());
2082 #endif /* CONFIG_STACKTRACE */
2084 /* created for use with alloc_percpu */
2085 struct trace_buffer_struct {
2086 char buffer[TRACE_BUF_SIZE];
2089 static struct trace_buffer_struct *trace_percpu_buffer;
2090 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
2091 static struct trace_buffer_struct *trace_percpu_irq_buffer;
2092 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
2095 * The buffer used is dependent on the context. There is a per cpu
2096 * buffer for normal context, softirq contex, hard irq context and
2097 * for NMI context. Thise allows for lockless recording.
2099 * Note, if the buffers failed to be allocated, then this returns NULL
2101 static char *get_trace_buf(void)
2103 struct trace_buffer_struct *percpu_buffer;
2106 * If we have allocated per cpu buffers, then we do not
2107 * need to do any locking.
2110 percpu_buffer = trace_percpu_nmi_buffer;
2112 percpu_buffer = trace_percpu_irq_buffer;
2113 else if (in_softirq())
2114 percpu_buffer = trace_percpu_sirq_buffer;
2116 percpu_buffer = trace_percpu_buffer;
2121 return this_cpu_ptr(&percpu_buffer->buffer[0]);
2124 static int alloc_percpu_trace_buffer(void)
2126 struct trace_buffer_struct *buffers;
2127 struct trace_buffer_struct *sirq_buffers;
2128 struct trace_buffer_struct *irq_buffers;
2129 struct trace_buffer_struct *nmi_buffers;
2131 buffers = alloc_percpu(struct trace_buffer_struct);
2135 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2139 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2143 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2147 trace_percpu_buffer = buffers;
2148 trace_percpu_sirq_buffer = sirq_buffers;
2149 trace_percpu_irq_buffer = irq_buffers;
2150 trace_percpu_nmi_buffer = nmi_buffers;
2155 free_percpu(irq_buffers);
2157 free_percpu(sirq_buffers);
2159 free_percpu(buffers);
2161 WARN(1, "Could not allocate percpu trace_printk buffer");
2165 static int buffers_allocated;
2167 void trace_printk_init_buffers(void)
2169 if (buffers_allocated)
2172 if (alloc_percpu_trace_buffer())
2175 /* trace_printk() is for debug use only. Don't use it in production. */
2178 pr_warn("**********************************************************\n");
2179 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2181 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2183 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2184 pr_warn("** unsafe for production use. **\n");
2186 pr_warn("** If you see this message and you are not debugging **\n");
2187 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2189 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2190 pr_warn("**********************************************************\n");
2192 /* Expand the buffers to set size */
2193 tracing_update_buffers();
2195 buffers_allocated = 1;
2198 * trace_printk_init_buffers() can be called by modules.
2199 * If that happens, then we need to start cmdline recording
2200 * directly here. If the global_trace.buffer is already
2201 * allocated here, then this was called by module code.
2203 if (global_trace.trace_buffer.buffer)
2204 tracing_start_cmdline_record();
2207 void trace_printk_start_comm(void)
2209 /* Start tracing comms if trace printk is set */
2210 if (!buffers_allocated)
2212 tracing_start_cmdline_record();
2215 static void trace_printk_start_stop_comm(int enabled)
2217 if (!buffers_allocated)
2221 tracing_start_cmdline_record();
2223 tracing_stop_cmdline_record();
2227 * trace_vbprintk - write binary msg to tracing buffer
2230 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2232 struct trace_event_call *call = &event_bprint;
2233 struct ring_buffer_event *event;
2234 struct ring_buffer *buffer;
2235 struct trace_array *tr = &global_trace;
2236 struct bprint_entry *entry;
2237 unsigned long flags;
2239 int len = 0, size, pc;
2241 if (unlikely(tracing_selftest_running || tracing_disabled))
2244 /* Don't pollute graph traces with trace_vprintk internals */
2245 pause_graph_tracing();
2247 pc = preempt_count();
2248 preempt_disable_notrace();
2250 tbuffer = get_trace_buf();
2256 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2258 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2261 local_save_flags(flags);
2262 size = sizeof(*entry) + sizeof(u32) * len;
2263 buffer = tr->trace_buffer.buffer;
2264 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2268 entry = ring_buffer_event_data(event);
2272 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2273 if (!call_filter_check_discard(call, entry, buffer, event)) {
2274 __buffer_unlock_commit(buffer, event);
2275 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2279 preempt_enable_notrace();
2280 unpause_graph_tracing();
2284 EXPORT_SYMBOL_GPL(trace_vbprintk);
2287 __trace_array_vprintk(struct ring_buffer *buffer,
2288 unsigned long ip, const char *fmt, va_list args)
2290 struct trace_event_call *call = &event_print;
2291 struct ring_buffer_event *event;
2292 int len = 0, size, pc;
2293 struct print_entry *entry;
2294 unsigned long flags;
2297 if (tracing_disabled || tracing_selftest_running)
2300 /* Don't pollute graph traces with trace_vprintk internals */
2301 pause_graph_tracing();
2303 pc = preempt_count();
2304 preempt_disable_notrace();
2307 tbuffer = get_trace_buf();
2313 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2315 local_save_flags(flags);
2316 size = sizeof(*entry) + len + 1;
2317 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2321 entry = ring_buffer_event_data(event);
2324 memcpy(&entry->buf, tbuffer, len + 1);
2325 if (!call_filter_check_discard(call, entry, buffer, event)) {
2326 __buffer_unlock_commit(buffer, event);
2327 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
2330 preempt_enable_notrace();
2331 unpause_graph_tracing();
2336 int trace_array_vprintk(struct trace_array *tr,
2337 unsigned long ip, const char *fmt, va_list args)
2339 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2342 int trace_array_printk(struct trace_array *tr,
2343 unsigned long ip, const char *fmt, ...)
2348 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2352 ret = trace_array_vprintk(tr, ip, fmt, ap);
2357 int trace_array_printk_buf(struct ring_buffer *buffer,
2358 unsigned long ip, const char *fmt, ...)
2363 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2367 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2372 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2374 return trace_array_vprintk(&global_trace, ip, fmt, args);
2376 EXPORT_SYMBOL_GPL(trace_vprintk);
2378 static void trace_iterator_increment(struct trace_iterator *iter)
2380 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2384 ring_buffer_read(buf_iter, NULL);
2387 static struct trace_entry *
2388 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2389 unsigned long *lost_events)
2391 struct ring_buffer_event *event;
2392 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2395 event = ring_buffer_iter_peek(buf_iter, ts);
2397 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2401 iter->ent_size = ring_buffer_event_length(event);
2402 return ring_buffer_event_data(event);
2408 static struct trace_entry *
2409 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2410 unsigned long *missing_events, u64 *ent_ts)
2412 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2413 struct trace_entry *ent, *next = NULL;
2414 unsigned long lost_events = 0, next_lost = 0;
2415 int cpu_file = iter->cpu_file;
2416 u64 next_ts = 0, ts;
2422 * If we are in a per_cpu trace file, don't bother by iterating over
2423 * all cpu and peek directly.
2425 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2426 if (ring_buffer_empty_cpu(buffer, cpu_file))
2428 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2430 *ent_cpu = cpu_file;
2435 for_each_tracing_cpu(cpu) {
2437 if (ring_buffer_empty_cpu(buffer, cpu))
2440 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2443 * Pick the entry with the smallest timestamp:
2445 if (ent && (!next || ts < next_ts)) {
2449 next_lost = lost_events;
2450 next_size = iter->ent_size;
2454 iter->ent_size = next_size;
2457 *ent_cpu = next_cpu;
2463 *missing_events = next_lost;
2468 /* Find the next real entry, without updating the iterator itself */
2469 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2470 int *ent_cpu, u64 *ent_ts)
2472 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2475 /* Find the next real entry, and increment the iterator to the next entry */
2476 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2478 iter->ent = __find_next_entry(iter, &iter->cpu,
2479 &iter->lost_events, &iter->ts);
2482 trace_iterator_increment(iter);
2484 return iter->ent ? iter : NULL;
2487 static void trace_consume(struct trace_iterator *iter)
2489 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2490 &iter->lost_events);
2493 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2495 struct trace_iterator *iter = m->private;
2499 WARN_ON_ONCE(iter->leftover);
2503 /* can't go backwards */
2508 ent = trace_find_next_entry_inc(iter);
2512 while (ent && iter->idx < i)
2513 ent = trace_find_next_entry_inc(iter);
2520 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2522 struct ring_buffer_event *event;
2523 struct ring_buffer_iter *buf_iter;
2524 unsigned long entries = 0;
2527 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2529 buf_iter = trace_buffer_iter(iter, cpu);
2533 ring_buffer_iter_reset(buf_iter);
2536 * We could have the case with the max latency tracers
2537 * that a reset never took place on a cpu. This is evident
2538 * by the timestamp being before the start of the buffer.
2540 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2541 if (ts >= iter->trace_buffer->time_start)
2544 ring_buffer_read(buf_iter, NULL);
2547 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2551 * The current tracer is copied to avoid a global locking
2554 static void *s_start(struct seq_file *m, loff_t *pos)
2556 struct trace_iterator *iter = m->private;
2557 struct trace_array *tr = iter->tr;
2558 int cpu_file = iter->cpu_file;
2564 * copy the tracer to avoid using a global lock all around.
2565 * iter->trace is a copy of current_trace, the pointer to the
2566 * name may be used instead of a strcmp(), as iter->trace->name
2567 * will point to the same string as current_trace->name.
2569 mutex_lock(&trace_types_lock);
2570 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2571 *iter->trace = *tr->current_trace;
2572 mutex_unlock(&trace_types_lock);
2574 #ifdef CONFIG_TRACER_MAX_TRACE
2575 if (iter->snapshot && iter->trace->use_max_tr)
2576 return ERR_PTR(-EBUSY);
2579 if (!iter->snapshot)
2580 atomic_inc(&trace_record_cmdline_disabled);
2582 if (*pos != iter->pos) {
2587 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2588 for_each_tracing_cpu(cpu)
2589 tracing_iter_reset(iter, cpu);
2591 tracing_iter_reset(iter, cpu_file);
2594 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2599 * If we overflowed the seq_file before, then we want
2600 * to just reuse the trace_seq buffer again.
2606 p = s_next(m, p, &l);
2610 trace_event_read_lock();
2611 trace_access_lock(cpu_file);
2615 static void s_stop(struct seq_file *m, void *p)
2617 struct trace_iterator *iter = m->private;
2619 #ifdef CONFIG_TRACER_MAX_TRACE
2620 if (iter->snapshot && iter->trace->use_max_tr)
2624 if (!iter->snapshot)
2625 atomic_dec(&trace_record_cmdline_disabled);
2627 trace_access_unlock(iter->cpu_file);
2628 trace_event_read_unlock();
2632 get_total_entries(struct trace_buffer *buf,
2633 unsigned long *total, unsigned long *entries)
2635 unsigned long count;
2641 for_each_tracing_cpu(cpu) {
2642 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2644 * If this buffer has skipped entries, then we hold all
2645 * entries for the trace and we need to ignore the
2646 * ones before the time stamp.
2648 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2649 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2650 /* total is the same as the entries */
2654 ring_buffer_overrun_cpu(buf->buffer, cpu);
2659 static void print_lat_help_header(struct seq_file *m)
2661 seq_puts(m, "# _------=> CPU# \n"
2662 "# / _-----=> irqs-off \n"
2663 "# | / _----=> need-resched \n"
2664 "# || / _---=> hardirq/softirq \n"
2665 "# ||| / _--=> preempt-depth \n"
2667 "# cmd pid ||||| time | caller \n"
2668 "# \\ / ||||| \\ | / \n");
2671 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2673 unsigned long total;
2674 unsigned long entries;
2676 get_total_entries(buf, &total, &entries);
2677 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2678 entries, total, num_online_cpus());
2682 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2684 print_event_info(buf, m);
2685 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2689 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2691 print_event_info(buf, m);
2692 seq_puts(m, "# _-----=> irqs-off\n"
2693 "# / _----=> need-resched\n"
2694 "# | / _---=> hardirq/softirq\n"
2695 "# || / _--=> preempt-depth\n"
2697 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2698 "# | | | |||| | |\n");
2702 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2704 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2705 struct trace_buffer *buf = iter->trace_buffer;
2706 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2707 struct tracer *type = iter->trace;
2708 unsigned long entries;
2709 unsigned long total;
2710 const char *name = "preemption";
2714 get_total_entries(buf, &total, &entries);
2716 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2718 seq_puts(m, "# -----------------------------------"
2719 "---------------------------------\n");
2720 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2721 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2722 nsecs_to_usecs(data->saved_latency),
2726 #if defined(CONFIG_PREEMPT_NONE)
2728 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2730 #elif defined(CONFIG_PREEMPT)
2735 /* These are reserved for later use */
2738 seq_printf(m, " #P:%d)\n", num_online_cpus());
2742 seq_puts(m, "# -----------------\n");
2743 seq_printf(m, "# | task: %.16s-%d "
2744 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2745 data->comm, data->pid,
2746 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2747 data->policy, data->rt_priority);
2748 seq_puts(m, "# -----------------\n");
2750 if (data->critical_start) {
2751 seq_puts(m, "# => started at: ");
2752 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2753 trace_print_seq(m, &iter->seq);
2754 seq_puts(m, "\n# => ended at: ");
2755 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2756 trace_print_seq(m, &iter->seq);
2757 seq_puts(m, "\n#\n");
2763 static void test_cpu_buff_start(struct trace_iterator *iter)
2765 struct trace_seq *s = &iter->seq;
2766 struct trace_array *tr = iter->tr;
2768 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
2771 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2774 if (iter->started && cpumask_test_cpu(iter->cpu, iter->started))
2777 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2781 cpumask_set_cpu(iter->cpu, iter->started);
2783 /* Don't print started cpu buffer for the first entry of the trace */
2785 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2789 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2791 struct trace_array *tr = iter->tr;
2792 struct trace_seq *s = &iter->seq;
2793 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
2794 struct trace_entry *entry;
2795 struct trace_event *event;
2799 test_cpu_buff_start(iter);
2801 event = ftrace_find_event(entry->type);
2803 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2804 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2805 trace_print_lat_context(iter);
2807 trace_print_context(iter);
2810 if (trace_seq_has_overflowed(s))
2811 return TRACE_TYPE_PARTIAL_LINE;
2814 return event->funcs->trace(iter, sym_flags, event);
2816 trace_seq_printf(s, "Unknown type %d\n", entry->type);
2818 return trace_handle_return(s);
2821 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2823 struct trace_array *tr = iter->tr;
2824 struct trace_seq *s = &iter->seq;
2825 struct trace_entry *entry;
2826 struct trace_event *event;
2830 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
2831 trace_seq_printf(s, "%d %d %llu ",
2832 entry->pid, iter->cpu, iter->ts);
2834 if (trace_seq_has_overflowed(s))
2835 return TRACE_TYPE_PARTIAL_LINE;
2837 event = ftrace_find_event(entry->type);
2839 return event->funcs->raw(iter, 0, event);
2841 trace_seq_printf(s, "%d ?\n", entry->type);
2843 return trace_handle_return(s);
2846 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2848 struct trace_array *tr = iter->tr;
2849 struct trace_seq *s = &iter->seq;
2850 unsigned char newline = '\n';
2851 struct trace_entry *entry;
2852 struct trace_event *event;
2856 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2857 SEQ_PUT_HEX_FIELD(s, entry->pid);
2858 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2859 SEQ_PUT_HEX_FIELD(s, iter->ts);
2860 if (trace_seq_has_overflowed(s))
2861 return TRACE_TYPE_PARTIAL_LINE;
2864 event = ftrace_find_event(entry->type);
2866 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2867 if (ret != TRACE_TYPE_HANDLED)
2871 SEQ_PUT_FIELD(s, newline);
2873 return trace_handle_return(s);
2876 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2878 struct trace_array *tr = iter->tr;
2879 struct trace_seq *s = &iter->seq;
2880 struct trace_entry *entry;
2881 struct trace_event *event;
2885 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2886 SEQ_PUT_FIELD(s, entry->pid);
2887 SEQ_PUT_FIELD(s, iter->cpu);
2888 SEQ_PUT_FIELD(s, iter->ts);
2889 if (trace_seq_has_overflowed(s))
2890 return TRACE_TYPE_PARTIAL_LINE;
2893 event = ftrace_find_event(entry->type);
2894 return event ? event->funcs->binary(iter, 0, event) :
2898 int trace_empty(struct trace_iterator *iter)
2900 struct ring_buffer_iter *buf_iter;
2903 /* If we are looking at one CPU buffer, only check that one */
2904 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2905 cpu = iter->cpu_file;
2906 buf_iter = trace_buffer_iter(iter, cpu);
2908 if (!ring_buffer_iter_empty(buf_iter))
2911 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2917 for_each_tracing_cpu(cpu) {
2918 buf_iter = trace_buffer_iter(iter, cpu);
2920 if (!ring_buffer_iter_empty(buf_iter))
2923 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2931 /* Called with trace_event_read_lock() held. */
2932 enum print_line_t print_trace_line(struct trace_iterator *iter)
2934 struct trace_array *tr = iter->tr;
2935 unsigned long trace_flags = tr->trace_flags;
2936 enum print_line_t ret;
2938 if (iter->lost_events) {
2939 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2940 iter->cpu, iter->lost_events);
2941 if (trace_seq_has_overflowed(&iter->seq))
2942 return TRACE_TYPE_PARTIAL_LINE;
2945 if (iter->trace && iter->trace->print_line) {
2946 ret = iter->trace->print_line(iter);
2947 if (ret != TRACE_TYPE_UNHANDLED)
2951 if (iter->ent->type == TRACE_BPUTS &&
2952 trace_flags & TRACE_ITER_PRINTK &&
2953 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2954 return trace_print_bputs_msg_only(iter);
2956 if (iter->ent->type == TRACE_BPRINT &&
2957 trace_flags & TRACE_ITER_PRINTK &&
2958 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2959 return trace_print_bprintk_msg_only(iter);
2961 if (iter->ent->type == TRACE_PRINT &&
2962 trace_flags & TRACE_ITER_PRINTK &&
2963 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2964 return trace_print_printk_msg_only(iter);
2966 if (trace_flags & TRACE_ITER_BIN)
2967 return print_bin_fmt(iter);
2969 if (trace_flags & TRACE_ITER_HEX)
2970 return print_hex_fmt(iter);
2972 if (trace_flags & TRACE_ITER_RAW)
2973 return print_raw_fmt(iter);
2975 return print_trace_fmt(iter);
2978 void trace_latency_header(struct seq_file *m)
2980 struct trace_iterator *iter = m->private;
2981 struct trace_array *tr = iter->tr;
2983 /* print nothing if the buffers are empty */
2984 if (trace_empty(iter))
2987 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2988 print_trace_header(m, iter);
2990 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
2991 print_lat_help_header(m);
2994 void trace_default_header(struct seq_file *m)
2996 struct trace_iterator *iter = m->private;
2997 struct trace_array *tr = iter->tr;
2998 unsigned long trace_flags = tr->trace_flags;
3000 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3003 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3004 /* print nothing if the buffers are empty */
3005 if (trace_empty(iter))
3007 print_trace_header(m, iter);
3008 if (!(trace_flags & TRACE_ITER_VERBOSE))
3009 print_lat_help_header(m);
3011 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3012 if (trace_flags & TRACE_ITER_IRQ_INFO)
3013 print_func_help_header_irq(iter->trace_buffer, m);
3015 print_func_help_header(iter->trace_buffer, m);
3020 static void test_ftrace_alive(struct seq_file *m)
3022 if (!ftrace_is_dead())
3024 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3025 "# MAY BE MISSING FUNCTION EVENTS\n");
3028 #ifdef CONFIG_TRACER_MAX_TRACE
3029 static void show_snapshot_main_help(struct seq_file *m)
3031 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3032 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3033 "# Takes a snapshot of the main buffer.\n"
3034 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3035 "# (Doesn't have to be '2' works with any number that\n"
3036 "# is not a '0' or '1')\n");
3039 static void show_snapshot_percpu_help(struct seq_file *m)
3041 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3042 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3043 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3044 "# Takes a snapshot of the main buffer for this cpu.\n");
3046 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3047 "# Must use main snapshot file to allocate.\n");
3049 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3050 "# (Doesn't have to be '2' works with any number that\n"
3051 "# is not a '0' or '1')\n");
3054 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3056 if (iter->tr->allocated_snapshot)
3057 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
3059 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
3061 seq_puts(m, "# Snapshot commands:\n");
3062 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3063 show_snapshot_main_help(m);
3065 show_snapshot_percpu_help(m);
3068 /* Should never be called */
3069 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3072 static int s_show(struct seq_file *m, void *v)
3074 struct trace_iterator *iter = v;
3077 if (iter->ent == NULL) {
3079 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3081 test_ftrace_alive(m);
3083 if (iter->snapshot && trace_empty(iter))
3084 print_snapshot_help(m, iter);
3085 else if (iter->trace && iter->trace->print_header)
3086 iter->trace->print_header(m);
3088 trace_default_header(m);
3090 } else if (iter->leftover) {
3092 * If we filled the seq_file buffer earlier, we
3093 * want to just show it now.
3095 ret = trace_print_seq(m, &iter->seq);
3097 /* ret should this time be zero, but you never know */
3098 iter->leftover = ret;
3101 print_trace_line(iter);
3102 ret = trace_print_seq(m, &iter->seq);
3104 * If we overflow the seq_file buffer, then it will
3105 * ask us for this data again at start up.
3107 * ret is 0 if seq_file write succeeded.
3110 iter->leftover = ret;
3117 * Should be used after trace_array_get(), trace_types_lock
3118 * ensures that i_cdev was already initialized.
3120 static inline int tracing_get_cpu(struct inode *inode)
3122 if (inode->i_cdev) /* See trace_create_cpu_file() */
3123 return (long)inode->i_cdev - 1;
3124 return RING_BUFFER_ALL_CPUS;
3127 static const struct seq_operations tracer_seq_ops = {
3134 static struct trace_iterator *
3135 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
3137 struct trace_array *tr = inode->i_private;
3138 struct trace_iterator *iter;
3141 if (tracing_disabled)
3142 return ERR_PTR(-ENODEV);
3144 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3146 return ERR_PTR(-ENOMEM);
3148 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3150 if (!iter->buffer_iter)
3154 * We make a copy of the current tracer to avoid concurrent
3155 * changes on it while we are reading.
3157 mutex_lock(&trace_types_lock);
3158 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3162 *iter->trace = *tr->current_trace;
3164 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3169 #ifdef CONFIG_TRACER_MAX_TRACE
3170 /* Currently only the top directory has a snapshot */
3171 if (tr->current_trace->print_max || snapshot)
3172 iter->trace_buffer = &tr->max_buffer;
3175 iter->trace_buffer = &tr->trace_buffer;
3176 iter->snapshot = snapshot;
3178 iter->cpu_file = tracing_get_cpu(inode);
3179 mutex_init(&iter->mutex);
3181 /* Notify the tracer early; before we stop tracing. */
3182 if (iter->trace && iter->trace->open)
3183 iter->trace->open(iter);
3185 /* Annotate start of buffers if we had overruns */
3186 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3187 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3189 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3190 if (trace_clocks[tr->clock_id].in_ns)
3191 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3193 /* stop the trace while dumping if we are not opening "snapshot" */
3194 if (!iter->snapshot)
3195 tracing_stop_tr(tr);
3197 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3198 for_each_tracing_cpu(cpu) {
3199 iter->buffer_iter[cpu] =
3200 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3202 ring_buffer_read_prepare_sync();
3203 for_each_tracing_cpu(cpu) {
3204 ring_buffer_read_start(iter->buffer_iter[cpu]);
3205 tracing_iter_reset(iter, cpu);
3208 cpu = iter->cpu_file;
3209 iter->buffer_iter[cpu] =
3210 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3211 ring_buffer_read_prepare_sync();
3212 ring_buffer_read_start(iter->buffer_iter[cpu]);
3213 tracing_iter_reset(iter, cpu);
3216 mutex_unlock(&trace_types_lock);
3221 mutex_unlock(&trace_types_lock);
3223 kfree(iter->buffer_iter);
3225 seq_release_private(inode, file);
3226 return ERR_PTR(-ENOMEM);
3229 int tracing_open_generic(struct inode *inode, struct file *filp)
3231 if (tracing_disabled)
3234 filp->private_data = inode->i_private;
3238 bool tracing_is_disabled(void)
3240 return (tracing_disabled) ? true: false;
3244 * Open and update trace_array ref count.
3245 * Must have the current trace_array passed to it.
3247 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3249 struct trace_array *tr = inode->i_private;
3251 if (tracing_disabled)
3254 if (trace_array_get(tr) < 0)
3257 filp->private_data = inode->i_private;
3262 static int tracing_release(struct inode *inode, struct file *file)
3264 struct trace_array *tr = inode->i_private;
3265 struct seq_file *m = file->private_data;
3266 struct trace_iterator *iter;
3269 if (!(file->f_mode & FMODE_READ)) {
3270 trace_array_put(tr);
3274 /* Writes do not use seq_file */
3276 mutex_lock(&trace_types_lock);
3278 for_each_tracing_cpu(cpu) {
3279 if (iter->buffer_iter[cpu])
3280 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3283 if (iter->trace && iter->trace->close)
3284 iter->trace->close(iter);
3286 if (!iter->snapshot)
3287 /* reenable tracing if it was previously enabled */
3288 tracing_start_tr(tr);
3290 __trace_array_put(tr);
3292 mutex_unlock(&trace_types_lock);
3294 mutex_destroy(&iter->mutex);
3295 free_cpumask_var(iter->started);
3297 kfree(iter->buffer_iter);
3298 seq_release_private(inode, file);
3303 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3305 struct trace_array *tr = inode->i_private;
3307 trace_array_put(tr);
3311 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3313 struct trace_array *tr = inode->i_private;
3315 trace_array_put(tr);
3317 return single_release(inode, file);
3320 static int tracing_open(struct inode *inode, struct file *file)
3322 struct trace_array *tr = inode->i_private;
3323 struct trace_iterator *iter;
3326 if (trace_array_get(tr) < 0)
3329 /* If this file was open for write, then erase contents */
3330 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3331 int cpu = tracing_get_cpu(inode);
3333 if (cpu == RING_BUFFER_ALL_CPUS)
3334 tracing_reset_online_cpus(&tr->trace_buffer);
3336 tracing_reset(&tr->trace_buffer, cpu);
3339 if (file->f_mode & FMODE_READ) {
3340 iter = __tracing_open(inode, file, false);
3342 ret = PTR_ERR(iter);
3343 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
3344 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3348 trace_array_put(tr);
3354 * Some tracers are not suitable for instance buffers.
3355 * A tracer is always available for the global array (toplevel)
3356 * or if it explicitly states that it is.
3359 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3361 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3364 /* Find the next tracer that this trace array may use */
3365 static struct tracer *
3366 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3368 while (t && !trace_ok_for_array(t, tr))
3375 t_next(struct seq_file *m, void *v, loff_t *pos)
3377 struct trace_array *tr = m->private;
3378 struct tracer *t = v;
3383 t = get_tracer_for_array(tr, t->next);
3388 static void *t_start(struct seq_file *m, loff_t *pos)
3390 struct trace_array *tr = m->private;
3394 mutex_lock(&trace_types_lock);
3396 t = get_tracer_for_array(tr, trace_types);
3397 for (; t && l < *pos; t = t_next(m, t, &l))
3403 static void t_stop(struct seq_file *m, void *p)
3405 mutex_unlock(&trace_types_lock);
3408 static int t_show(struct seq_file *m, void *v)
3410 struct tracer *t = v;
3415 seq_puts(m, t->name);
3424 static const struct seq_operations show_traces_seq_ops = {
3431 static int show_traces_open(struct inode *inode, struct file *file)
3433 struct trace_array *tr = inode->i_private;
3437 if (tracing_disabled)
3440 ret = seq_open(file, &show_traces_seq_ops);
3444 m = file->private_data;
3451 tracing_write_stub(struct file *filp, const char __user *ubuf,
3452 size_t count, loff_t *ppos)
3457 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3461 if (file->f_mode & FMODE_READ)
3462 ret = seq_lseek(file, offset, whence);
3464 file->f_pos = ret = 0;
3469 static const struct file_operations tracing_fops = {
3470 .open = tracing_open,
3472 .write = tracing_write_stub,
3473 .llseek = tracing_lseek,
3474 .release = tracing_release,
3477 static const struct file_operations show_traces_fops = {
3478 .open = show_traces_open,
3480 .release = seq_release,
3481 .llseek = seq_lseek,
3485 * The tracer itself will not take this lock, but still we want
3486 * to provide a consistent cpumask to user-space:
3488 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3491 * Temporary storage for the character representation of the
3492 * CPU bitmask (and one more byte for the newline):
3494 static char mask_str[NR_CPUS + 1];
3497 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3498 size_t count, loff_t *ppos)
3500 struct trace_array *tr = file_inode(filp)->i_private;
3503 mutex_lock(&tracing_cpumask_update_lock);
3505 len = snprintf(mask_str, count, "%*pb\n",
3506 cpumask_pr_args(tr->tracing_cpumask));
3511 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3514 mutex_unlock(&tracing_cpumask_update_lock);
3520 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3521 size_t count, loff_t *ppos)
3523 struct trace_array *tr = file_inode(filp)->i_private;
3524 cpumask_var_t tracing_cpumask_new;
3527 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3530 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3534 mutex_lock(&tracing_cpumask_update_lock);
3536 local_irq_disable();
3537 arch_spin_lock(&tr->max_lock);
3538 for_each_tracing_cpu(cpu) {
3540 * Increase/decrease the disabled counter if we are
3541 * about to flip a bit in the cpumask:
3543 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3544 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3545 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3546 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3548 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3549 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3550 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3551 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3554 arch_spin_unlock(&tr->max_lock);
3557 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3559 mutex_unlock(&tracing_cpumask_update_lock);
3560 free_cpumask_var(tracing_cpumask_new);
3565 free_cpumask_var(tracing_cpumask_new);
3570 static const struct file_operations tracing_cpumask_fops = {
3571 .open = tracing_open_generic_tr,
3572 .read = tracing_cpumask_read,
3573 .write = tracing_cpumask_write,
3574 .release = tracing_release_generic_tr,
3575 .llseek = generic_file_llseek,
3578 static int tracing_trace_options_show(struct seq_file *m, void *v)
3580 struct tracer_opt *trace_opts;
3581 struct trace_array *tr = m->private;
3585 mutex_lock(&trace_types_lock);
3586 tracer_flags = tr->current_trace->flags->val;
3587 trace_opts = tr->current_trace->flags->opts;
3589 for (i = 0; trace_options[i]; i++) {
3590 if (tr->trace_flags & (1 << i))
3591 seq_printf(m, "%s\n", trace_options[i]);
3593 seq_printf(m, "no%s\n", trace_options[i]);
3596 for (i = 0; trace_opts[i].name; i++) {
3597 if (tracer_flags & trace_opts[i].bit)
3598 seq_printf(m, "%s\n", trace_opts[i].name);
3600 seq_printf(m, "no%s\n", trace_opts[i].name);
3602 mutex_unlock(&trace_types_lock);
3607 static int __set_tracer_option(struct trace_array *tr,
3608 struct tracer_flags *tracer_flags,
3609 struct tracer_opt *opts, int neg)
3611 struct tracer *trace = tracer_flags->trace;
3614 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3619 tracer_flags->val &= ~opts->bit;
3621 tracer_flags->val |= opts->bit;
3625 /* Try to assign a tracer specific option */
3626 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3628 struct tracer *trace = tr->current_trace;
3629 struct tracer_flags *tracer_flags = trace->flags;
3630 struct tracer_opt *opts = NULL;
3633 for (i = 0; tracer_flags->opts[i].name; i++) {
3634 opts = &tracer_flags->opts[i];
3636 if (strcmp(cmp, opts->name) == 0)
3637 return __set_tracer_option(tr, trace->flags, opts, neg);
3643 /* Some tracers require overwrite to stay enabled */
3644 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3646 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3652 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3654 /* do nothing if flag is already set */
3655 if (!!(tr->trace_flags & mask) == !!enabled)
3658 /* Give the tracer a chance to approve the change */
3659 if (tr->current_trace->flag_changed)
3660 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3664 tr->trace_flags |= mask;
3666 tr->trace_flags &= ~mask;
3668 if (mask == TRACE_ITER_RECORD_CMD)
3669 trace_event_enable_cmd_record(enabled);
3671 if (mask == TRACE_ITER_EVENT_FORK)
3672 trace_event_follow_fork(tr, enabled);
3674 if (mask == TRACE_ITER_OVERWRITE) {
3675 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3676 #ifdef CONFIG_TRACER_MAX_TRACE
3677 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3681 if (mask == TRACE_ITER_PRINTK) {
3682 trace_printk_start_stop_comm(enabled);
3683 trace_printk_control(enabled);
3689 static int trace_set_options(struct trace_array *tr, char *option)
3695 size_t orig_len = strlen(option);
3697 cmp = strstrip(option);
3699 if (strncmp(cmp, "no", 2) == 0) {
3704 mutex_lock(&trace_types_lock);
3706 for (i = 0; trace_options[i]; i++) {
3707 if (strcmp(cmp, trace_options[i]) == 0) {
3708 ret = set_tracer_flag(tr, 1 << i, !neg);
3713 /* If no option could be set, test the specific tracer options */
3714 if (!trace_options[i])
3715 ret = set_tracer_option(tr, cmp, neg);
3717 mutex_unlock(&trace_types_lock);
3720 * If the first trailing whitespace is replaced with '\0' by strstrip,
3721 * turn it back into a space.
3723 if (orig_len > strlen(option))
3724 option[strlen(option)] = ' ';
3729 static void __init apply_trace_boot_options(void)
3731 char *buf = trace_boot_options_buf;
3735 option = strsep(&buf, ",");
3741 trace_set_options(&global_trace, option);
3743 /* Put back the comma to allow this to be called again */
3750 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3751 size_t cnt, loff_t *ppos)
3753 struct seq_file *m = filp->private_data;
3754 struct trace_array *tr = m->private;
3758 if (cnt >= sizeof(buf))
3761 if (copy_from_user(buf, ubuf, cnt))
3766 ret = trace_set_options(tr, buf);
3775 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3777 struct trace_array *tr = inode->i_private;
3780 if (tracing_disabled)
3783 if (trace_array_get(tr) < 0)
3786 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3788 trace_array_put(tr);
3793 static const struct file_operations tracing_iter_fops = {
3794 .open = tracing_trace_options_open,
3796 .llseek = seq_lseek,
3797 .release = tracing_single_release_tr,
3798 .write = tracing_trace_options_write,
3801 static const char readme_msg[] =
3802 "tracing mini-HOWTO:\n\n"
3803 "# echo 0 > tracing_on : quick way to disable tracing\n"
3804 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3805 " Important files:\n"
3806 " trace\t\t\t- The static contents of the buffer\n"
3807 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3808 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3809 " current_tracer\t- function and latency tracers\n"
3810 " available_tracers\t- list of configured tracers for current_tracer\n"
3811 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3812 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3813 " trace_clock\t\t-change the clock used to order events\n"
3814 " local: Per cpu clock but may not be synced across CPUs\n"
3815 " global: Synced across CPUs but slows tracing down.\n"
3816 " counter: Not a clock, but just an increment\n"
3817 " uptime: Jiffy counter from time of boot\n"
3818 " perf: Same clock that perf events use\n"
3819 #ifdef CONFIG_X86_64
3820 " x86-tsc: TSC cycle counter\n"
3822 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3823 " tracing_cpumask\t- Limit which CPUs to trace\n"
3824 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3825 "\t\t\t Remove sub-buffer with rmdir\n"
3826 " trace_options\t\t- Set format or modify how tracing happens\n"
3827 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3828 "\t\t\t option name\n"
3829 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3830 #ifdef CONFIG_DYNAMIC_FTRACE
3831 "\n available_filter_functions - list of functions that can be filtered on\n"
3832 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3833 "\t\t\t functions\n"
3834 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3835 "\t modules: Can select a group via module\n"
3836 "\t Format: :mod:<module-name>\n"
3837 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3838 "\t triggers: a command to perform when function is hit\n"
3839 "\t Format: <function>:<trigger>[:count]\n"
3840 "\t trigger: traceon, traceoff\n"
3841 "\t\t enable_event:<system>:<event>\n"
3842 "\t\t disable_event:<system>:<event>\n"
3843 #ifdef CONFIG_STACKTRACE
3846 #ifdef CONFIG_TRACER_SNAPSHOT
3851 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3852 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3853 "\t The first one will disable tracing every time do_fault is hit\n"
3854 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3855 "\t The first time do trap is hit and it disables tracing, the\n"
3856 "\t counter will decrement to 2. If tracing is already disabled,\n"
3857 "\t the counter will not decrement. It only decrements when the\n"
3858 "\t trigger did work\n"
3859 "\t To remove trigger without count:\n"
3860 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3861 "\t To remove trigger with a count:\n"
3862 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3863 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3864 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3865 "\t modules: Can select a group via module command :mod:\n"
3866 "\t Does not accept triggers\n"
3867 #endif /* CONFIG_DYNAMIC_FTRACE */
3868 #ifdef CONFIG_FUNCTION_TRACER
3869 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3872 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3873 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3874 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3875 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3877 #ifdef CONFIG_TRACER_SNAPSHOT
3878 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3879 "\t\t\t snapshot buffer. Read the contents for more\n"
3880 "\t\t\t information\n"
3882 #ifdef CONFIG_STACK_TRACER
3883 " stack_trace\t\t- Shows the max stack trace when active\n"
3884 " stack_max_size\t- Shows current max stack size that was traced\n"
3885 "\t\t\t Write into this file to reset the max size (trigger a\n"
3886 "\t\t\t new trace)\n"
3887 #ifdef CONFIG_DYNAMIC_FTRACE
3888 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3891 #endif /* CONFIG_STACK_TRACER */
3892 " events/\t\t- Directory containing all trace event subsystems:\n"
3893 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3894 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3895 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3897 " filter\t\t- If set, only events passing filter are traced\n"
3898 " events/<system>/<event>/\t- Directory containing control files for\n"
3900 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3901 " filter\t\t- If set, only events passing filter are traced\n"
3902 " trigger\t\t- If set, a command to perform when event is hit\n"
3903 "\t Format: <trigger>[:count][if <filter>]\n"
3904 "\t trigger: traceon, traceoff\n"
3905 "\t enable_event:<system>:<event>\n"
3906 "\t disable_event:<system>:<event>\n"
3907 #ifdef CONFIG_HIST_TRIGGERS
3908 "\t enable_hist:<system>:<event>\n"
3909 "\t disable_hist:<system>:<event>\n"
3911 #ifdef CONFIG_STACKTRACE
3914 #ifdef CONFIG_TRACER_SNAPSHOT
3917 #ifdef CONFIG_HIST_TRIGGERS
3918 "\t\t hist (see below)\n"
3920 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3921 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3922 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3923 "\t events/block/block_unplug/trigger\n"
3924 "\t The first disables tracing every time block_unplug is hit.\n"
3925 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3926 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3927 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3928 "\t Like function triggers, the counter is only decremented if it\n"
3929 "\t enabled or disabled tracing.\n"
3930 "\t To remove a trigger without a count:\n"
3931 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3932 "\t To remove a trigger with a count:\n"
3933 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3934 "\t Filters can be ignored when removing a trigger.\n"
3935 #ifdef CONFIG_HIST_TRIGGERS
3936 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
3937 "\t Format: hist:keys=<field1[,field2,...]>\n"
3938 "\t [:values=<field1[,field2,...]>]\n"
3939 "\t [:sort=<field1[,field2,...]>]\n"
3940 "\t [:size=#entries]\n"
3941 "\t [:pause][:continue][:clear]\n"
3942 "\t [:name=histname1]\n"
3943 "\t [if <filter>]\n\n"
3944 "\t When a matching event is hit, an entry is added to a hash\n"
3945 "\t table using the key(s) and value(s) named, and the value of a\n"
3946 "\t sum called 'hitcount' is incremented. Keys and values\n"
3947 "\t correspond to fields in the event's format description. Keys\n"
3948 "\t can be any field, or the special string 'stacktrace'.\n"
3949 "\t Compound keys consisting of up to two fields can be specified\n"
3950 "\t by the 'keys' keyword. Values must correspond to numeric\n"
3951 "\t fields. Sort keys consisting of up to two fields can be\n"
3952 "\t specified using the 'sort' keyword. The sort direction can\n"
3953 "\t be modified by appending '.descending' or '.ascending' to a\n"
3954 "\t sort field. The 'size' parameter can be used to specify more\n"
3955 "\t or fewer than the default 2048 entries for the hashtable size.\n"
3956 "\t If a hist trigger is given a name using the 'name' parameter,\n"
3957 "\t its histogram data will be shared with other triggers of the\n"
3958 "\t same name, and trigger hits will update this common data.\n\n"
3959 "\t Reading the 'hist' file for the event will dump the hash\n"
3960 "\t table in its entirety to stdout. If there are multiple hist\n"
3961 "\t triggers attached to an event, there will be a table for each\n"
3962 "\t trigger in the output. The table displayed for a named\n"
3963 "\t trigger will be the same as any other instance having the\n"
3964 "\t same name. The default format used to display a given field\n"
3965 "\t can be modified by appending any of the following modifiers\n"
3966 "\t to the field name, as applicable:\n\n"
3967 "\t .hex display a number as a hex value\n"
3968 "\t .sym display an address as a symbol\n"
3969 "\t .sym-offset display an address as a symbol and offset\n"
3970 "\t .execname display a common_pid as a program name\n"
3971 "\t .syscall display a syscall id as a syscall name\n\n"
3972 "\t .log2 display log2 value rather than raw number\n\n"
3973 "\t The 'pause' parameter can be used to pause an existing hist\n"
3974 "\t trigger or to start a hist trigger but not log any events\n"
3975 "\t until told to do so. 'continue' can be used to start or\n"
3976 "\t restart a paused hist trigger.\n\n"
3977 "\t The 'clear' parameter will clear the contents of a running\n"
3978 "\t hist trigger and leave its current paused/active state\n"
3980 "\t The enable_hist and disable_hist triggers can be used to\n"
3981 "\t have one event conditionally start and stop another event's\n"
3982 "\t already-attached hist trigger. The syntax is analagous to\n"
3983 "\t the enable_event and disable_event triggers.\n"
3988 tracing_readme_read(struct file *filp, char __user *ubuf,
3989 size_t cnt, loff_t *ppos)
3991 return simple_read_from_buffer(ubuf, cnt, ppos,
3992 readme_msg, strlen(readme_msg));
3995 static const struct file_operations tracing_readme_fops = {
3996 .open = tracing_open_generic,
3997 .read = tracing_readme_read,
3998 .llseek = generic_file_llseek,
4001 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
4003 unsigned int *ptr = v;
4005 if (*pos || m->count)
4010 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
4012 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
4021 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
4027 arch_spin_lock(&trace_cmdline_lock);
4029 v = &savedcmd->map_cmdline_to_pid[0];
4031 v = saved_cmdlines_next(m, v, &l);
4039 static void saved_cmdlines_stop(struct seq_file *m, void *v)
4041 arch_spin_unlock(&trace_cmdline_lock);
4045 static int saved_cmdlines_show(struct seq_file *m, void *v)
4047 char buf[TASK_COMM_LEN];
4048 unsigned int *pid = v;
4050 __trace_find_cmdline(*pid, buf);
4051 seq_printf(m, "%d %s\n", *pid, buf);
4055 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
4056 .start = saved_cmdlines_start,
4057 .next = saved_cmdlines_next,
4058 .stop = saved_cmdlines_stop,
4059 .show = saved_cmdlines_show,
4062 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
4064 if (tracing_disabled)
4067 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
4070 static const struct file_operations tracing_saved_cmdlines_fops = {
4071 .open = tracing_saved_cmdlines_open,
4073 .llseek = seq_lseek,
4074 .release = seq_release,
4078 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
4079 size_t cnt, loff_t *ppos)
4084 arch_spin_lock(&trace_cmdline_lock);
4085 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
4086 arch_spin_unlock(&trace_cmdline_lock);
4088 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4091 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
4093 kfree(s->saved_cmdlines);
4094 kfree(s->map_cmdline_to_pid);
4098 static int tracing_resize_saved_cmdlines(unsigned int val)
4100 struct saved_cmdlines_buffer *s, *savedcmd_temp;
4102 s = kmalloc(sizeof(*s), GFP_KERNEL);
4106 if (allocate_cmdlines_buffer(val, s) < 0) {
4111 arch_spin_lock(&trace_cmdline_lock);
4112 savedcmd_temp = savedcmd;
4114 arch_spin_unlock(&trace_cmdline_lock);
4115 free_saved_cmdlines_buffer(savedcmd_temp);
4121 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
4122 size_t cnt, loff_t *ppos)
4127 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4131 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4132 if (!val || val > PID_MAX_DEFAULT)
4135 ret = tracing_resize_saved_cmdlines((unsigned int)val);
4144 static const struct file_operations tracing_saved_cmdlines_size_fops = {
4145 .open = tracing_open_generic,
4146 .read = tracing_saved_cmdlines_size_read,
4147 .write = tracing_saved_cmdlines_size_write,
4150 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
4151 static union trace_enum_map_item *
4152 update_enum_map(union trace_enum_map_item *ptr)
4154 if (!ptr->map.enum_string) {
4155 if (ptr->tail.next) {
4156 ptr = ptr->tail.next;
4157 /* Set ptr to the next real item (skip head) */
4165 static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
4167 union trace_enum_map_item *ptr = v;
4170 * Paranoid! If ptr points to end, we don't want to increment past it.
4171 * This really should never happen.
4173 ptr = update_enum_map(ptr);
4174 if (WARN_ON_ONCE(!ptr))
4181 ptr = update_enum_map(ptr);
4186 static void *enum_map_start(struct seq_file *m, loff_t *pos)
4188 union trace_enum_map_item *v;
4191 mutex_lock(&trace_enum_mutex);
4193 v = trace_enum_maps;
4197 while (v && l < *pos) {
4198 v = enum_map_next(m, v, &l);
4204 static void enum_map_stop(struct seq_file *m, void *v)
4206 mutex_unlock(&trace_enum_mutex);
4209 static int enum_map_show(struct seq_file *m, void *v)
4211 union trace_enum_map_item *ptr = v;
4213 seq_printf(m, "%s %ld (%s)\n",
4214 ptr->map.enum_string, ptr->map.enum_value,
4220 static const struct seq_operations tracing_enum_map_seq_ops = {
4221 .start = enum_map_start,
4222 .next = enum_map_next,
4223 .stop = enum_map_stop,
4224 .show = enum_map_show,
4227 static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4229 if (tracing_disabled)
4232 return seq_open(filp, &tracing_enum_map_seq_ops);
4235 static const struct file_operations tracing_enum_map_fops = {
4236 .open = tracing_enum_map_open,
4238 .llseek = seq_lseek,
4239 .release = seq_release,
4242 static inline union trace_enum_map_item *
4243 trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4245 /* Return tail of array given the head */
4246 return ptr + ptr->head.length + 1;
4250 trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4253 struct trace_enum_map **stop;
4254 struct trace_enum_map **map;
4255 union trace_enum_map_item *map_array;
4256 union trace_enum_map_item *ptr;
4261 * The trace_enum_maps contains the map plus a head and tail item,
4262 * where the head holds the module and length of array, and the
4263 * tail holds a pointer to the next list.
4265 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4267 pr_warn("Unable to allocate trace enum mapping\n");
4271 mutex_lock(&trace_enum_mutex);
4273 if (!trace_enum_maps)
4274 trace_enum_maps = map_array;
4276 ptr = trace_enum_maps;
4278 ptr = trace_enum_jmp_to_tail(ptr);
4279 if (!ptr->tail.next)
4281 ptr = ptr->tail.next;
4284 ptr->tail.next = map_array;
4286 map_array->head.mod = mod;
4287 map_array->head.length = len;
4290 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4291 map_array->map = **map;
4294 memset(map_array, 0, sizeof(*map_array));
4296 mutex_unlock(&trace_enum_mutex);
4299 static void trace_create_enum_file(struct dentry *d_tracer)
4301 trace_create_file("enum_map", 0444, d_tracer,
4302 NULL, &tracing_enum_map_fops);
4305 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4306 static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4307 static inline void trace_insert_enum_map_file(struct module *mod,
4308 struct trace_enum_map **start, int len) { }
4309 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4311 static void trace_insert_enum_map(struct module *mod,
4312 struct trace_enum_map **start, int len)
4314 struct trace_enum_map **map;
4321 trace_event_enum_update(map, len);
4323 trace_insert_enum_map_file(mod, start, len);
4327 tracing_set_trace_read(struct file *filp, char __user *ubuf,
4328 size_t cnt, loff_t *ppos)
4330 struct trace_array *tr = filp->private_data;
4331 char buf[MAX_TRACER_SIZE+2];
4334 mutex_lock(&trace_types_lock);
4335 r = sprintf(buf, "%s\n", tr->current_trace->name);
4336 mutex_unlock(&trace_types_lock);
4338 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4341 int tracer_init(struct tracer *t, struct trace_array *tr)
4343 tracing_reset_online_cpus(&tr->trace_buffer);
4347 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4351 for_each_tracing_cpu(cpu)
4352 per_cpu_ptr(buf->data, cpu)->entries = val;
4355 #ifdef CONFIG_TRACER_MAX_TRACE
4356 /* resize @tr's buffer to the size of @size_tr's entries */
4357 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4358 struct trace_buffer *size_buf, int cpu_id)
4362 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4363 for_each_tracing_cpu(cpu) {
4364 ret = ring_buffer_resize(trace_buf->buffer,
4365 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4368 per_cpu_ptr(trace_buf->data, cpu)->entries =
4369 per_cpu_ptr(size_buf->data, cpu)->entries;
4372 ret = ring_buffer_resize(trace_buf->buffer,
4373 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4375 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4376 per_cpu_ptr(size_buf->data, cpu_id)->entries;
4381 #endif /* CONFIG_TRACER_MAX_TRACE */
4383 static int __tracing_resize_ring_buffer(struct trace_array *tr,
4384 unsigned long size, int cpu)
4389 * If kernel or user changes the size of the ring buffer
4390 * we use the size that was given, and we can forget about
4391 * expanding it later.
4393 ring_buffer_expanded = true;
4395 /* May be called before buffers are initialized */
4396 if (!tr->trace_buffer.buffer)
4399 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4403 #ifdef CONFIG_TRACER_MAX_TRACE
4404 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4405 !tr->current_trace->use_max_tr)
4408 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4410 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4411 &tr->trace_buffer, cpu);
4414 * AARGH! We are left with different
4415 * size max buffer!!!!
4416 * The max buffer is our "snapshot" buffer.
4417 * When a tracer needs a snapshot (one of the
4418 * latency tracers), it swaps the max buffer
4419 * with the saved snap shot. We succeeded to
4420 * update the size of the main buffer, but failed to
4421 * update the size of the max buffer. But when we tried
4422 * to reset the main buffer to the original size, we
4423 * failed there too. This is very unlikely to
4424 * happen, but if it does, warn and kill all
4428 tracing_disabled = 1;
4433 if (cpu == RING_BUFFER_ALL_CPUS)
4434 set_buffer_entries(&tr->max_buffer, size);
4436 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4439 #endif /* CONFIG_TRACER_MAX_TRACE */
4441 if (cpu == RING_BUFFER_ALL_CPUS)
4442 set_buffer_entries(&tr->trace_buffer, size);
4444 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4449 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4450 unsigned long size, int cpu_id)
4454 mutex_lock(&trace_types_lock);
4456 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4457 /* make sure, this cpu is enabled in the mask */
4458 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4464 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4469 mutex_unlock(&trace_types_lock);
4476 * tracing_update_buffers - used by tracing facility to expand ring buffers
4478 * To save on memory when the tracing is never used on a system with it
4479 * configured in. The ring buffers are set to a minimum size. But once
4480 * a user starts to use the tracing facility, then they need to grow
4481 * to their default size.
4483 * This function is to be called when a tracer is about to be used.
4485 int tracing_update_buffers(void)
4489 mutex_lock(&trace_types_lock);
4490 if (!ring_buffer_expanded)
4491 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4492 RING_BUFFER_ALL_CPUS);
4493 mutex_unlock(&trace_types_lock);
4498 struct trace_option_dentry;
4501 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4504 * Used to clear out the tracer before deletion of an instance.
4505 * Must have trace_types_lock held.
4507 static void tracing_set_nop(struct trace_array *tr)
4509 if (tr->current_trace == &nop_trace)
4512 tr->current_trace->enabled--;
4514 if (tr->current_trace->reset)
4515 tr->current_trace->reset(tr);
4517 tr->current_trace = &nop_trace;
4520 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
4522 /* Only enable if the directory has been created already. */
4526 create_trace_option_files(tr, t);
4529 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4532 #ifdef CONFIG_TRACER_MAX_TRACE
4537 mutex_lock(&trace_types_lock);
4539 if (!ring_buffer_expanded) {
4540 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4541 RING_BUFFER_ALL_CPUS);
4547 for (t = trace_types; t; t = t->next) {
4548 if (strcmp(t->name, buf) == 0)
4555 if (t == tr->current_trace)
4558 /* Some tracers are only allowed for the top level buffer */
4559 if (!trace_ok_for_array(t, tr)) {
4564 /* If trace pipe files are being read, we can't change the tracer */
4565 if (tr->current_trace->ref) {
4570 trace_branch_disable();
4572 tr->current_trace->enabled--;
4574 if (tr->current_trace->reset)
4575 tr->current_trace->reset(tr);
4577 /* Current trace needs to be nop_trace before synchronize_sched */
4578 tr->current_trace = &nop_trace;
4580 #ifdef CONFIG_TRACER_MAX_TRACE
4581 had_max_tr = tr->allocated_snapshot;
4583 if (had_max_tr && !t->use_max_tr) {
4585 * We need to make sure that the update_max_tr sees that
4586 * current_trace changed to nop_trace to keep it from
4587 * swapping the buffers after we resize it.
4588 * The update_max_tr is called from interrupts disabled
4589 * so a synchronized_sched() is sufficient.
4591 synchronize_sched();
4596 #ifdef CONFIG_TRACER_MAX_TRACE
4597 if (t->use_max_tr && !had_max_tr) {
4598 ret = alloc_snapshot(tr);
4605 ret = tracer_init(t, tr);
4610 tr->current_trace = t;
4611 tr->current_trace->enabled++;
4612 trace_branch_enable(tr);
4614 mutex_unlock(&trace_types_lock);
4620 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4621 size_t cnt, loff_t *ppos)
4623 struct trace_array *tr = filp->private_data;
4624 char buf[MAX_TRACER_SIZE+1];
4631 if (cnt > MAX_TRACER_SIZE)
4632 cnt = MAX_TRACER_SIZE;
4634 if (copy_from_user(buf, ubuf, cnt))
4639 /* strip ending whitespace. */
4640 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4643 err = tracing_set_tracer(tr, buf);
4653 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4654 size_t cnt, loff_t *ppos)
4659 r = snprintf(buf, sizeof(buf), "%ld\n",
4660 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4661 if (r > sizeof(buf))
4663 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4667 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4668 size_t cnt, loff_t *ppos)
4673 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4683 tracing_thresh_read(struct file *filp, char __user *ubuf,
4684 size_t cnt, loff_t *ppos)
4686 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4690 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4691 size_t cnt, loff_t *ppos)
4693 struct trace_array *tr = filp->private_data;
4696 mutex_lock(&trace_types_lock);
4697 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4701 if (tr->current_trace->update_thresh) {
4702 ret = tr->current_trace->update_thresh(tr);
4709 mutex_unlock(&trace_types_lock);
4714 #ifdef CONFIG_TRACER_MAX_TRACE
4717 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4718 size_t cnt, loff_t *ppos)
4720 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4724 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4725 size_t cnt, loff_t *ppos)
4727 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4732 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4734 struct trace_array *tr = inode->i_private;
4735 struct trace_iterator *iter;
4738 if (tracing_disabled)
4741 if (trace_array_get(tr) < 0)
4744 mutex_lock(&trace_types_lock);
4746 /* create a buffer to store the information to pass to userspace */
4747 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4750 __trace_array_put(tr);
4754 trace_seq_init(&iter->seq);
4755 iter->trace = tr->current_trace;
4757 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4762 /* trace pipe does not show start of buffer */
4763 cpumask_setall(iter->started);
4765 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4766 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4768 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4769 if (trace_clocks[tr->clock_id].in_ns)
4770 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4773 iter->trace_buffer = &tr->trace_buffer;
4774 iter->cpu_file = tracing_get_cpu(inode);
4775 mutex_init(&iter->mutex);
4776 filp->private_data = iter;
4778 if (iter->trace->pipe_open)
4779 iter->trace->pipe_open(iter);
4781 nonseekable_open(inode, filp);
4783 tr->current_trace->ref++;
4785 mutex_unlock(&trace_types_lock);
4791 __trace_array_put(tr);
4792 mutex_unlock(&trace_types_lock);
4796 static int tracing_release_pipe(struct inode *inode, struct file *file)
4798 struct trace_iterator *iter = file->private_data;
4799 struct trace_array *tr = inode->i_private;
4801 mutex_lock(&trace_types_lock);
4803 tr->current_trace->ref--;
4805 if (iter->trace->pipe_close)
4806 iter->trace->pipe_close(iter);
4808 mutex_unlock(&trace_types_lock);
4810 free_cpumask_var(iter->started);
4811 mutex_destroy(&iter->mutex);
4814 trace_array_put(tr);
4820 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4822 struct trace_array *tr = iter->tr;
4824 /* Iterators are static, they should be filled or empty */
4825 if (trace_buffer_iter(iter, iter->cpu_file))
4826 return POLLIN | POLLRDNORM;
4828 if (tr->trace_flags & TRACE_ITER_BLOCK)
4830 * Always select as readable when in blocking mode
4832 return POLLIN | POLLRDNORM;
4834 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4839 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4841 struct trace_iterator *iter = filp->private_data;
4843 return trace_poll(iter, filp, poll_table);
4846 /* Must be called with iter->mutex held. */
4847 static int tracing_wait_pipe(struct file *filp)
4849 struct trace_iterator *iter = filp->private_data;
4852 while (trace_empty(iter)) {
4854 if ((filp->f_flags & O_NONBLOCK)) {
4859 * We block until we read something and tracing is disabled.
4860 * We still block if tracing is disabled, but we have never
4861 * read anything. This allows a user to cat this file, and
4862 * then enable tracing. But after we have read something,
4863 * we give an EOF when tracing is again disabled.
4865 * iter->pos will be 0 if we haven't read anything.
4867 if (!tracing_is_on() && iter->pos)
4870 mutex_unlock(&iter->mutex);
4872 ret = wait_on_pipe(iter, false);
4874 mutex_lock(&iter->mutex);
4887 tracing_read_pipe(struct file *filp, char __user *ubuf,
4888 size_t cnt, loff_t *ppos)
4890 struct trace_iterator *iter = filp->private_data;
4893 /* return any leftover data */
4894 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4898 trace_seq_init(&iter->seq);
4901 * Avoid more than one consumer on a single file descriptor
4902 * This is just a matter of traces coherency, the ring buffer itself
4905 mutex_lock(&iter->mutex);
4906 if (iter->trace->read) {
4907 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4913 sret = tracing_wait_pipe(filp);
4917 /* stop when tracing is finished */
4918 if (trace_empty(iter)) {
4923 if (cnt >= PAGE_SIZE)
4924 cnt = PAGE_SIZE - 1;
4926 /* reset all but tr, trace, and overruns */
4927 memset(&iter->seq, 0,
4928 sizeof(struct trace_iterator) -
4929 offsetof(struct trace_iterator, seq));
4930 cpumask_clear(iter->started);
4933 trace_event_read_lock();
4934 trace_access_lock(iter->cpu_file);
4935 while (trace_find_next_entry_inc(iter) != NULL) {
4936 enum print_line_t ret;
4937 int save_len = iter->seq.seq.len;
4939 ret = print_trace_line(iter);
4940 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4941 /* don't print partial lines */
4942 iter->seq.seq.len = save_len;
4945 if (ret != TRACE_TYPE_NO_CONSUME)
4946 trace_consume(iter);
4948 if (trace_seq_used(&iter->seq) >= cnt)
4952 * Setting the full flag means we reached the trace_seq buffer
4953 * size and we should leave by partial output condition above.
4954 * One of the trace_seq_* functions is not used properly.
4956 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4959 trace_access_unlock(iter->cpu_file);
4960 trace_event_read_unlock();
4962 /* Now copy what we have to the user */
4963 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4964 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
4965 trace_seq_init(&iter->seq);
4968 * If there was nothing to send to user, in spite of consuming trace
4969 * entries, go back to wait for more entries.
4975 mutex_unlock(&iter->mutex);
4980 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4983 __free_page(spd->pages[idx]);
4986 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4988 .confirm = generic_pipe_buf_confirm,
4989 .release = generic_pipe_buf_release,
4990 .steal = generic_pipe_buf_steal,
4991 .get = generic_pipe_buf_get,
4995 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5001 /* Seq buffer is page-sized, exactly what we need. */
5003 save_len = iter->seq.seq.len;
5004 ret = print_trace_line(iter);
5006 if (trace_seq_has_overflowed(&iter->seq)) {
5007 iter->seq.seq.len = save_len;
5012 * This should not be hit, because it should only
5013 * be set if the iter->seq overflowed. But check it
5014 * anyway to be safe.
5016 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5017 iter->seq.seq.len = save_len;
5021 count = trace_seq_used(&iter->seq) - save_len;
5024 iter->seq.seq.len = save_len;
5028 if (ret != TRACE_TYPE_NO_CONSUME)
5029 trace_consume(iter);
5031 if (!trace_find_next_entry_inc(iter)) {
5041 static ssize_t tracing_splice_read_pipe(struct file *filp,
5043 struct pipe_inode_info *pipe,
5047 struct page *pages_def[PIPE_DEF_BUFFERS];
5048 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5049 struct trace_iterator *iter = filp->private_data;
5050 struct splice_pipe_desc spd = {
5052 .partial = partial_def,
5053 .nr_pages = 0, /* This gets updated below. */
5054 .nr_pages_max = PIPE_DEF_BUFFERS,
5056 .ops = &tracing_pipe_buf_ops,
5057 .spd_release = tracing_spd_release_pipe,
5063 if (splice_grow_spd(pipe, &spd))
5066 mutex_lock(&iter->mutex);
5068 if (iter->trace->splice_read) {
5069 ret = iter->trace->splice_read(iter, filp,
5070 ppos, pipe, len, flags);
5075 ret = tracing_wait_pipe(filp);
5079 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
5084 trace_event_read_lock();
5085 trace_access_lock(iter->cpu_file);
5087 /* Fill as many pages as possible. */
5088 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
5089 spd.pages[i] = alloc_page(GFP_KERNEL);
5093 rem = tracing_fill_pipe_page(rem, iter);
5095 /* Copy the data into the page, so we can start over. */
5096 ret = trace_seq_to_buffer(&iter->seq,
5097 page_address(spd.pages[i]),
5098 trace_seq_used(&iter->seq));
5100 __free_page(spd.pages[i]);
5103 spd.partial[i].offset = 0;
5104 spd.partial[i].len = trace_seq_used(&iter->seq);
5106 trace_seq_init(&iter->seq);
5109 trace_access_unlock(iter->cpu_file);
5110 trace_event_read_unlock();
5111 mutex_unlock(&iter->mutex);
5116 ret = splice_to_pipe(pipe, &spd);
5120 splice_shrink_spd(&spd);
5124 mutex_unlock(&iter->mutex);
5129 tracing_entries_read(struct file *filp, char __user *ubuf,
5130 size_t cnt, loff_t *ppos)
5132 struct inode *inode = file_inode(filp);
5133 struct trace_array *tr = inode->i_private;
5134 int cpu = tracing_get_cpu(inode);
5139 mutex_lock(&trace_types_lock);
5141 if (cpu == RING_BUFFER_ALL_CPUS) {
5142 int cpu, buf_size_same;
5147 /* check if all cpu sizes are same */
5148 for_each_tracing_cpu(cpu) {
5149 /* fill in the size from first enabled cpu */
5151 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5152 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
5158 if (buf_size_same) {
5159 if (!ring_buffer_expanded)
5160 r = sprintf(buf, "%lu (expanded: %lu)\n",
5162 trace_buf_size >> 10);
5164 r = sprintf(buf, "%lu\n", size >> 10);
5166 r = sprintf(buf, "X\n");
5168 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
5170 mutex_unlock(&trace_types_lock);
5172 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5177 tracing_entries_write(struct file *filp, const char __user *ubuf,
5178 size_t cnt, loff_t *ppos)
5180 struct inode *inode = file_inode(filp);
5181 struct trace_array *tr = inode->i_private;
5185 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5189 /* must have at least 1 entry */
5193 /* value is in KB */
5195 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5205 tracing_total_entries_read(struct file *filp, char __user *ubuf,
5206 size_t cnt, loff_t *ppos)
5208 struct trace_array *tr = filp->private_data;
5211 unsigned long size = 0, expanded_size = 0;
5213 mutex_lock(&trace_types_lock);
5214 for_each_tracing_cpu(cpu) {
5215 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5216 if (!ring_buffer_expanded)
5217 expanded_size += trace_buf_size >> 10;
5219 if (ring_buffer_expanded)
5220 r = sprintf(buf, "%lu\n", size);
5222 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5223 mutex_unlock(&trace_types_lock);
5225 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5229 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5230 size_t cnt, loff_t *ppos)
5233 * There is no need to read what the user has written, this function
5234 * is just to make sure that there is no error when "echo" is used
5243 tracing_free_buffer_release(struct inode *inode, struct file *filp)
5245 struct trace_array *tr = inode->i_private;
5247 /* disable tracing ? */
5248 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
5249 tracer_tracing_off(tr);
5250 /* resize the ring buffer to 0 */
5251 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5253 trace_array_put(tr);
5259 tracing_mark_write(struct file *filp, const char __user *ubuf,
5260 size_t cnt, loff_t *fpos)
5262 unsigned long addr = (unsigned long)ubuf;
5263 struct trace_array *tr = filp->private_data;
5264 struct ring_buffer_event *event;
5265 struct ring_buffer *buffer;
5266 struct print_entry *entry;
5267 unsigned long irq_flags;
5268 struct page *pages[2];
5278 if (tracing_disabled)
5281 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5284 if (cnt > TRACE_BUF_SIZE)
5285 cnt = TRACE_BUF_SIZE;
5288 * Userspace is injecting traces into the kernel trace buffer.
5289 * We want to be as non intrusive as possible.
5290 * To do so, we do not want to allocate any special buffers
5291 * or take any locks, but instead write the userspace data
5292 * straight into the ring buffer.
5294 * First we need to pin the userspace buffer into memory,
5295 * which, most likely it is, because it just referenced it.
5296 * But there's no guarantee that it is. By using get_user_pages_fast()
5297 * and kmap_atomic/kunmap_atomic() we can get access to the
5298 * pages directly. We then write the data directly into the
5301 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5303 /* check if we cross pages */
5304 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5307 offset = addr & (PAGE_SIZE - 1);
5310 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5311 if (ret < nr_pages) {
5313 put_page(pages[ret]);
5318 for (i = 0; i < nr_pages; i++)
5319 map_page[i] = kmap_atomic(pages[i]);
5321 local_save_flags(irq_flags);
5322 size = sizeof(*entry) + cnt + 2; /* possible \n added */
5323 buffer = tr->trace_buffer.buffer;
5324 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5325 irq_flags, preempt_count());
5327 /* Ring buffer disabled, return as if not open for write */
5332 entry = ring_buffer_event_data(event);
5333 entry->ip = _THIS_IP_;
5335 if (nr_pages == 2) {
5336 len = PAGE_SIZE - offset;
5337 memcpy(&entry->buf, map_page[0] + offset, len);
5338 memcpy(&entry->buf[len], map_page[1], cnt - len);
5340 memcpy(&entry->buf, map_page[0] + offset, cnt);
5342 if (entry->buf[cnt - 1] != '\n') {
5343 entry->buf[cnt] = '\n';
5344 entry->buf[cnt + 1] = '\0';
5346 entry->buf[cnt] = '\0';
5348 __buffer_unlock_commit(buffer, event);
5355 for (i = nr_pages - 1; i >= 0; i--) {
5356 kunmap_atomic(map_page[i]);
5363 static int tracing_clock_show(struct seq_file *m, void *v)
5365 struct trace_array *tr = m->private;
5368 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5370 "%s%s%s%s", i ? " " : "",
5371 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5372 i == tr->clock_id ? "]" : "");
5378 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5382 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5383 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5386 if (i == ARRAY_SIZE(trace_clocks))
5389 mutex_lock(&trace_types_lock);
5393 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5396 * New clock may not be consistent with the previous clock.
5397 * Reset the buffer so that it doesn't have incomparable timestamps.
5399 tracing_reset_online_cpus(&tr->trace_buffer);
5401 #ifdef CONFIG_TRACER_MAX_TRACE
5402 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5403 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5404 tracing_reset_online_cpus(&tr->max_buffer);
5407 mutex_unlock(&trace_types_lock);
5412 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5413 size_t cnt, loff_t *fpos)
5415 struct seq_file *m = filp->private_data;
5416 struct trace_array *tr = m->private;
5418 const char *clockstr;
5421 if (cnt >= sizeof(buf))
5424 if (copy_from_user(buf, ubuf, cnt))
5429 clockstr = strstrip(buf);
5431 ret = tracing_set_clock(tr, clockstr);
5440 static int tracing_clock_open(struct inode *inode, struct file *file)
5442 struct trace_array *tr = inode->i_private;
5445 if (tracing_disabled)
5448 if (trace_array_get(tr))
5451 ret = single_open(file, tracing_clock_show, inode->i_private);
5453 trace_array_put(tr);
5458 struct ftrace_buffer_info {
5459 struct trace_iterator iter;
5464 #ifdef CONFIG_TRACER_SNAPSHOT
5465 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5467 struct trace_array *tr = inode->i_private;
5468 struct trace_iterator *iter;
5472 if (trace_array_get(tr) < 0)
5475 if (file->f_mode & FMODE_READ) {
5476 iter = __tracing_open(inode, file, true);
5478 ret = PTR_ERR(iter);
5480 /* Writes still need the seq_file to hold the private data */
5482 m = kzalloc(sizeof(*m), GFP_KERNEL);
5485 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5493 iter->trace_buffer = &tr->max_buffer;
5494 iter->cpu_file = tracing_get_cpu(inode);
5496 file->private_data = m;
5500 trace_array_put(tr);
5506 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5509 struct seq_file *m = filp->private_data;
5510 struct trace_iterator *iter = m->private;
5511 struct trace_array *tr = iter->tr;
5515 ret = tracing_update_buffers();
5519 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5523 mutex_lock(&trace_types_lock);
5525 if (tr->current_trace->use_max_tr) {
5532 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5536 if (tr->allocated_snapshot)
5540 /* Only allow per-cpu swap if the ring buffer supports it */
5541 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5542 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5547 if (!tr->allocated_snapshot) {
5548 ret = alloc_snapshot(tr);
5552 local_irq_disable();
5553 /* Now, we're going to swap */
5554 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5555 update_max_tr(tr, current, smp_processor_id());
5557 update_max_tr_single(tr, current, iter->cpu_file);
5561 if (tr->allocated_snapshot) {
5562 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5563 tracing_reset_online_cpus(&tr->max_buffer);
5565 tracing_reset(&tr->max_buffer, iter->cpu_file);
5575 mutex_unlock(&trace_types_lock);
5579 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5581 struct seq_file *m = file->private_data;
5584 ret = tracing_release(inode, file);
5586 if (file->f_mode & FMODE_READ)
5589 /* If write only, the seq_file is just a stub */
5597 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5598 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5599 size_t count, loff_t *ppos);
5600 static int tracing_buffers_release(struct inode *inode, struct file *file);
5601 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5602 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5604 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5606 struct ftrace_buffer_info *info;
5609 ret = tracing_buffers_open(inode, filp);
5613 info = filp->private_data;
5615 if (info->iter.trace->use_max_tr) {
5616 tracing_buffers_release(inode, filp);
5620 info->iter.snapshot = true;
5621 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5626 #endif /* CONFIG_TRACER_SNAPSHOT */
5629 static const struct file_operations tracing_thresh_fops = {
5630 .open = tracing_open_generic,
5631 .read = tracing_thresh_read,
5632 .write = tracing_thresh_write,
5633 .llseek = generic_file_llseek,
5636 #ifdef CONFIG_TRACER_MAX_TRACE
5637 static const struct file_operations tracing_max_lat_fops = {
5638 .open = tracing_open_generic,
5639 .read = tracing_max_lat_read,
5640 .write = tracing_max_lat_write,
5641 .llseek = generic_file_llseek,
5645 static const struct file_operations set_tracer_fops = {
5646 .open = tracing_open_generic,
5647 .read = tracing_set_trace_read,
5648 .write = tracing_set_trace_write,
5649 .llseek = generic_file_llseek,
5652 static const struct file_operations tracing_pipe_fops = {
5653 .open = tracing_open_pipe,
5654 .poll = tracing_poll_pipe,
5655 .read = tracing_read_pipe,
5656 .splice_read = tracing_splice_read_pipe,
5657 .release = tracing_release_pipe,
5658 .llseek = no_llseek,
5661 static const struct file_operations tracing_entries_fops = {
5662 .open = tracing_open_generic_tr,
5663 .read = tracing_entries_read,
5664 .write = tracing_entries_write,
5665 .llseek = generic_file_llseek,
5666 .release = tracing_release_generic_tr,
5669 static const struct file_operations tracing_total_entries_fops = {
5670 .open = tracing_open_generic_tr,
5671 .read = tracing_total_entries_read,
5672 .llseek = generic_file_llseek,
5673 .release = tracing_release_generic_tr,
5676 static const struct file_operations tracing_free_buffer_fops = {
5677 .open = tracing_open_generic_tr,
5678 .write = tracing_free_buffer_write,
5679 .release = tracing_free_buffer_release,
5682 static const struct file_operations tracing_mark_fops = {
5683 .open = tracing_open_generic_tr,
5684 .write = tracing_mark_write,
5685 .llseek = generic_file_llseek,
5686 .release = tracing_release_generic_tr,
5689 static const struct file_operations trace_clock_fops = {
5690 .open = tracing_clock_open,
5692 .llseek = seq_lseek,
5693 .release = tracing_single_release_tr,
5694 .write = tracing_clock_write,
5697 #ifdef CONFIG_TRACER_SNAPSHOT
5698 static const struct file_operations snapshot_fops = {
5699 .open = tracing_snapshot_open,
5701 .write = tracing_snapshot_write,
5702 .llseek = tracing_lseek,
5703 .release = tracing_snapshot_release,
5706 static const struct file_operations snapshot_raw_fops = {
5707 .open = snapshot_raw_open,
5708 .read = tracing_buffers_read,
5709 .release = tracing_buffers_release,
5710 .splice_read = tracing_buffers_splice_read,
5711 .llseek = no_llseek,
5714 #endif /* CONFIG_TRACER_SNAPSHOT */
5716 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5718 struct trace_array *tr = inode->i_private;
5719 struct ftrace_buffer_info *info;
5722 if (tracing_disabled)
5725 if (trace_array_get(tr) < 0)
5728 info = kzalloc(sizeof(*info), GFP_KERNEL);
5730 trace_array_put(tr);
5734 mutex_lock(&trace_types_lock);
5737 info->iter.cpu_file = tracing_get_cpu(inode);
5738 info->iter.trace = tr->current_trace;
5739 info->iter.trace_buffer = &tr->trace_buffer;
5741 /* Force reading ring buffer for first read */
5742 info->read = (unsigned int)-1;
5744 filp->private_data = info;
5746 tr->current_trace->ref++;
5748 mutex_unlock(&trace_types_lock);
5750 ret = nonseekable_open(inode, filp);
5752 trace_array_put(tr);
5758 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5760 struct ftrace_buffer_info *info = filp->private_data;
5761 struct trace_iterator *iter = &info->iter;
5763 return trace_poll(iter, filp, poll_table);
5767 tracing_buffers_read(struct file *filp, char __user *ubuf,
5768 size_t count, loff_t *ppos)
5770 struct ftrace_buffer_info *info = filp->private_data;
5771 struct trace_iterator *iter = &info->iter;
5778 #ifdef CONFIG_TRACER_MAX_TRACE
5779 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5784 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5789 /* Do we have previous read data to read? */
5790 if (info->read < PAGE_SIZE)
5794 trace_access_lock(iter->cpu_file);
5795 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5799 trace_access_unlock(iter->cpu_file);
5802 if (trace_empty(iter)) {
5803 if ((filp->f_flags & O_NONBLOCK))
5806 ret = wait_on_pipe(iter, false);
5817 size = PAGE_SIZE - info->read;
5821 ret = copy_to_user(ubuf, info->spare + info->read, size);
5833 static int tracing_buffers_release(struct inode *inode, struct file *file)
5835 struct ftrace_buffer_info *info = file->private_data;
5836 struct trace_iterator *iter = &info->iter;
5838 mutex_lock(&trace_types_lock);
5840 iter->tr->current_trace->ref--;
5842 __trace_array_put(iter->tr);
5845 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5848 mutex_unlock(&trace_types_lock);
5854 struct ring_buffer *buffer;
5859 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5860 struct pipe_buffer *buf)
5862 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5867 ring_buffer_free_read_page(ref->buffer, ref->page);
5872 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5873 struct pipe_buffer *buf)
5875 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5880 /* Pipe buffer operations for a buffer. */
5881 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5883 .confirm = generic_pipe_buf_confirm,
5884 .release = buffer_pipe_buf_release,
5885 .steal = generic_pipe_buf_steal,
5886 .get = buffer_pipe_buf_get,
5890 * Callback from splice_to_pipe(), if we need to release some pages
5891 * at the end of the spd in case we error'ed out in filling the pipe.
5893 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5895 struct buffer_ref *ref =
5896 (struct buffer_ref *)spd->partial[i].private;
5901 ring_buffer_free_read_page(ref->buffer, ref->page);
5903 spd->partial[i].private = 0;
5907 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5908 struct pipe_inode_info *pipe, size_t len,
5911 struct ftrace_buffer_info *info = file->private_data;
5912 struct trace_iterator *iter = &info->iter;
5913 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5914 struct page *pages_def[PIPE_DEF_BUFFERS];
5915 struct splice_pipe_desc spd = {
5917 .partial = partial_def,
5918 .nr_pages_max = PIPE_DEF_BUFFERS,
5920 .ops = &buffer_pipe_buf_ops,
5921 .spd_release = buffer_spd_release,
5923 struct buffer_ref *ref;
5924 int entries, size, i;
5927 #ifdef CONFIG_TRACER_MAX_TRACE
5928 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5932 if (splice_grow_spd(pipe, &spd))
5935 if (*ppos & (PAGE_SIZE - 1))
5938 if (len & (PAGE_SIZE - 1)) {
5939 if (len < PAGE_SIZE)
5945 trace_access_lock(iter->cpu_file);
5946 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5948 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5952 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5959 ref->buffer = iter->trace_buffer->buffer;
5960 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5967 r = ring_buffer_read_page(ref->buffer, &ref->page,
5968 len, iter->cpu_file, 1);
5970 ring_buffer_free_read_page(ref->buffer, ref->page);
5976 * zero out any left over data, this is going to
5979 size = ring_buffer_page_len(ref->page);
5980 if (size < PAGE_SIZE)
5981 memset(ref->page + size, 0, PAGE_SIZE - size);
5983 page = virt_to_page(ref->page);
5985 spd.pages[i] = page;
5986 spd.partial[i].len = PAGE_SIZE;
5987 spd.partial[i].offset = 0;
5988 spd.partial[i].private = (unsigned long)ref;
5992 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5995 trace_access_unlock(iter->cpu_file);
5998 /* did we read anything? */
5999 if (!spd.nr_pages) {
6003 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
6006 ret = wait_on_pipe(iter, true);
6013 ret = splice_to_pipe(pipe, &spd);
6014 splice_shrink_spd(&spd);
6019 static const struct file_operations tracing_buffers_fops = {
6020 .open = tracing_buffers_open,
6021 .read = tracing_buffers_read,
6022 .poll = tracing_buffers_poll,
6023 .release = tracing_buffers_release,
6024 .splice_read = tracing_buffers_splice_read,
6025 .llseek = no_llseek,
6029 tracing_stats_read(struct file *filp, char __user *ubuf,
6030 size_t count, loff_t *ppos)
6032 struct inode *inode = file_inode(filp);
6033 struct trace_array *tr = inode->i_private;
6034 struct trace_buffer *trace_buf = &tr->trace_buffer;
6035 int cpu = tracing_get_cpu(inode);
6036 struct trace_seq *s;
6038 unsigned long long t;
6039 unsigned long usec_rem;
6041 s = kmalloc(sizeof(*s), GFP_KERNEL);
6047 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
6048 trace_seq_printf(s, "entries: %ld\n", cnt);
6050 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
6051 trace_seq_printf(s, "overrun: %ld\n", cnt);
6053 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
6054 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
6056 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
6057 trace_seq_printf(s, "bytes: %ld\n", cnt);
6059 if (trace_clocks[tr->clock_id].in_ns) {
6060 /* local or global for trace_clock */
6061 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6062 usec_rem = do_div(t, USEC_PER_SEC);
6063 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
6066 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
6067 usec_rem = do_div(t, USEC_PER_SEC);
6068 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
6070 /* counter or tsc mode for trace_clock */
6071 trace_seq_printf(s, "oldest event ts: %llu\n",
6072 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6074 trace_seq_printf(s, "now ts: %llu\n",
6075 ring_buffer_time_stamp(trace_buf->buffer, cpu));
6078 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
6079 trace_seq_printf(s, "dropped events: %ld\n", cnt);
6081 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
6082 trace_seq_printf(s, "read events: %ld\n", cnt);
6084 count = simple_read_from_buffer(ubuf, count, ppos,
6085 s->buffer, trace_seq_used(s));
6092 static const struct file_operations tracing_stats_fops = {
6093 .open = tracing_open_generic_tr,
6094 .read = tracing_stats_read,
6095 .llseek = generic_file_llseek,
6096 .release = tracing_release_generic_tr,
6099 #ifdef CONFIG_DYNAMIC_FTRACE
6101 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
6107 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
6108 size_t cnt, loff_t *ppos)
6110 static char ftrace_dyn_info_buffer[1024];
6111 static DEFINE_MUTEX(dyn_info_mutex);
6112 unsigned long *p = filp->private_data;
6113 char *buf = ftrace_dyn_info_buffer;
6114 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
6117 mutex_lock(&dyn_info_mutex);
6118 r = sprintf(buf, "%ld ", *p);
6120 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
6123 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6125 mutex_unlock(&dyn_info_mutex);
6130 static const struct file_operations tracing_dyn_info_fops = {
6131 .open = tracing_open_generic,
6132 .read = tracing_read_dyn_info,
6133 .llseek = generic_file_llseek,
6135 #endif /* CONFIG_DYNAMIC_FTRACE */
6137 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6139 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6145 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6147 unsigned long *count = (long *)data;
6159 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
6160 struct ftrace_probe_ops *ops, void *data)
6162 long count = (long)data;
6164 seq_printf(m, "%ps:", (void *)ip);
6166 seq_puts(m, "snapshot");
6169 seq_puts(m, ":unlimited\n");
6171 seq_printf(m, ":count=%ld\n", count);
6176 static struct ftrace_probe_ops snapshot_probe_ops = {
6177 .func = ftrace_snapshot,
6178 .print = ftrace_snapshot_print,
6181 static struct ftrace_probe_ops snapshot_count_probe_ops = {
6182 .func = ftrace_count_snapshot,
6183 .print = ftrace_snapshot_print,
6187 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6188 char *glob, char *cmd, char *param, int enable)
6190 struct ftrace_probe_ops *ops;
6191 void *count = (void *)-1;
6195 /* hash funcs only work with set_ftrace_filter */
6199 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6201 if (glob[0] == '!') {
6202 unregister_ftrace_function_probe_func(glob+1, ops);
6209 number = strsep(¶m, ":");
6211 if (!strlen(number))
6215 * We use the callback data field (which is a pointer)
6218 ret = kstrtoul(number, 0, (unsigned long *)&count);
6223 ret = register_ftrace_function_probe(glob, ops, count);
6226 alloc_snapshot(&global_trace);
6228 return ret < 0 ? ret : 0;
6231 static struct ftrace_func_command ftrace_snapshot_cmd = {
6233 .func = ftrace_trace_snapshot_callback,
6236 static __init int register_snapshot_cmd(void)
6238 return register_ftrace_command(&ftrace_snapshot_cmd);
6241 static inline __init int register_snapshot_cmd(void) { return 0; }
6242 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6244 static struct dentry *tracing_get_dentry(struct trace_array *tr)
6246 if (WARN_ON(!tr->dir))
6247 return ERR_PTR(-ENODEV);
6249 /* Top directory uses NULL as the parent */
6250 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6253 /* All sub buffers have a descriptor */
6257 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6259 struct dentry *d_tracer;
6262 return tr->percpu_dir;
6264 d_tracer = tracing_get_dentry(tr);
6265 if (IS_ERR(d_tracer))
6268 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6270 WARN_ONCE(!tr->percpu_dir,
6271 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6273 return tr->percpu_dir;
6276 static struct dentry *
6277 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6278 void *data, long cpu, const struct file_operations *fops)
6280 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6282 if (ret) /* See tracing_get_cpu() */
6283 d_inode(ret)->i_cdev = (void *)(cpu + 1);
6288 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6290 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6291 struct dentry *d_cpu;
6292 char cpu_dir[30]; /* 30 characters should be more than enough */
6297 snprintf(cpu_dir, 30, "cpu%ld", cpu);
6298 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6300 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
6304 /* per cpu trace_pipe */
6305 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6306 tr, cpu, &tracing_pipe_fops);
6309 trace_create_cpu_file("trace", 0644, d_cpu,
6310 tr, cpu, &tracing_fops);
6312 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6313 tr, cpu, &tracing_buffers_fops);
6315 trace_create_cpu_file("stats", 0444, d_cpu,
6316 tr, cpu, &tracing_stats_fops);
6318 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6319 tr, cpu, &tracing_entries_fops);
6321 #ifdef CONFIG_TRACER_SNAPSHOT
6322 trace_create_cpu_file("snapshot", 0644, d_cpu,
6323 tr, cpu, &snapshot_fops);
6325 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6326 tr, cpu, &snapshot_raw_fops);
6330 #ifdef CONFIG_FTRACE_SELFTEST
6331 /* Let selftest have access to static functions in this file */
6332 #include "trace_selftest.c"
6336 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6339 struct trace_option_dentry *topt = filp->private_data;
6342 if (topt->flags->val & topt->opt->bit)
6347 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6351 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6354 struct trace_option_dentry *topt = filp->private_data;
6358 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6362 if (val != 0 && val != 1)
6365 if (!!(topt->flags->val & topt->opt->bit) != val) {
6366 mutex_lock(&trace_types_lock);
6367 ret = __set_tracer_option(topt->tr, topt->flags,
6369 mutex_unlock(&trace_types_lock);
6380 static const struct file_operations trace_options_fops = {
6381 .open = tracing_open_generic,
6382 .read = trace_options_read,
6383 .write = trace_options_write,
6384 .llseek = generic_file_llseek,
6388 * In order to pass in both the trace_array descriptor as well as the index
6389 * to the flag that the trace option file represents, the trace_array
6390 * has a character array of trace_flags_index[], which holds the index
6391 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6392 * The address of this character array is passed to the flag option file
6393 * read/write callbacks.
6395 * In order to extract both the index and the trace_array descriptor,
6396 * get_tr_index() uses the following algorithm.
6400 * As the pointer itself contains the address of the index (remember
6403 * Then to get the trace_array descriptor, by subtracting that index
6404 * from the ptr, we get to the start of the index itself.
6406 * ptr - idx == &index[0]
6408 * Then a simple container_of() from that pointer gets us to the
6409 * trace_array descriptor.
6411 static void get_tr_index(void *data, struct trace_array **ptr,
6412 unsigned int *pindex)
6414 *pindex = *(unsigned char *)data;
6416 *ptr = container_of(data - *pindex, struct trace_array,
6421 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6424 void *tr_index = filp->private_data;
6425 struct trace_array *tr;
6429 get_tr_index(tr_index, &tr, &index);
6431 if (tr->trace_flags & (1 << index))
6436 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6440 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6443 void *tr_index = filp->private_data;
6444 struct trace_array *tr;
6449 get_tr_index(tr_index, &tr, &index);
6451 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6455 if (val != 0 && val != 1)
6458 mutex_lock(&trace_types_lock);
6459 ret = set_tracer_flag(tr, 1 << index, val);
6460 mutex_unlock(&trace_types_lock);
6470 static const struct file_operations trace_options_core_fops = {
6471 .open = tracing_open_generic,
6472 .read = trace_options_core_read,
6473 .write = trace_options_core_write,
6474 .llseek = generic_file_llseek,
6477 struct dentry *trace_create_file(const char *name,
6479 struct dentry *parent,
6481 const struct file_operations *fops)
6485 ret = tracefs_create_file(name, mode, parent, data, fops);
6487 pr_warn("Could not create tracefs '%s' entry\n", name);
6493 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6495 struct dentry *d_tracer;
6500 d_tracer = tracing_get_dentry(tr);
6501 if (IS_ERR(d_tracer))
6504 tr->options = tracefs_create_dir("options", d_tracer);
6506 pr_warn("Could not create tracefs directory 'options'\n");
6514 create_trace_option_file(struct trace_array *tr,
6515 struct trace_option_dentry *topt,
6516 struct tracer_flags *flags,
6517 struct tracer_opt *opt)
6519 struct dentry *t_options;
6521 t_options = trace_options_init_dentry(tr);
6525 topt->flags = flags;
6529 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6530 &trace_options_fops);
6535 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6537 struct trace_option_dentry *topts;
6538 struct trace_options *tr_topts;
6539 struct tracer_flags *flags;
6540 struct tracer_opt *opts;
6547 flags = tracer->flags;
6549 if (!flags || !flags->opts)
6553 * If this is an instance, only create flags for tracers
6554 * the instance may have.
6556 if (!trace_ok_for_array(tracer, tr))
6559 for (i = 0; i < tr->nr_topts; i++) {
6560 /* Make sure there's no duplicate flags. */
6561 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
6567 for (cnt = 0; opts[cnt].name; cnt++)
6570 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6574 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
6581 tr->topts = tr_topts;
6582 tr->topts[tr->nr_topts].tracer = tracer;
6583 tr->topts[tr->nr_topts].topts = topts;
6586 for (cnt = 0; opts[cnt].name; cnt++) {
6587 create_trace_option_file(tr, &topts[cnt], flags,
6589 WARN_ONCE(topts[cnt].entry == NULL,
6590 "Failed to create trace option: %s",
6595 static struct dentry *
6596 create_trace_option_core_file(struct trace_array *tr,
6597 const char *option, long index)
6599 struct dentry *t_options;
6601 t_options = trace_options_init_dentry(tr);
6605 return trace_create_file(option, 0644, t_options,
6606 (void *)&tr->trace_flags_index[index],
6607 &trace_options_core_fops);
6610 static void create_trace_options_dir(struct trace_array *tr)
6612 struct dentry *t_options;
6613 bool top_level = tr == &global_trace;
6616 t_options = trace_options_init_dentry(tr);
6620 for (i = 0; trace_options[i]; i++) {
6622 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6623 create_trace_option_core_file(tr, trace_options[i], i);
6628 rb_simple_read(struct file *filp, char __user *ubuf,
6629 size_t cnt, loff_t *ppos)
6631 struct trace_array *tr = filp->private_data;
6635 r = tracer_tracing_is_on(tr);
6636 r = sprintf(buf, "%d\n", r);
6638 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6642 rb_simple_write(struct file *filp, const char __user *ubuf,
6643 size_t cnt, loff_t *ppos)
6645 struct trace_array *tr = filp->private_data;
6646 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6650 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6655 mutex_lock(&trace_types_lock);
6657 tracer_tracing_on(tr);
6658 if (tr->current_trace->start)
6659 tr->current_trace->start(tr);
6661 tracer_tracing_off(tr);
6662 if (tr->current_trace->stop)
6663 tr->current_trace->stop(tr);
6665 mutex_unlock(&trace_types_lock);
6673 static const struct file_operations rb_simple_fops = {
6674 .open = tracing_open_generic_tr,
6675 .read = rb_simple_read,
6676 .write = rb_simple_write,
6677 .release = tracing_release_generic_tr,
6678 .llseek = default_llseek,
6681 struct dentry *trace_instance_dir;
6684 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
6687 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6689 enum ring_buffer_flags rb_flags;
6691 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6695 buf->buffer = ring_buffer_alloc(size, rb_flags);
6699 buf->data = alloc_percpu(struct trace_array_cpu);
6701 ring_buffer_free(buf->buffer);
6705 /* Allocate the first page for all buffers */
6706 set_buffer_entries(&tr->trace_buffer,
6707 ring_buffer_size(tr->trace_buffer.buffer, 0));
6712 static int allocate_trace_buffers(struct trace_array *tr, int size)
6716 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6720 #ifdef CONFIG_TRACER_MAX_TRACE
6721 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6722 allocate_snapshot ? size : 1);
6724 ring_buffer_free(tr->trace_buffer.buffer);
6725 free_percpu(tr->trace_buffer.data);
6728 tr->allocated_snapshot = allocate_snapshot;
6731 * Only the top level trace array gets its snapshot allocated
6732 * from the kernel command line.
6734 allocate_snapshot = false;
6739 static void free_trace_buffer(struct trace_buffer *buf)
6742 ring_buffer_free(buf->buffer);
6744 free_percpu(buf->data);
6749 static void free_trace_buffers(struct trace_array *tr)
6754 free_trace_buffer(&tr->trace_buffer);
6756 #ifdef CONFIG_TRACER_MAX_TRACE
6757 free_trace_buffer(&tr->max_buffer);
6761 static void init_trace_flags_index(struct trace_array *tr)
6765 /* Used by the trace options files */
6766 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
6767 tr->trace_flags_index[i] = i;
6770 static void __update_tracer_options(struct trace_array *tr)
6774 for (t = trace_types; t; t = t->next)
6775 add_tracer_options(tr, t);
6778 static void update_tracer_options(struct trace_array *tr)
6780 mutex_lock(&trace_types_lock);
6781 __update_tracer_options(tr);
6782 mutex_unlock(&trace_types_lock);
6785 static int instance_mkdir(const char *name)
6787 struct trace_array *tr;
6790 mutex_lock(&trace_types_lock);
6793 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6794 if (tr->name && strcmp(tr->name, name) == 0)
6799 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6803 tr->name = kstrdup(name, GFP_KERNEL);
6807 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6810 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
6812 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6814 raw_spin_lock_init(&tr->start_lock);
6816 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6818 tr->current_trace = &nop_trace;
6820 INIT_LIST_HEAD(&tr->systems);
6821 INIT_LIST_HEAD(&tr->events);
6823 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6826 tr->dir = tracefs_create_dir(name, trace_instance_dir);
6830 ret = event_trace_add_tracer(tr->dir, tr);
6832 tracefs_remove_recursive(tr->dir);
6836 init_tracer_tracefs(tr, tr->dir);
6837 init_trace_flags_index(tr);
6838 __update_tracer_options(tr);
6840 list_add(&tr->list, &ftrace_trace_arrays);
6842 mutex_unlock(&trace_types_lock);
6847 free_trace_buffers(tr);
6848 free_cpumask_var(tr->tracing_cpumask);
6853 mutex_unlock(&trace_types_lock);
6859 static int instance_rmdir(const char *name)
6861 struct trace_array *tr;
6866 mutex_lock(&trace_types_lock);
6869 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6870 if (tr->name && strcmp(tr->name, name) == 0) {
6879 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
6882 list_del(&tr->list);
6884 /* Disable all the flags that were enabled coming in */
6885 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
6886 if ((1 << i) & ZEROED_TRACE_FLAGS)
6887 set_tracer_flag(tr, 1 << i, 0);
6890 tracing_set_nop(tr);
6891 event_trace_del_tracer(tr);
6892 ftrace_destroy_function_files(tr);
6893 tracefs_remove_recursive(tr->dir);
6894 free_trace_buffers(tr);
6896 for (i = 0; i < tr->nr_topts; i++) {
6897 kfree(tr->topts[i].topts);
6907 mutex_unlock(&trace_types_lock);
6912 static __init void create_trace_instances(struct dentry *d_tracer)
6914 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6917 if (WARN_ON(!trace_instance_dir))
6922 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6926 trace_create_file("available_tracers", 0444, d_tracer,
6927 tr, &show_traces_fops);
6929 trace_create_file("current_tracer", 0644, d_tracer,
6930 tr, &set_tracer_fops);
6932 trace_create_file("tracing_cpumask", 0644, d_tracer,
6933 tr, &tracing_cpumask_fops);
6935 trace_create_file("trace_options", 0644, d_tracer,
6936 tr, &tracing_iter_fops);
6938 trace_create_file("trace", 0644, d_tracer,
6941 trace_create_file("trace_pipe", 0444, d_tracer,
6942 tr, &tracing_pipe_fops);
6944 trace_create_file("buffer_size_kb", 0644, d_tracer,
6945 tr, &tracing_entries_fops);
6947 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6948 tr, &tracing_total_entries_fops);
6950 trace_create_file("free_buffer", 0200, d_tracer,
6951 tr, &tracing_free_buffer_fops);
6953 trace_create_file("trace_marker", 0220, d_tracer,
6954 tr, &tracing_mark_fops);
6956 trace_create_file("trace_clock", 0644, d_tracer, tr,
6959 trace_create_file("tracing_on", 0644, d_tracer,
6960 tr, &rb_simple_fops);
6962 create_trace_options_dir(tr);
6964 #ifdef CONFIG_TRACER_MAX_TRACE
6965 trace_create_file("tracing_max_latency", 0644, d_tracer,
6966 &tr->max_latency, &tracing_max_lat_fops);
6969 if (ftrace_create_function_files(tr, d_tracer))
6970 WARN(1, "Could not allocate function filter files");
6972 #ifdef CONFIG_TRACER_SNAPSHOT
6973 trace_create_file("snapshot", 0644, d_tracer,
6974 tr, &snapshot_fops);
6977 for_each_tracing_cpu(cpu)
6978 tracing_init_tracefs_percpu(tr, cpu);
6982 static struct vfsmount *trace_automount(void *ingore)
6984 struct vfsmount *mnt;
6985 struct file_system_type *type;
6988 * To maintain backward compatibility for tools that mount
6989 * debugfs to get to the tracing facility, tracefs is automatically
6990 * mounted to the debugfs/tracing directory.
6992 type = get_fs_type("tracefs");
6995 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6996 put_filesystem(type);
7005 * tracing_init_dentry - initialize top level trace array
7007 * This is called when creating files or directories in the tracing
7008 * directory. It is called via fs_initcall() by any of the boot up code
7009 * and expects to return the dentry of the top level tracing directory.
7011 struct dentry *tracing_init_dentry(void)
7013 struct trace_array *tr = &global_trace;
7015 /* The top level trace array uses NULL as parent */
7019 if (WARN_ON(!tracefs_initialized()) ||
7020 (IS_ENABLED(CONFIG_DEBUG_FS) &&
7021 WARN_ON(!debugfs_initialized())))
7022 return ERR_PTR(-ENODEV);
7025 * As there may still be users that expect the tracing
7026 * files to exist in debugfs/tracing, we must automount
7027 * the tracefs file system there, so older tools still
7028 * work with the newer kerenl.
7030 tr->dir = debugfs_create_automount("tracing", NULL,
7031 trace_automount, NULL);
7033 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7034 return ERR_PTR(-ENOMEM);
7040 extern struct trace_enum_map *__start_ftrace_enum_maps[];
7041 extern struct trace_enum_map *__stop_ftrace_enum_maps[];
7043 static void __init trace_enum_init(void)
7047 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
7048 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
7051 #ifdef CONFIG_MODULES
7052 static void trace_module_add_enums(struct module *mod)
7054 if (!mod->num_trace_enums)
7058 * Modules with bad taint do not have events created, do
7059 * not bother with enums either.
7061 if (trace_module_has_bad_taint(mod))
7064 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
7067 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
7068 static void trace_module_remove_enums(struct module *mod)
7070 union trace_enum_map_item *map;
7071 union trace_enum_map_item **last = &trace_enum_maps;
7073 if (!mod->num_trace_enums)
7076 mutex_lock(&trace_enum_mutex);
7078 map = trace_enum_maps;
7081 if (map->head.mod == mod)
7083 map = trace_enum_jmp_to_tail(map);
7084 last = &map->tail.next;
7085 map = map->tail.next;
7090 *last = trace_enum_jmp_to_tail(map)->tail.next;
7093 mutex_unlock(&trace_enum_mutex);
7096 static inline void trace_module_remove_enums(struct module *mod) { }
7097 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
7099 static int trace_module_notify(struct notifier_block *self,
7100 unsigned long val, void *data)
7102 struct module *mod = data;
7105 case MODULE_STATE_COMING:
7106 trace_module_add_enums(mod);
7108 case MODULE_STATE_GOING:
7109 trace_module_remove_enums(mod);
7116 static struct notifier_block trace_module_nb = {
7117 .notifier_call = trace_module_notify,
7120 #endif /* CONFIG_MODULES */
7122 static __init int tracer_init_tracefs(void)
7124 struct dentry *d_tracer;
7126 trace_access_lock_init();
7128 d_tracer = tracing_init_dentry();
7129 if (IS_ERR(d_tracer))
7132 init_tracer_tracefs(&global_trace, d_tracer);
7134 trace_create_file("tracing_thresh", 0644, d_tracer,
7135 &global_trace, &tracing_thresh_fops);
7137 trace_create_file("README", 0444, d_tracer,
7138 NULL, &tracing_readme_fops);
7140 trace_create_file("saved_cmdlines", 0444, d_tracer,
7141 NULL, &tracing_saved_cmdlines_fops);
7143 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
7144 NULL, &tracing_saved_cmdlines_size_fops);
7148 trace_create_enum_file(d_tracer);
7150 #ifdef CONFIG_MODULES
7151 register_module_notifier(&trace_module_nb);
7154 #ifdef CONFIG_DYNAMIC_FTRACE
7155 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
7156 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
7159 create_trace_instances(d_tracer);
7161 update_tracer_options(&global_trace);
7166 static int trace_panic_handler(struct notifier_block *this,
7167 unsigned long event, void *unused)
7169 if (ftrace_dump_on_oops)
7170 ftrace_dump(ftrace_dump_on_oops);
7174 static struct notifier_block trace_panic_notifier = {
7175 .notifier_call = trace_panic_handler,
7177 .priority = 150 /* priority: INT_MAX >= x >= 0 */
7180 static int trace_die_handler(struct notifier_block *self,
7186 if (ftrace_dump_on_oops)
7187 ftrace_dump(ftrace_dump_on_oops);
7195 static struct notifier_block trace_die_notifier = {
7196 .notifier_call = trace_die_handler,
7201 * printk is set to max of 1024, we really don't need it that big.
7202 * Nothing should be printing 1000 characters anyway.
7204 #define TRACE_MAX_PRINT 1000
7207 * Define here KERN_TRACE so that we have one place to modify
7208 * it if we decide to change what log level the ftrace dump
7211 #define KERN_TRACE KERN_EMERG
7214 trace_printk_seq(struct trace_seq *s)
7216 /* Probably should print a warning here. */
7217 if (s->seq.len >= TRACE_MAX_PRINT)
7218 s->seq.len = TRACE_MAX_PRINT;
7221 * More paranoid code. Although the buffer size is set to
7222 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7223 * an extra layer of protection.
7225 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7226 s->seq.len = s->seq.size - 1;
7228 /* should be zero ended, but we are paranoid. */
7229 s->buffer[s->seq.len] = 0;
7231 printk(KERN_TRACE "%s", s->buffer);
7236 void trace_init_global_iter(struct trace_iterator *iter)
7238 iter->tr = &global_trace;
7239 iter->trace = iter->tr->current_trace;
7240 iter->cpu_file = RING_BUFFER_ALL_CPUS;
7241 iter->trace_buffer = &global_trace.trace_buffer;
7243 if (iter->trace && iter->trace->open)
7244 iter->trace->open(iter);
7246 /* Annotate start of buffers if we had overruns */
7247 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7248 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7250 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7251 if (trace_clocks[iter->tr->clock_id].in_ns)
7252 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
7255 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
7257 /* use static because iter can be a bit big for the stack */
7258 static struct trace_iterator iter;
7259 static atomic_t dump_running;
7260 struct trace_array *tr = &global_trace;
7261 unsigned int old_userobj;
7262 unsigned long flags;
7265 /* Only allow one dump user at a time. */
7266 if (atomic_inc_return(&dump_running) != 1) {
7267 atomic_dec(&dump_running);
7272 * Always turn off tracing when we dump.
7273 * We don't need to show trace output of what happens
7274 * between multiple crashes.
7276 * If the user does a sysrq-z, then they can re-enable
7277 * tracing with echo 1 > tracing_on.
7281 local_irq_save(flags);
7283 /* Simulate the iterator */
7284 trace_init_global_iter(&iter);
7286 for_each_tracing_cpu(cpu) {
7287 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7290 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
7292 /* don't look at user memory in panic mode */
7293 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
7295 switch (oops_dump_mode) {
7297 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7300 iter.cpu_file = raw_smp_processor_id();
7305 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7306 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7309 printk(KERN_TRACE "Dumping ftrace buffer:\n");
7311 /* Did function tracer already get disabled? */
7312 if (ftrace_is_dead()) {
7313 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7314 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7318 * We need to stop all tracing on all CPUS to read the
7319 * the next buffer. This is a bit expensive, but is
7320 * not done often. We fill all what we can read,
7321 * and then release the locks again.
7324 while (!trace_empty(&iter)) {
7327 printk(KERN_TRACE "---------------------------------\n");
7331 /* reset all but tr, trace, and overruns */
7332 memset(&iter.seq, 0,
7333 sizeof(struct trace_iterator) -
7334 offsetof(struct trace_iterator, seq));
7335 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7338 if (trace_find_next_entry_inc(&iter) != NULL) {
7341 ret = print_trace_line(&iter);
7342 if (ret != TRACE_TYPE_NO_CONSUME)
7343 trace_consume(&iter);
7345 touch_nmi_watchdog();
7347 trace_printk_seq(&iter.seq);
7351 printk(KERN_TRACE " (ftrace buffer empty)\n");
7353 printk(KERN_TRACE "---------------------------------\n");
7356 tr->trace_flags |= old_userobj;
7358 for_each_tracing_cpu(cpu) {
7359 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7361 atomic_dec(&dump_running);
7362 local_irq_restore(flags);
7364 EXPORT_SYMBOL_GPL(ftrace_dump);
7366 __init static int tracer_alloc_buffers(void)
7372 * Make sure we don't accidently add more trace options
7373 * than we have bits for.
7375 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
7377 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7380 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7381 goto out_free_buffer_mask;
7383 /* Only allocate trace_printk buffers if a trace_printk exists */
7384 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
7385 /* Must be called before global_trace.buffer is allocated */
7386 trace_printk_init_buffers();
7388 /* To save memory, keep the ring buffer size to its minimum */
7389 if (ring_buffer_expanded)
7390 ring_buf_size = trace_buf_size;
7394 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7395 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7397 raw_spin_lock_init(&global_trace.start_lock);
7399 /* Used for event triggers */
7400 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7402 goto out_free_cpumask;
7404 if (trace_create_savedcmd() < 0)
7405 goto out_free_temp_buffer;
7407 /* TODO: make the number of buffers hot pluggable with CPUS */
7408 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7409 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7411 goto out_free_savedcmd;
7414 if (global_trace.buffer_disabled)
7417 if (trace_boot_clock) {
7418 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7420 pr_warn("Trace clock %s not defined, going back to default\n",
7425 * register_tracer() might reference current_trace, so it
7426 * needs to be set before we register anything. This is
7427 * just a bootstrap of current_trace anyway.
7429 global_trace.current_trace = &nop_trace;
7431 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7433 ftrace_init_global_array_ops(&global_trace);
7435 init_trace_flags_index(&global_trace);
7437 register_tracer(&nop_trace);
7439 /* All seems OK, enable tracing */
7440 tracing_disabled = 0;
7442 atomic_notifier_chain_register(&panic_notifier_list,
7443 &trace_panic_notifier);
7445 register_die_notifier(&trace_die_notifier);
7447 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7449 INIT_LIST_HEAD(&global_trace.systems);
7450 INIT_LIST_HEAD(&global_trace.events);
7451 list_add(&global_trace.list, &ftrace_trace_arrays);
7453 apply_trace_boot_options();
7455 register_snapshot_cmd();
7460 free_saved_cmdlines_buffer(savedcmd);
7461 out_free_temp_buffer:
7462 ring_buffer_free(temp_buffer);
7464 free_cpumask_var(global_trace.tracing_cpumask);
7465 out_free_buffer_mask:
7466 free_cpumask_var(tracing_buffer_mask);
7471 void __init trace_init(void)
7473 if (tracepoint_printk) {
7474 tracepoint_print_iter =
7475 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7476 if (WARN_ON(!tracepoint_print_iter))
7477 tracepoint_printk = 0;
7479 tracer_alloc_buffers();
7483 __init static int clear_boot_tracer(void)
7486 * The default tracer at boot buffer is an init section.
7487 * This function is called in lateinit. If we did not
7488 * find the boot tracer, then clear it out, to prevent
7489 * later registration from accessing the buffer that is
7490 * about to be freed.
7492 if (!default_bootup_tracer)
7495 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7496 default_bootup_tracer);
7497 default_bootup_tracer = NULL;
7502 fs_initcall(tracer_init_tracefs);
7503 late_initcall(clear_boot_tracer);