Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
e5a81b62 SR |
2 | /* |
3 | * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com> | |
4 | * | |
5 | */ | |
68db0cf1 | 6 | #include <linux/sched/task_stack.h> |
e5a81b62 SR |
7 | #include <linux/stacktrace.h> |
8 | #include <linux/kallsyms.h> | |
9 | #include <linux/seq_file.h> | |
10 | #include <linux/spinlock.h> | |
11 | #include <linux/uaccess.h> | |
e5a81b62 SR |
12 | #include <linux/ftrace.h> |
13 | #include <linux/module.h> | |
f38f1d2a | 14 | #include <linux/sysctl.h> |
e5a81b62 | 15 | #include <linux/init.h> |
762e1207 SR |
16 | |
17 | #include <asm/setup.h> | |
18 | ||
e5a81b62 SR |
19 | #include "trace.h" |
20 | ||
3d9a8072 TG |
21 | #define STACK_TRACE_ENTRIES 500 |
22 | ||
23 | static unsigned long stack_dump_trace[STACK_TRACE_ENTRIES]; | |
24 | static unsigned stack_trace_index[STACK_TRACE_ENTRIES]; | |
1b6cced6 | 25 | |
9f50c91b | 26 | static unsigned int stack_trace_nr_entries; |
3d9a8072 TG |
27 | static unsigned long stack_trace_max_size; |
28 | static arch_spinlock_t stack_trace_max_lock = | |
edc35bd7 | 29 | (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; |
e5a81b62 | 30 | |
8aaf1ee7 | 31 | DEFINE_PER_CPU(int, disable_stack_tracer); |
f38f1d2a SR |
32 | static DEFINE_MUTEX(stack_sysctl_mutex); |
33 | ||
34 | int stack_tracer_enabled; | |
e5a81b62 | 35 | |
3d9a8072 | 36 | static void print_max_stack(void) |
e3172181 MK |
37 | { |
38 | long i; | |
39 | int size; | |
40 | ||
41 | pr_emerg(" Depth Size Location (%d entries)\n" | |
42 | " ----- ---- --------\n", | |
9f50c91b | 43 | stack_trace_nr_entries); |
e3172181 | 44 | |
9f50c91b TG |
45 | for (i = 0; i < stack_trace_nr_entries; i++) { |
46 | if (i + 1 == stack_trace_nr_entries) | |
bb99d8cc | 47 | size = stack_trace_index[i]; |
e3172181 | 48 | else |
bb99d8cc | 49 | size = stack_trace_index[i] - stack_trace_index[i+1]; |
e3172181 | 50 | |
bb99d8cc | 51 | pr_emerg("%3ld) %8d %5d %pS\n", i, stack_trace_index[i], |
e3172181 MK |
52 | size, (void *)stack_dump_trace[i]); |
53 | } | |
54 | } | |
55 | ||
3d9a8072 | 56 | static void check_stack(unsigned long ip, unsigned long *stack) |
e5a81b62 | 57 | { |
e3172181 | 58 | unsigned long this_size, flags; unsigned long *p, *top, *start; |
4df29712 | 59 | static int tracer_frame; |
6aa7de05 | 60 | int frame_size = READ_ONCE(tracer_frame); |
72ac426a | 61 | int i, x; |
e5a81b62 | 62 | |
87889501 | 63 | this_size = ((unsigned long)stack) & (THREAD_SIZE-1); |
e5a81b62 | 64 | this_size = THREAD_SIZE - this_size; |
4df29712 SRRH |
65 | /* Remove the frame of the tracer */ |
66 | this_size -= frame_size; | |
e5a81b62 | 67 | |
bb99d8cc | 68 | if (this_size <= stack_trace_max_size) |
e5a81b62 SR |
69 | return; |
70 | ||
81520a1b | 71 | /* we do not handle interrupt stacks yet */ |
87889501 | 72 | if (!object_is_on_stack(stack)) |
81520a1b SR |
73 | return; |
74 | ||
1904be1b SRRH |
75 | /* Can't do this from NMI context (can cause deadlocks) */ |
76 | if (in_nmi()) | |
77 | return; | |
78 | ||
a5e25883 | 79 | local_irq_save(flags); |
d332736d | 80 | arch_spin_lock(&stack_trace_max_lock); |
e5a81b62 | 81 | |
4df29712 SRRH |
82 | /* In case another CPU set the tracer_frame on us */ |
83 | if (unlikely(!frame_size)) | |
84 | this_size -= tracer_frame; | |
85 | ||
e5a81b62 | 86 | /* a race could have already updated it */ |
bb99d8cc | 87 | if (this_size <= stack_trace_max_size) |
e5a81b62 SR |
88 | goto out; |
89 | ||
bb99d8cc | 90 | stack_trace_max_size = this_size; |
e5a81b62 | 91 | |
9f50c91b TG |
92 | stack_trace_nr_entries = stack_trace_save(stack_dump_trace, |
93 | ARRAY_SIZE(stack_dump_trace) - 1, | |
94 | 0); | |
e5a81b62 | 95 | |
72ac426a | 96 | /* Skip over the overhead of the stack tracer itself */ |
9f50c91b | 97 | for (i = 0; i < stack_trace_nr_entries; i++) { |
72ac426a SRRH |
98 | if (stack_dump_trace[i] == ip) |
99 | break; | |
100 | } | |
d4ecbfc4 | 101 | |
6ccd8371 SR |
102 | /* |
103 | * Some archs may not have the passed in ip in the dump. | |
104 | * If that happens, we need to show everything. | |
105 | */ | |
9f50c91b | 106 | if (i == stack_trace_nr_entries) |
6ccd8371 SR |
107 | i = 0; |
108 | ||
1b6cced6 SR |
109 | /* |
110 | * Now find where in the stack these are. | |
111 | */ | |
72ac426a | 112 | x = 0; |
87889501 | 113 | start = stack; |
1b6cced6 SR |
114 | top = (unsigned long *) |
115 | (((unsigned long)start & ~(THREAD_SIZE-1)) + THREAD_SIZE); | |
116 | ||
117 | /* | |
118 | * Loop through all the entries. One of the entries may | |
119 | * for some reason be missed on the stack, so we may | |
120 | * have to account for them. If they are all there, this | |
121 | * loop will only happen once. This code only takes place | |
122 | * on a new max, so it is far from a fast path. | |
123 | */ | |
9f50c91b | 124 | while (i < stack_trace_nr_entries) { |
0a37119d | 125 | int found = 0; |
1b6cced6 | 126 | |
bb99d8cc | 127 | stack_trace_index[x] = this_size; |
1b6cced6 SR |
128 | p = start; |
129 | ||
9f50c91b | 130 | for (; p < top && i < stack_trace_nr_entries; p++) { |
6e22c836 YS |
131 | /* |
132 | * The READ_ONCE_NOCHECK is used to let KASAN know that | |
133 | * this is not a stack-out-of-bounds error. | |
134 | */ | |
135 | if ((READ_ONCE_NOCHECK(*p)) == stack_dump_trace[i]) { | |
72ac426a | 136 | stack_dump_trace[x] = stack_dump_trace[i++]; |
bb99d8cc | 137 | this_size = stack_trace_index[x++] = |
1b6cced6 | 138 | (top - p) * sizeof(unsigned long); |
0a37119d | 139 | found = 1; |
1b6cced6 SR |
140 | /* Start the search from here */ |
141 | start = p + 1; | |
4df29712 SRRH |
142 | /* |
143 | * We do not want to show the overhead | |
144 | * of the stack tracer stack in the | |
145 | * max stack. If we haven't figured | |
146 | * out what that is, then figure it out | |
147 | * now. | |
148 | */ | |
72ac426a | 149 | if (unlikely(!tracer_frame)) { |
4df29712 SRRH |
150 | tracer_frame = (p - stack) * |
151 | sizeof(unsigned long); | |
bb99d8cc | 152 | stack_trace_max_size -= tracer_frame; |
4df29712 | 153 | } |
1b6cced6 SR |
154 | } |
155 | } | |
156 | ||
0a37119d SR |
157 | if (!found) |
158 | i++; | |
1b6cced6 SR |
159 | } |
160 | ||
9f50c91b | 161 | stack_trace_nr_entries = x; |
72ac426a | 162 | |
a70857e4 | 163 | if (task_stack_end_corrupted(current)) { |
3d9a8072 | 164 | print_max_stack(); |
e3172181 MK |
165 | BUG(); |
166 | } | |
167 | ||
e5a81b62 | 168 | out: |
d332736d | 169 | arch_spin_unlock(&stack_trace_max_lock); |
a5e25883 | 170 | local_irq_restore(flags); |
e5a81b62 SR |
171 | } |
172 | ||
173 | static void | |
a1e2e31d SR |
174 | stack_trace_call(unsigned long ip, unsigned long parent_ip, |
175 | struct ftrace_ops *op, struct pt_regs *pt_regs) | |
e5a81b62 | 176 | { |
87889501 | 177 | unsigned long stack; |
e5a81b62 | 178 | |
5168ae50 | 179 | preempt_disable_notrace(); |
e5a81b62 | 180 | |
e5a81b62 | 181 | /* no atomic needed, we only modify this variable by this cpu */ |
8aaf1ee7 SRV |
182 | __this_cpu_inc(disable_stack_tracer); |
183 | if (__this_cpu_read(disable_stack_tracer) != 1) | |
e5a81b62 SR |
184 | goto out; |
185 | ||
b00d607b SRV |
186 | /* If rcu is not watching, then save stack trace can fail */ |
187 | if (!rcu_is_watching()) | |
188 | goto out; | |
189 | ||
72ac426a | 190 | ip += MCOUNT_INSN_SIZE; |
4df29712 SRRH |
191 | |
192 | check_stack(ip, &stack); | |
e5a81b62 SR |
193 | |
194 | out: | |
8aaf1ee7 | 195 | __this_cpu_dec(disable_stack_tracer); |
e5a81b62 | 196 | /* prevent recursion in schedule */ |
5168ae50 | 197 | preempt_enable_notrace(); |
e5a81b62 SR |
198 | } |
199 | ||
200 | static struct ftrace_ops trace_ops __read_mostly = | |
201 | { | |
202 | .func = stack_trace_call, | |
4740974a | 203 | .flags = FTRACE_OPS_FL_RECURSION_SAFE, |
e5a81b62 SR |
204 | }; |
205 | ||
206 | static ssize_t | |
207 | stack_max_size_read(struct file *filp, char __user *ubuf, | |
208 | size_t count, loff_t *ppos) | |
209 | { | |
210 | unsigned long *ptr = filp->private_data; | |
211 | char buf[64]; | |
212 | int r; | |
213 | ||
214 | r = snprintf(buf, sizeof(buf), "%ld\n", *ptr); | |
215 | if (r > sizeof(buf)) | |
216 | r = sizeof(buf); | |
217 | return simple_read_from_buffer(ubuf, count, ppos, buf, r); | |
218 | } | |
219 | ||
220 | static ssize_t | |
221 | stack_max_size_write(struct file *filp, const char __user *ubuf, | |
222 | size_t count, loff_t *ppos) | |
223 | { | |
224 | long *ptr = filp->private_data; | |
225 | unsigned long val, flags; | |
e5a81b62 SR |
226 | int ret; |
227 | ||
22fe9b54 PH |
228 | ret = kstrtoul_from_user(ubuf, count, 10, &val); |
229 | if (ret) | |
e5a81b62 SR |
230 | return ret; |
231 | ||
a5e25883 | 232 | local_irq_save(flags); |
4f48f8b7 LJ |
233 | |
234 | /* | |
235 | * In case we trace inside arch_spin_lock() or after (NMI), | |
236 | * we will cause circular lock, so we also need to increase | |
8aaf1ee7 | 237 | * the percpu disable_stack_tracer here. |
4f48f8b7 | 238 | */ |
8aaf1ee7 | 239 | __this_cpu_inc(disable_stack_tracer); |
4f48f8b7 | 240 | |
d332736d | 241 | arch_spin_lock(&stack_trace_max_lock); |
e5a81b62 | 242 | *ptr = val; |
d332736d | 243 | arch_spin_unlock(&stack_trace_max_lock); |
4f48f8b7 | 244 | |
8aaf1ee7 | 245 | __this_cpu_dec(disable_stack_tracer); |
a5e25883 | 246 | local_irq_restore(flags); |
e5a81b62 SR |
247 | |
248 | return count; | |
249 | } | |
250 | ||
f38f1d2a | 251 | static const struct file_operations stack_max_size_fops = { |
e5a81b62 SR |
252 | .open = tracing_open_generic, |
253 | .read = stack_max_size_read, | |
254 | .write = stack_max_size_write, | |
6038f373 | 255 | .llseek = default_llseek, |
e5a81b62 SR |
256 | }; |
257 | ||
258 | static void * | |
2fc5f0cf | 259 | __next(struct seq_file *m, loff_t *pos) |
e5a81b62 | 260 | { |
2fc5f0cf | 261 | long n = *pos - 1; |
e5a81b62 | 262 | |
9f50c91b | 263 | if (n >= stack_trace_nr_entries) |
e5a81b62 SR |
264 | return NULL; |
265 | ||
2fc5f0cf | 266 | m->private = (void *)n; |
1b6cced6 | 267 | return &m->private; |
e5a81b62 SR |
268 | } |
269 | ||
2fc5f0cf LZ |
270 | static void * |
271 | t_next(struct seq_file *m, void *v, loff_t *pos) | |
e5a81b62 | 272 | { |
2fc5f0cf LZ |
273 | (*pos)++; |
274 | return __next(m, pos); | |
275 | } | |
e5a81b62 | 276 | |
2fc5f0cf LZ |
277 | static void *t_start(struct seq_file *m, loff_t *pos) |
278 | { | |
e5a81b62 | 279 | local_irq_disable(); |
4f48f8b7 | 280 | |
8aaf1ee7 | 281 | __this_cpu_inc(disable_stack_tracer); |
4f48f8b7 | 282 | |
d332736d | 283 | arch_spin_lock(&stack_trace_max_lock); |
e5a81b62 | 284 | |
522a110b LW |
285 | if (*pos == 0) |
286 | return SEQ_START_TOKEN; | |
287 | ||
2fc5f0cf | 288 | return __next(m, pos); |
e5a81b62 SR |
289 | } |
290 | ||
291 | static void t_stop(struct seq_file *m, void *p) | |
292 | { | |
d332736d | 293 | arch_spin_unlock(&stack_trace_max_lock); |
4f48f8b7 | 294 | |
8aaf1ee7 | 295 | __this_cpu_dec(disable_stack_tracer); |
4f48f8b7 | 296 | |
e5a81b62 SR |
297 | local_irq_enable(); |
298 | } | |
299 | ||
962e3707 | 300 | static void trace_lookup_stack(struct seq_file *m, long i) |
e5a81b62 | 301 | { |
1b6cced6 | 302 | unsigned long addr = stack_dump_trace[i]; |
e5a81b62 | 303 | |
962e3707 | 304 | seq_printf(m, "%pS\n", (void *)addr); |
e5a81b62 SR |
305 | } |
306 | ||
e447e1df SR |
307 | static void print_disabled(struct seq_file *m) |
308 | { | |
309 | seq_puts(m, "#\n" | |
310 | "# Stack tracer disabled\n" | |
311 | "#\n" | |
312 | "# To enable the stack tracer, either add 'stacktrace' to the\n" | |
313 | "# kernel command line\n" | |
314 | "# or 'echo 1 > /proc/sys/kernel/stack_tracer_enabled'\n" | |
315 | "#\n"); | |
316 | } | |
317 | ||
e5a81b62 SR |
318 | static int t_show(struct seq_file *m, void *v) |
319 | { | |
522a110b | 320 | long i; |
1b6cced6 SR |
321 | int size; |
322 | ||
522a110b | 323 | if (v == SEQ_START_TOKEN) { |
eb1871f3 | 324 | seq_printf(m, " Depth Size Location" |
1b6cced6 | 325 | " (%d entries)\n" |
eb1871f3 | 326 | " ----- ---- --------\n", |
9f50c91b | 327 | stack_trace_nr_entries); |
e447e1df | 328 | |
bb99d8cc | 329 | if (!stack_tracer_enabled && !stack_trace_max_size) |
e447e1df SR |
330 | print_disabled(m); |
331 | ||
1b6cced6 SR |
332 | return 0; |
333 | } | |
e5a81b62 | 334 | |
522a110b LW |
335 | i = *(long *)v; |
336 | ||
9f50c91b | 337 | if (i >= stack_trace_nr_entries) |
e5a81b62 SR |
338 | return 0; |
339 | ||
9f50c91b | 340 | if (i + 1 == stack_trace_nr_entries) |
bb99d8cc | 341 | size = stack_trace_index[i]; |
1b6cced6 | 342 | else |
bb99d8cc | 343 | size = stack_trace_index[i] - stack_trace_index[i+1]; |
1b6cced6 | 344 | |
bb99d8cc | 345 | seq_printf(m, "%3ld) %8d %5d ", i, stack_trace_index[i], size); |
1b6cced6 SR |
346 | |
347 | trace_lookup_stack(m, i); | |
e5a81b62 SR |
348 | |
349 | return 0; | |
350 | } | |
351 | ||
f38f1d2a | 352 | static const struct seq_operations stack_trace_seq_ops = { |
e5a81b62 SR |
353 | .start = t_start, |
354 | .next = t_next, | |
355 | .stop = t_stop, | |
356 | .show = t_show, | |
357 | }; | |
358 | ||
359 | static int stack_trace_open(struct inode *inode, struct file *file) | |
360 | { | |
d8cc1ab7 | 361 | return seq_open(file, &stack_trace_seq_ops); |
e5a81b62 SR |
362 | } |
363 | ||
f38f1d2a | 364 | static const struct file_operations stack_trace_fops = { |
e5a81b62 SR |
365 | .open = stack_trace_open, |
366 | .read = seq_read, | |
367 | .llseek = seq_lseek, | |
d8cc1ab7 | 368 | .release = seq_release, |
e5a81b62 SR |
369 | }; |
370 | ||
bbd1d27d SRV |
371 | #ifdef CONFIG_DYNAMIC_FTRACE |
372 | ||
d2d45c7a SR |
373 | static int |
374 | stack_trace_filter_open(struct inode *inode, struct file *file) | |
375 | { | |
0f179765 SRV |
376 | struct ftrace_ops *ops = inode->i_private; |
377 | ||
378 | return ftrace_regex_open(ops, FTRACE_ITER_FILTER, | |
d2d45c7a SR |
379 | inode, file); |
380 | } | |
381 | ||
382 | static const struct file_operations stack_trace_filter_fops = { | |
383 | .open = stack_trace_filter_open, | |
384 | .read = seq_read, | |
385 | .write = ftrace_filter_write, | |
098c879e | 386 | .llseek = tracing_lseek, |
d2d45c7a SR |
387 | .release = ftrace_regex_release, |
388 | }; | |
389 | ||
bbd1d27d SRV |
390 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
391 | ||
f38f1d2a SR |
392 | int |
393 | stack_trace_sysctl(struct ctl_table *table, int write, | |
8d65af78 | 394 | void __user *buffer, size_t *lenp, |
f38f1d2a SR |
395 | loff_t *ppos) |
396 | { | |
3d9a8072 | 397 | int was_enabled; |
f38f1d2a SR |
398 | int ret; |
399 | ||
400 | mutex_lock(&stack_sysctl_mutex); | |
3d9a8072 | 401 | was_enabled = !!stack_tracer_enabled; |
f38f1d2a | 402 | |
8d65af78 | 403 | ret = proc_dointvec(table, write, buffer, lenp, ppos); |
f38f1d2a | 404 | |
3d9a8072 | 405 | if (ret || !write || (was_enabled == !!stack_tracer_enabled)) |
f38f1d2a SR |
406 | goto out; |
407 | ||
f38f1d2a SR |
408 | if (stack_tracer_enabled) |
409 | register_ftrace_function(&trace_ops); | |
410 | else | |
411 | unregister_ftrace_function(&trace_ops); | |
f38f1d2a SR |
412 | out: |
413 | mutex_unlock(&stack_sysctl_mutex); | |
414 | return ret; | |
415 | } | |
416 | ||
762e1207 SR |
417 | static char stack_trace_filter_buf[COMMAND_LINE_SIZE+1] __initdata; |
418 | ||
f38f1d2a SR |
419 | static __init int enable_stacktrace(char *str) |
420 | { | |
3d739c1f SRV |
421 | int len; |
422 | ||
423 | if ((len = str_has_prefix(str, "_filter="))) | |
424 | strncpy(stack_trace_filter_buf, str + len, COMMAND_LINE_SIZE); | |
762e1207 | 425 | |
e05a43b7 | 426 | stack_tracer_enabled = 1; |
f38f1d2a SR |
427 | return 1; |
428 | } | |
429 | __setup("stacktrace", enable_stacktrace); | |
430 | ||
e5a81b62 SR |
431 | static __init int stack_trace_init(void) |
432 | { | |
433 | struct dentry *d_tracer; | |
e5a81b62 SR |
434 | |
435 | d_tracer = tracing_init_dentry(); | |
14a5ae40 | 436 | if (IS_ERR(d_tracer)) |
ed6f1c99 | 437 | return 0; |
e5a81b62 | 438 | |
5452af66 | 439 | trace_create_file("stack_max_size", 0644, d_tracer, |
bb99d8cc | 440 | &stack_trace_max_size, &stack_max_size_fops); |
e5a81b62 | 441 | |
5452af66 FW |
442 | trace_create_file("stack_trace", 0444, d_tracer, |
443 | NULL, &stack_trace_fops); | |
e5a81b62 | 444 | |
bbd1d27d | 445 | #ifdef CONFIG_DYNAMIC_FTRACE |
0c5a9acc | 446 | trace_create_file("stack_trace_filter", 0644, d_tracer, |
0f179765 | 447 | &trace_ops, &stack_trace_filter_fops); |
bbd1d27d | 448 | #endif |
d2d45c7a | 449 | |
762e1207 SR |
450 | if (stack_trace_filter_buf[0]) |
451 | ftrace_set_early_filter(&trace_ops, stack_trace_filter_buf, 1); | |
452 | ||
e05a43b7 | 453 | if (stack_tracer_enabled) |
f38f1d2a | 454 | register_ftrace_function(&trace_ops); |
e5a81b62 SR |
455 | |
456 | return 0; | |
457 | } | |
458 | ||
459 | device_initcall(stack_trace_init); |