1 // SPDX-License-Identifier: GPL-2.0
3 * Kernel unwinding support
5 * (c) 2002-2004 Randolph Chung <tausq@debian.org>
7 * Derived partially from the IA64 implementation. The PA-RISC
8 * Runtime Architecture Document is also a useful reference to
9 * understand what is happening here
12 #include <linux/kernel.h>
13 #include <linux/init.h>
14 #include <linux/sched.h>
15 #include <linux/slab.h>
16 #include <linux/sort.h>
17 #include <linux/sched/task_stack.h>
19 #include <linux/uaccess.h>
20 #include <asm/assembly.h>
21 #include <asm/asm-offsets.h>
22 #include <asm/ptrace.h>
24 #include <asm/unwind.h>
25 #include <asm/switch_to.h>
26 #include <asm/sections.h>
27 #include <asm/ftrace.h>
31 #define dbg(x...) pr_debug(x)
33 #define dbg(x...) do { } while (0)
36 #define KERNEL_START (KERNEL_BINARY_TEXT_START)
38 extern struct unwind_table_entry __start___unwind[];
39 extern struct unwind_table_entry __stop___unwind[];
41 static DEFINE_SPINLOCK(unwind_lock);
43 * the kernel unwind block is not dynamically allocated so that
44 * we can call unwind_init as early in the bootup process as
45 * possible (before the slab allocator is initialized)
47 static struct unwind_table kernel_unwind_table __ro_after_init;
48 static LIST_HEAD(unwind_tables);
50 static inline const struct unwind_table_entry *
51 find_unwind_entry_in_table(const struct unwind_table *table, unsigned long addr)
53 const struct unwind_table_entry *e = NULL;
54 unsigned long lo, hi, mid;
57 hi = table->length - 1;
60 mid = (hi - lo) / 2 + lo;
61 e = &table->table[mid];
62 if (addr < e->region_start)
64 else if (addr > e->region_end)
73 static const struct unwind_table_entry *
74 find_unwind_entry(unsigned long addr)
76 struct unwind_table *table;
77 const struct unwind_table_entry *e = NULL;
79 if (addr >= kernel_unwind_table.start &&
80 addr <= kernel_unwind_table.end)
81 e = find_unwind_entry_in_table(&kernel_unwind_table, addr);
85 spin_lock_irqsave(&unwind_lock, flags);
86 list_for_each_entry(table, &unwind_tables, list) {
87 if (addr >= table->start &&
89 e = find_unwind_entry_in_table(table, addr);
91 /* Move-to-front to exploit common traces */
92 list_move(&table->list, &unwind_tables);
96 spin_unlock_irqrestore(&unwind_lock, flags);
103 unwind_table_init(struct unwind_table *table, const char *name,
104 unsigned long base_addr, unsigned long gp,
105 void *table_start, void *table_end)
107 struct unwind_table_entry *start = table_start;
108 struct unwind_table_entry *end =
109 (struct unwind_table_entry *)table_end - 1;
112 table->base_addr = base_addr;
114 table->start = base_addr + start->region_start;
115 table->end = base_addr + end->region_end;
116 table->table = (struct unwind_table_entry *)table_start;
117 table->length = end - start + 1;
118 INIT_LIST_HEAD(&table->list);
120 for (; start <= end; start++) {
122 start->region_end > (start+1)->region_start) {
123 pr_warn("Out of order unwind entry! %px and %px\n",
127 start->region_start += base_addr;
128 start->region_end += base_addr;
132 static int cmp_unwind_table_entry(const void *a, const void *b)
134 return ((const struct unwind_table_entry *)a)->region_start
135 - ((const struct unwind_table_entry *)b)->region_start;
139 unwind_table_sort(struct unwind_table_entry *start,
140 struct unwind_table_entry *finish)
142 sort(start, finish - start, sizeof(struct unwind_table_entry),
143 cmp_unwind_table_entry, NULL);
146 struct unwind_table *
147 unwind_table_add(const char *name, unsigned long base_addr,
149 void *start, void *end)
151 struct unwind_table *table;
153 struct unwind_table_entry *s = (struct unwind_table_entry *)start;
154 struct unwind_table_entry *e = (struct unwind_table_entry *)end;
156 unwind_table_sort(s, e);
158 table = kmalloc(sizeof(struct unwind_table), GFP_USER);
161 unwind_table_init(table, name, base_addr, gp, start, end);
162 spin_lock_irqsave(&unwind_lock, flags);
163 list_add_tail(&table->list, &unwind_tables);
164 spin_unlock_irqrestore(&unwind_lock, flags);
169 void unwind_table_remove(struct unwind_table *table)
173 spin_lock_irqsave(&unwind_lock, flags);
174 list_del(&table->list);
175 spin_unlock_irqrestore(&unwind_lock, flags);
180 /* Called from setup_arch to import the kernel unwind info */
181 int __init unwind_init(void)
183 long start __maybe_unused, stop __maybe_unused;
184 register unsigned long gp __asm__ ("r27");
186 start = (long)&__start___unwind[0];
187 stop = (long)&__stop___unwind[0];
189 dbg("unwind_init: start = 0x%lx, end = 0x%lx, entries = %lu\n",
191 (stop - start) / sizeof(struct unwind_table_entry));
193 unwind_table_init(&kernel_unwind_table, "kernel", KERNEL_START,
195 &__start___unwind[0], &__stop___unwind[0]);
199 for (i = 0; i < 10; i++)
201 printk("region 0x%x-0x%x\n",
202 __start___unwind[i].region_start,
203 __start___unwind[i].region_end);
210 static bool pc_is_kernel_fn(unsigned long pc, void *fn)
212 return (unsigned long)dereference_kernel_function_descriptor(fn) == pc;
215 static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int frame_size)
218 * We have to use void * instead of a function pointer, because
219 * function pointers aren't a pointer to the function on 64-bit.
220 * Make them const so the compiler knows they live in .text
221 * Note: We could use dereference_kernel_function_descriptor()
222 * instead but we want to keep it simple here.
224 extern void * const ret_from_kernel_thread;
225 extern void * const syscall_exit;
226 extern void * const intr_return;
227 extern void * const _switch_to_ret;
228 #ifdef CONFIG_IRQSTACKS
229 extern void * const _call_on_stack;
230 #endif /* CONFIG_IRQSTACKS */
233 ptr = dereference_kernel_function_descriptor(&handle_interruption);
234 if (pc_is_kernel_fn(pc, ptr)) {
235 struct pt_regs *regs = (struct pt_regs *)(info->sp - frame_size - PT_SZ_ALGN);
236 dbg("Unwinding through handle_interruption()\n");
237 info->prev_sp = regs->gr[30];
238 info->prev_ip = regs->iaoq[0];
242 if (pc_is_kernel_fn(pc, ret_from_kernel_thread) ||
243 pc_is_kernel_fn(pc, syscall_exit)) {
244 info->prev_sp = info->prev_ip = 0;
248 if (pc_is_kernel_fn(pc, intr_return)) {
249 struct pt_regs *regs;
251 dbg("Found intr_return()\n");
252 regs = (struct pt_regs *)(info->sp - PT_SZ_ALGN);
253 info->prev_sp = regs->gr[30];
254 info->prev_ip = regs->iaoq[0];
255 info->rp = regs->gr[2];
259 if (pc_is_kernel_fn(pc, _switch_to) ||
260 pc_is_kernel_fn(pc, _switch_to_ret)) {
261 info->prev_sp = info->sp - CALLEE_SAVE_FRAME_SIZE;
262 info->prev_ip = *(unsigned long *)(info->prev_sp - RP_OFFSET);
266 #ifdef CONFIG_IRQSTACKS
267 if (pc_is_kernel_fn(pc, _call_on_stack)) {
268 info->prev_sp = *(unsigned long *)(info->sp - FRAME_SIZE - REG_SZ);
269 info->prev_ip = *(unsigned long *)(info->sp - FRAME_SIZE - RP_OFFSET);
276 static void unwind_frame_regs(struct unwind_frame_info *info)
278 const struct unwind_table_entry *e;
282 int looking_for_rp, rpoffset = 0;
284 e = find_unwind_entry(info->ip);
288 dbg("Cannot find unwind entry for %pS; forced unwinding\n",
291 /* Since we are doing the unwinding blind, we don't know if
292 we are adjusting the stack correctly or extracting the rp
293 correctly. The rp is checked to see if it belongs to the
294 kernel text section, if not we assume we don't have a
295 correct stack frame and we continue to unwind the stack.
296 This is not quite correct, and will fail for loadable
302 info->prev_sp = sp - 64;
305 /* Check if stack is inside kernel stack area */
306 if ((info->prev_sp - (unsigned long) task_stack_page(info->t))
312 if (copy_from_kernel_nofault(&tmp,
313 (void *)info->prev_sp - RP_OFFSET, sizeof(tmp)))
317 } while (!kernel_text_address(info->prev_ip));
321 dbg("analyzing func @ %lx with no unwind info, setting "
322 "prev_sp=%lx prev_ip=%lx\n", info->ip,
323 info->prev_sp, info->prev_ip);
325 dbg("e->start = 0x%x, e->end = 0x%x, Save_SP = %d, "
326 "Save_RP = %d, Millicode = %d size = %u\n",
327 e->region_start, e->region_end, e->Save_SP, e->Save_RP,
328 e->Millicode, e->Total_frame_size);
330 looking_for_rp = e->Save_RP;
332 for (npc = e->region_start;
333 (frame_size < (e->Total_frame_size << 3) ||
338 insn = *(unsigned int *)npc;
340 if ((insn & 0xffffc001) == 0x37de0000 ||
341 (insn & 0xffe00001) == 0x6fc00000) {
342 /* ldo X(sp), sp, or stwm X,D(sp) */
343 frame_size += (insn & 0x3fff) >> 1;
344 dbg("analyzing func @ %lx, insn=%08x @ "
345 "%lx, frame_size = %ld\n", info->ip,
346 insn, npc, frame_size);
347 } else if ((insn & 0xffe00009) == 0x73c00008) {
349 frame_size += ((insn >> 4) & 0x3ff) << 3;
350 dbg("analyzing func @ %lx, insn=%08x @ "
351 "%lx, frame_size = %ld\n", info->ip,
352 insn, npc, frame_size);
353 } else if (insn == 0x6bc23fd9) {
357 dbg("analyzing func @ %lx, insn=stw rp,"
358 "-20(sp) @ %lx\n", info->ip, npc);
359 } else if (insn == 0x0fc212c1) {
360 /* std rp,-16(sr0,sp) */
363 dbg("analyzing func @ %lx, insn=std rp,"
364 "-16(sp) @ %lx\n", info->ip, npc);
368 if (frame_size > e->Total_frame_size << 3)
369 frame_size = e->Total_frame_size << 3;
371 if (!unwind_special(info, e->region_start, frame_size)) {
372 info->prev_sp = info->sp - frame_size;
374 info->rp = info->r31;
376 info->rp = *(unsigned long *)(info->prev_sp - rpoffset);
377 info->prev_ip = info->rp;
381 dbg("analyzing func @ %lx, setting prev_sp=%lx "
382 "prev_ip=%lx npc=%lx\n", info->ip, info->prev_sp,
387 void unwind_frame_init(struct unwind_frame_info *info, struct task_struct *t,
388 struct pt_regs *regs)
390 memset(info, 0, sizeof(struct unwind_frame_info));
392 info->sp = regs->gr[30];
393 info->ip = regs->iaoq[0];
394 info->rp = regs->gr[2];
395 info->r31 = regs->gr[31];
397 dbg("(%d) Start unwind from sp=%08lx ip=%08lx\n",
398 t ? (int)t->pid : -1, info->sp, info->ip);
401 void unwind_frame_init_from_blocked_task(struct unwind_frame_info *info, struct task_struct *t)
403 struct pt_regs *r = &t->thread.regs;
406 r2 = kmalloc(sizeof(struct pt_regs), GFP_ATOMIC);
411 r2->iaoq[0] = r->kpc;
412 unwind_frame_init(info, t, r2);
416 #define get_parisc_stackpointer() ({ \
418 __asm__("copy %%r30, %0" : "=r"(sp)); \
422 void unwind_frame_init_task(struct unwind_frame_info *info,
423 struct task_struct *task, struct pt_regs *regs)
425 task = task ? task : current;
427 if (task == current) {
431 memset(&r, 0, sizeof(r));
432 r.iaoq[0] = _THIS_IP_;
434 r.gr[30] = get_parisc_stackpointer();
437 unwind_frame_init(info, task, regs);
439 unwind_frame_init_from_blocked_task(info, task);
443 int unwind_once(struct unwind_frame_info *next_frame)
445 unwind_frame_regs(next_frame);
447 if (next_frame->prev_sp == 0 ||
448 next_frame->prev_ip == 0)
451 next_frame->sp = next_frame->prev_sp;
452 next_frame->ip = next_frame->prev_ip;
453 next_frame->prev_sp = 0;
454 next_frame->prev_ip = 0;
456 dbg("(%d) Continue unwind to sp=%08lx ip=%08lx\n",
457 next_frame->t ? (int)next_frame->t->pid : -1,
458 next_frame->sp, next_frame->ip);
463 int unwind_to_user(struct unwind_frame_info *info)
468 ret = unwind_once(info);
469 } while (!ret && !(info->ip & 3));
474 unsigned long return_address(unsigned int level)
476 struct unwind_frame_info info;
478 /* initialize unwind info */
479 unwind_frame_init_task(&info, current, NULL);
484 if (unwind_once(&info) < 0 || info.ip == 0)
486 if (!kernel_text_address(info.ip))
488 } while (info.ip && level--);