Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 LT |
2 | /* |
3 | * Kernel unwinding support | |
4 | * | |
5 | * (c) 2002-2004 Randolph Chung <tausq@debian.org> | |
6 | * | |
7 | * Derived partially from the IA64 implementation. The PA-RISC | |
8 | * Runtime Architecture Document is also a useful reference to | |
9 | * understand what is happening here | |
10 | */ | |
11 | ||
1da177e4 LT |
12 | #include <linux/kernel.h> |
13 | #include <linux/init.h> | |
e6fc0449 | 14 | #include <linux/sched.h> |
1da177e4 | 15 | #include <linux/slab.h> |
8f78df87 | 16 | #include <linux/sort.h> |
2214c0e7 | 17 | #include <linux/sched/task_stack.h> |
1da177e4 | 18 | |
7c0f6ba6 | 19 | #include <linux/uaccess.h> |
1da177e4 | 20 | #include <asm/assembly.h> |
05dc16d6 RC |
21 | #include <asm/asm-offsets.h> |
22 | #include <asm/ptrace.h> | |
1da177e4 LT |
23 | |
24 | #include <asm/unwind.h> | |
8e0ba125 SS |
25 | #include <asm/switch_to.h> |
26 | #include <asm/sections.h> | |
a7fde0bf | 27 | #include <asm/ftrace.h> |
1da177e4 LT |
28 | |
29 | /* #define DEBUG 1 */ | |
30 | #ifdef DEBUG | |
63ba82c0 | 31 | #define dbg(x...) pr_debug(x) |
1da177e4 | 32 | #else |
a7fde0bf | 33 | #define dbg(x...) do { } while (0) |
1da177e4 LT |
34 | #endif |
35 | ||
33a932d1 | 36 | #define KERNEL_START (KERNEL_BINARY_TEXT_START) |
e036306a | 37 | |
1da177e4 LT |
38 | extern struct unwind_table_entry __start___unwind[]; |
39 | extern struct unwind_table_entry __stop___unwind[]; | |
40 | ||
76cffeb6 | 41 | static DEFINE_SPINLOCK(unwind_lock); |
1da177e4 LT |
42 | /* |
43 | * the kernel unwind block is not dynamically allocated so that | |
44 | * we can call unwind_init as early in the bootup process as | |
45 | * possible (before the slab allocator is initialized) | |
46 | */ | |
47293774 | 47 | static struct unwind_table kernel_unwind_table __ro_after_init; |
1da177e4 LT |
48 | static LIST_HEAD(unwind_tables); |
49 | ||
50 | static inline const struct unwind_table_entry * | |
51 | find_unwind_entry_in_table(const struct unwind_table *table, unsigned long addr) | |
52 | { | |
53 | const struct unwind_table_entry *e = NULL; | |
54 | unsigned long lo, hi, mid; | |
55 | ||
56 | lo = 0; | |
57 | hi = table->length - 1; | |
58 | ||
59 | while (lo <= hi) { | |
60 | mid = (hi - lo) / 2 + lo; | |
61 | e = &table->table[mid]; | |
62 | if (addr < e->region_start) | |
63 | hi = mid - 1; | |
64 | else if (addr > e->region_end) | |
65 | lo = mid + 1; | |
66 | else | |
67 | return e; | |
68 | } | |
69 | ||
70 | return NULL; | |
71 | } | |
72 | ||
73 | static const struct unwind_table_entry * | |
74 | find_unwind_entry(unsigned long addr) | |
75 | { | |
76 | struct unwind_table *table; | |
77 | const struct unwind_table_entry *e = NULL; | |
78 | ||
79 | if (addr >= kernel_unwind_table.start && | |
80 | addr <= kernel_unwind_table.end) | |
81 | e = find_unwind_entry_in_table(&kernel_unwind_table, addr); | |
be24a897 MP |
82 | else { |
83 | unsigned long flags; | |
84 | ||
85 | spin_lock_irqsave(&unwind_lock, flags); | |
1da177e4 LT |
86 | list_for_each_entry(table, &unwind_tables, list) { |
87 | if (addr >= table->start && | |
88 | addr <= table->end) | |
89 | e = find_unwind_entry_in_table(table, addr); | |
b1b1d4a6 PC |
90 | if (e) { |
91 | /* Move-to-front to exploit common traces */ | |
92 | list_move(&table->list, &unwind_tables); | |
1da177e4 | 93 | break; |
b1b1d4a6 | 94 | } |
1da177e4 | 95 | } |
be24a897 MP |
96 | spin_unlock_irqrestore(&unwind_lock, flags); |
97 | } | |
1da177e4 LT |
98 | |
99 | return e; | |
100 | } | |
101 | ||
102 | static void | |
103 | unwind_table_init(struct unwind_table *table, const char *name, | |
104 | unsigned long base_addr, unsigned long gp, | |
105 | void *table_start, void *table_end) | |
106 | { | |
107 | struct unwind_table_entry *start = table_start; | |
108 | struct unwind_table_entry *end = | |
109 | (struct unwind_table_entry *)table_end - 1; | |
110 | ||
111 | table->name = name; | |
112 | table->base_addr = base_addr; | |
113 | table->gp = gp; | |
114 | table->start = base_addr + start->region_start; | |
115 | table->end = base_addr + end->region_end; | |
116 | table->table = (struct unwind_table_entry *)table_start; | |
117 | table->length = end - start + 1; | |
118 | INIT_LIST_HEAD(&table->list); | |
119 | ||
120 | for (; start <= end; start++) { | |
121 | if (start < end && | |
122 | start->region_end > (start+1)->region_start) { | |
c8921d72 HD |
123 | pr_warn("Out of order unwind entry! %px and %px\n", |
124 | start, start+1); | |
1da177e4 LT |
125 | } |
126 | ||
127 | start->region_start += base_addr; | |
128 | start->region_end += base_addr; | |
129 | } | |
130 | } | |
131 | ||
8f78df87 HD |
132 | static int cmp_unwind_table_entry(const void *a, const void *b) |
133 | { | |
134 | return ((const struct unwind_table_entry *)a)->region_start | |
135 | - ((const struct unwind_table_entry *)b)->region_start; | |
136 | } | |
137 | ||
1da177e4 LT |
138 | static void |
139 | unwind_table_sort(struct unwind_table_entry *start, | |
140 | struct unwind_table_entry *finish) | |
141 | { | |
8f78df87 HD |
142 | sort(start, finish - start, sizeof(struct unwind_table_entry), |
143 | cmp_unwind_table_entry, NULL); | |
1da177e4 LT |
144 | } |
145 | ||
146 | struct unwind_table * | |
147 | unwind_table_add(const char *name, unsigned long base_addr, | |
148 | unsigned long gp, | |
149 | void *start, void *end) | |
150 | { | |
151 | struct unwind_table *table; | |
152 | unsigned long flags; | |
153 | struct unwind_table_entry *s = (struct unwind_table_entry *)start; | |
154 | struct unwind_table_entry *e = (struct unwind_table_entry *)end; | |
155 | ||
156 | unwind_table_sort(s, e); | |
157 | ||
158 | table = kmalloc(sizeof(struct unwind_table), GFP_USER); | |
159 | if (table == NULL) | |
160 | return NULL; | |
161 | unwind_table_init(table, name, base_addr, gp, start, end); | |
162 | spin_lock_irqsave(&unwind_lock, flags); | |
163 | list_add_tail(&table->list, &unwind_tables); | |
164 | spin_unlock_irqrestore(&unwind_lock, flags); | |
165 | ||
166 | return table; | |
167 | } | |
168 | ||
169 | void unwind_table_remove(struct unwind_table *table) | |
170 | { | |
171 | unsigned long flags; | |
172 | ||
173 | spin_lock_irqsave(&unwind_lock, flags); | |
174 | list_del(&table->list); | |
175 | spin_unlock_irqrestore(&unwind_lock, flags); | |
176 | ||
177 | kfree(table); | |
178 | } | |
179 | ||
180 | /* Called from setup_arch to import the kernel unwind info */ | |
c790b41b | 181 | int __init unwind_init(void) |
1da177e4 | 182 | { |
a7fde0bf | 183 | long start __maybe_unused, stop __maybe_unused; |
1da177e4 LT |
184 | register unsigned long gp __asm__ ("r27"); |
185 | ||
186 | start = (long)&__start___unwind[0]; | |
187 | stop = (long)&__stop___unwind[0]; | |
188 | ||
63ba82c0 | 189 | dbg("unwind_init: start = 0x%lx, end = 0x%lx, entries = %lu\n", |
1da177e4 LT |
190 | start, stop, |
191 | (stop - start) / sizeof(struct unwind_table_entry)); | |
192 | ||
193 | unwind_table_init(&kernel_unwind_table, "kernel", KERNEL_START, | |
194 | gp, | |
195 | &__start___unwind[0], &__stop___unwind[0]); | |
196 | #if 0 | |
197 | { | |
198 | int i; | |
199 | for (i = 0; i < 10; i++) | |
200 | { | |
201 | printk("region 0x%x-0x%x\n", | |
202 | __start___unwind[i].region_start, | |
203 | __start___unwind[i].region_end); | |
204 | } | |
205 | } | |
206 | #endif | |
207 | return 0; | |
208 | } | |
209 | ||
8e0ba125 SS |
210 | static bool pc_is_kernel_fn(unsigned long pc, void *fn) |
211 | { | |
212 | return (unsigned long)dereference_kernel_function_descriptor(fn) == pc; | |
213 | } | |
214 | ||
05dc16d6 RC |
215 | static int unwind_special(struct unwind_frame_info *info, unsigned long pc, int frame_size) |
216 | { | |
c8921d72 HD |
217 | /* |
218 | * We have to use void * instead of a function pointer, because | |
219 | * function pointers aren't a pointer to the function on 64-bit. | |
220 | * Make them const so the compiler knows they live in .text | |
8801ccb9 HD |
221 | * Note: We could use dereference_kernel_function_descriptor() |
222 | * instead but we want to keep it simple here. | |
c8921d72 | 223 | */ |
c8921d72 HD |
224 | extern void * const ret_from_kernel_thread; |
225 | extern void * const syscall_exit; | |
226 | extern void * const intr_return; | |
227 | extern void * const _switch_to_ret; | |
228 | #ifdef CONFIG_IRQSTACKS | |
8801ccb9 | 229 | extern void * const _call_on_stack; |
c8921d72 | 230 | #endif /* CONFIG_IRQSTACKS */ |
6414b30b | 231 | void *ptr; |
c8921d72 | 232 | |
6414b30b HD |
233 | ptr = dereference_kernel_function_descriptor(&handle_interruption); |
234 | if (pc_is_kernel_fn(pc, ptr)) { | |
05dc16d6 RC |
235 | struct pt_regs *regs = (struct pt_regs *)(info->sp - frame_size - PT_SZ_ALGN); |
236 | dbg("Unwinding through handle_interruption()\n"); | |
237 | info->prev_sp = regs->gr[30]; | |
238 | info->prev_ip = regs->iaoq[0]; | |
c8921d72 HD |
239 | return 1; |
240 | } | |
241 | ||
8e0ba125 SS |
242 | if (pc_is_kernel_fn(pc, ret_from_kernel_thread) || |
243 | pc_is_kernel_fn(pc, syscall_exit)) { | |
c8921d72 HD |
244 | info->prev_sp = info->prev_ip = 0; |
245 | return 1; | |
246 | } | |
247 | ||
8e0ba125 | 248 | if (pc_is_kernel_fn(pc, intr_return)) { |
c8921d72 HD |
249 | struct pt_regs *regs; |
250 | ||
251 | dbg("Found intr_return()\n"); | |
252 | regs = (struct pt_regs *)(info->sp - PT_SZ_ALGN); | |
253 | info->prev_sp = regs->gr[30]; | |
254 | info->prev_ip = regs->iaoq[0]; | |
255 | info->rp = regs->gr[2]; | |
256 | return 1; | |
257 | } | |
258 | ||
8e0ba125 SS |
259 | if (pc_is_kernel_fn(pc, _switch_to) || |
260 | pc_is_kernel_fn(pc, _switch_to_ret)) { | |
c8921d72 HD |
261 | info->prev_sp = info->sp - CALLEE_SAVE_FRAME_SIZE; |
262 | info->prev_ip = *(unsigned long *)(info->prev_sp - RP_OFFSET); | |
263 | return 1; | |
264 | } | |
05dc16d6 | 265 | |
c8921d72 | 266 | #ifdef CONFIG_IRQSTACKS |
8e0ba125 | 267 | if (pc_is_kernel_fn(pc, _call_on_stack)) { |
c8921d72 HD |
268 | info->prev_sp = *(unsigned long *)(info->sp - FRAME_SIZE - REG_SZ); |
269 | info->prev_ip = *(unsigned long *)(info->sp - FRAME_SIZE - RP_OFFSET); | |
05dc16d6 RC |
270 | return 1; |
271 | } | |
c8921d72 | 272 | #endif |
05dc16d6 RC |
273 | return 0; |
274 | } | |
275 | ||
1da177e4 LT |
276 | static void unwind_frame_regs(struct unwind_frame_info *info) |
277 | { | |
278 | const struct unwind_table_entry *e; | |
279 | unsigned long npc; | |
280 | unsigned int insn; | |
281 | long frame_size = 0; | |
282 | int looking_for_rp, rpoffset = 0; | |
283 | ||
284 | e = find_unwind_entry(info->ip); | |
285 | if (e == NULL) { | |
286 | unsigned long sp; | |
1da177e4 | 287 | |
c8921d72 HD |
288 | dbg("Cannot find unwind entry for %pS; forced unwinding\n", |
289 | (void *) info->ip); | |
1da177e4 LT |
290 | |
291 | /* Since we are doing the unwinding blind, we don't know if | |
292 | we are adjusting the stack correctly or extracting the rp | |
293 | correctly. The rp is checked to see if it belongs to the | |
294 | kernel text section, if not we assume we don't have a | |
295 | correct stack frame and we continue to unwind the stack. | |
296 | This is not quite correct, and will fail for loadable | |
297 | modules. */ | |
298 | sp = info->sp & ~63; | |
299 | do { | |
300 | unsigned long tmp; | |
301 | ||
302 | info->prev_sp = sp - 64; | |
303 | info->prev_ip = 0; | |
e77900ab | 304 | |
2214c0e7 HD |
305 | /* Check if stack is inside kernel stack area */ |
306 | if ((info->prev_sp - (unsigned long) task_stack_page(info->t)) | |
307 | >= THREAD_SIZE) { | |
e77900ab HD |
308 | info->prev_sp = 0; |
309 | break; | |
310 | } | |
311 | ||
cf2ec789 SS |
312 | if (copy_from_kernel_nofault(&tmp, |
313 | (void *)info->prev_sp - RP_OFFSET, sizeof(tmp))) | |
1da177e4 LT |
314 | break; |
315 | info->prev_ip = tmp; | |
316 | sp = info->prev_sp; | |
c790b41b | 317 | } while (!kernel_text_address(info->prev_ip)); |
1da177e4 LT |
318 | |
319 | info->rp = 0; | |
320 | ||
321 | dbg("analyzing func @ %lx with no unwind info, setting " | |
322 | "prev_sp=%lx prev_ip=%lx\n", info->ip, | |
323 | info->prev_sp, info->prev_ip); | |
324 | } else { | |
325 | dbg("e->start = 0x%x, e->end = 0x%x, Save_SP = %d, " | |
326 | "Save_RP = %d, Millicode = %d size = %u\n", | |
327 | e->region_start, e->region_end, e->Save_SP, e->Save_RP, | |
328 | e->Millicode, e->Total_frame_size); | |
329 | ||
330 | looking_for_rp = e->Save_RP; | |
331 | ||
332 | for (npc = e->region_start; | |
333 | (frame_size < (e->Total_frame_size << 3) || | |
334 | looking_for_rp) && | |
335 | npc < info->ip; | |
336 | npc += 4) { | |
337 | ||
338 | insn = *(unsigned int *)npc; | |
339 | ||
be24a897 MP |
340 | if ((insn & 0xffffc001) == 0x37de0000 || |
341 | (insn & 0xffe00001) == 0x6fc00000) { | |
1da177e4 | 342 | /* ldo X(sp), sp, or stwm X,D(sp) */ |
be24a897 | 343 | frame_size += (insn & 0x3fff) >> 1; |
1da177e4 LT |
344 | dbg("analyzing func @ %lx, insn=%08x @ " |
345 | "%lx, frame_size = %ld\n", info->ip, | |
346 | insn, npc, frame_size); | |
be24a897 | 347 | } else if ((insn & 0xffe00009) == 0x73c00008) { |
1da177e4 | 348 | /* std,ma X,D(sp) */ |
be24a897 | 349 | frame_size += ((insn >> 4) & 0x3ff) << 3; |
1da177e4 LT |
350 | dbg("analyzing func @ %lx, insn=%08x @ " |
351 | "%lx, frame_size = %ld\n", info->ip, | |
352 | insn, npc, frame_size); | |
353 | } else if (insn == 0x6bc23fd9) { | |
354 | /* stw rp,-20(sp) */ | |
355 | rpoffset = 20; | |
356 | looking_for_rp = 0; | |
357 | dbg("analyzing func @ %lx, insn=stw rp," | |
358 | "-20(sp) @ %lx\n", info->ip, npc); | |
359 | } else if (insn == 0x0fc212c1) { | |
360 | /* std rp,-16(sr0,sp) */ | |
361 | rpoffset = 16; | |
362 | looking_for_rp = 0; | |
363 | dbg("analyzing func @ %lx, insn=std rp," | |
364 | "-16(sp) @ %lx\n", info->ip, npc); | |
365 | } | |
366 | } | |
367 | ||
be24a897 MP |
368 | if (frame_size > e->Total_frame_size << 3) |
369 | frame_size = e->Total_frame_size << 3; | |
370 | ||
05dc16d6 RC |
371 | if (!unwind_special(info, e->region_start, frame_size)) { |
372 | info->prev_sp = info->sp - frame_size; | |
373 | if (e->Millicode) | |
374 | info->rp = info->r31; | |
375 | else if (rpoffset) | |
376 | info->rp = *(unsigned long *)(info->prev_sp - rpoffset); | |
377 | info->prev_ip = info->rp; | |
378 | info->rp = 0; | |
379 | } | |
1da177e4 LT |
380 | |
381 | dbg("analyzing func @ %lx, setting prev_sp=%lx " | |
382 | "prev_ip=%lx npc=%lx\n", info->ip, info->prev_sp, | |
383 | info->prev_ip, npc); | |
384 | } | |
385 | } | |
386 | ||
387 | void unwind_frame_init(struct unwind_frame_info *info, struct task_struct *t, | |
388 | struct pt_regs *regs) | |
389 | { | |
390 | memset(info, 0, sizeof(struct unwind_frame_info)); | |
391 | info->t = t; | |
392 | info->sp = regs->gr[30]; | |
393 | info->ip = regs->iaoq[0]; | |
394 | info->rp = regs->gr[2]; | |
395 | info->r31 = regs->gr[31]; | |
396 | ||
397 | dbg("(%d) Start unwind from sp=%08lx ip=%08lx\n", | |
398 | t ? (int)t->pid : -1, info->sp, info->ip); | |
399 | } | |
400 | ||
401 | void unwind_frame_init_from_blocked_task(struct unwind_frame_info *info, struct task_struct *t) | |
402 | { | |
403 | struct pt_regs *r = &t->thread.regs; | |
404 | struct pt_regs *r2; | |
405 | ||
e0e7ed48 | 406 | r2 = kmalloc(sizeof(struct pt_regs), GFP_ATOMIC); |
1da177e4 LT |
407 | if (!r2) |
408 | return; | |
409 | *r2 = *r; | |
410 | r2->gr[30] = r->ksp; | |
411 | r2->iaoq[0] = r->kpc; | |
412 | unwind_frame_init(info, t, r2); | |
413 | kfree(r2); | |
414 | } | |
415 | ||
9e0d5c45 HD |
416 | #define get_parisc_stackpointer() ({ \ |
417 | unsigned long sp; \ | |
418 | __asm__("copy %%r30, %0" : "=r"(sp)); \ | |
419 | (sp); \ | |
420 | }) | |
421 | ||
422 | void unwind_frame_init_task(struct unwind_frame_info *info, | |
423 | struct task_struct *task, struct pt_regs *regs) | |
1da177e4 | 424 | { |
9e0d5c45 HD |
425 | task = task ? task : current; |
426 | ||
427 | if (task == current) { | |
428 | struct pt_regs r; | |
429 | ||
430 | if (!regs) { | |
431 | memset(&r, 0, sizeof(r)); | |
432 | r.iaoq[0] = _THIS_IP_; | |
433 | r.gr[2] = _RET_IP_; | |
434 | r.gr[30] = get_parisc_stackpointer(); | |
435 | regs = &r; | |
436 | } | |
cf8afe5c | 437 | unwind_frame_init(info, task, regs); |
9e0d5c45 HD |
438 | } else { |
439 | unwind_frame_init_from_blocked_task(info, task); | |
440 | } | |
1da177e4 LT |
441 | } |
442 | ||
443 | int unwind_once(struct unwind_frame_info *next_frame) | |
444 | { | |
445 | unwind_frame_regs(next_frame); | |
446 | ||
447 | if (next_frame->prev_sp == 0 || | |
448 | next_frame->prev_ip == 0) | |
449 | return -1; | |
450 | ||
451 | next_frame->sp = next_frame->prev_sp; | |
452 | next_frame->ip = next_frame->prev_ip; | |
453 | next_frame->prev_sp = 0; | |
454 | next_frame->prev_ip = 0; | |
455 | ||
456 | dbg("(%d) Continue unwind to sp=%08lx ip=%08lx\n", | |
457 | next_frame->t ? (int)next_frame->t->pid : -1, | |
458 | next_frame->sp, next_frame->ip); | |
459 | ||
460 | return 0; | |
461 | } | |
462 | ||
463 | int unwind_to_user(struct unwind_frame_info *info) | |
464 | { | |
465 | int ret; | |
466 | ||
467 | do { | |
468 | ret = unwind_once(info); | |
469 | } while (!ret && !(info->ip & 3)); | |
470 | ||
471 | return ret; | |
472 | } | |
11e17809 HD |
473 | |
474 | unsigned long return_address(unsigned int level) | |
475 | { | |
476 | struct unwind_frame_info info; | |
11e17809 HD |
477 | |
478 | /* initialize unwind info */ | |
9e0d5c45 | 479 | unwind_frame_init_task(&info, current, NULL); |
11e17809 HD |
480 | |
481 | /* unwind stack */ | |
9e0d5c45 | 482 | level += 2; |
11e17809 HD |
483 | do { |
484 | if (unwind_once(&info) < 0 || info.ip == 0) | |
485 | return 0; | |
c790b41b | 486 | if (!kernel_text_address(info.ip)) |
11e17809 | 487 | return 0; |
11e17809 HD |
488 | } while (info.ip && level--); |
489 | ||
490 | return info.ip; | |
491 | } |