Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
1da177e4 LT |
2 | /* Kernel module help for PPC64. |
3 | Copyright (C) 2001, 2003 Rusty Russell IBM Corporation. | |
4 | ||
1da177e4 | 5 | */ |
c7d1f6af AB |
6 | |
7 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
8 | ||
1da177e4 LT |
9 | #include <linux/module.h> |
10 | #include <linux/elf.h> | |
11 | #include <linux/moduleloader.h> | |
12 | #include <linux/err.h> | |
13 | #include <linux/vmalloc.h> | |
f48cb8b4 | 14 | #include <linux/ftrace.h> |
73c9ceab | 15 | #include <linux/bug.h> |
83775b85 | 16 | #include <linux/uaccess.h> |
1da177e4 | 17 | #include <asm/module.h> |
21c4ff80 | 18 | #include <asm/firmware.h> |
b7bcda63 | 19 | #include <asm/code-patching.h> |
eda09fbd | 20 | #include <linux/sort.h> |
b88c4767 | 21 | #include <asm/setup.h> |
336a7b5d | 22 | #include <asm/sections.h> |
75346251 | 23 | #include <asm/inst.h> |
1da177e4 LT |
24 | |
25 | /* FIXME: We don't do .init separately. To do this, we'd need to have | |
26 | a separate r2 value in the init and core section, and stub between | |
27 | them, too. | |
28 | ||
29 | Using a magic allocator which places modules within 32MB solves | |
30 | this, and makes other things simpler. Anton? | |
31 | --RR. */ | |
1da177e4 | 32 | |
f55d9665 | 33 | #ifdef PPC64_ELF_ABI_v2 |
008d7a91 RR |
34 | |
35 | /* An address is simply the address of the function. */ | |
36 | typedef unsigned long func_desc_t; | |
37 | ||
38 | static func_desc_t func_desc(unsigned long addr) | |
39 | { | |
40 | return addr; | |
41 | } | |
42 | static unsigned long func_addr(unsigned long addr) | |
43 | { | |
44 | return addr; | |
45 | } | |
46 | static unsigned long stub_func_addr(func_desc_t func) | |
47 | { | |
48 | return func; | |
49 | } | |
50 | ||
51 | /* PowerPC64 specific values for the Elf64_Sym st_other field. */ | |
52 | #define STO_PPC64_LOCAL_BIT 5 | |
53 | #define STO_PPC64_LOCAL_MASK (7 << STO_PPC64_LOCAL_BIT) | |
54 | #define PPC64_LOCAL_ENTRY_OFFSET(other) \ | |
55 | (((1 << (((other) & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT)) >> 2) << 2) | |
56 | ||
57 | static unsigned int local_entry_offset(const Elf64_Sym *sym) | |
58 | { | |
59 | /* sym->st_other indicates offset to local entry point | |
60 | * (otherwise it will assume r12 is the address of the start | |
61 | * of function and try to derive r2 from it). */ | |
62 | return PPC64_LOCAL_ENTRY_OFFSET(sym->st_other); | |
63 | } | |
d2fae548 | 64 | #else |
008d7a91 RR |
65 | |
66 | /* An address is address of the OPD entry, which contains address of fn. */ | |
67 | typedef struct ppc64_opd_entry func_desc_t; | |
68 | ||
69 | static func_desc_t func_desc(unsigned long addr) | |
70 | { | |
71 | return *(struct ppc64_opd_entry *)addr; | |
72 | } | |
73 | static unsigned long func_addr(unsigned long addr) | |
74 | { | |
75 | return func_desc(addr).funcaddr; | |
76 | } | |
77 | static unsigned long stub_func_addr(func_desc_t func) | |
78 | { | |
79 | return func.funcaddr; | |
80 | } | |
81 | static unsigned int local_entry_offset(const Elf64_Sym *sym) | |
82 | { | |
83 | return 0; | |
84 | } | |
5633e85b SS |
85 | |
86 | void *dereference_module_function_descriptor(struct module *mod, void *ptr) | |
87 | { | |
88 | if (ptr < (void *)mod->arch.start_opd || | |
89 | ptr >= (void *)mod->arch.end_opd) | |
90 | return ptr; | |
91 | ||
92 | return dereference_function_descriptor(ptr); | |
93 | } | |
d2fae548 RR |
94 | #endif |
95 | ||
f17c4e01 ME |
96 | #define STUB_MAGIC 0x73747562 /* stub */ |
97 | ||
1da177e4 LT |
98 | /* Like PPC32, we need little trampolines to do > 24-bit jumps (into |
99 | the kernel itself). But on PPC64, these need to be used for every | |
100 | jump, actually, to reset r2 (TOC+0x8000). */ | |
101 | struct ppc64_stub_entry | |
102 | { | |
83775b85 AB |
103 | /* 28 byte jump instruction sequence (7 instructions). We only |
104 | * need 6 instructions on ABIv2 but we always allocate 7 so | |
105 | * so we don't have to modify the trampoline load instruction. */ | |
0e60e46e | 106 | u32 jump[7]; |
f17c4e01 ME |
107 | /* Used by ftrace to identify stubs */ |
108 | u32 magic; | |
1da177e4 | 109 | /* Data for the above code */ |
008d7a91 | 110 | func_desc_t funcdata; |
1da177e4 LT |
111 | }; |
112 | ||
5c729a11 RR |
113 | /* |
114 | * PPC64 uses 24 bit jumps, but we need to jump into other modules or | |
115 | * the kernel which may be further. So we jump to a stub. | |
116 | * | |
117 | * For ELFv1 we need to use this to set up the new r2 value (aka TOC | |
118 | * pointer). For ELFv2 it's the callee's responsibility to set up the | |
119 | * new r2, but for both we need to save the old r2. | |
120 | * | |
121 | * We could simply patch the new r2 value and function pointer into | |
122 | * the stub, but it's significantly shorter to put these values at the | |
123 | * end of the stub code, and patch the stub address (32-bits relative | |
124 | * to the TOC ptr, r2) into the stub. | |
125 | */ | |
83775b85 | 126 | static u32 ppc64_stub_insns[] = { |
47b04699 CL |
127 | PPC_RAW_ADDIS(_R11, _R2, 0), |
128 | PPC_RAW_ADDI(_R11, _R11, 0), | |
fed8393e | 129 | /* Save current r2 value in magic place on the stack. */ |
47b04699 CL |
130 | PPC_RAW_STD(_R2, _R1, R2_STACK_OFFSET), |
131 | PPC_RAW_LD(_R12, _R11, 32), | |
f55d9665 | 132 | #ifdef PPC64_ELF_ABI_v1 |
5c729a11 | 133 | /* Set up new r2 from function descriptor */ |
47b04699 | 134 | PPC_RAW_LD(_R2, _R11, 40), |
5c729a11 | 135 | #endif |
47b04699 CL |
136 | PPC_RAW_MTCTR(_R12), |
137 | PPC_RAW_BCTR(), | |
83775b85 AB |
138 | }; |
139 | ||
1da177e4 LT |
140 | /* Count how many different 24-bit relocations (different symbol, |
141 | different addend) */ | |
142 | static unsigned int count_relocs(const Elf64_Rela *rela, unsigned int num) | |
143 | { | |
eda09fbd | 144 | unsigned int i, r_info, r_addend, _count_relocs; |
1da177e4 LT |
145 | |
146 | /* FIXME: Only count external ones --RR */ | |
eda09fbd EM |
147 | _count_relocs = 0; |
148 | r_info = 0; | |
149 | r_addend = 0; | |
150 | for (i = 0; i < num; i++) | |
1da177e4 | 151 | /* Only count 24-bit relocs, others don't need stubs */ |
eda09fbd EM |
152 | if (ELF64_R_TYPE(rela[i].r_info) == R_PPC_REL24 && |
153 | (r_info != ELF64_R_SYM(rela[i].r_info) || | |
154 | r_addend != rela[i].r_addend)) { | |
155 | _count_relocs++; | |
156 | r_info = ELF64_R_SYM(rela[i].r_info); | |
157 | r_addend = rela[i].r_addend; | |
1da177e4 | 158 | } |
eda09fbd EM |
159 | |
160 | return _count_relocs; | |
1da177e4 LT |
161 | } |
162 | ||
eda09fbd EM |
163 | static int relacmp(const void *_x, const void *_y) |
164 | { | |
165 | const Elf64_Rela *x, *y; | |
166 | ||
167 | y = (Elf64_Rela *)_x; | |
168 | x = (Elf64_Rela *)_y; | |
169 | ||
170 | /* Compare the entire r_info (as opposed to ELF64_R_SYM(r_info) only) to | |
171 | * make the comparison cheaper/faster. It won't affect the sorting or | |
172 | * the counting algorithms' performance | |
173 | */ | |
174 | if (x->r_info < y->r_info) | |
175 | return -1; | |
176 | else if (x->r_info > y->r_info) | |
177 | return 1; | |
178 | else if (x->r_addend < y->r_addend) | |
179 | return -1; | |
180 | else if (x->r_addend > y->r_addend) | |
181 | return 1; | |
182 | else | |
183 | return 0; | |
184 | } | |
185 | ||
1da177e4 LT |
186 | /* Get size of potential trampolines required. */ |
187 | static unsigned long get_stubs_size(const Elf64_Ehdr *hdr, | |
188 | const Elf64_Shdr *sechdrs) | |
189 | { | |
190 | /* One extra reloc so it's always 0-funcaddr terminated */ | |
191 | unsigned long relocs = 1; | |
192 | unsigned i; | |
193 | ||
194 | /* Every relocated section... */ | |
195 | for (i = 1; i < hdr->e_shnum; i++) { | |
196 | if (sechdrs[i].sh_type == SHT_RELA) { | |
c7d1f6af AB |
197 | pr_debug("Found relocations in section %u\n", i); |
198 | pr_debug("Ptr: %p. Number: %Lu\n", | |
1da177e4 LT |
199 | (void *)sechdrs[i].sh_addr, |
200 | sechdrs[i].sh_size / sizeof(Elf64_Rela)); | |
eda09fbd EM |
201 | |
202 | /* Sort the relocation information based on a symbol and | |
203 | * addend key. This is a stable O(n*log n) complexity | |
204 | * alogrithm but it will reduce the complexity of | |
205 | * count_relocs() to linear complexity O(n) | |
206 | */ | |
207 | sort((void *)sechdrs[i].sh_addr, | |
208 | sechdrs[i].sh_size / sizeof(Elf64_Rela), | |
bac7ca7b | 209 | sizeof(Elf64_Rela), relacmp, NULL); |
eda09fbd | 210 | |
1da177e4 LT |
211 | relocs += count_relocs((void *)sechdrs[i].sh_addr, |
212 | sechdrs[i].sh_size | |
213 | / sizeof(Elf64_Rela)); | |
214 | } | |
215 | } | |
216 | ||
f48cb8b4 SR |
217 | #ifdef CONFIG_DYNAMIC_FTRACE |
218 | /* make the trampoline to the ftrace_caller */ | |
219 | relocs++; | |
ae30cc05 NR |
220 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
221 | /* an additional one for ftrace_regs_caller */ | |
222 | relocs++; | |
223 | #endif | |
f48cb8b4 SR |
224 | #endif |
225 | ||
c7d1f6af | 226 | pr_debug("Looks like a total of %lu stubs, max\n", relocs); |
1da177e4 LT |
227 | return relocs * sizeof(struct ppc64_stub_entry); |
228 | } | |
229 | ||
5b12c5c6 | 230 | /* Still needed for ELFv2, for .TOC. */ |
1da177e4 LT |
231 | static void dedotify_versions(struct modversion_info *vers, |
232 | unsigned long size) | |
233 | { | |
234 | struct modversion_info *end; | |
235 | ||
236 | for (end = (void *)vers + size; vers < end; vers++) | |
c2cbcf53 | 237 | if (vers->name[0] == '.') { |
1da177e4 | 238 | memmove(vers->name, vers->name+1, strlen(vers->name)); |
c2cbcf53 | 239 | } |
1da177e4 LT |
240 | } |
241 | ||
c153693d AM |
242 | /* |
243 | * Undefined symbols which refer to .funcname, hack to funcname. Make .TOC. | |
244 | * seem to be defined (value set later). | |
245 | */ | |
1da177e4 LT |
246 | static void dedotify(Elf64_Sym *syms, unsigned int numsyms, char *strtab) |
247 | { | |
248 | unsigned int i; | |
249 | ||
250 | for (i = 1; i < numsyms; i++) { | |
251 | if (syms[i].st_shndx == SHN_UNDEF) { | |
252 | char *name = strtab + syms[i].st_name; | |
c153693d AM |
253 | if (name[0] == '.') { |
254 | if (strcmp(name+1, "TOC.") == 0) | |
255 | syms[i].st_shndx = SHN_ABS; | |
f15838e9 | 256 | syms[i].st_name++; |
c153693d | 257 | } |
1da177e4 LT |
258 | } |
259 | } | |
260 | } | |
261 | ||
4edebbea RR |
262 | static Elf64_Sym *find_dot_toc(Elf64_Shdr *sechdrs, |
263 | const char *strtab, | |
264 | unsigned int symindex) | |
265 | { | |
266 | unsigned int i, numsyms; | |
267 | Elf64_Sym *syms; | |
268 | ||
269 | syms = (Elf64_Sym *)sechdrs[symindex].sh_addr; | |
270 | numsyms = sechdrs[symindex].sh_size / sizeof(Elf64_Sym); | |
271 | ||
272 | for (i = 1; i < numsyms; i++) { | |
c153693d | 273 | if (syms[i].st_shndx == SHN_ABS |
008d7a91 | 274 | && strcmp(strtab + syms[i].st_name, "TOC.") == 0) |
4edebbea RR |
275 | return &syms[i]; |
276 | } | |
277 | return NULL; | |
278 | } | |
279 | ||
d4be60fe WAF |
280 | bool module_init_section(const char *name) |
281 | { | |
282 | /* We don't handle .init for the moment: always return false. */ | |
283 | return false; | |
284 | } | |
285 | ||
1da177e4 LT |
286 | int module_frob_arch_sections(Elf64_Ehdr *hdr, |
287 | Elf64_Shdr *sechdrs, | |
288 | char *secstrings, | |
289 | struct module *me) | |
290 | { | |
291 | unsigned int i; | |
292 | ||
293 | /* Find .toc and .stubs sections, symtab and strtab */ | |
294 | for (i = 1; i < hdr->e_shnum; i++) { | |
1da177e4 LT |
295 | if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0) |
296 | me->arch.stubs_section = i; | |
5c45b528 | 297 | else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0) { |
1da177e4 | 298 | me->arch.toc_section = i; |
5c45b528 AM |
299 | if (sechdrs[i].sh_addralign < 8) |
300 | sechdrs[i].sh_addralign = 8; | |
301 | } | |
1da177e4 LT |
302 | else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0) |
303 | dedotify_versions((void *)hdr + sechdrs[i].sh_offset, | |
304 | sechdrs[i].sh_size); | |
305 | ||
1da177e4 LT |
306 | if (sechdrs[i].sh_type == SHT_SYMTAB) |
307 | dedotify((void *)hdr + sechdrs[i].sh_offset, | |
308 | sechdrs[i].sh_size / sizeof(Elf64_Sym), | |
309 | (void *)hdr | |
310 | + sechdrs[sechdrs[i].sh_link].sh_offset); | |
311 | } | |
f749edae AM |
312 | |
313 | if (!me->arch.stubs_section) { | |
c7d1f6af | 314 | pr_err("%s: doesn't contain .stubs.\n", me->name); |
1da177e4 LT |
315 | return -ENOEXEC; |
316 | } | |
317 | ||
f749edae AM |
318 | /* If we don't have a .toc, just use .stubs. We need to set r2 |
319 | to some reasonable value in case the module calls out to | |
320 | other functions via a stub, or if a function pointer escapes | |
321 | the module by some means. */ | |
322 | if (!me->arch.toc_section) | |
323 | me->arch.toc_section = me->arch.stubs_section; | |
324 | ||
1da177e4 LT |
325 | /* Override the stubs size */ |
326 | sechdrs[me->arch.stubs_section].sh_size = get_stubs_size(hdr, sechdrs); | |
327 | return 0; | |
328 | } | |
329 | ||
bd55e792 NR |
330 | #ifdef CONFIG_MPROFILE_KERNEL |
331 | ||
bd55e792 | 332 | static u32 stub_insns[] = { |
47b04699 CL |
333 | PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernel_toc)), |
334 | PPC_RAW_ADDIS(_R12, _R12, 0), | |
335 | PPC_RAW_ADDI(_R12, _R12, 0), | |
336 | PPC_RAW_MTCTR(_R12), | |
337 | PPC_RAW_BCTR(), | |
bd55e792 NR |
338 | }; |
339 | ||
340 | /* | |
341 | * For mprofile-kernel we use a special stub for ftrace_caller() because we | |
342 | * can't rely on r2 containing this module's TOC when we enter the stub. | |
343 | * | |
344 | * That can happen if the function calling us didn't need to use the toc. In | |
345 | * that case it won't have setup r2, and the r2 value will be either the | |
346 | * kernel's toc, or possibly another modules toc. | |
347 | * | |
348 | * To deal with that this stub uses the kernel toc, which is always accessible | |
349 | * via the paca (in r13). The target (ftrace_caller()) is responsible for | |
350 | * saving and restoring the toc before returning. | |
351 | */ | |
352 | static inline int create_ftrace_stub(struct ppc64_stub_entry *entry, | |
353 | unsigned long addr, | |
354 | struct module *me) | |
355 | { | |
356 | long reladdr; | |
357 | ||
358 | memcpy(entry->jump, stub_insns, sizeof(stub_insns)); | |
359 | ||
360 | /* Stub uses address relative to kernel toc (from the paca) */ | |
361 | reladdr = addr - kernel_toc_addr(); | |
362 | if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { | |
363 | pr_err("%s: Address of %ps out of range of kernel_toc.\n", | |
364 | me->name, (void *)addr); | |
365 | return 0; | |
366 | } | |
367 | ||
368 | entry->jump[1] |= PPC_HA(reladdr); | |
369 | entry->jump[2] |= PPC_LO(reladdr); | |
370 | ||
371 | /* Eventhough we don't use funcdata in the stub, it's needed elsewhere. */ | |
372 | entry->funcdata = func_desc(addr); | |
373 | entry->magic = STUB_MAGIC; | |
374 | ||
375 | return 1; | |
376 | } | |
377 | ||
378 | static bool is_mprofile_ftrace_call(const char *name) | |
379 | { | |
380 | if (!strcmp("_mcount", name)) | |
381 | return true; | |
382 | #ifdef CONFIG_DYNAMIC_FTRACE | |
383 | if (!strcmp("ftrace_caller", name)) | |
384 | return true; | |
385 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS | |
386 | if (!strcmp("ftrace_regs_caller", name)) | |
387 | return true; | |
388 | #endif | |
389 | #endif | |
390 | ||
391 | return false; | |
392 | } | |
393 | #else | |
394 | static inline int create_ftrace_stub(struct ppc64_stub_entry *entry, | |
395 | unsigned long addr, | |
396 | struct module *me) | |
397 | { | |
398 | return 0; | |
399 | } | |
400 | ||
401 | static bool is_mprofile_ftrace_call(const char *name) | |
402 | { | |
403 | return false; | |
404 | } | |
405 | #endif | |
406 | ||
5c45b528 AM |
407 | /* |
408 | * r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the | |
409 | * value maximum span in an instruction which uses a signed offset). Round down | |
410 | * to a 256 byte boundary for the odd case where we are setting up r2 without a | |
411 | * .toc section. | |
412 | */ | |
136cd345 | 413 | static inline unsigned long my_r2(const Elf64_Shdr *sechdrs, struct module *me) |
1da177e4 | 414 | { |
5c45b528 | 415 | return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000; |
1da177e4 LT |
416 | } |
417 | ||
1da177e4 | 418 | /* Patch stub to reference function and correct r2 value. */ |
136cd345 | 419 | static inline int create_stub(const Elf64_Shdr *sechdrs, |
1da177e4 | 420 | struct ppc64_stub_entry *entry, |
008d7a91 | 421 | unsigned long addr, |
bd55e792 NR |
422 | struct module *me, |
423 | const char *name) | |
1da177e4 | 424 | { |
1da177e4 | 425 | long reladdr; |
8734b41b RC |
426 | func_desc_t desc; |
427 | int i; | |
1da177e4 | 428 | |
bd55e792 NR |
429 | if (is_mprofile_ftrace_call(name)) |
430 | return create_ftrace_stub(entry, addr, me); | |
431 | ||
8734b41b RC |
432 | for (i = 0; i < sizeof(ppc64_stub_insns) / sizeof(u32); i++) { |
433 | if (patch_instruction(&entry->jump[i], | |
434 | ppc_inst(ppc64_stub_insns[i]))) | |
435 | return 0; | |
436 | } | |
1da177e4 | 437 | |
1da177e4 LT |
438 | /* Stub uses address relative to r2. */ |
439 | reladdr = (unsigned long)entry - my_r2(sechdrs, me); | |
440 | if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { | |
c7d1f6af | 441 | pr_err("%s: Address %p of stub out of range of %p.\n", |
1da177e4 LT |
442 | me->name, (void *)reladdr, (void *)my_r2); |
443 | return 0; | |
444 | } | |
c7d1f6af | 445 | pr_debug("Stub %p get data from reladdr %li\n", entry, reladdr); |
1da177e4 | 446 | |
8734b41b RC |
447 | if (patch_instruction(&entry->jump[0], |
448 | ppc_inst(entry->jump[0] | PPC_HA(reladdr)))) | |
449 | return 0; | |
450 | ||
451 | if (patch_instruction(&entry->jump[1], | |
452 | ppc_inst(entry->jump[1] | PPC_LO(reladdr)))) | |
453 | return 0; | |
454 | ||
455 | // func_desc_t is 8 bytes if ABIv2, else 16 bytes | |
456 | desc = func_desc(addr); | |
457 | for (i = 0; i < sizeof(func_desc_t) / sizeof(u32); i++) { | |
458 | if (patch_instruction(((u32 *)&entry->funcdata) + i, | |
459 | ppc_inst(((u32 *)(&desc))[i]))) | |
460 | return 0; | |
461 | } | |
462 | ||
463 | if (patch_instruction(&entry->magic, ppc_inst(STUB_MAGIC))) | |
464 | return 0; | |
f17c4e01 | 465 | |
1da177e4 LT |
466 | return 1; |
467 | } | |
468 | ||
008d7a91 | 469 | /* Create stub to jump to function described in this OPD/ptr: we need the |
1da177e4 | 470 | stub to set up the TOC ptr (r2) for the function. */ |
136cd345 | 471 | static unsigned long stub_for_addr(const Elf64_Shdr *sechdrs, |
008d7a91 | 472 | unsigned long addr, |
bd55e792 NR |
473 | struct module *me, |
474 | const char *name) | |
1da177e4 LT |
475 | { |
476 | struct ppc64_stub_entry *stubs; | |
1da177e4 LT |
477 | unsigned int i, num_stubs; |
478 | ||
479 | num_stubs = sechdrs[me->arch.stubs_section].sh_size / sizeof(*stubs); | |
480 | ||
481 | /* Find this stub, or if that fails, the next avail. entry */ | |
482 | stubs = (void *)sechdrs[me->arch.stubs_section].sh_addr; | |
008d7a91 | 483 | for (i = 0; stub_func_addr(stubs[i].funcdata); i++) { |
1c0437af KB |
484 | if (WARN_ON(i >= num_stubs)) |
485 | return 0; | |
1da177e4 | 486 | |
008d7a91 | 487 | if (stub_func_addr(stubs[i].funcdata) == func_addr(addr)) |
1da177e4 LT |
488 | return (unsigned long)&stubs[i]; |
489 | } | |
490 | ||
bd55e792 | 491 | if (!create_stub(sechdrs, &stubs[i], addr, me, name)) |
1da177e4 LT |
492 | return 0; |
493 | ||
494 | return (unsigned long)&stubs[i]; | |
495 | } | |
496 | ||
497 | /* We expect a noop next: if it is, replace it with instruction to | |
498 | restore r2. */ | |
250122ba | 499 | static int restore_r2(const char *name, u32 *instruction, struct module *me) |
1da177e4 | 500 | { |
b9eab08d JP |
501 | u32 *prev_insn = instruction - 1; |
502 | ||
1f2aaed2 | 503 | if (is_mprofile_ftrace_call(name)) |
b9eab08d JP |
504 | return 1; |
505 | ||
506 | /* | |
507 | * Make sure the branch isn't a sibling call. Sibling calls aren't | |
508 | * "link" branches and they don't return, so they don't need the r2 | |
509 | * restore afterwards. | |
510 | */ | |
75346251 | 511 | if (!instr_is_relative_link_branch(ppc_inst(*prev_insn))) |
31278b17 ME |
512 | return 1; |
513 | ||
47b04699 | 514 | if (*instruction != PPC_RAW_NOP()) { |
1ea61ea2 JP |
515 | pr_err("%s: Expected nop after call, got %08x at %pS\n", |
516 | me->name, *instruction, instruction); | |
1da177e4 LT |
517 | return 0; |
518 | } | |
8734b41b | 519 | |
d2fae548 | 520 | /* ld r2,R2_STACK_OFFSET(r1) */ |
8734b41b RC |
521 | if (patch_instruction(instruction, ppc_inst(PPC_INST_LD_TOC))) |
522 | return 0; | |
523 | ||
1da177e4 LT |
524 | return 1; |
525 | } | |
526 | ||
527 | int apply_relocate_add(Elf64_Shdr *sechdrs, | |
528 | const char *strtab, | |
529 | unsigned int symindex, | |
530 | unsigned int relsec, | |
531 | struct module *me) | |
532 | { | |
533 | unsigned int i; | |
534 | Elf64_Rela *rela = (void *)sechdrs[relsec].sh_addr; | |
535 | Elf64_Sym *sym; | |
536 | unsigned long *location; | |
537 | unsigned long value; | |
538 | ||
c7d1f6af | 539 | pr_debug("Applying ADD relocate section %u to %u\n", relsec, |
1da177e4 | 540 | sechdrs[relsec].sh_info); |
4edebbea RR |
541 | |
542 | /* First time we're called, we can fix up .TOC. */ | |
543 | if (!me->arch.toc_fixed) { | |
544 | sym = find_dot_toc(sechdrs, strtab, symindex); | |
545 | /* It's theoretically possible that a module doesn't want a | |
546 | * .TOC. so don't fail it just for that. */ | |
547 | if (sym) | |
548 | sym->st_value = my_r2(sechdrs, me); | |
549 | me->arch.toc_fixed = true; | |
550 | } | |
551 | ||
1da177e4 LT |
552 | for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) { |
553 | /* This is where to make the change */ | |
554 | location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr | |
555 | + rela[i].r_offset; | |
556 | /* This is the symbol it is referring to */ | |
557 | sym = (Elf64_Sym *)sechdrs[symindex].sh_addr | |
558 | + ELF64_R_SYM(rela[i].r_info); | |
559 | ||
c7d1f6af | 560 | pr_debug("RELOC at %p: %li-type as %s (0x%lx) + %li\n", |
1da177e4 LT |
561 | location, (long)ELF64_R_TYPE(rela[i].r_info), |
562 | strtab + sym->st_name, (unsigned long)sym->st_value, | |
563 | (long)rela[i].r_addend); | |
564 | ||
565 | /* `Everything is relative'. */ | |
566 | value = sym->st_value + rela[i].r_addend; | |
567 | ||
568 | switch (ELF64_R_TYPE(rela[i].r_info)) { | |
569 | case R_PPC64_ADDR32: | |
570 | /* Simply set it */ | |
571 | *(u32 *)location = value; | |
572 | break; | |
eda09fbd | 573 | |
1da177e4 LT |
574 | case R_PPC64_ADDR64: |
575 | /* Simply set it */ | |
576 | *(unsigned long *)location = value; | |
577 | break; | |
578 | ||
579 | case R_PPC64_TOC: | |
580 | *(unsigned long *)location = my_r2(sechdrs, me); | |
581 | break; | |
582 | ||
9149ccfa | 583 | case R_PPC64_TOC16: |
f749edae | 584 | /* Subtract TOC pointer */ |
9149ccfa PB |
585 | value -= my_r2(sechdrs, me); |
586 | if (value + 0x8000 > 0xffff) { | |
c7d1f6af | 587 | pr_err("%s: bad TOC16 relocation (0x%lx)\n", |
9149ccfa PB |
588 | me->name, value); |
589 | return -ENOEXEC; | |
590 | } | |
591 | *((uint16_t *) location) | |
592 | = (*((uint16_t *) location) & ~0xffff) | |
593 | | (value & 0xffff); | |
594 | break; | |
595 | ||
1fbe9cf2 AB |
596 | case R_PPC64_TOC16_LO: |
597 | /* Subtract TOC pointer */ | |
598 | value -= my_r2(sechdrs, me); | |
599 | *((uint16_t *) location) | |
600 | = (*((uint16_t *) location) & ~0xffff) | |
601 | | (value & 0xffff); | |
602 | break; | |
603 | ||
1da177e4 | 604 | case R_PPC64_TOC16_DS: |
f749edae | 605 | /* Subtract TOC pointer */ |
1da177e4 LT |
606 | value -= my_r2(sechdrs, me); |
607 | if ((value & 3) != 0 || value + 0x8000 > 0xffff) { | |
c7d1f6af | 608 | pr_err("%s: bad TOC16_DS relocation (0x%lx)\n", |
1da177e4 LT |
609 | me->name, value); |
610 | return -ENOEXEC; | |
611 | } | |
612 | *((uint16_t *) location) | |
613 | = (*((uint16_t *) location) & ~0xfffc) | |
614 | | (value & 0xfffc); | |
615 | break; | |
616 | ||
1fbe9cf2 AB |
617 | case R_PPC64_TOC16_LO_DS: |
618 | /* Subtract TOC pointer */ | |
619 | value -= my_r2(sechdrs, me); | |
620 | if ((value & 3) != 0) { | |
c7d1f6af | 621 | pr_err("%s: bad TOC16_LO_DS relocation (0x%lx)\n", |
1fbe9cf2 AB |
622 | me->name, value); |
623 | return -ENOEXEC; | |
624 | } | |
625 | *((uint16_t *) location) | |
626 | = (*((uint16_t *) location) & ~0xfffc) | |
627 | | (value & 0xfffc); | |
628 | break; | |
629 | ||
630 | case R_PPC64_TOC16_HA: | |
631 | /* Subtract TOC pointer */ | |
632 | value -= my_r2(sechdrs, me); | |
633 | value = ((value + 0x8000) >> 16); | |
634 | *((uint16_t *) location) | |
635 | = (*((uint16_t *) location) & ~0xffff) | |
636 | | (value & 0xffff); | |
637 | break; | |
638 | ||
1da177e4 LT |
639 | case R_PPC_REL24: |
640 | /* FIXME: Handle weak symbols here --RR */ | |
a443bf6e KB |
641 | if (sym->st_shndx == SHN_UNDEF || |
642 | sym->st_shndx == SHN_LIVEPATCH) { | |
1da177e4 | 643 | /* External: go via stub */ |
bd55e792 NR |
644 | value = stub_for_addr(sechdrs, value, me, |
645 | strtab + sym->st_name); | |
1da177e4 LT |
646 | if (!value) |
647 | return -ENOENT; | |
250122ba NR |
648 | if (!restore_r2(strtab + sym->st_name, |
649 | (u32 *)location + 1, me)) | |
1da177e4 | 650 | return -ENOEXEC; |
008d7a91 RR |
651 | } else |
652 | value += local_entry_offset(sym); | |
1da177e4 LT |
653 | |
654 | /* Convert value to relative */ | |
655 | value -= (unsigned long)location; | |
656 | if (value + 0x2000000 > 0x3ffffff || (value & 3) != 0){ | |
c7d1f6af | 657 | pr_err("%s: REL24 %li out of range!\n", |
1da177e4 LT |
658 | me->name, (long int)value); |
659 | return -ENOEXEC; | |
660 | } | |
661 | ||
662 | /* Only replace bits 2 through 26 */ | |
8734b41b | 663 | value = (*(uint32_t *)location & ~0x03fffffc) |
1da177e4 | 664 | | (value & 0x03fffffc); |
8734b41b RC |
665 | |
666 | if (patch_instruction((u32 *)location, ppc_inst(value))) | |
667 | return -EFAULT; | |
668 | ||
1da177e4 LT |
669 | break; |
670 | ||
21c4ff80 BH |
671 | case R_PPC64_REL64: |
672 | /* 64 bits relative (used by features fixups) */ | |
673 | *location = value - (unsigned long)location; | |
674 | break; | |
675 | ||
9f751b82 ME |
676 | case R_PPC64_REL32: |
677 | /* 32 bits relative (used by relative exception tables) */ | |
b851ba02 NP |
678 | /* Convert value to relative */ |
679 | value -= (unsigned long)location; | |
680 | if (value + 0x80000000 > 0xffffffff) { | |
681 | pr_err("%s: REL32 %li out of range!\n", | |
682 | me->name, (long int)value); | |
683 | return -ENOEXEC; | |
684 | } | |
685 | *(u32 *)location = value; | |
9f751b82 ME |
686 | break; |
687 | ||
d247da0a RR |
688 | case R_PPC64_TOCSAVE: |
689 | /* | |
690 | * Marker reloc indicates we don't have to save r2. | |
691 | * That would only save us one instruction, so ignore | |
692 | * it. | |
693 | */ | |
694 | break; | |
695 | ||
a61674bd UW |
696 | case R_PPC64_ENTRY: |
697 | /* | |
698 | * Optimize ELFv2 large code model entry point if | |
699 | * the TOC is within 2GB range of current location. | |
700 | */ | |
701 | value = my_r2(sechdrs, me) - (unsigned long)location; | |
702 | if (value + 0x80008000 > 0xffffffff) | |
703 | break; | |
704 | /* | |
705 | * Check for the large code model prolog sequence: | |
706 | * ld r2, ...(r12) | |
707 | * add r2, r2, r12 | |
708 | */ | |
47b04699 | 709 | if ((((uint32_t *)location)[0] & ~0xfffc) != PPC_RAW_LD(_R2, _R12, 0)) |
a61674bd | 710 | break; |
47b04699 | 711 | if (((uint32_t *)location)[1] != PPC_RAW_ADD(_R2, _R2, _R12)) |
a61674bd UW |
712 | break; |
713 | /* | |
714 | * If found, replace it with: | |
715 | * addis r2, r12, (.TOC.-func)@ha | |
2fb0a2c9 | 716 | * addi r2, r2, (.TOC.-func)@l |
a61674bd | 717 | */ |
47b04699 CL |
718 | ((uint32_t *)location)[0] = PPC_RAW_ADDIS(_R2, _R12, PPC_HA(value)); |
719 | ((uint32_t *)location)[1] = PPC_RAW_ADDI(_R2, _R2, PPC_LO(value)); | |
a61674bd UW |
720 | break; |
721 | ||
0906584a RR |
722 | case R_PPC64_REL16_HA: |
723 | /* Subtract location pointer */ | |
724 | value -= (unsigned long)location; | |
725 | value = ((value + 0x8000) >> 16); | |
726 | *((uint16_t *) location) | |
727 | = (*((uint16_t *) location) & ~0xffff) | |
728 | | (value & 0xffff); | |
729 | break; | |
730 | ||
731 | case R_PPC64_REL16_LO: | |
732 | /* Subtract location pointer */ | |
733 | value -= (unsigned long)location; | |
734 | *((uint16_t *) location) | |
735 | = (*((uint16_t *) location) & ~0xffff) | |
736 | | (value & 0xffff); | |
737 | break; | |
738 | ||
1da177e4 | 739 | default: |
c7d1f6af | 740 | pr_err("%s: Unknown ADD relocation: %lu\n", |
1da177e4 LT |
741 | me->name, |
742 | (unsigned long)ELF64_R_TYPE(rela[i].r_info)); | |
743 | return -ENOEXEC; | |
744 | } | |
745 | } | |
746 | ||
136cd345 ME |
747 | return 0; |
748 | } | |
749 | ||
f48cb8b4 | 750 | #ifdef CONFIG_DYNAMIC_FTRACE |
03b51416 NR |
751 | int module_trampoline_target(struct module *mod, unsigned long addr, |
752 | unsigned long *target) | |
753 | { | |
754 | struct ppc64_stub_entry *stub; | |
755 | func_desc_t funcdata; | |
756 | u32 magic; | |
757 | ||
758 | if (!within_module_core(addr, mod)) { | |
759 | pr_err("%s: stub %lx not in module %s\n", __func__, addr, mod->name); | |
760 | return -EFAULT; | |
761 | } | |
762 | ||
763 | stub = (struct ppc64_stub_entry *)addr; | |
764 | ||
fe557319 CH |
765 | if (copy_from_kernel_nofault(&magic, &stub->magic, |
766 | sizeof(magic))) { | |
03b51416 NR |
767 | pr_err("%s: fault reading magic for stub %lx for %s\n", __func__, addr, mod->name); |
768 | return -EFAULT; | |
769 | } | |
770 | ||
771 | if (magic != STUB_MAGIC) { | |
772 | pr_err("%s: bad magic for stub %lx for %s\n", __func__, addr, mod->name); | |
773 | return -EFAULT; | |
774 | } | |
775 | ||
fe557319 CH |
776 | if (copy_from_kernel_nofault(&funcdata, &stub->funcdata, |
777 | sizeof(funcdata))) { | |
03b51416 NR |
778 | pr_err("%s: fault reading funcdata for stub %lx for %s\n", __func__, addr, mod->name); |
779 | return -EFAULT; | |
780 | } | |
781 | ||
782 | *target = stub_func_addr(funcdata); | |
783 | ||
784 | return 0; | |
785 | } | |
336a7b5d | 786 | |
136cd345 ME |
787 | int module_finalize_ftrace(struct module *mod, const Elf_Shdr *sechdrs) |
788 | { | |
bd55e792 NR |
789 | mod->arch.tramp = stub_for_addr(sechdrs, |
790 | (unsigned long)ftrace_caller, | |
791 | mod, | |
792 | "ftrace_caller"); | |
ae30cc05 | 793 | #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS |
bd55e792 NR |
794 | mod->arch.tramp_regs = stub_for_addr(sechdrs, |
795 | (unsigned long)ftrace_regs_caller, | |
796 | mod, | |
797 | "ftrace_regs_caller"); | |
ae30cc05 NR |
798 | if (!mod->arch.tramp_regs) |
799 | return -ENOENT; | |
800 | #endif | |
136cd345 ME |
801 | |
802 | if (!mod->arch.tramp) | |
803 | return -ENOENT; | |
f48cb8b4 | 804 | |
1da177e4 LT |
805 | return 0; |
806 | } | |
136cd345 | 807 | #endif |