Commit | Line | Data |
---|---|---|
1a59d1b8 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
f71d20e9 | 2 | /* |
24b9f0d2 SS |
3 | * Copyright (C) 2002 Richard Henderson |
4 | * Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM. | |
5 | */ | |
51161bfc LR |
6 | |
7 | #define INCLUDE_VERMAGIC | |
8 | ||
9984de1a | 9 | #include <linux/export.h> |
8a293be0 | 10 | #include <linux/extable.h> |
1da177e4 | 11 | #include <linux/moduleloader.h> |
c8424e77 | 12 | #include <linux/module_signature.h> |
af658dca | 13 | #include <linux/trace_events.h> |
1da177e4 | 14 | #include <linux/init.h> |
ae84e324 | 15 | #include <linux/kallsyms.h> |
9294523e | 16 | #include <linux/buildid.h> |
3b5d5c6b | 17 | #include <linux/fs.h> |
9f158333 | 18 | #include <linux/kernel.h> |
b89999d0 | 19 | #include <linux/kernel_read_file.h> |
1da177e4 LT |
20 | #include <linux/slab.h> |
21 | #include <linux/vmalloc.h> | |
22 | #include <linux/elf.h> | |
23 | #include <linux/seq_file.h> | |
24 | #include <linux/syscalls.h> | |
25 | #include <linux/fcntl.h> | |
26 | #include <linux/rcupdate.h> | |
c59ede7b | 27 | #include <linux/capability.h> |
1da177e4 LT |
28 | #include <linux/cpu.h> |
29 | #include <linux/moduleparam.h> | |
30 | #include <linux/errno.h> | |
31 | #include <linux/err.h> | |
32 | #include <linux/vermagic.h> | |
33 | #include <linux/notifier.h> | |
f6a57033 | 34 | #include <linux/sched.h> |
1da177e4 | 35 | #include <linux/device.h> |
c988d2b2 | 36 | #include <linux/string.h> |
97d1f15b | 37 | #include <linux/mutex.h> |
d72b3751 | 38 | #include <linux/rculist.h> |
7c0f6ba6 | 39 | #include <linux/uaccess.h> |
1da177e4 | 40 | #include <asm/cacheflush.h> |
563ec5cb | 41 | #include <linux/set_memory.h> |
eb8cdec4 | 42 | #include <asm/mmu_context.h> |
b817f6fe | 43 | #include <linux/license.h> |
6d762394 | 44 | #include <asm/sections.h> |
97e1c18e | 45 | #include <linux/tracepoint.h> |
90d595fe | 46 | #include <linux/ftrace.h> |
7e545d6e | 47 | #include <linux/livepatch.h> |
22a9d645 | 48 | #include <linux/async.h> |
fbf59bc9 | 49 | #include <linux/percpu.h> |
4f2294b6 | 50 | #include <linux/kmemleak.h> |
bf5438fc | 51 | #include <linux/jump_label.h> |
84e1c6bb | 52 | #include <linux/pfn.h> |
403ed278 | 53 | #include <linux/bsearch.h> |
9d5059c9 | 54 | #include <linux/dynamic_debug.h> |
ca86cad7 | 55 | #include <linux/audit.h> |
2f3238ae | 56 | #include <uapi/linux/module.h> |
cfc1d277 | 57 | #include "internal.h" |
1da177e4 | 58 | |
7ead8b83 LZ |
59 | #define CREATE_TRACE_POINTS |
60 | #include <trace/events/module.h> | |
61 | ||
75676500 RR |
62 | /* |
63 | * Mutex protects: | |
64 | * 1) List of modules (also safely readable with preempt_disable), | |
65 | * 2) module_use links, | |
55ce556d | 66 | * 3) mod_tree.addr_min/mod_tree.addr_max. |
24b9f0d2 SS |
67 | * (delete and add uses RCU list operations). |
68 | */ | |
8ab4ed08 AT |
69 | DEFINE_MUTEX(module_mutex); |
70 | LIST_HEAD(modules); | |
67fc4e0c | 71 | |
1a7b7d92 | 72 | /* Work queue for freeing init sections in success case */ |
fdf09ab8 DJ |
73 | static void do_free_init(struct work_struct *w); |
74 | static DECLARE_WORK(init_free_wq, do_free_init); | |
75 | static LLIST_HEAD(init_free_list); | |
1a7b7d92 | 76 | |
58d208de | 77 | struct mod_tree_root mod_tree __cacheline_aligned = { |
4f666546 | 78 | .addr_min = -1UL, |
106a4ee2 | 79 | }; |
106a4ee2 | 80 | |
01dc0386 CL |
81 | #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC |
82 | struct mod_tree_root mod_data_tree __cacheline_aligned = { | |
83 | .addr_min = -1UL, | |
84 | }; | |
85 | #endif | |
86 | ||
4f666546 PZ |
87 | #define module_addr_min mod_tree.addr_min |
88 | #define module_addr_max mod_tree.addr_max | |
89 | ||
47889798 AT |
90 | struct symsearch { |
91 | const struct kernel_symbol *start, *stop; | |
92 | const s32 *crcs; | |
93 | enum mod_license license; | |
94 | }; | |
95 | ||
4f666546 PZ |
96 | /* |
97 | * Bounds of module text, for speeding up __module_address. | |
98 | * Protected by module_mutex. | |
99 | */ | |
446d5566 | 100 | static void __mod_update_bounds(void *base, unsigned int size, struct mod_tree_root *tree) |
4f666546 PZ |
101 | { |
102 | unsigned long min = (unsigned long)base; | |
103 | unsigned long max = min + size; | |
104 | ||
446d5566 CL |
105 | if (min < tree->addr_min) |
106 | tree->addr_min = min; | |
107 | if (max > tree->addr_max) | |
108 | tree->addr_max = max; | |
4f666546 PZ |
109 | } |
110 | ||
111 | static void mod_update_bounds(struct module *mod) | |
112 | { | |
446d5566 | 113 | __mod_update_bounds(mod->core_layout.base, mod->core_layout.size, &mod_tree); |
7523e4dc | 114 | if (mod->init_layout.size) |
446d5566 | 115 | __mod_update_bounds(mod->init_layout.base, mod->init_layout.size, &mod_tree); |
01dc0386 CL |
116 | #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC |
117 | __mod_update_bounds(mod->data_layout.base, mod->data_layout.size, &mod_data_tree); | |
118 | #endif | |
4f666546 PZ |
119 | } |
120 | ||
19e4529e SR |
121 | /* Block module loading/unloading? */ |
122 | int modules_disabled = 0; | |
02608bef | 123 | core_param(nomodule, modules_disabled, bint, 0); |
19e4529e | 124 | |
c9a3ba55 RR |
125 | /* Waiting for a module to finish initializing? */ |
126 | static DECLARE_WAIT_QUEUE_HEAD(module_wq); | |
127 | ||
e041c683 | 128 | static BLOCKING_NOTIFIER_HEAD(module_notify_list); |
1da177e4 | 129 | |
6da0b565 | 130 | int register_module_notifier(struct notifier_block *nb) |
1da177e4 | 131 | { |
e041c683 | 132 | return blocking_notifier_chain_register(&module_notify_list, nb); |
1da177e4 LT |
133 | } |
134 | EXPORT_SYMBOL(register_module_notifier); | |
135 | ||
6da0b565 | 136 | int unregister_module_notifier(struct notifier_block *nb) |
1da177e4 | 137 | { |
e041c683 | 138 | return blocking_notifier_chain_unregister(&module_notify_list, nb); |
1da177e4 LT |
139 | } |
140 | EXPORT_SYMBOL(unregister_module_notifier); | |
141 | ||
71d9f507 MB |
142 | /* |
143 | * We require a truly strong try_module_get(): 0 means success. | |
144 | * Otherwise an error is returned due to ongoing or failed | |
145 | * initialization etc. | |
146 | */ | |
1da177e4 LT |
147 | static inline int strong_try_module_get(struct module *mod) |
148 | { | |
0d21b0e3 | 149 | BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED); |
1da177e4 | 150 | if (mod && mod->state == MODULE_STATE_COMING) |
c9a3ba55 RR |
151 | return -EBUSY; |
152 | if (try_module_get(mod)) | |
1da177e4 | 153 | return 0; |
c9a3ba55 RR |
154 | else |
155 | return -ENOENT; | |
1da177e4 LT |
156 | } |
157 | ||
373d4d09 RR |
158 | static inline void add_taint_module(struct module *mod, unsigned flag, |
159 | enum lockdep_ok lockdep_ok) | |
fa3ba2e8 | 160 | { |
373d4d09 | 161 | add_taint(flag, lockdep_ok); |
7fd8329b | 162 | set_bit(flag, &mod->taints); |
fa3ba2e8 FM |
163 | } |
164 | ||
02a3e59a RD |
165 | /* |
166 | * A thread that wants to hold a reference to a module only while it | |
f49169c9 | 167 | * is running can call this to safely exit. |
1da177e4 | 168 | */ |
ca3574bd | 169 | void __noreturn __module_put_and_kthread_exit(struct module *mod, long code) |
1da177e4 LT |
170 | { |
171 | module_put(mod); | |
ca3574bd | 172 | kthread_exit(code); |
1da177e4 | 173 | } |
ca3574bd | 174 | EXPORT_SYMBOL(__module_put_and_kthread_exit); |
22a8bdeb | 175 | |
1da177e4 | 176 | /* Find a module section: 0 means not found. */ |
49668688 | 177 | static unsigned int find_sec(const struct load_info *info, const char *name) |
1da177e4 LT |
178 | { |
179 | unsigned int i; | |
180 | ||
49668688 RR |
181 | for (i = 1; i < info->hdr->e_shnum; i++) { |
182 | Elf_Shdr *shdr = &info->sechdrs[i]; | |
1da177e4 | 183 | /* Alloc bit cleared means "ignore it." */ |
49668688 RR |
184 | if ((shdr->sh_flags & SHF_ALLOC) |
185 | && strcmp(info->secstrings + shdr->sh_name, name) == 0) | |
1da177e4 | 186 | return i; |
49668688 | 187 | } |
1da177e4 LT |
188 | return 0; |
189 | } | |
190 | ||
5e458cc0 | 191 | /* Find a module section, or NULL. */ |
49668688 | 192 | static void *section_addr(const struct load_info *info, const char *name) |
5e458cc0 RR |
193 | { |
194 | /* Section 0 has sh_addr 0. */ | |
49668688 | 195 | return (void *)info->sechdrs[find_sec(info, name)].sh_addr; |
5e458cc0 RR |
196 | } |
197 | ||
198 | /* Find a module section, or NULL. Fill in number of "objects" in section. */ | |
49668688 | 199 | static void *section_objs(const struct load_info *info, |
5e458cc0 RR |
200 | const char *name, |
201 | size_t object_size, | |
202 | unsigned int *num) | |
203 | { | |
49668688 | 204 | unsigned int sec = find_sec(info, name); |
5e458cc0 RR |
205 | |
206 | /* Section 0 has sh_addr 0 and sh_size 0. */ | |
49668688 RR |
207 | *num = info->sechdrs[sec].sh_size / object_size; |
208 | return (void *)info->sechdrs[sec].sh_addr; | |
5e458cc0 RR |
209 | } |
210 | ||
36e68442 AN |
211 | /* Find a module section: 0 means not found. Ignores SHF_ALLOC flag. */ |
212 | static unsigned int find_any_sec(const struct load_info *info, const char *name) | |
213 | { | |
214 | unsigned int i; | |
215 | ||
216 | for (i = 1; i < info->hdr->e_shnum; i++) { | |
217 | Elf_Shdr *shdr = &info->sechdrs[i]; | |
218 | if (strcmp(info->secstrings + shdr->sh_name, name) == 0) | |
219 | return i; | |
220 | } | |
221 | return 0; | |
222 | } | |
223 | ||
224 | /* | |
225 | * Find a module section, or NULL. Fill in number of "objects" in section. | |
226 | * Ignores SHF_ALLOC flag. | |
227 | */ | |
228 | static __maybe_unused void *any_section_objs(const struct load_info *info, | |
229 | const char *name, | |
230 | size_t object_size, | |
231 | unsigned int *num) | |
232 | { | |
233 | unsigned int sec = find_any_sec(info, name); | |
234 | ||
235 | /* Section 0 has sh_addr 0 and sh_size 0. */ | |
236 | *num = info->sechdrs[sec].sh_size / object_size; | |
237 | return (void *)info->sechdrs[sec].sh_addr; | |
238 | } | |
239 | ||
1da177e4 LT |
240 | #ifndef CONFIG_MODVERSIONS |
241 | #define symversion(base, idx) NULL | |
242 | #else | |
f83ca9fe | 243 | #define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL) |
1da177e4 LT |
244 | #endif |
245 | ||
7290d580 AB |
246 | static const char *kernel_symbol_name(const struct kernel_symbol *sym) |
247 | { | |
248 | #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS | |
249 | return offset_to_ptr(&sym->name_offset); | |
250 | #else | |
251 | return sym->name; | |
252 | #endif | |
253 | } | |
254 | ||
8651ec01 MM |
255 | static const char *kernel_symbol_namespace(const struct kernel_symbol *sym) |
256 | { | |
257 | #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS | |
069e1c07 WD |
258 | if (!sym->namespace_offset) |
259 | return NULL; | |
8651ec01 MM |
260 | return offset_to_ptr(&sym->namespace_offset); |
261 | #else | |
262 | return sym->namespace; | |
263 | #endif | |
264 | } | |
265 | ||
91fb02f3 | 266 | int cmp_name(const void *name, const void *sym) |
403ed278 | 267 | { |
b605be65 | 268 | return strcmp(name, kernel_symbol_name(sym)); |
403ed278 AIB |
269 | } |
270 | ||
2d25bc55 JY |
271 | static bool find_exported_symbol_in_section(const struct symsearch *syms, |
272 | struct module *owner, | |
c6eee9df | 273 | struct find_symbol_arg *fsa) |
de4d8d53 | 274 | { |
403ed278 AIB |
275 | struct kernel_symbol *sym; |
276 | ||
cdd66eb5 MY |
277 | if (!fsa->gplok && syms->license == GPL_ONLY) |
278 | return false; | |
279 | ||
403ed278 AIB |
280 | sym = bsearch(fsa->name, syms->start, syms->stop - syms->start, |
281 | sizeof(struct kernel_symbol), cmp_name); | |
7390b94a MY |
282 | if (!sym) |
283 | return false; | |
403ed278 | 284 | |
7390b94a MY |
285 | fsa->owner = owner; |
286 | fsa->crc = symversion(syms->crcs, sym - syms->start); | |
287 | fsa->sym = sym; | |
288 | fsa->license = syms->license; | |
de4d8d53 | 289 | |
7390b94a | 290 | return true; |
de4d8d53 RR |
291 | } |
292 | ||
24b9f0d2 SS |
293 | /* |
294 | * Find an exported symbol and return it, along with, (optional) crc and | |
295 | * (optional) module which owns it. Needs preempt disabled or module_mutex. | |
296 | */ | |
47889798 | 297 | bool find_symbol(struct find_symbol_arg *fsa) |
dafd0940 | 298 | { |
71e4b309 CH |
299 | static const struct symsearch arr[] = { |
300 | { __start___ksymtab, __stop___ksymtab, __start___kcrctab, | |
36794822 | 301 | NOT_GPL_ONLY }, |
71e4b309 CH |
302 | { __start___ksymtab_gpl, __stop___ksymtab_gpl, |
303 | __start___kcrctab_gpl, | |
36794822 | 304 | GPL_ONLY }, |
71e4b309 | 305 | }; |
71e4b309 CH |
306 | struct module *mod; |
307 | unsigned int i; | |
dafd0940 | 308 | |
71e4b309 | 309 | module_assert_mutex_or_preempt(); |
dafd0940 | 310 | |
71e4b309 | 311 | for (i = 0; i < ARRAY_SIZE(arr); i++) |
0b96615c CH |
312 | if (find_exported_symbol_in_section(&arr[i], NULL, fsa)) |
313 | return true; | |
71e4b309 CH |
314 | |
315 | list_for_each_entry_rcu(mod, &modules, list, | |
316 | lockdep_is_held(&module_mutex)) { | |
317 | struct symsearch arr[] = { | |
318 | { mod->syms, mod->syms + mod->num_syms, mod->crcs, | |
36794822 | 319 | NOT_GPL_ONLY }, |
71e4b309 CH |
320 | { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms, |
321 | mod->gpl_crcs, | |
36794822 | 322 | GPL_ONLY }, |
71e4b309 CH |
323 | }; |
324 | ||
325 | if (mod->state == MODULE_STATE_UNFORMED) | |
326 | continue; | |
327 | ||
328 | for (i = 0; i < ARRAY_SIZE(arr); i++) | |
0b96615c CH |
329 | if (find_exported_symbol_in_section(&arr[i], mod, fsa)) |
330 | return true; | |
dafd0940 RR |
331 | } |
332 | ||
0b96615c CH |
333 | pr_debug("Failed to find symbol %s\n", fsa->name); |
334 | return false; | |
1da177e4 LT |
335 | } |
336 | ||
fe0d34d2 RR |
337 | /* |
338 | * Search for module by name: must hold module_mutex (or preempt disabled | |
339 | * for read-only access). | |
340 | */ | |
91fb02f3 AT |
341 | struct module *find_module_all(const char *name, size_t len, |
342 | bool even_unformed) | |
1da177e4 LT |
343 | { |
344 | struct module *mod; | |
345 | ||
fe0d34d2 | 346 | module_assert_mutex_or_preempt(); |
0be964be | 347 | |
bf08949c MH |
348 | list_for_each_entry_rcu(mod, &modules, list, |
349 | lockdep_is_held(&module_mutex)) { | |
0d21b0e3 RR |
350 | if (!even_unformed && mod->state == MODULE_STATE_UNFORMED) |
351 | continue; | |
4f6de4d5 | 352 | if (strlen(mod->name) == len && !memcmp(mod->name, name, len)) |
1da177e4 LT |
353 | return mod; |
354 | } | |
355 | return NULL; | |
356 | } | |
0d21b0e3 RR |
357 | |
358 | struct module *find_module(const char *name) | |
359 | { | |
4f6de4d5 | 360 | return find_module_all(name, strlen(name), false); |
0d21b0e3 | 361 | } |
1da177e4 LT |
362 | |
363 | #ifdef CONFIG_SMP | |
fbf59bc9 | 364 | |
259354de | 365 | static inline void __percpu *mod_percpu(struct module *mod) |
fbf59bc9 | 366 | { |
259354de TH |
367 | return mod->percpu; |
368 | } | |
fbf59bc9 | 369 | |
9eb76d77 | 370 | static int percpu_modalloc(struct module *mod, struct load_info *info) |
259354de | 371 | { |
9eb76d77 RR |
372 | Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu]; |
373 | unsigned long align = pcpusec->sh_addralign; | |
374 | ||
375 | if (!pcpusec->sh_size) | |
376 | return 0; | |
377 | ||
fbf59bc9 | 378 | if (align > PAGE_SIZE) { |
bddb12b3 AM |
379 | pr_warn("%s: per-cpu alignment %li > %li\n", |
380 | mod->name, align, PAGE_SIZE); | |
fbf59bc9 TH |
381 | align = PAGE_SIZE; |
382 | } | |
383 | ||
9eb76d77 | 384 | mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align); |
259354de | 385 | if (!mod->percpu) { |
bddb12b3 AM |
386 | pr_warn("%s: Could not allocate %lu bytes percpu data\n", |
387 | mod->name, (unsigned long)pcpusec->sh_size); | |
259354de TH |
388 | return -ENOMEM; |
389 | } | |
9eb76d77 | 390 | mod->percpu_size = pcpusec->sh_size; |
259354de | 391 | return 0; |
fbf59bc9 TH |
392 | } |
393 | ||
259354de | 394 | static void percpu_modfree(struct module *mod) |
fbf59bc9 | 395 | { |
259354de | 396 | free_percpu(mod->percpu); |
fbf59bc9 TH |
397 | } |
398 | ||
49668688 | 399 | static unsigned int find_pcpusec(struct load_info *info) |
6b588c18 | 400 | { |
49668688 | 401 | return find_sec(info, ".data..percpu"); |
6b588c18 TH |
402 | } |
403 | ||
259354de TH |
404 | static void percpu_modcopy(struct module *mod, |
405 | const void *from, unsigned long size) | |
6b588c18 TH |
406 | { |
407 | int cpu; | |
408 | ||
409 | for_each_possible_cpu(cpu) | |
259354de | 410 | memcpy(per_cpu_ptr(mod->percpu, cpu), from, size); |
6b588c18 TH |
411 | } |
412 | ||
383776fa | 413 | bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr) |
10fad5e4 TH |
414 | { |
415 | struct module *mod; | |
416 | unsigned int cpu; | |
417 | ||
418 | preempt_disable(); | |
419 | ||
420 | list_for_each_entry_rcu(mod, &modules, list) { | |
0d21b0e3 RR |
421 | if (mod->state == MODULE_STATE_UNFORMED) |
422 | continue; | |
10fad5e4 TH |
423 | if (!mod->percpu_size) |
424 | continue; | |
425 | for_each_possible_cpu(cpu) { | |
426 | void *start = per_cpu_ptr(mod->percpu, cpu); | |
383776fa | 427 | void *va = (void *)addr; |
10fad5e4 | 428 | |
383776fa | 429 | if (va >= start && va < start + mod->percpu_size) { |
8ce371f9 | 430 | if (can_addr) { |
383776fa | 431 | *can_addr = (unsigned long) (va - start); |
8ce371f9 PZ |
432 | *can_addr += (unsigned long) |
433 | per_cpu_ptr(mod->percpu, | |
434 | get_boot_cpu_id()); | |
435 | } | |
10fad5e4 TH |
436 | preempt_enable(); |
437 | return true; | |
438 | } | |
439 | } | |
440 | } | |
441 | ||
442 | preempt_enable(); | |
443 | return false; | |
6b588c18 TH |
444 | } |
445 | ||
383776fa | 446 | /** |
24389b61 | 447 | * is_module_percpu_address() - test whether address is from module static percpu |
383776fa TG |
448 | * @addr: address to test |
449 | * | |
450 | * Test whether @addr belongs to module static percpu area. | |
451 | * | |
24389b61 | 452 | * Return: %true if @addr is from module static percpu area |
383776fa TG |
453 | */ |
454 | bool is_module_percpu_address(unsigned long addr) | |
455 | { | |
456 | return __is_module_percpu_address(addr, NULL); | |
457 | } | |
458 | ||
1da177e4 | 459 | #else /* ... !CONFIG_SMP */ |
6b588c18 | 460 | |
259354de | 461 | static inline void __percpu *mod_percpu(struct module *mod) |
1da177e4 LT |
462 | { |
463 | return NULL; | |
464 | } | |
9eb76d77 | 465 | static int percpu_modalloc(struct module *mod, struct load_info *info) |
259354de | 466 | { |
9eb76d77 RR |
467 | /* UP modules shouldn't have this section: ENOMEM isn't quite right */ |
468 | if (info->sechdrs[info->index.pcpu].sh_size != 0) | |
469 | return -ENOMEM; | |
470 | return 0; | |
259354de TH |
471 | } |
472 | static inline void percpu_modfree(struct module *mod) | |
1da177e4 | 473 | { |
1da177e4 | 474 | } |
49668688 | 475 | static unsigned int find_pcpusec(struct load_info *info) |
1da177e4 LT |
476 | { |
477 | return 0; | |
478 | } | |
259354de TH |
479 | static inline void percpu_modcopy(struct module *mod, |
480 | const void *from, unsigned long size) | |
1da177e4 LT |
481 | { |
482 | /* pcpusec should be 0, and size of that section should be 0. */ | |
483 | BUG_ON(size != 0); | |
484 | } | |
10fad5e4 TH |
485 | bool is_module_percpu_address(unsigned long addr) |
486 | { | |
487 | return false; | |
488 | } | |
6b588c18 | 489 | |
383776fa TG |
490 | bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr) |
491 | { | |
492 | return false; | |
493 | } | |
494 | ||
1da177e4 LT |
495 | #endif /* CONFIG_SMP */ |
496 | ||
c988d2b2 MD |
497 | #define MODINFO_ATTR(field) \ |
498 | static void setup_modinfo_##field(struct module *mod, const char *s) \ | |
499 | { \ | |
500 | mod->field = kstrdup(s, GFP_KERNEL); \ | |
501 | } \ | |
502 | static ssize_t show_modinfo_##field(struct module_attribute *mattr, \ | |
4befb026 | 503 | struct module_kobject *mk, char *buffer) \ |
c988d2b2 | 504 | { \ |
cc56ded3 | 505 | return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \ |
c988d2b2 MD |
506 | } \ |
507 | static int modinfo_##field##_exists(struct module *mod) \ | |
508 | { \ | |
509 | return mod->field != NULL; \ | |
510 | } \ | |
511 | static void free_modinfo_##field(struct module *mod) \ | |
512 | { \ | |
22a8bdeb DW |
513 | kfree(mod->field); \ |
514 | mod->field = NULL; \ | |
c988d2b2 MD |
515 | } \ |
516 | static struct module_attribute modinfo_##field = { \ | |
7b595756 | 517 | .attr = { .name = __stringify(field), .mode = 0444 }, \ |
c988d2b2 MD |
518 | .show = show_modinfo_##field, \ |
519 | .setup = setup_modinfo_##field, \ | |
520 | .test = modinfo_##field##_exists, \ | |
521 | .free = free_modinfo_##field, \ | |
522 | }; | |
523 | ||
524 | MODINFO_ATTR(version); | |
525 | MODINFO_ATTR(srcversion); | |
526 | ||
e14af7ee AV |
527 | static char last_unloaded_module[MODULE_NAME_LEN+1]; |
528 | ||
03e88ae1 | 529 | #ifdef CONFIG_MODULE_UNLOAD |
eb0c5377 SR |
530 | |
531 | EXPORT_TRACEPOINT_SYMBOL(module_get); | |
532 | ||
e513cc1c MH |
533 | /* MODULE_REF_BASE is the base reference count by kmodule loader. */ |
534 | #define MODULE_REF_BASE 1 | |
535 | ||
1da177e4 | 536 | /* Init the unload section of the module. */ |
9f85a4bb | 537 | static int module_unload_init(struct module *mod) |
1da177e4 | 538 | { |
e513cc1c MH |
539 | /* |
540 | * Initialize reference counter to MODULE_REF_BASE. | |
541 | * refcnt == 0 means module is going. | |
542 | */ | |
543 | atomic_set(&mod->refcnt, MODULE_REF_BASE); | |
9f85a4bb | 544 | |
2c02dfe7 LT |
545 | INIT_LIST_HEAD(&mod->source_list); |
546 | INIT_LIST_HEAD(&mod->target_list); | |
e1783a24 | 547 | |
1da177e4 | 548 | /* Hold reference count during initialization. */ |
e513cc1c | 549 | atomic_inc(&mod->refcnt); |
9f85a4bb RR |
550 | |
551 | return 0; | |
1da177e4 LT |
552 | } |
553 | ||
1da177e4 LT |
554 | /* Does a already use b? */ |
555 | static int already_uses(struct module *a, struct module *b) | |
556 | { | |
557 | struct module_use *use; | |
558 | ||
2c02dfe7 LT |
559 | list_for_each_entry(use, &b->source_list, source_list) { |
560 | if (use->source == a) { | |
5e124169 | 561 | pr_debug("%s uses %s!\n", a->name, b->name); |
1da177e4 LT |
562 | return 1; |
563 | } | |
564 | } | |
5e124169 | 565 | pr_debug("%s does not use %s!\n", a->name, b->name); |
1da177e4 LT |
566 | return 0; |
567 | } | |
568 | ||
2c02dfe7 LT |
569 | /* |
570 | * Module a uses b | |
571 | * - we add 'a' as a "source", 'b' as a "target" of module use | |
572 | * - the module_use is added to the list of 'b' sources (so | |
573 | * 'b' can walk the list to see who sourced them), and of 'a' | |
574 | * targets (so 'a' can see what modules it targets). | |
575 | */ | |
576 | static int add_module_usage(struct module *a, struct module *b) | |
577 | { | |
2c02dfe7 LT |
578 | struct module_use *use; |
579 | ||
5e124169 | 580 | pr_debug("Allocating new usage for %s.\n", a->name); |
2c02dfe7 | 581 | use = kmalloc(sizeof(*use), GFP_ATOMIC); |
9ad04574 | 582 | if (!use) |
2c02dfe7 | 583 | return -ENOMEM; |
2c02dfe7 LT |
584 | |
585 | use->source = a; | |
586 | use->target = b; | |
587 | list_add(&use->source_list, &b->source_list); | |
588 | list_add(&use->target_list, &a->target_list); | |
2c02dfe7 LT |
589 | return 0; |
590 | } | |
591 | ||
75676500 | 592 | /* Module a uses b: caller needs module_mutex() */ |
7ef5264d | 593 | static int ref_module(struct module *a, struct module *b) |
1da177e4 | 594 | { |
c8e21ced | 595 | int err; |
270a6c4c | 596 | |
9bea7f23 | 597 | if (b == NULL || already_uses(a, b)) |
218ce735 | 598 | return 0; |
218ce735 | 599 | |
9bea7f23 RR |
600 | /* If module isn't available, we fail. */ |
601 | err = strong_try_module_get(b); | |
c9a3ba55 | 602 | if (err) |
9bea7f23 | 603 | return err; |
1da177e4 | 604 | |
2c02dfe7 LT |
605 | err = add_module_usage(a, b); |
606 | if (err) { | |
1da177e4 | 607 | module_put(b); |
9bea7f23 | 608 | return err; |
1da177e4 | 609 | } |
9bea7f23 | 610 | return 0; |
1da177e4 LT |
611 | } |
612 | ||
613 | /* Clear the unload stuff of the module. */ | |
614 | static void module_unload_free(struct module *mod) | |
615 | { | |
2c02dfe7 | 616 | struct module_use *use, *tmp; |
1da177e4 | 617 | |
75676500 | 618 | mutex_lock(&module_mutex); |
2c02dfe7 LT |
619 | list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) { |
620 | struct module *i = use->target; | |
5e124169 | 621 | pr_debug("%s unusing %s\n", mod->name, i->name); |
2c02dfe7 LT |
622 | module_put(i); |
623 | list_del(&use->source_list); | |
624 | list_del(&use->target_list); | |
625 | kfree(use); | |
1da177e4 | 626 | } |
75676500 | 627 | mutex_unlock(&module_mutex); |
1da177e4 LT |
628 | } |
629 | ||
630 | #ifdef CONFIG_MODULE_FORCE_UNLOAD | |
fb169793 | 631 | static inline int try_force_unload(unsigned int flags) |
1da177e4 LT |
632 | { |
633 | int ret = (flags & O_TRUNC); | |
634 | if (ret) | |
373d4d09 | 635 | add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE); |
1da177e4 LT |
636 | return ret; |
637 | } | |
638 | #else | |
fb169793 | 639 | static inline int try_force_unload(unsigned int flags) |
1da177e4 LT |
640 | { |
641 | return 0; | |
642 | } | |
643 | #endif /* CONFIG_MODULE_FORCE_UNLOAD */ | |
644 | ||
e513cc1c MH |
645 | /* Try to release refcount of module, 0 means success. */ |
646 | static int try_release_module_ref(struct module *mod) | |
1da177e4 | 647 | { |
e513cc1c | 648 | int ret; |
1da177e4 | 649 | |
e513cc1c MH |
650 | /* Try to decrement refcnt which we set at loading */ |
651 | ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt); | |
652 | BUG_ON(ret < 0); | |
653 | if (ret) | |
654 | /* Someone can put this right now, recover with checking */ | |
655 | ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0); | |
1da177e4 | 656 | |
e513cc1c MH |
657 | return ret; |
658 | } | |
1da177e4 | 659 | |
e513cc1c MH |
660 | static int try_stop_module(struct module *mod, int flags, int *forced) |
661 | { | |
da39ba5e | 662 | /* If it's not unused, quit unless we're forcing. */ |
e513cc1c MH |
663 | if (try_release_module_ref(mod) != 0) { |
664 | *forced = try_force_unload(flags); | |
665 | if (!(*forced)) | |
1da177e4 LT |
666 | return -EWOULDBLOCK; |
667 | } | |
668 | ||
669 | /* Mark it as dying. */ | |
e513cc1c | 670 | mod->state = MODULE_STATE_GOING; |
1da177e4 | 671 | |
e513cc1c | 672 | return 0; |
1da177e4 LT |
673 | } |
674 | ||
d5db139a | 675 | /** |
24389b61 | 676 | * module_refcount() - return the refcount or -1 if unloading |
d5db139a RR |
677 | * @mod: the module we're checking |
678 | * | |
24389b61 | 679 | * Return: |
d5db139a RR |
680 | * -1 if the module is in the process of unloading |
681 | * otherwise the number of references in the kernel to the module | |
682 | */ | |
683 | int module_refcount(struct module *mod) | |
1da177e4 | 684 | { |
d5db139a | 685 | return atomic_read(&mod->refcnt) - MODULE_REF_BASE; |
1da177e4 LT |
686 | } |
687 | EXPORT_SYMBOL(module_refcount); | |
688 | ||
689 | /* This exists whether we can unload or not */ | |
690 | static void free_module(struct module *mod); | |
691 | ||
17da2bd9 HC |
692 | SYSCALL_DEFINE2(delete_module, const char __user *, name_user, |
693 | unsigned int, flags) | |
1da177e4 LT |
694 | { |
695 | struct module *mod; | |
dfff0a06 | 696 | char name[MODULE_NAME_LEN]; |
1da177e4 LT |
697 | int ret, forced = 0; |
698 | ||
3d43321b | 699 | if (!capable(CAP_SYS_MODULE) || modules_disabled) |
dfff0a06 GKH |
700 | return -EPERM; |
701 | ||
702 | if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0) | |
703 | return -EFAULT; | |
704 | name[MODULE_NAME_LEN-1] = '\0'; | |
705 | ||
f6276ac9 RGB |
706 | audit_log_kern_module(name); |
707 | ||
3fc1f1e2 TH |
708 | if (mutex_lock_interruptible(&module_mutex) != 0) |
709 | return -EINTR; | |
1da177e4 LT |
710 | |
711 | mod = find_module(name); | |
712 | if (!mod) { | |
713 | ret = -ENOENT; | |
714 | goto out; | |
715 | } | |
716 | ||
2c02dfe7 | 717 | if (!list_empty(&mod->source_list)) { |
1da177e4 LT |
718 | /* Other modules depend on us: get rid of them first. */ |
719 | ret = -EWOULDBLOCK; | |
720 | goto out; | |
721 | } | |
722 | ||
723 | /* Doing init or already dying? */ | |
724 | if (mod->state != MODULE_STATE_LIVE) { | |
3f2b9c9c | 725 | /* FIXME: if (force), slam module count damn the torpedoes */ |
5e124169 | 726 | pr_debug("%s already dying\n", mod->name); |
1da177e4 LT |
727 | ret = -EBUSY; |
728 | goto out; | |
729 | } | |
730 | ||
731 | /* If it has an init func, it must have an exit func to unload */ | |
af49d924 | 732 | if (mod->init && !mod->exit) { |
fb169793 | 733 | forced = try_force_unload(flags); |
1da177e4 LT |
734 | if (!forced) { |
735 | /* This module can't be removed */ | |
736 | ret = -EBUSY; | |
737 | goto out; | |
738 | } | |
739 | } | |
740 | ||
1da177e4 LT |
741 | ret = try_stop_module(mod, flags, &forced); |
742 | if (ret != 0) | |
743 | goto out; | |
744 | ||
df4b565e | 745 | mutex_unlock(&module_mutex); |
25985edc | 746 | /* Final destruction now no one is using it. */ |
df4b565e | 747 | if (mod->exit != NULL) |
1da177e4 | 748 | mod->exit(); |
df4b565e PO |
749 | blocking_notifier_call_chain(&module_notify_list, |
750 | MODULE_STATE_GOING, mod); | |
7e545d6e | 751 | klp_module_going(mod); |
7dcd182b JY |
752 | ftrace_release_mod(mod); |
753 | ||
22a9d645 | 754 | async_synchronize_full(); |
75676500 | 755 | |
e14af7ee | 756 | /* Store the name of the last unloaded module for diagnostic purposes */ |
efa5345e | 757 | strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module)); |
1da177e4 | 758 | |
75676500 | 759 | free_module(mod); |
5d603311 KK |
760 | /* someone could wait for the module in add_unformed_module() */ |
761 | wake_up_all(&module_wq); | |
75676500 RR |
762 | return 0; |
763 | out: | |
6389a385 | 764 | mutex_unlock(&module_mutex); |
1da177e4 LT |
765 | return ret; |
766 | } | |
767 | ||
1da177e4 LT |
768 | void __symbol_put(const char *symbol) |
769 | { | |
0b96615c CH |
770 | struct find_symbol_arg fsa = { |
771 | .name = symbol, | |
772 | .gplok = true, | |
773 | }; | |
1da177e4 | 774 | |
24da1cbf | 775 | preempt_disable(); |
02b2fb45 | 776 | BUG_ON(!find_symbol(&fsa)); |
0b96615c | 777 | module_put(fsa.owner); |
24da1cbf | 778 | preempt_enable(); |
1da177e4 LT |
779 | } |
780 | EXPORT_SYMBOL(__symbol_put); | |
781 | ||
7d1d16e4 | 782 | /* Note this assumes addr is a function, which it currently always is. */ |
1da177e4 LT |
783 | void symbol_put_addr(void *addr) |
784 | { | |
5e376613 | 785 | struct module *modaddr; |
7d1d16e4 | 786 | unsigned long a = (unsigned long)dereference_function_descriptor(addr); |
1da177e4 | 787 | |
7d1d16e4 | 788 | if (core_kernel_text(a)) |
5e376613 | 789 | return; |
1da177e4 | 790 | |
275d7d44 PZ |
791 | /* |
792 | * Even though we hold a reference on the module; we still need to | |
793 | * disable preemption in order to safely traverse the data structure. | |
794 | */ | |
795 | preempt_disable(); | |
7d1d16e4 | 796 | modaddr = __module_text_address(a); |
a6e6abd5 | 797 | BUG_ON(!modaddr); |
5e376613 | 798 | module_put(modaddr); |
275d7d44 | 799 | preempt_enable(); |
1da177e4 LT |
800 | } |
801 | EXPORT_SYMBOL_GPL(symbol_put_addr); | |
802 | ||
803 | static ssize_t show_refcnt(struct module_attribute *mattr, | |
4befb026 | 804 | struct module_kobject *mk, char *buffer) |
1da177e4 | 805 | { |
d5db139a | 806 | return sprintf(buffer, "%i\n", module_refcount(mk->mod)); |
1da177e4 LT |
807 | } |
808 | ||
cca3e707 KS |
809 | static struct module_attribute modinfo_refcnt = |
810 | __ATTR(refcnt, 0444, show_refcnt, NULL); | |
1da177e4 | 811 | |
d53799be SR |
812 | void __module_get(struct module *module) |
813 | { | |
814 | if (module) { | |
815 | preempt_disable(); | |
2f35c41f | 816 | atomic_inc(&module->refcnt); |
d53799be SR |
817 | trace_module_get(module, _RET_IP_); |
818 | preempt_enable(); | |
819 | } | |
820 | } | |
821 | EXPORT_SYMBOL(__module_get); | |
822 | ||
823 | bool try_module_get(struct module *module) | |
824 | { | |
825 | bool ret = true; | |
826 | ||
827 | if (module) { | |
828 | preempt_disable(); | |
e513cc1c MH |
829 | /* Note: here, we can fail to get a reference */ |
830 | if (likely(module_is_live(module) && | |
831 | atomic_inc_not_zero(&module->refcnt) != 0)) | |
d53799be | 832 | trace_module_get(module, _RET_IP_); |
e513cc1c | 833 | else |
d53799be SR |
834 | ret = false; |
835 | ||
836 | preempt_enable(); | |
837 | } | |
838 | return ret; | |
839 | } | |
840 | EXPORT_SYMBOL(try_module_get); | |
841 | ||
f6a57033 AV |
842 | void module_put(struct module *module) |
843 | { | |
e513cc1c MH |
844 | int ret; |
845 | ||
f6a57033 | 846 | if (module) { |
e1783a24 | 847 | preempt_disable(); |
e513cc1c MH |
848 | ret = atomic_dec_if_positive(&module->refcnt); |
849 | WARN_ON(ret < 0); /* Failed to put refcount */ | |
ae832d1e | 850 | trace_module_put(module, _RET_IP_); |
e1783a24 | 851 | preempt_enable(); |
f6a57033 AV |
852 | } |
853 | } | |
854 | EXPORT_SYMBOL(module_put); | |
855 | ||
1da177e4 | 856 | #else /* !CONFIG_MODULE_UNLOAD */ |
1da177e4 LT |
857 | static inline void module_unload_free(struct module *mod) |
858 | { | |
859 | } | |
860 | ||
7ef5264d | 861 | static int ref_module(struct module *a, struct module *b) |
1da177e4 | 862 | { |
9bea7f23 | 863 | return strong_try_module_get(b); |
1da177e4 LT |
864 | } |
865 | ||
9f85a4bb | 866 | static inline int module_unload_init(struct module *mod) |
1da177e4 | 867 | { |
9f85a4bb | 868 | return 0; |
1da177e4 LT |
869 | } |
870 | #endif /* CONFIG_MODULE_UNLOAD */ | |
871 | ||
c14e522b | 872 | size_t module_flags_taint(unsigned long taints, char *buf) |
53999bf3 KW |
873 | { |
874 | size_t l = 0; | |
7fd8329b PM |
875 | int i; |
876 | ||
877 | for (i = 0; i < TAINT_FLAGS_COUNT; i++) { | |
c14e522b | 878 | if (taint_flags[i].module && test_bit(i, &taints)) |
5eb7c0d0 | 879 | buf[l++] = taint_flags[i].c_true; |
7fd8329b | 880 | } |
53999bf3 | 881 | |
53999bf3 KW |
882 | return l; |
883 | } | |
884 | ||
1f71740a | 885 | static ssize_t show_initstate(struct module_attribute *mattr, |
4befb026 | 886 | struct module_kobject *mk, char *buffer) |
1f71740a KS |
887 | { |
888 | const char *state = "unknown"; | |
889 | ||
4befb026 | 890 | switch (mk->mod->state) { |
1f71740a KS |
891 | case MODULE_STATE_LIVE: |
892 | state = "live"; | |
893 | break; | |
894 | case MODULE_STATE_COMING: | |
895 | state = "coming"; | |
896 | break; | |
897 | case MODULE_STATE_GOING: | |
898 | state = "going"; | |
899 | break; | |
0d21b0e3 RR |
900 | default: |
901 | BUG(); | |
1f71740a KS |
902 | } |
903 | return sprintf(buffer, "%s\n", state); | |
904 | } | |
905 | ||
cca3e707 KS |
906 | static struct module_attribute modinfo_initstate = |
907 | __ATTR(initstate, 0444, show_initstate, NULL); | |
1f71740a | 908 | |
88bfa324 KS |
909 | static ssize_t store_uevent(struct module_attribute *mattr, |
910 | struct module_kobject *mk, | |
911 | const char *buffer, size_t count) | |
912 | { | |
df44b479 PR |
913 | int rc; |
914 | ||
915 | rc = kobject_synth_uevent(&mk->kobj, buffer, count); | |
916 | return rc ? rc : count; | |
88bfa324 KS |
917 | } |
918 | ||
cca3e707 KS |
919 | struct module_attribute module_uevent = |
920 | __ATTR(uevent, 0200, NULL, store_uevent); | |
921 | ||
922 | static ssize_t show_coresize(struct module_attribute *mattr, | |
923 | struct module_kobject *mk, char *buffer) | |
924 | { | |
7523e4dc | 925 | return sprintf(buffer, "%u\n", mk->mod->core_layout.size); |
cca3e707 KS |
926 | } |
927 | ||
928 | static struct module_attribute modinfo_coresize = | |
929 | __ATTR(coresize, 0444, show_coresize, NULL); | |
930 | ||
01dc0386 CL |
931 | #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC |
932 | static ssize_t show_datasize(struct module_attribute *mattr, | |
933 | struct module_kobject *mk, char *buffer) | |
934 | { | |
935 | return sprintf(buffer, "%u\n", mk->mod->data_layout.size); | |
936 | } | |
937 | ||
938 | static struct module_attribute modinfo_datasize = | |
939 | __ATTR(datasize, 0444, show_datasize, NULL); | |
940 | #endif | |
941 | ||
cca3e707 KS |
942 | static ssize_t show_initsize(struct module_attribute *mattr, |
943 | struct module_kobject *mk, char *buffer) | |
944 | { | |
7523e4dc | 945 | return sprintf(buffer, "%u\n", mk->mod->init_layout.size); |
cca3e707 KS |
946 | } |
947 | ||
948 | static struct module_attribute modinfo_initsize = | |
949 | __ATTR(initsize, 0444, show_initsize, NULL); | |
950 | ||
951 | static ssize_t show_taint(struct module_attribute *mattr, | |
952 | struct module_kobject *mk, char *buffer) | |
953 | { | |
954 | size_t l; | |
955 | ||
c14e522b | 956 | l = module_flags_taint(mk->mod->taints, buffer); |
cca3e707 KS |
957 | buffer[l++] = '\n'; |
958 | return l; | |
959 | } | |
960 | ||
961 | static struct module_attribute modinfo_taint = | |
962 | __ATTR(taint, 0444, show_taint, NULL); | |
88bfa324 | 963 | |
44c09535 | 964 | struct module_attribute *modinfo_attrs[] = { |
cca3e707 | 965 | &module_uevent, |
03e88ae1 GKH |
966 | &modinfo_version, |
967 | &modinfo_srcversion, | |
cca3e707 KS |
968 | &modinfo_initstate, |
969 | &modinfo_coresize, | |
01dc0386 CL |
970 | #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC |
971 | &modinfo_datasize, | |
972 | #endif | |
cca3e707 KS |
973 | &modinfo_initsize, |
974 | &modinfo_taint, | |
03e88ae1 | 975 | #ifdef CONFIG_MODULE_UNLOAD |
cca3e707 | 976 | &modinfo_refcnt, |
03e88ae1 GKH |
977 | #endif |
978 | NULL, | |
979 | }; | |
980 | ||
44c09535 AT |
981 | size_t modinfo_attrs_count = ARRAY_SIZE(modinfo_attrs); |
982 | ||
1da177e4 LT |
983 | static const char vermagic[] = VERMAGIC_STRING; |
984 | ||
47889798 | 985 | int try_to_force_load(struct module *mod, const char *reason) |
826e4506 LT |
986 | { |
987 | #ifdef CONFIG_MODULE_FORCE_LOAD | |
25ddbb18 | 988 | if (!test_taint(TAINT_FORCED_MODULE)) |
bddb12b3 | 989 | pr_warn("%s: %s: kernel tainted.\n", mod->name, reason); |
373d4d09 | 990 | add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE); |
826e4506 LT |
991 | return 0; |
992 | #else | |
993 | return -ENOEXEC; | |
994 | #endif | |
995 | } | |
996 | ||
8651ec01 MM |
997 | static char *get_modinfo(const struct load_info *info, const char *tag); |
998 | static char *get_next_modinfo(const struct load_info *info, const char *tag, | |
999 | char *prev); | |
1000 | ||
1001 | static int verify_namespace_is_imported(const struct load_info *info, | |
1002 | const struct kernel_symbol *sym, | |
1003 | struct module *mod) | |
1004 | { | |
1005 | const char *namespace; | |
1006 | char *imported_namespace; | |
1007 | ||
1008 | namespace = kernel_symbol_namespace(sym); | |
c3a6cf19 | 1009 | if (namespace && namespace[0]) { |
8651ec01 MM |
1010 | imported_namespace = get_modinfo(info, "import_ns"); |
1011 | while (imported_namespace) { | |
1012 | if (strcmp(namespace, imported_namespace) == 0) | |
1013 | return 0; | |
1014 | imported_namespace = get_next_modinfo( | |
1015 | info, "import_ns", imported_namespace); | |
1016 | } | |
3d52ec5e MM |
1017 | #ifdef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS |
1018 | pr_warn( | |
1019 | #else | |
1020 | pr_err( | |
1021 | #endif | |
1022 | "%s: module uses symbol (%s) from namespace %s, but does not import it.\n", | |
1023 | mod->name, kernel_symbol_name(sym), namespace); | |
1024 | #ifndef CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS | |
8651ec01 | 1025 | return -EINVAL; |
3d52ec5e | 1026 | #endif |
8651ec01 MM |
1027 | } |
1028 | return 0; | |
1029 | } | |
1030 | ||
8eac910a | 1031 | static bool inherit_taint(struct module *mod, struct module *owner, const char *name) |
262e6ae7 CH |
1032 | { |
1033 | if (!owner || !test_bit(TAINT_PROPRIETARY_MODULE, &owner->taints)) | |
1034 | return true; | |
1035 | ||
1036 | if (mod->using_gplonly_symbols) { | |
8eac910a LC |
1037 | pr_err("%s: module using GPL-only symbols uses symbols %s from proprietary module %s.\n", |
1038 | mod->name, name, owner->name); | |
262e6ae7 CH |
1039 | return false; |
1040 | } | |
1041 | ||
1042 | if (!test_bit(TAINT_PROPRIETARY_MODULE, &mod->taints)) { | |
8eac910a LC |
1043 | pr_warn("%s: module uses symbols %s from proprietary module %s, inheriting taint.\n", |
1044 | mod->name, name, owner->name); | |
262e6ae7 CH |
1045 | set_bit(TAINT_PROPRIETARY_MODULE, &mod->taints); |
1046 | } | |
1047 | return true; | |
1048 | } | |
8651ec01 | 1049 | |
75676500 | 1050 | /* Resolve a symbol for this module. I.e. if we find one, record usage. */ |
49668688 RR |
1051 | static const struct kernel_symbol *resolve_symbol(struct module *mod, |
1052 | const struct load_info *info, | |
414fd31b | 1053 | const char *name, |
9bea7f23 | 1054 | char ownername[]) |
1da177e4 | 1055 | { |
0b96615c CH |
1056 | struct find_symbol_arg fsa = { |
1057 | .name = name, | |
1058 | .gplok = !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), | |
1059 | .warn = true, | |
1060 | }; | |
9bea7f23 | 1061 | int err; |
1da177e4 | 1062 | |
d64810f5 PZ |
1063 | /* |
1064 | * The module_mutex should not be a heavily contended lock; | |
1065 | * if we get the occasional sleep here, we'll go an extra iteration | |
1066 | * in the wait_event_interruptible(), which is harmless. | |
1067 | */ | |
1068 | sched_annotate_sleep(); | |
75676500 | 1069 | mutex_lock(&module_mutex); |
0b96615c | 1070 | if (!find_symbol(&fsa)) |
9bea7f23 RR |
1071 | goto unlock; |
1072 | ||
0b96615c | 1073 | if (fsa.license == GPL_ONLY) |
262e6ae7 CH |
1074 | mod->using_gplonly_symbols = true; |
1075 | ||
8eac910a | 1076 | if (!inherit_taint(mod, fsa.owner, name)) { |
0b96615c | 1077 | fsa.sym = NULL; |
262e6ae7 CH |
1078 | goto getname; |
1079 | } | |
1080 | ||
0b96615c CH |
1081 | if (!check_version(info, name, mod, fsa.crc)) { |
1082 | fsa.sym = ERR_PTR(-EINVAL); | |
9bea7f23 | 1083 | goto getname; |
1da177e4 | 1084 | } |
9bea7f23 | 1085 | |
0b96615c | 1086 | err = verify_namespace_is_imported(info, fsa.sym, mod); |
8651ec01 | 1087 | if (err) { |
0b96615c | 1088 | fsa.sym = ERR_PTR(err); |
8651ec01 MM |
1089 | goto getname; |
1090 | } | |
1091 | ||
0b96615c | 1092 | err = ref_module(mod, fsa.owner); |
9bea7f23 | 1093 | if (err) { |
0b96615c | 1094 | fsa.sym = ERR_PTR(err); |
9bea7f23 RR |
1095 | goto getname; |
1096 | } | |
1097 | ||
1098 | getname: | |
1099 | /* We must make copy under the lock if we failed to get ref. */ | |
0b96615c | 1100 | strncpy(ownername, module_name(fsa.owner), MODULE_NAME_LEN); |
9bea7f23 | 1101 | unlock: |
75676500 | 1102 | mutex_unlock(&module_mutex); |
0b96615c | 1103 | return fsa.sym; |
1da177e4 LT |
1104 | } |
1105 | ||
49668688 RR |
1106 | static const struct kernel_symbol * |
1107 | resolve_symbol_wait(struct module *mod, | |
1108 | const struct load_info *info, | |
1109 | const char *name) | |
9bea7f23 RR |
1110 | { |
1111 | const struct kernel_symbol *ksym; | |
49668688 | 1112 | char owner[MODULE_NAME_LEN]; |
9bea7f23 RR |
1113 | |
1114 | if (wait_event_interruptible_timeout(module_wq, | |
49668688 RR |
1115 | !IS_ERR(ksym = resolve_symbol(mod, info, name, owner)) |
1116 | || PTR_ERR(ksym) != -EBUSY, | |
9bea7f23 | 1117 | 30 * HZ) <= 0) { |
bddb12b3 AM |
1118 | pr_warn("%s: gave up waiting for init of module %s.\n", |
1119 | mod->name, owner); | |
9bea7f23 RR |
1120 | } |
1121 | return ksym; | |
1122 | } | |
1123 | ||
be1f221c | 1124 | void __weak module_memfree(void *module_region) |
74e08fcf | 1125 | { |
1a7b7d92 RE |
1126 | /* |
1127 | * This memory may be RO, and freeing RO memory in an interrupt is not | |
1128 | * supported by vmalloc. | |
1129 | */ | |
1130 | WARN_ON(in_interrupt()); | |
74e08fcf JB |
1131 | vfree(module_region); |
1132 | } | |
1133 | ||
1134 | void __weak module_arch_cleanup(struct module *mod) | |
1135 | { | |
1136 | } | |
1137 | ||
d453cded RR |
1138 | void __weak module_arch_freeing_init(struct module *mod) |
1139 | { | |
1140 | } | |
1141 | ||
cf68fffb ST |
1142 | static void cfi_cleanup(struct module *mod); |
1143 | ||
75676500 | 1144 | /* Free a module, remove from lists, etc. */ |
1da177e4 LT |
1145 | static void free_module(struct module *mod) |
1146 | { | |
7ead8b83 LZ |
1147 | trace_module_free(mod); |
1148 | ||
36b0360d | 1149 | mod_sysfs_teardown(mod); |
1da177e4 | 1150 | |
24b9f0d2 SS |
1151 | /* |
1152 | * We leave it in list to prevent duplicate loads, but make sure | |
1153 | * that noone uses it while it's being deconstructed. | |
1154 | */ | |
d3051b48 | 1155 | mutex_lock(&module_mutex); |
944a1fa0 | 1156 | mod->state = MODULE_STATE_UNFORMED; |
d3051b48 | 1157 | mutex_unlock(&module_mutex); |
944a1fa0 | 1158 | |
b82bab4b JB |
1159 | /* Remove dynamic debug info */ |
1160 | ddebug_remove_module(mod->name); | |
1161 | ||
1da177e4 LT |
1162 | /* Arch-specific cleanup. */ |
1163 | module_arch_cleanup(mod); | |
1164 | ||
1165 | /* Module unload stuff */ | |
1166 | module_unload_free(mod); | |
1167 | ||
e180a6b7 RR |
1168 | /* Free any allocated parameters. */ |
1169 | destroy_params(mod->kp, mod->num_kp); | |
1170 | ||
1ce15ef4 JY |
1171 | if (is_livepatch_module(mod)) |
1172 | free_module_elf(mod); | |
1173 | ||
944a1fa0 RR |
1174 | /* Now we can delete it from the lists */ |
1175 | mutex_lock(&module_mutex); | |
461e34ae MH |
1176 | /* Unlink carefully: kallsyms could be walking list. */ |
1177 | list_del_rcu(&mod->list); | |
93c2e105 | 1178 | mod_tree_remove(mod); |
0286b5ea | 1179 | /* Remove this module from bug list, this uses list_del_rcu */ |
461e34ae | 1180 | module_bug_cleanup(mod); |
0be964be | 1181 | /* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */ |
cb2f5536 | 1182 | synchronize_rcu(); |
99bd9956 AT |
1183 | if (try_add_tainted_module(mod)) |
1184 | pr_err("%s: adding tainted module to the unloaded tainted modules list failed.\n", | |
1185 | mod->name); | |
944a1fa0 RR |
1186 | mutex_unlock(&module_mutex); |
1187 | ||
cf68fffb ST |
1188 | /* Clean up CFI for the module. */ |
1189 | cfi_cleanup(mod); | |
1190 | ||
85c898db | 1191 | /* This may be empty, but that's OK */ |
d453cded | 1192 | module_arch_freeing_init(mod); |
7523e4dc | 1193 | module_memfree(mod->init_layout.base); |
1da177e4 | 1194 | kfree(mod->args); |
259354de | 1195 | percpu_modfree(mod); |
9f85a4bb | 1196 | |
35a9393c | 1197 | /* Free lock-classes; relies on the preceding sync_rcu(). */ |
6ab9942c | 1198 | lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size); |
fbb9ce95 | 1199 | |
1da177e4 | 1200 | /* Finally, free the core (containing the module structure) */ |
7523e4dc | 1201 | module_memfree(mod->core_layout.base); |
01dc0386 CL |
1202 | #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC |
1203 | vfree(mod->data_layout.base); | |
1204 | #endif | |
1da177e4 LT |
1205 | } |
1206 | ||
1207 | void *__symbol_get(const char *symbol) | |
1208 | { | |
0b96615c CH |
1209 | struct find_symbol_arg fsa = { |
1210 | .name = symbol, | |
1211 | .gplok = true, | |
1212 | .warn = true, | |
1213 | }; | |
1da177e4 | 1214 | |
24da1cbf | 1215 | preempt_disable(); |
0b96615c CH |
1216 | if (!find_symbol(&fsa) || strong_try_module_get(fsa.owner)) { |
1217 | preempt_enable(); | |
1218 | return NULL; | |
1219 | } | |
24da1cbf | 1220 | preempt_enable(); |
0b96615c | 1221 | return (void *)kernel_symbol_value(fsa.sym); |
1da177e4 LT |
1222 | } |
1223 | EXPORT_SYMBOL_GPL(__symbol_get); | |
1224 | ||
eea8b54d AN |
1225 | /* |
1226 | * Ensure that an exported symbol [global namespace] does not already exist | |
02a3e59a | 1227 | * in the kernel or in some other module's exported symbol table. |
be593f4c RR |
1228 | * |
1229 | * You must hold the module_mutex. | |
eea8b54d | 1230 | */ |
2d25bc55 | 1231 | static int verify_exported_symbols(struct module *mod) |
eea8b54d | 1232 | { |
b211104d | 1233 | unsigned int i; |
b211104d RR |
1234 | const struct kernel_symbol *s; |
1235 | struct { | |
1236 | const struct kernel_symbol *sym; | |
1237 | unsigned int num; | |
1238 | } arr[] = { | |
1239 | { mod->syms, mod->num_syms }, | |
1240 | { mod->gpl_syms, mod->num_gpl_syms }, | |
b211104d | 1241 | }; |
eea8b54d | 1242 | |
b211104d RR |
1243 | for (i = 0; i < ARRAY_SIZE(arr); i++) { |
1244 | for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) { | |
0b96615c CH |
1245 | struct find_symbol_arg fsa = { |
1246 | .name = kernel_symbol_name(s), | |
1247 | .gplok = true, | |
1248 | }; | |
1249 | if (find_symbol(&fsa)) { | |
bddb12b3 | 1250 | pr_err("%s: exports duplicate symbol %s" |
b211104d | 1251 | " (owned by %s)\n", |
7290d580 | 1252 | mod->name, kernel_symbol_name(s), |
0b96615c | 1253 | module_name(fsa.owner)); |
b211104d RR |
1254 | return -ENOEXEC; |
1255 | } | |
eea8b54d | 1256 | } |
b211104d RR |
1257 | } |
1258 | return 0; | |
eea8b54d AN |
1259 | } |
1260 | ||
ebfac7b7 FS |
1261 | static bool ignore_undef_symbol(Elf_Half emachine, const char *name) |
1262 | { | |
1263 | /* | |
1264 | * On x86, PIC code and Clang non-PIC code may have call foo@PLT. GNU as | |
1265 | * before 2.37 produces an unreferenced _GLOBAL_OFFSET_TABLE_ on x86-64. | |
1266 | * i386 has a similar problem but may not deserve a fix. | |
1267 | * | |
1268 | * If we ever have to ignore many symbols, consider refactoring the code to | |
1269 | * only warn if referenced by a relocation. | |
1270 | */ | |
1271 | if (emachine == EM_386 || emachine == EM_X86_64) | |
1272 | return !strcmp(name, "_GLOBAL_OFFSET_TABLE_"); | |
1273 | return false; | |
1274 | } | |
1275 | ||
9a4b9708 | 1276 | /* Change all symbols so that st_value encodes the pointer directly. */ |
49668688 RR |
1277 | static int simplify_symbols(struct module *mod, const struct load_info *info) |
1278 | { | |
1279 | Elf_Shdr *symsec = &info->sechdrs[info->index.sym]; | |
1280 | Elf_Sym *sym = (void *)symsec->sh_addr; | |
1da177e4 | 1281 | unsigned long secbase; |
49668688 | 1282 | unsigned int i; |
1da177e4 | 1283 | int ret = 0; |
414fd31b | 1284 | const struct kernel_symbol *ksym; |
1da177e4 | 1285 | |
49668688 RR |
1286 | for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) { |
1287 | const char *name = info->strtab + sym[i].st_name; | |
1288 | ||
1da177e4 LT |
1289 | switch (sym[i].st_shndx) { |
1290 | case SHN_COMMON: | |
80375980 JM |
1291 | /* Ignore common symbols */ |
1292 | if (!strncmp(name, "__gnu_lto", 9)) | |
1293 | break; | |
1294 | ||
24b9f0d2 SS |
1295 | /* |
1296 | * We compiled with -fno-common. These are not | |
1297 | * supposed to happen. | |
1298 | */ | |
5e124169 | 1299 | pr_debug("Common symbol: %s\n", name); |
6da0b565 | 1300 | pr_warn("%s: please compile with -fno-common\n", |
1da177e4 LT |
1301 | mod->name); |
1302 | ret = -ENOEXEC; | |
1303 | break; | |
1304 | ||
1305 | case SHN_ABS: | |
1306 | /* Don't need to do anything */ | |
5e124169 | 1307 | pr_debug("Absolute symbol: 0x%08lx\n", |
1da177e4 LT |
1308 | (long)sym[i].st_value); |
1309 | break; | |
1310 | ||
1ce15ef4 JY |
1311 | case SHN_LIVEPATCH: |
1312 | /* Livepatch symbols are resolved by livepatch */ | |
1313 | break; | |
1314 | ||
1da177e4 | 1315 | case SHN_UNDEF: |
49668688 | 1316 | ksym = resolve_symbol_wait(mod, info, name); |
1da177e4 | 1317 | /* Ok if resolved. */ |
9bea7f23 | 1318 | if (ksym && !IS_ERR(ksym)) { |
7290d580 | 1319 | sym[i].st_value = kernel_symbol_value(ksym); |
1da177e4 | 1320 | break; |
414fd31b TA |
1321 | } |
1322 | ||
ebfac7b7 FS |
1323 | /* Ok if weak or ignored. */ |
1324 | if (!ksym && | |
1325 | (ELF_ST_BIND(sym[i].st_info) == STB_WEAK || | |
1326 | ignore_undef_symbol(info->hdr->e_machine, name))) | |
1da177e4 LT |
1327 | break; |
1328 | ||
9bea7f23 | 1329 | ret = PTR_ERR(ksym) ?: -ENOENT; |
62267e0e JD |
1330 | pr_warn("%s: Unknown symbol %s (err %d)\n", |
1331 | mod->name, name, ret); | |
1da177e4 LT |
1332 | break; |
1333 | ||
1334 | default: | |
1335 | /* Divert to percpu allocation if a percpu var. */ | |
49668688 | 1336 | if (sym[i].st_shndx == info->index.pcpu) |
259354de | 1337 | secbase = (unsigned long)mod_percpu(mod); |
1da177e4 | 1338 | else |
49668688 | 1339 | secbase = info->sechdrs[sym[i].st_shndx].sh_addr; |
1da177e4 LT |
1340 | sym[i].st_value += secbase; |
1341 | break; | |
1342 | } | |
1343 | } | |
1344 | ||
1345 | return ret; | |
1346 | } | |
1347 | ||
49668688 | 1348 | static int apply_relocations(struct module *mod, const struct load_info *info) |
22e268eb RR |
1349 | { |
1350 | unsigned int i; | |
1351 | int err = 0; | |
1352 | ||
1353 | /* Now do relocations. */ | |
49668688 RR |
1354 | for (i = 1; i < info->hdr->e_shnum; i++) { |
1355 | unsigned int infosec = info->sechdrs[i].sh_info; | |
22e268eb RR |
1356 | |
1357 | /* Not a valid relocation section? */ | |
49668688 | 1358 | if (infosec >= info->hdr->e_shnum) |
22e268eb RR |
1359 | continue; |
1360 | ||
1361 | /* Don't bother with non-allocated sections */ | |
49668688 | 1362 | if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC)) |
22e268eb RR |
1363 | continue; |
1364 | ||
1ce15ef4 | 1365 | if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH) |
7c8e2bdd JP |
1366 | err = klp_apply_section_relocs(mod, info->sechdrs, |
1367 | info->secstrings, | |
1368 | info->strtab, | |
1369 | info->index.sym, i, | |
1370 | NULL); | |
1371 | else if (info->sechdrs[i].sh_type == SHT_REL) | |
49668688 RR |
1372 | err = apply_relocate(info->sechdrs, info->strtab, |
1373 | info->index.sym, i, mod); | |
1374 | else if (info->sechdrs[i].sh_type == SHT_RELA) | |
1375 | err = apply_relocate_add(info->sechdrs, info->strtab, | |
1376 | info->index.sym, i, mod); | |
22e268eb RR |
1377 | if (err < 0) |
1378 | break; | |
1379 | } | |
1380 | return err; | |
1381 | } | |
1382 | ||
088af9a6 HD |
1383 | /* Additional bytes needed by arch in front of individual sections */ |
1384 | unsigned int __weak arch_mod_section_prepend(struct module *mod, | |
1385 | unsigned int section) | |
1386 | { | |
1387 | /* default implementation just returns zero */ | |
1388 | return 0; | |
1389 | } | |
1390 | ||
1da177e4 | 1391 | /* Update size with this section: return offset. */ |
91fb02f3 | 1392 | long module_get_offset(struct module *mod, unsigned int *size, |
088af9a6 | 1393 | Elf_Shdr *sechdr, unsigned int section) |
1da177e4 LT |
1394 | { |
1395 | long ret; | |
1396 | ||
088af9a6 | 1397 | *size += arch_mod_section_prepend(mod, section); |
1da177e4 LT |
1398 | ret = ALIGN(*size, sechdr->sh_addralign ?: 1); |
1399 | *size = ret + sechdr->sh_size; | |
1400 | return ret; | |
1401 | } | |
1402 | ||
055f23b7 JY |
1403 | static bool module_init_layout_section(const char *sname) |
1404 | { | |
1405 | #ifndef CONFIG_MODULE_UNLOAD | |
1406 | if (module_exit_section(sname)) | |
1407 | return true; | |
1408 | #endif | |
1409 | return module_init_section(sname); | |
1410 | } | |
1411 | ||
24b9f0d2 SS |
1412 | /* |
1413 | * Lay out the SHF_ALLOC sections in a way not dissimilar to how ld | |
1414 | * might -- code, read-only data, read-write data, small data. Tally | |
1415 | * sizes, and place the offsets into sh_entsize fields: high bit means it | |
1416 | * belongs in init. | |
1417 | */ | |
49668688 | 1418 | static void layout_sections(struct module *mod, struct load_info *info) |
1da177e4 LT |
1419 | { |
1420 | static unsigned long const masks[][2] = { | |
24b9f0d2 SS |
1421 | /* |
1422 | * NOTE: all executable code must be the first section | |
1da177e4 | 1423 | * in this array; otherwise modify the text_size |
24b9f0d2 SS |
1424 | * finder in the two loops below |
1425 | */ | |
1da177e4 LT |
1426 | { SHF_EXECINSTR | SHF_ALLOC, ARCH_SHF_SMALL }, |
1427 | { SHF_ALLOC, SHF_WRITE | ARCH_SHF_SMALL }, | |
444d13ff | 1428 | { SHF_RO_AFTER_INIT | SHF_ALLOC, ARCH_SHF_SMALL }, |
1da177e4 LT |
1429 | { SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL }, |
1430 | { ARCH_SHF_SMALL | SHF_ALLOC, 0 } | |
1431 | }; | |
1432 | unsigned int m, i; | |
1433 | ||
49668688 RR |
1434 | for (i = 0; i < info->hdr->e_shnum; i++) |
1435 | info->sechdrs[i].sh_entsize = ~0UL; | |
1da177e4 | 1436 | |
5e124169 | 1437 | pr_debug("Core section allocation order:\n"); |
1da177e4 | 1438 | for (m = 0; m < ARRAY_SIZE(masks); ++m) { |
49668688 RR |
1439 | for (i = 0; i < info->hdr->e_shnum; ++i) { |
1440 | Elf_Shdr *s = &info->sechdrs[i]; | |
1441 | const char *sname = info->secstrings + s->sh_name; | |
6ab9942c | 1442 | unsigned int *sizep; |
1da177e4 LT |
1443 | |
1444 | if ((s->sh_flags & masks[m][0]) != masks[m][0] | |
1445 | || (s->sh_flags & masks[m][1]) | |
1446 | || s->sh_entsize != ~0UL | |
055f23b7 | 1447 | || module_init_layout_section(sname)) |
1da177e4 | 1448 | continue; |
6ab9942c CL |
1449 | sizep = m ? &mod->data_layout.size : &mod->core_layout.size; |
1450 | s->sh_entsize = module_get_offset(mod, sizep, s, i); | |
5e124169 | 1451 | pr_debug("\t%s\n", sname); |
1da177e4 | 1452 | } |
84e1c6bb | 1453 | switch (m) { |
1454 | case 0: /* executable */ | |
7337f929 | 1455 | mod->core_layout.size = strict_align(mod->core_layout.size); |
7523e4dc | 1456 | mod->core_layout.text_size = mod->core_layout.size; |
84e1c6bb | 1457 | break; |
1458 | case 1: /* RO: text and ro-data */ | |
6ab9942c CL |
1459 | mod->data_layout.size = strict_align(mod->data_layout.size); |
1460 | mod->data_layout.ro_size = mod->data_layout.size; | |
84e1c6bb | 1461 | break; |
444d13ff | 1462 | case 2: /* RO after init */ |
6ab9942c CL |
1463 | mod->data_layout.size = strict_align(mod->data_layout.size); |
1464 | mod->data_layout.ro_after_init_size = mod->data_layout.size; | |
444d13ff JY |
1465 | break; |
1466 | case 4: /* whole core */ | |
6ab9942c | 1467 | mod->data_layout.size = strict_align(mod->data_layout.size); |
84e1c6bb | 1468 | break; |
1469 | } | |
1da177e4 LT |
1470 | } |
1471 | ||
5e124169 | 1472 | pr_debug("Init section allocation order:\n"); |
1da177e4 | 1473 | for (m = 0; m < ARRAY_SIZE(masks); ++m) { |
49668688 RR |
1474 | for (i = 0; i < info->hdr->e_shnum; ++i) { |
1475 | Elf_Shdr *s = &info->sechdrs[i]; | |
1476 | const char *sname = info->secstrings + s->sh_name; | |
1da177e4 LT |
1477 | |
1478 | if ((s->sh_flags & masks[m][0]) != masks[m][0] | |
1479 | || (s->sh_flags & masks[m][1]) | |
1480 | || s->sh_entsize != ~0UL | |
055f23b7 | 1481 | || !module_init_layout_section(sname)) |
1da177e4 | 1482 | continue; |
91fb02f3 | 1483 | s->sh_entsize = (module_get_offset(mod, &mod->init_layout.size, s, i) |
1da177e4 | 1484 | | INIT_OFFSET_MASK); |
5e124169 | 1485 | pr_debug("\t%s\n", sname); |
1da177e4 | 1486 | } |
84e1c6bb | 1487 | switch (m) { |
1488 | case 0: /* executable */ | |
7337f929 | 1489 | mod->init_layout.size = strict_align(mod->init_layout.size); |
7523e4dc | 1490 | mod->init_layout.text_size = mod->init_layout.size; |
84e1c6bb | 1491 | break; |
1492 | case 1: /* RO: text and ro-data */ | |
7337f929 | 1493 | mod->init_layout.size = strict_align(mod->init_layout.size); |
7523e4dc | 1494 | mod->init_layout.ro_size = mod->init_layout.size; |
84e1c6bb | 1495 | break; |
444d13ff JY |
1496 | case 2: |
1497 | /* | |
1498 | * RO after init doesn't apply to init_layout (only | |
1499 | * core_layout), so it just takes the value of ro_size. | |
1500 | */ | |
1501 | mod->init_layout.ro_after_init_size = mod->init_layout.ro_size; | |
1502 | break; | |
1503 | case 4: /* whole init */ | |
7337f929 | 1504 | mod->init_layout.size = strict_align(mod->init_layout.size); |
84e1c6bb | 1505 | break; |
1506 | } | |
1da177e4 LT |
1507 | } |
1508 | } | |
1509 | ||
1da177e4 LT |
1510 | static void set_license(struct module *mod, const char *license) |
1511 | { | |
1512 | if (!license) | |
1513 | license = "unspecified"; | |
1514 | ||
fa3ba2e8 | 1515 | if (!license_is_gpl_compatible(license)) { |
25ddbb18 | 1516 | if (!test_taint(TAINT_PROPRIETARY_MODULE)) |
bddb12b3 AM |
1517 | pr_warn("%s: module license '%s' taints kernel.\n", |
1518 | mod->name, license); | |
373d4d09 RR |
1519 | add_taint_module(mod, TAINT_PROPRIETARY_MODULE, |
1520 | LOCKDEP_NOW_UNRELIABLE); | |
1da177e4 LT |
1521 | } |
1522 | } | |
1523 | ||
1524 | /* Parse tag=value strings from .modinfo section */ | |
1525 | static char *next_string(char *string, unsigned long *secsize) | |
1526 | { | |
1527 | /* Skip non-zero chars */ | |
1528 | while (string[0]) { | |
1529 | string++; | |
1530 | if ((*secsize)-- <= 1) | |
1531 | return NULL; | |
1532 | } | |
1533 | ||
1534 | /* Skip any zero padding. */ | |
1535 | while (!string[0]) { | |
1536 | string++; | |
1537 | if ((*secsize)-- <= 1) | |
1538 | return NULL; | |
1539 | } | |
1540 | return string; | |
1541 | } | |
1542 | ||
c5e4a062 MM |
1543 | static char *get_next_modinfo(const struct load_info *info, const char *tag, |
1544 | char *prev) | |
1da177e4 LT |
1545 | { |
1546 | char *p; | |
1547 | unsigned int taglen = strlen(tag); | |
49668688 RR |
1548 | Elf_Shdr *infosec = &info->sechdrs[info->index.info]; |
1549 | unsigned long size = infosec->sh_size; | |
1da177e4 | 1550 | |
5fdc7db6 JY |
1551 | /* |
1552 | * get_modinfo() calls made before rewrite_section_headers() | |
1553 | * must use sh_offset, as sh_addr isn't set! | |
1554 | */ | |
c5e4a062 MM |
1555 | char *modinfo = (char *)info->hdr + infosec->sh_offset; |
1556 | ||
1557 | if (prev) { | |
1558 | size -= prev - modinfo; | |
1559 | modinfo = next_string(prev, &size); | |
1560 | } | |
1561 | ||
1562 | for (p = modinfo; p; p = next_string(p, &size)) { | |
1da177e4 LT |
1563 | if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=') |
1564 | return p + taglen + 1; | |
1565 | } | |
1566 | return NULL; | |
1567 | } | |
1568 | ||
c5e4a062 MM |
1569 | static char *get_modinfo(const struct load_info *info, const char *tag) |
1570 | { | |
1571 | return get_next_modinfo(info, tag, NULL); | |
1572 | } | |
1573 | ||
49668688 | 1574 | static void setup_modinfo(struct module *mod, struct load_info *info) |
c988d2b2 MD |
1575 | { |
1576 | struct module_attribute *attr; | |
1577 | int i; | |
1578 | ||
1579 | for (i = 0; (attr = modinfo_attrs[i]); i++) { | |
1580 | if (attr->setup) | |
49668688 | 1581 | attr->setup(mod, get_modinfo(info, attr->attr.name)); |
c988d2b2 MD |
1582 | } |
1583 | } | |
c988d2b2 | 1584 | |
a263f776 RR |
1585 | static void free_modinfo(struct module *mod) |
1586 | { | |
1587 | struct module_attribute *attr; | |
1588 | int i; | |
1589 | ||
1590 | for (i = 0; (attr = modinfo_attrs[i]); i++) { | |
1591 | if (attr->free) | |
1592 | attr->free(mod); | |
1593 | } | |
1594 | } | |
1595 | ||
52796312 | 1596 | static void dynamic_debug_setup(struct module *mod, struct _ddebug *debug, unsigned int num) |
346e15be | 1597 | { |
811d66a0 RR |
1598 | if (!debug) |
1599 | return; | |
513770f5 | 1600 | ddebug_add_module(debug, num, mod->name); |
5e458cc0 | 1601 | } |
346e15be | 1602 | |
52796312 | 1603 | static void dynamic_debug_remove(struct module *mod, struct _ddebug *debug) |
ff49d74a YS |
1604 | { |
1605 | if (debug) | |
52796312 | 1606 | ddebug_remove_module(mod->name); |
ff49d74a YS |
1607 | } |
1608 | ||
74e08fcf JB |
1609 | void * __weak module_alloc(unsigned long size) |
1610 | { | |
7a0e27b2 CH |
1611 | return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END, |
1612 | GFP_KERNEL, PAGE_KERNEL_EXEC, VM_FLUSH_RESET_PERMS, | |
a3a66c38 | 1613 | NUMA_NO_NODE, __builtin_return_address(0)); |
74e08fcf JB |
1614 | } |
1615 | ||
23189766 VW |
1616 | bool __weak module_init_section(const char *name) |
1617 | { | |
1618 | return strstarts(name, ".init"); | |
1619 | } | |
1620 | ||
38b37d63 MS |
1621 | bool __weak module_exit_section(const char *name) |
1622 | { | |
1623 | return strstarts(name, ".exit"); | |
1624 | } | |
1625 | ||
ec2a2959 | 1626 | static int validate_section_offset(struct load_info *info, Elf_Shdr *shdr) |
40dd2560 | 1627 | { |
d83d42d0 SK |
1628 | #if defined(CONFIG_64BIT) |
1629 | unsigned long long secend; | |
1630 | #else | |
ec2a2959 | 1631 | unsigned long secend; |
d83d42d0 | 1632 | #endif |
ec2a2959 FL |
1633 | |
1634 | /* | |
1635 | * Check for both overflow and offset/size being | |
1636 | * too large. | |
1637 | */ | |
1638 | secend = shdr->sh_offset + shdr->sh_size; | |
1639 | if (secend < shdr->sh_offset || secend > info->len) | |
1640 | return -ENOEXEC; | |
1641 | ||
1642 | return 0; | |
1643 | } | |
1644 | ||
1645 | /* | |
1646 | * Sanity checks against invalid binaries, wrong arch, weird elf version. | |
1647 | * | |
1648 | * Also do basic validity checks against section offsets and sizes, the | |
1649 | * section name string table, and the indices used for it (sh_name). | |
1650 | */ | |
1651 | static int elf_validity_check(struct load_info *info) | |
1652 | { | |
1653 | unsigned int i; | |
1654 | Elf_Shdr *shdr, *strhdr; | |
1655 | int err; | |
1656 | ||
7fd982f3 SK |
1657 | if (info->len < sizeof(*(info->hdr))) { |
1658 | pr_err("Invalid ELF header len %lu\n", info->len); | |
1659 | goto no_exec; | |
1660 | } | |
34e1169d | 1661 | |
7fd982f3 SK |
1662 | if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0) { |
1663 | pr_err("Invalid ELF header magic: != %s\n", ELFMAG); | |
1664 | goto no_exec; | |
1665 | } | |
1666 | if (info->hdr->e_type != ET_REL) { | |
1667 | pr_err("Invalid ELF header type: %u != %u\n", | |
1668 | info->hdr->e_type, ET_REL); | |
1669 | goto no_exec; | |
1670 | } | |
1671 | if (!elf_check_arch(info->hdr)) { | |
1672 | pr_err("Invalid architecture in ELF header: %u\n", | |
1673 | info->hdr->e_machine); | |
1674 | goto no_exec; | |
1675 | } | |
1676 | if (info->hdr->e_shentsize != sizeof(Elf_Shdr)) { | |
1677 | pr_err("Invalid ELF section header size\n"); | |
1678 | goto no_exec; | |
1679 | } | |
34e1169d | 1680 | |
ec2a2959 FL |
1681 | /* |
1682 | * e_shnum is 16 bits, and sizeof(Elf_Shdr) is | |
1683 | * known and small. So e_shnum * sizeof(Elf_Shdr) | |
1684 | * will not overflow unsigned long on any platform. | |
1685 | */ | |
34e1169d KC |
1686 | if (info->hdr->e_shoff >= info->len |
1687 | || (info->hdr->e_shnum * sizeof(Elf_Shdr) > | |
7fd982f3 SK |
1688 | info->len - info->hdr->e_shoff)) { |
1689 | pr_err("Invalid ELF section header overflow\n"); | |
1690 | goto no_exec; | |
1691 | } | |
40dd2560 | 1692 | |
ec2a2959 FL |
1693 | info->sechdrs = (void *)info->hdr + info->hdr->e_shoff; |
1694 | ||
1695 | /* | |
1696 | * Verify if the section name table index is valid. | |
1697 | */ | |
1698 | if (info->hdr->e_shstrndx == SHN_UNDEF | |
7fd982f3 SK |
1699 | || info->hdr->e_shstrndx >= info->hdr->e_shnum) { |
1700 | pr_err("Invalid ELF section name index: %d || e_shstrndx (%d) >= e_shnum (%d)\n", | |
1701 | info->hdr->e_shstrndx, info->hdr->e_shstrndx, | |
1702 | info->hdr->e_shnum); | |
1703 | goto no_exec; | |
1704 | } | |
ec2a2959 FL |
1705 | |
1706 | strhdr = &info->sechdrs[info->hdr->e_shstrndx]; | |
1707 | err = validate_section_offset(info, strhdr); | |
7fd982f3 SK |
1708 | if (err < 0) { |
1709 | pr_err("Invalid ELF section hdr(type %u)\n", strhdr->sh_type); | |
ec2a2959 | 1710 | return err; |
7fd982f3 | 1711 | } |
ec2a2959 FL |
1712 | |
1713 | /* | |
1714 | * The section name table must be NUL-terminated, as required | |
1715 | * by the spec. This makes strcmp and pr_* calls that access | |
1716 | * strings in the section safe. | |
1717 | */ | |
1718 | info->secstrings = (void *)info->hdr + strhdr->sh_offset; | |
391e982b AD |
1719 | if (strhdr->sh_size == 0) { |
1720 | pr_err("empty section name table\n"); | |
1721 | goto no_exec; | |
1722 | } | |
7fd982f3 SK |
1723 | if (info->secstrings[strhdr->sh_size - 1] != '\0') { |
1724 | pr_err("ELF Spec violation: section name table isn't null terminated\n"); | |
1725 | goto no_exec; | |
1726 | } | |
ec2a2959 FL |
1727 | |
1728 | /* | |
1729 | * The code assumes that section 0 has a length of zero and | |
1730 | * an addr of zero, so check for it. | |
1731 | */ | |
1732 | if (info->sechdrs[0].sh_type != SHT_NULL | |
1733 | || info->sechdrs[0].sh_size != 0 | |
7fd982f3 SK |
1734 | || info->sechdrs[0].sh_addr != 0) { |
1735 | pr_err("ELF Spec violation: section 0 type(%d)!=SH_NULL or non-zero len or addr\n", | |
1736 | info->sechdrs[0].sh_type); | |
1737 | goto no_exec; | |
1738 | } | |
ec2a2959 FL |
1739 | |
1740 | for (i = 1; i < info->hdr->e_shnum; i++) { | |
1741 | shdr = &info->sechdrs[i]; | |
1742 | switch (shdr->sh_type) { | |
1743 | case SHT_NULL: | |
1744 | case SHT_NOBITS: | |
1745 | continue; | |
1746 | case SHT_SYMTAB: | |
1747 | if (shdr->sh_link == SHN_UNDEF | |
7fd982f3 SK |
1748 | || shdr->sh_link >= info->hdr->e_shnum) { |
1749 | pr_err("Invalid ELF sh_link!=SHN_UNDEF(%d) or (sh_link(%d) >= hdr->e_shnum(%d)\n", | |
1750 | shdr->sh_link, shdr->sh_link, | |
1751 | info->hdr->e_shnum); | |
1752 | goto no_exec; | |
1753 | } | |
ec2a2959 FL |
1754 | fallthrough; |
1755 | default: | |
1756 | err = validate_section_offset(info, shdr); | |
1757 | if (err < 0) { | |
1758 | pr_err("Invalid ELF section in module (section %u type %u)\n", | |
1759 | i, shdr->sh_type); | |
1760 | return err; | |
1761 | } | |
1762 | ||
1763 | if (shdr->sh_flags & SHF_ALLOC) { | |
1764 | if (shdr->sh_name >= strhdr->sh_size) { | |
1765 | pr_err("Invalid ELF section name in module (section %u type %u)\n", | |
1766 | i, shdr->sh_type); | |
1767 | return -ENOEXEC; | |
1768 | } | |
1769 | } | |
1770 | break; | |
1771 | } | |
1772 | } | |
1773 | ||
34e1169d | 1774 | return 0; |
7fd982f3 SK |
1775 | |
1776 | no_exec: | |
1777 | return -ENOEXEC; | |
34e1169d KC |
1778 | } |
1779 | ||
3afe9f84 LT |
1780 | #define COPY_CHUNK_SIZE (16*PAGE_SIZE) |
1781 | ||
1782 | static int copy_chunked_from_user(void *dst, const void __user *usrc, unsigned long len) | |
1783 | { | |
1784 | do { | |
1785 | unsigned long n = min(len, COPY_CHUNK_SIZE); | |
1786 | ||
1787 | if (copy_from_user(dst, usrc, n) != 0) | |
1788 | return -EFAULT; | |
1789 | cond_resched(); | |
1790 | dst += n; | |
1791 | usrc += n; | |
1792 | len -= n; | |
1793 | } while (len); | |
1794 | return 0; | |
1795 | } | |
1796 | ||
2992ef29 | 1797 | static int check_modinfo_livepatch(struct module *mod, struct load_info *info) |
1ce15ef4 | 1798 | { |
1be9473e AT |
1799 | if (!get_modinfo(info, "livepatch")) |
1800 | /* Nothing more to do */ | |
1801 | return 0; | |
1802 | ||
1803 | if (set_livepatch_module(mod)) { | |
2992ef29 | 1804 | add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK); |
7598d167 | 1805 | pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n", |
1be9473e AT |
1806 | mod->name); |
1807 | return 0; | |
1ce15ef4 JY |
1808 | } |
1809 | ||
1be9473e AT |
1810 | pr_err("%s: module is marked as livepatch module, but livepatch support is disabled", |
1811 | mod->name); | |
1812 | return -ENOEXEC; | |
1ce15ef4 | 1813 | } |
1ce15ef4 | 1814 | |
caf7501a AK |
1815 | static void check_modinfo_retpoline(struct module *mod, struct load_info *info) |
1816 | { | |
1817 | if (retpoline_module_ok(get_modinfo(info, "retpoline"))) | |
1818 | return; | |
1819 | ||
1820 | pr_warn("%s: loading module not compiled with retpoline compiler.\n", | |
1821 | mod->name); | |
1822 | } | |
1823 | ||
34e1169d KC |
1824 | /* Sets info->hdr and info->len. */ |
1825 | static int copy_module_from_user(const void __user *umod, unsigned long len, | |
1826 | struct load_info *info) | |
40dd2560 RR |
1827 | { |
1828 | int err; | |
40dd2560 | 1829 | |
34e1169d KC |
1830 | info->len = len; |
1831 | if (info->len < sizeof(*(info->hdr))) | |
40dd2560 RR |
1832 | return -ENOEXEC; |
1833 | ||
38f90173 | 1834 | err = security_kernel_load_data(LOADING_MODULE, true); |
2e72d51b KC |
1835 | if (err) |
1836 | return err; | |
1837 | ||
40dd2560 | 1838 | /* Suck in entire file: we'll want most of it. */ |
88dca4ca | 1839 | info->hdr = __vmalloc(info->len, GFP_KERNEL | __GFP_NOWARN); |
34e1169d | 1840 | if (!info->hdr) |
40dd2560 RR |
1841 | return -ENOMEM; |
1842 | ||
3afe9f84 | 1843 | if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) { |
38f90173 KC |
1844 | err = -EFAULT; |
1845 | goto out; | |
40dd2560 RR |
1846 | } |
1847 | ||
38f90173 KC |
1848 | err = security_kernel_post_load_data((char *)info->hdr, info->len, |
1849 | LOADING_MODULE, "init_module"); | |
1850 | out: | |
1851 | if (err) | |
1852 | vfree(info->hdr); | |
1853 | ||
1854 | return err; | |
34e1169d KC |
1855 | } |
1856 | ||
b1ae6dc4 | 1857 | static void free_copy(struct load_info *info, int flags) |
d913188c | 1858 | { |
b1ae6dc4 DT |
1859 | if (flags & MODULE_INIT_COMPRESSED_FILE) |
1860 | module_decompress_cleanup(info); | |
1861 | else | |
1862 | vfree(info->hdr); | |
d913188c RR |
1863 | } |
1864 | ||
2f3238ae | 1865 | static int rewrite_section_headers(struct load_info *info, int flags) |
8b5f61a7 RR |
1866 | { |
1867 | unsigned int i; | |
1868 | ||
1869 | /* This should always be true, but let's be sure. */ | |
1870 | info->sechdrs[0].sh_addr = 0; | |
1871 | ||
1872 | for (i = 1; i < info->hdr->e_shnum; i++) { | |
1873 | Elf_Shdr *shdr = &info->sechdrs[i]; | |
8b5f61a7 | 1874 | |
24b9f0d2 SS |
1875 | /* |
1876 | * Mark all sections sh_addr with their address in the | |
1877 | * temporary image. | |
1878 | */ | |
8b5f61a7 RR |
1879 | shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset; |
1880 | ||
8b5f61a7 | 1881 | } |
d6df72a0 RR |
1882 | |
1883 | /* Track but don't keep modinfo and version sections. */ | |
3e2e857f | 1884 | info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC; |
d6df72a0 | 1885 | info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC; |
3e2e857f | 1886 | |
8b5f61a7 RR |
1887 | return 0; |
1888 | } | |
1889 | ||
3264d3f9 LT |
1890 | /* |
1891 | * Set up our basic convenience variables (pointers to section headers, | |
1892 | * search for module section index etc), and do some basic section | |
1893 | * verification. | |
1894 | * | |
81a0abd9 JY |
1895 | * Set info->mod to the temporary copy of the module in info->hdr. The final one |
1896 | * will be allocated in move_module(). | |
3264d3f9 | 1897 | */ |
81a0abd9 | 1898 | static int setup_load_info(struct load_info *info, int flags) |
3264d3f9 LT |
1899 | { |
1900 | unsigned int i; | |
3264d3f9 | 1901 | |
5fdc7db6 JY |
1902 | /* Try to find a name early so we can log errors with a module name */ |
1903 | info->index.info = find_sec(info, ".modinfo"); | |
708e0ada | 1904 | if (info->index.info) |
5fdc7db6 | 1905 | info->name = get_modinfo(info, "name"); |
3264d3f9 | 1906 | |
8b5f61a7 RR |
1907 | /* Find internal symbols and strings. */ |
1908 | for (i = 1; i < info->hdr->e_shnum; i++) { | |
3264d3f9 LT |
1909 | if (info->sechdrs[i].sh_type == SHT_SYMTAB) { |
1910 | info->index.sym = i; | |
1911 | info->index.str = info->sechdrs[i].sh_link; | |
8b5f61a7 RR |
1912 | info->strtab = (char *)info->hdr |
1913 | + info->sechdrs[info->index.str].sh_offset; | |
1914 | break; | |
3264d3f9 | 1915 | } |
3264d3f9 LT |
1916 | } |
1917 | ||
5fdc7db6 | 1918 | if (info->index.sym == 0) { |
708e0ada JY |
1919 | pr_warn("%s: module has no symbols (stripped?)\n", |
1920 | info->name ?: "(missing .modinfo section or name field)"); | |
5fdc7db6 JY |
1921 | return -ENOEXEC; |
1922 | } | |
1923 | ||
49668688 | 1924 | info->index.mod = find_sec(info, ".gnu.linkonce.this_module"); |
3264d3f9 | 1925 | if (!info->index.mod) { |
3e2e857f | 1926 | pr_warn("%s: No module found in object\n", |
708e0ada | 1927 | info->name ?: "(missing .modinfo section or name field)"); |
81a0abd9 | 1928 | return -ENOEXEC; |
3264d3f9 LT |
1929 | } |
1930 | /* This is temporary: point mod into copy of data. */ | |
5fdc7db6 | 1931 | info->mod = (void *)info->hdr + info->sechdrs[info->index.mod].sh_offset; |
3264d3f9 | 1932 | |
3e2e857f | 1933 | /* |
5fdc7db6 | 1934 | * If we didn't load the .modinfo 'name' field earlier, fall back to |
3e2e857f KC |
1935 | * on-disk struct mod 'name' field. |
1936 | */ | |
1937 | if (!info->name) | |
81a0abd9 | 1938 | info->name = info->mod->name; |
3e2e857f | 1939 | |
5fdc7db6 JY |
1940 | if (flags & MODULE_INIT_IGNORE_MODVERSIONS) |
1941 | info->index.vers = 0; /* Pretend no __versions section! */ | |
1942 | else | |
1943 | info->index.vers = find_sec(info, "__versions"); | |
3264d3f9 | 1944 | |
49668688 | 1945 | info->index.pcpu = find_pcpusec(info); |
3264d3f9 | 1946 | |
81a0abd9 | 1947 | return 0; |
3264d3f9 LT |
1948 | } |
1949 | ||
2f3238ae | 1950 | static int check_modinfo(struct module *mod, struct load_info *info, int flags) |
40dd2560 | 1951 | { |
49668688 | 1952 | const char *modmagic = get_modinfo(info, "vermagic"); |
40dd2560 RR |
1953 | int err; |
1954 | ||
2f3238ae RR |
1955 | if (flags & MODULE_INIT_IGNORE_VERMAGIC) |
1956 | modmagic = NULL; | |
1957 | ||
40dd2560 RR |
1958 | /* This is allowed: modprobe --force will invalidate it. */ |
1959 | if (!modmagic) { | |
1960 | err = try_to_force_load(mod, "bad vermagic"); | |
1961 | if (err) | |
1962 | return err; | |
49668688 | 1963 | } else if (!same_magic(modmagic, vermagic, info->index.vers)) { |
bddb12b3 | 1964 | pr_err("%s: version magic '%s' should be '%s'\n", |
3e2e857f | 1965 | info->name, modmagic, vermagic); |
40dd2560 RR |
1966 | return -ENOEXEC; |
1967 | } | |
1968 | ||
3205c36c LP |
1969 | if (!get_modinfo(info, "intree")) { |
1970 | if (!test_taint(TAINT_OOT_MODULE)) | |
1971 | pr_warn("%s: loading out-of-tree module taints kernel.\n", | |
1972 | mod->name); | |
373d4d09 | 1973 | add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK); |
3205c36c | 1974 | } |
2449b8ba | 1975 | |
caf7501a AK |
1976 | check_modinfo_retpoline(mod, info); |
1977 | ||
49668688 | 1978 | if (get_modinfo(info, "staging")) { |
373d4d09 | 1979 | add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK); |
bddb12b3 AM |
1980 | pr_warn("%s: module is from the staging directory, the quality " |
1981 | "is unknown, you have been warned.\n", mod->name); | |
40dd2560 | 1982 | } |
22e268eb | 1983 | |
2992ef29 | 1984 | err = check_modinfo_livepatch(mod, info); |
1ce15ef4 JY |
1985 | if (err) |
1986 | return err; | |
1987 | ||
22e268eb | 1988 | /* Set up license info based on the info section */ |
49668688 | 1989 | set_license(mod, get_modinfo(info, "license")); |
22e268eb | 1990 | |
40dd2560 RR |
1991 | return 0; |
1992 | } | |
1993 | ||
eb3057df | 1994 | static int find_module_sections(struct module *mod, struct load_info *info) |
f91a13bb | 1995 | { |
49668688 | 1996 | mod->kp = section_objs(info, "__param", |
f91a13bb | 1997 | sizeof(*mod->kp), &mod->num_kp); |
49668688 | 1998 | mod->syms = section_objs(info, "__ksymtab", |
f91a13bb | 1999 | sizeof(*mod->syms), &mod->num_syms); |
49668688 RR |
2000 | mod->crcs = section_addr(info, "__kcrctab"); |
2001 | mod->gpl_syms = section_objs(info, "__ksymtab_gpl", | |
f91a13bb LT |
2002 | sizeof(*mod->gpl_syms), |
2003 | &mod->num_gpl_syms); | |
49668688 | 2004 | mod->gpl_crcs = section_addr(info, "__kcrctab_gpl"); |
f91a13bb | 2005 | |
f91a13bb | 2006 | #ifdef CONFIG_CONSTRUCTORS |
49668688 | 2007 | mod->ctors = section_objs(info, ".ctors", |
f91a13bb | 2008 | sizeof(*mod->ctors), &mod->num_ctors); |
eb3057df FH |
2009 | if (!mod->ctors) |
2010 | mod->ctors = section_objs(info, ".init_array", | |
2011 | sizeof(*mod->ctors), &mod->num_ctors); | |
2012 | else if (find_sec(info, ".init_array")) { | |
2013 | /* | |
2014 | * This shouldn't happen with same compiler and binutils | |
2015 | * building all parts of the module. | |
2016 | */ | |
6da0b565 | 2017 | pr_warn("%s: has both .ctors and .init_array.\n", |
eb3057df FH |
2018 | mod->name); |
2019 | return -EINVAL; | |
2020 | } | |
f91a13bb LT |
2021 | #endif |
2022 | ||
66e9b071 TG |
2023 | mod->noinstr_text_start = section_objs(info, ".noinstr.text", 1, |
2024 | &mod->noinstr_text_size); | |
2025 | ||
f91a13bb | 2026 | #ifdef CONFIG_TRACEPOINTS |
65498646 MD |
2027 | mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs", |
2028 | sizeof(*mod->tracepoints_ptrs), | |
2029 | &mod->num_tracepoints); | |
f91a13bb | 2030 | #endif |
fe15b50c PM |
2031 | #ifdef CONFIG_TREE_SRCU |
2032 | mod->srcu_struct_ptrs = section_objs(info, "___srcu_struct_ptrs", | |
2033 | sizeof(*mod->srcu_struct_ptrs), | |
2034 | &mod->num_srcu_structs); | |
2035 | #endif | |
a38d1107 MM |
2036 | #ifdef CONFIG_BPF_EVENTS |
2037 | mod->bpf_raw_events = section_objs(info, "__bpf_raw_tp_map", | |
2038 | sizeof(*mod->bpf_raw_events), | |
2039 | &mod->num_bpf_raw_events); | |
2040 | #endif | |
36e68442 AN |
2041 | #ifdef CONFIG_DEBUG_INFO_BTF_MODULES |
2042 | mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size); | |
2043 | #endif | |
e9666d10 | 2044 | #ifdef CONFIG_JUMP_LABEL |
bf5438fc JB |
2045 | mod->jump_entries = section_objs(info, "__jump_table", |
2046 | sizeof(*mod->jump_entries), | |
2047 | &mod->num_jump_entries); | |
2048 | #endif | |
f91a13bb | 2049 | #ifdef CONFIG_EVENT_TRACING |
49668688 | 2050 | mod->trace_events = section_objs(info, "_ftrace_events", |
f91a13bb LT |
2051 | sizeof(*mod->trace_events), |
2052 | &mod->num_trace_events); | |
99be647c JL |
2053 | mod->trace_evals = section_objs(info, "_ftrace_eval_map", |
2054 | sizeof(*mod->trace_evals), | |
2055 | &mod->num_trace_evals); | |
f91a13bb | 2056 | #endif |
13b9b6e7 SR |
2057 | #ifdef CONFIG_TRACING |
2058 | mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt", | |
2059 | sizeof(*mod->trace_bprintk_fmt_start), | |
2060 | &mod->num_trace_bprintk_fmt); | |
13b9b6e7 | 2061 | #endif |
f91a13bb LT |
2062 | #ifdef CONFIG_FTRACE_MCOUNT_RECORD |
2063 | /* sechdrs[0].sh_size is always zero */ | |
a1326b17 | 2064 | mod->ftrace_callsites = section_objs(info, FTRACE_CALLSITE_SECTION, |
f91a13bb LT |
2065 | sizeof(*mod->ftrace_callsites), |
2066 | &mod->num_ftrace_callsites); | |
2067 | #endif | |
540adea3 MH |
2068 | #ifdef CONFIG_FUNCTION_ERROR_INJECTION |
2069 | mod->ei_funcs = section_objs(info, "_error_injection_whitelist", | |
2070 | sizeof(*mod->ei_funcs), | |
2071 | &mod->num_ei_funcs); | |
1e6769b0 MH |
2072 | #endif |
2073 | #ifdef CONFIG_KPROBES | |
2074 | mod->kprobes_text_start = section_objs(info, ".kprobes.text", 1, | |
2075 | &mod->kprobes_text_size); | |
16db6264 MH |
2076 | mod->kprobe_blacklist = section_objs(info, "_kprobe_blacklist", |
2077 | sizeof(unsigned long), | |
2078 | &mod->num_kprobe_blacklist); | |
9183c3f9 | 2079 | #endif |
33701557 CD |
2080 | #ifdef CONFIG_PRINTK_INDEX |
2081 | mod->printk_index_start = section_objs(info, ".printk_index", | |
2082 | sizeof(*mod->printk_index_start), | |
2083 | &mod->printk_index_size); | |
2084 | #endif | |
9183c3f9 JP |
2085 | #ifdef CONFIG_HAVE_STATIC_CALL_INLINE |
2086 | mod->static_call_sites = section_objs(info, ".static_call_sites", | |
2087 | sizeof(*mod->static_call_sites), | |
2088 | &mod->num_static_call_sites); | |
92ace999 | 2089 | #endif |
811d66a0 RR |
2090 | mod->extable = section_objs(info, "__ex_table", |
2091 | sizeof(*mod->extable), &mod->num_exentries); | |
2092 | ||
49668688 | 2093 | if (section_addr(info, "__obsparm")) |
bddb12b3 | 2094 | pr_warn("%s: Ignoring obsolete parameters\n", mod->name); |
811d66a0 | 2095 | |
e5ebffe1 | 2096 | info->debug = section_objs(info, "__dyndbg", |
811d66a0 | 2097 | sizeof(*info->debug), &info->num_debug); |
eb3057df FH |
2098 | |
2099 | return 0; | |
f91a13bb LT |
2100 | } |
2101 | ||
49668688 | 2102 | static int move_module(struct module *mod, struct load_info *info) |
65b8a9b4 LT |
2103 | { |
2104 | int i; | |
2105 | void *ptr; | |
2106 | ||
2107 | /* Do the allocs. */ | |
7523e4dc | 2108 | ptr = module_alloc(mod->core_layout.size); |
65b8a9b4 LT |
2109 | /* |
2110 | * The pointer to this block is stored in the module structure | |
2111 | * which is inside the block. Just mark it as not being a | |
2112 | * leak. | |
2113 | */ | |
2114 | kmemleak_not_leak(ptr); | |
2115 | if (!ptr) | |
d913188c | 2116 | return -ENOMEM; |
65b8a9b4 | 2117 | |
7523e4dc RR |
2118 | memset(ptr, 0, mod->core_layout.size); |
2119 | mod->core_layout.base = ptr; | |
65b8a9b4 | 2120 | |
7523e4dc RR |
2121 | if (mod->init_layout.size) { |
2122 | ptr = module_alloc(mod->init_layout.size); | |
82fab442 RR |
2123 | /* |
2124 | * The pointer to this block is stored in the module structure | |
2125 | * which is inside the block. This block doesn't need to be | |
2126 | * scanned as it contains data and code that will be freed | |
2127 | * after the module is initialized. | |
2128 | */ | |
2129 | kmemleak_ignore(ptr); | |
2130 | if (!ptr) { | |
7523e4dc | 2131 | module_memfree(mod->core_layout.base); |
82fab442 RR |
2132 | return -ENOMEM; |
2133 | } | |
7523e4dc RR |
2134 | memset(ptr, 0, mod->init_layout.size); |
2135 | mod->init_layout.base = ptr; | |
82fab442 | 2136 | } else |
7523e4dc | 2137 | mod->init_layout.base = NULL; |
65b8a9b4 | 2138 | |
01dc0386 CL |
2139 | #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC |
2140 | /* Do the allocs. */ | |
2141 | ptr = vmalloc(mod->data_layout.size); | |
2142 | /* | |
2143 | * The pointer to this block is stored in the module structure | |
2144 | * which is inside the block. Just mark it as not being a | |
2145 | * leak. | |
2146 | */ | |
2147 | kmemleak_not_leak(ptr); | |
2148 | if (!ptr) { | |
2149 | module_memfree(mod->core_layout.base); | |
2150 | module_memfree(mod->init_layout.base); | |
2151 | return -ENOMEM; | |
2152 | } | |
2153 | ||
2154 | memset(ptr, 0, mod->data_layout.size); | |
2155 | mod->data_layout.base = ptr; | |
2156 | #endif | |
65b8a9b4 | 2157 | /* Transfer each section which specifies SHF_ALLOC */ |
5e124169 | 2158 | pr_debug("final section addresses:\n"); |
49668688 | 2159 | for (i = 0; i < info->hdr->e_shnum; i++) { |
65b8a9b4 | 2160 | void *dest; |
49668688 | 2161 | Elf_Shdr *shdr = &info->sechdrs[i]; |
65b8a9b4 | 2162 | |
49668688 | 2163 | if (!(shdr->sh_flags & SHF_ALLOC)) |
65b8a9b4 LT |
2164 | continue; |
2165 | ||
49668688 | 2166 | if (shdr->sh_entsize & INIT_OFFSET_MASK) |
7523e4dc | 2167 | dest = mod->init_layout.base |
49668688 | 2168 | + (shdr->sh_entsize & ~INIT_OFFSET_MASK); |
6ab9942c CL |
2169 | else if (!(shdr->sh_flags & SHF_EXECINSTR)) |
2170 | dest = mod->data_layout.base + shdr->sh_entsize; | |
65b8a9b4 | 2171 | else |
7523e4dc | 2172 | dest = mod->core_layout.base + shdr->sh_entsize; |
65b8a9b4 | 2173 | |
49668688 RR |
2174 | if (shdr->sh_type != SHT_NOBITS) |
2175 | memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size); | |
65b8a9b4 | 2176 | /* Update sh_addr to point to copy in image. */ |
49668688 | 2177 | shdr->sh_addr = (unsigned long)dest; |
5e124169 JC |
2178 | pr_debug("\t0x%lx %s\n", |
2179 | (long)shdr->sh_addr, info->secstrings + shdr->sh_name); | |
65b8a9b4 | 2180 | } |
d913188c RR |
2181 | |
2182 | return 0; | |
65b8a9b4 LT |
2183 | } |
2184 | ||
49668688 | 2185 | static int check_module_license_and_versions(struct module *mod) |
22e268eb | 2186 | { |
3205c36c LP |
2187 | int prev_taint = test_taint(TAINT_PROPRIETARY_MODULE); |
2188 | ||
22e268eb RR |
2189 | /* |
2190 | * ndiswrapper is under GPL by itself, but loads proprietary modules. | |
2191 | * Don't use add_taint_module(), as it would prevent ndiswrapper from | |
2192 | * using GPL-only symbols it needs. | |
2193 | */ | |
2194 | if (strcmp(mod->name, "ndiswrapper") == 0) | |
373d4d09 | 2195 | add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE); |
22e268eb RR |
2196 | |
2197 | /* driverloader was caught wrongly pretending to be under GPL */ | |
2198 | if (strcmp(mod->name, "driverloader") == 0) | |
373d4d09 RR |
2199 | add_taint_module(mod, TAINT_PROPRIETARY_MODULE, |
2200 | LOCKDEP_NOW_UNRELIABLE); | |
22e268eb | 2201 | |
c99af375 MG |
2202 | /* lve claims to be GPL but upstream won't provide source */ |
2203 | if (strcmp(mod->name, "lve") == 0) | |
373d4d09 RR |
2204 | add_taint_module(mod, TAINT_PROPRIETARY_MODULE, |
2205 | LOCKDEP_NOW_UNRELIABLE); | |
c99af375 | 2206 | |
3205c36c LP |
2207 | if (!prev_taint && test_taint(TAINT_PROPRIETARY_MODULE)) |
2208 | pr_warn("%s: module license taints kernel.\n", mod->name); | |
2209 | ||
22e268eb | 2210 | #ifdef CONFIG_MODVERSIONS |
36794822 CH |
2211 | if ((mod->num_syms && !mod->crcs) || |
2212 | (mod->num_gpl_syms && !mod->gpl_crcs)) { | |
22e268eb RR |
2213 | return try_to_force_load(mod, |
2214 | "no versions for exported symbols"); | |
2215 | } | |
2216 | #endif | |
2217 | return 0; | |
2218 | } | |
2219 | ||
2220 | static void flush_module_icache(const struct module *mod) | |
2221 | { | |
22e268eb RR |
2222 | /* |
2223 | * Flush the instruction cache, since we've played with text. | |
2224 | * Do it before processing of module parameters, so the module | |
2225 | * can provide parameter accessor functions of its own. | |
2226 | */ | |
7523e4dc RR |
2227 | if (mod->init_layout.base) |
2228 | flush_icache_range((unsigned long)mod->init_layout.base, | |
2229 | (unsigned long)mod->init_layout.base | |
2230 | + mod->init_layout.size); | |
2231 | flush_icache_range((unsigned long)mod->core_layout.base, | |
2232 | (unsigned long)mod->core_layout.base + mod->core_layout.size); | |
22e268eb RR |
2233 | } |
2234 | ||
74e08fcf JB |
2235 | int __weak module_frob_arch_sections(Elf_Ehdr *hdr, |
2236 | Elf_Shdr *sechdrs, | |
2237 | char *secstrings, | |
2238 | struct module *mod) | |
2239 | { | |
2240 | return 0; | |
2241 | } | |
2242 | ||
be7de5f9 PB |
2243 | /* module_blacklist is a comma-separated list of module names */ |
2244 | static char *module_blacklist; | |
96b5b194 | 2245 | static bool blacklisted(const char *module_name) |
be7de5f9 PB |
2246 | { |
2247 | const char *p; | |
2248 | size_t len; | |
2249 | ||
2250 | if (!module_blacklist) | |
2251 | return false; | |
2252 | ||
2253 | for (p = module_blacklist; *p; p += len) { | |
2254 | len = strcspn(p, ","); | |
2255 | if (strlen(module_name) == len && !memcmp(module_name, p, len)) | |
2256 | return true; | |
2257 | if (p[len] == ',') | |
2258 | len++; | |
2259 | } | |
2260 | return false; | |
2261 | } | |
2262 | core_param(module_blacklist, module_blacklist, charp, 0400); | |
2263 | ||
2f3238ae | 2264 | static struct module *layout_and_allocate(struct load_info *info, int flags) |
1da177e4 | 2265 | { |
1da177e4 | 2266 | struct module *mod; |
444d13ff | 2267 | unsigned int ndx; |
d913188c | 2268 | int err; |
3ae91c21 | 2269 | |
81a0abd9 | 2270 | err = check_modinfo(info->mod, info, flags); |
40dd2560 RR |
2271 | if (err) |
2272 | return ERR_PTR(err); | |
1da177e4 | 2273 | |
1da177e4 | 2274 | /* Allow arches to frob section contents and sizes. */ |
49668688 | 2275 | err = module_frob_arch_sections(info->hdr, info->sechdrs, |
81a0abd9 | 2276 | info->secstrings, info->mod); |
1da177e4 | 2277 | if (err < 0) |
8d8022e8 | 2278 | return ERR_PTR(err); |
1da177e4 | 2279 | |
5c3a7db0 PZ |
2280 | err = module_enforce_rwx_sections(info->hdr, info->sechdrs, |
2281 | info->secstrings, info->mod); | |
2282 | if (err < 0) | |
2283 | return ERR_PTR(err); | |
2284 | ||
8d8022e8 RR |
2285 | /* We will do a special allocation for per-cpu sections later. */ |
2286 | info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC; | |
1da177e4 | 2287 | |
444d13ff JY |
2288 | /* |
2289 | * Mark ro_after_init section with SHF_RO_AFTER_INIT so that | |
2290 | * layout_sections() can put it in the right place. | |
2291 | * Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set. | |
2292 | */ | |
2293 | ndx = find_sec(info, ".data..ro_after_init"); | |
e872267b AB |
2294 | if (ndx) |
2295 | info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT; | |
2296 | /* | |
2297 | * Mark the __jump_table section as ro_after_init as well: these data | |
2298 | * structures are never modified, with the exception of entries that | |
2299 | * refer to code in the __init section, which are annotated as such | |
2300 | * at module load time. | |
2301 | */ | |
2302 | ndx = find_sec(info, "__jump_table"); | |
444d13ff JY |
2303 | if (ndx) |
2304 | info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT; | |
2305 | ||
24b9f0d2 SS |
2306 | /* |
2307 | * Determine total sizes, and put offsets in sh_entsize. For now | |
2308 | * this is done generically; there doesn't appear to be any | |
2309 | * special cases for the architectures. | |
2310 | */ | |
81a0abd9 JY |
2311 | layout_sections(info->mod, info); |
2312 | layout_symtab(info->mod, info); | |
1da177e4 | 2313 | |
65b8a9b4 | 2314 | /* Allocate and move to the final place */ |
81a0abd9 | 2315 | err = move_module(info->mod, info); |
d913188c | 2316 | if (err) |
8d8022e8 | 2317 | return ERR_PTR(err); |
d913188c RR |
2318 | |
2319 | /* Module has been copied to its final place now: return it. */ | |
2320 | mod = (void *)info->sechdrs[info->index.mod].sh_addr; | |
49668688 | 2321 | kmemleak_load_module(mod, info); |
d913188c | 2322 | return mod; |
d913188c RR |
2323 | } |
2324 | ||
2325 | /* mod is no longer valid after this! */ | |
2326 | static void module_deallocate(struct module *mod, struct load_info *info) | |
2327 | { | |
d913188c | 2328 | percpu_modfree(mod); |
d453cded | 2329 | module_arch_freeing_init(mod); |
7523e4dc RR |
2330 | module_memfree(mod->init_layout.base); |
2331 | module_memfree(mod->core_layout.base); | |
01dc0386 CL |
2332 | #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC |
2333 | vfree(mod->data_layout.base); | |
2334 | #endif | |
d913188c RR |
2335 | } |
2336 | ||
74e08fcf JB |
2337 | int __weak module_finalize(const Elf_Ehdr *hdr, |
2338 | const Elf_Shdr *sechdrs, | |
2339 | struct module *me) | |
2340 | { | |
2341 | return 0; | |
2342 | } | |
2343 | ||
811d66a0 RR |
2344 | static int post_relocation(struct module *mod, const struct load_info *info) |
2345 | { | |
51f3d0f4 | 2346 | /* Sort exception table now relocations are done. */ |
811d66a0 RR |
2347 | sort_extable(mod->extable, mod->extable + mod->num_exentries); |
2348 | ||
2349 | /* Copy relocated percpu area over. */ | |
2350 | percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr, | |
2351 | info->sechdrs[info->index.pcpu].sh_size); | |
2352 | ||
51f3d0f4 | 2353 | /* Setup kallsyms-specific fields. */ |
811d66a0 RR |
2354 | add_kallsyms(mod, info); |
2355 | ||
2356 | /* Arch-specific module finalizing. */ | |
2357 | return module_finalize(info->hdr, info->sechdrs, mod); | |
2358 | } | |
2359 | ||
9bb9c3be RR |
2360 | /* Is this module of this name done loading? No locks held. */ |
2361 | static bool finished_loading(const char *name) | |
2362 | { | |
2363 | struct module *mod; | |
2364 | bool ret; | |
2365 | ||
9cc019b8 PZ |
2366 | /* |
2367 | * The module_mutex should not be a heavily contended lock; | |
2368 | * if we get the occasional sleep here, we'll go an extra iteration | |
2369 | * in the wait_event_interruptible(), which is harmless. | |
2370 | */ | |
2371 | sched_annotate_sleep(); | |
9bb9c3be | 2372 | mutex_lock(&module_mutex); |
4f6de4d5 | 2373 | mod = find_module_all(name, strlen(name), true); |
6e6de3de | 2374 | ret = !mod || mod->state == MODULE_STATE_LIVE; |
9bb9c3be RR |
2375 | mutex_unlock(&module_mutex); |
2376 | ||
2377 | return ret; | |
2378 | } | |
2379 | ||
34e1169d KC |
2380 | /* Call module constructors. */ |
2381 | static void do_mod_ctors(struct module *mod) | |
2382 | { | |
2383 | #ifdef CONFIG_CONSTRUCTORS | |
2384 | unsigned long i; | |
2385 | ||
2386 | for (i = 0; i < mod->num_ctors; i++) | |
2387 | mod->ctors[i](); | |
2388 | #endif | |
2389 | } | |
2390 | ||
c7496379 RR |
2391 | /* For freeing module_init on success, in case kallsyms traversing */ |
2392 | struct mod_initfree { | |
1a7b7d92 | 2393 | struct llist_node node; |
c7496379 RR |
2394 | void *module_init; |
2395 | }; | |
2396 | ||
1a7b7d92 | 2397 | static void do_free_init(struct work_struct *w) |
c7496379 | 2398 | { |
1a7b7d92 RE |
2399 | struct llist_node *pos, *n, *list; |
2400 | struct mod_initfree *initfree; | |
2401 | ||
2402 | list = llist_del_all(&init_free_list); | |
2403 | ||
2404 | synchronize_rcu(); | |
2405 | ||
2406 | llist_for_each_safe(pos, n, list) { | |
2407 | initfree = container_of(pos, struct mod_initfree, node); | |
2408 | module_memfree(initfree->module_init); | |
2409 | kfree(initfree); | |
2410 | } | |
c7496379 RR |
2411 | } |
2412 | ||
be02a186 JK |
2413 | /* |
2414 | * This is where the real work happens. | |
2415 | * | |
2416 | * Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb | |
2417 | * helper command 'lx-symbols'. | |
2418 | */ | |
2419 | static noinline int do_init_module(struct module *mod) | |
34e1169d KC |
2420 | { |
2421 | int ret = 0; | |
c7496379 RR |
2422 | struct mod_initfree *freeinit; |
2423 | ||
2424 | freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL); | |
2425 | if (!freeinit) { | |
2426 | ret = -ENOMEM; | |
2427 | goto fail; | |
2428 | } | |
7523e4dc | 2429 | freeinit->module_init = mod->init_layout.base; |
34e1169d | 2430 | |
34e1169d KC |
2431 | do_mod_ctors(mod); |
2432 | /* Start the module */ | |
2433 | if (mod->init != NULL) | |
2434 | ret = do_one_initcall(mod->init); | |
2435 | if (ret < 0) { | |
c7496379 | 2436 | goto fail_free_freeinit; |
34e1169d KC |
2437 | } |
2438 | if (ret > 0) { | |
bddb12b3 AM |
2439 | pr_warn("%s: '%s'->init suspiciously returned %d, it should " |
2440 | "follow 0/-E convention\n" | |
2441 | "%s: loading module anyway...\n", | |
2442 | __func__, mod->name, ret, __func__); | |
34e1169d KC |
2443 | dump_stack(); |
2444 | } | |
2445 | ||
2446 | /* Now it's a first class citizen! */ | |
2447 | mod->state = MODULE_STATE_LIVE; | |
2448 | blocking_notifier_call_chain(&module_notify_list, | |
2449 | MODULE_STATE_LIVE, mod); | |
2450 | ||
38dc717e JY |
2451 | /* Delay uevent until module has finished its init routine */ |
2452 | kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD); | |
2453 | ||
774a1221 TH |
2454 | /* |
2455 | * We need to finish all async code before the module init sequence | |
67d6212a IP |
2456 | * is done. This has potential to deadlock if synchronous module |
2457 | * loading is requested from async (which is not allowed!). | |
774a1221 | 2458 | * |
67d6212a IP |
2459 | * See commit 0fdff3ec6d87 ("async, kmod: warn on synchronous |
2460 | * request_module() from async workers") for more details. | |
774a1221 | 2461 | */ |
67d6212a | 2462 | if (!mod->async_probe_requested) |
774a1221 | 2463 | async_synchronize_full(); |
34e1169d | 2464 | |
aba4b5c2 | 2465 | ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base + |
3e234289 | 2466 | mod->init_layout.size); |
34e1169d KC |
2467 | mutex_lock(&module_mutex); |
2468 | /* Drop initial reference. */ | |
2469 | module_put(mod); | |
2470 | trim_init_extable(mod); | |
2471 | #ifdef CONFIG_KALLSYMS | |
8244062e RR |
2472 | /* Switch to core kallsyms now init is done: kallsyms may be walking! */ |
2473 | rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms); | |
34e1169d | 2474 | #endif |
444d13ff | 2475 | module_enable_ro(mod, true); |
93c2e105 | 2476 | mod_tree_remove_init(mod); |
d453cded | 2477 | module_arch_freeing_init(mod); |
7523e4dc RR |
2478 | mod->init_layout.base = NULL; |
2479 | mod->init_layout.size = 0; | |
2480 | mod->init_layout.ro_size = 0; | |
444d13ff | 2481 | mod->init_layout.ro_after_init_size = 0; |
7523e4dc | 2482 | mod->init_layout.text_size = 0; |
607c543f AN |
2483 | #ifdef CONFIG_DEBUG_INFO_BTF_MODULES |
2484 | /* .BTF is not SHF_ALLOC and will get removed, so sanitize pointer */ | |
2485 | mod->btf_data = NULL; | |
607c543f | 2486 | #endif |
c7496379 RR |
2487 | /* |
2488 | * We want to free module_init, but be aware that kallsyms may be | |
0be964be | 2489 | * walking this with preempt disabled. In all the failure paths, we |
cb2f5536 | 2490 | * call synchronize_rcu(), but we don't want to slow down the success |
1a7b7d92 RE |
2491 | * path. module_memfree() cannot be called in an interrupt, so do the |
2492 | * work and call synchronize_rcu() in a work queue. | |
2493 | * | |
ae646f0b JH |
2494 | * Note that module_alloc() on most architectures creates W+X page |
2495 | * mappings which won't be cleaned up until do_free_init() runs. Any | |
2496 | * code such as mark_rodata_ro() which depends on those mappings to | |
2497 | * be cleaned up needs to sync with the queued work - ie | |
cb2f5536 | 2498 | * rcu_barrier() |
c7496379 | 2499 | */ |
1a7b7d92 RE |
2500 | if (llist_add(&freeinit->node, &init_free_list)) |
2501 | schedule_work(&init_free_wq); | |
2502 | ||
34e1169d KC |
2503 | mutex_unlock(&module_mutex); |
2504 | wake_up_all(&module_wq); | |
2505 | ||
2506 | return 0; | |
c7496379 RR |
2507 | |
2508 | fail_free_freeinit: | |
2509 | kfree(freeinit); | |
2510 | fail: | |
2511 | /* Try to protect us from buggy refcounters. */ | |
2512 | mod->state = MODULE_STATE_GOING; | |
cb2f5536 | 2513 | synchronize_rcu(); |
c7496379 RR |
2514 | module_put(mod); |
2515 | blocking_notifier_call_chain(&module_notify_list, | |
2516 | MODULE_STATE_GOING, mod); | |
7e545d6e | 2517 | klp_module_going(mod); |
7dcd182b | 2518 | ftrace_release_mod(mod); |
c7496379 RR |
2519 | free_module(mod); |
2520 | wake_up_all(&module_wq); | |
2521 | return ret; | |
34e1169d KC |
2522 | } |
2523 | ||
2524 | static int may_init_module(void) | |
2525 | { | |
2526 | if (!capable(CAP_SYS_MODULE) || modules_disabled) | |
2527 | return -EPERM; | |
2528 | ||
2529 | return 0; | |
2530 | } | |
2531 | ||
a3535c7e RR |
2532 | /* |
2533 | * We try to place it in the list now to make sure it's unique before | |
2534 | * we dedicate too many resources. In particular, temporary percpu | |
2535 | * memory exhaustion. | |
2536 | */ | |
2537 | static int add_unformed_module(struct module *mod) | |
2538 | { | |
2539 | int err; | |
2540 | struct module *old; | |
2541 | ||
2542 | mod->state = MODULE_STATE_UNFORMED; | |
2543 | ||
2544 | again: | |
2545 | mutex_lock(&module_mutex); | |
4f6de4d5 MK |
2546 | old = find_module_all(mod->name, strlen(mod->name), true); |
2547 | if (old != NULL) { | |
6e6de3de | 2548 | if (old->state != MODULE_STATE_LIVE) { |
a3535c7e RR |
2549 | /* Wait in case it fails to load. */ |
2550 | mutex_unlock(&module_mutex); | |
9cc019b8 PZ |
2551 | err = wait_event_interruptible(module_wq, |
2552 | finished_loading(mod->name)); | |
a3535c7e RR |
2553 | if (err) |
2554 | goto out_unlocked; | |
2555 | goto again; | |
2556 | } | |
2557 | err = -EEXIST; | |
2558 | goto out; | |
2559 | } | |
4f666546 | 2560 | mod_update_bounds(mod); |
a3535c7e | 2561 | list_add_rcu(&mod->list, &modules); |
93c2e105 | 2562 | mod_tree_insert(mod); |
a3535c7e RR |
2563 | err = 0; |
2564 | ||
2565 | out: | |
2566 | mutex_unlock(&module_mutex); | |
2567 | out_unlocked: | |
2568 | return err; | |
2569 | } | |
2570 | ||
2571 | static int complete_formation(struct module *mod, struct load_info *info) | |
2572 | { | |
2573 | int err; | |
2574 | ||
2575 | mutex_lock(&module_mutex); | |
2576 | ||
2577 | /* Find duplicate symbols (must be called under lock). */ | |
2d25bc55 | 2578 | err = verify_exported_symbols(mod); |
a3535c7e RR |
2579 | if (err < 0) |
2580 | goto out; | |
2581 | ||
2582 | /* This relies on module_mutex for list integrity. */ | |
2583 | module_bug_finalize(info->hdr, info->sechdrs, mod); | |
2584 | ||
ef505058 CL |
2585 | if (module_check_misalignment(mod)) |
2586 | goto out_misaligned; | |
2587 | ||
444d13ff | 2588 | module_enable_ro(mod, false); |
85c898db | 2589 | module_enable_nx(mod); |
af742623 | 2590 | module_enable_x(mod); |
4982223e | 2591 | |
24b9f0d2 SS |
2592 | /* |
2593 | * Mark state as coming so strong_try_module_get() ignores us, | |
2594 | * but kallsyms etc. can see us. | |
2595 | */ | |
a3535c7e | 2596 | mod->state = MODULE_STATE_COMING; |
4982223e RR |
2597 | mutex_unlock(&module_mutex); |
2598 | ||
4982223e | 2599 | return 0; |
a3535c7e | 2600 | |
ef505058 CL |
2601 | out_misaligned: |
2602 | err = -EINVAL; | |
a3535c7e RR |
2603 | out: |
2604 | mutex_unlock(&module_mutex); | |
2605 | return err; | |
2606 | } | |
2607 | ||
4c973d16 JY |
2608 | static int prepare_coming_module(struct module *mod) |
2609 | { | |
7e545d6e JY |
2610 | int err; |
2611 | ||
4c973d16 | 2612 | ftrace_module_enable(mod); |
7e545d6e JY |
2613 | err = klp_module_coming(mod); |
2614 | if (err) | |
2615 | return err; | |
2616 | ||
59cc8e0a PZ |
2617 | err = blocking_notifier_call_chain_robust(&module_notify_list, |
2618 | MODULE_STATE_COMING, MODULE_STATE_GOING, mod); | |
2619 | err = notifier_to_errno(err); | |
2620 | if (err) | |
2621 | klp_module_going(mod); | |
2622 | ||
2623 | return err; | |
4c973d16 JY |
2624 | } |
2625 | ||
ecc86170 LR |
2626 | static int unknown_module_param_cb(char *param, char *val, const char *modname, |
2627 | void *arg) | |
54041d8a | 2628 | { |
f2411da7 LR |
2629 | struct module *mod = arg; |
2630 | int ret; | |
2631 | ||
2632 | if (strcmp(param, "async_probe") == 0) { | |
2633 | mod->async_probe_requested = true; | |
2634 | return 0; | |
2635 | } | |
2636 | ||
6da0b565 | 2637 | /* Check for magic 'dyndbg' arg */ |
f2411da7 | 2638 | ret = ddebug_dyndbg_module_param_cb(param, val, modname); |
bddb12b3 AM |
2639 | if (ret != 0) |
2640 | pr_warn("%s: unknown parameter '%s' ignored\n", modname, param); | |
54041d8a RR |
2641 | return 0; |
2642 | } | |
2643 | ||
cf68fffb ST |
2644 | static void cfi_init(struct module *mod); |
2645 | ||
24b9f0d2 SS |
2646 | /* |
2647 | * Allocate and load the module: note that size of section 0 is always | |
2648 | * zero, and we rely on this for optional sections. | |
2649 | */ | |
2f3238ae RR |
2650 | static int load_module(struct load_info *info, const char __user *uargs, |
2651 | int flags) | |
d913188c | 2652 | { |
a3535c7e | 2653 | struct module *mod; |
5fdc7db6 | 2654 | long err = 0; |
51e158c1 | 2655 | char *after_dashes; |
d913188c | 2656 | |
ec2a2959 FL |
2657 | /* |
2658 | * Do the signature check (if any) first. All that | |
2659 | * the signature check needs is info->len, it does | |
2660 | * not need any of the section info. That can be | |
2661 | * set up later. This will minimize the chances | |
2662 | * of a corrupt module causing problems before | |
2663 | * we even get to the signature check. | |
2664 | * | |
2665 | * The check will also adjust info->len by stripping | |
2666 | * off the sig length at the end of the module, making | |
2667 | * checks against info->len more correct. | |
2668 | */ | |
2669 | err = module_sig_check(info, flags); | |
2670 | if (err) | |
2671 | goto free_copy; | |
2672 | ||
2673 | /* | |
2674 | * Do basic sanity checks against the ELF header and | |
2675 | * sections. | |
2676 | */ | |
2677 | err = elf_validity_check(info); | |
7fd982f3 | 2678 | if (err) |
5fdc7db6 JY |
2679 | goto free_copy; |
2680 | ||
ec2a2959 FL |
2681 | /* |
2682 | * Everything checks out, so set up the section info | |
2683 | * in the info structure. | |
2684 | */ | |
5fdc7db6 JY |
2685 | err = setup_load_info(info, flags); |
2686 | if (err) | |
2687 | goto free_copy; | |
2688 | ||
ec2a2959 FL |
2689 | /* |
2690 | * Now that we know we have the correct module name, check | |
2691 | * if it's blacklisted. | |
2692 | */ | |
5fdc7db6 JY |
2693 | if (blacklisted(info->name)) { |
2694 | err = -EPERM; | |
14721add | 2695 | pr_err("Module %s is blacklisted\n", info->name); |
5fdc7db6 JY |
2696 | goto free_copy; |
2697 | } | |
2698 | ||
5fdc7db6 | 2699 | err = rewrite_section_headers(info, flags); |
d913188c | 2700 | if (err) |
34e1169d | 2701 | goto free_copy; |
d913188c | 2702 | |
5fdc7db6 JY |
2703 | /* Check module struct version now, before we try to use module. */ |
2704 | if (!check_modstruct_version(info, info->mod)) { | |
2705 | err = -ENOEXEC; | |
2706 | goto free_copy; | |
2707 | } | |
2708 | ||
d913188c | 2709 | /* Figure out module layout, and allocate all the memory. */ |
2f3238ae | 2710 | mod = layout_and_allocate(info, flags); |
65b8a9b4 LT |
2711 | if (IS_ERR(mod)) { |
2712 | err = PTR_ERR(mod); | |
d913188c | 2713 | goto free_copy; |
1da177e4 | 2714 | } |
1da177e4 | 2715 | |
ca86cad7 RGB |
2716 | audit_log_kern_module(mod->name); |
2717 | ||
a3535c7e RR |
2718 | /* Reserve our place in the list. */ |
2719 | err = add_unformed_module(mod); | |
2720 | if (err) | |
1fb9341a | 2721 | goto free_module; |
1fb9341a | 2722 | |
106a4ee2 | 2723 | #ifdef CONFIG_MODULE_SIG |
34e1169d | 2724 | mod->sig_ok = info->sig_ok; |
64748a2c | 2725 | if (!mod->sig_ok) { |
bddb12b3 | 2726 | pr_notice_once("%s: module verification failed: signature " |
ab92ebbb | 2727 | "and/or required key missing - tainting " |
bddb12b3 | 2728 | "kernel\n", mod->name); |
66cc69e3 | 2729 | add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK); |
64748a2c | 2730 | } |
106a4ee2 RR |
2731 | #endif |
2732 | ||
8d8022e8 | 2733 | /* To avoid stressing percpu allocator, do this once we're unique. */ |
9eb76d77 | 2734 | err = percpu_modalloc(mod, info); |
8d8022e8 RR |
2735 | if (err) |
2736 | goto unlink_mod; | |
2737 | ||
49668688 | 2738 | /* Now module is in final location, initialize linked lists, etc. */ |
9f85a4bb RR |
2739 | err = module_unload_init(mod); |
2740 | if (err) | |
1fb9341a | 2741 | goto unlink_mod; |
1da177e4 | 2742 | |
cf2fde7b | 2743 | init_param_lock(mod); |
b51d23e4 | 2744 | |
24b9f0d2 SS |
2745 | /* |
2746 | * Now we've got everything in the final locations, we can | |
2747 | * find optional sections. | |
2748 | */ | |
eb3057df FH |
2749 | err = find_module_sections(mod, info); |
2750 | if (err) | |
2751 | goto free_unload; | |
9b37ccfc | 2752 | |
49668688 | 2753 | err = check_module_license_and_versions(mod); |
22e268eb RR |
2754 | if (err) |
2755 | goto free_unload; | |
9841d61d | 2756 | |
c988d2b2 | 2757 | /* Set up MODINFO_ATTR fields */ |
34e1169d | 2758 | setup_modinfo(mod, info); |
c988d2b2 | 2759 | |
1da177e4 | 2760 | /* Fix up syms, so that st_value is a pointer to location. */ |
34e1169d | 2761 | err = simplify_symbols(mod, info); |
1da177e4 | 2762 | if (err < 0) |
d913188c | 2763 | goto free_modinfo; |
1da177e4 | 2764 | |
34e1169d | 2765 | err = apply_relocations(mod, info); |
22e268eb | 2766 | if (err < 0) |
d913188c | 2767 | goto free_modinfo; |
1da177e4 | 2768 | |
34e1169d | 2769 | err = post_relocation(mod, info); |
1da177e4 | 2770 | if (err < 0) |
d913188c | 2771 | goto free_modinfo; |
1da177e4 | 2772 | |
22e268eb | 2773 | flush_module_icache(mod); |
378bac82 | 2774 | |
cf68fffb ST |
2775 | /* Setup CFI for the module. */ |
2776 | cfi_init(mod); | |
2777 | ||
6526c534 RR |
2778 | /* Now copy in args */ |
2779 | mod->args = strndup_user(uargs, ~0UL >> 1); | |
2780 | if (IS_ERR(mod->args)) { | |
2781 | err = PTR_ERR(mod->args); | |
2782 | goto free_arch_cleanup; | |
2783 | } | |
8d3b33f6 | 2784 | |
9294523e | 2785 | init_build_id(mod, info); |
52796312 | 2786 | dynamic_debug_setup(mod, info->debug, info->num_debug); |
ff49d74a | 2787 | |
a949ae56 SRRH |
2788 | /* Ftrace init must be called in the MODULE_STATE_UNFORMED state */ |
2789 | ftrace_module_init(mod); | |
2790 | ||
a3535c7e RR |
2791 | /* Finally it's fully formed, ready to start executing. */ |
2792 | err = complete_formation(mod, info); | |
2793 | if (err) | |
1fb9341a | 2794 | goto ddebug_cleanup; |
be593f4c | 2795 | |
4c973d16 JY |
2796 | err = prepare_coming_module(mod); |
2797 | if (err) | |
2798 | goto bug_cleanup; | |
2799 | ||
51f3d0f4 | 2800 | /* Module is ready to execute: parsing args may do that. */ |
51e158c1 | 2801 | after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp, |
4355efbd | 2802 | -32768, 32767, mod, |
ecc86170 | 2803 | unknown_module_param_cb); |
51e158c1 RR |
2804 | if (IS_ERR(after_dashes)) { |
2805 | err = PTR_ERR(after_dashes); | |
4c973d16 | 2806 | goto coming_cleanup; |
51e158c1 RR |
2807 | } else if (after_dashes) { |
2808 | pr_warn("%s: parameters '%s' after `--' ignored\n", | |
2809 | mod->name, after_dashes); | |
2810 | } | |
1da177e4 | 2811 | |
ca86cad7 | 2812 | /* Link in to sysfs. */ |
34e1169d | 2813 | err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp); |
1da177e4 | 2814 | if (err < 0) |
4c973d16 | 2815 | goto coming_cleanup; |
80a3d1bb | 2816 | |
1ce15ef4 JY |
2817 | if (is_livepatch_module(mod)) { |
2818 | err = copy_module_elf(mod, info); | |
2819 | if (err < 0) | |
2820 | goto sysfs_cleanup; | |
2821 | } | |
2822 | ||
48fd1188 | 2823 | /* Get rid of temporary copy. */ |
b1ae6dc4 | 2824 | free_copy(info, flags); |
1da177e4 LT |
2825 | |
2826 | /* Done! */ | |
51f3d0f4 | 2827 | trace_module_load(mod); |
34e1169d KC |
2828 | |
2829 | return do_init_module(mod); | |
1da177e4 | 2830 | |
1ce15ef4 JY |
2831 | sysfs_cleanup: |
2832 | mod_sysfs_teardown(mod); | |
4c973d16 | 2833 | coming_cleanup: |
885a78d4 | 2834 | mod->state = MODULE_STATE_GOING; |
a5544880 | 2835 | destroy_params(mod->kp, mod->num_kp); |
4c973d16 JY |
2836 | blocking_notifier_call_chain(&module_notify_list, |
2837 | MODULE_STATE_GOING, mod); | |
7e545d6e | 2838 | klp_module_going(mod); |
1fb9341a | 2839 | bug_cleanup: |
5e8ed280 | 2840 | mod->state = MODULE_STATE_GOING; |
1fb9341a | 2841 | /* module_bug_cleanup needs module_mutex protection */ |
75676500 | 2842 | mutex_lock(&module_mutex); |
5336377d | 2843 | module_bug_cleanup(mod); |
ee61abb3 | 2844 | mutex_unlock(&module_mutex); |
ff7e0055 | 2845 | |
a3535c7e | 2846 | ddebug_cleanup: |
1323eac7 | 2847 | ftrace_release_mod(mod); |
52796312 | 2848 | dynamic_debug_remove(mod, info->debug); |
cb2f5536 | 2849 | synchronize_rcu(); |
6526c534 RR |
2850 | kfree(mod->args); |
2851 | free_arch_cleanup: | |
cf68fffb | 2852 | cfi_cleanup(mod); |
1da177e4 | 2853 | module_arch_cleanup(mod); |
d913188c | 2854 | free_modinfo: |
a263f776 | 2855 | free_modinfo(mod); |
22e268eb | 2856 | free_unload: |
1da177e4 | 2857 | module_unload_free(mod); |
1fb9341a RR |
2858 | unlink_mod: |
2859 | mutex_lock(&module_mutex); | |
2860 | /* Unlink carefully: kallsyms could be walking list. */ | |
2861 | list_del_rcu(&mod->list); | |
758556bd | 2862 | mod_tree_remove(mod); |
1fb9341a | 2863 | wake_up_all(&module_wq); |
0be964be | 2864 | /* Wait for RCU-sched synchronizing before releasing mod->list. */ |
cb2f5536 | 2865 | synchronize_rcu(); |
1fb9341a | 2866 | mutex_unlock(&module_mutex); |
d913188c | 2867 | free_module: |
35a9393c | 2868 | /* Free lock-classes; relies on the preceding sync_rcu() */ |
6ab9942c | 2869 | lockdep_free_key_range(mod->data_layout.base, mod->data_layout.size); |
35a9393c | 2870 | |
34e1169d | 2871 | module_deallocate(mod, info); |
d913188c | 2872 | free_copy: |
b1ae6dc4 | 2873 | free_copy(info, flags); |
34e1169d | 2874 | return err; |
b99b87f7 PO |
2875 | } |
2876 | ||
17da2bd9 HC |
2877 | SYSCALL_DEFINE3(init_module, void __user *, umod, |
2878 | unsigned long, len, const char __user *, uargs) | |
1da177e4 | 2879 | { |
34e1169d KC |
2880 | int err; |
2881 | struct load_info info = { }; | |
1da177e4 | 2882 | |
34e1169d KC |
2883 | err = may_init_module(); |
2884 | if (err) | |
2885 | return err; | |
1da177e4 | 2886 | |
34e1169d KC |
2887 | pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n", |
2888 | umod, len, uargs); | |
1da177e4 | 2889 | |
34e1169d KC |
2890 | err = copy_module_from_user(umod, len, &info); |
2891 | if (err) | |
2892 | return err; | |
1da177e4 | 2893 | |
2f3238ae | 2894 | return load_module(&info, uargs, 0); |
34e1169d | 2895 | } |
94462ad3 | 2896 | |
2f3238ae | 2897 | SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags) |
34e1169d | 2898 | { |
34e1169d | 2899 | struct load_info info = { }; |
b1ae6dc4 DT |
2900 | void *buf = NULL; |
2901 | int len; | |
a1db7420 | 2902 | int err; |
94462ad3 | 2903 | |
34e1169d KC |
2904 | err = may_init_module(); |
2905 | if (err) | |
2906 | return err; | |
1da177e4 | 2907 | |
2f3238ae | 2908 | pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags); |
6c5db22d | 2909 | |
2f3238ae | 2910 | if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS |
b1ae6dc4 DT |
2911 | |MODULE_INIT_IGNORE_VERMAGIC |
2912 | |MODULE_INIT_COMPRESSED_FILE)) | |
2f3238ae | 2913 | return -EINVAL; |
d6de2c80 | 2914 | |
b1ae6dc4 | 2915 | len = kernel_read_file_from_fd(fd, 0, &buf, INT_MAX, NULL, |
a1db7420 | 2916 | READING_MODULE); |
b1ae6dc4 DT |
2917 | if (len < 0) |
2918 | return len; | |
2919 | ||
2920 | if (flags & MODULE_INIT_COMPRESSED_FILE) { | |
2921 | err = module_decompress(&info, buf, len); | |
2922 | vfree(buf); /* compressed data is no longer needed */ | |
2923 | if (err) | |
2924 | return err; | |
2925 | } else { | |
2926 | info.hdr = buf; | |
2927 | info.len = len; | |
2928 | } | |
1da177e4 | 2929 | |
2f3238ae | 2930 | return load_module(&info, uargs, flags); |
1da177e4 LT |
2931 | } |
2932 | ||
2933 | static inline int within(unsigned long addr, void *start, unsigned long size) | |
2934 | { | |
2935 | return ((void *)addr >= start && (void *)addr < start + size); | |
2936 | } | |
2937 | ||
cf68fffb ST |
2938 | static void cfi_init(struct module *mod) |
2939 | { | |
2940 | #ifdef CONFIG_CFI_CLANG | |
2941 | initcall_t *init; | |
f963ef12 | 2942 | #ifdef CONFIG_MODULE_UNLOAD |
cf68fffb | 2943 | exitcall_t *exit; |
f963ef12 | 2944 | #endif |
cf68fffb ST |
2945 | |
2946 | rcu_read_lock_sched(); | |
2947 | mod->cfi_check = (cfi_check_fn) | |
2948 | find_kallsyms_symbol_value(mod, "__cfi_check"); | |
2949 | init = (initcall_t *) | |
2950 | find_kallsyms_symbol_value(mod, "__cfi_jt_init_module"); | |
cf68fffb ST |
2951 | /* Fix init/exit functions to point to the CFI jump table */ |
2952 | if (init) | |
2953 | mod->init = *init; | |
0d67e332 | 2954 | #ifdef CONFIG_MODULE_UNLOAD |
f963ef12 CL |
2955 | exit = (exitcall_t *) |
2956 | find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module"); | |
cf68fffb ST |
2957 | if (exit) |
2958 | mod->exit = *exit; | |
0d67e332 | 2959 | #endif |
f963ef12 | 2960 | rcu_read_unlock_sched(); |
cf68fffb | 2961 | |
55ce556d | 2962 | cfi_module_add(mod, mod_tree.addr_min); |
cf68fffb ST |
2963 | #endif |
2964 | } | |
2965 | ||
2966 | static void cfi_cleanup(struct module *mod) | |
2967 | { | |
2968 | #ifdef CONFIG_CFI_CLANG | |
55ce556d | 2969 | cfi_module_remove(mod, mod_tree.addr_min); |
cf68fffb ST |
2970 | #endif |
2971 | } | |
2972 | ||
7fd8329b | 2973 | /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */ |
0ffc40f6 | 2974 | char *module_flags(struct module *mod, char *buf) |
fa3ba2e8 FM |
2975 | { |
2976 | int bx = 0; | |
2977 | ||
0d21b0e3 | 2978 | BUG_ON(mod->state == MODULE_STATE_UNFORMED); |
21aa9280 AV |
2979 | if (mod->taints || |
2980 | mod->state == MODULE_STATE_GOING || | |
2981 | mod->state == MODULE_STATE_COMING) { | |
fa3ba2e8 | 2982 | buf[bx++] = '('; |
c14e522b | 2983 | bx += module_flags_taint(mod->taints, buf + bx); |
21aa9280 AV |
2984 | /* Show a - for module-is-being-unloaded */ |
2985 | if (mod->state == MODULE_STATE_GOING) | |
2986 | buf[bx++] = '-'; | |
2987 | /* Show a + for module-is-being-loaded */ | |
2988 | if (mod->state == MODULE_STATE_COMING) | |
2989 | buf[bx++] = '+'; | |
fa3ba2e8 FM |
2990 | buf[bx++] = ')'; |
2991 | } | |
2992 | buf[bx] = '\0'; | |
2993 | ||
2994 | return buf; | |
2995 | } | |
2996 | ||
1da177e4 LT |
2997 | /* Given an address, look for it in the module exception tables. */ |
2998 | const struct exception_table_entry *search_module_extables(unsigned long addr) | |
2999 | { | |
1da177e4 LT |
3000 | const struct exception_table_entry *e = NULL; |
3001 | struct module *mod; | |
3002 | ||
24da1cbf | 3003 | preempt_disable(); |
5ff22646 PZ |
3004 | mod = __module_address(addr); |
3005 | if (!mod) | |
3006 | goto out; | |
22a8bdeb | 3007 | |
5ff22646 PZ |
3008 | if (!mod->num_exentries) |
3009 | goto out; | |
3010 | ||
3011 | e = search_extable(mod->extable, | |
a94c33dd | 3012 | mod->num_exentries, |
5ff22646 PZ |
3013 | addr); |
3014 | out: | |
24da1cbf | 3015 | preempt_enable(); |
1da177e4 | 3016 | |
5ff22646 PZ |
3017 | /* |
3018 | * Now, if we found one, we are running inside it now, hence | |
3019 | * we cannot unload the module, hence no refcnt needed. | |
3020 | */ | |
1da177e4 LT |
3021 | return e; |
3022 | } | |
3023 | ||
2541743e SS |
3024 | /** |
3025 | * is_module_address() - is this address inside a module? | |
e610499e RR |
3026 | * @addr: the address to check. |
3027 | * | |
3028 | * See is_module_text_address() if you simply want to see if the address | |
3029 | * is code (not data). | |
4d435f9d | 3030 | */ |
e610499e | 3031 | bool is_module_address(unsigned long addr) |
4d435f9d | 3032 | { |
e610499e | 3033 | bool ret; |
4d435f9d | 3034 | |
24da1cbf | 3035 | preempt_disable(); |
e610499e | 3036 | ret = __module_address(addr) != NULL; |
24da1cbf | 3037 | preempt_enable(); |
4d435f9d | 3038 | |
e610499e | 3039 | return ret; |
4d435f9d IM |
3040 | } |
3041 | ||
2541743e SS |
3042 | /** |
3043 | * __module_address() - get the module which contains an address. | |
e610499e RR |
3044 | * @addr: the address. |
3045 | * | |
3046 | * Must be called with preempt disabled or module mutex held so that | |
3047 | * module doesn't get freed during this. | |
3048 | */ | |
714f83d5 | 3049 | struct module *__module_address(unsigned long addr) |
1da177e4 LT |
3050 | { |
3051 | struct module *mod; | |
01dc0386 | 3052 | struct mod_tree_root *tree; |
1da177e4 | 3053 | |
01dc0386 CL |
3054 | if (addr >= mod_tree.addr_min && addr <= mod_tree.addr_max) |
3055 | tree = &mod_tree; | |
3056 | #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC | |
3057 | else if (addr >= mod_data_tree.addr_min && addr <= mod_data_tree.addr_max) | |
3058 | tree = &mod_data_tree; | |
3059 | #endif | |
3060 | else | |
3a642e99 RR |
3061 | return NULL; |
3062 | ||
0be964be PZ |
3063 | module_assert_mutex_or_preempt(); |
3064 | ||
01dc0386 | 3065 | mod = mod_find(addr, tree); |
93c2e105 PZ |
3066 | if (mod) { |
3067 | BUG_ON(!within_module(addr, mod)); | |
0d21b0e3 | 3068 | if (mod->state == MODULE_STATE_UNFORMED) |
93c2e105 | 3069 | mod = NULL; |
0d21b0e3 | 3070 | } |
93c2e105 | 3071 | return mod; |
1da177e4 LT |
3072 | } |
3073 | ||
2541743e SS |
3074 | /** |
3075 | * is_module_text_address() - is this address inside module code? | |
e610499e RR |
3076 | * @addr: the address to check. |
3077 | * | |
3078 | * See is_module_address() if you simply want to see if the address is | |
3079 | * anywhere in a module. See kernel_text_address() for testing if an | |
3080 | * address corresponds to kernel or module code. | |
3081 | */ | |
3082 | bool is_module_text_address(unsigned long addr) | |
3083 | { | |
3084 | bool ret; | |
3085 | ||
3086 | preempt_disable(); | |
3087 | ret = __module_text_address(addr) != NULL; | |
3088 | preempt_enable(); | |
3089 | ||
3090 | return ret; | |
3091 | } | |
3092 | ||
2541743e SS |
3093 | /** |
3094 | * __module_text_address() - get the module whose code contains an address. | |
e610499e RR |
3095 | * @addr: the address. |
3096 | * | |
3097 | * Must be called with preempt disabled or module mutex held so that | |
3098 | * module doesn't get freed during this. | |
3099 | */ | |
3100 | struct module *__module_text_address(unsigned long addr) | |
3101 | { | |
3102 | struct module *mod = __module_address(addr); | |
3103 | if (mod) { | |
3104 | /* Make sure it's within the text section. */ | |
7523e4dc RR |
3105 | if (!within(addr, mod->init_layout.base, mod->init_layout.text_size) |
3106 | && !within(addr, mod->core_layout.base, mod->core_layout.text_size)) | |
e610499e RR |
3107 | mod = NULL; |
3108 | } | |
3109 | return mod; | |
3110 | } | |
3111 | ||
1da177e4 LT |
3112 | /* Don't grab lock, we're oopsing. */ |
3113 | void print_modules(void) | |
3114 | { | |
3115 | struct module *mod; | |
7fd8329b | 3116 | char buf[MODULE_FLAGS_BUF_SIZE]; |
1da177e4 | 3117 | |
b231125a | 3118 | printk(KERN_DEFAULT "Modules linked in:"); |
d72b3751 AK |
3119 | /* Most callers should already have preempt disabled, but make sure */ |
3120 | preempt_disable(); | |
0d21b0e3 RR |
3121 | list_for_each_entry_rcu(mod, &modules, list) { |
3122 | if (mod->state == MODULE_STATE_UNFORMED) | |
3123 | continue; | |
27bba4d6 | 3124 | pr_cont(" %s%s", mod->name, module_flags(mod, buf)); |
0d21b0e3 | 3125 | } |
99bd9956 AT |
3126 | |
3127 | print_unloaded_tainted_modules(); | |
d72b3751 | 3128 | preempt_enable(); |
e14af7ee | 3129 | if (last_unloaded_module[0]) |
27bba4d6 JS |
3130 | pr_cont(" [last unloaded: %s]", last_unloaded_module); |
3131 | pr_cont("\n"); | |
1da177e4 | 3132 | } |