Commit | Line | Data |
---|---|---|
40b0b3f8 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
5234f5eb | 2 | /* |
835c34a1 | 3 | * handle transition of Linux booting another kernel |
5234f5eb | 4 | * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com> |
5234f5eb EB |
5 | */ |
6 | ||
12db5562 VG |
7 | #define pr_fmt(fmt) "kexec: " fmt |
8 | ||
5234f5eb EB |
9 | #include <linux/mm.h> |
10 | #include <linux/kexec.h> | |
5234f5eb | 11 | #include <linux/string.h> |
5a0e3ad6 | 12 | #include <linux/gfp.h> |
5234f5eb | 13 | #include <linux/reboot.h> |
fd59d231 | 14 | #include <linux/numa.h> |
f43fdad8 | 15 | #include <linux/ftrace.h> |
fef3a7a1 | 16 | #include <linux/io.h> |
fee7b0d8 | 17 | #include <linux/suspend.h> |
d6472302 | 18 | #include <linux/vmalloc.h> |
6bbeb276 | 19 | #include <linux/efi.h> |
f43fdad8 | 20 | |
9ebdc79f | 21 | #include <asm/init.h> |
5234f5eb EB |
22 | #include <asm/tlbflush.h> |
23 | #include <asm/mmu_context.h> | |
8643e28d | 24 | #include <asm/io_apic.h> |
17f557e5 | 25 | #include <asm/debugreg.h> |
27f48d3e | 26 | #include <asm/kexec-bzimage64.h> |
4545c898 | 27 | #include <asm/setup.h> |
d1163651 | 28 | #include <asm/set_memory.h> |
8bf27556 | 29 | |
6bbeb276 KS |
30 | #ifdef CONFIG_ACPI |
31 | /* | |
32 | * Used while adding mapping for ACPI tables. | |
33 | * Can be reused when other iomem regions need be mapped | |
34 | */ | |
35 | struct init_pgtable_data { | |
36 | struct x86_mapping_info *info; | |
37 | pgd_t *level4p; | |
38 | }; | |
39 | ||
40 | static int mem_region_callback(struct resource *res, void *arg) | |
41 | { | |
42 | struct init_pgtable_data *data = arg; | |
43 | unsigned long mstart, mend; | |
44 | ||
45 | mstart = res->start; | |
46 | mend = mstart + resource_size(res) - 1; | |
47 | ||
48 | return kernel_ident_mapping_init(data->info, data->level4p, mstart, mend); | |
49 | } | |
50 | ||
51 | static int | |
52 | map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p) | |
53 | { | |
6bbeb276 | 54 | struct init_pgtable_data data; |
5a949b38 KS |
55 | unsigned long flags; |
56 | int ret; | |
6bbeb276 KS |
57 | |
58 | data.info = info; | |
59 | data.level4p = level4p; | |
60 | flags = IORESOURCE_MEM | IORESOURCE_BUSY; | |
5a949b38 KS |
61 | |
62 | ret = walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1, | |
63 | &data, mem_region_callback); | |
64 | if (ret && ret != -EINVAL) | |
65 | return ret; | |
66 | ||
67 | /* ACPI tables could be located in ACPI Non-volatile Storage region */ | |
68 | ret = walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1, | |
69 | &data, mem_region_callback); | |
70 | if (ret && ret != -EINVAL) | |
71 | return ret; | |
72 | ||
73 | return 0; | |
6bbeb276 KS |
74 | } |
75 | #else | |
76 | static int map_acpi_tables(struct x86_mapping_info *info, pgd_t *level4p) { return 0; } | |
77 | #endif | |
78 | ||
74ca317c | 79 | #ifdef CONFIG_KEXEC_FILE |
9ec4ecef | 80 | const struct kexec_file_ops * const kexec_file_loaders[] = { |
27f48d3e | 81 | &kexec_bzImage64_ops, |
9ec4ecef | 82 | NULL |
cb105258 | 83 | }; |
74ca317c | 84 | #endif |
cb105258 | 85 | |
6bbeb276 KS |
86 | static int |
87 | map_efi_systab(struct x86_mapping_info *info, pgd_t *level4p) | |
88 | { | |
89 | #ifdef CONFIG_EFI | |
90 | unsigned long mstart, mend; | |
91 | ||
92 | if (!efi_enabled(EFI_BOOT)) | |
93 | return 0; | |
94 | ||
95 | mstart = (boot_params.efi_info.efi_systab | | |
96 | ((u64)boot_params.efi_info.efi_systab_hi<<32)); | |
97 | ||
98 | if (efi_enabled(EFI_64BIT)) | |
99 | mend = mstart + sizeof(efi_system_table_64_t); | |
100 | else | |
101 | mend = mstart + sizeof(efi_system_table_32_t); | |
102 | ||
103 | if (!mstart) | |
104 | return 0; | |
105 | ||
106 | return kernel_ident_mapping_init(info, level4p, mstart, mend); | |
107 | #endif | |
108 | return 0; | |
109 | } | |
110 | ||
f5deb796 HY |
111 | static void free_transition_pgtable(struct kimage *image) |
112 | { | |
7f689041 | 113 | free_page((unsigned long)image->arch.p4d); |
a466ef76 | 114 | image->arch.p4d = NULL; |
f5deb796 | 115 | free_page((unsigned long)image->arch.pud); |
a466ef76 | 116 | image->arch.pud = NULL; |
f5deb796 | 117 | free_page((unsigned long)image->arch.pmd); |
a466ef76 | 118 | image->arch.pmd = NULL; |
f5deb796 | 119 | free_page((unsigned long)image->arch.pte); |
a466ef76 | 120 | image->arch.pte = NULL; |
f5deb796 HY |
121 | } |
122 | ||
123 | static int init_transition_pgtable(struct kimage *image, pgd_t *pgd) | |
124 | { | |
85784d16 LJ |
125 | pgprot_t prot = PAGE_KERNEL_EXEC_NOENC; |
126 | unsigned long vaddr, paddr; | |
127 | int result = -ENOMEM; | |
7f689041 | 128 | p4d_t *p4d; |
f5deb796 HY |
129 | pud_t *pud; |
130 | pmd_t *pmd; | |
131 | pte_t *pte; | |
f5deb796 HY |
132 | |
133 | vaddr = (unsigned long)relocate_kernel; | |
134 | paddr = __pa(page_address(image->control_code_page)+PAGE_SIZE); | |
135 | pgd += pgd_index(vaddr); | |
136 | if (!pgd_present(*pgd)) { | |
7f689041 KS |
137 | p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL); |
138 | if (!p4d) | |
139 | goto err; | |
140 | image->arch.p4d = p4d; | |
141 | set_pgd(pgd, __pgd(__pa(p4d) | _KERNPG_TABLE)); | |
142 | } | |
143 | p4d = p4d_offset(pgd, vaddr); | |
144 | if (!p4d_present(*p4d)) { | |
f5deb796 HY |
145 | pud = (pud_t *)get_zeroed_page(GFP_KERNEL); |
146 | if (!pud) | |
147 | goto err; | |
148 | image->arch.pud = pud; | |
7f689041 | 149 | set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE)); |
f5deb796 | 150 | } |
7f689041 | 151 | pud = pud_offset(p4d, vaddr); |
f5deb796 HY |
152 | if (!pud_present(*pud)) { |
153 | pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL); | |
154 | if (!pmd) | |
155 | goto err; | |
156 | image->arch.pmd = pmd; | |
157 | set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE)); | |
158 | } | |
159 | pmd = pmd_offset(pud, vaddr); | |
160 | if (!pmd_present(*pmd)) { | |
161 | pte = (pte_t *)get_zeroed_page(GFP_KERNEL); | |
162 | if (!pte) | |
163 | goto err; | |
164 | image->arch.pte = pte; | |
165 | set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE)); | |
166 | } | |
167 | pte = pte_offset_kernel(pmd, vaddr); | |
85784d16 LJ |
168 | |
169 | if (sev_active()) | |
170 | prot = PAGE_KERNEL_EXEC; | |
171 | ||
172 | set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot)); | |
f5deb796 HY |
173 | return 0; |
174 | err: | |
f5deb796 HY |
175 | return result; |
176 | } | |
177 | ||
9ebdc79f YL |
178 | static void *alloc_pgt_page(void *data) |
179 | { | |
180 | struct kimage *image = (struct kimage *)data; | |
181 | struct page *page; | |
182 | void *p = NULL; | |
183 | ||
184 | page = kimage_alloc_control_pages(image, 0); | |
185 | if (page) { | |
186 | p = page_address(page); | |
187 | clear_page(p); | |
188 | } | |
189 | ||
190 | return p; | |
191 | } | |
192 | ||
5234f5eb EB |
193 | static int init_pgtable(struct kimage *image, unsigned long start_pgtable) |
194 | { | |
9ebdc79f YL |
195 | struct x86_mapping_info info = { |
196 | .alloc_pgt_page = alloc_pgt_page, | |
197 | .context = image, | |
66aad4fd | 198 | .page_flag = __PAGE_KERNEL_LARGE_EXEC, |
bba4ed01 | 199 | .kernpg_flag = _KERNPG_TABLE_NOENC, |
9ebdc79f | 200 | }; |
084d1283 | 201 | unsigned long mstart, mend; |
8bf27556 | 202 | pgd_t *level4p; |
f5deb796 | 203 | int result; |
084d1283 YL |
204 | int i; |
205 | ||
8bf27556 | 206 | level4p = (pgd_t *)__va(start_pgtable); |
9ebdc79f | 207 | clear_page(level4p); |
8638100c | 208 | |
85784d16 LJ |
209 | if (sev_active()) { |
210 | info.page_flag |= _PAGE_ENC; | |
211 | info.kernpg_flag |= _PAGE_ENC; | |
212 | } | |
213 | ||
8638100c XP |
214 | if (direct_gbpages) |
215 | info.direct_gbpages = true; | |
216 | ||
0e691cf8 YL |
217 | for (i = 0; i < nr_pfn_mapped; i++) { |
218 | mstart = pfn_mapped[i].start << PAGE_SHIFT; | |
219 | mend = pfn_mapped[i].end << PAGE_SHIFT; | |
220 | ||
221 | result = kernel_ident_mapping_init(&info, | |
222 | level4p, mstart, mend); | |
223 | if (result) | |
224 | return result; | |
225 | } | |
084d1283 | 226 | |
53594547 | 227 | /* |
084d1283 YL |
228 | * segments's mem ranges could be outside 0 ~ max_pfn, |
229 | * for example when jump back to original kernel from kexeced kernel. | |
230 | * or first kernel is booted with user mem map, and second kernel | |
231 | * could be loaded out of that range. | |
53594547 | 232 | */ |
084d1283 YL |
233 | for (i = 0; i < image->nr_segments; i++) { |
234 | mstart = image->segment[i].mem; | |
235 | mend = mstart + image->segment[i].memsz; | |
236 | ||
9ebdc79f YL |
237 | result = kernel_ident_mapping_init(&info, |
238 | level4p, mstart, mend); | |
084d1283 YL |
239 | |
240 | if (result) | |
241 | return result; | |
242 | } | |
243 | ||
6bbeb276 KS |
244 | /* |
245 | * Prepare EFI systab and ACPI tables for kexec kernel since they are | |
246 | * not covered by pfn_mapped. | |
247 | */ | |
248 | result = map_efi_systab(&info, level4p); | |
249 | if (result) | |
250 | return result; | |
251 | ||
252 | result = map_acpi_tables(&info, level4p); | |
253 | if (result) | |
254 | return result; | |
255 | ||
f5deb796 | 256 | return init_transition_pgtable(image, level4p); |
5234f5eb EB |
257 | } |
258 | ||
259 | static void set_idt(void *newidt, u16 limit) | |
260 | { | |
36c4fd23 | 261 | struct desc_ptr curidt; |
5234f5eb EB |
262 | |
263 | /* x86-64 supports unaliged loads & stores */ | |
36c4fd23 EB |
264 | curidt.size = limit; |
265 | curidt.address = (unsigned long)newidt; | |
5234f5eb EB |
266 | |
267 | __asm__ __volatile__ ( | |
36c4fd23 EB |
268 | "lidtq %0\n" |
269 | : : "m" (curidt) | |
5234f5eb EB |
270 | ); |
271 | }; | |
272 | ||
273 | ||
274 | static void set_gdt(void *newgdt, u16 limit) | |
275 | { | |
36c4fd23 | 276 | struct desc_ptr curgdt; |
5234f5eb EB |
277 | |
278 | /* x86-64 supports unaligned loads & stores */ | |
36c4fd23 EB |
279 | curgdt.size = limit; |
280 | curgdt.address = (unsigned long)newgdt; | |
5234f5eb EB |
281 | |
282 | __asm__ __volatile__ ( | |
36c4fd23 EB |
283 | "lgdtq %0\n" |
284 | : : "m" (curgdt) | |
5234f5eb EB |
285 | ); |
286 | }; | |
287 | ||
288 | static void load_segments(void) | |
289 | { | |
290 | __asm__ __volatile__ ( | |
36c4fd23 EB |
291 | "\tmovl %0,%%ds\n" |
292 | "\tmovl %0,%%es\n" | |
293 | "\tmovl %0,%%ss\n" | |
294 | "\tmovl %0,%%fs\n" | |
295 | "\tmovl %0,%%gs\n" | |
2ec5e3a8 | 296 | : : "a" (__KERNEL_DS) : "memory" |
5234f5eb | 297 | ); |
5234f5eb EB |
298 | } |
299 | ||
5234f5eb EB |
300 | int machine_kexec_prepare(struct kimage *image) |
301 | { | |
4bfaaef0 | 302 | unsigned long start_pgtable; |
5234f5eb EB |
303 | int result; |
304 | ||
305 | /* Calculate the offsets */ | |
72414d3f | 306 | start_pgtable = page_to_pfn(image->control_code_page) << PAGE_SHIFT; |
5234f5eb EB |
307 | |
308 | /* Setup the identity mapped 64bit page table */ | |
309 | result = init_pgtable(image, start_pgtable); | |
72414d3f | 310 | if (result) |
5234f5eb | 311 | return result; |
5234f5eb | 312 | |
5234f5eb EB |
313 | return 0; |
314 | } | |
315 | ||
316 | void machine_kexec_cleanup(struct kimage *image) | |
317 | { | |
f5deb796 | 318 | free_transition_pgtable(image); |
5234f5eb EB |
319 | } |
320 | ||
321 | /* | |
322 | * Do not allocate memory (or fail in any way) in machine_kexec(). | |
323 | * We are past the point of no return, committed to rebooting now. | |
324 | */ | |
3ab83521 | 325 | void machine_kexec(struct kimage *image) |
5234f5eb | 326 | { |
4bfaaef0 MD |
327 | unsigned long page_list[PAGES_NR]; |
328 | void *control_page; | |
fee7b0d8 | 329 | int save_ftrace_enabled; |
5234f5eb | 330 | |
fee7b0d8 | 331 | #ifdef CONFIG_KEXEC_JUMP |
6407df5c | 332 | if (image->preserve_context) |
fee7b0d8 HY |
333 | save_processor_state(); |
334 | #endif | |
335 | ||
336 | save_ftrace_enabled = __ftrace_enabled_save(); | |
f43fdad8 | 337 | |
5234f5eb EB |
338 | /* Interrupts aren't acceptable while we reboot */ |
339 | local_irq_disable(); | |
17f557e5 | 340 | hw_breakpoint_disable(); |
5234f5eb | 341 | |
fee7b0d8 HY |
342 | if (image->preserve_context) { |
343 | #ifdef CONFIG_X86_IO_APIC | |
344 | /* | |
345 | * We need to put APICs in legacy mode so that we can | |
346 | * get timer interrupts in second kernel. kexec/kdump | |
50374b96 BH |
347 | * paths already have calls to restore_boot_irq_mode() |
348 | * in one form or other. kexec jump path also need one. | |
fee7b0d8 | 349 | */ |
3c9e76db BH |
350 | clear_IO_APIC(); |
351 | restore_boot_irq_mode(); | |
fee7b0d8 HY |
352 | #endif |
353 | } | |
354 | ||
4bfaaef0 | 355 | control_page = page_address(image->control_code_page) + PAGE_SIZE; |
fee7b0d8 | 356 | memcpy(control_page, relocate_kernel, KEXEC_CONTROL_CODE_MAX_SIZE); |
4bfaaef0 | 357 | |
e3ebadd9 | 358 | page_list[PA_CONTROL_PAGE] = virt_to_phys(control_page); |
fee7b0d8 | 359 | page_list[VA_CONTROL_PAGE] = (unsigned long)control_page; |
4bfaaef0 MD |
360 | page_list[PA_TABLE_PAGE] = |
361 | (unsigned long)__pa(page_address(image->control_code_page)); | |
5234f5eb | 362 | |
fee7b0d8 HY |
363 | if (image->type == KEXEC_TYPE_DEFAULT) |
364 | page_list[PA_SWAP_PAGE] = (page_to_pfn(image->swap_page) | |
365 | << PAGE_SHIFT); | |
366 | ||
fef3a7a1 HY |
367 | /* |
368 | * The segment registers are funny things, they have both a | |
2a8a3d5b EB |
369 | * visible and an invisible part. Whenever the visible part is |
370 | * set to a specific selector, the invisible part is loaded | |
371 | * with from a table in memory. At no other time is the | |
372 | * descriptor table in memory accessed. | |
5234f5eb EB |
373 | * |
374 | * I take advantage of this here by force loading the | |
375 | * segments, before I zap the gdt with an invalid value. | |
376 | */ | |
377 | load_segments(); | |
fef3a7a1 HY |
378 | /* |
379 | * The gdt & idt are now invalid. | |
5234f5eb EB |
380 | * If you want to load them you must set up your own idt & gdt. |
381 | */ | |
fef3a7a1 HY |
382 | set_gdt(phys_to_virt(0), 0); |
383 | set_idt(phys_to_virt(0), 0); | |
4bfaaef0 | 384 | |
5234f5eb | 385 | /* now call it */ |
fee7b0d8 HY |
386 | image->start = relocate_kernel((unsigned long)image->head, |
387 | (unsigned long)page_list, | |
388 | image->start, | |
4e237903 TL |
389 | image->preserve_context, |
390 | sme_active()); | |
fee7b0d8 HY |
391 | |
392 | #ifdef CONFIG_KEXEC_JUMP | |
6407df5c | 393 | if (image->preserve_context) |
fee7b0d8 HY |
394 | restore_processor_state(); |
395 | #endif | |
396 | ||
397 | __ftrace_enabled_restore(save_ftrace_enabled); | |
5234f5eb | 398 | } |
2c8c0e6b | 399 | |
cb105258 VG |
400 | /* arch-dependent functionality related to kexec file-based syscall */ |
401 | ||
74ca317c | 402 | #ifdef CONFIG_KEXEC_FILE |
cb105258 VG |
403 | void *arch_kexec_kernel_image_load(struct kimage *image) |
404 | { | |
dd5f7260 VG |
405 | vfree(image->arch.elf_headers); |
406 | image->arch.elf_headers = NULL; | |
407 | ||
cb105258 VG |
408 | if (!image->fops || !image->fops->load) |
409 | return ERR_PTR(-ENOEXEC); | |
410 | ||
411 | return image->fops->load(image, image->kernel_buf, | |
412 | image->kernel_buf_len, image->initrd_buf, | |
413 | image->initrd_buf_len, image->cmdline_buf, | |
414 | image->cmdline_buf_len); | |
415 | } | |
416 | ||
12db5562 VG |
417 | /* |
418 | * Apply purgatory relocations. | |
419 | * | |
8aec395b PR |
420 | * @pi: Purgatory to be relocated. |
421 | * @section: Section relocations applying to. | |
422 | * @relsec: Section containing RELAs. | |
423 | * @symtabsec: Corresponding symtab. | |
12db5562 VG |
424 | * |
425 | * TODO: Some of the code belongs to generic code. Move that in kexec.c. | |
426 | */ | |
8aec395b PR |
427 | int arch_kexec_apply_relocations_add(struct purgatory_info *pi, |
428 | Elf_Shdr *section, const Elf_Shdr *relsec, | |
429 | const Elf_Shdr *symtabsec) | |
12db5562 VG |
430 | { |
431 | unsigned int i; | |
432 | Elf64_Rela *rel; | |
433 | Elf64_Sym *sym; | |
434 | void *location; | |
12db5562 VG |
435 | unsigned long address, sec_base, value; |
436 | const char *strtab, *name, *shstrtab; | |
8aec395b | 437 | const Elf_Shdr *sechdrs; |
12db5562 | 438 | |
8aec395b PR |
439 | /* String & section header string table */ |
440 | sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff; | |
441 | strtab = (char *)pi->ehdr + sechdrs[symtabsec->sh_link].sh_offset; | |
442 | shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset; | |
12db5562 | 443 | |
8aec395b | 444 | rel = (void *)pi->ehdr + relsec->sh_offset; |
12db5562 | 445 | |
8aec395b PR |
446 | pr_debug("Applying relocate section %s to %u\n", |
447 | shstrtab + relsec->sh_name, relsec->sh_info); | |
12db5562 | 448 | |
8aec395b | 449 | for (i = 0; i < relsec->sh_size / sizeof(*rel); i++) { |
12db5562 VG |
450 | |
451 | /* | |
452 | * rel[i].r_offset contains byte offset from beginning | |
453 | * of section to the storage unit affected. | |
454 | * | |
8da0b724 PR |
455 | * This is location to update. This is temporary buffer |
456 | * where section is currently loaded. This will finally be | |
457 | * loaded to a different address later, pointed to by | |
12db5562 VG |
458 | * ->sh_addr. kexec takes care of moving it |
459 | * (kexec_load_segment()). | |
460 | */ | |
8da0b724 PR |
461 | location = pi->purgatory_buf; |
462 | location += section->sh_offset; | |
463 | location += rel[i].r_offset; | |
12db5562 VG |
464 | |
465 | /* Final address of the location */ | |
466 | address = section->sh_addr + rel[i].r_offset; | |
467 | ||
468 | /* | |
469 | * rel[i].r_info contains information about symbol table index | |
470 | * w.r.t which relocation must be made and type of relocation | |
471 | * to apply. ELF64_R_SYM() and ELF64_R_TYPE() macros get | |
472 | * these respectively. | |
473 | */ | |
8aec395b PR |
474 | sym = (void *)pi->ehdr + symtabsec->sh_offset; |
475 | sym += ELF64_R_SYM(rel[i].r_info); | |
12db5562 VG |
476 | |
477 | if (sym->st_name) | |
478 | name = strtab + sym->st_name; | |
479 | else | |
480 | name = shstrtab + sechdrs[sym->st_shndx].sh_name; | |
481 | ||
482 | pr_debug("Symbol: %s info: %02x shndx: %02x value=%llx size: %llx\n", | |
483 | name, sym->st_info, sym->st_shndx, sym->st_value, | |
484 | sym->st_size); | |
485 | ||
486 | if (sym->st_shndx == SHN_UNDEF) { | |
487 | pr_err("Undefined symbol: %s\n", name); | |
488 | return -ENOEXEC; | |
489 | } | |
490 | ||
491 | if (sym->st_shndx == SHN_COMMON) { | |
492 | pr_err("symbol '%s' in common section\n", name); | |
493 | return -ENOEXEC; | |
494 | } | |
495 | ||
496 | if (sym->st_shndx == SHN_ABS) | |
497 | sec_base = 0; | |
8aec395b | 498 | else if (sym->st_shndx >= pi->ehdr->e_shnum) { |
12db5562 VG |
499 | pr_err("Invalid section %d for symbol %s\n", |
500 | sym->st_shndx, name); | |
501 | return -ENOEXEC; | |
502 | } else | |
8aec395b | 503 | sec_base = pi->sechdrs[sym->st_shndx].sh_addr; |
12db5562 VG |
504 | |
505 | value = sym->st_value; | |
506 | value += sec_base; | |
507 | value += rel[i].r_addend; | |
508 | ||
509 | switch (ELF64_R_TYPE(rel[i].r_info)) { | |
510 | case R_X86_64_NONE: | |
511 | break; | |
512 | case R_X86_64_64: | |
513 | *(u64 *)location = value; | |
514 | break; | |
515 | case R_X86_64_32: | |
516 | *(u32 *)location = value; | |
517 | if (value != *(u32 *)location) | |
518 | goto overflow; | |
519 | break; | |
520 | case R_X86_64_32S: | |
521 | *(s32 *)location = value; | |
522 | if ((s64)value != *(s32 *)location) | |
523 | goto overflow; | |
524 | break; | |
525 | case R_X86_64_PC32: | |
b21ebf2f | 526 | case R_X86_64_PLT32: |
12db5562 VG |
527 | value -= (u64)address; |
528 | *(u32 *)location = value; | |
529 | break; | |
530 | default: | |
531 | pr_err("Unknown rela relocation: %llu\n", | |
532 | ELF64_R_TYPE(rel[i].r_info)); | |
533 | return -ENOEXEC; | |
534 | } | |
535 | } | |
536 | return 0; | |
537 | ||
538 | overflow: | |
539 | pr_err("Overflow in relocation type %d value 0x%lx\n", | |
540 | (int)ELF64_R_TYPE(rel[i].r_info), value); | |
541 | return -ENOEXEC; | |
542 | } | |
74ca317c | 543 | #endif /* CONFIG_KEXEC_FILE */ |
1e5768ae XP |
544 | |
545 | static int | |
546 | kexec_mark_range(unsigned long start, unsigned long end, bool protect) | |
547 | { | |
548 | struct page *page; | |
549 | unsigned int nr_pages; | |
550 | ||
551 | /* | |
552 | * For physical range: [start, end]. We must skip the unassigned | |
553 | * crashk resource with zero-valued "end" member. | |
554 | */ | |
555 | if (!end || start > end) | |
556 | return 0; | |
557 | ||
558 | page = pfn_to_page(start >> PAGE_SHIFT); | |
559 | nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; | |
560 | if (protect) | |
561 | return set_pages_ro(page, nr_pages); | |
562 | else | |
563 | return set_pages_rw(page, nr_pages); | |
564 | } | |
565 | ||
566 | static void kexec_mark_crashkres(bool protect) | |
567 | { | |
568 | unsigned long control; | |
569 | ||
570 | kexec_mark_range(crashk_low_res.start, crashk_low_res.end, protect); | |
571 | ||
572 | /* Don't touch the control code page used in crash_kexec().*/ | |
573 | control = PFN_PHYS(page_to_pfn(kexec_crash_image->control_code_page)); | |
574 | /* Control code page is located in the 2nd page. */ | |
575 | kexec_mark_range(crashk_res.start, control + PAGE_SIZE - 1, protect); | |
576 | control += KEXEC_CONTROL_PAGE_SIZE; | |
577 | kexec_mark_range(control, crashk_res.end, protect); | |
578 | } | |
579 | ||
580 | void arch_kexec_protect_crashkres(void) | |
581 | { | |
582 | kexec_mark_crashkres(true); | |
583 | } | |
584 | ||
585 | void arch_kexec_unprotect_crashkres(void) | |
586 | { | |
587 | kexec_mark_crashkres(false); | |
588 | } | |
bba4ed01 | 589 | |
1a79c1b8 LJ |
590 | /* |
591 | * During a traditional boot under SME, SME will encrypt the kernel, | |
592 | * so the SME kexec kernel also needs to be un-encrypted in order to | |
593 | * replicate a normal SME boot. | |
594 | * | |
595 | * During a traditional boot under SEV, the kernel has already been | |
596 | * loaded encrypted, so the SEV kexec kernel needs to be encrypted in | |
597 | * order to replicate a normal SEV boot. | |
598 | */ | |
bba4ed01 TL |
599 | int arch_kexec_post_alloc_pages(void *vaddr, unsigned int pages, gfp_t gfp) |
600 | { | |
1a79c1b8 LJ |
601 | if (sev_active()) |
602 | return 0; | |
603 | ||
bba4ed01 TL |
604 | /* |
605 | * If SME is active we need to be sure that kexec pages are | |
606 | * not encrypted because when we boot to the new kernel the | |
607 | * pages won't be accessed encrypted (initially). | |
608 | */ | |
609 | return set_memory_decrypted((unsigned long)vaddr, pages); | |
610 | } | |
611 | ||
612 | void arch_kexec_pre_free_pages(void *vaddr, unsigned int pages) | |
613 | { | |
1a79c1b8 LJ |
614 | if (sev_active()) |
615 | return; | |
616 | ||
bba4ed01 TL |
617 | /* |
618 | * If SME is active we need to reset the pages back to being | |
619 | * an encrypted mapping before freeing them. | |
620 | */ | |
621 | set_memory_encrypted((unsigned long)vaddr, pages); | |
622 | } |