Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
5033cba0 | 2 | /* |
62a31a03 | 3 | * Architecture specific (i386/x86_64) functions for kexec based crash dumps. |
5033cba0 EB |
4 | * |
5 | * Created by: Hariprasad Nellitheertha (hari@in.ibm.com) | |
6 | * | |
7 | * Copyright (C) IBM Corporation, 2004. All rights reserved. | |
dd5f7260 VG |
8 | * Copyright (C) Red Hat Inc., 2014. All rights reserved. |
9 | * Authors: | |
10 | * Vivek Goyal <vgoyal@redhat.com> | |
5033cba0 EB |
11 | * |
12 | */ | |
13 | ||
dd5f7260 VG |
14 | #define pr_fmt(fmt) "kexec: " fmt |
15 | ||
5033cba0 EB |
16 | #include <linux/types.h> |
17 | #include <linux/kernel.h> | |
18 | #include <linux/smp.h> | |
5033cba0 EB |
19 | #include <linux/reboot.h> |
20 | #include <linux/kexec.h> | |
5033cba0 EB |
21 | #include <linux/delay.h> |
22 | #include <linux/elf.h> | |
23 | #include <linux/elfcore.h> | |
186f4360 | 24 | #include <linux/export.h> |
dd5f7260 | 25 | #include <linux/slab.h> |
d6472302 | 26 | #include <linux/vmalloc.h> |
6f599d84 | 27 | #include <linux/memblock.h> |
5033cba0 EB |
28 | |
29 | #include <asm/processor.h> | |
30 | #include <asm/hardirq.h> | |
31 | #include <asm/nmi.h> | |
32 | #include <asm/hw_irq.h> | |
19842d67 | 33 | #include <asm/apic.h> |
5520b7e7 | 34 | #include <asm/e820/types.h> |
8643e28d | 35 | #include <asm/io_apic.h> |
0c1b2724 | 36 | #include <asm/hpet.h> |
1eeb66a1 | 37 | #include <linux/kdebug.h> |
96b89dc6 | 38 | #include <asm/cpu.h> |
ed23dc6f | 39 | #include <asm/reboot.h> |
2340b62f | 40 | #include <asm/virtext.h> |
da06a43d | 41 | #include <asm/intel_pt.h> |
89f579ce | 42 | #include <asm/crash.h> |
6f599d84 | 43 | #include <asm/cmdline.h> |
8e294786 | 44 | |
dd5f7260 VG |
45 | /* Used while preparing memory map entries for second kernel */ |
46 | struct crash_memmap_data { | |
47 | struct boot_params *params; | |
48 | /* Type of memory */ | |
49 | unsigned int type; | |
50 | }; | |
51 | ||
f23d1f4a ZY |
52 | /* |
53 | * This is used to VMCLEAR all VMCSs loaded on the | |
54 | * processor. And when loading kvm_intel module, the | |
55 | * callback function pointer will be assigned. | |
56 | * | |
57 | * protected by rcu. | |
58 | */ | |
0ca0d818 | 59 | crash_vmclear_fn __rcu *crash_vmclear_loaded_vmcss = NULL; |
f23d1f4a ZY |
60 | EXPORT_SYMBOL_GPL(crash_vmclear_loaded_vmcss); |
61 | ||
62 | static inline void cpu_crash_vmclear_loaded_vmcss(void) | |
63 | { | |
0ca0d818 | 64 | crash_vmclear_fn *do_vmclear_operation = NULL; |
f23d1f4a ZY |
65 | |
66 | rcu_read_lock(); | |
67 | do_vmclear_operation = rcu_dereference(crash_vmclear_loaded_vmcss); | |
68 | if (do_vmclear_operation) | |
69 | do_vmclear_operation(); | |
70 | rcu_read_unlock(); | |
71 | } | |
72 | ||
6f599d84 LJ |
73 | /* |
74 | * When the crashkernel option is specified, only use the low | |
75 | * 1M for the real mode trampoline. | |
76 | */ | |
77 | void __init crash_reserve_low_1M(void) | |
78 | { | |
79 | if (cmdline_find_option(boot_command_line, "crashkernel", NULL, 0) < 0) | |
80 | return; | |
81 | ||
82 | memblock_reserve(0, 1<<20); | |
83 | pr_info("Reserving the low 1M of memory for crashkernel\n"); | |
84 | } | |
85 | ||
b2bbe71b EH |
86 | #if defined(CONFIG_SMP) && defined(CONFIG_X86_LOCAL_APIC) |
87 | ||
9c48f1c6 | 88 | static void kdump_nmi_callback(int cpu, struct pt_regs *regs) |
c4ac4263 | 89 | { |
a7d41820 EH |
90 | crash_save_cpu(regs, cpu); |
91 | ||
f23d1f4a ZY |
92 | /* |
93 | * VMCLEAR VMCSs loaded on all cpus if needed. | |
94 | */ | |
95 | cpu_crash_vmclear_loaded_vmcss(); | |
96 | ||
2340b62f EH |
97 | /* Disable VMX or SVM if needed. |
98 | * | |
99 | * We need to disable virtualization on all CPUs. | |
100 | * Having VMX or SVM enabled on any CPU may break rebooting | |
101 | * after the kdump kernel has finished its task. | |
102 | */ | |
103 | cpu_emergency_vmxoff(); | |
104 | cpu_emergency_svm_disable(); | |
105 | ||
da06a43d TI |
106 | /* |
107 | * Disable Intel PT to stop its logging | |
108 | */ | |
109 | cpu_emergency_stop_pt(); | |
110 | ||
a7d41820 EH |
111 | disable_local_APIC(); |
112 | } | |
113 | ||
0ee59413 | 114 | void kdump_nmi_shootdown_cpus(void) |
d1e7b91c | 115 | { |
8e294786 | 116 | nmi_shootdown_cpus(kdump_nmi_callback); |
d1e7b91c | 117 | |
19842d67 | 118 | disable_local_APIC(); |
c4ac4263 | 119 | } |
d1e7b91c | 120 | |
0ee59413 HK |
121 | /* Override the weak function in kernel/panic.c */ |
122 | void crash_smp_send_stop(void) | |
123 | { | |
124 | static int cpus_stopped; | |
125 | ||
126 | if (cpus_stopped) | |
127 | return; | |
128 | ||
129 | if (smp_ops.crash_stop_other_cpus) | |
130 | smp_ops.crash_stop_other_cpus(); | |
131 | else | |
132 | smp_send_stop(); | |
133 | ||
134 | cpus_stopped = 1; | |
135 | } | |
136 | ||
c4ac4263 | 137 | #else |
0ee59413 | 138 | void crash_smp_send_stop(void) |
c4ac4263 EB |
139 | { |
140 | /* There are no cpus to shootdown */ | |
141 | } | |
142 | #endif | |
143 | ||
ed23dc6f | 144 | void native_machine_crash_shutdown(struct pt_regs *regs) |
5033cba0 EB |
145 | { |
146 | /* This function is only called after the system | |
f18190bd | 147 | * has panicked or is otherwise in a critical state. |
5033cba0 EB |
148 | * The minimum amount of code to allow a kexec'd kernel |
149 | * to run successfully needs to happen here. | |
150 | * | |
151 | * In practice this means shooting down the other cpus in | |
152 | * an SMP system. | |
153 | */ | |
c4ac4263 EB |
154 | /* The kernel is broken so disable interrupts */ |
155 | local_irq_disable(); | |
a3ea8ac8 | 156 | |
0ee59413 | 157 | crash_smp_send_stop(); |
2340b62f | 158 | |
f23d1f4a ZY |
159 | /* |
160 | * VMCLEAR VMCSs loaded on this cpu if needed. | |
161 | */ | |
162 | cpu_crash_vmclear_loaded_vmcss(); | |
163 | ||
2340b62f EH |
164 | /* Booting kdump kernel with VMX or SVM enabled won't work, |
165 | * because (among other limitations) we can't disable paging | |
166 | * with the virt flags. | |
167 | */ | |
168 | cpu_emergency_vmxoff(); | |
169 | cpu_emergency_svm_disable(); | |
170 | ||
da06a43d TI |
171 | /* |
172 | * Disable Intel PT to stop its logging | |
173 | */ | |
174 | cpu_emergency_stop_pt(); | |
175 | ||
17405453 YY |
176 | #ifdef CONFIG_X86_IO_APIC |
177 | /* Prevent crash_kexec() from deadlocking on ioapic_lock. */ | |
178 | ioapic_zap_locks(); | |
339b2ae0 | 179 | clear_IO_APIC(); |
0c1b2724 | 180 | #endif |
522e6646 | 181 | lapic_shutdown(); |
339b2ae0 | 182 | restore_boot_irq_mode(); |
0c1b2724 OH |
183 | #ifdef CONFIG_HPET_TIMER |
184 | hpet_disable(); | |
19842d67 | 185 | #endif |
85916f81 | 186 | crash_save_cpu(regs, safe_smp_processor_id()); |
5033cba0 | 187 | } |
dd5f7260 | 188 | |
74ca317c | 189 | #ifdef CONFIG_KEXEC_FILE |
53b76073 | 190 | |
1d2e733b | 191 | static int get_nr_ram_ranges_callback(struct resource *res, void *arg) |
dd5f7260 | 192 | { |
e3c41e37 | 193 | unsigned int *nr_ranges = arg; |
dd5f7260 VG |
194 | |
195 | (*nr_ranges)++; | |
196 | return 0; | |
197 | } | |
198 | ||
dd5f7260 | 199 | /* Gather all the required information to prepare elf headers for ram regions */ |
8d5f894a | 200 | static struct crash_mem *fill_up_crash_elf_data(void) |
dd5f7260 VG |
201 | { |
202 | unsigned int nr_ranges = 0; | |
8d5f894a | 203 | struct crash_mem *cmem; |
dd5f7260 | 204 | |
9eff3037 | 205 | walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); |
8d5f894a AT |
206 | if (!nr_ranges) |
207 | return NULL; | |
dd5f7260 | 208 | |
8d5f894a AT |
209 | /* |
210 | * Exclusion of crash region and/or crashk_low_res may cause | |
211 | * another range split. So add extra two slots here. | |
212 | */ | |
213 | nr_ranges += 2; | |
4df43095 | 214 | cmem = vzalloc(struct_size(cmem, ranges, nr_ranges)); |
8d5f894a AT |
215 | if (!cmem) |
216 | return NULL; | |
dd5f7260 | 217 | |
8d5f894a AT |
218 | cmem->max_nr_ranges = nr_ranges; |
219 | cmem->nr_ranges = 0; | |
dd5f7260 | 220 | |
8d5f894a | 221 | return cmem; |
dd5f7260 VG |
222 | } |
223 | ||
dd5f7260 VG |
224 | /* |
225 | * Look for any unwanted ranges between mstart, mend and remove them. This | |
8d5f894a | 226 | * might lead to split and split ranges are put in cmem->ranges[] array |
dd5f7260 | 227 | */ |
8d5f894a | 228 | static int elf_header_exclude_ranges(struct crash_mem *cmem) |
dd5f7260 | 229 | { |
dd5f7260 VG |
230 | int ret = 0; |
231 | ||
7c321eb2 LJ |
232 | /* Exclude the low 1M because it is always reserved */ |
233 | ret = crash_exclude_mem_range(cmem, 0, 1<<20); | |
234 | if (ret) | |
235 | return ret; | |
236 | ||
dd5f7260 | 237 | /* Exclude crashkernel region */ |
babac4a8 | 238 | ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); |
dd5f7260 VG |
239 | if (ret) |
240 | return ret; | |
241 | ||
9eff3037 | 242 | if (crashk_low_res.end) |
babac4a8 | 243 | ret = crash_exclude_mem_range(cmem, crashk_low_res.start, |
9eff3037 | 244 | crashk_low_res.end); |
dd5f7260 | 245 | |
dd5f7260 VG |
246 | return ret; |
247 | } | |
248 | ||
1d2e733b | 249 | static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg) |
dd5f7260 | 250 | { |
8d5f894a | 251 | struct crash_mem *cmem = arg; |
dd5f7260 | 252 | |
cbe66016 AT |
253 | cmem->ranges[cmem->nr_ranges].start = res->start; |
254 | cmem->ranges[cmem->nr_ranges].end = res->end; | |
255 | cmem->nr_ranges++; | |
dd5f7260 | 256 | |
cbe66016 | 257 | return 0; |
dd5f7260 VG |
258 | } |
259 | ||
dd5f7260 VG |
260 | /* Prepare elf headers. Return addr and size */ |
261 | static int prepare_elf_headers(struct kimage *image, void **addr, | |
262 | unsigned long *sz) | |
263 | { | |
8d5f894a | 264 | struct crash_mem *cmem; |
7c321eb2 | 265 | int ret; |
dd5f7260 | 266 | |
8d5f894a AT |
267 | cmem = fill_up_crash_elf_data(); |
268 | if (!cmem) | |
dd5f7260 VG |
269 | return -ENOMEM; |
270 | ||
9eff3037 | 271 | ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback); |
cbe66016 AT |
272 | if (ret) |
273 | goto out; | |
274 | ||
275 | /* Exclude unwanted mem ranges */ | |
8d5f894a | 276 | ret = elf_header_exclude_ranges(cmem); |
cbe66016 AT |
277 | if (ret) |
278 | goto out; | |
279 | ||
dd5f7260 | 280 | /* By default prepare 64bit headers */ |
9eff3037 | 281 | ret = crash_prepare_elf64_headers(cmem, IS_ENABLED(CONFIG_X86_64), addr, sz); |
cbe66016 | 282 | |
cbe66016 | 283 | out: |
8d5f894a | 284 | vfree(cmem); |
dd5f7260 VG |
285 | return ret; |
286 | } | |
287 | ||
8ec67d97 | 288 | static int add_e820_entry(struct boot_params *params, struct e820_entry *entry) |
dd5f7260 VG |
289 | { |
290 | unsigned int nr_e820_entries; | |
291 | ||
292 | nr_e820_entries = params->e820_entries; | |
08b46d5d | 293 | if (nr_e820_entries >= E820_MAX_ENTRIES_ZEROPAGE) |
dd5f7260 VG |
294 | return 1; |
295 | ||
9eff3037 | 296 | memcpy(¶ms->e820_table[nr_e820_entries], entry, sizeof(struct e820_entry)); |
dd5f7260 VG |
297 | params->e820_entries++; |
298 | return 0; | |
299 | } | |
300 | ||
1d2e733b | 301 | static int memmap_entry_callback(struct resource *res, void *arg) |
dd5f7260 VG |
302 | { |
303 | struct crash_memmap_data *cmd = arg; | |
304 | struct boot_params *params = cmd->params; | |
8ec67d97 | 305 | struct e820_entry ei; |
dd5f7260 | 306 | |
1d2e733b | 307 | ei.addr = res->start; |
9275b933 | 308 | ei.size = resource_size(res); |
dd5f7260 VG |
309 | ei.type = cmd->type; |
310 | add_e820_entry(params, &ei); | |
311 | ||
312 | return 0; | |
313 | } | |
314 | ||
315 | static int memmap_exclude_ranges(struct kimage *image, struct crash_mem *cmem, | |
316 | unsigned long long mstart, | |
317 | unsigned long long mend) | |
318 | { | |
319 | unsigned long start, end; | |
dd5f7260 VG |
320 | |
321 | cmem->ranges[0].start = mstart; | |
322 | cmem->ranges[0].end = mend; | |
323 | cmem->nr_ranges = 1; | |
324 | ||
dd5f7260 VG |
325 | /* Exclude elf header region */ |
326 | start = image->arch.elf_load_addr; | |
327 | end = start + image->arch.elf_headers_sz - 1; | |
babac4a8 | 328 | return crash_exclude_mem_range(cmem, start, end); |
dd5f7260 VG |
329 | } |
330 | ||
331 | /* Prepare memory map for crash dump kernel */ | |
332 | int crash_setup_memmap_entries(struct kimage *image, struct boot_params *params) | |
333 | { | |
334 | int i, ret = 0; | |
335 | unsigned long flags; | |
8ec67d97 | 336 | struct e820_entry ei; |
dd5f7260 VG |
337 | struct crash_memmap_data cmd; |
338 | struct crash_mem *cmem; | |
339 | ||
340 | cmem = vzalloc(sizeof(struct crash_mem)); | |
341 | if (!cmem) | |
342 | return -ENOMEM; | |
343 | ||
344 | memset(&cmd, 0, sizeof(struct crash_memmap_data)); | |
345 | cmd.params = params; | |
346 | ||
7c321eb2 LJ |
347 | /* Add the low 1M */ |
348 | cmd.type = E820_TYPE_RAM; | |
349 | flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; | |
350 | walk_iomem_res_desc(IORES_DESC_NONE, flags, 0, (1<<20)-1, &cmd, | |
9eff3037 | 351 | memmap_entry_callback); |
dd5f7260 VG |
352 | |
353 | /* Add ACPI tables */ | |
09821ff1 | 354 | cmd.type = E820_TYPE_ACPI; |
dd5f7260 | 355 | flags = IORESOURCE_MEM | IORESOURCE_BUSY; |
f0f4711a | 356 | walk_iomem_res_desc(IORES_DESC_ACPI_TABLES, flags, 0, -1, &cmd, |
9eff3037 | 357 | memmap_entry_callback); |
dd5f7260 VG |
358 | |
359 | /* Add ACPI Non-volatile Storage */ | |
09821ff1 | 360 | cmd.type = E820_TYPE_NVS; |
f0f4711a | 361 | walk_iomem_res_desc(IORES_DESC_ACPI_NV_STORAGE, flags, 0, -1, &cmd, |
9eff3037 | 362 | memmap_entry_callback); |
dd5f7260 | 363 | |
980621da LJ |
364 | /* Add e820 reserved ranges */ |
365 | cmd.type = E820_TYPE_RESERVED; | |
366 | flags = IORESOURCE_MEM; | |
367 | walk_iomem_res_desc(IORES_DESC_RESERVED, flags, 0, -1, &cmd, | |
9eff3037 | 368 | memmap_entry_callback); |
980621da | 369 | |
dd5f7260 VG |
370 | /* Add crashk_low_res region */ |
371 | if (crashk_low_res.end) { | |
372 | ei.addr = crashk_low_res.start; | |
373 | ei.size = crashk_low_res.end - crashk_low_res.start + 1; | |
09821ff1 | 374 | ei.type = E820_TYPE_RAM; |
dd5f7260 VG |
375 | add_e820_entry(params, &ei); |
376 | } | |
377 | ||
378 | /* Exclude some ranges from crashk_res and add rest to memmap */ | |
9eff3037 | 379 | ret = memmap_exclude_ranges(image, cmem, crashk_res.start, crashk_res.end); |
dd5f7260 VG |
380 | if (ret) |
381 | goto out; | |
382 | ||
383 | for (i = 0; i < cmem->nr_ranges; i++) { | |
384 | ei.size = cmem->ranges[i].end - cmem->ranges[i].start + 1; | |
385 | ||
386 | /* If entry is less than a page, skip it */ | |
387 | if (ei.size < PAGE_SIZE) | |
388 | continue; | |
389 | ei.addr = cmem->ranges[i].start; | |
09821ff1 | 390 | ei.type = E820_TYPE_RAM; |
dd5f7260 VG |
391 | add_e820_entry(params, &ei); |
392 | } | |
393 | ||
394 | out: | |
395 | vfree(cmem); | |
396 | return ret; | |
397 | } | |
398 | ||
dd5f7260 VG |
399 | int crash_load_segments(struct kimage *image) |
400 | { | |
dd5f7260 | 401 | int ret; |
ec2b9bfa TJB |
402 | struct kexec_buf kbuf = { .image = image, .buf_min = 0, |
403 | .buf_max = ULONG_MAX, .top_down = false }; | |
dd5f7260 | 404 | |
dd5f7260 | 405 | /* Prepare elf headers and add a segment */ |
ec2b9bfa | 406 | ret = prepare_elf_headers(image, &kbuf.buffer, &kbuf.bufsz); |
dd5f7260 VG |
407 | if (ret) |
408 | return ret; | |
409 | ||
ec2b9bfa TJB |
410 | image->arch.elf_headers = kbuf.buffer; |
411 | image->arch.elf_headers_sz = kbuf.bufsz; | |
dd5f7260 | 412 | |
ec2b9bfa TJB |
413 | kbuf.memsz = kbuf.bufsz; |
414 | kbuf.buf_align = ELF_CORE_HEADER_ALIGN; | |
993a1103 | 415 | kbuf.mem = KEXEC_BUF_MEM_UNKNOWN; |
ec2b9bfa | 416 | ret = kexec_add_buffer(&kbuf); |
dd5f7260 VG |
417 | if (ret) { |
418 | vfree((void *)image->arch.elf_headers); | |
419 | return ret; | |
420 | } | |
ec2b9bfa | 421 | image->arch.elf_load_addr = kbuf.mem; |
dd5f7260 | 422 | pr_debug("Loaded ELF headers at 0x%lx bufsz=0x%lx memsz=0x%lx\n", |
ec2b9bfa | 423 | image->arch.elf_load_addr, kbuf.bufsz, kbuf.bufsz); |
dd5f7260 VG |
424 | |
425 | return ret; | |
426 | } | |
74ca317c | 427 | #endif /* CONFIG_KEXEC_FILE */ |