Merge tag 'pm-6.16-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm
[linux-2.6-block.git] / arch / arm64 / kernel / smp.c
CommitLineData
caab277b 1// SPDX-License-Identifier: GPL-2.0-only
08e875c1
CM
2/*
3 * SMP initialisation and IPI support
4 * Based on arch/arm/kernel/smp.c
5 *
6 * Copyright (C) 2012 ARM Ltd.
08e875c1
CM
7 */
8
0f078336 9#include <linux/acpi.h>
f5df2696 10#include <linux/arm_sdei.h>
08e875c1
CM
11#include <linux/delay.h>
12#include <linux/init.h>
13#include <linux/spinlock.h>
68e21be2 14#include <linux/sched/mm.h>
ef8bd77f 15#include <linux/sched/hotplug.h>
68db0cf1 16#include <linux/sched/task_stack.h>
08e875c1
CM
17#include <linux/interrupt.h>
18#include <linux/cache.h>
19#include <linux/profile.h>
20#include <linux/errno.h>
21#include <linux/mm.h>
22#include <linux/err.h>
23#include <linux/cpu.h>
24#include <linux/smp.h>
25#include <linux/seq_file.h>
26#include <linux/irq.h>
e7932188 27#include <linux/irqchip/arm-gic-v3.h>
08e875c1
CM
28#include <linux/percpu.h>
29#include <linux/clockchips.h>
30#include <linux/completion.h>
31#include <linux/of.h>
eb631bb5 32#include <linux/irq_work.h>
a2638815 33#include <linux/kernel_stat.h>
78fd584c 34#include <linux/kexec.h>
2f5cd0c7 35#include <linux/kgdb.h>
0492747c 36#include <linux/kvm_host.h>
331a1b3a 37#include <linux/nmi.h>
08e875c1 38
e039ee4e 39#include <asm/alternative.h>
08e875c1
CM
40#include <asm/atomic.h>
41#include <asm/cacheflush.h>
df857416 42#include <asm/cpu.h>
08e875c1 43#include <asm/cputype.h>
cd1aebf5 44#include <asm/cpu_ops.h>
0fbeb318 45#include <asm/daifflags.h>
0492747c 46#include <asm/kvm_mmu.h>
08e875c1 47#include <asm/mmu_context.h>
1a2db300 48#include <asm/numa.h>
08e875c1 49#include <asm/processor.h>
4c7aa002 50#include <asm/smp_plat.h>
08e875c1
CM
51#include <asm/sections.h>
52#include <asm/tlbflush.h>
53#include <asm/ptrace.h>
377bcff9 54#include <asm/virt.h>
08e875c1 55
45ed695a
NP
56#include <trace/events/ipi.h>
57
08e875c1
CM
58/*
59 * as from 2.5, kernels no longer have an init_tasks structure
60 * so we need some other way of telling a new secondary core
61 * where to place its SVC stack
62 */
63struct secondary_data secondary_data;
bb905274 64/* Number of CPUs which aren't online, but looping in kernel text. */
2eaf63ba 65static int cpus_stuck_in_kernel;
08e875c1
CM
66
67enum ipi_msg_type {
68 IPI_RESCHEDULE,
69 IPI_CALL_FUNC,
08e875c1 70 IPI_CPU_STOP,
fdfa5881 71 IPI_CPU_STOP_NMI,
1f85008e 72 IPI_TIMER,
eb631bb5 73 IPI_IRQ_WORK,
331a1b3a
DA
74 NR_IPI,
75 /*
76 * Any enum >= NR_IPI and < MAX_IPI is special and not tracable
77 * with trace_ipi_*
78 */
79 IPI_CPU_BACKTRACE = NR_IPI,
2f5cd0c7 80 IPI_KGDB_ROUNDUP,
331a1b3a 81 MAX_IPI
08e875c1
CM
82};
83
62817d5b
DA
84static int ipi_irq_base __ro_after_init;
85static int nr_ipi __ro_after_init = NR_IPI;
86static struct irq_desc *ipi_desc[MAX_IPI] __ro_after_init;
d3afc7f1 87
fdfa5881
DA
88static bool crash_stop;
89
d3afc7f1 90static void ipi_setup(int cpu);
d3afc7f1 91
bb905274 92#ifdef CONFIG_HOTPLUG_CPU
9d9edb96 93static void ipi_teardown(int cpu);
bb905274
SP
94static int op_cpu_kill(unsigned int cpu);
95#else
96static inline int op_cpu_kill(unsigned int cpu)
97{
98 return -ENOSYS;
99}
100#endif
101
102
08e875c1
CM
103/*
104 * Boot a secondary CPU, and assign it the specified idle task.
105 * This also gives us the initial stack to use for this CPU.
106 */
b8c6453a 107static int boot_secondary(unsigned int cpu, struct task_struct *idle)
08e875c1 108{
de58ed5e
GS
109 const struct cpu_operations *ops = get_cpu_ops(cpu);
110
111 if (ops->cpu_boot)
112 return ops->cpu_boot(cpu);
08e875c1 113
652af899 114 return -EOPNOTSUPP;
08e875c1
CM
115}
116
117static DECLARE_COMPLETION(cpu_running);
118
b8c6453a 119int __cpu_up(unsigned int cpu, struct task_struct *idle)
08e875c1
CM
120{
121 int ret;
bb905274 122 long status;
08e875c1
CM
123
124 /*
125 * We need to tell the secondary core where to find its stack and the
126 * page tables.
127 */
c02433dd 128 secondary_data.task = idle;
bb905274 129 update_cpu_boot_status(CPU_MMU_OFF);
08e875c1 130
d22b115c 131 /* Now bring the CPU into our world */
08e875c1 132 ret = boot_secondary(cpu, idle);
d22b115c 133 if (ret) {
643e12da
JPB
134 if (ret != -EPERM)
135 pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
f357b3a7 136 return ret;
08e875c1
CM
137 }
138
d22b115c
GS
139 /*
140 * CPU was successfully started, wait for it to come online or
141 * time out.
142 */
143 wait_for_completion_timeout(&cpu_running,
144 msecs_to_jiffies(5000));
145 if (cpu_online(cpu))
146 return 0;
147
148 pr_crit("CPU%u: failed to come online\n", cpu);
c02433dd 149 secondary_data.task = NULL;
bb905274 150 status = READ_ONCE(secondary_data.status);
d22b115c
GS
151 if (status == CPU_MMU_OFF)
152 status = READ_ONCE(__early_cpu_boot_status);
bb905274 153
d22b115c
GS
154 switch (status & CPU_BOOT_STATUS_MASK) {
155 default:
156 pr_err("CPU%u: failed in unknown state : 0x%lx\n",
157 cpu, status);
158 cpus_stuck_in_kernel++;
159 break;
160 case CPU_KILL_ME:
161 if (!op_cpu_kill(cpu)) {
162 pr_crit("CPU%u: died during early boot\n", cpu);
bb905274 163 break;
bb905274 164 }
d22b115c 165 pr_crit("CPU%u: may not have shut down cleanly\n", cpu);
df561f66 166 fallthrough;
d22b115c
GS
167 case CPU_STUCK_IN_KERNEL:
168 pr_crit("CPU%u: is stuck in kernel\n", cpu);
169 if (status & CPU_STUCK_REASON_52_BIT_VA)
170 pr_crit("CPU%u: does not support 52-bit VAs\n", cpu);
171 if (status & CPU_STUCK_REASON_NO_GRAN) {
172 pr_crit("CPU%u: does not support %luK granule\n",
173 cpu, PAGE_SIZE / SZ_1K);
174 }
175 cpus_stuck_in_kernel++;
176 break;
177 case CPU_PANIC_KERNEL:
178 panic("CPU%u detected unsupported configuration\n", cpu);
bb905274 179 }
08e875c1 180
ba051f09 181 return -EIO;
08e875c1
CM
182}
183
e7932188
JT
184static void init_gic_priority_masking(void)
185{
186 u32 cpuflags;
187
188 if (WARN_ON(!gic_enable_sre()))
189 return;
190
191 cpuflags = read_sysreg(daif);
192
193 WARN_ON(!(cpuflags & PSR_I_BIT));
f0098155 194 WARN_ON(!(cpuflags & PSR_F_BIT));
e7932188 195
e1d22385 196 gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
e7932188
JT
197}
198
08e875c1
CM
199/*
200 * This is the secondary CPU boot entry. We're using this CPUs
201 * idle thread stack, but a set of temporary page tables.
202 */
b154886f 203asmlinkage notrace void secondary_start_kernel(void)
08e875c1 204{
ccaac162 205 u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
08e875c1 206 struct mm_struct *mm = &init_mm;
de58ed5e 207 const struct cpu_operations *ops;
3d8c1a01 208 unsigned int cpu = smp_processor_id();
08e875c1 209
08e875c1
CM
210 /*
211 * All kernel threads share the same mm context; grab a
212 * reference and switch to it.
213 */
f1f10076 214 mmgrab(mm);
08e875c1 215 current->active_mm = mm;
08e875c1
CM
216
217 /*
218 * TTBR0 is only used for the identity mapping at this stage. Make it
219 * point to zero page to avoid speculatively fetching new entries.
220 */
9e8e865b 221 cpu_uninstall_idmap();
08e875c1 222
e7932188
JT
223 if (system_uses_irq_prio_masking())
224 init_gic_priority_masking();
225
448e9f34 226 rcutree_report_cpu_starting(cpu);
08e875c1
CM
227 trace_hardirqs_off();
228
dbb4e152
SP
229 /*
230 * If the system has established the capabilities, make sure
231 * this CPU ticks all of those. If it doesn't, the CPU will
232 * fail to come online.
233 */
c47a1900 234 check_local_cpu_capabilities();
dbb4e152 235
de58ed5e
GS
236 ops = get_cpu_ops(cpu);
237 if (ops->cpu_postboot)
238 ops->cpu_postboot();
08e875c1 239
df857416
MR
240 /*
241 * Log the CPU info before it is marked online and might get read.
242 */
243 cpuinfo_store_cpu();
5524cbb1 244 store_cpu_topology(cpu);
df857416 245
7ade67b5
MZ
246 /*
247 * Enable GIC and timers.
248 */
249 notify_cpu_starting(cpu);
250
d3afc7f1
MZ
251 ipi_setup(cpu);
252
97fd6016 253 numa_add_cpu(cpu);
f6e763b9 254
08e875c1
CM
255 /*
256 * OK, now it's safe to let the boot CPU continue. Wait for
257 * the CPU migration code to notice that the CPU is online
258 * before we continue.
259 */
ccaac162
MR
260 pr_info("CPU%u: Booted secondary processor 0x%010lx [0x%08x]\n",
261 cpu, (unsigned long)mpidr,
262 read_cpuid_id());
bb905274 263 update_cpu_boot_status(CPU_BOOT_SUCCESS);
08e875c1 264 set_cpu_online(cpu, true);
b3770b32 265 complete(&cpu_running);
08e875c1 266
080297be
MR
267 /*
268 * Secondary CPUs enter the kernel with all DAIF exceptions masked.
269 *
270 * As with setup_arch() we must unmask Debug and SError exceptions, and
271 * as the root irqchip has already been detected and initialized we can
272 * unmask IRQ and FIQ at the same time.
273 */
41bd5b5d 274 local_daif_restore(DAIF_PROCCTX);
53ae3acd 275
08e875c1
CM
276 /*
277 * OK, it's off to the idle thread for us
278 */
fc6d73d6 279 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
08e875c1
CM
280}
281
9327e2c6
MR
282#ifdef CONFIG_HOTPLUG_CPU
283static int op_cpu_disable(unsigned int cpu)
284{
de58ed5e
GS
285 const struct cpu_operations *ops = get_cpu_ops(cpu);
286
9327e2c6
MR
287 /*
288 * If we don't have a cpu_die method, abort before we reach the point
289 * of no return. CPU0 may not have an cpu_ops, so test for it.
290 */
de58ed5e 291 if (!ops || !ops->cpu_die)
9327e2c6
MR
292 return -EOPNOTSUPP;
293
294 /*
295 * We may need to abort a hot unplug for some other mechanism-specific
296 * reason.
297 */
de58ed5e
GS
298 if (ops->cpu_disable)
299 return ops->cpu_disable(cpu);
9327e2c6
MR
300
301 return 0;
302}
303
304/*
305 * __cpu_disable runs on the processor to be shutdown.
306 */
307int __cpu_disable(void)
308{
309 unsigned int cpu = smp_processor_id();
310 int ret;
311
312 ret = op_cpu_disable(cpu);
313 if (ret)
314 return ret;
315
7f9545aa
SH
316 remove_cpu_topology(cpu);
317 numa_remove_cpu(cpu);
318
9327e2c6
MR
319 /*
320 * Take this CPU offline. Once we clear this, we can't return,
321 * and we must not schedule until we're ready to give up the cpu.
322 */
323 set_cpu_online(cpu, false);
d3afc7f1 324 ipi_teardown(cpu);
9327e2c6
MR
325
326 /*
327 * OK - migrate IRQs away from this CPU
328 */
217d453d
YY
329 irq_migrate_all_off_this_cpu();
330
9327e2c6
MR
331 return 0;
332}
333
c814ca02
AC
334static int op_cpu_kill(unsigned int cpu)
335{
de58ed5e
GS
336 const struct cpu_operations *ops = get_cpu_ops(cpu);
337
c814ca02
AC
338 /*
339 * If we have no means of synchronising with the dying CPU, then assume
340 * that it is really dead. We can only wait for an arbitrary length of
341 * time and hope that it's dead, so let's skip the wait and just hope.
342 */
de58ed5e 343 if (!ops->cpu_kill)
6b99c68c 344 return 0;
c814ca02 345
de58ed5e 346 return ops->cpu_kill(cpu);
c814ca02
AC
347}
348
9327e2c6 349/*
b3091f17
TG
350 * Called on the thread which is asking for a CPU to be shutdown after the
351 * shutdown completed.
9327e2c6 352 */
b3091f17 353void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu)
9327e2c6 354{
6b99c68c
MR
355 int err;
356
cf814bcf 357 pr_debug("CPU%u: shutdown\n", cpu);
c814ca02
AC
358
359 /*
360 * Now that the dying CPU is beyond the point of no return w.r.t.
361 * in-kernel synchronisation, try to get the firwmare to help us to
362 * verify that it has really left the kernel before we consider
363 * clobbering anything it might still be using.
364 */
6b99c68c
MR
365 err = op_cpu_kill(cpu);
366 if (err)
a74ec64a 367 pr_warn("CPU%d may not have shut down cleanly: %d\n", cpu, err);
9327e2c6
MR
368}
369
370/*
371 * Called from the idle thread for the CPU which has been shutdown.
372 *
9327e2c6 373 */
9bdc61ef 374void __noreturn cpu_die(void)
9327e2c6
MR
375{
376 unsigned int cpu = smp_processor_id();
de58ed5e 377 const struct cpu_operations *ops = get_cpu_ops(cpu);
9327e2c6
MR
378
379 idle_task_exit();
380
0fbeb318 381 local_daif_mask();
9327e2c6 382
b3091f17
TG
383 /* Tell cpuhp_bp_sync_dead() that this CPU is now safe to dispose of */
384 cpuhp_ap_report_dead();
9327e2c6
MR
385
386 /*
387 * Actually shutdown the CPU. This must never fail. The specific hotplug
388 * mechanism must perform all required cache maintenance to ensure that
389 * no dirty lines are lost in the process of shutting down the CPU.
390 */
de58ed5e 391 ops->cpu_die(cpu);
9327e2c6
MR
392
393 BUG();
394}
395#endif
396
de58ed5e
GS
397static void __cpu_try_die(int cpu)
398{
399#ifdef CONFIG_HOTPLUG_CPU
400 const struct cpu_operations *ops = get_cpu_ops(cpu);
401
402 if (ops && ops->cpu_die)
403 ops->cpu_die(cpu);
404#endif
405}
406
fce6361f
SP
407/*
408 * Kill the calling secondary CPU, early in bringup before it is turned
409 * online.
410 */
5ab6876c 411void __noreturn cpu_die_early(void)
fce6361f
SP
412{
413 int cpu = smp_processor_id();
414
415 pr_crit("CPU%d: will not boot\n", cpu);
416
417 /* Mark this CPU absent */
418 set_cpu_present(cpu, 0);
448e9f34 419 rcutree_report_cpu_dead();
fce6361f 420
de58ed5e
GS
421 if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
422 update_cpu_boot_status(CPU_KILL_ME);
423 __cpu_try_die(cpu);
424 }
425
bb905274 426 update_cpu_boot_status(CPU_STUCK_IN_KERNEL);
fce6361f
SP
427
428 cpu_park_loop();
429}
430
377bcff9
JR
431static void __init hyp_mode_check(void)
432{
433 if (is_hyp_mode_available())
434 pr_info("CPU: All CPU(s) started at EL2\n");
435 else if (is_hyp_mode_mismatched())
436 WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
437 "CPU: CPUs started in inconsistent modes");
438 else
439 pr_info("CPU: All CPU(s) started at EL1\n");
6ec6259d 440 if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode()) {
0492747c 441 kvm_compute_layout();
6ec6259d
DB
442 kvm_apply_hyp_relocations();
443 }
377bcff9
JR
444}
445
08e875c1
CM
446void __init smp_cpus_done(unsigned int max_cpus)
447{
326b16db 448 pr_info("SMP: Total of %d processors activated.\n", num_online_cpus());
377bcff9 449 hyp_mode_check();
63a2d92e 450 setup_system_features();
075f48c9 451 setup_user_features();
5ea5306c 452 mark_linear_text_alias_ro();
08e875c1
CM
453}
454
455void __init smp_prepare_boot_cpu(void)
456{
3d8c1a01
MR
457 /*
458 * The runtime per-cpu areas have been allocated by
459 * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be
460 * freed shortly, so we must move over to the runtime per-cpu area.
461 */
9113c2aa 462 set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
0ceb0d56 463
eb15d707
MR
464 cpuinfo_store_boot_cpu();
465 setup_boot_cpu_features();
e7932188
JT
466
467 /* Conditionally switch to GIC PMR for interrupt masking */
468 if (system_uses_irq_prio_masking())
469 init_gic_priority_masking();
2e903b91
AK
470
471 kasan_init_hw_tags();
f75c2355
SH
472 /* Init percpu seeds for random tags after cpus are set up. */
473 kasan_init_sw_tags();
08e875c1
CM
474}
475
0f078336
LP
476/*
477 * Duplicate MPIDRs are a recipe for disaster. Scan all initialized
478 * entries and check for duplicates. If any is found just ignore the
479 * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid
480 * matching valid MPIDR values.
481 */
482static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid)
483{
484 unsigned int i;
485
486 for (i = 1; (i < cpu) && (i < NR_CPUS); i++)
487 if (cpu_logical_map(i) == hwid)
488 return true;
489 return false;
490}
491
819a8826
LP
492/*
493 * Initialize cpu operations for a logical cpu and
494 * set it in the possible mask on success
495 */
496static int __init smp_cpu_setup(int cpu)
497{
de58ed5e
GS
498 const struct cpu_operations *ops;
499
6885fb12 500 if (init_cpu_ops(cpu))
819a8826
LP
501 return -ENODEV;
502
de58ed5e
GS
503 ops = get_cpu_ops(cpu);
504 if (ops->cpu_init(cpu))
819a8826
LP
505 return -ENODEV;
506
507 set_cpu_possible(cpu, true);
508
509 return 0;
510}
511
0f078336
LP
512static bool bootcpu_valid __initdata;
513static unsigned int cpu_count = 1;
514
eba46750
JC
515int arch_register_cpu(int cpu)
516{
517 acpi_handle acpi_handle = acpi_get_processor_handle(cpu);
518 struct cpu *c = &per_cpu(cpu_devices, cpu);
519
520 if (!acpi_disabled && !acpi_handle &&
521 IS_ENABLED(CONFIG_ACPI_HOTPLUG_CPU))
522 return -EPROBE_DEFER;
523
524#ifdef CONFIG_ACPI_HOTPLUG_CPU
525 /* For now block anything that looks like physical CPU Hotplug */
526 if (invalid_logical_cpuid(cpu) || !cpu_present(cpu)) {
527 pr_err_once("Changing CPU present bit is not supported\n");
528 return -ENODEV;
529 }
530#endif
531
532 /*
533 * Availability of the acpi handle is sufficient to establish
534 * that _STA has aleady been checked. No need to recheck here.
535 */
536 c->hotpluggable = arch_cpu_is_hotpluggable(cpu);
537
538 return register_cpu(c, cpu);
539}
540
541#ifdef CONFIG_ACPI_HOTPLUG_CPU
542void arch_unregister_cpu(int cpu)
543{
544 acpi_handle acpi_handle = acpi_get_processor_handle(cpu);
545 struct cpu *c = &per_cpu(cpu_devices, cpu);
546 acpi_status status;
547 unsigned long long sta;
548
549 if (!acpi_handle) {
550 pr_err_once("Removing a CPU without associated ACPI handle\n");
551 return;
552 }
553
554 status = acpi_evaluate_integer(acpi_handle, "_STA", NULL, &sta);
555 if (ACPI_FAILURE(status))
556 return;
557
558 /* For now do not allow anything that looks like physical CPU HP */
559 if (cpu_present(cpu) && !(sta & ACPI_STA_DEVICE_PRESENT)) {
560 pr_err_once("Changing CPU present bit is not supported\n");
561 return;
562 }
563
564 unregister_cpu(c);
565}
566#endif /* CONFIG_ACPI_HOTPLUG_CPU */
567
0f078336 568#ifdef CONFIG_ACPI
e0013aed
MR
569static struct acpi_madt_generic_interrupt cpu_madt_gicc[NR_CPUS];
570
571struct acpi_madt_generic_interrupt *acpi_cpu_get_madt_gicc(int cpu)
572{
573 return &cpu_madt_gicc[cpu];
574}
d3c3db41 575EXPORT_SYMBOL_GPL(acpi_cpu_get_madt_gicc);
e0013aed 576
0f078336
LP
577/*
578 * acpi_map_gic_cpu_interface - parse processor MADT entry
579 *
580 * Carry out sanity checks on MADT processor entry and initialize
581 * cpu_logical_map on success
582 */
583static void __init
584acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor)
585{
586 u64 hwid = processor->arm_mpidr;
587
d633da5d
JM
588 if (!(processor->flags &
589 (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) {
f9058929 590 pr_debug("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid);
0f078336
LP
591 return;
592 }
593
f9058929
HG
594 if (hwid & ~MPIDR_HWID_BITMASK || hwid == INVALID_HWID) {
595 pr_err("skipping CPU entry with invalid MPIDR 0x%llx\n", hwid);
0f078336
LP
596 return;
597 }
598
599 if (is_mpidr_duplicate(cpu_count, hwid)) {
600 pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid);
601 return;
602 }
603
604 /* Check if GICC structure of boot CPU is available in the MADT */
605 if (cpu_logical_map(0) == hwid) {
606 if (bootcpu_valid) {
607 pr_err("duplicate boot CPU MPIDR: 0x%llx in MADT\n",
608 hwid);
609 return;
610 }
611 bootcpu_valid = true;
e0013aed 612 cpu_madt_gicc[0] = *processor;
0f078336
LP
613 return;
614 }
615
616 if (cpu_count >= NR_CPUS)
617 return;
618
619 /* map the logical cpu id to cpu MPIDR */
eaecca9e 620 set_cpu_logical_map(cpu_count, hwid);
0f078336 621
e0013aed
MR
622 cpu_madt_gicc[cpu_count] = *processor;
623
5e89c55e
LP
624 /*
625 * Set-up the ACPI parking protocol cpu entries
626 * while initializing the cpu_logical_map to
627 * avoid parsing MADT entries multiple times for
628 * nothing (ie a valid cpu_logical_map entry should
629 * contain a valid parking protocol data set to
630 * initialize the cpu if the parking protocol is
631 * the only available enable method).
632 */
633 acpi_set_mailbox_entry(cpu_count, processor);
634
0f078336
LP
635 cpu_count++;
636}
637
638static int __init
60574d1e 639acpi_parse_gic_cpu_interface(union acpi_subtable_headers *header,
0f078336
LP
640 const unsigned long end)
641{
642 struct acpi_madt_generic_interrupt *processor;
643
644 processor = (struct acpi_madt_generic_interrupt *)header;
99e3e3ae 645 if (BAD_MADT_GICC_ENTRY(processor, end))
0f078336
LP
646 return -EINVAL;
647
60574d1e 648 acpi_table_print_madt_entry(&header->common);
0f078336
LP
649
650 acpi_map_gic_cpu_interface(processor);
651
652 return 0;
653}
e1896249
LP
654
655static void __init acpi_parse_and_init_cpus(void)
656{
657 int i;
658
659 /*
660 * do a walk of MADT to determine how many CPUs
661 * we have including disabled CPUs, and get information
662 * we need for SMP init.
663 */
664 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
665 acpi_parse_gic_cpu_interface, 0);
666
667 /*
668 * In ACPI, SMP and CPU NUMA information is provided in separate
669 * static tables, namely the MADT and the SRAT.
670 *
671 * Thus, it is simpler to first create the cpu logical map through
672 * an MADT walk and then map the logical cpus to their node ids
673 * as separate steps.
674 */
675 acpi_map_cpus_to_nodes();
676
677 for (i = 0; i < nr_cpu_ids; i++)
678 early_map_cpu_to_node(i, acpi_numa_get_nid(i));
679}
0f078336 680#else
e1896249 681#define acpi_parse_and_init_cpus(...) do { } while (0)
0f078336
LP
682#endif
683
08e875c1 684/*
4c7aa002
JM
685 * Enumerate the possible CPU set from the device tree and build the
686 * cpu logical map array containing MPIDR values related to logical
687 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
08e875c1 688 */
29b8302b 689static void __init of_parse_and_init_cpus(void)
08e875c1 690{
3d29a9a0 691 struct device_node *dn;
08e875c1 692
de76e70a 693 for_each_of_cpu_node(dn) {
4d97b929 694 u64 hwid = of_get_cpu_hwid(dn, 0);
4c7aa002 695
4d97b929 696 if (hwid & ~MPIDR_HWID_BITMASK)
4c7aa002 697 goto next;
4c7aa002 698
0f078336 699 if (is_mpidr_duplicate(cpu_count, hwid)) {
a270f327
RH
700 pr_err("%pOF: duplicate cpu reg properties in the DT\n",
701 dn);
4c7aa002
JM
702 goto next;
703 }
704
4c7aa002
JM
705 /*
706 * The numbering scheme requires that the boot CPU
707 * must be assigned logical id 0. Record it so that
708 * the logical map built from DT is validated and can
709 * be used.
710 */
711 if (hwid == cpu_logical_map(0)) {
712 if (bootcpu_valid) {
a270f327
RH
713 pr_err("%pOF: duplicate boot cpu reg property in DT\n",
714 dn);
4c7aa002
JM
715 goto next;
716 }
717
718 bootcpu_valid = true;
7ba5f605 719 early_map_cpu_to_node(0, of_node_to_nid(dn));
4c7aa002
JM
720
721 /*
722 * cpu_logical_map has already been
723 * initialized and the boot cpu doesn't need
724 * the enable-method so continue without
725 * incrementing cpu.
726 */
727 continue;
728 }
729
0f078336 730 if (cpu_count >= NR_CPUS)
08e875c1
CM
731 goto next;
732
4c7aa002 733 pr_debug("cpu logical map 0x%llx\n", hwid);
eaecca9e 734 set_cpu_logical_map(cpu_count, hwid);
1a2db300
GK
735
736 early_map_cpu_to_node(cpu_count, of_node_to_nid(dn));
08e875c1 737next:
0f078336 738 cpu_count++;
08e875c1 739 }
0f078336
LP
740}
741
742/*
743 * Enumerate the possible CPU set from the device tree or ACPI and build the
744 * cpu logical map array containing MPIDR values related to logical
745 * cpus. Assumes that cpu_logical_map(0) has already been initialized.
746 */
747void __init smp_init_cpus(void)
748{
749 int i;
750
751 if (acpi_disabled)
752 of_parse_and_init_cpus();
753 else
e1896249 754 acpi_parse_and_init_cpus();
08e875c1 755
50ee91bd 756 if (cpu_count > nr_cpu_ids)
9b130ad5 757 pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n",
50ee91bd 758 cpu_count, nr_cpu_ids);
4c7aa002
JM
759
760 if (!bootcpu_valid) {
0f078336 761 pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
4c7aa002
JM
762 return;
763 }
764
765 /*
819a8826
LP
766 * We need to set the cpu_logical_map entries before enabling
767 * the cpus so that cpu processor description entries (DT cpu nodes
768 * and ACPI MADT entries) can be retrieved by matching the cpu hwid
769 * with entries in cpu_logical_map while initializing the cpus.
770 * If the cpu set-up fails, invalidate the cpu_logical_map entry.
4c7aa002 771 */
50ee91bd 772 for (i = 1; i < nr_cpu_ids; i++) {
819a8826
LP
773 if (cpu_logical_map(i) != INVALID_HWID) {
774 if (smp_cpu_setup(i))
eaecca9e 775 set_cpu_logical_map(i, INVALID_HWID);
819a8826
LP
776 }
777 }
08e875c1
CM
778}
779
780void __init smp_prepare_cpus(unsigned int max_cpus)
781{
de58ed5e 782 const struct cpu_operations *ops;
cd1aebf5 783 int err;
44dbcc93 784 unsigned int cpu;
c18df0ad 785 unsigned int this_cpu;
08e875c1 786
f6e763b9
MB
787 init_cpu_topology();
788
c18df0ad
DD
789 this_cpu = smp_processor_id();
790 store_cpu_topology(this_cpu);
791 numa_store_cpu_info(this_cpu);
97fd6016 792 numa_add_cpu(this_cpu);
f6e763b9 793
e75118a7
SP
794 /*
795 * If UP is mandated by "nosmp" (which implies "maxcpus=0"), don't set
796 * secondary CPUs present.
797 */
798 if (max_cpus == 0)
799 return;
800
08e875c1
CM
801 /*
802 * Initialise the present map (which describes the set of CPUs
803 * actually populated at the present time) and release the
804 * secondaries from the bootloader.
805 */
806 for_each_possible_cpu(cpu) {
08e875c1 807
d329de3f
MZ
808 if (cpu == smp_processor_id())
809 continue;
810
de58ed5e
GS
811 ops = get_cpu_ops(cpu);
812 if (!ops)
08e875c1
CM
813 continue;
814
de58ed5e 815 err = ops->cpu_prepare(cpu);
d329de3f
MZ
816 if (err)
817 continue;
08e875c1
CM
818
819 set_cpu_present(cpu, true);
c18df0ad 820 numa_store_cpu_info(cpu);
08e875c1 821 }
08e875c1
CM
822}
823
916b93f4 824static const char *ipi_types[MAX_IPI] __tracepoint_string = {
0edaee42
AK
825 [IPI_RESCHEDULE] = "Rescheduling interrupts",
826 [IPI_CALL_FUNC] = "Function call interrupts",
827 [IPI_CPU_STOP] = "CPU stop interrupts",
fdfa5881 828 [IPI_CPU_STOP_NMI] = "CPU stop NMIs",
0edaee42
AK
829 [IPI_TIMER] = "Timer broadcast interrupts",
830 [IPI_IRQ_WORK] = "IRQ work interrupts",
916b93f4
JR
831 [IPI_CPU_BACKTRACE] = "CPU backtrace interrupts",
832 [IPI_KGDB_ROUNDUP] = "KGDB roundup interrupts",
08e875c1
CM
833};
834
5cebfd2d 835static void smp_cross_call(const struct cpumask *target, unsigned int ipinr);
45ed695a 836
a2638815
MZ
837unsigned long irq_err_count;
838
839int arch_show_interrupts(struct seq_file *p, int prec)
08e875c1
CM
840{
841 unsigned int cpu, i;
842
916b93f4 843 for (i = 0; i < MAX_IPI; i++) {
45ed695a 844 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
08e875c1 845 prec >= 4 ? " " : "");
67317c26 846 for_each_online_cpu(cpu)
5089bc51 847 seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
08e875c1
CM
848 seq_printf(p, " %s\n", ipi_types[i]);
849 }
08e875c1 850
a2638815
MZ
851 seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
852 return 0;
08e875c1
CM
853}
854
45ed695a
NP
855void arch_send_call_function_ipi_mask(const struct cpumask *mask)
856{
857 smp_cross_call(mask, IPI_CALL_FUNC);
858}
859
860void arch_send_call_function_single_ipi(int cpu)
861{
0aaf0dae 862 smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
45ed695a
NP
863}
864
865#ifdef CONFIG_IRQ_WORK
866void arch_irq_work_raise(void)
867{
5cebfd2d 868 smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
45ed695a
NP
869}
870#endif
871
fdfa5881 872static void __noreturn local_cpu_stop(unsigned int cpu)
08e875c1 873{
fdfa5881 874 set_cpu_online(cpu, false);
08e875c1 875
0fbeb318 876 local_daif_mask();
f5df2696 877 sdei_mask_local_cpu();
dccc9da2 878 cpu_park_loop();
08e875c1 879}
08e875c1 880
d914d4d4
AK
881/*
882 * We need to implement panic_smp_self_stop() for parallel panic() calls, so
883 * that cpu_online_mask gets correctly updated and smp_send_stop() can skip
884 * CPUs that have already stopped themselves.
885 */
7412a60d 886void __noreturn panic_smp_self_stop(void)
d914d4d4 887{
fdfa5881 888 local_cpu_stop(smp_processor_id());
08e875c1
CM
889}
890
5ab6876c 891static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
78fd584c
AT
892{
893#ifdef CONFIG_KEXEC_CORE
fdfa5881
DA
894 /*
895 * Use local_daif_mask() instead of local_irq_disable() to make sure
896 * that pseudo-NMIs are disabled. The "crash stop" code starts with
897 * an IRQ and falls back to NMI (which might be pseudo). If the IRQ
898 * finally goes through right as we're timing out then the NMI could
899 * interrupt us. It's better to prevent the NMI and let the IRQ
900 * finish since the pt_regs will be better.
901 */
902 local_daif_mask();
903
78fd584c
AT
904 crash_save_cpu(regs, cpu);
905
fdfa5881 906 set_cpu_online(cpu, false);
78fd584c 907
f5df2696 908 sdei_mask_local_cpu();
78fd584c 909
de58ed5e
GS
910 if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
911 __cpu_try_die(cpu);
78fd584c
AT
912
913 /* just in case */
914 cpu_park_loop();
5ab6876c
JP
915#else
916 BUG();
78fd584c
AT
917#endif
918}
919
331a1b3a
DA
920static void arm64_backtrace_ipi(cpumask_t *mask)
921{
922 __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask);
923}
924
925void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)
926{
927 /*
928 * NOTE: though nmi_trigger_cpumask_backtrace() has "nmi_" in the name,
929 * nothing about it truly needs to be implemented using an NMI, it's
930 * just that it's _allowed_ to work with NMIs. If ipi_should_be_nmi()
931 * returned false our backtrace attempt will just use a regular IPI.
932 */
933 nmi_trigger_cpumask_backtrace(mask, exclude_cpu, arm64_backtrace_ipi);
934}
935
2f5cd0c7
DA
936#ifdef CONFIG_KGDB
937void kgdb_roundup_cpus(void)
938{
939 int this_cpu = raw_smp_processor_id();
940 int cpu;
941
942 for_each_online_cpu(cpu) {
943 /* No need to roundup ourselves */
944 if (cpu == this_cpu)
945 continue;
946
947 __ipi_send_single(ipi_desc[IPI_KGDB_ROUNDUP], cpu);
948 }
949}
950#endif
951
08e875c1
CM
952/*
953 * Main handler for inter-processor interrupts
954 */
d3afc7f1 955static void do_handle_IPI(int ipinr)
08e875c1
CM
956{
957 unsigned int cpu = smp_processor_id();
08e875c1 958
a2638815 959 if ((unsigned)ipinr < NR_IPI)
4a3182e6 960 trace_ipi_entry(ipi_types[ipinr]);
08e875c1
CM
961
962 switch (ipinr) {
963 case IPI_RESCHEDULE:
964 scheduler_ipi();
965 break;
966
967 case IPI_CALL_FUNC:
08e875c1 968 generic_smp_call_function_interrupt();
08e875c1
CM
969 break;
970
08e875c1 971 case IPI_CPU_STOP:
fdfa5881
DA
972 case IPI_CPU_STOP_NMI:
973 if (IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop) {
d3afc7f1 974 ipi_cpu_crash_stop(cpu, get_irq_regs());
78fd584c 975 unreachable();
fdfa5881
DA
976 } else {
977 local_cpu_stop(cpu);
78fd584c
AT
978 }
979 break;
980
1f85008e
LP
981#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
982 case IPI_TIMER:
1f85008e 983 tick_receive_broadcast();
1f85008e
LP
984 break;
985#endif
986
eb631bb5
LB
987#ifdef CONFIG_IRQ_WORK
988 case IPI_IRQ_WORK:
eb631bb5 989 irq_work_run();
eb631bb5
LB
990 break;
991#endif
992
331a1b3a
DA
993 case IPI_CPU_BACKTRACE:
994 /*
995 * NOTE: in some cases this _won't_ be NMI context. See the
996 * comment in arch_trigger_cpumask_backtrace().
997 */
998 nmi_cpu_backtrace(get_irq_regs());
999 break;
1000
2f5cd0c7
DA
1001 case IPI_KGDB_ROUNDUP:
1002 kgdb_nmicallback(cpu, get_irq_regs());
5e89c55e 1003 break;
5e89c55e 1004
08e875c1
CM
1005 default:
1006 pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
1007 break;
1008 }
45ed695a
NP
1009
1010 if ((unsigned)ipinr < NR_IPI)
4a3182e6 1011 trace_ipi_exit(ipi_types[ipinr]);
d3afc7f1
MZ
1012}
1013
d3afc7f1
MZ
1014static irqreturn_t ipi_handler(int irq, void *data)
1015{
1016 do_handle_IPI(irq - ipi_irq_base);
1017 return IRQ_HANDLED;
1018}
1019
5cebfd2d 1020static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
d3afc7f1 1021{
5cebfd2d
MZ
1022 trace_ipi_raise(target, ipi_types[ipinr]);
1023 __ipi_send_mask(ipi_desc[ipinr], target);
d3afc7f1
MZ
1024}
1025
331a1b3a
DA
1026static bool ipi_should_be_nmi(enum ipi_msg_type ipi)
1027{
4bb49009 1028 if (!system_uses_irq_prio_masking())
331a1b3a
DA
1029 return false;
1030
1031 switch (ipi) {
fdfa5881 1032 case IPI_CPU_STOP_NMI:
331a1b3a 1033 case IPI_CPU_BACKTRACE:
2f5cd0c7 1034 case IPI_KGDB_ROUNDUP:
331a1b3a
DA
1035 return true;
1036 default:
1037 return false;
1038 }
1039}
1040
d3afc7f1
MZ
1041static void ipi_setup(int cpu)
1042{
1043 int i;
1044
5cebfd2d 1045 if (WARN_ON_ONCE(!ipi_irq_base))
d3afc7f1
MZ
1046 return;
1047
331a1b3a
DA
1048 for (i = 0; i < nr_ipi; i++) {
1049 if (ipi_should_be_nmi(i)) {
1050 prepare_percpu_nmi(ipi_irq_base + i);
1051 enable_percpu_nmi(ipi_irq_base + i, 0);
1052 } else {
1053 enable_percpu_irq(ipi_irq_base + i, 0);
1054 }
1055 }
d3afc7f1
MZ
1056}
1057
9d9edb96 1058#ifdef CONFIG_HOTPLUG_CPU
d3afc7f1
MZ
1059static void ipi_teardown(int cpu)
1060{
1061 int i;
1062
5cebfd2d 1063 if (WARN_ON_ONCE(!ipi_irq_base))
d3afc7f1
MZ
1064 return;
1065
331a1b3a
DA
1066 for (i = 0; i < nr_ipi; i++) {
1067 if (ipi_should_be_nmi(i)) {
1068 disable_percpu_nmi(ipi_irq_base + i);
1069 teardown_percpu_nmi(ipi_irq_base + i);
1070 } else {
1071 disable_percpu_irq(ipi_irq_base + i);
1072 }
1073 }
d3afc7f1 1074}
9d9edb96 1075#endif
d3afc7f1
MZ
1076
1077void __init set_smp_ipi_range(int ipi_base, int n)
1078{
1079 int i;
1080
331a1b3a
DA
1081 WARN_ON(n < MAX_IPI);
1082 nr_ipi = min(n, MAX_IPI);
d3afc7f1
MZ
1083
1084 for (i = 0; i < nr_ipi; i++) {
1085 int err;
1086
331a1b3a
DA
1087 if (ipi_should_be_nmi(i)) {
1088 err = request_percpu_nmi(ipi_base + i, ipi_handler,
bf0baa5b 1089 "IPI", &irq_stat);
331a1b3a
DA
1090 WARN(err, "Could not request IPI %d as NMI, err=%d\n",
1091 i, err);
1092 } else {
1093 err = request_percpu_irq(ipi_base + i, ipi_handler,
bf0baa5b 1094 "IPI", &irq_stat);
331a1b3a
DA
1095 WARN(err, "Could not request IPI %d as IRQ, err=%d\n",
1096 i, err);
1097 }
d3afc7f1
MZ
1098
1099 ipi_desc[i] = irq_to_desc(ipi_base + i);
1100 irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
1101 }
1102
1103 ipi_irq_base = ipi_base;
d3afc7f1
MZ
1104
1105 /* Setup the boot CPU immediately */
1106 ipi_setup(smp_processor_id());
1107}
1108
4c8c3c7f 1109void arch_smp_send_reschedule(int cpu)
08e875c1
CM
1110{
1111 smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
1112}
1113
2b2d0a7a
MR
1114#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
1115void arch_send_wakeup_ipi(unsigned int cpu)
1116{
1117 /*
1118 * We use a scheduler IPI to wake the CPU as this avoids the need for a
1119 * dedicated IPI and we can safely handle spurious scheduler IPIs.
1120 */
ef31b8ce 1121 smp_send_reschedule(cpu);
2b2d0a7a
MR
1122}
1123#endif
1124
1f85008e
LP
1125#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
1126void tick_broadcast(const struct cpumask *mask)
1127{
1128 smp_cross_call(mask, IPI_TIMER);
1129}
1130#endif
1131
d0bab0c3
CM
1132/*
1133 * The number of CPUs online, not counting this CPU (which may not be
1134 * fully online and so not counted in num_online_cpus()).
1135 */
1136static inline unsigned int num_other_online_cpus(void)
1137{
1138 unsigned int this_cpu_online = cpu_online(smp_processor_id());
1139
1140 return num_online_cpus() - this_cpu_online;
1141}
1142
08e875c1
CM
1143void smp_send_stop(void)
1144{
fdfa5881
DA
1145 static unsigned long stop_in_progress;
1146 cpumask_t mask;
08e875c1
CM
1147 unsigned long timeout;
1148
fdfa5881
DA
1149 /*
1150 * If this cpu is the only one alive at this point in time, online or
1151 * not, there are no stop messages to be sent around, so just back out.
1152 */
1153 if (num_other_online_cpus() == 0)
1154 goto skip_ipi;
08e875c1 1155
fdfa5881
DA
1156 /* Only proceed if this is the first CPU to reach this code */
1157 if (test_and_set_bit(0, &stop_in_progress))
1158 return;
08e875c1 1159
fdfa5881
DA
1160 /*
1161 * Send an IPI to all currently online CPUs except the CPU running
1162 * this code.
1163 *
1164 * NOTE: we don't do anything here to prevent other CPUs from coming
1165 * online after we snapshot `cpu_online_mask`. Ideally, the calling code
1166 * should do something to prevent other CPUs from coming up. This code
1167 * can be called in the panic path and thus it doesn't seem wise to
1168 * grab the CPU hotplug mutex ourselves. Worst case:
1169 * - If a CPU comes online as we're running, we'll likely notice it
1170 * during the 1 second wait below and then we'll catch it when we try
1171 * with an NMI (assuming NMIs are enabled) since we re-snapshot the
1172 * mask before sending an NMI.
1173 * - If we leave the function and see that CPUs are still online we'll
1174 * at least print a warning. Especially without NMIs this function
1175 * isn't foolproof anyway so calling code will just have to accept
1176 * the fact that there could be cases where a CPU can't be stopped.
1177 */
1178 cpumask_copy(&mask, cpu_online_mask);
1179 cpumask_clear_cpu(smp_processor_id(), &mask);
08e875c1 1180
fdfa5881
DA
1181 if (system_state <= SYSTEM_RUNNING)
1182 pr_crit("SMP: stopping secondary CPUs\n");
1183
1184 /*
1185 * Start with a normal IPI and wait up to one second for other CPUs to
1186 * stop. We do this first because it gives other processors a chance
1187 * to exit critical sections / drop locks and makes the rest of the
1188 * stop process (especially console flush) more robust.
1189 */
1190 smp_cross_call(&mask, IPI_CPU_STOP);
08e875c1 1191 timeout = USEC_PER_SEC;
d0bab0c3 1192 while (num_other_online_cpus() && timeout--)
08e875c1
CM
1193 udelay(1);
1194
fdfa5881
DA
1195 /*
1196 * If CPUs are still online, try an NMI. There's no excuse for this to
1197 * be slow, so we only give them an extra 10 ms to respond.
1198 */
1199 if (num_other_online_cpus() && ipi_should_be_nmi(IPI_CPU_STOP_NMI)) {
1200 smp_rmb();
1201 cpumask_copy(&mask, cpu_online_mask);
1202 cpumask_clear_cpu(smp_processor_id(), &mask);
1203
1204 pr_info("SMP: retry stop with NMI for CPUs %*pbl\n",
1205 cpumask_pr_args(&mask));
1206
1207 smp_cross_call(&mask, IPI_CPU_STOP_NMI);
1208 timeout = USEC_PER_MSEC * 10;
1209 while (num_other_online_cpus() && timeout--)
1210 udelay(1);
1211 }
1212
1213 if (num_other_online_cpus()) {
1214 smp_rmb();
1215 cpumask_copy(&mask, cpu_online_mask);
1216 cpumask_clear_cpu(smp_processor_id(), &mask);
1217
a74ec64a 1218 pr_warn("SMP: failed to stop secondary CPUs %*pbl\n",
fdfa5881
DA
1219 cpumask_pr_args(&mask));
1220 }
f5df2696 1221
fdfa5881 1222skip_ipi:
f5df2696 1223 sdei_mask_local_cpu();
08e875c1
CM
1224}
1225
78fd584c 1226#ifdef CONFIG_KEXEC_CORE
a88ce63b 1227void crash_smp_send_stop(void)
78fd584c 1228{
a88ce63b
HR
1229 /*
1230 * This function can be called twice in panic path, but obviously
1231 * we execute this only once.
fdfa5881
DA
1232 *
1233 * We use this same boolean to tell whether the IPI we send was a
1234 * stop or a "crash stop".
a88ce63b 1235 */
fdfa5881 1236 if (crash_stop)
a88ce63b 1237 return;
fdfa5881 1238 crash_stop = 1;
a88ce63b 1239
fdfa5881 1240 smp_send_stop();
a88ce63b 1241
5cd474e5 1242 sdei_handler_abort();
78fd584c
AT
1243}
1244
1245bool smp_crash_stop_failed(void)
1246{
fdfa5881 1247 return num_other_online_cpus() != 0;
78fd584c
AT
1248}
1249#endif
1250
5c492c3f
JM
1251static bool have_cpu_die(void)
1252{
1253#ifdef CONFIG_HOTPLUG_CPU
1254 int any_cpu = raw_smp_processor_id();
de58ed5e 1255 const struct cpu_operations *ops = get_cpu_ops(any_cpu);
5c492c3f 1256
de58ed5e 1257 if (ops && ops->cpu_die)
5c492c3f
JM
1258 return true;
1259#endif
1260 return false;
1261}
1262
1263bool cpus_are_stuck_in_kernel(void)
1264{
1265 bool smp_spin_tables = (num_possible_cpus() > 1 && !have_cpu_die());
1266
8f4566f1
WD
1267 return !!cpus_stuck_in_kernel || smp_spin_tables ||
1268 is_protected_kvm_enabled();
5c492c3f 1269}