Commit | Line | Data |
---|---|---|
9ff554e9 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
c767a54b | 2 | /* |
4cedb334 GOC |
3 | * x86 SMP booting functions |
4 | * | |
87c6fe26 | 5 | * (c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk> |
8f47e163 | 6 | * (c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com> |
4cedb334 GOC |
7 | * Copyright 2001 Andi Kleen, SuSE Labs. |
8 | * | |
9 | * Much of the core SMP work is based on previous work by Thomas Radke, to | |
10 | * whom a great many thanks are extended. | |
11 | * | |
12 | * Thanks to Intel for making available several different Pentium, | |
13 | * Pentium Pro and Pentium-II/Xeon MP machines. | |
14 | * Original development of Linux SMP code supported by Caldera. | |
15 | * | |
4cedb334 GOC |
16 | * Fixes |
17 | * Felix Koop : NR_CPUS used properly | |
18 | * Jose Renau : Handle single CPU case. | |
19 | * Alan Cox : By repeated request 8) - Total BogoMIPS report. | |
20 | * Greg Wright : Fix for kernel stacks panic. | |
21 | * Erich Boleyn : MP v1.4 and additional changes. | |
22 | * Matthias Sattler : Changes for 2.1 kernel map. | |
23 | * Michel Lespinasse : Changes for 2.1 kernel map. | |
24 | * Michael Chastain : Change trampoline.S to gnu as. | |
25 | * Alan Cox : Dumb bug: 'B' step PPro's are fine | |
26 | * Ingo Molnar : Added APIC timers, based on code | |
27 | * from Jose Renau | |
28 | * Ingo Molnar : various cleanups and rewrites | |
29 | * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug. | |
30 | * Maciej W. Rozycki : Bits for genuine 82489DX APICs | |
31 | * Andi Kleen : Changed for SMP boot into long mode. | |
32 | * Martin J. Bligh : Added support for multi-quad systems | |
33 | * Dave Jones : Report invalid combinations of Athlon CPUs. | |
34 | * Rusty Russell : Hacked into shape for new "hotplug" boot process. | |
35 | * Andi Kleen : Converted to new state machine. | |
36 | * Ashok Raj : CPU hotplug support | |
37 | * Glauber Costa : i386 and x86_64 integration | |
38 | */ | |
39 | ||
c767a54b JP |
40 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
41 | ||
68a1c3f8 GC |
42 | #include <linux/init.h> |
43 | #include <linux/smp.h> | |
186f4360 | 44 | #include <linux/export.h> |
70708a18 | 45 | #include <linux/sched.h> |
105ab3d8 | 46 | #include <linux/sched/topology.h> |
ef8bd77f | 47 | #include <linux/sched/hotplug.h> |
68db0cf1 | 48 | #include <linux/sched/task_stack.h> |
69c18c15 | 49 | #include <linux/percpu.h> |
57c8a661 | 50 | #include <linux/memblock.h> |
cb3c8b90 GOC |
51 | #include <linux/err.h> |
52 | #include <linux/nmi.h> | |
69575d38 | 53 | #include <linux/tboot.h> |
5a0e3ad6 | 54 | #include <linux/gfp.h> |
1a022e3f | 55 | #include <linux/cpuidle.h> |
d7893093 | 56 | #include <linux/kexec.h> |
98fa15f3 | 57 | #include <linux/numa.h> |
65fddcfc | 58 | #include <linux/pgtable.h> |
e2b0d619 | 59 | #include <linux/overflow.h> |
b3883a9a | 60 | #include <linux/stackprotector.h> |
2711b8e2 | 61 | #include <linux/cpuhotplug.h> |
0c7ffa32 | 62 | #include <linux/mc146818rtc.h> |
69c18c15 | 63 | |
8aef135c | 64 | #include <asm/acpi.h> |
955d0e08 | 65 | #include <asm/cacheinfo.h> |
cb3c8b90 | 66 | #include <asm/desc.h> |
69c18c15 GC |
67 | #include <asm/nmi.h> |
68 | #include <asm/irq.h> | |
48927bbb | 69 | #include <asm/realmode.h> |
69c18c15 GC |
70 | #include <asm/cpu.h> |
71 | #include <asm/numa.h> | |
cb3c8b90 GOC |
72 | #include <asm/tlbflush.h> |
73 | #include <asm/mtrr.h> | |
ea530692 | 74 | #include <asm/mwait.h> |
7b6aa335 | 75 | #include <asm/apic.h> |
7167d08e | 76 | #include <asm/io_apic.h> |
b56d2795 | 77 | #include <asm/fpu/api.h> |
569712b2 | 78 | #include <asm/setup.h> |
bdbcdd48 | 79 | #include <asm/uv/uv.h> |
0c7ffa32 | 80 | #include <asm/microcode.h> |
b81bb373 | 81 | #include <asm/i8259.h> |
646e29a1 | 82 | #include <asm/misc.h> |
9043442b | 83 | #include <asm/qspinlock.h> |
1340ccfa AS |
84 | #include <asm/intel-family.h> |
85 | #include <asm/cpu_device_id.h> | |
1f50ddb4 | 86 | #include <asm/spec-ctrl.h> |
447ae316 | 87 | #include <asm/hw_irq.h> |
c9a1ff31 | 88 | #include <asm/stackprotector.h> |
0afb6b66 | 89 | #include <asm/sev.h> |
2743fe89 | 90 | #include <asm/spec-ctrl.h> |
48927bbb | 91 | |
a355352b | 92 | /* representing HT siblings of each logical CPU */ |
0816b0f0 | 93 | DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map); |
a355352b GC |
94 | EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); |
95 | ||
96 | /* representing HT and core siblings of each logical CPU */ | |
0816b0f0 | 97 | DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map); |
a355352b GC |
98 | EXPORT_PER_CPU_SYMBOL(cpu_core_map); |
99 | ||
2e4c54da LB |
100 | /* representing HT, core, and die siblings of each logical CPU */ |
101 | DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_die_map); | |
102 | EXPORT_PER_CPU_SYMBOL(cpu_die_map); | |
103 | ||
f54d4434 TG |
104 | /* CPUs which are the primary SMT threads */ |
105 | struct cpumask __cpu_primary_thread_mask __read_mostly; | |
106 | ||
d4f28f07 TG |
107 | /* Representing CPUs for which sibling maps can be computed */ |
108 | static cpumask_var_t cpu_sibling_setup_mask; | |
109 | ||
f9c9987b TG |
110 | struct mwait_cpu_dead { |
111 | unsigned int control; | |
112 | unsigned int status; | |
113 | }; | |
114 | ||
d7893093 TG |
115 | #define CPUDEAD_MWAIT_WAIT 0xDEADBEEF |
116 | #define CPUDEAD_MWAIT_KEXEC_HLT 0x4A17DEAD | |
117 | ||
f9c9987b TG |
118 | /* |
119 | * Cache line aligned data for mwait_play_dead(). Separate on purpose so | |
120 | * that it's unlikely to be touched by other CPUs. | |
121 | */ | |
122 | static DEFINE_PER_CPU_ALIGNED(struct mwait_cpu_dead, mwait_cpu_dead); | |
123 | ||
70b8301f | 124 | /* Maximum number of SMT threads on any online core */ |
947134d9 | 125 | int __read_mostly __max_smt_threads = 1; |
70b8301f | 126 | |
7d25127c TC |
127 | /* Flag to indicate if a complete sched domain rebuild is required */ |
128 | bool x86_topology_update; | |
129 | ||
130 | int arch_update_cpu_topology(void) | |
131 | { | |
132 | int retval = x86_topology_update; | |
133 | ||
134 | x86_topology_update = false; | |
135 | return retval; | |
136 | } | |
137 | ||
805ae9dc DW |
138 | static unsigned int smpboot_warm_reset_vector_count; |
139 | ||
f77aa308 TG |
140 | static inline void smpboot_setup_warm_reset_vector(unsigned long start_eip) |
141 | { | |
142 | unsigned long flags; | |
143 | ||
144 | spin_lock_irqsave(&rtc_lock, flags); | |
805ae9dc DW |
145 | if (!smpboot_warm_reset_vector_count++) { |
146 | CMOS_WRITE(0xa, 0xf); | |
147 | *((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_HIGH)) = start_eip >> 4; | |
148 | *((volatile unsigned short *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = start_eip & 0xf; | |
149 | } | |
f77aa308 | 150 | spin_unlock_irqrestore(&rtc_lock, flags); |
f77aa308 TG |
151 | } |
152 | ||
153 | static inline void smpboot_restore_warm_reset_vector(void) | |
154 | { | |
155 | unsigned long flags; | |
156 | ||
f77aa308 TG |
157 | /* |
158 | * Paranoid: Set warm reset code and vector here back | |
159 | * to default values. | |
160 | */ | |
161 | spin_lock_irqsave(&rtc_lock, flags); | |
805ae9dc DW |
162 | if (!--smpboot_warm_reset_vector_count) { |
163 | CMOS_WRITE(0, 0xf); | |
164 | *((volatile u32 *)phys_to_virt(TRAMPOLINE_PHYS_LOW)) = 0; | |
165 | } | |
f77aa308 TG |
166 | spin_unlock_irqrestore(&rtc_lock, flags); |
167 | ||
f77aa308 TG |
168 | } |
169 | ||
c8b7fb09 TG |
170 | /* Run the next set of setup steps for the upcoming CPU */ |
171 | static void ap_starting(void) | |
cb3c8b90 | 172 | { |
c8b7fb09 | 173 | int cpuid = smp_processor_id(); |
cb3c8b90 | 174 | |
d7893093 TG |
175 | /* Mop up eventual mwait_play_dead() wreckage */ |
176 | this_cpu_write(mwait_cpu_dead.status, 0); | |
177 | this_cpu_write(mwait_cpu_dead.control, 0); | |
178 | ||
cb3c8b90 | 179 | /* |
2711b8e2 TG |
180 | * If woken up by an INIT in an 82489DX configuration the alive |
181 | * synchronization guarantees that the CPU does not reach this | |
182 | * point before an INIT_deassert IPI reaches the local APIC, so it | |
183 | * is now safe to touch the local APIC. | |
c8b7fb09 TG |
184 | * |
185 | * Set up this CPU, first the APIC, which is probably redundant on | |
186 | * most boards. | |
cb3c8b90 | 187 | */ |
05f7e46d | 188 | apic_ap_setup(); |
cb3c8b90 | 189 | |
c8b7fb09 | 190 | /* Save the processor parameters. */ |
b565201c JS |
191 | smp_store_cpu_info(cpuid); |
192 | ||
76ce7cfe PT |
193 | /* |
194 | * The topology information must be up to date before | |
134a1282 | 195 | * notify_cpu_starting(). |
76ce7cfe | 196 | */ |
c8b7fb09 | 197 | set_cpu_sibling_map(cpuid); |
76ce7cfe | 198 | |
bb6e89df | 199 | ap_init_aperfmperf(); |
1567c3e3 | 200 | |
cfc1b9a6 | 201 | pr_debug("Stack at about %p\n", &cpuid); |
cb3c8b90 | 202 | |
5ef428c4 AK |
203 | wmb(); |
204 | ||
2b3be65d DW |
205 | /* |
206 | * This runs the AP through all the cpuhp states to its target | |
207 | * state CPUHP_ONLINE. | |
208 | */ | |
85257024 | 209 | notify_cpu_starting(cpuid); |
cb3c8b90 | 210 | } |
85257024 | 211 | |
134a1282 TG |
212 | static void ap_calibrate_delay(void) |
213 | { | |
cb3c8b90 | 214 | /* |
134a1282 TG |
215 | * Calibrate the delay loop and update loops_per_jiffy in cpu_data. |
216 | * smp_store_cpu_info() stored a value that is close but not as | |
217 | * accurate as the value just calculated. | |
218 | * | |
219 | * As this is invoked after the TSC synchronization check, | |
220 | * calibrate_delay_is_known() will skip the calibration routine | |
221 | * when TSC is synchronized across sockets. | |
cb3c8b90 | 222 | */ |
134a1282 TG |
223 | calibrate_delay(); |
224 | cpu_data(smp_processor_id()).loops_per_jiffy = loops_per_jiffy; | |
cb3c8b90 GOC |
225 | } |
226 | ||
bbc2ff6a GOC |
227 | /* |
228 | * Activate a secondary processor. | |
229 | */ | |
148f9bb8 | 230 | static void notrace start_secondary(void *unused) |
bbc2ff6a GOC |
231 | { |
232 | /* | |
c7ad5ad2 AL |
233 | * Don't put *anything* except direct CPU state initialization |
234 | * before cpu_init(), SMP booting is too fragile that we want to | |
235 | * limit the things done here to the most necessary things. | |
bbc2ff6a | 236 | */ |
7652ac92 | 237 | cr4_init(); |
e1c467e6 | 238 | |
0c7ffa32 TG |
239 | /* |
240 | * 32-bit specific. 64-bit reaches this code with the correct page | |
241 | * table established. Yet another historical divergence. | |
242 | */ | |
243 | if (IS_ENABLED(CONFIG_X86_32)) { | |
244 | /* switch away from the initial page table */ | |
245 | load_cr3(swapper_pg_dir); | |
246 | __flush_tlb_all(); | |
247 | } | |
248 | ||
e94cd150 TG |
249 | cpu_init_exception_handling(); |
250 | ||
2b3be65d | 251 | /* |
0b62f6cb TG |
252 | * Load the microcode before reaching the AP alive synchronization |
253 | * point below so it is not part of the full per CPU serialized | |
254 | * bringup part when "parallel" bringup is enabled. | |
0c7ffa32 TG |
255 | * |
256 | * That's even safe when hyperthreading is enabled in the CPU as | |
257 | * the core code starts the primary threads first and leaves the | |
258 | * secondary threads waiting for SIPI. Loading microcode on | |
259 | * physical cores concurrently is a safe operation. | |
260 | * | |
261 | * This covers both the Intel specific issue that concurrent | |
262 | * microcode loading on SMT siblings must be prohibited and the | |
263 | * vendor independent issue`that microcode loading which changes | |
264 | * CPUID, MSRs etc. must be strictly serialized to maintain | |
265 | * software state correctness. | |
266 | */ | |
0b62f6cb | 267 | load_ucode_ap(); |
0c7ffa32 TG |
268 | |
269 | /* | |
270 | * Synchronization point with the hotplug core. Sets this CPUs | |
271 | * synchronization state to ALIVE and spin-waits for the control CPU to | |
2711b8e2 | 272 | * release this CPU for further bringup. |
2b3be65d | 273 | */ |
2711b8e2 | 274 | cpuhp_ap_sync_alive(); |
e94cd150 TG |
275 | |
276 | cpu_init(); | |
9244724f | 277 | fpu__init_cpu(); |
448e9f34 | 278 | rcutree_report_cpu_starting(raw_smp_processor_id()); |
4ba55e65 | 279 | x86_cpuinit.early_percpu_clock_init(); |
4ba55e65 | 280 | |
c8b7fb09 | 281 | ap_starting(); |
4ba55e65 | 282 | |
9d349d47 | 283 | /* Check TSC synchronization with the control CPU. */ |
134a1282 | 284 | check_tsc_sync_target(); |
4ba55e65 | 285 | |
bbc2ff6a | 286 | /* |
134a1282 TG |
287 | * Calibrate the delay loop after the TSC synchronization check. |
288 | * This allows to skip the calibration when TSC is synchronized | |
289 | * across sockets. | |
bbc2ff6a | 290 | */ |
134a1282 | 291 | ap_calibrate_delay(); |
bbc2ff6a | 292 | |
1f50ddb4 TG |
293 | speculative_store_bypass_ht_init(); |
294 | ||
bbc2ff6a | 295 | /* |
8ed4f3e6 TG |
296 | * Lock vector_lock, set CPU online and bring the vector |
297 | * allocator online. Online must be set with vector_lock held | |
298 | * to prevent a concurrent irq setup/teardown from seeing a | |
299 | * half valid vector space. | |
bbc2ff6a | 300 | */ |
d388e5fd | 301 | lock_vector_lock(); |
c2d1cec1 | 302 | set_cpu_online(smp_processor_id(), true); |
8ed4f3e6 | 303 | lapic_online(); |
d388e5fd | 304 | unlock_vector_lock(); |
78c06176 | 305 | x86_platform.nmi_init(); |
bbc2ff6a | 306 | |
0cefa5b9 MS |
307 | /* enable local interrupts */ |
308 | local_irq_enable(); | |
309 | ||
736decac | 310 | x86_cpuinit.setup_percpu_clockev(); |
bbc2ff6a GOC |
311 | |
312 | wmb(); | |
fc6d73d6 | 313 | cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); |
bbc2ff6a GOC |
314 | } |
315 | ||
1d89a7f0 GOC |
316 | /* |
317 | * The bootstrap kernel entry code has set these up. Save them for | |
318 | * a given CPU | |
319 | */ | |
148f9bb8 | 320 | void smp_store_cpu_info(int id) |
1d89a7f0 GOC |
321 | { |
322 | struct cpuinfo_x86 *c = &cpu_data(id); | |
323 | ||
30bb9811 AK |
324 | /* Copy boot_cpu_data only on the first bringup */ |
325 | if (!c->initialized) | |
326 | *c = boot_cpu_data; | |
1d89a7f0 | 327 | c->cpu_index = id; |
30106c17 FY |
328 | /* |
329 | * During boot time, CPU0 has this setup already. Save the info when | |
5475abbd | 330 | * bringing up an AP. |
30106c17 FY |
331 | */ |
332 | identify_secondary_cpu(c); | |
30bb9811 | 333 | c->initialized = true; |
1d89a7f0 GOC |
334 | } |
335 | ||
cebf15eb DH |
336 | static bool |
337 | topology_same_node(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) | |
338 | { | |
339 | int cpu1 = c->cpu_index, cpu2 = o->cpu_index; | |
340 | ||
341 | return (cpu_to_node(cpu1) == cpu_to_node(cpu2)); | |
342 | } | |
343 | ||
148f9bb8 | 344 | static bool |
316ad248 | 345 | topology_sane(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o, const char *name) |
d4fbe4f0 | 346 | { |
316ad248 PZ |
347 | int cpu1 = c->cpu_index, cpu2 = o->cpu_index; |
348 | ||
cebf15eb | 349 | return !WARN_ONCE(!topology_same_node(c, o), |
316ad248 PZ |
350 | "sched: CPU #%d's %s-sibling CPU #%d is not on the same node! " |
351 | "[node: %d != %d]. Ignoring dependency.\n", | |
352 | cpu1, name, cpu2, cpu_to_node(cpu1), cpu_to_node(cpu2)); | |
353 | } | |
354 | ||
7d79a7bd | 355 | #define link_mask(mfunc, c1, c2) \ |
316ad248 | 356 | do { \ |
7d79a7bd BG |
357 | cpumask_set_cpu((c1), mfunc(c2)); \ |
358 | cpumask_set_cpu((c2), mfunc(c1)); \ | |
316ad248 PZ |
359 | } while (0) |
360 | ||
148f9bb8 | 361 | static bool match_smt(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) |
316ad248 | 362 | { |
362f924b | 363 | if (boot_cpu_has(X86_FEATURE_TOPOEXT)) { |
316ad248 PZ |
364 | int cpu1 = c->cpu_index, cpu2 = o->cpu_index; |
365 | ||
02fb601d | 366 | if (c->topo.pkg_id == o->topo.pkg_id && |
8a169ed4 | 367 | c->topo.die_id == o->topo.die_id && |
ace278e7 | 368 | c->topo.amd_node_id == o->topo.amd_node_id && |
6e290323 | 369 | per_cpu_llc_id(cpu1) == per_cpu_llc_id(cpu2)) { |
e9525633 | 370 | if (c->topo.core_id == o->topo.core_id) |
79a8b9aa BP |
371 | return topology_sane(c, o, "smt"); |
372 | ||
e3c0c5d5 TG |
373 | if ((c->topo.cu_id != 0xff) && |
374 | (o->topo.cu_id != 0xff) && | |
375 | (c->topo.cu_id == o->topo.cu_id)) | |
79a8b9aa BP |
376 | return topology_sane(c, o, "smt"); |
377 | } | |
316ad248 | 378 | |
02fb601d | 379 | } else if (c->topo.pkg_id == o->topo.pkg_id && |
8a169ed4 | 380 | c->topo.die_id == o->topo.die_id && |
e9525633 | 381 | c->topo.core_id == o->topo.core_id) { |
316ad248 PZ |
382 | return topology_sane(c, o, "smt"); |
383 | } | |
384 | ||
385 | return false; | |
386 | } | |
387 | ||
2c88d45e AS |
388 | static bool match_die(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) |
389 | { | |
ace278e7 TG |
390 | if (c->topo.pkg_id != o->topo.pkg_id || c->topo.die_id != o->topo.die_id) |
391 | return false; | |
392 | ||
393 | if (cpu_feature_enabled(X86_FEATURE_TOPOEXT) && topology_amd_nodes_per_pkg() > 1) | |
394 | return c->topo.amd_node_id == o->topo.amd_node_id; | |
395 | ||
396 | return true; | |
2c88d45e AS |
397 | } |
398 | ||
66558b73 TC |
399 | static bool match_l2c(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) |
400 | { | |
401 | int cpu1 = c->cpu_index, cpu2 = o->cpu_index; | |
402 | ||
55409ac5 | 403 | /* If the arch didn't set up l2c_id, fall back to SMT */ |
6e290323 | 404 | if (per_cpu_l2c_id(cpu1) == BAD_APICID) |
55409ac5 | 405 | return match_smt(c, o); |
66558b73 TC |
406 | |
407 | /* Do not match if L2 cache id does not match: */ | |
6e290323 | 408 | if (per_cpu_l2c_id(cpu1) != per_cpu_l2c_id(cpu2)) |
66558b73 TC |
409 | return false; |
410 | ||
411 | return topology_sane(c, o, "l2c"); | |
412 | } | |
413 | ||
1340ccfa | 414 | /* |
2c88d45e AS |
415 | * Unlike the other levels, we do not enforce keeping a |
416 | * multicore group inside a NUMA node. If this happens, we will | |
417 | * discard the MC level of the topology later. | |
418 | */ | |
419 | static bool match_pkg(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) | |
420 | { | |
02fb601d | 421 | if (c->topo.pkg_id == o->topo.pkg_id) |
2c88d45e AS |
422 | return true; |
423 | return false; | |
424 | } | |
425 | ||
426 | /* | |
427 | * Define intel_cod_cpu[] for Intel COD (Cluster-on-Die) CPUs. | |
1340ccfa | 428 | * |
2c88d45e AS |
429 | * Any Intel CPU that has multiple nodes per package and does not |
430 | * match intel_cod_cpu[] has the SNC (Sub-NUMA Cluster) topology. | |
1340ccfa | 431 | * |
2c88d45e AS |
432 | * When in SNC mode, these CPUs enumerate an LLC that is shared |
433 | * by multiple NUMA nodes. The LLC is shared for off-package data | |
434 | * access but private to the NUMA node (half of the package) for | |
435 | * on-package access. CPUID (the source of the information about | |
436 | * the LLC) can only enumerate the cache as shared or unshared, | |
437 | * but not this particular configuration. | |
1340ccfa AS |
438 | */ |
439 | ||
2c88d45e | 440 | static const struct x86_cpu_id intel_cod_cpu[] = { |
4db64279 TL |
441 | X86_MATCH_VFM(INTEL_HASWELL_X, 0), /* COD */ |
442 | X86_MATCH_VFM(INTEL_BROADWELL_X, 0), /* COD */ | |
443 | X86_MATCH_VFM(INTEL_ANY, 1), /* SNC */ | |
1340ccfa AS |
444 | {} |
445 | }; | |
446 | ||
148f9bb8 | 447 | static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o) |
316ad248 | 448 | { |
2c88d45e | 449 | const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu); |
316ad248 | 450 | int cpu1 = c->cpu_index, cpu2 = o->cpu_index; |
2c88d45e | 451 | bool intel_snc = id && id->driver_data; |
316ad248 | 452 | |
1340ccfa | 453 | /* Do not match if we do not have a valid APICID for cpu: */ |
6e290323 | 454 | if (per_cpu_llc_id(cpu1) == BAD_APICID) |
1340ccfa | 455 | return false; |
316ad248 | 456 | |
1340ccfa | 457 | /* Do not match if LLC id does not match: */ |
6e290323 | 458 | if (per_cpu_llc_id(cpu1) != per_cpu_llc_id(cpu2)) |
1340ccfa AS |
459 | return false; |
460 | ||
461 | /* | |
462 | * Allow the SNC topology without warning. Return of false | |
463 | * means 'c' does not share the LLC of 'o'. This will be | |
464 | * reflected to userspace. | |
465 | */ | |
2c88d45e | 466 | if (match_pkg(c, o) && !topology_same_node(c, o) && intel_snc) |
1340ccfa AS |
467 | return false; |
468 | ||
469 | return topology_sane(c, o, "llc"); | |
d4fbe4f0 AH |
470 | } |
471 | ||
2e4c54da | 472 | |
d3d37d85 TC |
473 | static inline int x86_sched_itmt_flags(void) |
474 | { | |
475 | return sysctl_sched_itmt_enabled ? SD_ASYM_PACKING : 0; | |
476 | } | |
477 | ||
478 | #ifdef CONFIG_SCHED_MC | |
479 | static int x86_core_flags(void) | |
480 | { | |
481 | return cpu_core_flags() | x86_sched_itmt_flags(); | |
482 | } | |
483 | #endif | |
484 | #ifdef CONFIG_SCHED_SMT | |
485 | static int x86_smt_flags(void) | |
486 | { | |
995998eb | 487 | return cpu_smt_flags(); |
d3d37d85 TC |
488 | } |
489 | #endif | |
66558b73 TC |
490 | #ifdef CONFIG_SCHED_CLUSTER |
491 | static int x86_cluster_flags(void) | |
492 | { | |
493 | return cpu_cluster_flags() | x86_sched_itmt_flags(); | |
494 | } | |
495 | #endif | |
108af4b4 RN |
496 | |
497 | static int x86_die_flags(void) | |
498 | { | |
499 | if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) | |
500 | return x86_sched_itmt_flags(); | |
501 | ||
502 | return 0; | |
503 | } | |
d3d37d85 | 504 | |
8f2d6c41 PZ |
505 | /* |
506 | * Set if a package/die has multiple NUMA nodes inside. | |
507 | * AMD Magny-Cours, Intel Cluster-on-Die, and Intel | |
508 | * Sub-NUMA Clustering have this. | |
509 | */ | |
510 | static bool x86_has_numa_in_package; | |
8f37961c | 511 | |
8f2d6c41 PZ |
512 | static struct sched_domain_topology_level x86_topology[6]; |
513 | ||
514 | static void __init build_sched_topology(void) | |
515 | { | |
516 | int i = 0; | |
cabdc3a8 | 517 | |
8f37961c | 518 | #ifdef CONFIG_SCHED_SMT |
8f2d6c41 PZ |
519 | x86_topology[i++] = (struct sched_domain_topology_level){ |
520 | cpu_smt_mask, x86_smt_flags, SD_INIT_NAME(SMT) | |
521 | }; | |
8f37961c | 522 | #endif |
66558b73 | 523 | #ifdef CONFIG_SCHED_CLUSTER |
17953249 PZ |
524 | x86_topology[i++] = (struct sched_domain_topology_level){ |
525 | cpu_clustergroup_mask, x86_cluster_flags, SD_INIT_NAME(CLS) | |
526 | }; | |
66558b73 | 527 | #endif |
8f37961c | 528 | #ifdef CONFIG_SCHED_MC |
8f2d6c41 PZ |
529 | x86_topology[i++] = (struct sched_domain_topology_level){ |
530 | cpu_coregroup_mask, x86_core_flags, SD_INIT_NAME(MC) | |
531 | }; | |
8f37961c | 532 | #endif |
8f2d6c41 | 533 | /* |
f577cd57 | 534 | * When there is NUMA topology inside the package skip the PKG domain |
8f2d6c41 PZ |
535 | * since the NUMA domains will auto-magically create the right spanning |
536 | * domains based on the SLIT. | |
537 | */ | |
538 | if (!x86_has_numa_in_package) { | |
539 | x86_topology[i++] = (struct sched_domain_topology_level){ | |
f577cd57 | 540 | cpu_cpu_mask, x86_die_flags, SD_INIT_NAME(PKG) |
8f2d6c41 PZ |
541 | }; |
542 | } | |
8f37961c | 543 | |
8f2d6c41 PZ |
544 | /* |
545 | * There must be one trailing NULL entry left. | |
546 | */ | |
547 | BUG_ON(i >= ARRAY_SIZE(x86_topology)-1); | |
548 | ||
549 | set_sched_topology(x86_topology); | |
550 | } | |
cebf15eb | 551 | |
148f9bb8 | 552 | void set_cpu_sibling_map(int cpu) |
768d9505 | 553 | { |
8078f4d6 | 554 | bool has_smt = __max_threads_per_core > 1; |
89b0f15f | 555 | bool has_mp = has_smt || topology_num_cores_per_package() > 1; |
768d9505 | 556 | struct cpuinfo_x86 *c = &cpu_data(cpu); |
316ad248 | 557 | struct cpuinfo_x86 *o; |
70b8301f | 558 | int i, threads; |
768d9505 | 559 | |
c2d1cec1 | 560 | cpumask_set_cpu(cpu, cpu_sibling_setup_mask); |
768d9505 | 561 | |
b0bc225d | 562 | if (!has_mp) { |
7d79a7bd | 563 | cpumask_set_cpu(cpu, topology_sibling_cpumask(cpu)); |
316ad248 | 564 | cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu)); |
66558b73 | 565 | cpumask_set_cpu(cpu, cpu_l2c_shared_mask(cpu)); |
7d79a7bd | 566 | cpumask_set_cpu(cpu, topology_core_cpumask(cpu)); |
2e4c54da | 567 | cpumask_set_cpu(cpu, topology_die_cpumask(cpu)); |
768d9505 GC |
568 | c->booted_cores = 1; |
569 | return; | |
570 | } | |
571 | ||
c2d1cec1 | 572 | for_each_cpu(i, cpu_sibling_setup_mask) { |
316ad248 PZ |
573 | o = &cpu_data(i); |
574 | ||
2c88d45e AS |
575 | if (match_pkg(c, o) && !topology_same_node(c, o)) |
576 | x86_has_numa_in_package = true; | |
577 | ||
316ad248 | 578 | if ((i == cpu) || (has_smt && match_smt(c, o))) |
7d79a7bd | 579 | link_mask(topology_sibling_cpumask, cpu, i); |
316ad248 | 580 | |
b0bc225d | 581 | if ((i == cpu) || (has_mp && match_llc(c, o))) |
7d79a7bd | 582 | link_mask(cpu_llc_shared_mask, cpu, i); |
316ad248 | 583 | |
66558b73 TC |
584 | if ((i == cpu) || (has_mp && match_l2c(c, o))) |
585 | link_mask(cpu_l2c_shared_mask, cpu, i); | |
586 | ||
2c88d45e AS |
587 | if ((i == cpu) || (has_mp && match_die(c, o))) |
588 | link_mask(topology_die_cpumask, cpu, i); | |
ceb1cbac KB |
589 | } |
590 | ||
2c88d45e AS |
591 | threads = cpumask_weight(topology_sibling_cpumask(cpu)); |
592 | if (threads > __max_smt_threads) | |
593 | __max_smt_threads = threads; | |
594 | ||
c52787b5 BS |
595 | for_each_cpu(i, topology_sibling_cpumask(cpu)) |
596 | cpu_data(i).smt_active = threads > 1; | |
597 | ||
ceb1cbac KB |
598 | /* |
599 | * This needs a separate iteration over the cpus because we rely on all | |
7d79a7bd | 600 | * topology_sibling_cpumask links to be set-up. |
ceb1cbac KB |
601 | */ |
602 | for_each_cpu(i, cpu_sibling_setup_mask) { | |
603 | o = &cpu_data(i); | |
604 | ||
169d0869 | 605 | if ((i == cpu) || (has_mp && match_pkg(c, o))) { |
7d79a7bd | 606 | link_mask(topology_core_cpumask, cpu, i); |
316ad248 | 607 | |
768d9505 GC |
608 | /* |
609 | * Does this new cpu bringup a new core? | |
610 | */ | |
2c88d45e | 611 | if (threads == 1) { |
768d9505 GC |
612 | /* |
613 | * for each core in package, increment | |
614 | * the booted_cores for this new cpu | |
615 | */ | |
7d79a7bd BG |
616 | if (cpumask_first( |
617 | topology_sibling_cpumask(i)) == i) | |
768d9505 GC |
618 | c->booted_cores++; |
619 | /* | |
620 | * increment the core count for all | |
621 | * the other cpus in this package | |
622 | */ | |
623 | if (i != cpu) | |
624 | cpu_data(i).booted_cores++; | |
625 | } else if (i != cpu && !c->booted_cores) | |
626 | c->booted_cores = cpu_data(i).booted_cores; | |
627 | } | |
628 | } | |
629 | } | |
630 | ||
70708a18 | 631 | /* maps the cpu to the sched domain representing multi-core */ |
030bb203 | 632 | const struct cpumask *cpu_coregroup_mask(int cpu) |
70708a18 | 633 | { |
9f646389 | 634 | return cpu_llc_shared_mask(cpu); |
030bb203 RR |
635 | } |
636 | ||
66558b73 TC |
637 | const struct cpumask *cpu_clustergroup_mask(int cpu) |
638 | { | |
639 | return cpu_l2c_shared_mask(cpu); | |
640 | } | |
c3dd1995 | 641 | EXPORT_SYMBOL_GPL(cpu_clustergroup_mask); |
66558b73 | 642 | |
a4928cff | 643 | static void impress_friends(void) |
904541e2 GOC |
644 | { |
645 | int cpu; | |
646 | unsigned long bogosum = 0; | |
647 | /* | |
648 | * Allow the user to impress friends. | |
649 | */ | |
c767a54b | 650 | pr_debug("Before bogomips\n"); |
2711b8e2 TG |
651 | for_each_online_cpu(cpu) |
652 | bogosum += cpu_data(cpu).loops_per_jiffy; | |
653 | ||
c767a54b | 654 | pr_info("Total of %d processors activated (%lu.%02lu BogoMIPS)\n", |
f68e00a3 | 655 | num_online_cpus(), |
904541e2 GOC |
656 | bogosum/(500000/HZ), |
657 | (bogosum/(5000/HZ))%100); | |
658 | ||
c767a54b | 659 | pr_debug("Before bogocount - setting activated=1\n"); |
904541e2 GOC |
660 | } |
661 | ||
d68921f9 LB |
662 | /* |
663 | * The Multiprocessor Specification 1.4 (1997) example code suggests | |
664 | * that there should be a 10ms delay between the BSP asserting INIT | |
665 | * and de-asserting INIT, when starting a remote processor. | |
666 | * But that slows boot and resume on modern processors, which include | |
667 | * many cores and don't require that delay. | |
668 | * | |
669 | * Cmdline "init_cpu_udelay=" is available to over-ride this delay. | |
1a744cb3 | 670 | * Modern processor families are quirked to remove the delay entirely. |
d68921f9 LB |
671 | */ |
672 | #define UDELAY_10MS_DEFAULT 10000 | |
673 | ||
656279a1 | 674 | static unsigned int init_udelay = UINT_MAX; |
d68921f9 LB |
675 | |
676 | static int __init cpu_init_udelay(char *str) | |
677 | { | |
678 | get_option(&str, &init_udelay); | |
679 | ||
680 | return 0; | |
681 | } | |
682 | early_param("cpu_init_udelay", cpu_init_udelay); | |
683 | ||
1a744cb3 LB |
684 | static void __init smp_quirk_init_udelay(void) |
685 | { | |
686 | /* if cmdline changed it from default, leave it alone */ | |
656279a1 | 687 | if (init_udelay != UINT_MAX) |
1a744cb3 LB |
688 | return; |
689 | ||
690 | /* if modern processor, use no delay */ | |
691 | if (((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 6)) || | |
0b13bec7 | 692 | ((boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) && (boot_cpu_data.x86 >= 0x18)) || |
656279a1 | 693 | ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && (boot_cpu_data.x86 >= 0xF))) { |
1a744cb3 | 694 | init_udelay = 0; |
656279a1 LB |
695 | return; |
696 | } | |
f1ccd249 LB |
697 | /* else, use legacy delay */ |
698 | init_udelay = UDELAY_10MS_DEFAULT; | |
1a744cb3 LB |
699 | } |
700 | ||
cb3c8b90 | 701 | /* |
5475abbd | 702 | * Wake up AP by INIT, INIT, STARTUP sequence. |
cb3c8b90 | 703 | */ |
8aa2a417 | 704 | static void send_init_sequence(u32 phys_apicid) |
cb3c8b90 | 705 | { |
6087dd5e | 706 | int maxlvt = lapic_get_maxlvt(); |
593f4a78 | 707 | |
6087dd5e | 708 | /* Be paranoid about clearing APIC errors. */ |
cff9ab2b | 709 | if (APIC_INTEGRATED(boot_cpu_apic_version)) { |
6087dd5e TG |
710 | /* Due to the Pentium erratum 3AP. */ |
711 | if (maxlvt > 3) | |
593f4a78 | 712 | apic_write(APIC_ESR, 0); |
cb3c8b90 GOC |
713 | apic_read(APIC_ESR); |
714 | } | |
715 | ||
6087dd5e TG |
716 | /* Assert INIT on the target CPU */ |
717 | apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT, phys_apicid); | |
718 | safe_apic_wait_icr_idle(); | |
cb3c8b90 | 719 | |
7cb68598 | 720 | udelay(init_udelay); |
cb3c8b90 | 721 | |
6087dd5e | 722 | /* Deassert INIT on the target CPU */ |
1b374e4d | 723 | apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid); |
6087dd5e TG |
724 | safe_apic_wait_icr_idle(); |
725 | } | |
cb3c8b90 | 726 | |
6087dd5e TG |
727 | /* |
728 | * Wake up AP by INIT, INIT, STARTUP sequence. | |
729 | */ | |
8aa2a417 | 730 | static int wakeup_secondary_cpu_via_init(u32 phys_apicid, unsigned long start_eip) |
6087dd5e TG |
731 | { |
732 | unsigned long send_status = 0, accept_status = 0; | |
88afbb21 | 733 | int num_starts, j, maxlvt; |
6087dd5e | 734 | |
88afbb21 LT |
735 | preempt_disable(); |
736 | maxlvt = lapic_get_maxlvt(); | |
6087dd5e | 737 | send_init_sequence(phys_apicid); |
cb3c8b90 GOC |
738 | |
739 | mb(); | |
cb3c8b90 GOC |
740 | |
741 | /* | |
742 | * Should we send STARTUP IPIs ? | |
743 | * | |
744 | * Determine this based on the APIC version. | |
745 | * If we don't have an integrated APIC, don't send the STARTUP IPIs. | |
746 | */ | |
cff9ab2b | 747 | if (APIC_INTEGRATED(boot_cpu_apic_version)) |
cb3c8b90 GOC |
748 | num_starts = 2; |
749 | else | |
750 | num_starts = 0; | |
751 | ||
cb3c8b90 GOC |
752 | /* |
753 | * Run STARTUP IPI loop. | |
754 | */ | |
c767a54b | 755 | pr_debug("#startup loops: %d\n", num_starts); |
cb3c8b90 | 756 | |
cb3c8b90 | 757 | for (j = 1; j <= num_starts; j++) { |
c767a54b | 758 | pr_debug("Sending STARTUP #%d\n", j); |
593f4a78 MR |
759 | if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ |
760 | apic_write(APIC_ESR, 0); | |
cb3c8b90 | 761 | apic_read(APIC_ESR); |
c767a54b | 762 | pr_debug("After apic_write\n"); |
cb3c8b90 GOC |
763 | |
764 | /* | |
765 | * STARTUP IPI | |
766 | */ | |
767 | ||
768 | /* Target chip */ | |
cb3c8b90 GOC |
769 | /* Boot on the stack */ |
770 | /* Kick the second */ | |
1b374e4d SS |
771 | apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12), |
772 | phys_apicid); | |
cb3c8b90 GOC |
773 | |
774 | /* | |
775 | * Give the other CPU some time to accept the IPI. | |
776 | */ | |
fcafddec LB |
777 | if (init_udelay == 0) |
778 | udelay(10); | |
779 | else | |
a9bcaa02 | 780 | udelay(300); |
cb3c8b90 | 781 | |
c767a54b | 782 | pr_debug("Startup point 1\n"); |
cb3c8b90 | 783 | |
cfc1b9a6 | 784 | pr_debug("Waiting for send to finish...\n"); |
cb3c8b90 GOC |
785 | send_status = safe_apic_wait_icr_idle(); |
786 | ||
787 | /* | |
788 | * Give the other CPU some time to accept the IPI. | |
789 | */ | |
fcafddec LB |
790 | if (init_udelay == 0) |
791 | udelay(10); | |
792 | else | |
a9bcaa02 | 793 | udelay(200); |
cb3c8b90 | 794 | |
593f4a78 | 795 | if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ |
cb3c8b90 | 796 | apic_write(APIC_ESR, 0); |
cb3c8b90 GOC |
797 | accept_status = (apic_read(APIC_ESR) & 0xEF); |
798 | if (send_status || accept_status) | |
799 | break; | |
800 | } | |
c767a54b | 801 | pr_debug("After Startup\n"); |
cb3c8b90 GOC |
802 | |
803 | if (send_status) | |
c767a54b | 804 | pr_err("APIC never delivered???\n"); |
cb3c8b90 | 805 | if (accept_status) |
c767a54b | 806 | pr_err("APIC delivery error (%lx)\n", accept_status); |
cb3c8b90 | 807 | |
5475abbd | 808 | preempt_enable(); |
cb3c8b90 GOC |
809 | return (send_status | accept_status); |
810 | } | |
cb3c8b90 | 811 | |
2eaad1fd | 812 | /* reduce the number of lines printed when booting a large cpu count system */ |
148f9bb8 | 813 | static void announce_cpu(int cpu, int apicid) |
2eaad1fd | 814 | { |
0c7ffa32 | 815 | static int width, node_width, first = 1; |
98fa15f3 | 816 | static int current_node = NUMA_NO_NODE; |
4adc8b71 | 817 | int node = early_cpu_to_node(cpu); |
646e29a1 BP |
818 | |
819 | if (!width) | |
820 | width = num_digits(num_possible_cpus()) + 1; /* + '#' sign */ | |
2eaad1fd | 821 | |
a17bce4d BP |
822 | if (!node_width) |
823 | node_width = num_digits(num_possible_nodes()) + 1; /* + '#' */ | |
824 | ||
719b3680 | 825 | if (system_state < SYSTEM_RUNNING) { |
0c7ffa32 TG |
826 | if (first) |
827 | pr_info("x86: Booting SMP configuration:\n"); | |
828 | ||
2eaad1fd MT |
829 | if (node != current_node) { |
830 | if (current_node > (-1)) | |
a17bce4d | 831 | pr_cont("\n"); |
2eaad1fd | 832 | current_node = node; |
a17bce4d BP |
833 | |
834 | printk(KERN_INFO ".... node %*s#%d, CPUs: ", | |
835 | node_width - num_digits(node), " ", node); | |
2eaad1fd | 836 | } |
646e29a1 BP |
837 | |
838 | /* Add padding for the BSP */ | |
0c7ffa32 | 839 | if (first) |
646e29a1 | 840 | pr_cont("%*s", width + 1, " "); |
0c7ffa32 | 841 | first = 0; |
646e29a1 BP |
842 | |
843 | pr_cont("%*s#%d", width - num_digits(cpu), " ", cpu); | |
2eaad1fd MT |
844 | } else |
845 | pr_info("Booting Node %d Processor %d APIC 0x%x\n", | |
846 | node, cpu, apicid); | |
847 | } | |
848 | ||
66c7ceb4 | 849 | int common_cpu_up(unsigned int cpu, struct task_struct *idle) |
3f85483b | 850 | { |
66c7ceb4 TG |
851 | int ret; |
852 | ||
3f85483b BO |
853 | /* Just in case we booted with a single CPU. */ |
854 | alternatives_enable_smp(); | |
855 | ||
e57ef2ed | 856 | per_cpu(pcpu_hot.current_task, cpu) = idle; |
c9a1ff31 | 857 | cpu_init_stack_canary(cpu, idle); |
3f85483b | 858 | |
66c7ceb4 TG |
859 | /* Initialize the interrupt stack(s) */ |
860 | ret = irq_init_percpu_irqstack(cpu); | |
861 | if (ret) | |
862 | return ret; | |
863 | ||
3f85483b BO |
864 | #ifdef CONFIG_X86_32 |
865 | /* Stack for startup_32 can be just as for start_secondary onwards */ | |
c063a217 | 866 | per_cpu(pcpu_hot.top_of_stack, cpu) = task_top_of_stack(idle); |
3f85483b | 867 | #endif |
66c7ceb4 | 868 | return 0; |
3f85483b BO |
869 | } |
870 | ||
cb3c8b90 GOC |
871 | /* |
872 | * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad | |
873 | * (ie clustered apic addressing mode), this is a LOGICAL apic ID. | |
2b3be65d | 874 | * Returns zero if startup was successfully sent, else error code from |
1f5bcabf | 875 | * ->wakeup_secondary_cpu. |
cb3c8b90 | 876 | */ |
8aa2a417 | 877 | static int do_boot_cpu(u32 apicid, int cpu, struct task_struct *idle) |
cb3c8b90 | 878 | { |
f37240f1 | 879 | unsigned long start_ip = real_mode_header->trampoline_start; |
2711b8e2 | 880 | int ret; |
cb3c8b90 | 881 | |
ff2e6468 SC |
882 | #ifdef CONFIG_X86_64 |
883 | /* If 64-bit wakeup method exists, use the 64-bit mode trampoline IP */ | |
884 | if (apic->wakeup_secondary_cpu_64) | |
885 | start_ip = real_mode_header->trampoline_start64; | |
886 | #endif | |
b9b1a9c3 | 887 | idle->thread.sp = (unsigned long)task_pt_regs(idle); |
3e970473 | 888 | initial_code = (unsigned long)start_secondary; |
3adee777 BG |
889 | |
890 | if (IS_ENABLED(CONFIG_X86_32)) { | |
c253b640 | 891 | early_gdt_descr.address = (unsigned long)get_cpu_gdt_rw(cpu); |
3adee777 | 892 | initial_stack = idle->thread.sp; |
7e75178a | 893 | } else if (!(smpboot_control & STARTUP_PARALLEL_MASK)) { |
3adee777 BG |
894 | smpboot_control = cpu; |
895 | } | |
cb3c8b90 | 896 | |
613e396b | 897 | /* Enable the espfix hack for this CPU */ |
20d5e4a9 | 898 | init_espfix_ap(cpu); |
20d5e4a9 | 899 | |
2eaad1fd MT |
900 | /* So we see what's up */ |
901 | announce_cpu(cpu, apicid); | |
cb3c8b90 GOC |
902 | |
903 | /* | |
904 | * This grunge runs the startup process for | |
905 | * the targeted processor. | |
906 | */ | |
e348caef | 907 | if (x86_platform.legacy.warm_reset) { |
cb3c8b90 | 908 | |
cfc1b9a6 | 909 | pr_debug("Setting warm reset code and vector.\n"); |
cb3c8b90 | 910 | |
34d05591 JS |
911 | smpboot_setup_warm_reset_vector(start_ip); |
912 | /* | |
913 | * Be paranoid about clearing APIC errors. | |
db96b0a0 | 914 | */ |
cff9ab2b | 915 | if (APIC_INTEGRATED(boot_cpu_apic_version)) { |
db96b0a0 CG |
916 | apic_write(APIC_ESR, 0); |
917 | apic_read(APIC_ESR); | |
918 | } | |
34d05591 | 919 | } |
cb3c8b90 | 920 | |
ce4b1b16 IM |
921 | smp_mb(); |
922 | ||
cb3c8b90 | 923 | /* |
e1c467e6 | 924 | * Wake up a CPU in difference cases: |
ff2e6468 SC |
925 | * - Use a method from the APIC driver if one defined, with wakeup |
926 | * straight to 64-bit mode preferred over wakeup to RM. | |
e1c467e6 | 927 | * Otherwise, |
5475abbd | 928 | * - Use an INIT boot APIC message |
cb3c8b90 | 929 | */ |
ff2e6468 | 930 | if (apic->wakeup_secondary_cpu_64) |
2711b8e2 | 931 | ret = apic->wakeup_secondary_cpu_64(apicid, start_ip); |
ff2e6468 | 932 | else if (apic->wakeup_secondary_cpu) |
2711b8e2 | 933 | ret = apic->wakeup_secondary_cpu(apicid, start_ip); |
1f5bcabf | 934 | else |
2711b8e2 | 935 | ret = wakeup_secondary_cpu_via_init(apicid, start_ip); |
e1c467e6 | 936 | |
2711b8e2 TG |
937 | /* If the wakeup mechanism failed, cleanup the warm reset vector */ |
938 | if (ret) | |
939 | arch_cpuhp_cleanup_kick_cpu(cpu); | |
940 | return ret; | |
cb3c8b90 GOC |
941 | } |
942 | ||
8b5a0f95 | 943 | int native_kick_ap(unsigned int cpu, struct task_struct *tidle) |
cb3c8b90 | 944 | { |
8aa2a417 | 945 | u32 apicid = apic->cpu_present_to_apicid(cpu); |
5475abbd | 946 | int err; |
cb3c8b90 | 947 | |
7a10e2a9 | 948 | lockdep_assert_irqs_enabled(); |
cb3c8b90 | 949 | |
cfc1b9a6 | 950 | pr_debug("++++++++++++++++++++=_---CPU UP %u\n", cpu); |
cb3c8b90 | 951 | |
6055f6cf TG |
952 | if (apicid == BAD_APICID || !apic_id_valid(apicid)) { |
953 | pr_err("CPU %u has invalid APIC ID %x. Aborting bringup\n", cpu, apicid); | |
954 | return -EINVAL; | |
955 | } | |
956 | ||
957 | if (!test_bit(apicid, phys_cpu_present_map)) { | |
958 | pr_err("CPU %u APIC ID %x is not present. Aborting bringup\n", cpu, apicid); | |
cb3c8b90 GOC |
959 | return -EINVAL; |
960 | } | |
961 | ||
cb3c8b90 GOC |
962 | /* |
963 | * Save current MTRR state in case it was changed since early boot | |
964 | * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync: | |
965 | */ | |
966 | mtrr_save_state(); | |
967 | ||
644c1541 | 968 | /* the FPU context is blank, nobody can own it */ |
317b622c | 969 | per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL; |
644c1541 | 970 | |
66c7ceb4 TG |
971 | err = common_cpu_up(cpu, tidle); |
972 | if (err) | |
973 | return err; | |
3f85483b | 974 | |
5475abbd | 975 | err = do_boot_cpu(apicid, cpu, tidle); |
2b3be65d | 976 | if (err) |
feef1e8e | 977 | pr_err("do_boot_cpu failed(%d) to wakeup CPU#%u\n", err, cpu); |
cb3c8b90 | 978 | |
2b3be65d DW |
979 | return err; |
980 | } | |
cb3c8b90 | 981 | |
8b5a0f95 | 982 | int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle) |
2b3be65d | 983 | { |
8b5a0f95 | 984 | return smp_ops.kick_ap_alive(cpu, tidle); |
2711b8e2 | 985 | } |
cb3c8b90 | 986 | |
2711b8e2 TG |
987 | void arch_cpuhp_cleanup_kick_cpu(unsigned int cpu) |
988 | { | |
2b3be65d | 989 | /* Cleanup possible dangling ends... */ |
8b5a0f95 | 990 | if (smp_ops.kick_ap_alive == native_kick_ap && x86_platform.legacy.warm_reset) |
2b3be65d | 991 | smpboot_restore_warm_reset_vector(); |
2711b8e2 | 992 | } |
10e66760 | 993 | |
2711b8e2 TG |
994 | void arch_cpuhp_cleanup_dead_cpu(unsigned int cpu) |
995 | { | |
996 | if (smp_ops.cleanup_dead_cpu) | |
997 | smp_ops.cleanup_dead_cpu(cpu); | |
998 | ||
999 | if (system_state == SYSTEM_RUNNING) | |
1000 | pr_info("CPU %u is now offline\n", cpu); | |
1001 | } | |
1002 | ||
1003 | void arch_cpuhp_sync_state_poll(void) | |
1004 | { | |
1005 | if (smp_ops.poll_sync_state) | |
1006 | smp_ops.poll_sync_state(); | |
cb3c8b90 GOC |
1007 | } |
1008 | ||
7167d08e | 1009 | /** |
ba831b7b | 1010 | * arch_disable_smp_support() - Disables SMP support for x86 at boottime |
7167d08e | 1011 | */ |
ba831b7b | 1012 | void __init arch_disable_smp_support(void) |
7167d08e HK |
1013 | { |
1014 | disable_ioapic_support(); | |
1015 | } | |
1016 | ||
8aef135c GOC |
1017 | /* |
1018 | * Fall back to non SMP mode after errors. | |
1019 | * | |
1020 | * RED-PEN audit/test this more. I bet there is more state messed up here. | |
1021 | */ | |
1022 | static __init void disable_smp(void) | |
1023 | { | |
613c25ef TG |
1024 | pr_info("SMP disabled\n"); |
1025 | ||
ef4c59a4 | 1026 | disable_ioapic_support(); |
7c0edad3 | 1027 | topology_reset_possible_cpus_up(); |
350b5e27 | 1028 | |
7d79a7bd BG |
1029 | cpumask_set_cpu(0, topology_sibling_cpumask(0)); |
1030 | cpumask_set_cpu(0, topology_core_cpumask(0)); | |
2e4c54da | 1031 | cpumask_set_cpu(0, topology_die_cpumask(0)); |
8aef135c GOC |
1032 | } |
1033 | ||
ce2612b6 | 1034 | void __init smp_prepare_cpus_common(void) |
8aef135c | 1035 | { |
dbbe13a6 | 1036 | unsigned int cpu, node; |
7ad728f9 | 1037 | |
c90399fb | 1038 | /* Mark all except the boot CPU as hotpluggable */ |
dbbe13a6 IM |
1039 | for_each_possible_cpu(cpu) { |
1040 | if (cpu) | |
1041 | per_cpu(cpu_info.cpu_index, cpu) = nr_cpu_ids; | |
c90399fb | 1042 | } |
bd22a2f1 | 1043 | |
dbbe13a6 IM |
1044 | for_each_possible_cpu(cpu) { |
1045 | node = cpu_to_node(cpu); | |
1046 | ||
1047 | zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu), GFP_KERNEL, node); | |
1048 | zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu), GFP_KERNEL, node); | |
1049 | zalloc_cpumask_var_node(&per_cpu(cpu_die_map, cpu), GFP_KERNEL, node); | |
1050 | zalloc_cpumask_var_node(&per_cpu(cpu_llc_shared_map, cpu), GFP_KERNEL, node); | |
1051 | zalloc_cpumask_var_node(&per_cpu(cpu_l2c_shared_map, cpu), GFP_KERNEL, node); | |
7ad728f9 | 1052 | } |
8f37961c | 1053 | |
8aef135c | 1054 | set_cpu_sibling_map(0); |
ce2612b6 BO |
1055 | } |
1056 | ||
71261072 TG |
1057 | void __init smp_prepare_boot_cpu(void) |
1058 | { | |
1059 | smp_ops.smp_prepare_boot_cpu(); | |
1060 | } | |
1061 | ||
0c7ffa32 TG |
1062 | #ifdef CONFIG_X86_64 |
1063 | /* Establish whether parallel bringup can be supported. */ | |
1064 | bool __init arch_cpuhp_init_parallel_bringup(void) | |
1065 | { | |
ff3cfcb0 TG |
1066 | if (!x86_cpuinit.parallel_bringup) { |
1067 | pr_info("Parallel CPU startup disabled by the platform\n"); | |
0c7ffa32 TG |
1068 | return false; |
1069 | } | |
1070 | ||
1071 | smpboot_control = STARTUP_READ_APICID; | |
1072 | pr_debug("Parallel CPU startup enabled: 0x%08x\n", smpboot_control); | |
1073 | return true; | |
1074 | } | |
1075 | #endif | |
1076 | ||
ce2612b6 BO |
1077 | /* |
1078 | * Prepare for SMP bootup. | |
1079 | * @max_cpus: configured maximum number of CPUs, It is a legacy parameter | |
1080 | * for common interface support. | |
1081 | */ | |
1082 | void __init native_smp_prepare_cpus(unsigned int max_cpus) | |
1083 | { | |
1084 | smp_prepare_cpus_common(); | |
1085 | ||
4f45ed9f DL |
1086 | switch (apic_intr_mode) { |
1087 | case APIC_PIC: | |
1088 | case APIC_VIRTUAL_WIRE_NO_CONFIG: | |
613c25ef TG |
1089 | disable_smp(); |
1090 | return; | |
4f45ed9f | 1091 | case APIC_SYMMETRIC_IO_NO_ROUTING: |
613c25ef | 1092 | disable_smp(); |
a2510d15 DL |
1093 | /* Setup local timer */ |
1094 | x86_init.timers.setup_percpu_clockev(); | |
250a1ac6 | 1095 | return; |
4f45ed9f DL |
1096 | case APIC_VIRTUAL_WIRE: |
1097 | case APIC_SYMMETRIC_IO: | |
613c25ef | 1098 | break; |
8aef135c GOC |
1099 | } |
1100 | ||
a2510d15 DL |
1101 | /* Setup local timer */ |
1102 | x86_init.timers.setup_percpu_clockev(); | |
8aef135c | 1103 | |
d54ff31d | 1104 | pr_info("CPU0: "); |
8aef135c | 1105 | print_cpu_info(&cpu_data(0)); |
c4bd1fda | 1106 | |
9ec808a0 | 1107 | uv_system_init(); |
d0af9eed | 1108 | |
1a744cb3 | 1109 | smp_quirk_init_udelay(); |
1f50ddb4 TG |
1110 | |
1111 | speculative_store_bypass_ht_init(); | |
0afb6b66 TL |
1112 | |
1113 | snp_set_wakeup_secondary_cpu(); | |
8aef135c | 1114 | } |
d0af9eed | 1115 | |
56555855 | 1116 | void arch_thaw_secondary_cpus_begin(void) |
d0af9eed | 1117 | { |
955d0e08 | 1118 | set_cache_aps_delayed_init(true); |
d0af9eed SS |
1119 | } |
1120 | ||
56555855 | 1121 | void arch_thaw_secondary_cpus_end(void) |
d0af9eed | 1122 | { |
0b9a6a8b | 1123 | cache_aps_init(); |
d0af9eed SS |
1124 | } |
1125 | ||
a8db8453 GOC |
1126 | /* |
1127 | * Early setup to make printk work. | |
1128 | */ | |
1129 | void __init native_smp_prepare_boot_cpu(void) | |
1130 | { | |
1131 | int me = smp_processor_id(); | |
1f19e2d5 TG |
1132 | |
1133 | /* SMP handles this from setup_per_cpu_areas() */ | |
1134 | if (!IS_ENABLED(CONFIG_SMP)) | |
1135 | switch_gdt_and_percpu_base(me); | |
1136 | ||
090d54bc | 1137 | native_pv_lock_init(); |
a8db8453 GOC |
1138 | } |
1139 | ||
63e708f8 PB |
1140 | void __init native_smp_cpus_done(unsigned int max_cpus) |
1141 | { | |
1142 | pr_debug("Boot done\n"); | |
1143 | ||
8f2d6c41 | 1144 | build_sched_topology(); |
99e8b9ca | 1145 | nmi_selftest(); |
83f7eb9c | 1146 | impress_friends(); |
0b9a6a8b | 1147 | cache_aps_init(); |
83f7eb9c GOC |
1148 | } |
1149 | ||
d4f28f07 TG |
1150 | /* correctly size the local cpu masks */ |
1151 | void __init setup_cpu_local_masks(void) | |
1152 | { | |
d4f28f07 TG |
1153 | alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask); |
1154 | } | |
1155 | ||
14adf855 CE |
1156 | #ifdef CONFIG_HOTPLUG_CPU |
1157 | ||
70b8301f AK |
1158 | /* Recompute SMT state for all CPUs on offline */ |
1159 | static void recompute_smt_state(void) | |
1160 | { | |
1161 | int max_threads, cpu; | |
1162 | ||
1163 | max_threads = 0; | |
1164 | for_each_online_cpu (cpu) { | |
1165 | int threads = cpumask_weight(topology_sibling_cpumask(cpu)); | |
1166 | ||
1167 | if (threads > max_threads) | |
1168 | max_threads = threads; | |
1169 | } | |
1170 | __max_smt_threads = max_threads; | |
1171 | } | |
1172 | ||
14adf855 CE |
1173 | static void remove_siblinginfo(int cpu) |
1174 | { | |
1175 | int sibling; | |
1176 | struct cpuinfo_x86 *c = &cpu_data(cpu); | |
1177 | ||
7d79a7bd BG |
1178 | for_each_cpu(sibling, topology_core_cpumask(cpu)) { |
1179 | cpumask_clear_cpu(cpu, topology_core_cpumask(sibling)); | |
14adf855 CE |
1180 | /*/ |
1181 | * last thread sibling in this cpu core going down | |
1182 | */ | |
7d79a7bd | 1183 | if (cpumask_weight(topology_sibling_cpumask(cpu)) == 1) |
14adf855 CE |
1184 | cpu_data(sibling).booted_cores--; |
1185 | } | |
1186 | ||
2e4c54da LB |
1187 | for_each_cpu(sibling, topology_die_cpumask(cpu)) |
1188 | cpumask_clear_cpu(cpu, topology_die_cpumask(sibling)); | |
c52787b5 BS |
1189 | |
1190 | for_each_cpu(sibling, topology_sibling_cpumask(cpu)) { | |
7d79a7bd | 1191 | cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling)); |
c52787b5 BS |
1192 | if (cpumask_weight(topology_sibling_cpumask(sibling)) == 1) |
1193 | cpu_data(sibling).smt_active = false; | |
1194 | } | |
1195 | ||
03bd4e1f WL |
1196 | for_each_cpu(sibling, cpu_llc_shared_mask(cpu)) |
1197 | cpumask_clear_cpu(cpu, cpu_llc_shared_mask(sibling)); | |
66558b73 TC |
1198 | for_each_cpu(sibling, cpu_l2c_shared_mask(cpu)) |
1199 | cpumask_clear_cpu(cpu, cpu_l2c_shared_mask(sibling)); | |
03bd4e1f | 1200 | cpumask_clear(cpu_llc_shared_mask(cpu)); |
66558b73 | 1201 | cpumask_clear(cpu_l2c_shared_mask(cpu)); |
7d79a7bd BG |
1202 | cpumask_clear(topology_sibling_cpumask(cpu)); |
1203 | cpumask_clear(topology_core_cpumask(cpu)); | |
2e4c54da | 1204 | cpumask_clear(topology_die_cpumask(cpu)); |
e9525633 | 1205 | c->topo.core_id = 0; |
45967493 | 1206 | c->booted_cores = 0; |
c2d1cec1 | 1207 | cpumask_clear_cpu(cpu, cpu_sibling_setup_mask); |
70b8301f | 1208 | recompute_smt_state(); |
14adf855 CE |
1209 | } |
1210 | ||
4daa832d | 1211 | static void remove_cpu_from_maps(int cpu) |
69c18c15 | 1212 | { |
c2d1cec1 | 1213 | set_cpu_online(cpu, false); |
23ca4bba | 1214 | numa_remove_cpu(cpu); |
69c18c15 GC |
1215 | } |
1216 | ||
8227dce7 | 1217 | void cpu_disable_common(void) |
69c18c15 GC |
1218 | { |
1219 | int cpu = smp_processor_id(); | |
69c18c15 | 1220 | |
69c18c15 GC |
1221 | remove_siblinginfo(cpu); |
1222 | ||
1223 | /* It's now safe to remove this processor from the online map */ | |
d388e5fd | 1224 | lock_vector_lock(); |
69c18c15 | 1225 | remove_cpu_from_maps(cpu); |
d388e5fd | 1226 | unlock_vector_lock(); |
d7b381bb | 1227 | fixup_irqs(); |
0fa115da | 1228 | lapic_offline(); |
8227dce7 AN |
1229 | } |
1230 | ||
1231 | int native_cpu_disable(void) | |
1232 | { | |
da6139e4 PB |
1233 | int ret; |
1234 | ||
2cffad7b | 1235 | ret = lapic_can_unplug_cpu(); |
da6139e4 PB |
1236 | if (ret) |
1237 | return ret; | |
1238 | ||
8227dce7 | 1239 | cpu_disable_common(); |
2ed53c0d | 1240 | |
52d6b926 AR |
1241 | /* |
1242 | * Disable the local APIC. Otherwise IPI broadcasts will reach | |
1243 | * it. It still responds normally to INIT, NMI, SMI, and SIPI | |
1244 | * messages. | |
1245 | * | |
1246 | * Disabling the APIC must happen after cpu_disable_common() | |
1247 | * which invokes fixup_irqs(). | |
1248 | * | |
1249 | * Disabling the APIC preserves already set bits in IRR, but | |
1250 | * an interrupt arriving after disabling the local APIC does not | |
1251 | * set the corresponding IRR bit. | |
1252 | * | |
1253 | * fixup_irqs() scans IRR for set bits so it can raise a not | |
1254 | * yet handled interrupt on the new destination CPU via an IPI | |
1255 | * but obviously it can't do so for IRR bits which are not set. | |
1256 | * IOW, interrupts arriving after disabling the local APIC will | |
1257 | * be lost. | |
1258 | */ | |
1259 | apic_soft_disable(); | |
1260 | ||
69c18c15 GC |
1261 | return 0; |
1262 | } | |
1263 | ||
a21f5d88 AN |
1264 | void play_dead_common(void) |
1265 | { | |
1266 | idle_task_exit(); | |
a21f5d88 | 1267 | |
2711b8e2 | 1268 | cpuhp_ap_report_dead(); |
52defa4a | 1269 | |
a21f5d88 AN |
1270 | local_irq_disable(); |
1271 | } | |
1272 | ||
ea530692 PA |
1273 | /* |
1274 | * We need to flush the caches before going to sleep, lest we have | |
1275 | * dirty data in our caches when we come back up. | |
1276 | */ | |
1277 | static inline void mwait_play_dead(void) | |
1278 | { | |
f9c9987b | 1279 | struct mwait_cpu_dead *md = this_cpu_ptr(&mwait_cpu_dead); |
ea530692 PA |
1280 | unsigned int eax, ebx, ecx, edx; |
1281 | unsigned int highest_cstate = 0; | |
1282 | unsigned int highest_subcstate = 0; | |
576cfb40 | 1283 | int i; |
ea530692 | 1284 | |
0b13bec7 PW |
1285 | if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || |
1286 | boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) | |
da6fa7ef | 1287 | return; |
69fb3676 | 1288 | if (!this_cpu_has(X86_FEATURE_MWAIT)) |
ea530692 | 1289 | return; |
840d2830 | 1290 | if (!this_cpu_has(X86_FEATURE_CLFLUSH)) |
ce5f6824 | 1291 | return; |
7b543a53 | 1292 | if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF) |
ea530692 PA |
1293 | return; |
1294 | ||
1295 | eax = CPUID_MWAIT_LEAF; | |
1296 | ecx = 0; | |
1297 | native_cpuid(&eax, &ebx, &ecx, &edx); | |
1298 | ||
1299 | /* | |
1300 | * eax will be 0 if EDX enumeration is not valid. | |
1301 | * Initialized below to cstate, sub_cstate value when EDX is valid. | |
1302 | */ | |
1303 | if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) { | |
1304 | eax = 0; | |
1305 | } else { | |
1306 | edx >>= MWAIT_SUBSTATE_SIZE; | |
1307 | for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) { | |
1308 | if (edx & MWAIT_SUBSTATE_MASK) { | |
1309 | highest_cstate = i; | |
1310 | highest_subcstate = edx & MWAIT_SUBSTATE_MASK; | |
1311 | } | |
1312 | } | |
1313 | eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) | | |
1314 | (highest_subcstate - 1); | |
1315 | } | |
1316 | ||
d7893093 TG |
1317 | /* Set up state for the kexec() hack below */ |
1318 | md->status = CPUDEAD_MWAIT_WAIT; | |
1319 | md->control = CPUDEAD_MWAIT_WAIT; | |
ce5f6824 | 1320 | |
a68e5c94 PA |
1321 | wbinvd(); |
1322 | ||
ea530692 | 1323 | while (1) { |
ce5f6824 PA |
1324 | /* |
1325 | * The CLFLUSH is a workaround for erratum AAI65 for | |
1326 | * the Xeon 7400 series. It's not clear it is actually | |
1327 | * needed, but it should be harmless in either case. | |
1328 | * The WBINVD is insufficient due to the spurious-wakeup | |
1329 | * case where we return around the loop. | |
1330 | */ | |
7d590cca | 1331 | mb(); |
f9c9987b | 1332 | clflush(md); |
7d590cca | 1333 | mb(); |
f9c9987b | 1334 | __monitor(md, 0, 0); |
ea530692 PA |
1335 | mb(); |
1336 | __mwait(eax, 0); | |
fa26d0c7 | 1337 | |
d7893093 TG |
1338 | if (READ_ONCE(md->control) == CPUDEAD_MWAIT_KEXEC_HLT) { |
1339 | /* | |
1340 | * Kexec is about to happen. Don't go back into mwait() as | |
1341 | * the kexec kernel might overwrite text and data including | |
1342 | * page tables and stack. So mwait() would resume when the | |
1343 | * monitor cache line is written to and then the CPU goes | |
1344 | * south due to overwritten text, page tables and stack. | |
1345 | * | |
1346 | * Note: This does _NOT_ protect against a stray MCE, NMI, | |
1347 | * SMI. They will resume execution at the instruction | |
1348 | * following the HLT instruction and run into the problem | |
1349 | * which this is trying to prevent. | |
1350 | */ | |
1351 | WRITE_ONCE(md->status, CPUDEAD_MWAIT_KEXEC_HLT); | |
1352 | while(1) | |
1353 | native_halt(); | |
1354 | } | |
ea530692 PA |
1355 | } |
1356 | } | |
1357 | ||
d7893093 TG |
1358 | /* |
1359 | * Kick all "offline" CPUs out of mwait on kexec(). See comment in | |
1360 | * mwait_play_dead(). | |
1361 | */ | |
1362 | void smp_kick_mwait_play_dead(void) | |
1363 | { | |
1364 | u32 newstate = CPUDEAD_MWAIT_KEXEC_HLT; | |
1365 | struct mwait_cpu_dead *md; | |
1366 | unsigned int cpu, i; | |
1367 | ||
1368 | for_each_cpu_andnot(cpu, cpu_present_mask, cpu_online_mask) { | |
1369 | md = per_cpu_ptr(&mwait_cpu_dead, cpu); | |
1370 | ||
1371 | /* Does it sit in mwait_play_dead() ? */ | |
1372 | if (READ_ONCE(md->status) != CPUDEAD_MWAIT_WAIT) | |
1373 | continue; | |
1374 | ||
1375 | /* Wait up to 5ms */ | |
1376 | for (i = 0; READ_ONCE(md->status) != newstate && i < 1000; i++) { | |
1377 | /* Bring it out of mwait */ | |
1378 | WRITE_ONCE(md->control, newstate); | |
1379 | udelay(5); | |
1380 | } | |
1381 | ||
1382 | if (READ_ONCE(md->status) != newstate) | |
1383 | pr_err_once("CPU%u is stuck in mwait_play_dead()\n", cpu); | |
ea530692 PA |
1384 | } |
1385 | } | |
1386 | ||
52668bad | 1387 | void __noreturn hlt_play_dead(void) |
ea530692 | 1388 | { |
7b543a53 | 1389 | if (__this_cpu_read(cpu_info.x86) >= 4) |
a68e5c94 PA |
1390 | wbinvd(); |
1391 | ||
5475abbd | 1392 | while (1) |
ea530692 | 1393 | native_halt(); |
ea530692 PA |
1394 | } |
1395 | ||
2743fe89 WL |
1396 | /* |
1397 | * native_play_dead() is essentially a __noreturn function, but it can't | |
1398 | * be marked as such as the compiler may complain about it. | |
1399 | */ | |
a21f5d88 AN |
1400 | void native_play_dead(void) |
1401 | { | |
2743fe89 WL |
1402 | if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS)) |
1403 | __update_spec_ctrl(0); | |
1404 | ||
a21f5d88 | 1405 | play_dead_common(); |
86886e55 | 1406 | tboot_shutdown(TB_SHUTDOWN_WFS); |
ea530692 | 1407 | |
fcb3a81d | 1408 | mwait_play_dead(); |
1a022e3f BO |
1409 | if (cpuidle_play_dead()) |
1410 | hlt_play_dead(); | |
a21f5d88 AN |
1411 | } |
1412 | ||
69c18c15 | 1413 | #else /* ... !CONFIG_HOTPLUG_CPU */ |
93be71b6 | 1414 | int native_cpu_disable(void) |
69c18c15 GC |
1415 | { |
1416 | return -ENOSYS; | |
1417 | } | |
1418 | ||
a21f5d88 AN |
1419 | void native_play_dead(void) |
1420 | { | |
1421 | BUG(); | |
1422 | } | |
1423 | ||
68a1c3f8 | 1424 | #endif |