x86/cpu: Take NUMA node into account when allocating per-CPU cpumasks

per-CPU cpumasks are dominantly accessed from their own local CPUs,
so allocate them node-local to improve performance.

[ mingo: Rewrote the changelog. ]

Signed-off-by: Li RongQing <lirongqing@baidu.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20240410030114.6201-1-lirongqing@baidu.com

diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index 567dbd2fe4b6978c0ffda146aa59e957bfb43919..afbb885ce2904eb6745573ebdb5164749abe5f5b 100644 (file)
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -183,7 +183,8 @@ static int x2apic_prepare_cpu(unsigned int cpu)
 
        if (alloc_clustermask(cpu, cluster, cpu_to_node(cpu)) < 0)
                return -ENOMEM;
-       if (!zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL))
+       if (!zalloc_cpumask_var_node(&per_cpu(ipi_mask, cpu), GFP_KERNEL,
+                               cpu_to_node(cpu)))
                return -ENOMEM;
        return 0;
 }

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 76bb65045c649a2c7d849c8a38c9e5d4ea92140d..536dad1440369d9e49790f331f23a931d43f8692 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1033,7 +1033,7 @@ static __init void disable_smp(void)
 
 void __init smp_prepare_cpus_common(void)
 {
-       unsigned int i;
+       unsigned int i, n;
 
        /* Mark all except the boot CPU as hotpluggable */
        for_each_possible_cpu(i) {
@@ -1042,11 +1042,12 @@ void __init smp_prepare_cpus_common(void)
        }
 
        for_each_possible_cpu(i) {
-               zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
-               zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
-               zalloc_cpumask_var(&per_cpu(cpu_die_map, i), GFP_KERNEL);
-               zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
-               zalloc_cpumask_var(&per_cpu(cpu_l2c_shared_map, i), GFP_KERNEL);
+               n = cpu_to_node(i);
+               zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, i), GFP_KERNEL, n);
+               zalloc_cpumask_var_node(&per_cpu(cpu_core_map, i), GFP_KERNEL, n);
+               zalloc_cpumask_var_node(&per_cpu(cpu_die_map, i), GFP_KERNEL, n);
+               zalloc_cpumask_var_node(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL, n);
+               zalloc_cpumask_var_node(&per_cpu(cpu_l2c_shared_map, i), GFP_KERNEL, n);
        }
 
        set_cpu_sibling_map(0);