powerpc/smp: poll cpu_callin_map more aggressively in __cpu_up()

At boot time, it is not necessary to delay between polls of
cpu_callin_map when waiting for a kicked CPU to come up. Remove the
delay intervals, but preserve the overall deadline (five seconds).

At run time, the first poll result is usually negative and we incur a
sleeping wait. If we spin on the callin word for a short time first,
we can reduce __cpu_up() from dozens of milliseconds to under 1ms in
the common case on a P9 LPAR:

$ ppc64_cpu --smt=off
$ bpftrace -e 'kprobe:__cpu_up {
                 @start[tid] = nsecs;
               }
               kretprobe:__cpu_up /@start[tid]/ {
                 @us = hist((nsecs - @start[tid]) / 1000);
                 delete(@start[tid]);
               }' -c 'ppc64_cpu --smt=on'

Before:

@us:
[16K, 32K)        85 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
[32K, 64K)        13 |@@@@@@@                                             |

After:

@us:
[128, 256)        95 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
[256, 512)         3 |@                                                   |

Signed-off-by: Nathan Lynch <nathanl@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20220926220250.157022-1-nathanl@linux.ibm.com

diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 11ded19186b9103eb8f664304e5e0be4e3d0d2f3..0da6e59161cd494678ebabce5feb8dd72e5fe017 100644 (file)
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -1257,7 +1257,12 @@ static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)
 
 int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 {
-       int rc, c;
+       const unsigned long boot_spin_ms = 5 * MSEC_PER_SEC;
+       const bool booting = system_state < SYSTEM_RUNNING;
+       const unsigned long hp_spin_ms = 1;
+       unsigned long deadline;
+       int rc;
+       const unsigned long spin_wait_ms = booting ? boot_spin_ms : hp_spin_ms;
 
        /*
         * Don't allow secondary threads to come online if inhibited
@@ -1302,22 +1307,23 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
        }
 
        /*
-        * wait to see if the cpu made a callin (is actually up).
-        * use this value that I found through experimentation.
-        * -- Cort
+        * At boot time, simply spin on the callin word until the
+        * deadline passes.
+        *
+        * At run time, spin for an optimistic amount of time to avoid
+        * sleeping in the common case.
         */
-       if (system_state < SYSTEM_RUNNING)
-               for (c = 50000; c && !cpu_callin_map[cpu]; c--)
-                       udelay(100);
-#ifdef CONFIG_HOTPLUG_CPU
-       else
-               /*
-                * CPUs can take much longer to come up in the
-                * hotplug case.  Wait five seconds.
-                */
-               for (c = 5000; c && !cpu_callin_map[cpu]; c--)
-                       msleep(1);
-#endif
+       deadline = jiffies + msecs_to_jiffies(spin_wait_ms);
+       spin_until_cond(cpu_callin_map[cpu] || time_is_before_jiffies(deadline));
+
+       if (!cpu_callin_map[cpu] && system_state >= SYSTEM_RUNNING) {
+               const unsigned long sleep_interval_us = 10 * USEC_PER_MSEC;
+               const unsigned long sleep_wait_ms = 100 * MSEC_PER_SEC;
+
+               deadline = jiffies + msecs_to_jiffies(sleep_wait_ms);
+               while (!cpu_callin_map[cpu] && time_is_after_jiffies(deadline))
+                       fsleep(sleep_interval_us);
+       }
 
        if (!cpu_callin_map[cpu]) {
                printk(KERN_ERR "Processor %u is stuck.\n", cpu);