x86/fpu: Fix irq_fpu_usable() to return false during CPU onlining

irq_fpu_usable() incorrectly returned true before the FPU is
initialized.  The x86 CPU onlining code can call sha256() to checksum
AMD microcode images, before the FPU is initialized.  Since sha256()
recently gained a kernel-mode FPU optimized code path, a crash occurred
in kernel_fpu_begin_mask() during hotplug CPU onlining.

(The crash did not occur during boot-time CPU onlining, since the
optimized sha256() code is not enabled until subsys_initcalls run.)

Fix this by making irq_fpu_usable() return false before fpu__init_cpu()
has run.  To do this without adding any additional overhead to
irq_fpu_usable(), replace the existing per-CPU bool in_kernel_fpu with
kernel_fpu_allowed which tracks both initialization and usage rather
than just usage.  The initial state is false; FPU initialization sets it
to true; kernel-mode FPU sections toggle it to false and then back to
true; and CPU offlining restores it to the initial state of false.

Fixes: 11d7956d526f ("crypto: x86/sha256 - implement library instead of shash")
Reported-by: Ayush Jain <Ayush.Jain3@amd.com>
Closes: https://lore.kernel.org/r/20250516112217.GBaCcf6Yoc6LkIIryP@fat_crate.local
Signed-off-by: Eric Biggers <ebiggers@google.com>
Tested-by: Ayush Jain <Ayush.Jain3@amd.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h
index f42de5f05e7eb6e3a851cf15506c093ce6a4f6e5..3ebeee2644de45abfe3022761112cd4d5256718c 100644 (file)
--- a/arch/x86/include/asm/fpu/api.h
+++ b/arch/x86/include/asm/fpu/api.h
@@ -126,6 +126,7 @@ static inline void fpstate_init_soft(struct swregs_state *soft) {}
 #endif
 
 /* State tracking */
+DECLARE_PER_CPU(bool, kernel_fpu_allowed);
 DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
 
 /* Process cleanup */

diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index 91d6341f281f80e5dd4653116ce1b3833e2eabfe..399f43aa78d5244ee3bd894da6cace92a11ea33c 100644 (file)
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -43,8 +43,11 @@ struct fpu_state_config fpu_user_cfg __ro_after_init;
  */
 struct fpstate init_fpstate __ro_after_init;
 
-/* Track in-kernel FPU usage */
-static DEFINE_PER_CPU(bool, in_kernel_fpu);
+/*
+ * Track FPU initialization and kernel-mode usage. 'true' means the FPU is
+ * initialized and is not currently being used by the kernel:
+ */
+DEFINE_PER_CPU(bool, kernel_fpu_allowed);
 
 /*
  * Track which context is using the FPU on the CPU:
@@ -61,15 +64,18 @@ bool irq_fpu_usable(void)
                return false;
 
        /*
-        * In kernel FPU usage already active?  This detects any explicitly
-        * nested usage in task or softirq context, which is unsupported.  It
-        * also detects attempted usage in a hardirq that has interrupted a
-        * kernel-mode FPU section.
+        * Return false in the following cases:
+        *
+        * - FPU is not yet initialized. This can happen only when the call is
+        *   coming from CPU onlining, for example for microcode checksumming.
+        * - The kernel is already using the FPU, either because of explicit
+        *   nesting (which should never be done), or because of implicit
+        *   nesting when a hardirq interrupted a kernel-mode FPU section.
+        *
+        * The single boolean check below handles both cases:
         */
-       if (this_cpu_read(in_kernel_fpu)) {
-               WARN_ON_FPU(!in_hardirq());
+       if (!this_cpu_read(kernel_fpu_allowed))
                return false;
-       }
 
        /*
         * When not in NMI or hard interrupt context, FPU can be used in:
@@ -431,9 +437,10 @@ void kernel_fpu_begin_mask(unsigned int kfpu_mask)
                fpregs_lock();
 
        WARN_ON_FPU(!irq_fpu_usable());
-       WARN_ON_FPU(this_cpu_read(in_kernel_fpu));
 
-       this_cpu_write(in_kernel_fpu, true);
+       /* Toggle kernel_fpu_allowed to false: */
+       WARN_ON_FPU(!this_cpu_read(kernel_fpu_allowed));
+       this_cpu_write(kernel_fpu_allowed, false);
 
        if (!(current->flags & (PF_KTHREAD | PF_USER_WORKER)) &&
            !test_thread_flag(TIF_NEED_FPU_LOAD)) {
@@ -453,9 +460,10 @@ EXPORT_SYMBOL_GPL(kernel_fpu_begin_mask);
 
 void kernel_fpu_end(void)
 {
-       WARN_ON_FPU(!this_cpu_read(in_kernel_fpu));
+       /* Toggle kernel_fpu_allowed back to true: */
+       WARN_ON_FPU(this_cpu_read(kernel_fpu_allowed));
+       this_cpu_write(kernel_fpu_allowed, true);
 
-       this_cpu_write(in_kernel_fpu, false);
        if (!irqs_disabled())
                fpregs_unlock();
 }

diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c
index 998a08f17e3317abde0d01961ebfe5d571681e62..1975c37c3668b1e3740fa37ede2d5e1ddc6f8d96 100644 (file)
--- a/arch/x86/kernel/fpu/init.c
+++ b/arch/x86/kernel/fpu/init.c
@@ -51,6 +51,9 @@ void fpu__init_cpu(void)
 {
        fpu__init_cpu_generic();
        fpu__init_cpu_xstate();
+
+       /* Start allowing kernel-mode FPU: */
+       this_cpu_write(kernel_fpu_allowed, true);
 }
 
 static bool __init fpu__probe_without_cpuid(void)

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index d6cf1e23c2a32677a40db13b8d0166b705d0d83f..2901f5cfd8252a2a58e50633840a42aa4e3b4602 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1188,6 +1188,12 @@ void cpu_disable_common(void)
 
        remove_siblinginfo(cpu);
 
+       /*
+        * Stop allowing kernel-mode FPU. This is needed so that if the CPU is
+        * brought online again, the initial state is not allowed:
+        */
+       this_cpu_write(kernel_fpu_allowed, false);
+
        /* It's now safe to remove this processor from the online map */
        lock_vector_lock();
        remove_cpu_from_maps(cpu);