arm64: entry: fix non-NMI kernel<->kernel transitions

There are periods in kernel mode when RCU is not watching and/or the
scheduler tick is disabled, but we can still take exceptions such as
interrupts. The arm64 exception handlers do not account for this, and
it's possible that RCU is not watching while an exception handler runs.

The x86/generic entry code handles this by ensuring that all (non-NMI)
kernel exception handlers call irqentry_enter() and irqentry_exit(),
which handle RCU, lockdep, and IRQ flag tracing. We can't yet move to
the generic entry code, and already hadnle the user<->kernel transitions
elsewhere, so we add new kernel<->kernel transition helpers alog the
lines of the generic entry code.

Since we now track interrupts becoming masked when an exception is
taken, local_daif_inherit() is modified to track interrupts becoming
re-enabled when the original context is inherited. To balance the
entry/exit paths, each handler masks all DAIF exceptions before
exit_to_kernel_mode().

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20201130115950.22492-10-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h
index ec213b4a165021be50b64f47d97b7018fb0326c7..1c26d7baa67f8d1e12a009bd0e22230f511abeea 100644 (file)
--- a/arch/arm64/include/asm/daifflags.h
+++ b/arch/arm64/include/asm/daifflags.h
@@ -128,6 +128,9 @@ static inline void local_daif_inherit(struct pt_regs *regs)
 {
        unsigned long flags = regs->pstate & DAIF_MASK;
 
+       if (interrupts_enabled(regs))
+               trace_hardirqs_on();
+
        /*
         * We can't use local_daif_restore(regs->pstate) here as
         * system_has_prio_mask_debugging() won't restore the I bit if it can

diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 49d1c1dd9bafa3248623c47131a55b803812cf4c..86622d8dd0d841b6df960b3e91045132e1c3c60e 100644 (file)
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -17,12 +17,58 @@
 #include <asm/mmu.h>
 #include <asm/sysreg.h>
 
+/*
+ * This is intended to match the logic in irqentry_enter(), handling the kernel
+ * mode transitions only.
+ */
+static void noinstr enter_from_kernel_mode(struct pt_regs *regs)
+{
+       regs->exit_rcu = false;
+
+       if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) {
+               lockdep_hardirqs_off(CALLER_ADDR0);
+               rcu_irq_enter();
+               trace_hardirqs_off_finish();
+
+               regs->exit_rcu = true;
+               return;
+       }
+
+       lockdep_hardirqs_off(CALLER_ADDR0);
+       rcu_irq_enter_check_tick();
+       trace_hardirqs_off_finish();
+}
+
+/*
+ * This is intended to match the logic in irqentry_exit(), handling the kernel
+ * mode transitions only, and with preemption handled elsewhere.
+ */
+static void noinstr exit_to_kernel_mode(struct pt_regs *regs)
+{
+       lockdep_assert_irqs_disabled();
+
+       if (interrupts_enabled(regs)) {
+               if (regs->exit_rcu) {
+                       trace_hardirqs_on_prepare();
+                       lockdep_hardirqs_on_prepare(CALLER_ADDR0);
+                       rcu_irq_exit();
+                       lockdep_hardirqs_on(CALLER_ADDR0);
+                       return;
+               }
+
+               trace_hardirqs_on();
+       } else {
+               if (regs->exit_rcu)
+                       rcu_irq_exit();
+       }
+}
+
 asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
 {
        if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
                nmi_enter();
-
-       trace_hardirqs_off();
+       else
+               enter_from_kernel_mode(regs);
 }
 
 asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
@@ -30,36 +76,48 @@ asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
        if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
                nmi_exit();
        else
-               trace_hardirqs_on();
+               exit_to_kernel_mode(regs);
 }
 
 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
 {
        unsigned long far = read_sysreg(far_el1);
 
+       enter_from_kernel_mode(regs);
        local_daif_inherit(regs);
        far = untagged_addr(far);
        do_mem_abort(far, esr, regs);
+       local_daif_mask();
+       exit_to_kernel_mode(regs);
 }
 
 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
 {
        unsigned long far = read_sysreg(far_el1);
 
+       enter_from_kernel_mode(regs);
        local_daif_inherit(regs);
        do_sp_pc_abort(far, esr, regs);
+       local_daif_mask();
+       exit_to_kernel_mode(regs);
 }
 
 static void noinstr el1_undef(struct pt_regs *regs)
 {
+       enter_from_kernel_mode(regs);
        local_daif_inherit(regs);
        do_undefinstr(regs);
+       local_daif_mask();
+       exit_to_kernel_mode(regs);
 }
 
 static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)
 {
+       enter_from_kernel_mode(regs);
        local_daif_inherit(regs);
        bad_mode(regs, 0, esr);
+       local_daif_mask();
+       exit_to_kernel_mode(regs);
 }
 
 static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
@@ -79,8 +137,11 @@ static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
 
 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
 {
+       enter_from_kernel_mode(regs);
        local_daif_inherit(regs);
        do_ptrauth_fault(regs, esr);
+       local_daif_mask();
+       exit_to_kernel_mode(regs);
 }
 
 asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)