arm64: Make PMR part of task context

In order to replace PSR.I interrupt disabling/enabling with ICC_PMR_EL1
interrupt masking, ICC_PMR_EL1 needs to be saved/restored when
taking/returning from an exception. This mimics the way hardware saves
and restores PSR.I bit in spsr_el1 for exceptions and ERET.

Add PMR to the registers to save in the pt_regs struct upon kernel entry,
and restore it before ERET. Also, initialize it to a sane value when
creating new tasks.

Signed-off-by: Julien Thierry <julien.thierry@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Dave Martin <Dave.Martin@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index f1a7ab18faf359f4a2b3b36805eb7852519cd3c0..5d9ce62bdebdebfb1c438f9c458cca6dcd4b711c 100644 (file)
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -191,6 +191,9 @@ static inline void start_thread_common(struct pt_regs *regs, unsigned long pc)
        memset(regs, 0, sizeof(*regs));
        forget_syscall(regs);
        regs->pc = pc;
+
+       if (system_uses_irq_prio_masking())
+               regs->pmr_save = GIC_PRIO_IRQON;
 }
 
 static inline void start_thread(struct pt_regs *regs, unsigned long pc,

diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h
index 8b131bc8984d054706a43dbaf59f92792435c587..ec60174c8c18417354fa9bac578b8dde5284f7bd 100644 (file)
--- a/arch/arm64/include/asm/ptrace.h
+++ b/arch/arm64/include/asm/ptrace.h
@@ -19,6 +19,8 @@
 #ifndef __ASM_PTRACE_H
 #define __ASM_PTRACE_H
 
+#include <asm/cpufeature.h>
+
 #include <uapi/asm/ptrace.h>
 
 /* Current Exception Level values, as contained in CurrentEL */
@@ -179,7 +181,8 @@ struct pt_regs {
 #endif
 
        u64 orig_addr_limit;
-       u64 unused;     // maintain 16 byte alignment
+       /* Only valid when ARM64_HAS_IRQ_PRIO_MASKING is enabled. */
+       u64 pmr_save;
        u64 stackframe[2];
 };
 
@@ -214,8 +217,13 @@ static inline void forget_syscall(struct pt_regs *regs)
 #define processor_mode(regs) \
        ((regs)->pstate & PSR_MODE_MASK)
 
-#define interrupts_enabled(regs) \
-       (!((regs)->pstate & PSR_I_BIT))
+#define irqs_priority_unmasked(regs)                                   \
+       (system_uses_irq_prio_masking() ?                               \
+               (regs)->pmr_save == GIC_PRIO_IRQON :                    \
+               true)
+
+#define interrupts_enabled(regs)                       \
+       (!((regs)->pstate & PSR_I_BIT) && irqs_priority_unmasked(regs))
 
 #define fast_interrupts_enabled(regs) \
        (!((regs)->pstate & PSR_F_BIT))

diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 0552b91d7666600d24e1be4451b3e4132eafe981..7f40dcbdd51d0a0d80d56a0b8315dcc2ed419d44 100644 (file)
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -73,6 +73,7 @@ int main(void)
   DEFINE(S_PC,                 offsetof(struct pt_regs, pc));
   DEFINE(S_SYSCALLNO,          offsetof(struct pt_regs, syscallno));
   DEFINE(S_ORIG_ADDR_LIMIT,    offsetof(struct pt_regs, orig_addr_limit));
+  DEFINE(S_PMR_SAVE,           offsetof(struct pt_regs, pmr_save));
   DEFINE(S_STACKFRAME,         offsetof(struct pt_regs, stackframe));
   DEFINE(S_FRAME_SIZE,         sizeof(struct pt_regs));
   BLANK();

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 4d0c81f29a60edf381434d3c1121edc9ed8037e6..02f809a5c8233ba2596b891a148970b5cb442734 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -249,6 +249,12 @@ alternative_else_nop_endif
        msr     sp_el0, tsk
        .endif
 
+       /* Save pmr */
+alternative_if ARM64_HAS_IRQ_PRIO_MASKING
+       mrs_s   x20, SYS_ICC_PMR_EL1
+       str     x20, [sp, #S_PMR_SAVE]
+alternative_else_nop_endif
+
        /*
         * Registers that may be useful after this macro is invoked:
         *
@@ -269,6 +275,14 @@ alternative_else_nop_endif
        /* No need to restore UAO, it will be restored from SPSR_EL1 */
        .endif
 
+       /* Restore pmr */
+alternative_if ARM64_HAS_IRQ_PRIO_MASKING
+       ldr     x20, [sp, #S_PMR_SAVE]
+       msr_s   SYS_ICC_PMR_EL1, x20
+       /* Ensure priority change is seen by redistributor */
+       dsb     sy
+alternative_else_nop_endif
+
        ldp     x21, x22, [sp, #S_PC]           // load ELR, SPSR
        .if     \el == 0
        ct_user_enter

diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index a0f985a6ac5055fbdd311811952d18335259c929..6d410fc2849babbf0746e214af495a996d17318e 100644 (file)
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -232,6 +232,9 @@ void __show_regs(struct pt_regs *regs)
 
        printk("sp : %016llx\n", sp);
 
+       if (system_uses_irq_prio_masking())
+               printk("pmr_save: %08llx\n", regs->pmr_save);
+
        i = top_reg;
 
        while (i >= 0) {
@@ -363,6 +366,9 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start,
                if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE)
                        childregs->pstate |= PSR_SSBS_BIT;
 
+               if (system_uses_irq_prio_masking())
+                       childregs->pmr_save = GIC_PRIO_IRQON;
+
                p->thread.cpu_context.x19 = stack_start;
                p->thread.cpu_context.x20 = stk_sz;
        }