arm64: scs: Store absolute SCS stack pointer value in thread_info

Storing the SCS information in thread_info as a {base,offset} pair
introduces an additional load instruction on the ret-to-user path,
since the SCS stack pointer in x18 has to be converted back to an offset
by subtracting the base.

Replace the offset with the absolute SCS stack pointer value instead
and avoid the redundant load.

Tested-by: Sami Tolvanen <samitolvanen@google.com>
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h
index 96549353b0cbdc41dfbb1619af720e3651c58f7b..6b8cf4352fe3d6f80127f65729431090e9bb39b9 100644 (file)
--- a/arch/arm64/include/asm/scs.h
+++ b/arch/arm64/include/asm/scs.h
@@ -4,16 +4,15 @@
 
 #ifdef __ASSEMBLY__
 
+#include <asm/asm-offsets.h>
+
 #ifdef CONFIG_SHADOW_CALL_STACK
        .macro scs_load tsk, tmp
-       ldp     x18, \tmp, [\tsk, #TSK_TI_SCS_BASE]
-       add     x18, x18, \tmp
+       ldr     x18, [\tsk, #TSK_TI_SCS_SP]
        .endm
 
        .macro scs_save tsk, tmp
-       ldr     \tmp, [\tsk, #TSK_TI_SCS_BASE]
-       sub     \tmp, x18, \tmp
-       str     \tmp, [\tsk, #TSK_TI_SCS_OFFSET]
+       str     x18, [\tsk, #TSK_TI_SCS_SP]
        .endm
 #else
        .macro scs_load tsk, tmp

diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 9df79c0a4c4368d4b6df44bb3a90893101638367..6ea8b6a26ae9b956bbdeafb36203e63022092867 100644 (file)
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -43,7 +43,7 @@ struct thread_info {
        };
 #ifdef CONFIG_SHADOW_CALL_STACK
        void                    *scs_base;
-       unsigned long           scs_offset;
+       void                    *scs_sp;
 #endif
 };
 
@@ -107,7 +107,7 @@ void arch_release_task_struct(struct task_struct *tsk);
 #ifdef CONFIG_SHADOW_CALL_STACK
 #define INIT_SCS                                                       \
        .scs_base       = init_shadow_call_stack,                       \
-       .scs_offset     = 0,
+       .scs_sp         = init_shadow_call_stack,
 #else
 #define INIT_SCS
 #endif

diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index d7934250b68c1b7451d88de38454283e94c9707a..a098a45f63d828f1f5d65028bd6fa80fefe4ede6 100644 (file)
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -36,7 +36,7 @@ int main(void)
 #endif
 #ifdef CONFIG_SHADOW_CALL_STACK
   DEFINE(TSK_TI_SCS_BASE,      offsetof(struct task_struct, thread_info.scs_base));
-  DEFINE(TSK_TI_SCS_OFFSET,    offsetof(struct task_struct, thread_info.scs_offset));
+  DEFINE(TSK_TI_SCS_SP,                offsetof(struct task_struct, thread_info.scs_sp));
 #endif
   DEFINE(TSK_STACK,            offsetof(struct task_struct, stack));
 #ifdef CONFIG_STACKPROTECTOR

diff --git a/include/linux/scs.h b/include/linux/scs.h
index 3f3662621a27ec9b1060f6284a484ad291616020..0eb2485ef832222f14e94590931d10765d286168 100644 (file)
--- a/include/linux/scs.h
+++ b/include/linux/scs.h
@@ -27,7 +27,7 @@
 #define SCS_END_MAGIC          (0x5f6UL + POISON_POINTER_DELTA)
 
 #define task_scs(tsk)          (task_thread_info(tsk)->scs_base)
-#define task_scs_offset(tsk)   (task_thread_info(tsk)->scs_offset)
+#define task_scs_sp(tsk)       (task_thread_info(tsk)->scs_sp)
 
 void scs_init(void);
 int scs_prepare(struct task_struct *tsk, int node);
@@ -39,7 +39,7 @@ static inline void scs_task_reset(struct task_struct *tsk)
         * Reset the shadow stack to the base address in case the task
         * is reused.
         */
-       task_scs_offset(tsk) = 0;
+       task_scs_sp(tsk) = task_scs(tsk);
 }
 
 static inline unsigned long *__scs_magic(void *s)
@@ -50,9 +50,9 @@ static inline unsigned long *__scs_magic(void *s)
 static inline bool scs_corrupted(struct task_struct *tsk)
 {
        unsigned long *magic = __scs_magic(task_scs(tsk));
+       unsigned long sz = task_scs_sp(tsk) - task_scs(tsk);
 
-       return (task_scs_offset(tsk) >= SCS_SIZE - 1 ||
-               READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC);
+       return sz >= SCS_SIZE - 1 || READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC;
 }
 
 #else /* CONFIG_SHADOW_CALL_STACK */

diff --git a/kernel/scs.c b/kernel/scs.c
index 9389c28f0853457032298615d6021cef97b8544f..5ff8663e4a67663efd40fde0c0605d23a6c2f79d 100644 (file)
--- a/kernel/scs.c
+++ b/kernel/scs.c
@@ -60,8 +60,7 @@ int scs_prepare(struct task_struct *tsk, int node)
        if (!s)
                return -ENOMEM;
 
-       task_scs(tsk) = s;
-       task_scs_offset(tsk) = 0;
+       task_scs(tsk) = task_scs_sp(tsk) = s;
        scs_account(tsk, 1);
        return 0;
 }