arm64: kpti: Avoid rewriting early page tables when KASLR is enabled

A side effect of commit c55191e96caa ("arm64: mm: apply r/o permissions
of VM areas to its linear alias as well") is that the linear map is
created with page granularity, which means that transitioning the early
page table from global to non-global mappings when enabling kpti can
take a significant amount of time during boot.

Given that most CPU implementations do not require kpti, this mainly
impacts KASLR builds where kpti is forcefully enabled. However, in these
situations we know early on that non-global mappings are required and
can avoid the use of global mappings from the beginning. The only gotcha
is Cavium erratum #27456, which we must detect based on the MIDR value
of the boot CPU.

Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reported-by: John Garry <john.garry@huawei.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 7689c7aa1d77fbf5fdb5bce5d2c454cb192c15c9..ac352accb3d98dd00b32f266ae71b3b9b4cb494b 100644 (file)
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -16,6 +16,8 @@
 #ifndef __ASM_MMU_H
 #define __ASM_MMU_H
 
+#include <asm/cputype.h>
+
 #define MMCF_AARCH32   0x1     /* mm context flag for AArch32 executables */
 #define USER_ASID_BIT  48
 #define USER_ASID_FLAG (UL(1) << USER_ASID_BIT)
@@ -44,6 +46,45 @@ static inline bool arm64_kernel_unmapped_at_el0(void)
               cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0);
 }
 
+static inline bool arm64_kernel_use_ng_mappings(void)
+{
+       bool tx1_bug;
+
+       /* What's a kpti? Use global mappings if we don't know. */
+       if (!IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0))
+               return false;
+
+       /*
+        * Note: this function is called before the CPU capabilities have
+        * been configured, so our early mappings will be global. If we
+        * later determine that kpti is required, then
+        * kpti_install_ng_mappings() will make them non-global.
+        */
+       if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE))
+               return arm64_kernel_unmapped_at_el0();
+
+       /*
+        * KASLR is enabled so we're going to be enabling kpti on non-broken
+        * CPUs regardless of their susceptibility to Meltdown. Rather
+        * than force everybody to go through the G -> nG dance later on,
+        * just put down non-global mappings from the beginning.
+        */
+       if (!IS_ENABLED(CONFIG_CAVIUM_ERRATUM_27456)) {
+               tx1_bug = false;
+#ifndef MODULE
+       } else if (!static_branch_likely(&arm64_const_caps_ready)) {
+               extern const struct midr_range cavium_erratum_27456_cpus[];
+
+               tx1_bug = is_midr_in_range_list(read_cpuid_id(),
+                                               cavium_erratum_27456_cpus);
+#endif
+       } else {
+               tx1_bug = __cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_27456);
+       }
+
+       return !tx1_bug && kaslr_offset() > 0;
+}
+
 typedef void (*bp_hardening_cb_t)(void);
 
 struct bp_hardening_data {

diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h
index 78b942c1bea437c5a0e9124b86afd4c48503d999..986e41c4c32b02bd9dacc4f3eb40d9d3e993fe97 100644 (file)
--- a/arch/arm64/include/asm/pgtable-prot.h
+++ b/arch/arm64/include/asm/pgtable-prot.h
@@ -37,8 +37,8 @@
 #define _PROT_DEFAULT          (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
 #define _PROT_SECT_DEFAULT     (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
 
-#define PTE_MAYBE_NG           (arm64_kernel_unmapped_at_el0() ? PTE_NG : 0)
-#define PMD_MAYBE_NG           (arm64_kernel_unmapped_at_el0() ? PMD_SECT_NG : 0)
+#define PTE_MAYBE_NG           (arm64_kernel_use_ng_mappings() ? PTE_NG : 0)
+#define PMD_MAYBE_NG           (arm64_kernel_use_ng_mappings() ? PMD_SECT_NG : 0)
 
 #define PROT_DEFAULT           (_PROT_DEFAULT | PTE_MAYBE_NG)
 #define PROT_SECT_DEFAULT      (_PROT_SECT_DEFAULT | PMD_MAYBE_NG)

diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 09ac548c9d444d18f07eb796d973b2e7cf44a680..9950bb0cbd52167c6b3b76e9d122291d70264df4 100644 (file)
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -553,7 +553,7 @@ static const struct midr_range arm64_repeat_tlbi_cpus[] = {
 #endif
 
 #ifdef CONFIG_CAVIUM_ERRATUM_27456
-static const struct midr_range cavium_erratum_27456_cpus[] = {
+const struct midr_range cavium_erratum_27456_cpus[] = {
        /* Cavium ThunderX, T88 pass 1.x - 2.1 */
        MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 1),
        /* Cavium ThunderX, T81 pass 1.0 */

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 4f272399de8943a488e17291efd653a6cec6814b..f6d84e2c92fe8bd65055c2c4927e704829e79c4f 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -983,7 +983,7 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
 
        /* Useful for KASLR robustness */
        if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
-               return true;
+               return kaslr_offset() > 0;
 
        /* Don't force KPTI for CPUs that are not vulnerable */
        if (is_midr_in_range_list(read_cpuid_id(), kpti_safe_list))
@@ -1003,7 +1003,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
        static bool kpti_applied = false;
        int cpu = smp_processor_id();
 
-       if (kpti_applied)
+       /*
+        * We don't need to rewrite the page-tables if either we've done
+        * it already or we have KASLR enabled and therefore have not
+        * created any global mappings at all.
+        */
+       if (kpti_applied || kaslr_offset() > 0)
                return;
 
        remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index c7213674cb24cb10c8a685794be9a8d3072cf932..15d79a8e5e5e414350b9f869dad48776767c762b 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -475,6 +475,7 @@ ENDPROC(__primary_switched)
 
 ENTRY(kimage_vaddr)
        .quad           _text - TEXT_OFFSET
+EXPORT_SYMBOL(kimage_vaddr)
 
 /*
  * If we're fortunate enough to boot at EL2, ensure that the world is