Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64...

[linux-2.6-block.git] / arch / arm64 / kernel / cpufeature.c

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 0fb6a31514430e4deecfce4c1f852691df3c4476..29b1f873e337fb6a423c3406c02d43af18bc4847 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -856,12 +856,23 @@ static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
 static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
                                int __unused)
 {
+       char const *str = "command line option";
        u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
 
-       /* Forced on command line? */
+       /*
+        * For reasons that aren't entirely clear, enabling KPTI on Cavium
+        * ThunderX leads to apparent I-cache corruption of kernel text, which
+        * ends as well as you might imagine. Don't even try.
+        */
+       if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_27456)) {
+               str = "ARM64_WORKAROUND_CAVIUM_27456";
+               __kpti_forced = -1;
+       }
+
+       /* Forced? */
        if (__kpti_forced) {
-               pr_info_once("kernel page table isolation forced %s by command line option\n",
-                            __kpti_forced > 0 ? "ON" : "OFF");
+               pr_info_once("kernel page table isolation forced %s by %s\n",
+                            __kpti_forced > 0 ? "ON" : "OFF", str);
                return __kpti_forced > 0;
        }
 
@@ -881,6 +892,30 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
                                                     ID_AA64PFR0_CSV3_SHIFT);
 }
 
+static int kpti_install_ng_mappings(void *__unused)
+{
+       typedef void (kpti_remap_fn)(int, int, phys_addr_t);
+       extern kpti_remap_fn idmap_kpti_install_ng_mappings;
+       kpti_remap_fn *remap_fn;
+
+       static bool kpti_applied = false;
+       int cpu = smp_processor_id();
+
+       if (kpti_applied)
+               return 0;
+
+       remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
+
+       cpu_install_idmap();
+       remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));
+       cpu_uninstall_idmap();
+
+       if (!cpu)
+               kpti_applied = true;
+
+       return 0;
+}
+
 static int __init parse_kpti(char *str)
 {
        bool enabled;
@@ -1004,6 +1039,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                .capability = ARM64_UNMAP_KERNEL_AT_EL0,
                .def_scope = SCOPE_SYSTEM,
                .matches = unmap_kernel_at_el0,
+               .enable = kpti_install_ng_mappings,
        },
 #endif
        {