x86/asm: Move native_write_cr0/4() out of line

[linux-2.6-block.git] / arch / x86 / kernel / cpu / common.c

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 309b6b9b49d4a92d74adae760d852a9293c9f826..11472178e17f5b45f6de5dfb20a8987cbba75a2f 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -366,10 +366,62 @@ out:
        cr4_clear_bits(X86_CR4_UMIP);
 }
 
-DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
-EXPORT_SYMBOL(cr_pinning);
-unsigned long cr4_pinned_bits __ro_after_init;
-EXPORT_SYMBOL(cr4_pinned_bits);
+static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
+static unsigned long cr4_pinned_bits __ro_after_init;
+
+void native_write_cr0(unsigned long val)
+{
+       unsigned long bits_missing = 0;
+
+set_register:
+       asm volatile("mov %0,%%cr0": "+r" (val), "+m" (__force_order));
+
+       if (static_branch_likely(&cr_pinning)) {
+               if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
+                       bits_missing = X86_CR0_WP;
+                       val |= bits_missing;
+                       goto set_register;
+               }
+               /* Warn after we've set the missing bits. */
+               WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
+       }
+}
+EXPORT_SYMBOL(native_write_cr0);
+
+void native_write_cr4(unsigned long val)
+{
+       unsigned long bits_missing = 0;
+
+set_register:
+       asm volatile("mov %0,%%cr4": "+r" (val), "+m" (cr4_pinned_bits));
+
+       if (static_branch_likely(&cr_pinning)) {
+               if (unlikely((val & cr4_pinned_bits) != cr4_pinned_bits)) {
+                       bits_missing = ~val & cr4_pinned_bits;
+                       val |= bits_missing;
+                       goto set_register;
+               }
+               /* Warn after we've set the missing bits. */
+               WARN_ONCE(bits_missing, "CR4 bits went missing: %lx!?\n",
+                         bits_missing);
+       }
+}
+EXPORT_SYMBOL(native_write_cr4);
+
+void cr4_init(void)
+{
+       unsigned long cr4 = __read_cr4();
+
+       if (boot_cpu_has(X86_FEATURE_PCID))
+               cr4 |= X86_CR4_PCIDE;
+       if (static_branch_likely(&cr_pinning))
+               cr4 |= cr4_pinned_bits;
+
+       __write_cr4(cr4);
+
+       /* Initialize cr4 shadow for this CPU. */
+       this_cpu_write(cpu_tlbstate.cr4, cr4);
+}
 
 /*
  * Once CPU feature detection is finished (and boot params have been
@@ -1723,12 +1775,6 @@ void cpu_init(void)
 
        wait_for_master_cpu(cpu);
 
-       /*
-        * Initialize the CR4 shadow before doing anything that could
-        * try to read it.
-        */
-       cr4_init_shadow();
-
        if (cpu)
                load_ucode_ap();
 
@@ -1823,12 +1869,6 @@ void cpu_init(void)
 
        wait_for_master_cpu(cpu);
 
-       /*
-        * Initialize the CR4 shadow before doing anything that could
-        * try to read it.
-        */
-       cr4_init_shadow();
-
        show_ucode_info_early();
 
        pr_info("Initializing CPU#%d\n", cpu);