arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/utsname.h>
  13 #include <linux/cpu.h>
  14 #include <linux/module.h>
  15 #include <linux/nospec.h>
  16 #include <linux/prctl.h>
  17 #include <linux/sched/smt.h>
  18
  19 #include <asm/spec-ctrl.h>
  20 #include <asm/cmdline.h>
  21 #include <asm/bugs.h>
  22 #include <asm/processor.h>
  23 #include <asm/processor-flags.h>
  24 #include <asm/fpu/internal.h>
  25 #include <asm/msr.h>
  26 #include <asm/vmx.h>
  27 #include <asm/paravirt.h>
  28 #include <asm/alternative.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/set_memory.h>
  31 #include <asm/intel-family.h>
  32 #include <asm/e820/api.h>
  33 #include <asm/hypervisor.h>
  34
  35 #include "cpu.h"
  36
  37 static void __init spectre_v1_select_mitigation(void);
  38 static void __init spectre_v2_select_mitigation(void);
  39 static void __init ssb_select_mitigation(void);
  40 static void __init l1tf_select_mitigation(void);
  41 static void __init mds_select_mitigation(void);
  42 static void __init taa_select_mitigation(void);
  43
  44 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
  45 u64 x86_spec_ctrl_base;
  46 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  47 static DEFINE_MUTEX(spec_ctrl_mutex);
  48
  49 /*
  50  * The vendor and possibly platform specific bits which can be modified in
  51  * x86_spec_ctrl_base.
  52  */
  53 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  54
  55 /*
  56  * AMD specific MSR info for Speculative Store Bypass control.
  57  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  58  */
  59 u64 __ro_after_init x86_amd_ls_cfg_base;
  60 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  61
  62 /* Control conditional STIBP in switch_to() */
  63 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
  64 /* Control conditional IBPB in switch_mm() */
  65 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
  66 /* Control unconditional IBPB in switch_mm() */
  67 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
  68
  69 /* Control MDS CPU buffer clear before returning to user space */
  70 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
  71 EXPORT_SYMBOL_GPL(mds_user_clear);
  72 /* Control MDS CPU buffer clear before idling (halt, mwait) */
  73 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
  74 EXPORT_SYMBOL_GPL(mds_idle_clear);
  75
  76 void __init check_bugs(void)
  77 {
  78         identify_boot_cpu();
  79
  80         /*
  81          * identify_boot_cpu() initialized SMT support information, let the
  82          * core code know.
  83          */
  84         cpu_smt_check_topology();
  85
  86         if (!IS_ENABLED(CONFIG_SMP)) {
  87                 pr_info("CPU: ");
  88                 print_cpu_info(&boot_cpu_data);
  89         }
  90
  91         /*
  92          * Read the SPEC_CTRL MSR to account for reserved bits which may
  93          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  94          * init code as it is not enumerated and depends on the family.
  95          */
  96         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  97                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  98
  99         /* Allow STIBP in MSR_SPEC_CTRL if supported */
 100         if (boot_cpu_has(X86_FEATURE_STIBP))
 101                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
 102
 103         /* Select the proper CPU mitigations before patching alternatives: */
 104         spectre_v1_select_mitigation();
 105         spectre_v2_select_mitigation();
 106         ssb_select_mitigation();
 107         l1tf_select_mitigation();
 108         mds_select_mitigation();
 109         taa_select_mitigation();
 110
 111         arch_smt_update();
 112
 113 #ifdef CONFIG_X86_32
 114         /*
 115          * Check whether we are able to run this kernel safely on SMP.
 116          *
 117          * - i386 is no longer supported.
 118          * - In order to run on anything without a TSC, we need to be
 119          *   compiled for a i486.
 120          */
 121         if (boot_cpu_data.x86 < 4)
 122                 panic("Kernel requires i486+ for 'invlpg' and other features");
 123
 124         init_utsname()->machine[1] =
 125                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 126         alternative_instructions();
 127
 128         fpu__init_check_bugs();
 129 #else /* CONFIG_X86_64 */
 130         alternative_instructions();
 131
 132         /*
 133          * Make sure the first 2MB area is not mapped by huge pages
 134          * There are typically fixed size MTRRs in there and overlapping
 135          * MTRRs into large pages causes slow downs.
 136          *
 137          * Right now we don't do that with gbpages because there seems
 138          * very little benefit for that case.
 139          */
 140         if (!direct_gbpages)
 141                 set_memory_4k((unsigned long)__va(0), 1);
 142 #endif
 143 }
 144
 145 void
 146 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 147 {
 148         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 149         struct thread_info *ti = current_thread_info();
 150
 151         /* Is MSR_SPEC_CTRL implemented ? */
 152         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 153                 /*
 154                  * Restrict guest_spec_ctrl to supported values. Clear the
 155                  * modifiable bits in the host base value and or the
 156                  * modifiable bits from the guest value.
 157                  */
 158                 guestval = hostval & ~x86_spec_ctrl_mask;
 159                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 160
 161                 /* SSBD controlled in MSR_SPEC_CTRL */
 162                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 163                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 164                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 165
 166                 /* Conditional STIBP enabled? */
 167                 if (static_branch_unlikely(&switch_to_cond_stibp))
 168                         hostval |= stibp_tif_to_spec_ctrl(ti->flags);
 169
 170                 if (hostval != guestval) {
 171                         msrval = setguest ? guestval : hostval;
 172                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 173                 }
 174         }
 175
 176         /*
 177          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 178          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 179          */
 180         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 181             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 182                 return;
 183
 184         /*
 185          * If the host has SSBD mitigation enabled, force it in the host's
 186          * virtual MSR value. If its not permanently enabled, evaluate
 187          * current's TIF_SSBD thread flag.
 188          */
 189         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 190                 hostval = SPEC_CTRL_SSBD;
 191         else
 192                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 193
 194         /* Sanitize the guest value */
 195         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 196
 197         if (hostval != guestval) {
 198                 unsigned long tif;
 199
 200                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 201                                  ssbd_spec_ctrl_to_tif(hostval);
 202
 203                 speculation_ctrl_update(tif);
 204         }
 205 }
 206 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 207
 208 static void x86_amd_ssb_disable(void)
 209 {
 210         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 211
 212         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 213                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 214         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 215                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 216 }
 217
 218 #undef pr_fmt
 219 #define pr_fmt(fmt)     "MDS: " fmt
 220
 221 /* Default mitigation for MDS-affected CPUs */
 222 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
 223 static bool mds_nosmt __ro_after_init = false;
 224
 225 static const char * const mds_strings[] = {
 226         [MDS_MITIGATION_OFF]    = "Vulnerable",
 227         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
 228         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
 229 };
 230
 231 static void __init mds_select_mitigation(void)
 232 {
 233         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
 234                 mds_mitigation = MDS_MITIGATION_OFF;
 235                 return;
 236         }
 237
 238         if (mds_mitigation == MDS_MITIGATION_FULL) {
 239                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
 240                         mds_mitigation = MDS_MITIGATION_VMWERV;
 241
 242                 static_branch_enable(&mds_user_clear);
 243
 244                 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
 245                     (mds_nosmt || cpu_mitigations_auto_nosmt()))
 246                         cpu_smt_disable(false);
 247         }
 248
 249         pr_info("%s\n", mds_strings[mds_mitigation]);
 250 }
 251
 252 static int __init mds_cmdline(char *str)
 253 {
 254         if (!boot_cpu_has_bug(X86_BUG_MDS))
 255                 return 0;
 256
 257         if (!str)
 258                 return -EINVAL;
 259
 260         if (!strcmp(str, "off"))
 261                 mds_mitigation = MDS_MITIGATION_OFF;
 262         else if (!strcmp(str, "full"))
 263                 mds_mitigation = MDS_MITIGATION_FULL;
 264         else if (!strcmp(str, "full,nosmt")) {
 265                 mds_mitigation = MDS_MITIGATION_FULL;
 266                 mds_nosmt = true;
 267         }
 268
 269         return 0;
 270 }
 271 early_param("mds", mds_cmdline);
 272
 273 #undef pr_fmt
 274 #define pr_fmt(fmt)     "TAA: " fmt
 275
 276 /* Default mitigation for TAA-affected CPUs */
 277 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
 278 static bool taa_nosmt __ro_after_init;
 279
 280 static const char * const taa_strings[] = {
 281         [TAA_MITIGATION_OFF]            = "Vulnerable",
 282         [TAA_MITIGATION_UCODE_NEEDED]   = "Vulnerable: Clear CPU buffers attempted, no microcode",
 283         [TAA_MITIGATION_VERW]           = "Mitigation: Clear CPU buffers",
 284         [TAA_MITIGATION_TSX_DISABLED]   = "Mitigation: TSX disabled",
 285 };
 286
 287 static void __init taa_select_mitigation(void)
 288 {
 289         u64 ia32_cap;
 290
 291         if (!boot_cpu_has_bug(X86_BUG_TAA)) {
 292                 taa_mitigation = TAA_MITIGATION_OFF;
 293                 return;
 294         }
 295
 296         /* TSX previously disabled by tsx=off */
 297         if (!boot_cpu_has(X86_FEATURE_RTM)) {
 298                 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
 299                 goto out;
 300         }
 301
 302         if (cpu_mitigations_off()) {
 303                 taa_mitigation = TAA_MITIGATION_OFF;
 304                 return;
 305         }
 306
 307         /* TAA mitigation is turned off on the cmdline (tsx_async_abort=off) */
 308         if (taa_mitigation == TAA_MITIGATION_OFF)
 309                 goto out;
 310
 311         if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
 312                 taa_mitigation = TAA_MITIGATION_VERW;
 313         else
 314                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 315
 316         /*
 317          * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
 318          * A microcode update fixes this behavior to clear CPU buffers. It also
 319          * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
 320          * ARCH_CAP_TSX_CTRL_MSR bit.
 321          *
 322          * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
 323          * update is required.
 324          */
 325         ia32_cap = x86_read_arch_cap_msr();
 326         if ( (ia32_cap & ARCH_CAP_MDS_NO) &&
 327             !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR))
 328                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 329
 330         /*
 331          * TSX is enabled, select alternate mitigation for TAA which is
 332          * the same as MDS. Enable MDS static branch to clear CPU buffers.
 333          *
 334          * For guests that can't determine whether the correct microcode is
 335          * present on host, enable the mitigation for UCODE_NEEDED as well.
 336          */
 337         static_branch_enable(&mds_user_clear);
 338
 339         if (taa_nosmt || cpu_mitigations_auto_nosmt())
 340                 cpu_smt_disable(false);
 341
 342 out:
 343         pr_info("%s\n", taa_strings[taa_mitigation]);
 344 }
 345
 346 static int __init tsx_async_abort_parse_cmdline(char *str)
 347 {
 348         if (!boot_cpu_has_bug(X86_BUG_TAA))
 349                 return 0;
 350
 351         if (!str)
 352                 return -EINVAL;
 353
 354         if (!strcmp(str, "off")) {
 355                 taa_mitigation = TAA_MITIGATION_OFF;
 356         } else if (!strcmp(str, "full")) {
 357                 taa_mitigation = TAA_MITIGATION_VERW;
 358         } else if (!strcmp(str, "full,nosmt")) {
 359                 taa_mitigation = TAA_MITIGATION_VERW;
 360                 taa_nosmt = true;
 361         }
 362
 363         return 0;
 364 }
 365 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
 366
 367 #undef pr_fmt
 368 #define pr_fmt(fmt)     "Spectre V1 : " fmt
 369
 370 enum spectre_v1_mitigation {
 371         SPECTRE_V1_MITIGATION_NONE,
 372         SPECTRE_V1_MITIGATION_AUTO,
 373 };
 374
 375 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
 376         SPECTRE_V1_MITIGATION_AUTO;
 377
 378 static const char * const spectre_v1_strings[] = {
 379         [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
 380         [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
 381 };
 382
 383 /*
 384  * Does SMAP provide full mitigation against speculative kernel access to
 385  * userspace?
 386  */
 387 static bool smap_works_speculatively(void)
 388 {
 389         if (!boot_cpu_has(X86_FEATURE_SMAP))
 390                 return false;
 391
 392         /*
 393          * On CPUs which are vulnerable to Meltdown, SMAP does not
 394          * prevent speculative access to user data in the L1 cache.
 395          * Consider SMAP to be non-functional as a mitigation on these
 396          * CPUs.
 397          */
 398         if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
 399                 return false;
 400
 401         return true;
 402 }
 403
 404 static void __init spectre_v1_select_mitigation(void)
 405 {
 406         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
 407                 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 408                 return;
 409         }
 410
 411         if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
 412                 /*
 413                  * With Spectre v1, a user can speculatively control either
 414                  * path of a conditional swapgs with a user-controlled GS
 415                  * value.  The mitigation is to add lfences to both code paths.
 416                  *
 417                  * If FSGSBASE is enabled, the user can put a kernel address in
 418                  * GS, in which case SMAP provides no protection.
 419                  *
 420                  * [ NOTE: Don't check for X86_FEATURE_FSGSBASE until the
 421                  *         FSGSBASE enablement patches have been merged. ]
 422                  *
 423                  * If FSGSBASE is disabled, the user can only put a user space
 424                  * address in GS.  That makes an attack harder, but still
 425                  * possible if there's no SMAP protection.
 426                  */
 427                 if (!smap_works_speculatively()) {
 428                         /*
 429                          * Mitigation can be provided from SWAPGS itself or
 430                          * PTI as the CR3 write in the Meltdown mitigation
 431                          * is serializing.
 432                          *
 433                          * If neither is there, mitigate with an LFENCE to
 434                          * stop speculation through swapgs.
 435                          */
 436                         if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
 437                             !boot_cpu_has(X86_FEATURE_PTI))
 438                                 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
 439
 440                         /*
 441                          * Enable lfences in the kernel entry (non-swapgs)
 442                          * paths, to prevent user entry from speculatively
 443                          * skipping swapgs.
 444                          */
 445                         setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
 446                 }
 447         }
 448
 449         pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
 450 }
 451
 452 static int __init nospectre_v1_cmdline(char *str)
 453 {
 454         spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 455         return 0;
 456 }
 457 early_param("nospectre_v1", nospectre_v1_cmdline);
 458
 459 #undef pr_fmt
 460 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 461
 462 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 463         SPECTRE_V2_NONE;
 464
 465 static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init =
 466         SPECTRE_V2_USER_NONE;
 467
 468 #ifdef CONFIG_RETPOLINE
 469 static bool spectre_v2_bad_module;
 470
 471 bool retpoline_module_ok(bool has_retpoline)
 472 {
 473         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 474                 return true;
 475
 476         pr_err("System may be vulnerable to spectre v2\n");
 477         spectre_v2_bad_module = true;
 478         return false;
 479 }
 480
 481 static inline const char *spectre_v2_module_string(void)
 482 {
 483         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 484 }
 485 #else
 486 static inline const char *spectre_v2_module_string(void) { return ""; }
 487 #endif
 488
 489 static inline bool match_option(const char *arg, int arglen, const char *opt)
 490 {
 491         int len = strlen(opt);
 492
 493         return len == arglen && !strncmp(arg, opt, len);
 494 }
 495
 496 /* The kernel command line selection for spectre v2 */
 497 enum spectre_v2_mitigation_cmd {
 498         SPECTRE_V2_CMD_NONE,
 499         SPECTRE_V2_CMD_AUTO,
 500         SPECTRE_V2_CMD_FORCE,
 501         SPECTRE_V2_CMD_RETPOLINE,
 502         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 503         SPECTRE_V2_CMD_RETPOLINE_AMD,
 504 };
 505
 506 enum spectre_v2_user_cmd {
 507         SPECTRE_V2_USER_CMD_NONE,
 508         SPECTRE_V2_USER_CMD_AUTO,
 509         SPECTRE_V2_USER_CMD_FORCE,
 510         SPECTRE_V2_USER_CMD_PRCTL,
 511         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
 512         SPECTRE_V2_USER_CMD_SECCOMP,
 513         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
 514 };
 515
 516 static const char * const spectre_v2_user_strings[] = {
 517         [SPECTRE_V2_USER_NONE]                  = "User space: Vulnerable",
 518         [SPECTRE_V2_USER_STRICT]                = "User space: Mitigation: STIBP protection",
 519         [SPECTRE_V2_USER_STRICT_PREFERRED]      = "User space: Mitigation: STIBP always-on protection",
 520         [SPECTRE_V2_USER_PRCTL]                 = "User space: Mitigation: STIBP via prctl",
 521         [SPECTRE_V2_USER_SECCOMP]               = "User space: Mitigation: STIBP via seccomp and prctl",
 522 };
 523
 524 static const struct {
 525         const char                      *option;
 526         enum spectre_v2_user_cmd        cmd;
 527         bool                            secure;
 528 } v2_user_options[] __initconst = {
 529         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
 530         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
 531         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
 532         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
 533         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
 534         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
 535         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
 536 };
 537
 538 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
 539 {
 540         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 541                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
 542 }
 543
 544 static enum spectre_v2_user_cmd __init
 545 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
 546 {
 547         char arg[20];
 548         int ret, i;
 549
 550         switch (v2_cmd) {
 551         case SPECTRE_V2_CMD_NONE:
 552                 return SPECTRE_V2_USER_CMD_NONE;
 553         case SPECTRE_V2_CMD_FORCE:
 554                 return SPECTRE_V2_USER_CMD_FORCE;
 555         default:
 556                 break;
 557         }
 558
 559         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
 560                                   arg, sizeof(arg));
 561         if (ret < 0)
 562                 return SPECTRE_V2_USER_CMD_AUTO;
 563
 564         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
 565                 if (match_option(arg, ret, v2_user_options[i].option)) {
 566                         spec_v2_user_print_cond(v2_user_options[i].option,
 567                                                 v2_user_options[i].secure);
 568                         return v2_user_options[i].cmd;
 569                 }
 570         }
 571
 572         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
 573         return SPECTRE_V2_USER_CMD_AUTO;
 574 }
 575
 576 static void __init
 577 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
 578 {
 579         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
 580         bool smt_possible = IS_ENABLED(CONFIG_SMP);
 581         enum spectre_v2_user_cmd cmd;
 582
 583         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
 584                 return;
 585
 586         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 587             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 588                 smt_possible = false;
 589
 590         cmd = spectre_v2_parse_user_cmdline(v2_cmd);
 591         switch (cmd) {
 592         case SPECTRE_V2_USER_CMD_NONE:
 593                 goto set_mode;
 594         case SPECTRE_V2_USER_CMD_FORCE:
 595                 mode = SPECTRE_V2_USER_STRICT;
 596                 break;
 597         case SPECTRE_V2_USER_CMD_PRCTL:
 598         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 599                 mode = SPECTRE_V2_USER_PRCTL;
 600                 break;
 601         case SPECTRE_V2_USER_CMD_AUTO:
 602         case SPECTRE_V2_USER_CMD_SECCOMP:
 603         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 604                 if (IS_ENABLED(CONFIG_SECCOMP))
 605                         mode = SPECTRE_V2_USER_SECCOMP;
 606                 else
 607                         mode = SPECTRE_V2_USER_PRCTL;
 608                 break;
 609         }
 610
 611         /*
 612          * At this point, an STIBP mode other than "off" has been set.
 613          * If STIBP support is not being forced, check if STIBP always-on
 614          * is preferred.
 615          */
 616         if (mode != SPECTRE_V2_USER_STRICT &&
 617             boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
 618                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
 619
 620         /* Initialize Indirect Branch Prediction Barrier */
 621         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 622                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 623
 624                 switch (cmd) {
 625                 case SPECTRE_V2_USER_CMD_FORCE:
 626                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 627                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 628                         static_branch_enable(&switch_mm_always_ibpb);
 629                         break;
 630                 case SPECTRE_V2_USER_CMD_PRCTL:
 631                 case SPECTRE_V2_USER_CMD_AUTO:
 632                 case SPECTRE_V2_USER_CMD_SECCOMP:
 633                         static_branch_enable(&switch_mm_cond_ibpb);
 634                         break;
 635                 default:
 636                         break;
 637                 }
 638
 639                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
 640                         static_key_enabled(&switch_mm_always_ibpb) ?
 641                         "always-on" : "conditional");
 642         }
 643
 644         /* If enhanced IBRS is enabled no STIBP required */
 645         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 646                 return;
 647
 648         /*
 649          * If SMT is not possible or STIBP is not available clear the STIBP
 650          * mode.
 651          */
 652         if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP))
 653                 mode = SPECTRE_V2_USER_NONE;
 654 set_mode:
 655         spectre_v2_user = mode;
 656         /* Only print the STIBP mode when SMT possible */
 657         if (smt_possible)
 658                 pr_info("%s\n", spectre_v2_user_strings[mode]);
 659 }
 660
 661 static const char * const spectre_v2_strings[] = {
 662         [SPECTRE_V2_NONE]                       = "Vulnerable",
 663         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 664         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 665         [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
 666 };
 667
 668 static const struct {
 669         const char *option;
 670         enum spectre_v2_mitigation_cmd cmd;
 671         bool secure;
 672 } mitigation_options[] __initconst = {
 673         { "off",                SPECTRE_V2_CMD_NONE,              false },
 674         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
 675         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
 676         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 677         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 678         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
 679 };
 680
 681 static void __init spec_v2_print_cond(const char *reason, bool secure)
 682 {
 683         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 684                 pr_info("%s selected on command line.\n", reason);
 685 }
 686
 687 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 688 {
 689         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 690         char arg[20];
 691         int ret, i;
 692
 693         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
 694             cpu_mitigations_off())
 695                 return SPECTRE_V2_CMD_NONE;
 696
 697         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 698         if (ret < 0)
 699                 return SPECTRE_V2_CMD_AUTO;
 700
 701         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 702                 if (!match_option(arg, ret, mitigation_options[i].option))
 703                         continue;
 704                 cmd = mitigation_options[i].cmd;
 705                 break;
 706         }
 707
 708         if (i >= ARRAY_SIZE(mitigation_options)) {
 709                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 710                 return SPECTRE_V2_CMD_AUTO;
 711         }
 712
 713         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 714              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 715              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 716             !IS_ENABLED(CONFIG_RETPOLINE)) {
 717                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 718                 return SPECTRE_V2_CMD_AUTO;
 719         }
 720
 721         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 722             boot_cpu_data.x86_vendor != X86_VENDOR_HYGON &&
 723             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 724                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 725                 return SPECTRE_V2_CMD_AUTO;
 726         }
 727
 728         spec_v2_print_cond(mitigation_options[i].option,
 729                            mitigation_options[i].secure);
 730         return cmd;
 731 }
 732
 733 static void __init spectre_v2_select_mitigation(void)
 734 {
 735         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 736         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 737
 738         /*
 739          * If the CPU is not affected and the command line mode is NONE or AUTO
 740          * then nothing to do.
 741          */
 742         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 743             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 744                 return;
 745
 746         switch (cmd) {
 747         case SPECTRE_V2_CMD_NONE:
 748                 return;
 749
 750         case SPECTRE_V2_CMD_FORCE:
 751         case SPECTRE_V2_CMD_AUTO:
 752                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 753                         mode = SPECTRE_V2_IBRS_ENHANCED;
 754                         /* Force it so VMEXIT will restore correctly */
 755                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
 756                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 757                         goto specv2_set_mode;
 758                 }
 759                 if (IS_ENABLED(CONFIG_RETPOLINE))
 760                         goto retpoline_auto;
 761                 break;
 762         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 763                 if (IS_ENABLED(CONFIG_RETPOLINE))
 764                         goto retpoline_amd;
 765                 break;
 766         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 767                 if (IS_ENABLED(CONFIG_RETPOLINE))
 768                         goto retpoline_generic;
 769                 break;
 770         case SPECTRE_V2_CMD_RETPOLINE:
 771                 if (IS_ENABLED(CONFIG_RETPOLINE))
 772                         goto retpoline_auto;
 773                 break;
 774         }
 775         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 776         return;
 777
 778 retpoline_auto:
 779         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
 780             boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
 781         retpoline_amd:
 782                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 783                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 784                         goto retpoline_generic;
 785                 }
 786                 mode = SPECTRE_V2_RETPOLINE_AMD;
 787                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 788                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 789         } else {
 790         retpoline_generic:
 791                 mode = SPECTRE_V2_RETPOLINE_GENERIC;
 792                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 793         }
 794
 795 specv2_set_mode:
 796         spectre_v2_enabled = mode;
 797         pr_info("%s\n", spectre_v2_strings[mode]);
 798
 799         /*
 800          * If spectre v2 protection has been enabled, unconditionally fill
 801          * RSB during a context switch; this protects against two independent
 802          * issues:
 803          *
 804          *      - RSB underflow (and switch to BTB) on Skylake+
 805          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
 806          */
 807         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 808         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
 809
 810         /*
 811          * Retpoline means the kernel is safe because it has no indirect
 812          * branches. Enhanced IBRS protects firmware too, so, enable restricted
 813          * speculation around firmware calls only when Enhanced IBRS isn't
 814          * supported.
 815          *
 816          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
 817          * the user might select retpoline on the kernel command line and if
 818          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
 819          * enable IBRS around firmware calls.
 820          */
 821         if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
 822                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 823                 pr_info("Enabling Restricted Speculation for firmware calls\n");
 824         }
 825
 826         /* Set up IBPB and STIBP depending on the general spectre V2 command */
 827         spectre_v2_user_select_mitigation(cmd);
 828 }
 829
 830 static void update_stibp_msr(void * __unused)
 831 {
 832         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 833 }
 834
 835 /* Update x86_spec_ctrl_base in case SMT state changed. */
 836 static void update_stibp_strict(void)
 837 {
 838         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
 839
 840         if (sched_smt_active())
 841                 mask |= SPEC_CTRL_STIBP;
 842
 843         if (mask == x86_spec_ctrl_base)
 844                 return;
 845
 846         pr_info("Update user space SMT mitigation: STIBP %s\n",
 847                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
 848         x86_spec_ctrl_base = mask;
 849         on_each_cpu(update_stibp_msr, NULL, 1);
 850 }
 851
 852 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
 853 static void update_indir_branch_cond(void)
 854 {
 855         if (sched_smt_active())
 856                 static_branch_enable(&switch_to_cond_stibp);
 857         else
 858                 static_branch_disable(&switch_to_cond_stibp);
 859 }
 860
 861 #undef pr_fmt
 862 #define pr_fmt(fmt) fmt
 863
 864 /* Update the static key controlling the MDS CPU buffer clear in idle */
 865 static void update_mds_branch_idle(void)
 866 {
 867         /*
 868          * Enable the idle clearing if SMT is active on CPUs which are
 869          * affected only by MSBDS and not any other MDS variant.
 870          *
 871          * The other variants cannot be mitigated when SMT is enabled, so
 872          * clearing the buffers on idle just to prevent the Store Buffer
 873          * repartitioning leak would be a window dressing exercise.
 874          */
 875         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
 876                 return;
 877
 878         if (sched_smt_active())
 879                 static_branch_enable(&mds_idle_clear);
 880         else
 881                 static_branch_disable(&mds_idle_clear);
 882 }
 883
 884 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
 885 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
 886
 887 void cpu_bugs_smt_update(void)
 888 {
 889         /* Enhanced IBRS implies STIBP. No update required. */
 890         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 891                 return;
 892
 893         mutex_lock(&spec_ctrl_mutex);
 894
 895         switch (spectre_v2_user) {
 896         case SPECTRE_V2_USER_NONE:
 897                 break;
 898         case SPECTRE_V2_USER_STRICT:
 899         case SPECTRE_V2_USER_STRICT_PREFERRED:
 900                 update_stibp_strict();
 901                 break;
 902         case SPECTRE_V2_USER_PRCTL:
 903         case SPECTRE_V2_USER_SECCOMP:
 904                 update_indir_branch_cond();
 905                 break;
 906         }
 907
 908         switch (mds_mitigation) {
 909         case MDS_MITIGATION_FULL:
 910         case MDS_MITIGATION_VMWERV:
 911                 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
 912                         pr_warn_once(MDS_MSG_SMT);
 913                 update_mds_branch_idle();
 914                 break;
 915         case MDS_MITIGATION_OFF:
 916                 break;
 917         }
 918
 919         switch (taa_mitigation) {
 920         case TAA_MITIGATION_VERW:
 921         case TAA_MITIGATION_UCODE_NEEDED:
 922                 if (sched_smt_active())
 923                         pr_warn_once(TAA_MSG_SMT);
 924                 break;
 925         case TAA_MITIGATION_TSX_DISABLED:
 926         case TAA_MITIGATION_OFF:
 927                 break;
 928         }
 929
 930         mutex_unlock(&spec_ctrl_mutex);
 931 }
 932
 933 #undef pr_fmt
 934 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 935
 936 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
 937
 938 /* The kernel command line selection */
 939 enum ssb_mitigation_cmd {
 940         SPEC_STORE_BYPASS_CMD_NONE,
 941         SPEC_STORE_BYPASS_CMD_AUTO,
 942         SPEC_STORE_BYPASS_CMD_ON,
 943         SPEC_STORE_BYPASS_CMD_PRCTL,
 944         SPEC_STORE_BYPASS_CMD_SECCOMP,
 945 };
 946
 947 static const char * const ssb_strings[] = {
 948         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 949         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
 950         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
 951         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
 952 };
 953
 954 static const struct {
 955         const char *option;
 956         enum ssb_mitigation_cmd cmd;
 957 } ssb_mitigation_options[]  __initconst = {
 958         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
 959         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
 960         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
 961         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
 962         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
 963 };
 964
 965 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 966 {
 967         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 968         char arg[20];
 969         int ret, i;
 970
 971         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
 972             cpu_mitigations_off()) {
 973                 return SPEC_STORE_BYPASS_CMD_NONE;
 974         } else {
 975                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 976                                           arg, sizeof(arg));
 977                 if (ret < 0)
 978                         return SPEC_STORE_BYPASS_CMD_AUTO;
 979
 980                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 981                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 982                                 continue;
 983
 984                         cmd = ssb_mitigation_options[i].cmd;
 985                         break;
 986                 }
 987
 988                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 989                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 990                         return SPEC_STORE_BYPASS_CMD_AUTO;
 991                 }
 992         }
 993
 994         return cmd;
 995 }
 996
 997 static enum ssb_mitigation __init __ssb_select_mitigation(void)
 998 {
 999         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
1000         enum ssb_mitigation_cmd cmd;
1001
1002         if (!boot_cpu_has(X86_FEATURE_SSBD))
1003                 return mode;
1004
1005         cmd = ssb_parse_cmdline();
1006         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
1007             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
1008              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
1009                 return mode;
1010
1011         switch (cmd) {
1012         case SPEC_STORE_BYPASS_CMD_AUTO:
1013         case SPEC_STORE_BYPASS_CMD_SECCOMP:
1014                 /*
1015                  * Choose prctl+seccomp as the default mode if seccomp is
1016                  * enabled.
1017                  */
1018                 if (IS_ENABLED(CONFIG_SECCOMP))
1019                         mode = SPEC_STORE_BYPASS_SECCOMP;
1020                 else
1021                         mode = SPEC_STORE_BYPASS_PRCTL;
1022                 break;
1023         case SPEC_STORE_BYPASS_CMD_ON:
1024                 mode = SPEC_STORE_BYPASS_DISABLE;
1025                 break;
1026         case SPEC_STORE_BYPASS_CMD_PRCTL:
1027                 mode = SPEC_STORE_BYPASS_PRCTL;
1028                 break;
1029         case SPEC_STORE_BYPASS_CMD_NONE:
1030                 break;
1031         }
1032
1033         /*
1034          * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper
1035          * bit in the mask to allow guests to use the mitigation even in the
1036          * case where the host does not enable it.
1037          */
1038         if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
1039             static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1040                 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
1041         }
1042
1043         /*
1044          * We have three CPU feature flags that are in play here:
1045          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1046          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1047          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1048          */
1049         if (mode == SPEC_STORE_BYPASS_DISABLE) {
1050                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
1051                 /*
1052                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1053                  * use a completely different MSR and bit dependent on family.
1054                  */
1055                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
1056                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1057                         x86_amd_ssb_disable();
1058                 } else {
1059                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
1060                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1061                 }
1062         }
1063
1064         return mode;
1065 }
1066
1067 static void ssb_select_mitigation(void)
1068 {
1069         ssb_mode = __ssb_select_mitigation();
1070
1071         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1072                 pr_info("%s\n", ssb_strings[ssb_mode]);
1073 }
1074
1075 #undef pr_fmt
1076 #define pr_fmt(fmt)     "Speculation prctl: " fmt
1077
1078 static void task_update_spec_tif(struct task_struct *tsk)
1079 {
1080         /* Force the update of the real TIF bits */
1081         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
1082
1083         /*
1084          * Immediately update the speculation control MSRs for the current
1085          * task, but for a non-current task delay setting the CPU
1086          * mitigation until it is scheduled next.
1087          *
1088          * This can only happen for SECCOMP mitigation. For PRCTL it's
1089          * always the current task.
1090          */
1091         if (tsk == current)
1092                 speculation_ctrl_update_current();
1093 }
1094
1095 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
1096 {
1097         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
1098             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
1099                 return -ENXIO;
1100
1101         switch (ctrl) {
1102         case PR_SPEC_ENABLE:
1103                 /* If speculation is force disabled, enable is not allowed */
1104                 if (task_spec_ssb_force_disable(task))
1105                         return -EPERM;
1106                 task_clear_spec_ssb_disable(task);
1107                 task_clear_spec_ssb_noexec(task);
1108                 task_update_spec_tif(task);
1109                 break;
1110         case PR_SPEC_DISABLE:
1111                 task_set_spec_ssb_disable(task);
1112                 task_clear_spec_ssb_noexec(task);
1113                 task_update_spec_tif(task);
1114                 break;
1115         case PR_SPEC_FORCE_DISABLE:
1116                 task_set_spec_ssb_disable(task);
1117                 task_set_spec_ssb_force_disable(task);
1118                 task_clear_spec_ssb_noexec(task);
1119                 task_update_spec_tif(task);
1120                 break;
1121         case PR_SPEC_DISABLE_NOEXEC:
1122                 if (task_spec_ssb_force_disable(task))
1123                         return -EPERM;
1124                 task_set_spec_ssb_disable(task);
1125                 task_set_spec_ssb_noexec(task);
1126                 task_update_spec_tif(task);
1127                 break;
1128         default:
1129                 return -ERANGE;
1130         }
1131         return 0;
1132 }
1133
1134 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
1135 {
1136         switch (ctrl) {
1137         case PR_SPEC_ENABLE:
1138                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
1139                         return 0;
1140                 /*
1141                  * Indirect branch speculation is always disabled in strict
1142                  * mode.
1143                  */
1144                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT ||
1145                     spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED)
1146                         return -EPERM;
1147                 task_clear_spec_ib_disable(task);
1148                 task_update_spec_tif(task);
1149                 break;
1150         case PR_SPEC_DISABLE:
1151         case PR_SPEC_FORCE_DISABLE:
1152                 /*
1153                  * Indirect branch speculation is always allowed when
1154                  * mitigation is force disabled.
1155                  */
1156                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
1157                         return -EPERM;
1158                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT ||
1159                     spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED)
1160                         return 0;
1161                 task_set_spec_ib_disable(task);
1162                 if (ctrl == PR_SPEC_FORCE_DISABLE)
1163                         task_set_spec_ib_force_disable(task);
1164                 task_update_spec_tif(task);
1165                 break;
1166         default:
1167                 return -ERANGE;
1168         }
1169         return 0;
1170 }
1171
1172 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
1173                              unsigned long ctrl)
1174 {
1175         switch (which) {
1176         case PR_SPEC_STORE_BYPASS:
1177                 return ssb_prctl_set(task, ctrl);
1178         case PR_SPEC_INDIRECT_BRANCH:
1179                 return ib_prctl_set(task, ctrl);
1180         default:
1181                 return -ENODEV;
1182         }
1183 }
1184
1185 #ifdef CONFIG_SECCOMP
1186 void arch_seccomp_spec_mitigate(struct task_struct *task)
1187 {
1188         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
1189                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1190         if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP)
1191                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1192 }
1193 #endif
1194
1195 static int ssb_prctl_get(struct task_struct *task)
1196 {
1197         switch (ssb_mode) {
1198         case SPEC_STORE_BYPASS_DISABLE:
1199                 return PR_SPEC_DISABLE;
1200         case SPEC_STORE_BYPASS_SECCOMP:
1201         case SPEC_STORE_BYPASS_PRCTL:
1202                 if (task_spec_ssb_force_disable(task))
1203                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1204                 if (task_spec_ssb_noexec(task))
1205                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
1206                 if (task_spec_ssb_disable(task))
1207                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1208                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1209         default:
1210                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1211                         return PR_SPEC_ENABLE;
1212                 return PR_SPEC_NOT_AFFECTED;
1213         }
1214 }
1215
1216 static int ib_prctl_get(struct task_struct *task)
1217 {
1218         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
1219                 return PR_SPEC_NOT_AFFECTED;
1220
1221         switch (spectre_v2_user) {
1222         case SPECTRE_V2_USER_NONE:
1223                 return PR_SPEC_ENABLE;
1224         case SPECTRE_V2_USER_PRCTL:
1225         case SPECTRE_V2_USER_SECCOMP:
1226                 if (task_spec_ib_force_disable(task))
1227                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1228                 if (task_spec_ib_disable(task))
1229                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1230                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1231         case SPECTRE_V2_USER_STRICT:
1232         case SPECTRE_V2_USER_STRICT_PREFERRED:
1233                 return PR_SPEC_DISABLE;
1234         default:
1235                 return PR_SPEC_NOT_AFFECTED;
1236         }
1237 }
1238
1239 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
1240 {
1241         switch (which) {
1242         case PR_SPEC_STORE_BYPASS:
1243                 return ssb_prctl_get(task);
1244         case PR_SPEC_INDIRECT_BRANCH:
1245                 return ib_prctl_get(task);
1246         default:
1247                 return -ENODEV;
1248         }
1249 }
1250
1251 void x86_spec_ctrl_setup_ap(void)
1252 {
1253         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
1254                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1255
1256         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
1257                 x86_amd_ssb_disable();
1258 }
1259
1260 #undef pr_fmt
1261 #define pr_fmt(fmt)     "L1TF: " fmt
1262
1263 /* Default mitigation for L1TF-affected CPUs */
1264 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
1265 #if IS_ENABLED(CONFIG_KVM_INTEL)
1266 EXPORT_SYMBOL_GPL(l1tf_mitigation);
1267 #endif
1268 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
1269 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
1270
1271 /*
1272  * These CPUs all support 44bits physical address space internally in the
1273  * cache but CPUID can report a smaller number of physical address bits.
1274  *
1275  * The L1TF mitigation uses the top most address bit for the inversion of
1276  * non present PTEs. When the installed memory reaches into the top most
1277  * address bit due to memory holes, which has been observed on machines
1278  * which report 36bits physical address bits and have 32G RAM installed,
1279  * then the mitigation range check in l1tf_select_mitigation() triggers.
1280  * This is a false positive because the mitigation is still possible due to
1281  * the fact that the cache uses 44bit internally. Use the cache bits
1282  * instead of the reported physical bits and adjust them on the affected
1283  * machines to 44bit if the reported bits are less than 44.
1284  */
1285 static void override_cache_bits(struct cpuinfo_x86 *c)
1286 {
1287         if (c->x86 != 6)
1288                 return;
1289
1290         switch (c->x86_model) {
1291         case INTEL_FAM6_NEHALEM:
1292         case INTEL_FAM6_WESTMERE:
1293         case INTEL_FAM6_SANDYBRIDGE:
1294         case INTEL_FAM6_IVYBRIDGE:
1295         case INTEL_FAM6_HASWELL:
1296         case INTEL_FAM6_HASWELL_L:
1297         case INTEL_FAM6_HASWELL_G:
1298         case INTEL_FAM6_BROADWELL:
1299         case INTEL_FAM6_BROADWELL_G:
1300         case INTEL_FAM6_SKYLAKE_L:
1301         case INTEL_FAM6_SKYLAKE:
1302         case INTEL_FAM6_KABYLAKE_L:
1303         case INTEL_FAM6_KABYLAKE:
1304                 if (c->x86_cache_bits < 44)
1305                         c->x86_cache_bits = 44;
1306                 break;
1307         }
1308 }
1309
1310 static void __init l1tf_select_mitigation(void)
1311 {
1312         u64 half_pa;
1313
1314         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1315                 return;
1316
1317         if (cpu_mitigations_off())
1318                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1319         else if (cpu_mitigations_auto_nosmt())
1320                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1321
1322         override_cache_bits(&boot_cpu_data);
1323
1324         switch (l1tf_mitigation) {
1325         case L1TF_MITIGATION_OFF:
1326         case L1TF_MITIGATION_FLUSH_NOWARN:
1327         case L1TF_MITIGATION_FLUSH:
1328                 break;
1329         case L1TF_MITIGATION_FLUSH_NOSMT:
1330         case L1TF_MITIGATION_FULL:
1331                 cpu_smt_disable(false);
1332                 break;
1333         case L1TF_MITIGATION_FULL_FORCE:
1334                 cpu_smt_disable(true);
1335                 break;
1336         }
1337
1338 #if CONFIG_PGTABLE_LEVELS == 2
1339         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
1340         return;
1341 #endif
1342
1343         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
1344         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
1345                         e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
1346                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
1347                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
1348                                 half_pa);
1349                 pr_info("However, doing so will make a part of your RAM unusable.\n");
1350                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
1351                 return;
1352         }
1353
1354         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
1355 }
1356
1357 static int __init l1tf_cmdline(char *str)
1358 {
1359         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1360                 return 0;
1361
1362         if (!str)
1363                 return -EINVAL;
1364
1365         if (!strcmp(str, "off"))
1366                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1367         else if (!strcmp(str, "flush,nowarn"))
1368                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
1369         else if (!strcmp(str, "flush"))
1370                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
1371         else if (!strcmp(str, "flush,nosmt"))
1372                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1373         else if (!strcmp(str, "full"))
1374                 l1tf_mitigation = L1TF_MITIGATION_FULL;
1375         else if (!strcmp(str, "full,force"))
1376                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
1377
1378         return 0;
1379 }
1380 early_param("l1tf", l1tf_cmdline);
1381
1382 #undef pr_fmt
1383 #define pr_fmt(fmt) fmt
1384
1385 #ifdef CONFIG_SYSFS
1386
1387 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
1388
1389 #if IS_ENABLED(CONFIG_KVM_INTEL)
1390 static const char * const l1tf_vmx_states[] = {
1391         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
1392         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
1393         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
1394         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
1395         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
1396         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
1397 };
1398
1399 static ssize_t l1tf_show_state(char *buf)
1400 {
1401         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
1402                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1403
1404         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
1405             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
1406              sched_smt_active())) {
1407                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
1408                                l1tf_vmx_states[l1tf_vmx_mitigation]);
1409         }
1410
1411         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
1412                        l1tf_vmx_states[l1tf_vmx_mitigation],
1413                        sched_smt_active() ? "vulnerable" : "disabled");
1414 }
1415 #else
1416 static ssize_t l1tf_show_state(char *buf)
1417 {
1418         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1419 }
1420 #endif
1421
1422 static ssize_t itlb_multihit_show_state(char *buf)
1423 {
1424         return sprintf(buf, "Processor vulnerable\n");
1425 }
1426
1427 static ssize_t mds_show_state(char *buf)
1428 {
1429         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1430                 return sprintf(buf, "%s; SMT Host state unknown\n",
1431                                mds_strings[mds_mitigation]);
1432         }
1433
1434         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
1435                 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1436                                (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
1437                                 sched_smt_active() ? "mitigated" : "disabled"));
1438         }
1439
1440         return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1441                        sched_smt_active() ? "vulnerable" : "disabled");
1442 }
1443
1444 static ssize_t tsx_async_abort_show_state(char *buf)
1445 {
1446         if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
1447             (taa_mitigation == TAA_MITIGATION_OFF))
1448                 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]);
1449
1450         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1451                 return sprintf(buf, "%s; SMT Host state unknown\n",
1452                                taa_strings[taa_mitigation]);
1453         }
1454
1455         return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
1456                        sched_smt_active() ? "vulnerable" : "disabled");
1457 }
1458
1459 static char *stibp_state(void)
1460 {
1461         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
1462                 return "";
1463
1464         switch (spectre_v2_user) {
1465         case SPECTRE_V2_USER_NONE:
1466                 return ", STIBP: disabled";
1467         case SPECTRE_V2_USER_STRICT:
1468                 return ", STIBP: forced";
1469         case SPECTRE_V2_USER_STRICT_PREFERRED:
1470                 return ", STIBP: always-on";
1471         case SPECTRE_V2_USER_PRCTL:
1472         case SPECTRE_V2_USER_SECCOMP:
1473                 if (static_key_enabled(&switch_to_cond_stibp))
1474                         return ", STIBP: conditional";
1475         }
1476         return "";
1477 }
1478
1479 static char *ibpb_state(void)
1480 {
1481         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1482                 if (static_key_enabled(&switch_mm_always_ibpb))
1483                         return ", IBPB: always-on";
1484                 if (static_key_enabled(&switch_mm_cond_ibpb))
1485                         return ", IBPB: conditional";
1486                 return ", IBPB: disabled";
1487         }
1488         return "";
1489 }
1490
1491 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
1492                                char *buf, unsigned int bug)
1493 {
1494         if (!boot_cpu_has_bug(bug))
1495                 return sprintf(buf, "Not affected\n");
1496
1497         switch (bug) {
1498         case X86_BUG_CPU_MELTDOWN:
1499                 if (boot_cpu_has(X86_FEATURE_PTI))
1500                         return sprintf(buf, "Mitigation: PTI\n");
1501
1502                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
1503                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
1504
1505                 break;
1506
1507         case X86_BUG_SPECTRE_V1:
1508                 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
1509
1510         case X86_BUG_SPECTRE_V2:
1511                 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
1512                                ibpb_state(),
1513                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
1514                                stibp_state(),
1515                                boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
1516                                spectre_v2_module_string());
1517
1518         case X86_BUG_SPEC_STORE_BYPASS:
1519                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
1520
1521         case X86_BUG_L1TF:
1522                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
1523                         return l1tf_show_state(buf);
1524                 break;
1525
1526         case X86_BUG_MDS:
1527                 return mds_show_state(buf);
1528
1529         case X86_BUG_TAA:
1530                 return tsx_async_abort_show_state(buf);
1531
1532         case X86_BUG_ITLB_MULTIHIT:
1533                 return itlb_multihit_show_state(buf);
1534
1535         default:
1536                 break;
1537         }
1538
1539         return sprintf(buf, "Vulnerable\n");
1540 }
1541
1542 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
1543 {
1544         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
1545 }
1546
1547 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
1548 {
1549         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
1550 }
1551
1552 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
1553 {
1554         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
1555 }
1556
1557 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
1558 {
1559         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
1560 }
1561
1562 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
1563 {
1564         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
1565 }
1566
1567 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
1568 {
1569         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
1570 }
1571
1572 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
1573 {
1574         return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
1575 }
1576
1577 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
1578 {
1579         return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
1580 }
1581 #endif