arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/utsname.h>
  13 #include <linux/cpu.h>
  14 #include <linux/module.h>
  15 #include <linux/nospec.h>
  16 #include <linux/prctl.h>
  17
  18 #include <asm/spec-ctrl.h>
  19 #include <asm/cmdline.h>
  20 #include <asm/bugs.h>
  21 #include <asm/processor.h>
  22 #include <asm/processor-flags.h>
  23 #include <asm/fpu/internal.h>
  24 #include <asm/msr.h>
  25 #include <asm/paravirt.h>
  26 #include <asm/alternative.h>
  27 #include <asm/pgtable.h>
  28 #include <asm/set_memory.h>
  29 #include <asm/intel-family.h>
  30
  31 static void __init spectre_v2_select_mitigation(void);
  32 static void __init ssb_select_mitigation(void);
  33
  34 /*
  35  * Our boot-time value of the SPEC_CTRL MSR. We read it once so that any
  36  * writes to SPEC_CTRL contain whatever reserved bits have been set.
  37  */
  38 u64 __ro_after_init x86_spec_ctrl_base;
  39 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  40
  41 /*
  42  * The vendor and possibly platform specific bits which can be modified in
  43  * x86_spec_ctrl_base.
  44  */
  45 static u64 __ro_after_init x86_spec_ctrl_mask = ~SPEC_CTRL_IBRS;
  46
  47 /*
  48  * AMD specific MSR info for Speculative Store Bypass control.
  49  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  50  */
  51 u64 __ro_after_init x86_amd_ls_cfg_base;
  52 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  53
  54 void __init check_bugs(void)
  55 {
  56         identify_boot_cpu();
  57
  58         if (!IS_ENABLED(CONFIG_SMP)) {
  59                 pr_info("CPU: ");
  60                 print_cpu_info(&boot_cpu_data);
  61         }
  62
  63         /*
  64          * Read the SPEC_CTRL MSR to account for reserved bits which may
  65          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  66          * init code as it is not enumerated and depends on the family.
  67          */
  68         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  69                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  70
  71         /* Select the proper spectre mitigation before patching alternatives */
  72         spectre_v2_select_mitigation();
  73
  74         /*
  75          * Select proper mitigation for any exposure to the Speculative Store
  76          * Bypass vulnerability.
  77          */
  78         ssb_select_mitigation();
  79
  80 #ifdef CONFIG_X86_32
  81         /*
  82          * Check whether we are able to run this kernel safely on SMP.
  83          *
  84          * - i386 is no longer supported.
  85          * - In order to run on anything without a TSC, we need to be
  86          *   compiled for a i486.
  87          */
  88         if (boot_cpu_data.x86 < 4)
  89                 panic("Kernel requires i486+ for 'invlpg' and other features");
  90
  91         init_utsname()->machine[1] =
  92                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
  93         alternative_instructions();
  94
  95         fpu__init_check_bugs();
  96 #else /* CONFIG_X86_64 */
  97         alternative_instructions();
  98
  99         /*
 100          * Make sure the first 2MB area is not mapped by huge pages
 101          * There are typically fixed size MTRRs in there and overlapping
 102          * MTRRs into large pages causes slow downs.
 103          *
 104          * Right now we don't do that with gbpages because there seems
 105          * very little benefit for that case.
 106          */
 107         if (!direct_gbpages)
 108                 set_memory_4k((unsigned long)__va(0), 1);
 109 #endif
 110 }
 111
 112 /* The kernel command line selection */
 113 enum spectre_v2_mitigation_cmd {
 114         SPECTRE_V2_CMD_NONE,
 115         SPECTRE_V2_CMD_AUTO,
 116         SPECTRE_V2_CMD_FORCE,
 117         SPECTRE_V2_CMD_RETPOLINE,
 118         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 119         SPECTRE_V2_CMD_RETPOLINE_AMD,
 120 };
 121
 122 static const char *spectre_v2_strings[] = {
 123         [SPECTRE_V2_NONE]                       = "Vulnerable",
 124         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
 125         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
 126         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 127         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 128 };
 129
 130 #undef pr_fmt
 131 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 132
 133 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 134         SPECTRE_V2_NONE;
 135
 136 void
 137 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 138 {
 139         struct thread_info *ti = current_thread_info();
 140         u64 msr, host = x86_spec_ctrl_base;
 141
 142         /* Is MSR_SPEC_CTRL implemented ? */
 143         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 144                 /* SSBD controlled in MSR_SPEC_CTRL */
 145                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD))
 146                         host |= ssbd_tif_to_spec_ctrl(ti->flags);
 147
 148                 if (host != guest_spec_ctrl) {
 149                         msr = setguest ? guest_spec_ctrl : host;
 150                         wrmsrl(MSR_IA32_SPEC_CTRL, msr);
 151                 }
 152         }
 153 }
 154 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 155
 156 static void x86_amd_ssb_disable(void)
 157 {
 158         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 159
 160         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 161                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 162         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 163                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 164 }
 165
 166 #ifdef RETPOLINE
 167 static bool spectre_v2_bad_module;
 168
 169 bool retpoline_module_ok(bool has_retpoline)
 170 {
 171         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 172                 return true;
 173
 174         pr_err("System may be vulnerable to spectre v2\n");
 175         spectre_v2_bad_module = true;
 176         return false;
 177 }
 178
 179 static inline const char *spectre_v2_module_string(void)
 180 {
 181         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 182 }
 183 #else
 184 static inline const char *spectre_v2_module_string(void) { return ""; }
 185 #endif
 186
 187 static void __init spec2_print_if_insecure(const char *reason)
 188 {
 189         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 190                 pr_info("%s selected on command line.\n", reason);
 191 }
 192
 193 static void __init spec2_print_if_secure(const char *reason)
 194 {
 195         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 196                 pr_info("%s selected on command line.\n", reason);
 197 }
 198
 199 static inline bool retp_compiler(void)
 200 {
 201         return __is_defined(RETPOLINE);
 202 }
 203
 204 static inline bool match_option(const char *arg, int arglen, const char *opt)
 205 {
 206         int len = strlen(opt);
 207
 208         return len == arglen && !strncmp(arg, opt, len);
 209 }
 210
 211 static const struct {
 212         const char *option;
 213         enum spectre_v2_mitigation_cmd cmd;
 214         bool secure;
 215 } mitigation_options[] = {
 216         { "off",               SPECTRE_V2_CMD_NONE,              false },
 217         { "on",                SPECTRE_V2_CMD_FORCE,             true },
 218         { "retpoline",         SPECTRE_V2_CMD_RETPOLINE,         false },
 219         { "retpoline,amd",     SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 220         { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 221         { "auto",              SPECTRE_V2_CMD_AUTO,              false },
 222 };
 223
 224 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 225 {
 226         char arg[20];
 227         int ret, i;
 228         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 229
 230         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
 231                 return SPECTRE_V2_CMD_NONE;
 232         else {
 233                 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 234                 if (ret < 0)
 235                         return SPECTRE_V2_CMD_AUTO;
 236
 237                 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 238                         if (!match_option(arg, ret, mitigation_options[i].option))
 239                                 continue;
 240                         cmd = mitigation_options[i].cmd;
 241                         break;
 242                 }
 243
 244                 if (i >= ARRAY_SIZE(mitigation_options)) {
 245                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 246                         return SPECTRE_V2_CMD_AUTO;
 247                 }
 248         }
 249
 250         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 251              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 252              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 253             !IS_ENABLED(CONFIG_RETPOLINE)) {
 254                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 255                 return SPECTRE_V2_CMD_AUTO;
 256         }
 257
 258         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 259             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 260                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 261                 return SPECTRE_V2_CMD_AUTO;
 262         }
 263
 264         if (mitigation_options[i].secure)
 265                 spec2_print_if_secure(mitigation_options[i].option);
 266         else
 267                 spec2_print_if_insecure(mitigation_options[i].option);
 268
 269         return cmd;
 270 }
 271
 272 /* Check for Skylake-like CPUs (for RSB handling) */
 273 static bool __init is_skylake_era(void)
 274 {
 275         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 276             boot_cpu_data.x86 == 6) {
 277                 switch (boot_cpu_data.x86_model) {
 278                 case INTEL_FAM6_SKYLAKE_MOBILE:
 279                 case INTEL_FAM6_SKYLAKE_DESKTOP:
 280                 case INTEL_FAM6_SKYLAKE_X:
 281                 case INTEL_FAM6_KABYLAKE_MOBILE:
 282                 case INTEL_FAM6_KABYLAKE_DESKTOP:
 283                         return true;
 284                 }
 285         }
 286         return false;
 287 }
 288
 289 static void __init spectre_v2_select_mitigation(void)
 290 {
 291         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 292         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 293
 294         /*
 295          * If the CPU is not affected and the command line mode is NONE or AUTO
 296          * then nothing to do.
 297          */
 298         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 299             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 300                 return;
 301
 302         switch (cmd) {
 303         case SPECTRE_V2_CMD_NONE:
 304                 return;
 305
 306         case SPECTRE_V2_CMD_FORCE:
 307         case SPECTRE_V2_CMD_AUTO:
 308                 if (IS_ENABLED(CONFIG_RETPOLINE))
 309                         goto retpoline_auto;
 310                 break;
 311         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 312                 if (IS_ENABLED(CONFIG_RETPOLINE))
 313                         goto retpoline_amd;
 314                 break;
 315         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 316                 if (IS_ENABLED(CONFIG_RETPOLINE))
 317                         goto retpoline_generic;
 318                 break;
 319         case SPECTRE_V2_CMD_RETPOLINE:
 320                 if (IS_ENABLED(CONFIG_RETPOLINE))
 321                         goto retpoline_auto;
 322                 break;
 323         }
 324         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 325         return;
 326
 327 retpoline_auto:
 328         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 329         retpoline_amd:
 330                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 331                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 332                         goto retpoline_generic;
 333                 }
 334                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
 335                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
 336                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 337                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 338         } else {
 339         retpoline_generic:
 340                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
 341                                          SPECTRE_V2_RETPOLINE_MINIMAL;
 342                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 343         }
 344
 345         spectre_v2_enabled = mode;
 346         pr_info("%s\n", spectre_v2_strings[mode]);
 347
 348         /*
 349          * If neither SMEP nor PTI are available, there is a risk of
 350          * hitting userspace addresses in the RSB after a context switch
 351          * from a shallow call stack to a deeper one. To prevent this fill
 352          * the entire RSB, even when using IBRS.
 353          *
 354          * Skylake era CPUs have a separate issue with *underflow* of the
 355          * RSB, when they will predict 'ret' targets from the generic BTB.
 356          * The proper mitigation for this is IBRS. If IBRS is not supported
 357          * or deactivated in favour of retpolines the RSB fill on context
 358          * switch is required.
 359          */
 360         if ((!boot_cpu_has(X86_FEATURE_PTI) &&
 361              !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) {
 362                 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 363                 pr_info("Spectre v2 mitigation: Filling RSB on context switch\n");
 364         }
 365
 366         /* Initialize Indirect Branch Prediction Barrier if supported */
 367         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 368                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 369                 pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
 370         }
 371
 372         /*
 373          * Retpoline means the kernel is safe because it has no indirect
 374          * branches. But firmware isn't, so use IBRS to protect that.
 375          */
 376         if (boot_cpu_has(X86_FEATURE_IBRS)) {
 377                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 378                 pr_info("Enabling Restricted Speculation for firmware calls\n");
 379         }
 380 }
 381
 382 #undef pr_fmt
 383 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 384
 385 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
 386
 387 /* The kernel command line selection */
 388 enum ssb_mitigation_cmd {
 389         SPEC_STORE_BYPASS_CMD_NONE,
 390         SPEC_STORE_BYPASS_CMD_AUTO,
 391         SPEC_STORE_BYPASS_CMD_ON,
 392         SPEC_STORE_BYPASS_CMD_PRCTL,
 393         SPEC_STORE_BYPASS_CMD_SECCOMP,
 394 };
 395
 396 static const char *ssb_strings[] = {
 397         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 398         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
 399         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
 400         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
 401 };
 402
 403 static const struct {
 404         const char *option;
 405         enum ssb_mitigation_cmd cmd;
 406 } ssb_mitigation_options[] = {
 407         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
 408         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
 409         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
 410         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
 411         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
 412 };
 413
 414 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 415 {
 416         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 417         char arg[20];
 418         int ret, i;
 419
 420         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
 421                 return SPEC_STORE_BYPASS_CMD_NONE;
 422         } else {
 423                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
 424                                           arg, sizeof(arg));
 425                 if (ret < 0)
 426                         return SPEC_STORE_BYPASS_CMD_AUTO;
 427
 428                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
 429                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
 430                                 continue;
 431
 432                         cmd = ssb_mitigation_options[i].cmd;
 433                         break;
 434                 }
 435
 436                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
 437                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 438                         return SPEC_STORE_BYPASS_CMD_AUTO;
 439                 }
 440         }
 441
 442         return cmd;
 443 }
 444
 445 static enum ssb_mitigation __init __ssb_select_mitigation(void)
 446 {
 447         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
 448         enum ssb_mitigation_cmd cmd;
 449
 450         if (!boot_cpu_has(X86_FEATURE_SSBD))
 451                 return mode;
 452
 453         cmd = ssb_parse_cmdline();
 454         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
 455             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
 456              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
 457                 return mode;
 458
 459         switch (cmd) {
 460         case SPEC_STORE_BYPASS_CMD_AUTO:
 461         case SPEC_STORE_BYPASS_CMD_SECCOMP:
 462                 /*
 463                  * Choose prctl+seccomp as the default mode if seccomp is
 464                  * enabled.
 465                  */
 466                 if (IS_ENABLED(CONFIG_SECCOMP))
 467                         mode = SPEC_STORE_BYPASS_SECCOMP;
 468                 else
 469                         mode = SPEC_STORE_BYPASS_PRCTL;
 470                 break;
 471         case SPEC_STORE_BYPASS_CMD_ON:
 472                 mode = SPEC_STORE_BYPASS_DISABLE;
 473                 break;
 474         case SPEC_STORE_BYPASS_CMD_PRCTL:
 475                 mode = SPEC_STORE_BYPASS_PRCTL;
 476                 break;
 477         case SPEC_STORE_BYPASS_CMD_NONE:
 478                 break;
 479         }
 480
 481         /*
 482          * We have three CPU feature flags that are in play here:
 483          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
 484          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
 485          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
 486          */
 487         if (mode == SPEC_STORE_BYPASS_DISABLE) {
 488                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
 489                 /*
 490                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD uses
 491                  * a completely different MSR and bit dependent on family.
 492                  */
 493                 switch (boot_cpu_data.x86_vendor) {
 494                 case X86_VENDOR_INTEL:
 495                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
 496                         x86_spec_ctrl_mask &= ~SPEC_CTRL_SSBD;
 497                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 498                         break;
 499                 case X86_VENDOR_AMD:
 500                         x86_amd_ssb_disable();
 501                         break;
 502                 }
 503         }
 504
 505         return mode;
 506 }
 507
 508 static void ssb_select_mitigation(void)
 509 {
 510         ssb_mode = __ssb_select_mitigation();
 511
 512         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 513                 pr_info("%s\n", ssb_strings[ssb_mode]);
 514 }
 515
 516 #undef pr_fmt
 517 #define pr_fmt(fmt)     "Speculation prctl: " fmt
 518
 519 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
 520 {
 521         bool update;
 522
 523         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
 524             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
 525                 return -ENXIO;
 526
 527         switch (ctrl) {
 528         case PR_SPEC_ENABLE:
 529                 /* If speculation is force disabled, enable is not allowed */
 530                 if (task_spec_ssb_force_disable(task))
 531                         return -EPERM;
 532                 task_clear_spec_ssb_disable(task);
 533                 update = test_and_clear_tsk_thread_flag(task, TIF_SSBD);
 534                 break;
 535         case PR_SPEC_DISABLE:
 536                 task_set_spec_ssb_disable(task);
 537                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 538                 break;
 539         case PR_SPEC_FORCE_DISABLE:
 540                 task_set_spec_ssb_disable(task);
 541                 task_set_spec_ssb_force_disable(task);
 542                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
 543                 break;
 544         default:
 545                 return -ERANGE;
 546         }
 547
 548         /*
 549          * If being set on non-current task, delay setting the CPU
 550          * mitigation until it is next scheduled.
 551          */
 552         if (task == current && update)
 553                 speculative_store_bypass_update_current();
 554
 555         return 0;
 556 }
 557
 558 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
 559                              unsigned long ctrl)
 560 {
 561         switch (which) {
 562         case PR_SPEC_STORE_BYPASS:
 563                 return ssb_prctl_set(task, ctrl);
 564         default:
 565                 return -ENODEV;
 566         }
 567 }
 568
 569 #ifdef CONFIG_SECCOMP
 570 void arch_seccomp_spec_mitigate(struct task_struct *task)
 571 {
 572         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
 573                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
 574 }
 575 #endif
 576
 577 static int ssb_prctl_get(struct task_struct *task)
 578 {
 579         switch (ssb_mode) {
 580         case SPEC_STORE_BYPASS_DISABLE:
 581                 return PR_SPEC_DISABLE;
 582         case SPEC_STORE_BYPASS_SECCOMP:
 583         case SPEC_STORE_BYPASS_PRCTL:
 584                 if (task_spec_ssb_force_disable(task))
 585                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 586                 if (task_spec_ssb_disable(task))
 587                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 588                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 589         default:
 590                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
 591                         return PR_SPEC_ENABLE;
 592                 return PR_SPEC_NOT_AFFECTED;
 593         }
 594 }
 595
 596 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
 597 {
 598         switch (which) {
 599         case PR_SPEC_STORE_BYPASS:
 600                 return ssb_prctl_get(task);
 601         default:
 602                 return -ENODEV;
 603         }
 604 }
 605
 606 void x86_spec_ctrl_setup_ap(void)
 607 {
 608         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
 609                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 610
 611         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
 612                 x86_amd_ssb_disable();
 613 }
 614
 615 #ifdef CONFIG_SYSFS
 616
 617 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
 618                                char *buf, unsigned int bug)
 619 {
 620         if (!boot_cpu_has_bug(bug))
 621                 return sprintf(buf, "Not affected\n");
 622
 623         switch (bug) {
 624         case X86_BUG_CPU_MELTDOWN:
 625                 if (boot_cpu_has(X86_FEATURE_PTI))
 626                         return sprintf(buf, "Mitigation: PTI\n");
 627
 628                 break;
 629
 630         case X86_BUG_SPECTRE_V1:
 631                 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
 632
 633         case X86_BUG_SPECTRE_V2:
 634                 return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
 635                                boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
 636                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
 637                                spectre_v2_module_string());
 638
 639         case X86_BUG_SPEC_STORE_BYPASS:
 640                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
 641
 642         default:
 643                 break;
 644         }
 645
 646         return sprintf(buf, "Vulnerable\n");
 647 }
 648
 649 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
 650 {
 651         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
 652 }
 653
 654 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
 655 {
 656         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
 657 }
 658
 659 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
 660 {
 661         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
 662 }
 663
 664 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
 665 {
 666         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
 667 }
 668 #endif