arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/cpu.h>
  13 #include <linux/module.h>
  14 #include <linux/nospec.h>
  15 #include <linux/prctl.h>
  16 #include <linux/sched/smt.h>
  17 #include <linux/pgtable.h>
  18 #include <linux/bpf.h>
  19
  20 #include <asm/spec-ctrl.h>
  21 #include <asm/cmdline.h>
  22 #include <asm/bugs.h>
  23 #include <asm/processor.h>
  24 #include <asm/processor-flags.h>
  25 #include <asm/fpu/api.h>
  26 #include <asm/msr.h>
  27 #include <asm/vmx.h>
  28 #include <asm/paravirt.h>
  29 #include <asm/intel-family.h>
  30 #include <asm/e820/api.h>
  31 #include <asm/hypervisor.h>
  32 #include <asm/tlbflush.h>
  33 #include <asm/cpu.h>
  34
  35 #include "cpu.h"
  36
  37 static void __init spectre_v1_select_mitigation(void);
  38 static void __init spectre_v2_select_mitigation(void);
  39 static void __init retbleed_select_mitigation(void);
  40 static void __init spectre_v2_user_select_mitigation(void);
  41 static void __init ssb_select_mitigation(void);
  42 static void __init l1tf_select_mitigation(void);
  43 static void __init mds_select_mitigation(void);
  44 static void __init md_clear_update_mitigation(void);
  45 static void __init md_clear_select_mitigation(void);
  46 static void __init taa_select_mitigation(void);
  47 static void __init mmio_select_mitigation(void);
  48 static void __init srbds_select_mitigation(void);
  49 static void __init l1d_flush_select_mitigation(void);
  50 static void __init srso_select_mitigation(void);
  51 static void __init gds_select_mitigation(void);
  52
  53 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
  54 u64 x86_spec_ctrl_base;
  55 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  56
  57 /* The current value of the SPEC_CTRL MSR with task-specific bits set */
  58 DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
  59 EXPORT_PER_CPU_SYMBOL_GPL(x86_spec_ctrl_current);
  60
  61 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
  62 EXPORT_SYMBOL_GPL(x86_pred_cmd);
  63
  64 static u64 __ro_after_init x86_arch_cap_msr;
  65
  66 static DEFINE_MUTEX(spec_ctrl_mutex);
  67
  68 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk;
  69
  70 /* Update SPEC_CTRL MSR and its cached copy unconditionally */
  71 static void update_spec_ctrl(u64 val)
  72 {
  73         this_cpu_write(x86_spec_ctrl_current, val);
  74         wrmsrl(MSR_IA32_SPEC_CTRL, val);
  75 }
  76
  77 /*
  78  * Keep track of the SPEC_CTRL MSR value for the current task, which may differ
  79  * from x86_spec_ctrl_base due to STIBP/SSB in __speculation_ctrl_update().
  80  */
  81 void update_spec_ctrl_cond(u64 val)
  82 {
  83         if (this_cpu_read(x86_spec_ctrl_current) == val)
  84                 return;
  85
  86         this_cpu_write(x86_spec_ctrl_current, val);
  87
  88         /*
  89          * When KERNEL_IBRS this MSR is written on return-to-user, unless
  90          * forced the update can be delayed until that time.
  91          */
  92         if (!cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS))
  93                 wrmsrl(MSR_IA32_SPEC_CTRL, val);
  94 }
  95
  96 noinstr u64 spec_ctrl_current(void)
  97 {
  98         return this_cpu_read(x86_spec_ctrl_current);
  99 }
 100 EXPORT_SYMBOL_GPL(spec_ctrl_current);
 101
 102 /*
 103  * AMD specific MSR info for Speculative Store Bypass control.
 104  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
 105  */
 106 u64 __ro_after_init x86_amd_ls_cfg_base;
 107 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
 108
 109 /* Control conditional STIBP in switch_to() */
 110 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
 111 /* Control conditional IBPB in switch_mm() */
 112 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
 113 /* Control unconditional IBPB in switch_mm() */
 114 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
 115
 116 /* Control MDS CPU buffer clear before idling (halt, mwait) */
 117 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
 118 EXPORT_SYMBOL_GPL(mds_idle_clear);
 119
 120 /*
 121  * Controls whether l1d flush based mitigations are enabled,
 122  * based on hw features and admin setting via boot parameter
 123  * defaults to false
 124  */
 125 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_l1d_flush);
 126
 127 /* Controls CPU Fill buffer clear before KVM guest MMIO accesses */
 128 DEFINE_STATIC_KEY_FALSE(mmio_stale_data_clear);
 129 EXPORT_SYMBOL_GPL(mmio_stale_data_clear);
 130
 131 void __init cpu_select_mitigations(void)
 132 {
 133         /*
 134          * Read the SPEC_CTRL MSR to account for reserved bits which may
 135          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
 136          * init code as it is not enumerated and depends on the family.
 137          */
 138         if (cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL)) {
 139                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 140
 141                 /*
 142                  * Previously running kernel (kexec), may have some controls
 143                  * turned ON. Clear them and let the mitigations setup below
 144                  * rediscover them based on configuration.
 145                  */
 146                 x86_spec_ctrl_base &= ~SPEC_CTRL_MITIGATIONS_MASK;
 147         }
 148
 149         x86_arch_cap_msr = x86_read_arch_cap_msr();
 150
 151         /* Select the proper CPU mitigations before patching alternatives: */
 152         spectre_v1_select_mitigation();
 153         spectre_v2_select_mitigation();
 154         /*
 155          * retbleed_select_mitigation() relies on the state set by
 156          * spectre_v2_select_mitigation(); specifically it wants to know about
 157          * spectre_v2=ibrs.
 158          */
 159         retbleed_select_mitigation();
 160         /*
 161          * spectre_v2_user_select_mitigation() relies on the state set by
 162          * retbleed_select_mitigation(); specifically the STIBP selection is
 163          * forced for UNRET or IBPB.
 164          */
 165         spectre_v2_user_select_mitigation();
 166         ssb_select_mitigation();
 167         l1tf_select_mitigation();
 168         md_clear_select_mitigation();
 169         srbds_select_mitigation();
 170         l1d_flush_select_mitigation();
 171
 172         /*
 173          * srso_select_mitigation() depends and must run after
 174          * retbleed_select_mitigation().
 175          */
 176         srso_select_mitigation();
 177         gds_select_mitigation();
 178 }
 179
 180 /*
 181  * NOTE: This function is *only* called for SVM, since Intel uses
 182  * MSR_IA32_SPEC_CTRL for SSBD.
 183  */
 184 void
 185 x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest)
 186 {
 187         u64 guestval, hostval;
 188         struct thread_info *ti = current_thread_info();
 189
 190         /*
 191          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 192          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 193          */
 194         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 195             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 196                 return;
 197
 198         /*
 199          * If the host has SSBD mitigation enabled, force it in the host's
 200          * virtual MSR value. If its not permanently enabled, evaluate
 201          * current's TIF_SSBD thread flag.
 202          */
 203         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 204                 hostval = SPEC_CTRL_SSBD;
 205         else
 206                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 207
 208         /* Sanitize the guest value */
 209         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 210
 211         if (hostval != guestval) {
 212                 unsigned long tif;
 213
 214                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 215                                  ssbd_spec_ctrl_to_tif(hostval);
 216
 217                 speculation_ctrl_update(tif);
 218         }
 219 }
 220 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 221
 222 static void x86_amd_ssb_disable(void)
 223 {
 224         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 225
 226         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 227                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 228         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 229                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 230 }
 231
 232 #undef pr_fmt
 233 #define pr_fmt(fmt)     "MDS: " fmt
 234
 235 /* Default mitigation for MDS-affected CPUs */
 236 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
 237 static bool mds_nosmt __ro_after_init = false;
 238
 239 static const char * const mds_strings[] = {
 240         [MDS_MITIGATION_OFF]    = "Vulnerable",
 241         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
 242         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
 243 };
 244
 245 static void __init mds_select_mitigation(void)
 246 {
 247         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
 248                 mds_mitigation = MDS_MITIGATION_OFF;
 249                 return;
 250         }
 251
 252         if (mds_mitigation == MDS_MITIGATION_FULL) {
 253                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
 254                         mds_mitigation = MDS_MITIGATION_VMWERV;
 255
 256                 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
 257
 258                 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
 259                     (mds_nosmt || cpu_mitigations_auto_nosmt()))
 260                         cpu_smt_disable(false);
 261         }
 262 }
 263
 264 static int __init mds_cmdline(char *str)
 265 {
 266         if (!boot_cpu_has_bug(X86_BUG_MDS))
 267                 return 0;
 268
 269         if (!str)
 270                 return -EINVAL;
 271
 272         if (!strcmp(str, "off"))
 273                 mds_mitigation = MDS_MITIGATION_OFF;
 274         else if (!strcmp(str, "full"))
 275                 mds_mitigation = MDS_MITIGATION_FULL;
 276         else if (!strcmp(str, "full,nosmt")) {
 277                 mds_mitigation = MDS_MITIGATION_FULL;
 278                 mds_nosmt = true;
 279         }
 280
 281         return 0;
 282 }
 283 early_param("mds", mds_cmdline);
 284
 285 #undef pr_fmt
 286 #define pr_fmt(fmt)     "TAA: " fmt
 287
 288 enum taa_mitigations {
 289         TAA_MITIGATION_OFF,
 290         TAA_MITIGATION_UCODE_NEEDED,
 291         TAA_MITIGATION_VERW,
 292         TAA_MITIGATION_TSX_DISABLED,
 293 };
 294
 295 /* Default mitigation for TAA-affected CPUs */
 296 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
 297 static bool taa_nosmt __ro_after_init;
 298
 299 static const char * const taa_strings[] = {
 300         [TAA_MITIGATION_OFF]            = "Vulnerable",
 301         [TAA_MITIGATION_UCODE_NEEDED]   = "Vulnerable: Clear CPU buffers attempted, no microcode",
 302         [TAA_MITIGATION_VERW]           = "Mitigation: Clear CPU buffers",
 303         [TAA_MITIGATION_TSX_DISABLED]   = "Mitigation: TSX disabled",
 304 };
 305
 306 static void __init taa_select_mitigation(void)
 307 {
 308         if (!boot_cpu_has_bug(X86_BUG_TAA)) {
 309                 taa_mitigation = TAA_MITIGATION_OFF;
 310                 return;
 311         }
 312
 313         /* TSX previously disabled by tsx=off */
 314         if (!boot_cpu_has(X86_FEATURE_RTM)) {
 315                 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
 316                 return;
 317         }
 318
 319         if (cpu_mitigations_off()) {
 320                 taa_mitigation = TAA_MITIGATION_OFF;
 321                 return;
 322         }
 323
 324         /*
 325          * TAA mitigation via VERW is turned off if both
 326          * tsx_async_abort=off and mds=off are specified.
 327          */
 328         if (taa_mitigation == TAA_MITIGATION_OFF &&
 329             mds_mitigation == MDS_MITIGATION_OFF)
 330                 return;
 331
 332         if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
 333                 taa_mitigation = TAA_MITIGATION_VERW;
 334         else
 335                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 336
 337         /*
 338          * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
 339          * A microcode update fixes this behavior to clear CPU buffers. It also
 340          * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
 341          * ARCH_CAP_TSX_CTRL_MSR bit.
 342          *
 343          * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
 344          * update is required.
 345          */
 346         if ( (x86_arch_cap_msr & ARCH_CAP_MDS_NO) &&
 347             !(x86_arch_cap_msr & ARCH_CAP_TSX_CTRL_MSR))
 348                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 349
 350         /*
 351          * TSX is enabled, select alternate mitigation for TAA which is
 352          * the same as MDS. Enable MDS static branch to clear CPU buffers.
 353          *
 354          * For guests that can't determine whether the correct microcode is
 355          * present on host, enable the mitigation for UCODE_NEEDED as well.
 356          */
 357         setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
 358
 359         if (taa_nosmt || cpu_mitigations_auto_nosmt())
 360                 cpu_smt_disable(false);
 361 }
 362
 363 static int __init tsx_async_abort_parse_cmdline(char *str)
 364 {
 365         if (!boot_cpu_has_bug(X86_BUG_TAA))
 366                 return 0;
 367
 368         if (!str)
 369                 return -EINVAL;
 370
 371         if (!strcmp(str, "off")) {
 372                 taa_mitigation = TAA_MITIGATION_OFF;
 373         } else if (!strcmp(str, "full")) {
 374                 taa_mitigation = TAA_MITIGATION_VERW;
 375         } else if (!strcmp(str, "full,nosmt")) {
 376                 taa_mitigation = TAA_MITIGATION_VERW;
 377                 taa_nosmt = true;
 378         }
 379
 380         return 0;
 381 }
 382 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
 383
 384 #undef pr_fmt
 385 #define pr_fmt(fmt)     "MMIO Stale Data: " fmt
 386
 387 enum mmio_mitigations {
 388         MMIO_MITIGATION_OFF,
 389         MMIO_MITIGATION_UCODE_NEEDED,
 390         MMIO_MITIGATION_VERW,
 391 };
 392
 393 /* Default mitigation for Processor MMIO Stale Data vulnerabilities */
 394 static enum mmio_mitigations mmio_mitigation __ro_after_init = MMIO_MITIGATION_VERW;
 395 static bool mmio_nosmt __ro_after_init = false;
 396
 397 static const char * const mmio_strings[] = {
 398         [MMIO_MITIGATION_OFF]           = "Vulnerable",
 399         [MMIO_MITIGATION_UCODE_NEEDED]  = "Vulnerable: Clear CPU buffers attempted, no microcode",
 400         [MMIO_MITIGATION_VERW]          = "Mitigation: Clear CPU buffers",
 401 };
 402
 403 static void __init mmio_select_mitigation(void)
 404 {
 405         if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) ||
 406              boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) ||
 407              cpu_mitigations_off()) {
 408                 mmio_mitigation = MMIO_MITIGATION_OFF;
 409                 return;
 410         }
 411
 412         if (mmio_mitigation == MMIO_MITIGATION_OFF)
 413                 return;
 414
 415         /*
 416          * Enable CPU buffer clear mitigation for host and VMM, if also affected
 417          * by MDS or TAA. Otherwise, enable mitigation for VMM only.
 418          */
 419         if (boot_cpu_has_bug(X86_BUG_MDS) || (boot_cpu_has_bug(X86_BUG_TAA) &&
 420                                               boot_cpu_has(X86_FEATURE_RTM)))
 421                 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
 422
 423         /*
 424          * X86_FEATURE_CLEAR_CPU_BUF could be enabled by other VERW based
 425          * mitigations, disable KVM-only mitigation in that case.
 426          */
 427         if (boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF))
 428                 static_branch_disable(&mmio_stale_data_clear);
 429         else
 430                 static_branch_enable(&mmio_stale_data_clear);
 431
 432         /*
 433          * If Processor-MMIO-Stale-Data bug is present and Fill Buffer data can
 434          * be propagated to uncore buffers, clearing the Fill buffers on idle
 435          * is required irrespective of SMT state.
 436          */
 437         if (!(x86_arch_cap_msr & ARCH_CAP_FBSDP_NO))
 438                 static_branch_enable(&mds_idle_clear);
 439
 440         /*
 441          * Check if the system has the right microcode.
 442          *
 443          * CPU Fill buffer clear mitigation is enumerated by either an explicit
 444          * FB_CLEAR or by the presence of both MD_CLEAR and L1D_FLUSH on MDS
 445          * affected systems.
 446          */
 447         if ((x86_arch_cap_msr & ARCH_CAP_FB_CLEAR) ||
 448             (boot_cpu_has(X86_FEATURE_MD_CLEAR) &&
 449              boot_cpu_has(X86_FEATURE_FLUSH_L1D) &&
 450              !(x86_arch_cap_msr & ARCH_CAP_MDS_NO)))
 451                 mmio_mitigation = MMIO_MITIGATION_VERW;
 452         else
 453                 mmio_mitigation = MMIO_MITIGATION_UCODE_NEEDED;
 454
 455         if (mmio_nosmt || cpu_mitigations_auto_nosmt())
 456                 cpu_smt_disable(false);
 457 }
 458
 459 static int __init mmio_stale_data_parse_cmdline(char *str)
 460 {
 461         if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 462                 return 0;
 463
 464         if (!str)
 465                 return -EINVAL;
 466
 467         if (!strcmp(str, "off")) {
 468                 mmio_mitigation = MMIO_MITIGATION_OFF;
 469         } else if (!strcmp(str, "full")) {
 470                 mmio_mitigation = MMIO_MITIGATION_VERW;
 471         } else if (!strcmp(str, "full,nosmt")) {
 472                 mmio_mitigation = MMIO_MITIGATION_VERW;
 473                 mmio_nosmt = true;
 474         }
 475
 476         return 0;
 477 }
 478 early_param("mmio_stale_data", mmio_stale_data_parse_cmdline);
 479
 480 #undef pr_fmt
 481 #define pr_fmt(fmt)     "Register File Data Sampling: " fmt
 482
 483 enum rfds_mitigations {
 484         RFDS_MITIGATION_OFF,
 485         RFDS_MITIGATION_VERW,
 486         RFDS_MITIGATION_UCODE_NEEDED,
 487 };
 488
 489 /* Default mitigation for Register File Data Sampling */
 490 static enum rfds_mitigations rfds_mitigation __ro_after_init =
 491         IS_ENABLED(CONFIG_MITIGATION_RFDS) ? RFDS_MITIGATION_VERW : RFDS_MITIGATION_OFF;
 492
 493 static const char * const rfds_strings[] = {
 494         [RFDS_MITIGATION_OFF]                   = "Vulnerable",
 495         [RFDS_MITIGATION_VERW]                  = "Mitigation: Clear Register File",
 496         [RFDS_MITIGATION_UCODE_NEEDED]          = "Vulnerable: No microcode",
 497 };
 498
 499 static void __init rfds_select_mitigation(void)
 500 {
 501         if (!boot_cpu_has_bug(X86_BUG_RFDS) || cpu_mitigations_off()) {
 502                 rfds_mitigation = RFDS_MITIGATION_OFF;
 503                 return;
 504         }
 505         if (rfds_mitigation == RFDS_MITIGATION_OFF)
 506                 return;
 507
 508         if (x86_arch_cap_msr & ARCH_CAP_RFDS_CLEAR)
 509                 setup_force_cpu_cap(X86_FEATURE_CLEAR_CPU_BUF);
 510         else
 511                 rfds_mitigation = RFDS_MITIGATION_UCODE_NEEDED;
 512 }
 513
 514 static __init int rfds_parse_cmdline(char *str)
 515 {
 516         if (!str)
 517                 return -EINVAL;
 518
 519         if (!boot_cpu_has_bug(X86_BUG_RFDS))
 520                 return 0;
 521
 522         if (!strcmp(str, "off"))
 523                 rfds_mitigation = RFDS_MITIGATION_OFF;
 524         else if (!strcmp(str, "on"))
 525                 rfds_mitigation = RFDS_MITIGATION_VERW;
 526
 527         return 0;
 528 }
 529 early_param("reg_file_data_sampling", rfds_parse_cmdline);
 530
 531 #undef pr_fmt
 532 #define pr_fmt(fmt)     "" fmt
 533
 534 static void __init md_clear_update_mitigation(void)
 535 {
 536         if (cpu_mitigations_off())
 537                 return;
 538
 539         if (!boot_cpu_has(X86_FEATURE_CLEAR_CPU_BUF))
 540                 goto out;
 541
 542         /*
 543          * X86_FEATURE_CLEAR_CPU_BUF is now enabled. Update MDS, TAA and MMIO
 544          * Stale Data mitigation, if necessary.
 545          */
 546         if (mds_mitigation == MDS_MITIGATION_OFF &&
 547             boot_cpu_has_bug(X86_BUG_MDS)) {
 548                 mds_mitigation = MDS_MITIGATION_FULL;
 549                 mds_select_mitigation();
 550         }
 551         if (taa_mitigation == TAA_MITIGATION_OFF &&
 552             boot_cpu_has_bug(X86_BUG_TAA)) {
 553                 taa_mitigation = TAA_MITIGATION_VERW;
 554                 taa_select_mitigation();
 555         }
 556         /*
 557          * MMIO_MITIGATION_OFF is not checked here so that mmio_stale_data_clear
 558          * gets updated correctly as per X86_FEATURE_CLEAR_CPU_BUF state.
 559          */
 560         if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) {
 561                 mmio_mitigation = MMIO_MITIGATION_VERW;
 562                 mmio_select_mitigation();
 563         }
 564         if (rfds_mitigation == RFDS_MITIGATION_OFF &&
 565             boot_cpu_has_bug(X86_BUG_RFDS)) {
 566                 rfds_mitigation = RFDS_MITIGATION_VERW;
 567                 rfds_select_mitigation();
 568         }
 569 out:
 570         if (boot_cpu_has_bug(X86_BUG_MDS))
 571                 pr_info("MDS: %s\n", mds_strings[mds_mitigation]);
 572         if (boot_cpu_has_bug(X86_BUG_TAA))
 573                 pr_info("TAA: %s\n", taa_strings[taa_mitigation]);
 574         if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 575                 pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]);
 576         else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
 577                 pr_info("MMIO Stale Data: Unknown: No mitigations\n");
 578         if (boot_cpu_has_bug(X86_BUG_RFDS))
 579                 pr_info("Register File Data Sampling: %s\n", rfds_strings[rfds_mitigation]);
 580 }
 581
 582 static void __init md_clear_select_mitigation(void)
 583 {
 584         mds_select_mitigation();
 585         taa_select_mitigation();
 586         mmio_select_mitigation();
 587         rfds_select_mitigation();
 588
 589         /*
 590          * As these mitigations are inter-related and rely on VERW instruction
 591          * to clear the microarchitural buffers, update and print their status
 592          * after mitigation selection is done for each of these vulnerabilities.
 593          */
 594         md_clear_update_mitigation();
 595 }
 596
 597 #undef pr_fmt
 598 #define pr_fmt(fmt)     "SRBDS: " fmt
 599
 600 enum srbds_mitigations {
 601         SRBDS_MITIGATION_OFF,
 602         SRBDS_MITIGATION_UCODE_NEEDED,
 603         SRBDS_MITIGATION_FULL,
 604         SRBDS_MITIGATION_TSX_OFF,
 605         SRBDS_MITIGATION_HYPERVISOR,
 606 };
 607
 608 static enum srbds_mitigations srbds_mitigation __ro_after_init = SRBDS_MITIGATION_FULL;
 609
 610 static const char * const srbds_strings[] = {
 611         [SRBDS_MITIGATION_OFF]          = "Vulnerable",
 612         [SRBDS_MITIGATION_UCODE_NEEDED] = "Vulnerable: No microcode",
 613         [SRBDS_MITIGATION_FULL]         = "Mitigation: Microcode",
 614         [SRBDS_MITIGATION_TSX_OFF]      = "Mitigation: TSX disabled",
 615         [SRBDS_MITIGATION_HYPERVISOR]   = "Unknown: Dependent on hypervisor status",
 616 };
 617
 618 static bool srbds_off;
 619
 620 void update_srbds_msr(void)
 621 {
 622         u64 mcu_ctrl;
 623
 624         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 625                 return;
 626
 627         if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 628                 return;
 629
 630         if (srbds_mitigation == SRBDS_MITIGATION_UCODE_NEEDED)
 631                 return;
 632
 633         /*
 634          * A MDS_NO CPU for which SRBDS mitigation is not needed due to TSX
 635          * being disabled and it hasn't received the SRBDS MSR microcode.
 636          */
 637         if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 638                 return;
 639
 640         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 641
 642         switch (srbds_mitigation) {
 643         case SRBDS_MITIGATION_OFF:
 644         case SRBDS_MITIGATION_TSX_OFF:
 645                 mcu_ctrl |= RNGDS_MITG_DIS;
 646                 break;
 647         case SRBDS_MITIGATION_FULL:
 648                 mcu_ctrl &= ~RNGDS_MITG_DIS;
 649                 break;
 650         default:
 651                 break;
 652         }
 653
 654         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 655 }
 656
 657 static void __init srbds_select_mitigation(void)
 658 {
 659         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 660                 return;
 661
 662         /*
 663          * Check to see if this is one of the MDS_NO systems supporting TSX that
 664          * are only exposed to SRBDS when TSX is enabled or when CPU is affected
 665          * by Processor MMIO Stale Data vulnerability.
 666          */
 667         if ((x86_arch_cap_msr & ARCH_CAP_MDS_NO) && !boot_cpu_has(X86_FEATURE_RTM) &&
 668             !boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA))
 669                 srbds_mitigation = SRBDS_MITIGATION_TSX_OFF;
 670         else if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 671                 srbds_mitigation = SRBDS_MITIGATION_HYPERVISOR;
 672         else if (!boot_cpu_has(X86_FEATURE_SRBDS_CTRL))
 673                 srbds_mitigation = SRBDS_MITIGATION_UCODE_NEEDED;
 674         else if (cpu_mitigations_off() || srbds_off)
 675                 srbds_mitigation = SRBDS_MITIGATION_OFF;
 676
 677         update_srbds_msr();
 678         pr_info("%s\n", srbds_strings[srbds_mitigation]);
 679 }
 680
 681 static int __init srbds_parse_cmdline(char *str)
 682 {
 683         if (!str)
 684                 return -EINVAL;
 685
 686         if (!boot_cpu_has_bug(X86_BUG_SRBDS))
 687                 return 0;
 688
 689         srbds_off = !strcmp(str, "off");
 690         return 0;
 691 }
 692 early_param("srbds", srbds_parse_cmdline);
 693
 694 #undef pr_fmt
 695 #define pr_fmt(fmt)     "L1D Flush : " fmt
 696
 697 enum l1d_flush_mitigations {
 698         L1D_FLUSH_OFF = 0,
 699         L1D_FLUSH_ON,
 700 };
 701
 702 static enum l1d_flush_mitigations l1d_flush_mitigation __initdata = L1D_FLUSH_OFF;
 703
 704 static void __init l1d_flush_select_mitigation(void)
 705 {
 706         if (!l1d_flush_mitigation || !boot_cpu_has(X86_FEATURE_FLUSH_L1D))
 707                 return;
 708
 709         static_branch_enable(&switch_mm_cond_l1d_flush);
 710         pr_info("Conditional flush on switch_mm() enabled\n");
 711 }
 712
 713 static int __init l1d_flush_parse_cmdline(char *str)
 714 {
 715         if (!strcmp(str, "on"))
 716                 l1d_flush_mitigation = L1D_FLUSH_ON;
 717
 718         return 0;
 719 }
 720 early_param("l1d_flush", l1d_flush_parse_cmdline);
 721
 722 #undef pr_fmt
 723 #define pr_fmt(fmt)     "GDS: " fmt
 724
 725 enum gds_mitigations {
 726         GDS_MITIGATION_OFF,
 727         GDS_MITIGATION_UCODE_NEEDED,
 728         GDS_MITIGATION_FORCE,
 729         GDS_MITIGATION_FULL,
 730         GDS_MITIGATION_FULL_LOCKED,
 731         GDS_MITIGATION_HYPERVISOR,
 732 };
 733
 734 #if IS_ENABLED(CONFIG_MITIGATION_GDS_FORCE)
 735 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FORCE;
 736 #else
 737 static enum gds_mitigations gds_mitigation __ro_after_init = GDS_MITIGATION_FULL;
 738 #endif
 739
 740 static const char * const gds_strings[] = {
 741         [GDS_MITIGATION_OFF]            = "Vulnerable",
 742         [GDS_MITIGATION_UCODE_NEEDED]   = "Vulnerable: No microcode",
 743         [GDS_MITIGATION_FORCE]          = "Mitigation: AVX disabled, no microcode",
 744         [GDS_MITIGATION_FULL]           = "Mitigation: Microcode",
 745         [GDS_MITIGATION_FULL_LOCKED]    = "Mitigation: Microcode (locked)",
 746         [GDS_MITIGATION_HYPERVISOR]     = "Unknown: Dependent on hypervisor status",
 747 };
 748
 749 bool gds_ucode_mitigated(void)
 750 {
 751         return (gds_mitigation == GDS_MITIGATION_FULL ||
 752                 gds_mitigation == GDS_MITIGATION_FULL_LOCKED);
 753 }
 754 EXPORT_SYMBOL_GPL(gds_ucode_mitigated);
 755
 756 void update_gds_msr(void)
 757 {
 758         u64 mcu_ctrl_after;
 759         u64 mcu_ctrl;
 760
 761         switch (gds_mitigation) {
 762         case GDS_MITIGATION_OFF:
 763                 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 764                 mcu_ctrl |= GDS_MITG_DIS;
 765                 break;
 766         case GDS_MITIGATION_FULL_LOCKED:
 767                 /*
 768                  * The LOCKED state comes from the boot CPU. APs might not have
 769                  * the same state. Make sure the mitigation is enabled on all
 770                  * CPUs.
 771                  */
 772         case GDS_MITIGATION_FULL:
 773                 rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 774                 mcu_ctrl &= ~GDS_MITG_DIS;
 775                 break;
 776         case GDS_MITIGATION_FORCE:
 777         case GDS_MITIGATION_UCODE_NEEDED:
 778         case GDS_MITIGATION_HYPERVISOR:
 779                 return;
 780         }
 781
 782         wrmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 783
 784         /*
 785          * Check to make sure that the WRMSR value was not ignored. Writes to
 786          * GDS_MITG_DIS will be ignored if this processor is locked but the boot
 787          * processor was not.
 788          */
 789         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl_after);
 790         WARN_ON_ONCE(mcu_ctrl != mcu_ctrl_after);
 791 }
 792
 793 static void __init gds_select_mitigation(void)
 794 {
 795         u64 mcu_ctrl;
 796
 797         if (!boot_cpu_has_bug(X86_BUG_GDS))
 798                 return;
 799
 800         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
 801                 gds_mitigation = GDS_MITIGATION_HYPERVISOR;
 802                 goto out;
 803         }
 804
 805         if (cpu_mitigations_off())
 806                 gds_mitigation = GDS_MITIGATION_OFF;
 807         /* Will verify below that mitigation _can_ be disabled */
 808
 809         /* No microcode */
 810         if (!(x86_arch_cap_msr & ARCH_CAP_GDS_CTRL)) {
 811                 if (gds_mitigation == GDS_MITIGATION_FORCE) {
 812                         /*
 813                          * This only needs to be done on the boot CPU so do it
 814                          * here rather than in update_gds_msr()
 815                          */
 816                         setup_clear_cpu_cap(X86_FEATURE_AVX);
 817                         pr_warn("Microcode update needed! Disabling AVX as mitigation.\n");
 818                 } else {
 819                         gds_mitigation = GDS_MITIGATION_UCODE_NEEDED;
 820                 }
 821                 goto out;
 822         }
 823
 824         /* Microcode has mitigation, use it */
 825         if (gds_mitigation == GDS_MITIGATION_FORCE)
 826                 gds_mitigation = GDS_MITIGATION_FULL;
 827
 828         rdmsrl(MSR_IA32_MCU_OPT_CTRL, mcu_ctrl);
 829         if (mcu_ctrl & GDS_MITG_LOCKED) {
 830                 if (gds_mitigation == GDS_MITIGATION_OFF)
 831                         pr_warn("Mitigation locked. Disable failed.\n");
 832
 833                 /*
 834                  * The mitigation is selected from the boot CPU. All other CPUs
 835                  * _should_ have the same state. If the boot CPU isn't locked
 836                  * but others are then update_gds_msr() will WARN() of the state
 837                  * mismatch. If the boot CPU is locked update_gds_msr() will
 838                  * ensure the other CPUs have the mitigation enabled.
 839                  */
 840                 gds_mitigation = GDS_MITIGATION_FULL_LOCKED;
 841         }
 842
 843         update_gds_msr();
 844 out:
 845         pr_info("%s\n", gds_strings[gds_mitigation]);
 846 }
 847
 848 static int __init gds_parse_cmdline(char *str)
 849 {
 850         if (!str)
 851                 return -EINVAL;
 852
 853         if (!boot_cpu_has_bug(X86_BUG_GDS))
 854                 return 0;
 855
 856         if (!strcmp(str, "off"))
 857                 gds_mitigation = GDS_MITIGATION_OFF;
 858         else if (!strcmp(str, "force"))
 859                 gds_mitigation = GDS_MITIGATION_FORCE;
 860
 861         return 0;
 862 }
 863 early_param("gather_data_sampling", gds_parse_cmdline);
 864
 865 #undef pr_fmt
 866 #define pr_fmt(fmt)     "Spectre V1 : " fmt
 867
 868 enum spectre_v1_mitigation {
 869         SPECTRE_V1_MITIGATION_NONE,
 870         SPECTRE_V1_MITIGATION_AUTO,
 871 };
 872
 873 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
 874         SPECTRE_V1_MITIGATION_AUTO;
 875
 876 static const char * const spectre_v1_strings[] = {
 877         [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
 878         [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
 879 };
 880
 881 /*
 882  * Does SMAP provide full mitigation against speculative kernel access to
 883  * userspace?
 884  */
 885 static bool smap_works_speculatively(void)
 886 {
 887         if (!boot_cpu_has(X86_FEATURE_SMAP))
 888                 return false;
 889
 890         /*
 891          * On CPUs which are vulnerable to Meltdown, SMAP does not
 892          * prevent speculative access to user data in the L1 cache.
 893          * Consider SMAP to be non-functional as a mitigation on these
 894          * CPUs.
 895          */
 896         if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
 897                 return false;
 898
 899         return true;
 900 }
 901
 902 static void __init spectre_v1_select_mitigation(void)
 903 {
 904         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
 905                 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 906                 return;
 907         }
 908
 909         if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
 910                 /*
 911                  * With Spectre v1, a user can speculatively control either
 912                  * path of a conditional swapgs with a user-controlled GS
 913                  * value.  The mitigation is to add lfences to both code paths.
 914                  *
 915                  * If FSGSBASE is enabled, the user can put a kernel address in
 916                  * GS, in which case SMAP provides no protection.
 917                  *
 918                  * If FSGSBASE is disabled, the user can only put a user space
 919                  * address in GS.  That makes an attack harder, but still
 920                  * possible if there's no SMAP protection.
 921                  */
 922                 if (boot_cpu_has(X86_FEATURE_FSGSBASE) ||
 923                     !smap_works_speculatively()) {
 924                         /*
 925                          * Mitigation can be provided from SWAPGS itself or
 926                          * PTI as the CR3 write in the Meltdown mitigation
 927                          * is serializing.
 928                          *
 929                          * If neither is there, mitigate with an LFENCE to
 930                          * stop speculation through swapgs.
 931                          */
 932                         if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
 933                             !boot_cpu_has(X86_FEATURE_PTI))
 934                                 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
 935
 936                         /*
 937                          * Enable lfences in the kernel entry (non-swapgs)
 938                          * paths, to prevent user entry from speculatively
 939                          * skipping swapgs.
 940                          */
 941                         setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
 942                 }
 943         }
 944
 945         pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
 946 }
 947
 948 static int __init nospectre_v1_cmdline(char *str)
 949 {
 950         spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 951         return 0;
 952 }
 953 early_param("nospectre_v1", nospectre_v1_cmdline);
 954
 955 enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE;
 956
 957 #undef pr_fmt
 958 #define pr_fmt(fmt)     "RETBleed: " fmt
 959
 960 enum retbleed_mitigation {
 961         RETBLEED_MITIGATION_NONE,
 962         RETBLEED_MITIGATION_UNRET,
 963         RETBLEED_MITIGATION_IBPB,
 964         RETBLEED_MITIGATION_IBRS,
 965         RETBLEED_MITIGATION_EIBRS,
 966         RETBLEED_MITIGATION_STUFF,
 967 };
 968
 969 enum retbleed_mitigation_cmd {
 970         RETBLEED_CMD_OFF,
 971         RETBLEED_CMD_AUTO,
 972         RETBLEED_CMD_UNRET,
 973         RETBLEED_CMD_IBPB,
 974         RETBLEED_CMD_STUFF,
 975 };
 976
 977 static const char * const retbleed_strings[] = {
 978         [RETBLEED_MITIGATION_NONE]      = "Vulnerable",
 979         [RETBLEED_MITIGATION_UNRET]     = "Mitigation: untrained return thunk",
 980         [RETBLEED_MITIGATION_IBPB]      = "Mitigation: IBPB",
 981         [RETBLEED_MITIGATION_IBRS]      = "Mitigation: IBRS",
 982         [RETBLEED_MITIGATION_EIBRS]     = "Mitigation: Enhanced IBRS",
 983         [RETBLEED_MITIGATION_STUFF]     = "Mitigation: Stuffing",
 984 };
 985
 986 static enum retbleed_mitigation retbleed_mitigation __ro_after_init =
 987         RETBLEED_MITIGATION_NONE;
 988 static enum retbleed_mitigation_cmd retbleed_cmd __ro_after_init =
 989         RETBLEED_CMD_AUTO;
 990
 991 static int __ro_after_init retbleed_nosmt = false;
 992
 993 static int __init retbleed_parse_cmdline(char *str)
 994 {
 995         if (!str)
 996                 return -EINVAL;
 997
 998         while (str) {
 999                 char *next = strchr(str, ',');
1000                 if (next) {
1001                         *next = 0;
1002                         next++;
1003                 }
1004
1005                 if (!strcmp(str, "off")) {
1006                         retbleed_cmd = RETBLEED_CMD_OFF;
1007                 } else if (!strcmp(str, "auto")) {
1008                         retbleed_cmd = RETBLEED_CMD_AUTO;
1009                 } else if (!strcmp(str, "unret")) {
1010                         retbleed_cmd = RETBLEED_CMD_UNRET;
1011                 } else if (!strcmp(str, "ibpb")) {
1012                         retbleed_cmd = RETBLEED_CMD_IBPB;
1013                 } else if (!strcmp(str, "stuff")) {
1014                         retbleed_cmd = RETBLEED_CMD_STUFF;
1015                 } else if (!strcmp(str, "nosmt")) {
1016                         retbleed_nosmt = true;
1017                 } else if (!strcmp(str, "force")) {
1018                         setup_force_cpu_bug(X86_BUG_RETBLEED);
1019                 } else {
1020                         pr_err("Ignoring unknown retbleed option (%s).", str);
1021                 }
1022
1023                 str = next;
1024         }
1025
1026         return 0;
1027 }
1028 early_param("retbleed", retbleed_parse_cmdline);
1029
1030 #define RETBLEED_UNTRAIN_MSG "WARNING: BTB untrained return thunk mitigation is only effective on AMD/Hygon!\n"
1031 #define RETBLEED_INTEL_MSG "WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!\n"
1032
1033 static void __init retbleed_select_mitigation(void)
1034 {
1035         bool mitigate_smt = false;
1036
1037         if (!boot_cpu_has_bug(X86_BUG_RETBLEED) || cpu_mitigations_off())
1038                 return;
1039
1040         switch (retbleed_cmd) {
1041         case RETBLEED_CMD_OFF:
1042                 return;
1043
1044         case RETBLEED_CMD_UNRET:
1045                 if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY)) {
1046                         retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
1047                 } else {
1048                         pr_err("WARNING: kernel not compiled with MITIGATION_UNRET_ENTRY.\n");
1049                         goto do_cmd_auto;
1050                 }
1051                 break;
1052
1053         case RETBLEED_CMD_IBPB:
1054                 if (!boot_cpu_has(X86_FEATURE_IBPB)) {
1055                         pr_err("WARNING: CPU does not support IBPB.\n");
1056                         goto do_cmd_auto;
1057                 } else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
1058                         retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
1059                 } else {
1060                         pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
1061                         goto do_cmd_auto;
1062                 }
1063                 break;
1064
1065         case RETBLEED_CMD_STUFF:
1066                 if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING) &&
1067                     spectre_v2_enabled == SPECTRE_V2_RETPOLINE) {
1068                         retbleed_mitigation = RETBLEED_MITIGATION_STUFF;
1069
1070                 } else {
1071                         if (IS_ENABLED(CONFIG_MITIGATION_CALL_DEPTH_TRACKING))
1072                                 pr_err("WARNING: retbleed=stuff depends on spectre_v2=retpoline\n");
1073                         else
1074                                 pr_err("WARNING: kernel not compiled with MITIGATION_CALL_DEPTH_TRACKING.\n");
1075
1076                         goto do_cmd_auto;
1077                 }
1078                 break;
1079
1080 do_cmd_auto:
1081         case RETBLEED_CMD_AUTO:
1082                 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1083                     boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
1084                         if (IS_ENABLED(CONFIG_MITIGATION_UNRET_ENTRY))
1085                                 retbleed_mitigation = RETBLEED_MITIGATION_UNRET;
1086                         else if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY) &&
1087                                  boot_cpu_has(X86_FEATURE_IBPB))
1088                                 retbleed_mitigation = RETBLEED_MITIGATION_IBPB;
1089                 }
1090
1091                 /*
1092                  * The Intel mitigation (IBRS or eIBRS) was already selected in
1093                  * spectre_v2_select_mitigation().  'retbleed_mitigation' will
1094                  * be set accordingly below.
1095                  */
1096
1097                 break;
1098         }
1099
1100         switch (retbleed_mitigation) {
1101         case RETBLEED_MITIGATION_UNRET:
1102                 setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1103                 setup_force_cpu_cap(X86_FEATURE_UNRET);
1104
1105                 x86_return_thunk = retbleed_return_thunk;
1106
1107                 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
1108                     boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
1109                         pr_err(RETBLEED_UNTRAIN_MSG);
1110
1111                 mitigate_smt = true;
1112                 break;
1113
1114         case RETBLEED_MITIGATION_IBPB:
1115                 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
1116                 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
1117                 mitigate_smt = true;
1118                 break;
1119
1120         case RETBLEED_MITIGATION_STUFF:
1121                 setup_force_cpu_cap(X86_FEATURE_RETHUNK);
1122                 setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH);
1123
1124                 x86_return_thunk = call_depth_return_thunk;
1125                 break;
1126
1127         default:
1128                 break;
1129         }
1130
1131         if (mitigate_smt && !boot_cpu_has(X86_FEATURE_STIBP) &&
1132             (retbleed_nosmt || cpu_mitigations_auto_nosmt()))
1133                 cpu_smt_disable(false);
1134
1135         /*
1136          * Let IBRS trump all on Intel without affecting the effects of the
1137          * retbleed= cmdline option except for call depth based stuffing
1138          */
1139         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1140                 switch (spectre_v2_enabled) {
1141                 case SPECTRE_V2_IBRS:
1142                         retbleed_mitigation = RETBLEED_MITIGATION_IBRS;
1143                         break;
1144                 case SPECTRE_V2_EIBRS:
1145                 case SPECTRE_V2_EIBRS_RETPOLINE:
1146                 case SPECTRE_V2_EIBRS_LFENCE:
1147                         retbleed_mitigation = RETBLEED_MITIGATION_EIBRS;
1148                         break;
1149                 default:
1150                         if (retbleed_mitigation != RETBLEED_MITIGATION_STUFF)
1151                                 pr_err(RETBLEED_INTEL_MSG);
1152                 }
1153         }
1154
1155         pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
1156 }
1157
1158 #undef pr_fmt
1159 #define pr_fmt(fmt)     "Spectre V2 : " fmt
1160
1161 static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init =
1162         SPECTRE_V2_USER_NONE;
1163 static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init =
1164         SPECTRE_V2_USER_NONE;
1165
1166 #ifdef CONFIG_MITIGATION_RETPOLINE
1167 static bool spectre_v2_bad_module;
1168
1169 bool retpoline_module_ok(bool has_retpoline)
1170 {
1171         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
1172                 return true;
1173
1174         pr_err("System may be vulnerable to spectre v2\n");
1175         spectre_v2_bad_module = true;
1176         return false;
1177 }
1178
1179 static inline const char *spectre_v2_module_string(void)
1180 {
1181         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
1182 }
1183 #else
1184 static inline const char *spectre_v2_module_string(void) { return ""; }
1185 #endif
1186
1187 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
1188 #define SPECTRE_V2_EIBRS_EBPF_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS on, data leaks possible via Spectre v2 BHB attacks!\n"
1189 #define SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG "WARNING: Unprivileged eBPF is enabled with eIBRS+LFENCE mitigation and SMT, data leaks possible via Spectre v2 BHB attacks!\n"
1190 #define SPECTRE_V2_IBRS_PERF_MSG "WARNING: IBRS mitigation selected on Enhanced IBRS CPU, this may cause unnecessary performance loss\n"
1191
1192 #ifdef CONFIG_BPF_SYSCALL
1193 void unpriv_ebpf_notify(int new_state)
1194 {
1195         if (new_state)
1196                 return;
1197
1198         /* Unprivileged eBPF is enabled */
1199
1200         switch (spectre_v2_enabled) {
1201         case SPECTRE_V2_EIBRS:
1202                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1203                 break;
1204         case SPECTRE_V2_EIBRS_LFENCE:
1205                 if (sched_smt_active())
1206                         pr_err(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1207                 break;
1208         default:
1209                 break;
1210         }
1211 }
1212 #endif
1213
1214 static inline bool match_option(const char *arg, int arglen, const char *opt)
1215 {
1216         int len = strlen(opt);
1217
1218         return len == arglen && !strncmp(arg, opt, len);
1219 }
1220
1221 /* The kernel command line selection for spectre v2 */
1222 enum spectre_v2_mitigation_cmd {
1223         SPECTRE_V2_CMD_NONE,
1224         SPECTRE_V2_CMD_AUTO,
1225         SPECTRE_V2_CMD_FORCE,
1226         SPECTRE_V2_CMD_RETPOLINE,
1227         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
1228         SPECTRE_V2_CMD_RETPOLINE_LFENCE,
1229         SPECTRE_V2_CMD_EIBRS,
1230         SPECTRE_V2_CMD_EIBRS_RETPOLINE,
1231         SPECTRE_V2_CMD_EIBRS_LFENCE,
1232         SPECTRE_V2_CMD_IBRS,
1233 };
1234
1235 enum spectre_v2_user_cmd {
1236         SPECTRE_V2_USER_CMD_NONE,
1237         SPECTRE_V2_USER_CMD_AUTO,
1238         SPECTRE_V2_USER_CMD_FORCE,
1239         SPECTRE_V2_USER_CMD_PRCTL,
1240         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
1241         SPECTRE_V2_USER_CMD_SECCOMP,
1242         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
1243 };
1244
1245 static const char * const spectre_v2_user_strings[] = {
1246         [SPECTRE_V2_USER_NONE]                  = "User space: Vulnerable",
1247         [SPECTRE_V2_USER_STRICT]                = "User space: Mitigation: STIBP protection",
1248         [SPECTRE_V2_USER_STRICT_PREFERRED]      = "User space: Mitigation: STIBP always-on protection",
1249         [SPECTRE_V2_USER_PRCTL]                 = "User space: Mitigation: STIBP via prctl",
1250         [SPECTRE_V2_USER_SECCOMP]               = "User space: Mitigation: STIBP via seccomp and prctl",
1251 };
1252
1253 static const struct {
1254         const char                      *option;
1255         enum spectre_v2_user_cmd        cmd;
1256         bool                            secure;
1257 } v2_user_options[] __initconst = {
1258         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
1259         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
1260         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
1261         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
1262         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
1263         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
1264         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
1265 };
1266
1267 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
1268 {
1269         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1270                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
1271 }
1272
1273 static __ro_after_init enum spectre_v2_mitigation_cmd spectre_v2_cmd;
1274
1275 static enum spectre_v2_user_cmd __init
1276 spectre_v2_parse_user_cmdline(void)
1277 {
1278         char arg[20];
1279         int ret, i;
1280
1281         switch (spectre_v2_cmd) {
1282         case SPECTRE_V2_CMD_NONE:
1283                 return SPECTRE_V2_USER_CMD_NONE;
1284         case SPECTRE_V2_CMD_FORCE:
1285                 return SPECTRE_V2_USER_CMD_FORCE;
1286         default:
1287                 break;
1288         }
1289
1290         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
1291                                   arg, sizeof(arg));
1292         if (ret < 0)
1293                 return SPECTRE_V2_USER_CMD_AUTO;
1294
1295         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
1296                 if (match_option(arg, ret, v2_user_options[i].option)) {
1297                         spec_v2_user_print_cond(v2_user_options[i].option,
1298                                                 v2_user_options[i].secure);
1299                         return v2_user_options[i].cmd;
1300                 }
1301         }
1302
1303         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
1304         return SPECTRE_V2_USER_CMD_AUTO;
1305 }
1306
1307 static inline bool spectre_v2_in_ibrs_mode(enum spectre_v2_mitigation mode)
1308 {
1309         return spectre_v2_in_eibrs_mode(mode) || mode == SPECTRE_V2_IBRS;
1310 }
1311
1312 static void __init
1313 spectre_v2_user_select_mitigation(void)
1314 {
1315         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
1316         bool smt_possible = IS_ENABLED(CONFIG_SMP);
1317         enum spectre_v2_user_cmd cmd;
1318
1319         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
1320                 return;
1321
1322         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
1323             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
1324                 smt_possible = false;
1325
1326         cmd = spectre_v2_parse_user_cmdline();
1327         switch (cmd) {
1328         case SPECTRE_V2_USER_CMD_NONE:
1329                 goto set_mode;
1330         case SPECTRE_V2_USER_CMD_FORCE:
1331                 mode = SPECTRE_V2_USER_STRICT;
1332                 break;
1333         case SPECTRE_V2_USER_CMD_AUTO:
1334         case SPECTRE_V2_USER_CMD_PRCTL:
1335         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1336                 mode = SPECTRE_V2_USER_PRCTL;
1337                 break;
1338         case SPECTRE_V2_USER_CMD_SECCOMP:
1339         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1340                 if (IS_ENABLED(CONFIG_SECCOMP))
1341                         mode = SPECTRE_V2_USER_SECCOMP;
1342                 else
1343                         mode = SPECTRE_V2_USER_PRCTL;
1344                 break;
1345         }
1346
1347         /* Initialize Indirect Branch Prediction Barrier */
1348         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1349                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
1350
1351                 spectre_v2_user_ibpb = mode;
1352                 switch (cmd) {
1353                 case SPECTRE_V2_USER_CMD_NONE:
1354                         break;
1355                 case SPECTRE_V2_USER_CMD_FORCE:
1356                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
1357                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
1358                         static_branch_enable(&switch_mm_always_ibpb);
1359                         spectre_v2_user_ibpb = SPECTRE_V2_USER_STRICT;
1360                         break;
1361                 case SPECTRE_V2_USER_CMD_PRCTL:
1362                 case SPECTRE_V2_USER_CMD_AUTO:
1363                 case SPECTRE_V2_USER_CMD_SECCOMP:
1364                         static_branch_enable(&switch_mm_cond_ibpb);
1365                         break;
1366                 }
1367
1368                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
1369                         static_key_enabled(&switch_mm_always_ibpb) ?
1370                         "always-on" : "conditional");
1371         }
1372
1373         /*
1374          * If no STIBP, Intel enhanced IBRS is enabled, or SMT impossible, STIBP
1375          * is not required.
1376          *
1377          * Intel's Enhanced IBRS also protects against cross-thread branch target
1378          * injection in user-mode as the IBRS bit remains always set which
1379          * implicitly enables cross-thread protections.  However, in legacy IBRS
1380          * mode, the IBRS bit is set only on kernel entry and cleared on return
1381          * to userspace.  AMD Automatic IBRS also does not protect userspace.
1382          * These modes therefore disable the implicit cross-thread protection,
1383          * so allow for STIBP to be selected in those cases.
1384          */
1385         if (!boot_cpu_has(X86_FEATURE_STIBP) ||
1386             !smt_possible ||
1387             (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
1388              !boot_cpu_has(X86_FEATURE_AUTOIBRS)))
1389                 return;
1390
1391         /*
1392          * At this point, an STIBP mode other than "off" has been set.
1393          * If STIBP support is not being forced, check if STIBP always-on
1394          * is preferred.
1395          */
1396         if (mode != SPECTRE_V2_USER_STRICT &&
1397             boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
1398                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1399
1400         if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
1401             retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
1402                 if (mode != SPECTRE_V2_USER_STRICT &&
1403                     mode != SPECTRE_V2_USER_STRICT_PREFERRED)
1404                         pr_info("Selecting STIBP always-on mode to complement retbleed mitigation\n");
1405                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
1406         }
1407
1408         spectre_v2_user_stibp = mode;
1409
1410 set_mode:
1411         pr_info("%s\n", spectre_v2_user_strings[mode]);
1412 }
1413
1414 static const char * const spectre_v2_strings[] = {
1415         [SPECTRE_V2_NONE]                       = "Vulnerable",
1416         [SPECTRE_V2_RETPOLINE]                  = "Mitigation: Retpolines",
1417         [SPECTRE_V2_LFENCE]                     = "Mitigation: LFENCE",
1418         [SPECTRE_V2_EIBRS]                      = "Mitigation: Enhanced / Automatic IBRS",
1419         [SPECTRE_V2_EIBRS_LFENCE]               = "Mitigation: Enhanced / Automatic IBRS + LFENCE",
1420         [SPECTRE_V2_EIBRS_RETPOLINE]            = "Mitigation: Enhanced / Automatic IBRS + Retpolines",
1421         [SPECTRE_V2_IBRS]                       = "Mitigation: IBRS",
1422 };
1423
1424 static const struct {
1425         const char *option;
1426         enum spectre_v2_mitigation_cmd cmd;
1427         bool secure;
1428 } mitigation_options[] __initconst = {
1429         { "off",                SPECTRE_V2_CMD_NONE,              false },
1430         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
1431         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
1432         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
1433         { "retpoline,lfence",   SPECTRE_V2_CMD_RETPOLINE_LFENCE,  false },
1434         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
1435         { "eibrs",              SPECTRE_V2_CMD_EIBRS,             false },
1436         { "eibrs,lfence",       SPECTRE_V2_CMD_EIBRS_LFENCE,      false },
1437         { "eibrs,retpoline",    SPECTRE_V2_CMD_EIBRS_RETPOLINE,   false },
1438         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
1439         { "ibrs",               SPECTRE_V2_CMD_IBRS,              false },
1440 };
1441
1442 static void __init spec_v2_print_cond(const char *reason, bool secure)
1443 {
1444         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
1445                 pr_info("%s selected on command line.\n", reason);
1446 }
1447
1448 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
1449 {
1450         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
1451         char arg[20];
1452         int ret, i;
1453
1454         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
1455             cpu_mitigations_off())
1456                 return SPECTRE_V2_CMD_NONE;
1457
1458         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
1459         if (ret < 0)
1460                 return SPECTRE_V2_CMD_AUTO;
1461
1462         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
1463                 if (!match_option(arg, ret, mitigation_options[i].option))
1464                         continue;
1465                 cmd = mitigation_options[i].cmd;
1466                 break;
1467         }
1468
1469         if (i >= ARRAY_SIZE(mitigation_options)) {
1470                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1471                 return SPECTRE_V2_CMD_AUTO;
1472         }
1473
1474         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
1475              cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1476              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC ||
1477              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1478              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1479             !IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) {
1480                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1481                        mitigation_options[i].option);
1482                 return SPECTRE_V2_CMD_AUTO;
1483         }
1484
1485         if ((cmd == SPECTRE_V2_CMD_EIBRS ||
1486              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE ||
1487              cmd == SPECTRE_V2_CMD_EIBRS_RETPOLINE) &&
1488             !boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1489                 pr_err("%s selected but CPU doesn't have Enhanced or Automatic IBRS. Switching to AUTO select\n",
1490                        mitigation_options[i].option);
1491                 return SPECTRE_V2_CMD_AUTO;
1492         }
1493
1494         if ((cmd == SPECTRE_V2_CMD_RETPOLINE_LFENCE ||
1495              cmd == SPECTRE_V2_CMD_EIBRS_LFENCE) &&
1496             !boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
1497                 pr_err("%s selected, but CPU doesn't have a serializing LFENCE. Switching to AUTO select\n",
1498                        mitigation_options[i].option);
1499                 return SPECTRE_V2_CMD_AUTO;
1500         }
1501
1502         if (cmd == SPECTRE_V2_CMD_IBRS && !IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY)) {
1503                 pr_err("%s selected but not compiled in. Switching to AUTO select\n",
1504                        mitigation_options[i].option);
1505                 return SPECTRE_V2_CMD_AUTO;
1506         }
1507
1508         if (cmd == SPECTRE_V2_CMD_IBRS && boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
1509                 pr_err("%s selected but not Intel CPU. Switching to AUTO select\n",
1510                        mitigation_options[i].option);
1511                 return SPECTRE_V2_CMD_AUTO;
1512         }
1513
1514         if (cmd == SPECTRE_V2_CMD_IBRS && !boot_cpu_has(X86_FEATURE_IBRS)) {
1515                 pr_err("%s selected but CPU doesn't have IBRS. Switching to AUTO select\n",
1516                        mitigation_options[i].option);
1517                 return SPECTRE_V2_CMD_AUTO;
1518         }
1519
1520         if (cmd == SPECTRE_V2_CMD_IBRS && cpu_feature_enabled(X86_FEATURE_XENPV)) {
1521                 pr_err("%s selected but running as XenPV guest. Switching to AUTO select\n",
1522                        mitigation_options[i].option);
1523                 return SPECTRE_V2_CMD_AUTO;
1524         }
1525
1526         spec_v2_print_cond(mitigation_options[i].option,
1527                            mitigation_options[i].secure);
1528         return cmd;
1529 }
1530
1531 static enum spectre_v2_mitigation __init spectre_v2_select_retpoline(void)
1532 {
1533         if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)) {
1534                 pr_err("Kernel not compiled with retpoline; no mitigation available!");
1535                 return SPECTRE_V2_NONE;
1536         }
1537
1538         return SPECTRE_V2_RETPOLINE;
1539 }
1540
1541 static bool __ro_after_init rrsba_disabled;
1542
1543 /* Disable in-kernel use of non-RSB RET predictors */
1544 static void __init spec_ctrl_disable_kernel_rrsba(void)
1545 {
1546         if (rrsba_disabled)
1547                 return;
1548
1549         if (!(x86_arch_cap_msr & ARCH_CAP_RRSBA)) {
1550                 rrsba_disabled = true;
1551                 return;
1552         }
1553
1554         if (!boot_cpu_has(X86_FEATURE_RRSBA_CTRL))
1555                 return;
1556
1557         x86_spec_ctrl_base |= SPEC_CTRL_RRSBA_DIS_S;
1558         update_spec_ctrl(x86_spec_ctrl_base);
1559         rrsba_disabled = true;
1560 }
1561
1562 static void __init spectre_v2_determine_rsb_fill_type_at_vmexit(enum spectre_v2_mitigation mode)
1563 {
1564         /*
1565          * Similar to context switches, there are two types of RSB attacks
1566          * after VM exit:
1567          *
1568          * 1) RSB underflow
1569          *
1570          * 2) Poisoned RSB entry
1571          *
1572          * When retpoline is enabled, both are mitigated by filling/clearing
1573          * the RSB.
1574          *
1575          * When IBRS is enabled, while #1 would be mitigated by the IBRS branch
1576          * prediction isolation protections, RSB still needs to be cleared
1577          * because of #2.  Note that SMEP provides no protection here, unlike
1578          * user-space-poisoned RSB entries.
1579          *
1580          * eIBRS should protect against RSB poisoning, but if the EIBRS_PBRSB
1581          * bug is present then a LITE version of RSB protection is required,
1582          * just a single call needs to retire before a RET is executed.
1583          */
1584         switch (mode) {
1585         case SPECTRE_V2_NONE:
1586                 return;
1587
1588         case SPECTRE_V2_EIBRS_LFENCE:
1589         case SPECTRE_V2_EIBRS:
1590                 if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
1591                         setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT_LITE);
1592                         pr_info("Spectre v2 / PBRSB-eIBRS: Retire a single CALL on VMEXIT\n");
1593                 }
1594                 return;
1595
1596         case SPECTRE_V2_EIBRS_RETPOLINE:
1597         case SPECTRE_V2_RETPOLINE:
1598         case SPECTRE_V2_LFENCE:
1599         case SPECTRE_V2_IBRS:
1600                 setup_force_cpu_cap(X86_FEATURE_RSB_VMEXIT);
1601                 pr_info("Spectre v2 / SpectreRSB : Filling RSB on VMEXIT\n");
1602                 return;
1603         }
1604
1605         pr_warn_once("Unknown Spectre v2 mode, disabling RSB mitigation at VM exit");
1606         dump_stack();
1607 }
1608
1609 /*
1610  * Set BHI_DIS_S to prevent indirect branches in kernel to be influenced by
1611  * branch history in userspace. Not needed if BHI_NO is set.
1612  */
1613 static bool __init spec_ctrl_bhi_dis(void)
1614 {
1615         if (!boot_cpu_has(X86_FEATURE_BHI_CTRL))
1616                 return false;
1617
1618         x86_spec_ctrl_base |= SPEC_CTRL_BHI_DIS_S;
1619         update_spec_ctrl(x86_spec_ctrl_base);
1620         setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_HW);
1621
1622         return true;
1623 }
1624
1625 enum bhi_mitigations {
1626         BHI_MITIGATION_OFF,
1627         BHI_MITIGATION_ON,
1628 };
1629
1630 static enum bhi_mitigations bhi_mitigation __ro_after_init =
1631         IS_ENABLED(CONFIG_MITIGATION_SPECTRE_BHI) ? BHI_MITIGATION_ON : BHI_MITIGATION_OFF;
1632
1633 static int __init spectre_bhi_parse_cmdline(char *str)
1634 {
1635         if (!str)
1636                 return -EINVAL;
1637
1638         if (!strcmp(str, "off"))
1639                 bhi_mitigation = BHI_MITIGATION_OFF;
1640         else if (!strcmp(str, "on"))
1641                 bhi_mitigation = BHI_MITIGATION_ON;
1642         else
1643                 pr_err("Ignoring unknown spectre_bhi option (%s)", str);
1644
1645         return 0;
1646 }
1647 early_param("spectre_bhi", spectre_bhi_parse_cmdline);
1648
1649 static void __init bhi_select_mitigation(void)
1650 {
1651         if (bhi_mitigation == BHI_MITIGATION_OFF)
1652                 return;
1653
1654         /* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
1655         if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
1656             !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
1657                 spec_ctrl_disable_kernel_rrsba();
1658                 if (rrsba_disabled)
1659                         return;
1660         }
1661
1662         if (spec_ctrl_bhi_dis())
1663                 return;
1664
1665         if (!IS_ENABLED(CONFIG_X86_64))
1666                 return;
1667
1668         /* Mitigate KVM by default */
1669         setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT);
1670         pr_info("Spectre BHI mitigation: SW BHB clearing on vm exit\n");
1671
1672         /* Mitigate syscalls when the mitigation is forced =on */
1673         setup_force_cpu_cap(X86_FEATURE_CLEAR_BHB_LOOP);
1674         pr_info("Spectre BHI mitigation: SW BHB clearing on syscall\n");
1675 }
1676
1677 static void __init spectre_v2_select_mitigation(void)
1678 {
1679         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
1680         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
1681
1682         /*
1683          * If the CPU is not affected and the command line mode is NONE or AUTO
1684          * then nothing to do.
1685          */
1686         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
1687             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
1688                 return;
1689
1690         switch (cmd) {
1691         case SPECTRE_V2_CMD_NONE:
1692                 return;
1693
1694         case SPECTRE_V2_CMD_FORCE:
1695         case SPECTRE_V2_CMD_AUTO:
1696                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
1697                         mode = SPECTRE_V2_EIBRS;
1698                         break;
1699                 }
1700
1701                 if (IS_ENABLED(CONFIG_MITIGATION_IBRS_ENTRY) &&
1702                     boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1703                     retbleed_cmd != RETBLEED_CMD_OFF &&
1704                     retbleed_cmd != RETBLEED_CMD_STUFF &&
1705                     boot_cpu_has(X86_FEATURE_IBRS) &&
1706                     boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
1707                         mode = SPECTRE_V2_IBRS;
1708                         break;
1709                 }
1710
1711                 mode = spectre_v2_select_retpoline();
1712                 break;
1713
1714         case SPECTRE_V2_CMD_RETPOLINE_LFENCE:
1715                 pr_err(SPECTRE_V2_LFENCE_MSG);
1716                 mode = SPECTRE_V2_LFENCE;
1717                 break;
1718
1719         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
1720                 mode = SPECTRE_V2_RETPOLINE;
1721                 break;
1722
1723         case SPECTRE_V2_CMD_RETPOLINE:
1724                 mode = spectre_v2_select_retpoline();
1725                 break;
1726
1727         case SPECTRE_V2_CMD_IBRS:
1728                 mode = SPECTRE_V2_IBRS;
1729                 break;
1730
1731         case SPECTRE_V2_CMD_EIBRS:
1732                 mode = SPECTRE_V2_EIBRS;
1733                 break;
1734
1735         case SPECTRE_V2_CMD_EIBRS_LFENCE:
1736                 mode = SPECTRE_V2_EIBRS_LFENCE;
1737                 break;
1738
1739         case SPECTRE_V2_CMD_EIBRS_RETPOLINE:
1740                 mode = SPECTRE_V2_EIBRS_RETPOLINE;
1741                 break;
1742         }
1743
1744         if (mode == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
1745                 pr_err(SPECTRE_V2_EIBRS_EBPF_MSG);
1746
1747         if (spectre_v2_in_ibrs_mode(mode)) {
1748                 if (boot_cpu_has(X86_FEATURE_AUTOIBRS)) {
1749                         msr_set_bit(MSR_EFER, _EFER_AUTOIBRS);
1750                 } else {
1751                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
1752                         update_spec_ctrl(x86_spec_ctrl_base);
1753                 }
1754         }
1755
1756         switch (mode) {
1757         case SPECTRE_V2_NONE:
1758         case SPECTRE_V2_EIBRS:
1759                 break;
1760
1761         case SPECTRE_V2_IBRS:
1762                 setup_force_cpu_cap(X86_FEATURE_KERNEL_IBRS);
1763                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED))
1764                         pr_warn(SPECTRE_V2_IBRS_PERF_MSG);
1765                 break;
1766
1767         case SPECTRE_V2_LFENCE:
1768         case SPECTRE_V2_EIBRS_LFENCE:
1769                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_LFENCE);
1770                 fallthrough;
1771
1772         case SPECTRE_V2_RETPOLINE:
1773         case SPECTRE_V2_EIBRS_RETPOLINE:
1774                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
1775                 break;
1776         }
1777
1778         /*
1779          * Disable alternate RSB predictions in kernel when indirect CALLs and
1780          * JMPs gets protection against BHI and Intramode-BTI, but RET
1781          * prediction from a non-RSB predictor is still a risk.
1782          */
1783         if (mode == SPECTRE_V2_EIBRS_LFENCE ||
1784             mode == SPECTRE_V2_EIBRS_RETPOLINE ||
1785             mode == SPECTRE_V2_RETPOLINE)
1786                 spec_ctrl_disable_kernel_rrsba();
1787
1788         if (boot_cpu_has(X86_BUG_BHI))
1789                 bhi_select_mitigation();
1790
1791         spectre_v2_enabled = mode;
1792         pr_info("%s\n", spectre_v2_strings[mode]);
1793
1794         /*
1795          * If Spectre v2 protection has been enabled, fill the RSB during a
1796          * context switch.  In general there are two types of RSB attacks
1797          * across context switches, for which the CALLs/RETs may be unbalanced.
1798          *
1799          * 1) RSB underflow
1800          *
1801          *    Some Intel parts have "bottomless RSB".  When the RSB is empty,
1802          *    speculated return targets may come from the branch predictor,
1803          *    which could have a user-poisoned BTB or BHB entry.
1804          *
1805          *    AMD has it even worse: *all* returns are speculated from the BTB,
1806          *    regardless of the state of the RSB.
1807          *
1808          *    When IBRS or eIBRS is enabled, the "user -> kernel" attack
1809          *    scenario is mitigated by the IBRS branch prediction isolation
1810          *    properties, so the RSB buffer filling wouldn't be necessary to
1811          *    protect against this type of attack.
1812          *
1813          *    The "user -> user" attack scenario is mitigated by RSB filling.
1814          *
1815          * 2) Poisoned RSB entry
1816          *
1817          *    If the 'next' in-kernel return stack is shorter than 'prev',
1818          *    'next' could be tricked into speculating with a user-poisoned RSB
1819          *    entry.
1820          *
1821          *    The "user -> kernel" attack scenario is mitigated by SMEP and
1822          *    eIBRS.
1823          *
1824          *    The "user -> user" scenario, also known as SpectreBHB, requires
1825          *    RSB clearing.
1826          *
1827          * So to mitigate all cases, unconditionally fill RSB on context
1828          * switches.
1829          *
1830          * FIXME: Is this pointless for retbleed-affected AMD?
1831          */
1832         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
1833         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
1834
1835         spectre_v2_determine_rsb_fill_type_at_vmexit(mode);
1836
1837         /*
1838          * Retpoline protects the kernel, but doesn't protect firmware.  IBRS
1839          * and Enhanced IBRS protect firmware too, so enable IBRS around
1840          * firmware calls only when IBRS / Enhanced / Automatic IBRS aren't
1841          * otherwise enabled.
1842          *
1843          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
1844          * the user might select retpoline on the kernel command line and if
1845          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
1846          * enable IBRS around firmware calls.
1847          */
1848         if (boot_cpu_has_bug(X86_BUG_RETBLEED) &&
1849             boot_cpu_has(X86_FEATURE_IBPB) &&
1850             (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
1851              boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)) {
1852
1853                 if (retbleed_cmd != RETBLEED_CMD_IBPB) {
1854                         setup_force_cpu_cap(X86_FEATURE_USE_IBPB_FW);
1855                         pr_info("Enabling Speculation Barrier for firmware calls\n");
1856                 }
1857
1858         } else if (boot_cpu_has(X86_FEATURE_IBRS) && !spectre_v2_in_ibrs_mode(mode)) {
1859                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
1860                 pr_info("Enabling Restricted Speculation for firmware calls\n");
1861         }
1862
1863         /* Set up IBPB and STIBP depending on the general spectre V2 command */
1864         spectre_v2_cmd = cmd;
1865 }
1866
1867 static void update_stibp_msr(void * __unused)
1868 {
1869         u64 val = spec_ctrl_current() | (x86_spec_ctrl_base & SPEC_CTRL_STIBP);
1870         update_spec_ctrl(val);
1871 }
1872
1873 /* Update x86_spec_ctrl_base in case SMT state changed. */
1874 static void update_stibp_strict(void)
1875 {
1876         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
1877
1878         if (sched_smt_active())
1879                 mask |= SPEC_CTRL_STIBP;
1880
1881         if (mask == x86_spec_ctrl_base)
1882                 return;
1883
1884         pr_info("Update user space SMT mitigation: STIBP %s\n",
1885                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
1886         x86_spec_ctrl_base = mask;
1887         on_each_cpu(update_stibp_msr, NULL, 1);
1888 }
1889
1890 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
1891 static void update_indir_branch_cond(void)
1892 {
1893         if (sched_smt_active())
1894                 static_branch_enable(&switch_to_cond_stibp);
1895         else
1896                 static_branch_disable(&switch_to_cond_stibp);
1897 }
1898
1899 #undef pr_fmt
1900 #define pr_fmt(fmt) fmt
1901
1902 /* Update the static key controlling the MDS CPU buffer clear in idle */
1903 static void update_mds_branch_idle(void)
1904 {
1905         /*
1906          * Enable the idle clearing if SMT is active on CPUs which are
1907          * affected only by MSBDS and not any other MDS variant.
1908          *
1909          * The other variants cannot be mitigated when SMT is enabled, so
1910          * clearing the buffers on idle just to prevent the Store Buffer
1911          * repartitioning leak would be a window dressing exercise.
1912          */
1913         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
1914                 return;
1915
1916         if (sched_smt_active()) {
1917                 static_branch_enable(&mds_idle_clear);
1918         } else if (mmio_mitigation == MMIO_MITIGATION_OFF ||
1919                    (x86_arch_cap_msr & ARCH_CAP_FBSDP_NO)) {
1920                 static_branch_disable(&mds_idle_clear);
1921         }
1922 }
1923
1924 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
1925 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
1926 #define MMIO_MSG_SMT "MMIO Stale Data CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/processor_mmio_stale_data.html for more details.\n"
1927
1928 void cpu_bugs_smt_update(void)
1929 {
1930         mutex_lock(&spec_ctrl_mutex);
1931
1932         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
1933             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
1934                 pr_warn_once(SPECTRE_V2_EIBRS_LFENCE_EBPF_SMT_MSG);
1935
1936         switch (spectre_v2_user_stibp) {
1937         case SPECTRE_V2_USER_NONE:
1938                 break;
1939         case SPECTRE_V2_USER_STRICT:
1940         case SPECTRE_V2_USER_STRICT_PREFERRED:
1941                 update_stibp_strict();
1942                 break;
1943         case SPECTRE_V2_USER_PRCTL:
1944         case SPECTRE_V2_USER_SECCOMP:
1945                 update_indir_branch_cond();
1946                 break;
1947         }
1948
1949         switch (mds_mitigation) {
1950         case MDS_MITIGATION_FULL:
1951         case MDS_MITIGATION_VMWERV:
1952                 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
1953                         pr_warn_once(MDS_MSG_SMT);
1954                 update_mds_branch_idle();
1955                 break;
1956         case MDS_MITIGATION_OFF:
1957                 break;
1958         }
1959
1960         switch (taa_mitigation) {
1961         case TAA_MITIGATION_VERW:
1962         case TAA_MITIGATION_UCODE_NEEDED:
1963                 if (sched_smt_active())
1964                         pr_warn_once(TAA_MSG_SMT);
1965                 break;
1966         case TAA_MITIGATION_TSX_DISABLED:
1967         case TAA_MITIGATION_OFF:
1968                 break;
1969         }
1970
1971         switch (mmio_mitigation) {
1972         case MMIO_MITIGATION_VERW:
1973         case MMIO_MITIGATION_UCODE_NEEDED:
1974                 if (sched_smt_active())
1975                         pr_warn_once(MMIO_MSG_SMT);
1976                 break;
1977         case MMIO_MITIGATION_OFF:
1978                 break;
1979         }
1980
1981         mutex_unlock(&spec_ctrl_mutex);
1982 }
1983
1984 #undef pr_fmt
1985 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
1986
1987 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
1988
1989 /* The kernel command line selection */
1990 enum ssb_mitigation_cmd {
1991         SPEC_STORE_BYPASS_CMD_NONE,
1992         SPEC_STORE_BYPASS_CMD_AUTO,
1993         SPEC_STORE_BYPASS_CMD_ON,
1994         SPEC_STORE_BYPASS_CMD_PRCTL,
1995         SPEC_STORE_BYPASS_CMD_SECCOMP,
1996 };
1997
1998 static const char * const ssb_strings[] = {
1999         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
2000         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
2001         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
2002         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
2003 };
2004
2005 static const struct {
2006         const char *option;
2007         enum ssb_mitigation_cmd cmd;
2008 } ssb_mitigation_options[]  __initconst = {
2009         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
2010         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
2011         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
2012         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
2013         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
2014 };
2015
2016 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
2017 {
2018         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
2019         char arg[20];
2020         int ret, i;
2021
2022         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
2023             cpu_mitigations_off()) {
2024                 return SPEC_STORE_BYPASS_CMD_NONE;
2025         } else {
2026                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
2027                                           arg, sizeof(arg));
2028                 if (ret < 0)
2029                         return SPEC_STORE_BYPASS_CMD_AUTO;
2030
2031                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
2032                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
2033                                 continue;
2034
2035                         cmd = ssb_mitigation_options[i].cmd;
2036                         break;
2037                 }
2038
2039                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
2040                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
2041                         return SPEC_STORE_BYPASS_CMD_AUTO;
2042                 }
2043         }
2044
2045         return cmd;
2046 }
2047
2048 static enum ssb_mitigation __init __ssb_select_mitigation(void)
2049 {
2050         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
2051         enum ssb_mitigation_cmd cmd;
2052
2053         if (!boot_cpu_has(X86_FEATURE_SSBD))
2054                 return mode;
2055
2056         cmd = ssb_parse_cmdline();
2057         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
2058             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
2059              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
2060                 return mode;
2061
2062         switch (cmd) {
2063         case SPEC_STORE_BYPASS_CMD_SECCOMP:
2064                 /*
2065                  * Choose prctl+seccomp as the default mode if seccomp is
2066                  * enabled.
2067                  */
2068                 if (IS_ENABLED(CONFIG_SECCOMP))
2069                         mode = SPEC_STORE_BYPASS_SECCOMP;
2070                 else
2071                         mode = SPEC_STORE_BYPASS_PRCTL;
2072                 break;
2073         case SPEC_STORE_BYPASS_CMD_ON:
2074                 mode = SPEC_STORE_BYPASS_DISABLE;
2075                 break;
2076         case SPEC_STORE_BYPASS_CMD_AUTO:
2077         case SPEC_STORE_BYPASS_CMD_PRCTL:
2078                 mode = SPEC_STORE_BYPASS_PRCTL;
2079                 break;
2080         case SPEC_STORE_BYPASS_CMD_NONE:
2081                 break;
2082         }
2083
2084         /*
2085          * We have three CPU feature flags that are in play here:
2086          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
2087          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
2088          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
2089          */
2090         if (mode == SPEC_STORE_BYPASS_DISABLE) {
2091                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
2092                 /*
2093                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
2094                  * use a completely different MSR and bit dependent on family.
2095                  */
2096                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
2097                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
2098                         x86_amd_ssb_disable();
2099                 } else {
2100                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
2101                         update_spec_ctrl(x86_spec_ctrl_base);
2102                 }
2103         }
2104
2105         return mode;
2106 }
2107
2108 static void ssb_select_mitigation(void)
2109 {
2110         ssb_mode = __ssb_select_mitigation();
2111
2112         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
2113                 pr_info("%s\n", ssb_strings[ssb_mode]);
2114 }
2115
2116 #undef pr_fmt
2117 #define pr_fmt(fmt)     "Speculation prctl: " fmt
2118
2119 static void task_update_spec_tif(struct task_struct *tsk)
2120 {
2121         /* Force the update of the real TIF bits */
2122         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
2123
2124         /*
2125          * Immediately update the speculation control MSRs for the current
2126          * task, but for a non-current task delay setting the CPU
2127          * mitigation until it is scheduled next.
2128          *
2129          * This can only happen for SECCOMP mitigation. For PRCTL it's
2130          * always the current task.
2131          */
2132         if (tsk == current)
2133                 speculation_ctrl_update_current();
2134 }
2135
2136 static int l1d_flush_prctl_set(struct task_struct *task, unsigned long ctrl)
2137 {
2138
2139         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
2140                 return -EPERM;
2141
2142         switch (ctrl) {
2143         case PR_SPEC_ENABLE:
2144                 set_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
2145                 return 0;
2146         case PR_SPEC_DISABLE:
2147                 clear_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH);
2148                 return 0;
2149         default:
2150                 return -ERANGE;
2151         }
2152 }
2153
2154 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
2155 {
2156         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
2157             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
2158                 return -ENXIO;
2159
2160         switch (ctrl) {
2161         case PR_SPEC_ENABLE:
2162                 /* If speculation is force disabled, enable is not allowed */
2163                 if (task_spec_ssb_force_disable(task))
2164                         return -EPERM;
2165                 task_clear_spec_ssb_disable(task);
2166                 task_clear_spec_ssb_noexec(task);
2167                 task_update_spec_tif(task);
2168                 break;
2169         case PR_SPEC_DISABLE:
2170                 task_set_spec_ssb_disable(task);
2171                 task_clear_spec_ssb_noexec(task);
2172                 task_update_spec_tif(task);
2173                 break;
2174         case PR_SPEC_FORCE_DISABLE:
2175                 task_set_spec_ssb_disable(task);
2176                 task_set_spec_ssb_force_disable(task);
2177                 task_clear_spec_ssb_noexec(task);
2178                 task_update_spec_tif(task);
2179                 break;
2180         case PR_SPEC_DISABLE_NOEXEC:
2181                 if (task_spec_ssb_force_disable(task))
2182                         return -EPERM;
2183                 task_set_spec_ssb_disable(task);
2184                 task_set_spec_ssb_noexec(task);
2185                 task_update_spec_tif(task);
2186                 break;
2187         default:
2188                 return -ERANGE;
2189         }
2190         return 0;
2191 }
2192
2193 static bool is_spec_ib_user_controlled(void)
2194 {
2195         return spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL ||
2196                 spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2197                 spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL ||
2198                 spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP;
2199 }
2200
2201 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
2202 {
2203         switch (ctrl) {
2204         case PR_SPEC_ENABLE:
2205                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2206                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2207                         return 0;
2208
2209                 /*
2210                  * With strict mode for both IBPB and STIBP, the instruction
2211                  * code paths avoid checking this task flag and instead,
2212                  * unconditionally run the instruction. However, STIBP and IBPB
2213                  * are independent and either can be set to conditionally
2214                  * enabled regardless of the mode of the other.
2215                  *
2216                  * If either is set to conditional, allow the task flag to be
2217                  * updated, unless it was force-disabled by a previous prctl
2218                  * call. Currently, this is possible on an AMD CPU which has the
2219                  * feature X86_FEATURE_AMD_STIBP_ALWAYS_ON. In this case, if the
2220                  * kernel is booted with 'spectre_v2_user=seccomp', then
2221                  * spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP and
2222                  * spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED.
2223                  */
2224                 if (!is_spec_ib_user_controlled() ||
2225                     task_spec_ib_force_disable(task))
2226                         return -EPERM;
2227
2228                 task_clear_spec_ib_disable(task);
2229                 task_update_spec_tif(task);
2230                 break;
2231         case PR_SPEC_DISABLE:
2232         case PR_SPEC_FORCE_DISABLE:
2233                 /*
2234                  * Indirect branch speculation is always allowed when
2235                  * mitigation is force disabled.
2236                  */
2237                 if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2238                     spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2239                         return -EPERM;
2240
2241                 if (!is_spec_ib_user_controlled())
2242                         return 0;
2243
2244                 task_set_spec_ib_disable(task);
2245                 if (ctrl == PR_SPEC_FORCE_DISABLE)
2246                         task_set_spec_ib_force_disable(task);
2247                 task_update_spec_tif(task);
2248                 if (task == current)
2249                         indirect_branch_prediction_barrier();
2250                 break;
2251         default:
2252                 return -ERANGE;
2253         }
2254         return 0;
2255 }
2256
2257 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
2258                              unsigned long ctrl)
2259 {
2260         switch (which) {
2261         case PR_SPEC_STORE_BYPASS:
2262                 return ssb_prctl_set(task, ctrl);
2263         case PR_SPEC_INDIRECT_BRANCH:
2264                 return ib_prctl_set(task, ctrl);
2265         case PR_SPEC_L1D_FLUSH:
2266                 return l1d_flush_prctl_set(task, ctrl);
2267         default:
2268                 return -ENODEV;
2269         }
2270 }
2271
2272 #ifdef CONFIG_SECCOMP
2273 void arch_seccomp_spec_mitigate(struct task_struct *task)
2274 {
2275         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
2276                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2277         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP ||
2278             spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP)
2279                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
2280 }
2281 #endif
2282
2283 static int l1d_flush_prctl_get(struct task_struct *task)
2284 {
2285         if (!static_branch_unlikely(&switch_mm_cond_l1d_flush))
2286                 return PR_SPEC_FORCE_DISABLE;
2287
2288         if (test_ti_thread_flag(&task->thread_info, TIF_SPEC_L1D_FLUSH))
2289                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2290         else
2291                 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2292 }
2293
2294 static int ssb_prctl_get(struct task_struct *task)
2295 {
2296         switch (ssb_mode) {
2297         case SPEC_STORE_BYPASS_NONE:
2298                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
2299                         return PR_SPEC_ENABLE;
2300                 return PR_SPEC_NOT_AFFECTED;
2301         case SPEC_STORE_BYPASS_DISABLE:
2302                 return PR_SPEC_DISABLE;
2303         case SPEC_STORE_BYPASS_SECCOMP:
2304         case SPEC_STORE_BYPASS_PRCTL:
2305                 if (task_spec_ssb_force_disable(task))
2306                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2307                 if (task_spec_ssb_noexec(task))
2308                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
2309                 if (task_spec_ssb_disable(task))
2310                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2311                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2312         }
2313         BUG();
2314 }
2315
2316 static int ib_prctl_get(struct task_struct *task)
2317 {
2318         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
2319                 return PR_SPEC_NOT_AFFECTED;
2320
2321         if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE &&
2322             spectre_v2_user_stibp == SPECTRE_V2_USER_NONE)
2323                 return PR_SPEC_ENABLE;
2324         else if (is_spec_ib_user_controlled()) {
2325                 if (task_spec_ib_force_disable(task))
2326                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
2327                 if (task_spec_ib_disable(task))
2328                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
2329                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
2330         } else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT ||
2331             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2332             spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED)
2333                 return PR_SPEC_DISABLE;
2334         else
2335                 return PR_SPEC_NOT_AFFECTED;
2336 }
2337
2338 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
2339 {
2340         switch (which) {
2341         case PR_SPEC_STORE_BYPASS:
2342                 return ssb_prctl_get(task);
2343         case PR_SPEC_INDIRECT_BRANCH:
2344                 return ib_prctl_get(task);
2345         case PR_SPEC_L1D_FLUSH:
2346                 return l1d_flush_prctl_get(task);
2347         default:
2348                 return -ENODEV;
2349         }
2350 }
2351
2352 void x86_spec_ctrl_setup_ap(void)
2353 {
2354         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
2355                 update_spec_ctrl(x86_spec_ctrl_base);
2356
2357         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
2358                 x86_amd_ssb_disable();
2359 }
2360
2361 bool itlb_multihit_kvm_mitigation;
2362 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
2363
2364 #undef pr_fmt
2365 #define pr_fmt(fmt)     "L1TF: " fmt
2366
2367 /* Default mitigation for L1TF-affected CPUs */
2368 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
2369 #if IS_ENABLED(CONFIG_KVM_INTEL)
2370 EXPORT_SYMBOL_GPL(l1tf_mitigation);
2371 #endif
2372 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
2373 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
2374
2375 /*
2376  * These CPUs all support 44bits physical address space internally in the
2377  * cache but CPUID can report a smaller number of physical address bits.
2378  *
2379  * The L1TF mitigation uses the top most address bit for the inversion of
2380  * non present PTEs. When the installed memory reaches into the top most
2381  * address bit due to memory holes, which has been observed on machines
2382  * which report 36bits physical address bits and have 32G RAM installed,
2383  * then the mitigation range check in l1tf_select_mitigation() triggers.
2384  * This is a false positive because the mitigation is still possible due to
2385  * the fact that the cache uses 44bit internally. Use the cache bits
2386  * instead of the reported physical bits and adjust them on the affected
2387  * machines to 44bit if the reported bits are less than 44.
2388  */
2389 static void override_cache_bits(struct cpuinfo_x86 *c)
2390 {
2391         if (c->x86 != 6)
2392                 return;
2393
2394         switch (c->x86_model) {
2395         case INTEL_FAM6_NEHALEM:
2396         case INTEL_FAM6_WESTMERE:
2397         case INTEL_FAM6_SANDYBRIDGE:
2398         case INTEL_FAM6_IVYBRIDGE:
2399         case INTEL_FAM6_HASWELL:
2400         case INTEL_FAM6_HASWELL_L:
2401         case INTEL_FAM6_HASWELL_G:
2402         case INTEL_FAM6_BROADWELL:
2403         case INTEL_FAM6_BROADWELL_G:
2404         case INTEL_FAM6_SKYLAKE_L:
2405         case INTEL_FAM6_SKYLAKE:
2406         case INTEL_FAM6_KABYLAKE_L:
2407         case INTEL_FAM6_KABYLAKE:
2408                 if (c->x86_cache_bits < 44)
2409                         c->x86_cache_bits = 44;
2410                 break;
2411         }
2412 }
2413
2414 static void __init l1tf_select_mitigation(void)
2415 {
2416         u64 half_pa;
2417
2418         if (!boot_cpu_has_bug(X86_BUG_L1TF))
2419                 return;
2420
2421         if (cpu_mitigations_off())
2422                 l1tf_mitigation = L1TF_MITIGATION_OFF;
2423         else if (cpu_mitigations_auto_nosmt())
2424                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2425
2426         override_cache_bits(&boot_cpu_data);
2427
2428         switch (l1tf_mitigation) {
2429         case L1TF_MITIGATION_OFF:
2430         case L1TF_MITIGATION_FLUSH_NOWARN:
2431         case L1TF_MITIGATION_FLUSH:
2432                 break;
2433         case L1TF_MITIGATION_FLUSH_NOSMT:
2434         case L1TF_MITIGATION_FULL:
2435                 cpu_smt_disable(false);
2436                 break;
2437         case L1TF_MITIGATION_FULL_FORCE:
2438                 cpu_smt_disable(true);
2439                 break;
2440         }
2441
2442 #if CONFIG_PGTABLE_LEVELS == 2
2443         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
2444         return;
2445 #endif
2446
2447         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
2448         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
2449                         e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
2450                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
2451                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
2452                                 half_pa);
2453                 pr_info("However, doing so will make a part of your RAM unusable.\n");
2454                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
2455                 return;
2456         }
2457
2458         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
2459 }
2460
2461 static int __init l1tf_cmdline(char *str)
2462 {
2463         if (!boot_cpu_has_bug(X86_BUG_L1TF))
2464                 return 0;
2465
2466         if (!str)
2467                 return -EINVAL;
2468
2469         if (!strcmp(str, "off"))
2470                 l1tf_mitigation = L1TF_MITIGATION_OFF;
2471         else if (!strcmp(str, "flush,nowarn"))
2472                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
2473         else if (!strcmp(str, "flush"))
2474                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
2475         else if (!strcmp(str, "flush,nosmt"))
2476                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
2477         else if (!strcmp(str, "full"))
2478                 l1tf_mitigation = L1TF_MITIGATION_FULL;
2479         else if (!strcmp(str, "full,force"))
2480                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
2481
2482         return 0;
2483 }
2484 early_param("l1tf", l1tf_cmdline);
2485
2486 #undef pr_fmt
2487 #define pr_fmt(fmt)     "Speculative Return Stack Overflow: " fmt
2488
2489 enum srso_mitigation {
2490         SRSO_MITIGATION_NONE,
2491         SRSO_MITIGATION_UCODE_NEEDED,
2492         SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED,
2493         SRSO_MITIGATION_MICROCODE,
2494         SRSO_MITIGATION_SAFE_RET,
2495         SRSO_MITIGATION_IBPB,
2496         SRSO_MITIGATION_IBPB_ON_VMEXIT,
2497 };
2498
2499 enum srso_mitigation_cmd {
2500         SRSO_CMD_OFF,
2501         SRSO_CMD_MICROCODE,
2502         SRSO_CMD_SAFE_RET,
2503         SRSO_CMD_IBPB,
2504         SRSO_CMD_IBPB_ON_VMEXIT,
2505 };
2506
2507 static const char * const srso_strings[] = {
2508         [SRSO_MITIGATION_NONE]                  = "Vulnerable",
2509         [SRSO_MITIGATION_UCODE_NEEDED]          = "Vulnerable: No microcode",
2510         [SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED] = "Vulnerable: Safe RET, no microcode",
2511         [SRSO_MITIGATION_MICROCODE]             = "Vulnerable: Microcode, no safe RET",
2512         [SRSO_MITIGATION_SAFE_RET]              = "Mitigation: Safe RET",
2513         [SRSO_MITIGATION_IBPB]                  = "Mitigation: IBPB",
2514         [SRSO_MITIGATION_IBPB_ON_VMEXIT]        = "Mitigation: IBPB on VMEXIT only"
2515 };
2516
2517 static enum srso_mitigation srso_mitigation __ro_after_init = SRSO_MITIGATION_NONE;
2518 static enum srso_mitigation_cmd srso_cmd __ro_after_init = SRSO_CMD_SAFE_RET;
2519
2520 static int __init srso_parse_cmdline(char *str)
2521 {
2522         if (!str)
2523                 return -EINVAL;
2524
2525         if (!strcmp(str, "off"))
2526                 srso_cmd = SRSO_CMD_OFF;
2527         else if (!strcmp(str, "microcode"))
2528                 srso_cmd = SRSO_CMD_MICROCODE;
2529         else if (!strcmp(str, "safe-ret"))
2530                 srso_cmd = SRSO_CMD_SAFE_RET;
2531         else if (!strcmp(str, "ibpb"))
2532                 srso_cmd = SRSO_CMD_IBPB;
2533         else if (!strcmp(str, "ibpb-vmexit"))
2534                 srso_cmd = SRSO_CMD_IBPB_ON_VMEXIT;
2535         else
2536                 pr_err("Ignoring unknown SRSO option (%s).", str);
2537
2538         return 0;
2539 }
2540 early_param("spec_rstack_overflow", srso_parse_cmdline);
2541
2542 #define SRSO_NOTICE "WARNING: See https://kernel.org/doc/html/latest/admin-guide/hw-vuln/srso.html for mitigation options."
2543
2544 static void __init srso_select_mitigation(void)
2545 {
2546         bool has_microcode = boot_cpu_has(X86_FEATURE_IBPB_BRTYPE);
2547
2548         if (cpu_mitigations_off())
2549                 return;
2550
2551         if (!boot_cpu_has_bug(X86_BUG_SRSO)) {
2552                 if (boot_cpu_has(X86_FEATURE_SBPB))
2553                         x86_pred_cmd = PRED_CMD_SBPB;
2554                 return;
2555         }
2556
2557         if (has_microcode) {
2558                 /*
2559                  * Zen1/2 with SMT off aren't vulnerable after the right
2560                  * IBPB microcode has been applied.
2561                  *
2562                  * Zen1/2 don't have SBPB, no need to try to enable it here.
2563                  */
2564                 if (boot_cpu_data.x86 < 0x19 && !cpu_smt_possible()) {
2565                         setup_force_cpu_cap(X86_FEATURE_SRSO_NO);
2566                         return;
2567                 }
2568
2569                 if (retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
2570                         srso_mitigation = SRSO_MITIGATION_IBPB;
2571                         goto out;
2572                 }
2573         } else {
2574                 pr_warn("IBPB-extending microcode not applied!\n");
2575                 pr_warn(SRSO_NOTICE);
2576
2577                 /* may be overwritten by SRSO_CMD_SAFE_RET below */
2578                 srso_mitigation = SRSO_MITIGATION_UCODE_NEEDED;
2579         }
2580
2581         switch (srso_cmd) {
2582         case SRSO_CMD_OFF:
2583                 if (boot_cpu_has(X86_FEATURE_SBPB))
2584                         x86_pred_cmd = PRED_CMD_SBPB;
2585                 return;
2586
2587         case SRSO_CMD_MICROCODE:
2588                 if (has_microcode) {
2589                         srso_mitigation = SRSO_MITIGATION_MICROCODE;
2590                         pr_warn(SRSO_NOTICE);
2591                 }
2592                 break;
2593
2594         case SRSO_CMD_SAFE_RET:
2595                 if (IS_ENABLED(CONFIG_MITIGATION_SRSO)) {
2596                         /*
2597                          * Enable the return thunk for generated code
2598                          * like ftrace, static_call, etc.
2599                          */
2600                         setup_force_cpu_cap(X86_FEATURE_RETHUNK);
2601                         setup_force_cpu_cap(X86_FEATURE_UNRET);
2602
2603                         if (boot_cpu_data.x86 == 0x19) {
2604                                 setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
2605                                 x86_return_thunk = srso_alias_return_thunk;
2606                         } else {
2607                                 setup_force_cpu_cap(X86_FEATURE_SRSO);
2608                                 x86_return_thunk = srso_return_thunk;
2609                         }
2610                         if (has_microcode)
2611                                 srso_mitigation = SRSO_MITIGATION_SAFE_RET;
2612                         else
2613                                 srso_mitigation = SRSO_MITIGATION_SAFE_RET_UCODE_NEEDED;
2614                 } else {
2615                         pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n");
2616                 }
2617                 break;
2618
2619         case SRSO_CMD_IBPB:
2620                 if (IS_ENABLED(CONFIG_MITIGATION_IBPB_ENTRY)) {
2621                         if (has_microcode) {
2622                                 setup_force_cpu_cap(X86_FEATURE_ENTRY_IBPB);
2623                                 srso_mitigation = SRSO_MITIGATION_IBPB;
2624                         }
2625                 } else {
2626                         pr_err("WARNING: kernel not compiled with MITIGATION_IBPB_ENTRY.\n");
2627                 }
2628                 break;
2629
2630         case SRSO_CMD_IBPB_ON_VMEXIT:
2631                 if (IS_ENABLED(CONFIG_MITIGATION_SRSO)) {
2632                         if (!boot_cpu_has(X86_FEATURE_ENTRY_IBPB) && has_microcode) {
2633                                 setup_force_cpu_cap(X86_FEATURE_IBPB_ON_VMEXIT);
2634                                 srso_mitigation = SRSO_MITIGATION_IBPB_ON_VMEXIT;
2635                         }
2636                 } else {
2637                         pr_err("WARNING: kernel not compiled with MITIGATION_SRSO.\n");
2638                 }
2639                 break;
2640         }
2641
2642 out:
2643         pr_info("%s\n", srso_strings[srso_mitigation]);
2644 }
2645
2646 #undef pr_fmt
2647 #define pr_fmt(fmt) fmt
2648
2649 #ifdef CONFIG_SYSFS
2650
2651 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
2652
2653 #if IS_ENABLED(CONFIG_KVM_INTEL)
2654 static const char * const l1tf_vmx_states[] = {
2655         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
2656         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
2657         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
2658         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
2659         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
2660         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
2661 };
2662
2663 static ssize_t l1tf_show_state(char *buf)
2664 {
2665         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
2666                 return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG);
2667
2668         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
2669             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
2670              sched_smt_active())) {
2671                 return sysfs_emit(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
2672                                   l1tf_vmx_states[l1tf_vmx_mitigation]);
2673         }
2674
2675         return sysfs_emit(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
2676                           l1tf_vmx_states[l1tf_vmx_mitigation],
2677                           sched_smt_active() ? "vulnerable" : "disabled");
2678 }
2679
2680 static ssize_t itlb_multihit_show_state(char *buf)
2681 {
2682         if (!boot_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) ||
2683             !boot_cpu_has(X86_FEATURE_VMX))
2684                 return sysfs_emit(buf, "KVM: Mitigation: VMX unsupported\n");
2685         else if (!(cr4_read_shadow() & X86_CR4_VMXE))
2686                 return sysfs_emit(buf, "KVM: Mitigation: VMX disabled\n");
2687         else if (itlb_multihit_kvm_mitigation)
2688                 return sysfs_emit(buf, "KVM: Mitigation: Split huge pages\n");
2689         else
2690                 return sysfs_emit(buf, "KVM: Vulnerable\n");
2691 }
2692 #else
2693 static ssize_t l1tf_show_state(char *buf)
2694 {
2695         return sysfs_emit(buf, "%s\n", L1TF_DEFAULT_MSG);
2696 }
2697
2698 static ssize_t itlb_multihit_show_state(char *buf)
2699 {
2700         return sysfs_emit(buf, "Processor vulnerable\n");
2701 }
2702 #endif
2703
2704 static ssize_t mds_show_state(char *buf)
2705 {
2706         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2707                 return sysfs_emit(buf, "%s; SMT Host state unknown\n",
2708                                   mds_strings[mds_mitigation]);
2709         }
2710
2711         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
2712                 return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2713                                   (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
2714                                    sched_smt_active() ? "mitigated" : "disabled"));
2715         }
2716
2717         return sysfs_emit(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
2718                           sched_smt_active() ? "vulnerable" : "disabled");
2719 }
2720
2721 static ssize_t tsx_async_abort_show_state(char *buf)
2722 {
2723         if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
2724             (taa_mitigation == TAA_MITIGATION_OFF))
2725                 return sysfs_emit(buf, "%s\n", taa_strings[taa_mitigation]);
2726
2727         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2728                 return sysfs_emit(buf, "%s; SMT Host state unknown\n",
2729                                   taa_strings[taa_mitigation]);
2730         }
2731
2732         return sysfs_emit(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
2733                           sched_smt_active() ? "vulnerable" : "disabled");
2734 }
2735
2736 static ssize_t mmio_stale_data_show_state(char *buf)
2737 {
2738         if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
2739                 return sysfs_emit(buf, "Unknown: No mitigations\n");
2740
2741         if (mmio_mitigation == MMIO_MITIGATION_OFF)
2742                 return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]);
2743
2744         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
2745                 return sysfs_emit(buf, "%s; SMT Host state unknown\n",
2746                                   mmio_strings[mmio_mitigation]);
2747         }
2748
2749         return sysfs_emit(buf, "%s; SMT %s\n", mmio_strings[mmio_mitigation],
2750                           sched_smt_active() ? "vulnerable" : "disabled");
2751 }
2752
2753 static ssize_t rfds_show_state(char *buf)
2754 {
2755         return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]);
2756 }
2757
2758 static char *stibp_state(void)
2759 {
2760         if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
2761             !boot_cpu_has(X86_FEATURE_AUTOIBRS))
2762                 return "";
2763
2764         switch (spectre_v2_user_stibp) {
2765         case SPECTRE_V2_USER_NONE:
2766                 return "; STIBP: disabled";
2767         case SPECTRE_V2_USER_STRICT:
2768                 return "; STIBP: forced";
2769         case SPECTRE_V2_USER_STRICT_PREFERRED:
2770                 return "; STIBP: always-on";
2771         case SPECTRE_V2_USER_PRCTL:
2772         case SPECTRE_V2_USER_SECCOMP:
2773                 if (static_key_enabled(&switch_to_cond_stibp))
2774                         return "; STIBP: conditional";
2775         }
2776         return "";
2777 }
2778
2779 static char *ibpb_state(void)
2780 {
2781         if (boot_cpu_has(X86_FEATURE_IBPB)) {
2782                 if (static_key_enabled(&switch_mm_always_ibpb))
2783                         return "; IBPB: always-on";
2784                 if (static_key_enabled(&switch_mm_cond_ibpb))
2785                         return "; IBPB: conditional";
2786                 return "; IBPB: disabled";
2787         }
2788         return "";
2789 }
2790
2791 static char *pbrsb_eibrs_state(void)
2792 {
2793         if (boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB)) {
2794                 if (boot_cpu_has(X86_FEATURE_RSB_VMEXIT_LITE) ||
2795                     boot_cpu_has(X86_FEATURE_RSB_VMEXIT))
2796                         return "; PBRSB-eIBRS: SW sequence";
2797                 else
2798                         return "; PBRSB-eIBRS: Vulnerable";
2799         } else {
2800                 return "; PBRSB-eIBRS: Not affected";
2801         }
2802 }
2803
2804 static const char *spectre_bhi_state(void)
2805 {
2806         if (!boot_cpu_has_bug(X86_BUG_BHI))
2807                 return "; BHI: Not affected";
2808         else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_HW))
2809                 return "; BHI: BHI_DIS_S";
2810         else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
2811                 return "; BHI: SW loop, KVM: SW loop";
2812         else if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
2813                  !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) &&
2814                  rrsba_disabled)
2815                 return "; BHI: Retpoline";
2816         else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP_ON_VMEXIT))
2817                 return "; BHI: Vulnerable, KVM: SW loop";
2818
2819         return "; BHI: Vulnerable";
2820 }
2821
2822 static ssize_t spectre_v2_show_state(char *buf)
2823 {
2824         if (spectre_v2_enabled == SPECTRE_V2_LFENCE)
2825                 return sysfs_emit(buf, "Vulnerable: LFENCE\n");
2826
2827         if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
2828                 return sysfs_emit(buf, "Vulnerable: eIBRS with unprivileged eBPF\n");
2829
2830         if (sched_smt_active() && unprivileged_ebpf_enabled() &&
2831             spectre_v2_enabled == SPECTRE_V2_EIBRS_LFENCE)
2832                 return sysfs_emit(buf, "Vulnerable: eIBRS+LFENCE with unprivileged eBPF and SMT\n");
2833
2834         return sysfs_emit(buf, "%s%s%s%s%s%s%s%s\n",
2835                           spectre_v2_strings[spectre_v2_enabled],
2836                           ibpb_state(),
2837                           boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? "; IBRS_FW" : "",
2838                           stibp_state(),
2839                           boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? "; RSB filling" : "",
2840                           pbrsb_eibrs_state(),
2841                           spectre_bhi_state(),
2842                           /* this should always be at the end */
2843                           spectre_v2_module_string());
2844 }
2845
2846 static ssize_t srbds_show_state(char *buf)
2847 {
2848         return sysfs_emit(buf, "%s\n", srbds_strings[srbds_mitigation]);
2849 }
2850
2851 static ssize_t retbleed_show_state(char *buf)
2852 {
2853         if (retbleed_mitigation == RETBLEED_MITIGATION_UNRET ||
2854             retbleed_mitigation == RETBLEED_MITIGATION_IBPB) {
2855                 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
2856                     boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
2857                         return sysfs_emit(buf, "Vulnerable: untrained return thunk / IBPB on non-AMD based uarch\n");
2858
2859                 return sysfs_emit(buf, "%s; SMT %s\n", retbleed_strings[retbleed_mitigation],
2860                                   !sched_smt_active() ? "disabled" :
2861                                   spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT ||
2862                                   spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED ?
2863                                   "enabled with STIBP protection" : "vulnerable");
2864         }
2865
2866         return sysfs_emit(buf, "%s\n", retbleed_strings[retbleed_mitigation]);
2867 }
2868
2869 static ssize_t srso_show_state(char *buf)
2870 {
2871         if (boot_cpu_has(X86_FEATURE_SRSO_NO))
2872                 return sysfs_emit(buf, "Mitigation: SMT disabled\n");
2873
2874         return sysfs_emit(buf, "%s\n", srso_strings[srso_mitigation]);
2875 }
2876
2877 static ssize_t gds_show_state(char *buf)
2878 {
2879         return sysfs_emit(buf, "%s\n", gds_strings[gds_mitigation]);
2880 }
2881
2882 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
2883                                char *buf, unsigned int bug)
2884 {
2885         if (!boot_cpu_has_bug(bug))
2886                 return sysfs_emit(buf, "Not affected\n");
2887
2888         switch (bug) {
2889         case X86_BUG_CPU_MELTDOWN:
2890                 if (boot_cpu_has(X86_FEATURE_PTI))
2891                         return sysfs_emit(buf, "Mitigation: PTI\n");
2892
2893                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
2894                         return sysfs_emit(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
2895
2896                 break;
2897
2898         case X86_BUG_SPECTRE_V1:
2899                 return sysfs_emit(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
2900
2901         case X86_BUG_SPECTRE_V2:
2902                 return spectre_v2_show_state(buf);
2903
2904         case X86_BUG_SPEC_STORE_BYPASS:
2905                 return sysfs_emit(buf, "%s\n", ssb_strings[ssb_mode]);
2906
2907         case X86_BUG_L1TF:
2908                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
2909                         return l1tf_show_state(buf);
2910                 break;
2911
2912         case X86_BUG_MDS:
2913                 return mds_show_state(buf);
2914
2915         case X86_BUG_TAA:
2916                 return tsx_async_abort_show_state(buf);
2917
2918         case X86_BUG_ITLB_MULTIHIT:
2919                 return itlb_multihit_show_state(buf);
2920
2921         case X86_BUG_SRBDS:
2922                 return srbds_show_state(buf);
2923
2924         case X86_BUG_MMIO_STALE_DATA:
2925         case X86_BUG_MMIO_UNKNOWN:
2926                 return mmio_stale_data_show_state(buf);
2927
2928         case X86_BUG_RETBLEED:
2929                 return retbleed_show_state(buf);
2930
2931         case X86_BUG_SRSO:
2932                 return srso_show_state(buf);
2933
2934         case X86_BUG_GDS:
2935                 return gds_show_state(buf);
2936
2937         case X86_BUG_RFDS:
2938                 return rfds_show_state(buf);
2939
2940         default:
2941                 break;
2942         }
2943
2944         return sysfs_emit(buf, "Vulnerable\n");
2945 }
2946
2947 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
2948 {
2949         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
2950 }
2951
2952 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
2953 {
2954         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
2955 }
2956
2957 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
2958 {
2959         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
2960 }
2961
2962 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
2963 {
2964         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
2965 }
2966
2967 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
2968 {
2969         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
2970 }
2971
2972 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
2973 {
2974         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
2975 }
2976
2977 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
2978 {
2979         return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
2980 }
2981
2982 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
2983 {
2984         return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
2985 }
2986
2987 ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char *buf)
2988 {
2989         return cpu_show_common(dev, attr, buf, X86_BUG_SRBDS);
2990 }
2991
2992 ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf)
2993 {
2994         if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN))
2995                 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN);
2996         else
2997                 return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA);
2998 }
2999
3000 ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf)
3001 {
3002         return cpu_show_common(dev, attr, buf, X86_BUG_RETBLEED);
3003 }
3004
3005 ssize_t cpu_show_spec_rstack_overflow(struct device *dev, struct device_attribute *attr, char *buf)
3006 {
3007         return cpu_show_common(dev, attr, buf, X86_BUG_SRSO);
3008 }
3009
3010 ssize_t cpu_show_gds(struct device *dev, struct device_attribute *attr, char *buf)
3011 {
3012         return cpu_show_common(dev, attr, buf, X86_BUG_GDS);
3013 }
3014
3015 ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attribute *attr, char *buf)
3016 {
3017         return cpu_show_common(dev, attr, buf, X86_BUG_RFDS);
3018 }
3019 #endif
3020
3021 void __warn_thunk(void)
3022 {
3023         WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n");
3024 }