x86/its: Enable Indirect Target Selection mitigation

Indirect Target Selection (ITS) is a bug in some pre-ADL Intel CPUs with
eIBRS. It affects prediction of indirect branch and RETs in the
lower half of cacheline. Due to ITS such branches may get wrongly predicted
to a target of (direct or indirect) branch that is located in the upper
half of the cacheline.

Scope of impact
===============

Guest/host isolation
--------------------
When eIBRS is used for guest/host isolation, the indirect branches in the
VMM may still be predicted with targets corresponding to branches in the
guest.

Intra-mode
----------
cBPF or other native gadgets can be used for intra-mode training and
disclosure using ITS.

User/kernel isolation
---------------------
When eIBRS is enabled user/kernel isolation is not impacted.

Indirect Branch Prediction Barrier (IBPB)
-----------------------------------------
After an IBPB, indirect branches may be predicted with targets
corresponding to direct branches which were executed prior to IBPB. This is
mitigated by a microcode update.

Add cmdline parameter indirect_target_selection=off|on|force to control the
mitigation to relocate the affected branches to an ITS-safe thunk i.e.
located in the upper half of cacheline. Also add the sysfs reporting.

When retpoline mitigation is deployed, ITS safe-thunks are not needed,
because retpoline sequence is already ITS-safe. Similarly, when call depth
tracking (CDT) mitigation is deployed (retbleed=stuff), ITS safe return
thunk is not used, as CDT prevents RSB-underflow.

To not overcomplicate things, ITS mitigation is not supported with
spectre-v2 lfence;jmp mitigation. Moreover, it is less practical to deploy
lfence;jmp mitigation on ITS affected parts anyways.

Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Josh Poimboeuf <jpoimboe@kernel.org>
Reviewed-by: Alexandre Chartre <alexandre.chartre@oracle.com>

diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu
index 206079d3bd5b127b0ccddfd4be50c978563a0b46..6a1acabb29d85f4954d424eff416e98474a0412d 100644 (file)
--- a/Documentation/ABI/testing/sysfs-devices-system-cpu
+++ b/Documentation/ABI/testing/sysfs-devices-system-cpu
@@ -511,6 +511,7 @@ Description:        information about CPUs heterogeneity.
 
 What:          /sys/devices/system/cpu/vulnerabilities
                /sys/devices/system/cpu/vulnerabilities/gather_data_sampling
+               /sys/devices/system/cpu/vulnerabilities/indirect_target_selection
                /sys/devices/system/cpu/vulnerabilities/itlb_multihit
                /sys/devices/system/cpu/vulnerabilities/l1tf
                /sys/devices/system/cpu/vulnerabilities/mds

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index d9fd26b95b340909101b80ab4b22a6bbd3d20ab6..d3e78e795e0d6fdd2ad286014b8b04480ea1ccba 100644 (file)
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2202,6 +2202,18 @@
                        different crypto accelerators. This option can be used
                        to achieve best performance for particular HW.
 
+       indirect_target_selection= [X86,Intel] Mitigation control for Indirect
+                       Target Selection(ITS) bug in Intel CPUs. Updated
+                       microcode is also required for a fix in IBPB.
+
+                       on:     Enable mitigation (default).
+                       off:    Disable mitigation.
+                       force:  Force the ITS bug and deploy default
+                               mitigation.
+
+                       For details see:
+                       Documentation/admin-guide/hw-vuln/indirect-target-selection.rst
+
        init=           [KNL]
                        Format: <full_path>
                        Run specified binary instead of /sbin/init as init
@@ -3693,6 +3705,7 @@
                                expose users to several CPU vulnerabilities.
                                Equivalent to: if nokaslr then kpti=0 [ARM64]
                                               gather_data_sampling=off [X86]
+                                              indirect_target_selection=off [X86]
                                               kvm.nx_huge_pages=off [X86]
                                               l1tf=off [X86]
                                               mds=off [X86]

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 362602b705cc43bd5e9df7c2157f44e7bfb304b9..417596ce5352b520f8190cda8633d09c6ff6ffc3 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -49,6 +49,7 @@ static void __init srbds_select_mitigation(void);
 static void __init l1d_flush_select_mitigation(void);
 static void __init srso_select_mitigation(void);
 static void __init gds_select_mitigation(void);
+static void __init its_select_mitigation(void);
 
 /* The base value of the SPEC_CTRL MSR without task-specific bits set */
 u64 x86_spec_ctrl_base;
@@ -66,6 +67,14 @@ static DEFINE_MUTEX(spec_ctrl_mutex);
 
 void (*x86_return_thunk)(void) __ro_after_init = __x86_return_thunk;
 
+static void __init set_return_thunk(void *thunk)
+{
+       if (x86_return_thunk != __x86_return_thunk)
+               pr_warn("x86/bugs: return thunk changed\n");
+
+       x86_return_thunk = thunk;
+}
+
 /* Update SPEC_CTRL MSR and its cached copy unconditionally */
 static void update_spec_ctrl(u64 val)
 {
@@ -178,6 +187,7 @@ void __init cpu_select_mitigations(void)
         */
        srso_select_mitigation();
        gds_select_mitigation();
+       its_select_mitigation();
 }
 
 /*
@@ -1118,7 +1128,7 @@ do_cmd_auto:
                setup_force_cpu_cap(X86_FEATURE_RETHUNK);
                setup_force_cpu_cap(X86_FEATURE_UNRET);
 
-               x86_return_thunk = retbleed_return_thunk;
+               set_return_thunk(retbleed_return_thunk);
 
                if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
                    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
@@ -1153,7 +1163,7 @@ do_cmd_auto:
                setup_force_cpu_cap(X86_FEATURE_RETHUNK);
                setup_force_cpu_cap(X86_FEATURE_CALL_DEPTH);
 
-               x86_return_thunk = call_depth_return_thunk;
+               set_return_thunk(call_depth_return_thunk);
                break;
 
        default:
@@ -1187,6 +1197,115 @@ do_cmd_auto:
        pr_info("%s\n", retbleed_strings[retbleed_mitigation]);
 }
 
+#undef pr_fmt
+#define pr_fmt(fmt)     "ITS: " fmt
+
+enum its_mitigation_cmd {
+       ITS_CMD_OFF,
+       ITS_CMD_ON,
+};
+
+enum its_mitigation {
+       ITS_MITIGATION_OFF,
+       ITS_MITIGATION_ALIGNED_THUNKS,
+       ITS_MITIGATION_RETPOLINE_STUFF,
+};
+
+static const char * const its_strings[] = {
+       [ITS_MITIGATION_OFF]                    = "Vulnerable",
+       [ITS_MITIGATION_ALIGNED_THUNKS]         = "Mitigation: Aligned branch/return thunks",
+       [ITS_MITIGATION_RETPOLINE_STUFF]        = "Mitigation: Retpolines, Stuffing RSB",
+};
+
+static enum its_mitigation its_mitigation __ro_after_init = ITS_MITIGATION_ALIGNED_THUNKS;
+
+static enum its_mitigation_cmd its_cmd __ro_after_init =
+       IS_ENABLED(CONFIG_MITIGATION_ITS) ? ITS_CMD_ON : ITS_CMD_OFF;
+
+static int __init its_parse_cmdline(char *str)
+{
+       if (!str)
+               return -EINVAL;
+
+       if (!IS_ENABLED(CONFIG_MITIGATION_ITS)) {
+               pr_err("Mitigation disabled at compile time, ignoring option (%s)", str);
+               return 0;
+       }
+
+       if (!strcmp(str, "off")) {
+               its_cmd = ITS_CMD_OFF;
+       } else if (!strcmp(str, "on")) {
+               its_cmd = ITS_CMD_ON;
+       } else if (!strcmp(str, "force")) {
+               its_cmd = ITS_CMD_ON;
+               setup_force_cpu_bug(X86_BUG_ITS);
+       } else {
+               pr_err("Ignoring unknown indirect_target_selection option (%s).", str);
+       }
+
+       return 0;
+}
+early_param("indirect_target_selection", its_parse_cmdline);
+
+static void __init its_select_mitigation(void)
+{
+       enum its_mitigation_cmd cmd = its_cmd;
+
+       if (!boot_cpu_has_bug(X86_BUG_ITS) || cpu_mitigations_off()) {
+               its_mitigation = ITS_MITIGATION_OFF;
+               return;
+       }
+
+       /* Retpoline+CDT mitigates ITS, bail out */
+       if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
+           boot_cpu_has(X86_FEATURE_CALL_DEPTH)) {
+               its_mitigation = ITS_MITIGATION_RETPOLINE_STUFF;
+               goto out;
+       }
+
+       /* Exit early to avoid irrelevant warnings */
+       if (cmd == ITS_CMD_OFF) {
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+       if (spectre_v2_enabled == SPECTRE_V2_NONE) {
+               pr_err("WARNING: Spectre-v2 mitigation is off, disabling ITS\n");
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+       if (!IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) ||
+           !IS_ENABLED(CONFIG_MITIGATION_RETHUNK)) {
+               pr_err("WARNING: ITS mitigation depends on retpoline and rethunk support\n");
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+       if (IS_ENABLED(CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B)) {
+               pr_err("WARNING: ITS mitigation is not compatible with CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_64B\n");
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+       if (boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
+               pr_err("WARNING: ITS mitigation is not compatible with lfence mitigation\n");
+               its_mitigation = ITS_MITIGATION_OFF;
+               goto out;
+       }
+
+       switch (cmd) {
+       case ITS_CMD_OFF:
+               its_mitigation = ITS_MITIGATION_OFF;
+               break;
+       case ITS_CMD_ON:
+               its_mitigation = ITS_MITIGATION_ALIGNED_THUNKS;
+               if (!boot_cpu_has(X86_FEATURE_RETPOLINE))
+                       setup_force_cpu_cap(X86_FEATURE_INDIRECT_THUNK_ITS);
+               setup_force_cpu_cap(X86_FEATURE_RETHUNK);
+               set_return_thunk(its_return_thunk);
+               break;
+       }
+out:
+       pr_info("%s\n", its_strings[its_mitigation]);
+}
+
 #undef pr_fmt
 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 
@@ -2607,10 +2726,10 @@ static void __init srso_select_mitigation(void)
 
                        if (boot_cpu_data.x86 == 0x19) {
                                setup_force_cpu_cap(X86_FEATURE_SRSO_ALIAS);
-                               x86_return_thunk = srso_alias_return_thunk;
+                               set_return_thunk(srso_alias_return_thunk);
                        } else {
                                setup_force_cpu_cap(X86_FEATURE_SRSO);
-                               x86_return_thunk = srso_return_thunk;
+                               set_return_thunk(srso_return_thunk);
                        }
                        if (has_microcode)
                                srso_mitigation = SRSO_MITIGATION_SAFE_RET;
@@ -2800,6 +2919,11 @@ static ssize_t rfds_show_state(char *buf)
        return sysfs_emit(buf, "%s\n", rfds_strings[rfds_mitigation]);
 }
 
+static ssize_t its_show_state(char *buf)
+{
+       return sysfs_emit(buf, "%s\n", its_strings[its_mitigation]);
+}
+
 static char *stibp_state(void)
 {
        if (spectre_v2_in_eibrs_mode(spectre_v2_enabled) &&
@@ -2982,6 +3106,9 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr
        case X86_BUG_RFDS:
                return rfds_show_state(buf);
 
+       case X86_BUG_ITS:
+               return its_show_state(buf);
+
        default:
                break;
        }
@@ -3061,6 +3188,11 @@ ssize_t cpu_show_reg_file_data_sampling(struct device *dev, struct device_attrib
 {
        return cpu_show_common(dev, attr, buf, X86_BUG_RFDS);
 }
+
+ssize_t cpu_show_indirect_target_selection(struct device *dev, struct device_attribute *attr, char *buf)
+{
+       return cpu_show_common(dev, attr, buf, X86_BUG_ITS);
+}
 #endif
 
 void __warn_thunk(void)

diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index a7e5118498758ef05cc98cd87655dc0b64110193..50651435577c8f52fedd86e0b6c9edec84c2545d 100644 (file)
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -600,6 +600,7 @@ CPU_SHOW_VULN_FALLBACK(spec_rstack_overflow);
 CPU_SHOW_VULN_FALLBACK(gds);
 CPU_SHOW_VULN_FALLBACK(reg_file_data_sampling);
 CPU_SHOW_VULN_FALLBACK(ghostwrite);
+CPU_SHOW_VULN_FALLBACK(indirect_target_selection);
 
 static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL);
 static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL);
@@ -616,6 +617,7 @@ static DEVICE_ATTR(spec_rstack_overflow, 0444, cpu_show_spec_rstack_overflow, NU
 static DEVICE_ATTR(gather_data_sampling, 0444, cpu_show_gds, NULL);
 static DEVICE_ATTR(reg_file_data_sampling, 0444, cpu_show_reg_file_data_sampling, NULL);
 static DEVICE_ATTR(ghostwrite, 0444, cpu_show_ghostwrite, NULL);
+static DEVICE_ATTR(indirect_target_selection, 0444, cpu_show_indirect_target_selection, NULL);
 
 static struct attribute *cpu_root_vulnerabilities_attrs[] = {
        &dev_attr_meltdown.attr,
@@ -633,6 +635,7 @@ static struct attribute *cpu_root_vulnerabilities_attrs[] = {
        &dev_attr_gather_data_sampling.attr,
        &dev_attr_reg_file_data_sampling.attr,
        &dev_attr_ghostwrite.attr,
+       &dev_attr_indirect_target_selection.attr,
        NULL
 };
 

diff --git a/include/linux/cpu.h b/include/linux/cpu.h
index e3049543008b9c8b8185ed202b4eef8f5b80234a..3aa955102b349a97d9f5776799dc57e74299cff0 100644 (file)
--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -78,6 +78,8 @@ extern ssize_t cpu_show_gds(struct device *dev,
 extern ssize_t cpu_show_reg_file_data_sampling(struct device *dev,
                                               struct device_attribute *attr, char *buf);
 extern ssize_t cpu_show_ghostwrite(struct device *dev, struct device_attribute *attr, char *buf);
+extern ssize_t cpu_show_indirect_target_selection(struct device *dev,
+                                                 struct device_attribute *attr, char *buf);
 
 extern __printf(4, 5)
 struct device *cpu_device_create(struct device *parent, void *drvdata,