Commit | Line | Data |
---|---|---|
f19fbd5e MS |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/module.h> | |
d424986f | 3 | #include <linux/device.h> |
0336e04a | 4 | #include <linux/cpu.h> |
f19fbd5e MS |
5 | #include <asm/nospec-branch.h> |
6 | ||
b2e2f43a MS |
7 | static int __init nobp_setup_early(char *str) |
8 | { | |
9 | bool enabled; | |
10 | int rc; | |
11 | ||
12 | rc = kstrtobool(str, &enabled); | |
13 | if (rc) | |
14 | return rc; | |
6e179d64 MS |
15 | if (enabled && test_facility(82)) { |
16 | /* | |
17 | * The user explicitely requested nobp=1, enable it and | |
18 | * disable the expoline support. | |
19 | */ | |
17e89e13 | 20 | __set_facility(82, alt_stfle_fac_list); |
6e179d64 MS |
21 | if (IS_ENABLED(CONFIG_EXPOLINE)) |
22 | nospec_disable = 1; | |
23 | } else { | |
17e89e13 | 24 | __clear_facility(82, alt_stfle_fac_list); |
6e179d64 | 25 | } |
b2e2f43a MS |
26 | return 0; |
27 | } | |
28 | early_param("nobp", nobp_setup_early); | |
29 | ||
30 | static int __init nospec_setup_early(char *str) | |
31 | { | |
17e89e13 | 32 | __clear_facility(82, alt_stfle_fac_list); |
b2e2f43a MS |
33 | return 0; |
34 | } | |
35 | early_param("nospec", nospec_setup_early); | |
36 | ||
bc035599 MS |
37 | static int __init nospec_report(void) |
38 | { | |
aeaf7002 MS |
39 | if (test_facility(156)) |
40 | pr_info("Spectre V2 mitigation: etokens\n"); | |
5d17d4ed | 41 | if (nospec_uses_trampoline()) |
b7e7f505 | 42 | pr_info("Spectre V2 mitigation: execute trampolines\n"); |
17e89e13 | 43 | if (__test_facility(82, alt_stfle_fac_list)) |
b7e7f505 | 44 | pr_info("Spectre V2 mitigation: limited branch prediction\n"); |
bc035599 MS |
45 | return 0; |
46 | } | |
47 | arch_initcall(nospec_report); | |
48 | ||
b2e2f43a MS |
49 | #ifdef CONFIG_EXPOLINE |
50 | ||
6e179d64 | 51 | int nospec_disable = IS_ENABLED(CONFIG_EXPOLINE_OFF); |
f19fbd5e MS |
52 | |
53 | static int __init nospectre_v2_setup_early(char *str) | |
54 | { | |
6e179d64 | 55 | nospec_disable = 1; |
f19fbd5e MS |
56 | return 0; |
57 | } | |
58 | early_param("nospectre_v2", nospectre_v2_setup_early); | |
59 | ||
6a3d1e81 | 60 | void __init nospec_auto_detect(void) |
6e179d64 | 61 | { |
0336e04a | 62 | if (test_facility(156) || cpu_mitigations_off()) { |
aeaf7002 MS |
63 | /* |
64 | * The machine supports etokens. | |
65 | * Disable expolines and disable nobp. | |
66 | */ | |
475c8e9e | 67 | if (__is_defined(CC_USING_EXPOLINE)) |
aeaf7002 | 68 | nospec_disable = 1; |
17e89e13 | 69 | __clear_facility(82, alt_stfle_fac_list); |
475c8e9e | 70 | } else if (__is_defined(CC_USING_EXPOLINE)) { |
6e179d64 MS |
71 | /* |
72 | * The kernel has been compiled with expolines. | |
73 | * Keep expolines enabled and disable nobp. | |
74 | */ | |
75 | nospec_disable = 0; | |
17e89e13 | 76 | __clear_facility(82, alt_stfle_fac_list); |
6e179d64 MS |
77 | } |
78 | /* | |
79 | * If the kernel has not been compiled with expolines the | |
80 | * nobp setting decides what is done, this depends on the | |
81 | * CONFIG_KERNEL_NP option and the nobp/nospec parameters. | |
82 | */ | |
6e179d64 | 83 | } |
6e179d64 | 84 | |
f19fbd5e MS |
85 | static int __init spectre_v2_setup_early(char *str) |
86 | { | |
87 | if (str && !strncmp(str, "on", 2)) { | |
6e179d64 | 88 | nospec_disable = 0; |
17e89e13 | 89 | __clear_facility(82, alt_stfle_fac_list); |
f19fbd5e | 90 | } |
6e179d64 MS |
91 | if (str && !strncmp(str, "off", 3)) |
92 | nospec_disable = 1; | |
93 | if (str && !strncmp(str, "auto", 4)) | |
6a3d1e81 | 94 | nospec_auto_detect(); |
f19fbd5e MS |
95 | return 0; |
96 | } | |
97 | early_param("spectre_v2", spectre_v2_setup_early); | |
98 | ||
99 | static void __init_or_module __nospec_revert(s32 *start, s32 *end) | |
100 | { | |
101 | enum { BRCL_EXPOLINE, BRASL_EXPOLINE } type; | |
c74d3c18 | 102 | static const u8 branch[] = { 0x47, 0x00, 0x07, 0x00 }; |
f19fbd5e MS |
103 | u8 *instr, *thunk, *br; |
104 | u8 insnbuf[6]; | |
105 | s32 *epo; | |
106 | ||
107 | /* Second part of the instruction replace is always a nop */ | |
2268169c | 108 | memcpy(insnbuf + 2, branch, sizeof(branch)); |
f19fbd5e MS |
109 | for (epo = start; epo < end; epo++) { |
110 | instr = (u8 *) epo + *epo; | |
111 | if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x04) | |
112 | type = BRCL_EXPOLINE; /* brcl instruction */ | |
113 | else if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x05) | |
114 | type = BRASL_EXPOLINE; /* brasl instruction */ | |
115 | else | |
116 | continue; | |
117 | thunk = instr + (*(int *)(instr + 2)) * 2; | |
118 | if (thunk[0] == 0xc6 && thunk[1] == 0x00) | |
119 | /* exrl %r0,<target-br> */ | |
120 | br = thunk + (*(int *)(thunk + 2)) * 2; | |
f19fbd5e MS |
121 | else |
122 | continue; | |
2268169c | 123 | if (br[0] != 0x07 || (br[1] & 0xf0) != 0xf0) |
f19fbd5e MS |
124 | continue; |
125 | switch (type) { | |
126 | case BRCL_EXPOLINE: | |
2268169c | 127 | /* brcl to thunk, replace with br + nop */ |
f19fbd5e MS |
128 | insnbuf[0] = br[0]; |
129 | insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f); | |
130 | break; | |
131 | case BRASL_EXPOLINE: | |
2268169c VG |
132 | /* brasl to thunk, replace with basr + nop */ |
133 | insnbuf[0] = 0x0d; | |
f19fbd5e MS |
134 | insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f); |
135 | break; | |
136 | } | |
137 | ||
138 | s390_kernel_write(instr, insnbuf, 6); | |
139 | } | |
140 | } | |
141 | ||
6e179d64 | 142 | void __init_or_module nospec_revert(s32 *start, s32 *end) |
f19fbd5e | 143 | { |
6e179d64 | 144 | if (nospec_disable) |
f19fbd5e MS |
145 | __nospec_revert(start, end); |
146 | } | |
147 | ||
148 | extern s32 __nospec_call_start[], __nospec_call_end[]; | |
149 | extern s32 __nospec_return_start[], __nospec_return_end[]; | |
150 | void __init nospec_init_branches(void) | |
151 | { | |
6e179d64 MS |
152 | nospec_revert(__nospec_call_start, __nospec_call_end); |
153 | nospec_revert(__nospec_return_start, __nospec_return_end); | |
f19fbd5e | 154 | } |
b2e2f43a MS |
155 | |
156 | #endif /* CONFIG_EXPOLINE */ |