Commit | Line | Data |
---|---|---|
f19fbd5e MS |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/module.h> | |
d424986f | 3 | #include <linux/device.h> |
0336e04a | 4 | #include <linux/cpu.h> |
f19fbd5e MS |
5 | #include <asm/nospec-branch.h> |
6 | ||
b2e2f43a MS |
7 | static int __init nobp_setup_early(char *str) |
8 | { | |
9 | bool enabled; | |
10 | int rc; | |
11 | ||
12 | rc = kstrtobool(str, &enabled); | |
13 | if (rc) | |
14 | return rc; | |
6e179d64 MS |
15 | if (enabled && test_facility(82)) { |
16 | /* | |
17 | * The user explicitely requested nobp=1, enable it and | |
18 | * disable the expoline support. | |
19 | */ | |
17e89e13 | 20 | __set_facility(82, alt_stfle_fac_list); |
6e179d64 MS |
21 | if (IS_ENABLED(CONFIG_EXPOLINE)) |
22 | nospec_disable = 1; | |
23 | } else { | |
17e89e13 | 24 | __clear_facility(82, alt_stfle_fac_list); |
6e179d64 | 25 | } |
b2e2f43a MS |
26 | return 0; |
27 | } | |
28 | early_param("nobp", nobp_setup_early); | |
29 | ||
30 | static int __init nospec_setup_early(char *str) | |
31 | { | |
17e89e13 | 32 | __clear_facility(82, alt_stfle_fac_list); |
b2e2f43a MS |
33 | return 0; |
34 | } | |
35 | early_param("nospec", nospec_setup_early); | |
36 | ||
bc035599 MS |
37 | static int __init nospec_report(void) |
38 | { | |
aeaf7002 MS |
39 | if (test_facility(156)) |
40 | pr_info("Spectre V2 mitigation: etokens\n"); | |
475c8e9e | 41 | if (__is_defined(CC_USING_EXPOLINE) && !nospec_disable) |
b7e7f505 | 42 | pr_info("Spectre V2 mitigation: execute trampolines\n"); |
17e89e13 | 43 | if (__test_facility(82, alt_stfle_fac_list)) |
b7e7f505 | 44 | pr_info("Spectre V2 mitigation: limited branch prediction\n"); |
bc035599 MS |
45 | return 0; |
46 | } | |
47 | arch_initcall(nospec_report); | |
48 | ||
b2e2f43a MS |
49 | #ifdef CONFIG_EXPOLINE |
50 | ||
6e179d64 | 51 | int nospec_disable = IS_ENABLED(CONFIG_EXPOLINE_OFF); |
f19fbd5e MS |
52 | |
53 | static int __init nospectre_v2_setup_early(char *str) | |
54 | { | |
6e179d64 | 55 | nospec_disable = 1; |
f19fbd5e MS |
56 | return 0; |
57 | } | |
58 | early_param("nospectre_v2", nospectre_v2_setup_early); | |
59 | ||
6a3d1e81 | 60 | void __init nospec_auto_detect(void) |
6e179d64 | 61 | { |
0336e04a | 62 | if (test_facility(156) || cpu_mitigations_off()) { |
aeaf7002 MS |
63 | /* |
64 | * The machine supports etokens. | |
65 | * Disable expolines and disable nobp. | |
66 | */ | |
475c8e9e | 67 | if (__is_defined(CC_USING_EXPOLINE)) |
aeaf7002 | 68 | nospec_disable = 1; |
17e89e13 | 69 | __clear_facility(82, alt_stfle_fac_list); |
475c8e9e | 70 | } else if (__is_defined(CC_USING_EXPOLINE)) { |
6e179d64 MS |
71 | /* |
72 | * The kernel has been compiled with expolines. | |
73 | * Keep expolines enabled and disable nobp. | |
74 | */ | |
75 | nospec_disable = 0; | |
17e89e13 | 76 | __clear_facility(82, alt_stfle_fac_list); |
6e179d64 MS |
77 | } |
78 | /* | |
79 | * If the kernel has not been compiled with expolines the | |
80 | * nobp setting decides what is done, this depends on the | |
81 | * CONFIG_KERNEL_NP option and the nobp/nospec parameters. | |
82 | */ | |
6e179d64 | 83 | } |
6e179d64 | 84 | |
f19fbd5e MS |
85 | static int __init spectre_v2_setup_early(char *str) |
86 | { | |
87 | if (str && !strncmp(str, "on", 2)) { | |
6e179d64 | 88 | nospec_disable = 0; |
17e89e13 | 89 | __clear_facility(82, alt_stfle_fac_list); |
f19fbd5e | 90 | } |
6e179d64 MS |
91 | if (str && !strncmp(str, "off", 3)) |
92 | nospec_disable = 1; | |
93 | if (str && !strncmp(str, "auto", 4)) | |
6a3d1e81 | 94 | nospec_auto_detect(); |
f19fbd5e MS |
95 | return 0; |
96 | } | |
97 | early_param("spectre_v2", spectre_v2_setup_early); | |
98 | ||
99 | static void __init_or_module __nospec_revert(s32 *start, s32 *end) | |
100 | { | |
101 | enum { BRCL_EXPOLINE, BRASL_EXPOLINE } type; | |
102 | u8 *instr, *thunk, *br; | |
103 | u8 insnbuf[6]; | |
104 | s32 *epo; | |
105 | ||
106 | /* Second part of the instruction replace is always a nop */ | |
f19fbd5e MS |
107 | for (epo = start; epo < end; epo++) { |
108 | instr = (u8 *) epo + *epo; | |
109 | if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x04) | |
110 | type = BRCL_EXPOLINE; /* brcl instruction */ | |
111 | else if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x05) | |
112 | type = BRASL_EXPOLINE; /* brasl instruction */ | |
113 | else | |
114 | continue; | |
115 | thunk = instr + (*(int *)(instr + 2)) * 2; | |
116 | if (thunk[0] == 0xc6 && thunk[1] == 0x00) | |
117 | /* exrl %r0,<target-br> */ | |
118 | br = thunk + (*(int *)(thunk + 2)) * 2; | |
119 | else if (thunk[0] == 0xc0 && (thunk[1] & 0x0f) == 0x00 && | |
120 | thunk[6] == 0x44 && thunk[7] == 0x00 && | |
121 | (thunk[8] & 0x0f) == 0x00 && thunk[9] == 0x00 && | |
122 | (thunk[1] & 0xf0) == (thunk[8] & 0xf0)) | |
123 | /* larl %rx,<target br> + ex %r0,0(%rx) */ | |
124 | br = thunk + (*(int *)(thunk + 2)) * 2; | |
125 | else | |
126 | continue; | |
6deaa3bb MS |
127 | /* Check for unconditional branch 0x07f? or 0x47f???? */ |
128 | if ((br[0] & 0xbf) != 0x07 || (br[1] & 0xf0) != 0xf0) | |
f19fbd5e | 129 | continue; |
6deaa3bb MS |
130 | |
131 | memcpy(insnbuf + 2, (char[]) { 0x47, 0x00, 0x07, 0x00 }, 4); | |
f19fbd5e MS |
132 | switch (type) { |
133 | case BRCL_EXPOLINE: | |
f19fbd5e MS |
134 | insnbuf[0] = br[0]; |
135 | insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f); | |
6deaa3bb MS |
136 | if (br[0] == 0x47) { |
137 | /* brcl to b, replace with bc + nopr */ | |
138 | insnbuf[2] = br[2]; | |
139 | insnbuf[3] = br[3]; | |
140 | } else { | |
141 | /* brcl to br, replace with bcr + nop */ | |
142 | } | |
f19fbd5e MS |
143 | break; |
144 | case BRASL_EXPOLINE: | |
f19fbd5e | 145 | insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f); |
6deaa3bb MS |
146 | if (br[0] == 0x47) { |
147 | /* brasl to b, replace with bas + nopr */ | |
148 | insnbuf[0] = 0x4d; | |
149 | insnbuf[2] = br[2]; | |
150 | insnbuf[3] = br[3]; | |
151 | } else { | |
152 | /* brasl to br, replace with basr + nop */ | |
153 | insnbuf[0] = 0x0d; | |
154 | } | |
f19fbd5e MS |
155 | break; |
156 | } | |
157 | ||
158 | s390_kernel_write(instr, insnbuf, 6); | |
159 | } | |
160 | } | |
161 | ||
6e179d64 | 162 | void __init_or_module nospec_revert(s32 *start, s32 *end) |
f19fbd5e | 163 | { |
6e179d64 | 164 | if (nospec_disable) |
f19fbd5e MS |
165 | __nospec_revert(start, end); |
166 | } | |
167 | ||
168 | extern s32 __nospec_call_start[], __nospec_call_end[]; | |
169 | extern s32 __nospec_return_start[], __nospec_return_end[]; | |
170 | void __init nospec_init_branches(void) | |
171 | { | |
6e179d64 MS |
172 | nospec_revert(__nospec_call_start, __nospec_call_end); |
173 | nospec_revert(__nospec_return_start, __nospec_return_end); | |
f19fbd5e | 174 | } |
b2e2f43a MS |
175 | |
176 | #endif /* CONFIG_EXPOLINE */ |