Commit | Line | Data |
---|---|---|
f19fbd5e MS |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/module.h> | |
d424986f | 3 | #include <linux/device.h> |
f19fbd5e MS |
4 | #include <asm/nospec-branch.h> |
5 | ||
b2e2f43a MS |
6 | static int __init nobp_setup_early(char *str) |
7 | { | |
8 | bool enabled; | |
9 | int rc; | |
10 | ||
11 | rc = kstrtobool(str, &enabled); | |
12 | if (rc) | |
13 | return rc; | |
6e179d64 MS |
14 | if (enabled && test_facility(82)) { |
15 | /* | |
16 | * The user explicitely requested nobp=1, enable it and | |
17 | * disable the expoline support. | |
18 | */ | |
b2e2f43a | 19 | __set_facility(82, S390_lowcore.alt_stfle_fac_list); |
6e179d64 MS |
20 | if (IS_ENABLED(CONFIG_EXPOLINE)) |
21 | nospec_disable = 1; | |
22 | } else { | |
b2e2f43a | 23 | __clear_facility(82, S390_lowcore.alt_stfle_fac_list); |
6e179d64 | 24 | } |
b2e2f43a MS |
25 | return 0; |
26 | } | |
27 | early_param("nobp", nobp_setup_early); | |
28 | ||
29 | static int __init nospec_setup_early(char *str) | |
30 | { | |
31 | __clear_facility(82, S390_lowcore.alt_stfle_fac_list); | |
32 | return 0; | |
33 | } | |
34 | early_param("nospec", nospec_setup_early); | |
35 | ||
bc035599 MS |
36 | static int __init nospec_report(void) |
37 | { | |
38 | if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) | |
39 | pr_info("Spectre V2 mitigation: execute trampolines.\n"); | |
40 | if (__test_facility(82, S390_lowcore.alt_stfle_fac_list)) | |
41 | pr_info("Spectre V2 mitigation: limited branch prediction.\n"); | |
42 | return 0; | |
43 | } | |
44 | arch_initcall(nospec_report); | |
45 | ||
b2e2f43a MS |
46 | #ifdef CONFIG_EXPOLINE |
47 | ||
6e179d64 | 48 | int nospec_disable = IS_ENABLED(CONFIG_EXPOLINE_OFF); |
f19fbd5e MS |
49 | |
50 | static int __init nospectre_v2_setup_early(char *str) | |
51 | { | |
6e179d64 | 52 | nospec_disable = 1; |
f19fbd5e MS |
53 | return 0; |
54 | } | |
55 | early_param("nospectre_v2", nospectre_v2_setup_early); | |
56 | ||
6a3d1e81 | 57 | void __init nospec_auto_detect(void) |
6e179d64 MS |
58 | { |
59 | if (IS_ENABLED(CC_USING_EXPOLINE)) { | |
60 | /* | |
61 | * The kernel has been compiled with expolines. | |
62 | * Keep expolines enabled and disable nobp. | |
63 | */ | |
64 | nospec_disable = 0; | |
65 | __clear_facility(82, S390_lowcore.alt_stfle_fac_list); | |
66 | } | |
67 | /* | |
68 | * If the kernel has not been compiled with expolines the | |
69 | * nobp setting decides what is done, this depends on the | |
70 | * CONFIG_KERNEL_NP option and the nobp/nospec parameters. | |
71 | */ | |
6e179d64 | 72 | } |
6e179d64 | 73 | |
f19fbd5e MS |
74 | static int __init spectre_v2_setup_early(char *str) |
75 | { | |
76 | if (str && !strncmp(str, "on", 2)) { | |
6e179d64 MS |
77 | nospec_disable = 0; |
78 | __clear_facility(82, S390_lowcore.alt_stfle_fac_list); | |
f19fbd5e | 79 | } |
6e179d64 MS |
80 | if (str && !strncmp(str, "off", 3)) |
81 | nospec_disable = 1; | |
82 | if (str && !strncmp(str, "auto", 4)) | |
6a3d1e81 | 83 | nospec_auto_detect(); |
f19fbd5e MS |
84 | return 0; |
85 | } | |
86 | early_param("spectre_v2", spectre_v2_setup_early); | |
87 | ||
88 | static void __init_or_module __nospec_revert(s32 *start, s32 *end) | |
89 | { | |
90 | enum { BRCL_EXPOLINE, BRASL_EXPOLINE } type; | |
91 | u8 *instr, *thunk, *br; | |
92 | u8 insnbuf[6]; | |
93 | s32 *epo; | |
94 | ||
95 | /* Second part of the instruction replace is always a nop */ | |
f19fbd5e MS |
96 | for (epo = start; epo < end; epo++) { |
97 | instr = (u8 *) epo + *epo; | |
98 | if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x04) | |
99 | type = BRCL_EXPOLINE; /* brcl instruction */ | |
100 | else if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x05) | |
101 | type = BRASL_EXPOLINE; /* brasl instruction */ | |
102 | else | |
103 | continue; | |
104 | thunk = instr + (*(int *)(instr + 2)) * 2; | |
105 | if (thunk[0] == 0xc6 && thunk[1] == 0x00) | |
106 | /* exrl %r0,<target-br> */ | |
107 | br = thunk + (*(int *)(thunk + 2)) * 2; | |
108 | else if (thunk[0] == 0xc0 && (thunk[1] & 0x0f) == 0x00 && | |
109 | thunk[6] == 0x44 && thunk[7] == 0x00 && | |
110 | (thunk[8] & 0x0f) == 0x00 && thunk[9] == 0x00 && | |
111 | (thunk[1] & 0xf0) == (thunk[8] & 0xf0)) | |
112 | /* larl %rx,<target br> + ex %r0,0(%rx) */ | |
113 | br = thunk + (*(int *)(thunk + 2)) * 2; | |
114 | else | |
115 | continue; | |
6deaa3bb MS |
116 | /* Check for unconditional branch 0x07f? or 0x47f???? */ |
117 | if ((br[0] & 0xbf) != 0x07 || (br[1] & 0xf0) != 0xf0) | |
f19fbd5e | 118 | continue; |
6deaa3bb MS |
119 | |
120 | memcpy(insnbuf + 2, (char[]) { 0x47, 0x00, 0x07, 0x00 }, 4); | |
f19fbd5e MS |
121 | switch (type) { |
122 | case BRCL_EXPOLINE: | |
f19fbd5e MS |
123 | insnbuf[0] = br[0]; |
124 | insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f); | |
6deaa3bb MS |
125 | if (br[0] == 0x47) { |
126 | /* brcl to b, replace with bc + nopr */ | |
127 | insnbuf[2] = br[2]; | |
128 | insnbuf[3] = br[3]; | |
129 | } else { | |
130 | /* brcl to br, replace with bcr + nop */ | |
131 | } | |
f19fbd5e MS |
132 | break; |
133 | case BRASL_EXPOLINE: | |
f19fbd5e | 134 | insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f); |
6deaa3bb MS |
135 | if (br[0] == 0x47) { |
136 | /* brasl to b, replace with bas + nopr */ | |
137 | insnbuf[0] = 0x4d; | |
138 | insnbuf[2] = br[2]; | |
139 | insnbuf[3] = br[3]; | |
140 | } else { | |
141 | /* brasl to br, replace with basr + nop */ | |
142 | insnbuf[0] = 0x0d; | |
143 | } | |
f19fbd5e MS |
144 | break; |
145 | } | |
146 | ||
147 | s390_kernel_write(instr, insnbuf, 6); | |
148 | } | |
149 | } | |
150 | ||
6e179d64 | 151 | void __init_or_module nospec_revert(s32 *start, s32 *end) |
f19fbd5e | 152 | { |
6e179d64 | 153 | if (nospec_disable) |
f19fbd5e MS |
154 | __nospec_revert(start, end); |
155 | } | |
156 | ||
157 | extern s32 __nospec_call_start[], __nospec_call_end[]; | |
158 | extern s32 __nospec_return_start[], __nospec_return_end[]; | |
159 | void __init nospec_init_branches(void) | |
160 | { | |
6e179d64 MS |
161 | nospec_revert(__nospec_call_start, __nospec_call_end); |
162 | nospec_revert(__nospec_return_start, __nospec_return_end); | |
f19fbd5e | 163 | } |
b2e2f43a MS |
164 | |
165 | #endif /* CONFIG_EXPOLINE */ |