Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
e2c0cdfb PD |
2 | /* |
3 | * Copied from arch/arm64/kernel/cpufeature.c | |
4 | * | |
5 | * Copyright (C) 2015 ARM Ltd. | |
6 | * Copyright (C) 2017 SiFive | |
e2c0cdfb PD |
7 | */ |
8 | ||
6bcff515 | 9 | #include <linux/bitmap.h> |
2a31c54b | 10 | #include <linux/ctype.h> |
ff689fd2 HS |
11 | #include <linux/libfdt.h> |
12 | #include <linux/module.h> | |
e2c0cdfb | 13 | #include <linux/of.h> |
ff689fd2 | 14 | #include <asm/alternative.h> |
1631ba12 | 15 | #include <asm/cacheflush.h> |
ff689fd2 | 16 | #include <asm/errata_list.h> |
e2c0cdfb | 17 | #include <asm/hwcap.h> |
ff689fd2 HS |
18 | #include <asm/patch.h> |
19 | #include <asm/pgtable.h> | |
20 | #include <asm/processor.h> | |
fbdc6193 | 21 | #include <asm/smp.h> |
5ed881bc | 22 | #include <asm/switch_to.h> |
e2c0cdfb | 23 | |
58004f26 TO |
24 | #define NUM_ALPHA_EXTS ('z' - 'a' + 1) |
25 | ||
e2c0cdfb | 26 | unsigned long elf_hwcap __read_mostly; |
6bcff515 AP |
27 | |
28 | /* Host ISA bitmap */ | |
29 | static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; | |
30 | ||
eb6354e1 | 31 | DEFINE_STATIC_KEY_ARRAY_FALSE(riscv_isa_ext_keys, RISCV_ISA_EXT_KEY_MAX); |
c360cbec | 32 | EXPORT_SYMBOL(riscv_isa_ext_keys); |
e2c0cdfb | 33 | |
6bcff515 AP |
34 | /** |
35 | * riscv_isa_extension_base() - Get base extension word | |
36 | * | |
37 | * @isa_bitmap: ISA bitmap to use | |
38 | * Return: base extension word as unsigned long value | |
39 | * | |
40 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
41 | */ | |
42 | unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) | |
43 | { | |
44 | if (!isa_bitmap) | |
45 | return riscv_isa[0]; | |
46 | return isa_bitmap[0]; | |
47 | } | |
48 | EXPORT_SYMBOL_GPL(riscv_isa_extension_base); | |
49 | ||
50 | /** | |
51 | * __riscv_isa_extension_available() - Check whether given extension | |
52 | * is available or not | |
53 | * | |
54 | * @isa_bitmap: ISA bitmap to use | |
55 | * @bit: bit position of the desired extension | |
56 | * Return: true or false | |
57 | * | |
58 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
59 | */ | |
60 | bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) | |
61 | { | |
62 | const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; | |
63 | ||
64 | if (bit >= RISCV_ISA_EXT_MAX) | |
65 | return false; | |
66 | ||
67 | return test_bit(bit, bmap) ? true : false; | |
68 | } | |
69 | EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); | |
70 | ||
3df952ae | 71 | void __init riscv_fill_hwcap(void) |
e2c0cdfb | 72 | { |
dd81c8ab | 73 | struct device_node *node; |
e2c0cdfb | 74 | const char *isa; |
58004f26 | 75 | char print_str[NUM_ALPHA_EXTS + 1]; |
ad635e72 | 76 | int i, j, rc; |
e2c0cdfb | 77 | static unsigned long isa2hwcap[256] = {0}; |
ad635e72 | 78 | unsigned long hartid; |
e2c0cdfb PD |
79 | |
80 | isa2hwcap['i'] = isa2hwcap['I'] = COMPAT_HWCAP_ISA_I; | |
81 | isa2hwcap['m'] = isa2hwcap['M'] = COMPAT_HWCAP_ISA_M; | |
82 | isa2hwcap['a'] = isa2hwcap['A'] = COMPAT_HWCAP_ISA_A; | |
83 | isa2hwcap['f'] = isa2hwcap['F'] = COMPAT_HWCAP_ISA_F; | |
84 | isa2hwcap['d'] = isa2hwcap['D'] = COMPAT_HWCAP_ISA_D; | |
85 | isa2hwcap['c'] = isa2hwcap['C'] = COMPAT_HWCAP_ISA_C; | |
86 | ||
87 | elf_hwcap = 0; | |
88 | ||
6bcff515 AP |
89 | bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); |
90 | ||
dd81c8ab | 91 | for_each_of_cpu_node(node) { |
fbdc6193 | 92 | unsigned long this_hwcap = 0; |
02d52fbd | 93 | DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX); |
3f96db12 | 94 | const char *temp; |
e2c0cdfb | 95 | |
ad635e72 S |
96 | rc = riscv_of_processor_hartid(node, &hartid); |
97 | if (rc < 0) | |
fbdc6193 | 98 | continue; |
e2c0cdfb | 99 | |
fbdc6193 AP |
100 | if (of_property_read_string(node, "riscv,isa", &isa)) { |
101 | pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); | |
102 | continue; | |
103 | } | |
104 | ||
3f96db12 | 105 | temp = isa; |
6bcff515 AP |
106 | #if IS_ENABLED(CONFIG_32BIT) |
107 | if (!strncmp(isa, "rv32", 4)) | |
2a31c54b | 108 | isa += 4; |
6bcff515 AP |
109 | #elif IS_ENABLED(CONFIG_64BIT) |
110 | if (!strncmp(isa, "rv64", 4)) | |
2a31c54b | 111 | isa += 4; |
6bcff515 | 112 | #endif |
3f96db12 AP |
113 | /* The riscv,isa DT property must start with rv64 or rv32 */ |
114 | if (temp == isa) | |
115 | continue; | |
02d52fbd | 116 | bitmap_zero(this_isa, RISCV_ISA_EXT_MAX); |
2a31c54b TO |
117 | for (; *isa; ++isa) { |
118 | const char *ext = isa++; | |
119 | const char *ext_end = isa; | |
120 | bool ext_long = false, ext_err = false; | |
121 | ||
122 | switch (*ext) { | |
123 | case 's': | |
124 | /** | |
125 | * Workaround for invalid single-letter 's' & 'u'(QEMU). | |
126 | * No need to set the bit in riscv_isa as 's' & 'u' are | |
127 | * not valid ISA extensions. It works until multi-letter | |
128 | * extension starting with "Su" appears. | |
129 | */ | |
130 | if (ext[-1] != '_' && ext[1] == 'u') { | |
131 | ++isa; | |
132 | ext_err = true; | |
133 | break; | |
134 | } | |
135 | fallthrough; | |
136 | case 'x': | |
137 | case 'z': | |
138 | ext_long = true; | |
139 | /* Multi-letter extension must be delimited */ | |
140 | for (; *isa && *isa != '_'; ++isa) | |
40a4d0df TO |
141 | if (unlikely(!islower(*isa) |
142 | && !isdigit(*isa))) | |
2a31c54b | 143 | ext_err = true; |
40a4d0df TO |
144 | /* Parse backwards */ |
145 | ext_end = isa; | |
146 | if (unlikely(ext_err)) | |
147 | break; | |
148 | if (!isdigit(ext_end[-1])) | |
149 | break; | |
150 | /* Skip the minor version */ | |
151 | while (isdigit(*--ext_end)) | |
152 | ; | |
153 | if (ext_end[0] != 'p' | |
154 | || !isdigit(ext_end[-1])) { | |
155 | /* Advance it to offset the pre-decrement */ | |
156 | ++ext_end; | |
157 | break; | |
158 | } | |
159 | /* Skip the major version */ | |
160 | while (isdigit(*--ext_end)) | |
161 | ; | |
162 | ++ext_end; | |
2a31c54b TO |
163 | break; |
164 | default: | |
165 | if (unlikely(!islower(*ext))) { | |
166 | ext_err = true; | |
167 | break; | |
168 | } | |
169 | /* Find next extension */ | |
170 | if (!isdigit(*isa)) | |
171 | break; | |
172 | /* Skip the minor version */ | |
173 | while (isdigit(*++isa)) | |
174 | ; | |
175 | if (*isa != 'p') | |
176 | break; | |
177 | if (!isdigit(*++isa)) { | |
178 | --isa; | |
179 | break; | |
180 | } | |
181 | /* Skip the major version */ | |
182 | while (isdigit(*++isa)) | |
183 | ; | |
184 | break; | |
185 | } | |
186 | if (*isa != '_') | |
187 | --isa; | |
40a4d0df | 188 | |
02d52fbd AP |
189 | #define SET_ISA_EXT_MAP(name, bit) \ |
190 | do { \ | |
191 | if ((ext_end - ext == sizeof(name) - 1) && \ | |
192 | !memcmp(ext, name, sizeof(name) - 1)) \ | |
193 | set_bit(bit, this_isa); \ | |
194 | } while (false) \ | |
195 | ||
40a4d0df | 196 | if (unlikely(ext_err)) |
2a31c54b | 197 | continue; |
40a4d0df TO |
198 | if (!ext_long) { |
199 | this_hwcap |= isa2hwcap[(unsigned char)(*ext)]; | |
02d52fbd | 200 | set_bit(*ext - 'a', this_isa); |
4905ec2f AP |
201 | } else { |
202 | SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF); | |
ff689fd2 | 203 | SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); |
1631ba12 | 204 | SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM); |
8eb060e1 | 205 | SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE); |
464b0187 | 206 | SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC); |
122979aa | 207 | SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL); |
40a4d0df | 208 | } |
02d52fbd | 209 | #undef SET_ISA_EXT_MAP |
6bcff515 | 210 | } |
fbdc6193 AP |
211 | |
212 | /* | |
213 | * All "okay" hart should have same isa. Set HWCAP based on | |
214 | * common capabilities of every "okay" hart, in case they don't | |
215 | * have. | |
216 | */ | |
217 | if (elf_hwcap) | |
218 | elf_hwcap &= this_hwcap; | |
219 | else | |
220 | elf_hwcap = this_hwcap; | |
6bcff515 | 221 | |
8f51558e | 222 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) |
02d52fbd | 223 | bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX); |
8f51558e YN |
224 | else |
225 | bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX); | |
fbdc6193 | 226 | } |
e2c0cdfb | 227 | |
86e581e3 PD |
228 | /* We don't support systems with F but without D, so mask those out |
229 | * here. */ | |
230 | if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { | |
7265d103 | 231 | pr_info("This kernel does not support systems with F but not D\n"); |
86e581e3 PD |
232 | elf_hwcap &= ~COMPAT_HWCAP_ISA_F; |
233 | } | |
234 | ||
6bcff515 | 235 | memset(print_str, 0, sizeof(print_str)); |
58004f26 | 236 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
237 | if (riscv_isa[0] & BIT_MASK(i)) |
238 | print_str[j++] = (char)('a' + i); | |
02d52fbd | 239 | pr_info("riscv: base ISA extensions %s\n", print_str); |
6bcff515 AP |
240 | |
241 | memset(print_str, 0, sizeof(print_str)); | |
58004f26 | 242 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
243 | if (elf_hwcap & BIT_MASK(i)) |
244 | print_str[j++] = (char)('a' + i); | |
245 | pr_info("riscv: ELF capabilities %s\n", print_str); | |
9411ec60 | 246 | |
c360cbec JZ |
247 | for_each_set_bit(i, riscv_isa, RISCV_ISA_EXT_MAX) { |
248 | j = riscv_isa_ext2key(i); | |
249 | if (j >= 0) | |
250 | static_branch_enable(&riscv_isa_ext_keys[j]); | |
251 | } | |
e2c0cdfb | 252 | } |
ff689fd2 HS |
253 | |
254 | #ifdef CONFIG_RISCV_ALTERNATIVE | |
1771c8c9 | 255 | static bool __init_or_module cpufeature_probe_svpbmt(unsigned int stage) |
ff689fd2 | 256 | { |
e47bddcb | 257 | if (!IS_ENABLED(CONFIG_RISCV_ISA_SVPBMT)) |
a35707c3 | 258 | return false; |
ff689fd2 | 259 | |
e47bddcb | 260 | if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) |
a35707c3 | 261 | return false; |
ff689fd2 | 262 | |
e47bddcb | 263 | return riscv_isa_extension_available(NULL, SVPBMT); |
ff689fd2 HS |
264 | } |
265 | ||
1631ba12 HS |
266 | static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage) |
267 | { | |
f055268e HS |
268 | if (!IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM)) |
269 | return false; | |
270 | ||
271 | if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) | |
272 | return false; | |
273 | ||
274 | if (!riscv_isa_extension_available(NULL, ZICBOM)) | |
1631ba12 | 275 | return false; |
1631ba12 | 276 | |
f055268e HS |
277 | riscv_noncoherent_supported(); |
278 | return true; | |
1631ba12 HS |
279 | } |
280 | ||
1771c8c9 HS |
281 | /* |
282 | * Probe presence of individual extensions. | |
283 | * | |
284 | * This code may also be executed before kernel relocation, so we cannot use | |
285 | * addresses generated by the address-of operator as they won't be valid in | |
286 | * this context. | |
287 | */ | |
ff689fd2 HS |
288 | static u32 __init_or_module cpufeature_probe(unsigned int stage) |
289 | { | |
ff689fd2 | 290 | u32 cpu_req_feature = 0; |
ff689fd2 | 291 | |
1771c8c9 | 292 | if (cpufeature_probe_svpbmt(stage)) |
e283187c | 293 | cpu_req_feature |= BIT(CPUFEATURE_SVPBMT); |
ff689fd2 | 294 | |
1631ba12 | 295 | if (cpufeature_probe_zicbom(stage)) |
e283187c | 296 | cpu_req_feature |= BIT(CPUFEATURE_ZICBOM); |
1631ba12 | 297 | |
ff689fd2 HS |
298 | return cpu_req_feature; |
299 | } | |
300 | ||
301 | void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, | |
302 | struct alt_entry *end, | |
303 | unsigned int stage) | |
304 | { | |
305 | u32 cpu_req_feature = cpufeature_probe(stage); | |
ff689fd2 HS |
306 | struct alt_entry *alt; |
307 | u32 tmp; | |
308 | ||
309 | for (alt = begin; alt < end; alt++) { | |
310 | if (alt->vendor_id != 0) | |
311 | continue; | |
312 | if (alt->errata_id >= CPUFEATURE_NUMBER) { | |
313 | WARN(1, "This feature id:%d is not in kernel cpufeature list", | |
314 | alt->errata_id); | |
315 | continue; | |
316 | } | |
317 | ||
318 | tmp = (1U << alt->errata_id); | |
237c0ee4 | 319 | if (cpu_req_feature & tmp) |
ff689fd2 | 320 | patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len); |
ff689fd2 HS |
321 | } |
322 | } | |
323 | #endif |