Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
e2c0cdfb PD |
2 | /* |
3 | * Copied from arch/arm64/kernel/cpufeature.c | |
4 | * | |
5 | * Copyright (C) 2015 ARM Ltd. | |
6 | * Copyright (C) 2017 SiFive | |
e2c0cdfb PD |
7 | */ |
8 | ||
6bcff515 | 9 | #include <linux/bitmap.h> |
2a31c54b | 10 | #include <linux/ctype.h> |
9daaca4a | 11 | #include <linux/log2.h> |
9493e6f3 | 12 | #include <linux/memory.h> |
ff689fd2 | 13 | #include <linux/module.h> |
e2c0cdfb | 14 | #include <linux/of.h> |
ff689fd2 | 15 | #include <asm/alternative.h> |
1631ba12 | 16 | #include <asm/cacheflush.h> |
c2d3c844 | 17 | #include <asm/cpufeature.h> |
e2c0cdfb | 18 | #include <asm/hwcap.h> |
ff689fd2 | 19 | #include <asm/patch.h> |
ff689fd2 | 20 | #include <asm/processor.h> |
e2c0cdfb | 21 | |
58004f26 TO |
22 | #define NUM_ALPHA_EXTS ('z' - 'a' + 1) |
23 | ||
e2c0cdfb | 24 | unsigned long elf_hwcap __read_mostly; |
6bcff515 AP |
25 | |
26 | /* Host ISA bitmap */ | |
27 | static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; | |
28 | ||
62a31d6e EG |
29 | /* Performance information */ |
30 | DEFINE_PER_CPU(long, misaligned_access_speed); | |
31 | ||
6bcff515 AP |
32 | /** |
33 | * riscv_isa_extension_base() - Get base extension word | |
34 | * | |
35 | * @isa_bitmap: ISA bitmap to use | |
36 | * Return: base extension word as unsigned long value | |
37 | * | |
38 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
39 | */ | |
40 | unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) | |
41 | { | |
42 | if (!isa_bitmap) | |
43 | return riscv_isa[0]; | |
44 | return isa_bitmap[0]; | |
45 | } | |
46 | EXPORT_SYMBOL_GPL(riscv_isa_extension_base); | |
47 | ||
48 | /** | |
49 | * __riscv_isa_extension_available() - Check whether given extension | |
50 | * is available or not | |
51 | * | |
52 | * @isa_bitmap: ISA bitmap to use | |
53 | * @bit: bit position of the desired extension | |
54 | * Return: true or false | |
55 | * | |
56 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
57 | */ | |
58 | bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) | |
59 | { | |
60 | const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; | |
61 | ||
62 | if (bit >= RISCV_ISA_EXT_MAX) | |
63 | return false; | |
64 | ||
65 | return test_bit(bit, bmap) ? true : false; | |
66 | } | |
67 | EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); | |
68 | ||
fb0ff0a9 AJ |
69 | static bool riscv_isa_extension_check(int id) |
70 | { | |
9daaca4a AJ |
71 | switch (id) { |
72 | case RISCV_ISA_EXT_ZICBOM: | |
73 | if (!riscv_cbom_block_size) { | |
74 | pr_err("Zicbom detected in ISA string, but no cbom-block-size found\n"); | |
75 | return false; | |
76 | } else if (!is_power_of_2(riscv_cbom_block_size)) { | |
77 | pr_err("cbom-block-size present, but is not a power-of-2\n"); | |
78 | return false; | |
79 | } | |
80 | return true; | |
7ea5a736 AJ |
81 | case RISCV_ISA_EXT_ZICBOZ: |
82 | if (!riscv_cboz_block_size) { | |
83 | pr_err("Zicboz detected in ISA string, but no cboz-block-size found\n"); | |
84 | return false; | |
85 | } else if (!is_power_of_2(riscv_cboz_block_size)) { | |
86 | pr_err("cboz-block-size present, but is not a power-of-2\n"); | |
87 | return false; | |
88 | } | |
89 | return true; | |
9daaca4a AJ |
90 | } |
91 | ||
fb0ff0a9 AJ |
92 | return true; |
93 | } | |
94 | ||
3df952ae | 95 | void __init riscv_fill_hwcap(void) |
e2c0cdfb | 96 | { |
dd81c8ab | 97 | struct device_node *node; |
e2c0cdfb | 98 | const char *isa; |
58004f26 | 99 | char print_str[NUM_ALPHA_EXTS + 1]; |
ad635e72 | 100 | int i, j, rc; |
72685554 | 101 | unsigned long isa2hwcap[26] = {0}; |
ad635e72 | 102 | unsigned long hartid; |
e2c0cdfb | 103 | |
72685554 AJ |
104 | isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; |
105 | isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; | |
106 | isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; | |
107 | isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; | |
108 | isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; | |
109 | isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; | |
e2c0cdfb PD |
110 | |
111 | elf_hwcap = 0; | |
112 | ||
6bcff515 AP |
113 | bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); |
114 | ||
dd81c8ab | 115 | for_each_of_cpu_node(node) { |
fbdc6193 | 116 | unsigned long this_hwcap = 0; |
02d52fbd | 117 | DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX); |
3f96db12 | 118 | const char *temp; |
e2c0cdfb | 119 | |
ad635e72 S |
120 | rc = riscv_of_processor_hartid(node, &hartid); |
121 | if (rc < 0) | |
fbdc6193 | 122 | continue; |
e2c0cdfb | 123 | |
fbdc6193 AP |
124 | if (of_property_read_string(node, "riscv,isa", &isa)) { |
125 | pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); | |
126 | continue; | |
127 | } | |
128 | ||
3f96db12 | 129 | temp = isa; |
6bcff515 AP |
130 | #if IS_ENABLED(CONFIG_32BIT) |
131 | if (!strncmp(isa, "rv32", 4)) | |
2a31c54b | 132 | isa += 4; |
6bcff515 AP |
133 | #elif IS_ENABLED(CONFIG_64BIT) |
134 | if (!strncmp(isa, "rv64", 4)) | |
2a31c54b | 135 | isa += 4; |
6bcff515 | 136 | #endif |
3f96db12 AP |
137 | /* The riscv,isa DT property must start with rv64 or rv32 */ |
138 | if (temp == isa) | |
139 | continue; | |
02d52fbd | 140 | bitmap_zero(this_isa, RISCV_ISA_EXT_MAX); |
2a31c54b TO |
141 | for (; *isa; ++isa) { |
142 | const char *ext = isa++; | |
143 | const char *ext_end = isa; | |
144 | bool ext_long = false, ext_err = false; | |
145 | ||
146 | switch (*ext) { | |
147 | case 's': | |
148 | /** | |
149 | * Workaround for invalid single-letter 's' & 'u'(QEMU). | |
150 | * No need to set the bit in riscv_isa as 's' & 'u' are | |
151 | * not valid ISA extensions. It works until multi-letter | |
152 | * extension starting with "Su" appears. | |
153 | */ | |
154 | if (ext[-1] != '_' && ext[1] == 'u') { | |
155 | ++isa; | |
156 | ext_err = true; | |
157 | break; | |
158 | } | |
159 | fallthrough; | |
160 | case 'x': | |
161 | case 'z': | |
162 | ext_long = true; | |
163 | /* Multi-letter extension must be delimited */ | |
164 | for (; *isa && *isa != '_'; ++isa) | |
40a4d0df TO |
165 | if (unlikely(!islower(*isa) |
166 | && !isdigit(*isa))) | |
2a31c54b | 167 | ext_err = true; |
40a4d0df TO |
168 | /* Parse backwards */ |
169 | ext_end = isa; | |
170 | if (unlikely(ext_err)) | |
171 | break; | |
172 | if (!isdigit(ext_end[-1])) | |
173 | break; | |
174 | /* Skip the minor version */ | |
175 | while (isdigit(*--ext_end)) | |
176 | ; | |
177 | if (ext_end[0] != 'p' | |
178 | || !isdigit(ext_end[-1])) { | |
179 | /* Advance it to offset the pre-decrement */ | |
180 | ++ext_end; | |
181 | break; | |
182 | } | |
183 | /* Skip the major version */ | |
184 | while (isdigit(*--ext_end)) | |
185 | ; | |
186 | ++ext_end; | |
2a31c54b TO |
187 | break; |
188 | default: | |
189 | if (unlikely(!islower(*ext))) { | |
190 | ext_err = true; | |
191 | break; | |
192 | } | |
193 | /* Find next extension */ | |
194 | if (!isdigit(*isa)) | |
195 | break; | |
196 | /* Skip the minor version */ | |
197 | while (isdigit(*++isa)) | |
198 | ; | |
199 | if (*isa != 'p') | |
200 | break; | |
201 | if (!isdigit(*++isa)) { | |
202 | --isa; | |
203 | break; | |
204 | } | |
205 | /* Skip the major version */ | |
206 | while (isdigit(*++isa)) | |
207 | ; | |
208 | break; | |
209 | } | |
210 | if (*isa != '_') | |
211 | --isa; | |
40a4d0df | 212 | |
02d52fbd AP |
213 | #define SET_ISA_EXT_MAP(name, bit) \ |
214 | do { \ | |
215 | if ((ext_end - ext == sizeof(name) - 1) && \ | |
fb0ff0a9 AJ |
216 | !memcmp(ext, name, sizeof(name) - 1) && \ |
217 | riscv_isa_extension_check(bit)) \ | |
02d52fbd AP |
218 | set_bit(bit, this_isa); \ |
219 | } while (false) \ | |
220 | ||
40a4d0df | 221 | if (unlikely(ext_err)) |
2a31c54b | 222 | continue; |
40a4d0df | 223 | if (!ext_long) { |
72685554 AJ |
224 | int nr = *ext - 'a'; |
225 | ||
fb0ff0a9 AJ |
226 | if (riscv_isa_extension_check(nr)) { |
227 | this_hwcap |= isa2hwcap[nr]; | |
228 | set_bit(nr, this_isa); | |
229 | } | |
4905ec2f | 230 | } else { |
80c200b3 | 231 | /* sorted alphabetically */ |
8fe6f7e1 AP |
232 | SET_ISA_EXT_MAP("smaia", RISCV_ISA_EXT_SMAIA); |
233 | SET_ISA_EXT_MAP("ssaia", RISCV_ISA_EXT_SSAIA); | |
4905ec2f | 234 | SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF); |
80c200b3 CD |
235 | SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC); |
236 | SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL); | |
23ad288a | 237 | SET_ISA_EXT_MAP("svnapot", RISCV_ISA_EXT_SVNAPOT); |
ff689fd2 | 238 | SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT); |
b6fcdb19 | 239 | SET_ISA_EXT_MAP("zbb", RISCV_ISA_EXT_ZBB); |
1631ba12 | 240 | SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM); |
7ea5a736 | 241 | SET_ISA_EXT_MAP("zicboz", RISCV_ISA_EXT_ZICBOZ); |
8eb060e1 | 242 | SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE); |
40a4d0df | 243 | } |
02d52fbd | 244 | #undef SET_ISA_EXT_MAP |
6bcff515 | 245 | } |
fbdc6193 AP |
246 | |
247 | /* | |
248 | * All "okay" hart should have same isa. Set HWCAP based on | |
249 | * common capabilities of every "okay" hart, in case they don't | |
250 | * have. | |
251 | */ | |
252 | if (elf_hwcap) | |
253 | elf_hwcap &= this_hwcap; | |
254 | else | |
255 | elf_hwcap = this_hwcap; | |
6bcff515 | 256 | |
8f51558e | 257 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) |
02d52fbd | 258 | bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX); |
8f51558e YN |
259 | else |
260 | bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX); | |
fbdc6193 | 261 | } |
e2c0cdfb | 262 | |
86e581e3 PD |
263 | /* We don't support systems with F but without D, so mask those out |
264 | * here. */ | |
265 | if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { | |
7265d103 | 266 | pr_info("This kernel does not support systems with F but not D\n"); |
86e581e3 PD |
267 | elf_hwcap &= ~COMPAT_HWCAP_ISA_F; |
268 | } | |
269 | ||
6bcff515 | 270 | memset(print_str, 0, sizeof(print_str)); |
58004f26 | 271 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
272 | if (riscv_isa[0] & BIT_MASK(i)) |
273 | print_str[j++] = (char)('a' + i); | |
02d52fbd | 274 | pr_info("riscv: base ISA extensions %s\n", print_str); |
6bcff515 AP |
275 | |
276 | memset(print_str, 0, sizeof(print_str)); | |
58004f26 | 277 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
278 | if (elf_hwcap & BIT_MASK(i)) |
279 | print_str[j++] = (char)('a' + i); | |
280 | pr_info("riscv: ELF capabilities %s\n", print_str); | |
e2c0cdfb | 281 | } |
ff689fd2 HS |
282 | |
283 | #ifdef CONFIG_RISCV_ALTERNATIVE | |
d25f2563 AJ |
284 | /* |
285 | * Alternative patch sites consider 48 bits when determining when to patch | |
286 | * the old instruction sequence with the new. These bits are broken into a | |
287 | * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the | |
288 | * patch site is for an erratum, identified by the 32-bit patch ID. When | |
289 | * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures | |
290 | * further break down patch ID into two 16-bit numbers. The lower 16 bits | |
291 | * are the cpufeature ID and the upper 16 bits are used for a value specific | |
292 | * to the cpufeature and patch site. If the upper 16 bits are zero, then it | |
293 | * implies no specific value is specified. cpufeatures that want to control | |
294 | * patching on a per-site basis will provide non-zero values and implement | |
295 | * checks here. The checks return true when patching should be done, and | |
296 | * false otherwise. | |
297 | */ | |
298 | static bool riscv_cpufeature_patch_check(u16 id, u16 value) | |
299 | { | |
300 | if (!value) | |
301 | return true; | |
302 | ||
ab0f7746 AJ |
303 | switch (id) { |
304 | case RISCV_ISA_EXT_ZICBOZ: | |
305 | /* | |
306 | * Zicboz alternative applications provide the maximum | |
307 | * supported block size order, or zero when it doesn't | |
308 | * matter. If the current block size exceeds the maximum, | |
309 | * then the alternative cannot be applied. | |
310 | */ | |
311 | return riscv_cboz_block_size <= (1U << value); | |
312 | } | |
313 | ||
d25f2563 AJ |
314 | return false; |
315 | } | |
316 | ||
ff689fd2 HS |
317 | void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, |
318 | struct alt_entry *end, | |
319 | unsigned int stage) | |
320 | { | |
ff689fd2 | 321 | struct alt_entry *alt; |
8d23e94a | 322 | void *oldptr, *altptr; |
d25f2563 | 323 | u16 id, value; |
ff689fd2 | 324 | |
191b27c7 JZ |
325 | if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) |
326 | return; | |
ff689fd2 HS |
327 | |
328 | for (alt = begin; alt < end; alt++) { | |
329 | if (alt->vendor_id != 0) | |
330 | continue; | |
d25f2563 AJ |
331 | |
332 | id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); | |
333 | ||
334 | if (id >= RISCV_ISA_EXT_MAX) { | |
335 | WARN(1, "This extension id:%d is not in ISA extension list", id); | |
ff689fd2 HS |
336 | continue; |
337 | } | |
338 | ||
d25f2563 AJ |
339 | if (!__riscv_isa_extension_available(NULL, id)) |
340 | continue; | |
341 | ||
342 | value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); | |
343 | if (!riscv_cpufeature_patch_check(id, value)) | |
4bf88607 JZ |
344 | continue; |
345 | ||
8d23e94a JZ |
346 | oldptr = ALT_OLD_PTR(alt); |
347 | altptr = ALT_ALT_PTR(alt); | |
9493e6f3 CD |
348 | |
349 | mutex_lock(&text_mutex); | |
8d23e94a JZ |
350 | patch_text_nosync(oldptr, altptr, alt->alt_len); |
351 | riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); | |
9493e6f3 | 352 | mutex_unlock(&text_mutex); |
ff689fd2 HS |
353 | } |
354 | } | |
355 | #endif |