Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
e2c0cdfb PD |
2 | /* |
3 | * Copied from arch/arm64/kernel/cpufeature.c | |
4 | * | |
5 | * Copyright (C) 2015 ARM Ltd. | |
6 | * Copyright (C) 2017 SiFive | |
e2c0cdfb PD |
7 | */ |
8 | ||
396c0183 | 9 | #include <linux/acpi.h> |
6bcff515 | 10 | #include <linux/bitmap.h> |
2ce5729f | 11 | #include <linux/cpu.h> |
55e0bf49 | 12 | #include <linux/cpuhotplug.h> |
2a31c54b | 13 | #include <linux/ctype.h> |
9daaca4a | 14 | #include <linux/log2.h> |
9493e6f3 | 15 | #include <linux/memory.h> |
ff689fd2 | 16 | #include <linux/module.h> |
e2c0cdfb | 17 | #include <linux/of.h> |
396c0183 | 18 | #include <asm/acpi.h> |
ff689fd2 | 19 | #include <asm/alternative.h> |
1631ba12 | 20 | #include <asm/cacheflush.h> |
c2d3c844 | 21 | #include <asm/cpufeature.h> |
e2c0cdfb | 22 | #include <asm/hwcap.h> |
ff689fd2 | 23 | #include <asm/patch.h> |
ff689fd2 | 24 | #include <asm/processor.h> |
7017858e | 25 | #include <asm/vector.h> |
e2c0cdfb | 26 | |
58004f26 TO |
27 | #define NUM_ALPHA_EXTS ('z' - 'a' + 1) |
28 | ||
e2c0cdfb | 29 | unsigned long elf_hwcap __read_mostly; |
6bcff515 AP |
30 | |
31 | /* Host ISA bitmap */ | |
32 | static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; | |
33 | ||
82e9c66e EG |
34 | /* Per-cpu ISA extensions. */ |
35 | struct riscv_isainfo hart_isa[NR_CPUS]; | |
36 | ||
6bcff515 AP |
37 | /** |
38 | * riscv_isa_extension_base() - Get base extension word | |
39 | * | |
40 | * @isa_bitmap: ISA bitmap to use | |
41 | * Return: base extension word as unsigned long value | |
42 | * | |
43 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
44 | */ | |
45 | unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) | |
46 | { | |
47 | if (!isa_bitmap) | |
48 | return riscv_isa[0]; | |
49 | return isa_bitmap[0]; | |
50 | } | |
51 | EXPORT_SYMBOL_GPL(riscv_isa_extension_base); | |
52 | ||
53 | /** | |
54 | * __riscv_isa_extension_available() - Check whether given extension | |
55 | * is available or not | |
56 | * | |
57 | * @isa_bitmap: ISA bitmap to use | |
58 | * @bit: bit position of the desired extension | |
59 | * Return: true or false | |
60 | * | |
61 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
62 | */ | |
0d8295ed | 63 | bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit) |
6bcff515 AP |
64 | { |
65 | const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; | |
66 | ||
67 | if (bit >= RISCV_ISA_EXT_MAX) | |
68 | return false; | |
69 | ||
70 | return test_bit(bit, bmap) ? true : false; | |
71 | } | |
72 | EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); | |
73 | ||
fb0ff0a9 AJ |
74 | static bool riscv_isa_extension_check(int id) |
75 | { | |
9daaca4a AJ |
76 | switch (id) { |
77 | case RISCV_ISA_EXT_ZICBOM: | |
78 | if (!riscv_cbom_block_size) { | |
c818fea8 | 79 | pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); |
9daaca4a AJ |
80 | return false; |
81 | } else if (!is_power_of_2(riscv_cbom_block_size)) { | |
c818fea8 | 82 | pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); |
9daaca4a AJ |
83 | return false; |
84 | } | |
85 | return true; | |
7ea5a736 AJ |
86 | case RISCV_ISA_EXT_ZICBOZ: |
87 | if (!riscv_cboz_block_size) { | |
181f2a28 | 88 | pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n"); |
7ea5a736 AJ |
89 | return false; |
90 | } else if (!is_power_of_2(riscv_cboz_block_size)) { | |
181f2a28 | 91 | pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n"); |
7ea5a736 AJ |
92 | return false; |
93 | } | |
94 | return true; | |
0d8295ed EG |
95 | case RISCV_ISA_EXT_INVALID: |
96 | return false; | |
9daaca4a AJ |
97 | } |
98 | ||
fb0ff0a9 AJ |
99 | return true; |
100 | } | |
101 | ||
0d8295ed EG |
102 | #define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size) { \ |
103 | .name = #_name, \ | |
104 | .property = #_name, \ | |
105 | .id = _id, \ | |
106 | .subset_ext_ids = _subset_exts, \ | |
107 | .subset_ext_size = _subset_exts_size \ | |
37f988dc | 108 | } |
8135ade3 | 109 | |
0d8295ed EG |
110 | #define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0) |
111 | ||
112 | /* Used to declare pure "lasso" extension (Zk for instance) */ | |
113 | #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \ | |
114 | _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, ARRAY_SIZE(_bundled_exts)) | |
115 | ||
aec33539 CL |
116 | /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */ |
117 | #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \ | |
118 | _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts)) | |
119 | ||
0d8295ed EG |
120 | static const unsigned int riscv_zk_bundled_exts[] = { |
121 | RISCV_ISA_EXT_ZBKB, | |
122 | RISCV_ISA_EXT_ZBKC, | |
123 | RISCV_ISA_EXT_ZBKX, | |
124 | RISCV_ISA_EXT_ZKND, | |
125 | RISCV_ISA_EXT_ZKNE, | |
126 | RISCV_ISA_EXT_ZKR, | |
127 | RISCV_ISA_EXT_ZKT, | |
128 | }; | |
129 | ||
130 | static const unsigned int riscv_zkn_bundled_exts[] = { | |
131 | RISCV_ISA_EXT_ZBKB, | |
132 | RISCV_ISA_EXT_ZBKC, | |
133 | RISCV_ISA_EXT_ZBKX, | |
134 | RISCV_ISA_EXT_ZKND, | |
135 | RISCV_ISA_EXT_ZKNE, | |
136 | RISCV_ISA_EXT_ZKNH, | |
137 | }; | |
138 | ||
139 | static const unsigned int riscv_zks_bundled_exts[] = { | |
140 | RISCV_ISA_EXT_ZBKB, | |
141 | RISCV_ISA_EXT_ZBKC, | |
142 | RISCV_ISA_EXT_ZKSED, | |
143 | RISCV_ISA_EXT_ZKSH | |
144 | }; | |
145 | ||
aec33539 CL |
146 | #define RISCV_ISA_EXT_ZVKN \ |
147 | RISCV_ISA_EXT_ZVKNED, \ | |
148 | RISCV_ISA_EXT_ZVKNHB, \ | |
149 | RISCV_ISA_EXT_ZVKB, \ | |
150 | RISCV_ISA_EXT_ZVKT | |
151 | ||
152 | static const unsigned int riscv_zvkn_bundled_exts[] = { | |
153 | RISCV_ISA_EXT_ZVKN | |
154 | }; | |
155 | ||
156 | static const unsigned int riscv_zvknc_bundled_exts[] = { | |
157 | RISCV_ISA_EXT_ZVKN, | |
158 | RISCV_ISA_EXT_ZVBC | |
159 | }; | |
160 | ||
161 | static const unsigned int riscv_zvkng_bundled_exts[] = { | |
162 | RISCV_ISA_EXT_ZVKN, | |
163 | RISCV_ISA_EXT_ZVKG | |
164 | }; | |
165 | ||
166 | #define RISCV_ISA_EXT_ZVKS \ | |
167 | RISCV_ISA_EXT_ZVKSED, \ | |
168 | RISCV_ISA_EXT_ZVKSH, \ | |
169 | RISCV_ISA_EXT_ZVKB, \ | |
170 | RISCV_ISA_EXT_ZVKT | |
171 | ||
172 | static const unsigned int riscv_zvks_bundled_exts[] = { | |
173 | RISCV_ISA_EXT_ZVKS | |
174 | }; | |
175 | ||
176 | static const unsigned int riscv_zvksc_bundled_exts[] = { | |
177 | RISCV_ISA_EXT_ZVKS, | |
178 | RISCV_ISA_EXT_ZVBC | |
179 | }; | |
180 | ||
181 | static const unsigned int riscv_zvksg_bundled_exts[] = { | |
182 | RISCV_ISA_EXT_ZVKS, | |
183 | RISCV_ISA_EXT_ZVKG | |
184 | }; | |
185 | ||
186 | static const unsigned int riscv_zvbb_exts[] = { | |
187 | RISCV_ISA_EXT_ZVKB | |
188 | }; | |
189 | ||
8135ade3 CD |
190 | /* |
191 | * The canonical order of ISA extension names in the ISA string is defined in | |
192 | * chapter 27 of the unprivileged specification. | |
193 | * | |
194 | * Ordinarily, for in-kernel data structures, this order is unimportant but | |
195 | * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. | |
196 | * | |
197 | * The specification uses vague wording, such as should, when it comes to | |
198 | * ordering, so for our purposes the following rules apply: | |
199 | * | |
200 | * 1. All multi-letter extensions must be separated from other extensions by an | |
201 | * underscore. | |
202 | * | |
203 | * 2. Additional standard extensions (starting with 'Z') must be sorted after | |
204 | * single-letter extensions and before any higher-privileged extensions. | |
205 | * | |
206 | * 3. The first letter following the 'Z' conventionally indicates the most | |
207 | * closely related alphabetical extension category, IMAFDQLCBKJTPVH. | |
208 | * If multiple 'Z' extensions are named, they must be ordered first by | |
209 | * category, then alphabetically within a category. | |
210 | * | |
211 | * 3. Standard supervisor-level extensions (starting with 'S') must be listed | |
212 | * after standard unprivileged extensions. If multiple supervisor-level | |
213 | * extensions are listed, they must be ordered alphabetically. | |
214 | * | |
215 | * 4. Standard machine-level extensions (starting with 'Zxm') must be listed | |
216 | * after any lower-privileged, standard extensions. If multiple | |
217 | * machine-level extensions are listed, they must be ordered | |
218 | * alphabetically. | |
219 | * | |
220 | * 5. Non-standard extensions (starting with 'X') must be listed after all | |
221 | * standard extensions. If multiple non-standard extensions are listed, they | |
222 | * must be ordered alphabetically. | |
223 | * | |
224 | * An example string following the order is: | |
225 | * rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux | |
226 | * | |
227 | * New entries to this struct should follow the ordering rules described above. | |
228 | */ | |
229 | const struct riscv_isa_ext_data riscv_isa_ext[] = { | |
effc122a CD |
230 | __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), |
231 | __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), | |
232 | __RISCV_ISA_EXT_DATA(a, RISCV_ISA_EXT_a), | |
233 | __RISCV_ISA_EXT_DATA(f, RISCV_ISA_EXT_f), | |
234 | __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), | |
235 | __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), | |
236 | __RISCV_ISA_EXT_DATA(c, RISCV_ISA_EXT_c), | |
effc122a CD |
237 | __RISCV_ISA_EXT_DATA(v, RISCV_ISA_EXT_v), |
238 | __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), | |
8135ade3 CD |
239 | __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), |
240 | __RISCV_ISA_EXT_DATA(zicboz, RISCV_ISA_EXT_ZICBOZ), | |
241 | __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), | |
662a601a | 242 | __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), |
8135ade3 CD |
243 | __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), |
244 | __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), | |
eddbfa0d | 245 | __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL), |
8135ade3 CD |
246 | __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), |
247 | __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), | |
188a2122 | 248 | __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS), |
fe987e84 | 249 | __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA), |
11e8e1ee CL |
250 | __RISCV_ISA_EXT_DATA(zfh, RISCV_ISA_EXT_ZFH), |
251 | __RISCV_ISA_EXT_DATA(zfhmin, RISCV_ISA_EXT_ZFHMIN), | |
8135ade3 CD |
252 | __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), |
253 | __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), | |
e45f463a | 254 | __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), |
0d8295ed EG |
255 | __RISCV_ISA_EXT_DATA(zbkb, RISCV_ISA_EXT_ZBKB), |
256 | __RISCV_ISA_EXT_DATA(zbkc, RISCV_ISA_EXT_ZBKC), | |
257 | __RISCV_ISA_EXT_DATA(zbkx, RISCV_ISA_EXT_ZBKX), | |
8135ade3 | 258 | __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), |
0d8295ed EG |
259 | __RISCV_ISA_EXT_BUNDLE(zk, riscv_zk_bundled_exts), |
260 | __RISCV_ISA_EXT_BUNDLE(zkn, riscv_zkn_bundled_exts), | |
261 | __RISCV_ISA_EXT_DATA(zknd, RISCV_ISA_EXT_ZKND), | |
262 | __RISCV_ISA_EXT_DATA(zkne, RISCV_ISA_EXT_ZKNE), | |
263 | __RISCV_ISA_EXT_DATA(zknh, RISCV_ISA_EXT_ZKNH), | |
264 | __RISCV_ISA_EXT_DATA(zkr, RISCV_ISA_EXT_ZKR), | |
265 | __RISCV_ISA_EXT_BUNDLE(zks, riscv_zks_bundled_exts), | |
266 | __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT), | |
267 | __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), | |
268 | __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), | |
1ec9f381 | 269 | __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO), |
aec33539 CL |
270 | __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), |
271 | __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), | |
f4961b78 CL |
272 | __RISCV_ISA_EXT_DATA(zvfh, RISCV_ISA_EXT_ZVFH), |
273 | __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN), | |
aec33539 CL |
274 | __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), |
275 | __RISCV_ISA_EXT_DATA(zvkg, RISCV_ISA_EXT_ZVKG), | |
276 | __RISCV_ISA_EXT_BUNDLE(zvkn, riscv_zvkn_bundled_exts), | |
277 | __RISCV_ISA_EXT_BUNDLE(zvknc, riscv_zvknc_bundled_exts), | |
278 | __RISCV_ISA_EXT_DATA(zvkned, RISCV_ISA_EXT_ZVKNED), | |
279 | __RISCV_ISA_EXT_BUNDLE(zvkng, riscv_zvkng_bundled_exts), | |
280 | __RISCV_ISA_EXT_DATA(zvknha, RISCV_ISA_EXT_ZVKNHA), | |
281 | __RISCV_ISA_EXT_DATA(zvknhb, RISCV_ISA_EXT_ZVKNHB), | |
282 | __RISCV_ISA_EXT_BUNDLE(zvks, riscv_zvks_bundled_exts), | |
283 | __RISCV_ISA_EXT_BUNDLE(zvksc, riscv_zvksc_bundled_exts), | |
284 | __RISCV_ISA_EXT_DATA(zvksed, RISCV_ISA_EXT_ZVKSED), | |
285 | __RISCV_ISA_EXT_DATA(zvksh, RISCV_ISA_EXT_ZVKSH), | |
286 | __RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts), | |
287 | __RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT), | |
8135ade3 | 288 | __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), |
9dbaf381 | 289 | __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), |
8135ade3 CD |
290 | __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), |
291 | __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), | |
292 | __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), | |
293 | __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), | |
294 | __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), | |
295 | __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), | |
296 | }; | |
297 | ||
298 | const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); | |
299 | ||
0d8295ed EG |
300 | static void __init match_isa_ext(const struct riscv_isa_ext_data *ext, const char *name, |
301 | const char *name_end, struct riscv_isainfo *isainfo) | |
302 | { | |
303 | if ((name_end - name == strlen(ext->name)) && | |
304 | !strncasecmp(name, ext->name, name_end - name)) { | |
305 | /* | |
306 | * If this is a bundle, enable all the ISA extensions that | |
307 | * comprise the bundle. | |
308 | */ | |
309 | if (ext->subset_ext_size) { | |
310 | for (int i = 0; i < ext->subset_ext_size; i++) { | |
311 | if (riscv_isa_extension_check(ext->subset_ext_ids[i])) | |
312 | set_bit(ext->subset_ext_ids[i], isainfo->isa); | |
313 | } | |
314 | } | |
315 | ||
316 | /* | |
317 | * This is valid even for bundle extensions which uses the RISCV_ISA_EXT_INVALID id | |
318 | * (rejected by riscv_isa_extension_check()). | |
319 | */ | |
320 | if (riscv_isa_extension_check(ext->id)) | |
321 | set_bit(ext->id, isainfo->isa); | |
322 | } | |
323 | } | |
324 | ||
4265b0ec CD |
325 | static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct riscv_isainfo *isainfo, |
326 | unsigned long *isa2hwcap, const char *isa) | |
327 | { | |
328 | /* | |
329 | * For all possible cpus, we have already validated in | |
330 | * the boot process that they at least contain "rv" and | |
331 | * whichever of "32"/"64" this kernel supports, and so this | |
332 | * section can be skipped. | |
333 | */ | |
334 | isa += 4; | |
335 | ||
336 | while (*isa) { | |
337 | const char *ext = isa++; | |
338 | const char *ext_end = isa; | |
339 | bool ext_long = false, ext_err = false; | |
340 | ||
341 | switch (*ext) { | |
342 | case 's': | |
343 | /* | |
8f501be8 | 344 | * Workaround for invalid single-letter 's' & 'u' (QEMU). |
4265b0ec | 345 | * No need to set the bit in riscv_isa as 's' & 'u' are |
8f501be8 TO |
346 | * not valid ISA extensions. It works unless the first |
347 | * multi-letter extension in the ISA string begins with | |
348 | * "Su" and is not prefixed with an underscore. | |
4265b0ec CD |
349 | */ |
350 | if (ext[-1] != '_' && ext[1] == 'u') { | |
351 | ++isa; | |
352 | ext_err = true; | |
353 | break; | |
354 | } | |
355 | fallthrough; | |
356 | case 'S': | |
357 | case 'x': | |
358 | case 'X': | |
359 | case 'z': | |
360 | case 'Z': | |
361 | /* | |
362 | * Before attempting to parse the extension itself, we find its end. | |
363 | * As multi-letter extensions must be split from other multi-letter | |
364 | * extensions with an "_", the end of a multi-letter extension will | |
365 | * either be the null character or the "_" at the start of the next | |
366 | * multi-letter extension. | |
367 | * | |
368 | * Next, as the extensions version is currently ignored, we | |
369 | * eliminate that portion. This is done by parsing backwards from | |
370 | * the end of the extension, removing any numbers. This may be a | |
371 | * major or minor number however, so the process is repeated if a | |
372 | * minor number was found. | |
373 | * | |
374 | * ext_end is intended to represent the first character *after* the | |
375 | * name portion of an extension, but will be decremented to the last | |
376 | * character itself while eliminating the extensions version number. | |
377 | * A simple re-increment solves this problem. | |
378 | */ | |
379 | ext_long = true; | |
380 | for (; *isa && *isa != '_'; ++isa) | |
381 | if (unlikely(!isalnum(*isa))) | |
382 | ext_err = true; | |
383 | ||
384 | ext_end = isa; | |
385 | if (unlikely(ext_err)) | |
386 | break; | |
387 | ||
388 | if (!isdigit(ext_end[-1])) | |
389 | break; | |
390 | ||
391 | while (isdigit(*--ext_end)) | |
392 | ; | |
393 | ||
394 | if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) { | |
395 | ++ext_end; | |
396 | break; | |
397 | } | |
398 | ||
399 | while (isdigit(*--ext_end)) | |
400 | ; | |
401 | ||
402 | ++ext_end; | |
403 | break; | |
404 | default: | |
405 | /* | |
406 | * Things are a little easier for single-letter extensions, as they | |
407 | * are parsed forwards. | |
408 | * | |
409 | * After checking that our starting position is valid, we need to | |
410 | * ensure that, when isa was incremented at the start of the loop, | |
411 | * that it arrived at the start of the next extension. | |
412 | * | |
413 | * If we are already on a non-digit, there is nothing to do. Either | |
414 | * we have a multi-letter extension's _, or the start of an | |
415 | * extension. | |
416 | * | |
417 | * Otherwise we have found the current extension's major version | |
418 | * number. Parse past it, and a subsequent p/minor version number | |
419 | * if present. The `p` extension must not appear immediately after | |
420 | * a number, so there is no fear of missing it. | |
421 | * | |
422 | */ | |
423 | if (unlikely(!isalpha(*ext))) { | |
424 | ext_err = true; | |
425 | break; | |
426 | } | |
427 | ||
428 | if (!isdigit(*isa)) | |
429 | break; | |
430 | ||
431 | while (isdigit(*++isa)) | |
432 | ; | |
433 | ||
434 | if (tolower(*isa) != 'p') | |
435 | break; | |
436 | ||
437 | if (!isdigit(*++isa)) { | |
438 | --isa; | |
439 | break; | |
440 | } | |
441 | ||
442 | while (isdigit(*++isa)) | |
443 | ; | |
444 | ||
445 | break; | |
446 | } | |
447 | ||
448 | /* | |
449 | * The parser expects that at the start of an iteration isa points to the | |
450 | * first character of the next extension. As we stop parsing an extension | |
451 | * on meeting a non-alphanumeric character, an extra increment is needed | |
452 | * where the succeeding extension is a multi-letter prefixed with an "_". | |
453 | */ | |
454 | if (*isa == '_') | |
455 | ++isa; | |
456 | ||
4265b0ec CD |
457 | if (unlikely(ext_err)) |
458 | continue; | |
459 | if (!ext_long) { | |
460 | int nr = tolower(*ext) - 'a'; | |
461 | ||
462 | if (riscv_isa_extension_check(nr)) { | |
463 | *this_hwcap |= isa2hwcap[nr]; | |
464 | set_bit(nr, isainfo->isa); | |
465 | } | |
466 | } else { | |
467 | for (int i = 0; i < riscv_isa_ext_count; i++) | |
0d8295ed | 468 | match_isa_ext(&riscv_isa_ext[i], ext, ext_end, isainfo); |
4265b0ec | 469 | } |
4265b0ec CD |
470 | } |
471 | } | |
472 | ||
473 | static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) | |
e2c0cdfb | 474 | { |
dd81c8ab | 475 | struct device_node *node; |
e2c0cdfb | 476 | const char *isa; |
4265b0ec | 477 | int rc; |
396c0183 S |
478 | struct acpi_table_header *rhct; |
479 | acpi_status status; | |
914d6f44 | 480 | unsigned int cpu; |
e2c0cdfb | 481 | |
396c0183 S |
482 | if (!acpi_disabled) { |
483 | status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); | |
484 | if (ACPI_FAILURE(status)) | |
485 | return; | |
486 | } | |
487 | ||
914d6f44 | 488 | for_each_possible_cpu(cpu) { |
16252e01 | 489 | struct riscv_isainfo *isainfo = &hart_isa[cpu]; |
fbdc6193 | 490 | unsigned long this_hwcap = 0; |
e2c0cdfb | 491 | |
396c0183 S |
492 | if (acpi_disabled) { |
493 | node = of_cpu_device_node_get(cpu); | |
494 | if (!node) { | |
495 | pr_warn("Unable to find cpu node\n"); | |
496 | continue; | |
497 | } | |
e2c0cdfb | 498 | |
396c0183 S |
499 | rc = of_property_read_string(node, "riscv,isa", &isa); |
500 | of_node_put(node); | |
501 | if (rc) { | |
502 | pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); | |
503 | continue; | |
504 | } | |
505 | } else { | |
506 | rc = acpi_get_riscv_isa(rhct, cpu, &isa); | |
507 | if (rc < 0) { | |
508 | pr_warn("Unable to get ISA for the hart - %d\n", cpu); | |
509 | continue; | |
510 | } | |
fbdc6193 AP |
511 | } |
512 | ||
4265b0ec | 513 | riscv_parse_isa_string(&this_hwcap, isainfo, isa2hwcap, isa); |
fbdc6193 | 514 | |
07edc327 CD |
515 | /* |
516 | * These ones were as they were part of the base ISA when the | |
517 | * port & dt-bindings were upstreamed, and so can be set | |
518 | * unconditionally where `i` is in riscv,isa on DT systems. | |
519 | */ | |
520 | if (acpi_disabled) { | |
ab2dbc7a PD |
521 | set_bit(RISCV_ISA_EXT_ZICSR, isainfo->isa); |
522 | set_bit(RISCV_ISA_EXT_ZIFENCEI, isainfo->isa); | |
42b89447 PD |
523 | set_bit(RISCV_ISA_EXT_ZICNTR, isainfo->isa); |
524 | set_bit(RISCV_ISA_EXT_ZIHPM, isainfo->isa); | |
07edc327 CD |
525 | } |
526 | ||
fbdc6193 AP |
527 | /* |
528 | * All "okay" hart should have same isa. Set HWCAP based on | |
529 | * common capabilities of every "okay" hart, in case they don't | |
530 | * have. | |
531 | */ | |
532 | if (elf_hwcap) | |
533 | elf_hwcap &= this_hwcap; | |
534 | else | |
535 | elf_hwcap = this_hwcap; | |
6bcff515 | 536 | |
8f51558e | 537 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) |
82e9c66e | 538 | bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); |
8f51558e | 539 | else |
82e9c66e | 540 | bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); |
fbdc6193 | 541 | } |
e2c0cdfb | 542 | |
396c0183 S |
543 | if (!acpi_disabled && rhct) |
544 | acpi_put_table((struct acpi_table_header *)rhct); | |
4265b0ec CD |
545 | } |
546 | ||
90700a4f CD |
547 | static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) |
548 | { | |
549 | unsigned int cpu; | |
550 | ||
551 | for_each_possible_cpu(cpu) { | |
552 | unsigned long this_hwcap = 0; | |
553 | struct device_node *cpu_node; | |
554 | struct riscv_isainfo *isainfo = &hart_isa[cpu]; | |
555 | ||
556 | cpu_node = of_cpu_device_node_get(cpu); | |
557 | if (!cpu_node) { | |
558 | pr_warn("Unable to find cpu node\n"); | |
559 | continue; | |
560 | } | |
561 | ||
562 | if (!of_property_present(cpu_node, "riscv,isa-extensions")) { | |
563 | of_node_put(cpu_node); | |
564 | continue; | |
565 | } | |
566 | ||
567 | for (int i = 0; i < riscv_isa_ext_count; i++) { | |
0d8295ed EG |
568 | const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; |
569 | ||
90700a4f | 570 | if (of_property_match_string(cpu_node, "riscv,isa-extensions", |
0d8295ed | 571 | ext->property) < 0) |
90700a4f CD |
572 | continue; |
573 | ||
0d8295ed EG |
574 | if (ext->subset_ext_size) { |
575 | for (int j = 0; j < ext->subset_ext_size; j++) { | |
576 | if (riscv_isa_extension_check(ext->subset_ext_ids[i])) | |
577 | set_bit(ext->subset_ext_ids[j], isainfo->isa); | |
578 | } | |
579 | } | |
90700a4f | 580 | |
0d8295ed EG |
581 | if (riscv_isa_extension_check(ext->id)) { |
582 | set_bit(ext->id, isainfo->isa); | |
90700a4f | 583 | |
0d8295ed EG |
584 | /* Only single letter extensions get set in hwcap */ |
585 | if (strnlen(riscv_isa_ext[i].name, 2) == 1) | |
586 | this_hwcap |= isa2hwcap[riscv_isa_ext[i].id]; | |
587 | } | |
90700a4f CD |
588 | } |
589 | ||
590 | of_node_put(cpu_node); | |
591 | ||
592 | /* | |
593 | * All "okay" harts should have same isa. Set HWCAP based on | |
594 | * common capabilities of every "okay" hart, in case they don't. | |
595 | */ | |
596 | if (elf_hwcap) | |
597 | elf_hwcap &= this_hwcap; | |
598 | else | |
599 | elf_hwcap = this_hwcap; | |
600 | ||
601 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) | |
602 | bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); | |
603 | else | |
604 | bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); | |
605 | } | |
606 | ||
607 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) | |
608 | return -ENOENT; | |
609 | ||
610 | return 0; | |
611 | } | |
612 | ||
496ea826 CD |
613 | #ifdef CONFIG_RISCV_ISA_FALLBACK |
614 | bool __initdata riscv_isa_fallback = true; | |
615 | #else | |
616 | bool __initdata riscv_isa_fallback; | |
617 | static int __init riscv_isa_fallback_setup(char *__unused) | |
618 | { | |
619 | riscv_isa_fallback = true; | |
620 | return 1; | |
621 | } | |
622 | early_param("riscv_isa_fallback", riscv_isa_fallback_setup); | |
623 | #endif | |
624 | ||
4265b0ec CD |
625 | void __init riscv_fill_hwcap(void) |
626 | { | |
627 | char print_str[NUM_ALPHA_EXTS + 1]; | |
4265b0ec | 628 | unsigned long isa2hwcap[26] = {0}; |
90700a4f | 629 | int i, j; |
4265b0ec CD |
630 | |
631 | isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; | |
632 | isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; | |
633 | isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; | |
634 | isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; | |
635 | isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; | |
636 | isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; | |
637 | isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V; | |
638 | ||
90700a4f CD |
639 | if (!acpi_disabled) { |
640 | riscv_fill_hwcap_from_isa_string(isa2hwcap); | |
641 | } else { | |
642 | int ret = riscv_fill_hwcap_from_ext_list(isa2hwcap); | |
396c0183 | 643 | |
496ea826 | 644 | if (ret && riscv_isa_fallback) { |
90700a4f CD |
645 | pr_info("Falling back to deprecated \"riscv,isa\"\n"); |
646 | riscv_fill_hwcap_from_isa_string(isa2hwcap); | |
647 | } | |
648 | } | |
396c0183 | 649 | |
90700a4f CD |
650 | /* |
651 | * We don't support systems with F but without D, so mask those out | |
652 | * here. | |
653 | */ | |
86e581e3 | 654 | if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { |
7265d103 | 655 | pr_info("This kernel does not support systems with F but not D\n"); |
86e581e3 PD |
656 | elf_hwcap &= ~COMPAT_HWCAP_ISA_F; |
657 | } | |
658 | ||
dc6667a4 | 659 | if (elf_hwcap & COMPAT_HWCAP_ISA_V) { |
7017858e | 660 | riscv_v_setup_vsize(); |
dc6667a4 GR |
661 | /* |
662 | * ISA string in device tree might have 'v' flag, but | |
663 | * CONFIG_RISCV_ISA_V is disabled in kernel. | |
664 | * Clear V flag in elf_hwcap if CONFIG_RISCV_ISA_V is disabled. | |
665 | */ | |
666 | if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) | |
667 | elf_hwcap &= ~COMPAT_HWCAP_ISA_V; | |
668 | } | |
669 | ||
6bcff515 | 670 | memset(print_str, 0, sizeof(print_str)); |
58004f26 | 671 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
672 | if (riscv_isa[0] & BIT_MASK(i)) |
673 | print_str[j++] = (char)('a' + i); | |
02d52fbd | 674 | pr_info("riscv: base ISA extensions %s\n", print_str); |
6bcff515 AP |
675 | |
676 | memset(print_str, 0, sizeof(print_str)); | |
58004f26 | 677 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
678 | if (elf_hwcap & BIT_MASK(i)) |
679 | print_str[j++] = (char)('a' + i); | |
680 | pr_info("riscv: ELF capabilities %s\n", print_str); | |
e2c0cdfb | 681 | } |
ff689fd2 | 682 | |
50724efc AC |
683 | unsigned long riscv_get_elf_hwcap(void) |
684 | { | |
1fd96a3e AC |
685 | unsigned long hwcap; |
686 | ||
687 | hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)); | |
688 | ||
689 | if (!riscv_v_vstate_ctrl_user_allowed()) | |
690 | hwcap &= ~COMPAT_HWCAP_ISA_V; | |
691 | ||
692 | return hwcap; | |
50724efc AC |
693 | } |
694 | ||
43c16d51 AJ |
695 | void riscv_user_isa_enable(void) |
696 | { | |
697 | if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_ZICBOZ)) | |
698 | csr_set(CSR_SENVCFG, ENVCFG_CBZE); | |
699 | } | |
700 | ||
ff689fd2 | 701 | #ifdef CONFIG_RISCV_ALTERNATIVE |
d25f2563 AJ |
702 | /* |
703 | * Alternative patch sites consider 48 bits when determining when to patch | |
704 | * the old instruction sequence with the new. These bits are broken into a | |
705 | * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the | |
706 | * patch site is for an erratum, identified by the 32-bit patch ID. When | |
707 | * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures | |
708 | * further break down patch ID into two 16-bit numbers. The lower 16 bits | |
709 | * are the cpufeature ID and the upper 16 bits are used for a value specific | |
710 | * to the cpufeature and patch site. If the upper 16 bits are zero, then it | |
711 | * implies no specific value is specified. cpufeatures that want to control | |
712 | * patching on a per-site basis will provide non-zero values and implement | |
713 | * checks here. The checks return true when patching should be done, and | |
714 | * false otherwise. | |
715 | */ | |
716 | static bool riscv_cpufeature_patch_check(u16 id, u16 value) | |
717 | { | |
718 | if (!value) | |
719 | return true; | |
720 | ||
ab0f7746 AJ |
721 | switch (id) { |
722 | case RISCV_ISA_EXT_ZICBOZ: | |
723 | /* | |
724 | * Zicboz alternative applications provide the maximum | |
725 | * supported block size order, or zero when it doesn't | |
726 | * matter. If the current block size exceeds the maximum, | |
727 | * then the alternative cannot be applied. | |
728 | */ | |
729 | return riscv_cboz_block_size <= (1U << value); | |
730 | } | |
731 | ||
d25f2563 AJ |
732 | return false; |
733 | } | |
734 | ||
ff689fd2 HS |
735 | void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, |
736 | struct alt_entry *end, | |
737 | unsigned int stage) | |
738 | { | |
ff689fd2 | 739 | struct alt_entry *alt; |
8d23e94a | 740 | void *oldptr, *altptr; |
d25f2563 | 741 | u16 id, value; |
ff689fd2 | 742 | |
191b27c7 JZ |
743 | if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) |
744 | return; | |
ff689fd2 HS |
745 | |
746 | for (alt = begin; alt < end; alt++) { | |
747 | if (alt->vendor_id != 0) | |
748 | continue; | |
d25f2563 AJ |
749 | |
750 | id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); | |
751 | ||
752 | if (id >= RISCV_ISA_EXT_MAX) { | |
753 | WARN(1, "This extension id:%d is not in ISA extension list", id); | |
ff689fd2 HS |
754 | continue; |
755 | } | |
756 | ||
d25f2563 AJ |
757 | if (!__riscv_isa_extension_available(NULL, id)) |
758 | continue; | |
759 | ||
760 | value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); | |
761 | if (!riscv_cpufeature_patch_check(id, value)) | |
4bf88607 JZ |
762 | continue; |
763 | ||
8d23e94a JZ |
764 | oldptr = ALT_OLD_PTR(alt); |
765 | altptr = ALT_ALT_PTR(alt); | |
9493e6f3 CD |
766 | |
767 | mutex_lock(&text_mutex); | |
8d23e94a JZ |
768 | patch_text_nosync(oldptr, altptr, alt->alt_len); |
769 | riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); | |
9493e6f3 | 770 | mutex_unlock(&text_mutex); |
ff689fd2 HS |
771 | } |
772 | } | |
773 | #endif |