Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
e2c0cdfb PD |
2 | /* |
3 | * Copied from arch/arm64/kernel/cpufeature.c | |
4 | * | |
5 | * Copyright (C) 2015 ARM Ltd. | |
6 | * Copyright (C) 2017 SiFive | |
e2c0cdfb PD |
7 | */ |
8 | ||
396c0183 | 9 | #include <linux/acpi.h> |
6bcff515 | 10 | #include <linux/bitmap.h> |
55e0bf49 | 11 | #include <linux/cpuhotplug.h> |
2a31c54b | 12 | #include <linux/ctype.h> |
9daaca4a | 13 | #include <linux/log2.h> |
9493e6f3 | 14 | #include <linux/memory.h> |
ff689fd2 | 15 | #include <linux/module.h> |
e2c0cdfb | 16 | #include <linux/of.h> |
396c0183 | 17 | #include <asm/acpi.h> |
ff689fd2 | 18 | #include <asm/alternative.h> |
1631ba12 | 19 | #include <asm/cacheflush.h> |
c2d3c844 | 20 | #include <asm/cpufeature.h> |
e2c0cdfb | 21 | #include <asm/hwcap.h> |
584ea656 | 22 | #include <asm/hwprobe.h> |
ff689fd2 | 23 | #include <asm/patch.h> |
ff689fd2 | 24 | #include <asm/processor.h> |
7017858e | 25 | #include <asm/vector.h> |
e2c0cdfb | 26 | |
584ea656 EG |
27 | #include "copy-unaligned.h" |
28 | ||
58004f26 TO |
29 | #define NUM_ALPHA_EXTS ('z' - 'a' + 1) |
30 | ||
584ea656 EG |
31 | #define MISALIGNED_ACCESS_JIFFIES_LG2 1 |
32 | #define MISALIGNED_BUFFER_SIZE 0x4000 | |
55e0bf49 | 33 | #define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE) |
584ea656 EG |
34 | #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80) |
35 | ||
e2c0cdfb | 36 | unsigned long elf_hwcap __read_mostly; |
6bcff515 AP |
37 | |
38 | /* Host ISA bitmap */ | |
39 | static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; | |
40 | ||
82e9c66e EG |
41 | /* Per-cpu ISA extensions. */ |
42 | struct riscv_isainfo hart_isa[NR_CPUS]; | |
43 | ||
62a31d6e EG |
44 | /* Performance information */ |
45 | DEFINE_PER_CPU(long, misaligned_access_speed); | |
46 | ||
6bcff515 AP |
47 | /** |
48 | * riscv_isa_extension_base() - Get base extension word | |
49 | * | |
50 | * @isa_bitmap: ISA bitmap to use | |
51 | * Return: base extension word as unsigned long value | |
52 | * | |
53 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
54 | */ | |
55 | unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) | |
56 | { | |
57 | if (!isa_bitmap) | |
58 | return riscv_isa[0]; | |
59 | return isa_bitmap[0]; | |
60 | } | |
61 | EXPORT_SYMBOL_GPL(riscv_isa_extension_base); | |
62 | ||
63 | /** | |
64 | * __riscv_isa_extension_available() - Check whether given extension | |
65 | * is available or not | |
66 | * | |
67 | * @isa_bitmap: ISA bitmap to use | |
68 | * @bit: bit position of the desired extension | |
69 | * Return: true or false | |
70 | * | |
71 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
72 | */ | |
0d8295ed | 73 | bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit) |
6bcff515 AP |
74 | { |
75 | const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; | |
76 | ||
77 | if (bit >= RISCV_ISA_EXT_MAX) | |
78 | return false; | |
79 | ||
80 | return test_bit(bit, bmap) ? true : false; | |
81 | } | |
82 | EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); | |
83 | ||
fb0ff0a9 AJ |
84 | static bool riscv_isa_extension_check(int id) |
85 | { | |
9daaca4a AJ |
86 | switch (id) { |
87 | case RISCV_ISA_EXT_ZICBOM: | |
88 | if (!riscv_cbom_block_size) { | |
c818fea8 | 89 | pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); |
9daaca4a AJ |
90 | return false; |
91 | } else if (!is_power_of_2(riscv_cbom_block_size)) { | |
c818fea8 | 92 | pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); |
9daaca4a AJ |
93 | return false; |
94 | } | |
95 | return true; | |
7ea5a736 AJ |
96 | case RISCV_ISA_EXT_ZICBOZ: |
97 | if (!riscv_cboz_block_size) { | |
181f2a28 | 98 | pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n"); |
7ea5a736 AJ |
99 | return false; |
100 | } else if (!is_power_of_2(riscv_cboz_block_size)) { | |
181f2a28 | 101 | pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n"); |
7ea5a736 AJ |
102 | return false; |
103 | } | |
104 | return true; | |
0d8295ed EG |
105 | case RISCV_ISA_EXT_INVALID: |
106 | return false; | |
9daaca4a AJ |
107 | } |
108 | ||
fb0ff0a9 AJ |
109 | return true; |
110 | } | |
111 | ||
0d8295ed EG |
112 | #define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size) { \ |
113 | .name = #_name, \ | |
114 | .property = #_name, \ | |
115 | .id = _id, \ | |
116 | .subset_ext_ids = _subset_exts, \ | |
117 | .subset_ext_size = _subset_exts_size \ | |
37f988dc | 118 | } |
8135ade3 | 119 | |
0d8295ed EG |
120 | #define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0) |
121 | ||
122 | /* Used to declare pure "lasso" extension (Zk for instance) */ | |
123 | #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \ | |
124 | _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, ARRAY_SIZE(_bundled_exts)) | |
125 | ||
aec33539 CL |
126 | /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */ |
127 | #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \ | |
128 | _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts)) | |
129 | ||
0d8295ed EG |
130 | static const unsigned int riscv_zk_bundled_exts[] = { |
131 | RISCV_ISA_EXT_ZBKB, | |
132 | RISCV_ISA_EXT_ZBKC, | |
133 | RISCV_ISA_EXT_ZBKX, | |
134 | RISCV_ISA_EXT_ZKND, | |
135 | RISCV_ISA_EXT_ZKNE, | |
136 | RISCV_ISA_EXT_ZKR, | |
137 | RISCV_ISA_EXT_ZKT, | |
138 | }; | |
139 | ||
140 | static const unsigned int riscv_zkn_bundled_exts[] = { | |
141 | RISCV_ISA_EXT_ZBKB, | |
142 | RISCV_ISA_EXT_ZBKC, | |
143 | RISCV_ISA_EXT_ZBKX, | |
144 | RISCV_ISA_EXT_ZKND, | |
145 | RISCV_ISA_EXT_ZKNE, | |
146 | RISCV_ISA_EXT_ZKNH, | |
147 | }; | |
148 | ||
149 | static const unsigned int riscv_zks_bundled_exts[] = { | |
150 | RISCV_ISA_EXT_ZBKB, | |
151 | RISCV_ISA_EXT_ZBKC, | |
152 | RISCV_ISA_EXT_ZKSED, | |
153 | RISCV_ISA_EXT_ZKSH | |
154 | }; | |
155 | ||
aec33539 CL |
156 | #define RISCV_ISA_EXT_ZVKN \ |
157 | RISCV_ISA_EXT_ZVKNED, \ | |
158 | RISCV_ISA_EXT_ZVKNHB, \ | |
159 | RISCV_ISA_EXT_ZVKB, \ | |
160 | RISCV_ISA_EXT_ZVKT | |
161 | ||
162 | static const unsigned int riscv_zvkn_bundled_exts[] = { | |
163 | RISCV_ISA_EXT_ZVKN | |
164 | }; | |
165 | ||
166 | static const unsigned int riscv_zvknc_bundled_exts[] = { | |
167 | RISCV_ISA_EXT_ZVKN, | |
168 | RISCV_ISA_EXT_ZVBC | |
169 | }; | |
170 | ||
171 | static const unsigned int riscv_zvkng_bundled_exts[] = { | |
172 | RISCV_ISA_EXT_ZVKN, | |
173 | RISCV_ISA_EXT_ZVKG | |
174 | }; | |
175 | ||
176 | #define RISCV_ISA_EXT_ZVKS \ | |
177 | RISCV_ISA_EXT_ZVKSED, \ | |
178 | RISCV_ISA_EXT_ZVKSH, \ | |
179 | RISCV_ISA_EXT_ZVKB, \ | |
180 | RISCV_ISA_EXT_ZVKT | |
181 | ||
182 | static const unsigned int riscv_zvks_bundled_exts[] = { | |
183 | RISCV_ISA_EXT_ZVKS | |
184 | }; | |
185 | ||
186 | static const unsigned int riscv_zvksc_bundled_exts[] = { | |
187 | RISCV_ISA_EXT_ZVKS, | |
188 | RISCV_ISA_EXT_ZVBC | |
189 | }; | |
190 | ||
191 | static const unsigned int riscv_zvksg_bundled_exts[] = { | |
192 | RISCV_ISA_EXT_ZVKS, | |
193 | RISCV_ISA_EXT_ZVKG | |
194 | }; | |
195 | ||
196 | static const unsigned int riscv_zvbb_exts[] = { | |
197 | RISCV_ISA_EXT_ZVKB | |
198 | }; | |
199 | ||
8135ade3 CD |
200 | /* |
201 | * The canonical order of ISA extension names in the ISA string is defined in | |
202 | * chapter 27 of the unprivileged specification. | |
203 | * | |
204 | * Ordinarily, for in-kernel data structures, this order is unimportant but | |
205 | * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. | |
206 | * | |
207 | * The specification uses vague wording, such as should, when it comes to | |
208 | * ordering, so for our purposes the following rules apply: | |
209 | * | |
210 | * 1. All multi-letter extensions must be separated from other extensions by an | |
211 | * underscore. | |
212 | * | |
213 | * 2. Additional standard extensions (starting with 'Z') must be sorted after | |
214 | * single-letter extensions and before any higher-privileged extensions. | |
215 | * | |
216 | * 3. The first letter following the 'Z' conventionally indicates the most | |
217 | * closely related alphabetical extension category, IMAFDQLCBKJTPVH. | |
218 | * If multiple 'Z' extensions are named, they must be ordered first by | |
219 | * category, then alphabetically within a category. | |
220 | * | |
221 | * 3. Standard supervisor-level extensions (starting with 'S') must be listed | |
222 | * after standard unprivileged extensions. If multiple supervisor-level | |
223 | * extensions are listed, they must be ordered alphabetically. | |
224 | * | |
225 | * 4. Standard machine-level extensions (starting with 'Zxm') must be listed | |
226 | * after any lower-privileged, standard extensions. If multiple | |
227 | * machine-level extensions are listed, they must be ordered | |
228 | * alphabetically. | |
229 | * | |
230 | * 5. Non-standard extensions (starting with 'X') must be listed after all | |
231 | * standard extensions. If multiple non-standard extensions are listed, they | |
232 | * must be ordered alphabetically. | |
233 | * | |
234 | * An example string following the order is: | |
235 | * rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux | |
236 | * | |
237 | * New entries to this struct should follow the ordering rules described above. | |
238 | */ | |
239 | const struct riscv_isa_ext_data riscv_isa_ext[] = { | |
effc122a CD |
240 | __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), |
241 | __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), | |
242 | __RISCV_ISA_EXT_DATA(a, RISCV_ISA_EXT_a), | |
243 | __RISCV_ISA_EXT_DATA(f, RISCV_ISA_EXT_f), | |
244 | __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), | |
245 | __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), | |
246 | __RISCV_ISA_EXT_DATA(c, RISCV_ISA_EXT_c), | |
247 | __RISCV_ISA_EXT_DATA(b, RISCV_ISA_EXT_b), | |
248 | __RISCV_ISA_EXT_DATA(k, RISCV_ISA_EXT_k), | |
249 | __RISCV_ISA_EXT_DATA(j, RISCV_ISA_EXT_j), | |
250 | __RISCV_ISA_EXT_DATA(p, RISCV_ISA_EXT_p), | |
251 | __RISCV_ISA_EXT_DATA(v, RISCV_ISA_EXT_v), | |
252 | __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), | |
8135ade3 CD |
253 | __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), |
254 | __RISCV_ISA_EXT_DATA(zicboz, RISCV_ISA_EXT_ZICBOZ), | |
255 | __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), | |
662a601a | 256 | __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), |
8135ade3 CD |
257 | __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), |
258 | __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), | |
259 | __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), | |
260 | __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), | |
261 | __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), | |
262 | __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), | |
e45f463a | 263 | __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC), |
0d8295ed EG |
264 | __RISCV_ISA_EXT_DATA(zbkb, RISCV_ISA_EXT_ZBKB), |
265 | __RISCV_ISA_EXT_DATA(zbkc, RISCV_ISA_EXT_ZBKC), | |
266 | __RISCV_ISA_EXT_DATA(zbkx, RISCV_ISA_EXT_ZBKX), | |
8135ade3 | 267 | __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), |
0d8295ed EG |
268 | __RISCV_ISA_EXT_BUNDLE(zk, riscv_zk_bundled_exts), |
269 | __RISCV_ISA_EXT_BUNDLE(zkn, riscv_zkn_bundled_exts), | |
270 | __RISCV_ISA_EXT_DATA(zknd, RISCV_ISA_EXT_ZKND), | |
271 | __RISCV_ISA_EXT_DATA(zkne, RISCV_ISA_EXT_ZKNE), | |
272 | __RISCV_ISA_EXT_DATA(zknh, RISCV_ISA_EXT_ZKNH), | |
273 | __RISCV_ISA_EXT_DATA(zkr, RISCV_ISA_EXT_ZKR), | |
274 | __RISCV_ISA_EXT_BUNDLE(zks, riscv_zks_bundled_exts), | |
275 | __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT), | |
276 | __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED), | |
277 | __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH), | |
aec33539 CL |
278 | __RISCV_ISA_EXT_SUPERSET(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts), |
279 | __RISCV_ISA_EXT_DATA(zvbc, RISCV_ISA_EXT_ZVBC), | |
280 | __RISCV_ISA_EXT_DATA(zvkb, RISCV_ISA_EXT_ZVKB), | |
281 | __RISCV_ISA_EXT_DATA(zvkg, RISCV_ISA_EXT_ZVKG), | |
282 | __RISCV_ISA_EXT_BUNDLE(zvkn, riscv_zvkn_bundled_exts), | |
283 | __RISCV_ISA_EXT_BUNDLE(zvknc, riscv_zvknc_bundled_exts), | |
284 | __RISCV_ISA_EXT_DATA(zvkned, RISCV_ISA_EXT_ZVKNED), | |
285 | __RISCV_ISA_EXT_BUNDLE(zvkng, riscv_zvkng_bundled_exts), | |
286 | __RISCV_ISA_EXT_DATA(zvknha, RISCV_ISA_EXT_ZVKNHA), | |
287 | __RISCV_ISA_EXT_DATA(zvknhb, RISCV_ISA_EXT_ZVKNHB), | |
288 | __RISCV_ISA_EXT_BUNDLE(zvks, riscv_zvks_bundled_exts), | |
289 | __RISCV_ISA_EXT_BUNDLE(zvksc, riscv_zvksc_bundled_exts), | |
290 | __RISCV_ISA_EXT_DATA(zvksed, RISCV_ISA_EXT_ZVKSED), | |
291 | __RISCV_ISA_EXT_DATA(zvksh, RISCV_ISA_EXT_ZVKSH), | |
292 | __RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts), | |
293 | __RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT), | |
8135ade3 | 294 | __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), |
9dbaf381 | 295 | __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN), |
8135ade3 CD |
296 | __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), |
297 | __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), | |
298 | __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), | |
299 | __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), | |
300 | __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), | |
301 | __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), | |
302 | }; | |
303 | ||
304 | const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); | |
305 | ||
0d8295ed EG |
306 | static void __init match_isa_ext(const struct riscv_isa_ext_data *ext, const char *name, |
307 | const char *name_end, struct riscv_isainfo *isainfo) | |
308 | { | |
309 | if ((name_end - name == strlen(ext->name)) && | |
310 | !strncasecmp(name, ext->name, name_end - name)) { | |
311 | /* | |
312 | * If this is a bundle, enable all the ISA extensions that | |
313 | * comprise the bundle. | |
314 | */ | |
315 | if (ext->subset_ext_size) { | |
316 | for (int i = 0; i < ext->subset_ext_size; i++) { | |
317 | if (riscv_isa_extension_check(ext->subset_ext_ids[i])) | |
318 | set_bit(ext->subset_ext_ids[i], isainfo->isa); | |
319 | } | |
320 | } | |
321 | ||
322 | /* | |
323 | * This is valid even for bundle extensions which uses the RISCV_ISA_EXT_INVALID id | |
324 | * (rejected by riscv_isa_extension_check()). | |
325 | */ | |
326 | if (riscv_isa_extension_check(ext->id)) | |
327 | set_bit(ext->id, isainfo->isa); | |
328 | } | |
329 | } | |
330 | ||
4265b0ec CD |
331 | static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct riscv_isainfo *isainfo, |
332 | unsigned long *isa2hwcap, const char *isa) | |
333 | { | |
334 | /* | |
335 | * For all possible cpus, we have already validated in | |
336 | * the boot process that they at least contain "rv" and | |
337 | * whichever of "32"/"64" this kernel supports, and so this | |
338 | * section can be skipped. | |
339 | */ | |
340 | isa += 4; | |
341 | ||
342 | while (*isa) { | |
343 | const char *ext = isa++; | |
344 | const char *ext_end = isa; | |
345 | bool ext_long = false, ext_err = false; | |
346 | ||
347 | switch (*ext) { | |
348 | case 's': | |
349 | /* | |
8f501be8 | 350 | * Workaround for invalid single-letter 's' & 'u' (QEMU). |
4265b0ec | 351 | * No need to set the bit in riscv_isa as 's' & 'u' are |
8f501be8 TO |
352 | * not valid ISA extensions. It works unless the first |
353 | * multi-letter extension in the ISA string begins with | |
354 | * "Su" and is not prefixed with an underscore. | |
4265b0ec CD |
355 | */ |
356 | if (ext[-1] != '_' && ext[1] == 'u') { | |
357 | ++isa; | |
358 | ext_err = true; | |
359 | break; | |
360 | } | |
361 | fallthrough; | |
362 | case 'S': | |
363 | case 'x': | |
364 | case 'X': | |
365 | case 'z': | |
366 | case 'Z': | |
367 | /* | |
368 | * Before attempting to parse the extension itself, we find its end. | |
369 | * As multi-letter extensions must be split from other multi-letter | |
370 | * extensions with an "_", the end of a multi-letter extension will | |
371 | * either be the null character or the "_" at the start of the next | |
372 | * multi-letter extension. | |
373 | * | |
374 | * Next, as the extensions version is currently ignored, we | |
375 | * eliminate that portion. This is done by parsing backwards from | |
376 | * the end of the extension, removing any numbers. This may be a | |
377 | * major or minor number however, so the process is repeated if a | |
378 | * minor number was found. | |
379 | * | |
380 | * ext_end is intended to represent the first character *after* the | |
381 | * name portion of an extension, but will be decremented to the last | |
382 | * character itself while eliminating the extensions version number. | |
383 | * A simple re-increment solves this problem. | |
384 | */ | |
385 | ext_long = true; | |
386 | for (; *isa && *isa != '_'; ++isa) | |
387 | if (unlikely(!isalnum(*isa))) | |
388 | ext_err = true; | |
389 | ||
390 | ext_end = isa; | |
391 | if (unlikely(ext_err)) | |
392 | break; | |
393 | ||
394 | if (!isdigit(ext_end[-1])) | |
395 | break; | |
396 | ||
397 | while (isdigit(*--ext_end)) | |
398 | ; | |
399 | ||
400 | if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) { | |
401 | ++ext_end; | |
402 | break; | |
403 | } | |
404 | ||
405 | while (isdigit(*--ext_end)) | |
406 | ; | |
407 | ||
408 | ++ext_end; | |
409 | break; | |
410 | default: | |
411 | /* | |
412 | * Things are a little easier for single-letter extensions, as they | |
413 | * are parsed forwards. | |
414 | * | |
415 | * After checking that our starting position is valid, we need to | |
416 | * ensure that, when isa was incremented at the start of the loop, | |
417 | * that it arrived at the start of the next extension. | |
418 | * | |
419 | * If we are already on a non-digit, there is nothing to do. Either | |
420 | * we have a multi-letter extension's _, or the start of an | |
421 | * extension. | |
422 | * | |
423 | * Otherwise we have found the current extension's major version | |
424 | * number. Parse past it, and a subsequent p/minor version number | |
425 | * if present. The `p` extension must not appear immediately after | |
426 | * a number, so there is no fear of missing it. | |
427 | * | |
428 | */ | |
429 | if (unlikely(!isalpha(*ext))) { | |
430 | ext_err = true; | |
431 | break; | |
432 | } | |
433 | ||
434 | if (!isdigit(*isa)) | |
435 | break; | |
436 | ||
437 | while (isdigit(*++isa)) | |
438 | ; | |
439 | ||
440 | if (tolower(*isa) != 'p') | |
441 | break; | |
442 | ||
443 | if (!isdigit(*++isa)) { | |
444 | --isa; | |
445 | break; | |
446 | } | |
447 | ||
448 | while (isdigit(*++isa)) | |
449 | ; | |
450 | ||
451 | break; | |
452 | } | |
453 | ||
454 | /* | |
455 | * The parser expects that at the start of an iteration isa points to the | |
456 | * first character of the next extension. As we stop parsing an extension | |
457 | * on meeting a non-alphanumeric character, an extra increment is needed | |
458 | * where the succeeding extension is a multi-letter prefixed with an "_". | |
459 | */ | |
460 | if (*isa == '_') | |
461 | ++isa; | |
462 | ||
4265b0ec CD |
463 | if (unlikely(ext_err)) |
464 | continue; | |
465 | if (!ext_long) { | |
466 | int nr = tolower(*ext) - 'a'; | |
467 | ||
468 | if (riscv_isa_extension_check(nr)) { | |
469 | *this_hwcap |= isa2hwcap[nr]; | |
470 | set_bit(nr, isainfo->isa); | |
471 | } | |
472 | } else { | |
473 | for (int i = 0; i < riscv_isa_ext_count; i++) | |
0d8295ed | 474 | match_isa_ext(&riscv_isa_ext[i], ext, ext_end, isainfo); |
4265b0ec | 475 | } |
4265b0ec CD |
476 | } |
477 | } | |
478 | ||
479 | static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) | |
e2c0cdfb | 480 | { |
dd81c8ab | 481 | struct device_node *node; |
e2c0cdfb | 482 | const char *isa; |
4265b0ec | 483 | int rc; |
396c0183 S |
484 | struct acpi_table_header *rhct; |
485 | acpi_status status; | |
914d6f44 | 486 | unsigned int cpu; |
e2c0cdfb | 487 | |
396c0183 S |
488 | if (!acpi_disabled) { |
489 | status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); | |
490 | if (ACPI_FAILURE(status)) | |
491 | return; | |
492 | } | |
493 | ||
914d6f44 | 494 | for_each_possible_cpu(cpu) { |
16252e01 | 495 | struct riscv_isainfo *isainfo = &hart_isa[cpu]; |
fbdc6193 | 496 | unsigned long this_hwcap = 0; |
e2c0cdfb | 497 | |
396c0183 S |
498 | if (acpi_disabled) { |
499 | node = of_cpu_device_node_get(cpu); | |
500 | if (!node) { | |
501 | pr_warn("Unable to find cpu node\n"); | |
502 | continue; | |
503 | } | |
e2c0cdfb | 504 | |
396c0183 S |
505 | rc = of_property_read_string(node, "riscv,isa", &isa); |
506 | of_node_put(node); | |
507 | if (rc) { | |
508 | pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); | |
509 | continue; | |
510 | } | |
511 | } else { | |
512 | rc = acpi_get_riscv_isa(rhct, cpu, &isa); | |
513 | if (rc < 0) { | |
514 | pr_warn("Unable to get ISA for the hart - %d\n", cpu); | |
515 | continue; | |
516 | } | |
fbdc6193 AP |
517 | } |
518 | ||
4265b0ec | 519 | riscv_parse_isa_string(&this_hwcap, isainfo, isa2hwcap, isa); |
fbdc6193 | 520 | |
07edc327 CD |
521 | /* |
522 | * These ones were as they were part of the base ISA when the | |
523 | * port & dt-bindings were upstreamed, and so can be set | |
524 | * unconditionally where `i` is in riscv,isa on DT systems. | |
525 | */ | |
526 | if (acpi_disabled) { | |
ab2dbc7a PD |
527 | set_bit(RISCV_ISA_EXT_ZICSR, isainfo->isa); |
528 | set_bit(RISCV_ISA_EXT_ZIFENCEI, isainfo->isa); | |
42b89447 PD |
529 | set_bit(RISCV_ISA_EXT_ZICNTR, isainfo->isa); |
530 | set_bit(RISCV_ISA_EXT_ZIHPM, isainfo->isa); | |
07edc327 CD |
531 | } |
532 | ||
fbdc6193 AP |
533 | /* |
534 | * All "okay" hart should have same isa. Set HWCAP based on | |
535 | * common capabilities of every "okay" hart, in case they don't | |
536 | * have. | |
537 | */ | |
538 | if (elf_hwcap) | |
539 | elf_hwcap &= this_hwcap; | |
540 | else | |
541 | elf_hwcap = this_hwcap; | |
6bcff515 | 542 | |
8f51558e | 543 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) |
82e9c66e | 544 | bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); |
8f51558e | 545 | else |
82e9c66e | 546 | bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); |
fbdc6193 | 547 | } |
e2c0cdfb | 548 | |
396c0183 S |
549 | if (!acpi_disabled && rhct) |
550 | acpi_put_table((struct acpi_table_header *)rhct); | |
4265b0ec CD |
551 | } |
552 | ||
90700a4f CD |
553 | static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) |
554 | { | |
555 | unsigned int cpu; | |
556 | ||
557 | for_each_possible_cpu(cpu) { | |
558 | unsigned long this_hwcap = 0; | |
559 | struct device_node *cpu_node; | |
560 | struct riscv_isainfo *isainfo = &hart_isa[cpu]; | |
561 | ||
562 | cpu_node = of_cpu_device_node_get(cpu); | |
563 | if (!cpu_node) { | |
564 | pr_warn("Unable to find cpu node\n"); | |
565 | continue; | |
566 | } | |
567 | ||
568 | if (!of_property_present(cpu_node, "riscv,isa-extensions")) { | |
569 | of_node_put(cpu_node); | |
570 | continue; | |
571 | } | |
572 | ||
573 | for (int i = 0; i < riscv_isa_ext_count; i++) { | |
0d8295ed EG |
574 | const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i]; |
575 | ||
90700a4f | 576 | if (of_property_match_string(cpu_node, "riscv,isa-extensions", |
0d8295ed | 577 | ext->property) < 0) |
90700a4f CD |
578 | continue; |
579 | ||
0d8295ed EG |
580 | if (ext->subset_ext_size) { |
581 | for (int j = 0; j < ext->subset_ext_size; j++) { | |
582 | if (riscv_isa_extension_check(ext->subset_ext_ids[i])) | |
583 | set_bit(ext->subset_ext_ids[j], isainfo->isa); | |
584 | } | |
585 | } | |
90700a4f | 586 | |
0d8295ed EG |
587 | if (riscv_isa_extension_check(ext->id)) { |
588 | set_bit(ext->id, isainfo->isa); | |
90700a4f | 589 | |
0d8295ed EG |
590 | /* Only single letter extensions get set in hwcap */ |
591 | if (strnlen(riscv_isa_ext[i].name, 2) == 1) | |
592 | this_hwcap |= isa2hwcap[riscv_isa_ext[i].id]; | |
593 | } | |
90700a4f CD |
594 | } |
595 | ||
596 | of_node_put(cpu_node); | |
597 | ||
598 | /* | |
599 | * All "okay" harts should have same isa. Set HWCAP based on | |
600 | * common capabilities of every "okay" hart, in case they don't. | |
601 | */ | |
602 | if (elf_hwcap) | |
603 | elf_hwcap &= this_hwcap; | |
604 | else | |
605 | elf_hwcap = this_hwcap; | |
606 | ||
607 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) | |
608 | bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); | |
609 | else | |
610 | bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); | |
611 | } | |
612 | ||
613 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) | |
614 | return -ENOENT; | |
615 | ||
616 | return 0; | |
617 | } | |
618 | ||
496ea826 CD |
619 | #ifdef CONFIG_RISCV_ISA_FALLBACK |
620 | bool __initdata riscv_isa_fallback = true; | |
621 | #else | |
622 | bool __initdata riscv_isa_fallback; | |
623 | static int __init riscv_isa_fallback_setup(char *__unused) | |
624 | { | |
625 | riscv_isa_fallback = true; | |
626 | return 1; | |
627 | } | |
628 | early_param("riscv_isa_fallback", riscv_isa_fallback_setup); | |
629 | #endif | |
630 | ||
4265b0ec CD |
631 | void __init riscv_fill_hwcap(void) |
632 | { | |
633 | char print_str[NUM_ALPHA_EXTS + 1]; | |
4265b0ec | 634 | unsigned long isa2hwcap[26] = {0}; |
90700a4f | 635 | int i, j; |
4265b0ec CD |
636 | |
637 | isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; | |
638 | isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; | |
639 | isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; | |
640 | isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; | |
641 | isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; | |
642 | isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; | |
643 | isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V; | |
644 | ||
90700a4f CD |
645 | if (!acpi_disabled) { |
646 | riscv_fill_hwcap_from_isa_string(isa2hwcap); | |
647 | } else { | |
648 | int ret = riscv_fill_hwcap_from_ext_list(isa2hwcap); | |
396c0183 | 649 | |
496ea826 | 650 | if (ret && riscv_isa_fallback) { |
90700a4f CD |
651 | pr_info("Falling back to deprecated \"riscv,isa\"\n"); |
652 | riscv_fill_hwcap_from_isa_string(isa2hwcap); | |
653 | } | |
654 | } | |
396c0183 | 655 | |
90700a4f CD |
656 | /* |
657 | * We don't support systems with F but without D, so mask those out | |
658 | * here. | |
659 | */ | |
86e581e3 | 660 | if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { |
7265d103 | 661 | pr_info("This kernel does not support systems with F but not D\n"); |
86e581e3 PD |
662 | elf_hwcap &= ~COMPAT_HWCAP_ISA_F; |
663 | } | |
664 | ||
dc6667a4 | 665 | if (elf_hwcap & COMPAT_HWCAP_ISA_V) { |
7017858e | 666 | riscv_v_setup_vsize(); |
dc6667a4 GR |
667 | /* |
668 | * ISA string in device tree might have 'v' flag, but | |
669 | * CONFIG_RISCV_ISA_V is disabled in kernel. | |
670 | * Clear V flag in elf_hwcap if CONFIG_RISCV_ISA_V is disabled. | |
671 | */ | |
672 | if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) | |
673 | elf_hwcap &= ~COMPAT_HWCAP_ISA_V; | |
674 | } | |
675 | ||
6bcff515 | 676 | memset(print_str, 0, sizeof(print_str)); |
58004f26 | 677 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
678 | if (riscv_isa[0] & BIT_MASK(i)) |
679 | print_str[j++] = (char)('a' + i); | |
02d52fbd | 680 | pr_info("riscv: base ISA extensions %s\n", print_str); |
6bcff515 AP |
681 | |
682 | memset(print_str, 0, sizeof(print_str)); | |
58004f26 | 683 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
684 | if (elf_hwcap & BIT_MASK(i)) |
685 | print_str[j++] = (char)('a' + i); | |
686 | pr_info("riscv: ELF capabilities %s\n", print_str); | |
e2c0cdfb | 687 | } |
ff689fd2 | 688 | |
50724efc AC |
689 | unsigned long riscv_get_elf_hwcap(void) |
690 | { | |
1fd96a3e AC |
691 | unsigned long hwcap; |
692 | ||
693 | hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)); | |
694 | ||
695 | if (!riscv_v_vstate_ctrl_user_allowed()) | |
696 | hwcap &= ~COMPAT_HWCAP_ISA_V; | |
697 | ||
698 | return hwcap; | |
50724efc AC |
699 | } |
700 | ||
55e0bf49 | 701 | static int check_unaligned_access(void *param) |
584ea656 | 702 | { |
55e0bf49 | 703 | int cpu = smp_processor_id(); |
584ea656 EG |
704 | u64 start_cycles, end_cycles; |
705 | u64 word_cycles; | |
706 | u64 byte_cycles; | |
707 | int ratio; | |
708 | unsigned long start_jiffies, now; | |
55e0bf49 | 709 | struct page *page = param; |
584ea656 EG |
710 | void *dst; |
711 | void *src; | |
712 | long speed = RISCV_HWPROBE_MISALIGNED_SLOW; | |
713 | ||
71c54b3d | 714 | if (check_unaligned_access_emulated(cpu)) |
55e0bf49 | 715 | return 0; |
584ea656 EG |
716 | |
717 | /* Make an unaligned destination buffer. */ | |
718 | dst = (void *)((unsigned long)page_address(page) | 0x1); | |
719 | /* Unalign src as well, but differently (off by 1 + 2 = 3). */ | |
720 | src = dst + (MISALIGNED_BUFFER_SIZE / 2); | |
721 | src += 2; | |
722 | word_cycles = -1ULL; | |
723 | /* Do a warmup. */ | |
724 | __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); | |
725 | preempt_disable(); | |
726 | start_jiffies = jiffies; | |
727 | while ((now = jiffies) == start_jiffies) | |
728 | cpu_relax(); | |
729 | ||
730 | /* | |
731 | * For a fixed amount of time, repeatedly try the function, and take | |
732 | * the best time in cycles as the measurement. | |
733 | */ | |
734 | while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { | |
735 | start_cycles = get_cycles64(); | |
736 | /* Ensure the CSR read can't reorder WRT to the copy. */ | |
737 | mb(); | |
738 | __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); | |
739 | /* Ensure the copy ends before the end time is snapped. */ | |
740 | mb(); | |
741 | end_cycles = get_cycles64(); | |
742 | if ((end_cycles - start_cycles) < word_cycles) | |
743 | word_cycles = end_cycles - start_cycles; | |
744 | } | |
745 | ||
746 | byte_cycles = -1ULL; | |
747 | __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); | |
748 | start_jiffies = jiffies; | |
749 | while ((now = jiffies) == start_jiffies) | |
750 | cpu_relax(); | |
751 | ||
752 | while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { | |
753 | start_cycles = get_cycles64(); | |
754 | mb(); | |
755 | __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); | |
756 | mb(); | |
757 | end_cycles = get_cycles64(); | |
758 | if ((end_cycles - start_cycles) < byte_cycles) | |
759 | byte_cycles = end_cycles - start_cycles; | |
760 | } | |
761 | ||
762 | preempt_enable(); | |
763 | ||
764 | /* Don't divide by zero. */ | |
765 | if (!word_cycles || !byte_cycles) { | |
766 | pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned access speed\n", | |
767 | cpu); | |
768 | ||
55e0bf49 | 769 | return 0; |
584ea656 EG |
770 | } |
771 | ||
772 | if (word_cycles < byte_cycles) | |
773 | speed = RISCV_HWPROBE_MISALIGNED_FAST; | |
774 | ||
775 | ratio = div_u64((byte_cycles * 100), word_cycles); | |
776 | pr_info("cpu%d: Ratio of byte access time to unaligned word access is %d.%02d, unaligned accesses are %s\n", | |
777 | cpu, | |
778 | ratio / 100, | |
779 | ratio % 100, | |
780 | (speed == RISCV_HWPROBE_MISALIGNED_FAST) ? "fast" : "slow"); | |
781 | ||
782 | per_cpu(misaligned_access_speed, cpu) = speed; | |
55e0bf49 EG |
783 | return 0; |
784 | } | |
584ea656 | 785 | |
55e0bf49 EG |
786 | static void check_unaligned_access_nonboot_cpu(void *param) |
787 | { | |
788 | unsigned int cpu = smp_processor_id(); | |
789 | struct page **pages = param; | |
790 | ||
791 | if (smp_processor_id() != 0) | |
792 | check_unaligned_access(pages[cpu]); | |
584ea656 EG |
793 | } |
794 | ||
55e0bf49 | 795 | static int riscv_online_cpu(unsigned int cpu) |
584ea656 | 796 | { |
55e0bf49 EG |
797 | static struct page *buf; |
798 | ||
799 | /* We are already set since the last check */ | |
800 | if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_UNKNOWN) | |
801 | return 0; | |
802 | ||
803 | buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); | |
804 | if (!buf) { | |
805 | pr_warn("Allocation failure, not measuring misaligned performance\n"); | |
806 | return -ENOMEM; | |
807 | } | |
808 | ||
809 | check_unaligned_access(buf); | |
810 | __free_pages(buf, MISALIGNED_BUFFER_ORDER); | |
811 | return 0; | |
812 | } | |
813 | ||
814 | /* Measure unaligned access on all CPUs present at boot in parallel. */ | |
815 | static int check_unaligned_access_all_cpus(void) | |
816 | { | |
817 | unsigned int cpu; | |
818 | unsigned int cpu_count = num_possible_cpus(); | |
819 | struct page **bufs = kzalloc(cpu_count * sizeof(struct page *), | |
820 | GFP_KERNEL); | |
821 | ||
822 | if (!bufs) { | |
823 | pr_warn("Allocation failure, not measuring misaligned performance\n"); | |
824 | return 0; | |
825 | } | |
826 | ||
827 | /* | |
828 | * Allocate separate buffers for each CPU so there's no fighting over | |
829 | * cache lines. | |
830 | */ | |
831 | for_each_cpu(cpu, cpu_online_mask) { | |
832 | bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); | |
833 | if (!bufs[cpu]) { | |
834 | pr_warn("Allocation failure, not measuring misaligned performance\n"); | |
835 | goto out; | |
836 | } | |
837 | } | |
838 | ||
839 | /* Check everybody except 0, who stays behind to tend jiffies. */ | |
840 | on_each_cpu(check_unaligned_access_nonboot_cpu, bufs, 1); | |
841 | ||
842 | /* Check core 0. */ | |
843 | smp_call_on_cpu(0, check_unaligned_access, bufs[0], true); | |
844 | ||
845 | /* Setup hotplug callback for any new CPUs that come online. */ | |
846 | cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", | |
847 | riscv_online_cpu, NULL); | |
848 | ||
849 | out: | |
71c54b3d | 850 | unaligned_emulation_finish(); |
55e0bf49 EG |
851 | for_each_cpu(cpu, cpu_online_mask) { |
852 | if (bufs[cpu]) | |
853 | __free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER); | |
854 | } | |
855 | ||
856 | kfree(bufs); | |
584ea656 EG |
857 | return 0; |
858 | } | |
859 | ||
55e0bf49 | 860 | arch_initcall(check_unaligned_access_all_cpus); |
584ea656 | 861 | |
43c16d51 AJ |
862 | void riscv_user_isa_enable(void) |
863 | { | |
864 | if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_ZICBOZ)) | |
865 | csr_set(CSR_SENVCFG, ENVCFG_CBZE); | |
866 | } | |
867 | ||
ff689fd2 | 868 | #ifdef CONFIG_RISCV_ALTERNATIVE |
d25f2563 AJ |
869 | /* |
870 | * Alternative patch sites consider 48 bits when determining when to patch | |
871 | * the old instruction sequence with the new. These bits are broken into a | |
872 | * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the | |
873 | * patch site is for an erratum, identified by the 32-bit patch ID. When | |
874 | * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures | |
875 | * further break down patch ID into two 16-bit numbers. The lower 16 bits | |
876 | * are the cpufeature ID and the upper 16 bits are used for a value specific | |
877 | * to the cpufeature and patch site. If the upper 16 bits are zero, then it | |
878 | * implies no specific value is specified. cpufeatures that want to control | |
879 | * patching on a per-site basis will provide non-zero values and implement | |
880 | * checks here. The checks return true when patching should be done, and | |
881 | * false otherwise. | |
882 | */ | |
883 | static bool riscv_cpufeature_patch_check(u16 id, u16 value) | |
884 | { | |
885 | if (!value) | |
886 | return true; | |
887 | ||
ab0f7746 AJ |
888 | switch (id) { |
889 | case RISCV_ISA_EXT_ZICBOZ: | |
890 | /* | |
891 | * Zicboz alternative applications provide the maximum | |
892 | * supported block size order, or zero when it doesn't | |
893 | * matter. If the current block size exceeds the maximum, | |
894 | * then the alternative cannot be applied. | |
895 | */ | |
896 | return riscv_cboz_block_size <= (1U << value); | |
897 | } | |
898 | ||
d25f2563 AJ |
899 | return false; |
900 | } | |
901 | ||
ff689fd2 HS |
902 | void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, |
903 | struct alt_entry *end, | |
904 | unsigned int stage) | |
905 | { | |
ff689fd2 | 906 | struct alt_entry *alt; |
8d23e94a | 907 | void *oldptr, *altptr; |
d25f2563 | 908 | u16 id, value; |
ff689fd2 | 909 | |
191b27c7 JZ |
910 | if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) |
911 | return; | |
ff689fd2 HS |
912 | |
913 | for (alt = begin; alt < end; alt++) { | |
914 | if (alt->vendor_id != 0) | |
915 | continue; | |
d25f2563 AJ |
916 | |
917 | id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); | |
918 | ||
919 | if (id >= RISCV_ISA_EXT_MAX) { | |
920 | WARN(1, "This extension id:%d is not in ISA extension list", id); | |
ff689fd2 HS |
921 | continue; |
922 | } | |
923 | ||
d25f2563 AJ |
924 | if (!__riscv_isa_extension_available(NULL, id)) |
925 | continue; | |
926 | ||
927 | value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); | |
928 | if (!riscv_cpufeature_patch_check(id, value)) | |
4bf88607 JZ |
929 | continue; |
930 | ||
8d23e94a JZ |
931 | oldptr = ALT_OLD_PTR(alt); |
932 | altptr = ALT_ALT_PTR(alt); | |
9493e6f3 CD |
933 | |
934 | mutex_lock(&text_mutex); | |
8d23e94a JZ |
935 | patch_text_nosync(oldptr, altptr, alt->alt_len); |
936 | riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); | |
9493e6f3 | 937 | mutex_unlock(&text_mutex); |
ff689fd2 HS |
938 | } |
939 | } | |
940 | #endif |