Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
e2c0cdfb PD |
2 | /* |
3 | * Copied from arch/arm64/kernel/cpufeature.c | |
4 | * | |
5 | * Copyright (C) 2015 ARM Ltd. | |
6 | * Copyright (C) 2017 SiFive | |
e2c0cdfb PD |
7 | */ |
8 | ||
396c0183 | 9 | #include <linux/acpi.h> |
6bcff515 | 10 | #include <linux/bitmap.h> |
2a31c54b | 11 | #include <linux/ctype.h> |
9daaca4a | 12 | #include <linux/log2.h> |
9493e6f3 | 13 | #include <linux/memory.h> |
ff689fd2 | 14 | #include <linux/module.h> |
e2c0cdfb | 15 | #include <linux/of.h> |
396c0183 | 16 | #include <asm/acpi.h> |
ff689fd2 | 17 | #include <asm/alternative.h> |
1631ba12 | 18 | #include <asm/cacheflush.h> |
c2d3c844 | 19 | #include <asm/cpufeature.h> |
e2c0cdfb | 20 | #include <asm/hwcap.h> |
584ea656 | 21 | #include <asm/hwprobe.h> |
ff689fd2 | 22 | #include <asm/patch.h> |
ff689fd2 | 23 | #include <asm/processor.h> |
7017858e | 24 | #include <asm/vector.h> |
e2c0cdfb | 25 | |
584ea656 EG |
26 | #include "copy-unaligned.h" |
27 | ||
58004f26 TO |
28 | #define NUM_ALPHA_EXTS ('z' - 'a' + 1) |
29 | ||
584ea656 EG |
30 | #define MISALIGNED_ACCESS_JIFFIES_LG2 1 |
31 | #define MISALIGNED_BUFFER_SIZE 0x4000 | |
32 | #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80) | |
33 | ||
e2c0cdfb | 34 | unsigned long elf_hwcap __read_mostly; |
6bcff515 AP |
35 | |
36 | /* Host ISA bitmap */ | |
37 | static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; | |
38 | ||
82e9c66e EG |
39 | /* Per-cpu ISA extensions. */ |
40 | struct riscv_isainfo hart_isa[NR_CPUS]; | |
41 | ||
62a31d6e EG |
42 | /* Performance information */ |
43 | DEFINE_PER_CPU(long, misaligned_access_speed); | |
44 | ||
6bcff515 AP |
45 | /** |
46 | * riscv_isa_extension_base() - Get base extension word | |
47 | * | |
48 | * @isa_bitmap: ISA bitmap to use | |
49 | * Return: base extension word as unsigned long value | |
50 | * | |
51 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
52 | */ | |
53 | unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap) | |
54 | { | |
55 | if (!isa_bitmap) | |
56 | return riscv_isa[0]; | |
57 | return isa_bitmap[0]; | |
58 | } | |
59 | EXPORT_SYMBOL_GPL(riscv_isa_extension_base); | |
60 | ||
61 | /** | |
62 | * __riscv_isa_extension_available() - Check whether given extension | |
63 | * is available or not | |
64 | * | |
65 | * @isa_bitmap: ISA bitmap to use | |
66 | * @bit: bit position of the desired extension | |
67 | * Return: true or false | |
68 | * | |
69 | * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used. | |
70 | */ | |
71 | bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit) | |
72 | { | |
73 | const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa; | |
74 | ||
75 | if (bit >= RISCV_ISA_EXT_MAX) | |
76 | return false; | |
77 | ||
78 | return test_bit(bit, bmap) ? true : false; | |
79 | } | |
80 | EXPORT_SYMBOL_GPL(__riscv_isa_extension_available); | |
81 | ||
fb0ff0a9 AJ |
82 | static bool riscv_isa_extension_check(int id) |
83 | { | |
9daaca4a AJ |
84 | switch (id) { |
85 | case RISCV_ISA_EXT_ZICBOM: | |
86 | if (!riscv_cbom_block_size) { | |
c818fea8 | 87 | pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n"); |
9daaca4a AJ |
88 | return false; |
89 | } else if (!is_power_of_2(riscv_cbom_block_size)) { | |
c818fea8 | 90 | pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n"); |
9daaca4a AJ |
91 | return false; |
92 | } | |
93 | return true; | |
7ea5a736 AJ |
94 | case RISCV_ISA_EXT_ZICBOZ: |
95 | if (!riscv_cboz_block_size) { | |
181f2a28 | 96 | pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n"); |
7ea5a736 AJ |
97 | return false; |
98 | } else if (!is_power_of_2(riscv_cboz_block_size)) { | |
181f2a28 | 99 | pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n"); |
7ea5a736 AJ |
100 | return false; |
101 | } | |
102 | return true; | |
9daaca4a AJ |
103 | } |
104 | ||
fb0ff0a9 AJ |
105 | return true; |
106 | } | |
107 | ||
37f988dc CD |
108 | #define __RISCV_ISA_EXT_DATA(_name, _id) { \ |
109 | .name = #_name, \ | |
90700a4f | 110 | .property = #_name, \ |
37f988dc CD |
111 | .id = _id, \ |
112 | } | |
8135ade3 CD |
113 | |
114 | /* | |
115 | * The canonical order of ISA extension names in the ISA string is defined in | |
116 | * chapter 27 of the unprivileged specification. | |
117 | * | |
118 | * Ordinarily, for in-kernel data structures, this order is unimportant but | |
119 | * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. | |
120 | * | |
121 | * The specification uses vague wording, such as should, when it comes to | |
122 | * ordering, so for our purposes the following rules apply: | |
123 | * | |
124 | * 1. All multi-letter extensions must be separated from other extensions by an | |
125 | * underscore. | |
126 | * | |
127 | * 2. Additional standard extensions (starting with 'Z') must be sorted after | |
128 | * single-letter extensions and before any higher-privileged extensions. | |
129 | * | |
130 | * 3. The first letter following the 'Z' conventionally indicates the most | |
131 | * closely related alphabetical extension category, IMAFDQLCBKJTPVH. | |
132 | * If multiple 'Z' extensions are named, they must be ordered first by | |
133 | * category, then alphabetically within a category. | |
134 | * | |
135 | * 3. Standard supervisor-level extensions (starting with 'S') must be listed | |
136 | * after standard unprivileged extensions. If multiple supervisor-level | |
137 | * extensions are listed, they must be ordered alphabetically. | |
138 | * | |
139 | * 4. Standard machine-level extensions (starting with 'Zxm') must be listed | |
140 | * after any lower-privileged, standard extensions. If multiple | |
141 | * machine-level extensions are listed, they must be ordered | |
142 | * alphabetically. | |
143 | * | |
144 | * 5. Non-standard extensions (starting with 'X') must be listed after all | |
145 | * standard extensions. If multiple non-standard extensions are listed, they | |
146 | * must be ordered alphabetically. | |
147 | * | |
148 | * An example string following the order is: | |
149 | * rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux | |
150 | * | |
151 | * New entries to this struct should follow the ordering rules described above. | |
152 | */ | |
153 | const struct riscv_isa_ext_data riscv_isa_ext[] = { | |
effc122a CD |
154 | __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_i), |
155 | __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_m), | |
156 | __RISCV_ISA_EXT_DATA(a, RISCV_ISA_EXT_a), | |
157 | __RISCV_ISA_EXT_DATA(f, RISCV_ISA_EXT_f), | |
158 | __RISCV_ISA_EXT_DATA(d, RISCV_ISA_EXT_d), | |
159 | __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_q), | |
160 | __RISCV_ISA_EXT_DATA(c, RISCV_ISA_EXT_c), | |
161 | __RISCV_ISA_EXT_DATA(b, RISCV_ISA_EXT_b), | |
162 | __RISCV_ISA_EXT_DATA(k, RISCV_ISA_EXT_k), | |
163 | __RISCV_ISA_EXT_DATA(j, RISCV_ISA_EXT_j), | |
164 | __RISCV_ISA_EXT_DATA(p, RISCV_ISA_EXT_p), | |
165 | __RISCV_ISA_EXT_DATA(v, RISCV_ISA_EXT_v), | |
166 | __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), | |
8135ade3 CD |
167 | __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), |
168 | __RISCV_ISA_EXT_DATA(zicboz, RISCV_ISA_EXT_ZICBOZ), | |
169 | __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), | |
170 | __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), | |
171 | __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI), | |
172 | __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), | |
173 | __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM), | |
174 | __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA), | |
175 | __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB), | |
176 | __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS), | |
177 | __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA), | |
178 | __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA), | |
179 | __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), | |
180 | __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), | |
181 | __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), | |
182 | __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT), | |
183 | __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), | |
184 | }; | |
185 | ||
186 | const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); | |
187 | ||
4265b0ec CD |
188 | static void __init riscv_parse_isa_string(unsigned long *this_hwcap, struct riscv_isainfo *isainfo, |
189 | unsigned long *isa2hwcap, const char *isa) | |
190 | { | |
191 | /* | |
192 | * For all possible cpus, we have already validated in | |
193 | * the boot process that they at least contain "rv" and | |
194 | * whichever of "32"/"64" this kernel supports, and so this | |
195 | * section can be skipped. | |
196 | */ | |
197 | isa += 4; | |
198 | ||
199 | while (*isa) { | |
200 | const char *ext = isa++; | |
201 | const char *ext_end = isa; | |
202 | bool ext_long = false, ext_err = false; | |
203 | ||
204 | switch (*ext) { | |
205 | case 's': | |
206 | /* | |
8f501be8 | 207 | * Workaround for invalid single-letter 's' & 'u' (QEMU). |
4265b0ec | 208 | * No need to set the bit in riscv_isa as 's' & 'u' are |
8f501be8 TO |
209 | * not valid ISA extensions. It works unless the first |
210 | * multi-letter extension in the ISA string begins with | |
211 | * "Su" and is not prefixed with an underscore. | |
4265b0ec CD |
212 | */ |
213 | if (ext[-1] != '_' && ext[1] == 'u') { | |
214 | ++isa; | |
215 | ext_err = true; | |
216 | break; | |
217 | } | |
218 | fallthrough; | |
219 | case 'S': | |
220 | case 'x': | |
221 | case 'X': | |
222 | case 'z': | |
223 | case 'Z': | |
224 | /* | |
225 | * Before attempting to parse the extension itself, we find its end. | |
226 | * As multi-letter extensions must be split from other multi-letter | |
227 | * extensions with an "_", the end of a multi-letter extension will | |
228 | * either be the null character or the "_" at the start of the next | |
229 | * multi-letter extension. | |
230 | * | |
231 | * Next, as the extensions version is currently ignored, we | |
232 | * eliminate that portion. This is done by parsing backwards from | |
233 | * the end of the extension, removing any numbers. This may be a | |
234 | * major or minor number however, so the process is repeated if a | |
235 | * minor number was found. | |
236 | * | |
237 | * ext_end is intended to represent the first character *after* the | |
238 | * name portion of an extension, but will be decremented to the last | |
239 | * character itself while eliminating the extensions version number. | |
240 | * A simple re-increment solves this problem. | |
241 | */ | |
242 | ext_long = true; | |
243 | for (; *isa && *isa != '_'; ++isa) | |
244 | if (unlikely(!isalnum(*isa))) | |
245 | ext_err = true; | |
246 | ||
247 | ext_end = isa; | |
248 | if (unlikely(ext_err)) | |
249 | break; | |
250 | ||
251 | if (!isdigit(ext_end[-1])) | |
252 | break; | |
253 | ||
254 | while (isdigit(*--ext_end)) | |
255 | ; | |
256 | ||
257 | if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) { | |
258 | ++ext_end; | |
259 | break; | |
260 | } | |
261 | ||
262 | while (isdigit(*--ext_end)) | |
263 | ; | |
264 | ||
265 | ++ext_end; | |
266 | break; | |
267 | default: | |
268 | /* | |
269 | * Things are a little easier for single-letter extensions, as they | |
270 | * are parsed forwards. | |
271 | * | |
272 | * After checking that our starting position is valid, we need to | |
273 | * ensure that, when isa was incremented at the start of the loop, | |
274 | * that it arrived at the start of the next extension. | |
275 | * | |
276 | * If we are already on a non-digit, there is nothing to do. Either | |
277 | * we have a multi-letter extension's _, or the start of an | |
278 | * extension. | |
279 | * | |
280 | * Otherwise we have found the current extension's major version | |
281 | * number. Parse past it, and a subsequent p/minor version number | |
282 | * if present. The `p` extension must not appear immediately after | |
283 | * a number, so there is no fear of missing it. | |
284 | * | |
285 | */ | |
286 | if (unlikely(!isalpha(*ext))) { | |
287 | ext_err = true; | |
288 | break; | |
289 | } | |
290 | ||
291 | if (!isdigit(*isa)) | |
292 | break; | |
293 | ||
294 | while (isdigit(*++isa)) | |
295 | ; | |
296 | ||
297 | if (tolower(*isa) != 'p') | |
298 | break; | |
299 | ||
300 | if (!isdigit(*++isa)) { | |
301 | --isa; | |
302 | break; | |
303 | } | |
304 | ||
305 | while (isdigit(*++isa)) | |
306 | ; | |
307 | ||
308 | break; | |
309 | } | |
310 | ||
311 | /* | |
312 | * The parser expects that at the start of an iteration isa points to the | |
313 | * first character of the next extension. As we stop parsing an extension | |
314 | * on meeting a non-alphanumeric character, an extra increment is needed | |
315 | * where the succeeding extension is a multi-letter prefixed with an "_". | |
316 | */ | |
317 | if (*isa == '_') | |
318 | ++isa; | |
319 | ||
320 | #define SET_ISA_EXT_MAP(name, bit) \ | |
321 | do { \ | |
322 | if ((ext_end - ext == strlen(name)) && \ | |
323 | !strncasecmp(ext, name, strlen(name)) && \ | |
324 | riscv_isa_extension_check(bit)) \ | |
325 | set_bit(bit, isainfo->isa); \ | |
326 | } while (false) \ | |
327 | ||
328 | if (unlikely(ext_err)) | |
329 | continue; | |
330 | if (!ext_long) { | |
331 | int nr = tolower(*ext) - 'a'; | |
332 | ||
333 | if (riscv_isa_extension_check(nr)) { | |
334 | *this_hwcap |= isa2hwcap[nr]; | |
335 | set_bit(nr, isainfo->isa); | |
336 | } | |
337 | } else { | |
338 | for (int i = 0; i < riscv_isa_ext_count; i++) | |
339 | SET_ISA_EXT_MAP(riscv_isa_ext[i].name, | |
340 | riscv_isa_ext[i].id); | |
341 | } | |
342 | #undef SET_ISA_EXT_MAP | |
343 | } | |
344 | } | |
345 | ||
346 | static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) | |
e2c0cdfb | 347 | { |
dd81c8ab | 348 | struct device_node *node; |
e2c0cdfb | 349 | const char *isa; |
4265b0ec | 350 | int rc; |
396c0183 S |
351 | struct acpi_table_header *rhct; |
352 | acpi_status status; | |
914d6f44 | 353 | unsigned int cpu; |
e2c0cdfb | 354 | |
396c0183 S |
355 | if (!acpi_disabled) { |
356 | status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); | |
357 | if (ACPI_FAILURE(status)) | |
358 | return; | |
359 | } | |
360 | ||
914d6f44 | 361 | for_each_possible_cpu(cpu) { |
16252e01 | 362 | struct riscv_isainfo *isainfo = &hart_isa[cpu]; |
fbdc6193 | 363 | unsigned long this_hwcap = 0; |
e2c0cdfb | 364 | |
396c0183 S |
365 | if (acpi_disabled) { |
366 | node = of_cpu_device_node_get(cpu); | |
367 | if (!node) { | |
368 | pr_warn("Unable to find cpu node\n"); | |
369 | continue; | |
370 | } | |
e2c0cdfb | 371 | |
396c0183 S |
372 | rc = of_property_read_string(node, "riscv,isa", &isa); |
373 | of_node_put(node); | |
374 | if (rc) { | |
375 | pr_warn("Unable to find \"riscv,isa\" devicetree entry\n"); | |
376 | continue; | |
377 | } | |
378 | } else { | |
379 | rc = acpi_get_riscv_isa(rhct, cpu, &isa); | |
380 | if (rc < 0) { | |
381 | pr_warn("Unable to get ISA for the hart - %d\n", cpu); | |
382 | continue; | |
383 | } | |
fbdc6193 AP |
384 | } |
385 | ||
4265b0ec | 386 | riscv_parse_isa_string(&this_hwcap, isainfo, isa2hwcap, isa); |
fbdc6193 | 387 | |
07edc327 CD |
388 | /* |
389 | * These ones were as they were part of the base ISA when the | |
390 | * port & dt-bindings were upstreamed, and so can be set | |
391 | * unconditionally where `i` is in riscv,isa on DT systems. | |
392 | */ | |
393 | if (acpi_disabled) { | |
ab2dbc7a PD |
394 | set_bit(RISCV_ISA_EXT_ZICSR, isainfo->isa); |
395 | set_bit(RISCV_ISA_EXT_ZIFENCEI, isainfo->isa); | |
42b89447 PD |
396 | set_bit(RISCV_ISA_EXT_ZICNTR, isainfo->isa); |
397 | set_bit(RISCV_ISA_EXT_ZIHPM, isainfo->isa); | |
07edc327 CD |
398 | } |
399 | ||
fbdc6193 AP |
400 | /* |
401 | * All "okay" hart should have same isa. Set HWCAP based on | |
402 | * common capabilities of every "okay" hart, in case they don't | |
403 | * have. | |
404 | */ | |
405 | if (elf_hwcap) | |
406 | elf_hwcap &= this_hwcap; | |
407 | else | |
408 | elf_hwcap = this_hwcap; | |
6bcff515 | 409 | |
8f51558e | 410 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) |
82e9c66e | 411 | bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); |
8f51558e | 412 | else |
82e9c66e | 413 | bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); |
fbdc6193 | 414 | } |
e2c0cdfb | 415 | |
396c0183 S |
416 | if (!acpi_disabled && rhct) |
417 | acpi_put_table((struct acpi_table_header *)rhct); | |
4265b0ec CD |
418 | } |
419 | ||
90700a4f CD |
420 | static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) |
421 | { | |
422 | unsigned int cpu; | |
423 | ||
424 | for_each_possible_cpu(cpu) { | |
425 | unsigned long this_hwcap = 0; | |
426 | struct device_node *cpu_node; | |
427 | struct riscv_isainfo *isainfo = &hart_isa[cpu]; | |
428 | ||
429 | cpu_node = of_cpu_device_node_get(cpu); | |
430 | if (!cpu_node) { | |
431 | pr_warn("Unable to find cpu node\n"); | |
432 | continue; | |
433 | } | |
434 | ||
435 | if (!of_property_present(cpu_node, "riscv,isa-extensions")) { | |
436 | of_node_put(cpu_node); | |
437 | continue; | |
438 | } | |
439 | ||
440 | for (int i = 0; i < riscv_isa_ext_count; i++) { | |
441 | if (of_property_match_string(cpu_node, "riscv,isa-extensions", | |
442 | riscv_isa_ext[i].property) < 0) | |
443 | continue; | |
444 | ||
445 | if (!riscv_isa_extension_check(riscv_isa_ext[i].id)) | |
446 | continue; | |
447 | ||
448 | /* Only single letter extensions get set in hwcap */ | |
449 | if (strnlen(riscv_isa_ext[i].name, 2) == 1) | |
450 | this_hwcap |= isa2hwcap[riscv_isa_ext[i].id]; | |
451 | ||
452 | set_bit(riscv_isa_ext[i].id, isainfo->isa); | |
453 | } | |
454 | ||
455 | of_node_put(cpu_node); | |
456 | ||
457 | /* | |
458 | * All "okay" harts should have same isa. Set HWCAP based on | |
459 | * common capabilities of every "okay" hart, in case they don't. | |
460 | */ | |
461 | if (elf_hwcap) | |
462 | elf_hwcap &= this_hwcap; | |
463 | else | |
464 | elf_hwcap = this_hwcap; | |
465 | ||
466 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) | |
467 | bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); | |
468 | else | |
469 | bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX); | |
470 | } | |
471 | ||
472 | if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX)) | |
473 | return -ENOENT; | |
474 | ||
475 | return 0; | |
476 | } | |
477 | ||
496ea826 CD |
478 | #ifdef CONFIG_RISCV_ISA_FALLBACK |
479 | bool __initdata riscv_isa_fallback = true; | |
480 | #else | |
481 | bool __initdata riscv_isa_fallback; | |
482 | static int __init riscv_isa_fallback_setup(char *__unused) | |
483 | { | |
484 | riscv_isa_fallback = true; | |
485 | return 1; | |
486 | } | |
487 | early_param("riscv_isa_fallback", riscv_isa_fallback_setup); | |
488 | #endif | |
489 | ||
4265b0ec CD |
490 | void __init riscv_fill_hwcap(void) |
491 | { | |
492 | char print_str[NUM_ALPHA_EXTS + 1]; | |
4265b0ec | 493 | unsigned long isa2hwcap[26] = {0}; |
90700a4f | 494 | int i, j; |
4265b0ec CD |
495 | |
496 | isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I; | |
497 | isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M; | |
498 | isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A; | |
499 | isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F; | |
500 | isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D; | |
501 | isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C; | |
502 | isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V; | |
503 | ||
90700a4f CD |
504 | if (!acpi_disabled) { |
505 | riscv_fill_hwcap_from_isa_string(isa2hwcap); | |
506 | } else { | |
507 | int ret = riscv_fill_hwcap_from_ext_list(isa2hwcap); | |
396c0183 | 508 | |
496ea826 | 509 | if (ret && riscv_isa_fallback) { |
90700a4f CD |
510 | pr_info("Falling back to deprecated \"riscv,isa\"\n"); |
511 | riscv_fill_hwcap_from_isa_string(isa2hwcap); | |
512 | } | |
513 | } | |
396c0183 | 514 | |
90700a4f CD |
515 | /* |
516 | * We don't support systems with F but without D, so mask those out | |
517 | * here. | |
518 | */ | |
86e581e3 | 519 | if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) { |
7265d103 | 520 | pr_info("This kernel does not support systems with F but not D\n"); |
86e581e3 PD |
521 | elf_hwcap &= ~COMPAT_HWCAP_ISA_F; |
522 | } | |
523 | ||
dc6667a4 | 524 | if (elf_hwcap & COMPAT_HWCAP_ISA_V) { |
7017858e | 525 | riscv_v_setup_vsize(); |
dc6667a4 GR |
526 | /* |
527 | * ISA string in device tree might have 'v' flag, but | |
528 | * CONFIG_RISCV_ISA_V is disabled in kernel. | |
529 | * Clear V flag in elf_hwcap if CONFIG_RISCV_ISA_V is disabled. | |
530 | */ | |
531 | if (!IS_ENABLED(CONFIG_RISCV_ISA_V)) | |
532 | elf_hwcap &= ~COMPAT_HWCAP_ISA_V; | |
533 | } | |
534 | ||
6bcff515 | 535 | memset(print_str, 0, sizeof(print_str)); |
58004f26 | 536 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
537 | if (riscv_isa[0] & BIT_MASK(i)) |
538 | print_str[j++] = (char)('a' + i); | |
02d52fbd | 539 | pr_info("riscv: base ISA extensions %s\n", print_str); |
6bcff515 AP |
540 | |
541 | memset(print_str, 0, sizeof(print_str)); | |
58004f26 | 542 | for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) |
6bcff515 AP |
543 | if (elf_hwcap & BIT_MASK(i)) |
544 | print_str[j++] = (char)('a' + i); | |
545 | pr_info("riscv: ELF capabilities %s\n", print_str); | |
e2c0cdfb | 546 | } |
ff689fd2 | 547 | |
50724efc AC |
548 | unsigned long riscv_get_elf_hwcap(void) |
549 | { | |
1fd96a3e AC |
550 | unsigned long hwcap; |
551 | ||
552 | hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1)); | |
553 | ||
554 | if (!riscv_v_vstate_ctrl_user_allowed()) | |
555 | hwcap &= ~COMPAT_HWCAP_ISA_V; | |
556 | ||
557 | return hwcap; | |
50724efc AC |
558 | } |
559 | ||
584ea656 EG |
560 | void check_unaligned_access(int cpu) |
561 | { | |
562 | u64 start_cycles, end_cycles; | |
563 | u64 word_cycles; | |
564 | u64 byte_cycles; | |
565 | int ratio; | |
566 | unsigned long start_jiffies, now; | |
567 | struct page *page; | |
568 | void *dst; | |
569 | void *src; | |
570 | long speed = RISCV_HWPROBE_MISALIGNED_SLOW; | |
571 | ||
572 | page = alloc_pages(GFP_NOWAIT, get_order(MISALIGNED_BUFFER_SIZE)); | |
573 | if (!page) { | |
574 | pr_warn("Can't alloc pages to measure memcpy performance"); | |
575 | return; | |
576 | } | |
577 | ||
578 | /* Make an unaligned destination buffer. */ | |
579 | dst = (void *)((unsigned long)page_address(page) | 0x1); | |
580 | /* Unalign src as well, but differently (off by 1 + 2 = 3). */ | |
581 | src = dst + (MISALIGNED_BUFFER_SIZE / 2); | |
582 | src += 2; | |
583 | word_cycles = -1ULL; | |
584 | /* Do a warmup. */ | |
585 | __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); | |
586 | preempt_disable(); | |
587 | start_jiffies = jiffies; | |
588 | while ((now = jiffies) == start_jiffies) | |
589 | cpu_relax(); | |
590 | ||
591 | /* | |
592 | * For a fixed amount of time, repeatedly try the function, and take | |
593 | * the best time in cycles as the measurement. | |
594 | */ | |
595 | while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { | |
596 | start_cycles = get_cycles64(); | |
597 | /* Ensure the CSR read can't reorder WRT to the copy. */ | |
598 | mb(); | |
599 | __riscv_copy_words_unaligned(dst, src, MISALIGNED_COPY_SIZE); | |
600 | /* Ensure the copy ends before the end time is snapped. */ | |
601 | mb(); | |
602 | end_cycles = get_cycles64(); | |
603 | if ((end_cycles - start_cycles) < word_cycles) | |
604 | word_cycles = end_cycles - start_cycles; | |
605 | } | |
606 | ||
607 | byte_cycles = -1ULL; | |
608 | __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); | |
609 | start_jiffies = jiffies; | |
610 | while ((now = jiffies) == start_jiffies) | |
611 | cpu_relax(); | |
612 | ||
613 | while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) { | |
614 | start_cycles = get_cycles64(); | |
615 | mb(); | |
616 | __riscv_copy_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE); | |
617 | mb(); | |
618 | end_cycles = get_cycles64(); | |
619 | if ((end_cycles - start_cycles) < byte_cycles) | |
620 | byte_cycles = end_cycles - start_cycles; | |
621 | } | |
622 | ||
623 | preempt_enable(); | |
624 | ||
625 | /* Don't divide by zero. */ | |
626 | if (!word_cycles || !byte_cycles) { | |
627 | pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned access speed\n", | |
628 | cpu); | |
629 | ||
630 | goto out; | |
631 | } | |
632 | ||
633 | if (word_cycles < byte_cycles) | |
634 | speed = RISCV_HWPROBE_MISALIGNED_FAST; | |
635 | ||
636 | ratio = div_u64((byte_cycles * 100), word_cycles); | |
637 | pr_info("cpu%d: Ratio of byte access time to unaligned word access is %d.%02d, unaligned accesses are %s\n", | |
638 | cpu, | |
639 | ratio / 100, | |
640 | ratio % 100, | |
641 | (speed == RISCV_HWPROBE_MISALIGNED_FAST) ? "fast" : "slow"); | |
642 | ||
643 | per_cpu(misaligned_access_speed, cpu) = speed; | |
644 | ||
645 | out: | |
646 | __free_pages(page, get_order(MISALIGNED_BUFFER_SIZE)); | |
647 | } | |
648 | ||
649 | static int check_unaligned_access_boot_cpu(void) | |
650 | { | |
651 | check_unaligned_access(0); | |
652 | return 0; | |
653 | } | |
654 | ||
655 | arch_initcall(check_unaligned_access_boot_cpu); | |
656 | ||
43c16d51 AJ |
657 | void riscv_user_isa_enable(void) |
658 | { | |
659 | if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_ZICBOZ)) | |
660 | csr_set(CSR_SENVCFG, ENVCFG_CBZE); | |
661 | } | |
662 | ||
ff689fd2 | 663 | #ifdef CONFIG_RISCV_ALTERNATIVE |
d25f2563 AJ |
664 | /* |
665 | * Alternative patch sites consider 48 bits when determining when to patch | |
666 | * the old instruction sequence with the new. These bits are broken into a | |
667 | * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the | |
668 | * patch site is for an erratum, identified by the 32-bit patch ID. When | |
669 | * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures | |
670 | * further break down patch ID into two 16-bit numbers. The lower 16 bits | |
671 | * are the cpufeature ID and the upper 16 bits are used for a value specific | |
672 | * to the cpufeature and patch site. If the upper 16 bits are zero, then it | |
673 | * implies no specific value is specified. cpufeatures that want to control | |
674 | * patching on a per-site basis will provide non-zero values and implement | |
675 | * checks here. The checks return true when patching should be done, and | |
676 | * false otherwise. | |
677 | */ | |
678 | static bool riscv_cpufeature_patch_check(u16 id, u16 value) | |
679 | { | |
680 | if (!value) | |
681 | return true; | |
682 | ||
ab0f7746 AJ |
683 | switch (id) { |
684 | case RISCV_ISA_EXT_ZICBOZ: | |
685 | /* | |
686 | * Zicboz alternative applications provide the maximum | |
687 | * supported block size order, or zero when it doesn't | |
688 | * matter. If the current block size exceeds the maximum, | |
689 | * then the alternative cannot be applied. | |
690 | */ | |
691 | return riscv_cboz_block_size <= (1U << value); | |
692 | } | |
693 | ||
d25f2563 AJ |
694 | return false; |
695 | } | |
696 | ||
ff689fd2 HS |
697 | void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, |
698 | struct alt_entry *end, | |
699 | unsigned int stage) | |
700 | { | |
ff689fd2 | 701 | struct alt_entry *alt; |
8d23e94a | 702 | void *oldptr, *altptr; |
d25f2563 | 703 | u16 id, value; |
ff689fd2 | 704 | |
191b27c7 JZ |
705 | if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) |
706 | return; | |
ff689fd2 HS |
707 | |
708 | for (alt = begin; alt < end; alt++) { | |
709 | if (alt->vendor_id != 0) | |
710 | continue; | |
d25f2563 AJ |
711 | |
712 | id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); | |
713 | ||
714 | if (id >= RISCV_ISA_EXT_MAX) { | |
715 | WARN(1, "This extension id:%d is not in ISA extension list", id); | |
ff689fd2 HS |
716 | continue; |
717 | } | |
718 | ||
d25f2563 AJ |
719 | if (!__riscv_isa_extension_available(NULL, id)) |
720 | continue; | |
721 | ||
722 | value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id); | |
723 | if (!riscv_cpufeature_patch_check(id, value)) | |
4bf88607 JZ |
724 | continue; |
725 | ||
8d23e94a JZ |
726 | oldptr = ALT_OLD_PTR(alt); |
727 | altptr = ALT_ALT_PTR(alt); | |
9493e6f3 CD |
728 | |
729 | mutex_lock(&text_mutex); | |
8d23e94a JZ |
730 | patch_text_nosync(oldptr, altptr, alt->alt_len); |
731 | riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); | |
9493e6f3 | 732 | mutex_unlock(&text_mutex); |
ff689fd2 HS |
733 | } |
734 | } | |
735 | #endif |