arch/riscv/kernel/cpufeature.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copied from arch/arm64/kernel/cpufeature.c
   4  *
   5  * Copyright (C) 2015 ARM Ltd.
   6  * Copyright (C) 2017 SiFive
   7  */
   8
   9 #include <linux/bitmap.h>
  10 #include <linux/ctype.h>
  11 #include <linux/libfdt.h>
  12 #include <linux/module.h>
  13 #include <linux/of.h>
  14 #include <asm/alternative.h>
  15 #include <asm/errata_list.h>
  16 #include <asm/hwcap.h>
  17 #include <asm/patch.h>
  18 #include <asm/pgtable.h>
  19 #include <asm/processor.h>
  20 #include <asm/smp.h>
  21 #include <asm/switch_to.h>
  22
  23 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
  24
  25 unsigned long elf_hwcap __read_mostly;
  26
  27 /* Host ISA bitmap */
  28 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
  29
  30 #ifdef CONFIG_FPU
  31 __ro_after_init DEFINE_STATIC_KEY_FALSE(cpu_hwcap_fpu);
  32 #endif
  33
  34 /**
  35  * riscv_isa_extension_base() - Get base extension word
  36  *
  37  * @isa_bitmap: ISA bitmap to use
  38  * Return: base extension word as unsigned long value
  39  *
  40  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
  41  */
  42 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap)
  43 {
  44         if (!isa_bitmap)
  45                 return riscv_isa[0];
  46         return isa_bitmap[0];
  47 }
  48 EXPORT_SYMBOL_GPL(riscv_isa_extension_base);
  49
  50 /**
  51  * __riscv_isa_extension_available() - Check whether given extension
  52  * is available or not
  53  *
  54  * @isa_bitmap: ISA bitmap to use
  55  * @bit: bit position of the desired extension
  56  * Return: true or false
  57  *
  58  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
  59  */
  60 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit)
  61 {
  62         const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa;
  63
  64         if (bit >= RISCV_ISA_EXT_MAX)
  65                 return false;
  66
  67         return test_bit(bit, bmap) ? true : false;
  68 }
  69 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
  70
  71 void __init riscv_fill_hwcap(void)
  72 {
  73         struct device_node *node;
  74         const char *isa;
  75         char print_str[NUM_ALPHA_EXTS + 1];
  76         int i, j;
  77         static unsigned long isa2hwcap[256] = {0};
  78
  79         isa2hwcap['i'] = isa2hwcap['I'] = COMPAT_HWCAP_ISA_I;
  80         isa2hwcap['m'] = isa2hwcap['M'] = COMPAT_HWCAP_ISA_M;
  81         isa2hwcap['a'] = isa2hwcap['A'] = COMPAT_HWCAP_ISA_A;
  82         isa2hwcap['f'] = isa2hwcap['F'] = COMPAT_HWCAP_ISA_F;
  83         isa2hwcap['d'] = isa2hwcap['D'] = COMPAT_HWCAP_ISA_D;
  84         isa2hwcap['c'] = isa2hwcap['C'] = COMPAT_HWCAP_ISA_C;
  85
  86         elf_hwcap = 0;
  87
  88         bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX);
  89
  90         for_each_of_cpu_node(node) {
  91                 unsigned long this_hwcap = 0;
  92                 DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX);
  93                 const char *temp;
  94
  95                 if (riscv_of_processor_hartid(node) < 0)
  96                         continue;
  97
  98                 if (of_property_read_string(node, "riscv,isa", &isa)) {
  99                         pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");
 100                         continue;
 101                 }
 102
 103                 temp = isa;
 104 #if IS_ENABLED(CONFIG_32BIT)
 105                 if (!strncmp(isa, "rv32", 4))
 106                         isa += 4;
 107 #elif IS_ENABLED(CONFIG_64BIT)
 108                 if (!strncmp(isa, "rv64", 4))
 109                         isa += 4;
 110 #endif
 111                 /* The riscv,isa DT property must start with rv64 or rv32 */
 112                 if (temp == isa)
 113                         continue;
 114                 bitmap_zero(this_isa, RISCV_ISA_EXT_MAX);
 115                 for (; *isa; ++isa) {
 116                         const char *ext = isa++;
 117                         const char *ext_end = isa;
 118                         bool ext_long = false, ext_err = false;
 119
 120                         switch (*ext) {
 121                         case 's':
 122                                 /**
 123                                  * Workaround for invalid single-letter 's' & 'u'(QEMU).
 124                                  * No need to set the bit in riscv_isa as 's' & 'u' are
 125                                  * not valid ISA extensions. It works until multi-letter
 126                                  * extension starting with "Su" appears.
 127                                  */
 128                                 if (ext[-1] != '_' && ext[1] == 'u') {
 129                                         ++isa;
 130                                         ext_err = true;
 131                                         break;
 132                                 }
 133                                 fallthrough;
 134                         case 'x':
 135                         case 'z':
 136                                 ext_long = true;
 137                                 /* Multi-letter extension must be delimited */
 138                                 for (; *isa && *isa != '_'; ++isa)
 139                                         if (unlikely(!islower(*isa)
 140                                                      && !isdigit(*isa)))
 141                                                 ext_err = true;
 142                                 /* Parse backwards */
 143                                 ext_end = isa;
 144                                 if (unlikely(ext_err))
 145                                         break;
 146                                 if (!isdigit(ext_end[-1]))
 147                                         break;
 148                                 /* Skip the minor version */
 149                                 while (isdigit(*--ext_end))
 150                                         ;
 151                                 if (ext_end[0] != 'p'
 152                                     || !isdigit(ext_end[-1])) {
 153                                         /* Advance it to offset the pre-decrement */
 154                                         ++ext_end;
 155                                         break;
 156                                 }
 157                                 /* Skip the major version */
 158                                 while (isdigit(*--ext_end))
 159                                         ;
 160                                 ++ext_end;
 161                                 break;
 162                         default:
 163                                 if (unlikely(!islower(*ext))) {
 164                                         ext_err = true;
 165                                         break;
 166                                 }
 167                                 /* Find next extension */
 168                                 if (!isdigit(*isa))
 169                                         break;
 170                                 /* Skip the minor version */
 171                                 while (isdigit(*++isa))
 172                                         ;
 173                                 if (*isa != 'p')
 174                                         break;
 175                                 if (!isdigit(*++isa)) {
 176                                         --isa;
 177                                         break;
 178                                 }
 179                                 /* Skip the major version */
 180                                 while (isdigit(*++isa))
 181                                         ;
 182                                 break;
 183                         }
 184                         if (*isa != '_')
 185                                 --isa;
 186
 187 #define SET_ISA_EXT_MAP(name, bit)                                              \
 188                         do {                                                    \
 189                                 if ((ext_end - ext == sizeof(name) - 1) &&      \
 190                                      !memcmp(ext, name, sizeof(name) - 1))      \
 191                                         set_bit(bit, this_isa);                 \
 192                         } while (false)                                         \
 193
 194                         if (unlikely(ext_err))
 195                                 continue;
 196                         if (!ext_long) {
 197                                 this_hwcap |= isa2hwcap[(unsigned char)(*ext)];
 198                                 set_bit(*ext - 'a', this_isa);
 199                         } else {
 200                                 SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF);
 201                                 SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT);
 202                         }
 203 #undef SET_ISA_EXT_MAP
 204                 }
 205
 206                 /*
 207                  * All "okay" hart should have same isa. Set HWCAP based on
 208                  * common capabilities of every "okay" hart, in case they don't
 209                  * have.
 210                  */
 211                 if (elf_hwcap)
 212                         elf_hwcap &= this_hwcap;
 213                 else
 214                         elf_hwcap = this_hwcap;
 215
 216                 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
 217                         bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
 218                 else
 219                         bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
 220         }
 221
 222         /* We don't support systems with F but without D, so mask those out
 223          * here. */
 224         if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) {
 225                 pr_info("This kernel does not support systems with F but not D\n");
 226                 elf_hwcap &= ~COMPAT_HWCAP_ISA_F;
 227         }
 228
 229         memset(print_str, 0, sizeof(print_str));
 230         for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
 231                 if (riscv_isa[0] & BIT_MASK(i))
 232                         print_str[j++] = (char)('a' + i);
 233         pr_info("riscv: base ISA extensions %s\n", print_str);
 234
 235         memset(print_str, 0, sizeof(print_str));
 236         for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
 237                 if (elf_hwcap & BIT_MASK(i))
 238                         print_str[j++] = (char)('a' + i);
 239         pr_info("riscv: ELF capabilities %s\n", print_str);
 240
 241 #ifdef CONFIG_FPU
 242         if (elf_hwcap & (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D))
 243                 static_branch_enable(&cpu_hwcap_fpu);
 244 #endif
 245 }
 246
 247 #ifdef CONFIG_RISCV_ALTERNATIVE
 248 struct cpufeature_info {
 249         char name[ERRATA_STRING_LENGTH_MAX];
 250         bool (*check_func)(unsigned int stage);
 251 };
 252
 253 static bool __init_or_module cpufeature_svpbmt_check_func(unsigned int stage)
 254 {
 255 #ifdef CONFIG_RISCV_ISA_SVPBMT
 256         switch (stage) {
 257         case RISCV_ALTERNATIVES_EARLY_BOOT:
 258                 return false;
 259         default:
 260                 return riscv_isa_extension_available(NULL, SVPBMT);
 261         }
 262 #endif
 263
 264         return false;
 265 }
 266
 267 static const struct cpufeature_info __initdata_or_module
 268 cpufeature_list[CPUFEATURE_NUMBER] = {
 269         {
 270                 .name = "svpbmt",
 271                 .check_func = cpufeature_svpbmt_check_func
 272         },
 273 };
 274
 275 static u32 __init_or_module cpufeature_probe(unsigned int stage)
 276 {
 277         const struct cpufeature_info *info;
 278         u32 cpu_req_feature = 0;
 279         int idx;
 280
 281         for (idx = 0; idx < CPUFEATURE_NUMBER; idx++) {
 282                 info = &cpufeature_list[idx];
 283
 284                 if (info->check_func(stage))
 285                         cpu_req_feature |= (1U << idx);
 286         }
 287
 288         return cpu_req_feature;
 289 }
 290
 291 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
 292                                                   struct alt_entry *end,
 293                                                   unsigned int stage)
 294 {
 295         u32 cpu_req_feature = cpufeature_probe(stage);
 296         u32 cpu_apply_feature = 0;
 297         struct alt_entry *alt;
 298         u32 tmp;
 299
 300         for (alt = begin; alt < end; alt++) {
 301                 if (alt->vendor_id != 0)
 302                         continue;
 303                 if (alt->errata_id >= CPUFEATURE_NUMBER) {
 304                         WARN(1, "This feature id:%d is not in kernel cpufeature list",
 305                                 alt->errata_id);
 306                         continue;
 307                 }
 308
 309                 tmp = (1U << alt->errata_id);
 310                 if (cpu_req_feature & tmp) {
 311                         patch_text_nosync(alt->old_ptr, alt->alt_ptr, alt->alt_len);
 312                         cpu_apply_feature |= tmp;
 313                 }
 314         }
 315 }
 316 #endif