arch/riscv/errata/thead/errata.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2021 Heiko Stuebner <heiko@sntech.de>
   4  */
   5
   6 #include <linux/bug.h>
   7 #include <linux/kernel.h>
   8 #include <linux/memory.h>
   9 #include <linux/module.h>
  10 #include <linux/string.h>
  11 #include <linux/uaccess.h>
  12 #include <asm/alternative.h>
  13 #include <asm/cacheflush.h>
  14 #include <asm/cpufeature.h>
  15 #include <asm/dma-noncoherent.h>
  16 #include <asm/errata_list.h>
  17 #include <asm/hwprobe.h>
  18 #include <asm/io.h>
  19 #include <asm/patch.h>
  20 #include <asm/vendorid_list.h>
  21
  22 static bool errata_probe_pbmt(unsigned int stage,
  23                               unsigned long arch_id, unsigned long impid)
  24 {
  25         if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PBMT))
  26                 return false;
  27
  28         if (arch_id != 0 || impid != 0)
  29                 return false;
  30
  31         if (stage == RISCV_ALTERNATIVES_EARLY_BOOT ||
  32             stage == RISCV_ALTERNATIVES_MODULE)
  33                 return true;
  34
  35         return false;
  36 }
  37
  38 /*
  39  * th.dcache.ipa rs1 (invalidate, physical address)
  40  * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
  41  *   0000001    01010      rs1       000      00000  0001011
  42  * th.dcache.iva rs1 (invalidate, virtual address)
  43  *   0000001    00110      rs1       000      00000  0001011
  44  *
  45  * th.dcache.cpa rs1 (clean, physical address)
  46  * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
  47  *   0000001    01001      rs1       000      00000  0001011
  48  * th.dcache.cva rs1 (clean, virtual address)
  49  *   0000001    00101      rs1       000      00000  0001011
  50  *
  51  * th.dcache.cipa rs1 (clean then invalidate, physical address)
  52  * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
  53  *   0000001    01011      rs1       000      00000  0001011
  54  * th.dcache.civa rs1 (clean then invalidate, virtual address)
  55  *   0000001    00111      rs1       000      00000  0001011
  56  *
  57  * th.sync.s (make sure all cache operations finished)
  58  * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
  59  *   0000000    11001     00000      000      00000  0001011
  60  */
  61 #define THEAD_INVAL_A0  ".long 0x02a5000b"
  62 #define THEAD_CLEAN_A0  ".long 0x0295000b"
  63 #define THEAD_FLUSH_A0  ".long 0x02b5000b"
  64 #define THEAD_SYNC_S    ".long 0x0190000b"
  65
  66 #define THEAD_CMO_OP(_op, _start, _size, _cachesize)                    \
  67 asm volatile("mv a0, %1\n\t"                                            \
  68              "j 2f\n\t"                                                 \
  69              "3:\n\t"                                                   \
  70              THEAD_##_op##_A0 "\n\t"                                    \
  71              "add a0, a0, %0\n\t"                                       \
  72              "2:\n\t"                                                   \
  73              "bltu a0, %2, 3b\n\t"                                      \
  74              THEAD_SYNC_S                                               \
  75              : : "r"(_cachesize),                                       \
  76                  "r"((unsigned long)(_start) & ~((_cachesize) - 1UL)),  \
  77                  "r"((unsigned long)(_start) + (_size))                 \
  78              : "a0")
  79
  80 static void thead_errata_cache_inv(phys_addr_t paddr, size_t size)
  81 {
  82         THEAD_CMO_OP(INVAL, paddr, size, riscv_cbom_block_size);
  83 }
  84
  85 static void thead_errata_cache_wback(phys_addr_t paddr, size_t size)
  86 {
  87         THEAD_CMO_OP(CLEAN, paddr, size, riscv_cbom_block_size);
  88 }
  89
  90 static void thead_errata_cache_wback_inv(phys_addr_t paddr, size_t size)
  91 {
  92         THEAD_CMO_OP(FLUSH, paddr, size, riscv_cbom_block_size);
  93 }
  94
  95 static const struct riscv_nonstd_cache_ops thead_errata_cmo_ops = {
  96         .wback = &thead_errata_cache_wback,
  97         .inv = &thead_errata_cache_inv,
  98         .wback_inv = &thead_errata_cache_wback_inv,
  99 };
 100
 101 static bool errata_probe_cmo(unsigned int stage,
 102                              unsigned long arch_id, unsigned long impid)
 103 {
 104         if (!IS_ENABLED(CONFIG_ERRATA_THEAD_CMO))
 105                 return false;
 106
 107         if (arch_id != 0 || impid != 0)
 108                 return false;
 109
 110         if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
 111                 return false;
 112
 113         if (stage == RISCV_ALTERNATIVES_BOOT) {
 114                 riscv_cbom_block_size = L1_CACHE_BYTES;
 115                 riscv_noncoherent_supported();
 116                 riscv_noncoherent_register_cache_ops(&thead_errata_cmo_ops);
 117         }
 118
 119         return true;
 120 }
 121
 122 static bool errata_probe_pmu(unsigned int stage,
 123                              unsigned long arch_id, unsigned long impid)
 124 {
 125         if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PMU))
 126                 return false;
 127
 128         /* target-c9xx cores report arch_id and impid as 0 */
 129         if (arch_id != 0 || impid != 0)
 130                 return false;
 131
 132         if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
 133                 return false;
 134
 135         return true;
 136 }
 137
 138 static u32 thead_errata_probe(unsigned int stage,
 139                               unsigned long archid, unsigned long impid)
 140 {
 141         u32 cpu_req_errata = 0;
 142
 143         if (errata_probe_pbmt(stage, archid, impid))
 144                 cpu_req_errata |= BIT(ERRATA_THEAD_PBMT);
 145
 146         errata_probe_cmo(stage, archid, impid);
 147
 148         if (errata_probe_pmu(stage, archid, impid))
 149                 cpu_req_errata |= BIT(ERRATA_THEAD_PMU);
 150
 151         return cpu_req_errata;
 152 }
 153
 154 void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
 155                              unsigned long archid, unsigned long impid,
 156                              unsigned int stage)
 157 {
 158         struct alt_entry *alt;
 159         u32 cpu_req_errata = thead_errata_probe(stage, archid, impid);
 160         u32 tmp;
 161         void *oldptr, *altptr;
 162
 163         for (alt = begin; alt < end; alt++) {
 164                 if (alt->vendor_id != THEAD_VENDOR_ID)
 165                         continue;
 166                 if (alt->patch_id >= ERRATA_THEAD_NUMBER)
 167                         continue;
 168
 169                 tmp = (1U << alt->patch_id);
 170                 if (cpu_req_errata & tmp) {
 171                         oldptr = ALT_OLD_PTR(alt);
 172                         altptr = ALT_ALT_PTR(alt);
 173
 174                         /* On vm-alternatives, the mmu isn't running yet */
 175                         if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) {
 176                                 memcpy(oldptr, altptr, alt->alt_len);
 177                         } else {
 178                                 mutex_lock(&text_mutex);
 179                                 patch_text_nosync(oldptr, altptr, alt->alt_len);
 180                                 mutex_unlock(&text_mutex);
 181                         }
 182                 }
 183         }
 184
 185         if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
 186                 local_flush_icache_all();
 187 }