projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
arm64: Kconfig: Reword UNMAP_KERNEL_AT_EL0 kconfig entry
[linux-2.6-block.git] / arch / arm64 / kernel / cpufeature.c
CommitLineData
359b7064
MZ		 1/*
2 * Contains CPU feature definitions
3 *
4 * Copyright (C) 2015 ARM Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
9cdf8ec4 19#define pr_fmt(fmt) "CPU features: " fmt
359b7064 20
3c739b57 21#include <linux/bsearch.h>
2a6dcb2b 22#include <linux/cpumask.h>
3c739b57 23#include <linux/sort.h>
2a6dcb2b 24#include <linux/stop_machine.h>
359b7064 25#include <linux/types.h>
2077be67 26#include <linux/mm.h>
359b7064
MZ		 27#include <asm/cpu.h>
28#include <asm/cpufeature.h>
dbb4e152 29#include <asm/cpu_ops.h>
2e0f2478 30#include <asm/fpsimd.h>
13f417f3 31#include <asm/mmu_context.h>
338d4f49 32#include <asm/processor.h>
cdcf817b 33#include <asm/sysreg.h>
77c97b4e 34#include <asm/traps.h>
d88701be 35#include <asm/virt.h>
359b7064 36
9cdf8ec4
SP		 37unsigned long elf_hwcap __read_mostly;
38EXPORT_SYMBOL_GPL(elf_hwcap);
39
40#ifdef CONFIG_COMPAT
41#define COMPAT_ELF_HWCAP_DEFAULT \
42 (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
43 COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
44 COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\
45 COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\
46 COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\
47 COMPAT_HWCAP_LPAE)
48unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
49unsigned int compat_elf_hwcap2 __read_mostly;
50#endif
51
52DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
4b65a5db 53EXPORT_SYMBOL(cpu_hwcaps);
9cdf8ec4 54
8f1eec57
DM		 55/*
56 * Flag to indicate if we have computed the system wide
57 * capabilities based on the boot time active CPUs. This
58 * will be used to determine if a new booting CPU should
59 * go through the verification process to make sure that it
60 * supports the system capabilities, without using a hotplug
61 * notifier.
62 */
63static bool sys_caps_initialised;
64
65static inline void set_sys_caps_initialised(void)
66{
67 sys_caps_initialised = true;
68}
69
8effeaaf
MR		 70static int dump_cpu_hwcaps(struct notifier_block *self, unsigned long v, void *p)
71{
72 /* file-wide pr_fmt adds "CPU features: " prefix */
73 pr_emerg("0x%*pb\n", ARM64_NCAPS, &cpu_hwcaps);
74 return 0;
75}
76
77static struct notifier_block cpu_hwcaps_notifier = {
78 .notifier_call = dump_cpu_hwcaps
79};
80
81static int __init register_cpu_hwcaps_dumper(void)
82{
83 atomic_notifier_chain_register(&panic_notifier_list,
84 &cpu_hwcaps_notifier);
85 return 0;
86}
87__initcall(register_cpu_hwcaps_dumper);
88
efd9e03f
CM		 89DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
90EXPORT_SYMBOL(cpu_hwcap_keys);
91
fe4fbdbc 92#define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
3c739b57 93 { \
4f0a606b 94 .sign = SIGNED, \
fe4fbdbc 95 .visible = VISIBLE, \
3c739b57
SP		 96 .strict = STRICT, \
97 .type = TYPE, \
98 .shift = SHIFT, \
99 .width = WIDTH, \
100 .safe_val = SAFE_VAL, \
101 }
102
0710cfdb 103/* Define a feature with unsigned values */
fe4fbdbc
SP		 104#define ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
105 __ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
4f0a606b 106
0710cfdb 107/* Define a feature with a signed value */
fe4fbdbc
SP		 108#define S_ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
109 __ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
0710cfdb 110
3c739b57
SP		 111#define ARM64_FTR_END \
112 { \
113 .width = 0, \
114 }
115
70544196
JM		 116/* meta feature for alternatives */
117static bool __maybe_unused
92406f0c
SP		 118cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused);
119
70544196 120
4aa8a472
SP		 121/*
122 * NOTE: Any changes to the visibility of features should be kept in
123 * sync with the documentation of the CPU feature register ABI.
124 */
5e49d73c 125static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
3b3b6810 126 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_FHM_SHIFT, 4, 0),
5bdecb79
SP		 127 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_DP_SHIFT, 4, 0),
128 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM4_SHIFT, 4, 0),
129 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM3_SHIFT, 4, 0),
130 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA3_SHIFT, 4, 0),
131 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
fe4fbdbc
SP		 132 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0),
133 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0),
134 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0),
135 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA1_SHIFT, 4, 0),
136 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_AES_SHIFT, 4, 0),
3c739b57
SP		 137 ARM64_FTR_END,
138};
139
c8c3798d 140static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
5bdecb79
SP		 141 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
142 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
143 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
144 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
c8c3798d
SP		 145 ARM64_FTR_END,
146};
147
5e49d73c 148static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
43994d82 149 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
5bdecb79 150 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_GIC_SHIFT, 4, 0),
fe4fbdbc
SP		 151 S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
152 S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
3c739b57 153 /* Linux doesn't care about the EL3 */
5bdecb79
SP		 154 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL3_SHIFT, 4, 0),
155 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL2_SHIFT, 4, 0),
156 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
157 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
3c739b57
SP		 158 ARM64_FTR_END,
159};
160
5e49d73c 161static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
5bdecb79
SP		 162 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
163 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
164 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
165 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
3c739b57 166 /* Linux shouldn't care about secure memory */
5bdecb79
SP		 167 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
168 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
169 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
3c739b57
SP		 170 /*
171 * Differing PARange is fine as long as all peripherals and memory are mapped
172 * within the minimum PARange of all CPUs
173 */
fe4fbdbc 174 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0),
3c739b57
SP		 175 ARM64_FTR_END,
176};
177
5e49d73c 178static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
fe4fbdbc 179 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
5bdecb79
SP		 180 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
181 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
182 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
183 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
184 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
3c739b57
SP		 185 ARM64_FTR_END,
186};
187
5e49d73c 188static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
5bdecb79
SP		 189 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
190 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
191 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
192 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
193 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
406e3087
JM		 194 ARM64_FTR_END,
195};
196
5e49d73c 197static const struct arm64_ftr_bits ftr_ctr[] = {
fe4fbdbc
SP		 198 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RAO */
199 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, 24, 4, 0), /* CWG */
200 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0), /* ERG */
201 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 1), /* DminLine */
3c739b57
SP		 202 /*
203 * Linux can handle differing I-cache policies. Userspace JITs will
ee7bc638 204 * make use of *minLine.
155433cb 205 * If we have differing I-cache policies, report it as the weakest - VIPT.
3c739b57 206 */
155433cb 207 ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, 14, 2, ICACHE_POLICY_VIPT), /* L1Ip */
fe4fbdbc 208 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* IminLine */
3c739b57
SP		 209 ARM64_FTR_END,
210};
211
675b0563
AB		 212struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = {
213 .name = "SYS_CTR_EL0",
214 .ftr_bits = ftr_ctr
215};
216
5e49d73c 217static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
5bdecb79
SP		 218 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0xf), /* InnerShr */
219 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0), /* FCSE */
fe4fbdbc 220 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, 20, 4, 0), /* AuxReg */
5bdecb79
SP		 221 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0), /* TCM */
222 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0), /* ShareLvl */
223 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0xf), /* OuterShr */
224 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), /* PMSA */
225 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* VMSA */
3c739b57
SP		 226 ARM64_FTR_END,
227};
228
5e49d73c 229static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
fe4fbdbc
SP		 230 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 36, 28, 0),
231 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_PMSVER_SHIFT, 4, 0),
232 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0),
233 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0),
234 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0),
b20d1ba3
WD		 235 /*
236 * We can instantiate multiple PMU instances with different levels
237 * of support.
fe4fbdbc
SP		 238 */
239 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0),
240 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_TRACEVER_SHIFT, 4, 0),
241 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6),
3c739b57
SP		 242 ARM64_FTR_END,
243};
244
5e49d73c 245static const struct arm64_ftr_bits ftr_mvfr2[] = {
5bdecb79
SP		 246 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), /* FPMisc */
247 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* SIMDMisc */
3c739b57
SP		 248 ARM64_FTR_END,
249};
250
5e49d73c 251static const struct arm64_ftr_bits ftr_dczid[] = {
fe4fbdbc
SP		 252 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 4, 1, 1), /* DZP */
253 ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* BS */
3c739b57
SP		 254 ARM64_FTR_END,
255};
256
257
5e49d73c 258static const struct arm64_ftr_bits ftr_id_isar5[] = {
5bdecb79
SP		 259 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),
260 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),
261 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),
262 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),
263 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),
264 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),
3c739b57
SP		 265 ARM64_FTR_END,
266};
267
5e49d73c 268static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
5bdecb79 269 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), /* ac2 */
3c739b57
SP		 270 ARM64_FTR_END,
271};
272
5e49d73c 273static const struct arm64_ftr_bits ftr_id_pfr0[] = {
5bdecb79
SP		 274 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0), /* State3 */
275 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0), /* State2 */
276 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0), /* State1 */
277 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* State0 */
3c739b57
SP		 278 ARM64_FTR_END,
279};
280
5e49d73c 281static const struct arm64_ftr_bits ftr_id_dfr0[] = {
fe4fbdbc
SP		 282 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
283 S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0xf), /* PerfMon */
284 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
285 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
286 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
287 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
288 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
289 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
e5343503
SP		 290 ARM64_FTR_END,
291};
292
2e0f2478
DM		 293static const struct arm64_ftr_bits ftr_zcr[] = {
294 ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,
295 ZCR_ELx_LEN_SHIFT, ZCR_ELx_LEN_SIZE, 0), /* LEN */
296 ARM64_FTR_END,
297};
298
3c739b57
SP		 299/*
300 * Common ftr bits for a 32bit register with all hidden, strict
301 * attributes, with 4bit feature fields and a default safe value of
302 * 0. Covers the following 32bit registers:
303 * id_isar[0-4], id_mmfr[1-3], id_pfr1, mvfr[0-1]
304 */
5e49d73c 305static const struct arm64_ftr_bits ftr_generic_32bits[] = {
fe4fbdbc
SP		 306 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
307 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),
308 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
309 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
310 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
311 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
312 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
313 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
3c739b57
SP		 314 ARM64_FTR_END,
315};
316
eab43e88
SP		 317/* Table for a single 32bit feature value */
318static const struct arm64_ftr_bits ftr_single32[] = {
fe4fbdbc 319 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 32, 0),
3c739b57
SP		 320 ARM64_FTR_END,
321};
322
eab43e88 323static const struct arm64_ftr_bits ftr_raz[] = {
3c739b57
SP		 324 ARM64_FTR_END,
325};
326
6f2b7eef
AB		 327#define ARM64_FTR_REG(id, table) { \
328 .sys_id = id, \
329 .reg = &(struct arm64_ftr_reg){ \
3c739b57
SP		 330 .name = #id, \
331 .ftr_bits = &((table)[0]), \
6f2b7eef 332 }}
3c739b57 333
6f2b7eef
AB		 334static const struct __ftr_reg_entry {
335 u32 sys_id;
336 struct arm64_ftr_reg *reg;
337} arm64_ftr_regs[] = {
3c739b57
SP		 338
339 /* Op1 = 0, CRn = 0, CRm = 1 */
340 ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0),
341 ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_generic_32bits),
e5343503 342 ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_id_dfr0),
3c739b57
SP		 343 ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0),
344 ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits),
345 ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits),
346 ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits),
347
348 /* Op1 = 0, CRn = 0, CRm = 2 */
349 ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_generic_32bits),
350 ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits),
351 ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits),
352 ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits),
353 ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_generic_32bits),
354 ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5),
355 ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4),
356
357 /* Op1 = 0, CRn = 0, CRm = 3 */
358 ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_generic_32bits),
359 ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_generic_32bits),
360 ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2),
361
362 /* Op1 = 0, CRn = 0, CRm = 4 */
363 ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
eab43e88 364 ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_raz),
2e0f2478 365 ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_raz),
3c739b57
SP		 366
367 /* Op1 = 0, CRn = 0, CRm = 5 */
368 ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
eab43e88 369 ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_raz),
3c739b57
SP		 370
371 /* Op1 = 0, CRn = 0, CRm = 6 */
372 ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
c8c3798d 373 ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1),
3c739b57
SP		 374
375 /* Op1 = 0, CRn = 0, CRm = 7 */
376 ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
377 ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
406e3087 378 ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
3c739b57 379
2e0f2478
DM		 380 /* Op1 = 0, CRn = 1, CRm = 2 */
381 ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
382
3c739b57 383 /* Op1 = 3, CRn = 0, CRm = 0 */
675b0563 384 { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
3c739b57
SP		 385 ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
386
387 /* Op1 = 3, CRn = 14, CRm = 0 */
eab43e88 388 ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_single32),
3c739b57
SP		 389};
390
391static int search_cmp_ftr_reg(const void *id, const void *regp)
392{
6f2b7eef 393 return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id;
3c739b57
SP		 394}
395
396/*
397 * get_arm64_ftr_reg - Lookup a feature register entry using its
398 * sys_reg() encoding. With the array arm64_ftr_regs sorted in the
399 * ascending order of sys_id , we use binary search to find a matching
400 * entry.
401 *
402 * returns - Upon success, matching ftr_reg entry for id.
403 * - NULL on failure. It is upto the caller to decide
404 * the impact of a failure.
405 */
406static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
407{
6f2b7eef
AB		 408 const struct __ftr_reg_entry *ret;
409
410 ret = bsearch((const void *)(unsigned long)sys_id,
3c739b57
SP		 411 arm64_ftr_regs,
412 ARRAY_SIZE(arm64_ftr_regs),
413 sizeof(arm64_ftr_regs[0]),
414 search_cmp_ftr_reg);
6f2b7eef
AB		 415 if (ret)
416 return ret->reg;
417 return NULL;
3c739b57
SP		 418}
419
5e49d73c
AB		 420static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg,
421 s64 ftr_val)
3c739b57
SP		 422{
423 u64 mask = arm64_ftr_mask(ftrp);
424
425 reg &= ~mask;
426 reg |= (ftr_val << ftrp->shift) & mask;
427 return reg;
428}
429
5e49d73c
AB		 430static s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new,
431 s64 cur)
3c739b57
SP		 432{
433 s64 ret = 0;
434
435 switch (ftrp->type) {
436 case FTR_EXACT:
437 ret = ftrp->safe_val;
438 break;
439 case FTR_LOWER_SAFE:
440 ret = new < cur ? new : cur;
441 break;
442 case FTR_HIGHER_SAFE:
443 ret = new > cur ? new : cur;
444 break;
445 default:
446 BUG();
447 }
448
449 return ret;
450}
451
3c739b57
SP		 452static void __init sort_ftr_regs(void)
453{
6f2b7eef
AB		 454 int i;
455
456 /* Check that the array is sorted so that we can do the binary search */
457 for (i = 1; i < ARRAY_SIZE(arm64_ftr_regs); i++)
458 BUG_ON(arm64_ftr_regs[i].sys_id < arm64_ftr_regs[i - 1].sys_id);
3c739b57
SP		 459}
460
461/*
462 * Initialise the CPU feature register from Boot CPU values.
463 * Also initiliases the strict_mask for the register.
b389d799
MR		 464 * Any bits that are not covered by an arm64_ftr_bits entry are considered
465 * RES0 for the system-wide value, and must strictly match.
3c739b57
SP		 466 */
467static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
468{
469 u64 val = 0;
470 u64 strict_mask = ~0x0ULL;
fe4fbdbc 471 u64 user_mask = 0;
b389d799
MR		 472 u64 valid_mask = 0;
473
5e49d73c 474 const struct arm64_ftr_bits *ftrp;
3c739b57
SP		 475 struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg);
476
477 BUG_ON(!reg);
478
479 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
b389d799 480 u64 ftr_mask = arm64_ftr_mask(ftrp);
3c739b57
SP		 481 s64 ftr_new = arm64_ftr_value(ftrp, new);
482
483 val = arm64_ftr_set_value(ftrp, val, ftr_new);
b389d799
MR		 484
485 valid_mask |= ftr_mask;
3c739b57 486 if (!ftrp->strict)
b389d799 487 strict_mask &= ~ftr_mask;
fe4fbdbc
SP		 488 if (ftrp->visible)
489 user_mask |= ftr_mask;
490 else
491 reg->user_val = arm64_ftr_set_value(ftrp,
492 reg->user_val,
493 ftrp->safe_val);
3c739b57 494 }
b389d799
MR		 495
496 val &= valid_mask;
497
3c739b57
SP		 498 reg->sys_val = val;
499 reg->strict_mask = strict_mask;
fe4fbdbc 500 reg->user_mask = user_mask;
3c739b57
SP		 501}
502
503void __init init_cpu_features(struct cpuinfo_arm64 *info)
504{
505 /* Before we start using the tables, make sure it is sorted */
506 sort_ftr_regs();
507
508 init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr);
509 init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid);
510 init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq);
511 init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0);
512 init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
513 init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
514 init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
515 init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
516 init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
406e3087 517 init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
3c739b57
SP		 518 init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
519 init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
2e0f2478 520 init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
a6dc3cd7
SP		 521
522 if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
523 init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
524 init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
525 init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
526 init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
527 init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
528 init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
529 init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
530 init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
531 init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
532 init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
533 init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
534 init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
535 init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
536 init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
537 init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
538 init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
539 }
540
2e0f2478
DM		 541 if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
542 init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
543 sve_init_vq_map();
544 }
3c739b57
SP		 545}
546
3086d391 547static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
3c739b57 548{
5e49d73c 549 const struct arm64_ftr_bits *ftrp;
3c739b57
SP		 550
551 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
552 s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val);
553 s64 ftr_new = arm64_ftr_value(ftrp, new);
554
555 if (ftr_cur == ftr_new)
556 continue;
557 /* Find a safe value */
558 ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur);
559 reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new);
560 }
561
562}
563
3086d391 564static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot)
cdcf817b 565{
3086d391
SP		 566 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
567
568 BUG_ON(!regp);
569 update_cpu_ftr_reg(regp, val);
570 if ((boot & regp->strict_mask) == (val & regp->strict_mask))
571 return 0;
572 pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n",
573 regp->name, boot, cpu, val);
574 return 1;
575}
576
577/*
578 * Update system wide CPU feature registers with the values from a
579 * non-boot CPU. Also performs SANITY checks to make sure that there
580 * aren't any insane variations from that of the boot CPU.
581 */
582void update_cpu_features(int cpu,
583 struct cpuinfo_arm64 *info,
584 struct cpuinfo_arm64 *boot)
585{
586 int taint = 0;
587
588 /*
589 * The kernel can handle differing I-cache policies, but otherwise
590 * caches should look identical. Userspace JITs will make use of
591 * *minLine.
592 */
593 taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu,
594 info->reg_ctr, boot->reg_ctr);
595
596 /*
597 * Userspace may perform DC ZVA instructions. Mismatched block sizes
598 * could result in too much or too little memory being zeroed if a
599 * process is preempted and migrated between CPUs.
600 */
601 taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu,
602 info->reg_dczid, boot->reg_dczid);
603
604 /* If different, timekeeping will be broken (especially with KVM) */
605 taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu,
606 info->reg_cntfrq, boot->reg_cntfrq);
607
608 /*
609 * The kernel uses self-hosted debug features and expects CPUs to
610 * support identical debug features. We presently need CTX_CMPs, WRPs,
611 * and BRPs to be identical.
612 * ID_AA64DFR1 is currently RES0.
613 */
614 taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu,
615 info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0);
616 taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu,
617 info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1);
618 /*
619 * Even in big.LITTLE, processors should be identical instruction-set
620 * wise.
621 */
622 taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu,
623 info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
624 taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
625 info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
626
627 /*
628 * Differing PARange support is fine as long as all peripherals and
629 * memory are mapped within the minimum PARange of all CPUs.
630 * Linux should not care about secure memory.
631 */
632 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu,
633 info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0);
634 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu,
635 info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1);
406e3087
JM		 636 taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu,
637 info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2);
3086d391
SP		 638
639 /*
640 * EL3 is not our concern.
641 * ID_AA64PFR1 is currently RES0.
642 */
643 taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu,
644 info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
645 taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
646 info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
647
2e0f2478
DM		 648 taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,
649 info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);
650
3086d391 651 /*
a6dc3cd7
SP		 652 * If we have AArch32, we care about 32-bit features for compat.
653 * If the system doesn't support AArch32, don't update them.
3086d391 654 */
46823dd1 655 if (id_aa64pfr0_32bit_el0(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
a6dc3cd7
SP		 656 id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
657
658 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
3086d391 659 info->reg_id_dfr0, boot->reg_id_dfr0);
a6dc3cd7 660 taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu,
3086d391 661 info->reg_id_isar0, boot->reg_id_isar0);
a6dc3cd7 662 taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu,
3086d391 663 info->reg_id_isar1, boot->reg_id_isar1);
a6dc3cd7 664 taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu,
3086d391 665 info->reg_id_isar2, boot->reg_id_isar2);
a6dc3cd7 666 taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu,
3086d391 667 info->reg_id_isar3, boot->reg_id_isar3);
a6dc3cd7 668 taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu,
3086d391 669 info->reg_id_isar4, boot->reg_id_isar4);
a6dc3cd7 670 taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu,
3086d391
SP		 671 info->reg_id_isar5, boot->reg_id_isar5);
672
a6dc3cd7
SP		 673 /*
674 * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
675 * ACTLR formats could differ across CPUs and therefore would have to
676 * be trapped for virtualization anyway.
677 */
678 taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu,
3086d391 679 info->reg_id_mmfr0, boot->reg_id_mmfr0);
a6dc3cd7 680 taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu,
3086d391 681 info->reg_id_mmfr1, boot->reg_id_mmfr1);
a6dc3cd7 682 taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu,
3086d391 683 info->reg_id_mmfr2, boot->reg_id_mmfr2);
a6dc3cd7 684 taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu,
3086d391 685 info->reg_id_mmfr3, boot->reg_id_mmfr3);
a6dc3cd7 686 taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu,
3086d391 687 info->reg_id_pfr0, boot->reg_id_pfr0);
a6dc3cd7 688 taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu,
3086d391 689 info->reg_id_pfr1, boot->reg_id_pfr1);
a6dc3cd7 690 taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu,
3086d391 691 info->reg_mvfr0, boot->reg_mvfr0);
a6dc3cd7 692 taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu,
3086d391 693 info->reg_mvfr1, boot->reg_mvfr1);
a6dc3cd7 694 taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu,
3086d391 695 info->reg_mvfr2, boot->reg_mvfr2);
a6dc3cd7 696 }
3086d391 697
2e0f2478
DM		 698 if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
699 taint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu,
700 info->reg_zcr, boot->reg_zcr);
701
702 /* Probe vector lengths, unless we already gave up on SVE */
703 if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
704 !sys_caps_initialised)
705 sve_update_vq_map();
706 }
707
3086d391
SP		 708 /*
709 * Mismatched CPU features are a recipe for disaster. Don't even
710 * pretend to support them.
711 */
8dd0ee65
WD		 712 if (taint) {
713 pr_warn_once("Unsupported CPU feature variation detected.\n");
714 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
715 }
cdcf817b
SP		 716}
717
46823dd1 718u64 read_sanitised_ftr_reg(u32 id)
b3f15378
SP		 719{
720 struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
721
722 /* We shouldn't get a request for an unsupported register */
723 BUG_ON(!regp);
724 return regp->sys_val;
725}
359b7064 726
965861d6
MR		 727#define read_sysreg_case(r) \
728 case r: return read_sysreg_s(r)
729
92406f0c 730/*
46823dd1 731 * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated.
92406f0c
SP		 732 * Read the system register on the current CPU
733 */
46823dd1 734static u64 __read_sysreg_by_encoding(u32 sys_id)
92406f0c
SP		 735{
736 switch (sys_id) {
965861d6
MR		 737 read_sysreg_case(SYS_ID_PFR0_EL1);
738 read_sysreg_case(SYS_ID_PFR1_EL1);
739 read_sysreg_case(SYS_ID_DFR0_EL1);
740 read_sysreg_case(SYS_ID_MMFR0_EL1);
741 read_sysreg_case(SYS_ID_MMFR1_EL1);
742 read_sysreg_case(SYS_ID_MMFR2_EL1);
743 read_sysreg_case(SYS_ID_MMFR3_EL1);
744 read_sysreg_case(SYS_ID_ISAR0_EL1);
745 read_sysreg_case(SYS_ID_ISAR1_EL1);
746 read_sysreg_case(SYS_ID_ISAR2_EL1);
747 read_sysreg_case(SYS_ID_ISAR3_EL1);
748 read_sysreg_case(SYS_ID_ISAR4_EL1);
749 read_sysreg_case(SYS_ID_ISAR5_EL1);
750 read_sysreg_case(SYS_MVFR0_EL1);
751 read_sysreg_case(SYS_MVFR1_EL1);
752 read_sysreg_case(SYS_MVFR2_EL1);
753
754 read_sysreg_case(SYS_ID_AA64PFR0_EL1);
755 read_sysreg_case(SYS_ID_AA64PFR1_EL1);
756 read_sysreg_case(SYS_ID_AA64DFR0_EL1);
757 read_sysreg_case(SYS_ID_AA64DFR1_EL1);
758 read_sysreg_case(SYS_ID_AA64MMFR0_EL1);
759 read_sysreg_case(SYS_ID_AA64MMFR1_EL1);
760 read_sysreg_case(SYS_ID_AA64MMFR2_EL1);
761 read_sysreg_case(SYS_ID_AA64ISAR0_EL1);
762 read_sysreg_case(SYS_ID_AA64ISAR1_EL1);
763
764 read_sysreg_case(SYS_CNTFRQ_EL0);
765 read_sysreg_case(SYS_CTR_EL0);
766 read_sysreg_case(SYS_DCZID_EL0);
767
92406f0c
SP		 768 default:
769 BUG();
770 return 0;
771 }
772}
773
963fcd40
MZ		 774#include <linux/irqchip/arm-gic-v3.h>
775
18ffa046
JM		 776static bool
777feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
778{
28c5dcb2 779 int val = cpuid_feature_extract_field(reg, entry->field_pos, entry->sign);
18ffa046
JM		 780
781 return val >= entry->min_field_value;
782}
783
da8d02d1 784static bool
92406f0c 785has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
da8d02d1
SP		 786{
787 u64 val;
94a9e04a 788
92406f0c
SP		 789 WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
790 if (scope == SCOPE_SYSTEM)
46823dd1 791 val = read_sanitised_ftr_reg(entry->sys_reg);
92406f0c 792 else
46823dd1 793 val = __read_sysreg_by_encoding(entry->sys_reg);
92406f0c 794
da8d02d1
SP		 795 return feature_matches(val, entry);
796}
338d4f49 797
92406f0c 798static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope)
963fcd40
MZ		 799{
800 bool has_sre;
801
92406f0c 802 if (!has_cpuid_feature(entry, scope))
963fcd40
MZ		 803 return false;
804
805 has_sre = gic_enable_sre();
806 if (!has_sre)
807 pr_warn_once("%s present but disabled by higher exception level\n",
808 entry->desc);
809
810 return has_sre;
811}
812
92406f0c 813static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused)
d5370f75
WD		 814{
815 u32 midr = read_cpuid_id();
d5370f75
WD		 816
817 /* Cavium ThunderX pass 1.x and 2.x */
fa5ce3d1
RR		 818 return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX,
819 MIDR_CPU_VAR_REV(0, 0),
820 MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK));
d5370f75
WD		 821}
822
92406f0c 823static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
d88701be
MZ		 824{
825 return is_kernel_in_hyp_mode();
826}
827
d1745910
MZ		 828static bool hyp_offset_low(const struct arm64_cpu_capabilities *entry,
829 int __unused)
830{
2077be67 831 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
d1745910
MZ		 832
833 /*
834 * Activate the lower HYP offset only if:
835 * - the idmap doesn't clash with it,
836 * - the kernel is not running at EL2.
837 */
838 return idmap_addr > GENMASK(VA_BITS - 2, 0) && !is_kernel_in_hyp_mode();
839}
840
82e0191a
SP		 841static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
842{
46823dd1 843 u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
82e0191a
SP		 844
845 return cpuid_feature_extract_signed_field(pfr0,
846 ID_AA64PFR0_FP_SHIFT) < 0;
847}
848
ea1e3de8
WD		 849#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
850static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
851
852static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
853 int __unused)
854{
855 /* Forced on command line? */
856 if (__kpti_forced) {
857 pr_info_once("kernel page table isolation forced %s by command line option\n",
858 __kpti_forced > 0 ? "ON" : "OFF");
859 return __kpti_forced > 0;
860 }
861
862 /* Useful for KASLR robustness */
863 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
864 return true;
865
866 return false;
867}
868
869static int __init parse_kpti(char *str)
870{
871 bool enabled;
872 int ret = strtobool(str, &enabled);
873
874 if (ret)
875 return ret;
876
877 __kpti_forced = enabled ? 1 : -1;
878 return 0;
879}
880__setup("kpti=", parse_kpti);
881#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
882
359b7064 883static const struct arm64_cpu_capabilities arm64_features[] = {
94a9e04a
MZ		 884 {
885 .desc = "GIC system register CPU interface",
886 .capability = ARM64_HAS_SYSREG_GIC_CPUIF,
92406f0c 887 .def_scope = SCOPE_SYSTEM,
963fcd40 888 .matches = has_useable_gicv3_cpuif,
da8d02d1
SP		 889 .sys_reg = SYS_ID_AA64PFR0_EL1,
890 .field_pos = ID_AA64PFR0_GIC_SHIFT,
ff96f7bc 891 .sign = FTR_UNSIGNED,
18ffa046 892 .min_field_value = 1,
94a9e04a 893 },
338d4f49
JM		 894#ifdef CONFIG_ARM64_PAN
895 {
896 .desc = "Privileged Access Never",
897 .capability = ARM64_HAS_PAN,
92406f0c 898 .def_scope = SCOPE_SYSTEM,
da8d02d1
SP		 899 .matches = has_cpuid_feature,
900 .sys_reg = SYS_ID_AA64MMFR1_EL1,
901 .field_pos = ID_AA64MMFR1_PAN_SHIFT,
ff96f7bc 902 .sign = FTR_UNSIGNED,
338d4f49
JM		 903 .min_field_value = 1,
904 .enable = cpu_enable_pan,
905 },
906#endif /* CONFIG_ARM64_PAN */
2e94da13
WD		 907#if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
908 {
909 .desc = "LSE atomic instructions",
910 .capability = ARM64_HAS_LSE_ATOMICS,
92406f0c 911 .def_scope = SCOPE_SYSTEM,
da8d02d1
SP		 912 .matches = has_cpuid_feature,
913 .sys_reg = SYS_ID_AA64ISAR0_EL1,
914 .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT,
ff96f7bc 915 .sign = FTR_UNSIGNED,
2e94da13
WD		 916 .min_field_value = 2,
917 },
918#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
d5370f75
WD		 919 {
920 .desc = "Software prefetching using PRFM",
921 .capability = ARM64_HAS_NO_HW_PREFETCH,
92406f0c 922 .def_scope = SCOPE_SYSTEM,
d5370f75
WD		 923 .matches = has_no_hw_prefetch,
924 },
57f4959b
JM		 925#ifdef CONFIG_ARM64_UAO
926 {
927 .desc = "User Access Override",
928 .capability = ARM64_HAS_UAO,
92406f0c 929 .def_scope = SCOPE_SYSTEM,
57f4959b
JM		 930 .matches = has_cpuid_feature,
931 .sys_reg = SYS_ID_AA64MMFR2_EL1,
932 .field_pos = ID_AA64MMFR2_UAO_SHIFT,
933 .min_field_value = 1,
c8b06e3f
JM		 934 /*
935 * We rely on stop_machine() calling uao_thread_switch() to set
936 * UAO immediately after patching.
937 */
57f4959b
JM		 938 },
939#endif /* CONFIG_ARM64_UAO */
70544196
JM		 940#ifdef CONFIG_ARM64_PAN
941 {
942 .capability = ARM64_ALT_PAN_NOT_UAO,
92406f0c 943 .def_scope = SCOPE_SYSTEM,
70544196
JM		 944 .matches = cpufeature_pan_not_uao,
945 },
946#endif /* CONFIG_ARM64_PAN */
d88701be
MZ		 947 {
948 .desc = "Virtualization Host Extensions",
949 .capability = ARM64_HAS_VIRT_HOST_EXTN,
92406f0c 950 .def_scope = SCOPE_SYSTEM,
d88701be
MZ		 951 .matches = runs_at_el2,
952 },
042446a3
SP		 953 {
954 .desc = "32-bit EL0 Support",
955 .capability = ARM64_HAS_32BIT_EL0,
92406f0c 956 .def_scope = SCOPE_SYSTEM,
042446a3
SP		 957 .matches = has_cpuid_feature,
958 .sys_reg = SYS_ID_AA64PFR0_EL1,
959 .sign = FTR_UNSIGNED,
960 .field_pos = ID_AA64PFR0_EL0_SHIFT,
961 .min_field_value = ID_AA64PFR0_EL0_32BIT_64BIT,
962 },
d1745910
MZ		 963 {
964 .desc = "Reduced HYP mapping offset",
965 .capability = ARM64_HYP_OFFSET_LOW,
966 .def_scope = SCOPE_SYSTEM,
967 .matches = hyp_offset_low,
968 },
ea1e3de8
WD		 969#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
970 {
971 .capability = ARM64_UNMAP_KERNEL_AT_EL0,
972 .def_scope = SCOPE_SYSTEM,
973 .matches = unmap_kernel_at_el0,
974 },
975#endif
82e0191a
SP		 976 {
977 /* FP/SIMD is not implemented */
978 .capability = ARM64_HAS_NO_FPSIMD,
979 .def_scope = SCOPE_SYSTEM,
980 .min_field_value = 0,
981 .matches = has_no_fpsimd,
982 },
d50e071f
RM		 983#ifdef CONFIG_ARM64_PMEM
984 {
985 .desc = "Data cache clean to Point of Persistence",
986 .capability = ARM64_HAS_DCPOP,
987 .def_scope = SCOPE_SYSTEM,
988 .matches = has_cpuid_feature,
989 .sys_reg = SYS_ID_AA64ISAR1_EL1,
990 .field_pos = ID_AA64ISAR1_DPB_SHIFT,
991 .min_field_value = 1,
992 },
993#endif
43994d82
DM		 994#ifdef CONFIG_ARM64_SVE
995 {
996 .desc = "Scalable Vector Extension",
997 .capability = ARM64_SVE,
998 .def_scope = SCOPE_SYSTEM,
999 .sys_reg = SYS_ID_AA64PFR0_EL1,
1000 .sign = FTR_UNSIGNED,
1001 .field_pos = ID_AA64PFR0_SVE_SHIFT,
1002 .min_field_value = ID_AA64PFR0_SVE,
1003 .matches = has_cpuid_feature,
1004 .enable = sve_kernel_enable,
1005 },
1006#endif /* CONFIG_ARM64_SVE */
359b7064
MZ		 1007 {},
1008};
1009
ff96f7bc 1010#define HWCAP_CAP(reg, field, s, min_value, type, cap) \
37b01d53
SP		 1011 { \
1012 .desc = #cap, \
92406f0c 1013 .def_scope = SCOPE_SYSTEM, \
37b01d53
SP		 1014 .matches = has_cpuid_feature, \
1015 .sys_reg = reg, \
1016 .field_pos = field, \
ff96f7bc 1017 .sign = s, \
37b01d53
SP		 1018 .min_field_value = min_value, \
1019 .hwcap_type = type, \
1020 .hwcap = cap, \
1021 }
1022
f3efb675 1023static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
ff96f7bc
SP		 1024 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_PMULL),
1025 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_AES),
1026 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA1),
1027 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA2),
f5e035f8 1028 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_SHA512),
ff96f7bc
SP		 1029 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_CRC32),
1030 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ATOMICS),
f92f5ce0 1031 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RDM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDRDM),
f5e035f8
SP		 1032 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA3),
1033 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM3),
1034 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM4_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM4),
1035 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_DP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDDP),
3b3b6810 1036 HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_FHM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDFHM),
ff96f7bc 1037 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_FP),
bf500618 1038 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
ff96f7bc 1039 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
bf500618 1040 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP),
7aac405e 1041 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_DCPOP),
c8c3798d 1042 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT),
cb567e79 1043 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
c651aae5 1044 HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
43994d82
DM		 1045#ifdef CONFIG_ARM64_SVE
1046 HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE),
1047#endif
75283501
SP		 1048 {},
1049};
1050
1051static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
37b01d53 1052#ifdef CONFIG_COMPAT
ff96f7bc
SP		 1053 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
1054 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
1055 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
1056 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
1057 HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
37b01d53
SP		 1058#endif
1059 {},
1060};
1061
f3efb675 1062static void __init cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap)
37b01d53
SP		 1063{
1064 switch (cap->hwcap_type) {
1065 case CAP_HWCAP:
1066 elf_hwcap |= cap->hwcap;
1067 break;
1068#ifdef CONFIG_COMPAT
1069 case CAP_COMPAT_HWCAP:
1070 compat_elf_hwcap |= (u32)cap->hwcap;
1071 break;
1072 case CAP_COMPAT_HWCAP2:
1073 compat_elf_hwcap2 |= (u32)cap->hwcap;
1074 break;
1075#endif
1076 default:
1077 WARN_ON(1);
1078 break;
1079 }
1080}
1081
1082/* Check if we have a particular HWCAP enabled */
f3efb675 1083static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap)
37b01d53
SP		 1084{
1085 bool rc;
1086
1087 switch (cap->hwcap_type) {
1088 case CAP_HWCAP:
1089 rc = (elf_hwcap & cap->hwcap) != 0;
1090 break;
1091#ifdef CONFIG_COMPAT
1092 case CAP_COMPAT_HWCAP:
1093 rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0;
1094 break;
1095 case CAP_COMPAT_HWCAP2:
1096 rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0;
1097 break;
1098#endif
1099 default:
1100 WARN_ON(1);
1101 rc = false;
1102 }
1103
1104 return rc;
1105}
1106
75283501 1107static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
37b01d53 1108{
77c97b4e
SP		 1109 /* We support emulation of accesses to CPU ID feature registers */
1110 elf_hwcap |= HWCAP_CPUID;
75283501 1111 for (; hwcaps->matches; hwcaps++)
92406f0c 1112 if (hwcaps->matches(hwcaps, hwcaps->def_scope))
75283501 1113 cap_set_elf_hwcap(hwcaps);
37b01d53
SP		 1114}
1115
ce8b602c 1116void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
359b7064
MZ		 1117 const char *info)
1118{
75283501 1119 for (; caps->matches; caps++) {
92406f0c 1120 if (!caps->matches(caps, caps->def_scope))
359b7064
MZ		 1121 continue;
1122
75283501
SP		 1123 if (!cpus_have_cap(caps->capability) && caps->desc)
1124 pr_info("%s %s\n", info, caps->desc);
1125 cpus_set_cap(caps->capability);
359b7064 1126 }
ce8b602c
SP		 1127}
1128
1129/*
dbb4e152
SP		 1130 * Run through the enabled capabilities and enable() it on all active
1131 * CPUs
ce8b602c 1132 */
8e231852 1133void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
ce8b602c 1134{
63a1e1c9
MR		 1135 for (; caps->matches; caps++) {
1136 unsigned int num = caps->capability;
1137
1138 if (!cpus_have_cap(num))
1139 continue;
1140
1141 /* Ensure cpus_have_const_cap(num) works */
1142 static_branch_enable(&cpu_hwcap_keys[num]);
1143
1144 if (caps->enable) {
2a6dcb2b
JM		 1145 /*
1146 * Use stop_machine() as it schedules the work allowing
1147 * us to modify PSTATE, instead of on_each_cpu() which
1148 * uses an IPI, giving us a PSTATE that disappears when
1149 * we return.
1150 */
1151 stop_machine(caps->enable, NULL, cpu_online_mask);
63a1e1c9
MR		 1152 }
1153 }
dbb4e152
SP		 1154}
1155
dbb4e152 1156/*
13f417f3
SP		 1157 * Check for CPU features that are used in early boot
1158 * based on the Boot CPU value.
dbb4e152 1159 */
13f417f3 1160static void check_early_cpu_features(void)
dbb4e152 1161{
ac1ad20f 1162 verify_cpu_run_el();
13f417f3 1163 verify_cpu_asid_bits();
dbb4e152 1164}
1c076303 1165
75283501
SP		 1166static void
1167verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
1168{
1169
92406f0c
SP		 1170 for (; caps->matches; caps++)
1171 if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) {
75283501
SP		 1172 pr_crit("CPU%d: missing HWCAP: %s\n",
1173 smp_processor_id(), caps->desc);
1174 cpu_die_early();
1175 }
75283501
SP		 1176}
1177
1178static void
1179verify_local_cpu_features(const struct arm64_cpu_capabilities *caps)
1180{
1181 for (; caps->matches; caps++) {
92406f0c 1182 if (!cpus_have_cap(caps->capability))
75283501
SP		 1183 continue;
1184 /*
1185 * If the new CPU misses an advertised feature, we cannot proceed
1186 * further, park the cpu.
1187 */
92406f0c 1188 if (!caps->matches(caps, SCOPE_LOCAL_CPU)) {
75283501
SP		 1189 pr_crit("CPU%d: missing feature: %s\n",
1190 smp_processor_id(), caps->desc);
1191 cpu_die_early();
1192 }
1193 if (caps->enable)
1194 caps->enable(NULL);
1195 }
1196}
1197
2e0f2478
DM		 1198static void verify_sve_features(void)
1199{
1200 u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
1201 u64 zcr = read_zcr_features();
1202
1203 unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;
1204 unsigned int len = zcr & ZCR_ELx_LEN_MASK;
1205
1206 if (len < safe_len || sve_verify_vq_map()) {
1207 pr_crit("CPU%d: SVE: required vector length(s) missing\n",
1208 smp_processor_id());
1209 cpu_die_early();
1210 }
1211
1212 /* Add checks on other ZCR bits here if necessary */
1213}
1214
dbb4e152
SP		 1215/*
1216 * Run through the enabled system capabilities and enable() it on this CPU.
1217 * The capabilities were decided based on the available CPUs at the boot time.
1218 * Any new CPU should match the system wide status of the capability. If the
1219 * new CPU doesn't have a capability which the system now has enabled, we
1220 * cannot do anything to fix it up and could cause unexpected failures. So
1221 * we park the CPU.
1222 */
c47a1900 1223static void verify_local_cpu_capabilities(void)
dbb4e152 1224{
c47a1900
SP		 1225 verify_local_cpu_errata_workarounds();
1226 verify_local_cpu_features(arm64_features);
1227 verify_local_elf_hwcaps(arm64_elf_hwcaps);
2e0f2478 1228
c47a1900
SP		 1229 if (system_supports_32bit_el0())
1230 verify_local_elf_hwcaps(compat_elf_hwcaps);
2e0f2478
DM		 1231
1232 if (system_supports_sve())
1233 verify_sve_features();
c47a1900 1234}
dbb4e152 1235
c47a1900
SP		 1236void check_local_cpu_capabilities(void)
1237{
1238 /*
1239 * All secondary CPUs should conform to the early CPU features
1240 * in use by the kernel based on boot CPU.
1241 */
13f417f3
SP		 1242 check_early_cpu_features();
1243
dbb4e152 1244 /*
c47a1900
SP		 1245 * If we haven't finalised the system capabilities, this CPU gets
1246 * a chance to update the errata work arounds.
1247 * Otherwise, this CPU should verify that it has all the system
1248 * advertised capabilities.
dbb4e152
SP		 1249 */
1250 if (!sys_caps_initialised)
c47a1900
SP		 1251 update_cpu_errata_workarounds();
1252 else
1253 verify_local_cpu_capabilities();
359b7064
MZ		 1254}
1255
a7c61a34 1256static void __init setup_feature_capabilities(void)
359b7064 1257{
ce8b602c
SP		 1258 update_cpu_capabilities(arm64_features, "detected feature:");
1259 enable_cpu_capabilities(arm64_features);
359b7064
MZ		 1260}
1261
63a1e1c9
MR		 1262DEFINE_STATIC_KEY_FALSE(arm64_const_caps_ready);
1263EXPORT_SYMBOL(arm64_const_caps_ready);
1264
1265static void __init mark_const_caps_ready(void)
1266{
1267 static_branch_enable(&arm64_const_caps_ready);
1268}
1269
e3661b12
MZ		 1270/*
1271 * Check if the current CPU has a given feature capability.
1272 * Should be called from non-preemptible context.
1273 */
8f413758
MZ		 1274static bool __this_cpu_has_cap(const struct arm64_cpu_capabilities *cap_array,
1275 unsigned int cap)
e3661b12
MZ		 1276{
1277 const struct arm64_cpu_capabilities *caps;
1278
1279 if (WARN_ON(preemptible()))
1280 return false;
1281
8f413758 1282 for (caps = cap_array; caps->desc; caps++)
e3661b12
MZ		 1283 if (caps->capability == cap && caps->matches)
1284 return caps->matches(caps, SCOPE_LOCAL_CPU);
1285
1286 return false;
1287}
1288
8f413758
MZ		 1289extern const struct arm64_cpu_capabilities arm64_errata[];
1290
1291bool this_cpu_has_cap(unsigned int cap)
1292{
1293 return (__this_cpu_has_cap(arm64_features, cap) ||
1294 __this_cpu_has_cap(arm64_errata, cap));
1295}
1296
9cdf8ec4 1297void __init setup_cpu_features(void)
359b7064 1298{
9cdf8ec4
SP		 1299 u32 cwg;
1300 int cls;
1301
dbb4e152
SP		 1302 /* Set the CPU feature capabilies */
1303 setup_feature_capabilities();
8e231852 1304 enable_errata_workarounds();
63a1e1c9 1305 mark_const_caps_ready();
75283501 1306 setup_elf_hwcaps(arm64_elf_hwcaps);
643d703d
SP		 1307
1308 if (system_supports_32bit_el0())
1309 setup_elf_hwcaps(compat_elf_hwcaps);
dbb4e152 1310
2e0f2478
DM		 1311 sve_setup();
1312
dbb4e152
SP		 1313 /* Advertise that we have computed the system capabilities */
1314 set_sys_caps_initialised();
1315
9cdf8ec4
SP		 1316 /*
1317 * Check for sane CTR_EL0.CWG value.
1318 */
1319 cwg = cache_type_cwg();
1320 cls = cache_line_size();
1321 if (!cwg)
1322 pr_warn("No Cache Writeback Granule information, assuming cache line size %d\n",
1323 cls);
1324 if (L1_CACHE_BYTES < cls)
1325 pr_warn("L1_CACHE_BYTES smaller than the Cache Writeback Granule (%d < %d)\n",
1326 L1_CACHE_BYTES, cls);
359b7064 1327}
70544196
JM		 1328
1329static bool __maybe_unused
92406f0c 1330cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused)
70544196 1331{
a4023f68 1332 return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO));
70544196 1333}
77c97b4e
SP		 1334
1335/*
1336 * We emulate only the following system register space.
1337 * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 4 - 7]
1338 * See Table C5-6 System instruction encodings for System register accesses,
1339 * ARMv8 ARM(ARM DDI 0487A.f) for more details.
1340 */
1341static inline bool __attribute_const__ is_emulated(u32 id)
1342{
1343 return (sys_reg_Op0(id) == 0x3 &&
1344 sys_reg_CRn(id) == 0x0 &&
1345 sys_reg_Op1(id) == 0x0 &&
1346 (sys_reg_CRm(id) == 0 ||
1347 ((sys_reg_CRm(id) >= 4) && (sys_reg_CRm(id) <= 7))));
1348}
1349
1350/*
1351 * With CRm == 0, reg should be one of :
1352 * MIDR_EL1, MPIDR_EL1 or REVIDR_EL1.
1353 */
1354static inline int emulate_id_reg(u32 id, u64 *valp)
1355{
1356 switch (id) {
1357 case SYS_MIDR_EL1:
1358 *valp = read_cpuid_id();
1359 break;
1360 case SYS_MPIDR_EL1:
1361 *valp = SYS_MPIDR_SAFE_VAL;
1362 break;
1363 case SYS_REVIDR_EL1:
1364 /* IMPLEMENTATION DEFINED values are emulated with 0 */
1365 *valp = 0;
1366 break;
1367 default:
1368 return -EINVAL;
1369 }
1370
1371 return 0;
1372}
1373
1374static int emulate_sys_reg(u32 id, u64 *valp)
1375{
1376 struct arm64_ftr_reg *regp;
1377
1378 if (!is_emulated(id))
1379 return -EINVAL;
1380
1381 if (sys_reg_CRm(id) == 0)
1382 return emulate_id_reg(id, valp);
1383
1384 regp = get_arm64_ftr_reg(id);
1385 if (regp)
1386 *valp = arm64_ftr_reg_user_value(regp);
1387 else
1388 /*
1389 * The untracked registers are either IMPLEMENTATION DEFINED
1390 * (e.g, ID_AFR0_EL1) or reserved RAZ.
1391 */
1392 *valp = 0;
1393 return 0;
1394}
1395
1396static int emulate_mrs(struct pt_regs *regs, u32 insn)
1397{
1398 int rc;
1399 u32 sys_reg, dst;
1400 u64 val;
1401
1402 /*
1403 * sys_reg values are defined as used in mrs/msr instruction.
1404 * shift the imm value to get the encoding.
1405 */
1406 sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
1407 rc = emulate_sys_reg(sys_reg, &val);
1408 if (!rc) {
1409 dst = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
521c6461 1410 pt_regs_write_reg(regs, dst, val);
6436beee 1411 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
77c97b4e
SP		 1412 }
1413
1414 return rc;
1415}
1416
1417static struct undef_hook mrs_hook = {
1418 .instr_mask = 0xfff00000,
1419 .instr_val = 0xd5300000,
1420 .pstate_mask = COMPAT_PSR_MODE_MASK,
1421 .pstate_val = PSR_MODE_EL0t,
1422 .fn = emulate_mrs,
1423};
1424
1425static int __init enable_mrs_emulation(void)
1426{
1427 register_undef_hook(&mrs_hook);
1428 return 0;
1429}
1430
c0d8832e 1431core_initcall(enable_mrs_emulation);
Linux 6.x block layer and io_uring tree(s)