projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'kvm-x86-misc-6.9' of https://github.com/kvm-x86/linux into HEAD
[linux-2.6-block.git] / arch / arm64 / kvm / sys_regs.c
CommitLineData
caab277b 1// SPDX-License-Identifier: GPL-2.0-only
7c8c5e6a
MZ		 2/*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 *
6 * Derived from arch/arm/kvm/coproc.c:
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Authors: Rusty Russell <rusty@rustcorp.com.au>
9 * Christoffer Dall <c.dall@virtualopensystems.com>
7c8c5e6a
MZ		 10 */
11
c8857935 12#include <linux/bitfield.h>
623eefa8 13#include <linux/bsearch.h>
7af0c253 14#include <linux/cacheinfo.h>
89176658 15#include <linux/debugfs.h>
7c8c5e6a 16#include <linux/kvm_host.h>
c6d01a94 17#include <linux/mm.h>
07d79fe7 18#include <linux/printk.h>
7c8c5e6a 19#include <linux/uaccess.h>
c6d01a94 20
7c8c5e6a
MZ		 21#include <asm/cacheflush.h>
22#include <asm/cputype.h>
0c557ed4 23#include <asm/debug-monitors.h>
c6d01a94
MR		 24#include <asm/esr.h>
25#include <asm/kvm_arm.h>
c6d01a94 26#include <asm/kvm_emulate.h>
d47533da 27#include <asm/kvm_hyp.h>
c6d01a94 28#include <asm/kvm_mmu.h>
6ff9dc23 29#include <asm/kvm_nested.h>
ab946834 30#include <asm/perf_event.h>
1f3d8699 31#include <asm/sysreg.h>
c6d01a94 32
7c8c5e6a
MZ		 33#include <trace/events/kvm.h>
34
b80b701d 35#include "check-res-bits.h"
7c8c5e6a
MZ		 36#include "sys_regs.h"
37
eef8c85a
AB		 38#include "trace.h"
39
7c8c5e6a 40/*
62a89c44
MZ		 41 * For AArch32, we only take care of what is being trapped. Anything
42 * that has to do with init and userspace access has to go via the
43 * 64bit interface.
7c8c5e6a
MZ		 44 */
45
f24adc65 46static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
c118cead
JZ		 47static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
48 u64 val);
f24adc65 49
2733dd10
MZ		 50static bool bad_trap(struct kvm_vcpu *vcpu,
51 struct sys_reg_params *params,
52 const struct sys_reg_desc *r,
53 const char *msg)
7b5b4df1 54{
2733dd10 55 WARN_ONCE(1, "Unexpected %s\n", msg);
7b5b4df1
MZ		 56 print_sys_reg_instr(params);
57 kvm_inject_undefined(vcpu);
58 return false;
59}
60
2733dd10
MZ		 61static bool read_from_write_only(struct kvm_vcpu *vcpu,
62 struct sys_reg_params *params,
63 const struct sys_reg_desc *r)
64{
65 return bad_trap(vcpu, params, r,
66 "sys_reg read to write-only register");
67}
68
7b1dba1f
MZ		 69static bool write_to_read_only(struct kvm_vcpu *vcpu,
70 struct sys_reg_params *params,
71 const struct sys_reg_desc *r)
72{
2733dd10
MZ		 73 return bad_trap(vcpu, params, r,
74 "sys_reg write to read-only register");
7b1dba1f
MZ		 75}
76
fedc6123
MZ		 77#define PURE_EL2_SYSREG(el2) \
78 case el2: { \
79 *el1r = el2; \
80 return true; \
81 }
82
83#define MAPPED_EL2_SYSREG(el2, el1, fn) \
84 case el2: { \
85 *xlate = fn; \
86 *el1r = el1; \
87 return true; \
88 }
89
90static bool get_el2_to_el1_mapping(unsigned int reg,
91 unsigned int *el1r, u64 (**xlate)(u64))
92{
93 switch (reg) {
94 PURE_EL2_SYSREG( VPIDR_EL2 );
95 PURE_EL2_SYSREG( VMPIDR_EL2 );
96 PURE_EL2_SYSREG( ACTLR_EL2 );
97 PURE_EL2_SYSREG( HCR_EL2 );
98 PURE_EL2_SYSREG( MDCR_EL2 );
99 PURE_EL2_SYSREG( HSTR_EL2 );
100 PURE_EL2_SYSREG( HACR_EL2 );
101 PURE_EL2_SYSREG( VTTBR_EL2 );
102 PURE_EL2_SYSREG( VTCR_EL2 );
103 PURE_EL2_SYSREG( RVBAR_EL2 );
104 PURE_EL2_SYSREG( TPIDR_EL2 );
105 PURE_EL2_SYSREG( HPFAR_EL2 );
106 PURE_EL2_SYSREG( CNTHCTL_EL2 );
107 MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1,
108 translate_sctlr_el2_to_sctlr_el1 );
109 MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1,
110 translate_cptr_el2_to_cpacr_el1 );
111 MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1,
112 translate_ttbr0_el2_to_ttbr0_el1 );
113 MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1, NULL );
114 MAPPED_EL2_SYSREG(TCR_EL2, TCR_EL1,
115 translate_tcr_el2_to_tcr_el1 );
116 MAPPED_EL2_SYSREG(VBAR_EL2, VBAR_EL1, NULL );
117 MAPPED_EL2_SYSREG(AFSR0_EL2, AFSR0_EL1, NULL );
118 MAPPED_EL2_SYSREG(AFSR1_EL2, AFSR1_EL1, NULL );
119 MAPPED_EL2_SYSREG(ESR_EL2, ESR_EL1, NULL );
120 MAPPED_EL2_SYSREG(FAR_EL2, FAR_EL1, NULL );
121 MAPPED_EL2_SYSREG(MAIR_EL2, MAIR_EL1, NULL );
122 MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL );
123 MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL );
124 MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
125 default:
126 return false;
127 }
7b1dba1f
MZ		 128}
129
7ea90bdd
MZ		 130u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
131{
132 u64 val = 0x8badf00d8badf00d;
fedc6123
MZ		 133 u64 (*xlate)(u64) = NULL;
134 unsigned int el1r;
135
136 if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
137 goto memory_read;
7ea90bdd 138
fedc6123
MZ		 139 if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
140 if (!is_hyp_ctxt(vcpu))
141 goto memory_read;
142
143 /*
144 * If this register does not have an EL1 counterpart,
145 * then read the stored EL2 version.
146 */
147 if (reg == el1r)
148 goto memory_read;
149
150 /*
151 * If we have a non-VHE guest and that the sysreg
152 * requires translation to be used at EL1, use the
153 * in-memory copy instead.
154 */
155 if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
156 goto memory_read;
157
158 /* Get the current version of the EL1 counterpart. */
159 WARN_ON(!__vcpu_read_sys_reg_from_cpu(el1r, &val));
7ea90bdd 160 return val;
fedc6123 161 }
7ea90bdd 162
fedc6123
MZ		 163 /* EL1 register can't be on the CPU if the guest is in vEL2. */
164 if (unlikely(is_hyp_ctxt(vcpu)))
165 goto memory_read;
166
167 if (__vcpu_read_sys_reg_from_cpu(reg, &val))
168 return val;
169
170memory_read:
7ea90bdd
MZ		 171 return __vcpu_sys_reg(vcpu, reg);
172}
173
174void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
175{
fedc6123
MZ		 176 u64 (*xlate)(u64) = NULL;
177 unsigned int el1r;
178
179 if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
180 goto memory_write;
181
182 if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
183 if (!is_hyp_ctxt(vcpu))
184 goto memory_write;
185
186 /*
187 * Always store a copy of the write to memory to avoid having
188 * to reverse-translate virtual EL2 system registers for a
189 * non-VHE guest hypervisor.
190 */
191 __vcpu_sys_reg(vcpu, reg) = val;
192
193 /* No EL1 counterpart? We're done here.? */
194 if (reg == el1r)
195 return;
196
197 if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
198 val = xlate(val);
199
200 /* Redirect this to the EL1 version of the register. */
201 WARN_ON(!__vcpu_write_sys_reg_to_cpu(val, el1r));
202 return;
203 }
204
205 /* EL1 register can't be on the CPU if the guest is in vEL2. */
206 if (unlikely(is_hyp_ctxt(vcpu)))
207 goto memory_write;
208
209 if (__vcpu_write_sys_reg_to_cpu(val, reg))
7ea90bdd
MZ		 210 return;
211
fedc6123
MZ		 212memory_write:
213 __vcpu_sys_reg(vcpu, reg) = val;
d47533da
CD		 214}
215
7c8c5e6a 216/* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
c73a4416 217#define CSSELR_MAX 14
7c8c5e6a 218
7af0c253
AO		 219/*
220 * Returns the minimum line size for the selected cache, expressed as
221 * Log2(bytes).
222 */
223static u8 get_min_cache_line_size(bool icache)
224{
225 u64 ctr = read_sanitised_ftr_reg(SYS_CTR_EL0);
226 u8 field;
227
228 if (icache)
229 field = SYS_FIELD_GET(CTR_EL0, IminLine, ctr);
230 else
231 field = SYS_FIELD_GET(CTR_EL0, DminLine, ctr);
232
233 /*
234 * Cache line size is represented as Log2(words) in CTR_EL0.
235 * Log2(bytes) can be derived with the following:
236 *
237 * Log2(words) + 2 = Log2(bytes / 4) + 2
238 * = Log2(bytes) - 2 + 2
239 * = Log2(bytes)
240 */
241 return field + 2;
242}
243
7c8c5e6a 244/* Which cache CCSIDR represents depends on CSSELR value. */
7af0c253
AO		 245static u32 get_ccsidr(struct kvm_vcpu *vcpu, u32 csselr)
246{
247 u8 line_size;
248
249 if (vcpu->arch.ccsidr)
250 return vcpu->arch.ccsidr[csselr];
251
252 line_size = get_min_cache_line_size(csselr & CSSELR_EL1_InD);
253
254 /*
255 * Fabricate a CCSIDR value as the overriding value does not exist.
256 * The real CCSIDR value will not be used as it can vary by the
257 * physical CPU which the vcpu currently resides in.
258 *
259 * The line size is determined with get_min_cache_line_size(), which
260 * should be valid for all CPUs even if they have different cache
261 * configuration.
262 *
263 * The associativity bits are cleared, meaning the geometry of all data
264 * and unified caches (which are guaranteed to be PIPT and thus
265 * non-aliasing) are 1 set and 1 way.
266 * Guests should not be doing cache operations by set/way at all, and
267 * for this reason, we trap them and attempt to infer the intent, so
268 * that we can flush the entire guest's address space at the appropriate
269 * time. The exposed geometry minimizes the number of the traps.
270 * [If guests should attempt to infer aliasing properties from the
271 * geometry (which is not permitted by the architecture), they would
272 * only do so for virtually indexed caches.]
273 *
274 * We don't check if the cache level exists as it is allowed to return
275 * an UNKNOWN value if not.
276 */
277 return SYS_FIELD_PREP(CCSIDR_EL1, LineSize, line_size - 4);
278}
279
280static int set_ccsidr(struct kvm_vcpu *vcpu, u32 csselr, u32 val)
7c8c5e6a 281{
7af0c253
AO		 282 u8 line_size = FIELD_GET(CCSIDR_EL1_LineSize, val) + 4;
283 u32 *ccsidr = vcpu->arch.ccsidr;
284 u32 i;
285
286 if ((val & CCSIDR_EL1_RES0) ||
287 line_size < get_min_cache_line_size(csselr & CSSELR_EL1_InD))
288 return -EINVAL;
289
290 if (!ccsidr) {
291 if (val == get_ccsidr(vcpu, csselr))
292 return 0;
7c8c5e6a 293
5f623a59 294 ccsidr = kmalloc_array(CSSELR_MAX, sizeof(u32), GFP_KERNEL_ACCOUNT);
7af0c253
AO		 295 if (!ccsidr)
296 return -ENOMEM;
7c8c5e6a 297
7af0c253
AO		 298 for (i = 0; i < CSSELR_MAX; i++)
299 ccsidr[i] = get_ccsidr(vcpu, i);
300
301 vcpu->arch.ccsidr = ccsidr;
302 }
7c8c5e6a 303
7af0c253 304 ccsidr[csselr] = val;
7c8c5e6a 305
7af0c253 306 return 0;
7c8c5e6a
MZ		 307}
308
6ff9dc23
JL		 309static bool access_rw(struct kvm_vcpu *vcpu,
310 struct sys_reg_params *p,
311 const struct sys_reg_desc *r)
312{
313 if (p->is_write)
314 vcpu_write_sys_reg(vcpu, p->regval, r->reg);
315 else
316 p->regval = vcpu_read_sys_reg(vcpu, r->reg);
317
318 return true;
319}
320
3c1e7165
MZ		 321/*
322 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
323 */
7c8c5e6a 324static bool access_dcsw(struct kvm_vcpu *vcpu,
3fec037d 325 struct sys_reg_params *p,
7c8c5e6a
MZ		 326 const struct sys_reg_desc *r)
327{
7c8c5e6a 328 if (!p->is_write)
e7f1d1ee 329 return read_from_write_only(vcpu, p, r);
7c8c5e6a 330
09605e94
MZ		 331 /*
332 * Only track S/W ops if we don't have FWB. It still indicates
333 * that the guest is a bit broken (S/W operations should only
334 * be done by firmware, knowing that there is only a single
335 * CPU left in the system, and certainly not from non-secure
336 * software).
337 */
d8569fba 338 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
09605e94
MZ		 339 kvm_set_way_flush(vcpu);
340
7c8c5e6a
MZ		 341 return true;
342}
343
d282fa3c
MZ		 344static bool access_dcgsw(struct kvm_vcpu *vcpu,
345 struct sys_reg_params *p,
346 const struct sys_reg_desc *r)
347{
348 if (!kvm_has_mte(vcpu->kvm)) {
349 kvm_inject_undefined(vcpu);
350 return false;
351 }
352
353 /* Treat MTE S/W ops as we treat the classic ones: with contempt */
354 return access_dcsw(vcpu, p, r);
355}
356
b1ea1d76
MZ		 357static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
358{
359 switch (r->aarch32_map) {
360 case AA32_LO:
361 *mask = GENMASK_ULL(31, 0);
362 *shift = 0;
363 break;
364 case AA32_HI:
365 *mask = GENMASK_ULL(63, 32);
366 *shift = 32;
367 break;
368 default:
369 *mask = GENMASK_ULL(63, 0);
370 *shift = 0;
371 break;
372 }
373}
374
4d44923b
MZ		 375/*
376 * Generic accessor for VM registers. Only called as long as HCR_TVM
3c1e7165
MZ		 377 * is set. If the guest enables the MMU, we stop trapping the VM
378 * sys_regs and leave it in complete control of the caches.
4d44923b
MZ		 379 */
380static bool access_vm_reg(struct kvm_vcpu *vcpu,
3fec037d 381 struct sys_reg_params *p,
4d44923b
MZ		 382 const struct sys_reg_desc *r)
383{
3c1e7165 384 bool was_enabled = vcpu_has_cache_enabled(vcpu);
b1ea1d76 385 u64 val, mask, shift;
4d44923b
MZ		 386
387 BUG_ON(!p->is_write);
388
b1ea1d76 389 get_access_mask(r, &mask, &shift);
52f6c4f0 390
b1ea1d76
MZ		 391 if (~mask) {
392 val = vcpu_read_sys_reg(vcpu, r->reg);
393 val &= ~mask;
dedf97e8 394 } else {
b1ea1d76 395 val = 0;
dedf97e8 396 }
b1ea1d76
MZ		 397
398 val |= (p->regval & (mask >> shift)) << shift;
399 vcpu_write_sys_reg(vcpu, val, r->reg);
f0a3eaff 400
3c1e7165 401 kvm_toggle_cache(vcpu, was_enabled);
4d44923b
MZ		 402 return true;
403}
404
af473829
JM		 405static bool access_actlr(struct kvm_vcpu *vcpu,
406 struct sys_reg_params *p,
407 const struct sys_reg_desc *r)
408{
b1ea1d76
MZ		 409 u64 mask, shift;
410
af473829
JM		 411 if (p->is_write)
412 return ignore_write(vcpu, p);
413
b1ea1d76
MZ		 414 get_access_mask(r, &mask, &shift);
415 p->regval = (vcpu_read_sys_reg(vcpu, r->reg) & mask) >> shift;
af473829
JM		 416
417 return true;
418}
419
6d52f35a
AP		 420/*
421 * Trap handler for the GICv3 SGI generation system register.
422 * Forward the request to the VGIC emulation.
423 * The cp15_64 code makes sure this automatically works
424 * for both AArch64 and AArch32 accesses.
425 */
426static bool access_gic_sgi(struct kvm_vcpu *vcpu,
3fec037d 427 struct sys_reg_params *p,
6d52f35a
AP		 428 const struct sys_reg_desc *r)
429{
03bd646d
MZ		 430 bool g1;
431
6d52f35a 432 if (!p->is_write)
e7f1d1ee 433 return read_from_write_only(vcpu, p, r);
6d52f35a 434
03bd646d
MZ		 435 /*
436 * In a system where GICD_CTLR.DS=1, a ICC_SGI0R_EL1 access generates
437 * Group0 SGIs only, while ICC_SGI1R_EL1 can generate either group,
438 * depending on the SGI configuration. ICC_ASGI1R_EL1 is effectively
439 * equivalent to ICC_SGI0R_EL1, as there is no "alternative" secure
440 * group.
441 */
50f30453 442 if (p->Op0 == 0) { /* AArch32 */
03bd646d
MZ		 443 switch (p->Op1) {
444 default: /* Keep GCC quiet */
445 case 0: /* ICC_SGI1R */
446 g1 = true;
447 break;
448 case 1: /* ICC_ASGI1R */
449 case 2: /* ICC_SGI0R */
450 g1 = false;
451 break;
452 }
50f30453 453 } else { /* AArch64 */
03bd646d
MZ		 454 switch (p->Op2) {
455 default: /* Keep GCC quiet */
456 case 5: /* ICC_SGI1R_EL1 */
457 g1 = true;
458 break;
459 case 6: /* ICC_ASGI1R_EL1 */
460 case 7: /* ICC_SGI0R_EL1 */
461 g1 = false;
462 break;
463 }
464 }
465
466 vgic_v3_dispatch_sgi(vcpu, p->regval, g1);
6d52f35a
AP		 467
468 return true;
469}
470
b34f2bcb
MZ		 471static bool access_gic_sre(struct kvm_vcpu *vcpu,
472 struct sys_reg_params *p,
473 const struct sys_reg_desc *r)
474{
475 if (p->is_write)
476 return ignore_write(vcpu, p);
477
478 p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
479 return true;
480}
481
7609c125 482static bool trap_raz_wi(struct kvm_vcpu *vcpu,
3fec037d 483 struct sys_reg_params *p,
7609c125 484 const struct sys_reg_desc *r)
7c8c5e6a
MZ		 485{
486 if (p->is_write)
487 return ignore_write(vcpu, p);
488 else
489 return read_zero(vcpu, p);
490}
491
6ff9dc23
JL		 492static bool trap_undef(struct kvm_vcpu *vcpu,
493 struct sys_reg_params *p,
494 const struct sys_reg_desc *r)
495{
496 kvm_inject_undefined(vcpu);
497 return false;
498}
499
22925521
MZ		 500/*
501 * ARMv8.1 mandates at least a trivial LORegion implementation, where all the
502 * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0
503 * system, these registers should UNDEF. LORID_EL1 being a RO register, we
504 * treat it separately.
505 */
506static bool trap_loregion(struct kvm_vcpu *vcpu,
507 struct sys_reg_params *p,
508 const struct sys_reg_desc *r)
cc33c4e2 509{
7ba8b438 510 u32 sr = reg_to_encoding(r);
22925521 511
c62d7a23 512 if (!kvm_has_feat(vcpu->kvm, ID_AA64MMFR1_EL1, LO, IMP)) {
22925521
MZ		 513 kvm_inject_undefined(vcpu);
514 return false;
515 }
516
517 if (p->is_write && sr == SYS_LORID_EL1)
518 return write_to_read_only(vcpu, p, r);
519
520 return trap_raz_wi(vcpu, p, r);
cc33c4e2
MR		 521}
522
f24adc65
OU		 523static bool trap_oslar_el1(struct kvm_vcpu *vcpu,
524 struct sys_reg_params *p,
525 const struct sys_reg_desc *r)
526{
527 u64 oslsr;
528
529 if (!p->is_write)
530 return read_from_write_only(vcpu, p, r);
531
532 /* Forward the OSLK bit to OSLSR */
187de7c2
MB		 533 oslsr = __vcpu_sys_reg(vcpu, OSLSR_EL1) & ~OSLSR_EL1_OSLK;
534 if (p->regval & OSLAR_EL1_OSLK)
535 oslsr |= OSLSR_EL1_OSLK;
f24adc65
OU		 536
537 __vcpu_sys_reg(vcpu, OSLSR_EL1) = oslsr;
538 return true;
539}
540
0c557ed4 541static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
3fec037d 542 struct sys_reg_params *p,
0c557ed4
MZ		 543 const struct sys_reg_desc *r)
544{
d42e2671 545 if (p->is_write)
e2ffceaa 546 return write_to_read_only(vcpu, p, r);
d42e2671
OU		 547
548 p->regval = __vcpu_sys_reg(vcpu, r->reg);
549 return true;
550}
551
552static int set_oslsr_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 553 u64 val)
d42e2671 554{
f24adc65
OU		 555 /*
556 * The only modifiable bit is the OSLK bit. Refuse the write if
557 * userspace attempts to change any other bit in the register.
558 */
187de7c2 559 if ((val ^ rd->val) & ~OSLSR_EL1_OSLK)
d42e2671
OU		 560 return -EINVAL;
561
f24adc65 562 __vcpu_sys_reg(vcpu, rd->reg) = val;
d42e2671 563 return 0;
0c557ed4
MZ		 564}
565
566static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
3fec037d 567 struct sys_reg_params *p,
0c557ed4
MZ		 568 const struct sys_reg_desc *r)
569{
570 if (p->is_write) {
571 return ignore_write(vcpu, p);
572 } else {
1f3d8699 573 p->regval = read_sysreg(dbgauthstatus_el1);
0c557ed4
MZ		 574 return true;
575 }
576}
577
578/*
579 * We want to avoid world-switching all the DBG registers all the
580 * time:
e6bc555c 581 *
0c557ed4
MZ		 582 * - If we've touched any debug register, it is likely that we're
583 * going to touch more of them. It then makes sense to disable the
584 * traps and start doing the save/restore dance
585 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
586 * then mandatory to save/restore the registers, as the guest
587 * depends on them.
e6bc555c 588 *
0c557ed4
MZ		 589 * For this, we use a DIRTY bit, indicating the guest has modified the
590 * debug registers, used as follow:
591 *
592 * On guest entry:
593 * - If the dirty bit is set (because we're coming back from trapping),
594 * disable the traps, save host registers, restore guest registers.
595 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
596 * set the dirty bit, disable the traps, save host registers,
597 * restore guest registers.
598 * - Otherwise, enable the traps
599 *
600 * On guest exit:
601 * - If the dirty bit is set, save guest registers, restore host
602 * registers and clear the dirty bit. This ensure that the host can
603 * now use the debug registers.
604 */
605static bool trap_debug_regs(struct kvm_vcpu *vcpu,
3fec037d 606 struct sys_reg_params *p,
0c557ed4
MZ		 607 const struct sys_reg_desc *r)
608{
6ff9dc23
JL		 609 access_rw(vcpu, p, r);
610 if (p->is_write)
b1da4908 611 vcpu_set_flag(vcpu, DEBUG_DIRTY);
0c557ed4 612
2ec5be3d 613 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
eef8c85a 614
0c557ed4
MZ		 615 return true;
616}
617
84e690bf
AB		 618/*
619 * reg_to_dbg/dbg_to_reg
620 *
621 * A 32 bit write to a debug register leave top bits alone
622 * A 32 bit read from a debug register only returns the bottom bits
623 *
b1da4908
MZ		 624 * All writes will set the DEBUG_DIRTY flag to ensure the hyp code
625 * switches between host and guest values in future.
84e690bf 626 */
281243cb
MZ		 627static void reg_to_dbg(struct kvm_vcpu *vcpu,
628 struct sys_reg_params *p,
1da42c34 629 const struct sys_reg_desc *rd,
281243cb 630 u64 *dbg_reg)
84e690bf 631{
1da42c34 632 u64 mask, shift, val;
84e690bf 633
1da42c34 634 get_access_mask(rd, &mask, &shift);
84e690bf 635
1da42c34
MZ		 636 val = *dbg_reg;
637 val &= ~mask;
638 val |= (p->regval & (mask >> shift)) << shift;
84e690bf 639 *dbg_reg = val;
1da42c34 640
b1da4908 641 vcpu_set_flag(vcpu, DEBUG_DIRTY);
84e690bf
AB		 642}
643
281243cb
MZ		 644static void dbg_to_reg(struct kvm_vcpu *vcpu,
645 struct sys_reg_params *p,
1da42c34 646 const struct sys_reg_desc *rd,
281243cb 647 u64 *dbg_reg)
84e690bf 648{
1da42c34
MZ		 649 u64 mask, shift;
650
651 get_access_mask(rd, &mask, &shift);
652 p->regval = (*dbg_reg & mask) >> shift;
84e690bf
AB		 653}
654
281243cb
MZ		 655static bool trap_bvr(struct kvm_vcpu *vcpu,
656 struct sys_reg_params *p,
657 const struct sys_reg_desc *rd)
84e690bf 658{
cb853ded 659 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
84e690bf
AB		 660
661 if (p->is_write)
1da42c34 662 reg_to_dbg(vcpu, p, rd, dbg_reg);
84e690bf 663 else
1da42c34 664 dbg_to_reg(vcpu, p, rd, dbg_reg);
84e690bf 665
cb853ded 666 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
eef8c85a 667
84e690bf
AB		 668 return true;
669}
670
671static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 672 u64 val)
84e690bf 673{
978ceeb3 674 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = val;
84e690bf
AB		 675 return 0;
676}
677
678static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 679 u64 *val)
84e690bf 680{
978ceeb3 681 *val = vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
84e690bf
AB		 682 return 0;
683}
684
d86cde6e 685static u64 reset_bvr(struct kvm_vcpu *vcpu,
281243cb 686 const struct sys_reg_desc *rd)
84e690bf 687{
cb853ded 688 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = rd->val;
d86cde6e 689 return rd->val;
84e690bf
AB		 690}
691
281243cb
MZ		 692static bool trap_bcr(struct kvm_vcpu *vcpu,
693 struct sys_reg_params *p,
694 const struct sys_reg_desc *rd)
84e690bf 695{
cb853ded 696 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
84e690bf
AB		 697
698 if (p->is_write)
1da42c34 699 reg_to_dbg(vcpu, p, rd, dbg_reg);
84e690bf 700 else
1da42c34 701 dbg_to_reg(vcpu, p, rd, dbg_reg);
84e690bf 702
cb853ded 703 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
eef8c85a 704
84e690bf
AB		 705 return true;
706}
707
708static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 709 u64 val)
84e690bf 710{
978ceeb3 711 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = val;
84e690bf
AB		 712 return 0;
713}
714
715static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 716 u64 *val)
84e690bf 717{
978ceeb3 718 *val = vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
84e690bf
AB		 719 return 0;
720}
721
d86cde6e 722static u64 reset_bcr(struct kvm_vcpu *vcpu,
281243cb 723 const struct sys_reg_desc *rd)
84e690bf 724{
cb853ded 725 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = rd->val;
d86cde6e 726 return rd->val;
84e690bf
AB		 727}
728
281243cb
MZ		 729static bool trap_wvr(struct kvm_vcpu *vcpu,
730 struct sys_reg_params *p,
731 const struct sys_reg_desc *rd)
84e690bf 732{
cb853ded 733 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
84e690bf
AB		 734
735 if (p->is_write)
1da42c34 736 reg_to_dbg(vcpu, p, rd, dbg_reg);
84e690bf 737 else
1da42c34 738 dbg_to_reg(vcpu, p, rd, dbg_reg);
84e690bf 739
cb853ded
MZ		 740 trace_trap_reg(__func__, rd->CRm, p->is_write,
741 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm]);
eef8c85a 742
84e690bf
AB		 743 return true;
744}
745
746static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 747 u64 val)
84e690bf 748{
978ceeb3 749 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = val;
84e690bf
AB		 750 return 0;
751}
752
753static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 754 u64 *val)
84e690bf 755{
978ceeb3 756 *val = vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
84e690bf
AB		 757 return 0;
758}
759
d86cde6e 760static u64 reset_wvr(struct kvm_vcpu *vcpu,
281243cb 761 const struct sys_reg_desc *rd)
84e690bf 762{
cb853ded 763 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = rd->val;
d86cde6e 764 return rd->val;
84e690bf
AB		 765}
766
281243cb
MZ		 767static bool trap_wcr(struct kvm_vcpu *vcpu,
768 struct sys_reg_params *p,
769 const struct sys_reg_desc *rd)
84e690bf 770{
cb853ded 771 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
84e690bf
AB		 772
773 if (p->is_write)
1da42c34 774 reg_to_dbg(vcpu, p, rd, dbg_reg);
84e690bf 775 else
1da42c34 776 dbg_to_reg(vcpu, p, rd, dbg_reg);
84e690bf 777
cb853ded 778 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
eef8c85a 779
84e690bf
AB		 780 return true;
781}
782
783static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 784 u64 val)
84e690bf 785{
978ceeb3 786 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = val;
84e690bf
AB		 787 return 0;
788}
789
790static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 791 u64 *val)
84e690bf 792{
978ceeb3 793 *val = vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
84e690bf
AB		 794 return 0;
795}
796
d86cde6e 797static u64 reset_wcr(struct kvm_vcpu *vcpu,
281243cb 798 const struct sys_reg_desc *rd)
84e690bf 799{
cb853ded 800 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = rd->val;
d86cde6e 801 return rd->val;
84e690bf
AB		 802}
803
d86cde6e 804static u64 reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
7c8c5e6a 805{
8d404c4c
CD		 806 u64 amair = read_sysreg(amair_el1);
807 vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
d86cde6e 808 return amair;
7c8c5e6a
MZ		 809}
810
d86cde6e 811static u64 reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
af473829
JM		 812{
813 u64 actlr = read_sysreg(actlr_el1);
814 vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1);
d86cde6e 815 return actlr;
af473829
JM		 816}
817
d86cde6e 818static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
7c8c5e6a 819{
4429fc64
AP		 820 u64 mpidr;
821
7c8c5e6a 822 /*
4429fc64
AP		 823 * Map the vcpu_id into the first three affinity level fields of
824 * the MPIDR. We limit the number of VCPUs in level 0 due to a
825 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
826 * of the GICv3 to be able to address each CPU directly when
827 * sending IPIs.
7c8c5e6a 828 */
4429fc64
AP		 829 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
830 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
831 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
d86cde6e
JZ		 832 mpidr |= (1ULL << 31);
833 vcpu_write_sys_reg(vcpu, mpidr, MPIDR_EL1);
834
835 return mpidr;
7c8c5e6a
MZ		 836}
837
11663111
MZ		 838static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
839 const struct sys_reg_desc *r)
840{
841 if (kvm_vcpu_has_pmu(vcpu))
842 return 0;
843
844 return REG_HIDDEN;
845}
846
d86cde6e 847static u64 reset_pmu_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
0ab410a9 848{
ea9ca904
RW		 849 u64 mask = BIT(ARMV8_PMU_CYCLE_IDX);
850 u8 n = vcpu->kvm->arch.pmcr_n;
0ab410a9 851
0ab410a9
MZ		 852 if (n)
853 mask |= GENMASK(n - 1, 0);
854
855 reset_unknown(vcpu, r);
856 __vcpu_sys_reg(vcpu, r->reg) &= mask;
d86cde6e
JZ		 857
858 return __vcpu_sys_reg(vcpu, r->reg);
0ab410a9
MZ		 859}
860
d86cde6e 861static u64 reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
0ab410a9
MZ		 862{
863 reset_unknown(vcpu, r);
864 __vcpu_sys_reg(vcpu, r->reg) &= GENMASK(31, 0);
d86cde6e
JZ		 865
866 return __vcpu_sys_reg(vcpu, r->reg);
0ab410a9
MZ		 867}
868
d86cde6e 869static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
0ab410a9 870{
bc512d6a
OU		 871 /* This thing will UNDEF, who cares about the reset value? */
872 if (!kvm_vcpu_has_pmu(vcpu))
873 return 0;
874
0ab410a9 875 reset_unknown(vcpu, r);
bc512d6a 876 __vcpu_sys_reg(vcpu, r->reg) &= kvm_pmu_evtyper_mask(vcpu->kvm);
d86cde6e
JZ		 877
878 return __vcpu_sys_reg(vcpu, r->reg);
0ab410a9
MZ		 879}
880
d86cde6e 881static u64 reset_pmselr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
0ab410a9
MZ		 882{
883 reset_unknown(vcpu, r);
884 __vcpu_sys_reg(vcpu, r->reg) &= ARMV8_PMU_COUNTER_MASK;
d86cde6e
JZ		 885
886 return __vcpu_sys_reg(vcpu, r->reg);
0ab410a9
MZ		 887}
888
d86cde6e 889static u64 reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
ab946834 890{
4d20debf 891 u64 pmcr = 0;
ab946834 892
f3c6efc7 893 if (!kvm_supports_32bit_el0())
292e8f14
MZ		 894 pmcr |= ARMV8_PMU_PMCR_LC;
895
4d20debf
RRA		 896 /*
897 * The value of PMCR.N field is included when the
898 * vCPU register is read via kvm_vcpu_read_pmcr().
899 */
292e8f14 900 __vcpu_sys_reg(vcpu, r->reg) = pmcr;
d86cde6e
JZ		 901
902 return __vcpu_sys_reg(vcpu, r->reg);
ab946834
SZ		 903}
904
6c007036 905static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
d692b8ad 906{
8d404c4c 907 u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
7ded92e2 908 bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
d692b8ad 909
24d5950f
MZ		 910 if (!enabled)
911 kvm_inject_undefined(vcpu);
d692b8ad 912
6c007036 913 return !enabled;
d692b8ad
SZ		 914}
915
6c007036 916static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
d692b8ad 917{
6c007036
MZ		 918 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
919}
d692b8ad 920
6c007036
MZ		 921static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
922{
923 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
d692b8ad
SZ		 924}
925
926static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
927{
6c007036 928 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
d692b8ad
SZ		 929}
930
931static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
932{
6c007036 933 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
d692b8ad
SZ		 934}
935
ab946834
SZ		 936static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
937 const struct sys_reg_desc *r)
938{
939 u64 val;
940
d692b8ad
SZ		 941 if (pmu_access_el0_disabled(vcpu))
942 return false;
943
ab946834 944 if (p->is_write) {
64d6820d
MZ		 945 /*
946 * Only update writeable bits of PMCR (continuing into
947 * kvm_pmu_handle_pmcr() as well)
948 */
57fc267f 949 val = kvm_vcpu_read_pmcr(vcpu);
ab946834
SZ		 950 val &= ~ARMV8_PMU_PMCR_MASK;
951 val |= p->regval & ARMV8_PMU_PMCR_MASK;
f3c6efc7 952 if (!kvm_supports_32bit_el0())
6f163714 953 val |= ARMV8_PMU_PMCR_LC;
76993739 954 kvm_pmu_handle_pmcr(vcpu, val);
ab946834
SZ		 955 } else {
956 /* PMCR.P & PMCR.C are RAZ */
57fc267f 957 val = kvm_vcpu_read_pmcr(vcpu)
ab946834
SZ		 958 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
959 p->regval = val;
960 }
961
962 return true;
963}
964
3965c3ce
SZ		 965static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
966 const struct sys_reg_desc *r)
967{
d692b8ad
SZ		 968 if (pmu_access_event_counter_el0_disabled(vcpu))
969 return false;
970
3965c3ce 971 if (p->is_write)
8d404c4c 972 __vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
3965c3ce
SZ		 973 else
974 /* return PMSELR.SEL field */
8d404c4c 975 p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
3965c3ce
SZ		 976 & ARMV8_PMU_COUNTER_MASK;
977
978 return true;
979}
980
a86b5505
SZ		 981static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
982 const struct sys_reg_desc *r)
983{
99b6a401 984 u64 pmceid, mask, shift;
a86b5505 985
a86b5505
SZ		 986 BUG_ON(p->is_write);
987
d692b8ad
SZ		 988 if (pmu_access_el0_disabled(vcpu))
989 return false;
990
99b6a401
MZ		 991 get_access_mask(r, &mask, &shift);
992
88865bec 993 pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
99b6a401
MZ		 994 pmceid &= mask;
995 pmceid >>= shift;
a86b5505
SZ		 996
997 p->regval = pmceid;
998
999 return true;
1000}
1001
051ff581
SZ		 1002static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
1003{
1004 u64 pmcr, val;
1005
57fc267f 1006 pmcr = kvm_vcpu_read_pmcr(vcpu);
62e1f212 1007 val = FIELD_GET(ARMV8_PMU_PMCR_N, pmcr);
24d5950f
MZ		 1008 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
1009 kvm_inject_undefined(vcpu);
051ff581 1010 return false;
24d5950f 1011 }
051ff581
SZ		 1012
1013 return true;
1014}
1015
9228b261
RW		 1016static int get_pmu_evcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1017 u64 *val)
1018{
1019 u64 idx;
1020
1021 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 0)
1022 /* PMCCNTR_EL0 */
1023 idx = ARMV8_PMU_CYCLE_IDX;
1024 else
1025 /* PMEVCNTRn_EL0 */
1026 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1027
1028 *val = kvm_pmu_get_counter_value(vcpu, idx);
1029 return 0;
1030}
1031
051ff581
SZ		 1032static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
1033 struct sys_reg_params *p,
1034 const struct sys_reg_desc *r)
1035{
a3da9358 1036 u64 idx = ~0UL;
051ff581
SZ		 1037
1038 if (r->CRn == 9 && r->CRm == 13) {
1039 if (r->Op2 == 2) {
1040 /* PMXEVCNTR_EL0 */
d692b8ad
SZ		 1041 if (pmu_access_event_counter_el0_disabled(vcpu))
1042 return false;
1043
8d404c4c 1044 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
051ff581
SZ		 1045 & ARMV8_PMU_COUNTER_MASK;
1046 } else if (r->Op2 == 0) {
1047 /* PMCCNTR_EL0 */
d692b8ad
SZ		 1048 if (pmu_access_cycle_counter_el0_disabled(vcpu))
1049 return false;
1050
051ff581 1051 idx = ARMV8_PMU_CYCLE_IDX;
051ff581 1052 }
9e3f7a29
WH		 1053 } else if (r->CRn == 0 && r->CRm == 9) {
1054 /* PMCCNTR */
1055 if (pmu_access_event_counter_el0_disabled(vcpu))
1056 return false;
1057
1058 idx = ARMV8_PMU_CYCLE_IDX;
051ff581
SZ		 1059 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
1060 /* PMEVCNTRn_EL0 */
d692b8ad
SZ		 1061 if (pmu_access_event_counter_el0_disabled(vcpu))
1062 return false;
1063
051ff581 1064 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
051ff581
SZ		 1065 }
1066
a3da9358
MZ		 1067 /* Catch any decoding mistake */
1068 WARN_ON(idx == ~0UL);
1069
051ff581
SZ		 1070 if (!pmu_counter_idx_valid(vcpu, idx))
1071 return false;
1072
d692b8ad
SZ		 1073 if (p->is_write) {
1074 if (pmu_access_el0_disabled(vcpu))
1075 return false;
1076
051ff581 1077 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
d692b8ad 1078 } else {
051ff581 1079 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
d692b8ad 1080 }
051ff581
SZ		 1081
1082 return true;
1083}
1084
9feb21ac
SZ		 1085static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1086 const struct sys_reg_desc *r)
1087{
1088 u64 idx, reg;
1089
d692b8ad
SZ		 1090 if (pmu_access_el0_disabled(vcpu))
1091 return false;
1092
9feb21ac
SZ		 1093 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
1094 /* PMXEVTYPER_EL0 */
8d404c4c 1095 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
9feb21ac
SZ		 1096 reg = PMEVTYPER0_EL0 + idx;
1097 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
1098 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1099 if (idx == ARMV8_PMU_CYCLE_IDX)
1100 reg = PMCCFILTR_EL0;
1101 else
1102 /* PMEVTYPERn_EL0 */
1103 reg = PMEVTYPER0_EL0 + idx;
1104 } else {
1105 BUG();
1106 }
1107
1108 if (!pmu_counter_idx_valid(vcpu, idx))
1109 return false;
1110
1111 if (p->is_write) {
1112 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
435e53fb 1113 kvm_vcpu_pmu_restore_guest(vcpu);
9feb21ac 1114 } else {
bc512d6a 1115 p->regval = __vcpu_sys_reg(vcpu, reg);
9feb21ac
SZ		 1116 }
1117
1118 return true;
1119}
1120
a45f41d7
RRA		 1121static int set_pmreg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, u64 val)
1122{
1123 bool set;
1124
1125 val &= kvm_pmu_valid_counter_mask(vcpu);
1126
1127 switch (r->reg) {
1128 case PMOVSSET_EL0:
1129 /* CRm[1] being set indicates a SET register, and CLR otherwise */
1130 set = r->CRm & 2;
1131 break;
1132 default:
1133 /* Op2[0] being set indicates a SET register, and CLR otherwise */
1134 set = r->Op2 & 1;
1135 break;
1136 }
1137
1138 if (set)
1139 __vcpu_sys_reg(vcpu, r->reg) |= val;
1140 else
1141 __vcpu_sys_reg(vcpu, r->reg) &= ~val;
1142
1143 return 0;
1144}
1145
1146static int get_pmreg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, u64 *val)
1147{
1148 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1149
1150 *val = __vcpu_sys_reg(vcpu, r->reg) & mask;
1151 return 0;
1152}
1153
96b0eebc
SZ		 1154static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1155 const struct sys_reg_desc *r)
1156{
1157 u64 val, mask;
1158
d692b8ad
SZ		 1159 if (pmu_access_el0_disabled(vcpu))
1160 return false;
1161
96b0eebc
SZ		 1162 mask = kvm_pmu_valid_counter_mask(vcpu);
1163 if (p->is_write) {
1164 val = p->regval & mask;
1165 if (r->Op2 & 0x1) {
1166 /* accessing PMCNTENSET_EL0 */
8d404c4c 1167 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
418e5ca8 1168 kvm_pmu_enable_counter_mask(vcpu, val);
435e53fb 1169 kvm_vcpu_pmu_restore_guest(vcpu);
96b0eebc
SZ		 1170 } else {
1171 /* accessing PMCNTENCLR_EL0 */
8d404c4c 1172 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
418e5ca8 1173 kvm_pmu_disable_counter_mask(vcpu, val);
96b0eebc
SZ		 1174 }
1175 } else {
f5eff400 1176 p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
96b0eebc
SZ		 1177 }
1178
1179 return true;
1180}
1181
9db52c78
SZ		 1182static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1183 const struct sys_reg_desc *r)
1184{
1185 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1186
b0737e99 1187 if (check_pmu_access_disabled(vcpu, 0))
d692b8ad
SZ		 1188 return false;
1189
9db52c78
SZ		 1190 if (p->is_write) {
1191 u64 val = p->regval & mask;
1192
1193 if (r->Op2 & 0x1)
1194 /* accessing PMINTENSET_EL1 */
8d404c4c 1195 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
9db52c78
SZ		 1196 else
1197 /* accessing PMINTENCLR_EL1 */
8d404c4c 1198 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
9db52c78 1199 } else {
f5eff400 1200 p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1);
9db52c78
SZ		 1201 }
1202
1203 return true;
1204}
1205
76d883c4
SZ		 1206static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1207 const struct sys_reg_desc *r)
1208{
1209 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1210
d692b8ad
SZ		 1211 if (pmu_access_el0_disabled(vcpu))
1212 return false;
1213
76d883c4
SZ		 1214 if (p->is_write) {
1215 if (r->CRm & 0x2)
1216 /* accessing PMOVSSET_EL0 */
8d404c4c 1217 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
76d883c4
SZ		 1218 else
1219 /* accessing PMOVSCLR_EL0 */
8d404c4c 1220 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
76d883c4 1221 } else {
f5eff400 1222 p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0);
76d883c4
SZ		 1223 }
1224
1225 return true;
1226}
1227
7a0adc70
SZ		 1228static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1229 const struct sys_reg_desc *r)
1230{
1231 u64 mask;
1232
e0443230 1233 if (!p->is_write)
e7f1d1ee 1234 return read_from_write_only(vcpu, p, r);
e0443230 1235
d692b8ad
SZ		 1236 if (pmu_write_swinc_el0_disabled(vcpu))
1237 return false;
1238
e0443230
MZ		 1239 mask = kvm_pmu_valid_counter_mask(vcpu);
1240 kvm_pmu_software_increment(vcpu, p->regval & mask);
1241 return true;
7a0adc70
SZ		 1242}
1243
d692b8ad
SZ		 1244static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1245 const struct sys_reg_desc *r)
1246{
d692b8ad 1247 if (p->is_write) {
9008c235
MZ		 1248 if (!vcpu_mode_priv(vcpu)) {
1249 kvm_inject_undefined(vcpu);
d692b8ad 1250 return false;
9008c235 1251 }
d692b8ad 1252
8d404c4c
CD		 1253 __vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
1254 p->regval & ARMV8_PMU_USERENR_MASK;
d692b8ad 1255 } else {
8d404c4c 1256 p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
d692b8ad
SZ		 1257 & ARMV8_PMU_USERENR_MASK;
1258 }
1259
1260 return true;
1261}
1262
4d20debf
RRA		 1263static int get_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1264 u64 *val)
1265{
1266 *val = kvm_vcpu_read_pmcr(vcpu);
1267 return 0;
1268}
1269
ea9ca904
RW		 1270static int set_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1271 u64 val)
1272{
62e1f212 1273 u8 new_n = FIELD_GET(ARMV8_PMU_PMCR_N, val);
ea9ca904
RW		 1274 struct kvm *kvm = vcpu->kvm;
1275
1276 mutex_lock(&kvm->arch.config_lock);
1277
1278 /*
1279 * The vCPU can't have more counters than the PMU hardware
1280 * implements. Ignore this error to maintain compatibility
1281 * with the existing KVM behavior.
1282 */
1283 if (!kvm_vm_has_ran_once(kvm) &&
1284 new_n <= kvm_arm_pmu_get_max_counters(kvm))
1285 kvm->arch.pmcr_n = new_n;
1286
1287 mutex_unlock(&kvm->arch.config_lock);
1288
1289 /*
1290 * Ignore writes to RES0 bits, read only bits that are cleared on
1291 * vCPU reset, and writable bits that KVM doesn't support yet.
1292 * (i.e. only PMCR.N and bits [7:0] are mutable from userspace)
1293 * The LP bit is RES0 when FEAT_PMUv3p5 is not supported on the vCPU.
1294 * But, we leave the bit as it is here, as the vCPU's PMUver might
1295 * be changed later (NOTE: the bit will be cleared on first vCPU run
1296 * if necessary).
1297 */
1298 val &= ARMV8_PMU_PMCR_MASK;
1299
1300 /* The LC bit is RES1 when AArch32 is not supported */
1301 if (!kvm_supports_32bit_el0())
1302 val |= ARMV8_PMU_PMCR_LC;
1303
1304 __vcpu_sys_reg(vcpu, r->reg) = val;
1305 return 0;
1306}
1307
0c557ed4
MZ		 1308/* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
1309#define DBG_BCR_BVR_WCR_WVR_EL1(n) \
ee1b64e6 1310 { SYS_DESC(SYS_DBGBVRn_EL1(n)), \
03fdfb26 1311 trap_bvr, reset_bvr, 0, 0, get_bvr, set_bvr }, \
ee1b64e6 1312 { SYS_DESC(SYS_DBGBCRn_EL1(n)), \
03fdfb26 1313 trap_bcr, reset_bcr, 0, 0, get_bcr, set_bcr }, \
ee1b64e6 1314 { SYS_DESC(SYS_DBGWVRn_EL1(n)), \
03fdfb26 1315 trap_wvr, reset_wvr, 0, 0, get_wvr, set_wvr }, \
ee1b64e6 1316 { SYS_DESC(SYS_DBGWCRn_EL1(n)), \
03fdfb26 1317 trap_wcr, reset_wcr, 0, 0, get_wcr, set_wcr }
0c557ed4 1318
9d2a55b4
XC		 1319#define PMU_SYS_REG(name) \
1320 SYS_DESC(SYS_##name), .reset = reset_pmu_reg, \
1321 .visibility = pmu_visibility
11663111 1322
051ff581
SZ		 1323/* Macro to expand the PMEVCNTRn_EL0 register */
1324#define PMU_PMEVCNTR_EL0(n) \
9d2a55b4 1325 { PMU_SYS_REG(PMEVCNTRn_EL0(n)), \
9228b261 1326 .reset = reset_pmevcntr, .get_user = get_pmu_evcntr, \
11663111 1327 .access = access_pmu_evcntr, .reg = (PMEVCNTR0_EL0 + n), }
051ff581 1328
9feb21ac
SZ		 1329/* Macro to expand the PMEVTYPERn_EL0 register */
1330#define PMU_PMEVTYPER_EL0(n) \
9d2a55b4 1331 { PMU_SYS_REG(PMEVTYPERn_EL0(n)), \
0ab410a9 1332 .reset = reset_pmevtyper, \
11663111 1333 .access = access_pmu_evtyper, .reg = (PMEVTYPER0_EL0 + n), }
9feb21ac 1334
338b1793
MZ		 1335static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1336 const struct sys_reg_desc *r)
4fcdf106
IV		 1337{
1338 kvm_inject_undefined(vcpu);
1339
1340 return false;
1341}
1342
1343/* Macro to expand the AMU counter and type registers*/
338b1793
MZ		 1344#define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), undef_access }
1345#define AMU_AMEVTYPER0_EL0(n) { SYS_DESC(SYS_AMEVTYPER0_EL0(n)), undef_access }
1346#define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), undef_access }
1347#define AMU_AMEVTYPER1_EL0(n) { SYS_DESC(SYS_AMEVTYPER1_EL0(n)), undef_access }
384b40ca
MR		 1348
1349static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
1350 const struct sys_reg_desc *rd)
1351{
01fe5ace 1352 return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN;
384b40ca
MR		 1353}
1354
338b1793
MZ		 1355/*
1356 * If we land here on a PtrAuth access, that is because we didn't
1357 * fixup the access on exit by allowing the PtrAuth sysregs. The only
1358 * way this happens is when the guest does not have PtrAuth support
1359 * enabled.
1360 */
384b40ca 1361#define __PTRAUTH_KEY(k) \
338b1793 1362 { SYS_DESC(SYS_## k), undef_access, reset_unknown, k, \
384b40ca
MR		 1363 .visibility = ptrauth_visibility}
1364
1365#define PTRAUTH_KEY(k) \
1366 __PTRAUTH_KEY(k ## KEYLO_EL1), \
1367 __PTRAUTH_KEY(k ## KEYHI_EL1)
1368
84135d3d
AP		 1369static bool access_arch_timer(struct kvm_vcpu *vcpu,
1370 struct sys_reg_params *p,
1371 const struct sys_reg_desc *r)
c9a3c58f 1372{
84135d3d
AP		 1373 enum kvm_arch_timers tmr;
1374 enum kvm_arch_timer_regs treg;
1375 u64 reg = reg_to_encoding(r);
7b6b4631 1376
84135d3d
AP		 1377 switch (reg) {
1378 case SYS_CNTP_TVAL_EL0:
1379 case SYS_AARCH32_CNTP_TVAL:
1380 tmr = TIMER_PTIMER;
1381 treg = TIMER_REG_TVAL;
1382 break;
1383 case SYS_CNTP_CTL_EL0:
1384 case SYS_AARCH32_CNTP_CTL:
1385 tmr = TIMER_PTIMER;
1386 treg = TIMER_REG_CTL;
1387 break;
1388 case SYS_CNTP_CVAL_EL0:
1389 case SYS_AARCH32_CNTP_CVAL:
1390 tmr = TIMER_PTIMER;
1391 treg = TIMER_REG_CVAL;
1392 break;
c605ee24
MZ		 1393 case SYS_CNTPCT_EL0:
1394 case SYS_CNTPCTSS_EL0:
1395 case SYS_AARCH32_CNTPCT:
1396 tmr = TIMER_PTIMER;
1397 treg = TIMER_REG_CNT;
1398 break;
84135d3d 1399 default:
ba82e06c
MZ		 1400 print_sys_reg_msg(p, "%s", "Unhandled trapped timer register");
1401 kvm_inject_undefined(vcpu);
1402 return false;
c1b135af 1403 }
7b6b4631 1404
7b6b4631 1405 if (p->is_write)
84135d3d 1406 kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
7b6b4631 1407 else
84135d3d 1408 p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
7b6b4631 1409
c9a3c58f
JL		 1410 return true;
1411}
1412
2e8bf0cb
JZ		 1413static s64 kvm_arm64_ftr_safe_value(u32 id, const struct arm64_ftr_bits *ftrp,
1414 s64 new, s64 cur)
3d0dba57 1415{
2e8bf0cb
JZ		 1416 struct arm64_ftr_bits kvm_ftr = *ftrp;
1417
1418 /* Some features have different safe value type in KVM than host features */
1419 switch (id) {
1420 case SYS_ID_AA64DFR0_EL1:
a9bc4a1c
OU		 1421 switch (kvm_ftr.shift) {
1422 case ID_AA64DFR0_EL1_PMUVer_SHIFT:
2e8bf0cb 1423 kvm_ftr.type = FTR_LOWER_SAFE;
a9bc4a1c
OU		 1424 break;
1425 case ID_AA64DFR0_EL1_DebugVer_SHIFT:
2e8bf0cb 1426 kvm_ftr.type = FTR_LOWER_SAFE;
a9bc4a1c
OU		 1427 break;
1428 }
2e8bf0cb
JZ		 1429 break;
1430 case SYS_ID_DFR0_EL1:
1431 if (kvm_ftr.shift == ID_DFR0_EL1_PerfMon_SHIFT)
1432 kvm_ftr.type = FTR_LOWER_SAFE;
1433 break;
1434 }
3d0dba57 1435
2e8bf0cb 1436 return arm64_ftr_safe_value(&kvm_ftr, new, cur);
3d0dba57
MZ		 1437}
1438
7b424ffc 1439/*
2e8bf0cb
JZ		 1440 * arm64_check_features() - Check if a feature register value constitutes
1441 * a subset of features indicated by the idreg's KVM sanitised limit.
1442 *
1443 * This function will check if each feature field of @val is the "safe" value
1444 * against idreg's KVM sanitised limit return from reset() callback.
1445 * If a field value in @val is the same as the one in limit, it is always
1446 * considered the safe value regardless For register fields that are not in
1447 * writable, only the value in limit is considered the safe value.
1448 *
1449 * Return: 0 if all the fields are safe. Otherwise, return negative errno.
1450 */
1451static int arm64_check_features(struct kvm_vcpu *vcpu,
1452 const struct sys_reg_desc *rd,
1453 u64 val)
d82e0dfd 1454{
2e8bf0cb
JZ		 1455 const struct arm64_ftr_reg *ftr_reg;
1456 const struct arm64_ftr_bits *ftrp = NULL;
1457 u32 id = reg_to_encoding(rd);
1458 u64 writable_mask = rd->val;
1459 u64 limit = rd->reset(vcpu, rd);
1460 u64 mask = 0;
1461
1462 /*
1463 * Hidden and unallocated ID registers may not have a corresponding
1464 * struct arm64_ftr_reg. Of course, if the register is RAZ we know the
1465 * only safe value is 0.
1466 */
1467 if (sysreg_visible_as_raz(vcpu, rd))
1468 return val ? -E2BIG : 0;
1469
1470 ftr_reg = get_arm64_ftr_reg(id);
1471 if (!ftr_reg)
1472 return -EINVAL;
1473
1474 ftrp = ftr_reg->ftr_bits;
1475
1476 for (; ftrp && ftrp->width; ftrp++) {
1477 s64 f_val, f_lim, safe_val;
1478 u64 ftr_mask;
1479
1480 ftr_mask = arm64_ftr_mask(ftrp);
1481 if ((ftr_mask & writable_mask) != ftr_mask)
1482 continue;
1483
1484 f_val = arm64_ftr_value(ftrp, val);
1485 f_lim = arm64_ftr_value(ftrp, limit);
1486 mask |= ftr_mask;
1487
1488 if (f_val == f_lim)
1489 safe_val = f_val;
1490 else
1491 safe_val = kvm_arm64_ftr_safe_value(id, ftrp, f_val, f_lim);
1492
1493 if (safe_val != f_val)
1494 return -E2BIG;
d82e0dfd 1495 }
2e8bf0cb
JZ		 1496
1497 /* For fields that are not writable, values in limit are the safe values. */
1498 if ((val & ~mask) != (limit & ~mask))
1499 return -E2BIG;
1500
1501 return 0;
d82e0dfd
MZ		 1502}
1503
3d0dba57
MZ		 1504static u8 pmuver_to_perfmon(u8 pmuver)
1505{
1506 switch (pmuver) {
1507 case ID_AA64DFR0_EL1_PMUVer_IMP:
753d734f 1508 return ID_DFR0_EL1_PerfMon_PMUv3;
3d0dba57 1509 case ID_AA64DFR0_EL1_PMUVer_IMP_DEF:
753d734f 1510 return ID_DFR0_EL1_PerfMon_IMPDEF;
3d0dba57
MZ		 1511 default:
1512 /* Anything ARMv8.1+ and NI have the same value. For now. */
1513 return pmuver;
1514 }
1515}
1516
93390c0a 1517/* Read a sanitised cpufeature ID register by sys_reg_desc */
d86cde6e
JZ		 1518static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
1519 const struct sys_reg_desc *r)
93390c0a 1520{
7ba8b438 1521 u32 id = reg_to_encoding(r);
00d5101b
AE		 1522 u64 val;
1523
cdd5036d 1524 if (sysreg_visible_as_raz(vcpu, r))
00d5101b
AE		 1525 return 0;
1526
1527 val = read_sanitised_ftr_reg(id);
93390c0a 1528
c8857935 1529 switch (id) {
c8857935 1530 case SYS_ID_AA64PFR1_EL1:
16dd1fbb 1531 if (!kvm_has_mte(vcpu->kvm))
6ca2b9ca 1532 val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE);
90807748 1533
6ca2b9ca 1534 val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME);
c8857935
MZ		 1535 break;
1536 case SYS_ID_AA64ISAR1_EL1:
1537 if (!vcpu_has_ptrauth(vcpu))
aa50479b
MB		 1538 val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA) |
1539 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API) |
1540 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA) |
1541 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI));
c8857935 1542 break;
def8c222
VM		 1543 case SYS_ID_AA64ISAR2_EL1:
1544 if (!vcpu_has_ptrauth(vcpu))
b2d71f27
MB		 1545 val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) |
1546 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
06e0b802 1547 if (!cpus_have_final_cap(ARM64_HAS_WFXT))
b2d71f27 1548 val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT);
def8c222 1549 break;
bf48040c
AO		 1550 case SYS_ID_AA64MMFR2_EL1:
1551 val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK;
1552 break;
1553 case SYS_ID_MMFR4_EL1:
1554 val &= ~ARM64_FEATURE_MASK(ID_MMFR4_EL1_CCIDX);
1555 break;
07d79fe7
DM		 1556 }
1557
1558 return val;
93390c0a
DM		 1559}
1560
d86cde6e
JZ		 1561static u64 kvm_read_sanitised_id_reg(struct kvm_vcpu *vcpu,
1562 const struct sys_reg_desc *r)
1563{
1564 return __kvm_read_sanitised_id_reg(vcpu, r);
1565}
1566
1567static u64 read_id_reg(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
1568{
6db7af0d 1569 return IDREG(vcpu->kvm, reg_to_encoding(r));
d86cde6e
JZ		 1570}
1571
47334146
JZ		 1572/*
1573 * Return true if the register's (Op0, Op1, CRn, CRm, Op2) is
1574 * (3, 0, 0, crm, op2), where 1<=crm<8, 0<=op2<8.
1575 */
1576static inline bool is_id_reg(u32 id)
1577{
1578 return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 &&
1579 sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 1 &&
1580 sys_reg_CRm(id) < 8);
1581}
1582
3f9cd0ca
JZ		 1583static inline bool is_aa32_id_reg(u32 id)
1584{
1585 return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 &&
1586 sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 1 &&
1587 sys_reg_CRm(id) <= 3);
1588}
1589
912dee57
AJ		 1590static unsigned int id_visibility(const struct kvm_vcpu *vcpu,
1591 const struct sys_reg_desc *r)
1592{
7ba8b438 1593 u32 id = reg_to_encoding(r);
c512298e
AJ		 1594
1595 switch (id) {
1596 case SYS_ID_AA64ZFR0_EL1:
1597 if (!vcpu_has_sve(vcpu))
1598 return REG_RAZ;
1599 break;
1600 }
1601
912dee57
AJ		 1602 return 0;
1603}
1604
d5efec7e
OU		 1605static unsigned int aa32_id_visibility(const struct kvm_vcpu *vcpu,
1606 const struct sys_reg_desc *r)
1607{
1608 /*
1609 * AArch32 ID registers are UNKNOWN if AArch32 isn't implemented at any
1610 * EL. Promote to RAZ/WI in order to guarantee consistency between
1611 * systems.
1612 */
1613 if (!kvm_supports_32bit_el0())
1614 return REG_RAZ | REG_USER_WI;
1615
1616 return id_visibility(vcpu, r);
1617}
1618
34b4d203
OU		 1619static unsigned int raz_visibility(const struct kvm_vcpu *vcpu,
1620 const struct sys_reg_desc *r)
1621{
1622 return REG_RAZ;
1623}
1624
93390c0a
DM		 1625/* cpufeature ID register access trap handlers */
1626
93390c0a
DM		 1627static bool access_id_reg(struct kvm_vcpu *vcpu,
1628 struct sys_reg_params *p,
1629 const struct sys_reg_desc *r)
1630{
4782ccc8
OU		 1631 if (p->is_write)
1632 return write_to_read_only(vcpu, p, r);
1633
cdd5036d 1634 p->regval = read_id_reg(vcpu, r);
9f75b6d4 1635
4782ccc8 1636 return true;
93390c0a
DM		 1637}
1638
73433762
DM		 1639/* Visibility overrides for SVE-specific control registers */
1640static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1641 const struct sys_reg_desc *rd)
1642{
1643 if (vcpu_has_sve(vcpu))
1644 return 0;
1645
01fe5ace 1646 return REG_HIDDEN;
73433762
DM		 1647}
1648
c39f5974
JZ		 1649static u64 read_sanitised_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
1650 const struct sys_reg_desc *rd)
23711a5e 1651{
c39f5974
JZ		 1652 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1653
1654 if (!vcpu_has_sve(vcpu))
1655 val &= ~ID_AA64PFR0_EL1_SVE_MASK;
23711a5e
MZ		 1656
1657 /*
c39f5974
JZ		 1658 * The default is to expose CSV2 == 1 if the HW isn't affected.
1659 * Although this is a per-CPU feature, we make it global because
1660 * asymmetric systems are just a nuisance.
1661 *
1662 * Userspace can override this as long as it doesn't promise
1663 * the impossible.
23711a5e 1664 */
c39f5974
JZ		 1665 if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED) {
1666 val &= ~ID_AA64PFR0_EL1_CSV2_MASK;
1667 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV2, IMP);
1668 }
1669 if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED) {
1670 val &= ~ID_AA64PFR0_EL1_CSV3_MASK;
1671 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV3, IMP);
1672 }
23711a5e 1673
c39f5974
JZ		 1674 if (kvm_vgic_global_state.type == VGIC_V3) {
1675 val &= ~ID_AA64PFR0_EL1_GIC_MASK;
1676 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
1677 }
4f1df628 1678
c39f5974 1679 val &= ~ID_AA64PFR0_EL1_AMU_MASK;
23711a5e 1680
c39f5974
JZ		 1681 return val;
1682}
23711a5e 1683
a9bc4a1c
OU		 1684#define ID_REG_LIMIT_FIELD_ENUM(val, reg, field, limit) \
1685({ \
1686 u64 __f_val = FIELD_GET(reg##_##field##_MASK, val); \
1687 (val) &= ~reg##_##field##_MASK; \
1688 (val) |= FIELD_PREP(reg##_##field##_MASK, \
53eaeb7f
MZ		 1689 min(__f_val, \
1690 (u64)SYS_FIELD_VALUE(reg, field, limit))); \
a9bc4a1c
OU		 1691 (val); \
1692})
1693
c118cead
JZ		 1694static u64 read_sanitised_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1695 const struct sys_reg_desc *rd)
1696{
1697 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1698
9f9917bc 1699 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8);
c118cead
JZ		 1700
1701 /*
1702 * Only initialize the PMU version if the vCPU was configured with one.
1703 */
1704 val &= ~ID_AA64DFR0_EL1_PMUVer_MASK;
1705 if (kvm_vcpu_has_pmu(vcpu))
1706 val |= SYS_FIELD_PREP(ID_AA64DFR0_EL1, PMUVer,
1707 kvm_arm_pmu_get_pmuver_limit());
1708
1709 /* Hide SPE from guests */
1710 val &= ~ID_AA64DFR0_EL1_PMSVer_MASK;
1711
1712 return val;
23711a5e
MZ		 1713}
1714
60e651ff
MZ		 1715static int set_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1716 const struct sys_reg_desc *rd,
1717 u64 val)
1718{
a9bc4a1c 1719 u8 debugver = SYS_FIELD_GET(ID_AA64DFR0_EL1, DebugVer, val);
c118cead 1720 u8 pmuver = SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, val);
60e651ff
MZ		 1721
1722 /*
f90f9360
OU		 1723 * Prior to commit 3d0dba5764b9 ("KVM: arm64: PMU: Move the
1724 * ID_AA64DFR0_EL1.PMUver limit to VM creation"), KVM erroneously
1725 * exposed an IMP_DEF PMU to userspace and the guest on systems w/
1726 * non-architectural PMUs. Of course, PMUv3 is the only game in town for
1727 * PMU virtualization, so the IMP_DEF value was rather user-hostile.
1728 *
1729 * At minimum, we're on the hook to allow values that were given to
1730 * userspace by KVM. Cover our tracks here and replace the IMP_DEF value
1731 * with a more sensible NI. The value of an ID register changing under
1732 * the nose of the guest is unfortunate, but is certainly no more
1733 * surprising than an ill-guided PMU driver poking at impdef system
1734 * registers that end in an UNDEF...
60e651ff 1735 */
68667240 1736 if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
f90f9360 1737 val &= ~ID_AA64DFR0_EL1_PMUVer_MASK;
60e651ff 1738
a9bc4a1c
OU		 1739 /*
1740 * ID_AA64DFR0_EL1.DebugVer is one of those awkward fields with a
1741 * nonzero minimum safe value.
1742 */
1743 if (debugver < ID_AA64DFR0_EL1_DebugVer_IMP)
1744 return -EINVAL;
1745
68667240 1746 return set_id_reg(vcpu, rd, val);
c118cead 1747}
60e651ff 1748
c118cead
JZ		 1749static u64 read_sanitised_id_dfr0_el1(struct kvm_vcpu *vcpu,
1750 const struct sys_reg_desc *rd)
1751{
1752 u8 perfmon = pmuver_to_perfmon(kvm_arm_pmu_get_pmuver_limit());
1753 u64 val = read_sanitised_ftr_reg(SYS_ID_DFR0_EL1);
60e651ff 1754
c118cead
JZ		 1755 val &= ~ID_DFR0_EL1_PerfMon_MASK;
1756 if (kvm_vcpu_has_pmu(vcpu))
1757 val |= SYS_FIELD_PREP(ID_DFR0_EL1, PerfMon, perfmon);
60e651ff 1758
9f9917bc
OU		 1759 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_DFR0_EL1, CopDbg, Debugv8p8);
1760
c118cead 1761 return val;
60e651ff
MZ		 1762}
1763
d82e0dfd
MZ		 1764static int set_id_dfr0_el1(struct kvm_vcpu *vcpu,
1765 const struct sys_reg_desc *rd,
1766 u64 val)
1767{
c118cead 1768 u8 perfmon = SYS_FIELD_GET(ID_DFR0_EL1, PerfMon, val);
a9bc4a1c 1769 u8 copdbg = SYS_FIELD_GET(ID_DFR0_EL1, CopDbg, val);
d82e0dfd 1770
f90f9360
OU		 1771 if (perfmon == ID_DFR0_EL1_PerfMon_IMPDEF) {
1772 val &= ~ID_DFR0_EL1_PerfMon_MASK;
1773 perfmon = 0;
1774 }
d82e0dfd
MZ		 1775
1776 /*
1777 * Allow DFR0_EL1.PerfMon to be set from userspace as long as
1778 * it doesn't promise more than what the HW gives us on the
1779 * AArch64 side (as everything is emulated with that), and
1780 * that this is a PMUv3.
1781 */
c118cead 1782 if (perfmon != 0 && perfmon < ID_DFR0_EL1_PerfMon_PMUv3)
d82e0dfd
MZ		 1783 return -EINVAL;
1784
a9bc4a1c
OU		 1785 if (copdbg < ID_DFR0_EL1_CopDbg_Armv8)
1786 return -EINVAL;
1787
68667240 1788 return set_id_reg(vcpu, rd, val);
d82e0dfd
MZ		 1789}
1790
93390c0a
DM		 1791/*
1792 * cpufeature ID register user accessors
1793 *
1794 * For now, these registers are immutable for userspace, so no values
1795 * are stored, and for set_id_reg() we don't allow the effective value
1796 * to be changed.
1797 */
93390c0a 1798static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 1799 u64 *val)
93390c0a 1800{
6db7af0d
OU		 1801 /*
1802 * Avoid locking if the VM has already started, as the ID registers are
1803 * guaranteed to be invariant at that point.
1804 */
1805 if (kvm_vm_has_ran_once(vcpu->kvm)) {
1806 *val = read_id_reg(vcpu, rd);
1807 return 0;
1808 }
1809
1810 mutex_lock(&vcpu->kvm->arch.config_lock);
cdd5036d 1811 *val = read_id_reg(vcpu, rd);
6db7af0d
OU		 1812 mutex_unlock(&vcpu->kvm->arch.config_lock);
1813
4782ccc8 1814 return 0;
93390c0a
DM		 1815}
1816
1817static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 1818 u64 val)
93390c0a 1819{
2e8bf0cb
JZ		 1820 u32 id = reg_to_encoding(rd);
1821 int ret;
4782ccc8 1822
2e8bf0cb
JZ		 1823 mutex_lock(&vcpu->kvm->arch.config_lock);
1824
1825 /*
1826 * Once the VM has started the ID registers are immutable. Reject any
1827 * write that does not match the final register value.
1828 */
1829 if (kvm_vm_has_ran_once(vcpu->kvm)) {
1830 if (val != read_id_reg(vcpu, rd))
1831 ret = -EBUSY;
1832 else
1833 ret = 0;
1834
1835 mutex_unlock(&vcpu->kvm->arch.config_lock);
1836 return ret;
1837 }
1838
1839 ret = arm64_check_features(vcpu, rd, val);
1840 if (!ret)
1841 IDREG(vcpu->kvm, id) = val;
1842
1843 mutex_unlock(&vcpu->kvm->arch.config_lock);
1844
1845 /*
1846 * arm64_check_features() returns -E2BIG to indicate the register's
1847 * feature set is a superset of the maximally-allowed register value.
1848 * While it would be nice to precisely describe this to userspace, the
1849 * existing UAPI for KVM_SET_ONE_REG has it that invalid register
1850 * writes return -EINVAL.
1851 */
1852 if (ret == -E2BIG)
1853 ret = -EINVAL;
1854 return ret;
93390c0a
DM		 1855}
1856
5a430976 1857static int get_raz_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 1858 u64 *val)
5a430976 1859{
978ceeb3
MZ		 1860 *val = 0;
1861 return 0;
5a430976
AE		 1862}
1863
7a3ba309 1864static int set_wi_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
978ceeb3 1865 u64 val)
7a3ba309 1866{
7a3ba309
MZ		 1867 return 0;
1868}
1869
f7f2b15c
AB		 1870static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1871 const struct sys_reg_desc *r)
1872{
1873 if (p->is_write)
1874 return write_to_read_only(vcpu, p, r);
1875
1876 p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1877 return true;
1878}
1879
1880static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1881 const struct sys_reg_desc *r)
1882{
1883 if (p->is_write)
1884 return write_to_read_only(vcpu, p, r);
1885
7af0c253 1886 p->regval = __vcpu_sys_reg(vcpu, r->reg);
f7f2b15c
AB		 1887 return true;
1888}
1889
7af0c253
AO		 1890/*
1891 * Fabricate a CLIDR_EL1 value instead of using the real value, which can vary
1892 * by the physical CPU which the vcpu currently resides in.
1893 */
d86cde6e 1894static u64 reset_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
7af0c253
AO		 1895{
1896 u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
1897 u64 clidr;
1898 u8 loc;
1899
1900 if ((ctr_el0 & CTR_EL0_IDC)) {
1901 /*
1902 * Data cache clean to the PoU is not required so LoUU and LoUIS
1903 * will not be set and a unified cache, which will be marked as
1904 * LoC, will be added.
1905 *
1906 * If not DIC, let the unified cache L2 so that an instruction
1907 * cache can be added as L1 later.
1908 */
1909 loc = (ctr_el0 & CTR_EL0_DIC) ? 1 : 2;
1910 clidr = CACHE_TYPE_UNIFIED << CLIDR_CTYPE_SHIFT(loc);
1911 } else {
1912 /*
1913 * Data cache clean to the PoU is required so let L1 have a data
1914 * cache and mark it as LoUU and LoUIS. As L1 has a data cache,
1915 * it can be marked as LoC too.
1916 */
1917 loc = 1;
1918 clidr = 1 << CLIDR_LOUU_SHIFT;
1919 clidr |= 1 << CLIDR_LOUIS_SHIFT;
1920 clidr |= CACHE_TYPE_DATA << CLIDR_CTYPE_SHIFT(1);
1921 }
1922
1923 /*
1924 * Instruction cache invalidation to the PoU is required so let L1 have
1925 * an instruction cache. If L1 already has a data cache, it will be
1926 * CACHE_TYPE_SEPARATE.
1927 */
1928 if (!(ctr_el0 & CTR_EL0_DIC))
1929 clidr |= CACHE_TYPE_INST << CLIDR_CTYPE_SHIFT(1);
1930
1931 clidr |= loc << CLIDR_LOC_SHIFT;
1932
1933 /*
1934 * Add tag cache unified to data cache. Allocation tags and data are
1935 * unified in a cache line so that it looks valid even if there is only
1936 * one cache line.
1937 */
1938 if (kvm_has_mte(vcpu->kvm))
1939 clidr |= 2 << CLIDR_TTYPE_SHIFT(loc);
1940
1941 __vcpu_sys_reg(vcpu, r->reg) = clidr;
d86cde6e
JZ		 1942
1943 return __vcpu_sys_reg(vcpu, r->reg);
7af0c253
AO		 1944}
1945
1946static int set_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1947 u64 val)
1948{
1949 u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
1950 u64 idc = !CLIDR_LOC(val) || (!CLIDR_LOUIS(val) && !CLIDR_LOUU(val));
1951
1952 if ((val & CLIDR_EL1_RES0) || (!(ctr_el0 & CTR_EL0_IDC) && idc))
1953 return -EINVAL;
1954
1955 __vcpu_sys_reg(vcpu, rd->reg) = val;
1956
1957 return 0;
1958}
1959
f7f2b15c
AB		 1960static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1961 const struct sys_reg_desc *r)
1962{
7c582bf4
JM		 1963 int reg = r->reg;
1964
f7f2b15c 1965 if (p->is_write)
7c582bf4 1966 vcpu_write_sys_reg(vcpu, p->regval, reg);
f7f2b15c 1967 else
7c582bf4 1968 p->regval = vcpu_read_sys_reg(vcpu, reg);
f7f2b15c
AB		 1969 return true;
1970}
1971
1972static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1973 const struct sys_reg_desc *r)
1974{
1975 u32 csselr;
1976
1977 if (p->is_write)
1978 return write_to_read_only(vcpu, p, r);
1979
1980 csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
7af0c253
AO		 1981 csselr &= CSSELR_EL1_Level | CSSELR_EL1_InD;
1982 if (csselr < CSSELR_MAX)
1983 p->regval = get_ccsidr(vcpu, csselr);
793acf87 1984
f7f2b15c
AB		 1985 return true;
1986}
1987
e1f358b5
SP		 1988static unsigned int mte_visibility(const struct kvm_vcpu *vcpu,
1989 const struct sys_reg_desc *rd)
1990{
673638f4
SP		 1991 if (kvm_has_mte(vcpu->kvm))
1992 return 0;
1993
e1f358b5
SP		 1994 return REG_HIDDEN;
1995}
1996
1997#define MTE_REG(name) { \
1998 SYS_DESC(SYS_##name), \
1999 .access = undef_access, \
2000 .reset = reset_unknown, \
2001 .reg = name, \
2002 .visibility = mte_visibility, \
2003}
2004
6ff9dc23
JL		 2005static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
2006 const struct sys_reg_desc *rd)
2007{
2008 if (vcpu_has_nv(vcpu))
2009 return 0;
2010
2011 return REG_HIDDEN;
2012}
2013
9b9cce60
MZ		 2014static bool bad_vncr_trap(struct kvm_vcpu *vcpu,
2015 struct sys_reg_params *p,
2016 const struct sys_reg_desc *r)
2017{
2018 /*
2019 * We really shouldn't be here, and this is likely the result
2020 * of a misconfigured trap, as this register should target the
2021 * VNCR page, and nothing else.
2022 */
2023 return bad_trap(vcpu, p, r,
2024 "trap of VNCR-backed register");
2025}
2026
2027static bool bad_redir_trap(struct kvm_vcpu *vcpu,
2028 struct sys_reg_params *p,
2029 const struct sys_reg_desc *r)
2030{
2031 /*
2032 * We really shouldn't be here, and this is likely the result
2033 * of a misconfigured trap, as this register should target the
2034 * corresponding EL1, and nothing else.
2035 */
2036 return bad_trap(vcpu, p, r,
2037 "trap of EL2 register redirected to EL1");
2038}
2039
6ff9dc23
JL		 2040#define EL2_REG(name, acc, rst, v) { \
2041 SYS_DESC(SYS_##name), \
2042 .access = acc, \
2043 .reset = rst, \
2044 .reg = name, \
2045 .visibility = el2_visibility, \
2046 .val = v, \
2047}
2048
9b9cce60
MZ		 2049#define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
2050#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
2051
280b748e
JL		 2052/*
2053 * EL{0,1}2 registers are the EL2 view on an EL0 or EL1 register when
2054 * HCR_EL2.E2H==1, and only in the sysreg table for convenience of
2055 * handling traps. Given that, they are always hidden from userspace.
2056 */
3f7915cc
MZ		 2057static unsigned int hidden_user_visibility(const struct kvm_vcpu *vcpu,
2058 const struct sys_reg_desc *rd)
280b748e
JL		 2059{
2060 return REG_HIDDEN_USER;
2061}
2062
2063#define EL12_REG(name, acc, rst, v) { \
2064 SYS_DESC(SYS_##name##_EL12), \
2065 .access = acc, \
2066 .reset = rst, \
2067 .reg = name##_EL1, \
2068 .val = v, \
3f7915cc 2069 .visibility = hidden_user_visibility, \
280b748e
JL		 2070}
2071
d86cde6e
JZ		 2072/*
2073 * Since reset() callback and field val are not used for idregs, they will be
2074 * used for specific purposes for idregs.
2075 * The reset() would return KVM sanitised register value. The value would be the
2076 * same as the host kernel sanitised value if there is no KVM sanitisation.
2077 * The val would be used as a mask indicating writable fields for the idreg.
2078 * Only bits with 1 are writable from userspace. This mask might not be
2079 * necessary in the future whenever all ID registers are enabled as writable
2080 * from userspace.
2081 */
2082
56d77aa8 2083#define ID_DESC(name) \
93390c0a
DM		 2084 SYS_DESC(SYS_##name), \
2085 .access = access_id_reg, \
56d77aa8
OU		 2086 .get_user = get_id_reg \
2087
2088/* sys_reg_desc initialiser for known cpufeature ID registers */
2089#define ID_SANITISED(name) { \
2090 ID_DESC(name), \
93390c0a 2091 .set_user = set_id_reg, \
912dee57 2092 .visibility = id_visibility, \
d86cde6e
JZ		 2093 .reset = kvm_read_sanitised_id_reg, \
2094 .val = 0, \
93390c0a
DM		 2095}
2096
d5efec7e
OU		 2097/* sys_reg_desc initialiser for known cpufeature ID registers */
2098#define AA32_ID_SANITISED(name) { \
56d77aa8 2099 ID_DESC(name), \
d5efec7e
OU		 2100 .set_user = set_id_reg, \
2101 .visibility = aa32_id_visibility, \
d86cde6e
JZ		 2102 .reset = kvm_read_sanitised_id_reg, \
2103 .val = 0, \
d5efec7e
OU		 2104}
2105
56d77aa8
OU		 2106/* sys_reg_desc initialiser for writable ID registers */
2107#define ID_WRITABLE(name, mask) { \
2108 ID_DESC(name), \
2109 .set_user = set_id_reg, \
2110 .visibility = id_visibility, \
2111 .reset = kvm_read_sanitised_id_reg, \
2112 .val = mask, \
2113}
2114
93390c0a
DM		 2115/*
2116 * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
2117 * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
2118 * (1 <= crm < 8, 0 <= Op2 < 8).
2119 */
2120#define ID_UNALLOCATED(crm, op2) { \
2121 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
34b4d203
OU		 2122 .access = access_id_reg, \
2123 .get_user = get_id_reg, \
2124 .set_user = set_id_reg, \
d86cde6e
JZ		 2125 .visibility = raz_visibility, \
2126 .reset = kvm_read_sanitised_id_reg, \
2127 .val = 0, \
93390c0a
DM		 2128}
2129
2130/*
2131 * sys_reg_desc initialiser for known ID registers that we hide from guests.
2132 * For now, these are exposed just like unallocated ID regs: they appear
2133 * RAZ for the guest.
2134 */
2135#define ID_HIDDEN(name) { \
56d77aa8 2136 ID_DESC(name), \
34b4d203
OU		 2137 .set_user = set_id_reg, \
2138 .visibility = raz_visibility, \
d86cde6e
JZ		 2139 .reset = kvm_read_sanitised_id_reg, \
2140 .val = 0, \
93390c0a
DM		 2141}
2142
6ff9dc23
JL		 2143static bool access_sp_el1(struct kvm_vcpu *vcpu,
2144 struct sys_reg_params *p,
2145 const struct sys_reg_desc *r)
2146{
2147 if (p->is_write)
2148 __vcpu_sys_reg(vcpu, SP_EL1) = p->regval;
2149 else
2150 p->regval = __vcpu_sys_reg(vcpu, SP_EL1);
2151
2152 return true;
2153}
2154
9da117ee
JL		 2155static bool access_elr(struct kvm_vcpu *vcpu,
2156 struct sys_reg_params *p,
2157 const struct sys_reg_desc *r)
2158{
2159 if (p->is_write)
2160 vcpu_write_sys_reg(vcpu, p->regval, ELR_EL1);
2161 else
2162 p->regval = vcpu_read_sys_reg(vcpu, ELR_EL1);
2163
2164 return true;
2165}
2166
2167static bool access_spsr(struct kvm_vcpu *vcpu,
2168 struct sys_reg_params *p,
2169 const struct sys_reg_desc *r)
2170{
2171 if (p->is_write)
2172 __vcpu_sys_reg(vcpu, SPSR_EL1) = p->regval;
2173 else
2174 p->regval = __vcpu_sys_reg(vcpu, SPSR_EL1);
2175
2176 return true;
2177}
2178
94f29ab2
MZ		 2179static u64 reset_hcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
2180{
2181 u64 val = r->val;
2182
2183 if (!cpus_have_final_cap(ARM64_HAS_HCR_NV1))
2184 val |= HCR_E2H;
2185
2186 return __vcpu_sys_reg(vcpu, r->reg) = val;
2187}
2188
7c8c5e6a
MZ		 2189/*
2190 * Architected system registers.
2191 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
7609c125 2192 *
0c557ed4
MZ		 2193 * Debug handling: We do trap most, if not all debug related system
2194 * registers. The implementation is good enough to ensure that a guest
2195 * can use these with minimal performance degradation. The drawback is
7dabf02f
OU		 2196 * that we don't implement any of the external debug architecture.
2197 * This should be revisited if we ever encounter a more demanding
2198 * guest...
7c8c5e6a
MZ		 2199 */
2200static const struct sys_reg_desc sys_reg_descs[] = {
0c557ed4
MZ		 2201 DBG_BCR_BVR_WCR_WVR_EL1(0),
2202 DBG_BCR_BVR_WCR_WVR_EL1(1),
ee1b64e6
MR		 2203 { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
2204 { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
0c557ed4
MZ		 2205 DBG_BCR_BVR_WCR_WVR_EL1(2),
2206 DBG_BCR_BVR_WCR_WVR_EL1(3),
2207 DBG_BCR_BVR_WCR_WVR_EL1(4),
2208 DBG_BCR_BVR_WCR_WVR_EL1(5),
2209 DBG_BCR_BVR_WCR_WVR_EL1(6),
2210 DBG_BCR_BVR_WCR_WVR_EL1(7),
2211 DBG_BCR_BVR_WCR_WVR_EL1(8),
2212 DBG_BCR_BVR_WCR_WVR_EL1(9),
2213 DBG_BCR_BVR_WCR_WVR_EL1(10),
2214 DBG_BCR_BVR_WCR_WVR_EL1(11),
2215 DBG_BCR_BVR_WCR_WVR_EL1(12),
2216 DBG_BCR_BVR_WCR_WVR_EL1(13),
2217 DBG_BCR_BVR_WCR_WVR_EL1(14),
2218 DBG_BCR_BVR_WCR_WVR_EL1(15),
2219
ee1b64e6 2220 { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
f24adc65 2221 { SYS_DESC(SYS_OSLAR_EL1), trap_oslar_el1 },
d42e2671 2222 { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1, reset_val, OSLSR_EL1,
187de7c2 2223 OSLSR_EL1_OSLM_IMPLEMENTED, .set_user = set_oslsr_el1, },
ee1b64e6
MR		 2224 { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
2225 { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
2226 { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
2227 { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
2228 { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
2229
2230 { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
2231 { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
2232 // DBGDTR[TR]X_EL0 share the same encoding
2233 { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
2234
c7d11a61 2235 { SYS_DESC(SYS_DBGVCR32_EL2), trap_undef, reset_val, DBGVCR32_EL2, 0 },
62a89c44 2236
851050a5 2237 { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
93390c0a
DM		 2238
2239 /*
2240 * ID regs: all ID_SANITISED() entries here must have corresponding
2241 * entries in arm64_ftr_regs[].
2242 */
2243
2244 /* AArch64 mappings of the AArch32 ID registers */
2245 /* CRm=1 */
d5efec7e
OU		 2246 AA32_ID_SANITISED(ID_PFR0_EL1),
2247 AA32_ID_SANITISED(ID_PFR1_EL1),
c118cead
JZ		 2248 { SYS_DESC(SYS_ID_DFR0_EL1),
2249 .access = access_id_reg,
2250 .get_user = get_id_reg,
2251 .set_user = set_id_dfr0_el1,
2252 .visibility = aa32_id_visibility,
2253 .reset = read_sanitised_id_dfr0_el1,
9f9917bc
OU		 2254 .val = ID_DFR0_EL1_PerfMon_MASK |
2255 ID_DFR0_EL1_CopDbg_MASK, },
93390c0a 2256 ID_HIDDEN(ID_AFR0_EL1),
d5efec7e
OU		 2257 AA32_ID_SANITISED(ID_MMFR0_EL1),
2258 AA32_ID_SANITISED(ID_MMFR1_EL1),
2259 AA32_ID_SANITISED(ID_MMFR2_EL1),
2260 AA32_ID_SANITISED(ID_MMFR3_EL1),
93390c0a
DM		 2261
2262 /* CRm=2 */
d5efec7e
OU		 2263 AA32_ID_SANITISED(ID_ISAR0_EL1),
2264 AA32_ID_SANITISED(ID_ISAR1_EL1),
2265 AA32_ID_SANITISED(ID_ISAR2_EL1),
2266 AA32_ID_SANITISED(ID_ISAR3_EL1),
2267 AA32_ID_SANITISED(ID_ISAR4_EL1),
2268 AA32_ID_SANITISED(ID_ISAR5_EL1),
2269 AA32_ID_SANITISED(ID_MMFR4_EL1),
2270 AA32_ID_SANITISED(ID_ISAR6_EL1),
93390c0a
DM		 2271
2272 /* CRm=3 */
d5efec7e
OU		 2273 AA32_ID_SANITISED(MVFR0_EL1),
2274 AA32_ID_SANITISED(MVFR1_EL1),
2275 AA32_ID_SANITISED(MVFR2_EL1),
93390c0a 2276 ID_UNALLOCATED(3,3),
d5efec7e 2277 AA32_ID_SANITISED(ID_PFR2_EL1),
dd35ec07 2278 ID_HIDDEN(ID_DFR1_EL1),
d5efec7e 2279 AA32_ID_SANITISED(ID_MMFR5_EL1),
93390c0a
DM		 2280 ID_UNALLOCATED(3,7),
2281
2282 /* AArch64 ID registers */
2283 /* CRm=4 */
c39f5974
JZ		 2284 { SYS_DESC(SYS_ID_AA64PFR0_EL1),
2285 .access = access_id_reg,
2286 .get_user = get_id_reg,
68667240 2287 .set_user = set_id_reg,
c39f5974 2288 .reset = read_sanitised_id_aa64pfr0_el1,
8cfd5be8
JZ		 2289 .val = ~(ID_AA64PFR0_EL1_AMU |
2290 ID_AA64PFR0_EL1_MPAM |
2291 ID_AA64PFR0_EL1_SVE |
2292 ID_AA64PFR0_EL1_RAS |
2293 ID_AA64PFR0_EL1_GIC |
2294 ID_AA64PFR0_EL1_AdvSIMD |
2295 ID_AA64PFR0_EL1_FP), },
93390c0a
DM		 2296 ID_SANITISED(ID_AA64PFR1_EL1),
2297 ID_UNALLOCATED(4,2),
2298 ID_UNALLOCATED(4,3),
f89fbb35 2299 ID_WRITABLE(ID_AA64ZFR0_EL1, ~ID_AA64ZFR0_EL1_RES0),
90807748 2300 ID_HIDDEN(ID_AA64SMFR0_EL1),
93390c0a
DM		 2301 ID_UNALLOCATED(4,6),
2302 ID_UNALLOCATED(4,7),
2303
2304 /* CRm=5 */
c118cead
JZ		 2305 { SYS_DESC(SYS_ID_AA64DFR0_EL1),
2306 .access = access_id_reg,
2307 .get_user = get_id_reg,
2308 .set_user = set_id_aa64dfr0_el1,
2309 .reset = read_sanitised_id_aa64dfr0_el1,
9f9917bc
OU		 2310 .val = ID_AA64DFR0_EL1_PMUVer_MASK |
2311 ID_AA64DFR0_EL1_DebugVer_MASK, },
93390c0a
DM		 2312 ID_SANITISED(ID_AA64DFR1_EL1),
2313 ID_UNALLOCATED(5,2),
2314 ID_UNALLOCATED(5,3),
2315 ID_HIDDEN(ID_AA64AFR0_EL1),
2316 ID_HIDDEN(ID_AA64AFR1_EL1),
2317 ID_UNALLOCATED(5,6),
2318 ID_UNALLOCATED(5,7),
2319
2320 /* CRm=6 */
56d77aa8
OU		 2321 ID_WRITABLE(ID_AA64ISAR0_EL1, ~ID_AA64ISAR0_EL1_RES0),
2322 ID_WRITABLE(ID_AA64ISAR1_EL1, ~(ID_AA64ISAR1_EL1_GPI |
2323 ID_AA64ISAR1_EL1_GPA |
2324 ID_AA64ISAR1_EL1_API |
2325 ID_AA64ISAR1_EL1_APA)),
2326 ID_WRITABLE(ID_AA64ISAR2_EL1, ~(ID_AA64ISAR2_EL1_RES0 |
56d77aa8
OU		 2327 ID_AA64ISAR2_EL1_APA3 |
2328 ID_AA64ISAR2_EL1_GPA3)),
93390c0a
DM		 2329 ID_UNALLOCATED(6,3),
2330 ID_UNALLOCATED(6,4),
2331 ID_UNALLOCATED(6,5),
2332 ID_UNALLOCATED(6,6),
2333 ID_UNALLOCATED(6,7),
2334
2335 /* CRm=7 */
d5a32b60
JZ		 2336 ID_WRITABLE(ID_AA64MMFR0_EL1, ~(ID_AA64MMFR0_EL1_RES0 |
2337 ID_AA64MMFR0_EL1_TGRAN4_2 |
2338 ID_AA64MMFR0_EL1_TGRAN64_2 |
2339 ID_AA64MMFR0_EL1_TGRAN16_2)),
2340 ID_WRITABLE(ID_AA64MMFR1_EL1, ~(ID_AA64MMFR1_EL1_RES0 |
2341 ID_AA64MMFR1_EL1_HCX |
2342 ID_AA64MMFR1_EL1_XNX |
2343 ID_AA64MMFR1_EL1_TWED |
2344 ID_AA64MMFR1_EL1_XNX |
2345 ID_AA64MMFR1_EL1_VH |
2346 ID_AA64MMFR1_EL1_VMIDBits)),
2347 ID_WRITABLE(ID_AA64MMFR2_EL1, ~(ID_AA64MMFR2_EL1_RES0 |
2348 ID_AA64MMFR2_EL1_EVT |
2349 ID_AA64MMFR2_EL1_FWB |
2350 ID_AA64MMFR2_EL1_IDS |
2351 ID_AA64MMFR2_EL1_NV |
2352 ID_AA64MMFR2_EL1_CCIDX)),
8ef67c67 2353 ID_SANITISED(ID_AA64MMFR3_EL1),
c21df6e4 2354 ID_SANITISED(ID_AA64MMFR4_EL1),
93390c0a
DM		 2355 ID_UNALLOCATED(7,5),
2356 ID_UNALLOCATED(7,6),
2357 ID_UNALLOCATED(7,7),
2358
851050a5 2359 { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
af473829 2360 { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
851050a5 2361 { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
2ac638fc 2362
e1f358b5
SP		 2363 MTE_REG(RGSR_EL1),
2364 MTE_REG(GCR_EL1),
2ac638fc 2365
73433762 2366 { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
cc427cbb 2367 { SYS_DESC(SYS_TRFCR_EL1), undef_access },
90807748
MB		 2368 { SYS_DESC(SYS_SMPRI_EL1), undef_access },
2369 { SYS_DESC(SYS_SMCR_EL1), undef_access },
851050a5
MR		 2370 { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
2371 { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
2372 { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
fbff5606 2373 { SYS_DESC(SYS_TCR2_EL1), access_vm_reg, reset_val, TCR2_EL1, 0 },
851050a5 2374
384b40ca
MR		 2375 PTRAUTH_KEY(APIA),
2376 PTRAUTH_KEY(APIB),
2377 PTRAUTH_KEY(APDA),
2378 PTRAUTH_KEY(APDB),
2379 PTRAUTH_KEY(APGA),
2380
9da117ee
JL		 2381 { SYS_DESC(SYS_SPSR_EL1), access_spsr},
2382 { SYS_DESC(SYS_ELR_EL1), access_elr},
2383
851050a5
MR		 2384 { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
2385 { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
2386 { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
558daf69
DG		 2387
2388 { SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
2389 { SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
2390 { SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
2391 { SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
2392 { SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
2393 { SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
2394 { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
2395 { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
2396
e1f358b5
SP		 2397 MTE_REG(TFSR_EL1),
2398 MTE_REG(TFSRE0_EL1),
2ac638fc 2399
851050a5
MR		 2400 { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
2401 { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
7c8c5e6a 2402
13611bc8
AE		 2403 { SYS_DESC(SYS_PMSCR_EL1), undef_access },
2404 { SYS_DESC(SYS_PMSNEVFR_EL1), undef_access },
2405 { SYS_DESC(SYS_PMSICR_EL1), undef_access },
2406 { SYS_DESC(SYS_PMSIRR_EL1), undef_access },
2407 { SYS_DESC(SYS_PMSFCR_EL1), undef_access },
2408 { SYS_DESC(SYS_PMSEVFR_EL1), undef_access },
2409 { SYS_DESC(SYS_PMSLATFR_EL1), undef_access },
2410 { SYS_DESC(SYS_PMSIDR_EL1), undef_access },
2411 { SYS_DESC(SYS_PMBLIMITR_EL1), undef_access },
2412 { SYS_DESC(SYS_PMBPTR_EL1), undef_access },
2413 { SYS_DESC(SYS_PMBSR_EL1), undef_access },
2414 /* PMBIDR_EL1 is not trapped */
2415
9d2a55b4 2416 { PMU_SYS_REG(PMINTENSET_EL1),
a45f41d7
RRA		 2417 .access = access_pminten, .reg = PMINTENSET_EL1,
2418 .get_user = get_pmreg, .set_user = set_pmreg },
9d2a55b4 2419 { PMU_SYS_REG(PMINTENCLR_EL1),
a45f41d7
RRA		 2420 .access = access_pminten, .reg = PMINTENSET_EL1,
2421 .get_user = get_pmreg, .set_user = set_pmreg },
46081078 2422 { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi },
7c8c5e6a 2423
851050a5 2424 { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
839d9035
JG		 2425 { SYS_DESC(SYS_PIRE0_EL1), NULL, reset_unknown, PIRE0_EL1 },
2426 { SYS_DESC(SYS_PIR_EL1), NULL, reset_unknown, PIR_EL1 },
851050a5 2427 { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
7c8c5e6a 2428
22925521
MZ		 2429 { SYS_DESC(SYS_LORSA_EL1), trap_loregion },
2430 { SYS_DESC(SYS_LOREA_EL1), trap_loregion },
2431 { SYS_DESC(SYS_LORN_EL1), trap_loregion },
2432 { SYS_DESC(SYS_LORC_EL1), trap_loregion },
2433 { SYS_DESC(SYS_LORID_EL1), trap_loregion },
cc33c4e2 2434
9da117ee 2435 { SYS_DESC(SYS_VBAR_EL1), access_rw, reset_val, VBAR_EL1, 0 },
c773ae2b 2436 { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
db7dedd0 2437
7b1dba1f 2438 { SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
e7f1d1ee 2439 { SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
7b1dba1f 2440 { SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
e7f1d1ee 2441 { SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
7b1dba1f 2442 { SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
e804d208 2443 { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
03bd646d
MZ		 2444 { SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
2445 { SYS_DESC(SYS_ICC_SGI0R_EL1), access_gic_sgi },
7b1dba1f 2446 { SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
e7f1d1ee 2447 { SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
7b1dba1f 2448 { SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
e804d208 2449 { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
db7dedd0 2450
851050a5
MR		 2451 { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
2452 { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
7c8c5e6a 2453
484f8682
MZ		 2454 { SYS_DESC(SYS_ACCDATA_EL1), undef_access },
2455
ed4ffaf4
MZ		 2456 { SYS_DESC(SYS_SCXTNUM_EL1), undef_access },
2457
851050a5 2458 { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
7c8c5e6a 2459
f7f2b15c 2460 { SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
7af0c253
AO		 2461 { SYS_DESC(SYS_CLIDR_EL1), access_clidr, reset_clidr, CLIDR_EL1,
2462 .set_user = set_clidr },
bf48040c 2463 { SYS_DESC(SYS_CCSIDR2_EL1), undef_access },
90807748 2464 { SYS_DESC(SYS_SMIDR_EL1), undef_access },
f7f2b15c
AB		 2465 { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
2466 { SYS_DESC(SYS_CTR_EL0), access_ctr },
ec0067a6 2467 { SYS_DESC(SYS_SVCR), undef_access },
7c8c5e6a 2468
ea9ca904
RW		 2469 { PMU_SYS_REG(PMCR_EL0), .access = access_pmcr, .reset = reset_pmcr,
2470 .reg = PMCR_EL0, .get_user = get_pmcr, .set_user = set_pmcr },
9d2a55b4 2471 { PMU_SYS_REG(PMCNTENSET_EL0),
a45f41d7
RRA		 2472 .access = access_pmcnten, .reg = PMCNTENSET_EL0,
2473 .get_user = get_pmreg, .set_user = set_pmreg },
9d2a55b4 2474 { PMU_SYS_REG(PMCNTENCLR_EL0),
a45f41d7
RRA		 2475 .access = access_pmcnten, .reg = PMCNTENSET_EL0,
2476 .get_user = get_pmreg, .set_user = set_pmreg },
9d2a55b4 2477 { PMU_SYS_REG(PMOVSCLR_EL0),
a45f41d7
RRA		 2478 .access = access_pmovs, .reg = PMOVSSET_EL0,
2479 .get_user = get_pmreg, .set_user = set_pmreg },
7a3ba309
MZ		 2480 /*
2481 * PM_SWINC_EL0 is exposed to userspace as RAZ/WI, as it was
2482 * previously (and pointlessly) advertised in the past...
2483 */
9d2a55b4 2484 { PMU_SYS_REG(PMSWINC_EL0),
5a430976 2485 .get_user = get_raz_reg, .set_user = set_wi_reg,
7a3ba309 2486 .access = access_pmswinc, .reset = NULL },
9d2a55b4 2487 { PMU_SYS_REG(PMSELR_EL0),
0ab410a9 2488 .access = access_pmselr, .reset = reset_pmselr, .reg = PMSELR_EL0 },
9d2a55b4 2489 { PMU_SYS_REG(PMCEID0_EL0),
11663111 2490 .access = access_pmceid, .reset = NULL },
9d2a55b4 2491 { PMU_SYS_REG(PMCEID1_EL0),
11663111 2492 .access = access_pmceid, .reset = NULL },
9d2a55b4 2493 { PMU_SYS_REG(PMCCNTR_EL0),
9228b261
RW		 2494 .access = access_pmu_evcntr, .reset = reset_unknown,
2495 .reg = PMCCNTR_EL0, .get_user = get_pmu_evcntr},
9d2a55b4 2496 { PMU_SYS_REG(PMXEVTYPER_EL0),
11663111 2497 .access = access_pmu_evtyper, .reset = NULL },
9d2a55b4 2498 { PMU_SYS_REG(PMXEVCNTR_EL0),
11663111 2499 .access = access_pmu_evcntr, .reset = NULL },
174ed3e4
MR		 2500 /*
2501 * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
d692b8ad
SZ		 2502 * in 32bit mode. Here we choose to reset it as zero for consistency.
2503 */
9d2a55b4 2504 { PMU_SYS_REG(PMUSERENR_EL0), .access = access_pmuserenr,
11663111 2505 .reset = reset_val, .reg = PMUSERENR_EL0, .val = 0 },
9d2a55b4 2506 { PMU_SYS_REG(PMOVSSET_EL0),
a45f41d7
RRA		 2507 .access = access_pmovs, .reg = PMOVSSET_EL0,
2508 .get_user = get_pmreg, .set_user = set_pmreg },
7c8c5e6a 2509
851050a5
MR		 2510 { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
2511 { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
90807748 2512 { SYS_DESC(SYS_TPIDR2_EL0), undef_access },
4fcdf106 2513
ed4ffaf4
MZ		 2514 { SYS_DESC(SYS_SCXTNUM_EL0), undef_access },
2515
338b1793
MZ		 2516 { SYS_DESC(SYS_AMCR_EL0), undef_access },
2517 { SYS_DESC(SYS_AMCFGR_EL0), undef_access },
2518 { SYS_DESC(SYS_AMCGCR_EL0), undef_access },
2519 { SYS_DESC(SYS_AMUSERENR_EL0), undef_access },
2520 { SYS_DESC(SYS_AMCNTENCLR0_EL0), undef_access },
2521 { SYS_DESC(SYS_AMCNTENSET0_EL0), undef_access },
2522 { SYS_DESC(SYS_AMCNTENCLR1_EL0), undef_access },
2523 { SYS_DESC(SYS_AMCNTENSET1_EL0), undef_access },
4fcdf106
IV		 2524 AMU_AMEVCNTR0_EL0(0),
2525 AMU_AMEVCNTR0_EL0(1),
2526 AMU_AMEVCNTR0_EL0(2),
2527 AMU_AMEVCNTR0_EL0(3),
2528 AMU_AMEVCNTR0_EL0(4),
2529 AMU_AMEVCNTR0_EL0(5),
2530 AMU_AMEVCNTR0_EL0(6),
2531 AMU_AMEVCNTR0_EL0(7),
2532 AMU_AMEVCNTR0_EL0(8),
2533 AMU_AMEVCNTR0_EL0(9),
2534 AMU_AMEVCNTR0_EL0(10),
2535 AMU_AMEVCNTR0_EL0(11),
2536 AMU_AMEVCNTR0_EL0(12),
2537 AMU_AMEVCNTR0_EL0(13),
2538 AMU_AMEVCNTR0_EL0(14),
2539 AMU_AMEVCNTR0_EL0(15),
493cf9b7
VM		 2540 AMU_AMEVTYPER0_EL0(0),
2541 AMU_AMEVTYPER0_EL0(1),
2542 AMU_AMEVTYPER0_EL0(2),
2543 AMU_AMEVTYPER0_EL0(3),
2544 AMU_AMEVTYPER0_EL0(4),
2545 AMU_AMEVTYPER0_EL0(5),
2546 AMU_AMEVTYPER0_EL0(6),
2547 AMU_AMEVTYPER0_EL0(7),
2548 AMU_AMEVTYPER0_EL0(8),
2549 AMU_AMEVTYPER0_EL0(9),
2550 AMU_AMEVTYPER0_EL0(10),
2551 AMU_AMEVTYPER0_EL0(11),
2552 AMU_AMEVTYPER0_EL0(12),
2553 AMU_AMEVTYPER0_EL0(13),
2554 AMU_AMEVTYPER0_EL0(14),
2555 AMU_AMEVTYPER0_EL0(15),
4fcdf106
IV		 2556 AMU_AMEVCNTR1_EL0(0),
2557 AMU_AMEVCNTR1_EL0(1),
2558 AMU_AMEVCNTR1_EL0(2),
2559 AMU_AMEVCNTR1_EL0(3),
2560 AMU_AMEVCNTR1_EL0(4),
2561 AMU_AMEVCNTR1_EL0(5),
2562 AMU_AMEVCNTR1_EL0(6),
2563 AMU_AMEVCNTR1_EL0(7),
2564 AMU_AMEVCNTR1_EL0(8),
2565 AMU_AMEVCNTR1_EL0(9),
2566 AMU_AMEVCNTR1_EL0(10),
2567 AMU_AMEVCNTR1_EL0(11),
2568 AMU_AMEVCNTR1_EL0(12),
2569 AMU_AMEVCNTR1_EL0(13),
2570 AMU_AMEVCNTR1_EL0(14),
2571 AMU_AMEVCNTR1_EL0(15),
493cf9b7
VM		 2572 AMU_AMEVTYPER1_EL0(0),
2573 AMU_AMEVTYPER1_EL0(1),
2574 AMU_AMEVTYPER1_EL0(2),
2575 AMU_AMEVTYPER1_EL0(3),
2576 AMU_AMEVTYPER1_EL0(4),
2577 AMU_AMEVTYPER1_EL0(5),
2578 AMU_AMEVTYPER1_EL0(6),
2579 AMU_AMEVTYPER1_EL0(7),
2580 AMU_AMEVTYPER1_EL0(8),
2581 AMU_AMEVTYPER1_EL0(9),
2582 AMU_AMEVTYPER1_EL0(10),
2583 AMU_AMEVTYPER1_EL0(11),
2584 AMU_AMEVTYPER1_EL0(12),
2585 AMU_AMEVTYPER1_EL0(13),
2586 AMU_AMEVTYPER1_EL0(14),
2587 AMU_AMEVTYPER1_EL0(15),
62a89c44 2588
c605ee24
MZ		 2589 { SYS_DESC(SYS_CNTPCT_EL0), access_arch_timer },
2590 { SYS_DESC(SYS_CNTPCTSS_EL0), access_arch_timer },
84135d3d
AP		 2591 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
2592 { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
2593 { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
c9a3c58f 2594
051ff581
SZ		 2595 /* PMEVCNTRn_EL0 */
2596 PMU_PMEVCNTR_EL0(0),
2597 PMU_PMEVCNTR_EL0(1),
2598 PMU_PMEVCNTR_EL0(2),
2599 PMU_PMEVCNTR_EL0(3),
2600 PMU_PMEVCNTR_EL0(4),
2601 PMU_PMEVCNTR_EL0(5),
2602 PMU_PMEVCNTR_EL0(6),
2603 PMU_PMEVCNTR_EL0(7),
2604 PMU_PMEVCNTR_EL0(8),
2605 PMU_PMEVCNTR_EL0(9),
2606 PMU_PMEVCNTR_EL0(10),
2607 PMU_PMEVCNTR_EL0(11),
2608 PMU_PMEVCNTR_EL0(12),
2609 PMU_PMEVCNTR_EL0(13),
2610 PMU_PMEVCNTR_EL0(14),
2611 PMU_PMEVCNTR_EL0(15),
2612 PMU_PMEVCNTR_EL0(16),
2613 PMU_PMEVCNTR_EL0(17),
2614 PMU_PMEVCNTR_EL0(18),
2615 PMU_PMEVCNTR_EL0(19),
2616 PMU_PMEVCNTR_EL0(20),
2617 PMU_PMEVCNTR_EL0(21),
2618 PMU_PMEVCNTR_EL0(22),
2619 PMU_PMEVCNTR_EL0(23),
2620 PMU_PMEVCNTR_EL0(24),
2621 PMU_PMEVCNTR_EL0(25),
2622 PMU_PMEVCNTR_EL0(26),
2623 PMU_PMEVCNTR_EL0(27),
2624 PMU_PMEVCNTR_EL0(28),
2625 PMU_PMEVCNTR_EL0(29),
2626 PMU_PMEVCNTR_EL0(30),
9feb21ac
SZ		 2627 /* PMEVTYPERn_EL0 */
2628 PMU_PMEVTYPER_EL0(0),
2629 PMU_PMEVTYPER_EL0(1),
2630 PMU_PMEVTYPER_EL0(2),
2631 PMU_PMEVTYPER_EL0(3),
2632 PMU_PMEVTYPER_EL0(4),
2633 PMU_PMEVTYPER_EL0(5),
2634 PMU_PMEVTYPER_EL0(6),
2635 PMU_PMEVTYPER_EL0(7),
2636 PMU_PMEVTYPER_EL0(8),
2637 PMU_PMEVTYPER_EL0(9),
2638 PMU_PMEVTYPER_EL0(10),
2639 PMU_PMEVTYPER_EL0(11),
2640 PMU_PMEVTYPER_EL0(12),
2641 PMU_PMEVTYPER_EL0(13),
2642 PMU_PMEVTYPER_EL0(14),
2643 PMU_PMEVTYPER_EL0(15),
2644 PMU_PMEVTYPER_EL0(16),
2645 PMU_PMEVTYPER_EL0(17),
2646 PMU_PMEVTYPER_EL0(18),
2647 PMU_PMEVTYPER_EL0(19),
2648 PMU_PMEVTYPER_EL0(20),
2649 PMU_PMEVTYPER_EL0(21),
2650 PMU_PMEVTYPER_EL0(22),
2651 PMU_PMEVTYPER_EL0(23),
2652 PMU_PMEVTYPER_EL0(24),
2653 PMU_PMEVTYPER_EL0(25),
2654 PMU_PMEVTYPER_EL0(26),
2655 PMU_PMEVTYPER_EL0(27),
2656 PMU_PMEVTYPER_EL0(28),
2657 PMU_PMEVTYPER_EL0(29),
2658 PMU_PMEVTYPER_EL0(30),
174ed3e4
MR		 2659 /*
2660 * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
9feb21ac
SZ		 2661 * in 32bit mode. Here we choose to reset it as zero for consistency.
2662 */
9d2a55b4 2663 { PMU_SYS_REG(PMCCFILTR_EL0), .access = access_pmu_evtyper,
11663111 2664 .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
051ff581 2665
9b9cce60
MZ		 2666 EL2_REG_VNCR(VPIDR_EL2, reset_unknown, 0),
2667 EL2_REG_VNCR(VMPIDR_EL2, reset_unknown, 0),
6ff9dc23
JL		 2668 EL2_REG(SCTLR_EL2, access_rw, reset_val, SCTLR_EL2_RES1),
2669 EL2_REG(ACTLR_EL2, access_rw, reset_val, 0),
94f29ab2 2670 EL2_REG_VNCR(HCR_EL2, reset_hcr, 0),
6ff9dc23 2671 EL2_REG(MDCR_EL2, access_rw, reset_val, 0),
75c76ab5 2672 EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1),
9b9cce60
MZ		 2673 EL2_REG_VNCR(HSTR_EL2, reset_val, 0),
2674 EL2_REG_VNCR(HFGRTR_EL2, reset_val, 0),
2675 EL2_REG_VNCR(HFGWTR_EL2, reset_val, 0),
2676 EL2_REG_VNCR(HFGITR_EL2, reset_val, 0),
2677 EL2_REG_VNCR(HACR_EL2, reset_val, 0),
6ff9dc23 2678
9b9cce60 2679 EL2_REG_VNCR(HCRX_EL2, reset_val, 0),
03fb54d0 2680
6ff9dc23
JL		 2681 EL2_REG(TTBR0_EL2, access_rw, reset_val, 0),
2682 EL2_REG(TTBR1_EL2, access_rw, reset_val, 0),
2683 EL2_REG(TCR_EL2, access_rw, reset_val, TCR_EL2_RES1),
9b9cce60
MZ		 2684 EL2_REG_VNCR(VTTBR_EL2, reset_val, 0),
2685 EL2_REG_VNCR(VTCR_EL2, reset_val, 0),
6ff9dc23 2686
c7d11a61 2687 { SYS_DESC(SYS_DACR32_EL2), trap_undef, reset_unknown, DACR32_EL2 },
9b9cce60
MZ		 2688 EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
2689 EL2_REG_VNCR(HDFGWTR_EL2, reset_val, 0),
d016264d 2690 EL2_REG_VNCR(HAFGRTR_EL2, reset_val, 0),
9b9cce60
MZ		 2691 EL2_REG_REDIR(SPSR_EL2, reset_val, 0),
2692 EL2_REG_REDIR(ELR_EL2, reset_val, 0),
6ff9dc23
JL		 2693 { SYS_DESC(SYS_SP_EL1), access_sp_el1},
2694
3f7915cc
MZ		 2695 /* AArch32 SPSR_* are RES0 if trapped from a NV guest */
2696 { SYS_DESC(SYS_SPSR_irq), .access = trap_raz_wi,
2697 .visibility = hidden_user_visibility },
2698 { SYS_DESC(SYS_SPSR_abt), .access = trap_raz_wi,
2699 .visibility = hidden_user_visibility },
2700 { SYS_DESC(SYS_SPSR_und), .access = trap_raz_wi,
2701 .visibility = hidden_user_visibility },
2702 { SYS_DESC(SYS_SPSR_fiq), .access = trap_raz_wi,
2703 .visibility = hidden_user_visibility },
2704
c7d11a61 2705 { SYS_DESC(SYS_IFSR32_EL2), trap_undef, reset_unknown, IFSR32_EL2 },
6ff9dc23
JL		 2706 EL2_REG(AFSR0_EL2, access_rw, reset_val, 0),
2707 EL2_REG(AFSR1_EL2, access_rw, reset_val, 0),
9b9cce60 2708 EL2_REG_REDIR(ESR_EL2, reset_val, 0),
c7d11a61 2709 { SYS_DESC(SYS_FPEXC32_EL2), trap_undef, reset_val, FPEXC32_EL2, 0x700 },
6ff9dc23 2710
9b9cce60 2711 EL2_REG_REDIR(FAR_EL2, reset_val, 0),
6ff9dc23
JL		 2712 EL2_REG(HPFAR_EL2, access_rw, reset_val, 0),
2713
2714 EL2_REG(MAIR_EL2, access_rw, reset_val, 0),
2715 EL2_REG(AMAIR_EL2, access_rw, reset_val, 0),
2716
2717 EL2_REG(VBAR_EL2, access_rw, reset_val, 0),
2718 EL2_REG(RVBAR_EL2, access_rw, reset_val, 0),
2719 { SYS_DESC(SYS_RMR_EL2), trap_undef },
2720
2721 EL2_REG(CONTEXTIDR_EL2, access_rw, reset_val, 0),
2722 EL2_REG(TPIDR_EL2, access_rw, reset_val, 0),
2723
9b9cce60 2724 EL2_REG_VNCR(CNTVOFF_EL2, reset_val, 0),
6ff9dc23
JL		 2725 EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0),
2726
280b748e
JL		 2727 EL12_REG(CNTKCTL, access_rw, reset_val, 0),
2728
6ff9dc23 2729 EL2_REG(SP_EL2, NULL, reset_unknown, 0),
62a89c44
MZ		 2730};
2731
89bc63fa
MZ		 2732static struct sys_reg_desc sys_insn_descs[] = {
2733 { SYS_DESC(SYS_DC_ISW), access_dcsw },
2734 { SYS_DESC(SYS_DC_IGSW), access_dcgsw },
2735 { SYS_DESC(SYS_DC_IGDSW), access_dcgsw },
2736 { SYS_DESC(SYS_DC_CSW), access_dcsw },
2737 { SYS_DESC(SYS_DC_CGSW), access_dcgsw },
2738 { SYS_DESC(SYS_DC_CGDSW), access_dcgsw },
2739 { SYS_DESC(SYS_DC_CISW), access_dcsw },
2740 { SYS_DESC(SYS_DC_CIGSW), access_dcgsw },
2741 { SYS_DESC(SYS_DC_CIGDSW), access_dcgsw },
2742};
2743
47334146
JZ		 2744static const struct sys_reg_desc *first_idreg;
2745
8c358b29 2746static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
3fec037d 2747 struct sys_reg_params *p,
bdfb4b38
MZ		 2748 const struct sys_reg_desc *r)
2749{
2750 if (p->is_write) {
2751 return ignore_write(vcpu, p);
2752 } else {
8b6958d6 2753 u64 dfr = IDREG(vcpu->kvm, SYS_ID_AA64DFR0_EL1);
c62d7a23 2754 u32 el3 = kvm_has_feat(vcpu->kvm, ID_AA64PFR0_EL1, EL3, IMP);
bdfb4b38 2755
5a23e5c7
OU		 2756 p->regval = ((SYS_FIELD_GET(ID_AA64DFR0_EL1, WRPs, dfr) << 28) |
2757 (SYS_FIELD_GET(ID_AA64DFR0_EL1, BRPs, dfr) << 24) |
2758 (SYS_FIELD_GET(ID_AA64DFR0_EL1, CTX_CMPs, dfr) << 20) |
2759 (SYS_FIELD_GET(ID_AA64DFR0_EL1, DebugVer, dfr) << 16) |
2760 (1 << 15) | (el3 << 14) | (el3 << 12));
bdfb4b38
MZ		 2761 return true;
2762 }
2763}
2764
1da42c34
MZ		 2765/*
2766 * AArch32 debug register mappings
84e690bf
AB		 2767 *
2768 * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
2769 * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
2770 *
1da42c34
MZ		 2771 * None of the other registers share their location, so treat them as
2772 * if they were 64bit.
84e690bf 2773 */
1da42c34
MZ		 2774#define DBG_BCR_BVR_WCR_WVR(n) \
2775 /* DBGBVRn */ \
2776 { AA32(LO), Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
2777 /* DBGBCRn */ \
2778 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
2779 /* DBGWVRn */ \
2780 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
2781 /* DBGWCRn */ \
84e690bf
AB		 2782 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
2783
1da42c34
MZ		 2784#define DBGBXVR(n) \
2785 { AA32(HI), Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_bvr, NULL, n }
bdfb4b38
MZ		 2786
2787/*
2788 * Trapped cp14 registers. We generally ignore most of the external
2789 * debug, on the principle that they don't really make sense to a
84e690bf 2790 * guest. Revisit this one day, would this principle change.
bdfb4b38 2791 */
72564016 2792static const struct sys_reg_desc cp14_regs[] = {
8c358b29
AE		 2793 /* DBGDIDR */
2794 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgdidr },
bdfb4b38
MZ		 2795 /* DBGDTRRXext */
2796 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
2797
2798 DBG_BCR_BVR_WCR_WVR(0),
2799 /* DBGDSCRint */
2800 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
2801 DBG_BCR_BVR_WCR_WVR(1),
2802 /* DBGDCCINT */
1da42c34 2803 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug_regs, NULL, MDCCINT_EL1 },
bdfb4b38 2804 /* DBGDSCRext */
1da42c34 2805 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug_regs, NULL, MDSCR_EL1 },
bdfb4b38
MZ		 2806 DBG_BCR_BVR_WCR_WVR(2),
2807 /* DBGDTR[RT]Xint */
2808 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
2809 /* DBGDTR[RT]Xext */
2810 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
2811 DBG_BCR_BVR_WCR_WVR(3),
2812 DBG_BCR_BVR_WCR_WVR(4),
2813 DBG_BCR_BVR_WCR_WVR(5),
2814 /* DBGWFAR */
2815 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
2816 /* DBGOSECCR */
2817 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
2818 DBG_BCR_BVR_WCR_WVR(6),
2819 /* DBGVCR */
1da42c34 2820 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug_regs, NULL, DBGVCR32_EL2 },
bdfb4b38
MZ		 2821 DBG_BCR_BVR_WCR_WVR(7),
2822 DBG_BCR_BVR_WCR_WVR(8),
2823 DBG_BCR_BVR_WCR_WVR(9),
2824 DBG_BCR_BVR_WCR_WVR(10),
2825 DBG_BCR_BVR_WCR_WVR(11),
2826 DBG_BCR_BVR_WCR_WVR(12),
2827 DBG_BCR_BVR_WCR_WVR(13),
2828 DBG_BCR_BVR_WCR_WVR(14),
2829 DBG_BCR_BVR_WCR_WVR(15),
2830
2831 /* DBGDRAR (32bit) */
2832 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
2833
2834 DBGBXVR(0),
2835 /* DBGOSLAR */
f24adc65 2836 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_oslar_el1 },
bdfb4b38
MZ		 2837 DBGBXVR(1),
2838 /* DBGOSLSR */
d42e2671 2839 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1, NULL, OSLSR_EL1 },
bdfb4b38
MZ		 2840 DBGBXVR(2),
2841 DBGBXVR(3),
2842 /* DBGOSDLR */
2843 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
2844 DBGBXVR(4),
2845 /* DBGPRCR */
2846 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
2847 DBGBXVR(5),
2848 DBGBXVR(6),
2849 DBGBXVR(7),
2850 DBGBXVR(8),
2851 DBGBXVR(9),
2852 DBGBXVR(10),
2853 DBGBXVR(11),
2854 DBGBXVR(12),
2855 DBGBXVR(13),
2856 DBGBXVR(14),
2857 DBGBXVR(15),
2858
2859 /* DBGDSAR (32bit) */
2860 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
2861
2862 /* DBGDEVID2 */
2863 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
2864 /* DBGDEVID1 */
2865 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
2866 /* DBGDEVID */
2867 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
2868 /* DBGCLAIMSET */
2869 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
2870 /* DBGCLAIMCLR */
2871 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
2872 /* DBGAUTHSTATUS */
2873 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
72564016
MZ		 2874};
2875
a9866ba0
MZ		 2876/* Trapped cp14 64bit registers */
2877static const struct sys_reg_desc cp14_64_regs[] = {
bdfb4b38
MZ		 2878 /* DBGDRAR (64bit) */
2879 { Op1( 0), CRm( 1), .access = trap_raz_wi },
2880
2881 /* DBGDSAR (64bit) */
2882 { Op1( 0), CRm( 2), .access = trap_raz_wi },
a9866ba0
MZ		 2883};
2884
a9e192cd
AE		 2885#define CP15_PMU_SYS_REG(_map, _Op1, _CRn, _CRm, _Op2) \
2886 AA32(_map), \
2887 Op1(_Op1), CRn(_CRn), CRm(_CRm), Op2(_Op2), \
2888 .visibility = pmu_visibility
2889
051ff581
SZ		 2890/* Macro to expand the PMEVCNTRn register */
2891#define PMU_PMEVCNTR(n) \
a9e192cd
AE		 2892 { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \
2893 (0b1000 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \
2894 .access = access_pmu_evcntr }
051ff581 2895
9feb21ac
SZ		 2896/* Macro to expand the PMEVTYPERn register */
2897#define PMU_PMEVTYPER(n) \
a9e192cd
AE		 2898 { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \
2899 (0b1100 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \
2900 .access = access_pmu_evtyper }
4d44923b
MZ		 2901/*
2902 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
2903 * depending on the way they are accessed (as a 32bit or a 64bit
2904 * register).
2905 */
62a89c44 2906static const struct sys_reg_desc cp15_regs[] = {
f7f2b15c 2907 { Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
b1ea1d76
MZ		 2908 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, SCTLR_EL1 },
2909 /* ACTLR */
2910 { AA32(LO), Op1( 0), CRn( 1), CRm( 0), Op2( 1), access_actlr, NULL, ACTLR_EL1 },
2911 /* ACTLR2 */
2912 { AA32(HI), Op1( 0), CRn( 1), CRm( 0), Op2( 3), access_actlr, NULL, ACTLR_EL1 },
2913 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
2914 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, TTBR1_EL1 },
2915 /* TTBCR */
2916 { AA32(LO), Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, TCR_EL1 },
2917 /* TTBCR2 */
2918 { AA32(HI), Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, TCR_EL1 },
2919 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, DACR32_EL2 },
2920 /* DFSR */
2921 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, ESR_EL1 },
2922 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, IFSR32_EL2 },
2923 /* ADFSR */
2924 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, AFSR0_EL1 },
2925 /* AIFSR */
2926 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, AFSR1_EL1 },
2927 /* DFAR */
2928 { AA32(LO), Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, FAR_EL1 },
2929 /* IFAR */
2930 { AA32(HI), Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, FAR_EL1 },
4d44923b 2931
62a89c44
MZ		 2932 /*
2933 * DC{C,I,CI}SW operations:
2934 */
2935 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
2936 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
2937 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
4d44923b 2938
7609c125 2939 /* PMU */
a9e192cd
AE		 2940 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 0), .access = access_pmcr },
2941 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 1), .access = access_pmcnten },
2942 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 2), .access = access_pmcnten },
2943 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 3), .access = access_pmovs },
2944 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 4), .access = access_pmswinc },
2945 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 5), .access = access_pmselr },
2946 { CP15_PMU_SYS_REG(LO, 0, 9, 12, 6), .access = access_pmceid },
2947 { CP15_PMU_SYS_REG(LO, 0, 9, 12, 7), .access = access_pmceid },
2948 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 0), .access = access_pmu_evcntr },
2949 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 1), .access = access_pmu_evtyper },
2950 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 2), .access = access_pmu_evcntr },
2951 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 0), .access = access_pmuserenr },
2952 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 1), .access = access_pminten },
2953 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 2), .access = access_pminten },
2954 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 3), .access = access_pmovs },
2955 { CP15_PMU_SYS_REG(HI, 0, 9, 14, 4), .access = access_pmceid },
2956 { CP15_PMU_SYS_REG(HI, 0, 9, 14, 5), .access = access_pmceid },
46081078 2957 /* PMMIR */
a9e192cd 2958 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = trap_raz_wi },
4d44923b 2959
b1ea1d76
MZ		 2960 /* PRRR/MAIR0 */
2961 { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },
2962 /* NMRR/MAIR1 */
2963 { AA32(HI), Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, MAIR_EL1 },
2964 /* AMAIR0 */
2965 { AA32(LO), Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, AMAIR_EL1 },
2966 /* AMAIR1 */
2967 { AA32(HI), Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, AMAIR_EL1 },
db7dedd0
CD		 2968
2969 /* ICC_SRE */
f7f6f2d9 2970 { Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
db7dedd0 2971
b1ea1d76 2972 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, CONTEXTIDR_EL1 },
051ff581 2973
84135d3d
AP		 2974 /* Arch Tmers */
2975 { SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
2976 { SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
eac137b4 2977
051ff581
SZ		 2978 /* PMEVCNTRn */
2979 PMU_PMEVCNTR(0),
2980 PMU_PMEVCNTR(1),
2981 PMU_PMEVCNTR(2),
2982 PMU_PMEVCNTR(3),
2983 PMU_PMEVCNTR(4),
2984 PMU_PMEVCNTR(5),
2985 PMU_PMEVCNTR(6),
2986 PMU_PMEVCNTR(7),
2987 PMU_PMEVCNTR(8),
2988 PMU_PMEVCNTR(9),
2989 PMU_PMEVCNTR(10),
2990 PMU_PMEVCNTR(11),
2991 PMU_PMEVCNTR(12),
2992 PMU_PMEVCNTR(13),
2993 PMU_PMEVCNTR(14),
2994 PMU_PMEVCNTR(15),
2995 PMU_PMEVCNTR(16),
2996 PMU_PMEVCNTR(17),
2997 PMU_PMEVCNTR(18),
2998 PMU_PMEVCNTR(19),
2999 PMU_PMEVCNTR(20),
3000 PMU_PMEVCNTR(21),
3001 PMU_PMEVCNTR(22),
3002 PMU_PMEVCNTR(23),
3003 PMU_PMEVCNTR(24),
3004 PMU_PMEVCNTR(25),
3005 PMU_PMEVCNTR(26),
3006 PMU_PMEVCNTR(27),
3007 PMU_PMEVCNTR(28),
3008 PMU_PMEVCNTR(29),
3009 PMU_PMEVCNTR(30),
9feb21ac
SZ		 3010 /* PMEVTYPERn */
3011 PMU_PMEVTYPER(0),
3012 PMU_PMEVTYPER(1),
3013 PMU_PMEVTYPER(2),
3014 PMU_PMEVTYPER(3),
3015 PMU_PMEVTYPER(4),
3016 PMU_PMEVTYPER(5),
3017 PMU_PMEVTYPER(6),
3018 PMU_PMEVTYPER(7),
3019 PMU_PMEVTYPER(8),
3020 PMU_PMEVTYPER(9),
3021 PMU_PMEVTYPER(10),
3022 PMU_PMEVTYPER(11),
3023 PMU_PMEVTYPER(12),
3024 PMU_PMEVTYPER(13),
3025 PMU_PMEVTYPER(14),
3026 PMU_PMEVTYPER(15),
3027 PMU_PMEVTYPER(16),
3028 PMU_PMEVTYPER(17),
3029 PMU_PMEVTYPER(18),
3030 PMU_PMEVTYPER(19),
3031 PMU_PMEVTYPER(20),
3032 PMU_PMEVTYPER(21),
3033 PMU_PMEVTYPER(22),
3034 PMU_PMEVTYPER(23),
3035 PMU_PMEVTYPER(24),
3036 PMU_PMEVTYPER(25),
3037 PMU_PMEVTYPER(26),
3038 PMU_PMEVTYPER(27),
3039 PMU_PMEVTYPER(28),
3040 PMU_PMEVTYPER(29),
3041 PMU_PMEVTYPER(30),
3042 /* PMCCFILTR */
a9e192cd 3043 { CP15_PMU_SYS_REG(DIRECT, 0, 14, 15, 7), .access = access_pmu_evtyper },
f7f2b15c
AB		 3044
3045 { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
3046 { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
bf48040c
AO		 3047
3048 /* CCSIDR2 */
3049 { Op1(1), CRn( 0), CRm( 0), Op2(2), undef_access },
3050
b1ea1d76 3051 { Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, CSSELR_EL1 },
a9866ba0
MZ		 3052};
3053
3054static const struct sys_reg_desc cp15_64_regs[] = {
b1ea1d76 3055 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
a9e192cd 3056 { CP15_PMU_SYS_REG(DIRECT, 0, 0, 9, 0), .access = access_pmu_evcntr },
03bd646d 3057 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */
c605ee24 3058 { SYS_DESC(SYS_AARCH32_CNTPCT), access_arch_timer },
b1ea1d76 3059 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 },
03bd646d
MZ		 3060 { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */
3061 { Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */
84135d3d 3062 { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer },
a6610435 3063 { SYS_DESC(SYS_AARCH32_CNTPCTSS), access_arch_timer },
7c8c5e6a
MZ		 3064};
3065
f1f0c0cf
AE		 3066static bool check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
3067 bool is_32)
bb44a8db
MZ		 3068{
3069 unsigned int i;
3070
3071 for (i = 0; i < n; i++) {
3072 if (!is_32 && table[i].reg && !table[i].reset) {
325031d4 3073 kvm_err("sys_reg table %pS entry %d lacks reset\n", &table[i], i);
f1f0c0cf 3074 return false;
bb44a8db
MZ		 3075 }
3076
3077 if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
325031d4 3078 kvm_err("sys_reg table %pS entry %d out of order\n", &table[i - 1], i - 1);
f1f0c0cf 3079 return false;
bb44a8db
MZ		 3080 }
3081 }
3082
f1f0c0cf 3083 return true;
bb44a8db
MZ		 3084}
3085
74cc7e0c 3086int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu)
62a89c44
MZ		 3087{
3088 kvm_inject_undefined(vcpu);
3089 return 1;
3090}
3091
e70b9522
MZ		 3092static void perform_access(struct kvm_vcpu *vcpu,
3093 struct sys_reg_params *params,
3094 const struct sys_reg_desc *r)
3095{
599d79dc
MZ		 3096 trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
3097
7f34e409 3098 /* Check for regs disabled by runtime config */
01fe5ace 3099 if (sysreg_hidden(vcpu, r)) {
7f34e409
DM		 3100 kvm_inject_undefined(vcpu);
3101 return;
3102 }
3103
e70b9522
MZ		 3104 /*
3105 * Not having an accessor means that we have configured a trap
3106 * that we don't know how to handle. This certainly qualifies
3107 * as a gross bug that should be fixed right away.
3108 */
3109 BUG_ON(!r->access);
3110
3111 /* Skip instruction if instructed so */
3112 if (likely(r->access(vcpu, params, r)))
cdb5e02e 3113 kvm_incr_pc(vcpu);
e70b9522
MZ		 3114}
3115
72564016
MZ		 3116/*
3117 * emulate_cp -- tries to match a sys_reg access in a handling table, and
3118 * call the corresponding trap handler.
3119 *
3120 * @params: pointer to the descriptor of the access
3121 * @table: array of trap descriptors
3122 * @num: size of the trap descriptor array
3123 *
001bb819 3124 * Return true if the access has been handled, false if not.
72564016 3125 */
001bb819
OU		 3126static bool emulate_cp(struct kvm_vcpu *vcpu,
3127 struct sys_reg_params *params,
3128 const struct sys_reg_desc *table,
3129 size_t num)
62a89c44 3130{
72564016 3131 const struct sys_reg_desc *r;
62a89c44 3132
72564016 3133 if (!table)
001bb819 3134 return false; /* Not handled */
62a89c44 3135
62a89c44 3136 r = find_reg(params, table, num);
62a89c44 3137
72564016 3138 if (r) {
e70b9522 3139 perform_access(vcpu, params, r);
001bb819 3140 return true;
72564016
MZ		 3141 }
3142
3143 /* Not handled */
001bb819 3144 return false;
72564016
MZ		 3145}
3146
3147static void unhandled_cp_access(struct kvm_vcpu *vcpu,
3148 struct sys_reg_params *params)
3149{
3a949f4c 3150 u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
40c4f8d2 3151 int cp = -1;
72564016 3152
3a949f4c 3153 switch (esr_ec) {
c6d01a94
MR		 3154 case ESR_ELx_EC_CP15_32:
3155 case ESR_ELx_EC_CP15_64:
72564016
MZ		 3156 cp = 15;
3157 break;
c6d01a94
MR		 3158 case ESR_ELx_EC_CP14_MR:
3159 case ESR_ELx_EC_CP14_64:
72564016
MZ		 3160 cp = 14;
3161 break;
3162 default:
40c4f8d2 3163 WARN_ON(1);
62a89c44
MZ		 3164 }
3165
bf4b96bb
MR		 3166 print_sys_reg_msg(params,
3167 "Unsupported guest CP%d access at: %08lx [%08lx]\n",
3168 cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
62a89c44
MZ		 3169 kvm_inject_undefined(vcpu);
3170}
3171
3172/**
7769db90 3173 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
62a89c44 3174 * @vcpu: The VCPU pointer
8ce78392
RD		 3175 * @global: &struct sys_reg_desc
3176 * @nr_global: size of the @global array
62a89c44 3177 */
72564016
MZ		 3178static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
3179 const struct sys_reg_desc *global,
dcaffa7b 3180 size_t nr_global)
62a89c44
MZ		 3181{
3182 struct sys_reg_params params;
0b12620f 3183 u64 esr = kvm_vcpu_get_esr(vcpu);
c667186f 3184 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3a949f4c 3185 int Rt2 = (esr >> 10) & 0x1f;
62a89c44 3186
3a949f4c
GS		 3187 params.CRm = (esr >> 1) & 0xf;
3188 params.is_write = ((esr & 1) == 0);
62a89c44
MZ		 3189
3190 params.Op0 = 0;
3a949f4c 3191 params.Op1 = (esr >> 16) & 0xf;
62a89c44
MZ		 3192 params.Op2 = 0;
3193 params.CRn = 0;
3194
3195 /*
2ec5be3d 3196 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
62a89c44
MZ		 3197 * backends between AArch32 and AArch64, we get away with it.
3198 */
3199 if (params.is_write) {
2ec5be3d
PF		 3200 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
3201 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
62a89c44
MZ		 3202 }
3203
b6b7a806 3204 /*
dcaffa7b 3205 * If the table contains a handler, handle the
b6b7a806
MZ		 3206 * potential register operation in the case of a read and return
3207 * with success.
3208 */
001bb819 3209 if (emulate_cp(vcpu, &params, global, nr_global)) {
b6b7a806
MZ		 3210 /* Split up the value between registers for the read side */
3211 if (!params.is_write) {
3212 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
3213 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
3214 }
62a89c44 3215
b6b7a806 3216 return 1;
62a89c44
MZ		 3217 }
3218
b6b7a806 3219 unhandled_cp_access(vcpu, &params);
62a89c44
MZ		 3220 return 1;
3221}
3222
e6519766
OU		 3223static bool emulate_sys_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *params);
3224
9369bc5c
OU		 3225/*
3226 * The CP10 ID registers are architecturally mapped to AArch64 feature
3227 * registers. Abuse that fact so we can rely on the AArch64 handler for accesses
3228 * from AArch32.
3229 */
ee87a9bd 3230static bool kvm_esr_cp10_id_to_sys64(u64 esr, struct sys_reg_params *params)
9369bc5c
OU		 3231{
3232 u8 reg_id = (esr >> 10) & 0xf;
3233 bool valid;
3234
3235 params->is_write = ((esr & 1) == 0);
3236 params->Op0 = 3;
3237 params->Op1 = 0;
3238 params->CRn = 0;
3239 params->CRm = 3;
3240
3241 /* CP10 ID registers are read-only */
3242 valid = !params->is_write;
3243
3244 switch (reg_id) {
3245 /* MVFR0 */
3246 case 0b0111:
3247 params->Op2 = 0;
3248 break;
3249 /* MVFR1 */
3250 case 0b0110:
3251 params->Op2 = 1;
3252 break;
3253 /* MVFR2 */
3254 case 0b0101:
3255 params->Op2 = 2;
3256 break;
3257 default:
3258 valid = false;
3259 }
3260
3261 if (valid)
3262 return true;
3263
3264 kvm_pr_unimpl("Unhandled cp10 register %s: %u\n",
3265 params->is_write ? "write" : "read", reg_id);
3266 return false;
3267}
3268
3269/**
3270 * kvm_handle_cp10_id() - Handles a VMRS trap on guest access to a 'Media and
3271 * VFP Register' from AArch32.
3272 * @vcpu: The vCPU pointer
3273 *
3274 * MVFR{0-2} are architecturally mapped to the AArch64 MVFR{0-2}_EL1 registers.
3275 * Work out the correct AArch64 system register encoding and reroute to the
3276 * AArch64 system register emulation.
3277 */
3278int kvm_handle_cp10_id(struct kvm_vcpu *vcpu)
3279{
3280 int Rt = kvm_vcpu_sys_get_rt(vcpu);
ee87a9bd 3281 u64 esr = kvm_vcpu_get_esr(vcpu);
9369bc5c
OU		 3282 struct sys_reg_params params;
3283
3284 /* UNDEF on any unhandled register access */
3285 if (!kvm_esr_cp10_id_to_sys64(esr, &params)) {
3286 kvm_inject_undefined(vcpu);
3287 return 1;
3288 }
3289
3290 if (emulate_sys_reg(vcpu, &params))
3291 vcpu_set_reg(vcpu, Rt, params.regval);
3292
3293 return 1;
3294}
3295
e6519766
OU		 3296/**
3297 * kvm_emulate_cp15_id_reg() - Handles an MRC trap on a guest CP15 access where
3298 * CRn=0, which corresponds to the AArch32 feature
3299 * registers.
3300 * @vcpu: the vCPU pointer
3301 * @params: the system register access parameters.
3302 *
3303 * Our cp15 system register tables do not enumerate the AArch32 feature
3304 * registers. Conveniently, our AArch64 table does, and the AArch32 system
3305 * register encoding can be trivially remapped into the AArch64 for the feature
3306 * registers: Append op0=3, leaving op1, CRn, CRm, and op2 the same.
3307 *
3308 * According to DDI0487G.b G7.3.1, paragraph "Behavior of VMSAv8-32 32-bit
3309 * System registers with (coproc=0b1111, CRn==c0)", read accesses from this
3310 * range are either UNKNOWN or RES0. Rerouting remains architectural as we
3311 * treat undefined registers in this range as RAZ.
3312 */
3313static int kvm_emulate_cp15_id_reg(struct kvm_vcpu *vcpu,
3314 struct sys_reg_params *params)
3315{
3316 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3317
3318 /* Treat impossible writes to RO registers as UNDEFINED */
3319 if (params->is_write) {
3320 unhandled_cp_access(vcpu, params);
3321 return 1;
3322 }
3323
3324 params->Op0 = 3;
3325
3326 /*
3327 * All registers where CRm > 3 are known to be UNKNOWN/RAZ from AArch32.
3328 * Avoid conflicting with future expansion of AArch64 feature registers
3329 * and simply treat them as RAZ here.
3330 */
3331 if (params->CRm > 3)
3332 params->regval = 0;
3333 else if (!emulate_sys_reg(vcpu, params))
3334 return 1;
3335
3336 vcpu_set_reg(vcpu, Rt, params->regval);
3337 return 1;
3338}
3339
62a89c44 3340/**
7769db90 3341 * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
62a89c44 3342 * @vcpu: The VCPU pointer
8ce78392
RD		 3343 * @params: &struct sys_reg_params
3344 * @global: &struct sys_reg_desc
3345 * @nr_global: size of the @global array
62a89c44 3346 */
72564016 3347static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
e6519766 3348 struct sys_reg_params *params,
72564016 3349 const struct sys_reg_desc *global,
dcaffa7b 3350 size_t nr_global)
62a89c44 3351{
c667186f 3352 int Rt = kvm_vcpu_sys_get_rt(vcpu);
62a89c44 3353
e6519766 3354 params->regval = vcpu_get_reg(vcpu, Rt);
62a89c44 3355
e6519766
OU		 3356 if (emulate_cp(vcpu, params, global, nr_global)) {
3357 if (!params->is_write)
3358 vcpu_set_reg(vcpu, Rt, params->regval);
72564016 3359 return 1;
2ec5be3d 3360 }
72564016 3361
e6519766 3362 unhandled_cp_access(vcpu, params);
62a89c44
MZ		 3363 return 1;
3364}
3365
74cc7e0c 3366int kvm_handle_cp15_64(struct kvm_vcpu *vcpu)
72564016 3367{
dcaffa7b 3368 return kvm_handle_cp_64(vcpu, cp15_64_regs, ARRAY_SIZE(cp15_64_regs));
72564016
MZ		 3369}
3370
74cc7e0c 3371int kvm_handle_cp15_32(struct kvm_vcpu *vcpu)
72564016 3372{
e6519766
OU		 3373 struct sys_reg_params params;
3374
3375 params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu));
3376
3377 /*
3378 * Certain AArch32 ID registers are handled by rerouting to the AArch64
3379 * system register table. Registers in the ID range where CRm=0 are
3380 * excluded from this scheme as they do not trivially map into AArch64
3381 * system register encodings.
3382 */
3383 if (params.Op1 == 0 && params.CRn == 0 && params.CRm)
3384 return kvm_emulate_cp15_id_reg(vcpu, &params);
3385
3386 return kvm_handle_cp_32(vcpu, &params, cp15_regs, ARRAY_SIZE(cp15_regs));
72564016
MZ		 3387}
3388
74cc7e0c 3389int kvm_handle_cp14_64(struct kvm_vcpu *vcpu)
72564016 3390{
dcaffa7b 3391 return kvm_handle_cp_64(vcpu, cp14_64_regs, ARRAY_SIZE(cp14_64_regs));
72564016
MZ		 3392}
3393
74cc7e0c 3394int kvm_handle_cp14_32(struct kvm_vcpu *vcpu)
72564016 3395{
e6519766
OU		 3396 struct sys_reg_params params;
3397
3398 params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu));
3399
3400 return kvm_handle_cp_32(vcpu, &params, cp14_regs, ARRAY_SIZE(cp14_regs));
72564016
MZ		 3401}
3402
28eda7b5
OU		 3403/**
3404 * emulate_sys_reg - Emulate a guest access to an AArch64 system register
3405 * @vcpu: The VCPU pointer
3406 * @params: Decoded system register parameters
3407 *
3408 * Return: true if the system register access was successful, false otherwise.
3409 */
3410static bool emulate_sys_reg(struct kvm_vcpu *vcpu,
cc5f84fb 3411 struct sys_reg_params *params)
7c8c5e6a 3412{
dcaffa7b 3413 const struct sys_reg_desc *r;
7c8c5e6a 3414
dcaffa7b 3415 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
7c8c5e6a 3416 if (likely(r)) {
e70b9522 3417 perform_access(vcpu, params, r);
28eda7b5
OU		 3418 return true;
3419 }
3420
cc5f84fb
MZ		 3421 print_sys_reg_msg(params,
3422 "Unsupported guest sys_reg access at: %lx [%08lx]\n",
3423 *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
3424 kvm_inject_undefined(vcpu);
89bc63fa 3425
cc5f84fb 3426 return false;
89bc63fa
MZ		 3427}
3428
89176658
MZ		 3429static void *idregs_debug_start(struct seq_file *s, loff_t *pos)
3430{
3431 struct kvm *kvm = s->private;
3432 u8 *iter;
3433
3434 mutex_lock(&kvm->arch.config_lock);
3435
3436 iter = &kvm->arch.idreg_debugfs_iter;
29ef55ce
OU		 3437 if (test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags) &&
3438 *iter == (u8)~0) {
89176658
MZ		 3439 *iter = *pos;
3440 if (*iter >= KVM_ARM_ID_REG_NUM)
3441 iter = NULL;
3442 } else {
3443 iter = ERR_PTR(-EBUSY);
3444 }
3445
3446 mutex_unlock(&kvm->arch.config_lock);
3447
3448 return iter;
3449}
3450
3451static void *idregs_debug_next(struct seq_file *s, void *v, loff_t *pos)
3452{
3453 struct kvm *kvm = s->private;
3454
3455 (*pos)++;
3456
3457 if ((kvm->arch.idreg_debugfs_iter + 1) < KVM_ARM_ID_REG_NUM) {
3458 kvm->arch.idreg_debugfs_iter++;
3459
3460 return &kvm->arch.idreg_debugfs_iter;
3461 }
3462
3463 return NULL;
3464}
3465
3466static void idregs_debug_stop(struct seq_file *s, void *v)
3467{
3468 struct kvm *kvm = s->private;
3469
3470 if (IS_ERR(v))
3471 return;
3472
3473 mutex_lock(&kvm->arch.config_lock);
3474
3475 kvm->arch.idreg_debugfs_iter = ~0;
3476
3477 mutex_unlock(&kvm->arch.config_lock);
3478}
3479
3480static int idregs_debug_show(struct seq_file *s, void *v)
3481{
3482 struct kvm *kvm = s->private;
3483 const struct sys_reg_desc *desc;
3484
3485 desc = first_idreg + kvm->arch.idreg_debugfs_iter;
3486
3487 if (!desc->name)
3488 return 0;
3489
3490 seq_printf(s, "%20s:\t%016llx\n",
3491 desc->name, IDREG(kvm, IDX_IDREG(kvm->arch.idreg_debugfs_iter)));
3492
3493 return 0;
3494}
3495
3496static const struct seq_operations idregs_debug_sops = {
3497 .start = idregs_debug_start,
3498 .next = idregs_debug_next,
3499 .stop = idregs_debug_stop,
3500 .show = idregs_debug_show,
3501};
3502
3503DEFINE_SEQ_ATTRIBUTE(idregs_debug);
3504
5c1ebe9a
OU		 3505void kvm_sys_regs_create_debugfs(struct kvm *kvm)
3506{
3507 kvm->arch.idreg_debugfs_iter = ~0;
3508
3509 debugfs_create_file("idregs", 0444, kvm->debugfs_dentry, kvm,
3510 &idregs_debug_fops);
3511}
3512
47334146
JZ		 3513static void kvm_reset_id_regs(struct kvm_vcpu *vcpu)
3514{
3515 const struct sys_reg_desc *idreg = first_idreg;
3516 u32 id = reg_to_encoding(idreg);
3517 struct kvm *kvm = vcpu->kvm;
3518
3519 if (test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags))
3520 return;
3521
3522 lockdep_assert_held(&kvm->arch.config_lock);
3523
3524 /* Initialize all idregs */
3525 while (is_id_reg(id)) {
3526 IDREG(kvm, id) = idreg->reset(vcpu, idreg);
3527
3528 idreg++;
3529 id = reg_to_encoding(idreg);
3530 }
3531
3532 set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags);
3533}
3534
750ed566
JM		 3535/**
3536 * kvm_reset_sys_regs - sets system registers to reset value
3537 * @vcpu: The VCPU pointer
3538 *
3539 * This function finds the right table above and sets the registers on the
3540 * virtual CPU struct to their architecturally defined reset values.
3541 */
3542void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
7c8c5e6a
MZ		 3543{
3544 unsigned long i;
3545
47334146
JZ		 3546 kvm_reset_id_regs(vcpu);
3547
3548 for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) {
3549 const struct sys_reg_desc *r = &sys_reg_descs[i];
3550
3551 if (is_id_reg(reg_to_encoding(r)))
3552 continue;
3553
3554 if (r->reset)
3555 r->reset(vcpu, r);
3556 }
7c8c5e6a
MZ		 3557}
3558
3559/**
89bc63fa
MZ		 3560 * kvm_handle_sys_reg -- handles a system instruction or mrs/msr instruction
3561 * trap on a guest execution
7c8c5e6a 3562 * @vcpu: The VCPU pointer
7c8c5e6a 3563 */
74cc7e0c 3564int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
7c8c5e6a 3565{
cc5f84fb 3566 const struct sys_reg_desc *desc = NULL;
7c8c5e6a 3567 struct sys_reg_params params;
3a949f4c 3568 unsigned long esr = kvm_vcpu_get_esr(vcpu);
c667186f 3569 int Rt = kvm_vcpu_sys_get_rt(vcpu);
cc5f84fb 3570 int sr_idx;
7c8c5e6a 3571
eef8c85a
AB		 3572 trace_kvm_handle_sys_reg(esr);
3573
085eabaa 3574 if (triage_sysreg_trap(vcpu, &sr_idx))
e58ec47b
MZ		 3575 return 1;
3576
f76f89e2 3577 params = esr_sys64_to_params(esr);
2ec5be3d 3578 params.regval = vcpu_get_reg(vcpu, Rt);
7c8c5e6a 3579
89bc63fa 3580 /* System registers have Op0=={2,3}, as per DDI487 J.a C5.1.2 */
cc5f84fb
MZ		 3581 if (params.Op0 == 2 || params.Op0 == 3)
3582 desc = &sys_reg_descs[sr_idx];
3583 else
3584 desc = &sys_insn_descs[sr_idx];
89bc63fa 3585
cc5f84fb 3586 perform_access(vcpu, &params, desc);
89bc63fa 3587
cc5f84fb
MZ		 3588 /* Read from system register? */
3589 if (!params.is_write &&
3590 (params.Op0 == 2 || params.Op0 == 3))
3591 vcpu_set_reg(vcpu, Rt, params.regval);
2ec5be3d 3592
cc5f84fb 3593 return 1;
7c8c5e6a
MZ		 3594}
3595
3596/******************************************************************************
3597 * Userspace API
3598 *****************************************************************************/
3599
3600static bool index_to_params(u64 id, struct sys_reg_params *params)
3601{
3602 switch (id & KVM_REG_SIZE_MASK) {
3603 case KVM_REG_SIZE_U64:
3604 /* Any unused index bits means it's not valid. */
3605 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
3606 | KVM_REG_ARM_COPROC_MASK
3607 | KVM_REG_ARM64_SYSREG_OP0_MASK
3608 | KVM_REG_ARM64_SYSREG_OP1_MASK
3609 | KVM_REG_ARM64_SYSREG_CRN_MASK
3610 | KVM_REG_ARM64_SYSREG_CRM_MASK
3611 | KVM_REG_ARM64_SYSREG_OP2_MASK))
3612 return false;
3613 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
3614 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
3615 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
3616 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
3617 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
3618 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
3619 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
3620 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
3621 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
3622 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
3623 return true;
3624 default:
3625 return false;
3626 }
3627}
3628
da8d120f
MZ		 3629const struct sys_reg_desc *get_reg_by_id(u64 id,
3630 const struct sys_reg_desc table[],
3631 unsigned int num)
4b927b94 3632{
da8d120f
MZ		 3633 struct sys_reg_params params;
3634
3635 if (!index_to_params(id, &params))
4b927b94
VK		 3636 return NULL;
3637
da8d120f 3638 return find_reg(&params, table, num);
4b927b94
VK		 3639}
3640
7c8c5e6a 3641/* Decode an index value, and find the sys_reg_desc entry. */
ba23aec9
MZ		 3642static const struct sys_reg_desc *
3643id_to_sys_reg_desc(struct kvm_vcpu *vcpu, u64 id,
3644 const struct sys_reg_desc table[], unsigned int num)
3645
7c8c5e6a 3646{
dcaffa7b 3647 const struct sys_reg_desc *r;
7c8c5e6a
MZ		 3648
3649 /* We only do sys_reg for now. */
3650 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
3651 return NULL;
3652
ba23aec9 3653 r = get_reg_by_id(id, table, num);
7c8c5e6a 3654
93390c0a 3655 /* Not saved in the sys_reg array and not otherwise accessible? */
ba23aec9 3656 if (r && (!(r->reg || r->get_user) || sysreg_hidden(vcpu, r)))
7c8c5e6a
MZ		 3657 r = NULL;
3658
3659 return r;
3660}
3661
3662/*
3663 * These are the invariant sys_reg registers: we let the guest see the
3664 * host versions of these, so they're part of the guest state.
3665 *
3666 * A future CPU may provide a mechanism to present different values to
3667 * the guest, or a future kvm may trap them.
3668 */
3669
3670#define FUNCTION_INVARIANT(reg) \
d86cde6e 3671 static u64 get_##reg(struct kvm_vcpu *v, \
7c8c5e6a
MZ		 3672 const struct sys_reg_desc *r) \
3673 { \
1f3d8699 3674 ((struct sys_reg_desc *)r)->val = read_sysreg(reg); \
d86cde6e 3675 return ((struct sys_reg_desc *)r)->val; \
7c8c5e6a
MZ		 3676 }
3677
3678FUNCTION_INVARIANT(midr_el1)
7c8c5e6a 3679FUNCTION_INVARIANT(revidr_el1)
7c8c5e6a
MZ		 3680FUNCTION_INVARIANT(aidr_el1)
3681
d86cde6e 3682static u64 get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
f7f2b15c
AB		 3683{
3684 ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
d86cde6e 3685 return ((struct sys_reg_desc *)r)->val;
f7f2b15c
AB		 3686}
3687
7c8c5e6a 3688/* ->val is filled in by kvm_sys_reg_table_init() */
8d20bd63 3689static struct sys_reg_desc invariant_sys_regs[] __ro_after_init = {
0d449541
MR		 3690 { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
3691 { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
0d449541
MR		 3692 { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
3693 { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
7c8c5e6a
MZ		 3694};
3695
5a420ed9 3696static int get_invariant_sys_reg(u64 id, u64 __user *uaddr)
7c8c5e6a 3697{
7c8c5e6a
MZ		 3698 const struct sys_reg_desc *r;
3699
da8d120f
MZ		 3700 r = get_reg_by_id(id, invariant_sys_regs,
3701 ARRAY_SIZE(invariant_sys_regs));
7c8c5e6a
MZ		 3702 if (!r)
3703 return -ENOENT;
3704
5a420ed9 3705 return put_user(r->val, uaddr);
7c8c5e6a
MZ		 3706}
3707
5a420ed9 3708static int set_invariant_sys_reg(u64 id, u64 __user *uaddr)
7c8c5e6a 3709{
7c8c5e6a 3710 const struct sys_reg_desc *r;
5a420ed9 3711 u64 val;
7c8c5e6a 3712
da8d120f
MZ		 3713 r = get_reg_by_id(id, invariant_sys_regs,
3714 ARRAY_SIZE(invariant_sys_regs));
7c8c5e6a
MZ		 3715 if (!r)
3716 return -ENOENT;
3717
5a420ed9
MZ		 3718 if (get_user(val, uaddr))
3719 return -EFAULT;
7c8c5e6a
MZ		 3720
3721 /* This is what we mean by invariant: you can't change it. */
3722 if (r->val != val)
3723 return -EINVAL;
3724
3725 return 0;
3726}
3727
7af0c253 3728static int demux_c15_get(struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
7c8c5e6a
MZ		 3729{
3730 u32 val;
3731 u32 __user *uval = uaddr;
3732
3733 /* Fail if we have unknown bits set. */
3734 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
3735 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
3736 return -ENOENT;
3737
3738 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
3739 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
3740 if (KVM_REG_SIZE(id) != 4)
3741 return -ENOENT;
3742 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
3743 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
7af0c253 3744 if (val >= CSSELR_MAX)
7c8c5e6a
MZ		 3745 return -ENOENT;
3746
7af0c253 3747 return put_user(get_ccsidr(vcpu, val), uval);
7c8c5e6a
MZ		 3748 default:
3749 return -ENOENT;
3750 }
3751}
3752
7af0c253 3753static int demux_c15_set(struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
7c8c5e6a
MZ		 3754{
3755 u32 val, newval;
3756 u32 __user *uval = uaddr;
3757
3758 /* Fail if we have unknown bits set. */
3759 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
3760 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
3761 return -ENOENT;
3762
3763 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
3764 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
3765 if (KVM_REG_SIZE(id) != 4)
3766 return -ENOENT;
3767 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
3768 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
7af0c253 3769 if (val >= CSSELR_MAX)
7c8c5e6a
MZ		 3770 return -ENOENT;
3771
3772 if (get_user(newval, uval))
3773 return -EFAULT;
3774
7af0c253 3775 return set_ccsidr(vcpu, val, newval);
7c8c5e6a
MZ		 3776 default:
3777 return -ENOENT;
3778 }
3779}
3780
ba23aec9
MZ		 3781int kvm_sys_reg_get_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
3782 const struct sys_reg_desc table[], unsigned int num)
7c8c5e6a 3783{
978ceeb3 3784 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
7c8c5e6a 3785 const struct sys_reg_desc *r;
978ceeb3
MZ		 3786 u64 val;
3787 int ret;
ba23aec9
MZ		 3788
3789 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
e6b367db 3790 if (!r || sysreg_hidden_user(vcpu, r))
ba23aec9
MZ		 3791 return -ENOENT;
3792
978ceeb3
MZ		 3793 if (r->get_user) {
3794 ret = (r->get_user)(vcpu, r, &val);
3795 } else {
3796 val = __vcpu_sys_reg(vcpu, r->reg);
3797 ret = 0;
3798 }
3799
3800 if (!ret)
3801 ret = put_user(val, uaddr);
ba23aec9 3802
978ceeb3 3803 return ret;
ba23aec9
MZ		 3804}
3805
3806int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
3807{
7c8c5e6a 3808 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
1deeffb5 3809 int err;
7c8c5e6a
MZ		 3810
3811 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
7af0c253 3812 return demux_c15_get(vcpu, reg->id, uaddr);
7c8c5e6a 3813
1deeffb5
MZ		 3814 err = get_invariant_sys_reg(reg->id, uaddr);
3815 if (err != -ENOENT)
3816 return err;
7c8c5e6a 3817
ba23aec9
MZ		 3818 return kvm_sys_reg_get_user(vcpu, reg,
3819 sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3820}
7c8c5e6a 3821
ba23aec9
MZ		 3822int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
3823 const struct sys_reg_desc table[], unsigned int num)
3824{
978ceeb3 3825 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
ba23aec9 3826 const struct sys_reg_desc *r;
978ceeb3
MZ		 3827 u64 val;
3828 int ret;
3829
3830 if (get_user(val, uaddr))
3831 return -EFAULT;
ba23aec9
MZ		 3832
3833 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
e6b367db 3834 if (!r || sysreg_hidden_user(vcpu, r))
7f34e409
DM		 3835 return -ENOENT;
3836
4de06e4c
OU		 3837 if (sysreg_user_write_ignore(vcpu, r))
3838 return 0;
3839
978ceeb3
MZ		 3840 if (r->set_user) {
3841 ret = (r->set_user)(vcpu, r, val);
3842 } else {
3843 __vcpu_sys_reg(vcpu, r->reg) = val;
3844 ret = 0;
3845 }
84e690bf 3846
978ceeb3 3847 return ret;
7c8c5e6a
MZ		 3848}
3849
3850int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
3851{
7c8c5e6a 3852 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
1deeffb5 3853 int err;
7c8c5e6a
MZ		 3854
3855 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
7af0c253 3856 return demux_c15_set(vcpu, reg->id, uaddr);
7c8c5e6a 3857
1deeffb5
MZ		 3858 err = set_invariant_sys_reg(reg->id, uaddr);
3859 if (err != -ENOENT)
3860 return err;
84e690bf 3861
ba23aec9
MZ		 3862 return kvm_sys_reg_set_user(vcpu, reg,
3863 sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
7c8c5e6a
MZ		 3864}
3865
3866static unsigned int num_demux_regs(void)
3867{
7af0c253 3868 return CSSELR_MAX;
7c8c5e6a
MZ		 3869}
3870
3871static int write_demux_regids(u64 __user *uindices)
3872{
efd48cea 3873 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
7c8c5e6a
MZ		 3874 unsigned int i;
3875
3876 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
3877 for (i = 0; i < CSSELR_MAX; i++) {
7c8c5e6a
MZ		 3878 if (put_user(val | i, uindices))
3879 return -EFAULT;
3880 uindices++;
3881 }
3882 return 0;
3883}
3884
3885static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
3886{
3887 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
3888 KVM_REG_ARM64_SYSREG |
3889 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
3890 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
3891 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
3892 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
3893 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
3894}
3895
3896static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
3897{
3898 if (!*uind)
3899 return true;
3900
3901 if (put_user(sys_reg_to_index(reg), *uind))
3902 return false;
3903
3904 (*uind)++;
3905 return true;
3906}
3907
7f34e409
DM		 3908static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
3909 const struct sys_reg_desc *rd,
93390c0a
DM		 3910 u64 __user **uind,
3911 unsigned int *total)
3912{
3913 /*
3914 * Ignore registers we trap but don't save,
3915 * and for which no custom user accessor is provided.
3916 */
3917 if (!(rd->reg || rd->get_user))
3918 return 0;
3919
e6b367db 3920 if (sysreg_hidden_user(vcpu, rd))
7f34e409
DM		 3921 return 0;
3922
93390c0a
DM		 3923 if (!copy_reg_to_user(rd, uind))
3924 return -EFAULT;
3925
3926 (*total)++;
3927 return 0;
3928}
3929
7c8c5e6a
MZ		 3930/* Assumed ordered tables, see kvm_sys_reg_table_init. */
3931static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
3932{
dcaffa7b 3933 const struct sys_reg_desc *i2, *end2;
7c8c5e6a 3934 unsigned int total = 0;
93390c0a 3935 int err;
7c8c5e6a 3936
7c8c5e6a
MZ		 3937 i2 = sys_reg_descs;
3938 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
3939
dcaffa7b
JM		 3940 while (i2 != end2) {
3941 err = walk_one_sys_reg(vcpu, i2++, &uind, &total);
93390c0a
DM		 3942 if (err)
3943 return err;
7c8c5e6a
MZ		 3944 }
3945 return total;
3946}
3947
3948unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
3949{
3950 return ARRAY_SIZE(invariant_sys_regs)
3951 + num_demux_regs()
3952 + walk_sys_regs(vcpu, (u64 __user *)NULL);
3953}
3954
3955int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
3956{
3957 unsigned int i;
3958 int err;
3959
3960 /* Then give them all the invariant registers' indices. */
3961 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
3962 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
3963 return -EFAULT;
3964 uindices++;
3965 }
3966
3967 err = walk_sys_regs(vcpu, uindices);
3968 if (err < 0)
3969 return err;
3970 uindices += err;
3971
3972 return write_demux_regids(uindices);
3973}
3974
3f9cd0ca
JZ		 3975#define KVM_ARM_FEATURE_ID_RANGE_INDEX(r) \
3976 KVM_ARM_FEATURE_ID_RANGE_IDX(sys_reg_Op0(r), \
3977 sys_reg_Op1(r), \
3978 sys_reg_CRn(r), \
3979 sys_reg_CRm(r), \
3980 sys_reg_Op2(r))
3981
3982static bool is_feature_id_reg(u32 encoding)
3983{
3984 return (sys_reg_Op0(encoding) == 3 &&
3985 (sys_reg_Op1(encoding) < 2 || sys_reg_Op1(encoding) == 3) &&
3986 sys_reg_CRn(encoding) == 0 &&
3987 sys_reg_CRm(encoding) <= 7);
3988}
3989
3990int kvm_vm_ioctl_get_reg_writable_masks(struct kvm *kvm, struct reg_mask_range *range)
3991{
3992 const void *zero_page = page_to_virt(ZERO_PAGE(0));
3993 u64 __user *masks = (u64 __user *)range->addr;
3994
3995 /* Only feature id range is supported, reserved[13] must be zero. */
3996 if (range->range ||
3997 memcmp(range->reserved, zero_page, sizeof(range->reserved)))
3998 return -EINVAL;
3999
4000 /* Wipe the whole thing first */
4001 if (clear_user(masks, KVM_ARM_FEATURE_ID_RANGE_SIZE * sizeof(__u64)))
4002 return -EFAULT;
4003
4004 for (int i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) {
4005 const struct sys_reg_desc *reg = &sys_reg_descs[i];
4006 u32 encoding = reg_to_encoding(reg);
4007 u64 val;
4008
4009 if (!is_feature_id_reg(encoding) || !reg->set_user)
4010 continue;
4011
4012 /*
4013 * For ID registers, we return the writable mask. Other feature
4014 * registers return a full 64bit mask. That's not necessary
4015 * compliant with a given revision of the architecture, but the
4016 * RES0/RES1 definitions allow us to do that.
4017 */
4018 if (is_id_reg(encoding)) {
4019 if (!reg->val ||
4020 (is_aa32_id_reg(encoding) && !kvm_supports_32bit_el0()))
4021 continue;
4022 val = reg->val;
4023 } else {
4024 val = ~0UL;
4025 }
4026
4027 if (put_user(val, (masks + KVM_ARM_FEATURE_ID_RANGE_INDEX(encoding))))
4028 return -EFAULT;
4029 }
4030
4031 return 0;
4032}
4033
c5bac1ef
MZ		 4034void kvm_init_sysreg(struct kvm_vcpu *vcpu)
4035{
4036 struct kvm *kvm = vcpu->kvm;
4037
4038 mutex_lock(&kvm->arch.config_lock);
4039
8ecdccb9
MZ		 4040 /*
4041 * In the absence of FGT, we cannot independently trap TLBI
4042 * Range instructions. This isn't great, but trapping all
4043 * TLBIs would be far worse. Live with it...
4044 */
4045 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
4046 vcpu->arch.hcr_el2 |= HCR_TTLBOS;
4047
84de212d
MZ		 4048 if (cpus_have_final_cap(ARM64_HAS_HCX)) {
4049 vcpu->arch.hcrx_el2 = HCRX_GUEST_FLAGS;
4050
4051 if (kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP))
4052 vcpu->arch.hcrx_el2 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2);
4053 }
4054
c5bac1ef
MZ		 4055 if (test_bit(KVM_ARCH_FLAG_FGU_INITIALIZED, &kvm->arch.flags))
4056 goto out;
4057
4058 kvm->arch.fgu[HFGxTR_GROUP] = (HFGxTR_EL2_nAMAIR2_EL1 |
4059 HFGxTR_EL2_nMAIR2_EL1 |
4060 HFGxTR_EL2_nS2POR_EL1 |
4061 HFGxTR_EL2_nPOR_EL1 |
4062 HFGxTR_EL2_nPOR_EL0 |
4063 HFGxTR_EL2_nACCDATA_EL1 |
4064 HFGxTR_EL2_nSMPRI_EL1_MASK |
4065 HFGxTR_EL2_nTPIDR2_EL0_MASK);
4066
8ecdccb9
MZ		 4067 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
4068 kvm->arch.fgu[HFGITR_GROUP] |= (HFGITR_EL2_TLBIRVAALE1OS|
4069 HFGITR_EL2_TLBIRVALE1OS |
4070 HFGITR_EL2_TLBIRVAAE1OS |
4071 HFGITR_EL2_TLBIRVAE1OS |
4072 HFGITR_EL2_TLBIVAALE1OS |
4073 HFGITR_EL2_TLBIVALE1OS |
4074 HFGITR_EL2_TLBIVAAE1OS |
4075 HFGITR_EL2_TLBIASIDE1OS |
4076 HFGITR_EL2_TLBIVAE1OS |
4077 HFGITR_EL2_TLBIVMALLE1OS);
4078
4079 if (!kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
4080 kvm->arch.fgu[HFGITR_GROUP] |= (HFGITR_EL2_TLBIRVAALE1 |
4081 HFGITR_EL2_TLBIRVALE1 |
4082 HFGITR_EL2_TLBIRVAAE1 |
4083 HFGITR_EL2_TLBIRVAE1 |
4084 HFGITR_EL2_TLBIRVAALE1IS|
4085 HFGITR_EL2_TLBIRVALE1IS |
4086 HFGITR_EL2_TLBIRVAAE1IS |
4087 HFGITR_EL2_TLBIRVAE1IS |
4088 HFGITR_EL2_TLBIRVAALE1OS|
4089 HFGITR_EL2_TLBIRVALE1OS |
4090 HFGITR_EL2_TLBIRVAAE1OS |
4091 HFGITR_EL2_TLBIRVAE1OS);
4092
58627b72
MZ		 4093 if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, S1PIE, IMP))
4094 kvm->arch.fgu[HFGxTR_GROUP] |= (HFGxTR_EL2_nPIRE0_EL1 |
4095 HFGxTR_EL2_nPIR_EL1);
4096
b03e8bb5
MZ		 4097 if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, AMU, IMP))
4098 kvm->arch.fgu[HAFGRTR_GROUP] |= ~(HAFGRTR_EL2_RES0 |
4099 HAFGRTR_EL2_RES1);
4100
c5bac1ef
MZ		 4101 set_bit(KVM_ARCH_FLAG_FGU_INITIALIZED, &kvm->arch.flags);
4102out:
4103 mutex_unlock(&kvm->arch.config_lock);
4104}
4105
8d20bd63 4106int __init kvm_sys_reg_table_init(void)
7c8c5e6a 4107{
47334146 4108 struct sys_reg_params params;
f1f0c0cf 4109 bool valid = true;
7c8c5e6a 4110 unsigned int i;
19f3e7ea 4111 int ret = 0;
7c8c5e6a 4112
b80b701d
MZ		 4113 check_res_bits();
4114
7c8c5e6a 4115 /* Make sure tables are unique and in order. */
f1f0c0cf
AE		 4116 valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false);
4117 valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true);
4118 valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true);
4119 valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true);
4120 valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true);
4121 valid &= check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false);
89bc63fa 4122 valid &= check_sysreg_table(sys_insn_descs, ARRAY_SIZE(sys_insn_descs), false);
f1f0c0cf
AE		 4123
4124 if (!valid)
4125 return -EINVAL;
7c8c5e6a
MZ		 4126
4127 /* We abuse the reset function to overwrite the table itself. */
4128 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
4129 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
4130
47334146
JZ		 4131 /* Find the first idreg (SYS_ID_PFR0_EL1) in sys_reg_descs. */
4132 params = encoding_to_params(SYS_ID_PFR0_EL1);
4133 first_idreg = find_reg(&params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
4134 if (!first_idreg)
4135 return -EINVAL;
4136
19f3e7ea
MZ		 4137 ret = populate_nv_trap_config();
4138
4139 for (i = 0; !ret && i < ARRAY_SIZE(sys_reg_descs); i++)
4140 ret = populate_sysreg_config(sys_reg_descs + i, i);
4141
4142 for (i = 0; !ret && i < ARRAY_SIZE(sys_insn_descs); i++)
4143 ret = populate_sysreg_config(sys_insn_descs + i, i);
4144
4145 return ret;
7c8c5e6a 4146}
Linux 6.x block layer and io_uring tree(s)