projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: arm64: Convert vcpu sysregs_loaded_on_cpu to a state flag
[linux-2.6-block.git] / arch / arm64 / include / asm / kvm_emulate.h
CommitLineData
caab277b 1/* SPDX-License-Identifier: GPL-2.0-only */
83a49794
MZ		 2/*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 *
6 * Derived from arch/arm/include/kvm_emulate.h
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
83a49794
MZ		 9 */
10
11#ifndef __ARM64_KVM_EMULATE_H__
12#define __ARM64_KVM_EMULATE_H__
13
14#include <linux/kvm_host.h>
c6d01a94 15
bd7d95ca 16#include <asm/debug-monitors.h>
c6d01a94 17#include <asm/esr.h>
83a49794 18#include <asm/kvm_arm.h>
00536ec4 19#include <asm/kvm_hyp.h>
83a49794 20#include <asm/ptrace.h>
4429fc64 21#include <asm/cputype.h>
68908bf7 22#include <asm/virt.h>
83a49794 23
bb666c47
MZ		 24#define CURRENT_EL_SP_EL0_VECTOR 0x0
25#define CURRENT_EL_SP_ELx_VECTOR 0x200
26#define LOWER_EL_AArch64_VECTOR 0x400
27#define LOWER_EL_AArch32_VECTOR 0x600
28
29enum exception_type {
30 except_type_sync = 0,
31 except_type_irq = 0x80,
32 except_type_fiq = 0x100,
33 except_type_serror = 0x180,
34};
b547631f 35
27b190bd 36bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
6ddbc281 37void kvm_skip_instr32(struct kvm_vcpu *vcpu);
27b190bd 38
83a49794 39void kvm_inject_undefined(struct kvm_vcpu *vcpu);
10cf3390 40void kvm_inject_vabt(struct kvm_vcpu *vcpu);
83a49794
MZ		 41void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
42void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
85ea6b1e 43void kvm_inject_size_fault(struct kvm_vcpu *vcpu);
83a49794 44
6109c5a6
SC		 45void kvm_vcpu_wfi(struct kvm_vcpu *vcpu);
46
26bf74bd 47#if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__)
5c37f1ae 48static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
e72341c5
CD		 49{
50 return !(vcpu->arch.hcr_el2 & HCR_RW);
51}
26bf74bd
RW		 52#else
53static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
54{
55 struct kvm *kvm = vcpu->kvm;
56
57 WARN_ON_ONCE(!test_bit(KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED,
58 &kvm->arch.flags));
59
60 return test_bit(KVM_ARCH_FLAG_EL1_32BIT, &kvm->arch.flags);
61}
62#endif
e72341c5 63
b856a591
CD		 64static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
65{
66 vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
68908bf7
MZ		 67 if (is_kernel_in_hyp_mode())
68 vcpu->arch.hcr_el2 |= HCR_E2H;
558daf69
DG		 69 if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) {
70 /* route synchronous external abort exceptions to EL2 */
71 vcpu->arch.hcr_el2 |= HCR_TEA;
72 /* trap error record accesses */
73 vcpu->arch.hcr_el2 |= HCR_TERR;
74 }
5c401308
CD		 75
76 if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) {
e48d53a9 77 vcpu->arch.hcr_el2 |= HCR_FWB;
5c401308
CD		 78 } else {
79 /*
80 * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C
81 * get set in SCTLR_EL1 such that we can detect when the guest
82 * MMU gets turned on and do the necessary cache maintenance
83 * then.
84 */
85 vcpu->arch.hcr_el2 |= HCR_TVM;
86 }
558daf69 87
26bf74bd 88 if (vcpu_el1_is_32bit(vcpu))
801f6772 89 vcpu->arch.hcr_el2 &= ~HCR_RW;
f7f2b15c 90
793acf87
AB		 91 if (cpus_have_const_cap(ARM64_MISMATCHED_CACHE_TYPE) ||
92 vcpu_el1_is_32bit(vcpu))
f7f2b15c 93 vcpu->arch.hcr_el2 |= HCR_TID2;
ea7fc1bb
SP		 94
95 if (kvm_has_mte(vcpu->kvm))
96 vcpu->arch.hcr_el2 |= HCR_ATA;
b856a591
CD		 97}
98
3df59d8d 99static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu)
3c1e7165 100{
3df59d8d 101 return (unsigned long *)&vcpu->arch.hcr_el2;
3c1e7165
MZ		 102}
103
ef2e78dd 104static inline void vcpu_clear_wfx_traps(struct kvm_vcpu *vcpu)
de737089
MZ		 105{
106 vcpu->arch.hcr_el2 &= ~HCR_TWE;
7bdabad1
MZ		 107 if (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
108 vcpu->kvm->arch.vgic.nassgireq)
ef2e78dd
MZ		 109 vcpu->arch.hcr_el2 &= ~HCR_TWI;
110 else
111 vcpu->arch.hcr_el2 |= HCR_TWI;
de737089
MZ		 112}
113
ef2e78dd 114static inline void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu)
de737089
MZ		 115{
116 vcpu->arch.hcr_el2 |= HCR_TWE;
ef2e78dd 117 vcpu->arch.hcr_el2 |= HCR_TWI;
de737089
MZ		 118}
119
384b40ca
MR		 120static inline void vcpu_ptrauth_enable(struct kvm_vcpu *vcpu)
121{
122 vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
123}
124
125static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
126{
127 vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
128}
129
b7b27fac
DG		 130static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
131{
132 return vcpu->arch.vsesr_el2;
133}
134
4715c14b
JM		 135static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr)
136{
137 vcpu->arch.vsesr_el2 = vsesr;
138}
139
5c37f1ae 140static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
83a49794 141{
e47c2055 142 return (unsigned long *)&vcpu_gp_regs(vcpu)->pc;
83a49794
MZ		 143}
144
5c37f1ae 145static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu)
83a49794 146{
e47c2055 147 return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate;
83a49794
MZ		 148}
149
5c37f1ae 150static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
83a49794 151{
b547631f 152 return !!(*vcpu_cpsr(vcpu) & PSR_MODE32_BIT);
83a49794
MZ		 153}
154
5c37f1ae 155static __always_inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu)
83a49794 156{
27b190bd
MZ		 157 if (vcpu_mode_is_32bit(vcpu))
158 return kvm_condition_valid32(vcpu);
159
160 return true;
83a49794
MZ		 161}
162
83a49794
MZ		 163static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
164{
256c0960 165 *vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT;
83a49794
MZ		 166}
167
c0f09634 168/*
f6be563a
PF		 169 * vcpu_get_reg and vcpu_set_reg should always be passed a register number
170 * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on
171 * AArch32 with banked registers.
c0f09634 172 */
5c37f1ae 173static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu,
bc45a516
PF		 174 u8 reg_num)
175{
e47c2055 176 return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num];
bc45a516
PF		 177}
178
5c37f1ae 179static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num,
bc45a516
PF		 180 unsigned long val)
181{
182 if (reg_num != 31)
e47c2055 183 vcpu_gp_regs(vcpu)->regs[reg_num] = val;
bc45a516
PF		 184}
185
1cfbb484
MR		 186/*
187 * The layout of SPSR for an AArch32 state is different when observed from an
188 * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32
189 * view given an AArch64 view.
190 *
191 * In ARM DDI 0487E.a see:
192 *
193 * - The AArch64 view (SPSR_EL2) in section C5.2.18, page C5-426
194 * - The AArch32 view (SPSR_abt) in section G8.2.126, page G8-6256
195 * - The AArch32 view (SPSR_und) in section G8.2.132, page G8-6280
196 *
197 * Which show the following differences:
198 *
199 * | Bit | AA64 | AA32 | Notes |
200 * +-----+------+------+-----------------------------|
201 * | 24 | DIT | J | J is RES0 in ARMv8 |
202 * | 21 | SS | DIT | SS doesn't exist in AArch32 |
203 *
204 * ... and all other bits are (currently) common.
205 */
206static inline unsigned long host_spsr_to_spsr32(unsigned long spsr)
207{
208 const unsigned long overlap = BIT(24) | BIT(21);
209 unsigned long dit = !!(spsr & PSR_AA32_DIT_BIT);
210
211 spsr &= ~overlap;
212
213 spsr |= dit << 21;
214
215 return spsr;
216}
217
83a49794
MZ		 218static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu)
219{
9586a2ea 220 u32 mode;
83a49794 221
9586a2ea 222 if (vcpu_mode_is_32bit(vcpu)) {
256c0960
MR		 223 mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK;
224 return mode > PSR_AA32_MODE_USR;
9586a2ea
SZ		 225 }
226
227 mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK;
b547631f 228
83a49794
MZ		 229 return mode != PSR_MODE_EL0t;
230}
231
0b12620f 232static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu)
83a49794
MZ		 233{
234 return vcpu->arch.fault.esr_el2;
235}
236
5c37f1ae 237static __always_inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
3e51d435 238{
0b12620f 239 u64 esr = kvm_vcpu_get_esr(vcpu);
3e51d435
MZ		 240
241 if (esr & ESR_ELx_CV)
242 return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
243
244 return -1;
245}
246
5c37f1ae 247static __always_inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu)
83a49794
MZ		 248{
249 return vcpu->arch.fault.far_el2;
250}
251
5c37f1ae 252static __always_inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
83a49794
MZ		 253{
254 return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8;
255}
256
0067df41
JM		 257static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu)
258{
259 return vcpu->arch.fault.disr_el1;
260}
261
0d97f884
WH		 262static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu)
263{
3a949f4c 264 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_xVC_IMM_MASK;
0d97f884
WH		 265}
266
5c37f1ae 267static __always_inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
83a49794 268{
3a949f4c 269 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_ISV);
83a49794
MZ		 270}
271
c726200d
CD		 272static inline unsigned long kvm_vcpu_dabt_iss_nisv_sanitized(const struct kvm_vcpu *vcpu)
273{
3a949f4c 274 return kvm_vcpu_get_esr(vcpu) & (ESR_ELx_CM | ESR_ELx_WNR | ESR_ELx_FSC);
c726200d
CD		 275}
276
83a49794
MZ		 277static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
278{
3a949f4c 279 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SSE);
83a49794
MZ		 280}
281
b6ae256a
CD		 282static inline bool kvm_vcpu_dabt_issf(const struct kvm_vcpu *vcpu)
283{
3a949f4c 284 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SF);
b6ae256a
CD		 285}
286
5c37f1ae 287static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
83a49794 288{
3a949f4c 289 return (kvm_vcpu_get_esr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
83a49794
MZ		 290}
291
c4ad98e4 292static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu)
83a49794 293{
3a949f4c 294 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW);
83a49794
MZ		 295}
296
620cf45f 297/* Always check for S1PTW *before* using this. */
5c37f1ae 298static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
60e21a0e 299{
620cf45f 300 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR;
60e21a0e
WD		 301}
302
57c841f1
MZ		 303static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu)
304{
3a949f4c 305 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM);
57c841f1
MZ		 306}
307
5c37f1ae 308static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
83a49794 309{
3a949f4c 310 return 1 << ((kvm_vcpu_get_esr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
83a49794
MZ		 311}
312
313/* This one is not specific to Data Abort */
5c37f1ae 314static __always_inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
83a49794 315{
3a949f4c 316 return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_IL);
83a49794
MZ		 317}
318
5c37f1ae 319static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
83a49794 320{
3a949f4c 321 return ESR_ELx_EC(kvm_vcpu_get_esr(vcpu));
83a49794
MZ		 322}
323
324static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
325{
c6d01a94 326 return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
83a49794
MZ		 327}
328
c4ad98e4
MZ		 329static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu)
330{
331 return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu);
332}
333
5c37f1ae 334static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
0496daa5 335{
3a949f4c 336 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC;
0496daa5
CD		 337}
338
5c37f1ae 339static __always_inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
83a49794 340{
3a949f4c 341 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_TYPE;
83a49794
MZ		 342}
343
7d894834
YW		 344static __always_inline u8 kvm_vcpu_trap_get_fault_level(const struct kvm_vcpu *vcpu)
345{
346 return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC_LEVEL;
347}
348
c9a636f2 349static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu)
bb428921 350{
a2b83133 351 switch (kvm_vcpu_trap_get_fault(vcpu)) {
bb428921
JM		 352 case FSC_SEA:
353 case FSC_SEA_TTW0:
354 case FSC_SEA_TTW1:
355 case FSC_SEA_TTW2:
356 case FSC_SEA_TTW3:
357 case FSC_SECC:
358 case FSC_SECC_TTW0:
359 case FSC_SECC_TTW1:
360 case FSC_SECC_TTW2:
361 case FSC_SECC_TTW3:
362 return true;
363 default:
364 return false;
365 }
366}
367
5c37f1ae 368static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
c667186f 369{
0b12620f 370 u64 esr = kvm_vcpu_get_esr(vcpu);
1c839141 371 return ESR_ELx_SYS64_ISS_RT(esr);
c667186f
MZ		 372}
373
64cf98fa
CD		 374static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
375{
c4ad98e4
MZ		 376 if (kvm_vcpu_abt_iss1tw(vcpu))
377 return true;
378
64cf98fa
CD		 379 if (kvm_vcpu_trap_is_iabt(vcpu))
380 return false;
381
382 return kvm_vcpu_dabt_iswrite(vcpu);
383}
384
4429fc64 385static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
79c64880 386{
8d404c4c 387 return vcpu_read_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
79c64880
MZ		 388}
389
ce94fe93
MZ		 390static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
391{
8d404c4c 392 if (vcpu_mode_is_32bit(vcpu)) {
256c0960 393 *vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT;
8d404c4c
CD		 394 } else {
395 u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
500ca524 396 sctlr |= SCTLR_ELx_EE;
1975fa56 397 vcpu_write_sys_reg(vcpu, sctlr, SCTLR_EL1);
8d404c4c 398 }
ce94fe93
MZ		 399}
400
6d89d2d9
MZ		 401static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu)
402{
403 if (vcpu_mode_is_32bit(vcpu))
256c0960 404 return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT);
6d89d2d9 405
69adec18
MZ		 406 if (vcpu_mode_priv(vcpu))
407 return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_ELx_EE);
408 else
409 return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & SCTLR_EL1_E0E);
6d89d2d9
MZ		 410}
411
412static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu,
413 unsigned long data,
414 unsigned int len)
415{
416 if (kvm_vcpu_is_be(vcpu)) {
417 switch (len) {
418 case 1:
419 return data & 0xff;
420 case 2:
421 return be16_to_cpu(data & 0xffff);
422 case 4:
423 return be32_to_cpu(data & 0xffffffff);
424 default:
425 return be64_to_cpu(data);
426 }
b3007086
VK		 427 } else {
428 switch (len) {
429 case 1:
430 return data & 0xff;
431 case 2:
432 return le16_to_cpu(data & 0xffff);
433 case 4:
434 return le32_to_cpu(data & 0xffffffff);
435 default:
436 return le64_to_cpu(data);
437 }
6d89d2d9
MZ		 438 }
439
440 return data; /* Leave LE untouched */
441}
442
443static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
444 unsigned long data,
445 unsigned int len)
446{
447 if (kvm_vcpu_is_be(vcpu)) {
448 switch (len) {
449 case 1:
450 return data & 0xff;
451 case 2:
452 return cpu_to_be16(data & 0xffff);
453 case 4:
454 return cpu_to_be32(data & 0xffffffff);
455 default:
456 return cpu_to_be64(data);
457 }
b3007086
VK		 458 } else {
459 switch (len) {
460 case 1:
461 return data & 0xff;
462 case 2:
463 return cpu_to_le16(data & 0xffff);
464 case 4:
465 return cpu_to_le32(data & 0xffffffff);
466 default:
467 return cpu_to_le64(data);
468 }
6d89d2d9
MZ		 469 }
470
471 return data; /* Leave LE untouched */
472}
473
cdb5e02e 474static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)
bd7d95ca 475{
699bb2e0 476 vcpu_set_flag(vcpu, INCREMENT_PC);
bd7d95ca
MR		 477}
478
699bb2e0
MZ		 479#define kvm_pend_exception(v, e) \
480 do { \
481 vcpu_set_flag((v), PENDING_EXCEPTION); \
482 vcpu_set_flag((v), e); \
483 } while (0)
484
485
66e94d5c
MZ		 486static inline bool vcpu_has_feature(struct kvm_vcpu *vcpu, int feature)
487{
488 return test_bit(feature, vcpu->arch.features);
489}
490
83a49794 491#endif /* __ARM64_KVM_EMULATE_H__ */
Linux 6.x block layer and io_uring tree(s)