projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[linux-2.6-block.git] / arch / arm64 / include / asm / kvm_emulate.h
CommitLineData
83a49794
MZ		 1/*
2 * Copyright (C) 2012,2013 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * Derived from arch/arm/include/kvm_emulate.h
6 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
7 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 */
21
22#ifndef __ARM64_KVM_EMULATE_H__
23#define __ARM64_KVM_EMULATE_H__
24
25#include <linux/kvm_host.h>
c6d01a94 26
bd7d95ca 27#include <asm/debug-monitors.h>
c6d01a94 28#include <asm/esr.h>
83a49794 29#include <asm/kvm_arm.h>
00536ec4 30#include <asm/kvm_hyp.h>
83a49794
MZ		 31#include <asm/kvm_mmio.h>
32#include <asm/ptrace.h>
4429fc64 33#include <asm/cputype.h>
68908bf7 34#include <asm/virt.h>
83a49794 35
b547631f 36unsigned long *vcpu_reg32(const struct kvm_vcpu *vcpu, u8 reg_num);
a8928195
CD		 37unsigned long vcpu_read_spsr32(const struct kvm_vcpu *vcpu);
38void vcpu_write_spsr32(struct kvm_vcpu *vcpu, unsigned long v);
b547631f 39
27b190bd
MZ		 40bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
41void kvm_skip_instr32(struct kvm_vcpu *vcpu, bool is_wide_instr);
42
83a49794 43void kvm_inject_undefined(struct kvm_vcpu *vcpu);
10cf3390 44void kvm_inject_vabt(struct kvm_vcpu *vcpu);
83a49794
MZ		 45void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
46void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
74a64a98
MZ		 47void kvm_inject_undef32(struct kvm_vcpu *vcpu);
48void kvm_inject_dabt32(struct kvm_vcpu *vcpu, unsigned long addr);
49void kvm_inject_pabt32(struct kvm_vcpu *vcpu, unsigned long addr);
83a49794 50
e72341c5
CD		 51static inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
52{
53 return !(vcpu->arch.hcr_el2 & HCR_RW);
54}
55
b856a591
CD		 56static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
57{
58 vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
68908bf7
MZ		 59 if (is_kernel_in_hyp_mode())
60 vcpu->arch.hcr_el2 |= HCR_E2H;
558daf69
DG		 61 if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) {
62 /* route synchronous external abort exceptions to EL2 */
63 vcpu->arch.hcr_el2 |= HCR_TEA;
64 /* trap error record accesses */
65 vcpu->arch.hcr_el2 |= HCR_TERR;
66 }
e48d53a9
MZ		 67 if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
68 vcpu->arch.hcr_el2 |= HCR_FWB;
558daf69 69
801f6772
MZ		 70 if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features))
71 vcpu->arch.hcr_el2 &= ~HCR_RW;
005781be
DM		 72
73 /*
74 * TID3: trap feature register accesses that we virtualise.
75 * For now this is conditional, since no AArch32 feature regs
76 * are currently virtualised.
77 */
e72341c5 78 if (!vcpu_el1_is_32bit(vcpu))
005781be 79 vcpu->arch.hcr_el2 |= HCR_TID3;
f7f2b15c 80
793acf87
AB		 81 if (cpus_have_const_cap(ARM64_MISMATCHED_CACHE_TYPE) ||
82 vcpu_el1_is_32bit(vcpu))
f7f2b15c 83 vcpu->arch.hcr_el2 |= HCR_TID2;
b856a591
CD		 84}
85
3df59d8d 86static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu)
3c1e7165 87{
3df59d8d 88 return (unsigned long *)&vcpu->arch.hcr_el2;
3c1e7165
MZ		 89}
90
de737089
MZ		 91static inline void vcpu_clear_wfe_traps(struct kvm_vcpu *vcpu)
92{
93 vcpu->arch.hcr_el2 &= ~HCR_TWE;
94}
95
96static inline void vcpu_set_wfe_traps(struct kvm_vcpu *vcpu)
97{
98 vcpu->arch.hcr_el2 |= HCR_TWE;
99}
100
b7b27fac
DG		 101static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
102{
103 return vcpu->arch.vsesr_el2;
104}
105
4715c14b
JM		 106static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr)
107{
108 vcpu->arch.vsesr_el2 = vsesr;
109}
110
83a49794
MZ		 111static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
112{
113 return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc;
114}
115
6d4bd909 116static inline unsigned long *__vcpu_elr_el1(const struct kvm_vcpu *vcpu)
83a49794
MZ		 117{
118 return (unsigned long *)&vcpu_gp_regs(vcpu)->elr_el1;
119}
120
6d4bd909
CD		 121static inline unsigned long vcpu_read_elr_el1(const struct kvm_vcpu *vcpu)
122{
123 if (vcpu->arch.sysregs_loaded_on_cpu)
124 return read_sysreg_el1(elr);
125 else
126 return *__vcpu_elr_el1(vcpu);
127}
128
129static inline void vcpu_write_elr_el1(const struct kvm_vcpu *vcpu, unsigned long v)
130{
131 if (vcpu->arch.sysregs_loaded_on_cpu)
132 write_sysreg_el1(v, elr);
133 else
134 *__vcpu_elr_el1(vcpu) = v;
135}
136
83a49794
MZ		 137static inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu)
138{
139 return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pstate;
140}
141
142static inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
143{
b547631f 144 return !!(*vcpu_cpsr(vcpu) & PSR_MODE32_BIT);
83a49794
MZ		 145}
146
147static inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu)
148{
27b190bd
MZ		 149 if (vcpu_mode_is_32bit(vcpu))
150 return kvm_condition_valid32(vcpu);
151
152 return true;
83a49794
MZ		 153}
154
83a49794
MZ		 155static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
156{
256c0960 157 *vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT;
83a49794
MZ		 158}
159
c0f09634 160/*
f6be563a
PF		 161 * vcpu_get_reg and vcpu_set_reg should always be passed a register number
162 * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on
163 * AArch32 with banked registers.
c0f09634 164 */
bc45a516
PF		 165static inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu,
166 u8 reg_num)
167{
168 return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs.regs[reg_num];
169}
170
171static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num,
172 unsigned long val)
173{
174 if (reg_num != 31)
175 vcpu_gp_regs(vcpu)->regs.regs[reg_num] = val;
176}
177
00536ec4 178static inline unsigned long vcpu_read_spsr(const struct kvm_vcpu *vcpu)
83a49794 179{
a8928195
CD		 180 if (vcpu_mode_is_32bit(vcpu))
181 return vcpu_read_spsr32(vcpu);
00536ec4
CD		 182
183 if (vcpu->arch.sysregs_loaded_on_cpu)
184 return read_sysreg_el1(spsr);
185 else
a8928195 186 return vcpu_gp_regs(vcpu)->spsr[KVM_SPSR_EL1];
00536ec4 187}
b547631f 188
a8928195 189static inline void vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long v)
00536ec4 190{
00536ec4 191 if (vcpu_mode_is_32bit(vcpu)) {
a8928195
CD		 192 vcpu_write_spsr32(vcpu, v);
193 return;
00536ec4
CD		 194 }
195
196 if (vcpu->arch.sysregs_loaded_on_cpu)
197 write_sysreg_el1(v, spsr);
198 else
a8928195 199 vcpu_gp_regs(vcpu)->spsr[KVM_SPSR_EL1] = v;
83a49794
MZ		 200}
201
202static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu)
203{
9586a2ea 204 u32 mode;
83a49794 205
9586a2ea 206 if (vcpu_mode_is_32bit(vcpu)) {
256c0960
MR		 207 mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK;
208 return mode > PSR_AA32_MODE_USR;
9586a2ea
SZ		 209 }
210
211 mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK;
b547631f 212
83a49794
MZ		 213 return mode != PSR_MODE_EL0t;
214}
215
216static inline u32 kvm_vcpu_get_hsr(const struct kvm_vcpu *vcpu)
217{
218 return vcpu->arch.fault.esr_el2;
219}
220
3e51d435
MZ		 221static inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
222{
223 u32 esr = kvm_vcpu_get_hsr(vcpu);
224
225 if (esr & ESR_ELx_CV)
226 return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
227
228 return -1;
229}
230
83a49794
MZ		 231static inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu)
232{
233 return vcpu->arch.fault.far_el2;
234}
235
236static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
237{
238 return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8;
239}
240
0067df41
JM		 241static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu)
242{
243 return vcpu->arch.fault.disr_el1;
244}
245
0d97f884
WH		 246static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu)
247{
1c6007d5 248 return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_xVC_IMM_MASK;
0d97f884
WH		 249}
250
83a49794
MZ		 251static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
252{
c6d01a94 253 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_ISV);
83a49794
MZ		 254}
255
83a49794
MZ		 256static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
257{
c6d01a94 258 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SSE);
83a49794
MZ		 259}
260
261static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
262{
c6d01a94 263 return (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
83a49794
MZ		 264}
265
83a49794
MZ		 266static inline bool kvm_vcpu_dabt_iss1tw(const struct kvm_vcpu *vcpu)
267{
c6d01a94 268 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_S1PTW);
83a49794
MZ		 269}
270
60e21a0e
WD		 271static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
272{
273 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WNR) ||
274 kvm_vcpu_dabt_iss1tw(vcpu); /* AF/DBM update */
275}
276
57c841f1
MZ		 277static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu)
278{
279 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_CM);
280}
281
83a49794
MZ		 282static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
283{
c6d01a94 284 return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
83a49794
MZ		 285}
286
287/* This one is not specific to Data Abort */
288static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
289{
c6d01a94 290 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_IL);
83a49794
MZ		 291}
292
293static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
294{
561454e2 295 return ESR_ELx_EC(kvm_vcpu_get_hsr(vcpu));
83a49794
MZ		 296}
297
298static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
299{
c6d01a94 300 return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
83a49794
MZ		 301}
302
303static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
0496daa5 304{
c6d01a94 305 return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC;
0496daa5
CD		 306}
307
308static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
83a49794 309{
c6d01a94 310 return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC_TYPE;
83a49794
MZ		 311}
312
bb428921
JM		 313static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu)
314{
a2b83133 315 switch (kvm_vcpu_trap_get_fault(vcpu)) {
bb428921
JM		 316 case FSC_SEA:
317 case FSC_SEA_TTW0:
318 case FSC_SEA_TTW1:
319 case FSC_SEA_TTW2:
320 case FSC_SEA_TTW3:
321 case FSC_SECC:
322 case FSC_SECC_TTW0:
323 case FSC_SECC_TTW1:
324 case FSC_SECC_TTW2:
325 case FSC_SECC_TTW3:
326 return true;
327 default:
328 return false;
329 }
330}
331
c667186f
MZ		 332static inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
333{
334 u32 esr = kvm_vcpu_get_hsr(vcpu);
1c839141 335 return ESR_ELx_SYS64_ISS_RT(esr);
c667186f
MZ		 336}
337
64cf98fa
CD		 338static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
339{
340 if (kvm_vcpu_trap_is_iabt(vcpu))
341 return false;
342
343 return kvm_vcpu_dabt_iswrite(vcpu);
344}
345
4429fc64 346static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
79c64880 347{
8d404c4c 348 return vcpu_read_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
79c64880
MZ		 349}
350
ce94fe93
MZ		 351static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
352{
8d404c4c 353 if (vcpu_mode_is_32bit(vcpu)) {
256c0960 354 *vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT;
8d404c4c
CD		 355 } else {
356 u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
357 sctlr |= (1 << 25);
1975fa56 358 vcpu_write_sys_reg(vcpu, sctlr, SCTLR_EL1);
8d404c4c 359 }
ce94fe93
MZ		 360}
361
6d89d2d9
MZ		 362static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu)
363{
364 if (vcpu_mode_is_32bit(vcpu))
256c0960 365 return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT);
6d89d2d9 366
8d404c4c 367 return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & (1 << 25));
6d89d2d9
MZ		 368}
369
370static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu,
371 unsigned long data,
372 unsigned int len)
373{
374 if (kvm_vcpu_is_be(vcpu)) {
375 switch (len) {
376 case 1:
377 return data & 0xff;
378 case 2:
379 return be16_to_cpu(data & 0xffff);
380 case 4:
381 return be32_to_cpu(data & 0xffffffff);
382 default:
383 return be64_to_cpu(data);
384 }
b3007086
VK		 385 } else {
386 switch (len) {
387 case 1:
388 return data & 0xff;
389 case 2:
390 return le16_to_cpu(data & 0xffff);
391 case 4:
392 return le32_to_cpu(data & 0xffffffff);
393 default:
394 return le64_to_cpu(data);
395 }
6d89d2d9
MZ		 396 }
397
398 return data; /* Leave LE untouched */
399}
400
401static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
402 unsigned long data,
403 unsigned int len)
404{
405 if (kvm_vcpu_is_be(vcpu)) {
406 switch (len) {
407 case 1:
408 return data & 0xff;
409 case 2:
410 return cpu_to_be16(data & 0xffff);
411 case 4:
412 return cpu_to_be32(data & 0xffffffff);
413 default:
414 return cpu_to_be64(data);
415 }
b3007086
VK		 416 } else {
417 switch (len) {
418 case 1:
419 return data & 0xff;
420 case 2:
421 return cpu_to_le16(data & 0xffff);
422 case 4:
423 return cpu_to_le32(data & 0xffffffff);
424 default:
425 return cpu_to_le64(data);
426 }
6d89d2d9
MZ		 427 }
428
429 return data; /* Leave LE untouched */
430}
431
bd7d95ca
MR		 432static inline void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr)
433{
434 if (vcpu_mode_is_32bit(vcpu))
435 kvm_skip_instr32(vcpu, is_wide_instr);
436 else
437 *vcpu_pc(vcpu) += 4;
438
439 /* advance the singlestep state machine */
440 *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
441}
442
443/*
444 * Skip an instruction which has been emulated at hyp while most guest sysregs
445 * are live.
446 */
447static inline void __hyp_text __kvm_skip_instr(struct kvm_vcpu *vcpu)
448{
449 *vcpu_pc(vcpu) = read_sysreg_el2(elr);
450 vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(spsr);
451
452 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
453
454 write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, spsr);
455 write_sysreg_el2(*vcpu_pc(vcpu), elr);
456}
457
83a49794 458#endif /* __ARM64_KVM_EMULATE_H__ */
Linux 6.x block layer and io_uring tree(s)