Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
be901e9b MZ |
2 | /* |
3 | * Copyright (C) 2015 - ARM Ltd | |
4 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
be901e9b MZ |
5 | */ |
6 | ||
55e3748e | 7 | #include <linux/arm-smccc.h> |
b7c50fab | 8 | #include <linux/kvm_host.h> |
5f05a72a | 9 | #include <linux/types.h> |
5a7a8426 | 10 | #include <linux/jump_label.h> |
90348689 | 11 | #include <uapi/linux/psci.h> |
5a7a8426 | 12 | |
a4097b35 MZ |
13 | #include <kvm/arm_psci.h> |
14 | ||
f2266504 | 15 | #include <asm/barrier.h> |
85acda3b | 16 | #include <asm/cpufeature.h> |
7d826029 | 17 | #include <asm/kprobes.h> |
68908bf7 | 18 | #include <asm/kvm_asm.h> |
fb5ee369 | 19 | #include <asm/kvm_emulate.h> |
e6b673b7 | 20 | #include <asm/kvm_host.h> |
13720a56 | 21 | #include <asm/kvm_hyp.h> |
d6811986 | 22 | #include <asm/kvm_mmu.h> |
82e0191a | 23 | #include <asm/fpsimd.h> |
e3feebf8 | 24 | #include <asm/debug-monitors.h> |
85acda3b | 25 | #include <asm/processor.h> |
e6b673b7 | 26 | #include <asm/thread_info.h> |
be901e9b | 27 | |
e6b673b7 DM |
28 | /* Check whether the FP regs were dirtied while in the host-side run loop: */ |
29 | static bool __hyp_text update_fp_enabled(struct kvm_vcpu *vcpu) | |
32876224 | 30 | { |
e6b673b7 DM |
31 | if (vcpu->arch.host_thread_info->flags & _TIF_FOREIGN_FPSTATE) |
32 | vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED | | |
33 | KVM_ARM64_FP_HOST); | |
32876224 | 34 | |
e6b673b7 | 35 | return !!(vcpu->arch.flags & KVM_ARM64_FP_ENABLED); |
32876224 MZ |
36 | } |
37 | ||
b9f8ca4d CD |
38 | /* Save the 32-bit only FPSIMD system register state */ |
39 | static void __hyp_text __fpsimd_save_fpexc32(struct kvm_vcpu *vcpu) | |
40 | { | |
41 | if (!vcpu_el1_is_32bit(vcpu)) | |
42 | return; | |
43 | ||
44 | vcpu->arch.ctxt.sys_regs[FPEXC32_EL2] = read_sysreg(fpexc32_el2); | |
45 | } | |
46 | ||
d5a21bcc CD |
47 | static void __hyp_text __activate_traps_fpsimd32(struct kvm_vcpu *vcpu) |
48 | { | |
49 | /* | |
50 | * We are about to set CPTR_EL2.TFP to trap all floating point | |
51 | * register accesses to EL2, however, the ARM ARM clearly states that | |
52 | * traps are only taken to EL2 if the operation would not otherwise | |
53 | * trap to EL1. Therefore, always make sure that for 32-bit guests, | |
54 | * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit. | |
55 | * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to | |
56 | * it will cause an exception. | |
57 | */ | |
58 | if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) { | |
59 | write_sysreg(1 << 30, fpexc32_el2); | |
60 | isb(); | |
61 | } | |
62 | } | |
63 | ||
64 | static void __hyp_text __activate_traps_common(struct kvm_vcpu *vcpu) | |
65 | { | |
66 | /* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */ | |
67 | write_sysreg(1 << 15, hstr_el2); | |
68 | ||
69 | /* | |
70 | * Make sure we trap PMU access from EL0 to EL2. Also sanitize | |
71 | * PMSELR_EL0 to make sure it never contains the cycle | |
72 | * counter, which could make a PMXEVCNTR_EL0 access UNDEF at | |
73 | * EL1 instead of being trapped to EL2. | |
74 | */ | |
75 | write_sysreg(0, pmselr_el0); | |
76 | write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0); | |
77 | write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2); | |
78 | } | |
79 | ||
80 | static void __hyp_text __deactivate_traps_common(void) | |
81 | { | |
82 | write_sysreg(0, hstr_el2); | |
83 | write_sysreg(0, pmuserenr_el0); | |
84 | } | |
85 | ||
b7787e66 | 86 | static void activate_traps_vhe(struct kvm_vcpu *vcpu) |
68908bf7 MZ |
87 | { |
88 | u64 val; | |
89 | ||
90 | val = read_sysreg(cpacr_el1); | |
91 | val |= CPACR_EL1_TTA; | |
e6b673b7 | 92 | val &= ~CPACR_EL1_ZEN; |
b43b5dd9 DM |
93 | if (update_fp_enabled(vcpu)) { |
94 | if (vcpu_has_sve(vcpu)) | |
95 | val |= CPACR_EL1_ZEN; | |
96 | } else { | |
e6b673b7 | 97 | val &= ~CPACR_EL1_FPEN; |
7d14919c MZ |
98 | __activate_traps_fpsimd32(vcpu); |
99 | } | |
e6b673b7 | 100 | |
68908bf7 MZ |
101 | write_sysreg(val, cpacr_el1); |
102 | ||
6840bdd7 | 103 | write_sysreg(kvm_get_hyp_vector(), vbar_el1); |
68908bf7 | 104 | } |
7d826029 | 105 | NOKPROBE_SYMBOL(activate_traps_vhe); |
68908bf7 | 106 | |
d5a21bcc | 107 | static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu) |
68908bf7 MZ |
108 | { |
109 | u64 val; | |
110 | ||
a2465629 CD |
111 | __activate_traps_common(vcpu); |
112 | ||
68908bf7 | 113 | val = CPTR_EL2_DEFAULT; |
e6b673b7 | 114 | val |= CPTR_EL2_TTA | CPTR_EL2_TZ; |
7d14919c | 115 | if (!update_fp_enabled(vcpu)) { |
e6b673b7 | 116 | val |= CPTR_EL2_TFP; |
7d14919c MZ |
117 | __activate_traps_fpsimd32(vcpu); |
118 | } | |
e6b673b7 | 119 | |
68908bf7 | 120 | write_sysreg(val, cptr_el2); |
bd227553 MZ |
121 | |
122 | if (cpus_have_const_cap(ARM64_WORKAROUND_1319367)) { | |
123 | struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt; | |
124 | ||
125 | isb(); | |
126 | /* | |
127 | * At this stage, and thanks to the above isb(), S2 is | |
128 | * configured and enabled. We can now restore the guest's S1 | |
129 | * configuration: SCTLR, and only then TCR. | |
130 | */ | |
131 | write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1], SYS_SCTLR); | |
132 | isb(); | |
133 | write_sysreg_el1(ctxt->sys_regs[TCR_EL1], SYS_TCR); | |
134 | } | |
68908bf7 MZ |
135 | } |
136 | ||
be901e9b MZ |
137 | static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu) |
138 | { | |
e72341c5 | 139 | u64 hcr = vcpu->arch.hcr_el2; |
be901e9b | 140 | |
d3ec3a08 MZ |
141 | if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM)) |
142 | hcr |= HCR_TVM; | |
143 | ||
d5a21bcc | 144 | write_sysreg(hcr, hcr_el2); |
93390c0a | 145 | |
e72341c5 | 146 | if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) |
4715c14b JM |
147 | write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2); |
148 | ||
b7787e66 CD |
149 | if (has_vhe()) |
150 | activate_traps_vhe(vcpu); | |
151 | else | |
152 | __activate_traps_nvhe(vcpu); | |
68908bf7 | 153 | } |
a7e0ac29 | 154 | |
b7787e66 | 155 | static void deactivate_traps_vhe(void) |
68908bf7 MZ |
156 | { |
157 | extern char vectors[]; /* kernel exception vectors */ | |
68908bf7 | 158 | write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2); |
1e4448c5 MZ |
159 | |
160 | /* | |
161 | * ARM erratum 1165522 requires the actual execution of the above | |
162 | * before we can switch to the EL2/EL0 translation regime used by | |
163 | * the host. | |
164 | */ | |
165 | asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_1165522)); | |
166 | ||
17eed27b | 167 | write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); |
68908bf7 | 168 | write_sysreg(vectors, vbar_el1); |
be901e9b | 169 | } |
7d826029 | 170 | NOKPROBE_SYMBOL(deactivate_traps_vhe); |
be901e9b | 171 | |
68908bf7 | 172 | static void __hyp_text __deactivate_traps_nvhe(void) |
be901e9b | 173 | { |
f85279b4 WD |
174 | u64 mdcr_el2 = read_sysreg(mdcr_el2); |
175 | ||
bd227553 MZ |
176 | if (cpus_have_const_cap(ARM64_WORKAROUND_1319367)) { |
177 | u64 val; | |
178 | ||
179 | /* | |
180 | * Set the TCR and SCTLR registers in the exact opposite | |
181 | * sequence as __activate_traps_nvhe (first prevent walks, | |
182 | * then force the MMU on). A generous sprinkling of isb() | |
183 | * ensure that things happen in this exact order. | |
184 | */ | |
185 | val = read_sysreg_el1(SYS_TCR); | |
186 | write_sysreg_el1(val | TCR_EPD1_MASK | TCR_EPD0_MASK, SYS_TCR); | |
187 | isb(); | |
188 | val = read_sysreg_el1(SYS_SCTLR); | |
189 | write_sysreg_el1(val | SCTLR_ELx_M, SYS_SCTLR); | |
190 | isb(); | |
191 | } | |
192 | ||
a2465629 CD |
193 | __deactivate_traps_common(); |
194 | ||
f85279b4 WD |
195 | mdcr_el2 &= MDCR_EL2_HPMN_MASK; |
196 | mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT; | |
197 | ||
198 | write_sysreg(mdcr_el2, mdcr_el2); | |
4eaed6aa | 199 | write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2); |
68908bf7 MZ |
200 | write_sysreg(CPTR_EL2_DEFAULT, cptr_el2); |
201 | } | |
202 | ||
68908bf7 MZ |
203 | static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu) |
204 | { | |
44636f97 MZ |
205 | /* |
206 | * If we pended a virtual abort, preserve it until it gets | |
207 | * cleared. See D1.14.3 (Virtual Interrupts) for details, but | |
208 | * the crucial bit is "On taking a vSError interrupt, | |
209 | * HCR_EL2.VSE is cleared to 0." | |
210 | */ | |
d3ec3a08 MZ |
211 | if (vcpu->arch.hcr_el2 & HCR_VSE) { |
212 | vcpu->arch.hcr_el2 &= ~HCR_VSE; | |
213 | vcpu->arch.hcr_el2 |= read_sysreg(hcr_el2) & HCR_VSE; | |
214 | } | |
44636f97 | 215 | |
b7787e66 CD |
216 | if (has_vhe()) |
217 | deactivate_traps_vhe(); | |
218 | else | |
219 | __deactivate_traps_nvhe(); | |
be901e9b MZ |
220 | } |
221 | ||
a2465629 CD |
222 | void activate_traps_vhe_load(struct kvm_vcpu *vcpu) |
223 | { | |
224 | __activate_traps_common(vcpu); | |
225 | } | |
226 | ||
227 | void deactivate_traps_vhe_put(void) | |
228 | { | |
229 | u64 mdcr_el2 = read_sysreg(mdcr_el2); | |
230 | ||
231 | mdcr_el2 &= MDCR_EL2_HPMN_MASK | | |
232 | MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT | | |
233 | MDCR_EL2_TPMS; | |
234 | ||
235 | write_sysreg(mdcr_el2, mdcr_el2); | |
236 | ||
237 | __deactivate_traps_common(); | |
238 | } | |
239 | ||
34f8cdf1 | 240 | static void __hyp_text __activate_vm(struct kvm *kvm) |
be901e9b | 241 | { |
9f98ddd6 | 242 | __load_guest_stage2(kvm); |
be901e9b MZ |
243 | } |
244 | ||
245 | static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu) | |
246 | { | |
247 | write_sysreg(0, vttbr_el2); | |
248 | } | |
249 | ||
771621b0 CD |
250 | /* Save VGICv3 state on non-VHE systems */ |
251 | static void __hyp_text __hyp_vgic_save_state(struct kvm_vcpu *vcpu) | |
be901e9b | 252 | { |
2d0e63e0 | 253 | if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { |
5a7a8426 | 254 | __vgic_v3_save_state(vcpu); |
2d0e63e0 CD |
255 | __vgic_v3_deactivate_traps(vcpu); |
256 | } | |
be901e9b MZ |
257 | } |
258 | ||
771621b0 CD |
259 | /* Restore VGICv3 state on non_VEH systems */ |
260 | static void __hyp_text __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) | |
be901e9b | 261 | { |
2d0e63e0 CD |
262 | if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { |
263 | __vgic_v3_activate_traps(vcpu); | |
5a7a8426 | 264 | __vgic_v3_restore_state(vcpu); |
2d0e63e0 | 265 | } |
be901e9b MZ |
266 | } |
267 | ||
5f05a72a MZ |
268 | static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar) |
269 | { | |
270 | u64 par, tmp; | |
271 | ||
272 | /* | |
273 | * Resolve the IPA the hard way using the guest VA. | |
274 | * | |
275 | * Stage-1 translation already validated the memory access | |
276 | * rights. As such, we can use the EL1 translation regime, and | |
277 | * don't have to distinguish between EL0 and EL1 access. | |
278 | * | |
279 | * We do need to save/restore PAR_EL1 though, as we haven't | |
280 | * saved the guest context yet, and we may return early... | |
281 | */ | |
282 | par = read_sysreg(par_el1); | |
283 | asm volatile("at s1e1r, %0" : : "r" (far)); | |
284 | isb(); | |
285 | ||
286 | tmp = read_sysreg(par_el1); | |
287 | write_sysreg(par, par_el1); | |
288 | ||
5c062ef4 | 289 | if (unlikely(tmp & SYS_PAR_EL1_F)) |
5f05a72a MZ |
290 | return false; /* Translation failed, back to guest */ |
291 | ||
292 | /* Convert PAR to HPFAR format */ | |
bc1d7de8 | 293 | *hpfar = PAR_TO_HPFAR(tmp); |
5f05a72a MZ |
294 | return true; |
295 | } | |
296 | ||
297 | static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu) | |
298 | { | |
c60590b5 JM |
299 | u8 ec; |
300 | u64 esr; | |
5f05a72a MZ |
301 | u64 hpfar, far; |
302 | ||
c60590b5 JM |
303 | esr = vcpu->arch.fault.esr_el2; |
304 | ec = ESR_ELx_EC(esr); | |
5f05a72a MZ |
305 | |
306 | if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW) | |
307 | return true; | |
308 | ||
fdec2a9e | 309 | far = read_sysreg_el2(SYS_FAR); |
5f05a72a MZ |
310 | |
311 | /* | |
312 | * The HPFAR can be invalid if the stage 2 fault did not | |
313 | * happen during a stage 1 page table walk (the ESR_EL2.S1PTW | |
314 | * bit is clear) and one of the two following cases are true: | |
315 | * 1. The fault was due to a permission fault | |
316 | * 2. The processor carries errata 834220 | |
317 | * | |
318 | * Therefore, for all non S1PTW faults where we either have a | |
319 | * permission fault or the errata workaround is enabled, we | |
320 | * resolve the IPA using the AT instruction. | |
321 | */ | |
322 | if (!(esr & ESR_ELx_S1PTW) && | |
b6749e20 MZ |
323 | (cpus_have_const_cap(ARM64_WORKAROUND_834220) || |
324 | (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) { | |
5f05a72a MZ |
325 | if (!__translate_far_to_hpfar(far, &hpfar)) |
326 | return false; | |
327 | } else { | |
328 | hpfar = read_sysreg(hpfar_el2); | |
329 | } | |
330 | ||
331 | vcpu->arch.fault.far_el2 = far; | |
332 | vcpu->arch.fault.hpfar_el2 = hpfar; | |
333 | return true; | |
334 | } | |
335 | ||
b43b5dd9 DM |
336 | /* Check for an FPSIMD/SVE trap and handle as appropriate */ |
337 | static bool __hyp_text __hyp_handle_fpsimd(struct kvm_vcpu *vcpu) | |
ceda9fff | 338 | { |
b43b5dd9 DM |
339 | bool vhe, sve_guest, sve_host; |
340 | u8 hsr_ec; | |
85acda3b | 341 | |
b43b5dd9 DM |
342 | if (!system_supports_fpsimd()) |
343 | return false; | |
344 | ||
345 | if (system_supports_sve()) { | |
346 | sve_guest = vcpu_has_sve(vcpu); | |
347 | sve_host = vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE; | |
348 | vhe = true; | |
349 | } else { | |
350 | sve_guest = false; | |
351 | sve_host = false; | |
352 | vhe = has_vhe(); | |
353 | } | |
354 | ||
355 | hsr_ec = kvm_vcpu_trap_get_class(vcpu); | |
356 | if (hsr_ec != ESR_ELx_EC_FP_ASIMD && | |
357 | hsr_ec != ESR_ELx_EC_SVE) | |
358 | return false; | |
359 | ||
360 | /* Don't handle SVE traps for non-SVE vcpus here: */ | |
361 | if (!sve_guest) | |
362 | if (hsr_ec != ESR_ELx_EC_FP_ASIMD) | |
363 | return false; | |
364 | ||
365 | /* Valid trap. Switch the context: */ | |
366 | ||
367 | if (vhe) { | |
368 | u64 reg = read_sysreg(cpacr_el1) | CPACR_EL1_FPEN; | |
369 | ||
370 | if (sve_guest) | |
371 | reg |= CPACR_EL1_ZEN; | |
372 | ||
373 | write_sysreg(reg, cpacr_el1); | |
374 | } else { | |
ceda9fff DM |
375 | write_sysreg(read_sysreg(cptr_el2) & ~(u64)CPTR_EL2_TFP, |
376 | cptr_el2); | |
b43b5dd9 | 377 | } |
ceda9fff DM |
378 | |
379 | isb(); | |
380 | ||
e6b673b7 | 381 | if (vcpu->arch.flags & KVM_ARM64_FP_HOST) { |
85acda3b DM |
382 | /* |
383 | * In the SVE case, VHE is assumed: it is enforced by | |
384 | * Kconfig and kvm_arch_init(). | |
385 | */ | |
b43b5dd9 | 386 | if (sve_host) { |
85acda3b | 387 | struct thread_struct *thread = container_of( |
b43b5dd9 | 388 | vcpu->arch.host_fpsimd_state, |
85acda3b DM |
389 | struct thread_struct, uw.fpsimd_state); |
390 | ||
b43b5dd9 DM |
391 | sve_save_state(sve_pffr(thread), |
392 | &vcpu->arch.host_fpsimd_state->fpsr); | |
85acda3b | 393 | } else { |
b43b5dd9 | 394 | __fpsimd_save_state(vcpu->arch.host_fpsimd_state); |
85acda3b DM |
395 | } |
396 | ||
e6b673b7 DM |
397 | vcpu->arch.flags &= ~KVM_ARM64_FP_HOST; |
398 | } | |
399 | ||
b43b5dd9 DM |
400 | if (sve_guest) { |
401 | sve_load_state(vcpu_sve_pffr(vcpu), | |
402 | &vcpu->arch.ctxt.gp_regs.fp_regs.fpsr, | |
403 | sve_vq_from_vl(vcpu->arch.sve_max_vl) - 1); | |
73433762 | 404 | write_sysreg_s(vcpu->arch.ctxt.sys_regs[ZCR_EL1], SYS_ZCR_EL12); |
b43b5dd9 DM |
405 | } else { |
406 | __fpsimd_restore_state(&vcpu->arch.ctxt.gp_regs.fp_regs); | |
407 | } | |
73433762 | 408 | |
ceda9fff DM |
409 | /* Skip restoring fpexc32 for AArch64 guests */ |
410 | if (!(read_sysreg(hcr_el2) & HCR_RW)) | |
411 | write_sysreg(vcpu->arch.ctxt.sys_regs[FPEXC32_EL2], | |
412 | fpexc32_el2); | |
e6b673b7 DM |
413 | |
414 | vcpu->arch.flags |= KVM_ARM64_FP_ENABLED; | |
cf412b00 DM |
415 | |
416 | return true; | |
ceda9fff DM |
417 | } |
418 | ||
d3ec3a08 MZ |
419 | static bool __hyp_text handle_tx2_tvm(struct kvm_vcpu *vcpu) |
420 | { | |
421 | u32 sysreg = esr_sys64_to_sysreg(kvm_vcpu_get_hsr(vcpu)); | |
422 | int rt = kvm_vcpu_sys_get_rt(vcpu); | |
423 | u64 val = vcpu_get_reg(vcpu, rt); | |
424 | ||
425 | /* | |
426 | * The normal sysreg handling code expects to see the traps, | |
427 | * let's not do anything here. | |
428 | */ | |
429 | if (vcpu->arch.hcr_el2 & HCR_TVM) | |
430 | return false; | |
431 | ||
432 | switch (sysreg) { | |
433 | case SYS_SCTLR_EL1: | |
434 | write_sysreg_el1(val, SYS_SCTLR); | |
435 | break; | |
436 | case SYS_TTBR0_EL1: | |
437 | write_sysreg_el1(val, SYS_TTBR0); | |
438 | break; | |
439 | case SYS_TTBR1_EL1: | |
440 | write_sysreg_el1(val, SYS_TTBR1); | |
441 | break; | |
442 | case SYS_TCR_EL1: | |
443 | write_sysreg_el1(val, SYS_TCR); | |
444 | break; | |
445 | case SYS_ESR_EL1: | |
446 | write_sysreg_el1(val, SYS_ESR); | |
447 | break; | |
448 | case SYS_FAR_EL1: | |
449 | write_sysreg_el1(val, SYS_FAR); | |
450 | break; | |
451 | case SYS_AFSR0_EL1: | |
452 | write_sysreg_el1(val, SYS_AFSR0); | |
453 | break; | |
454 | case SYS_AFSR1_EL1: | |
455 | write_sysreg_el1(val, SYS_AFSR1); | |
456 | break; | |
457 | case SYS_MAIR_EL1: | |
458 | write_sysreg_el1(val, SYS_MAIR); | |
459 | break; | |
460 | case SYS_AMAIR_EL1: | |
461 | write_sysreg_el1(val, SYS_AMAIR); | |
462 | break; | |
463 | case SYS_CONTEXTIDR_EL1: | |
464 | write_sysreg_el1(val, SYS_CONTEXTIDR); | |
465 | break; | |
466 | default: | |
467 | return false; | |
468 | } | |
469 | ||
470 | __kvm_skip_instr(vcpu); | |
471 | return true; | |
472 | } | |
473 | ||
dc251406 CD |
474 | /* |
475 | * Return true when we were able to fixup the guest exit and should return to | |
476 | * the guest, false when we should restore the host state and return to the | |
477 | * main run loop. | |
478 | */ | |
479 | static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) | |
be901e9b | 480 | { |
dc251406 | 481 | if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ) |
fdec2a9e | 482 | vcpu->arch.fault.esr_el2 = read_sysreg_el2(SYS_ESR); |
dc251406 | 483 | |
395ea79e MZ |
484 | /* |
485 | * We're using the raw exception code in order to only process | |
486 | * the trap if no SError is pending. We will come back to the | |
487 | * same PC once the SError has been injected, and replay the | |
488 | * trapping instruction. | |
489 | */ | |
7846b311 DM |
490 | if (*exit_code != ARM_EXCEPTION_TRAP) |
491 | goto exit; | |
492 | ||
d3ec3a08 MZ |
493 | if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) && |
494 | kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 && | |
495 | handle_tx2_tvm(vcpu)) | |
496 | return true; | |
497 | ||
cf412b00 DM |
498 | /* |
499 | * We trap the first access to the FP/SIMD to save the host context | |
500 | * and restore the guest context lazily. | |
501 | * If FP/SIMD is not implemented, handle the trap and inject an | |
502 | * undefined instruction exception to the guest. | |
b43b5dd9 | 503 | * Similarly for trapped SVE accesses. |
cf412b00 | 504 | */ |
b43b5dd9 DM |
505 | if (__hyp_handle_fpsimd(vcpu)) |
506 | return true; | |
cf412b00 | 507 | |
7846b311 | 508 | if (!__populate_fault_info(vcpu)) |
dc251406 | 509 | return true; |
5f05a72a | 510 | |
7846b311 | 511 | if (static_branch_unlikely(&vgic_v2_cpuif_trap)) { |
fb5ee369 MZ |
512 | bool valid; |
513 | ||
514 | valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW && | |
515 | kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT && | |
516 | kvm_vcpu_dabt_isvalid(vcpu) && | |
517 | !kvm_vcpu_dabt_isextabt(vcpu) && | |
518 | !kvm_vcpu_dabt_iss1tw(vcpu); | |
519 | ||
3272f0d0 MZ |
520 | if (valid) { |
521 | int ret = __vgic_v2_perform_cpuif_access(vcpu); | |
522 | ||
bd7d95ca | 523 | if (ret == 1) |
ba4f4cb0 | 524 | return true; |
3272f0d0 | 525 | |
bd7d95ca MR |
526 | /* Promote an illegal access to an SError.*/ |
527 | if (ret == -1) | |
dc251406 | 528 | *exit_code = ARM_EXCEPTION_EL1_SERROR; |
7846b311 DM |
529 | |
530 | goto exit; | |
fb5ee369 MZ |
531 | } |
532 | } | |
533 | ||
59da1cbf | 534 | if (static_branch_unlikely(&vgic_v3_cpuif_trap) && |
59da1cbf MZ |
535 | (kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 || |
536 | kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) { | |
537 | int ret = __vgic_v3_perform_cpuif_access(vcpu); | |
538 | ||
bd7d95ca | 539 | if (ret == 1) |
ba4f4cb0 | 540 | return true; |
59da1cbf MZ |
541 | } |
542 | ||
7846b311 | 543 | exit: |
dc251406 CD |
544 | /* Return to the host kernel and handle the exit */ |
545 | return false; | |
546 | } | |
547 | ||
55e3748e MZ |
548 | static inline bool __hyp_text __needs_ssbd_off(struct kvm_vcpu *vcpu) |
549 | { | |
550 | if (!cpus_have_const_cap(ARM64_SSBD)) | |
551 | return false; | |
552 | ||
553 | return !(vcpu->arch.workaround_flags & VCPU_WORKAROUND_2_FLAG); | |
554 | } | |
555 | ||
556 | static void __hyp_text __set_guest_arch_workaround_state(struct kvm_vcpu *vcpu) | |
557 | { | |
558 | #ifdef CONFIG_ARM64_SSBD | |
559 | /* | |
560 | * The host runs with the workaround always present. If the | |
561 | * guest wants it disabled, so be it... | |
562 | */ | |
563 | if (__needs_ssbd_off(vcpu) && | |
564 | __hyp_this_cpu_read(arm64_ssbd_callback_required)) | |
565 | arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, 0, NULL); | |
566 | #endif | |
567 | } | |
568 | ||
569 | static void __hyp_text __set_host_arch_workaround_state(struct kvm_vcpu *vcpu) | |
570 | { | |
571 | #ifdef CONFIG_ARM64_SSBD | |
572 | /* | |
573 | * If the guest has disabled the workaround, bring it back on. | |
574 | */ | |
575 | if (__needs_ssbd_off(vcpu) && | |
576 | __hyp_this_cpu_read(arm64_ssbd_callback_required)) | |
577 | arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, 1, NULL); | |
578 | #endif | |
579 | } | |
580 | ||
b7c50fab JM |
581 | /** |
582 | * Disable host events, enable guest events | |
583 | */ | |
584 | static bool __hyp_text __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt) | |
585 | { | |
586 | struct kvm_host_data *host; | |
587 | struct kvm_pmu_events *pmu; | |
588 | ||
589 | host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); | |
590 | pmu = &host->pmu_events; | |
591 | ||
592 | if (pmu->events_host) | |
593 | write_sysreg(pmu->events_host, pmcntenclr_el0); | |
594 | ||
595 | if (pmu->events_guest) | |
596 | write_sysreg(pmu->events_guest, pmcntenset_el0); | |
597 | ||
598 | return (pmu->events_host || pmu->events_guest); | |
599 | } | |
600 | ||
601 | /** | |
602 | * Disable guest events, enable host events | |
603 | */ | |
604 | static void __hyp_text __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) | |
605 | { | |
606 | struct kvm_host_data *host; | |
607 | struct kvm_pmu_events *pmu; | |
608 | ||
609 | host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); | |
610 | pmu = &host->pmu_events; | |
611 | ||
612 | if (pmu->events_guest) | |
613 | write_sysreg(pmu->events_guest, pmcntenclr_el0); | |
614 | ||
615 | if (pmu->events_host) | |
616 | write_sysreg(pmu->events_host, pmcntenset_el0); | |
617 | } | |
618 | ||
3f5c90b8 CD |
619 | /* Switch to the guest for VHE systems running in EL2 */ |
620 | int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) | |
621 | { | |
622 | struct kvm_cpu_context *host_ctxt; | |
623 | struct kvm_cpu_context *guest_ctxt; | |
3f5c90b8 CD |
624 | u64 exit_code; |
625 | ||
86d05682 | 626 | host_ctxt = vcpu->arch.host_cpu_context; |
3f5c90b8 CD |
627 | host_ctxt->__hyp_running_vcpu = vcpu; |
628 | guest_ctxt = &vcpu->arch.ctxt; | |
629 | ||
f837453d | 630 | sysreg_save_host_state_vhe(host_ctxt); |
3f5c90b8 | 631 | |
1e4448c5 MZ |
632 | /* |
633 | * ARM erratum 1165522 requires us to configure both stage 1 and | |
634 | * stage 2 translation for the guest context before we clear | |
635 | * HCR_EL2.TGE. | |
636 | * | |
637 | * We have already configured the guest's stage 1 translation in | |
638 | * kvm_vcpu_load_sysregs above. We must now call __activate_vm | |
639 | * before __activate_traps, because __activate_vm configures | |
640 | * stage 2 translation, and __activate_traps clear HCR_EL2.TGE | |
641 | * (among other things). | |
642 | */ | |
34f8cdf1 | 643 | __activate_vm(vcpu->kvm); |
bfae1b98 | 644 | __activate_traps(vcpu); |
3f5c90b8 | 645 | |
f837453d | 646 | sysreg_restore_guest_state_vhe(guest_ctxt); |
3f5c90b8 CD |
647 | __debug_switch_to_guest(vcpu); |
648 | ||
55e3748e MZ |
649 | __set_guest_arch_workaround_state(vcpu); |
650 | ||
3f5c90b8 CD |
651 | do { |
652 | /* Jump in the fire! */ | |
653 | exit_code = __guest_enter(vcpu, host_ctxt); | |
654 | ||
655 | /* And we're baaack! */ | |
656 | } while (fixup_guest_exit(vcpu, &exit_code)); | |
657 | ||
55e3748e MZ |
658 | __set_host_arch_workaround_state(vcpu); |
659 | ||
f837453d | 660 | sysreg_save_guest_state_vhe(guest_ctxt); |
3f5c90b8 CD |
661 | |
662 | __deactivate_traps(vcpu); | |
3f5c90b8 | 663 | |
f837453d | 664 | sysreg_restore_host_state_vhe(host_ctxt); |
3f5c90b8 | 665 | |
e6b673b7 | 666 | if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) |
b9f8ca4d | 667 | __fpsimd_save_fpexc32(vcpu); |
3f5c90b8 | 668 | |
3f5c90b8 CD |
669 | __debug_switch_to_host(vcpu); |
670 | ||
671 | return exit_code; | |
672 | } | |
7d826029 | 673 | NOKPROBE_SYMBOL(kvm_vcpu_run_vhe); |
3f5c90b8 CD |
674 | |
675 | /* Switch to the guest for legacy non-VHE systems */ | |
676 | int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu) | |
dc251406 CD |
677 | { |
678 | struct kvm_cpu_context *host_ctxt; | |
679 | struct kvm_cpu_context *guest_ctxt; | |
3d91befb | 680 | bool pmu_switch_needed; |
dc251406 CD |
681 | u64 exit_code; |
682 | ||
85738e05 JT |
683 | /* |
684 | * Having IRQs masked via PMR when entering the guest means the GIC | |
685 | * will not signal the CPU of interrupts of lower priority, and the | |
686 | * only way to get out will be via guest exceptions. | |
687 | * Naturally, we want to avoid this. | |
688 | */ | |
689 | if (system_uses_irq_prio_masking()) { | |
bd82d4bd | 690 | gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); |
f2266504 | 691 | pmr_sync(); |
85738e05 JT |
692 | } |
693 | ||
dc251406 CD |
694 | vcpu = kern_hyp_va(vcpu); |
695 | ||
696 | host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); | |
697 | host_ctxt->__hyp_running_vcpu = vcpu; | |
698 | guest_ctxt = &vcpu->arch.ctxt; | |
699 | ||
3d91befb AM |
700 | pmu_switch_needed = __pmu_switch_to_guest(host_ctxt); |
701 | ||
4cdecaba | 702 | __sysreg_save_state_nvhe(host_ctxt); |
dc251406 | 703 | |
dc251406 CD |
704 | /* |
705 | * We must restore the 32-bit state before the sysregs, thanks | |
706 | * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72). | |
1d8cd06a MZ |
707 | * |
708 | * Also, and in order to be able to deal with erratum #1319537 (A57) | |
709 | * and #1319367 (A72), we must ensure that all VM-related sysreg are | |
710 | * restored before we enable S2 translation. | |
dc251406 CD |
711 | */ |
712 | __sysreg32_restore_state(vcpu); | |
4cdecaba | 713 | __sysreg_restore_state_nvhe(guest_ctxt); |
1d8cd06a MZ |
714 | |
715 | __activate_vm(kern_hyp_va(vcpu->kvm)); | |
716 | __activate_traps(vcpu); | |
717 | ||
718 | __hyp_vgic_restore_state(vcpu); | |
719 | __timer_enable_traps(vcpu); | |
720 | ||
dc251406 CD |
721 | __debug_switch_to_guest(vcpu); |
722 | ||
55e3748e MZ |
723 | __set_guest_arch_workaround_state(vcpu); |
724 | ||
dc251406 CD |
725 | do { |
726 | /* Jump in the fire! */ | |
727 | exit_code = __guest_enter(vcpu, host_ctxt); | |
728 | ||
729 | /* And we're baaack! */ | |
730 | } while (fixup_guest_exit(vcpu, &exit_code)); | |
731 | ||
55e3748e MZ |
732 | __set_host_arch_workaround_state(vcpu); |
733 | ||
4cdecaba | 734 | __sysreg_save_state_nvhe(guest_ctxt); |
be901e9b | 735 | __sysreg32_save_state(vcpu); |
688c50aa | 736 | __timer_disable_traps(vcpu); |
771621b0 | 737 | __hyp_vgic_save_state(vcpu); |
be901e9b MZ |
738 | |
739 | __deactivate_traps(vcpu); | |
740 | __deactivate_vm(vcpu); | |
741 | ||
4cdecaba | 742 | __sysreg_restore_state_nvhe(host_ctxt); |
be901e9b | 743 | |
e6b673b7 | 744 | if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) |
b9f8ca4d | 745 | __fpsimd_save_fpexc32(vcpu); |
c13d1683 | 746 | |
f85279b4 WD |
747 | /* |
748 | * This must come after restoring the host sysregs, since a non-VHE | |
749 | * system may enable SPE here and make use of the TTBRs. | |
750 | */ | |
014c4c77 | 751 | __debug_switch_to_host(vcpu); |
be901e9b | 752 | |
3d91befb AM |
753 | if (pmu_switch_needed) |
754 | __pmu_switch_to_host(host_ctxt); | |
755 | ||
85738e05 JT |
756 | /* Returning to host will clear PSR.I, remask PMR if needed */ |
757 | if (system_uses_irq_prio_masking()) | |
758 | gic_write_pmr(GIC_PRIO_IRQOFF); | |
759 | ||
be901e9b MZ |
760 | return exit_code; |
761 | } | |
53fd5b64 MZ |
762 | |
763 | static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n"; | |
764 | ||
c97e166e | 765 | static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par, |
8f17f5e4 | 766 | struct kvm_cpu_context *__host_ctxt) |
53fd5b64 | 767 | { |
8f17f5e4 | 768 | struct kvm_vcpu *vcpu; |
cf7df13d | 769 | unsigned long str_va; |
253dcbd3 | 770 | |
8f17f5e4 CD |
771 | vcpu = __host_ctxt->__hyp_running_vcpu; |
772 | ||
773 | if (read_sysreg(vttbr_el2)) { | |
774 | __timer_disable_traps(vcpu); | |
775 | __deactivate_traps(vcpu); | |
776 | __deactivate_vm(vcpu); | |
4cdecaba | 777 | __sysreg_restore_state_nvhe(__host_ctxt); |
8f17f5e4 CD |
778 | } |
779 | ||
cf7df13d MZ |
780 | /* |
781 | * Force the panic string to be loaded from the literal pool, | |
782 | * making sure it is a kernel address and not a PC-relative | |
783 | * reference. | |
784 | */ | |
785 | asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va)); | |
786 | ||
787 | __hyp_do_panic(str_va, | |
fdec2a9e DM |
788 | spsr, elr, |
789 | read_sysreg(esr_el2), read_sysreg_el2(SYS_FAR), | |
c97e166e | 790 | read_sysreg(hpfar_el2), par, vcpu); |
253dcbd3 MZ |
791 | } |
792 | ||
8f17f5e4 CD |
793 | static void __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par, |
794 | struct kvm_cpu_context *host_ctxt) | |
253dcbd3 | 795 | { |
8f17f5e4 CD |
796 | struct kvm_vcpu *vcpu; |
797 | vcpu = host_ctxt->__hyp_running_vcpu; | |
798 | ||
799 | __deactivate_traps(vcpu); | |
f837453d | 800 | sysreg_restore_host_state_vhe(host_ctxt); |
8f17f5e4 | 801 | |
253dcbd3 MZ |
802 | panic(__hyp_panic_string, |
803 | spsr, elr, | |
fdec2a9e | 804 | read_sysreg_el2(SYS_ESR), read_sysreg_el2(SYS_FAR), |
c97e166e | 805 | read_sysreg(hpfar_el2), par, vcpu); |
253dcbd3 | 806 | } |
7d826029 | 807 | NOKPROBE_SYMBOL(__hyp_call_panic_vhe); |
253dcbd3 | 808 | |
4464e210 | 809 | void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt) |
253dcbd3 | 810 | { |
fdec2a9e DM |
811 | u64 spsr = read_sysreg_el2(SYS_SPSR); |
812 | u64 elr = read_sysreg_el2(SYS_ELR); | |
53fd5b64 MZ |
813 | u64 par = read_sysreg(par_el1); |
814 | ||
8f17f5e4 CD |
815 | if (!has_vhe()) |
816 | __hyp_call_panic_nvhe(spsr, elr, par, host_ctxt); | |
817 | else | |
818 | __hyp_call_panic_vhe(spsr, elr, par, host_ctxt); | |
53fd5b64 MZ |
819 | |
820 | unreachable(); | |
821 | } |