Commit | Line | Data |
---|---|---|
09cf57eb DB |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Copyright (C) 2015 - ARM Ltd | |
4 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
5 | */ | |
6 | ||
7 | #include <hyp/switch.h> | |
8 | ||
9 | #include <linux/arm-smccc.h> | |
10 | #include <linux/kvm_host.h> | |
11 | #include <linux/types.h> | |
12 | #include <linux/jump_label.h> | |
bd09128d | 13 | #include <linux/percpu.h> |
09cf57eb DB |
14 | #include <uapi/linux/psci.h> |
15 | ||
16 | #include <kvm/arm_psci.h> | |
17 | ||
18 | #include <asm/barrier.h> | |
19 | #include <asm/cpufeature.h> | |
20 | #include <asm/kprobes.h> | |
21 | #include <asm/kvm_asm.h> | |
22 | #include <asm/kvm_emulate.h> | |
23 | #include <asm/kvm_hyp.h> | |
24 | #include <asm/kvm_mmu.h> | |
25 | #include <asm/fpsimd.h> | |
26 | #include <asm/debug-monitors.h> | |
27 | #include <asm/processor.h> | |
bd09128d JM |
28 | #include <asm/thread_info.h> |
29 | #include <asm/vectors.h> | |
09cf57eb | 30 | |
14ef9d04 MZ |
31 | /* VHE specific context */ |
32 | DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data); | |
33 | DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); | |
34 | DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); | |
2a1198c9 | 35 | |
09cf57eb DB |
36 | static void __activate_traps(struct kvm_vcpu *vcpu) |
37 | { | |
38 | u64 val; | |
39 | ||
40 | ___activate_traps(vcpu); | |
41 | ||
94046732 MZ |
42 | if (has_cntpoff()) { |
43 | struct timer_map map; | |
44 | ||
45 | get_timer_map(vcpu, &map); | |
46 | ||
47 | /* | |
48 | * We're entrering the guest. Reload the correct | |
49 | * values from memory now that TGE is clear. | |
50 | */ | |
51 | if (map.direct_ptimer == vcpu_ptimer(vcpu)) | |
52 | val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); | |
53 | if (map.direct_ptimer == vcpu_hptimer(vcpu)) | |
54 | val = __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2); | |
55 | ||
56 | if (map.direct_ptimer) { | |
57 | write_sysreg_el0(val, SYS_CNTP_CVAL); | |
58 | isb(); | |
59 | } | |
60 | } | |
61 | ||
09cf57eb | 62 | val = read_sysreg(cpacr_el1); |
7a5e9c8f | 63 | val |= CPACR_ELx_TTA; |
51729fb1 MB |
64 | val &= ~(CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN | |
65 | CPACR_EL1_SMEN_EL0EN | CPACR_EL1_SMEN_EL1EN); | |
09cf57eb DB |
66 | |
67 | /* | |
68 | * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to | |
69 | * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2, | |
70 | * except for some missing controls, such as TAM. | |
71 | * In this case, CPTR_EL2.TAM has the same position with or without | |
72 | * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM | |
73 | * shift value for trapping the AMU accesses. | |
74 | */ | |
75 | ||
76 | val |= CPTR_EL2_TAM; | |
77 | ||
e9ada6c2 | 78 | if (guest_owns_fp_regs(vcpu)) { |
09cf57eb | 79 | if (vcpu_has_sve(vcpu)) |
3bb72d86 | 80 | val |= CPACR_EL1_ZEN_EL0EN | CPACR_EL1_ZEN_EL1EN; |
09cf57eb | 81 | } else { |
3bb72d86 | 82 | val &= ~(CPACR_EL1_FPEN_EL0EN | CPACR_EL1_FPEN_EL1EN); |
09cf57eb DB |
83 | __activate_traps_fpsimd32(vcpu); |
84 | } | |
85 | ||
86 | write_sysreg(val, cpacr_el1); | |
87 | ||
a0e47952 | 88 | write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el1); |
09cf57eb DB |
89 | } |
90 | NOKPROBE_SYMBOL(__activate_traps); | |
91 | ||
92 | static void __deactivate_traps(struct kvm_vcpu *vcpu) | |
93 | { | |
bd09128d | 94 | const char *host_vectors = vectors; |
09cf57eb DB |
95 | |
96 | ___deactivate_traps(vcpu); | |
97 | ||
98 | write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2); | |
99 | ||
94046732 MZ |
100 | if (has_cntpoff()) { |
101 | struct timer_map map; | |
102 | u64 val, offset; | |
103 | ||
104 | get_timer_map(vcpu, &map); | |
105 | ||
106 | /* | |
107 | * We're exiting the guest. Save the latest CVAL value | |
108 | * to memory and apply the offset now that TGE is set. | |
109 | */ | |
110 | val = read_sysreg_el0(SYS_CNTP_CVAL); | |
111 | if (map.direct_ptimer == vcpu_ptimer(vcpu)) | |
112 | __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0) = val; | |
113 | if (map.direct_ptimer == vcpu_hptimer(vcpu)) | |
114 | __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2) = val; | |
115 | ||
116 | offset = read_sysreg_s(SYS_CNTPOFF_EL2); | |
117 | ||
118 | if (map.direct_ptimer && offset) { | |
119 | write_sysreg_el0(val + offset, SYS_CNTP_CVAL); | |
120 | isb(); | |
121 | } | |
122 | } | |
123 | ||
09cf57eb DB |
124 | /* |
125 | * ARM errata 1165522 and 1530923 require the actual execution of the | |
126 | * above before we can switch to the EL2/EL0 translation regime used by | |
127 | * the host. | |
128 | */ | |
129 | asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); | |
130 | ||
75c76ab5 | 131 | kvm_reset_cptr_el2(vcpu); |
bd09128d JM |
132 | |
133 | if (!arm64_kernel_unmapped_at_el0()) | |
134 | host_vectors = __this_cpu_read(this_cpu_vector); | |
135 | write_sysreg(host_vectors, vbar_el1); | |
09cf57eb DB |
136 | } |
137 | NOKPROBE_SYMBOL(__deactivate_traps); | |
138 | ||
0c2f9acf RW |
139 | /* |
140 | * Disable IRQs in {activate,deactivate}_traps_vhe_{load,put}() to | |
141 | * prevent a race condition between context switching of PMUSERENR_EL0 | |
142 | * in __{activate,deactivate}_traps_common() and IPIs that attempts to | |
143 | * update PMUSERENR_EL0. See also kvm_set_pmuserenr(). | |
144 | */ | |
09cf57eb DB |
145 | void activate_traps_vhe_load(struct kvm_vcpu *vcpu) |
146 | { | |
0c2f9acf RW |
147 | unsigned long flags; |
148 | ||
149 | local_irq_save(flags); | |
09cf57eb | 150 | __activate_traps_common(vcpu); |
0c2f9acf | 151 | local_irq_restore(flags); |
09cf57eb DB |
152 | } |
153 | ||
1460b4b2 | 154 | void deactivate_traps_vhe_put(struct kvm_vcpu *vcpu) |
09cf57eb | 155 | { |
0c2f9acf RW |
156 | unsigned long flags; |
157 | ||
158 | local_irq_save(flags); | |
1460b4b2 | 159 | __deactivate_traps_common(vcpu); |
0c2f9acf | 160 | local_irq_restore(flags); |
09cf57eb DB |
161 | } |
162 | ||
8fb20461 MZ |
163 | static const exit_handler_fn hyp_exit_handlers[] = { |
164 | [0 ... ESR_ELx_EC_MAX] = NULL, | |
165 | [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32, | |
166 | [ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg, | |
167 | [ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd, | |
168 | [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, | |
169 | [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low, | |
170 | [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low, | |
811154e2 | 171 | [ESR_ELx_EC_WATCHPT_LOW] = kvm_hyp_handle_watchpt_low, |
8fb20461 MZ |
172 | [ESR_ELx_EC_PAC] = kvm_hyp_handle_ptrauth, |
173 | }; | |
174 | ||
0c7639cc | 175 | static const exit_handler_fn *kvm_get_exit_handler_array(struct kvm_vcpu *vcpu) |
8fb20461 MZ |
176 | { |
177 | return hyp_exit_handlers; | |
178 | } | |
179 | ||
7183b2b5 MZ |
180 | static void early_exit_filter(struct kvm_vcpu *vcpu, u64 *exit_code) |
181 | { | |
d9552fe1 MZ |
182 | /* |
183 | * If we were in HYP context on entry, adjust the PSTATE view | |
184 | * so that the usual helpers work correctly. | |
185 | */ | |
186 | if (unlikely(vcpu_get_flag(vcpu, VCPU_HYP_CONTEXT))) { | |
187 | u64 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); | |
188 | ||
189 | switch (mode) { | |
190 | case PSR_MODE_EL1t: | |
191 | mode = PSR_MODE_EL2t; | |
192 | break; | |
193 | case PSR_MODE_EL1h: | |
194 | mode = PSR_MODE_EL2h; | |
195 | break; | |
196 | } | |
197 | ||
198 | *vcpu_cpsr(vcpu) &= ~(PSR_MODE_MASK | PSR_MODE32_BIT); | |
199 | *vcpu_cpsr(vcpu) |= mode; | |
200 | } | |
7183b2b5 MZ |
201 | } |
202 | ||
09cf57eb DB |
203 | /* Switch to the guest for VHE systems running in EL2 */ |
204 | static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) | |
205 | { | |
206 | struct kvm_cpu_context *host_ctxt; | |
207 | struct kvm_cpu_context *guest_ctxt; | |
208 | u64 exit_code; | |
209 | ||
717cf94a | 210 | host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; |
09cf57eb DB |
211 | host_ctxt->__hyp_running_vcpu = vcpu; |
212 | guest_ctxt = &vcpu->arch.ctxt; | |
213 | ||
214 | sysreg_save_host_state_vhe(host_ctxt); | |
215 | ||
216 | /* | |
217 | * ARM erratum 1165522 requires us to configure both stage 1 and | |
218 | * stage 2 translation for the guest context before we clear | |
219 | * HCR_EL2.TGE. | |
220 | * | |
221 | * We have already configured the guest's stage 1 translation in | |
501a67a2 | 222 | * kvm_vcpu_load_sysregs_vhe above. We must now call |
4efc0ede MZ |
223 | * __load_stage2 before __activate_traps, because |
224 | * __load_stage2 configures stage 2 translation, and | |
501a67a2 | 225 | * __activate_traps clear HCR_EL2.TGE (among other things). |
09cf57eb | 226 | */ |
4efc0ede | 227 | __load_stage2(vcpu->arch.hw_mmu, vcpu->arch.hw_mmu->arch); |
09cf57eb DB |
228 | __activate_traps(vcpu); |
229 | ||
f5e30680 | 230 | __kvm_adjust_pc(vcpu); |
cdb5e02e | 231 | |
09cf57eb DB |
232 | sysreg_restore_guest_state_vhe(guest_ctxt); |
233 | __debug_switch_to_guest(vcpu); | |
234 | ||
d9552fe1 MZ |
235 | if (is_hyp_ctxt(vcpu)) |
236 | vcpu_set_flag(vcpu, VCPU_HYP_CONTEXT); | |
237 | else | |
238 | vcpu_clear_flag(vcpu, VCPU_HYP_CONTEXT); | |
239 | ||
09cf57eb DB |
240 | do { |
241 | /* Jump in the fire! */ | |
b619d9aa | 242 | exit_code = __guest_enter(vcpu); |
09cf57eb DB |
243 | |
244 | /* And we're baaack! */ | |
245 | } while (fixup_guest_exit(vcpu, &exit_code)); | |
246 | ||
09cf57eb DB |
247 | sysreg_save_guest_state_vhe(guest_ctxt); |
248 | ||
249 | __deactivate_traps(vcpu); | |
250 | ||
251 | sysreg_restore_host_state_vhe(host_ctxt); | |
252 | ||
f8077b0d | 253 | if (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED) |
09cf57eb DB |
254 | __fpsimd_save_fpexc32(vcpu); |
255 | ||
256 | __debug_switch_to_host(vcpu); | |
257 | ||
258 | return exit_code; | |
259 | } | |
260 | NOKPROBE_SYMBOL(__kvm_vcpu_run_vhe); | |
261 | ||
262 | int __kvm_vcpu_run(struct kvm_vcpu *vcpu) | |
263 | { | |
264 | int ret; | |
265 | ||
266 | local_daif_mask(); | |
267 | ||
268 | /* | |
269 | * Having IRQs masked via PMR when entering the guest means the GIC | |
270 | * will not signal the CPU of interrupts of lower priority, and the | |
271 | * only way to get out will be via guest exceptions. | |
272 | * Naturally, we want to avoid this. | |
273 | * | |
274 | * local_daif_mask() already sets GIC_PRIO_PSR_I_SET, we just need a | |
275 | * dsb to ensure the redistributor is forwards EL2 IRQs to the CPU. | |
276 | */ | |
277 | pmr_sync(); | |
278 | ||
279 | ret = __kvm_vcpu_run_vhe(vcpu); | |
280 | ||
281 | /* | |
282 | * local_daif_restore() takes care to properly restore PSTATE.DAIF | |
283 | * and the GIC PMR if the host is using IRQ priorities. | |
284 | */ | |
285 | local_daif_restore(DAIF_PROCCTX_NOIRQ); | |
286 | ||
287 | /* | |
288 | * When we exit from the guest we change a number of CPU configuration | |
bcf3e7da MZ |
289 | * parameters, such as traps. We rely on the isb() in kvm_call_hyp*() |
290 | * to make sure these changes take effect before running the host or | |
291 | * additional guests. | |
09cf57eb | 292 | */ |
09cf57eb DB |
293 | return ret; |
294 | } | |
295 | ||
6a0259ed | 296 | static void __hyp_call_panic(u64 spsr, u64 elr, u64 par) |
09cf57eb | 297 | { |
6a0259ed | 298 | struct kvm_cpu_context *host_ctxt; |
09cf57eb | 299 | struct kvm_vcpu *vcpu; |
6a0259ed | 300 | |
14ef9d04 | 301 | host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; |
09cf57eb DB |
302 | vcpu = host_ctxt->__hyp_running_vcpu; |
303 | ||
304 | __deactivate_traps(vcpu); | |
305 | sysreg_restore_host_state_vhe(host_ctxt); | |
306 | ||
aec0fae6 | 307 | panic("HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n", |
09cf57eb DB |
308 | spsr, elr, |
309 | read_sysreg_el2(SYS_ESR), read_sysreg_el2(SYS_FAR), | |
310 | read_sysreg(hpfar_el2), par, vcpu); | |
311 | } | |
312 | NOKPROBE_SYMBOL(__hyp_call_panic); | |
313 | ||
6a0259ed | 314 | void __noreturn hyp_panic(void) |
09cf57eb DB |
315 | { |
316 | u64 spsr = read_sysreg_el2(SYS_SPSR); | |
317 | u64 elr = read_sysreg_el2(SYS_ELR); | |
96d389ca | 318 | u64 par = read_sysreg_par(); |
09cf57eb | 319 | |
6a0259ed | 320 | __hyp_call_panic(spsr, elr, par); |
09cf57eb DB |
321 | unreachable(); |
322 | } | |
e9ee186b JM |
323 | |
324 | asmlinkage void kvm_unexpected_el2_exception(void) | |
325 | { | |
1c3ace2b | 326 | __kvm_unexpected_el2_exception(); |
e9ee186b | 327 | } |