Commit | Line | Data |
---|---|---|
09cf57eb DB |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Copyright (C) 2015 - ARM Ltd | |
4 | * Author: Marc Zyngier <marc.zyngier@arm.com> | |
5 | */ | |
6 | ||
7 | #include <hyp/switch.h> | |
8 | ||
9 | #include <linux/arm-smccc.h> | |
10 | #include <linux/kvm_host.h> | |
11 | #include <linux/types.h> | |
12 | #include <linux/jump_label.h> | |
13 | #include <uapi/linux/psci.h> | |
14 | ||
15 | #include <kvm/arm_psci.h> | |
16 | ||
17 | #include <asm/barrier.h> | |
18 | #include <asm/cpufeature.h> | |
19 | #include <asm/kprobes.h> | |
20 | #include <asm/kvm_asm.h> | |
21 | #include <asm/kvm_emulate.h> | |
22 | #include <asm/kvm_hyp.h> | |
23 | #include <asm/kvm_mmu.h> | |
24 | #include <asm/fpsimd.h> | |
25 | #include <asm/debug-monitors.h> | |
26 | #include <asm/processor.h> | |
27 | #include <asm/thread_info.h> | |
28 | ||
29 | const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n"; | |
30 | ||
14ef9d04 MZ |
31 | /* VHE specific context */ |
32 | DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data); | |
33 | DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); | |
34 | DEFINE_PER_CPU(unsigned long, kvm_hyp_vector); | |
2a1198c9 | 35 | |
09cf57eb DB |
36 | static void __activate_traps(struct kvm_vcpu *vcpu) |
37 | { | |
38 | u64 val; | |
39 | ||
40 | ___activate_traps(vcpu); | |
41 | ||
42 | val = read_sysreg(cpacr_el1); | |
43 | val |= CPACR_EL1_TTA; | |
44 | val &= ~CPACR_EL1_ZEN; | |
45 | ||
46 | /* | |
47 | * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to | |
48 | * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2, | |
49 | * except for some missing controls, such as TAM. | |
50 | * In this case, CPTR_EL2.TAM has the same position with or without | |
51 | * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM | |
52 | * shift value for trapping the AMU accesses. | |
53 | */ | |
54 | ||
55 | val |= CPTR_EL2_TAM; | |
56 | ||
57 | if (update_fp_enabled(vcpu)) { | |
58 | if (vcpu_has_sve(vcpu)) | |
59 | val |= CPACR_EL1_ZEN; | |
60 | } else { | |
61 | val &= ~CPACR_EL1_FPEN; | |
62 | __activate_traps_fpsimd32(vcpu); | |
63 | } | |
64 | ||
65 | write_sysreg(val, cpacr_el1); | |
66 | ||
a0e47952 | 67 | write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el1); |
09cf57eb DB |
68 | } |
69 | NOKPROBE_SYMBOL(__activate_traps); | |
70 | ||
71 | static void __deactivate_traps(struct kvm_vcpu *vcpu) | |
72 | { | |
73 | extern char vectors[]; /* kernel exception vectors */ | |
74 | ||
75 | ___deactivate_traps(vcpu); | |
76 | ||
77 | write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2); | |
78 | ||
79 | /* | |
80 | * ARM errata 1165522 and 1530923 require the actual execution of the | |
81 | * above before we can switch to the EL2/EL0 translation regime used by | |
82 | * the host. | |
83 | */ | |
84 | asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); | |
85 | ||
86 | write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); | |
87 | write_sysreg(vectors, vbar_el1); | |
88 | } | |
89 | NOKPROBE_SYMBOL(__deactivate_traps); | |
90 | ||
91 | void activate_traps_vhe_load(struct kvm_vcpu *vcpu) | |
92 | { | |
93 | __activate_traps_common(vcpu); | |
94 | } | |
95 | ||
96 | void deactivate_traps_vhe_put(void) | |
97 | { | |
98 | u64 mdcr_el2 = read_sysreg(mdcr_el2); | |
99 | ||
100 | mdcr_el2 &= MDCR_EL2_HPMN_MASK | | |
101 | MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT | | |
102 | MDCR_EL2_TPMS; | |
103 | ||
104 | write_sysreg(mdcr_el2, mdcr_el2); | |
105 | ||
106 | __deactivate_traps_common(); | |
107 | } | |
108 | ||
109 | /* Switch to the guest for VHE systems running in EL2 */ | |
110 | static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) | |
111 | { | |
112 | struct kvm_cpu_context *host_ctxt; | |
113 | struct kvm_cpu_context *guest_ctxt; | |
114 | u64 exit_code; | |
115 | ||
717cf94a | 116 | host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; |
09cf57eb DB |
117 | host_ctxt->__hyp_running_vcpu = vcpu; |
118 | guest_ctxt = &vcpu->arch.ctxt; | |
119 | ||
120 | sysreg_save_host_state_vhe(host_ctxt); | |
121 | ||
122 | /* | |
123 | * ARM erratum 1165522 requires us to configure both stage 1 and | |
124 | * stage 2 translation for the guest context before we clear | |
125 | * HCR_EL2.TGE. | |
126 | * | |
127 | * We have already configured the guest's stage 1 translation in | |
501a67a2 AS |
128 | * kvm_vcpu_load_sysregs_vhe above. We must now call |
129 | * __load_guest_stage2 before __activate_traps, because | |
130 | * __load_guest_stage2 configures stage 2 translation, and | |
131 | * __activate_traps clear HCR_EL2.TGE (among other things). | |
09cf57eb | 132 | */ |
501a67a2 | 133 | __load_guest_stage2(vcpu->arch.hw_mmu); |
09cf57eb DB |
134 | __activate_traps(vcpu); |
135 | ||
136 | sysreg_restore_guest_state_vhe(guest_ctxt); | |
137 | __debug_switch_to_guest(vcpu); | |
138 | ||
09cf57eb DB |
139 | do { |
140 | /* Jump in the fire! */ | |
b619d9aa | 141 | exit_code = __guest_enter(vcpu); |
09cf57eb DB |
142 | |
143 | /* And we're baaack! */ | |
144 | } while (fixup_guest_exit(vcpu, &exit_code)); | |
145 | ||
09cf57eb DB |
146 | sysreg_save_guest_state_vhe(guest_ctxt); |
147 | ||
148 | __deactivate_traps(vcpu); | |
149 | ||
150 | sysreg_restore_host_state_vhe(host_ctxt); | |
151 | ||
152 | if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) | |
153 | __fpsimd_save_fpexc32(vcpu); | |
154 | ||
155 | __debug_switch_to_host(vcpu); | |
156 | ||
157 | return exit_code; | |
158 | } | |
159 | NOKPROBE_SYMBOL(__kvm_vcpu_run_vhe); | |
160 | ||
161 | int __kvm_vcpu_run(struct kvm_vcpu *vcpu) | |
162 | { | |
163 | int ret; | |
164 | ||
165 | local_daif_mask(); | |
166 | ||
167 | /* | |
168 | * Having IRQs masked via PMR when entering the guest means the GIC | |
169 | * will not signal the CPU of interrupts of lower priority, and the | |
170 | * only way to get out will be via guest exceptions. | |
171 | * Naturally, we want to avoid this. | |
172 | * | |
173 | * local_daif_mask() already sets GIC_PRIO_PSR_I_SET, we just need a | |
174 | * dsb to ensure the redistributor is forwards EL2 IRQs to the CPU. | |
175 | */ | |
176 | pmr_sync(); | |
177 | ||
178 | ret = __kvm_vcpu_run_vhe(vcpu); | |
179 | ||
180 | /* | |
181 | * local_daif_restore() takes care to properly restore PSTATE.DAIF | |
182 | * and the GIC PMR if the host is using IRQ priorities. | |
183 | */ | |
184 | local_daif_restore(DAIF_PROCCTX_NOIRQ); | |
185 | ||
186 | /* | |
187 | * When we exit from the guest we change a number of CPU configuration | |
188 | * parameters, such as traps. Make sure these changes take effect | |
189 | * before running the host or additional guests. | |
190 | */ | |
191 | isb(); | |
192 | ||
193 | return ret; | |
194 | } | |
195 | ||
6a0259ed | 196 | static void __hyp_call_panic(u64 spsr, u64 elr, u64 par) |
09cf57eb | 197 | { |
6a0259ed | 198 | struct kvm_cpu_context *host_ctxt; |
09cf57eb | 199 | struct kvm_vcpu *vcpu; |
6a0259ed | 200 | |
14ef9d04 | 201 | host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt; |
09cf57eb DB |
202 | vcpu = host_ctxt->__hyp_running_vcpu; |
203 | ||
204 | __deactivate_traps(vcpu); | |
205 | sysreg_restore_host_state_vhe(host_ctxt); | |
206 | ||
207 | panic(__hyp_panic_string, | |
208 | spsr, elr, | |
209 | read_sysreg_el2(SYS_ESR), read_sysreg_el2(SYS_FAR), | |
210 | read_sysreg(hpfar_el2), par, vcpu); | |
211 | } | |
212 | NOKPROBE_SYMBOL(__hyp_call_panic); | |
213 | ||
6a0259ed | 214 | void __noreturn hyp_panic(void) |
09cf57eb DB |
215 | { |
216 | u64 spsr = read_sysreg_el2(SYS_SPSR); | |
217 | u64 elr = read_sysreg_el2(SYS_ELR); | |
96d389ca | 218 | u64 par = read_sysreg_par(); |
09cf57eb | 219 | |
6a0259ed | 220 | __hyp_call_panic(spsr, elr, par); |
09cf57eb DB |
221 | unreachable(); |
222 | } | |
e9ee186b JM |
223 | |
224 | asmlinkage void kvm_unexpected_el2_exception(void) | |
225 | { | |
226 | return __kvm_unexpected_el2_exception(); | |
227 | } |