Commit | Line | Data |
---|---|---|
f3601156 NP |
1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | #include <asm/asm-offsets.h> | |
3 | #include <asm/cache.h> | |
89d35b23 | 4 | #include <asm/code-patching-asm.h> |
f33e0702 | 5 | #include <asm/exception-64s.h> |
89d35b23 | 6 | #include <asm/export.h> |
f3601156 NP |
7 | #include <asm/kvm_asm.h> |
8 | #include <asm/kvm_book3s_asm.h> | |
89d35b23 | 9 | #include <asm/mmu.h> |
f3601156 | 10 | #include <asm/ppc_asm.h> |
89d35b23 | 11 | #include <asm/ptrace.h> |
f3601156 | 12 | #include <asm/reg.h> |
89d35b23 | 13 | #include <asm/ultravisor-api.h> |
f3601156 NP |
14 | |
15 | /* | |
31c67cfe | 16 | * These are branched to from interrupt handlers in exception-64s.S which set |
f3601156 NP |
17 | * IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero. |
18 | */ | |
e2762743 NP |
19 | |
20 | /* | |
21 | * This is a hcall, so register convention is as | |
22 | * Documentation/powerpc/papr_hcalls.rst. | |
23 | * | |
24 | * This may also be a syscall from PR-KVM userspace that is to be | |
25 | * reflected to the PR guest kernel, so registers may be set up for | |
26 | * a system call rather than hcall. We don't currently clobber | |
27 | * anything here, but the 0xc00 handler has already clobbered CTR | |
28 | * and CR0, so PR-KVM can not support a guest kernel that preserves | |
29 | * those registers across its system calls. | |
30 | * | |
31 | * The state of registers is as kvmppc_interrupt, except CFAR is not | |
32 | * saved, R13 is not in SCRATCH0, and R10 does not contain the trap. | |
33 | */ | |
31c67cfe NP |
34 | .global kvmppc_hcall |
35 | .balign IFETCH_ALIGN_BYTES | |
36 | kvmppc_hcall: | |
89d35b23 NP |
37 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
38 | lbz r10,HSTATE_IN_GUEST(r13) | |
0bf7e1b2 | 39 | cmpwi r10,KVM_GUEST_MODE_HV_P9 |
89d35b23 NP |
40 | beq kvmppc_p9_exit_hcall |
41 | #endif | |
e2762743 NP |
42 | ld r10,PACA_EXGEN+EX_R13(r13) |
43 | SET_SCRATCH0(r10) | |
44 | li r10,0xc00 | |
45 | /* Now we look like kvmppc_interrupt */ | |
46 | li r11,PACA_EXGEN | |
47 | b .Lgot_save_area | |
31c67cfe | 48 | |
69fdd674 NP |
49 | /* |
50 | * KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that | |
51 | * call convention: | |
52 | * | |
53 | * guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area | |
54 | * guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts | |
55 | * guest R13 also saved in SCRATCH0 | |
56 | * R13 = PACA | |
57 | * R11 = (H)SRR0 | |
58 | * R12 = (H)SRR1 | |
59 | * R9 = guest CR | |
60 | * PPR is set to medium | |
61 | * | |
62 | * With the addition for KVM: | |
63 | * R10 = trap vector | |
64 | */ | |
f3601156 NP |
65 | .global kvmppc_interrupt |
66 | .balign IFETCH_ALIGN_BYTES | |
67 | kvmppc_interrupt: | |
89d35b23 NP |
68 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE |
69 | std r10,HSTATE_SCRATCH0(r13) | |
70 | lbz r10,HSTATE_IN_GUEST(r13) | |
0bf7e1b2 | 71 | cmpwi r10,KVM_GUEST_MODE_HV_P9 |
89d35b23 NP |
72 | beq kvmppc_p9_exit_interrupt |
73 | ld r10,HSTATE_SCRATCH0(r13) | |
74 | #endif | |
69fdd674 NP |
75 | li r11,PACA_EXGEN |
76 | cmpdi r10,0x200 | |
e2762743 | 77 | bgt+ .Lgot_save_area |
69fdd674 | 78 | li r11,PACA_EXMC |
e2762743 | 79 | beq .Lgot_save_area |
69fdd674 | 80 | li r11,PACA_EXNMI |
e2762743 NP |
81 | .Lgot_save_area: |
82 | add r11,r11,r13 | |
69fdd674 NP |
83 | BEGIN_FTR_SECTION |
84 | ld r12,EX_CFAR(r11) | |
85 | std r12,HSTATE_CFAR(r13) | |
86 | END_FTR_SECTION_IFSET(CPU_FTR_CFAR) | |
87 | ld r12,EX_CTR(r11) | |
88 | mtctr r12 | |
89 | BEGIN_FTR_SECTION | |
90 | ld r12,EX_PPR(r11) | |
91 | std r12,HSTATE_PPR(r13) | |
92 | END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) | |
93 | ld r12,EX_R12(r11) | |
94 | std r12,HSTATE_SCRATCH0(r13) | |
95 | sldi r12,r9,32 | |
96 | or r12,r12,r10 | |
97 | ld r9,EX_R9(r11) | |
98 | ld r10,EX_R10(r11) | |
99 | ld r11,EX_R11(r11) | |
100 | ||
f3601156 | 101 | /* |
69fdd674 NP |
102 | * Hcalls and other interrupts come here after normalising register |
103 | * contents and save locations: | |
104 | * | |
f3601156 NP |
105 | * R12 = (guest CR << 32) | interrupt vector |
106 | * R13 = PACA | |
69fdd674 | 107 | * guest R12 saved in shadow HSTATE_SCRATCH0 |
f3601156 NP |
108 | * guest R13 saved in SPRN_SCRATCH0 |
109 | */ | |
f3601156 NP |
110 | std r9,HSTATE_SCRATCH2(r13) |
111 | lbz r9,HSTATE_IN_GUEST(r13) | |
f33e0702 NP |
112 | cmpwi r9,KVM_GUEST_MODE_SKIP |
113 | beq- .Lmaybe_skip | |
114 | .Lno_skip: | |
115 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE | |
f3601156 NP |
116 | #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE |
117 | cmpwi r9,KVM_GUEST_MODE_GUEST | |
f3601156 NP |
118 | beq kvmppc_interrupt_pr |
119 | #endif | |
120 | b kvmppc_interrupt_hv | |
121 | #else | |
122 | b kvmppc_interrupt_pr | |
123 | #endif | |
f33e0702 NP |
124 | |
125 | /* | |
126 | * "Skip" interrupts are part of a trick KVM uses a with hash guests to load | |
127 | * the faulting instruction in guest memory from the the hypervisor without | |
128 | * walking page tables. | |
129 | * | |
130 | * When the guest takes a fault that requires the hypervisor to load the | |
131 | * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1 | |
132 | * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets | |
133 | * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions | |
134 | * but loads and stores will access the guest context. This is used to load | |
135 | * the faulting instruction using the faulting guest effective address. | |
136 | * | |
137 | * However the guest context may not be able to translate, or it may cause a | |
138 | * machine check or other issue, which results in a fault in the host | |
139 | * (even with KVM-HV). | |
140 | * | |
141 | * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they | |
142 | * are (or are likely) caused by that load, the instruction is skipped by | |
143 | * just returning with the PC advanced +4, where it is noticed the load did | |
144 | * not execute and it goes to the slow path which walks the page tables to | |
145 | * read guest memory. | |
146 | */ | |
147 | .Lmaybe_skip: | |
148 | cmpwi r12,BOOK3S_INTERRUPT_MACHINE_CHECK | |
149 | beq 1f | |
150 | cmpwi r12,BOOK3S_INTERRUPT_DATA_STORAGE | |
151 | beq 1f | |
152 | cmpwi r12,BOOK3S_INTERRUPT_DATA_SEGMENT | |
153 | beq 1f | |
154 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE | |
155 | /* HSRR interrupts get 2 added to interrupt number */ | |
156 | cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2 | |
157 | beq 2f | |
158 | #endif | |
159 | b .Lno_skip | |
160 | 1: mfspr r9,SPRN_SRR0 | |
161 | addi r9,r9,4 | |
162 | mtspr SPRN_SRR0,r9 | |
163 | ld r12,HSTATE_SCRATCH0(r13) | |
164 | ld r9,HSTATE_SCRATCH2(r13) | |
165 | GET_SCRATCH0(r13) | |
166 | RFI_TO_KERNEL | |
167 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE | |
168 | 2: mfspr r9,SPRN_HSRR0 | |
169 | addi r9,r9,4 | |
170 | mtspr SPRN_HSRR0,r9 | |
171 | ld r12,HSTATE_SCRATCH0(r13) | |
172 | ld r9,HSTATE_SCRATCH2(r13) | |
173 | GET_SCRATCH0(r13) | |
174 | HRFI_TO_KERNEL | |
175 | #endif | |
89d35b23 NP |
176 | |
177 | #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE | |
178 | ||
179 | /* Stack frame offsets for kvmppc_p9_enter_guest */ | |
180 | #define SFS (144 + STACK_FRAME_MIN_SIZE) | |
181 | #define STACK_SLOT_NVGPRS (SFS - 144) /* 18 gprs */ | |
182 | ||
183 | /* | |
184 | * void kvmppc_p9_enter_guest(struct vcpu *vcpu); | |
185 | * | |
0bf7e1b2 | 186 | * Enter the guest on a ISAv3.0 or later system. |
89d35b23 NP |
187 | */ |
188 | .balign IFETCH_ALIGN_BYTES | |
189 | _GLOBAL(kvmppc_p9_enter_guest) | |
190 | EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest) | |
191 | mflr r0 | |
192 | std r0,PPC_LR_STKOFF(r1) | |
193 | stdu r1,-SFS(r1) | |
194 | ||
195 | std r1,HSTATE_HOST_R1(r13) | |
196 | ||
197 | mfcr r4 | |
198 | stw r4,SFS+8(r1) | |
199 | ||
200 | reg = 14 | |
201 | .rept 18 | |
202 | std reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) | |
203 | reg = reg + 1 | |
204 | .endr | |
205 | ||
206 | ld r4,VCPU_LR(r3) | |
207 | mtlr r4 | |
208 | ld r4,VCPU_CTR(r3) | |
209 | mtctr r4 | |
210 | ld r4,VCPU_XER(r3) | |
211 | mtspr SPRN_XER,r4 | |
212 | ||
213 | ld r1,VCPU_CR(r3) | |
214 | ||
215 | BEGIN_FTR_SECTION | |
216 | ld r4,VCPU_CFAR(r3) | |
217 | mtspr SPRN_CFAR,r4 | |
218 | END_FTR_SECTION_IFSET(CPU_FTR_CFAR) | |
219 | BEGIN_FTR_SECTION | |
220 | ld r4,VCPU_PPR(r3) | |
221 | mtspr SPRN_PPR,r4 | |
222 | END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) | |
223 | ||
224 | reg = 4 | |
225 | .rept 28 | |
226 | ld reg,__VCPU_GPR(reg)(r3) | |
227 | reg = reg + 1 | |
228 | .endr | |
229 | ||
230 | ld r4,VCPU_KVM(r3) | |
231 | lbz r4,KVM_SECURE_GUEST(r4) | |
232 | cmpdi r4,0 | |
233 | ld r4,VCPU_GPR(R4)(r3) | |
234 | bne .Lret_to_ultra | |
235 | ||
236 | mtcr r1 | |
237 | ||
238 | ld r0,VCPU_GPR(R0)(r3) | |
239 | ld r1,VCPU_GPR(R1)(r3) | |
240 | ld r2,VCPU_GPR(R2)(r3) | |
241 | ld r3,VCPU_GPR(R3)(r3) | |
242 | ||
243 | HRFI_TO_GUEST | |
244 | b . | |
245 | ||
246 | /* | |
247 | * Use UV_RETURN ultracall to return control back to the Ultravisor | |
248 | * after processing an hypercall or interrupt that was forwarded | |
249 | * (a.k.a. reflected) to the Hypervisor. | |
250 | * | |
251 | * All registers have already been reloaded except the ucall requires: | |
252 | * R0 = hcall result | |
253 | * R2 = SRR1, so UV can detect a synthesized interrupt (if any) | |
254 | * R3 = UV_RETURN | |
255 | */ | |
256 | .Lret_to_ultra: | |
257 | mtcr r1 | |
258 | ld r1,VCPU_GPR(R1)(r3) | |
259 | ||
260 | ld r0,VCPU_GPR(R3)(r3) | |
261 | mfspr r2,SPRN_SRR1 | |
262 | LOAD_REG_IMMEDIATE(r3, UV_RETURN) | |
263 | sc 2 | |
264 | ||
265 | /* | |
266 | * kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from | |
267 | * above if the interrupt was taken for a guest that was entered via | |
268 | * kvmppc_p9_enter_guest(). | |
269 | * | |
270 | * The exit code recovers the host stack and vcpu pointer, saves all guest GPRs | |
271 | * and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re- | |
272 | * establishes the host stack and registers to return from the | |
273 | * kvmppc_p9_enter_guest() function, which saves CTR and other guest registers | |
274 | * (SPRs and FP, VEC, etc). | |
275 | */ | |
276 | .balign IFETCH_ALIGN_BYTES | |
277 | kvmppc_p9_exit_hcall: | |
278 | mfspr r11,SPRN_SRR0 | |
279 | mfspr r12,SPRN_SRR1 | |
280 | li r10,0xc00 | |
281 | std r10,HSTATE_SCRATCH0(r13) | |
282 | ||
283 | .balign IFETCH_ALIGN_BYTES | |
284 | kvmppc_p9_exit_interrupt: | |
285 | /* | |
0bf7e1b2 | 286 | * If set to KVM_GUEST_MODE_HV_P9 but we're still in the |
89d35b23 NP |
287 | * hypervisor, that means we can't return from the entry stack. |
288 | */ | |
289 | rldicl. r10,r12,64-MSR_HV_LG,63 | |
290 | bne- kvmppc_p9_bad_interrupt | |
291 | ||
292 | std r1,HSTATE_SCRATCH1(r13) | |
293 | std r3,HSTATE_SCRATCH2(r13) | |
294 | ld r1,HSTATE_HOST_R1(r13) | |
295 | ld r3,HSTATE_KVM_VCPU(r13) | |
296 | ||
297 | std r9,VCPU_CR(r3) | |
298 | ||
299 | 1: | |
300 | std r11,VCPU_PC(r3) | |
301 | std r12,VCPU_MSR(r3) | |
302 | ||
303 | reg = 14 | |
304 | .rept 18 | |
305 | std reg,__VCPU_GPR(reg)(r3) | |
306 | reg = reg + 1 | |
307 | .endr | |
308 | ||
309 | /* r1, r3, r9-r13 are saved to vcpu by C code */ | |
310 | std r0,VCPU_GPR(R0)(r3) | |
311 | std r2,VCPU_GPR(R2)(r3) | |
312 | reg = 4 | |
313 | .rept 5 | |
314 | std reg,__VCPU_GPR(reg)(r3) | |
315 | reg = reg + 1 | |
316 | .endr | |
317 | ||
318 | ld r2,PACATOC(r13) | |
319 | ||
320 | mflr r4 | |
321 | std r4,VCPU_LR(r3) | |
322 | mfspr r4,SPRN_XER | |
323 | std r4,VCPU_XER(r3) | |
324 | ||
325 | reg = 14 | |
326 | .rept 18 | |
327 | ld reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) | |
328 | reg = reg + 1 | |
329 | .endr | |
330 | ||
331 | lwz r4,SFS+8(r1) | |
332 | mtcr r4 | |
333 | ||
334 | /* | |
335 | * Flush the link stack here, before executing the first blr on the | |
336 | * way out of the guest. | |
337 | * | |
338 | * The link stack won't match coming out of the guest anyway so the | |
339 | * only cost is the flush itself. The call clobbers r0. | |
340 | */ | |
341 | 1: nop | |
342 | patch_site 1b patch__call_kvm_flush_link_stack_p9 | |
343 | ||
344 | addi r1,r1,SFS | |
345 | ld r0,PPC_LR_STKOFF(r1) | |
346 | mtlr r0 | |
347 | blr | |
348 | ||
349 | /* | |
350 | * Took an interrupt somewhere right before HRFID to guest, so registers are | |
351 | * in a bad way. Return things hopefully enough to run host virtual code and | |
352 | * run the Linux interrupt handler (SRESET or MCE) to print something useful. | |
353 | * | |
354 | * We could be really clever and save all host registers in known locations | |
355 | * before setting HSTATE_IN_GUEST, then restoring them all here, and setting | |
356 | * return address to a fixup that sets them up again. But that's a lot of | |
357 | * effort for a small bit of code. Lots of other things to do first. | |
358 | */ | |
359 | kvmppc_p9_bad_interrupt: | |
0bf7e1b2 NP |
360 | BEGIN_MMU_FTR_SECTION |
361 | /* | |
362 | * Hash host doesn't try to recover MMU (requires host SLB reload) | |
363 | */ | |
364 | b . | |
365 | END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) | |
89d35b23 NP |
366 | /* |
367 | * Clean up guest registers to give host a chance to run. | |
368 | */ | |
369 | li r10,0 | |
370 | mtspr SPRN_AMR,r10 | |
371 | mtspr SPRN_IAMR,r10 | |
372 | mtspr SPRN_CIABR,r10 | |
373 | mtspr SPRN_DAWRX0,r10 | |
374 | BEGIN_FTR_SECTION | |
375 | mtspr SPRN_DAWRX1,r10 | |
376 | END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) | |
89d35b23 NP |
377 | |
378 | /* | |
cf3b16cf NP |
379 | * Switch to host MMU mode (don't have the real host PID but we aren't |
380 | * going back to userspace). | |
89d35b23 | 381 | */ |
cf3b16cf NP |
382 | hwsync |
383 | isync | |
384 | ||
385 | mtspr SPRN_PID,r10 | |
386 | ||
89d35b23 NP |
387 | ld r10, HSTATE_KVM_VCPU(r13) |
388 | ld r10, VCPU_KVM(r10) | |
389 | lwz r10, KVM_HOST_LPID(r10) | |
390 | mtspr SPRN_LPID,r10 | |
391 | ||
392 | ld r10, HSTATE_KVM_VCPU(r13) | |
393 | ld r10, VCPU_KVM(r10) | |
394 | ld r10, KVM_HOST_LPCR(r10) | |
395 | mtspr SPRN_LPCR,r10 | |
396 | ||
cf3b16cf NP |
397 | isync |
398 | ||
89d35b23 NP |
399 | /* |
400 | * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear | |
401 | * MSR_RI in r12 ([H]SRR1) so the handler won't try to return. | |
402 | */ | |
403 | li r10,KVM_GUEST_MODE_NONE | |
404 | stb r10,HSTATE_IN_GUEST(r13) | |
405 | li r10,MSR_RI | |
406 | andc r12,r12,r10 | |
407 | ||
408 | /* | |
409 | * Go back to interrupt handler. MCE and SRESET have their specific | |
410 | * PACA save area so they should be used directly. They set up their | |
411 | * own stack. The other handlers all use EXGEN. They will use the | |
412 | * guest r1 if it looks like a kernel stack, so just load the | |
413 | * emergency stack and go to program check for all other interrupts. | |
414 | */ | |
415 | ld r10,HSTATE_SCRATCH0(r13) | |
416 | cmpwi r10,BOOK3S_INTERRUPT_MACHINE_CHECK | |
af41d286 | 417 | beq .Lcall_machine_check_common |
89d35b23 NP |
418 | |
419 | cmpwi r10,BOOK3S_INTERRUPT_SYSTEM_RESET | |
af41d286 | 420 | beq .Lcall_system_reset_common |
89d35b23 NP |
421 | |
422 | b . | |
af41d286 CL |
423 | |
424 | .Lcall_machine_check_common: | |
425 | b machine_check_common | |
426 | ||
427 | .Lcall_system_reset_common: | |
428 | b system_reset_common | |
89d35b23 | 429 | #endif |