2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 #include <linux/linkage.h>
20 #include <linux/const.h>
21 #include <asm/unified.h>
23 #include <asm/ptrace.h>
24 #include <asm/asm-offsets.h>
25 #include <asm/kvm_asm.h>
26 #include <asm/kvm_arm.h>
27 #include <asm/vfpmacros.h>
28 #include "interrupts_head.S"
31 .pushsection .hyp.text, "ax"
33 /********************************************************************
36 * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
38 * We rely on the hardware to broadcast the TLB invalidation to all CPUs
39 * inside the inner-shareable domain (which is the case for all v7
40 * implementations). If we come across a non-IS SMP implementation, we'll
41 * have to use an IPI based mechanism. Until then, we stick to the simple
42 * hardware assisted version.
44 * As v7 does not support flushing per IPA, just nuke the whole TLB
45 * instead, ignoring the ipa value.
47 ENTRY(__kvm_tlb_flush_vmid_ipa)
51 add r0, r0, #KVM_VTTBR
53 mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR
55 mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
60 mcrr p15, 6, r2, r3, c2 @ Back to VMID #0
61 isb @ Not necessary if followed by eret
65 ENDPROC(__kvm_tlb_flush_vmid_ipa)
68 * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs
70 * Reuses __kvm_tlb_flush_vmid_ipa() for ARMv7, without passing address
74 ENTRY(__kvm_tlb_flush_vmid)
75 b __kvm_tlb_flush_vmid_ipa
76 ENDPROC(__kvm_tlb_flush_vmid)
78 /********************************************************************
79 * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
80 * domain, for all VMIDs
82 * void __kvm_flush_vm_context(void);
84 ENTRY(__kvm_flush_vm_context)
85 mov r0, #0 @ rn parameter for c15 flushes is SBZ
87 /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
88 mcr p15, 4, r0, c8, c3, 4
89 /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
90 mcr p15, 0, r0, c7, c1, 0
92 isb @ Not necessary if followed by eret
95 ENDPROC(__kvm_flush_vm_context)
98 /********************************************************************
99 * Hypervisor world-switch code
102 * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
104 ENTRY(__kvm_vcpu_run)
105 @ Save the vcpu pointer
106 mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR
113 @ Store hardware CP15 state and load guest state
114 read_cp15_state store_to_vcpu = 0
115 write_cp15_state read_from_vcpu = 1
117 @ If the host kernel has not been configured with VFPv3 support,
118 @ then it is safer if we deny guests from using it as well.
120 @ Set FPEXC_EN so the guest doesn't trap floating point instructions
121 VFPFMRX r2, FPEXC @ VMRS
123 orr r2, r2, #FPEXC_EN
124 VFPFMXR FPEXC, r2 @ VMSR
128 configure_hyp_role vmentry
130 @ Trap coprocessor CRx accesses
132 set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
135 @ Write configured ID register into MIDR alias
136 ldr r1, [vcpu, #VCPU_MIDR]
137 mcr p15, 4, r1, c0, c0, 0
139 @ Write guest view of MPIDR into VMPIDR
140 ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
141 mcr p15, 4, r1, c0, c0, 5
143 @ Set up guest memory translation
144 ldr r1, [vcpu, #VCPU_KVM]
145 add r1, r1, #KVM_VTTBR
147 mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR
149 @ We're all done, just restore the GPRs and go to the guest
151 clrex @ Clear exclusive monitor
157 * guest r0, r1, r2 saved on the stack
166 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
168 @ Don't trap coprocessor accesses for host kernel
171 set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)), after_vfp_restore
174 @ Switch VFP/NEON hardware state to the host's
175 add r7, vcpu, #VCPU_VFP_GUEST
177 add r7, vcpu, #VCPU_VFP_HOST
182 @ Restore FPEXC_EN which we clobbered on entry
190 configure_hyp_role vmexit
192 @ Let host read hardware MIDR
193 mrc p15, 0, r2, c0, c0, 0
194 mcr p15, 4, r2, c0, c0, 0
196 @ Back to hardware MPIDR
197 mrc p15, 0, r2, c0, c0, 5
198 mcr p15, 4, r2, c0, c0, 5
200 @ Store guest CP15 state and restore host state
201 read_cp15_state store_to_vcpu = 1
202 write_cp15_state read_from_vcpu = 0
208 clrex @ Clear exclusive monitor
209 #ifndef CONFIG_CPU_ENDIAN_BE8
210 mov r0, r1 @ Return the return code
211 mov r1, #0 @ Clear upper bits in return value
213 @ r1 already has return code
214 mov r0, #0 @ Clear upper bits in return value
215 #endif /* CONFIG_CPU_ENDIAN_BE8 */
216 bx lr @ return to IOCTL
218 /********************************************************************
219 * Call function in Hyp mode
222 * u64 kvm_call_hyp(void *hypfn, ...);
224 * This is not really a variadic function in the classic C-way and care must
225 * be taken when calling this to ensure parameters are passed in registers
226 * only, since the stack will change between the caller and the callee.
228 * Call the function with the first argument containing a pointer to the
229 * function you wish to call in Hyp mode, and subsequent arguments will be
230 * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
231 * function pointer can be passed). The function being called must be mapped
232 * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
233 * passed in r0 and r1.
235 * A function pointer with a value of 0xffffffff has a special meaning,
236 * and is used to implement __hyp_get_vectors in the same way as in
237 * arch/arm/kernel/hyp_stub.S.
239 * The calling convention follows the standard AAPCS:
240 * r0 - r3: caller save
248 /********************************************************************
249 * Hypervisor exception vector and handlers
252 * The KVM/ARM Hypervisor ABI is defined as follows:
254 * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
255 * instruction is issued since all traps are disabled when running the host
256 * kernel as per the Hyp-mode initialization at boot time.
258 * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
259 * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
260 * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
261 * instructions are called from within Hyp-mode.
263 * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
264 * Switching to Hyp mode is done through a simple HVC #0 instruction. The
265 * exception vector code will check that the HVC comes from VMID==0 and if
266 * so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
267 * - r0 contains a pointer to a HYP function
268 * - r1, r2, and r3 contain arguments to the above function.
269 * - The HYP function will be called with its arguments in r0, r1 and r2.
270 * On HYP function return, we return directly to SVC.
272 * Note that the above is used to execute code in Hyp-mode from a host-kernel
273 * point of view, and is a different concept from performing a world-switch and
274 * executing guest code SVC mode (with a VMID != 0).
277 /* Handle undef, svc, pabt, or dabt by crashing with a user notice */
278 .macro bad_exception exception_code, panic_str
280 mrrc p15, 6, r0, r1, c2 @ Read VTTBR
285 load_vcpu @ Load VCPU pointer
286 .if \exception_code == ARM_EXCEPTION_DATA_ABORT
287 mrc p15, 4, r2, c5, c2, 0 @ HSR
288 mrc p15, 4, r1, c6, c0, 0 @ HDFAR
289 str r2, [vcpu, #VCPU_HSR]
290 str r1, [vcpu, #VCPU_HxFAR]
292 .if \exception_code == ARM_EXCEPTION_PREF_ABORT
293 mrc p15, 4, r2, c5, c2, 0 @ HSR
294 mrc p15, 4, r1, c6, c0, 2 @ HIFAR
295 str r2, [vcpu, #VCPU_HSR]
296 str r1, [vcpu, #VCPU_HxFAR]
298 mov r1, #\exception_code
301 @ We were in the host already. Let's craft a panic-ing return to SVC.
303 bic r2, r2, #MODE_MASK
304 orr r2, r2, #SVC_MODE
305 THUMB( orr r2, r2, #PSR_T_BIT )
311 clrex @ Clear exclusive monitor
317 .globl __kvm_hyp_vector
319 @ Hyp-mode exception vector
335 bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
339 bad_exception ARM_EXCEPTION_HVC, svc_die_str
343 bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
347 bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
352 * Getting here is either becuase of a trap from a guest or from calling
353 * HVC from the host kernel, which means "switch to Hyp mode".
357 @ Check syndrome register
358 mrc p15, 4, r1, c5, c2, 0 @ HSR
359 lsr r0, r1, #HSR_EC_SHIFT
361 bne guest_trap @ Not HVC instr.
364 * Let's check if the HVC came from VMID 0 and allow simple
367 mrrc p15, 6, r0, r2, c2
371 bne guest_trap @ Guest called HVC
374 * Getting here means host called HVC, we shift parameters and branch
379 /* Check for __hyp_get_vectors */
381 mrceq p15, 4, r0, c12, c0, 0 @ get HVBAR
394 blx lr @ Call the HYP function
402 load_vcpu @ Load VCPU pointer to r0
403 str r1, [vcpu, #VCPU_HSR]
405 @ Check if we need the fault information
406 lsr r1, r1, #HSR_EC_SHIFT
408 cmp r1, #HSR_EC_CP_0_13
409 beq switch_to_guest_vfp
412 mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
416 mrc p15, 4, r2, c6, c0, 0 @ HDFAR
418 2: str r2, [vcpu, #VCPU_HxFAR]
421 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
423 * Abort on the stage 2 translation for a memory access from a
424 * Non-secure PL1 or PL0 mode:
426 * For any Access flag fault or Translation fault, and also for any
427 * Permission fault on the stage 2 translation of a memory access
428 * made as part of a translation table walk for a stage 1 translation,
429 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
433 /* Check for permission fault, and S1PTW */
434 mrc p15, 4, r1, c5, c2, 0 @ HSR
435 and r0, r1, #HSR_FSC_TYPE
437 tsteq r1, #(1 << 7) @ S1PTW
438 mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
442 mrrc p15, 0, r0, r1, c7 @ PAR
445 /* Resolve IPA using the xFAR */
446 mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
448 mrrc p15, 0, r0, r1, c7 @ PAR
450 bne 4f @ Failed translation
451 ubfx r2, r0, #12, #20
453 orr r2, r2, r1, lsl #24
457 mcrr p15, 0, r0, r1, c7 @ PAR
459 3: load_vcpu @ Load VCPU pointer to r0
460 str r2, [r0, #VCPU_HPFAR]
462 1: mov r1, #ARM_EXCEPTION_HVC
465 4: pop {r0, r1} @ Failed translation, return to guest
466 mcrr p15, 0, r0, r1, c7 @ PAR
472 * If VFPv3 support is not available, then we will not switch the VFP
473 * registers; however cp10 and cp11 accesses will still trap and fallback
474 * to the regular coprocessor emulation code, which currently will
475 * inject an undefined exception to the guest.
481 @ NEON/VFP used. Turn on VFP access.
482 set_hcptr vmtrap, (HCPTR_TCP(10) | HCPTR_TCP(11))
484 @ Switch VFP/NEON hardware state to the guest's
485 add r7, r0, #VCPU_VFP_HOST
488 add r7, r0, #VCPU_VFP_GUEST
500 mov r1, #ARM_EXCEPTION_IRQ
501 load_vcpu @ Load VCPU pointer to r0
512 .pushsection ".rodata"
515 .ascii "unexpected undefined exception in Hyp mode at: %#08x\n"
517 .ascii "unexpected prefetch abort in Hyp mode at: %#08x\n"
519 .ascii "unexpected data abort in Hyp mode at: %#08x\n"
521 .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x\n"