[linux-2.6-block.git] / arch / arm / kvm / hyp / hyp-entry.S

/*
 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License, version 2, as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */

#include <linux/arm-smccc.h>
#include <linux/linkage.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>

	.arch_extension     virt

	.text
	.pushsection	.hyp.text, "ax"

.macro load_vcpu	reg
	mrc	p15, 4, \reg, c13, c0, 2	@ HTPIDR
.endm

/********************************************************************
 * Hypervisor exception vector and handlers
 *
 *
 * The KVM/ARM Hypervisor ABI is defined as follows:
 *
 * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
 * instruction is issued since all traps are disabled when running the host
 * kernel as per the Hyp-mode initialization at boot time.
 *
 * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
 * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
 * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
 * instructions are called from within Hyp-mode.
 *
 * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
 *    Switching to Hyp mode is done through a simple HVC #0 instruction. The
 *    exception vector code will check that the HVC comes from VMID==0.
 *    - r0 contains a pointer to a HYP function
 *    - r1, r2, and r3 contain arguments to the above function.
 *    - The HYP function will be called with its arguments in r0, r1 and r2.
 *    On HYP function return, we return directly to SVC.
 *
 * Note that the above is used to execute code in Hyp-mode from a host-kernel
 * point of view, and is a different concept from performing a world-switch and
 * executing guest code SVC mode (with a VMID != 0).
 */

	.align 5
__kvm_hyp_vector:
	.global __kvm_hyp_vector

	@ Hyp-mode exception vector
	W(b)	hyp_reset
	W(b)	hyp_undef
	W(b)	hyp_svc
	W(b)	hyp_pabt
	W(b)	hyp_dabt
	W(b)	hyp_hvc
	W(b)	hyp_irq
	W(b)	hyp_fiq

#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
	.align 5
__kvm_hyp_vector_ic_inv:
	.global __kvm_hyp_vector_ic_inv

	/*
	 * We encode the exception entry in the bottom 3 bits of
	 * SP, and we have to guarantee to be 8 bytes aligned.
	 */
	W(add)	sp, sp, #1	/* Reset 	  7 */
	W(add)	sp, sp, #1	/* Undef	  6 */
	W(add)	sp, sp, #1	/* Syscall	  5 */
	W(add)	sp, sp, #1	/* Prefetch abort 4 */
	W(add)	sp, sp, #1	/* Data abort	  3 */
	W(add)	sp, sp, #1	/* HVC		  2 */
	W(add)	sp, sp, #1	/* IRQ		  1 */
	W(nop)			/* FIQ		  0 */

	mcr	p15, 0, r0, c7, c5, 0	/* ICIALLU */
	isb

	b	decode_vectors

	.align 5
__kvm_hyp_vector_bp_inv:
	.global __kvm_hyp_vector_bp_inv

	/*
	 * We encode the exception entry in the bottom 3 bits of
	 * SP, and we have to guarantee to be 8 bytes aligned.
	 */
	W(add)	sp, sp, #1	/* Reset 	  7 */
	W(add)	sp, sp, #1	/* Undef	  6 */
	W(add)	sp, sp, #1	/* Syscall	  5 */
	W(add)	sp, sp, #1	/* Prefetch abort 4 */
	W(add)	sp, sp, #1	/* Data abort	  3 */
	W(add)	sp, sp, #1	/* HVC		  2 */
	W(add)	sp, sp, #1	/* IRQ		  1 */
	W(nop)			/* FIQ		  0 */

	mcr	p15, 0, r0, c7, c5, 6	/* BPIALL */
	isb

decode_vectors:

#ifdef CONFIG_THUMB2_KERNEL
	/*
	 * Yet another silly hack: Use VPIDR as a temp register.
	 * Thumb2 is really a pain, as SP cannot be used with most
	 * of the bitwise instructions. The vect_br macro ensures
	 * things gets cleaned-up.
	 */
	mcr	p15, 4, r0, c0, c0, 0	/* VPIDR */
	mov	r0, sp
	and	r0, r0, #7
	sub	sp, sp, r0
	push	{r1, r2}
	mov	r1, r0
	mrc	p15, 4, r0, c0, c0, 0	/* VPIDR */
	mrc	p15, 0, r2, c0, c0, 0	/* MIDR  */
	mcr	p15, 4, r2, c0, c0, 0	/* VPIDR */
#endif

.macro vect_br val, targ
ARM(	eor	sp, sp, #\val	)
ARM(	tst	sp, #7		)
ARM(	eorne	sp, sp, #\val	)

THUMB(	cmp	r1, #\val	)
THUMB(	popeq	{r1, r2}	)

	beq	\targ
.endm

	vect_br	0, hyp_fiq
	vect_br	1, hyp_irq
	vect_br	2, hyp_hvc
	vect_br	3, hyp_dabt
	vect_br	4, hyp_pabt
	vect_br	5, hyp_svc
	vect_br	6, hyp_undef
	vect_br	7, hyp_reset
#endif

.macro invalid_vector label, cause
	.align
\label:	mov	r0, #\cause
	b	__hyp_panic
.endm

	invalid_vector	hyp_reset	ARM_EXCEPTION_RESET
	invalid_vector	hyp_undef	ARM_EXCEPTION_UNDEFINED
	invalid_vector	hyp_svc		ARM_EXCEPTION_SOFTWARE
	invalid_vector	hyp_pabt	ARM_EXCEPTION_PREF_ABORT
	invalid_vector	hyp_fiq		ARM_EXCEPTION_FIQ

ENTRY(__hyp_do_panic)
	mrs	lr, cpsr
	bic	lr, lr, #MODE_MASK
	orr	lr, lr, #SVC_MODE
THUMB(	orr	lr, lr, #PSR_T_BIT	)
	msr	spsr_cxsf, lr
	ldr	lr, =panic
	msr	ELR_hyp, lr
	ldr	lr, =__kvm_call_hyp
	clrex
	eret
ENDPROC(__hyp_do_panic)

hyp_hvc:
	/*
	 * Getting here is either because of a trap from a guest,
	 * or from executing HVC from the host kernel, which means
	 * "do something in Hyp mode".
	 */
	push	{r0, r1, r2}

	@ Check syndrome register
	mrc	p15, 4, r1, c5, c2, 0	@ HSR
	lsr	r0, r1, #HSR_EC_SHIFT
	cmp	r0, #HSR_EC_HVC
	bne	guest_trap		@ Not HVC instr.

	/*
	 * Let's check if the HVC came from VMID 0 and allow simple
	 * switch to Hyp mode
	 */
	mrrc    p15, 6, r0, r2, c2
	lsr     r2, r2, #16
	and     r2, r2, #0xff
	cmp     r2, #0
	bne	guest_hvc_trap		@ Guest called HVC

	/*
	 * Getting here means host called HVC, we shift parameters and branch
	 * to Hyp function.
	 */
	pop	{r0, r1, r2}

	/*
	 * Check if we have a kernel function, which is guaranteed to be
	 * bigger than the maximum hyp stub hypercall
	 */
	cmp	r0, #HVC_STUB_HCALL_NR
	bhs	1f

	/*
	 * Not a kernel function, treat it as a stub hypercall.
	 * Compute the physical address for __kvm_handle_stub_hvc
	 * (as the code lives in the idmaped page) and branch there.
	 * We hijack ip (r12) as a tmp register.
	 */
	push	{r1}
	ldr	r1, =kimage_voffset
	ldr	r1, [r1]
	ldr	ip, =__kvm_handle_stub_hvc
	sub	ip, ip, r1
	pop	{r1}

	bx	ip

1:
	/*
	 * Pushing r2 here is just a way of keeping the stack aligned to
	 * 8 bytes on any path that can trigger a HYP exception. Here,
	 * we may well be about to jump into the guest, and the guest
	 * exit would otherwise be badly decoded by our fancy
	 * "decode-exception-without-a-branch" code...
	 */
	push	{r2, lr}

	mov	lr, r0
	mov	r0, r1
	mov	r1, r2
	mov	r2, r3

THUMB(	orr	lr, #1)
	blx	lr			@ Call the HYP function

	pop	{r2, lr}
	eret

guest_hvc_trap:
	movw	r2, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
	movt	r2, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
	ldr	r0, [sp]		@ Guest's r0
	teq	r0, r2
	bne	guest_trap
	add	sp, sp, #12
	@ Returns:
	@ r0 = 0
	@ r1 = HSR value (perfectly predictable)
	@ r2 = ARM_SMCCC_ARCH_WORKAROUND_1
	mov	r0, #0
	eret

guest_trap:
	load_vcpu r0			@ Load VCPU pointer to r0

#ifdef CONFIG_VFPv3
	@ Check for a VFP access
	lsr	r1, r1, #HSR_EC_SHIFT
	cmp	r1, #HSR_EC_CP_0_13
	beq	__vfp_guest_restore
#endif

	mov	r1, #ARM_EXCEPTION_HVC
	b	__guest_exit

hyp_irq:
	push	{r0, r1, r2}
	mov	r1, #ARM_EXCEPTION_IRQ
	load_vcpu r0			@ Load VCPU pointer to r0
	b	__guest_exit

hyp_dabt:
	push	{r0, r1}
	mrs	r0, ELR_hyp
	ldr	r1, =abort_guest_exit_start
THUMB(	add	r1, r1, #1)
	cmp	r0, r1
	ldrne	r1, =abort_guest_exit_end
THUMB(	addne	r1, r1, #1)
	cmpne	r0, r1
	pop	{r0, r1}
	bne	__hyp_panic

	orr	r0, r0, #(1 << ARM_EXIT_WITH_ABORT_BIT)
	eret

	.ltorg

	.popsection
Commit	Line	Data
bafc6c2a MZ	1	/*
	2	* Copyright (C) 2012 - Virtual Open Systems and Columbia University
	3	* Author: Christoffer Dall <c.dall@virtualopensystems.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License, version 2, as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program; if not, write to the Free Software
	16	* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
	17	*/
	18
b800acfc	19	#include <linux/arm-smccc.h>
bafc6c2a MZ	20	#include <linux/linkage.h>
	21	#include <asm/kvm_arm.h>
	22	#include <asm/kvm_asm.h>
	23
	24	.arch_extension virt
	25
	26	.text
	27	.pushsection .hyp.text, "ax"
	28
	29	.macro load_vcpu reg
	30	mrc p15, 4, \reg, c13, c0, 2 @ HTPIDR
	31	.endm
	32
	33	/********************************************************************
	34	* Hypervisor exception vector and handlers
	35	*
	36	*
	37	* The KVM/ARM Hypervisor ABI is defined as follows:
	38	*
	39	* Entry to Hyp mode from the host kernel will happen _only_ when an HVC
	40	* instruction is issued since all traps are disabled when running the host
	41	* kernel as per the Hyp-mode initialization at boot time.
	42	*
	43	* HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
	44	* below) when the HVC instruction is called from SVC mode (i.e. a guest or the
	45	* host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
	46	* instructions are called from within Hyp-mode.
	47	*
	48	* Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
	49	* Switching to Hyp mode is done through a simple HVC #0 instruction. The
	50	* exception vector code will check that the HVC comes from VMID==0.
	51	* - r0 contains a pointer to a HYP function
	52	* - r1, r2, and r3 contain arguments to the above function.
	53	* - The HYP function will be called with its arguments in r0, r1 and r2.
	54	* On HYP function return, we return directly to SVC.
	55	*
	56	* Note that the above is used to execute code in Hyp-mode from a host-kernel
	57	* point of view, and is a different concept from performing a world-switch and
	58	* executing guest code SVC mode (with a VMID != 0).
	59	*/
	60
	61	.align 5
bafc6c2a	62	__kvm_hyp_vector:
fa85e25d	63	.global __kvm_hyp_vector
bafc6c2a MZ	64
	65	@ Hyp-mode exception vector
	66	W(b) hyp_reset
	67	W(b) hyp_undef
	68	W(b) hyp_svc
	69	W(b) hyp_pabt
	70	W(b) hyp_dabt
	71	W(b) hyp_hvc
	72	W(b) hyp_irq
	73	W(b) hyp_fiq
	74
3f7e8e2e	75	#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
0c47ac8c MZ	76	.align 5
	77	__kvm_hyp_vector_ic_inv:
	78	.global __kvm_hyp_vector_ic_inv
	79
	80	/*
	81	* We encode the exception entry in the bottom 3 bits of
	82	* SP, and we have to guarantee to be 8 bytes aligned.
	83	*/
	84	W(add) sp, sp, #1 /* Reset 7 */
	85	W(add) sp, sp, #1 /* Undef 6 */
	86	W(add) sp, sp, #1 /* Syscall 5 */
	87	W(add) sp, sp, #1 /* Prefetch abort 4 */
	88	W(add) sp, sp, #1 /* Data abort 3 */
	89	W(add) sp, sp, #1 /* HVC 2 */
	90	W(add) sp, sp, #1 /* IRQ 1 */
	91	W(nop) /* FIQ 0 */
	92
	93	mcr p15, 0, r0, c7, c5, 0 /* ICIALLU */
	94	isb
	95
	96	b decode_vectors
	97
3f7e8e2e MZ	98	.align 5
	99	__kvm_hyp_vector_bp_inv:
	100	.global __kvm_hyp_vector_bp_inv
	101
	102	/*
	103	* We encode the exception entry in the bottom 3 bits of
	104	* SP, and we have to guarantee to be 8 bytes aligned.
	105	*/
	106	W(add) sp, sp, #1 /* Reset 7 */
	107	W(add) sp, sp, #1 /* Undef 6 */
	108	W(add) sp, sp, #1 /* Syscall 5 */
	109	W(add) sp, sp, #1 /* Prefetch abort 4 */
	110	W(add) sp, sp, #1 /* Data abort 3 */
	111	W(add) sp, sp, #1 /* HVC 2 */
	112	W(add) sp, sp, #1 /* IRQ 1 */
	113	W(nop) /* FIQ 0 */
	114
	115	mcr p15, 0, r0, c7, c5, 6 /* BPIALL */
	116	isb
	117
0c47ac8c MZ	118	decode_vectors:
0c47ac8c MZ	119
3f7e8e2e MZ	120	#ifdef CONFIG_THUMB2_KERNEL
	121	/*
	122	* Yet another silly hack: Use VPIDR as a temp register.
	123	* Thumb2 is really a pain, as SP cannot be used with most
	124	* of the bitwise instructions. The vect_br macro ensures
	125	* things gets cleaned-up.
	126	*/
	127	mcr p15, 4, r0, c0, c0, 0 /* VPIDR */
	128	mov r0, sp
	129	and r0, r0, #7
	130	sub sp, sp, r0
	131	push {r1, r2}
	132	mov r1, r0
	133	mrc p15, 4, r0, c0, c0, 0 /* VPIDR */
	134	mrc p15, 0, r2, c0, c0, 0 /* MIDR */
	135	mcr p15, 4, r2, c0, c0, 0 /* VPIDR */
	136	#endif
	137
	138	.macro vect_br val, targ
	139	ARM( eor sp, sp, #\val )
	140	ARM( tst sp, #7 )
	141	ARM( eorne sp, sp, #\val )
	142
	143	THUMB( cmp r1, #\val )
	144	THUMB( popeq {r1, r2} )
	145
	146	beq \targ
	147	.endm
	148
	149	vect_br 0, hyp_fiq
	150	vect_br 1, hyp_irq
	151	vect_br 2, hyp_hvc
	152	vect_br 3, hyp_dabt
	153	vect_br 4, hyp_pabt
	154	vect_br 5, hyp_svc
	155	vect_br 6, hyp_undef
	156	vect_br 7, hyp_reset
	157	#endif
	158
bafc6c2a MZ	159	.macro invalid_vector label, cause
bafc6c2a MZ	160	.align
c36b6db5 MZ	161	\label: mov r0, #\cause
c36b6db5 MZ	162	b __hyp_panic
bafc6c2a MZ	163	.endm
bafc6c2a MZ	164
c36b6db5 MZ	165	invalid_vector hyp_reset ARM_EXCEPTION_RESET
	166	invalid_vector hyp_undef ARM_EXCEPTION_UNDEFINED
	167	invalid_vector hyp_svc ARM_EXCEPTION_SOFTWARE
	168	invalid_vector hyp_pabt ARM_EXCEPTION_PREF_ABORT
c36b6db5 MZ	169	invalid_vector hyp_fiq ARM_EXCEPTION_FIQ
	170
	171	ENTRY(__hyp_do_panic)
	172	mrs lr, cpsr
	173	bic lr, lr, #MODE_MASK
	174	orr lr, lr, #SVC_MODE
	175	THUMB( orr lr, lr, #PSR_T_BIT )
	176	msr spsr_cxsf, lr
	177	ldr lr, =panic
	178	msr ELR_hyp, lr
d18232ea	179	ldr lr, =__kvm_call_hyp
c36b6db5 MZ	180	clrex
	181	eret
	182	ENDPROC(__hyp_do_panic)
bafc6c2a MZ	183
	184	hyp_hvc:
	185	/*
	186	* Getting here is either because of a trap from a guest,
	187	* or from executing HVC from the host kernel, which means
	188	* "do something in Hyp mode".
	189	*/
	190	push {r0, r1, r2}
	191
	192	@ Check syndrome register
	193	mrc p15, 4, r1, c5, c2, 0 @ HSR
	194	lsr r0, r1, #HSR_EC_SHIFT
	195	cmp r0, #HSR_EC_HVC
	196	bne guest_trap @ Not HVC instr.
	197
	198	/*
	199	* Let's check if the HVC came from VMID 0 and allow simple
	200	* switch to Hyp mode
	201	*/
	202	mrrc p15, 6, r0, r2, c2
	203	lsr r2, r2, #16
	204	and r2, r2, #0xff
	205	cmp r2, #0
b800acfc	206	bne guest_hvc_trap @ Guest called HVC
bafc6c2a MZ	207
	208	/*
	209	* Getting here means host called HVC, we shift parameters and branch
	210	* to Hyp function.
	211	*/
	212	pop {r0, r1, r2}
	213
6bebcecb MZ	214	/*
	215	* Check if we have a kernel function, which is guaranteed to be
	216	* bigger than the maximum hyp stub hypercall
	217	*/
	218	cmp r0, #HVC_STUB_HCALL_NR
	219	bhs 1f
bafc6c2a	220
6bebcecb MZ	221	/*
	222	* Not a kernel function, treat it as a stub hypercall.
	223	* Compute the physical address for __kvm_handle_stub_hvc
	224	* (as the code lives in the idmaped page) and branch there.
	225	* We hijack ip (r12) as a tmp register.
	226	*/
	227	push {r1}
	228	ldr r1, =kimage_voffset
	229	ldr r1, [r1]
	230	ldr ip, =__kvm_handle_stub_hvc
	231	sub ip, ip, r1
6bebcecb MZ	232	pop {r1}
	233
	234	bx ip
	235
	236	1:
3f7e8e2e MZ	237	/*
	238	* Pushing r2 here is just a way of keeping the stack aligned to
	239	* 8 bytes on any path that can trigger a HYP exception. Here,
	240	* we may well be about to jump into the guest, and the guest
	241	* exit would otherwise be badly decoded by our fancy
	242	* "decode-exception-without-a-branch" code...
	243	*/
	244	push {r2, lr}
bafc6c2a MZ	245
	246	mov lr, r0
	247	mov r0, r1
	248	mov r1, r2
	249	mov r2, r3
	250
	251	THUMB( orr lr, #1)
	252	blx lr @ Call the HYP function
	253
3f7e8e2e	254	pop {r2, lr}
6bebcecb	255	eret
bafc6c2a	256
b800acfc RK	257	guest_hvc_trap:
	258	movw r2, #:lower16:ARM_SMCCC_ARCH_WORKAROUND_1
	259	movt r2, #:upper16:ARM_SMCCC_ARCH_WORKAROUND_1
	260	ldr r0, [sp] @ Guest's r0
	261	teq r0, r2
	262	bne guest_trap
	263	add sp, sp, #12
	264	@ Returns:
	265	@ r0 = 0
	266	@ r1 = HSR value (perfectly predictable)
	267	@ r2 = ARM_SMCCC_ARCH_WORKAROUND_1
	268	mov r0, #0
	269	eret
	270
bafc6c2a MZ	271	guest_trap:
	272	load_vcpu r0 @ Load VCPU pointer to r0
	273
	274	#ifdef CONFIG_VFPv3
	275	@ Check for a VFP access
	276	lsr r1, r1, #HSR_EC_SHIFT
	277	cmp r1, #HSR_EC_CP_0_13
	278	beq __vfp_guest_restore
	279	#endif
	280
	281	mov r1, #ARM_EXCEPTION_HVC
	282	b __guest_exit
	283
	284	hyp_irq:
	285	push {r0, r1, r2}
	286	mov r1, #ARM_EXCEPTION_IRQ
	287	load_vcpu r0 @ Load VCPU pointer to r0
	288	b __guest_exit
	289
c39798f4 MZ	290	hyp_dabt:
	291	push {r0, r1}
	292	mrs r0, ELR_hyp
	293	ldr r1, =abort_guest_exit_start
	294	THUMB( add r1, r1, #1)
	295	cmp r0, r1
	296	ldrne r1, =abort_guest_exit_end
	297	THUMB( addne r1, r1, #1)
	298	cmpne r0, r1
	299	pop {r0, r1}
	300	bne __hyp_panic
	301
	302	orr r0, r0, #(1 << ARM_EXIT_WITH_ABORT_BIT)
	303	eret
	304
bafc6c2a MZ	305	.ltorg
	306
	307	.popsection