[linux-2.6-block.git] / arch / arm64 / kvm / hyp / switch.c

/*
 * Copyright (C) 2015 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/types.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>

static bool __hyp_text __fpsimd_enabled_nvhe(void)
{
	return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
}

static bool __hyp_text __fpsimd_enabled_vhe(void)
{
	return !!(read_sysreg(cpacr_el1) & CPACR_EL1_FPEN);
}

static hyp_alternate_select(__fpsimd_is_enabled,
			    __fpsimd_enabled_nvhe, __fpsimd_enabled_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

bool __hyp_text __fpsimd_enabled(void)
{
	return __fpsimd_is_enabled()();
}

static void __hyp_text __activate_traps_vhe(void)
{
	u64 val;

	val = read_sysreg(cpacr_el1);
	val |= CPACR_EL1_TTA;
	val &= ~CPACR_EL1_FPEN;
	write_sysreg(val, cpacr_el1);

	write_sysreg(__kvm_hyp_vector, vbar_el1);
}

static void __hyp_text __activate_traps_nvhe(void)
{
	u64 val;

	val = CPTR_EL2_DEFAULT;
	val |= CPTR_EL2_TTA | CPTR_EL2_TFP;
	write_sysreg(val, cptr_el2);
}

static hyp_alternate_select(__activate_traps_arch,
			    __activate_traps_nvhe, __activate_traps_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
{
	u64 val;

	/*
	 * We are about to set CPTR_EL2.TFP to trap all floating point
	 * register accesses to EL2, however, the ARM ARM clearly states that
	 * traps are only taken to EL2 if the operation would not otherwise
	 * trap to EL1.  Therefore, always make sure that for 32-bit guests,
	 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
	 */
	val = vcpu->arch.hcr_el2;
	if (!(val & HCR_RW)) {
		write_sysreg(1 << 30, fpexc32_el2);
		isb();
	}
	write_sysreg(val, hcr_el2);
	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
	write_sysreg(1 << 15, hstr_el2);
	/* Make sure we trap PMU access from EL0 to EL2 */
	write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
	__activate_traps_arch()();
}

static void __hyp_text __deactivate_traps_vhe(void)
{
	extern char vectors[];	/* kernel exception vectors */

	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
	write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
	write_sysreg(vectors, vbar_el1);
}

static void __hyp_text __deactivate_traps_nvhe(void)
{
	write_sysreg(HCR_RW, hcr_el2);
	write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
}

static hyp_alternate_select(__deactivate_traps_arch,
			    __deactivate_traps_nvhe, __deactivate_traps_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
{
	/*
	 * If we pended a virtual abort, preserve it until it gets
	 * cleared. See D1.14.3 (Virtual Interrupts) for details, but
	 * the crucial bit is "On taking a vSError interrupt,
	 * HCR_EL2.VSE is cleared to 0."
	 */
	if (vcpu->arch.hcr_el2 & HCR_VSE)
		vcpu->arch.hcr_el2 = read_sysreg(hcr_el2);

	__deactivate_traps_arch()();
	write_sysreg(0, hstr_el2);
	write_sysreg(read_sysreg(mdcr_el2) & MDCR_EL2_HPMN_MASK, mdcr_el2);
	write_sysreg(0, pmuserenr_el0);
}

static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
{
	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	write_sysreg(kvm->arch.vttbr, vttbr_el2);
}

static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
{
	write_sysreg(0, vttbr_el2);
}

static hyp_alternate_select(__vgic_call_save_state,
			    __vgic_v2_save_state, __vgic_v3_save_state,
			    ARM64_HAS_SYSREG_GIC_CPUIF);

static hyp_alternate_select(__vgic_call_restore_state,
			    __vgic_v2_restore_state, __vgic_v3_restore_state,
			    ARM64_HAS_SYSREG_GIC_CPUIF);

static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
{
	__vgic_call_save_state()(vcpu);
	write_sysreg(read_sysreg(hcr_el2) & ~HCR_INT_OVERRIDE, hcr_el2);
}

static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
{
	u64 val;

	val = read_sysreg(hcr_el2);
	val |= 	HCR_INT_OVERRIDE;
	val |= vcpu->arch.irq_lines;
	write_sysreg(val, hcr_el2);

	__vgic_call_restore_state()(vcpu);
}

static bool __hyp_text __true_value(void)
{
	return true;
}

static bool __hyp_text __false_value(void)
{
	return false;
}

static hyp_alternate_select(__check_arm_834220,
			    __false_value, __true_value,
			    ARM64_WORKAROUND_834220);

static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
{
	u64 par, tmp;

	/*
	 * Resolve the IPA the hard way using the guest VA.
	 *
	 * Stage-1 translation already validated the memory access
	 * rights. As such, we can use the EL1 translation regime, and
	 * don't have to distinguish between EL0 and EL1 access.
	 *
	 * We do need to save/restore PAR_EL1 though, as we haven't
	 * saved the guest context yet, and we may return early...
	 */
	par = read_sysreg(par_el1);
	asm volatile("at s1e1r, %0" : : "r" (far));
	isb();

	tmp = read_sysreg(par_el1);
	write_sysreg(par, par_el1);

	if (unlikely(tmp & 1))
		return false; /* Translation failed, back to guest */

	/* Convert PAR to HPFAR format */
	*hpfar = ((tmp >> 12) & ((1UL << 36) - 1)) << 4;
	return true;
}

static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
{
	u64 esr = read_sysreg_el2(esr);
	u8 ec = ESR_ELx_EC(esr);
	u64 hpfar, far;

	vcpu->arch.fault.esr_el2 = esr;

	if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
		return true;

	far = read_sysreg_el2(far);

	/*
	 * The HPFAR can be invalid if the stage 2 fault did not
	 * happen during a stage 1 page table walk (the ESR_EL2.S1PTW
	 * bit is clear) and one of the two following cases are true:
	 *   1. The fault was due to a permission fault
	 *   2. The processor carries errata 834220
	 *
	 * Therefore, for all non S1PTW faults where we either have a
	 * permission fault or the errata workaround is enabled, we
	 * resolve the IPA using the AT instruction.
	 */
	if (!(esr & ESR_ELx_S1PTW) &&
	    (__check_arm_834220()() || (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
		if (!__translate_far_to_hpfar(far, &hpfar))
			return false;
	} else {
		hpfar = read_sysreg(hpfar_el2);
	}

	vcpu->arch.fault.far_el2 = far;
	vcpu->arch.fault.hpfar_el2 = hpfar;
	return true;
}

static void __hyp_text __skip_instr(struct kvm_vcpu *vcpu)
{
	*vcpu_pc(vcpu) = read_sysreg_el2(elr);

	if (vcpu_mode_is_32bit(vcpu)) {
		vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(spsr);
		kvm_skip_instr32(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
		write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, spsr);
	} else {
		*vcpu_pc(vcpu) += 4;
	}

	write_sysreg_el2(*vcpu_pc(vcpu), elr);
}

int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
{
	struct kvm_cpu_context *host_ctxt;
	struct kvm_cpu_context *guest_ctxt;
	bool fp_enabled;
	u64 exit_code;

	vcpu = kern_hyp_va(vcpu);
	write_sysreg(vcpu, tpidr_el2);

	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
	guest_ctxt = &vcpu->arch.ctxt;

	__sysreg_save_host_state(host_ctxt);
	__debug_cond_save_host_state(vcpu);

	__activate_traps(vcpu);
	__activate_vm(vcpu);

	__vgic_restore_state(vcpu);
	__timer_restore_state(vcpu);

	/*
	 * We must restore the 32-bit state before the sysregs, thanks
	 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
	 */
	__sysreg32_restore_state(vcpu);
	__sysreg_restore_guest_state(guest_ctxt);
	__debug_restore_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);

	/* Jump in the fire! */
again:
	exit_code = __guest_enter(vcpu, host_ctxt);
	/* And we're baaack! */

	if (exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
		goto again;

	if (static_branch_unlikely(&vgic_v2_cpuif_trap) &&
	    exit_code == ARM_EXCEPTION_TRAP) {
		bool valid;

		valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
			kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&
			kvm_vcpu_dabt_isvalid(vcpu) &&
			!kvm_vcpu_dabt_isextabt(vcpu) &&
			!kvm_vcpu_dabt_iss1tw(vcpu);

		if (valid && __vgic_v2_perform_cpuif_access(vcpu)) {
			__skip_instr(vcpu);
			goto again;
		}
	}

	fp_enabled = __fpsimd_enabled();

	__sysreg_save_guest_state(guest_ctxt);
	__sysreg32_save_state(vcpu);
	__timer_save_state(vcpu);
	__vgic_save_state(vcpu);

	__deactivate_traps(vcpu);
	__deactivate_vm(vcpu);

	__sysreg_restore_host_state(host_ctxt);

	if (fp_enabled) {
		__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
		__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
	}

	__debug_save_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
	__debug_cond_restore_host_state(vcpu);

	return exit_code;
}

static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";

static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par)
{
	unsigned long str_va;

	/*
	 * Force the panic string to be loaded from the literal pool,
	 * making sure it is a kernel address and not a PC-relative
	 * reference.
	 */
	asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va));

	__hyp_do_panic(str_va,
		       spsr,  elr,
		       read_sysreg(esr_el2),   read_sysreg_el2(far),
		       read_sysreg(hpfar_el2), par,
		       (void *)read_sysreg(tpidr_el2));
}

static void __hyp_text __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par)
{
	panic(__hyp_panic_string,
	      spsr,  elr,
	      read_sysreg_el2(esr),   read_sysreg_el2(far),
	      read_sysreg(hpfar_el2), par,
	      (void *)read_sysreg(tpidr_el2));
}

static hyp_alternate_select(__hyp_call_panic,
			    __hyp_call_panic_nvhe, __hyp_call_panic_vhe,
			    ARM64_HAS_VIRT_HOST_EXTN);

void __hyp_text __noreturn __hyp_panic(void)
{
	u64 spsr = read_sysreg_el2(spsr);
	u64 elr = read_sysreg_el2(elr);
	u64 par = read_sysreg(par_el1);

	if (read_sysreg(vttbr_el2)) {
		struct kvm_vcpu *vcpu;
		struct kvm_cpu_context *host_ctxt;

		vcpu = (struct kvm_vcpu *)read_sysreg(tpidr_el2);
		host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
		__deactivate_traps(vcpu);
		__deactivate_vm(vcpu);
		__sysreg_restore_host_state(host_ctxt);
	}

	/* Call panic for real */
	__hyp_call_panic()(spsr, elr, par);

	unreachable();
}
Commit	Line	Data
be901e9b MZ	1	/*
	2	* Copyright (C) 2015 - ARM Ltd
	3	* Author: Marc Zyngier <marc.zyngier@arm.com>
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License version 2 as
	7	* published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful,
	10	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	* GNU General Public License for more details.
	13	*
	14	* You should have received a copy of the GNU General Public License
	15	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	16	*/
	17
5f05a72a	18	#include <linux/types.h>
68908bf7	19	#include <asm/kvm_asm.h>
fb5ee369	20	#include <asm/kvm_emulate.h>
13720a56	21	#include <asm/kvm_hyp.h>
be901e9b	22
32876224 MZ	23	static bool __hyp_text __fpsimd_enabled_nvhe(void)
	24	{
	25	return !(read_sysreg(cptr_el2) & CPTR_EL2_TFP);
	26	}
	27
	28	static bool __hyp_text __fpsimd_enabled_vhe(void)
	29	{
	30	return !!(read_sysreg(cpacr_el1) & CPACR_EL1_FPEN);
	31	}
	32
	33	static hyp_alternate_select(__fpsimd_is_enabled,
	34	__fpsimd_enabled_nvhe, __fpsimd_enabled_vhe,
	35	ARM64_HAS_VIRT_HOST_EXTN);
	36
	37	bool __hyp_text __fpsimd_enabled(void)
	38	{
	39	return __fpsimd_is_enabled()();
	40	}
	41
68908bf7 MZ	42	static void __hyp_text __activate_traps_vhe(void)
	43	{
	44	u64 val;
	45
	46	val = read_sysreg(cpacr_el1);
	47	val \|= CPACR_EL1_TTA;
	48	val &= ~CPACR_EL1_FPEN;
	49	write_sysreg(val, cpacr_el1);
	50
	51	write_sysreg(__kvm_hyp_vector, vbar_el1);
	52	}
	53
	54	static void __hyp_text __activate_traps_nvhe(void)
	55	{
	56	u64 val;
	57
	58	val = CPTR_EL2_DEFAULT;
	59	val \|= CPTR_EL2_TTA \| CPTR_EL2_TFP;
	60	write_sysreg(val, cptr_el2);
	61	}
	62
	63	static hyp_alternate_select(__activate_traps_arch,
	64	__activate_traps_nvhe, __activate_traps_vhe,
	65	ARM64_HAS_VIRT_HOST_EXTN);
	66
be901e9b MZ	67	static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
	68	{
	69	u64 val;
	70
	71	/*
	72	* We are about to set CPTR_EL2.TFP to trap all floating point
	73	* register accesses to EL2, however, the ARM ARM clearly states that
	74	* traps are only taken to EL2 if the operation would not otherwise
	75	* trap to EL1. Therefore, always make sure that for 32-bit guests,
	76	* we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
	77	*/
	78	val = vcpu->arch.hcr_el2;
	79	if (!(val & HCR_RW)) {
	80	write_sysreg(1 << 30, fpexc32_el2);
	81	isb();
	82	}
	83	write_sysreg(val, hcr_el2);
	84	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
	85	write_sysreg(1 << 15, hstr_el2);
d692b8ad SZ	86	/* Make sure we trap PMU access from EL0 to EL2 */
d692b8ad SZ	87	write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
68908bf7 MZ	88	write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
	89	__activate_traps_arch()();
	90	}
a7e0ac29	91
68908bf7 MZ	92	static void __hyp_text __deactivate_traps_vhe(void)
	93	{
	94	extern char vectors[]; /* kernel exception vectors */
a7e0ac29	95
68908bf7 MZ	96	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
	97	write_sysreg(CPACR_EL1_FPEN, cpacr_el1);
	98	write_sysreg(vectors, vbar_el1);
be901e9b MZ	99	}
be901e9b MZ	100
68908bf7	101	static void __hyp_text __deactivate_traps_nvhe(void)
be901e9b MZ	102	{
be901e9b MZ	103	write_sysreg(HCR_RW, hcr_el2);
68908bf7 MZ	104	write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
	105	}
	106
	107	static hyp_alternate_select(__deactivate_traps_arch,
	108	__deactivate_traps_nvhe, __deactivate_traps_vhe,
	109	ARM64_HAS_VIRT_HOST_EXTN);
	110
	111	static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
	112	{
44636f97 MZ	113	/*
	114	* If we pended a virtual abort, preserve it until it gets
	115	* cleared. See D1.14.3 (Virtual Interrupts) for details, but
	116	* the crucial bit is "On taking a vSError interrupt,
	117	* HCR_EL2.VSE is cleared to 0."
	118	*/
	119	if (vcpu->arch.hcr_el2 & HCR_VSE)
	120	vcpu->arch.hcr_el2 = read_sysreg(hcr_el2);
	121
68908bf7	122	__deactivate_traps_arch()();
be901e9b MZ	123	write_sysreg(0, hstr_el2);
be901e9b MZ	124	write_sysreg(read_sysreg(mdcr_el2) & MDCR_EL2_HPMN_MASK, mdcr_el2);
d692b8ad	125	write_sysreg(0, pmuserenr_el0);
be901e9b MZ	126	}
	127
	128	static void __hyp_text __activate_vm(struct kvm_vcpu *vcpu)
	129	{
	130	struct kvm *kvm = kern_hyp_va(vcpu->kvm);
	131	write_sysreg(kvm->arch.vttbr, vttbr_el2);
	132	}
	133
	134	static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
	135	{
	136	write_sysreg(0, vttbr_el2);
	137	}
	138
	139	static hyp_alternate_select(__vgic_call_save_state,
	140	__vgic_v2_save_state, __vgic_v3_save_state,
	141	ARM64_HAS_SYSREG_GIC_CPUIF);
	142
	143	static hyp_alternate_select(__vgic_call_restore_state,
	144	__vgic_v2_restore_state, __vgic_v3_restore_state,
	145	ARM64_HAS_SYSREG_GIC_CPUIF);
	146
	147	static void __hyp_text __vgic_save_state(struct kvm_vcpu *vcpu)
	148	{
	149	__vgic_call_save_state()(vcpu);
	150	write_sysreg(read_sysreg(hcr_el2) & ~HCR_INT_OVERRIDE, hcr_el2);
	151	}
	152
	153	static void __hyp_text __vgic_restore_state(struct kvm_vcpu *vcpu)
	154	{
	155	u64 val;
	156
	157	val = read_sysreg(hcr_el2);
	158	val \|= HCR_INT_OVERRIDE;
	159	val \|= vcpu->arch.irq_lines;
	160	write_sysreg(val, hcr_el2);
	161
	162	__vgic_call_restore_state()(vcpu);
	163	}
	164
5f05a72a MZ	165	static bool __hyp_text __true_value(void)
	166	{
	167	return true;
	168	}
	169
	170	static bool __hyp_text __false_value(void)
	171	{
	172	return false;
	173	}
	174
	175	static hyp_alternate_select(__check_arm_834220,
	176	__false_value, __true_value,
	177	ARM64_WORKAROUND_834220);
	178
	179	static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
	180	{
	181	u64 par, tmp;
	182
	183	/*
	184	* Resolve the IPA the hard way using the guest VA.
	185	*
	186	* Stage-1 translation already validated the memory access
	187	* rights. As such, we can use the EL1 translation regime, and
	188	* don't have to distinguish between EL0 and EL1 access.
	189	*
	190	* We do need to save/restore PAR_EL1 though, as we haven't
	191	* saved the guest context yet, and we may return early...
	192	*/
	193	par = read_sysreg(par_el1);
	194	asm volatile("at s1e1r, %0" : : "r" (far));
	195	isb();
	196
	197	tmp = read_sysreg(par_el1);
	198	write_sysreg(par, par_el1);
	199
	200	if (unlikely(tmp & 1))
	201	return false; /* Translation failed, back to guest */
	202
	203	/* Convert PAR to HPFAR format */
	204	*hpfar = ((tmp >> 12) & ((1UL << 36) - 1)) << 4;
	205	return true;
	206	}
	207
	208	static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
	209	{
	210	u64 esr = read_sysreg_el2(esr);
561454e2	211	u8 ec = ESR_ELx_EC(esr);
5f05a72a MZ	212	u64 hpfar, far;
	213
	214	vcpu->arch.fault.esr_el2 = esr;
	215
	216	if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
	217	return true;
	218
	219	far = read_sysreg_el2(far);
	220
	221	/*
	222	* The HPFAR can be invalid if the stage 2 fault did not
	223	* happen during a stage 1 page table walk (the ESR_EL2.S1PTW
	224	* bit is clear) and one of the two following cases are true:
	225	* 1. The fault was due to a permission fault
	226	* 2. The processor carries errata 834220
	227	*
	228	* Therefore, for all non S1PTW faults where we either have a
	229	* permission fault or the errata workaround is enabled, we
	230	* resolve the IPA using the AT instruction.
	231	*/
	232	if (!(esr & ESR_ELx_S1PTW) &&
	233	(__check_arm_834220()() \|\| (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
	234	if (!__translate_far_to_hpfar(far, &hpfar))
	235	return false;
	236	} else {
	237	hpfar = read_sysreg(hpfar_el2);
	238	}
	239
	240	vcpu->arch.fault.far_el2 = far;
	241	vcpu->arch.fault.hpfar_el2 = hpfar;
	242	return true;
	243	}
	244
fb5ee369 MZ	245	static void __hyp_text __skip_instr(struct kvm_vcpu *vcpu)
	246	{
	247	*vcpu_pc(vcpu) = read_sysreg_el2(elr);
	248
	249	if (vcpu_mode_is_32bit(vcpu)) {
	250	vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(spsr);
	251	kvm_skip_instr32(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
	252	write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, spsr);
	253	} else {
	254	*vcpu_pc(vcpu) += 4;
	255	}
	256
	257	write_sysreg_el2(*vcpu_pc(vcpu), elr);
	258	}
	259
cf0ba18a	260	int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
be901e9b MZ	261	{
	262	struct kvm_cpu_context *host_ctxt;
	263	struct kvm_cpu_context *guest_ctxt;
c13d1683	264	bool fp_enabled;
be901e9b MZ	265	u64 exit_code;
	266
	267	vcpu = kern_hyp_va(vcpu);
	268	write_sysreg(vcpu, tpidr_el2);
	269
	270	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
	271	guest_ctxt = &vcpu->arch.ctxt;
	272
edef528d	273	__sysreg_save_host_state(host_ctxt);
be901e9b MZ	274	__debug_cond_save_host_state(vcpu);
	275
	276	__activate_traps(vcpu);
	277	__activate_vm(vcpu);
	278
	279	__vgic_restore_state(vcpu);
	280	__timer_restore_state(vcpu);
	281
	282	/*
	283	* We must restore the 32-bit state before the sysregs, thanks
674e7012	284	* to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
be901e9b MZ	285	*/
be901e9b MZ	286	__sysreg32_restore_state(vcpu);
edef528d	287	__sysreg_restore_guest_state(guest_ctxt);
be901e9b MZ	288	__debug_restore_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
	289
	290	/* Jump in the fire! */
5f05a72a	291	again:
be901e9b MZ	292	exit_code = __guest_enter(vcpu, host_ctxt);
	293	/* And we're baaack! */
	294
5f05a72a MZ	295	if (exit_code == ARM_EXCEPTION_TRAP && !__populate_fault_info(vcpu))
	296	goto again;
	297
fb5ee369 MZ	298	if (static_branch_unlikely(&vgic_v2_cpuif_trap) &&
	299	exit_code == ARM_EXCEPTION_TRAP) {
	300	bool valid;
	301
	302	valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
	303	kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&
	304	kvm_vcpu_dabt_isvalid(vcpu) &&
	305	!kvm_vcpu_dabt_isextabt(vcpu) &&
	306	!kvm_vcpu_dabt_iss1tw(vcpu);
	307
	308	if (valid && __vgic_v2_perform_cpuif_access(vcpu)) {
	309	__skip_instr(vcpu);
	310	goto again;
	311	}
	312	}
	313
c13d1683 MZ	314	fp_enabled = __fpsimd_enabled();
c13d1683 MZ	315
edef528d	316	__sysreg_save_guest_state(guest_ctxt);
be901e9b MZ	317	__sysreg32_save_state(vcpu);
	318	__timer_save_state(vcpu);
	319	__vgic_save_state(vcpu);
	320
	321	__deactivate_traps(vcpu);
	322	__deactivate_vm(vcpu);
	323
edef528d	324	__sysreg_restore_host_state(host_ctxt);
be901e9b	325
c13d1683 MZ	326	if (fp_enabled) {
	327	__fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
	328	__fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
	329	}
	330
be901e9b MZ	331	__debug_save_state(vcpu, kern_hyp_va(vcpu->arch.debug_ptr), guest_ctxt);
	332	__debug_cond_restore_host_state(vcpu);
	333
	334	return exit_code;
	335	}
53fd5b64 MZ	336
	337	static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";
	338
253dcbd3	339	static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par)
53fd5b64	340	{
cf7df13d	341	unsigned long str_va;
253dcbd3	342
cf7df13d MZ	343	/*
	344	* Force the panic string to be loaded from the literal pool,
	345	* making sure it is a kernel address and not a PC-relative
	346	* reference.
	347	*/
	348	asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va));
	349
	350	__hyp_do_panic(str_va,
253dcbd3 MZ	351	spsr, elr,
	352	read_sysreg(esr_el2), read_sysreg_el2(far),
	353	read_sysreg(hpfar_el2), par,
	354	(void *)read_sysreg(tpidr_el2));
	355	}
	356
	357	static void __hyp_text __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par)
	358	{
	359	panic(__hyp_panic_string,
	360	spsr, elr,
	361	read_sysreg_el2(esr), read_sysreg_el2(far),
	362	read_sysreg(hpfar_el2), par,
	363	(void *)read_sysreg(tpidr_el2));
	364	}
	365
	366	static hyp_alternate_select(__hyp_call_panic,
	367	__hyp_call_panic_nvhe, __hyp_call_panic_vhe,
	368	ARM64_HAS_VIRT_HOST_EXTN);
	369
	370	void __hyp_text __noreturn __hyp_panic(void)
	371	{
	372	u64 spsr = read_sysreg_el2(spsr);
	373	u64 elr = read_sysreg_el2(elr);
53fd5b64 MZ	374	u64 par = read_sysreg(par_el1);
	375
	376	if (read_sysreg(vttbr_el2)) {
	377	struct kvm_vcpu *vcpu;
	378	struct kvm_cpu_context *host_ctxt;
	379
	380	vcpu = (struct kvm_vcpu *)read_sysreg(tpidr_el2);
	381	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
	382	__deactivate_traps(vcpu);
	383	__deactivate_vm(vcpu);
edef528d	384	__sysreg_restore_host_state(host_ctxt);
53fd5b64 MZ	385	}
	386
	387	/* Call panic for real */
253dcbd3	388	__hyp_call_panic()(spsr, elr, par);
53fd5b64 MZ	389
	390	unreachable();
	391	}