[linux-2.6-block.git] / arch / arm64 / kvm / fpsimd.c

// SPDX-License-Identifier: GPL-2.0
/*
 * arch/arm64/kvm/fpsimd.c: Guest/host FPSIMD context coordination helpers
 *
 * Copyright 2018 Arm Limited
 * Author: Dave Martin <Dave.Martin@arm.com>
 */
#include <linux/irqflags.h>
#include <linux/sched.h>
#include <linux/kvm_host.h>
#include <asm/fpsimd.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <asm/sysreg.h>

void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu)
{
	struct task_struct *p = vcpu->arch.parent_task;
	struct user_fpsimd_state *fpsimd;

	if (!is_protected_kvm_enabled() || !p)
		return;

	fpsimd = &p->thread.uw.fpsimd_state;
	kvm_unshare_hyp(fpsimd, fpsimd + 1);
	put_task_struct(p);
}

/*
 * Called on entry to KVM_RUN unless this vcpu previously ran at least
 * once and the most recent prior KVM_RUN for this vcpu was called from
 * the same task as current (highly likely).
 *
 * This is guaranteed to execute before kvm_arch_vcpu_load_fp(vcpu),
 * such that on entering hyp the relevant parts of current are already
 * mapped.
 */
int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu)
{
	int ret;

	struct user_fpsimd_state *fpsimd = &current->thread.uw.fpsimd_state;

	kvm_vcpu_unshare_task_fp(vcpu);

	/* Make sure the host task fpsimd state is visible to hyp: */
	ret = kvm_share_hyp(fpsimd, fpsimd + 1);
	if (ret)
		return ret;

	vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);

	/*
	 * We need to keep current's task_struct pinned until its data has been
	 * unshared with the hypervisor to make sure it is not re-used by the
	 * kernel and donated to someone else while already shared -- see
	 * kvm_vcpu_unshare_task_fp() for the matching put_task_struct().
	 */
	if (is_protected_kvm_enabled()) {
		get_task_struct(current);
		vcpu->arch.parent_task = current;
	}

	return 0;
}

/*
 * Prepare vcpu for saving the host's FPSIMD state and loading the guest's.
 * The actual loading is done by the FPSIMD access trap taken to hyp.
 *
 * Here, we just set the correct metadata to indicate that the FPSIMD
 * state in the cpu regs (if any) belongs to current on the host.
 */
void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
{
	BUG_ON(!current->mm);
	BUG_ON(test_thread_flag(TIF_SVE));

	vcpu->arch.flags &= ~KVM_ARM64_FP_ENABLED;
	vcpu->arch.flags |= KVM_ARM64_FP_HOST;

	vcpu->arch.flags &= ~KVM_ARM64_HOST_SVE_ENABLED;
	if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN)
		vcpu->arch.flags |= KVM_ARM64_HOST_SVE_ENABLED;

	/*
	 * We don't currently support SME guests but if we leave
	 * things in streaming mode then when the guest starts running
	 * FPSIMD or SVE code it may generate SME traps so as a
	 * special case if we are in streaming mode we force the host
	 * state to be saved now and exit streaming mode so that we
	 * don't have to handle any SME traps for valid guest
	 * operations. Do this for ZA as well for now for simplicity.
	 */
	if (system_supports_sme()) {
		vcpu->arch.flags &= ~KVM_ARM64_HOST_SME_ENABLED;
		if (read_sysreg(cpacr_el1) & CPACR_EL1_SMEN_EL0EN)
			vcpu->arch.flags |= KVM_ARM64_HOST_SME_ENABLED;

		if (read_sysreg_s(SYS_SVCR) &
		    (SVCR_SM_MASK | SVCR_ZA_MASK)) {
			vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
			fpsimd_save_and_flush_cpu_state();
		}
	}
}

/*
 * Called just before entering the guest once we are no longer
 * preemptable. Syncs the host's TIF_FOREIGN_FPSTATE with the KVM
 * mirror of the flag used by the hypervisor.
 */
void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu)
{
	if (test_thread_flag(TIF_FOREIGN_FPSTATE))
		vcpu->arch.flags |= KVM_ARM64_FP_FOREIGN_FPSTATE;
	else
		vcpu->arch.flags &= ~KVM_ARM64_FP_FOREIGN_FPSTATE;
}

/*
 * Called just after exiting the guest. If the guest FPSIMD state
 * was loaded, update the host's context tracking data mark the CPU
 * FPSIMD regs as dirty and belonging to vcpu so that they will be
 * written back if the kernel clobbers them due to kernel-mode NEON
 * before re-entry into the guest.
 */
void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
{
	WARN_ON_ONCE(!irqs_disabled());

	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
		/*
		 * Currently we do not support SME guests so SVCR is
		 * always 0 and we just need a variable to point to.
		 */
		fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.fp_regs,
					 vcpu->arch.sve_state,
					 vcpu->arch.sve_max_vl,
					 NULL, 0, &vcpu->arch.svcr);

		clear_thread_flag(TIF_FOREIGN_FPSTATE);
		update_thread_flag(TIF_SVE, vcpu_has_sve(vcpu));
	}
}

/*
 * Write back the vcpu FPSIMD regs if they are dirty, and invalidate the
 * cpu FPSIMD regs so that they can't be spuriously reused if this vcpu
 * disappears and another task or vcpu appears that recycles the same
 * struct fpsimd_state.
 */
void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
{
	unsigned long flags;

	local_irq_save(flags);

	/*
	 * If we have VHE then the Hyp code will reset CPACR_EL1 to
	 * CPACR_EL1_DEFAULT and we need to reenable SME.
	 */
	if (has_vhe() && system_supports_sme()) {
		/* Also restore EL0 state seen on entry */
		if (vcpu->arch.flags & KVM_ARM64_HOST_SME_ENABLED)
			sysreg_clear_set(CPACR_EL1, 0,
					 CPACR_EL1_SMEN_EL0EN |
					 CPACR_EL1_SMEN_EL1EN);
		else
			sysreg_clear_set(CPACR_EL1,
					 CPACR_EL1_SMEN_EL0EN,
					 CPACR_EL1_SMEN_EL1EN);
	}

	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
		if (vcpu_has_sve(vcpu)) {
			__vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);

			/* Restore the VL that was saved when bound to the CPU */
			if (!has_vhe())
				sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1,
						       SYS_ZCR_EL1);
		}

		fpsimd_save_and_flush_cpu_state();
	} else if (has_vhe() && system_supports_sve()) {
		/*
		 * The FPSIMD/SVE state in the CPU has not been touched, and we
		 * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been
		 * reset to CPACR_EL1_DEFAULT by the Hyp code, disabling SVE
		 * for EL0.  To avoid spurious traps, restore the trap state
		 * seen by kvm_arch_vcpu_load_fp():
		 */
		if (vcpu->arch.flags & KVM_ARM64_HOST_SVE_ENABLED)
			sysreg_clear_set(CPACR_EL1, 0, CPACR_EL1_ZEN_EL0EN);
		else
			sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0);
	}

	update_thread_flag(TIF_SVE, 0);

	local_irq_restore(flags);
}
Commit	Line	Data
e6b673b7 DM	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* arch/arm64/kvm/fpsimd.c: Guest/host FPSIMD context coordination helpers
	4	*
	5	* Copyright 2018 Arm Limited
	6	* Author: Dave Martin <Dave.Martin@arm.com>
	7	*/
b045e4d0	8	#include <linux/irqflags.h>
e6b673b7	9	#include <linux/sched.h>
e6b673b7	10	#include <linux/kvm_host.h>
04950674	11	#include <asm/fpsimd.h>
e6b673b7	12	#include <asm/kvm_asm.h>
83857371	13	#include <asm/kvm_hyp.h>
e6b673b7	14	#include <asm/kvm_mmu.h>
b3eb56b6	15	#include <asm/sysreg.h>
e6b673b7	16
52b28657 QP	17	void kvm_vcpu_unshare_task_fp(struct kvm_vcpu *vcpu)
	18	{
	19	struct task_struct *p = vcpu->arch.parent_task;
	20	struct user_fpsimd_state *fpsimd;
	21
	22	if (!is_protected_kvm_enabled() \|\| !p)
	23	return;
	24
	25	fpsimd = &p->thread.uw.fpsimd_state;
	26	kvm_unshare_hyp(fpsimd, fpsimd + 1);
	27	put_task_struct(p);
	28	}
	29
e6b673b7 DM	30	/*
	31	* Called on entry to KVM_RUN unless this vcpu previously ran at least
	32	* once and the most recent prior KVM_RUN for this vcpu was called from
	33	* the same task as current (highly likely).
	34	*
	35	* This is guaranteed to execute before kvm_arch_vcpu_load_fp(vcpu),
	36	* such that on entering hyp the relevant parts of current are already
	37	* mapped.
	38	*/
	39	int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu)
	40	{
	41	int ret;
	42
e6b673b7 DM	43	struct user_fpsimd_state *fpsimd = &current->thread.uw.fpsimd_state;
e6b673b7 DM	44
52b28657 QP	45	kvm_vcpu_unshare_task_fp(vcpu);
52b28657 QP	46
bee14bca	47	/* Make sure the host task fpsimd state is visible to hyp: */
3f868e14	48	ret = kvm_share_hyp(fpsimd, fpsimd + 1);
52b28657 QP	49	if (ret)
	50	return ret;
	51
	52	vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);
	53
	54	/*
	55	* We need to keep current's task_struct pinned until its data has been
	56	* unshared with the hypervisor to make sure it is not re-used by the
	57	* kernel and donated to someone else while already shared -- see
	58	* kvm_vcpu_unshare_task_fp() for the matching put_task_struct().
	59	*/
	60	if (is_protected_kvm_enabled()) {
	61	get_task_struct(current);
	62	vcpu->arch.parent_task = current;
	63	}
e6b673b7	64
52b28657	65	return 0;
e6b673b7 DM	66	}
	67
	68	/*
	69	* Prepare vcpu for saving the host's FPSIMD state and loading the guest's.
	70	* The actual loading is done by the FPSIMD access trap taken to hyp.
	71	*
	72	* Here, we just set the correct metadata to indicate that the FPSIMD
	73	* state in the cpu regs (if any) belongs to current on the host.
e6b673b7 DM	74	*/
	75	void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
	76	{
e6b673b7	77	BUG_ON(!current->mm);
8383741a	78	BUG_ON(test_thread_flag(TIF_SVE));
e6b673b7	79
8383741a	80	vcpu->arch.flags &= ~KVM_ARM64_FP_ENABLED;
e6b673b7	81	vcpu->arch.flags \|= KVM_ARM64_FP_HOST;
b3eb56b6	82
d52d165d	83	vcpu->arch.flags &= ~KVM_ARM64_HOST_SVE_ENABLED;
b3eb56b6 DM	84	if (read_sysreg(cpacr_el1) & CPACR_EL1_ZEN_EL0EN)
b3eb56b6 DM	85	vcpu->arch.flags \|= KVM_ARM64_HOST_SVE_ENABLED;
861262ab MB	86
	87	/*
	88	* We don't currently support SME guests but if we leave
	89	* things in streaming mode then when the guest starts running
	90	* FPSIMD or SVE code it may generate SME traps so as a
	91	* special case if we are in streaming mode we force the host
	92	* state to be saved now and exit streaming mode so that we
	93	* don't have to handle any SME traps for valid guest
	94	* operations. Do this for ZA as well for now for simplicity.
	95	*/
	96	if (system_supports_sme()) {
039f49c4	97	vcpu->arch.flags &= ~KVM_ARM64_HOST_SME_ENABLED;
861262ab MB	98	if (read_sysreg(cpacr_el1) & CPACR_EL1_SMEN_EL0EN)
	99	vcpu->arch.flags \|= KVM_ARM64_HOST_SME_ENABLED;
	100
ec0067a6 MB	101	if (read_sysreg_s(SYS_SVCR) &
ec0067a6 MB	102	(SVCR_SM_MASK \| SVCR_ZA_MASK)) {
861262ab MB	103	vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
	104	fpsimd_save_and_flush_cpu_state();
	105	}
	106	}
e6b673b7 DM	107	}
e6b673b7 DM	108
23afc825 MB	109	/*
	110	* Called just before entering the guest once we are no longer
	111	* preemptable. Syncs the host's TIF_FOREIGN_FPSTATE with the KVM
	112	* mirror of the flag used by the hypervisor.
	113	*/
af9a0e21 MZ	114	void kvm_arch_vcpu_ctxflush_fp(struct kvm_vcpu *vcpu)
	115	{
	116	if (test_thread_flag(TIF_FOREIGN_FPSTATE))
	117	vcpu->arch.flags \|= KVM_ARM64_FP_FOREIGN_FPSTATE;
	118	else
	119	vcpu->arch.flags &= ~KVM_ARM64_FP_FOREIGN_FPSTATE;
	120	}
	121
e6b673b7	122	/*
23afc825 MB	123	* Called just after exiting the guest. If the guest FPSIMD state
	124	* was loaded, update the host's context tracking data mark the CPU
	125	* FPSIMD regs as dirty and belonging to vcpu so that they will be
	126	* written back if the kernel clobbers them due to kernel-mode NEON
	127	* before re-entry into the guest.
e6b673b7 DM	128	*/
	129	void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
	130	{
	131	WARN_ON_ONCE(!irqs_disabled());
	132
	133	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
b40c559b MB	134	/*
	135	* Currently we do not support SME guests so SVCR is
	136	* always 0 and we just need a variable to point to.
	137	*/
e47c2055	138	fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.fp_regs,
b43b5dd9	139	vcpu->arch.sve_state,
b40c559b	140	vcpu->arch.sve_max_vl,
0033cd93	141	NULL, 0, &vcpu->arch.svcr);
04950674	142
e6b673b7	143	clear_thread_flag(TIF_FOREIGN_FPSTATE);
b43b5dd9	144	update_thread_flag(TIF_SVE, vcpu_has_sve(vcpu));
e6b673b7 DM	145	}
	146	}
	147
	148	/*
	149	* Write back the vcpu FPSIMD regs if they are dirty, and invalidate the
	150	* cpu FPSIMD regs so that they can't be spuriously reused if this vcpu
	151	* disappears and another task or vcpu appears that recycles the same
	152	* struct fpsimd_state.
	153	*/
	154	void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
	155	{
b045e4d0 DM	156	unsigned long flags;
	157
	158	local_irq_save(flags);
e6b673b7	159
861262ab MB	160	/*
	161	* If we have VHE then the Hyp code will reset CPACR_EL1 to
	162	* CPACR_EL1_DEFAULT and we need to reenable SME.
	163	*/
	164	if (has_vhe() && system_supports_sme()) {
	165	/* Also restore EL0 state seen on entry */
	166	if (vcpu->arch.flags & KVM_ARM64_HOST_SME_ENABLED)
	167	sysreg_clear_set(CPACR_EL1, 0,
	168	CPACR_EL1_SMEN_EL0EN \|
	169	CPACR_EL1_SMEN_EL1EN);
	170	else
	171	sysreg_clear_set(CPACR_EL1,
	172	CPACR_EL1_SMEN_EL0EN,
	173	CPACR_EL1_SMEN_EL1EN);
	174	}
	175
e6b673b7	176	if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
8383741a	177	if (vcpu_has_sve(vcpu)) {
83857371	178	__vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
b145a843	179
8c8010d6 MZ	180	/* Restore the VL that was saved when bound to the CPU */
	181	if (!has_vhe())
	182	sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1,
	183	SYS_ZCR_EL1);
	184	}
	185
b145a843	186	fpsimd_save_and_flush_cpu_state();
8383741a	187	} else if (has_vhe() && system_supports_sve()) {
b3eb56b6 DM	188	/*
	189	* The FPSIMD/SVE state in the CPU has not been touched, and we
	190	* have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been
	191	* reset to CPACR_EL1_DEFAULT by the Hyp code, disabling SVE
	192	* for EL0. To avoid spurious traps, restore the trap state
	193	* seen by kvm_arch_vcpu_load_fp():
	194	*/
	195	if (vcpu->arch.flags & KVM_ARM64_HOST_SVE_ENABLED)
	196	sysreg_clear_set(CPACR_EL1, 0, CPACR_EL1_ZEN_EL0EN);
	197	else
	198	sysreg_clear_set(CPACR_EL1, CPACR_EL1_ZEN_EL0EN, 0);
e6b673b7 DM	199	}
e6b673b7 DM	200
8383741a	201	update_thread_flag(TIF_SVE, 0);
2955bcc8	202
b045e4d0	203	local_irq_restore(flags);
e6b673b7	204	}