[linux-block.git] / arch / riscv / kvm / vmid.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
 *
 * Authors:
 *     Anup Patel <anup.patel@wdc.com>
 */

#include <linux/bitops.h>
#include <linux/cpumask.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/smp.h>
#include <linux/kvm_host.h>
#include <asm/csr.h>

static unsigned long vmid_version = 1;
static unsigned long vmid_next;
static unsigned long vmid_bits __ro_after_init;
static DEFINE_SPINLOCK(vmid_lock);

void __init kvm_riscv_gstage_vmid_detect(void)
{
	unsigned long old;

	/* Figure-out number of VMID bits in HW */
	old = csr_read(CSR_HGATP);
	csr_write(CSR_HGATP, old | HGATP_VMID);
	vmid_bits = csr_read(CSR_HGATP);
	vmid_bits = (vmid_bits & HGATP_VMID) >> HGATP_VMID_SHIFT;
	vmid_bits = fls_long(vmid_bits);
	csr_write(CSR_HGATP, old);

	/* We polluted local TLB so flush all guest TLB */
	kvm_riscv_local_hfence_gvma_all();

	/* We don't use VMID bits if they are not sufficient */
	if ((1UL << vmid_bits) < num_possible_cpus())
		vmid_bits = 0;
}

unsigned long kvm_riscv_gstage_vmid_bits(void)
{
	return vmid_bits;
}

int kvm_riscv_gstage_vmid_init(struct kvm *kvm)
{
	/* Mark the initial VMID and VMID version invalid */
	kvm->arch.vmid.vmid_version = 0;
	kvm->arch.vmid.vmid = 0;

	return 0;
}

bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid)
{
	if (!vmid_bits)
		return false;

	return unlikely(READ_ONCE(vmid->vmid_version) !=
			READ_ONCE(vmid_version));
}

static void __local_hfence_gvma_all(void *info)
{
	kvm_riscv_local_hfence_gvma_all();
}

void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu)
{
	unsigned long i;
	struct kvm_vcpu *v;
	struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid;

	if (!kvm_riscv_gstage_vmid_ver_changed(vmid))
		return;

	spin_lock(&vmid_lock);

	/*
	 * We need to re-check the vmid_version here to ensure that if
	 * another vcpu already allocated a valid vmid for this vm.
	 */
	if (!kvm_riscv_gstage_vmid_ver_changed(vmid)) {
		spin_unlock(&vmid_lock);
		return;
	}

	/* First user of a new VMID version? */
	if (unlikely(vmid_next == 0)) {
		WRITE_ONCE(vmid_version, READ_ONCE(vmid_version) + 1);
		vmid_next = 1;

		/*
		 * We ran out of VMIDs so we increment vmid_version and
		 * start assigning VMIDs from 1.
		 *
		 * This also means existing VMIDs assignment to all Guest
		 * instances is invalid and we have force VMID re-assignement
		 * for all Guest instances. The Guest instances that were not
		 * running will automatically pick-up new VMIDs because will
		 * call kvm_riscv_gstage_vmid_update() whenever they enter
		 * in-kernel run loop. For Guest instances that are already
		 * running, we force VM exits on all host CPUs using IPI and
		 * flush all Guest TLBs.
		 */
		on_each_cpu_mask(cpu_online_mask, __local_hfence_gvma_all,
				 NULL, 1);
	}

	vmid->vmid = vmid_next;
	vmid_next++;
	vmid_next &= (1 << vmid_bits) - 1;

	WRITE_ONCE(vmid->vmid_version, READ_ONCE(vmid_version));

	spin_unlock(&vmid_lock);

	/* Request G-stage page table update for all VCPUs */
	kvm_for_each_vcpu(i, v, vcpu->kvm)
		kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
}
Commit	Line	Data
fd7bb4a2 AP	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (C) 2019 Western Digital Corporation or its affiliates.
	4	*
	5	* Authors:
	6	* Anup Patel <anup.patel@wdc.com>
	7	*/
	8
	9	#include <linux/bitops.h>
	10	#include <linux/cpumask.h>
	11	#include <linux/errno.h>
	12	#include <linux/err.h>
	13	#include <linux/module.h>
13acfec2	14	#include <linux/smp.h>
fd7bb4a2 AP	15	#include <linux/kvm_host.h>
fd7bb4a2 AP	16	#include <asm/csr.h>
fd7bb4a2 AP	17
	18	static unsigned long vmid_version = 1;
	19	static unsigned long vmid_next;
45b66dc1	20	static unsigned long vmid_bits __ro_after_init;
fd7bb4a2 AP	21	static DEFINE_SPINLOCK(vmid_lock);
fd7bb4a2 AP	22
45b66dc1	23	void __init kvm_riscv_gstage_vmid_detect(void)
fd7bb4a2 AP	24	{
	25	unsigned long old;
	26
	27	/* Figure-out number of VMID bits in HW */
	28	old = csr_read(CSR_HGATP);
e290dbb7	29	csr_write(CSR_HGATP, old \| HGATP_VMID);
fd7bb4a2	30	vmid_bits = csr_read(CSR_HGATP);
e290dbb7	31	vmid_bits = (vmid_bits & HGATP_VMID) >> HGATP_VMID_SHIFT;
fd7bb4a2 AP	32	vmid_bits = fls_long(vmid_bits);
	33	csr_write(CSR_HGATP, old);
	34
	35	/* We polluted local TLB so flush all guest TLB */
2415e46e	36	kvm_riscv_local_hfence_gvma_all();
fd7bb4a2 AP	37
	38	/* We don't use VMID bits if they are not sufficient */
	39	if ((1UL << vmid_bits) < num_possible_cpus())
	40	vmid_bits = 0;
	41	}
	42
26708234	43	unsigned long kvm_riscv_gstage_vmid_bits(void)
fd7bb4a2 AP	44	{
	45	return vmid_bits;
	46	}
	47
26708234	48	int kvm_riscv_gstage_vmid_init(struct kvm *kvm)
fd7bb4a2 AP	49	{
	50	/* Mark the initial VMID and VMID version invalid */
	51	kvm->arch.vmid.vmid_version = 0;
	52	kvm->arch.vmid.vmid = 0;
	53
	54	return 0;
	55	}
	56
26708234	57	bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid)
fd7bb4a2 AP	58	{
	59	if (!vmid_bits)
	60	return false;
	61
	62	return unlikely(READ_ONCE(vmid->vmid_version) !=
	63	READ_ONCE(vmid_version));
	64	}
	65
13acfec2 AP	66	static void __local_hfence_gvma_all(void *info)
	67	{
	68	kvm_riscv_local_hfence_gvma_all();
	69	}
	70
26708234	71	void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu)
fd7bb4a2	72	{
46808a4c	73	unsigned long i;
fd7bb4a2	74	struct kvm_vcpu *v;
fd7bb4a2 AP	75	struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid;
fd7bb4a2 AP	76
26708234	77	if (!kvm_riscv_gstage_vmid_ver_changed(vmid))
fd7bb4a2 AP	78	return;
	79
	80	spin_lock(&vmid_lock);
	81
	82	/*
	83	* We need to re-check the vmid_version here to ensure that if
	84	* another vcpu already allocated a valid vmid for this vm.
	85	*/
26708234	86	if (!kvm_riscv_gstage_vmid_ver_changed(vmid)) {
fd7bb4a2 AP	87	spin_unlock(&vmid_lock);
	88	return;
	89	}
	90
	91	/* First user of a new VMID version? */
	92	if (unlikely(vmid_next == 0)) {
	93	WRITE_ONCE(vmid_version, READ_ONCE(vmid_version) + 1);
	94	vmid_next = 1;
	95
	96	/*
	97	* We ran out of VMIDs so we increment vmid_version and
	98	* start assigning VMIDs from 1.
	99	*
ea6c1213	100	* This also means existing VMIDs assignment to all Guest
fd7bb4a2 AP	101	* instances is invalid and we have force VMID re-assignement
	102	* for all Guest instances. The Guest instances that were not
	103	* running will automatically pick-up new VMIDs because will
26708234	104	* call kvm_riscv_gstage_vmid_update() whenever they enter
fd7bb4a2 AP	105	* in-kernel run loop. For Guest instances that are already
	106	* running, we force VM exits on all host CPUs using IPI and
	107	* flush all Guest TLBs.
	108	*/
13acfec2 AP	109	on_each_cpu_mask(cpu_online_mask, __local_hfence_gvma_all,
13acfec2 AP	110	NULL, 1);
fd7bb4a2 AP	111	}
	112
	113	vmid->vmid = vmid_next;
	114	vmid_next++;
	115	vmid_next &= (1 << vmid_bits) - 1;
	116
	117	WRITE_ONCE(vmid->vmid_version, READ_ONCE(vmid_version));
	118
	119	spin_unlock(&vmid_lock);
	120
26708234	121	/* Request G-stage page table update for all VCPUs */
fd7bb4a2 AP	122	kvm_for_each_vcpu(i, v, vcpu->kvm)
	123	kvm_make_request(KVM_REQ_UPDATE_HGATP, v);
	124	}