KVM: arm64: nv: Add basic emulation of AT S1E{0,1}{R,W}

Emulating AT instructions is one the tasks devolved to the host
hypervisor when NV is on.

Here, we take the basic approach of emulating AT S1E{0,1}{R,W}
using the AT instructions themselves. While this mostly work,
it doesn't *always* work:

- S1 page tables can be swapped out

- shadow S2 can be incomplete and not contain mappings for
  the S1 page tables

We are not trying to handle these case here, and defer it to
a later patch. Suitable comments indicate where we are in dire
need of better handling.

Co-developed-by: Jintack Lim <jintack.lim@linaro.org>
Signed-off-by: Jintack Lim <jintack.lim@linaro.org>
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 25f49f5fc4a635cd0537b4f8eae58f3f4b5a235a..9b6c9f4f4d88596d3502d02f1000da5abe119039 100644 (file)
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -236,6 +236,7 @@ extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu);
 extern int __kvm_tlbi_s1e2(struct kvm_s2_mmu *mmu, u64 va, u64 sys_encoding);
 
 extern void __kvm_timer_set_cntvoff(u64 cntvoff);
+extern void __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr);
 
 extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
 

diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index a6497228c5a8c37d9159a6f99b162b22f18192ca..8a3ae76b4da225eb9d874391e7aaceb930fe1bae 100644 (file)
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -14,7 +14,7 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \
         inject_fault.o va_layout.o handle_exit.o \
         guest.o debug.o reset.o sys_regs.o stacktrace.o \
         vgic-sys-reg-v3.o fpsimd.o pkvm.o \
-        arch_timer.o trng.o vmid.o emulate-nested.o nested.o \
+        arch_timer.o trng.o vmid.o emulate-nested.o nested.o at.o \
         vgic/vgic.o vgic/vgic-init.o \
         vgic/vgic-irqfd.o vgic/vgic-v2.o \
         vgic/vgic-v3.o vgic/vgic-v4.o \

diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c
new file mode 100644 (file)
index 0000000..da378ad
--- /dev/null
+++ b/arch/arm64/kvm/at.c
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2017 - Linaro Ltd
+ * Author: Jintack Lim <jintack.lim@linaro.org>
+ */
+
+#include <asm/kvm_hyp.h>
+#include <asm/kvm_mmu.h>
+
+struct mmu_config {
+       u64     ttbr0;
+       u64     ttbr1;
+       u64     tcr;
+       u64     mair;
+       u64     sctlr;
+       u64     vttbr;
+       u64     vtcr;
+       u64     hcr;
+};
+
+static void __mmu_config_save(struct mmu_config *config)
+{
+       config->ttbr0   = read_sysreg_el1(SYS_TTBR0);
+       config->ttbr1   = read_sysreg_el1(SYS_TTBR1);
+       config->tcr     = read_sysreg_el1(SYS_TCR);
+       config->mair    = read_sysreg_el1(SYS_MAIR);
+       config->sctlr   = read_sysreg_el1(SYS_SCTLR);
+       config->vttbr   = read_sysreg(vttbr_el2);
+       config->vtcr    = read_sysreg(vtcr_el2);
+       config->hcr     = read_sysreg(hcr_el2);
+}
+
+static void __mmu_config_restore(struct mmu_config *config)
+{
+       write_sysreg(config->hcr,       hcr_el2);
+
+       /*
+        * ARM errata 1165522 and 1530923 require TGE to be 1 before
+        * we update the guest state.
+        */
+       asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
+
+       write_sysreg_el1(config->ttbr0, SYS_TTBR0);
+       write_sysreg_el1(config->ttbr1, SYS_TTBR1);
+       write_sysreg_el1(config->tcr,   SYS_TCR);
+       write_sysreg_el1(config->mair,  SYS_MAIR);
+       write_sysreg_el1(config->sctlr, SYS_SCTLR);
+       write_sysreg(config->vttbr,     vttbr_el2);
+       write_sysreg(config->vtcr,      vtcr_el2);
+}
+
+/*
+ * Return the PAR_EL1 value as the result of a valid translation.
+ *
+ * If the translation is unsuccessful, the value may only contain
+ * PAR_EL1.F, and cannot be taken at face value. It isn't an
+ * indication of the translation having failed, only that the fast
+ * path did not succeed, *unless* it indicates a S1 permission fault.
+ */
+static u64 __kvm_at_s1e01_fast(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
+{
+       struct mmu_config config;
+       struct kvm_s2_mmu *mmu;
+       bool fail;
+       u64 par;
+
+       par = SYS_PAR_EL1_F;
+
+       /*
+        * We've trapped, so everything is live on the CPU. As we will
+        * be switching contexts behind everybody's back, disable
+        * interrupts while holding the mmu lock.
+        */
+       guard(write_lock_irqsave)(&vcpu->kvm->mmu_lock);
+
+       /*
+        * If HCR_EL2.{E2H,TGE} == {1,1}, the MMU context is already
+        * the right one (as we trapped from vEL2). If not, save the
+        * full MMU context.
+        */
+       if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu))
+               goto skip_mmu_switch;
+
+       /*
+        * Obtaining the S2 MMU for a L2 is horribly racy, and we may not
+        * find it (recycled by another vcpu, for example). When this
+        * happens, admit defeat immediately and use the SW (slow) path.
+        */
+       mmu = lookup_s2_mmu(vcpu);
+       if (!mmu)
+               return par;
+
+       __mmu_config_save(&config);
+
+       write_sysreg_el1(vcpu_read_sys_reg(vcpu, TTBR0_EL1),    SYS_TTBR0);
+       write_sysreg_el1(vcpu_read_sys_reg(vcpu, TTBR1_EL1),    SYS_TTBR1);
+       write_sysreg_el1(vcpu_read_sys_reg(vcpu, TCR_EL1),      SYS_TCR);
+       write_sysreg_el1(vcpu_read_sys_reg(vcpu, MAIR_EL1),     SYS_MAIR);
+       write_sysreg_el1(vcpu_read_sys_reg(vcpu, SCTLR_EL1),    SYS_SCTLR);
+       __load_stage2(mmu, mmu->arch);
+
+skip_mmu_switch:
+       /* Clear TGE, enable S2 translation, we're rolling */
+       write_sysreg((config.hcr & ~HCR_TGE) | HCR_VM,  hcr_el2);
+       isb();
+
+       switch (op) {
+       case OP_AT_S1E1R:
+               fail = __kvm_at(OP_AT_S1E1R, vaddr);
+               break;
+       case OP_AT_S1E1W:
+               fail = __kvm_at(OP_AT_S1E1W, vaddr);
+               break;
+       case OP_AT_S1E0R:
+               fail = __kvm_at(OP_AT_S1E0R, vaddr);
+               break;
+       case OP_AT_S1E0W:
+               fail = __kvm_at(OP_AT_S1E0W, vaddr);
+               break;
+       default:
+               WARN_ON_ONCE(1);
+               fail = true;
+               break;
+       }
+
+       if (!fail)
+               par = read_sysreg_par();
+
+       if (!(vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)))
+               __mmu_config_restore(&config);
+
+       return par;
+}
+
+void __kvm_at_s1e01(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
+{
+       u64 par = __kvm_at_s1e01_fast(vcpu, op, vaddr);
+
+       vcpu_write_sys_reg(vcpu, par, PAR_EL1);
+}