LoongArch: KVM: Add LSX (128bit SIMD) support
authorTianrui Zhao <zhaotianrui@loongson.cn>
Tue, 19 Dec 2023 02:48:28 +0000 (10:48 +0800)
committerHuacai Chen <chenhuacai@loongson.cn>
Tue, 19 Dec 2023 02:48:28 +0000 (10:48 +0800)
This patch adds LSX (128bit SIMD) support for LoongArch KVM.

There will be LSX exception in KVM when guest use the LSX instructions.
KVM will enable LSX and restore the vector registers for guest and then
return to guest to continue running.

Signed-off-by: Tianrui Zhao <zhaotianrui@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
arch/loongarch/include/asm/kvm_host.h
arch/loongarch/include/asm/kvm_vcpu.h
arch/loongarch/include/uapi/asm/kvm.h
arch/loongarch/kernel/fpu.S
arch/loongarch/kvm/exit.c
arch/loongarch/kvm/switch.S
arch/loongarch/kvm/trace.h
arch/loongarch/kvm/vcpu.c

index 0e89db020481c6652260e8ba29609c21851975ba..b0c5cdd8014c21a408bae0e3852103a4d180d26b 100644 (file)
@@ -95,8 +95,9 @@ enum emulation_result {
 };
 
 #define KVM_LARCH_FPU          (0x1 << 0)
-#define KVM_LARCH_SWCSR_LATEST (0x1 << 1)
-#define KVM_LARCH_HWCSR_USABLE (0x1 << 2)
+#define KVM_LARCH_LSX          (0x1 << 1)
+#define KVM_LARCH_SWCSR_LATEST (0x1 << 2)
+#define KVM_LARCH_HWCSR_USABLE (0x1 << 3)
 
 struct kvm_vcpu_arch {
        /*
@@ -178,6 +179,16 @@ static inline void writel_sw_gcsr(struct loongarch_csrs *csr, int reg, unsigned
        csr->csrs[reg] = val;
 }
 
+static inline bool kvm_guest_has_fpu(struct kvm_vcpu_arch *arch)
+{
+       return arch->cpucfg[2] & CPUCFG2_FP;
+}
+
+static inline bool kvm_guest_has_lsx(struct kvm_vcpu_arch *arch)
+{
+       return arch->cpucfg[2] & CPUCFG2_LSX;
+}
+
 /* Debug: dump vcpu state */
 int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu);
 
index 0e87652f780a023b93ef19b045c06e047f398f86..db08dd46b525ab4d71d2142f32a3b9bac2dc6f1a 100644 (file)
@@ -55,6 +55,16 @@ void kvm_save_fpu(struct loongarch_fpu *fpu);
 void kvm_restore_fpu(struct loongarch_fpu *fpu);
 void kvm_restore_fcsr(struct loongarch_fpu *fpu);
 
+#ifdef CONFIG_CPU_HAS_LSX
+int kvm_own_lsx(struct kvm_vcpu *vcpu);
+void kvm_save_lsx(struct loongarch_fpu *fpu);
+void kvm_restore_lsx(struct loongarch_fpu *fpu);
+#else
+static inline int kvm_own_lsx(struct kvm_vcpu *vcpu) { }
+static inline void kvm_save_lsx(struct loongarch_fpu *fpu) { }
+static inline void kvm_restore_lsx(struct loongarch_fpu *fpu) { }
+#endif
+
 void kvm_init_timer(struct kvm_vcpu *vcpu, unsigned long hz);
 void kvm_reset_timer(struct kvm_vcpu *vcpu);
 void kvm_save_timer(struct kvm_vcpu *vcpu);
index c6ad2ee6106cb0389f0ce626c6d0df65647d351b..923d0bd382941acc5794d7622f7adfe1b2533422 100644 (file)
@@ -79,6 +79,7 @@ struct kvm_fpu {
 #define LOONGARCH_REG_64(TYPE, REG)    (TYPE | KVM_REG_SIZE_U64 | (REG << LOONGARCH_REG_SHIFT))
 #define KVM_IOC_CSRID(REG)             LOONGARCH_REG_64(KVM_REG_LOONGARCH_CSR, REG)
 #define KVM_IOC_CPUCFG(REG)            LOONGARCH_REG_64(KVM_REG_LOONGARCH_CPUCFG, REG)
+#define KVM_LOONGARCH_VCPU_CPUCFG      0
 
 struct kvm_debug_exit_arch {
 };
index d53ab10f464465e3f88910614afefce92b5af607..a400924c0348bffeca7ee6d7ad7e2628d41fb5c7 100644 (file)
@@ -349,6 +349,7 @@ SYM_FUNC_START(_restore_lsx_upper)
        lsx_restore_all_upper a0 t0 t1
        jr      ra
 SYM_FUNC_END(_restore_lsx_upper)
+EXPORT_SYMBOL(_restore_lsx_upper)
 
 SYM_FUNC_START(_init_lsx_upper)
        lsx_init_all_upper t1
index e708a1786d6bdc9f8affa4ab690eb5dc24de9e12..676f7a3a335c297e90b0d6f9a8a2493bf218ef43 100644 (file)
@@ -634,6 +634,11 @@ static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu)
 {
        struct kvm_run *run = vcpu->run;
 
+       if (!kvm_guest_has_fpu(&vcpu->arch)) {
+               kvm_queue_exception(vcpu, EXCCODE_INE, 0);
+               return RESUME_GUEST;
+       }
+
        /*
         * If guest FPU not present, the FPU operation should have been
         * treated as a reserved instruction!
@@ -650,6 +655,21 @@ static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu)
        return RESUME_GUEST;
 }
 
+/*
+ * kvm_handle_lsx_disabled() - Guest used LSX while disabled in root.
+ * @vcpu:      Virtual CPU context.
+ *
+ * Handle when the guest attempts to use LSX when it is disabled in the root
+ * context.
+ */
+static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu)
+{
+       if (kvm_own_lsx(vcpu))
+               kvm_queue_exception(vcpu, EXCCODE_INE, 0);
+
+       return RESUME_GUEST;
+}
+
 /*
  * LoongArch KVM callback handling for unimplemented guest exiting
  */
@@ -678,6 +698,7 @@ static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = {
        [EXCCODE_TLBS]                  = kvm_handle_write_fault,
        [EXCCODE_TLBM]                  = kvm_handle_write_fault,
        [EXCCODE_FPDIS]                 = kvm_handle_fpu_disabled,
+       [EXCCODE_LSXDIS]                = kvm_handle_lsx_disabled,
        [EXCCODE_GSPR]                  = kvm_handle_gspr,
 };
 
index 0ed9040307b71511b42aa6913358ec1acdd7b25b..00fbf772d16f9969f16f72c127178de0f2d460f7 100644 (file)
@@ -245,6 +245,22 @@ SYM_FUNC_START(kvm_restore_fpu)
        jr                 ra
 SYM_FUNC_END(kvm_restore_fpu)
 
+#ifdef CONFIG_CPU_HAS_LSX
+SYM_FUNC_START(kvm_save_lsx)
+       fpu_save_csr    a0 t1
+       fpu_save_cc     a0 t1 t2
+       lsx_save_data   a0 t1
+       jr              ra
+SYM_FUNC_END(kvm_save_lsx)
+
+SYM_FUNC_START(kvm_restore_lsx)
+       lsx_restore_data a0 t1
+       fpu_restore_cc   a0 t1 t2
+       fpu_restore_csr  a0 t1 t2
+       jr               ra
+SYM_FUNC_END(kvm_restore_lsx)
+#endif
+
        .section ".rodata"
 SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry)
 SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest)
index a1e35d6554185cf69912cae41e0b26ff20e70739..7da4e230e896f49fca7721140b9cfa6ab761d0e1 100644 (file)
@@ -102,6 +102,7 @@ TRACE_EVENT(kvm_exit_gspr,
 #define KVM_TRACE_AUX_DISCARD          4
 
 #define KVM_TRACE_AUX_FPU              1
+#define KVM_TRACE_AUX_LSX              2
 
 #define kvm_trace_symbol_aux_op                                \
        { KVM_TRACE_AUX_SAVE,           "save" },       \
@@ -111,7 +112,8 @@ TRACE_EVENT(kvm_exit_gspr,
        { KVM_TRACE_AUX_DISCARD,        "discard" }
 
 #define kvm_trace_symbol_aux_state                     \
-       { KVM_TRACE_AUX_FPU,     "FPU" }
+       { KVM_TRACE_AUX_FPU,     "FPU" },               \
+       { KVM_TRACE_AUX_LSX,     "LSX" }
 
 TRACE_EVENT(kvm_aux,
            TP_PROTO(struct kvm_vcpu *vcpu, unsigned int op,
index 53fcef8b24a10241b49bbe578d813fa3581d075c..80487d177ca4a32c00777c74c8b4bb80f1e06a44 100644 (file)
@@ -298,6 +298,69 @@ static int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 val)
        return ret;
 }
 
+static int _kvm_get_cpucfg(int id, u64 *v)
+{
+       int ret = 0;
+
+       if (id < 0 && id >= KVM_MAX_CPUCFG_REGS)
+               return -EINVAL;
+
+       switch (id) {
+       case 2:
+               /* Return CPUCFG2 features which have been supported by KVM */
+               *v = CPUCFG2_FP     | CPUCFG2_FPSP  | CPUCFG2_FPDP     |
+                    CPUCFG2_FPVERS | CPUCFG2_LLFTP | CPUCFG2_LLFTPREV |
+                    CPUCFG2_LAM;
+               /*
+                * If LSX is supported by CPU, it is also supported by KVM,
+                * as we implement it.
+                */
+               if (cpu_has_lsx)
+                       *v |= CPUCFG2_LSX;
+               break;
+       default:
+               ret = -EINVAL;
+               break;
+       }
+       return ret;
+}
+
+static int kvm_check_cpucfg(int id, u64 val)
+{
+       u64 mask;
+       int ret = 0;
+
+       if (id < 0 && id >= KVM_MAX_CPUCFG_REGS)
+               return -EINVAL;
+
+       if (_kvm_get_cpucfg(id, &mask))
+               return ret;
+
+       switch (id) {
+       case 2:
+               /* CPUCFG2 features checking */
+               if (val & ~mask)
+                       /* The unsupported features should not be set */
+                       ret = -EINVAL;
+               else if (!(val & CPUCFG2_LLFTP))
+                       /* The LLFTP must be set, as guest must has a constant timer */
+                       ret = -EINVAL;
+               else if ((val & CPUCFG2_FP) && (!(val & CPUCFG2_FPSP) || !(val & CPUCFG2_FPDP)))
+                       /* Single and double float point must both be set when enable FP */
+                       ret = -EINVAL;
+               else if ((val & CPUCFG2_LSX) && !(val & CPUCFG2_FP))
+                       /* FP should be set when enable LSX */
+                       ret = -EINVAL;
+               else if ((val & CPUCFG2_LASX) && !(val & CPUCFG2_LSX))
+                       /* LSX, FP should be set when enable LASX, and FP has been checked before. */
+                       ret = -EINVAL;
+               break;
+       default:
+               break;
+       }
+       return ret;
+}
+
 static int kvm_get_one_reg(struct kvm_vcpu *vcpu,
                const struct kvm_one_reg *reg, u64 *v)
 {
@@ -367,10 +430,10 @@ static int kvm_set_one_reg(struct kvm_vcpu *vcpu,
                break;
        case KVM_REG_LOONGARCH_CPUCFG:
                id = KVM_GET_IOC_CPUCFG_IDX(reg->id);
-               if (id >= 0 && id < KVM_MAX_CPUCFG_REGS)
-                       vcpu->arch.cpucfg[id] = (u32)v;
-               else
-                       ret = -EINVAL;
+               ret = kvm_check_cpucfg(id, v);
+               if (ret)
+                       break;
+               vcpu->arch.cpucfg[id] = (u32)v;
                break;
        case KVM_REG_LOONGARCH_KVM:
                switch (reg->id) {
@@ -460,10 +523,94 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
        return -EINVAL;
 }
 
+static int kvm_loongarch_cpucfg_has_attr(struct kvm_vcpu *vcpu,
+                                        struct kvm_device_attr *attr)
+{
+       switch (attr->attr) {
+       case 2:
+               return 0;
+       default:
+               return -ENXIO;
+       }
+
+       return -ENXIO;
+}
+
+static int kvm_loongarch_vcpu_has_attr(struct kvm_vcpu *vcpu,
+                                      struct kvm_device_attr *attr)
+{
+       int ret = -ENXIO;
+
+       switch (attr->group) {
+       case KVM_LOONGARCH_VCPU_CPUCFG:
+               ret = kvm_loongarch_cpucfg_has_attr(vcpu, attr);
+               break;
+       default:
+               break;
+       }
+
+       return ret;
+}
+
+static int kvm_loongarch_get_cpucfg_attr(struct kvm_vcpu *vcpu,
+                                        struct kvm_device_attr *attr)
+{
+       int ret = 0;
+       uint64_t val;
+       uint64_t __user *uaddr = (uint64_t __user *)attr->addr;
+
+       ret = _kvm_get_cpucfg(attr->attr, &val);
+       if (ret)
+               return ret;
+
+       put_user(val, uaddr);
+
+       return ret;
+}
+
+static int kvm_loongarch_vcpu_get_attr(struct kvm_vcpu *vcpu,
+                                      struct kvm_device_attr *attr)
+{
+       int ret = -ENXIO;
+
+       switch (attr->group) {
+       case KVM_LOONGARCH_VCPU_CPUCFG:
+               ret = kvm_loongarch_get_cpucfg_attr(vcpu, attr);
+               break;
+       default:
+               break;
+       }
+
+       return ret;
+}
+
+static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu,
+                                        struct kvm_device_attr *attr)
+{
+       return -ENXIO;
+}
+
+static int kvm_loongarch_vcpu_set_attr(struct kvm_vcpu *vcpu,
+                                      struct kvm_device_attr *attr)
+{
+       int ret = -ENXIO;
+
+       switch (attr->group) {
+       case KVM_LOONGARCH_VCPU_CPUCFG:
+               ret = kvm_loongarch_cpucfg_set_attr(vcpu, attr);
+               break;
+       default:
+               break;
+       }
+
+       return ret;
+}
+
 long kvm_arch_vcpu_ioctl(struct file *filp,
                         unsigned int ioctl, unsigned long arg)
 {
        long r;
+       struct kvm_device_attr attr;
        void __user *argp = (void __user *)arg;
        struct kvm_vcpu *vcpu = filp->private_data;
 
@@ -503,6 +650,27 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
                r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
                break;
        }
+       case KVM_HAS_DEVICE_ATTR: {
+               r = -EFAULT;
+               if (copy_from_user(&attr, argp, sizeof(attr)))
+                       break;
+               r = kvm_loongarch_vcpu_has_attr(vcpu, &attr);
+               break;
+       }
+       case KVM_GET_DEVICE_ATTR: {
+               r = -EFAULT;
+               if (copy_from_user(&attr, argp, sizeof(attr)))
+                       break;
+               r = kvm_loongarch_vcpu_get_attr(vcpu, &attr);
+               break;
+       }
+       case KVM_SET_DEVICE_ATTR: {
+               r = -EFAULT;
+               if (copy_from_user(&attr, argp, sizeof(attr)))
+                       break;
+               r = kvm_loongarch_vcpu_set_attr(vcpu, &attr);
+               break;
+       }
        default:
                r = -ENOIOCTLCMD;
                break;
@@ -550,12 +718,54 @@ void kvm_own_fpu(struct kvm_vcpu *vcpu)
        preempt_enable();
 }
 
+#ifdef CONFIG_CPU_HAS_LSX
+/* Enable LSX and restore context */
+int kvm_own_lsx(struct kvm_vcpu *vcpu)
+{
+       if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch))
+               return -EINVAL;
+
+       preempt_disable();
+
+       /* Enable LSX for guest */
+       set_csr_euen(CSR_EUEN_LSXEN | CSR_EUEN_FPEN);
+       switch (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
+       case KVM_LARCH_FPU:
+               /*
+                * Guest FPU state already loaded,
+                * only restore upper LSX state
+                */
+               _restore_lsx_upper(&vcpu->arch.fpu);
+               break;
+       default:
+               /* Neither FP or LSX already active,
+                * restore full LSX state
+                */
+               kvm_restore_lsx(&vcpu->arch.fpu);
+               break;
+       }
+
+       trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LSX);
+       vcpu->arch.aux_inuse |= KVM_LARCH_LSX | KVM_LARCH_FPU;
+       preempt_enable();
+
+       return 0;
+}
+#endif
+
 /* Save context and disable FPU */
 void kvm_lose_fpu(struct kvm_vcpu *vcpu)
 {
        preempt_disable();
 
-       if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
+       if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) {
+               kvm_save_lsx(&vcpu->arch.fpu);
+               vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU);
+               trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX);
+
+               /* Disable LSX & FPU */
+               clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN);
+       } else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
                kvm_save_fpu(&vcpu->arch.fpu);
                vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU;
                trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU);