1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
7 * Anup Patel <apatel@ventanamicro.com>
10 #include <linux/bits.h>
11 #include <linux/kvm_host.h>
12 #include <linux/uaccess.h>
13 #include <asm/kvm_aia_imsic.h>
15 static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx)
17 struct kvm_vcpu *tmp_vcpu;
19 for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) {
20 tmp_vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx);
21 mutex_unlock(&tmp_vcpu->mutex);
25 static void unlock_all_vcpus(struct kvm *kvm)
27 unlock_vcpus(kvm, atomic_read(&kvm->online_vcpus) - 1);
30 static bool lock_all_vcpus(struct kvm *kvm)
32 struct kvm_vcpu *tmp_vcpu;
35 kvm_for_each_vcpu(c, tmp_vcpu, kvm) {
36 if (!mutex_trylock(&tmp_vcpu->mutex)) {
37 unlock_vcpus(kvm, c - 1);
45 static int aia_create(struct kvm_device *dev, u32 type)
49 struct kvm *kvm = dev->kvm;
50 struct kvm_vcpu *vcpu;
52 if (irqchip_in_kernel(kvm))
56 if (!lock_all_vcpus(kvm))
59 kvm_for_each_vcpu(i, vcpu, kvm) {
60 if (vcpu->arch.ran_atleast_once)
65 kvm->arch.aia.in_kernel = true;
68 unlock_all_vcpus(kvm);
72 static void aia_destroy(struct kvm_device *dev)
77 static int aia_config(struct kvm *kvm, unsigned long type,
80 struct kvm_aia *aia = &kvm->arch.aia;
82 /* Writes can only be done before irqchip is initialized */
83 if (write && kvm_riscv_aia_initialized(kvm))
87 case KVM_DEV_RISCV_AIA_CONFIG_MODE:
90 case KVM_DEV_RISCV_AIA_MODE_EMUL:
92 case KVM_DEV_RISCV_AIA_MODE_HWACCEL:
93 case KVM_DEV_RISCV_AIA_MODE_AUTO:
95 * HW Acceleration and Auto modes only
96 * supported on host with non-zero guest
97 * external interrupts (i.e. non-zero
98 * VS-level IMSIC pages).
100 if (!kvm_riscv_aia_nr_hgei)
110 case KVM_DEV_RISCV_AIA_CONFIG_IDS:
112 if ((*nr < KVM_DEV_RISCV_AIA_IDS_MIN) ||
113 (*nr >= KVM_DEV_RISCV_AIA_IDS_MAX) ||
114 ((*nr & KVM_DEV_RISCV_AIA_IDS_MIN) !=
115 KVM_DEV_RISCV_AIA_IDS_MIN) ||
116 (kvm_riscv_aia_max_ids <= *nr))
122 case KVM_DEV_RISCV_AIA_CONFIG_SRCS:
124 if ((*nr >= KVM_DEV_RISCV_AIA_SRCS_MAX) ||
125 (*nr >= kvm_riscv_aia_max_ids))
127 aia->nr_sources = *nr;
129 *nr = aia->nr_sources;
131 case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS:
133 if (*nr >= KVM_DEV_RISCV_AIA_GROUP_BITS_MAX)
135 aia->nr_group_bits = *nr;
137 *nr = aia->nr_group_bits;
139 case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT:
141 if ((*nr < KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN) ||
142 (*nr >= KVM_DEV_RISCV_AIA_GROUP_SHIFT_MAX))
144 aia->nr_group_shift = *nr;
146 *nr = aia->nr_group_shift;
148 case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS:
150 if (*nr >= KVM_DEV_RISCV_AIA_HART_BITS_MAX)
152 aia->nr_hart_bits = *nr;
154 *nr = aia->nr_hart_bits;
156 case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS:
158 if (*nr >= KVM_DEV_RISCV_AIA_GUEST_BITS_MAX)
160 aia->nr_guest_bits = *nr;
162 *nr = aia->nr_guest_bits;
171 static int aia_aplic_addr(struct kvm *kvm, u64 *addr, bool write)
173 struct kvm_aia *aia = &kvm->arch.aia;
176 /* Writes can only be done before irqchip is initialized */
177 if (kvm_riscv_aia_initialized(kvm))
180 if (*addr & (KVM_DEV_RISCV_APLIC_ALIGN - 1))
183 aia->aplic_addr = *addr;
185 *addr = aia->aplic_addr;
190 static int aia_imsic_addr(struct kvm *kvm, u64 *addr,
191 unsigned long vcpu_idx, bool write)
193 struct kvm_vcpu *vcpu;
194 struct kvm_vcpu_aia *vcpu_aia;
196 vcpu = kvm_get_vcpu(kvm, vcpu_idx);
199 vcpu_aia = &vcpu->arch.aia_context;
202 /* Writes can only be done before irqchip is initialized */
203 if (kvm_riscv_aia_initialized(kvm))
206 if (*addr & (KVM_DEV_RISCV_IMSIC_ALIGN - 1))
210 mutex_lock(&vcpu->mutex);
212 vcpu_aia->imsic_addr = *addr;
214 *addr = vcpu_aia->imsic_addr;
215 mutex_unlock(&vcpu->mutex);
220 static gpa_t aia_imsic_ppn(struct kvm_aia *aia, gpa_t addr)
225 h = aia->nr_hart_bits + aia->nr_guest_bits +
226 IMSIC_MMIO_PAGE_SHIFT - 1;
227 mask = GENMASK_ULL(h, 0);
229 if (aia->nr_group_bits) {
230 h = aia->nr_group_bits + aia->nr_group_shift - 1;
231 l = aia->nr_group_shift;
232 mask |= GENMASK_ULL(h, l);
235 return (addr & ~mask) >> IMSIC_MMIO_PAGE_SHIFT;
238 static u32 aia_imsic_hart_index(struct kvm_aia *aia, gpa_t addr)
242 hart = (addr >> (aia->nr_guest_bits + IMSIC_MMIO_PAGE_SHIFT)) &
243 GENMASK_ULL(aia->nr_hart_bits - 1, 0);
244 if (aia->nr_group_bits)
245 group = (addr >> aia->nr_group_shift) &
246 GENMASK_ULL(aia->nr_group_bits - 1, 0);
248 return (group << aia->nr_hart_bits) | hart;
251 static int aia_init(struct kvm *kvm)
255 struct kvm_vcpu *vcpu;
256 struct kvm_vcpu_aia *vaia;
257 struct kvm_aia *aia = &kvm->arch.aia;
258 gpa_t base_ppn = KVM_RISCV_AIA_UNDEF_ADDR;
260 /* Irqchip can be initialized only once */
261 if (kvm_riscv_aia_initialized(kvm))
264 /* We might be in the middle of creating a VCPU? */
265 if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
268 /* Number of sources should be less than or equals number of IDs */
269 if (aia->nr_ids < aia->nr_sources)
272 /* APLIC base is required for non-zero number of sources */
273 if (aia->nr_sources && aia->aplic_addr == KVM_RISCV_AIA_UNDEF_ADDR)
276 /* Initialize APLIC */
277 ret = kvm_riscv_aia_aplic_init(kvm);
281 /* Iterate over each VCPU */
282 kvm_for_each_vcpu(idx, vcpu, kvm) {
283 vaia = &vcpu->arch.aia_context;
285 /* IMSIC base is required */
286 if (vaia->imsic_addr == KVM_RISCV_AIA_UNDEF_ADDR) {
288 goto fail_cleanup_imsics;
291 /* All IMSICs should have matching base PPN */
292 if (base_ppn == KVM_RISCV_AIA_UNDEF_ADDR)
293 base_ppn = aia_imsic_ppn(aia, vaia->imsic_addr);
294 if (base_ppn != aia_imsic_ppn(aia, vaia->imsic_addr)) {
296 goto fail_cleanup_imsics;
299 /* Update HART index of the IMSIC based on IMSIC base */
300 vaia->hart_index = aia_imsic_hart_index(aia,
303 /* Initialize IMSIC for this VCPU */
304 ret = kvm_riscv_vcpu_aia_imsic_init(vcpu);
306 goto fail_cleanup_imsics;
309 /* Set the initialized flag */
310 kvm->arch.aia.initialized = true;
315 for (i = idx - 1; i >= 0; i--) {
316 vcpu = kvm_get_vcpu(kvm, i);
319 kvm_riscv_vcpu_aia_imsic_cleanup(vcpu);
321 kvm_riscv_aia_aplic_cleanup(kvm);
325 static int aia_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
329 int nr_vcpus, r = -ENXIO;
330 unsigned long v, type = (unsigned long)attr->attr;
331 void __user *uaddr = (void __user *)(long)attr->addr;
333 switch (attr->group) {
334 case KVM_DEV_RISCV_AIA_GRP_CONFIG:
335 if (copy_from_user(&nr, uaddr, sizeof(nr)))
338 mutex_lock(&dev->kvm->lock);
339 r = aia_config(dev->kvm, type, &nr, true);
340 mutex_unlock(&dev->kvm->lock);
344 case KVM_DEV_RISCV_AIA_GRP_ADDR:
345 if (copy_from_user(&addr, uaddr, sizeof(addr)))
348 nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
349 mutex_lock(&dev->kvm->lock);
350 if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC)
351 r = aia_aplic_addr(dev->kvm, &addr, true);
352 else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
353 r = aia_imsic_addr(dev->kvm, &addr,
354 type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), true);
355 mutex_unlock(&dev->kvm->lock);
359 case KVM_DEV_RISCV_AIA_GRP_CTRL:
361 case KVM_DEV_RISCV_AIA_CTRL_INIT:
362 mutex_lock(&dev->kvm->lock);
363 r = aia_init(dev->kvm);
364 mutex_unlock(&dev->kvm->lock);
369 case KVM_DEV_RISCV_AIA_GRP_APLIC:
370 if (copy_from_user(&nr, uaddr, sizeof(nr)))
373 mutex_lock(&dev->kvm->lock);
374 r = kvm_riscv_aia_aplic_set_attr(dev->kvm, type, nr);
375 mutex_unlock(&dev->kvm->lock);
378 case KVM_DEV_RISCV_AIA_GRP_IMSIC:
379 if (copy_from_user(&v, uaddr, sizeof(v)))
382 mutex_lock(&dev->kvm->lock);
383 r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, true, &v);
384 mutex_unlock(&dev->kvm->lock);
392 static int aia_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
396 int nr_vcpus, r = -ENXIO;
397 void __user *uaddr = (void __user *)(long)attr->addr;
398 unsigned long v, type = (unsigned long)attr->attr;
400 switch (attr->group) {
401 case KVM_DEV_RISCV_AIA_GRP_CONFIG:
402 if (copy_from_user(&nr, uaddr, sizeof(nr)))
405 mutex_lock(&dev->kvm->lock);
406 r = aia_config(dev->kvm, type, &nr, false);
407 mutex_unlock(&dev->kvm->lock);
411 if (copy_to_user(uaddr, &nr, sizeof(nr)))
415 case KVM_DEV_RISCV_AIA_GRP_ADDR:
416 if (copy_from_user(&addr, uaddr, sizeof(addr)))
419 nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
420 mutex_lock(&dev->kvm->lock);
421 if (type == KVM_DEV_RISCV_AIA_ADDR_APLIC)
422 r = aia_aplic_addr(dev->kvm, &addr, false);
423 else if (type < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
424 r = aia_imsic_addr(dev->kvm, &addr,
425 type - KVM_DEV_RISCV_AIA_ADDR_IMSIC(0), false);
426 mutex_unlock(&dev->kvm->lock);
430 if (copy_to_user(uaddr, &addr, sizeof(addr)))
434 case KVM_DEV_RISCV_AIA_GRP_APLIC:
435 if (copy_from_user(&nr, uaddr, sizeof(nr)))
438 mutex_lock(&dev->kvm->lock);
439 r = kvm_riscv_aia_aplic_get_attr(dev->kvm, type, &nr);
440 mutex_unlock(&dev->kvm->lock);
444 if (copy_to_user(uaddr, &nr, sizeof(nr)))
448 case KVM_DEV_RISCV_AIA_GRP_IMSIC:
449 if (copy_from_user(&v, uaddr, sizeof(v)))
452 mutex_lock(&dev->kvm->lock);
453 r = kvm_riscv_aia_imsic_rw_attr(dev->kvm, type, false, &v);
454 mutex_unlock(&dev->kvm->lock);
458 if (copy_to_user(uaddr, &v, sizeof(v)))
467 static int aia_has_attr(struct kvm_device *dev, struct kvm_device_attr *attr)
471 switch (attr->group) {
472 case KVM_DEV_RISCV_AIA_GRP_CONFIG:
473 switch (attr->attr) {
474 case KVM_DEV_RISCV_AIA_CONFIG_MODE:
475 case KVM_DEV_RISCV_AIA_CONFIG_IDS:
476 case KVM_DEV_RISCV_AIA_CONFIG_SRCS:
477 case KVM_DEV_RISCV_AIA_CONFIG_GROUP_BITS:
478 case KVM_DEV_RISCV_AIA_CONFIG_GROUP_SHIFT:
479 case KVM_DEV_RISCV_AIA_CONFIG_HART_BITS:
480 case KVM_DEV_RISCV_AIA_CONFIG_GUEST_BITS:
484 case KVM_DEV_RISCV_AIA_GRP_ADDR:
485 nr_vcpus = atomic_read(&dev->kvm->online_vcpus);
486 if (attr->attr == KVM_DEV_RISCV_AIA_ADDR_APLIC)
488 else if (attr->attr < KVM_DEV_RISCV_AIA_ADDR_IMSIC(nr_vcpus))
491 case KVM_DEV_RISCV_AIA_GRP_CTRL:
492 switch (attr->attr) {
493 case KVM_DEV_RISCV_AIA_CTRL_INIT:
497 case KVM_DEV_RISCV_AIA_GRP_APLIC:
498 return kvm_riscv_aia_aplic_has_attr(dev->kvm, attr->attr);
499 case KVM_DEV_RISCV_AIA_GRP_IMSIC:
500 return kvm_riscv_aia_imsic_has_attr(dev->kvm, attr->attr);
506 struct kvm_device_ops kvm_riscv_aia_device_ops = {
507 .name = "kvm-riscv-aia",
508 .create = aia_create,
509 .destroy = aia_destroy,
510 .set_attr = aia_set_attr,
511 .get_attr = aia_get_attr,
512 .has_attr = aia_has_attr,
515 int kvm_riscv_vcpu_aia_update(struct kvm_vcpu *vcpu)
517 /* Proceed only if AIA was initialized successfully */
518 if (!kvm_riscv_aia_initialized(vcpu->kvm))
521 /* Update the IMSIC HW state before entering guest mode */
522 return kvm_riscv_vcpu_aia_imsic_update(vcpu);
525 void kvm_riscv_vcpu_aia_reset(struct kvm_vcpu *vcpu)
527 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
528 struct kvm_vcpu_aia_csr *reset_csr =
529 &vcpu->arch.aia_context.guest_reset_csr;
531 if (!kvm_riscv_aia_available())
533 memcpy(csr, reset_csr, sizeof(*csr));
535 /* Proceed only if AIA was initialized successfully */
536 if (!kvm_riscv_aia_initialized(vcpu->kvm))
539 /* Reset the IMSIC context */
540 kvm_riscv_vcpu_aia_imsic_reset(vcpu);
543 int kvm_riscv_vcpu_aia_init(struct kvm_vcpu *vcpu)
545 struct kvm_vcpu_aia *vaia = &vcpu->arch.aia_context;
547 if (!kvm_riscv_aia_available())
551 * We don't do any memory allocations over here because these
552 * will be done after AIA device is initialized by the user-space.
554 * Refer, aia_init() implementation for more details.
557 /* Initialize default values in AIA vcpu context */
558 vaia->imsic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
559 vaia->hart_index = vcpu->vcpu_idx;
564 void kvm_riscv_vcpu_aia_deinit(struct kvm_vcpu *vcpu)
566 /* Proceed only if AIA was initialized successfully */
567 if (!kvm_riscv_aia_initialized(vcpu->kvm))
570 /* Cleanup IMSIC context */
571 kvm_riscv_vcpu_aia_imsic_cleanup(vcpu);
574 int kvm_riscv_aia_inject_msi_by_id(struct kvm *kvm, u32 hart_index,
575 u32 guest_index, u32 iid)
578 struct kvm_vcpu *vcpu;
580 /* Proceed only if AIA was initialized successfully */
581 if (!kvm_riscv_aia_initialized(kvm))
584 /* Inject MSI to matching VCPU */
585 kvm_for_each_vcpu(idx, vcpu, kvm) {
586 if (vcpu->arch.aia_context.hart_index == hart_index)
587 return kvm_riscv_vcpu_aia_imsic_inject(vcpu,
595 int kvm_riscv_aia_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
599 struct kvm_vcpu *vcpu;
600 u32 g, toff, iid = msi->data;
601 struct kvm_aia *aia = &kvm->arch.aia;
602 gpa_t target = (((gpa_t)msi->address_hi) << 32) | msi->address_lo;
604 /* Proceed only if AIA was initialized successfully */
605 if (!kvm_riscv_aia_initialized(kvm))
608 /* Convert target address to target PPN */
609 tppn = target >> IMSIC_MMIO_PAGE_SHIFT;
611 /* Extract and clear Guest ID from target PPN */
612 g = tppn & (BIT(aia->nr_guest_bits) - 1);
613 tppn &= ~((gpa_t)(BIT(aia->nr_guest_bits) - 1));
615 /* Inject MSI to matching VCPU */
616 kvm_for_each_vcpu(idx, vcpu, kvm) {
617 ippn = vcpu->arch.aia_context.imsic_addr >>
618 IMSIC_MMIO_PAGE_SHIFT;
620 toff = target & (IMSIC_MMIO_PAGE_SZ - 1);
621 return kvm_riscv_vcpu_aia_imsic_inject(vcpu, g,
629 int kvm_riscv_aia_inject_irq(struct kvm *kvm, unsigned int irq, bool level)
631 /* Proceed only if AIA was initialized successfully */
632 if (!kvm_riscv_aia_initialized(kvm))
635 /* Inject interrupt level change in APLIC */
636 return kvm_riscv_aia_aplic_inject(kvm, irq, level);
639 void kvm_riscv_aia_init_vm(struct kvm *kvm)
641 struct kvm_aia *aia = &kvm->arch.aia;
643 if (!kvm_riscv_aia_available())
647 * We don't do any memory allocations over here because these
648 * will be done after AIA device is initialized by the user-space.
650 * Refer, aia_init() implementation for more details.
653 /* Initialize default values in AIA global context */
654 aia->mode = (kvm_riscv_aia_nr_hgei) ?
655 KVM_DEV_RISCV_AIA_MODE_AUTO : KVM_DEV_RISCV_AIA_MODE_EMUL;
656 aia->nr_ids = kvm_riscv_aia_max_ids - 1;
658 aia->nr_group_bits = 0;
659 aia->nr_group_shift = KVM_DEV_RISCV_AIA_GROUP_SHIFT_MIN;
660 aia->nr_hart_bits = 0;
661 aia->nr_guest_bits = 0;
662 aia->aplic_addr = KVM_RISCV_AIA_UNDEF_ADDR;
665 void kvm_riscv_aia_destroy_vm(struct kvm *kvm)
667 /* Proceed only if AIA was initialized successfully */
668 if (!kvm_riscv_aia_initialized(kvm))
671 /* Cleanup APLIC context */
672 kvm_riscv_aia_aplic_cleanup(kvm);