1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
7 * Anup Patel <apatel@ventanamicro.com>
10 #include <linux/kvm_host.h>
11 #include <linux/math.h>
12 #include <linux/spinlock.h>
13 #include <linux/swab.h>
14 #include <kvm/iodev.h>
15 #include <asm/kvm_aia_aplic.h>
21 #define APLIC_IRQ_STATE_PENDING BIT(0)
22 #define APLIC_IRQ_STATE_ENABLED BIT(1)
23 #define APLIC_IRQ_STATE_ENPEND (APLIC_IRQ_STATE_PENDING | \
24 APLIC_IRQ_STATE_ENABLED)
25 #define APLIC_IRQ_STATE_INPUT BIT(8)
30 struct kvm_io_device iodev;
37 struct aplic_irq *irqs;
40 static u32 aplic_read_sourcecfg(struct aplic *aplic, u32 irq)
44 struct aplic_irq *irqd;
46 if (!irq || aplic->nr_irqs <= irq)
48 irqd = &aplic->irqs[irq];
50 raw_spin_lock_irqsave(&irqd->lock, flags);
51 ret = irqd->sourcecfg;
52 raw_spin_unlock_irqrestore(&irqd->lock, flags);
57 static void aplic_write_sourcecfg(struct aplic *aplic, u32 irq, u32 val)
60 struct aplic_irq *irqd;
62 if (!irq || aplic->nr_irqs <= irq)
64 irqd = &aplic->irqs[irq];
66 if (val & APLIC_SOURCECFG_D)
69 val &= APLIC_SOURCECFG_SM_MASK;
71 raw_spin_lock_irqsave(&irqd->lock, flags);
72 irqd->sourcecfg = val;
73 raw_spin_unlock_irqrestore(&irqd->lock, flags);
76 static u32 aplic_read_target(struct aplic *aplic, u32 irq)
80 struct aplic_irq *irqd;
82 if (!irq || aplic->nr_irqs <= irq)
84 irqd = &aplic->irqs[irq];
86 raw_spin_lock_irqsave(&irqd->lock, flags);
88 raw_spin_unlock_irqrestore(&irqd->lock, flags);
93 static void aplic_write_target(struct aplic *aplic, u32 irq, u32 val)
96 struct aplic_irq *irqd;
98 if (!irq || aplic->nr_irqs <= irq)
100 irqd = &aplic->irqs[irq];
102 val &= APLIC_TARGET_EIID_MASK |
103 (APLIC_TARGET_HART_IDX_MASK << APLIC_TARGET_HART_IDX_SHIFT) |
104 (APLIC_TARGET_GUEST_IDX_MASK << APLIC_TARGET_GUEST_IDX_SHIFT);
106 raw_spin_lock_irqsave(&irqd->lock, flags);
108 raw_spin_unlock_irqrestore(&irqd->lock, flags);
111 static bool aplic_read_pending(struct aplic *aplic, u32 irq)
115 struct aplic_irq *irqd;
117 if (!irq || aplic->nr_irqs <= irq)
119 irqd = &aplic->irqs[irq];
121 raw_spin_lock_irqsave(&irqd->lock, flags);
122 ret = (irqd->state & APLIC_IRQ_STATE_PENDING) ? true : false;
123 raw_spin_unlock_irqrestore(&irqd->lock, flags);
128 static void aplic_write_pending(struct aplic *aplic, u32 irq, bool pending)
130 unsigned long flags, sm;
131 struct aplic_irq *irqd;
133 if (!irq || aplic->nr_irqs <= irq)
135 irqd = &aplic->irqs[irq];
137 raw_spin_lock_irqsave(&irqd->lock, flags);
139 sm = irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK;
140 if (sm == APLIC_SOURCECFG_SM_INACTIVE)
141 goto skip_write_pending;
143 if (sm == APLIC_SOURCECFG_SM_LEVEL_HIGH ||
144 sm == APLIC_SOURCECFG_SM_LEVEL_LOW) {
146 goto skip_write_pending;
147 if ((irqd->state & APLIC_IRQ_STATE_INPUT) &&
148 sm == APLIC_SOURCECFG_SM_LEVEL_LOW)
149 goto skip_write_pending;
150 if (!(irqd->state & APLIC_IRQ_STATE_INPUT) &&
151 sm == APLIC_SOURCECFG_SM_LEVEL_HIGH)
152 goto skip_write_pending;
156 irqd->state |= APLIC_IRQ_STATE_PENDING;
158 irqd->state &= ~APLIC_IRQ_STATE_PENDING;
161 raw_spin_unlock_irqrestore(&irqd->lock, flags);
164 static bool aplic_read_enabled(struct aplic *aplic, u32 irq)
168 struct aplic_irq *irqd;
170 if (!irq || aplic->nr_irqs <= irq)
172 irqd = &aplic->irqs[irq];
174 raw_spin_lock_irqsave(&irqd->lock, flags);
175 ret = (irqd->state & APLIC_IRQ_STATE_ENABLED) ? true : false;
176 raw_spin_unlock_irqrestore(&irqd->lock, flags);
181 static void aplic_write_enabled(struct aplic *aplic, u32 irq, bool enabled)
184 struct aplic_irq *irqd;
186 if (!irq || aplic->nr_irqs <= irq)
188 irqd = &aplic->irqs[irq];
190 raw_spin_lock_irqsave(&irqd->lock, flags);
192 irqd->state |= APLIC_IRQ_STATE_ENABLED;
194 irqd->state &= ~APLIC_IRQ_STATE_ENABLED;
195 raw_spin_unlock_irqrestore(&irqd->lock, flags);
198 static bool aplic_read_input(struct aplic *aplic, u32 irq)
200 u32 sourcecfg, sm, raw_input, irq_inverted;
201 struct aplic_irq *irqd;
205 if (!irq || aplic->nr_irqs <= irq)
207 irqd = &aplic->irqs[irq];
209 raw_spin_lock_irqsave(&irqd->lock, flags);
211 sourcecfg = irqd->sourcecfg;
212 if (sourcecfg & APLIC_SOURCECFG_D)
215 sm = sourcecfg & APLIC_SOURCECFG_SM_MASK;
216 if (sm == APLIC_SOURCECFG_SM_INACTIVE)
219 raw_input = (irqd->state & APLIC_IRQ_STATE_INPUT) ? 1 : 0;
220 irq_inverted = (sm == APLIC_SOURCECFG_SM_LEVEL_LOW ||
221 sm == APLIC_SOURCECFG_SM_EDGE_FALL) ? 1 : 0;
222 ret = !!(raw_input ^ irq_inverted);
225 raw_spin_unlock_irqrestore(&irqd->lock, flags);
230 static void aplic_inject_msi(struct kvm *kvm, u32 irq, u32 target)
232 u32 hart_idx, guest_idx, eiid;
234 hart_idx = target >> APLIC_TARGET_HART_IDX_SHIFT;
235 hart_idx &= APLIC_TARGET_HART_IDX_MASK;
236 guest_idx = target >> APLIC_TARGET_GUEST_IDX_SHIFT;
237 guest_idx &= APLIC_TARGET_GUEST_IDX_MASK;
238 eiid = target & APLIC_TARGET_EIID_MASK;
239 kvm_riscv_aia_inject_msi_by_id(kvm, hart_idx, guest_idx, eiid);
242 static void aplic_update_irq_range(struct kvm *kvm, u32 first, u32 last)
247 struct aplic_irq *irqd;
248 struct aplic *aplic = kvm->arch.aia.aplic_state;
250 if (!(aplic->domaincfg & APLIC_DOMAINCFG_IE))
253 for (irq = first; irq <= last; irq++) {
254 if (!irq || aplic->nr_irqs <= irq)
256 irqd = &aplic->irqs[irq];
258 raw_spin_lock_irqsave(&irqd->lock, flags);
261 target = irqd->target;
262 if ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
263 APLIC_IRQ_STATE_ENPEND) {
264 irqd->state &= ~APLIC_IRQ_STATE_PENDING;
268 raw_spin_unlock_irqrestore(&irqd->lock, flags);
271 aplic_inject_msi(kvm, irq, target);
275 int kvm_riscv_aia_aplic_inject(struct kvm *kvm, u32 source, bool level)
278 bool inject = false, ie;
280 struct aplic_irq *irqd;
281 struct aplic *aplic = kvm->arch.aia.aplic_state;
283 if (!aplic || !source || (aplic->nr_irqs <= source))
285 irqd = &aplic->irqs[source];
286 ie = (aplic->domaincfg & APLIC_DOMAINCFG_IE) ? true : false;
288 raw_spin_lock_irqsave(&irqd->lock, flags);
290 if (irqd->sourcecfg & APLIC_SOURCECFG_D)
293 switch (irqd->sourcecfg & APLIC_SOURCECFG_SM_MASK) {
294 case APLIC_SOURCECFG_SM_EDGE_RISE:
295 if (level && !(irqd->state & APLIC_IRQ_STATE_INPUT) &&
296 !(irqd->state & APLIC_IRQ_STATE_PENDING))
297 irqd->state |= APLIC_IRQ_STATE_PENDING;
299 case APLIC_SOURCECFG_SM_EDGE_FALL:
300 if (!level && (irqd->state & APLIC_IRQ_STATE_INPUT) &&
301 !(irqd->state & APLIC_IRQ_STATE_PENDING))
302 irqd->state |= APLIC_IRQ_STATE_PENDING;
304 case APLIC_SOURCECFG_SM_LEVEL_HIGH:
305 if (level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
306 irqd->state |= APLIC_IRQ_STATE_PENDING;
308 case APLIC_SOURCECFG_SM_LEVEL_LOW:
309 if (!level && !(irqd->state & APLIC_IRQ_STATE_PENDING))
310 irqd->state |= APLIC_IRQ_STATE_PENDING;
315 irqd->state |= APLIC_IRQ_STATE_INPUT;
317 irqd->state &= ~APLIC_IRQ_STATE_INPUT;
319 target = irqd->target;
320 if (ie && ((irqd->state & APLIC_IRQ_STATE_ENPEND) ==
321 APLIC_IRQ_STATE_ENPEND)) {
322 irqd->state &= ~APLIC_IRQ_STATE_PENDING;
327 raw_spin_unlock_irqrestore(&irqd->lock, flags);
330 aplic_inject_msi(kvm, source, target);
335 static u32 aplic_read_input_word(struct aplic *aplic, u32 word)
339 for (i = 0; i < 32; i++)
340 ret |= aplic_read_input(aplic, word * 32 + i) ? BIT(i) : 0;
345 static u32 aplic_read_pending_word(struct aplic *aplic, u32 word)
349 for (i = 0; i < 32; i++)
350 ret |= aplic_read_pending(aplic, word * 32 + i) ? BIT(i) : 0;
355 static void aplic_write_pending_word(struct aplic *aplic, u32 word,
356 u32 val, bool pending)
360 for (i = 0; i < 32; i++) {
362 aplic_write_pending(aplic, word * 32 + i, pending);
366 static u32 aplic_read_enabled_word(struct aplic *aplic, u32 word)
370 for (i = 0; i < 32; i++)
371 ret |= aplic_read_enabled(aplic, word * 32 + i) ? BIT(i) : 0;
376 static void aplic_write_enabled_word(struct aplic *aplic, u32 word,
377 u32 val, bool enabled)
381 for (i = 0; i < 32; i++) {
383 aplic_write_enabled(aplic, word * 32 + i, enabled);
387 static int aplic_mmio_read_offset(struct kvm *kvm, gpa_t off, u32 *val32)
390 struct aplic *aplic = kvm->arch.aia.aplic_state;
392 if ((off & 0x3) != 0)
395 if (off == APLIC_DOMAINCFG) {
396 *val32 = APLIC_DOMAINCFG_RDONLY |
397 aplic->domaincfg | APLIC_DOMAINCFG_DM;
398 } else if ((off >= APLIC_SOURCECFG_BASE) &&
399 (off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
400 i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
401 *val32 = aplic_read_sourcecfg(aplic, i);
402 } else if ((off >= APLIC_SETIP_BASE) &&
403 (off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
404 i = (off - APLIC_SETIP_BASE) >> 2;
405 *val32 = aplic_read_pending_word(aplic, i);
406 } else if (off == APLIC_SETIPNUM) {
408 } else if ((off >= APLIC_CLRIP_BASE) &&
409 (off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
410 i = (off - APLIC_CLRIP_BASE) >> 2;
411 *val32 = aplic_read_input_word(aplic, i);
412 } else if (off == APLIC_CLRIPNUM) {
414 } else if ((off >= APLIC_SETIE_BASE) &&
415 (off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
416 i = (off - APLIC_SETIE_BASE) >> 2;
417 *val32 = aplic_read_enabled_word(aplic, i);
418 } else if (off == APLIC_SETIENUM) {
420 } else if ((off >= APLIC_CLRIE_BASE) &&
421 (off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
423 } else if (off == APLIC_CLRIENUM) {
425 } else if (off == APLIC_SETIPNUM_LE) {
427 } else if (off == APLIC_SETIPNUM_BE) {
429 } else if (off == APLIC_GENMSI) {
430 *val32 = aplic->genmsi;
431 } else if ((off >= APLIC_TARGET_BASE) &&
432 (off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
433 i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
434 *val32 = aplic_read_target(aplic, i);
441 static int aplic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
442 gpa_t addr, int len, void *val)
447 return aplic_mmio_read_offset(vcpu->kvm,
448 addr - vcpu->kvm->arch.aia.aplic_addr,
452 static int aplic_mmio_write_offset(struct kvm *kvm, gpa_t off, u32 val32)
455 struct aplic *aplic = kvm->arch.aia.aplic_state;
457 if ((off & 0x3) != 0)
460 if (off == APLIC_DOMAINCFG) {
461 /* Only IE bit writeable */
462 aplic->domaincfg = val32 & APLIC_DOMAINCFG_IE;
463 } else if ((off >= APLIC_SOURCECFG_BASE) &&
464 (off < (APLIC_SOURCECFG_BASE + (aplic->nr_irqs - 1) * 4))) {
465 i = ((off - APLIC_SOURCECFG_BASE) >> 2) + 1;
466 aplic_write_sourcecfg(aplic, i, val32);
467 } else if ((off >= APLIC_SETIP_BASE) &&
468 (off < (APLIC_SETIP_BASE + aplic->nr_words * 4))) {
469 i = (off - APLIC_SETIP_BASE) >> 2;
470 aplic_write_pending_word(aplic, i, val32, true);
471 } else if (off == APLIC_SETIPNUM) {
472 aplic_write_pending(aplic, val32, true);
473 } else if ((off >= APLIC_CLRIP_BASE) &&
474 (off < (APLIC_CLRIP_BASE + aplic->nr_words * 4))) {
475 i = (off - APLIC_CLRIP_BASE) >> 2;
476 aplic_write_pending_word(aplic, i, val32, false);
477 } else if (off == APLIC_CLRIPNUM) {
478 aplic_write_pending(aplic, val32, false);
479 } else if ((off >= APLIC_SETIE_BASE) &&
480 (off < (APLIC_SETIE_BASE + aplic->nr_words * 4))) {
481 i = (off - APLIC_SETIE_BASE) >> 2;
482 aplic_write_enabled_word(aplic, i, val32, true);
483 } else if (off == APLIC_SETIENUM) {
484 aplic_write_enabled(aplic, val32, true);
485 } else if ((off >= APLIC_CLRIE_BASE) &&
486 (off < (APLIC_CLRIE_BASE + aplic->nr_words * 4))) {
487 i = (off - APLIC_CLRIE_BASE) >> 2;
488 aplic_write_enabled_word(aplic, i, val32, false);
489 } else if (off == APLIC_CLRIENUM) {
490 aplic_write_enabled(aplic, val32, false);
491 } else if (off == APLIC_SETIPNUM_LE) {
492 aplic_write_pending(aplic, val32, true);
493 } else if (off == APLIC_SETIPNUM_BE) {
494 aplic_write_pending(aplic, __swab32(val32), true);
495 } else if (off == APLIC_GENMSI) {
496 aplic->genmsi = val32 & ~(APLIC_TARGET_GUEST_IDX_MASK <<
497 APLIC_TARGET_GUEST_IDX_SHIFT);
498 kvm_riscv_aia_inject_msi_by_id(kvm,
499 val32 >> APLIC_TARGET_HART_IDX_SHIFT, 0,
500 val32 & APLIC_TARGET_EIID_MASK);
501 } else if ((off >= APLIC_TARGET_BASE) &&
502 (off < (APLIC_TARGET_BASE + (aplic->nr_irqs - 1) * 4))) {
503 i = ((off - APLIC_TARGET_BASE) >> 2) + 1;
504 aplic_write_target(aplic, i, val32);
508 aplic_update_irq_range(kvm, 1, aplic->nr_irqs - 1);
513 static int aplic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
514 gpa_t addr, int len, const void *val)
519 return aplic_mmio_write_offset(vcpu->kvm,
520 addr - vcpu->kvm->arch.aia.aplic_addr,
521 *((const u32 *)val));
524 static struct kvm_io_device_ops aplic_iodoev_ops = {
525 .read = aplic_mmio_read,
526 .write = aplic_mmio_write,
529 int kvm_riscv_aia_aplic_set_attr(struct kvm *kvm, unsigned long type, u32 v)
533 if (!kvm->arch.aia.aplic_state)
536 rc = aplic_mmio_write_offset(kvm, type, v);
543 int kvm_riscv_aia_aplic_get_attr(struct kvm *kvm, unsigned long type, u32 *v)
547 if (!kvm->arch.aia.aplic_state)
550 rc = aplic_mmio_read_offset(kvm, type, v);
557 int kvm_riscv_aia_aplic_has_attr(struct kvm *kvm, unsigned long type)
562 if (!kvm->arch.aia.aplic_state)
565 rc = aplic_mmio_read_offset(kvm, type, &val);
572 int kvm_riscv_aia_aplic_init(struct kvm *kvm)
577 /* Do nothing if we have zero sources */
578 if (!kvm->arch.aia.nr_sources)
581 /* Allocate APLIC global state */
582 aplic = kzalloc(sizeof(*aplic), GFP_KERNEL);
585 kvm->arch.aia.aplic_state = aplic;
587 /* Setup APLIC IRQs */
588 aplic->nr_irqs = kvm->arch.aia.nr_sources + 1;
589 aplic->nr_words = DIV_ROUND_UP(aplic->nr_irqs, 32);
590 aplic->irqs = kcalloc(aplic->nr_irqs,
591 sizeof(*aplic->irqs), GFP_KERNEL);
594 goto fail_free_aplic;
596 for (i = 0; i < aplic->nr_irqs; i++)
597 raw_spin_lock_init(&aplic->irqs[i].lock);
599 /* Setup IO device */
600 kvm_iodevice_init(&aplic->iodev, &aplic_iodoev_ops);
601 mutex_lock(&kvm->slots_lock);
602 ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
603 kvm->arch.aia.aplic_addr,
604 KVM_DEV_RISCV_APLIC_SIZE,
606 mutex_unlock(&kvm->slots_lock);
608 goto fail_free_aplic_irqs;
610 /* Setup default IRQ routing */
611 ret = kvm_riscv_setup_default_irq_routing(kvm, aplic->nr_irqs);
613 goto fail_unreg_iodev;
618 mutex_lock(&kvm->slots_lock);
619 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
620 mutex_unlock(&kvm->slots_lock);
621 fail_free_aplic_irqs:
624 kvm->arch.aia.aplic_state = NULL;
629 void kvm_riscv_aia_aplic_cleanup(struct kvm *kvm)
631 struct aplic *aplic = kvm->arch.aia.aplic_state;
636 mutex_lock(&kvm->slots_lock);
637 kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS, &aplic->iodev);
638 mutex_unlock(&kvm->slots_lock);
642 kvm->arch.aia.aplic_state = NULL;