1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2015 Linaro Ltd.
4 * Author: Shannon Zhao <shannon.zhao@linaro.org>
9 #include <linux/kvm_host.h>
10 #include <linux/list.h>
11 #include <linux/perf_event.h>
12 #include <linux/perf/arm_pmu.h>
13 #include <linux/uaccess.h>
14 #include <asm/kvm_emulate.h>
15 #include <kvm/arm_pmu.h>
16 #include <kvm/arm_vgic.h>
18 #define PERF_ATTR_CFG1_COUNTER_64BIT BIT(0)
20 DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available);
22 static LIST_HEAD(arm_pmus);
23 static DEFINE_MUTEX(arm_pmus_lock);
25 static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc);
26 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc);
28 static struct kvm_vcpu *kvm_pmc_to_vcpu(const struct kvm_pmc *pmc)
30 return container_of(pmc, struct kvm_vcpu, arch.pmu.pmc[pmc->idx]);
33 static struct kvm_pmc *kvm_vcpu_idx_to_pmc(struct kvm_vcpu *vcpu, int cnt_idx)
35 return &vcpu->arch.pmu.pmc[cnt_idx];
38 static u32 kvm_pmu_event_mask(struct kvm *kvm)
42 pmuver = kvm->arch.arm_pmu->pmuver;
45 case ID_AA64DFR0_EL1_PMUVer_IMP:
47 case ID_AA64DFR0_EL1_PMUVer_V3P1:
48 case ID_AA64DFR0_EL1_PMUVer_V3P4:
49 case ID_AA64DFR0_EL1_PMUVer_V3P5:
50 case ID_AA64DFR0_EL1_PMUVer_V3P7:
51 return GENMASK(15, 0);
52 default: /* Shouldn't be here, just for sanity */
53 WARN_ONCE(1, "Unknown PMU version %d\n", pmuver);
59 * kvm_pmc_is_64bit - determine if counter is 64bit
60 * @pmc: counter context
62 static bool kvm_pmc_is_64bit(struct kvm_pmc *pmc)
64 return (pmc->idx == ARMV8_PMU_CYCLE_IDX ||
65 kvm_pmu_is_3p5(kvm_pmc_to_vcpu(pmc)));
68 static bool kvm_pmc_has_64bit_overflow(struct kvm_pmc *pmc)
70 u64 val = __vcpu_sys_reg(kvm_pmc_to_vcpu(pmc), PMCR_EL0);
72 return (pmc->idx < ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LP)) ||
73 (pmc->idx == ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LC));
76 static bool kvm_pmu_counter_can_chain(struct kvm_pmc *pmc)
78 return (!(pmc->idx & 1) && (pmc->idx + 1) < ARMV8_PMU_CYCLE_IDX &&
79 !kvm_pmc_has_64bit_overflow(pmc));
82 static u32 counter_index_to_reg(u64 idx)
84 return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + idx;
87 static u32 counter_index_to_evtreg(u64 idx)
89 return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + idx;
92 static u64 kvm_pmu_get_pmc_value(struct kvm_pmc *pmc)
94 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
95 u64 counter, reg, enabled, running;
97 reg = counter_index_to_reg(pmc->idx);
98 counter = __vcpu_sys_reg(vcpu, reg);
101 * The real counter value is equal to the value of counter register plus
102 * the value perf event counts.
105 counter += perf_event_read_value(pmc->perf_event, &enabled,
108 if (!kvm_pmc_is_64bit(pmc))
109 counter = lower_32_bits(counter);
115 * kvm_pmu_get_counter_value - get PMU counter value
116 * @vcpu: The vcpu pointer
117 * @select_idx: The counter index
119 u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
121 if (!kvm_vcpu_has_pmu(vcpu))
124 return kvm_pmu_get_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx));
127 static void kvm_pmu_set_pmc_value(struct kvm_pmc *pmc, u64 val, bool force)
129 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
132 kvm_pmu_release_perf_event(pmc);
134 reg = counter_index_to_reg(pmc->idx);
136 if (vcpu_mode_is_32bit(vcpu) && pmc->idx != ARMV8_PMU_CYCLE_IDX &&
139 * Even with PMUv3p5, AArch32 cannot write to the top
140 * 32bit of the counters. The only possible course of
141 * action is to use PMCR.P, which will reset them to
142 * 0 (the only use of the 'force' parameter).
144 val = __vcpu_sys_reg(vcpu, reg) & GENMASK(63, 32);
145 val |= lower_32_bits(val);
148 __vcpu_sys_reg(vcpu, reg) = val;
150 /* Recreate the perf event to reflect the updated sample_period */
151 kvm_pmu_create_perf_event(pmc);
155 * kvm_pmu_set_counter_value - set PMU counter value
156 * @vcpu: The vcpu pointer
157 * @select_idx: The counter index
158 * @val: The counter value
160 void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
162 if (!kvm_vcpu_has_pmu(vcpu))
165 kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx), val, false);
169 * kvm_pmu_release_perf_event - remove the perf event
170 * @pmc: The PMU counter pointer
172 static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc)
174 if (pmc->perf_event) {
175 perf_event_disable(pmc->perf_event);
176 perf_event_release_kernel(pmc->perf_event);
177 pmc->perf_event = NULL;
182 * kvm_pmu_stop_counter - stop PMU counter
183 * @pmc: The PMU counter pointer
185 * If this counter has been configured to monitor some event, release it here.
187 static void kvm_pmu_stop_counter(struct kvm_pmc *pmc)
189 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
192 if (!pmc->perf_event)
195 val = kvm_pmu_get_pmc_value(pmc);
197 reg = counter_index_to_reg(pmc->idx);
199 __vcpu_sys_reg(vcpu, reg) = val;
201 kvm_pmu_release_perf_event(pmc);
205 * kvm_pmu_vcpu_init - assign pmu counter idx for cpu
206 * @vcpu: The vcpu pointer
209 void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu)
212 struct kvm_pmu *pmu = &vcpu->arch.pmu;
214 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
219 * kvm_pmu_vcpu_reset - reset pmu state for cpu
220 * @vcpu: The vcpu pointer
223 void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
225 unsigned long mask = kvm_pmu_valid_counter_mask(vcpu);
228 for_each_set_bit(i, &mask, 32)
229 kvm_pmu_stop_counter(kvm_vcpu_idx_to_pmc(vcpu, i));
233 * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu
234 * @vcpu: The vcpu pointer
237 void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)
241 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++)
242 kvm_pmu_release_perf_event(kvm_vcpu_idx_to_pmc(vcpu, i));
243 irq_work_sync(&vcpu->arch.pmu.overflow_work);
246 u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
248 u64 val = __vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;
250 val &= ARMV8_PMU_PMCR_N_MASK;
252 return BIT(ARMV8_PMU_CYCLE_IDX);
254 return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX);
258 * kvm_pmu_enable_counter_mask - enable selected PMU counters
259 * @vcpu: The vcpu pointer
260 * @val: the value guest writes to PMCNTENSET register
262 * Call perf_event_enable to start counting the perf event
264 void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
267 if (!kvm_vcpu_has_pmu(vcpu))
270 if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val)
273 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
279 pmc = kvm_vcpu_idx_to_pmc(vcpu, i);
281 if (!pmc->perf_event) {
282 kvm_pmu_create_perf_event(pmc);
284 perf_event_enable(pmc->perf_event);
285 if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
286 kvm_debug("fail to enable perf event\n");
292 * kvm_pmu_disable_counter_mask - disable selected PMU counters
293 * @vcpu: The vcpu pointer
294 * @val: the value guest writes to PMCNTENCLR register
296 * Call perf_event_disable to stop counting the perf event
298 void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val)
302 if (!kvm_vcpu_has_pmu(vcpu) || !val)
305 for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
311 pmc = kvm_vcpu_idx_to_pmc(vcpu, i);
314 perf_event_disable(pmc->perf_event);
318 static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
322 if ((__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) {
323 reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0);
324 reg &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
325 reg &= __vcpu_sys_reg(vcpu, PMINTENSET_EL1);
331 static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
333 struct kvm_pmu *pmu = &vcpu->arch.pmu;
336 if (!kvm_vcpu_has_pmu(vcpu))
339 overflow = !!kvm_pmu_overflow_status(vcpu);
340 if (pmu->irq_level == overflow)
343 pmu->irq_level = overflow;
345 if (likely(irqchip_in_kernel(vcpu->kvm))) {
346 int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
347 pmu->irq_num, overflow, pmu);
352 bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
354 struct kvm_pmu *pmu = &vcpu->arch.pmu;
355 struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
356 bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU;
358 if (likely(irqchip_in_kernel(vcpu->kvm)))
361 return pmu->irq_level != run_level;
365 * Reflect the PMU overflow interrupt output level into the kvm_run structure
367 void kvm_pmu_update_run(struct kvm_vcpu *vcpu)
369 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
371 /* Populate the timer bitmap for user space */
372 regs->device_irq_level &= ~KVM_ARM_DEV_PMU;
373 if (vcpu->arch.pmu.irq_level)
374 regs->device_irq_level |= KVM_ARM_DEV_PMU;
378 * kvm_pmu_flush_hwstate - flush pmu state to cpu
379 * @vcpu: The vcpu pointer
381 * Check if the PMU has overflowed while we were running in the host, and inject
382 * an interrupt if that was the case.
384 void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)
386 kvm_pmu_update_state(vcpu);
390 * kvm_pmu_sync_hwstate - sync pmu state from cpu
391 * @vcpu: The vcpu pointer
393 * Check if the PMU has overflowed while we were running in the guest, and
394 * inject an interrupt if that was the case.
396 void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
398 kvm_pmu_update_state(vcpu);
402 * When perf interrupt is an NMI, we cannot safely notify the vcpu corresponding
404 * This is why we need a callback to do it once outside of the NMI context.
406 static void kvm_pmu_perf_overflow_notify_vcpu(struct irq_work *work)
408 struct kvm_vcpu *vcpu;
410 vcpu = container_of(work, struct kvm_vcpu, arch.pmu.overflow_work);
415 * Perform an increment on any of the counters described in @mask,
416 * generating the overflow if required, and propagate it as a chained
419 static void kvm_pmu_counter_increment(struct kvm_vcpu *vcpu,
420 unsigned long mask, u32 event)
424 if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
427 /* Weed out disabled counters */
428 mask &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
430 for_each_set_bit(i, &mask, ARMV8_PMU_CYCLE_IDX) {
431 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i);
434 /* Filter on event type */
435 type = __vcpu_sys_reg(vcpu, counter_index_to_evtreg(i));
436 type &= kvm_pmu_event_mask(vcpu->kvm);
440 /* Increment this counter */
441 reg = __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) + 1;
442 if (!kvm_pmc_is_64bit(pmc))
443 reg = lower_32_bits(reg);
444 __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) = reg;
446 /* No overflow? move on */
447 if (kvm_pmc_has_64bit_overflow(pmc) ? reg : lower_32_bits(reg))
451 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i);
453 if (kvm_pmu_counter_can_chain(pmc))
454 kvm_pmu_counter_increment(vcpu, BIT(i + 1),
455 ARMV8_PMUV3_PERFCTR_CHAIN);
459 /* Compute the sample period for a given counter value */
460 static u64 compute_period(struct kvm_pmc *pmc, u64 counter)
464 if (kvm_pmc_is_64bit(pmc) && kvm_pmc_has_64bit_overflow(pmc))
465 val = (-counter) & GENMASK(63, 0);
467 val = (-counter) & GENMASK(31, 0);
473 * When the perf event overflows, set the overflow status and inform the vcpu.
475 static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
476 struct perf_sample_data *data,
477 struct pt_regs *regs)
479 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
480 struct arm_pmu *cpu_pmu = to_arm_pmu(perf_event->pmu);
481 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
485 cpu_pmu->pmu.stop(perf_event, PERF_EF_UPDATE);
488 * Reset the sample period to the architectural limit,
489 * i.e. the point where the counter overflows.
491 period = compute_period(pmc, local64_read(&perf_event->count));
493 local64_set(&perf_event->hw.period_left, 0);
494 perf_event->attr.sample_period = period;
495 perf_event->hw.sample_period = period;
497 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx);
499 if (kvm_pmu_counter_can_chain(pmc))
500 kvm_pmu_counter_increment(vcpu, BIT(idx + 1),
501 ARMV8_PMUV3_PERFCTR_CHAIN);
503 if (kvm_pmu_overflow_status(vcpu)) {
504 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
509 irq_work_queue(&vcpu->arch.pmu.overflow_work);
512 cpu_pmu->pmu.start(perf_event, PERF_EF_RELOAD);
516 * kvm_pmu_software_increment - do software increment
517 * @vcpu: The vcpu pointer
518 * @val: the value guest writes to PMSWINC register
520 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
522 kvm_pmu_counter_increment(vcpu, val, ARMV8_PMUV3_PERFCTR_SW_INCR);
526 * kvm_pmu_handle_pmcr - handle PMCR register
527 * @vcpu: The vcpu pointer
528 * @val: the value guest writes to PMCR register
530 void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)
534 if (!kvm_vcpu_has_pmu(vcpu))
537 /* Fixup PMCR_EL0 to reconcile the PMU version and the LP bit */
538 if (!kvm_pmu_is_3p5(vcpu))
539 val &= ~ARMV8_PMU_PMCR_LP;
541 __vcpu_sys_reg(vcpu, PMCR_EL0) = val;
543 if (val & ARMV8_PMU_PMCR_E) {
544 kvm_pmu_enable_counter_mask(vcpu,
545 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0));
547 kvm_pmu_disable_counter_mask(vcpu,
548 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0));
551 if (val & ARMV8_PMU_PMCR_C)
552 kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);
554 if (val & ARMV8_PMU_PMCR_P) {
555 unsigned long mask = kvm_pmu_valid_counter_mask(vcpu);
556 mask &= ~BIT(ARMV8_PMU_CYCLE_IDX);
557 for_each_set_bit(i, &mask, 32)
558 kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, i), 0, true);
562 static bool kvm_pmu_counter_is_enabled(struct kvm_pmc *pmc)
564 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
565 return (__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
566 (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(pmc->idx));
570 * kvm_pmu_create_perf_event - create a perf event for a counter
571 * @pmc: Counter context
573 static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc)
575 struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
576 struct arm_pmu *arm_pmu = vcpu->kvm->arch.arm_pmu;
577 struct perf_event *event;
578 struct perf_event_attr attr;
579 u64 eventsel, reg, data;
581 reg = counter_index_to_evtreg(pmc->idx);
582 data = __vcpu_sys_reg(vcpu, reg);
584 kvm_pmu_stop_counter(pmc);
585 if (pmc->idx == ARMV8_PMU_CYCLE_IDX)
586 eventsel = ARMV8_PMUV3_PERFCTR_CPU_CYCLES;
588 eventsel = data & kvm_pmu_event_mask(vcpu->kvm);
591 * Neither SW increment nor chained events need to be backed
594 if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR ||
595 eventsel == ARMV8_PMUV3_PERFCTR_CHAIN)
599 * If we have a filter in place and that the event isn't allowed, do
600 * not install a perf event either.
602 if (vcpu->kvm->arch.pmu_filter &&
603 !test_bit(eventsel, vcpu->kvm->arch.pmu_filter))
606 memset(&attr, 0, sizeof(struct perf_event_attr));
607 attr.type = arm_pmu->pmu.type;
608 attr.size = sizeof(attr);
610 attr.disabled = !kvm_pmu_counter_is_enabled(pmc);
611 attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
612 attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
613 attr.exclude_hv = 1; /* Don't count EL2 events */
614 attr.exclude_host = 1; /* Don't count host events */
615 attr.config = eventsel;
618 * If counting with a 64bit counter, advertise it to the perf
619 * code, carefully dealing with the initial sample period
620 * which also depends on the overflow.
622 if (kvm_pmc_is_64bit(pmc))
623 attr.config1 |= PERF_ATTR_CFG1_COUNTER_64BIT;
625 attr.sample_period = compute_period(pmc, kvm_pmu_get_pmc_value(pmc));
627 event = perf_event_create_kernel_counter(&attr, -1, current,
628 kvm_pmu_perf_overflow, pmc);
631 pr_err_once("kvm: pmu event creation failed %ld\n",
636 pmc->perf_event = event;
640 * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
641 * @vcpu: The vcpu pointer
642 * @data: The data guest writes to PMXEVTYPER_EL0
643 * @select_idx: The number of selected counter
645 * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
646 * event with given hardware event number. Here we call perf_event API to
647 * emulate this action and create a kernel perf event for it.
649 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
652 struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, select_idx);
655 if (!kvm_vcpu_has_pmu(vcpu))
658 mask = ARMV8_PMU_EVTYPE_MASK;
659 mask &= ~ARMV8_PMU_EVTYPE_EVENT;
660 mask |= kvm_pmu_event_mask(vcpu->kvm);
662 reg = counter_index_to_evtreg(pmc->idx);
664 __vcpu_sys_reg(vcpu, reg) = data & mask;
666 kvm_pmu_create_perf_event(pmc);
669 void kvm_host_pmu_init(struct arm_pmu *pmu)
671 struct arm_pmu_entry *entry;
673 if (pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_NI ||
674 pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
677 mutex_lock(&arm_pmus_lock);
679 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
683 entry->arm_pmu = pmu;
684 list_add_tail(&entry->entry, &arm_pmus);
686 if (list_is_singular(&arm_pmus))
687 static_branch_enable(&kvm_arm_pmu_available);
690 mutex_unlock(&arm_pmus_lock);
693 static struct arm_pmu *kvm_pmu_probe_armpmu(void)
695 struct perf_event_attr attr = { };
696 struct perf_event *event;
697 struct arm_pmu *pmu = NULL;
700 * Create a dummy event that only counts user cycles. As we'll never
701 * leave this function with the event being live, it will never
702 * count anything. But it allows us to probe some of the PMU
703 * details. Yes, this is terrible.
705 attr.type = PERF_TYPE_RAW;
706 attr.size = sizeof(attr);
709 attr.exclude_user = 0;
710 attr.exclude_kernel = 1;
712 attr.exclude_host = 1;
713 attr.config = ARMV8_PMUV3_PERFCTR_CPU_CYCLES;
714 attr.sample_period = GENMASK(63, 0);
716 event = perf_event_create_kernel_counter(&attr, -1, current,
717 kvm_pmu_perf_overflow, &attr);
720 pr_err_once("kvm: pmu event creation failed %ld\n",
726 pmu = to_arm_pmu(event->pmu);
727 if (pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_NI ||
728 pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
732 perf_event_disable(event);
733 perf_event_release_kernel(event);
738 u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1)
740 unsigned long *bmap = vcpu->kvm->arch.pmu_filter;
742 int base, i, nr_events;
744 if (!kvm_vcpu_has_pmu(vcpu))
748 val = read_sysreg(pmceid0_el0);
749 /* always support CHAIN */
750 val |= BIT(ARMV8_PMUV3_PERFCTR_CHAIN);
753 val = read_sysreg(pmceid1_el0);
755 * Don't advertise STALL_SLOT, as PMMIR_EL0 is handled
758 if (vcpu->kvm->arch.arm_pmu->pmuver >= ID_AA64DFR0_EL1_PMUVer_V3P4)
759 val &= ~BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32);
766 nr_events = kvm_pmu_event_mask(vcpu->kvm) + 1;
768 for (i = 0; i < 32; i += 8) {
771 byte = bitmap_get_value8(bmap, base + i);
773 if (nr_events >= (0x4000 + base + 32)) {
774 byte = bitmap_get_value8(bmap, 0x4000 + base + i);
775 mask |= byte << (32 + i);
782 int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu)
784 if (!kvm_vcpu_has_pmu(vcpu))
787 if (!vcpu->arch.pmu.created)
791 * A valid interrupt configuration for the PMU is either to have a
792 * properly configured interrupt number and using an in-kernel
793 * irqchip, or to not have an in-kernel GIC and not set an IRQ.
795 if (irqchip_in_kernel(vcpu->kvm)) {
796 int irq = vcpu->arch.pmu.irq_num;
798 * If we are using an in-kernel vgic, at this point we know
799 * the vgic will be initialized, so we can check the PMU irq
800 * number against the dimensions of the vgic and make sure
803 if (!irq_is_ppi(irq) && !vgic_valid_spi(vcpu->kvm, irq))
805 } else if (kvm_arm_pmu_irq_initialized(vcpu)) {
809 /* One-off reload of the PMU on first run */
810 kvm_make_request(KVM_REQ_RELOAD_PMU, vcpu);
815 static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu)
817 if (irqchip_in_kernel(vcpu->kvm)) {
821 * If using the PMU with an in-kernel virtual GIC
822 * implementation, we require the GIC to be already
823 * initialized when initializing the PMU.
825 if (!vgic_initialized(vcpu->kvm))
828 if (!kvm_arm_pmu_irq_initialized(vcpu))
831 ret = kvm_vgic_set_owner(vcpu, vcpu->arch.pmu.irq_num,
837 init_irq_work(&vcpu->arch.pmu.overflow_work,
838 kvm_pmu_perf_overflow_notify_vcpu);
840 vcpu->arch.pmu.created = true;
845 * For one VM the interrupt type must be same for each vcpu.
846 * As a PPI, the interrupt number is the same for all vcpus,
847 * while as an SPI it must be a separate number per vcpu.
849 static bool pmu_irq_is_valid(struct kvm *kvm, int irq)
852 struct kvm_vcpu *vcpu;
854 kvm_for_each_vcpu(i, vcpu, kvm) {
855 if (!kvm_arm_pmu_irq_initialized(vcpu))
858 if (irq_is_ppi(irq)) {
859 if (vcpu->arch.pmu.irq_num != irq)
862 if (vcpu->arch.pmu.irq_num == irq)
870 static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id)
872 struct kvm *kvm = vcpu->kvm;
873 struct arm_pmu_entry *entry;
874 struct arm_pmu *arm_pmu;
877 mutex_lock(&kvm->lock);
878 mutex_lock(&arm_pmus_lock);
880 list_for_each_entry(entry, &arm_pmus, entry) {
881 arm_pmu = entry->arm_pmu;
882 if (arm_pmu->pmu.type == pmu_id) {
883 if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags) ||
884 (kvm->arch.pmu_filter && kvm->arch.arm_pmu != arm_pmu)) {
889 kvm->arch.arm_pmu = arm_pmu;
890 cpumask_copy(kvm->arch.supported_cpus, &arm_pmu->supported_cpus);
896 mutex_unlock(&arm_pmus_lock);
897 mutex_unlock(&kvm->lock);
901 int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
903 struct kvm *kvm = vcpu->kvm;
905 if (!kvm_vcpu_has_pmu(vcpu))
908 if (vcpu->arch.pmu.created)
911 mutex_lock(&kvm->lock);
912 if (!kvm->arch.arm_pmu) {
913 /* No PMU set, get the default one */
914 kvm->arch.arm_pmu = kvm_pmu_probe_armpmu();
915 if (!kvm->arch.arm_pmu) {
916 mutex_unlock(&kvm->lock);
920 mutex_unlock(&kvm->lock);
922 switch (attr->attr) {
923 case KVM_ARM_VCPU_PMU_V3_IRQ: {
924 int __user *uaddr = (int __user *)(long)attr->addr;
927 if (!irqchip_in_kernel(kvm))
930 if (get_user(irq, uaddr))
933 /* The PMU overflow interrupt can be a PPI or a valid SPI. */
934 if (!(irq_is_ppi(irq) || irq_is_spi(irq)))
937 if (!pmu_irq_is_valid(kvm, irq))
940 if (kvm_arm_pmu_irq_initialized(vcpu))
943 kvm_debug("Set kvm ARM PMU irq: %d\n", irq);
944 vcpu->arch.pmu.irq_num = irq;
947 case KVM_ARM_VCPU_PMU_V3_FILTER: {
948 struct kvm_pmu_event_filter __user *uaddr;
949 struct kvm_pmu_event_filter filter;
952 nr_events = kvm_pmu_event_mask(kvm) + 1;
954 uaddr = (struct kvm_pmu_event_filter __user *)(long)attr->addr;
956 if (copy_from_user(&filter, uaddr, sizeof(filter)))
959 if (((u32)filter.base_event + filter.nevents) > nr_events ||
960 (filter.action != KVM_PMU_EVENT_ALLOW &&
961 filter.action != KVM_PMU_EVENT_DENY))
964 mutex_lock(&kvm->lock);
966 if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags)) {
967 mutex_unlock(&kvm->lock);
971 if (!kvm->arch.pmu_filter) {
972 kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT);
973 if (!kvm->arch.pmu_filter) {
974 mutex_unlock(&kvm->lock);
979 * The default depends on the first applied filter.
980 * If it allows events, the default is to deny.
981 * Conversely, if the first filter denies a set of
982 * events, the default is to allow.
984 if (filter.action == KVM_PMU_EVENT_ALLOW)
985 bitmap_zero(kvm->arch.pmu_filter, nr_events);
987 bitmap_fill(kvm->arch.pmu_filter, nr_events);
990 if (filter.action == KVM_PMU_EVENT_ALLOW)
991 bitmap_set(kvm->arch.pmu_filter, filter.base_event, filter.nevents);
993 bitmap_clear(kvm->arch.pmu_filter, filter.base_event, filter.nevents);
995 mutex_unlock(&kvm->lock);
999 case KVM_ARM_VCPU_PMU_V3_SET_PMU: {
1000 int __user *uaddr = (int __user *)(long)attr->addr;
1003 if (get_user(pmu_id, uaddr))
1006 return kvm_arm_pmu_v3_set_pmu(vcpu, pmu_id);
1008 case KVM_ARM_VCPU_PMU_V3_INIT:
1009 return kvm_arm_pmu_v3_init(vcpu);
1015 int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1017 switch (attr->attr) {
1018 case KVM_ARM_VCPU_PMU_V3_IRQ: {
1019 int __user *uaddr = (int __user *)(long)attr->addr;
1022 if (!irqchip_in_kernel(vcpu->kvm))
1025 if (!kvm_vcpu_has_pmu(vcpu))
1028 if (!kvm_arm_pmu_irq_initialized(vcpu))
1031 irq = vcpu->arch.pmu.irq_num;
1032 return put_user(irq, uaddr);
1039 int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1041 switch (attr->attr) {
1042 case KVM_ARM_VCPU_PMU_V3_IRQ:
1043 case KVM_ARM_VCPU_PMU_V3_INIT:
1044 case KVM_ARM_VCPU_PMU_V3_FILTER:
1045 case KVM_ARM_VCPU_PMU_V3_SET_PMU:
1046 if (kvm_vcpu_has_pmu(vcpu))
1053 u8 kvm_arm_pmu_get_pmuver_limit(void)
1057 tmp = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1058 tmp = cpuid_feature_cap_perfmon_field(tmp,
1059 ID_AA64DFR0_EL1_PMUVer_SHIFT,
1060 ID_AA64DFR0_EL1_PMUVer_V3P5);
1061 return FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), tmp);