1 // SPDX-License-Identifier: GPL-2.0-only
3 * Kernel-based Virtual Machine -- Performance Monitoring Unit support
5 * Copyright 2015 Red Hat, Inc. and/or its affiliates.
8 * Avi Kivity <avi@redhat.com>
9 * Gleb Natapov <gleb@redhat.com>
10 * Wei Huang <wei@redhat.com>
12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 #include <linux/types.h>
15 #include <linux/kvm_host.h>
16 #include <linux/perf_event.h>
17 #include <linux/bsearch.h>
18 #include <linux/sort.h>
19 #include <asm/perf_event.h>
20 #include <asm/cpu_device_id.h>
26 /* This is enough to filter the vast majority of currently defined events. */
27 #define KVM_PMU_EVENT_FILTER_MAX_EVENTS 300
29 struct x86_pmu_capability __read_mostly kvm_pmu_cap;
30 EXPORT_SYMBOL_GPL(kvm_pmu_cap);
32 /* Precise Distribution of Instructions Retired (PDIR) */
33 static const struct x86_cpu_id vmx_pebs_pdir_cpu[] = {
34 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, NULL),
35 X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, NULL),
36 /* Instruction-Accurate PDIR (PDIR++) */
37 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
41 /* Precise Distribution (PDist) */
42 static const struct x86_cpu_id vmx_pebs_pdist_cpu[] = {
43 X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, NULL),
48 * - Each perf counter is defined as "struct kvm_pmc";
49 * - There are two types of perf counters: general purpose (gp) and fixed.
50 * gp counters are stored in gp_counters[] and fixed counters are stored
51 * in fixed_counters[] respectively. Both of them are part of "struct
53 * - pmu.c understands the difference between gp counters and fixed counters.
54 * However AMD doesn't support fixed-counters;
55 * - There are three types of index to access perf counters (PMC):
56 * 1. MSR (named msr): For example Intel has MSR_IA32_PERFCTRn and AMD
57 * has MSR_K7_PERFCTRn and, for families 15H and later,
58 * MSR_F15H_PERF_CTRn, where MSR_F15H_PERF_CTR[0-3] are
59 * aliased to MSR_K7_PERFCTRn.
60 * 2. MSR Index (named idx): This normally is used by RDPMC instruction.
61 * For instance AMD RDPMC instruction uses 0000_0003h in ECX to access
62 * C001_0007h (MSR_K7_PERCTR3). Intel has a similar mechanism, except
63 * that it also supports fixed counters. idx can be used to as index to
64 * gp and fixed counters.
65 * 3. Global PMC Index (named pmc): pmc is an index specific to PMU
66 * code. Each pmc, stored in kvm_pmc.idx field, is unique across
67 * all perf counters (both gp and fixed). The mapping relationship
68 * between pmc and perf counters is as the following:
69 * * Intel: [0 .. KVM_INTEL_PMC_MAX_GENERIC-1] <=> gp counters
70 * [INTEL_PMC_IDX_FIXED .. INTEL_PMC_IDX_FIXED + 2] <=> fixed
71 * * AMD: [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H
72 * and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters
75 static struct kvm_pmu_ops kvm_pmu_ops __read_mostly;
77 #define KVM_X86_PMU_OP(func) \
78 DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \
79 *(((struct kvm_pmu_ops *)0)->func));
80 #define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP
81 #include <asm/kvm-x86-pmu-ops.h>
83 void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops)
85 memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops));
87 #define __KVM_X86_PMU_OP(func) \
88 static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func);
89 #define KVM_X86_PMU_OP(func) \
90 WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func)
91 #define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP
92 #include <asm/kvm-x86-pmu-ops.h>
93 #undef __KVM_X86_PMU_OP
96 static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi)
98 struct kvm_pmu *pmu = pmc_to_pmu(pmc);
99 bool skip_pmi = false;
101 if (pmc->perf_event && pmc->perf_event->attr.precise_ip) {
104 * TODO: KVM is currently _choosing_ to not generate records
105 * for emulated instructions, avoiding BUFFER_OVF PMI when
106 * there are no records. Strictly speaking, it should be done
107 * as well in the right context to improve sampling accuracy.
111 /* Indicate PEBS overflow PMI to guest. */
112 skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT,
113 (unsigned long *)&pmu->global_status);
116 __set_bit(pmc->idx, (unsigned long *)&pmu->global_status);
119 if (pmc->intr && !skip_pmi)
120 kvm_make_request(KVM_REQ_PMI, pmc->vcpu);
123 static void kvm_perf_overflow(struct perf_event *perf_event,
124 struct perf_sample_data *data,
125 struct pt_regs *regs)
127 struct kvm_pmc *pmc = perf_event->overflow_handler_context;
130 * Ignore overflow events for counters that are scheduled to be
131 * reprogrammed, e.g. if a PMI for the previous event races with KVM's
132 * handling of a related guest WRMSR.
134 if (test_and_set_bit(pmc->idx, pmc_to_pmu(pmc)->reprogram_pmi))
137 __kvm_perf_overflow(pmc, true);
139 kvm_make_request(KVM_REQ_PMU, pmc->vcpu);
142 static u64 pmc_get_pebs_precise_level(struct kvm_pmc *pmc)
145 * For some model specific pebs counters with special capabilities
146 * (PDIR, PDIR++, PDIST), KVM needs to raise the event precise
147 * level to the maximum value (currently 3, backwards compatible)
148 * so that the perf subsystem would assign specific hardware counter
149 * with that capability for vPMC.
151 if ((pmc->idx == 0 && x86_match_cpu(vmx_pebs_pdist_cpu)) ||
152 (pmc->idx == 32 && x86_match_cpu(vmx_pebs_pdir_cpu)))
156 * The non-zero precision level of guest event makes the ordinary
157 * guest event becomes a guest PEBS event and triggers the host
158 * PEBS PMI handler to determine whether the PEBS overflow PMI
159 * comes from the host counters or the guest.
164 static int pmc_reprogram_counter(struct kvm_pmc *pmc, u32 type, u64 config,
165 bool exclude_user, bool exclude_kernel,
168 struct kvm_pmu *pmu = pmc_to_pmu(pmc);
169 struct perf_event *event;
170 struct perf_event_attr attr = {
172 .size = sizeof(attr),
174 .exclude_idle = true,
176 .exclude_user = exclude_user,
177 .exclude_kernel = exclude_kernel,
180 bool pebs = test_bit(pmc->idx, (unsigned long *)&pmu->pebs_enable);
182 attr.sample_period = get_sample_period(pmc, pmc->counter);
184 if ((attr.config & HSW_IN_TX_CHECKPOINTED) &&
185 guest_cpuid_is_intel(pmc->vcpu)) {
187 * HSW_IN_TX_CHECKPOINTED is not supported with nonzero
188 * period. Just clear the sample period so at least
189 * allocating the counter doesn't fail.
191 attr.sample_period = 0;
195 * For most PEBS hardware events, the difference in the software
196 * precision levels of guest and host PEBS events will not affect
197 * the accuracy of the PEBS profiling result, because the "event IP"
198 * in the PEBS record is calibrated on the guest side.
200 attr.precise_ip = pmc_get_pebs_precise_level(pmc);
203 event = perf_event_create_kernel_counter(&attr, -1, current,
204 kvm_perf_overflow, pmc);
206 pr_debug_ratelimited("kvm_pmu: event creation failed %ld for pmc->idx = %d\n",
207 PTR_ERR(event), pmc->idx);
208 return PTR_ERR(event);
211 pmc->perf_event = event;
212 pmc_to_pmu(pmc)->event_count++;
213 pmc->is_paused = false;
214 pmc->intr = intr || pebs;
218 static void pmc_pause_counter(struct kvm_pmc *pmc)
220 u64 counter = pmc->counter;
222 if (!pmc->perf_event || pmc->is_paused)
225 /* update counter, reset event value to avoid redundant accumulation */
226 counter += perf_event_pause(pmc->perf_event, true);
227 pmc->counter = counter & pmc_bitmask(pmc);
228 pmc->is_paused = true;
231 static bool pmc_resume_counter(struct kvm_pmc *pmc)
233 if (!pmc->perf_event)
236 /* recalibrate sample period and check if it's accepted by perf core */
237 if (is_sampling_event(pmc->perf_event) &&
238 perf_event_period(pmc->perf_event,
239 get_sample_period(pmc, pmc->counter)))
242 if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) !=
243 (!!pmc->perf_event->attr.precise_ip))
246 /* reuse perf_event to serve as pmc_reprogram_counter() does*/
247 perf_event_enable(pmc->perf_event);
248 pmc->is_paused = false;
253 static int filter_cmp(const void *pa, const void *pb, u64 mask)
255 u64 a = *(u64 *)pa & mask;
256 u64 b = *(u64 *)pb & mask;
258 return (a > b) - (a < b);
262 static int filter_sort_cmp(const void *pa, const void *pb)
264 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT |
265 KVM_PMU_MASKED_ENTRY_EXCLUDE));
269 * For the event filter, searching is done on the 'includes' list and
270 * 'excludes' list separately rather than on the 'events' list (which
271 * has both). As a result the exclude bit can be ignored.
273 static int filter_event_cmp(const void *pa, const void *pb)
275 return filter_cmp(pa, pb, (KVM_PMU_MASKED_ENTRY_EVENT_SELECT));
278 static int find_filter_index(u64 *events, u64 nevents, u64 key)
280 u64 *fe = bsearch(&key, events, nevents, sizeof(events[0]),
289 static bool is_filter_entry_match(u64 filter_event, u64 umask)
291 u64 mask = filter_event >> (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8);
292 u64 match = filter_event & KVM_PMU_MASKED_ENTRY_UMASK_MATCH;
294 BUILD_BUG_ON((KVM_PMU_ENCODE_MASKED_ENTRY(0, 0xff, 0, false) >>
295 (KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT - 8)) !=
296 ARCH_PERFMON_EVENTSEL_UMASK);
298 return (umask & mask) == match;
301 static bool filter_contains_match(u64 *events, u64 nevents, u64 eventsel)
303 u64 event_select = eventsel & kvm_pmu_ops.EVENTSEL_EVENT;
304 u64 umask = eventsel & ARCH_PERFMON_EVENTSEL_UMASK;
307 index = find_filter_index(events, nevents, event_select);
312 * Entries are sorted by the event select. Walk the list in both
313 * directions to process all entries with the targeted event select.
315 for (i = index; i < nevents; i++) {
316 if (filter_event_cmp(&events[i], &event_select))
319 if (is_filter_entry_match(events[i], umask))
323 for (i = index - 1; i >= 0; i--) {
324 if (filter_event_cmp(&events[i], &event_select))
327 if (is_filter_entry_match(events[i], umask))
334 static bool is_gp_event_allowed(struct kvm_x86_pmu_event_filter *f,
337 if (filter_contains_match(f->includes, f->nr_includes, eventsel) &&
338 !filter_contains_match(f->excludes, f->nr_excludes, eventsel))
339 return f->action == KVM_PMU_EVENT_ALLOW;
341 return f->action == KVM_PMU_EVENT_DENY;
344 static bool is_fixed_event_allowed(struct kvm_x86_pmu_event_filter *filter,
347 int fixed_idx = idx - INTEL_PMC_IDX_FIXED;
349 if (filter->action == KVM_PMU_EVENT_DENY &&
350 test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap))
352 if (filter->action == KVM_PMU_EVENT_ALLOW &&
353 !test_bit(fixed_idx, (ulong *)&filter->fixed_counter_bitmap))
359 static bool check_pmu_event_filter(struct kvm_pmc *pmc)
361 struct kvm_x86_pmu_event_filter *filter;
362 struct kvm *kvm = pmc->vcpu->kvm;
364 filter = srcu_dereference(kvm->arch.pmu_event_filter, &kvm->srcu);
369 return is_gp_event_allowed(filter, pmc->eventsel);
371 return is_fixed_event_allowed(filter, pmc->idx);
374 static bool pmc_event_is_allowed(struct kvm_pmc *pmc)
376 return pmc_is_globally_enabled(pmc) && pmc_speculative_in_use(pmc) &&
377 static_call(kvm_x86_pmu_hw_event_available)(pmc) &&
378 check_pmu_event_filter(pmc);
381 static void reprogram_counter(struct kvm_pmc *pmc)
383 struct kvm_pmu *pmu = pmc_to_pmu(pmc);
384 u64 eventsel = pmc->eventsel;
385 u64 new_config = eventsel;
388 pmc_pause_counter(pmc);
390 if (!pmc_event_is_allowed(pmc))
391 goto reprogram_complete;
393 if (pmc->counter < pmc->prev_counter)
394 __kvm_perf_overflow(pmc, false);
396 if (eventsel & ARCH_PERFMON_EVENTSEL_PIN_CONTROL)
397 printk_once("kvm pmu: pin control bit is ignored\n");
399 if (pmc_is_fixed(pmc)) {
400 fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl,
401 pmc->idx - INTEL_PMC_IDX_FIXED);
402 if (fixed_ctr_ctrl & 0x1)
403 eventsel |= ARCH_PERFMON_EVENTSEL_OS;
404 if (fixed_ctr_ctrl & 0x2)
405 eventsel |= ARCH_PERFMON_EVENTSEL_USR;
406 if (fixed_ctr_ctrl & 0x8)
407 eventsel |= ARCH_PERFMON_EVENTSEL_INT;
408 new_config = (u64)fixed_ctr_ctrl;
411 if (pmc->current_config == new_config && pmc_resume_counter(pmc))
412 goto reprogram_complete;
414 pmc_release_perf_event(pmc);
416 pmc->current_config = new_config;
419 * If reprogramming fails, e.g. due to contention, leave the counter's
420 * regprogram bit set, i.e. opportunistically try again on the next PMU
421 * refresh. Don't make a new request as doing so can stall the guest
422 * if reprogramming repeatedly fails.
424 if (pmc_reprogram_counter(pmc, PERF_TYPE_RAW,
425 (eventsel & pmu->raw_event_mask),
426 !(eventsel & ARCH_PERFMON_EVENTSEL_USR),
427 !(eventsel & ARCH_PERFMON_EVENTSEL_OS),
428 eventsel & ARCH_PERFMON_EVENTSEL_INT))
432 clear_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->reprogram_pmi);
433 pmc->prev_counter = 0;
436 void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
438 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
441 for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) {
442 struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit);
444 if (unlikely(!pmc)) {
445 clear_bit(bit, pmu->reprogram_pmi);
449 reprogram_counter(pmc);
453 * Unused perf_events are only released if the corresponding MSRs
454 * weren't accessed during the last vCPU time slice. kvm_arch_sched_in
455 * triggers KVM_REQ_PMU if cleanup is needed.
457 if (unlikely(pmu->need_cleanup))
458 kvm_pmu_cleanup(vcpu);
461 /* check if idx is a valid index to access PMU */
462 bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx)
464 return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx);
467 bool is_vmware_backdoor_pmc(u32 pmc_idx)
470 case VMWARE_BACKDOOR_PMC_HOST_TSC:
471 case VMWARE_BACKDOOR_PMC_REAL_TIME:
472 case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
478 static int kvm_pmu_rdpmc_vmware(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
483 case VMWARE_BACKDOOR_PMC_HOST_TSC:
486 case VMWARE_BACKDOOR_PMC_REAL_TIME:
487 ctr_val = ktime_get_boottime_ns();
489 case VMWARE_BACKDOOR_PMC_APPARENT_TIME:
490 ctr_val = ktime_get_boottime_ns() +
491 vcpu->kvm->arch.kvmclock_offset;
501 int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data)
503 bool fast_mode = idx & (1u << 31);
504 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
506 u64 mask = fast_mode ? ~0u : ~0ull;
511 if (is_vmware_backdoor_pmc(idx))
512 return kvm_pmu_rdpmc_vmware(vcpu, idx, data);
514 pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask);
518 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_PCE) &&
519 (static_call(kvm_x86_get_cpl)(vcpu) != 0) &&
520 kvm_is_cr0_bit_set(vcpu, X86_CR0_PE))
523 *data = pmc_read_counter(pmc) & mask;
527 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu)
529 if (lapic_in_kernel(vcpu)) {
530 static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu);
531 kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC);
535 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr)
538 case MSR_CORE_PERF_GLOBAL_STATUS:
539 case MSR_CORE_PERF_GLOBAL_CTRL:
540 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
541 return kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu));
545 return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) ||
546 static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr);
549 static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr)
551 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
552 struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr);
555 __set_bit(pmc->idx, pmu->pmc_in_use);
558 int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
560 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
561 u32 msr = msr_info->index;
564 case MSR_CORE_PERF_GLOBAL_STATUS:
565 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
566 msr_info->data = pmu->global_status;
568 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
569 case MSR_CORE_PERF_GLOBAL_CTRL:
570 msr_info->data = pmu->global_ctrl;
572 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
573 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
577 return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info);
583 int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
585 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
586 u32 msr = msr_info->index;
587 u64 data = msr_info->data;
591 * Note, AMD ignores writes to reserved bits and read-only PMU MSRs,
592 * whereas Intel generates #GP on attempts to write reserved/RO MSRs.
595 case MSR_CORE_PERF_GLOBAL_STATUS:
596 if (!msr_info->host_initiated)
597 return 1; /* RO MSR */
599 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS:
600 /* Per PPR, Read-only MSR. Writes are ignored. */
601 if (!msr_info->host_initiated)
604 if (data & pmu->global_status_mask)
607 pmu->global_status = data;
609 case MSR_AMD64_PERF_CNTR_GLOBAL_CTL:
610 data &= ~pmu->global_ctrl_mask;
612 case MSR_CORE_PERF_GLOBAL_CTRL:
613 if (!kvm_valid_perf_global_ctrl(pmu, data))
616 if (pmu->global_ctrl != data) {
617 diff = pmu->global_ctrl ^ data;
618 pmu->global_ctrl = data;
619 reprogram_counters(pmu, diff);
622 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
624 * GLOBAL_OVF_CTRL, a.k.a. GLOBAL STATUS_RESET, clears bits in
625 * GLOBAL_STATUS, and so the set of reserved bits is the same.
627 if (data & pmu->global_status_mask)
630 case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR:
631 if (!msr_info->host_initiated)
632 pmu->global_status &= ~data;
635 kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index);
636 return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info);
642 /* refresh PMU settings. This function generally is called when underlying
643 * settings are changed (such as changes of PMU CPUID by guest VMs), which
644 * should rarely happen.
646 void kvm_pmu_refresh(struct kvm_vcpu *vcpu)
648 if (KVM_BUG_ON(kvm_vcpu_has_run(vcpu), vcpu->kvm))
651 bitmap_zero(vcpu_to_pmu(vcpu)->all_valid_pmc_idx, X86_PMC_IDX_MAX);
652 static_call(kvm_x86_pmu_refresh)(vcpu);
655 void kvm_pmu_reset(struct kvm_vcpu *vcpu)
657 static_call(kvm_x86_pmu_reset)(vcpu);
660 void kvm_pmu_init(struct kvm_vcpu *vcpu)
662 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
664 memset(pmu, 0, sizeof(*pmu));
665 static_call(kvm_x86_pmu_init)(vcpu);
666 pmu->event_count = 0;
667 pmu->need_cleanup = false;
668 kvm_pmu_refresh(vcpu);
671 /* Release perf_events for vPMCs that have been unused for a full time slice. */
672 void kvm_pmu_cleanup(struct kvm_vcpu *vcpu)
674 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
675 struct kvm_pmc *pmc = NULL;
676 DECLARE_BITMAP(bitmask, X86_PMC_IDX_MAX);
679 pmu->need_cleanup = false;
681 bitmap_andnot(bitmask, pmu->all_valid_pmc_idx,
682 pmu->pmc_in_use, X86_PMC_IDX_MAX);
684 for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) {
685 pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
687 if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc))
688 pmc_stop_counter(pmc);
691 static_call_cond(kvm_x86_pmu_cleanup)(vcpu);
693 bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX);
696 void kvm_pmu_destroy(struct kvm_vcpu *vcpu)
701 static void kvm_pmu_incr_counter(struct kvm_pmc *pmc)
703 pmc->prev_counter = pmc->counter;
704 pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc);
705 kvm_pmu_request_counter_reprogram(pmc);
708 static inline bool eventsel_match_perf_hw_id(struct kvm_pmc *pmc,
709 unsigned int perf_hw_id)
711 return !((pmc->eventsel ^ perf_get_hw_event_config(perf_hw_id)) &
712 AMD64_RAW_EVENT_MASK_NB);
715 static inline bool cpl_is_matched(struct kvm_pmc *pmc)
717 bool select_os, select_user;
720 if (pmc_is_gp(pmc)) {
721 config = pmc->eventsel;
722 select_os = config & ARCH_PERFMON_EVENTSEL_OS;
723 select_user = config & ARCH_PERFMON_EVENTSEL_USR;
725 config = fixed_ctrl_field(pmc_to_pmu(pmc)->fixed_ctr_ctrl,
726 pmc->idx - INTEL_PMC_IDX_FIXED);
727 select_os = config & 0x1;
728 select_user = config & 0x2;
731 return (static_call(kvm_x86_get_cpl)(pmc->vcpu) == 0) ? select_os : select_user;
734 void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 perf_hw_id)
736 struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
740 for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) {
741 pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i);
743 if (!pmc || !pmc_event_is_allowed(pmc))
746 /* Ignore checks for edge detect, pin control, invert and CMASK bits */
747 if (eventsel_match_perf_hw_id(pmc, perf_hw_id) && cpl_is_matched(pmc))
748 kvm_pmu_incr_counter(pmc);
751 EXPORT_SYMBOL_GPL(kvm_pmu_trigger_event);
753 static bool is_masked_filter_valid(const struct kvm_x86_pmu_event_filter *filter)
755 u64 mask = kvm_pmu_ops.EVENTSEL_EVENT |
756 KVM_PMU_MASKED_ENTRY_UMASK_MASK |
757 KVM_PMU_MASKED_ENTRY_UMASK_MATCH |
758 KVM_PMU_MASKED_ENTRY_EXCLUDE;
761 for (i = 0; i < filter->nevents; i++) {
762 if (filter->events[i] & ~mask)
769 static void convert_to_masked_filter(struct kvm_x86_pmu_event_filter *filter)
773 for (i = 0, j = 0; i < filter->nevents; i++) {
775 * Skip events that are impossible to match against a guest
776 * event. When filtering, only the event select + unit mask
777 * of the guest event is used. To maintain backwards
778 * compatibility, impossible filters can't be rejected :-(
780 if (filter->events[i] & ~(kvm_pmu_ops.EVENTSEL_EVENT |
781 ARCH_PERFMON_EVENTSEL_UMASK))
784 * Convert userspace events to a common in-kernel event so
785 * only one code path is needed to support both events. For
786 * the in-kernel events use masked events because they are
787 * flexible enough to handle both cases. To convert to masked
788 * events all that's needed is to add an "all ones" umask_mask,
789 * (unmasked filter events don't support EXCLUDE).
791 filter->events[j++] = filter->events[i] |
792 (0xFFULL << KVM_PMU_MASKED_ENTRY_UMASK_MASK_SHIFT);
798 static int prepare_filter_lists(struct kvm_x86_pmu_event_filter *filter)
802 if (!(filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS))
803 convert_to_masked_filter(filter);
804 else if (!is_masked_filter_valid(filter))
808 * Sort entries by event select and includes vs. excludes so that all
809 * entries for a given event select can be processed efficiently during
810 * filtering. The EXCLUDE flag uses a more significant bit than the
811 * event select, and so the sorted list is also effectively split into
812 * includes and excludes sub-lists.
814 sort(&filter->events, filter->nevents, sizeof(filter->events[0]),
815 filter_sort_cmp, NULL);
818 /* Find the first EXCLUDE event (only supported for masked events). */
819 if (filter->flags & KVM_PMU_EVENT_FLAG_MASKED_EVENTS) {
820 for (i = 0; i < filter->nevents; i++) {
821 if (filter->events[i] & KVM_PMU_MASKED_ENTRY_EXCLUDE)
826 filter->nr_includes = i;
827 filter->nr_excludes = filter->nevents - filter->nr_includes;
828 filter->includes = filter->events;
829 filter->excludes = filter->events + filter->nr_includes;
834 int kvm_vm_ioctl_set_pmu_event_filter(struct kvm *kvm, void __user *argp)
836 struct kvm_pmu_event_filter __user *user_filter = argp;
837 struct kvm_x86_pmu_event_filter *filter;
838 struct kvm_pmu_event_filter tmp;
839 struct kvm_vcpu *vcpu;
844 if (copy_from_user(&tmp, user_filter, sizeof(tmp)))
847 if (tmp.action != KVM_PMU_EVENT_ALLOW &&
848 tmp.action != KVM_PMU_EVENT_DENY)
851 if (tmp.flags & ~KVM_PMU_EVENT_FLAGS_VALID_MASK)
854 if (tmp.nevents > KVM_PMU_EVENT_FILTER_MAX_EVENTS)
857 size = struct_size(filter, events, tmp.nevents);
858 filter = kzalloc(size, GFP_KERNEL_ACCOUNT);
862 filter->action = tmp.action;
863 filter->nevents = tmp.nevents;
864 filter->fixed_counter_bitmap = tmp.fixed_counter_bitmap;
865 filter->flags = tmp.flags;
868 if (copy_from_user(filter->events, user_filter->events,
869 sizeof(filter->events[0]) * filter->nevents))
872 r = prepare_filter_lists(filter);
876 mutex_lock(&kvm->lock);
877 filter = rcu_replace_pointer(kvm->arch.pmu_event_filter, filter,
878 mutex_is_locked(&kvm->lock));
879 mutex_unlock(&kvm->lock);
880 synchronize_srcu_expedited(&kvm->srcu);
882 BUILD_BUG_ON(sizeof(((struct kvm_pmu *)0)->reprogram_pmi) >
883 sizeof(((struct kvm_pmu *)0)->__reprogram_pmi));
885 kvm_for_each_vcpu(i, vcpu, kvm)
886 atomic64_set(&vcpu_to_pmu(vcpu)->__reprogram_pmi, -1ull);
888 kvm_make_all_cpus_request(kvm, KVM_REQ_PMU);