Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
051ff581 SZ |
2 | /* |
3 | * Copyright (C) 2015 Linaro Ltd. | |
4 | * Author: Shannon Zhao <shannon.zhao@linaro.org> | |
051ff581 SZ |
5 | */ |
6 | ||
7 | #include <linux/cpu.h> | |
8 | #include <linux/kvm.h> | |
9 | #include <linux/kvm_host.h> | |
db858060 | 10 | #include <linux/list.h> |
051ff581 | 11 | #include <linux/perf_event.h> |
8c3252c0 | 12 | #include <linux/perf/arm_pmu.h> |
bb0c70bc | 13 | #include <linux/uaccess.h> |
051ff581 SZ |
14 | #include <asm/kvm_emulate.h> |
15 | #include <kvm/arm_pmu.h> | |
b02386eb | 16 | #include <kvm/arm_vgic.h> |
051ff581 | 17 | |
bead0220 MZ |
18 | #define PERF_ATTR_CFG1_COUNTER_64BIT BIT(0) |
19 | ||
be399d82 SC |
20 | DEFINE_STATIC_KEY_FALSE(kvm_arm_pmu_available); |
21 | ||
db858060 AE |
22 | static LIST_HEAD(arm_pmus); |
23 | static DEFINE_MUTEX(arm_pmus_lock); | |
24 | ||
d56bdce5 | 25 | static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc); |
9917264d | 26 | static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc); |
80f393a2 | 27 | |
d56bdce5 MZ |
28 | static struct kvm_vcpu *kvm_pmc_to_vcpu(const struct kvm_pmc *pmc) |
29 | { | |
30 | return container_of(pmc, struct kvm_vcpu, arch.pmu.pmc[pmc->idx]); | |
31 | } | |
32 | ||
33 | static struct kvm_pmc *kvm_vcpu_idx_to_pmc(struct kvm_vcpu *vcpu, int cnt_idx) | |
34 | { | |
35 | return &vcpu->arch.pmu.pmc[cnt_idx]; | |
36 | } | |
37 | ||
fd65a3b5 MZ |
38 | static u32 kvm_pmu_event_mask(struct kvm *kvm) |
39 | { | |
46b18782 MZ |
40 | unsigned int pmuver; |
41 | ||
42 | pmuver = kvm->arch.arm_pmu->pmuver; | |
43 | ||
44 | switch (pmuver) { | |
121a8fc0 | 45 | case ID_AA64DFR0_EL1_PMUVer_IMP: |
fd65a3b5 | 46 | return GENMASK(9, 0); |
121a8fc0 MB |
47 | case ID_AA64DFR0_EL1_PMUVer_V3P1: |
48 | case ID_AA64DFR0_EL1_PMUVer_V3P4: | |
49 | case ID_AA64DFR0_EL1_PMUVer_V3P5: | |
50 | case ID_AA64DFR0_EL1_PMUVer_V3P7: | |
fd65a3b5 MZ |
51 | return GENMASK(15, 0); |
52 | default: /* Shouldn't be here, just for sanity */ | |
46b18782 | 53 | WARN_ONCE(1, "Unknown PMU version %d\n", pmuver); |
fd65a3b5 MZ |
54 | return 0; |
55 | } | |
56 | } | |
57 | ||
218907cb | 58 | /** |
d56bdce5 MZ |
59 | * kvm_pmc_is_64bit - determine if counter is 64bit |
60 | * @pmc: counter context | |
218907cb | 61 | */ |
d56bdce5 | 62 | static bool kvm_pmc_is_64bit(struct kvm_pmc *pmc) |
c82d28cb | 63 | { |
d56bdce5 MZ |
64 | return (pmc->idx == ARMV8_PMU_CYCLE_IDX || |
65 | kvm_pmu_is_3p5(kvm_pmc_to_vcpu(pmc))); | |
c82d28cb MZ |
66 | } |
67 | ||
d56bdce5 | 68 | static bool kvm_pmc_has_64bit_overflow(struct kvm_pmc *pmc) |
218907cb | 69 | { |
d56bdce5 | 70 | u64 val = __vcpu_sys_reg(kvm_pmc_to_vcpu(pmc), PMCR_EL0); |
11af4c37 | 71 | |
d56bdce5 MZ |
72 | return (pmc->idx < ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LP)) || |
73 | (pmc->idx == ARMV8_PMU_CYCLE_IDX && (val & ARMV8_PMU_PMCR_LC)); | |
218907cb AM |
74 | } |
75 | ||
d56bdce5 | 76 | static bool kvm_pmu_counter_can_chain(struct kvm_pmc *pmc) |
bead0220 | 77 | { |
d56bdce5 MZ |
78 | return (!(pmc->idx & 1) && (pmc->idx + 1) < ARMV8_PMU_CYCLE_IDX && |
79 | !kvm_pmc_has_64bit_overflow(pmc)); | |
80f393a2 AM |
80 | } |
81 | ||
0cb9c3c8 MZ |
82 | static u32 counter_index_to_reg(u64 idx) |
83 | { | |
84 | return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + idx; | |
85 | } | |
86 | ||
87 | static u32 counter_index_to_evtreg(u64 idx) | |
88 | { | |
89 | return (idx == ARMV8_PMU_CYCLE_IDX) ? PMCCFILTR_EL0 : PMEVTYPER0_EL0 + idx; | |
90 | } | |
91 | ||
d56bdce5 | 92 | static u64 kvm_pmu_get_pmc_value(struct kvm_pmc *pmc) |
80f393a2 | 93 | { |
d56bdce5 | 94 | struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); |
bead0220 | 95 | u64 counter, reg, enabled, running; |
80f393a2 | 96 | |
d56bdce5 | 97 | reg = counter_index_to_reg(pmc->idx); |
bead0220 | 98 | counter = __vcpu_sys_reg(vcpu, reg); |
80f393a2 AM |
99 | |
100 | /* | |
101 | * The real counter value is equal to the value of counter register plus | |
051ff581 SZ |
102 | * the value perf event counts. |
103 | */ | |
104 | if (pmc->perf_event) | |
105 | counter += perf_event_read_value(pmc->perf_event, &enabled, | |
106 | &running); | |
107 | ||
d56bdce5 | 108 | if (!kvm_pmc_is_64bit(pmc)) |
218907cb AM |
109 | counter = lower_32_bits(counter); |
110 | ||
111 | return counter; | |
051ff581 SZ |
112 | } |
113 | ||
d56bdce5 MZ |
114 | /** |
115 | * kvm_pmu_get_counter_value - get PMU counter value | |
116 | * @vcpu: The vcpu pointer | |
117 | * @select_idx: The counter index | |
118 | */ | |
119 | u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx) | |
051ff581 | 120 | { |
8f6379e2 | 121 | if (!kvm_vcpu_has_pmu(vcpu)) |
d56bdce5 | 122 | return 0; |
8f6379e2 | 123 | |
d56bdce5 MZ |
124 | return kvm_pmu_get_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx)); |
125 | } | |
9917264d | 126 | |
d56bdce5 MZ |
127 | static void kvm_pmu_set_pmc_value(struct kvm_pmc *pmc, u64 val, bool force) |
128 | { | |
129 | struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); | |
130 | u64 reg; | |
26d2d059 | 131 | |
d56bdce5 MZ |
132 | kvm_pmu_release_perf_event(pmc); |
133 | ||
134 | reg = counter_index_to_reg(pmc->idx); | |
135 | ||
136 | if (vcpu_mode_is_32bit(vcpu) && pmc->idx != ARMV8_PMU_CYCLE_IDX && | |
26d2d059 MZ |
137 | !force) { |
138 | /* | |
139 | * Even with PMUv3p5, AArch32 cannot write to the top | |
140 | * 32bit of the counters. The only possible course of | |
141 | * action is to use PMCR.P, which will reset them to | |
142 | * 0 (the only use of the 'force' parameter). | |
143 | */ | |
144 | val = __vcpu_sys_reg(vcpu, reg) & GENMASK(63, 32); | |
145 | val |= lower_32_bits(val); | |
146 | } | |
147 | ||
9917264d | 148 | __vcpu_sys_reg(vcpu, reg) = val; |
30d97754 AM |
149 | |
150 | /* Recreate the perf event to reflect the updated sample_period */ | |
d56bdce5 | 151 | kvm_pmu_create_perf_event(pmc); |
051ff581 | 152 | } |
96b0eebc | 153 | |
26d2d059 MZ |
154 | /** |
155 | * kvm_pmu_set_counter_value - set PMU counter value | |
156 | * @vcpu: The vcpu pointer | |
157 | * @select_idx: The counter index | |
158 | * @val: The counter value | |
159 | */ | |
160 | void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val) | |
161 | { | |
d56bdce5 MZ |
162 | if (!kvm_vcpu_has_pmu(vcpu)) |
163 | return; | |
164 | ||
165 | kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, select_idx), val, false); | |
26d2d059 MZ |
166 | } |
167 | ||
6f4d2a0b AM |
168 | /** |
169 | * kvm_pmu_release_perf_event - remove the perf event | |
170 | * @pmc: The PMU counter pointer | |
171 | */ | |
172 | static void kvm_pmu_release_perf_event(struct kvm_pmc *pmc) | |
173 | { | |
174 | if (pmc->perf_event) { | |
175 | perf_event_disable(pmc->perf_event); | |
176 | perf_event_release_kernel(pmc->perf_event); | |
177 | pmc->perf_event = NULL; | |
178 | } | |
179 | } | |
180 | ||
7f766358 SZ |
181 | /** |
182 | * kvm_pmu_stop_counter - stop PMU counter | |
183 | * @pmc: The PMU counter pointer | |
184 | * | |
185 | * If this counter has been configured to monitor some event, release it here. | |
186 | */ | |
d56bdce5 | 187 | static void kvm_pmu_stop_counter(struct kvm_pmc *pmc) |
7f766358 | 188 | { |
d56bdce5 | 189 | struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); |
0f1e172b | 190 | u64 reg, val; |
7f766358 | 191 | |
80f393a2 AM |
192 | if (!pmc->perf_event) |
193 | return; | |
194 | ||
d56bdce5 | 195 | val = kvm_pmu_get_pmc_value(pmc); |
80f393a2 | 196 | |
0cb9c3c8 | 197 | reg = counter_index_to_reg(pmc->idx); |
80f393a2 | 198 | |
f4e23cf9 MZ |
199 | __vcpu_sys_reg(vcpu, reg) = val; |
200 | ||
80f393a2 | 201 | kvm_pmu_release_perf_event(pmc); |
7f766358 SZ |
202 | } |
203 | ||
bca031e2 ZY |
204 | /** |
205 | * kvm_pmu_vcpu_init - assign pmu counter idx for cpu | |
206 | * @vcpu: The vcpu pointer | |
207 | * | |
208 | */ | |
209 | void kvm_pmu_vcpu_init(struct kvm_vcpu *vcpu) | |
210 | { | |
211 | int i; | |
212 | struct kvm_pmu *pmu = &vcpu->arch.pmu; | |
213 | ||
214 | for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) | |
215 | pmu->pmc[i].idx = i; | |
216 | } | |
217 | ||
2aa36e98 SZ |
218 | /** |
219 | * kvm_pmu_vcpu_reset - reset pmu state for cpu | |
220 | * @vcpu: The vcpu pointer | |
221 | * | |
222 | */ | |
223 | void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu) | |
224 | { | |
c01d6a18 | 225 | unsigned long mask = kvm_pmu_valid_counter_mask(vcpu); |
c01d6a18 | 226 | int i; |
2aa36e98 | 227 | |
c01d6a18 | 228 | for_each_set_bit(i, &mask, 32) |
d56bdce5 | 229 | kvm_pmu_stop_counter(kvm_vcpu_idx_to_pmc(vcpu, i)); |
2aa36e98 SZ |
230 | } |
231 | ||
5f0a714a SZ |
232 | /** |
233 | * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu | |
234 | * @vcpu: The vcpu pointer | |
235 | * | |
236 | */ | |
237 | void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu) | |
238 | { | |
239 | int i; | |
5f0a714a | 240 | |
6f4d2a0b | 241 | for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) |
d56bdce5 | 242 | kvm_pmu_release_perf_event(kvm_vcpu_idx_to_pmc(vcpu, i)); |
95e92e45 | 243 | irq_work_sync(&vcpu->arch.pmu.overflow_work); |
5f0a714a SZ |
244 | } |
245 | ||
96b0eebc SZ |
246 | u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu) |
247 | { | |
8d404c4c | 248 | u64 val = __vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT; |
96b0eebc SZ |
249 | |
250 | val &= ARMV8_PMU_PMCR_N_MASK; | |
251 | if (val == 0) | |
252 | return BIT(ARMV8_PMU_CYCLE_IDX); | |
253 | else | |
254 | return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX); | |
255 | } | |
256 | ||
257 | /** | |
418e5ca8 | 258 | * kvm_pmu_enable_counter_mask - enable selected PMU counters |
96b0eebc SZ |
259 | * @vcpu: The vcpu pointer |
260 | * @val: the value guest writes to PMCNTENSET register | |
261 | * | |
262 | * Call perf_event_enable to start counting the perf event | |
263 | */ | |
418e5ca8 | 264 | void kvm_pmu_enable_counter_mask(struct kvm_vcpu *vcpu, u64 val) |
96b0eebc SZ |
265 | { |
266 | int i; | |
8f6379e2 AE |
267 | if (!kvm_vcpu_has_pmu(vcpu)) |
268 | return; | |
269 | ||
8d404c4c | 270 | if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val) |
96b0eebc SZ |
271 | return; |
272 | ||
273 | for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) { | |
d56bdce5 MZ |
274 | struct kvm_pmc *pmc; |
275 | ||
96b0eebc SZ |
276 | if (!(val & BIT(i))) |
277 | continue; | |
278 | ||
d56bdce5 | 279 | pmc = kvm_vcpu_idx_to_pmc(vcpu, i); |
80f393a2 | 280 | |
bead0220 | 281 | if (!pmc->perf_event) { |
d56bdce5 | 282 | kvm_pmu_create_perf_event(pmc); |
bead0220 | 283 | } else { |
96b0eebc SZ |
284 | perf_event_enable(pmc->perf_event); |
285 | if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE) | |
286 | kvm_debug("fail to enable perf event\n"); | |
287 | } | |
288 | } | |
289 | } | |
290 | ||
291 | /** | |
418e5ca8 | 292 | * kvm_pmu_disable_counter_mask - disable selected PMU counters |
96b0eebc SZ |
293 | * @vcpu: The vcpu pointer |
294 | * @val: the value guest writes to PMCNTENCLR register | |
295 | * | |
296 | * Call perf_event_disable to stop counting the perf event | |
297 | */ | |
418e5ca8 | 298 | void kvm_pmu_disable_counter_mask(struct kvm_vcpu *vcpu, u64 val) |
96b0eebc SZ |
299 | { |
300 | int i; | |
96b0eebc | 301 | |
8f6379e2 | 302 | if (!kvm_vcpu_has_pmu(vcpu) || !val) |
96b0eebc SZ |
303 | return; |
304 | ||
305 | for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) { | |
d56bdce5 MZ |
306 | struct kvm_pmc *pmc; |
307 | ||
96b0eebc SZ |
308 | if (!(val & BIT(i))) |
309 | continue; | |
310 | ||
d56bdce5 | 311 | pmc = kvm_vcpu_idx_to_pmc(vcpu, i); |
80f393a2 | 312 | |
96b0eebc SZ |
313 | if (pmc->perf_event) |
314 | perf_event_disable(pmc->perf_event); | |
315 | } | |
316 | } | |
7f766358 | 317 | |
76d883c4 SZ |
318 | static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu) |
319 | { | |
320 | u64 reg = 0; | |
321 | ||
8d404c4c CD |
322 | if ((__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) { |
323 | reg = __vcpu_sys_reg(vcpu, PMOVSSET_EL0); | |
324 | reg &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0); | |
325 | reg &= __vcpu_sys_reg(vcpu, PMINTENSET_EL1); | |
7d4bd1d2 | 326 | } |
76d883c4 SZ |
327 | |
328 | return reg; | |
329 | } | |
330 | ||
d9f89b4e | 331 | static void kvm_pmu_update_state(struct kvm_vcpu *vcpu) |
b7484931 AJ |
332 | { |
333 | struct kvm_pmu *pmu = &vcpu->arch.pmu; | |
d9f89b4e AJ |
334 | bool overflow; |
335 | ||
46acf89d | 336 | if (!kvm_vcpu_has_pmu(vcpu)) |
d9f89b4e | 337 | return; |
b7484931 | 338 | |
d9f89b4e | 339 | overflow = !!kvm_pmu_overflow_status(vcpu); |
b7484931 AJ |
340 | if (pmu->irq_level == overflow) |
341 | return; | |
342 | ||
343 | pmu->irq_level = overflow; | |
344 | ||
345 | if (likely(irqchip_in_kernel(vcpu->kvm))) { | |
346 | int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id, | |
d9f89b4e | 347 | pmu->irq_num, overflow, pmu); |
b7484931 AJ |
348 | WARN_ON(ret); |
349 | } | |
350 | } | |
351 | ||
3dbbdf78 CD |
352 | bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu) |
353 | { | |
354 | struct kvm_pmu *pmu = &vcpu->arch.pmu; | |
355 | struct kvm_sync_regs *sregs = &vcpu->run->s.regs; | |
356 | bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU; | |
357 | ||
358 | if (likely(irqchip_in_kernel(vcpu->kvm))) | |
359 | return false; | |
360 | ||
361 | return pmu->irq_level != run_level; | |
362 | } | |
363 | ||
364 | /* | |
365 | * Reflect the PMU overflow interrupt output level into the kvm_run structure | |
366 | */ | |
367 | void kvm_pmu_update_run(struct kvm_vcpu *vcpu) | |
368 | { | |
369 | struct kvm_sync_regs *regs = &vcpu->run->s.regs; | |
370 | ||
371 | /* Populate the timer bitmap for user space */ | |
372 | regs->device_irq_level &= ~KVM_ARM_DEV_PMU; | |
373 | if (vcpu->arch.pmu.irq_level) | |
374 | regs->device_irq_level |= KVM_ARM_DEV_PMU; | |
375 | } | |
376 | ||
b02386eb SZ |
377 | /** |
378 | * kvm_pmu_flush_hwstate - flush pmu state to cpu | |
379 | * @vcpu: The vcpu pointer | |
380 | * | |
381 | * Check if the PMU has overflowed while we were running in the host, and inject | |
382 | * an interrupt if that was the case. | |
383 | */ | |
384 | void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu) | |
385 | { | |
386 | kvm_pmu_update_state(vcpu); | |
387 | } | |
388 | ||
389 | /** | |
390 | * kvm_pmu_sync_hwstate - sync pmu state from cpu | |
391 | * @vcpu: The vcpu pointer | |
392 | * | |
393 | * Check if the PMU has overflowed while we were running in the guest, and | |
394 | * inject an interrupt if that was the case. | |
395 | */ | |
396 | void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu) | |
397 | { | |
398 | kvm_pmu_update_state(vcpu); | |
399 | } | |
400 | ||
95e92e45 JT |
401 | /** |
402 | * When perf interrupt is an NMI, we cannot safely notify the vcpu corresponding | |
403 | * to the event. | |
404 | * This is why we need a callback to do it once outside of the NMI context. | |
405 | */ | |
406 | static void kvm_pmu_perf_overflow_notify_vcpu(struct irq_work *work) | |
407 | { | |
408 | struct kvm_vcpu *vcpu; | |
95e92e45 | 409 | |
9bad925d | 410 | vcpu = container_of(work, struct kvm_vcpu, arch.pmu.overflow_work); |
95e92e45 JT |
411 | kvm_vcpu_kick(vcpu); |
412 | } | |
413 | ||
bead0220 MZ |
414 | /* |
415 | * Perform an increment on any of the counters described in @mask, | |
416 | * generating the overflow if required, and propagate it as a chained | |
417 | * event if possible. | |
418 | */ | |
419 | static void kvm_pmu_counter_increment(struct kvm_vcpu *vcpu, | |
420 | unsigned long mask, u32 event) | |
421 | { | |
422 | int i; | |
423 | ||
424 | if (!(__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E)) | |
425 | return; | |
426 | ||
427 | /* Weed out disabled counters */ | |
428 | mask &= __vcpu_sys_reg(vcpu, PMCNTENSET_EL0); | |
429 | ||
430 | for_each_set_bit(i, &mask, ARMV8_PMU_CYCLE_IDX) { | |
d56bdce5 | 431 | struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, i); |
bead0220 MZ |
432 | u64 type, reg; |
433 | ||
434 | /* Filter on event type */ | |
0cb9c3c8 | 435 | type = __vcpu_sys_reg(vcpu, counter_index_to_evtreg(i)); |
bead0220 MZ |
436 | type &= kvm_pmu_event_mask(vcpu->kvm); |
437 | if (type != event) | |
438 | continue; | |
439 | ||
440 | /* Increment this counter */ | |
0cb9c3c8 | 441 | reg = __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) + 1; |
d56bdce5 | 442 | if (!kvm_pmc_is_64bit(pmc)) |
0f1e172b | 443 | reg = lower_32_bits(reg); |
0cb9c3c8 | 444 | __vcpu_sys_reg(vcpu, counter_index_to_reg(i)) = reg; |
bead0220 | 445 | |
001d85bd | 446 | /* No overflow? move on */ |
d56bdce5 | 447 | if (kvm_pmc_has_64bit_overflow(pmc) ? reg : lower_32_bits(reg)) |
bead0220 MZ |
448 | continue; |
449 | ||
450 | /* Mark overflow */ | |
451 | __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(i); | |
452 | ||
d56bdce5 | 453 | if (kvm_pmu_counter_can_chain(pmc)) |
bead0220 MZ |
454 | kvm_pmu_counter_increment(vcpu, BIT(i + 1), |
455 | ARMV8_PMUV3_PERFCTR_CHAIN); | |
456 | } | |
457 | } | |
458 | ||
c82d28cb | 459 | /* Compute the sample period for a given counter value */ |
d56bdce5 | 460 | static u64 compute_period(struct kvm_pmc *pmc, u64 counter) |
c82d28cb MZ |
461 | { |
462 | u64 val; | |
463 | ||
d56bdce5 MZ |
464 | if (kvm_pmc_is_64bit(pmc)) { |
465 | if (!kvm_pmc_has_64bit_overflow(pmc)) | |
c82d28cb MZ |
466 | val = -(counter & GENMASK(31, 0)); |
467 | else | |
468 | val = (-counter) & GENMASK(63, 0); | |
469 | } else { | |
470 | val = (-counter) & GENMASK(31, 0); | |
471 | } | |
472 | ||
473 | return val; | |
474 | } | |
475 | ||
b02386eb | 476 | /** |
d9f89b4e | 477 | * When the perf event overflows, set the overflow status and inform the vcpu. |
b02386eb SZ |
478 | */ |
479 | static void kvm_pmu_perf_overflow(struct perf_event *perf_event, | |
480 | struct perf_sample_data *data, | |
481 | struct pt_regs *regs) | |
482 | { | |
483 | struct kvm_pmc *pmc = perf_event->overflow_handler_context; | |
8c3252c0 | 484 | struct arm_pmu *cpu_pmu = to_arm_pmu(perf_event->pmu); |
b02386eb SZ |
485 | struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); |
486 | int idx = pmc->idx; | |
8c3252c0 MZ |
487 | u64 period; |
488 | ||
489 | cpu_pmu->pmu.stop(perf_event, PERF_EF_UPDATE); | |
490 | ||
491 | /* | |
492 | * Reset the sample period to the architectural limit, | |
493 | * i.e. the point where the counter overflows. | |
494 | */ | |
d56bdce5 | 495 | period = compute_period(pmc, local64_read(&perf_event->count)); |
8c3252c0 MZ |
496 | |
497 | local64_set(&perf_event->hw.period_left, 0); | |
498 | perf_event->attr.sample_period = period; | |
499 | perf_event->hw.sample_period = period; | |
b02386eb | 500 | |
8d404c4c | 501 | __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= BIT(idx); |
d9f89b4e | 502 | |
d56bdce5 | 503 | if (kvm_pmu_counter_can_chain(pmc)) |
bead0220 MZ |
504 | kvm_pmu_counter_increment(vcpu, BIT(idx + 1), |
505 | ARMV8_PMUV3_PERFCTR_CHAIN); | |
506 | ||
d9f89b4e AJ |
507 | if (kvm_pmu_overflow_status(vcpu)) { |
508 | kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); | |
95e92e45 JT |
509 | |
510 | if (!in_nmi()) | |
511 | kvm_vcpu_kick(vcpu); | |
512 | else | |
513 | irq_work_queue(&vcpu->arch.pmu.overflow_work); | |
d9f89b4e | 514 | } |
8c3252c0 MZ |
515 | |
516 | cpu_pmu->pmu.start(perf_event, PERF_EF_RELOAD); | |
b02386eb SZ |
517 | } |
518 | ||
7a0adc70 SZ |
519 | /** |
520 | * kvm_pmu_software_increment - do software increment | |
521 | * @vcpu: The vcpu pointer | |
522 | * @val: the value guest writes to PMSWINC register | |
523 | */ | |
524 | void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) | |
525 | { | |
bead0220 | 526 | kvm_pmu_counter_increment(vcpu, val, ARMV8_PMUV3_PERFCTR_SW_INCR); |
7a0adc70 SZ |
527 | } |
528 | ||
76993739 SZ |
529 | /** |
530 | * kvm_pmu_handle_pmcr - handle PMCR register | |
531 | * @vcpu: The vcpu pointer | |
532 | * @val: the value guest writes to PMCR register | |
533 | */ | |
534 | void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) | |
535 | { | |
76993739 SZ |
536 | int i; |
537 | ||
8f6379e2 AE |
538 | if (!kvm_vcpu_has_pmu(vcpu)) |
539 | return; | |
540 | ||
76993739 | 541 | if (val & ARMV8_PMU_PMCR_E) { |
418e5ca8 | 542 | kvm_pmu_enable_counter_mask(vcpu, |
f5eff400 | 543 | __vcpu_sys_reg(vcpu, PMCNTENSET_EL0)); |
76993739 | 544 | } else { |
ca4f202d AC |
545 | kvm_pmu_disable_counter_mask(vcpu, |
546 | __vcpu_sys_reg(vcpu, PMCNTENSET_EL0)); | |
76993739 SZ |
547 | } |
548 | ||
549 | if (val & ARMV8_PMU_PMCR_C) | |
550 | kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0); | |
551 | ||
552 | if (val & ARMV8_PMU_PMCR_P) { | |
ca4f202d | 553 | unsigned long mask = kvm_pmu_valid_counter_mask(vcpu); |
2a71fabf | 554 | mask &= ~BIT(ARMV8_PMU_CYCLE_IDX); |
c01d6a18 | 555 | for_each_set_bit(i, &mask, 32) |
d56bdce5 | 556 | kvm_pmu_set_pmc_value(kvm_vcpu_idx_to_pmc(vcpu, i), 0, true); |
76993739 | 557 | } |
76993739 SZ |
558 | } |
559 | ||
d56bdce5 | 560 | static bool kvm_pmu_counter_is_enabled(struct kvm_pmc *pmc) |
7f766358 | 561 | { |
d56bdce5 | 562 | struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); |
8d404c4c | 563 | return (__vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) && |
d56bdce5 | 564 | (__vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(pmc->idx)); |
7f766358 SZ |
565 | } |
566 | ||
567 | /** | |
30d97754 | 568 | * kvm_pmu_create_perf_event - create a perf event for a counter |
d56bdce5 | 569 | * @pmc: Counter context |
7f766358 | 570 | */ |
d56bdce5 | 571 | static void kvm_pmu_create_perf_event(struct kvm_pmc *pmc) |
7f766358 | 572 | { |
d56bdce5 | 573 | struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc); |
46b18782 | 574 | struct arm_pmu *arm_pmu = vcpu->kvm->arch.arm_pmu; |
7f766358 SZ |
575 | struct perf_event *event; |
576 | struct perf_event_attr attr; | |
d56bdce5 | 577 | u64 eventsel, reg, data; |
30d97754 | 578 | |
d56bdce5 | 579 | reg = counter_index_to_evtreg(pmc->idx); |
30d97754 | 580 | data = __vcpu_sys_reg(vcpu, reg); |
7f766358 | 581 | |
d56bdce5 | 582 | kvm_pmu_stop_counter(pmc); |
d7eec236 MZ |
583 | if (pmc->idx == ARMV8_PMU_CYCLE_IDX) |
584 | eventsel = ARMV8_PMUV3_PERFCTR_CPU_CYCLES; | |
585 | else | |
586 | eventsel = data & kvm_pmu_event_mask(vcpu->kvm); | |
587 | ||
bead0220 MZ |
588 | /* |
589 | * Neither SW increment nor chained events need to be backed | |
590 | * by a perf event. | |
591 | */ | |
592 | if (eventsel == ARMV8_PMUV3_PERFCTR_SW_INCR || | |
593 | eventsel == ARMV8_PMUV3_PERFCTR_CHAIN) | |
d7eec236 | 594 | return; |
7f766358 | 595 | |
d7eec236 MZ |
596 | /* |
597 | * If we have a filter in place and that the event isn't allowed, do | |
598 | * not install a perf event either. | |
599 | */ | |
600 | if (vcpu->kvm->arch.pmu_filter && | |
601 | !test_bit(eventsel, vcpu->kvm->arch.pmu_filter)) | |
7a0adc70 SZ |
602 | return; |
603 | ||
7f766358 | 604 | memset(&attr, 0, sizeof(struct perf_event_attr)); |
46b18782 | 605 | attr.type = arm_pmu->pmu.type; |
7f766358 SZ |
606 | attr.size = sizeof(attr); |
607 | attr.pinned = 1; | |
d56bdce5 | 608 | attr.disabled = !kvm_pmu_counter_is_enabled(pmc); |
7f766358 SZ |
609 | attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0; |
610 | attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0; | |
611 | attr.exclude_hv = 1; /* Don't count EL2 events */ | |
612 | attr.exclude_host = 1; /* Don't count host events */ | |
d7eec236 | 613 | attr.config = eventsel; |
7f766358 | 614 | |
bead0220 MZ |
615 | /* |
616 | * If counting with a 64bit counter, advertise it to the perf | |
c82d28cb MZ |
617 | * code, carefully dealing with the initial sample period |
618 | * which also depends on the overflow. | |
bead0220 | 619 | */ |
d56bdce5 | 620 | if (kvm_pmc_is_64bit(pmc)) |
bead0220 | 621 | attr.config1 |= PERF_ATTR_CFG1_COUNTER_64BIT; |
c82d28cb | 622 | |
d56bdce5 | 623 | attr.sample_period = compute_period(pmc, kvm_pmu_get_pmc_value(pmc)); |
7f766358 | 624 | |
bead0220 | 625 | event = perf_event_create_kernel_counter(&attr, -1, current, |
b02386eb | 626 | kvm_pmu_perf_overflow, pmc); |
80f393a2 | 627 | |
7f766358 SZ |
628 | if (IS_ERR(event)) { |
629 | pr_err_once("kvm: pmu event creation failed %ld\n", | |
630 | PTR_ERR(event)); | |
631 | return; | |
632 | } | |
633 | ||
634 | pmc->perf_event = event; | |
635 | } | |
808e7381 | 636 | |
30d97754 AM |
637 | /** |
638 | * kvm_pmu_set_counter_event_type - set selected counter to monitor some event | |
639 | * @vcpu: The vcpu pointer | |
640 | * @data: The data guest writes to PMXEVTYPER_EL0 | |
641 | * @select_idx: The number of selected counter | |
642 | * | |
643 | * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an | |
644 | * event with given hardware event number. Here we call perf_event API to | |
645 | * emulate this action and create a kernel perf event for it. | |
646 | */ | |
647 | void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data, | |
648 | u64 select_idx) | |
649 | { | |
d56bdce5 | 650 | struct kvm_pmc *pmc = kvm_vcpu_idx_to_pmc(vcpu, select_idx); |
fd65a3b5 MZ |
651 | u64 reg, mask; |
652 | ||
8f6379e2 AE |
653 | if (!kvm_vcpu_has_pmu(vcpu)) |
654 | return; | |
655 | ||
fd65a3b5 MZ |
656 | mask = ARMV8_PMU_EVTYPE_MASK; |
657 | mask &= ~ARMV8_PMU_EVTYPE_EVENT; | |
658 | mask |= kvm_pmu_event_mask(vcpu->kvm); | |
30d97754 | 659 | |
d56bdce5 | 660 | reg = counter_index_to_evtreg(pmc->idx); |
30d97754 | 661 | |
fd65a3b5 | 662 | __vcpu_sys_reg(vcpu, reg) = data & mask; |
80f393a2 | 663 | |
d56bdce5 | 664 | kvm_pmu_create_perf_event(pmc); |
30d97754 AM |
665 | } |
666 | ||
e840f42a MZ |
667 | void kvm_host_pmu_init(struct arm_pmu *pmu) |
668 | { | |
db858060 AE |
669 | struct arm_pmu_entry *entry; |
670 | ||
fcf37b38 | 671 | if (pmu->pmuver == 0 || pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF) |
db858060 AE |
672 | return; |
673 | ||
674 | mutex_lock(&arm_pmus_lock); | |
675 | ||
676 | entry = kmalloc(sizeof(*entry), GFP_KERNEL); | |
677 | if (!entry) | |
678 | goto out_unlock; | |
679 | ||
680 | entry->arm_pmu = pmu; | |
681 | list_add_tail(&entry->entry, &arm_pmus); | |
682 | ||
683 | if (list_is_singular(&arm_pmus)) | |
e840f42a | 684 | static_branch_enable(&kvm_arm_pmu_available); |
db858060 AE |
685 | |
686 | out_unlock: | |
687 | mutex_unlock(&arm_pmus_lock); | |
e840f42a MZ |
688 | } |
689 | ||
46b18782 | 690 | static struct arm_pmu *kvm_pmu_probe_armpmu(void) |
fd65a3b5 MZ |
691 | { |
692 | struct perf_event_attr attr = { }; | |
693 | struct perf_event *event; | |
46b18782 | 694 | struct arm_pmu *pmu = NULL; |
fd65a3b5 MZ |
695 | |
696 | /* | |
697 | * Create a dummy event that only counts user cycles. As we'll never | |
698 | * leave this function with the event being live, it will never | |
699 | * count anything. But it allows us to probe some of the PMU | |
700 | * details. Yes, this is terrible. | |
701 | */ | |
702 | attr.type = PERF_TYPE_RAW; | |
703 | attr.size = sizeof(attr); | |
704 | attr.pinned = 1; | |
705 | attr.disabled = 0; | |
706 | attr.exclude_user = 0; | |
707 | attr.exclude_kernel = 1; | |
708 | attr.exclude_hv = 1; | |
709 | attr.exclude_host = 1; | |
710 | attr.config = ARMV8_PMUV3_PERFCTR_CPU_CYCLES; | |
711 | attr.sample_period = GENMASK(63, 0); | |
712 | ||
713 | event = perf_event_create_kernel_counter(&attr, -1, current, | |
714 | kvm_pmu_perf_overflow, &attr); | |
715 | ||
716 | if (IS_ERR(event)) { | |
717 | pr_err_once("kvm: pmu event creation failed %ld\n", | |
718 | PTR_ERR(event)); | |
46b18782 | 719 | return NULL; |
fd65a3b5 MZ |
720 | } |
721 | ||
722 | if (event->pmu) { | |
723 | pmu = to_arm_pmu(event->pmu); | |
46b18782 | 724 | if (pmu->pmuver == 0 || |
fcf37b38 | 725 | pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF) |
46b18782 | 726 | pmu = NULL; |
fd65a3b5 MZ |
727 | } |
728 | ||
729 | perf_event_disable(event); | |
730 | perf_event_release_kernel(event); | |
731 | ||
46b18782 | 732 | return pmu; |
fd65a3b5 MZ |
733 | } |
734 | ||
88865bec MZ |
735 | u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) |
736 | { | |
737 | unsigned long *bmap = vcpu->kvm->arch.pmu_filter; | |
738 | u64 val, mask = 0; | |
9529aaa0 | 739 | int base, i, nr_events; |
88865bec | 740 | |
8f6379e2 AE |
741 | if (!kvm_vcpu_has_pmu(vcpu)) |
742 | return 0; | |
743 | ||
88865bec MZ |
744 | if (!pmceid1) { |
745 | val = read_sysreg(pmceid0_el0); | |
acdd8a4e MZ |
746 | /* always support CHAIN */ |
747 | val |= BIT(ARMV8_PMUV3_PERFCTR_CHAIN); | |
88865bec MZ |
748 | base = 0; |
749 | } else { | |
750 | val = read_sysreg(pmceid1_el0); | |
46081078 MZ |
751 | /* |
752 | * Don't advertise STALL_SLOT, as PMMIR_EL0 is handled | |
753 | * as RAZ | |
754 | */ | |
121a8fc0 | 755 | if (vcpu->kvm->arch.arm_pmu->pmuver >= ID_AA64DFR0_EL1_PMUVer_V3P4) |
46081078 | 756 | val &= ~BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32); |
88865bec MZ |
757 | base = 32; |
758 | } | |
759 | ||
760 | if (!bmap) | |
761 | return val; | |
762 | ||
9529aaa0 MZ |
763 | nr_events = kvm_pmu_event_mask(vcpu->kvm) + 1; |
764 | ||
88865bec MZ |
765 | for (i = 0; i < 32; i += 8) { |
766 | u64 byte; | |
767 | ||
768 | byte = bitmap_get_value8(bmap, base + i); | |
769 | mask |= byte << i; | |
9529aaa0 MZ |
770 | if (nr_events >= (0x4000 + base + 32)) { |
771 | byte = bitmap_get_value8(bmap, 0x4000 + base + i); | |
772 | mask |= byte << (32 + i); | |
773 | } | |
88865bec MZ |
774 | } |
775 | ||
776 | return val & mask; | |
777 | } | |
778 | ||
a2befacf | 779 | int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu) |
bb0c70bc | 780 | { |
9bbfa4b5 | 781 | if (!kvm_vcpu_has_pmu(vcpu)) |
a2befacf | 782 | return 0; |
bb0c70bc | 783 | |
9bbfa4b5 AE |
784 | if (!vcpu->arch.pmu.created) |
785 | return -EINVAL; | |
786 | ||
6fe407f2 | 787 | /* |
a2befacf CD |
788 | * A valid interrupt configuration for the PMU is either to have a |
789 | * properly configured interrupt number and using an in-kernel | |
ebb127f2 | 790 | * irqchip, or to not have an in-kernel GIC and not set an IRQ. |
6fe407f2 | 791 | */ |
ebb127f2 CD |
792 | if (irqchip_in_kernel(vcpu->kvm)) { |
793 | int irq = vcpu->arch.pmu.irq_num; | |
ebb127f2 CD |
794 | /* |
795 | * If we are using an in-kernel vgic, at this point we know | |
796 | * the vgic will be initialized, so we can check the PMU irq | |
797 | * number against the dimensions of the vgic and make sure | |
798 | * it's valid. | |
799 | */ | |
800 | if (!irq_is_ppi(irq) && !vgic_valid_spi(vcpu->kvm, irq)) | |
801 | return -EINVAL; | |
802 | } else if (kvm_arm_pmu_irq_initialized(vcpu)) { | |
803 | return -EINVAL; | |
804 | } | |
a2befacf | 805 | |
d0c94c49 MZ |
806 | /* One-off reload of the PMU on first run */ |
807 | kvm_make_request(KVM_REQ_RELOAD_PMU, vcpu); | |
808 | ||
a2befacf CD |
809 | return 0; |
810 | } | |
811 | ||
812 | static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu) | |
813 | { | |
a2befacf | 814 | if (irqchip_in_kernel(vcpu->kvm)) { |
abcb851d CD |
815 | int ret; |
816 | ||
a2befacf CD |
817 | /* |
818 | * If using the PMU with an in-kernel virtual GIC | |
819 | * implementation, we require the GIC to be already | |
820 | * initialized when initializing the PMU. | |
821 | */ | |
822 | if (!vgic_initialized(vcpu->kvm)) | |
823 | return -ENODEV; | |
824 | ||
825 | if (!kvm_arm_pmu_irq_initialized(vcpu)) | |
826 | return -ENXIO; | |
abcb851d CD |
827 | |
828 | ret = kvm_vgic_set_owner(vcpu, vcpu->arch.pmu.irq_num, | |
829 | &vcpu->arch.pmu); | |
830 | if (ret) | |
831 | return ret; | |
a2befacf | 832 | } |
bb0c70bc | 833 | |
95e92e45 JT |
834 | init_irq_work(&vcpu->arch.pmu.overflow_work, |
835 | kvm_pmu_perf_overflow_notify_vcpu); | |
836 | ||
a2befacf | 837 | vcpu->arch.pmu.created = true; |
bb0c70bc SZ |
838 | return 0; |
839 | } | |
840 | ||
2defaff4 AP |
841 | /* |
842 | * For one VM the interrupt type must be same for each vcpu. | |
843 | * As a PPI, the interrupt number is the same for all vcpus, | |
844 | * while as an SPI it must be a separate number per vcpu. | |
845 | */ | |
846 | static bool pmu_irq_is_valid(struct kvm *kvm, int irq) | |
bb0c70bc | 847 | { |
46808a4c | 848 | unsigned long i; |
bb0c70bc SZ |
849 | struct kvm_vcpu *vcpu; |
850 | ||
851 | kvm_for_each_vcpu(i, vcpu, kvm) { | |
852 | if (!kvm_arm_pmu_irq_initialized(vcpu)) | |
853 | continue; | |
854 | ||
2defaff4 | 855 | if (irq_is_ppi(irq)) { |
bb0c70bc SZ |
856 | if (vcpu->arch.pmu.irq_num != irq) |
857 | return false; | |
858 | } else { | |
859 | if (vcpu->arch.pmu.irq_num == irq) | |
860 | return false; | |
861 | } | |
862 | } | |
863 | ||
864 | return true; | |
865 | } | |
866 | ||
6ee7fca2 AE |
867 | static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id) |
868 | { | |
869 | struct kvm *kvm = vcpu->kvm; | |
870 | struct arm_pmu_entry *entry; | |
871 | struct arm_pmu *arm_pmu; | |
872 | int ret = -ENXIO; | |
873 | ||
874 | mutex_lock(&kvm->lock); | |
875 | mutex_lock(&arm_pmus_lock); | |
876 | ||
877 | list_for_each_entry(entry, &arm_pmus, entry) { | |
878 | arm_pmu = entry->arm_pmu; | |
879 | if (arm_pmu->pmu.type == pmu_id) { | |
06394531 | 880 | if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags) || |
6ee7fca2 AE |
881 | (kvm->arch.pmu_filter && kvm->arch.arm_pmu != arm_pmu)) { |
882 | ret = -EBUSY; | |
883 | break; | |
884 | } | |
885 | ||
886 | kvm->arch.arm_pmu = arm_pmu; | |
583cda1b | 887 | cpumask_copy(kvm->arch.supported_cpus, &arm_pmu->supported_cpus); |
6ee7fca2 AE |
888 | ret = 0; |
889 | break; | |
890 | } | |
891 | } | |
892 | ||
893 | mutex_unlock(&arm_pmus_lock); | |
894 | mutex_unlock(&kvm->lock); | |
895 | return ret; | |
896 | } | |
897 | ||
bb0c70bc SZ |
898 | int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) |
899 | { | |
5177fe91 MZ |
900 | struct kvm *kvm = vcpu->kvm; |
901 | ||
77da4303 | 902 | if (!kvm_vcpu_has_pmu(vcpu)) |
42223fb1 MZ |
903 | return -ENODEV; |
904 | ||
905 | if (vcpu->arch.pmu.created) | |
906 | return -EBUSY; | |
907 | ||
46b18782 MZ |
908 | mutex_lock(&kvm->lock); |
909 | if (!kvm->arch.arm_pmu) { | |
910 | /* No PMU set, get the default one */ | |
911 | kvm->arch.arm_pmu = kvm_pmu_probe_armpmu(); | |
912 | if (!kvm->arch.arm_pmu) { | |
913 | mutex_unlock(&kvm->lock); | |
914 | return -ENODEV; | |
915 | } | |
916 | } | |
917 | mutex_unlock(&kvm->lock); | |
fd65a3b5 | 918 | |
bb0c70bc SZ |
919 | switch (attr->attr) { |
920 | case KVM_ARM_VCPU_PMU_V3_IRQ: { | |
921 | int __user *uaddr = (int __user *)(long)attr->addr; | |
922 | int irq; | |
923 | ||
5177fe91 | 924 | if (!irqchip_in_kernel(kvm)) |
a2befacf CD |
925 | return -EINVAL; |
926 | ||
bb0c70bc SZ |
927 | if (get_user(irq, uaddr)) |
928 | return -EFAULT; | |
929 | ||
2defaff4 | 930 | /* The PMU overflow interrupt can be a PPI or a valid SPI. */ |
ebb127f2 | 931 | if (!(irq_is_ppi(irq) || irq_is_spi(irq))) |
2defaff4 AP |
932 | return -EINVAL; |
933 | ||
5177fe91 | 934 | if (!pmu_irq_is_valid(kvm, irq)) |
bb0c70bc SZ |
935 | return -EINVAL; |
936 | ||
937 | if (kvm_arm_pmu_irq_initialized(vcpu)) | |
938 | return -EBUSY; | |
939 | ||
940 | kvm_debug("Set kvm ARM PMU irq: %d\n", irq); | |
941 | vcpu->arch.pmu.irq_num = irq; | |
942 | return 0; | |
943 | } | |
d7eec236 MZ |
944 | case KVM_ARM_VCPU_PMU_V3_FILTER: { |
945 | struct kvm_pmu_event_filter __user *uaddr; | |
946 | struct kvm_pmu_event_filter filter; | |
947 | int nr_events; | |
948 | ||
5177fe91 | 949 | nr_events = kvm_pmu_event_mask(kvm) + 1; |
d7eec236 MZ |
950 | |
951 | uaddr = (struct kvm_pmu_event_filter __user *)(long)attr->addr; | |
952 | ||
953 | if (copy_from_user(&filter, uaddr, sizeof(filter))) | |
954 | return -EFAULT; | |
955 | ||
956 | if (((u32)filter.base_event + filter.nevents) > nr_events || | |
957 | (filter.action != KVM_PMU_EVENT_ALLOW && | |
958 | filter.action != KVM_PMU_EVENT_DENY)) | |
959 | return -EINVAL; | |
960 | ||
5177fe91 MZ |
961 | mutex_lock(&kvm->lock); |
962 | ||
06394531 | 963 | if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags)) { |
5177fe91 MZ |
964 | mutex_unlock(&kvm->lock); |
965 | return -EBUSY; | |
966 | } | |
d7eec236 | 967 | |
5177fe91 MZ |
968 | if (!kvm->arch.pmu_filter) { |
969 | kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT); | |
970 | if (!kvm->arch.pmu_filter) { | |
971 | mutex_unlock(&kvm->lock); | |
d7eec236 MZ |
972 | return -ENOMEM; |
973 | } | |
974 | ||
975 | /* | |
976 | * The default depends on the first applied filter. | |
977 | * If it allows events, the default is to deny. | |
978 | * Conversely, if the first filter denies a set of | |
979 | * events, the default is to allow. | |
980 | */ | |
981 | if (filter.action == KVM_PMU_EVENT_ALLOW) | |
5177fe91 | 982 | bitmap_zero(kvm->arch.pmu_filter, nr_events); |
d7eec236 | 983 | else |
5177fe91 | 984 | bitmap_fill(kvm->arch.pmu_filter, nr_events); |
d7eec236 MZ |
985 | } |
986 | ||
987 | if (filter.action == KVM_PMU_EVENT_ALLOW) | |
5177fe91 | 988 | bitmap_set(kvm->arch.pmu_filter, filter.base_event, filter.nevents); |
d7eec236 | 989 | else |
5177fe91 | 990 | bitmap_clear(kvm->arch.pmu_filter, filter.base_event, filter.nevents); |
d7eec236 | 991 | |
5177fe91 | 992 | mutex_unlock(&kvm->lock); |
d7eec236 MZ |
993 | |
994 | return 0; | |
995 | } | |
6ee7fca2 AE |
996 | case KVM_ARM_VCPU_PMU_V3_SET_PMU: { |
997 | int __user *uaddr = (int __user *)(long)attr->addr; | |
998 | int pmu_id; | |
999 | ||
1000 | if (get_user(pmu_id, uaddr)) | |
1001 | return -EFAULT; | |
1002 | ||
1003 | return kvm_arm_pmu_v3_set_pmu(vcpu, pmu_id); | |
1004 | } | |
bb0c70bc SZ |
1005 | case KVM_ARM_VCPU_PMU_V3_INIT: |
1006 | return kvm_arm_pmu_v3_init(vcpu); | |
1007 | } | |
1008 | ||
1009 | return -ENXIO; | |
1010 | } | |
1011 | ||
1012 | int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) | |
1013 | { | |
1014 | switch (attr->attr) { | |
1015 | case KVM_ARM_VCPU_PMU_V3_IRQ: { | |
1016 | int __user *uaddr = (int __user *)(long)attr->addr; | |
1017 | int irq; | |
1018 | ||
a2befacf CD |
1019 | if (!irqchip_in_kernel(vcpu->kvm)) |
1020 | return -EINVAL; | |
1021 | ||
14bda7a9 | 1022 | if (!kvm_vcpu_has_pmu(vcpu)) |
bb0c70bc SZ |
1023 | return -ENODEV; |
1024 | ||
1025 | if (!kvm_arm_pmu_irq_initialized(vcpu)) | |
1026 | return -ENXIO; | |
1027 | ||
1028 | irq = vcpu->arch.pmu.irq_num; | |
1029 | return put_user(irq, uaddr); | |
1030 | } | |
1031 | } | |
1032 | ||
1033 | return -ENXIO; | |
1034 | } | |
1035 | ||
1036 | int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) | |
1037 | { | |
1038 | switch (attr->attr) { | |
1039 | case KVM_ARM_VCPU_PMU_V3_IRQ: | |
1040 | case KVM_ARM_VCPU_PMU_V3_INIT: | |
d7eec236 | 1041 | case KVM_ARM_VCPU_PMU_V3_FILTER: |
6ee7fca2 | 1042 | case KVM_ARM_VCPU_PMU_V3_SET_PMU: |
77da4303 | 1043 | if (kvm_vcpu_has_pmu(vcpu)) |
bb0c70bc SZ |
1044 | return 0; |
1045 | } | |
1046 | ||
1047 | return -ENXIO; | |
1048 | } | |
3d0dba57 MZ |
1049 | |
1050 | u8 kvm_arm_pmu_get_pmuver_limit(void) | |
1051 | { | |
1052 | u64 tmp; | |
1053 | ||
1054 | tmp = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1); | |
1055 | tmp = cpuid_feature_cap_perfmon_field(tmp, | |
1056 | ID_AA64DFR0_EL1_PMUVer_SHIFT, | |
1f7c9782 | 1057 | ID_AA64DFR0_EL1_PMUVer_V3P5); |
3d0dba57 MZ |
1058 | return FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), tmp); |
1059 | } |