virt/kvm/arm/pmu.c

   1 /*
   2  * Copyright (C) 2015 Linaro Ltd.
   3  * Author: Shannon Zhao <shannon.zhao@linaro.org>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/cpu.h>
  19 #include <linux/kvm.h>
  20 #include <linux/kvm_host.h>
  21 #include <linux/perf_event.h>
  22 #include <linux/uaccess.h>
  23 #include <asm/kvm_emulate.h>
  24 #include <kvm/arm_pmu.h>
  25 #include <kvm/arm_vgic.h>
  26
  27 /**
  28  * kvm_pmu_get_counter_value - get PMU counter value
  29  * @vcpu: The vcpu pointer
  30  * @select_idx: The counter index
  31  */
  32 u64 kvm_pmu_get_counter_value(struct kvm_vcpu *vcpu, u64 select_idx)
  33 {
  34         u64 counter, reg, enabled, running;
  35         struct kvm_pmu *pmu = &vcpu->arch.pmu;
  36         struct kvm_pmc *pmc = &pmu->pmc[select_idx];
  37
  38         reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
  39               ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
  40         counter = vcpu_sys_reg(vcpu, reg);
  41
  42         /* The real counter value is equal to the value of counter register plus
  43          * the value perf event counts.
  44          */
  45         if (pmc->perf_event)
  46                 counter += perf_event_read_value(pmc->perf_event, &enabled,
  47                                                  &running);
  48
  49         return counter & pmc->bitmask;
  50 }
  51
  52 /**
  53  * kvm_pmu_set_counter_value - set PMU counter value
  54  * @vcpu: The vcpu pointer
  55  * @select_idx: The counter index
  56  * @val: The counter value
  57  */
  58 void kvm_pmu_set_counter_value(struct kvm_vcpu *vcpu, u64 select_idx, u64 val)
  59 {
  60         u64 reg;
  61
  62         reg = (select_idx == ARMV8_PMU_CYCLE_IDX)
  63               ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + select_idx;
  64         vcpu_sys_reg(vcpu, reg) += (s64)val - kvm_pmu_get_counter_value(vcpu, select_idx);
  65 }
  66
  67 /**
  68  * kvm_pmu_stop_counter - stop PMU counter
  69  * @pmc: The PMU counter pointer
  70  *
  71  * If this counter has been configured to monitor some event, release it here.
  72  */
  73 static void kvm_pmu_stop_counter(struct kvm_vcpu *vcpu, struct kvm_pmc *pmc)
  74 {
  75         u64 counter, reg;
  76
  77         if (pmc->perf_event) {
  78                 counter = kvm_pmu_get_counter_value(vcpu, pmc->idx);
  79                 reg = (pmc->idx == ARMV8_PMU_CYCLE_IDX)
  80                        ? PMCCNTR_EL0 : PMEVCNTR0_EL0 + pmc->idx;
  81                 vcpu_sys_reg(vcpu, reg) = counter;
  82                 perf_event_disable(pmc->perf_event);
  83                 perf_event_release_kernel(pmc->perf_event);
  84                 pmc->perf_event = NULL;
  85         }
  86 }
  87
  88 /**
  89  * kvm_pmu_vcpu_reset - reset pmu state for cpu
  90  * @vcpu: The vcpu pointer
  91  *
  92  */
  93 void kvm_pmu_vcpu_reset(struct kvm_vcpu *vcpu)
  94 {
  95         int i;
  96         struct kvm_pmu *pmu = &vcpu->arch.pmu;
  97
  98         for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
  99                 kvm_pmu_stop_counter(vcpu, &pmu->pmc[i]);
 100                 pmu->pmc[i].idx = i;
 101                 pmu->pmc[i].bitmask = 0xffffffffUL;
 102         }
 103 }
 104
 105 /**
 106  * kvm_pmu_vcpu_destroy - free perf event of PMU for cpu
 107  * @vcpu: The vcpu pointer
 108  *
 109  */
 110 void kvm_pmu_vcpu_destroy(struct kvm_vcpu *vcpu)
 111 {
 112         int i;
 113         struct kvm_pmu *pmu = &vcpu->arch.pmu;
 114
 115         for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
 116                 struct kvm_pmc *pmc = &pmu->pmc[i];
 117
 118                 if (pmc->perf_event) {
 119                         perf_event_disable(pmc->perf_event);
 120                         perf_event_release_kernel(pmc->perf_event);
 121                         pmc->perf_event = NULL;
 122                 }
 123         }
 124 }
 125
 126 u64 kvm_pmu_valid_counter_mask(struct kvm_vcpu *vcpu)
 127 {
 128         u64 val = vcpu_sys_reg(vcpu, PMCR_EL0) >> ARMV8_PMU_PMCR_N_SHIFT;
 129
 130         val &= ARMV8_PMU_PMCR_N_MASK;
 131         if (val == 0)
 132                 return BIT(ARMV8_PMU_CYCLE_IDX);
 133         else
 134                 return GENMASK(val - 1, 0) | BIT(ARMV8_PMU_CYCLE_IDX);
 135 }
 136
 137 /**
 138  * kvm_pmu_enable_counter - enable selected PMU counter
 139  * @vcpu: The vcpu pointer
 140  * @val: the value guest writes to PMCNTENSET register
 141  *
 142  * Call perf_event_enable to start counting the perf event
 143  */
 144 void kvm_pmu_enable_counter(struct kvm_vcpu *vcpu, u64 val)
 145 {
 146         int i;
 147         struct kvm_pmu *pmu = &vcpu->arch.pmu;
 148         struct kvm_pmc *pmc;
 149
 150         if (!(vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) || !val)
 151                 return;
 152
 153         for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
 154                 if (!(val & BIT(i)))
 155                         continue;
 156
 157                 pmc = &pmu->pmc[i];
 158                 if (pmc->perf_event) {
 159                         perf_event_enable(pmc->perf_event);
 160                         if (pmc->perf_event->state != PERF_EVENT_STATE_ACTIVE)
 161                                 kvm_debug("fail to enable perf event\n");
 162                 }
 163         }
 164 }
 165
 166 /**
 167  * kvm_pmu_disable_counter - disable selected PMU counter
 168  * @vcpu: The vcpu pointer
 169  * @val: the value guest writes to PMCNTENCLR register
 170  *
 171  * Call perf_event_disable to stop counting the perf event
 172  */
 173 void kvm_pmu_disable_counter(struct kvm_vcpu *vcpu, u64 val)
 174 {
 175         int i;
 176         struct kvm_pmu *pmu = &vcpu->arch.pmu;
 177         struct kvm_pmc *pmc;
 178
 179         if (!val)
 180                 return;
 181
 182         for (i = 0; i < ARMV8_PMU_MAX_COUNTERS; i++) {
 183                 if (!(val & BIT(i)))
 184                         continue;
 185
 186                 pmc = &pmu->pmc[i];
 187                 if (pmc->perf_event)
 188                         perf_event_disable(pmc->perf_event);
 189         }
 190 }
 191
 192 static u64 kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
 193 {
 194         u64 reg = 0;
 195
 196         if ((vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E))
 197                 reg = vcpu_sys_reg(vcpu, PMOVSSET_EL0);
 198                 reg &= vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
 199                 reg &= vcpu_sys_reg(vcpu, PMINTENSET_EL1);
 200                 reg &= kvm_pmu_valid_counter_mask(vcpu);
 201
 202         return reg;
 203 }
 204
 205 /**
 206  * kvm_pmu_overflow_set - set PMU overflow interrupt
 207  * @vcpu: The vcpu pointer
 208  * @val: the value guest writes to PMOVSSET register
 209  */
 210 void kvm_pmu_overflow_set(struct kvm_vcpu *vcpu, u64 val)
 211 {
 212         u64 reg;
 213
 214         if (val == 0)
 215                 return;
 216
 217         vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= val;
 218         reg = kvm_pmu_overflow_status(vcpu);
 219         if (reg != 0)
 220                 kvm_vcpu_kick(vcpu);
 221 }
 222
 223 static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
 224 {
 225         struct kvm_pmu *pmu = &vcpu->arch.pmu;
 226         bool overflow;
 227
 228         if (!kvm_arm_pmu_v3_ready(vcpu))
 229                 return;
 230
 231         overflow = !!kvm_pmu_overflow_status(vcpu);
 232         if (pmu->irq_level != overflow) {
 233                 pmu->irq_level = overflow;
 234                 kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
 235                                     pmu->irq_num, overflow);
 236         }
 237 }
 238
 239 /**
 240  * kvm_pmu_flush_hwstate - flush pmu state to cpu
 241  * @vcpu: The vcpu pointer
 242  *
 243  * Check if the PMU has overflowed while we were running in the host, and inject
 244  * an interrupt if that was the case.
 245  */
 246 void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu)
 247 {
 248         kvm_pmu_update_state(vcpu);
 249 }
 250
 251 /**
 252  * kvm_pmu_sync_hwstate - sync pmu state from cpu
 253  * @vcpu: The vcpu pointer
 254  *
 255  * Check if the PMU has overflowed while we were running in the guest, and
 256  * inject an interrupt if that was the case.
 257  */
 258 void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu)
 259 {
 260         kvm_pmu_update_state(vcpu);
 261 }
 262
 263 static inline struct kvm_vcpu *kvm_pmc_to_vcpu(struct kvm_pmc *pmc)
 264 {
 265         struct kvm_pmu *pmu;
 266         struct kvm_vcpu_arch *vcpu_arch;
 267
 268         pmc -= pmc->idx;
 269         pmu = container_of(pmc, struct kvm_pmu, pmc[0]);
 270         vcpu_arch = container_of(pmu, struct kvm_vcpu_arch, pmu);
 271         return container_of(vcpu_arch, struct kvm_vcpu, arch);
 272 }
 273
 274 /**
 275  * When perf event overflows, call kvm_pmu_overflow_set to set overflow status.
 276  */
 277 static void kvm_pmu_perf_overflow(struct perf_event *perf_event,
 278                                   struct perf_sample_data *data,
 279                                   struct pt_regs *regs)
 280 {
 281         struct kvm_pmc *pmc = perf_event->overflow_handler_context;
 282         struct kvm_vcpu *vcpu = kvm_pmc_to_vcpu(pmc);
 283         int idx = pmc->idx;
 284
 285         kvm_pmu_overflow_set(vcpu, BIT(idx));
 286 }
 287
 288 /**
 289  * kvm_pmu_software_increment - do software increment
 290  * @vcpu: The vcpu pointer
 291  * @val: the value guest writes to PMSWINC register
 292  */
 293 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val)
 294 {
 295         int i;
 296         u64 type, enable, reg;
 297
 298         if (val == 0)
 299                 return;
 300
 301         enable = vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
 302         for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++) {
 303                 if (!(val & BIT(i)))
 304                         continue;
 305                 type = vcpu_sys_reg(vcpu, PMEVTYPER0_EL0 + i)
 306                        & ARMV8_PMU_EVTYPE_EVENT;
 307                 if ((type == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
 308                     && (enable & BIT(i))) {
 309                         reg = vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) + 1;
 310                         reg = lower_32_bits(reg);
 311                         vcpu_sys_reg(vcpu, PMEVCNTR0_EL0 + i) = reg;
 312                         if (!reg)
 313                                 kvm_pmu_overflow_set(vcpu, BIT(i));
 314                 }
 315         }
 316 }
 317
 318 /**
 319  * kvm_pmu_handle_pmcr - handle PMCR register
 320  * @vcpu: The vcpu pointer
 321  * @val: the value guest writes to PMCR register
 322  */
 323 void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val)
 324 {
 325         struct kvm_pmu *pmu = &vcpu->arch.pmu;
 326         struct kvm_pmc *pmc;
 327         u64 mask;
 328         int i;
 329
 330         mask = kvm_pmu_valid_counter_mask(vcpu);
 331         if (val & ARMV8_PMU_PMCR_E) {
 332                 kvm_pmu_enable_counter(vcpu,
 333                                 vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & mask);
 334         } else {
 335                 kvm_pmu_disable_counter(vcpu, mask);
 336         }
 337
 338         if (val & ARMV8_PMU_PMCR_C)
 339                 kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0);
 340
 341         if (val & ARMV8_PMU_PMCR_P) {
 342                 for (i = 0; i < ARMV8_PMU_CYCLE_IDX; i++)
 343                         kvm_pmu_set_counter_value(vcpu, i, 0);
 344         }
 345
 346         if (val & ARMV8_PMU_PMCR_LC) {
 347                 pmc = &pmu->pmc[ARMV8_PMU_CYCLE_IDX];
 348                 pmc->bitmask = 0xffffffffffffffffUL;
 349         }
 350 }
 351
 352 static bool kvm_pmu_counter_is_enabled(struct kvm_vcpu *vcpu, u64 select_idx)
 353 {
 354         return (vcpu_sys_reg(vcpu, PMCR_EL0) & ARMV8_PMU_PMCR_E) &&
 355                (vcpu_sys_reg(vcpu, PMCNTENSET_EL0) & BIT(select_idx));
 356 }
 357
 358 /**
 359  * kvm_pmu_set_counter_event_type - set selected counter to monitor some event
 360  * @vcpu: The vcpu pointer
 361  * @data: The data guest writes to PMXEVTYPER_EL0
 362  * @select_idx: The number of selected counter
 363  *
 364  * When OS accesses PMXEVTYPER_EL0, that means it wants to set a PMC to count an
 365  * event with given hardware event number. Here we call perf_event API to
 366  * emulate this action and create a kernel perf event for it.
 367  */
 368 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
 369                                     u64 select_idx)
 370 {
 371         struct kvm_pmu *pmu = &vcpu->arch.pmu;
 372         struct kvm_pmc *pmc = &pmu->pmc[select_idx];
 373         struct perf_event *event;
 374         struct perf_event_attr attr;
 375         u64 eventsel, counter;
 376
 377         kvm_pmu_stop_counter(vcpu, pmc);
 378         eventsel = data & ARMV8_PMU_EVTYPE_EVENT;
 379
 380         /* Software increment event does't need to be backed by a perf event */
 381         if (eventsel == ARMV8_PMU_EVTYPE_EVENT_SW_INCR)
 382                 return;
 383
 384         memset(&attr, 0, sizeof(struct perf_event_attr));
 385         attr.type = PERF_TYPE_RAW;
 386         attr.size = sizeof(attr);
 387         attr.pinned = 1;
 388         attr.disabled = !kvm_pmu_counter_is_enabled(vcpu, select_idx);
 389         attr.exclude_user = data & ARMV8_PMU_EXCLUDE_EL0 ? 1 : 0;
 390         attr.exclude_kernel = data & ARMV8_PMU_EXCLUDE_EL1 ? 1 : 0;
 391         attr.exclude_hv = 1; /* Don't count EL2 events */
 392         attr.exclude_host = 1; /* Don't count host events */
 393         attr.config = eventsel;
 394
 395         counter = kvm_pmu_get_counter_value(vcpu, select_idx);
 396         /* The initial sample period (overflow count) of an event. */
 397         attr.sample_period = (-counter) & pmc->bitmask;
 398
 399         event = perf_event_create_kernel_counter(&attr, -1, current,
 400                                                  kvm_pmu_perf_overflow, pmc);
 401         if (IS_ERR(event)) {
 402                 pr_err_once("kvm: pmu event creation failed %ld\n",
 403                             PTR_ERR(event));
 404                 return;
 405         }
 406
 407         pmc->perf_event = event;
 408 }
 409
 410 bool kvm_arm_support_pmu_v3(void)
 411 {
 412         /*
 413          * Check if HW_PERF_EVENTS are supported by checking the number of
 414          * hardware performance counters. This could ensure the presence of
 415          * a physical PMU and CONFIG_PERF_EVENT is selected.
 416          */
 417         return (perf_num_counters() > 0);
 418 }
 419
 420 static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu)
 421 {
 422         if (!kvm_arm_support_pmu_v3())
 423                 return -ENODEV;
 424
 425         if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features) ||
 426             !kvm_arm_pmu_irq_initialized(vcpu))
 427                 return -ENXIO;
 428
 429         if (kvm_arm_pmu_v3_ready(vcpu))
 430                 return -EBUSY;
 431
 432         kvm_pmu_vcpu_reset(vcpu);
 433         vcpu->arch.pmu.ready = true;
 434
 435         return 0;
 436 }
 437
 438 static bool irq_is_valid(struct kvm *kvm, int irq, bool is_ppi)
 439 {
 440         int i;
 441         struct kvm_vcpu *vcpu;
 442
 443         kvm_for_each_vcpu(i, vcpu, kvm) {
 444                 if (!kvm_arm_pmu_irq_initialized(vcpu))
 445                         continue;
 446
 447                 if (is_ppi) {
 448                         if (vcpu->arch.pmu.irq_num != irq)
 449                                 return false;
 450                 } else {
 451                         if (vcpu->arch.pmu.irq_num == irq)
 452                                 return false;
 453                 }
 454         }
 455
 456         return true;
 457 }
 458
 459
 460 int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 461 {
 462         switch (attr->attr) {
 463         case KVM_ARM_VCPU_PMU_V3_IRQ: {
 464                 int __user *uaddr = (int __user *)(long)attr->addr;
 465                 int irq;
 466
 467                 if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
 468                         return -ENODEV;
 469
 470                 if (get_user(irq, uaddr))
 471                         return -EFAULT;
 472
 473                 /*
 474                  * The PMU overflow interrupt could be a PPI or SPI, but for one
 475                  * VM the interrupt type must be same for each vcpu. As a PPI,
 476                  * the interrupt number is the same for all vcpus, while as an
 477                  * SPI it must be a separate number per vcpu.
 478                  */
 479                 if (irq < VGIC_NR_SGIS || irq >= vcpu->kvm->arch.vgic.nr_irqs ||
 480                     !irq_is_valid(vcpu->kvm, irq, irq < VGIC_NR_PRIVATE_IRQS))
 481                         return -EINVAL;
 482
 483                 if (kvm_arm_pmu_irq_initialized(vcpu))
 484                         return -EBUSY;
 485
 486                 kvm_debug("Set kvm ARM PMU irq: %d\n", irq);
 487                 vcpu->arch.pmu.irq_num = irq;
 488                 return 0;
 489         }
 490         case KVM_ARM_VCPU_PMU_V3_INIT:
 491                 return kvm_arm_pmu_v3_init(vcpu);
 492         }
 493
 494         return -ENXIO;
 495 }
 496
 497 int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 498 {
 499         switch (attr->attr) {
 500         case KVM_ARM_VCPU_PMU_V3_IRQ: {
 501                 int __user *uaddr = (int __user *)(long)attr->addr;
 502                 int irq;
 503
 504                 if (!test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
 505                         return -ENODEV;
 506
 507                 if (!kvm_arm_pmu_irq_initialized(vcpu))
 508                         return -ENXIO;
 509
 510                 irq = vcpu->arch.pmu.irq_num;
 511                 return put_user(irq, uaddr);
 512         }
 513         }
 514
 515         return -ENXIO;
 516 }
 517
 518 int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 519 {
 520         switch (attr->attr) {
 521         case KVM_ARM_VCPU_PMU_V3_IRQ:
 522         case KVM_ARM_VCPU_PMU_V3_INIT:
 523                 if (kvm_arm_support_pmu_v3() &&
 524                     test_bit(KVM_ARM_VCPU_PMU_V3, vcpu->arch.features))
 525                         return 0;
 526         }
 527
 528         return -ENXIO;
 529 }