virt/kvm/arm/arch_timer.c

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/irq.h>
  24 #include <linux/uaccess.h>
  25
  26 #include <clocksource/arm_arch_timer.h>
  27 #include <asm/arch_timer.h>
  28 #include <asm/kvm_hyp.h>
  29
  30 #include <kvm/arm_vgic.h>
  31 #include <kvm/arm_arch_timer.h>
  32
  33 #include "trace.h"
  34
  35 static struct timecounter *timecounter;
  36 static unsigned int host_vtimer_irq;
  37 static u32 host_vtimer_irq_flags;
  38
  39 static DEFINE_STATIC_KEY_FALSE(has_gic_active_state);
  40
  41 static const struct kvm_irq_level default_ptimer_irq = {
  42         .irq    = 30,
  43         .level  = 1,
  44 };
  45
  46 static const struct kvm_irq_level default_vtimer_irq = {
  47         .irq    = 27,
  48         .level  = 1,
  49 };
  50
  51 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
  52 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
  53                                  struct arch_timer_context *timer_ctx);
  54 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
  55
  56 u64 kvm_phys_timer_read(void)
  57 {
  58         return timecounter->cc->read(timecounter->cc);
  59 }
  60
  61 static inline bool userspace_irqchip(struct kvm *kvm)
  62 {
  63         return static_branch_unlikely(&userspace_irqchip_in_use) &&
  64                 unlikely(!irqchip_in_kernel(kvm));
  65 }
  66
  67 static void soft_timer_start(struct hrtimer *hrt, u64 ns)
  68 {
  69         hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns),
  70                       HRTIMER_MODE_ABS);
  71 }
  72
  73 static void soft_timer_cancel(struct hrtimer *hrt)
  74 {
  75         hrtimer_cancel(hrt);
  76 }
  77
  78 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  79 {
  80         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  81         struct arch_timer_context *vtimer;
  82
  83         /*
  84          * We may see a timer interrupt after vcpu_put() has been called which
  85          * sets the CPU's vcpu pointer to NULL, because even though the timer
  86          * has been disabled in vtimer_save_state(), the hardware interrupt
  87          * signal may not have been retired from the interrupt controller yet.
  88          */
  89         if (!vcpu)
  90                 return IRQ_HANDLED;
  91
  92         vtimer = vcpu_vtimer(vcpu);
  93         if (kvm_timer_should_fire(vtimer))
  94                 kvm_timer_update_irq(vcpu, true, vtimer);
  95
  96         if (userspace_irqchip(vcpu->kvm) &&
  97             !static_branch_unlikely(&has_gic_active_state))
  98                 disable_percpu_irq(host_vtimer_irq);
  99
 100         return IRQ_HANDLED;
 101 }
 102
 103 static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
 104 {
 105         u64 cval, now;
 106
 107         cval = timer_ctx->cnt_cval;
 108         now = kvm_phys_timer_read() - timer_ctx->cntvoff;
 109
 110         if (now < cval) {
 111                 u64 ns;
 112
 113                 ns = cyclecounter_cyc2ns(timecounter->cc,
 114                                          cval - now,
 115                                          timecounter->mask,
 116                                          &timecounter->frac);
 117                 return ns;
 118         }
 119
 120         return 0;
 121 }
 122
 123 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
 124 {
 125         return !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
 126                 (timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE);
 127 }
 128
 129 /*
 130  * Returns the earliest expiration time in ns among guest timers.
 131  * Note that it will return 0 if none of timers can fire.
 132  */
 133 static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
 134 {
 135         u64 min_virt = ULLONG_MAX, min_phys = ULLONG_MAX;
 136         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 137         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 138
 139         if (kvm_timer_irq_can_fire(vtimer))
 140                 min_virt = kvm_timer_compute_delta(vtimer);
 141
 142         if (kvm_timer_irq_can_fire(ptimer))
 143                 min_phys = kvm_timer_compute_delta(ptimer);
 144
 145         /* If none of timers can fire, then return 0 */
 146         if ((min_virt == ULLONG_MAX) && (min_phys == ULLONG_MAX))
 147                 return 0;
 148
 149         return min(min_virt, min_phys);
 150 }
 151
 152 static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt)
 153 {
 154         struct arch_timer_cpu *timer;
 155         struct kvm_vcpu *vcpu;
 156         u64 ns;
 157
 158         timer = container_of(hrt, struct arch_timer_cpu, bg_timer);
 159         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 160
 161         /*
 162          * Check that the timer has really expired from the guest's
 163          * PoV (NTP on the host may have forced it to expire
 164          * early). If we should have slept longer, restart it.
 165          */
 166         ns = kvm_timer_earliest_exp(vcpu);
 167         if (unlikely(ns)) {
 168                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 169                 return HRTIMER_RESTART;
 170         }
 171
 172         kvm_vcpu_wake_up(vcpu);
 173         return HRTIMER_NORESTART;
 174 }
 175
 176 static enum hrtimer_restart kvm_phys_timer_expire(struct hrtimer *hrt)
 177 {
 178         struct arch_timer_context *ptimer;
 179         struct arch_timer_cpu *timer;
 180         struct kvm_vcpu *vcpu;
 181         u64 ns;
 182
 183         timer = container_of(hrt, struct arch_timer_cpu, phys_timer);
 184         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 185         ptimer = vcpu_ptimer(vcpu);
 186
 187         /*
 188          * Check that the timer has really expired from the guest's
 189          * PoV (NTP on the host may have forced it to expire
 190          * early). If not ready, schedule for a later time.
 191          */
 192         ns = kvm_timer_compute_delta(ptimer);
 193         if (unlikely(ns)) {
 194                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 195                 return HRTIMER_RESTART;
 196         }
 197
 198         kvm_timer_update_irq(vcpu, true, ptimer);
 199         return HRTIMER_NORESTART;
 200 }
 201
 202 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
 203 {
 204         u64 cval, now;
 205
 206         if (timer_ctx->loaded) {
 207                 u32 cnt_ctl;
 208
 209                 /* Only the virtual timer can be loaded so far */
 210                 cnt_ctl = read_sysreg_el0(cntv_ctl);
 211                 return  (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) &&
 212                         (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) &&
 213                        !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
 214         }
 215
 216         if (!kvm_timer_irq_can_fire(timer_ctx))
 217                 return false;
 218
 219         cval = timer_ctx->cnt_cval;
 220         now = kvm_phys_timer_read() - timer_ctx->cntvoff;
 221
 222         return cval <= now;
 223 }
 224
 225 bool kvm_timer_is_pending(struct kvm_vcpu *vcpu)
 226 {
 227         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 228         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 229
 230         if (kvm_timer_should_fire(vtimer))
 231                 return true;
 232
 233         return kvm_timer_should_fire(ptimer);
 234 }
 235
 236 /*
 237  * Reflect the timer output level into the kvm_run structure
 238  */
 239 void kvm_timer_update_run(struct kvm_vcpu *vcpu)
 240 {
 241         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 242         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 243         struct kvm_sync_regs *regs = &vcpu->run->s.regs;
 244
 245         /* Populate the device bitmap with the timer states */
 246         regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
 247                                     KVM_ARM_DEV_EL1_PTIMER);
 248         if (kvm_timer_should_fire(vtimer))
 249                 regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
 250         if (kvm_timer_should_fire(ptimer))
 251                 regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
 252 }
 253
 254 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 255                                  struct arch_timer_context *timer_ctx)
 256 {
 257         int ret;
 258
 259         timer_ctx->irq.level = new_level;
 260         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
 261                                    timer_ctx->irq.level);
 262
 263         if (!userspace_irqchip(vcpu->kvm)) {
 264                 ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
 265                                           timer_ctx->irq.irq,
 266                                           timer_ctx->irq.level,
 267                                           timer_ctx);
 268                 WARN_ON(ret);
 269         }
 270 }
 271
 272 /* Schedule the background timer for the emulated timer. */
 273 static void phys_timer_emulate(struct kvm_vcpu *vcpu)
 274 {
 275         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 276         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 277
 278         /*
 279          * If the timer can fire now, we don't need to have a soft timer
 280          * scheduled for the future.  If the timer cannot fire at all,
 281          * then we also don't need a soft timer.
 282          */
 283         if (kvm_timer_should_fire(ptimer) || !kvm_timer_irq_can_fire(ptimer)) {
 284                 soft_timer_cancel(&timer->phys_timer);
 285                 return;
 286         }
 287
 288         soft_timer_start(&timer->phys_timer, kvm_timer_compute_delta(ptimer));
 289 }
 290
 291 /*
 292  * Check if there was a change in the timer state, so that we should either
 293  * raise or lower the line level to the GIC or schedule a background timer to
 294  * emulate the physical timer.
 295  */
 296 static void kvm_timer_update_state(struct kvm_vcpu *vcpu)
 297 {
 298         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 299         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 300         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 301         bool level;
 302
 303         if (unlikely(!timer->enabled))
 304                 return;
 305
 306         /*
 307          * The vtimer virtual interrupt is a 'mapped' interrupt, meaning part
 308          * of its lifecycle is offloaded to the hardware, and we therefore may
 309          * not have lowered the irq.level value before having to signal a new
 310          * interrupt, but have to signal an interrupt every time the level is
 311          * asserted.
 312          */
 313         level = kvm_timer_should_fire(vtimer);
 314         kvm_timer_update_irq(vcpu, level, vtimer);
 315
 316         phys_timer_emulate(vcpu);
 317
 318         if (kvm_timer_should_fire(ptimer) != ptimer->irq.level)
 319                 kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer);
 320 }
 321
 322 static void vtimer_save_state(struct kvm_vcpu *vcpu)
 323 {
 324         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 325         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 326         unsigned long flags;
 327
 328         local_irq_save(flags);
 329
 330         if (!vtimer->loaded)
 331                 goto out;
 332
 333         if (timer->enabled) {
 334                 vtimer->cnt_ctl = read_sysreg_el0(cntv_ctl);
 335                 vtimer->cnt_cval = read_sysreg_el0(cntv_cval);
 336         }
 337
 338         /* Disable the virtual timer */
 339         write_sysreg_el0(0, cntv_ctl);
 340         isb();
 341
 342         vtimer->loaded = false;
 343 out:
 344         local_irq_restore(flags);
 345 }
 346
 347 /*
 348  * Schedule the background timer before calling kvm_vcpu_block, so that this
 349  * thread is removed from its waitqueue and made runnable when there's a timer
 350  * interrupt to handle.
 351  */
 352 void kvm_timer_schedule(struct kvm_vcpu *vcpu)
 353 {
 354         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 355         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 356         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 357
 358         vtimer_save_state(vcpu);
 359
 360         /*
 361          * No need to schedule a background timer if any guest timer has
 362          * already expired, because kvm_vcpu_block will return before putting
 363          * the thread to sleep.
 364          */
 365         if (kvm_timer_should_fire(vtimer) || kvm_timer_should_fire(ptimer))
 366                 return;
 367
 368         /*
 369          * If both timers are not capable of raising interrupts (disabled or
 370          * masked), then there's no more work for us to do.
 371          */
 372         if (!kvm_timer_irq_can_fire(vtimer) && !kvm_timer_irq_can_fire(ptimer))
 373                 return;
 374
 375         /*
 376          * The guest timers have not yet expired, schedule a background timer.
 377          * Set the earliest expiration time among the guest timers.
 378          */
 379         soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));
 380 }
 381
 382 static void vtimer_restore_state(struct kvm_vcpu *vcpu)
 383 {
 384         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 385         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 386         unsigned long flags;
 387
 388         local_irq_save(flags);
 389
 390         if (vtimer->loaded)
 391                 goto out;
 392
 393         if (timer->enabled) {
 394                 write_sysreg_el0(vtimer->cnt_cval, cntv_cval);
 395                 isb();
 396                 write_sysreg_el0(vtimer->cnt_ctl, cntv_ctl);
 397         }
 398
 399         vtimer->loaded = true;
 400 out:
 401         local_irq_restore(flags);
 402 }
 403
 404 void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
 405 {
 406         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 407
 408         vtimer_restore_state(vcpu);
 409
 410         soft_timer_cancel(&timer->bg_timer);
 411 }
 412
 413 static void set_cntvoff(u64 cntvoff)
 414 {
 415         u32 low = lower_32_bits(cntvoff);
 416         u32 high = upper_32_bits(cntvoff);
 417
 418         /*
 419          * Since kvm_call_hyp doesn't fully support the ARM PCS especially on
 420          * 32-bit systems, but rather passes register by register shifted one
 421          * place (we put the function address in r0/x0), we cannot simply pass
 422          * a 64-bit value as an argument, but have to split the value in two
 423          * 32-bit halves.
 424          */
 425         kvm_call_hyp(__kvm_timer_set_cntvoff, low, high);
 426 }
 427
 428 static inline void set_vtimer_irq_phys_active(struct kvm_vcpu *vcpu, bool active)
 429 {
 430         int r;
 431         r = irq_set_irqchip_state(host_vtimer_irq, IRQCHIP_STATE_ACTIVE, active);
 432         WARN_ON(r);
 433 }
 434
 435 static void kvm_timer_vcpu_load_gic(struct kvm_vcpu *vcpu)
 436 {
 437         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 438         bool phys_active;
 439
 440         if (irqchip_in_kernel(vcpu->kvm))
 441                 phys_active = kvm_vgic_map_is_active(vcpu, vtimer->irq.irq);
 442         else
 443                 phys_active = vtimer->irq.level;
 444         set_vtimer_irq_phys_active(vcpu, phys_active);
 445 }
 446
 447 static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
 448 {
 449         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 450
 451         /*
 452          * When using a userspace irqchip with the architected timers and a
 453          * host interrupt controller that doesn't support an active state, we
 454          * must still prevent continuously exiting from the guest, and
 455          * therefore mask the physical interrupt by disabling it on the host
 456          * interrupt controller when the virtual level is high, such that the
 457          * guest can make forward progress.  Once we detect the output level
 458          * being de-asserted, we unmask the interrupt again so that we exit
 459          * from the guest when the timer fires.
 460          */
 461         if (vtimer->irq.level)
 462                 disable_percpu_irq(host_vtimer_irq);
 463         else
 464                 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 465 }
 466
 467 void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
 468 {
 469         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 470         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 471         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 472
 473         if (unlikely(!timer->enabled))
 474                 return;
 475
 476         if (static_branch_likely(&has_gic_active_state))
 477                 kvm_timer_vcpu_load_gic(vcpu);
 478         else
 479                 kvm_timer_vcpu_load_nogic(vcpu);
 480
 481         set_cntvoff(vtimer->cntvoff);
 482
 483         vtimer_restore_state(vcpu);
 484
 485         /* Set the background timer for the physical timer emulation. */
 486         phys_timer_emulate(vcpu);
 487
 488         /* If the timer fired while we weren't running, inject it now */
 489         if (kvm_timer_should_fire(ptimer) != ptimer->irq.level)
 490                 kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer);
 491 }
 492
 493 bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
 494 {
 495         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 496         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 497         struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
 498         bool vlevel, plevel;
 499
 500         if (likely(irqchip_in_kernel(vcpu->kvm)))
 501                 return false;
 502
 503         vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
 504         plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
 505
 506         return kvm_timer_should_fire(vtimer) != vlevel ||
 507                kvm_timer_should_fire(ptimer) != plevel;
 508 }
 509
 510 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
 511 {
 512         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 513
 514         if (unlikely(!timer->enabled))
 515                 return;
 516
 517         vtimer_save_state(vcpu);
 518
 519         /*
 520          * Cancel the physical timer emulation, because the only case where we
 521          * need it after a vcpu_put is in the context of a sleeping VCPU, and
 522          * in that case we already factor in the deadline for the physical
 523          * timer when scheduling the bg_timer.
 524          *
 525          * In any case, we re-schedule the hrtimer for the physical timer when
 526          * coming back to the VCPU thread in kvm_timer_vcpu_load().
 527          */
 528         soft_timer_cancel(&timer->phys_timer);
 529
 530         /*
 531          * The kernel may decide to run userspace after calling vcpu_put, so
 532          * we reset cntvoff to 0 to ensure a consistent read between user
 533          * accesses to the virtual counter and kernel access to the physical
 534          * counter of non-VHE case. For VHE, the virtual counter uses a fixed
 535          * virtual offset of zero, so no need to zero CNTVOFF_EL2 register.
 536          */
 537         if (!has_vhe())
 538                 set_cntvoff(0);
 539 }
 540
 541 /*
 542  * With a userspace irqchip we have to check if the guest de-asserted the
 543  * timer and if so, unmask the timer irq signal on the host interrupt
 544  * controller to ensure that we see future timer signals.
 545  */
 546 static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
 547 {
 548         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 549
 550         if (!kvm_timer_should_fire(vtimer)) {
 551                 kvm_timer_update_irq(vcpu, false, vtimer);
 552                 if (static_branch_likely(&has_gic_active_state))
 553                         set_vtimer_irq_phys_active(vcpu, false);
 554                 else
 555                         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 556         }
 557 }
 558
 559 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 560 {
 561         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 562
 563         if (unlikely(!timer->enabled))
 564                 return;
 565
 566         if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
 567                 unmask_vtimer_irq_user(vcpu);
 568 }
 569
 570 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
 571 {
 572         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 573         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 574         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 575
 576         /*
 577          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 578          * and to 0 for ARMv7.  We provide an implementation that always
 579          * resets the timer to be disabled and unmasked and is compliant with
 580          * the ARMv7 architecture.
 581          */
 582         vtimer->cnt_ctl = 0;
 583         ptimer->cnt_ctl = 0;
 584         kvm_timer_update_state(vcpu);
 585
 586         if (timer->enabled && irqchip_in_kernel(vcpu->kvm))
 587                 kvm_vgic_reset_mapped_irq(vcpu, vtimer->irq.irq);
 588
 589         return 0;
 590 }
 591
 592 /* Make the updates of cntvoff for all vtimer contexts atomic */
 593 static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff)
 594 {
 595         int i;
 596         struct kvm *kvm = vcpu->kvm;
 597         struct kvm_vcpu *tmp;
 598
 599         mutex_lock(&kvm->lock);
 600         kvm_for_each_vcpu(i, tmp, kvm)
 601                 vcpu_vtimer(tmp)->cntvoff = cntvoff;
 602
 603         /*
 604          * When called from the vcpu create path, the CPU being created is not
 605          * included in the loop above, so we just set it here as well.
 606          */
 607         vcpu_vtimer(vcpu)->cntvoff = cntvoff;
 608         mutex_unlock(&kvm->lock);
 609 }
 610
 611 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 612 {
 613         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 614         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 615         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 616
 617         /* Synchronize cntvoff across all vtimers of a VM. */
 618         update_vtimer_cntvoff(vcpu, kvm_phys_timer_read());
 619         vcpu_ptimer(vcpu)->cntvoff = 0;
 620
 621         hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 622         timer->bg_timer.function = kvm_bg_timer_expire;
 623
 624         hrtimer_init(&timer->phys_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 625         timer->phys_timer.function = kvm_phys_timer_expire;
 626
 627         vtimer->irq.irq = default_vtimer_irq.irq;
 628         ptimer->irq.irq = default_ptimer_irq.irq;
 629 }
 630
 631 static void kvm_timer_init_interrupt(void *info)
 632 {
 633         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 634 }
 635
 636 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 637 {
 638         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 639         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 640
 641         switch (regid) {
 642         case KVM_REG_ARM_TIMER_CTL:
 643                 vtimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT;
 644                 break;
 645         case KVM_REG_ARM_TIMER_CNT:
 646                 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value);
 647                 break;
 648         case KVM_REG_ARM_TIMER_CVAL:
 649                 vtimer->cnt_cval = value;
 650                 break;
 651         case KVM_REG_ARM_PTIMER_CTL:
 652                 ptimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT;
 653                 break;
 654         case KVM_REG_ARM_PTIMER_CVAL:
 655                 ptimer->cnt_cval = value;
 656                 break;
 657
 658         default:
 659                 return -1;
 660         }
 661
 662         kvm_timer_update_state(vcpu);
 663         return 0;
 664 }
 665
 666 static u64 read_timer_ctl(struct arch_timer_context *timer)
 667 {
 668         /*
 669          * Set ISTATUS bit if it's expired.
 670          * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is
 671          * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit
 672          * regardless of ENABLE bit for our implementation convenience.
 673          */
 674         if (!kvm_timer_compute_delta(timer))
 675                 return timer->cnt_ctl | ARCH_TIMER_CTRL_IT_STAT;
 676         else
 677                 return timer->cnt_ctl;
 678 }
 679
 680 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 681 {
 682         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 683         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 684
 685         switch (regid) {
 686         case KVM_REG_ARM_TIMER_CTL:
 687                 return read_timer_ctl(vtimer);
 688         case KVM_REG_ARM_TIMER_CNT:
 689                 return kvm_phys_timer_read() - vtimer->cntvoff;
 690         case KVM_REG_ARM_TIMER_CVAL:
 691                 return vtimer->cnt_cval;
 692         case KVM_REG_ARM_PTIMER_CTL:
 693                 return read_timer_ctl(ptimer);
 694         case KVM_REG_ARM_PTIMER_CVAL:
 695                 return ptimer->cnt_cval;
 696         case KVM_REG_ARM_PTIMER_CNT:
 697                 return kvm_phys_timer_read();
 698         }
 699         return (u64)-1;
 700 }
 701
 702 static int kvm_timer_starting_cpu(unsigned int cpu)
 703 {
 704         kvm_timer_init_interrupt(NULL);
 705         return 0;
 706 }
 707
 708 static int kvm_timer_dying_cpu(unsigned int cpu)
 709 {
 710         disable_percpu_irq(host_vtimer_irq);
 711         return 0;
 712 }
 713
 714 int kvm_timer_hyp_init(bool has_gic)
 715 {
 716         struct arch_timer_kvm_info *info;
 717         int err;
 718
 719         info = arch_timer_get_kvm_info();
 720         timecounter = &info->timecounter;
 721
 722         if (!timecounter->cc) {
 723                 kvm_err("kvm_arch_timer: uninitialized timecounter\n");
 724                 return -ENODEV;
 725         }
 726
 727         if (info->virtual_irq <= 0) {
 728                 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
 729                         info->virtual_irq);
 730                 return -ENODEV;
 731         }
 732         host_vtimer_irq = info->virtual_irq;
 733
 734         host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
 735         if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
 736             host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
 737                 kvm_err("Invalid trigger for IRQ%d, assuming level low\n",
 738                         host_vtimer_irq);
 739                 host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
 740         }
 741
 742         err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
 743                                  "kvm guest timer", kvm_get_running_vcpus());
 744         if (err) {
 745                 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
 746                         host_vtimer_irq, err);
 747                 return err;
 748         }
 749
 750         if (has_gic) {
 751                 err = irq_set_vcpu_affinity(host_vtimer_irq,
 752                                             kvm_get_running_vcpus());
 753                 if (err) {
 754                         kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
 755                         goto out_free_irq;
 756                 }
 757
 758                 static_branch_enable(&has_gic_active_state);
 759         }
 760
 761         kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
 762
 763         cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
 764                           "kvm/arm/timer:starting", kvm_timer_starting_cpu,
 765                           kvm_timer_dying_cpu);
 766         return 0;
 767 out_free_irq:
 768         free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
 769         return err;
 770 }
 771
 772 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
 773 {
 774         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 775
 776         soft_timer_cancel(&timer->bg_timer);
 777 }
 778
 779 static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
 780 {
 781         int vtimer_irq, ptimer_irq;
 782         int i, ret;
 783
 784         vtimer_irq = vcpu_vtimer(vcpu)->irq.irq;
 785         ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu));
 786         if (ret)
 787                 return false;
 788
 789         ptimer_irq = vcpu_ptimer(vcpu)->irq.irq;
 790         ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu));
 791         if (ret)
 792                 return false;
 793
 794         kvm_for_each_vcpu(i, vcpu, vcpu->kvm) {
 795                 if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq ||
 796                     vcpu_ptimer(vcpu)->irq.irq != ptimer_irq)
 797                         return false;
 798         }
 799
 800         return true;
 801 }
 802
 803 bool kvm_arch_timer_get_input_level(int vintid)
 804 {
 805         struct kvm_vcpu *vcpu = kvm_arm_get_running_vcpu();
 806         struct arch_timer_context *timer;
 807
 808         if (vintid == vcpu_vtimer(vcpu)->irq.irq)
 809                 timer = vcpu_vtimer(vcpu);
 810         else
 811                 BUG(); /* We only map the vtimer so far */
 812
 813         return kvm_timer_should_fire(timer);
 814 }
 815
 816 int kvm_timer_enable(struct kvm_vcpu *vcpu)
 817 {
 818         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 819         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 820         int ret;
 821
 822         if (timer->enabled)
 823                 return 0;
 824
 825         /* Without a VGIC we do not map virtual IRQs to physical IRQs */
 826         if (!irqchip_in_kernel(vcpu->kvm))
 827                 goto no_vgic;
 828
 829         if (!vgic_initialized(vcpu->kvm))
 830                 return -ENODEV;
 831
 832         if (!timer_irqs_are_valid(vcpu)) {
 833                 kvm_debug("incorrectly configured timer irqs\n");
 834                 return -EINVAL;
 835         }
 836
 837         ret = kvm_vgic_map_phys_irq(vcpu, host_vtimer_irq, vtimer->irq.irq,
 838                                     kvm_arch_timer_get_input_level);
 839         if (ret)
 840                 return ret;
 841
 842 no_vgic:
 843         timer->enabled = 1;
 844         return 0;
 845 }
 846
 847 /*
 848  * On VHE system, we only need to configure trap on physical timer and counter
 849  * accesses in EL0 and EL1 once, not for every world switch.
 850  * The host kernel runs at EL2 with HCR_EL2.TGE == 1,
 851  * and this makes those bits have no effect for the host kernel execution.
 852  */
 853 void kvm_timer_init_vhe(void)
 854 {
 855         /* When HCR_EL2.E2H ==1, EL1PCEN and EL1PCTEN are shifted by 10 */
 856         u32 cnthctl_shift = 10;
 857         u64 val;
 858
 859         /*
 860          * Disallow physical timer access for the guest.
 861          * Physical counter access is allowed.
 862          */
 863         val = read_sysreg(cnthctl_el2);
 864         val &= ~(CNTHCTL_EL1PCEN << cnthctl_shift);
 865         val |= (CNTHCTL_EL1PCTEN << cnthctl_shift);
 866         write_sysreg(val, cnthctl_el2);
 867 }
 868
 869 static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq)
 870 {
 871         struct kvm_vcpu *vcpu;
 872         int i;
 873
 874         kvm_for_each_vcpu(i, vcpu, kvm) {
 875                 vcpu_vtimer(vcpu)->irq.irq = vtimer_irq;
 876                 vcpu_ptimer(vcpu)->irq.irq = ptimer_irq;
 877         }
 878 }
 879
 880 int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 881 {
 882         int __user *uaddr = (int __user *)(long)attr->addr;
 883         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 884         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 885         int irq;
 886
 887         if (!irqchip_in_kernel(vcpu->kvm))
 888                 return -EINVAL;
 889
 890         if (get_user(irq, uaddr))
 891                 return -EFAULT;
 892
 893         if (!(irq_is_ppi(irq)))
 894                 return -EINVAL;
 895
 896         if (vcpu->arch.timer_cpu.enabled)
 897                 return -EBUSY;
 898
 899         switch (attr->attr) {
 900         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
 901                 set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq);
 902                 break;
 903         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
 904                 set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq);
 905                 break;
 906         default:
 907                 return -ENXIO;
 908         }
 909
 910         return 0;
 911 }
 912
 913 int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 914 {
 915         int __user *uaddr = (int __user *)(long)attr->addr;
 916         struct arch_timer_context *timer;
 917         int irq;
 918
 919         switch (attr->attr) {
 920         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
 921                 timer = vcpu_vtimer(vcpu);
 922                 break;
 923         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
 924                 timer = vcpu_ptimer(vcpu);
 925                 break;
 926         default:
 927                 return -ENXIO;
 928         }
 929
 930         irq = timer->irq.irq;
 931         return put_user(irq, uaddr);
 932 }
 933
 934 int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 935 {
 936         switch (attr->attr) {
 937         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
 938         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
 939                 return 0;
 940         }
 941
 942         return -ENXIO;
 943 }