virt/kvm/arm/arch_timer.c

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/irq.h>
  24 #include <linux/uaccess.h>
  25
  26 #include <clocksource/arm_arch_timer.h>
  27 #include <asm/arch_timer.h>
  28 #include <asm/kvm_hyp.h>
  29
  30 #include <kvm/arm_vgic.h>
  31 #include <kvm/arm_arch_timer.h>
  32
  33 #include "trace.h"
  34
  35 static struct timecounter *timecounter;
  36 static unsigned int host_vtimer_irq;
  37 static u32 host_vtimer_irq_flags;
  38
  39 static const struct kvm_irq_level default_ptimer_irq = {
  40         .irq    = 30,
  41         .level  = 1,
  42 };
  43
  44 static const struct kvm_irq_level default_vtimer_irq = {
  45         .irq    = 27,
  46         .level  = 1,
  47 };
  48
  49 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
  50 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
  51                                  struct arch_timer_context *timer_ctx);
  52 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
  53
  54 u64 kvm_phys_timer_read(void)
  55 {
  56         return timecounter->cc->read(timecounter->cc);
  57 }
  58
  59 static void soft_timer_start(struct hrtimer *hrt, u64 ns)
  60 {
  61         hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns),
  62                       HRTIMER_MODE_ABS);
  63 }
  64
  65 static void soft_timer_cancel(struct hrtimer *hrt, struct work_struct *work)
  66 {
  67         hrtimer_cancel(hrt);
  68         if (work)
  69                 cancel_work_sync(work);
  70 }
  71
  72 static void kvm_vtimer_update_mask_user(struct kvm_vcpu *vcpu)
  73 {
  74         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
  75
  76         /*
  77          * When using a userspace irqchip with the architected timers, we must
  78          * prevent continuously exiting from the guest, and therefore mask the
  79          * physical interrupt by disabling it on the host interrupt controller
  80          * when the virtual level is high, such that the guest can make
  81          * forward progress.  Once we detect the output level being
  82          * de-asserted, we unmask the interrupt again so that we exit from the
  83          * guest when the timer fires.
  84          */
  85         if (vtimer->irq.level)
  86                 disable_percpu_irq(host_vtimer_irq);
  87         else
  88                 enable_percpu_irq(host_vtimer_irq, 0);
  89 }
  90
  91 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  92 {
  93         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  94         struct arch_timer_context *vtimer;
  95
  96         /*
  97          * We may see a timer interrupt after vcpu_put() has been called which
  98          * sets the CPU's vcpu pointer to NULL, because even though the timer
  99          * has been disabled in vtimer_save_state(), the hardware interrupt
 100          * signal may not have been retired from the interrupt controller yet.
 101          */
 102         if (!vcpu)
 103                 return IRQ_HANDLED;
 104
 105         vtimer = vcpu_vtimer(vcpu);
 106         if (kvm_timer_should_fire(vtimer))
 107                 kvm_timer_update_irq(vcpu, true, vtimer);
 108
 109         if (static_branch_unlikely(&userspace_irqchip_in_use) &&
 110             unlikely(!irqchip_in_kernel(vcpu->kvm)))
 111                 kvm_vtimer_update_mask_user(vcpu);
 112
 113         return IRQ_HANDLED;
 114 }
 115
 116 /*
 117  * Work function for handling the backup timer that we schedule when a vcpu is
 118  * no longer running, but had a timer programmed to fire in the future.
 119  */
 120 static void kvm_timer_inject_irq_work(struct work_struct *work)
 121 {
 122         struct kvm_vcpu *vcpu;
 123
 124         vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
 125
 126         /*
 127          * If the vcpu is blocked we want to wake it up so that it will see
 128          * the timer has expired when entering the guest.
 129          */
 130         kvm_vcpu_wake_up(vcpu);
 131 }
 132
 133 static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
 134 {
 135         u64 cval, now;
 136
 137         cval = timer_ctx->cnt_cval;
 138         now = kvm_phys_timer_read() - timer_ctx->cntvoff;
 139
 140         if (now < cval) {
 141                 u64 ns;
 142
 143                 ns = cyclecounter_cyc2ns(timecounter->cc,
 144                                          cval - now,
 145                                          timecounter->mask,
 146                                          &timecounter->frac);
 147                 return ns;
 148         }
 149
 150         return 0;
 151 }
 152
 153 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
 154 {
 155         return !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
 156                 (timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE);
 157 }
 158
 159 /*
 160  * Returns the earliest expiration time in ns among guest timers.
 161  * Note that it will return 0 if none of timers can fire.
 162  */
 163 static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
 164 {
 165         u64 min_virt = ULLONG_MAX, min_phys = ULLONG_MAX;
 166         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 167         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 168
 169         if (kvm_timer_irq_can_fire(vtimer))
 170                 min_virt = kvm_timer_compute_delta(vtimer);
 171
 172         if (kvm_timer_irq_can_fire(ptimer))
 173                 min_phys = kvm_timer_compute_delta(ptimer);
 174
 175         /* If none of timers can fire, then return 0 */
 176         if ((min_virt == ULLONG_MAX) && (min_phys == ULLONG_MAX))
 177                 return 0;
 178
 179         return min(min_virt, min_phys);
 180 }
 181
 182 static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt)
 183 {
 184         struct arch_timer_cpu *timer;
 185         struct kvm_vcpu *vcpu;
 186         u64 ns;
 187
 188         timer = container_of(hrt, struct arch_timer_cpu, bg_timer);
 189         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 190
 191         /*
 192          * Check that the timer has really expired from the guest's
 193          * PoV (NTP on the host may have forced it to expire
 194          * early). If we should have slept longer, restart it.
 195          */
 196         ns = kvm_timer_earliest_exp(vcpu);
 197         if (unlikely(ns)) {
 198                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 199                 return HRTIMER_RESTART;
 200         }
 201
 202         schedule_work(&timer->expired);
 203         return HRTIMER_NORESTART;
 204 }
 205
 206 static enum hrtimer_restart kvm_phys_timer_expire(struct hrtimer *hrt)
 207 {
 208         struct arch_timer_context *ptimer;
 209         struct arch_timer_cpu *timer;
 210         struct kvm_vcpu *vcpu;
 211         u64 ns;
 212
 213         timer = container_of(hrt, struct arch_timer_cpu, phys_timer);
 214         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 215         ptimer = vcpu_ptimer(vcpu);
 216
 217         /*
 218          * Check that the timer has really expired from the guest's
 219          * PoV (NTP on the host may have forced it to expire
 220          * early). If not ready, schedule for a later time.
 221          */
 222         ns = kvm_timer_compute_delta(ptimer);
 223         if (unlikely(ns)) {
 224                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 225                 return HRTIMER_RESTART;
 226         }
 227
 228         kvm_timer_update_irq(vcpu, true, ptimer);
 229         return HRTIMER_NORESTART;
 230 }
 231
 232 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
 233 {
 234         u64 cval, now;
 235
 236         if (timer_ctx->loaded) {
 237                 u32 cnt_ctl;
 238
 239                 /* Only the virtual timer can be loaded so far */
 240                 cnt_ctl = read_sysreg_el0(cntv_ctl);
 241                 return  (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) &&
 242                         (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) &&
 243                        !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
 244         }
 245
 246         if (!kvm_timer_irq_can_fire(timer_ctx))
 247                 return false;
 248
 249         cval = timer_ctx->cnt_cval;
 250         now = kvm_phys_timer_read() - timer_ctx->cntvoff;
 251
 252         return cval <= now;
 253 }
 254
 255 bool kvm_timer_is_pending(struct kvm_vcpu *vcpu)
 256 {
 257         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 258         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 259
 260         if (kvm_timer_should_fire(vtimer))
 261                 return true;
 262
 263         return kvm_timer_should_fire(ptimer);
 264 }
 265
 266 /*
 267  * Reflect the timer output level into the kvm_run structure
 268  */
 269 void kvm_timer_update_run(struct kvm_vcpu *vcpu)
 270 {
 271         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 272         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 273         struct kvm_sync_regs *regs = &vcpu->run->s.regs;
 274
 275         /* Populate the device bitmap with the timer states */
 276         regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
 277                                     KVM_ARM_DEV_EL1_PTIMER);
 278         if (kvm_timer_should_fire(vtimer))
 279                 regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
 280         if (kvm_timer_should_fire(ptimer))
 281                 regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
 282 }
 283
 284 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 285                                  struct arch_timer_context *timer_ctx)
 286 {
 287         int ret;
 288
 289         timer_ctx->irq.level = new_level;
 290         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
 291                                    timer_ctx->irq.level);
 292
 293         if (!static_branch_unlikely(&userspace_irqchip_in_use) ||
 294             likely(irqchip_in_kernel(vcpu->kvm))) {
 295                 ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
 296                                           timer_ctx->irq.irq,
 297                                           timer_ctx->irq.level,
 298                                           timer_ctx);
 299                 WARN_ON(ret);
 300         }
 301 }
 302
 303 /* Schedule the background timer for the emulated timer. */
 304 static void phys_timer_emulate(struct kvm_vcpu *vcpu)
 305 {
 306         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 307         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 308
 309         /*
 310          * If the timer can fire now we have just raised the IRQ line and we
 311          * don't need to have a soft timer scheduled for the future.  If the
 312          * timer cannot fire at all, then we also don't need a soft timer.
 313          */
 314         if (kvm_timer_should_fire(ptimer) || !kvm_timer_irq_can_fire(ptimer)) {
 315                 soft_timer_cancel(&timer->phys_timer, NULL);
 316                 return;
 317         }
 318
 319         soft_timer_start(&timer->phys_timer, kvm_timer_compute_delta(ptimer));
 320 }
 321
 322 /*
 323  * Check if there was a change in the timer state, so that we should either
 324  * raise or lower the line level to the GIC or schedule a background timer to
 325  * emulate the physical timer.
 326  */
 327 static void kvm_timer_update_state(struct kvm_vcpu *vcpu)
 328 {
 329         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 330         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 331         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 332         bool level;
 333
 334         if (unlikely(!timer->enabled))
 335                 return;
 336
 337         /*
 338          * The vtimer virtual interrupt is a 'mapped' interrupt, meaning part
 339          * of its lifecycle is offloaded to the hardware, and we therefore may
 340          * not have lowered the irq.level value before having to signal a new
 341          * interrupt, but have to signal an interrupt every time the level is
 342          * asserted.
 343          */
 344         level = kvm_timer_should_fire(vtimer);
 345         kvm_timer_update_irq(vcpu, level, vtimer);
 346
 347         if (kvm_timer_should_fire(ptimer) != ptimer->irq.level)
 348                 kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer);
 349
 350         phys_timer_emulate(vcpu);
 351 }
 352
 353 static void __timer_snapshot_state(struct arch_timer_context *timer)
 354 {
 355         timer->cnt_ctl = read_sysreg_el0(cntv_ctl);
 356         timer->cnt_cval = read_sysreg_el0(cntv_cval);
 357 }
 358
 359 static void vtimer_save_state(struct kvm_vcpu *vcpu)
 360 {
 361         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 362         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 363         unsigned long flags;
 364
 365         local_irq_save(flags);
 366
 367         if (!vtimer->loaded)
 368                 goto out;
 369
 370         if (timer->enabled)
 371                 __timer_snapshot_state(vtimer);
 372
 373         /* Disable the virtual timer */
 374         write_sysreg_el0(0, cntv_ctl);
 375         isb();
 376
 377         vtimer->loaded = false;
 378 out:
 379         local_irq_restore(flags);
 380 }
 381
 382 /*
 383  * Schedule the background timer before calling kvm_vcpu_block, so that this
 384  * thread is removed from its waitqueue and made runnable when there's a timer
 385  * interrupt to handle.
 386  */
 387 void kvm_timer_schedule(struct kvm_vcpu *vcpu)
 388 {
 389         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 390         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 391         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 392
 393         vtimer_save_state(vcpu);
 394
 395         /*
 396          * No need to schedule a background timer if any guest timer has
 397          * already expired, because kvm_vcpu_block will return before putting
 398          * the thread to sleep.
 399          */
 400         if (kvm_timer_should_fire(vtimer) || kvm_timer_should_fire(ptimer))
 401                 return;
 402
 403         /*
 404          * If both timers are not capable of raising interrupts (disabled or
 405          * masked), then there's no more work for us to do.
 406          */
 407         if (!kvm_timer_irq_can_fire(vtimer) && !kvm_timer_irq_can_fire(ptimer))
 408                 return;
 409
 410         /*
 411          * The guest timers have not yet expired, schedule a background timer.
 412          * Set the earliest expiration time among the guest timers.
 413          */
 414         soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));
 415 }
 416
 417 static void vtimer_restore_state(struct kvm_vcpu *vcpu)
 418 {
 419         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 420         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 421         unsigned long flags;
 422
 423         local_irq_save(flags);
 424
 425         if (vtimer->loaded)
 426                 goto out;
 427
 428         if (timer->enabled) {
 429                 write_sysreg_el0(vtimer->cnt_cval, cntv_cval);
 430                 isb();
 431                 write_sysreg_el0(vtimer->cnt_ctl, cntv_ctl);
 432         }
 433
 434         vtimer->loaded = true;
 435 out:
 436         local_irq_restore(flags);
 437 }
 438
 439 void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
 440 {
 441         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 442
 443         vtimer_restore_state(vcpu);
 444
 445         soft_timer_cancel(&timer->bg_timer, &timer->expired);
 446 }
 447
 448 static void set_cntvoff(u64 cntvoff)
 449 {
 450         u32 low = lower_32_bits(cntvoff);
 451         u32 high = upper_32_bits(cntvoff);
 452
 453         /*
 454          * Since kvm_call_hyp doesn't fully support the ARM PCS especially on
 455          * 32-bit systems, but rather passes register by register shifted one
 456          * place (we put the function address in r0/x0), we cannot simply pass
 457          * a 64-bit value as an argument, but have to split the value in two
 458          * 32-bit halves.
 459          */
 460         kvm_call_hyp(__kvm_timer_set_cntvoff, low, high);
 461 }
 462
 463 static void kvm_timer_vcpu_load_vgic(struct kvm_vcpu *vcpu)
 464 {
 465         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 466         bool phys_active;
 467         int ret;
 468
 469         phys_active = kvm_vgic_map_is_active(vcpu, vtimer->irq.irq);
 470
 471         ret = irq_set_irqchip_state(host_vtimer_irq,
 472                                     IRQCHIP_STATE_ACTIVE,
 473                                     phys_active);
 474         WARN_ON(ret);
 475 }
 476
 477 static void kvm_timer_vcpu_load_user(struct kvm_vcpu *vcpu)
 478 {
 479         kvm_vtimer_update_mask_user(vcpu);
 480 }
 481
 482 void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
 483 {
 484         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 485         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 486
 487         if (unlikely(!timer->enabled))
 488                 return;
 489
 490         if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
 491                 kvm_timer_vcpu_load_user(vcpu);
 492         else
 493                 kvm_timer_vcpu_load_vgic(vcpu);
 494
 495         set_cntvoff(vtimer->cntvoff);
 496
 497         vtimer_restore_state(vcpu);
 498
 499         /* Set the background timer for the physical timer emulation. */
 500         phys_timer_emulate(vcpu);
 501 }
 502
 503 bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
 504 {
 505         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 506         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 507         struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
 508         bool vlevel, plevel;
 509
 510         if (likely(irqchip_in_kernel(vcpu->kvm)))
 511                 return false;
 512
 513         vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
 514         plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
 515
 516         return kvm_timer_should_fire(vtimer) != vlevel ||
 517                kvm_timer_should_fire(ptimer) != plevel;
 518 }
 519
 520 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
 521 {
 522         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 523
 524         if (unlikely(!timer->enabled))
 525                 return;
 526
 527         vtimer_save_state(vcpu);
 528
 529         /*
 530          * Cancel the physical timer emulation, because the only case where we
 531          * need it after a vcpu_put is in the context of a sleeping VCPU, and
 532          * in that case we already factor in the deadline for the physical
 533          * timer when scheduling the bg_timer.
 534          *
 535          * In any case, we re-schedule the hrtimer for the physical timer when
 536          * coming back to the VCPU thread in kvm_timer_vcpu_load().
 537          */
 538         soft_timer_cancel(&timer->phys_timer, NULL);
 539
 540         /*
 541          * The kernel may decide to run userspace after calling vcpu_put, so
 542          * we reset cntvoff to 0 to ensure a consistent read between user
 543          * accesses to the virtual counter and kernel access to the physical
 544          * counter.
 545          */
 546         set_cntvoff(0);
 547 }
 548
 549 /*
 550  * With a userspace irqchip we have to check if the guest de-asserted the
 551  * timer and if so, unmask the timer irq signal on the host interrupt
 552  * controller to ensure that we see future timer signals.
 553  */
 554 static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
 555 {
 556         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 557
 558         if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
 559                 __timer_snapshot_state(vtimer);
 560                 if (!kvm_timer_should_fire(vtimer)) {
 561                         kvm_timer_update_irq(vcpu, false, vtimer);
 562                         kvm_vtimer_update_mask_user(vcpu);
 563                 }
 564         }
 565 }
 566
 567 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 568 {
 569         unmask_vtimer_irq_user(vcpu);
 570 }
 571
 572 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
 573 {
 574         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 575         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 576
 577         /*
 578          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 579          * and to 0 for ARMv7.  We provide an implementation that always
 580          * resets the timer to be disabled and unmasked and is compliant with
 581          * the ARMv7 architecture.
 582          */
 583         vtimer->cnt_ctl = 0;
 584         ptimer->cnt_ctl = 0;
 585         kvm_timer_update_state(vcpu);
 586
 587         return 0;
 588 }
 589
 590 /* Make the updates of cntvoff for all vtimer contexts atomic */
 591 static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff)
 592 {
 593         int i;
 594         struct kvm *kvm = vcpu->kvm;
 595         struct kvm_vcpu *tmp;
 596
 597         mutex_lock(&kvm->lock);
 598         kvm_for_each_vcpu(i, tmp, kvm)
 599                 vcpu_vtimer(tmp)->cntvoff = cntvoff;
 600
 601         /*
 602          * When called from the vcpu create path, the CPU being created is not
 603          * included in the loop above, so we just set it here as well.
 604          */
 605         vcpu_vtimer(vcpu)->cntvoff = cntvoff;
 606         mutex_unlock(&kvm->lock);
 607 }
 608
 609 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 610 {
 611         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 612         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 613         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 614
 615         /* Synchronize cntvoff across all vtimers of a VM. */
 616         update_vtimer_cntvoff(vcpu, kvm_phys_timer_read());
 617         vcpu_ptimer(vcpu)->cntvoff = 0;
 618
 619         INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
 620         hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 621         timer->bg_timer.function = kvm_bg_timer_expire;
 622
 623         hrtimer_init(&timer->phys_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 624         timer->phys_timer.function = kvm_phys_timer_expire;
 625
 626         vtimer->irq.irq = default_vtimer_irq.irq;
 627         ptimer->irq.irq = default_ptimer_irq.irq;
 628 }
 629
 630 static void kvm_timer_init_interrupt(void *info)
 631 {
 632         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 633 }
 634
 635 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 636 {
 637         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 638         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 639
 640         switch (regid) {
 641         case KVM_REG_ARM_TIMER_CTL:
 642                 vtimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT;
 643                 break;
 644         case KVM_REG_ARM_TIMER_CNT:
 645                 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value);
 646                 break;
 647         case KVM_REG_ARM_TIMER_CVAL:
 648                 vtimer->cnt_cval = value;
 649                 break;
 650         case KVM_REG_ARM_PTIMER_CTL:
 651                 ptimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT;
 652                 break;
 653         case KVM_REG_ARM_PTIMER_CVAL:
 654                 ptimer->cnt_cval = value;
 655                 break;
 656
 657         default:
 658                 return -1;
 659         }
 660
 661         kvm_timer_update_state(vcpu);
 662         return 0;
 663 }
 664
 665 static u64 read_timer_ctl(struct arch_timer_context *timer)
 666 {
 667         /*
 668          * Set ISTATUS bit if it's expired.
 669          * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is
 670          * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit
 671          * regardless of ENABLE bit for our implementation convenience.
 672          */
 673         if (!kvm_timer_compute_delta(timer))
 674                 return timer->cnt_ctl | ARCH_TIMER_CTRL_IT_STAT;
 675         else
 676                 return timer->cnt_ctl;
 677 }
 678
 679 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 680 {
 681         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 682         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 683
 684         switch (regid) {
 685         case KVM_REG_ARM_TIMER_CTL:
 686                 return read_timer_ctl(vtimer);
 687         case KVM_REG_ARM_TIMER_CNT:
 688                 return kvm_phys_timer_read() - vtimer->cntvoff;
 689         case KVM_REG_ARM_TIMER_CVAL:
 690                 return vtimer->cnt_cval;
 691         case KVM_REG_ARM_PTIMER_CTL:
 692                 return read_timer_ctl(ptimer);
 693         case KVM_REG_ARM_PTIMER_CVAL:
 694                 return ptimer->cnt_cval;
 695         case KVM_REG_ARM_PTIMER_CNT:
 696                 return kvm_phys_timer_read();
 697         }
 698         return (u64)-1;
 699 }
 700
 701 static int kvm_timer_starting_cpu(unsigned int cpu)
 702 {
 703         kvm_timer_init_interrupt(NULL);
 704         return 0;
 705 }
 706
 707 static int kvm_timer_dying_cpu(unsigned int cpu)
 708 {
 709         disable_percpu_irq(host_vtimer_irq);
 710         return 0;
 711 }
 712
 713 int kvm_timer_hyp_init(bool has_gic)
 714 {
 715         struct arch_timer_kvm_info *info;
 716         int err;
 717
 718         info = arch_timer_get_kvm_info();
 719         timecounter = &info->timecounter;
 720
 721         if (!timecounter->cc) {
 722                 kvm_err("kvm_arch_timer: uninitialized timecounter\n");
 723                 return -ENODEV;
 724         }
 725
 726         if (info->virtual_irq <= 0) {
 727                 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
 728                         info->virtual_irq);
 729                 return -ENODEV;
 730         }
 731         host_vtimer_irq = info->virtual_irq;
 732
 733         host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
 734         if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
 735             host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
 736                 kvm_err("Invalid trigger for IRQ%d, assuming level low\n",
 737                         host_vtimer_irq);
 738                 host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
 739         }
 740
 741         err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
 742                                  "kvm guest timer", kvm_get_running_vcpus());
 743         if (err) {
 744                 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
 745                         host_vtimer_irq, err);
 746                 return err;
 747         }
 748
 749         if (has_gic) {
 750                 err = irq_set_vcpu_affinity(host_vtimer_irq,
 751                                             kvm_get_running_vcpus());
 752                 if (err) {
 753                         kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
 754                         goto out_free_irq;
 755                 }
 756         }
 757
 758         kvm_info("virtual timer IRQ%d\n", host_vtimer_irq);
 759
 760         cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
 761                           "kvm/arm/timer:starting", kvm_timer_starting_cpu,
 762                           kvm_timer_dying_cpu);
 763         return 0;
 764 out_free_irq:
 765         free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
 766         return err;
 767 }
 768
 769 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
 770 {
 771         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 772         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 773
 774         soft_timer_cancel(&timer->bg_timer, &timer->expired);
 775         soft_timer_cancel(&timer->phys_timer, NULL);
 776         kvm_vgic_unmap_phys_irq(vcpu, vtimer->irq.irq);
 777 }
 778
 779 static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
 780 {
 781         int vtimer_irq, ptimer_irq;
 782         int i, ret;
 783
 784         vtimer_irq = vcpu_vtimer(vcpu)->irq.irq;
 785         ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu));
 786         if (ret)
 787                 return false;
 788
 789         ptimer_irq = vcpu_ptimer(vcpu)->irq.irq;
 790         ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu));
 791         if (ret)
 792                 return false;
 793
 794         kvm_for_each_vcpu(i, vcpu, vcpu->kvm) {
 795                 if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq ||
 796                     vcpu_ptimer(vcpu)->irq.irq != ptimer_irq)
 797                         return false;
 798         }
 799
 800         return true;
 801 }
 802
 803 bool kvm_arch_timer_get_input_level(int vintid)
 804 {
 805         struct kvm_vcpu *vcpu = kvm_arm_get_running_vcpu();
 806         struct arch_timer_context *timer;
 807
 808         if (vintid == vcpu_vtimer(vcpu)->irq.irq)
 809                 timer = vcpu_vtimer(vcpu);
 810         else
 811                 BUG(); /* We only map the vtimer so far */
 812
 813         return kvm_timer_should_fire(timer);
 814 }
 815
 816 int kvm_timer_enable(struct kvm_vcpu *vcpu)
 817 {
 818         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 819         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 820         int ret;
 821
 822         if (timer->enabled)
 823                 return 0;
 824
 825         /* Without a VGIC we do not map virtual IRQs to physical IRQs */
 826         if (!irqchip_in_kernel(vcpu->kvm))
 827                 goto no_vgic;
 828
 829         if (!vgic_initialized(vcpu->kvm))
 830                 return -ENODEV;
 831
 832         if (!timer_irqs_are_valid(vcpu)) {
 833                 kvm_debug("incorrectly configured timer irqs\n");
 834                 return -EINVAL;
 835         }
 836
 837         ret = kvm_vgic_map_phys_irq(vcpu, host_vtimer_irq, vtimer->irq.irq,
 838                                     kvm_arch_timer_get_input_level);
 839         if (ret)
 840                 return ret;
 841
 842 no_vgic:
 843         preempt_disable();
 844         timer->enabled = 1;
 845         kvm_timer_vcpu_load(vcpu);
 846         preempt_enable();
 847
 848         return 0;
 849 }
 850
 851 /*
 852  * On VHE system, we only need to configure trap on physical timer and counter
 853  * accesses in EL0 and EL1 once, not for every world switch.
 854  * The host kernel runs at EL2 with HCR_EL2.TGE == 1,
 855  * and this makes those bits have no effect for the host kernel execution.
 856  */
 857 void kvm_timer_init_vhe(void)
 858 {
 859         /* When HCR_EL2.E2H ==1, EL1PCEN and EL1PCTEN are shifted by 10 */
 860         u32 cnthctl_shift = 10;
 861         u64 val;
 862
 863         /*
 864          * Disallow physical timer access for the guest.
 865          * Physical counter access is allowed.
 866          */
 867         val = read_sysreg(cnthctl_el2);
 868         val &= ~(CNTHCTL_EL1PCEN << cnthctl_shift);
 869         val |= (CNTHCTL_EL1PCTEN << cnthctl_shift);
 870         write_sysreg(val, cnthctl_el2);
 871 }
 872
 873 static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq)
 874 {
 875         struct kvm_vcpu *vcpu;
 876         int i;
 877
 878         kvm_for_each_vcpu(i, vcpu, kvm) {
 879                 vcpu_vtimer(vcpu)->irq.irq = vtimer_irq;
 880                 vcpu_ptimer(vcpu)->irq.irq = ptimer_irq;
 881         }
 882 }
 883
 884 int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 885 {
 886         int __user *uaddr = (int __user *)(long)attr->addr;
 887         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 888         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 889         int irq;
 890
 891         if (!irqchip_in_kernel(vcpu->kvm))
 892                 return -EINVAL;
 893
 894         if (get_user(irq, uaddr))
 895                 return -EFAULT;
 896
 897         if (!(irq_is_ppi(irq)))
 898                 return -EINVAL;
 899
 900         if (vcpu->arch.timer_cpu.enabled)
 901                 return -EBUSY;
 902
 903         switch (attr->attr) {
 904         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
 905                 set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq);
 906                 break;
 907         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
 908                 set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq);
 909                 break;
 910         default:
 911                 return -ENXIO;
 912         }
 913
 914         return 0;
 915 }
 916
 917 int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 918 {
 919         int __user *uaddr = (int __user *)(long)attr->addr;
 920         struct arch_timer_context *timer;
 921         int irq;
 922
 923         switch (attr->attr) {
 924         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
 925                 timer = vcpu_vtimer(vcpu);
 926                 break;
 927         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
 928                 timer = vcpu_ptimer(vcpu);
 929                 break;
 930         default:
 931                 return -ENXIO;
 932         }
 933
 934         irq = timer->irq.irq;
 935         return put_user(irq, uaddr);
 936 }
 937
 938 int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 939 {
 940         switch (attr->attr) {
 941         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
 942         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
 943                 return 0;
 944         }
 945
 946         return -ENXIO;
 947 }