arch/riscv/kvm/vcpu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2019 Western Digital Corporation or its affiliates.
   4  *
   5  * Authors:
   6  *     Anup Patel <anup.patel@wdc.com>
   7  */
   8
   9 #include <linux/bitops.h>
  10 #include <linux/entry-kvm.h>
  11 #include <linux/errno.h>
  12 #include <linux/err.h>
  13 #include <linux/kdebug.h>
  14 #include <linux/module.h>
  15 #include <linux/percpu.h>
  16 #include <linux/uaccess.h>
  17 #include <linux/vmalloc.h>
  18 #include <linux/sched/signal.h>
  19 #include <linux/fs.h>
  20 #include <linux/kvm_host.h>
  21 #include <asm/csr.h>
  22 #include <asm/cacheflush.h>
  23 #include <asm/hwcap.h>
  24 #include <asm/sbi.h>
  25
  26 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
  27         KVM_GENERIC_VCPU_STATS(),
  28         STATS_DESC_COUNTER(VCPU, ecall_exit_stat),
  29         STATS_DESC_COUNTER(VCPU, wfi_exit_stat),
  30         STATS_DESC_COUNTER(VCPU, mmio_exit_user),
  31         STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
  32         STATS_DESC_COUNTER(VCPU, csr_exit_user),
  33         STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
  34         STATS_DESC_COUNTER(VCPU, signal_exits),
  35         STATS_DESC_COUNTER(VCPU, exits)
  36 };
  37
  38 const struct kvm_stats_header kvm_vcpu_stats_header = {
  39         .name_size = KVM_STATS_NAME_SIZE,
  40         .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
  41         .id_offset = sizeof(struct kvm_stats_header),
  42         .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  43         .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  44                        sizeof(kvm_vcpu_stats_desc),
  45 };
  46
  47 #define KVM_RISCV_BASE_ISA_MASK         GENMASK(25, 0)
  48
  49 #define KVM_ISA_EXT_ARR(ext)            [KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext
  50
  51 /* Mapping between KVM ISA Extension ID & Host ISA extension ID */
  52 static const unsigned long kvm_isa_ext_arr[] = {
  53         [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a,
  54         [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c,
  55         [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d,
  56         [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f,
  57         [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h,
  58         [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i,
  59         [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,
  60
  61         KVM_ISA_EXT_ARR(SSTC),
  62         KVM_ISA_EXT_ARR(SVINVAL),
  63         KVM_ISA_EXT_ARR(SVPBMT),
  64         KVM_ISA_EXT_ARR(ZIHINTPAUSE),
  65         KVM_ISA_EXT_ARR(ZICBOM),
  66 };
  67
  68 static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext)
  69 {
  70         unsigned long i;
  71
  72         for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) {
  73                 if (kvm_isa_ext_arr[i] == base_ext)
  74                         return i;
  75         }
  76
  77         return KVM_RISCV_ISA_EXT_MAX;
  78 }
  79
  80 static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext)
  81 {
  82         switch (ext) {
  83         case KVM_RISCV_ISA_EXT_H:
  84                 return false;
  85         default:
  86                 break;
  87         }
  88
  89         return true;
  90 }
  91
  92 static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext)
  93 {
  94         switch (ext) {
  95         case KVM_RISCV_ISA_EXT_A:
  96         case KVM_RISCV_ISA_EXT_C:
  97         case KVM_RISCV_ISA_EXT_I:
  98         case KVM_RISCV_ISA_EXT_M:
  99         case KVM_RISCV_ISA_EXT_SSTC:
 100         case KVM_RISCV_ISA_EXT_SVINVAL:
 101         case KVM_RISCV_ISA_EXT_ZIHINTPAUSE:
 102                 return false;
 103         default:
 104                 break;
 105         }
 106
 107         return true;
 108 }
 109
 110 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu)
 111 {
 112         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 113         struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;
 114         struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
 115         struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context;
 116         bool loaded;
 117
 118         /**
 119          * The preemption should be disabled here because it races with
 120          * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which
 121          * also calls vcpu_load/put.
 122          */
 123         get_cpu();
 124         loaded = (vcpu->cpu != -1);
 125         if (loaded)
 126                 kvm_arch_vcpu_put(vcpu);
 127
 128         vcpu->arch.last_exit_cpu = -1;
 129
 130         memcpy(csr, reset_csr, sizeof(*csr));
 131
 132         memcpy(cntx, reset_cntx, sizeof(*cntx));
 133
 134         kvm_riscv_vcpu_fp_reset(vcpu);
 135
 136         kvm_riscv_vcpu_timer_reset(vcpu);
 137
 138         WRITE_ONCE(vcpu->arch.irqs_pending, 0);
 139         WRITE_ONCE(vcpu->arch.irqs_pending_mask, 0);
 140
 141         kvm_riscv_vcpu_pmu_reset(vcpu);
 142
 143         vcpu->arch.hfence_head = 0;
 144         vcpu->arch.hfence_tail = 0;
 145         memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue));
 146
 147         /* Reset the guest CSRs for hotplug usecase */
 148         if (loaded)
 149                 kvm_arch_vcpu_load(vcpu, smp_processor_id());
 150         put_cpu();
 151 }
 152
 153 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
 154 {
 155         return 0;
 156 }
 157
 158 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 159 {
 160         struct kvm_cpu_context *cntx;
 161         struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr;
 162         unsigned long host_isa, i;
 163
 164         /* Mark this VCPU never ran */
 165         vcpu->arch.ran_atleast_once = false;
 166         vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
 167         bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX);
 168
 169         /* Setup ISA features available to VCPU */
 170         for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) {
 171                 host_isa = kvm_isa_ext_arr[i];
 172                 if (__riscv_isa_extension_available(NULL, host_isa) &&
 173                     kvm_riscv_vcpu_isa_enable_allowed(i))
 174                         set_bit(host_isa, vcpu->arch.isa);
 175         }
 176
 177         /* Setup vendor, arch, and implementation details */
 178         vcpu->arch.mvendorid = sbi_get_mvendorid();
 179         vcpu->arch.marchid = sbi_get_marchid();
 180         vcpu->arch.mimpid = sbi_get_mimpid();
 181
 182         /* Setup VCPU hfence queue */
 183         spin_lock_init(&vcpu->arch.hfence_lock);
 184
 185         /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */
 186         cntx = &vcpu->arch.guest_reset_context;
 187         cntx->sstatus = SR_SPP | SR_SPIE;
 188         cntx->hstatus = 0;
 189         cntx->hstatus |= HSTATUS_VTW;
 190         cntx->hstatus |= HSTATUS_SPVP;
 191         cntx->hstatus |= HSTATUS_SPV;
 192
 193         /* By default, make CY, TM, and IR counters accessible in VU mode */
 194         reset_csr->scounteren = 0x7;
 195
 196         /* Setup VCPU timer */
 197         kvm_riscv_vcpu_timer_init(vcpu);
 198
 199         /* setup performance monitoring */
 200         kvm_riscv_vcpu_pmu_init(vcpu);
 201
 202         /* Reset VCPU */
 203         kvm_riscv_reset_vcpu(vcpu);
 204
 205         return 0;
 206 }
 207
 208 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 209 {
 210         /**
 211          * vcpu with id 0 is the designated boot cpu.
 212          * Keep all vcpus with non-zero id in power-off state so that
 213          * they can be brought up using SBI HSM extension.
 214          */
 215         if (vcpu->vcpu_idx != 0)
 216                 kvm_riscv_vcpu_power_off(vcpu);
 217 }
 218
 219 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 220 {
 221         /* Cleanup VCPU timer */
 222         kvm_riscv_vcpu_timer_deinit(vcpu);
 223
 224         kvm_riscv_vcpu_pmu_deinit(vcpu);
 225
 226         /* Free unused pages pre-allocated for G-stage page table mappings */
 227         kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
 228 }
 229
 230 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 231 {
 232         return kvm_riscv_vcpu_timer_pending(vcpu);
 233 }
 234
 235 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
 236 {
 237 }
 238
 239 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
 240 {
 241 }
 242
 243 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 244 {
 245         return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) &&
 246                 !vcpu->arch.power_off && !vcpu->arch.pause);
 247 }
 248
 249 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 250 {
 251         return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
 252 }
 253
 254 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
 255 {
 256         return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false;
 257 }
 258
 259 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
 260 {
 261         return VM_FAULT_SIGBUS;
 262 }
 263
 264 static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu,
 265                                          const struct kvm_one_reg *reg)
 266 {
 267         unsigned long __user *uaddr =
 268                         (unsigned long __user *)(unsigned long)reg->addr;
 269         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 270                                             KVM_REG_SIZE_MASK |
 271                                             KVM_REG_RISCV_CONFIG);
 272         unsigned long reg_val;
 273
 274         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 275                 return -EINVAL;
 276
 277         switch (reg_num) {
 278         case KVM_REG_RISCV_CONFIG_REG(isa):
 279                 reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK;
 280                 break;
 281         case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
 282                 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM))
 283                         return -EINVAL;
 284                 reg_val = riscv_cbom_block_size;
 285                 break;
 286         case KVM_REG_RISCV_CONFIG_REG(mvendorid):
 287                 reg_val = vcpu->arch.mvendorid;
 288                 break;
 289         case KVM_REG_RISCV_CONFIG_REG(marchid):
 290                 reg_val = vcpu->arch.marchid;
 291                 break;
 292         case KVM_REG_RISCV_CONFIG_REG(mimpid):
 293                 reg_val = vcpu->arch.mimpid;
 294                 break;
 295         default:
 296                 return -EINVAL;
 297         }
 298
 299         if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 300                 return -EFAULT;
 301
 302         return 0;
 303 }
 304
 305 static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu,
 306                                          const struct kvm_one_reg *reg)
 307 {
 308         unsigned long __user *uaddr =
 309                         (unsigned long __user *)(unsigned long)reg->addr;
 310         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 311                                             KVM_REG_SIZE_MASK |
 312                                             KVM_REG_RISCV_CONFIG);
 313         unsigned long i, isa_ext, reg_val;
 314
 315         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 316                 return -EINVAL;
 317
 318         if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 319                 return -EFAULT;
 320
 321         switch (reg_num) {
 322         case KVM_REG_RISCV_CONFIG_REG(isa):
 323                 /*
 324                  * This ONE REG interface is only defined for
 325                  * single letter extensions.
 326                  */
 327                 if (fls(reg_val) >= RISCV_ISA_EXT_BASE)
 328                         return -EINVAL;
 329
 330                 if (!vcpu->arch.ran_atleast_once) {
 331                         /* Ignore the enable/disable request for certain extensions */
 332                         for (i = 0; i < RISCV_ISA_EXT_BASE; i++) {
 333                                 isa_ext = kvm_riscv_vcpu_base2isa_ext(i);
 334                                 if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) {
 335                                         reg_val &= ~BIT(i);
 336                                         continue;
 337                                 }
 338                                 if (!kvm_riscv_vcpu_isa_enable_allowed(isa_ext))
 339                                         if (reg_val & BIT(i))
 340                                                 reg_val &= ~BIT(i);
 341                                 if (!kvm_riscv_vcpu_isa_disable_allowed(isa_ext))
 342                                         if (!(reg_val & BIT(i)))
 343                                                 reg_val |= BIT(i);
 344                         }
 345                         reg_val &= riscv_isa_extension_base(NULL);
 346                         /* Do not modify anything beyond single letter extensions */
 347                         reg_val = (vcpu->arch.isa[0] & ~KVM_RISCV_BASE_ISA_MASK) |
 348                                   (reg_val & KVM_RISCV_BASE_ISA_MASK);
 349                         vcpu->arch.isa[0] = reg_val;
 350                         kvm_riscv_vcpu_fp_reset(vcpu);
 351                 } else {
 352                         return -EOPNOTSUPP;
 353                 }
 354                 break;
 355         case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size):
 356                 return -EOPNOTSUPP;
 357         case KVM_REG_RISCV_CONFIG_REG(mvendorid):
 358                 if (!vcpu->arch.ran_atleast_once)
 359                         vcpu->arch.mvendorid = reg_val;
 360                 else
 361                         return -EBUSY;
 362                 break;
 363         case KVM_REG_RISCV_CONFIG_REG(marchid):
 364                 if (!vcpu->arch.ran_atleast_once)
 365                         vcpu->arch.marchid = reg_val;
 366                 else
 367                         return -EBUSY;
 368                 break;
 369         case KVM_REG_RISCV_CONFIG_REG(mimpid):
 370                 if (!vcpu->arch.ran_atleast_once)
 371                         vcpu->arch.mimpid = reg_val;
 372                 else
 373                         return -EBUSY;
 374                 break;
 375         default:
 376                 return -EINVAL;
 377         }
 378
 379         return 0;
 380 }
 381
 382 static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu,
 383                                        const struct kvm_one_reg *reg)
 384 {
 385         struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
 386         unsigned long __user *uaddr =
 387                         (unsigned long __user *)(unsigned long)reg->addr;
 388         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 389                                             KVM_REG_SIZE_MASK |
 390                                             KVM_REG_RISCV_CORE);
 391         unsigned long reg_val;
 392
 393         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 394                 return -EINVAL;
 395         if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long))
 396                 return -EINVAL;
 397
 398         if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc))
 399                 reg_val = cntx->sepc;
 400         else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num &&
 401                  reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6))
 402                 reg_val = ((unsigned long *)cntx)[reg_num];
 403         else if (reg_num == KVM_REG_RISCV_CORE_REG(mode))
 404                 reg_val = (cntx->sstatus & SR_SPP) ?
 405                                 KVM_RISCV_MODE_S : KVM_RISCV_MODE_U;
 406         else
 407                 return -EINVAL;
 408
 409         if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 410                 return -EFAULT;
 411
 412         return 0;
 413 }
 414
 415 static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu,
 416                                        const struct kvm_one_reg *reg)
 417 {
 418         struct kvm_cpu_context *cntx = &vcpu->arch.guest_context;
 419         unsigned long __user *uaddr =
 420                         (unsigned long __user *)(unsigned long)reg->addr;
 421         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 422                                             KVM_REG_SIZE_MASK |
 423                                             KVM_REG_RISCV_CORE);
 424         unsigned long reg_val;
 425
 426         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 427                 return -EINVAL;
 428         if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long))
 429                 return -EINVAL;
 430
 431         if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 432                 return -EFAULT;
 433
 434         if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc))
 435                 cntx->sepc = reg_val;
 436         else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num &&
 437                  reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6))
 438                 ((unsigned long *)cntx)[reg_num] = reg_val;
 439         else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) {
 440                 if (reg_val == KVM_RISCV_MODE_S)
 441                         cntx->sstatus |= SR_SPP;
 442                 else
 443                         cntx->sstatus &= ~SR_SPP;
 444         } else
 445                 return -EINVAL;
 446
 447         return 0;
 448 }
 449
 450 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu,
 451                                       const struct kvm_one_reg *reg)
 452 {
 453         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 454         unsigned long __user *uaddr =
 455                         (unsigned long __user *)(unsigned long)reg->addr;
 456         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 457                                             KVM_REG_SIZE_MASK |
 458                                             KVM_REG_RISCV_CSR);
 459         unsigned long reg_val;
 460
 461         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 462                 return -EINVAL;
 463         if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long))
 464                 return -EINVAL;
 465
 466         if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
 467                 kvm_riscv_vcpu_flush_interrupts(vcpu);
 468                 reg_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK;
 469         } else
 470                 reg_val = ((unsigned long *)csr)[reg_num];
 471
 472         if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 473                 return -EFAULT;
 474
 475         return 0;
 476 }
 477
 478 static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu,
 479                                       const struct kvm_one_reg *reg)
 480 {
 481         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 482         unsigned long __user *uaddr =
 483                         (unsigned long __user *)(unsigned long)reg->addr;
 484         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 485                                             KVM_REG_SIZE_MASK |
 486                                             KVM_REG_RISCV_CSR);
 487         unsigned long reg_val;
 488
 489         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 490                 return -EINVAL;
 491         if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long))
 492                 return -EINVAL;
 493
 494         if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 495                 return -EFAULT;
 496
 497         if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) {
 498                 reg_val &= VSIP_VALID_MASK;
 499                 reg_val <<= VSIP_TO_HVIP_SHIFT;
 500         }
 501
 502         ((unsigned long *)csr)[reg_num] = reg_val;
 503
 504         if (reg_num == KVM_REG_RISCV_CSR_REG(sip))
 505                 WRITE_ONCE(vcpu->arch.irqs_pending_mask, 0);
 506
 507         return 0;
 508 }
 509
 510 static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu,
 511                                           const struct kvm_one_reg *reg)
 512 {
 513         unsigned long __user *uaddr =
 514                         (unsigned long __user *)(unsigned long)reg->addr;
 515         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 516                                             KVM_REG_SIZE_MASK |
 517                                             KVM_REG_RISCV_ISA_EXT);
 518         unsigned long reg_val = 0;
 519         unsigned long host_isa_ext;
 520
 521         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 522                 return -EINVAL;
 523
 524         if (reg_num >= KVM_RISCV_ISA_EXT_MAX ||
 525             reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
 526                 return -EINVAL;
 527
 528         host_isa_ext = kvm_isa_ext_arr[reg_num];
 529         if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext))
 530                 reg_val = 1; /* Mark the given extension as available */
 531
 532         if (copy_to_user(uaddr, &reg_val, KVM_REG_SIZE(reg->id)))
 533                 return -EFAULT;
 534
 535         return 0;
 536 }
 537
 538 static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu,
 539                                           const struct kvm_one_reg *reg)
 540 {
 541         unsigned long __user *uaddr =
 542                         (unsigned long __user *)(unsigned long)reg->addr;
 543         unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK |
 544                                             KVM_REG_SIZE_MASK |
 545                                             KVM_REG_RISCV_ISA_EXT);
 546         unsigned long reg_val;
 547         unsigned long host_isa_ext;
 548
 549         if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long))
 550                 return -EINVAL;
 551
 552         if (reg_num >= KVM_RISCV_ISA_EXT_MAX ||
 553             reg_num >= ARRAY_SIZE(kvm_isa_ext_arr))
 554                 return -EINVAL;
 555
 556         if (copy_from_user(&reg_val, uaddr, KVM_REG_SIZE(reg->id)))
 557                 return -EFAULT;
 558
 559         host_isa_ext = kvm_isa_ext_arr[reg_num];
 560         if (!__riscv_isa_extension_available(NULL, host_isa_ext))
 561                 return  -EOPNOTSUPP;
 562
 563         if (!vcpu->arch.ran_atleast_once) {
 564                 /*
 565                  * All multi-letter extension and a few single letter
 566                  * extension can be disabled
 567                  */
 568                 if (reg_val == 1 &&
 569                     kvm_riscv_vcpu_isa_enable_allowed(reg_num))
 570                         set_bit(host_isa_ext, vcpu->arch.isa);
 571                 else if (!reg_val &&
 572                          kvm_riscv_vcpu_isa_disable_allowed(reg_num))
 573                         clear_bit(host_isa_ext, vcpu->arch.isa);
 574                 else
 575                         return -EINVAL;
 576                 kvm_riscv_vcpu_fp_reset(vcpu);
 577         } else {
 578                 return -EOPNOTSUPP;
 579         }
 580
 581         return 0;
 582 }
 583
 584 static int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu,
 585                                   const struct kvm_one_reg *reg)
 586 {
 587         switch (reg->id & KVM_REG_RISCV_TYPE_MASK) {
 588         case KVM_REG_RISCV_CONFIG:
 589                 return kvm_riscv_vcpu_set_reg_config(vcpu, reg);
 590         case KVM_REG_RISCV_CORE:
 591                 return kvm_riscv_vcpu_set_reg_core(vcpu, reg);
 592         case KVM_REG_RISCV_CSR:
 593                 return kvm_riscv_vcpu_set_reg_csr(vcpu, reg);
 594         case KVM_REG_RISCV_TIMER:
 595                 return kvm_riscv_vcpu_set_reg_timer(vcpu, reg);
 596         case KVM_REG_RISCV_FP_F:
 597                 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg,
 598                                                  KVM_REG_RISCV_FP_F);
 599         case KVM_REG_RISCV_FP_D:
 600                 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg,
 601                                                  KVM_REG_RISCV_FP_D);
 602         case KVM_REG_RISCV_ISA_EXT:
 603                 return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg);
 604         default:
 605                 break;
 606         }
 607
 608         return -EINVAL;
 609 }
 610
 611 static int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu,
 612                                   const struct kvm_one_reg *reg)
 613 {
 614         switch (reg->id & KVM_REG_RISCV_TYPE_MASK) {
 615         case KVM_REG_RISCV_CONFIG:
 616                 return kvm_riscv_vcpu_get_reg_config(vcpu, reg);
 617         case KVM_REG_RISCV_CORE:
 618                 return kvm_riscv_vcpu_get_reg_core(vcpu, reg);
 619         case KVM_REG_RISCV_CSR:
 620                 return kvm_riscv_vcpu_get_reg_csr(vcpu, reg);
 621         case KVM_REG_RISCV_TIMER:
 622                 return kvm_riscv_vcpu_get_reg_timer(vcpu, reg);
 623         case KVM_REG_RISCV_FP_F:
 624                 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg,
 625                                                  KVM_REG_RISCV_FP_F);
 626         case KVM_REG_RISCV_FP_D:
 627                 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg,
 628                                                  KVM_REG_RISCV_FP_D);
 629         case KVM_REG_RISCV_ISA_EXT:
 630                 return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg);
 631         default:
 632                 break;
 633         }
 634
 635         return -EINVAL;
 636 }
 637
 638 long kvm_arch_vcpu_async_ioctl(struct file *filp,
 639                                unsigned int ioctl, unsigned long arg)
 640 {
 641         struct kvm_vcpu *vcpu = filp->private_data;
 642         void __user *argp = (void __user *)arg;
 643
 644         if (ioctl == KVM_INTERRUPT) {
 645                 struct kvm_interrupt irq;
 646
 647                 if (copy_from_user(&irq, argp, sizeof(irq)))
 648                         return -EFAULT;
 649
 650                 if (irq.irq == KVM_INTERRUPT_SET)
 651                         return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT);
 652                 else
 653                         return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT);
 654         }
 655
 656         return -ENOIOCTLCMD;
 657 }
 658
 659 long kvm_arch_vcpu_ioctl(struct file *filp,
 660                          unsigned int ioctl, unsigned long arg)
 661 {
 662         struct kvm_vcpu *vcpu = filp->private_data;
 663         void __user *argp = (void __user *)arg;
 664         long r = -EINVAL;
 665
 666         switch (ioctl) {
 667         case KVM_SET_ONE_REG:
 668         case KVM_GET_ONE_REG: {
 669                 struct kvm_one_reg reg;
 670
 671                 r = -EFAULT;
 672                 if (copy_from_user(&reg, argp, sizeof(reg)))
 673                         break;
 674
 675                 if (ioctl == KVM_SET_ONE_REG)
 676                         r = kvm_riscv_vcpu_set_reg(vcpu, &reg);
 677                 else
 678                         r = kvm_riscv_vcpu_get_reg(vcpu, &reg);
 679                 break;
 680         }
 681         default:
 682                 break;
 683         }
 684
 685         return r;
 686 }
 687
 688 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 689                                   struct kvm_sregs *sregs)
 690 {
 691         return -EINVAL;
 692 }
 693
 694 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 695                                   struct kvm_sregs *sregs)
 696 {
 697         return -EINVAL;
 698 }
 699
 700 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 701 {
 702         return -EINVAL;
 703 }
 704
 705 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 706 {
 707         return -EINVAL;
 708 }
 709
 710 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 711                                   struct kvm_translation *tr)
 712 {
 713         return -EINVAL;
 714 }
 715
 716 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 717 {
 718         return -EINVAL;
 719 }
 720
 721 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 722 {
 723         return -EINVAL;
 724 }
 725
 726 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu)
 727 {
 728         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 729         unsigned long mask, val;
 730
 731         if (READ_ONCE(vcpu->arch.irqs_pending_mask)) {
 732                 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask, 0);
 733                 val = READ_ONCE(vcpu->arch.irqs_pending) & mask;
 734
 735                 csr->hvip &= ~mask;
 736                 csr->hvip |= val;
 737         }
 738 }
 739
 740 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu)
 741 {
 742         unsigned long hvip;
 743         struct kvm_vcpu_arch *v = &vcpu->arch;
 744         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 745
 746         /* Read current HVIP and VSIE CSRs */
 747         csr->vsie = csr_read(CSR_VSIE);
 748
 749         /* Sync-up HVIP.VSSIP bit changes does by Guest */
 750         hvip = csr_read(CSR_HVIP);
 751         if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) {
 752                 if (hvip & (1UL << IRQ_VS_SOFT)) {
 753                         if (!test_and_set_bit(IRQ_VS_SOFT,
 754                                               &v->irqs_pending_mask))
 755                                 set_bit(IRQ_VS_SOFT, &v->irqs_pending);
 756                 } else {
 757                         if (!test_and_set_bit(IRQ_VS_SOFT,
 758                                               &v->irqs_pending_mask))
 759                                 clear_bit(IRQ_VS_SOFT, &v->irqs_pending);
 760                 }
 761         }
 762
 763         /* Sync-up timer CSRs */
 764         kvm_riscv_vcpu_timer_sync(vcpu);
 765 }
 766
 767 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
 768 {
 769         if (irq != IRQ_VS_SOFT &&
 770             irq != IRQ_VS_TIMER &&
 771             irq != IRQ_VS_EXT)
 772                 return -EINVAL;
 773
 774         set_bit(irq, &vcpu->arch.irqs_pending);
 775         smp_mb__before_atomic();
 776         set_bit(irq, &vcpu->arch.irqs_pending_mask);
 777
 778         kvm_vcpu_kick(vcpu);
 779
 780         return 0;
 781 }
 782
 783 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq)
 784 {
 785         if (irq != IRQ_VS_SOFT &&
 786             irq != IRQ_VS_TIMER &&
 787             irq != IRQ_VS_EXT)
 788                 return -EINVAL;
 789
 790         clear_bit(irq, &vcpu->arch.irqs_pending);
 791         smp_mb__before_atomic();
 792         set_bit(irq, &vcpu->arch.irqs_pending_mask);
 793
 794         return 0;
 795 }
 796
 797 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, unsigned long mask)
 798 {
 799         unsigned long ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK)
 800                             << VSIP_TO_HVIP_SHIFT) & mask;
 801
 802         return (READ_ONCE(vcpu->arch.irqs_pending) & ie) ? true : false;
 803 }
 804
 805 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu)
 806 {
 807         vcpu->arch.power_off = true;
 808         kvm_make_request(KVM_REQ_SLEEP, vcpu);
 809         kvm_vcpu_kick(vcpu);
 810 }
 811
 812 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu)
 813 {
 814         vcpu->arch.power_off = false;
 815         kvm_vcpu_wake_up(vcpu);
 816 }
 817
 818 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 819                                     struct kvm_mp_state *mp_state)
 820 {
 821         if (vcpu->arch.power_off)
 822                 mp_state->mp_state = KVM_MP_STATE_STOPPED;
 823         else
 824                 mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
 825
 826         return 0;
 827 }
 828
 829 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 830                                     struct kvm_mp_state *mp_state)
 831 {
 832         int ret = 0;
 833
 834         switch (mp_state->mp_state) {
 835         case KVM_MP_STATE_RUNNABLE:
 836                 vcpu->arch.power_off = false;
 837                 break;
 838         case KVM_MP_STATE_STOPPED:
 839                 kvm_riscv_vcpu_power_off(vcpu);
 840                 break;
 841         default:
 842                 ret = -EINVAL;
 843         }
 844
 845         return ret;
 846 }
 847
 848 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 849                                         struct kvm_guest_debug *dbg)
 850 {
 851         /* TODO; To be implemented later. */
 852         return -EINVAL;
 853 }
 854
 855 static void kvm_riscv_vcpu_update_config(const unsigned long *isa)
 856 {
 857         u64 henvcfg = 0;
 858
 859         if (riscv_isa_extension_available(isa, SVPBMT))
 860                 henvcfg |= ENVCFG_PBMTE;
 861
 862         if (riscv_isa_extension_available(isa, SSTC))
 863                 henvcfg |= ENVCFG_STCE;
 864
 865         if (riscv_isa_extension_available(isa, ZICBOM))
 866                 henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE);
 867
 868         csr_write(CSR_HENVCFG, henvcfg);
 869 #ifdef CONFIG_32BIT
 870         csr_write(CSR_HENVCFGH, henvcfg >> 32);
 871 #endif
 872 }
 873
 874 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 875 {
 876         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 877
 878         csr_write(CSR_VSSTATUS, csr->vsstatus);
 879         csr_write(CSR_VSIE, csr->vsie);
 880         csr_write(CSR_VSTVEC, csr->vstvec);
 881         csr_write(CSR_VSSCRATCH, csr->vsscratch);
 882         csr_write(CSR_VSEPC, csr->vsepc);
 883         csr_write(CSR_VSCAUSE, csr->vscause);
 884         csr_write(CSR_VSTVAL, csr->vstval);
 885         csr_write(CSR_HVIP, csr->hvip);
 886         csr_write(CSR_VSATP, csr->vsatp);
 887
 888         kvm_riscv_vcpu_update_config(vcpu->arch.isa);
 889
 890         kvm_riscv_gstage_update_hgatp(vcpu);
 891
 892         kvm_riscv_vcpu_timer_restore(vcpu);
 893
 894         kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context);
 895         kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context,
 896                                         vcpu->arch.isa);
 897
 898         vcpu->cpu = cpu;
 899 }
 900
 901 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 902 {
 903         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 904
 905         vcpu->cpu = -1;
 906
 907         kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context,
 908                                      vcpu->arch.isa);
 909         kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context);
 910
 911         kvm_riscv_vcpu_timer_save(vcpu);
 912
 913         csr->vsstatus = csr_read(CSR_VSSTATUS);
 914         csr->vsie = csr_read(CSR_VSIE);
 915         csr->vstvec = csr_read(CSR_VSTVEC);
 916         csr->vsscratch = csr_read(CSR_VSSCRATCH);
 917         csr->vsepc = csr_read(CSR_VSEPC);
 918         csr->vscause = csr_read(CSR_VSCAUSE);
 919         csr->vstval = csr_read(CSR_VSTVAL);
 920         csr->hvip = csr_read(CSR_HVIP);
 921         csr->vsatp = csr_read(CSR_VSATP);
 922 }
 923
 924 static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu)
 925 {
 926         struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
 927
 928         if (kvm_request_pending(vcpu)) {
 929                 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) {
 930                         kvm_vcpu_srcu_read_unlock(vcpu);
 931                         rcuwait_wait_event(wait,
 932                                 (!vcpu->arch.power_off) && (!vcpu->arch.pause),
 933                                 TASK_INTERRUPTIBLE);
 934                         kvm_vcpu_srcu_read_lock(vcpu);
 935
 936                         if (vcpu->arch.power_off || vcpu->arch.pause) {
 937                                 /*
 938                                  * Awaken to handle a signal, request to
 939                                  * sleep again later.
 940                                  */
 941                                 kvm_make_request(KVM_REQ_SLEEP, vcpu);
 942                         }
 943                 }
 944
 945                 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
 946                         kvm_riscv_reset_vcpu(vcpu);
 947
 948                 if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu))
 949                         kvm_riscv_gstage_update_hgatp(vcpu);
 950
 951                 if (kvm_check_request(KVM_REQ_FENCE_I, vcpu))
 952                         kvm_riscv_fence_i_process(vcpu);
 953
 954                 /*
 955                  * The generic KVM_REQ_TLB_FLUSH is same as
 956                  * KVM_REQ_HFENCE_GVMA_VMID_ALL
 957                  */
 958                 if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu))
 959                         kvm_riscv_hfence_gvma_vmid_all_process(vcpu);
 960
 961                 if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu))
 962                         kvm_riscv_hfence_vvma_all_process(vcpu);
 963
 964                 if (kvm_check_request(KVM_REQ_HFENCE, vcpu))
 965                         kvm_riscv_hfence_process(vcpu);
 966         }
 967 }
 968
 969 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu)
 970 {
 971         struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
 972
 973         csr_write(CSR_HVIP, csr->hvip);
 974 }
 975
 976 /*
 977  * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
 978  * the vCPU is running.
 979  *
 980  * This must be noinstr as instrumentation may make use of RCU, and this is not
 981  * safe during the EQS.
 982  */
 983 static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu)
 984 {
 985         guest_state_enter_irqoff();
 986         __kvm_riscv_switch_to(&vcpu->arch);
 987         vcpu->arch.last_exit_cpu = vcpu->cpu;
 988         guest_state_exit_irqoff();
 989 }
 990
 991 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 992 {
 993         int ret;
 994         struct kvm_cpu_trap trap;
 995         struct kvm_run *run = vcpu->run;
 996
 997         /* Mark this VCPU ran at least once */
 998         vcpu->arch.ran_atleast_once = true;
 999
1000         kvm_vcpu_srcu_read_lock(vcpu);
1001
1002         switch (run->exit_reason) {
1003         case KVM_EXIT_MMIO:
1004                 /* Process MMIO value returned from user-space */
1005                 ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run);
1006                 break;
1007         case KVM_EXIT_RISCV_SBI:
1008                 /* Process SBI value returned from user-space */
1009                 ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run);
1010                 break;
1011         case KVM_EXIT_RISCV_CSR:
1012                 /* Process CSR value returned from user-space */
1013                 ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run);
1014                 break;
1015         default:
1016                 ret = 0;
1017                 break;
1018         }
1019         if (ret) {
1020                 kvm_vcpu_srcu_read_unlock(vcpu);
1021                 return ret;
1022         }
1023
1024         if (run->immediate_exit) {
1025                 kvm_vcpu_srcu_read_unlock(vcpu);
1026                 return -EINTR;
1027         }
1028
1029         vcpu_load(vcpu);
1030
1031         kvm_sigset_activate(vcpu);
1032
1033         ret = 1;
1034         run->exit_reason = KVM_EXIT_UNKNOWN;
1035         while (ret > 0) {
1036                 /* Check conditions before entering the guest */
1037                 ret = xfer_to_guest_mode_handle_work(vcpu);
1038                 if (ret)
1039                         continue;
1040                 ret = 1;
1041
1042                 kvm_riscv_gstage_vmid_update(vcpu);
1043
1044                 kvm_riscv_check_vcpu_requests(vcpu);
1045
1046                 local_irq_disable();
1047
1048                 /*
1049                  * Ensure we set mode to IN_GUEST_MODE after we disable
1050                  * interrupts and before the final VCPU requests check.
1051                  * See the comment in kvm_vcpu_exiting_guest_mode() and
1052                  * Documentation/virt/kvm/vcpu-requests.rst
1053                  */
1054                 vcpu->mode = IN_GUEST_MODE;
1055
1056                 kvm_vcpu_srcu_read_unlock(vcpu);
1057                 smp_mb__after_srcu_read_unlock();
1058
1059                 /*
1060                  * We might have got VCPU interrupts updated asynchronously
1061                  * so update it in HW.
1062                  */
1063                 kvm_riscv_vcpu_flush_interrupts(vcpu);
1064
1065                 /* Update HVIP CSR for current CPU */
1066                 kvm_riscv_update_hvip(vcpu);
1067
1068                 if (ret <= 0 ||
1069                     kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
1070                     kvm_request_pending(vcpu) ||
1071                     xfer_to_guest_mode_work_pending()) {
1072                         vcpu->mode = OUTSIDE_GUEST_MODE;
1073                         local_irq_enable();
1074                         kvm_vcpu_srcu_read_lock(vcpu);
1075                         continue;
1076                 }
1077
1078                 /*
1079                  * Cleanup stale TLB enteries
1080                  *
1081                  * Note: This should be done after G-stage VMID has been
1082                  * updated using kvm_riscv_gstage_vmid_ver_changed()
1083                  */
1084                 kvm_riscv_local_tlb_sanitize(vcpu);
1085
1086                 guest_timing_enter_irqoff();
1087
1088                 kvm_riscv_vcpu_enter_exit(vcpu);
1089
1090                 vcpu->mode = OUTSIDE_GUEST_MODE;
1091                 vcpu->stat.exits++;
1092
1093                 /*
1094                  * Save SCAUSE, STVAL, HTVAL, and HTINST because we might
1095                  * get an interrupt between __kvm_riscv_switch_to() and
1096                  * local_irq_enable() which can potentially change CSRs.
1097                  */
1098                 trap.sepc = vcpu->arch.guest_context.sepc;
1099                 trap.scause = csr_read(CSR_SCAUSE);
1100                 trap.stval = csr_read(CSR_STVAL);
1101                 trap.htval = csr_read(CSR_HTVAL);
1102                 trap.htinst = csr_read(CSR_HTINST);
1103
1104                 /* Syncup interrupts state with HW */
1105                 kvm_riscv_vcpu_sync_interrupts(vcpu);
1106
1107                 preempt_disable();
1108
1109                 /*
1110                  * We must ensure that any pending interrupts are taken before
1111                  * we exit guest timing so that timer ticks are accounted as
1112                  * guest time. Transiently unmask interrupts so that any
1113                  * pending interrupts are taken.
1114                  *
1115                  * There's no barrier which ensures that pending interrupts are
1116                  * recognised, so we just hope that the CPU takes any pending
1117                  * interrupts between the enable and disable.
1118                  */
1119                 local_irq_enable();
1120                 local_irq_disable();
1121
1122                 guest_timing_exit_irqoff();
1123
1124                 local_irq_enable();
1125
1126                 preempt_enable();
1127
1128                 kvm_vcpu_srcu_read_lock(vcpu);
1129
1130                 ret = kvm_riscv_vcpu_exit(vcpu, run, &trap);
1131         }
1132
1133         kvm_sigset_deactivate(vcpu);
1134
1135         vcpu_put(vcpu);
1136
1137         kvm_vcpu_srcu_read_unlock(vcpu);
1138
1139         return ret;
1140 }