arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/random.h>
  26 #include <linux/slab.h>
  27 #include <linux/timer.h>
  28 #include <linux/vmalloc.h>
  29 #include <asm/asm-offsets.h>
  30 #include <asm/lowcore.h>
  31 #include <asm/pgtable.h>
  32 #include <asm/nmi.h>
  33 #include <asm/switch_to.h>
  34 #include <asm/isc.h>
  35 #include <asm/sclp.h>
  36 #include "kvm-s390.h"
  37 #include "gaccess.h"
  38
  39 #define KMSG_COMPONENT "kvm-s390"
  40 #undef pr_fmt
  41 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  42
  43 #define CREATE_TRACE_POINTS
  44 #include "trace.h"
  45 #include "trace-s390.h"
  46
  47 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  48 #define LOCAL_IRQS 32
  49 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  50                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  51
  52 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  53
  54 struct kvm_stats_debugfs_item debugfs_entries[] = {
  55         { "userspace_handled", VCPU_STAT(exit_userspace) },
  56         { "exit_null", VCPU_STAT(exit_null) },
  57         { "exit_validity", VCPU_STAT(exit_validity) },
  58         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  59         { "exit_external_request", VCPU_STAT(exit_external_request) },
  60         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  61         { "exit_instruction", VCPU_STAT(exit_instruction) },
  62         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  63         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  64         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  65         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  66         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  67         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  68         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  69         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  70         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  71         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  72         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  73         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  74         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  75         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  76         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  77         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  78         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  79         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  80         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  81         { "instruction_spx", VCPU_STAT(instruction_spx) },
  82         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  83         { "instruction_stap", VCPU_STAT(instruction_stap) },
  84         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  85         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  86         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  87         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  88         { "instruction_essa", VCPU_STAT(instruction_essa) },
  89         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  90         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  91         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  92         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  93         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  94         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  95         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  96         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
  97         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
  98         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
  99         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 100         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 101         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 102         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 103         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 104         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 105         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 106         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 107         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 108         { "diagnose_10", VCPU_STAT(diagnose_10) },
 109         { "diagnose_44", VCPU_STAT(diagnose_44) },
 110         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 111         { NULL }
 112 };
 113
 114 /* upper facilities limit for kvm */
 115 unsigned long kvm_s390_fac_list_mask[] = {
 116         0xffe6fffbfcfdfc40UL,
 117         0x005e800000000000UL,
 118 };
 119
 120 unsigned long kvm_s390_fac_list_mask_size(void)
 121 {
 122         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 123         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 124 }
 125
 126 static struct gmap_notifier gmap_notifier;
 127
 128 /* Section: not file related */
 129 int kvm_arch_hardware_enable(void)
 130 {
 131         /* every s390 is virtualization enabled ;-) */
 132         return 0;
 133 }
 134
 135 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 136
 137 int kvm_arch_hardware_setup(void)
 138 {
 139         gmap_notifier.notifier_call = kvm_gmap_notifier;
 140         gmap_register_ipte_notifier(&gmap_notifier);
 141         return 0;
 142 }
 143
 144 void kvm_arch_hardware_unsetup(void)
 145 {
 146         gmap_unregister_ipte_notifier(&gmap_notifier);
 147 }
 148
 149 int kvm_arch_init(void *opaque)
 150 {
 151         /* Register floating interrupt controller interface. */
 152         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 153 }
 154
 155 /* Section: device related */
 156 long kvm_arch_dev_ioctl(struct file *filp,
 157                         unsigned int ioctl, unsigned long arg)
 158 {
 159         if (ioctl == KVM_S390_ENABLE_SIE)
 160                 return s390_enable_sie();
 161         return -EINVAL;
 162 }
 163
 164 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 165 {
 166         int r;
 167
 168         switch (ext) {
 169         case KVM_CAP_S390_PSW:
 170         case KVM_CAP_S390_GMAP:
 171         case KVM_CAP_SYNC_MMU:
 172 #ifdef CONFIG_KVM_S390_UCONTROL
 173         case KVM_CAP_S390_UCONTROL:
 174 #endif
 175         case KVM_CAP_ASYNC_PF:
 176         case KVM_CAP_SYNC_REGS:
 177         case KVM_CAP_ONE_REG:
 178         case KVM_CAP_ENABLE_CAP:
 179         case KVM_CAP_S390_CSS_SUPPORT:
 180         case KVM_CAP_IOEVENTFD:
 181         case KVM_CAP_DEVICE_CTRL:
 182         case KVM_CAP_ENABLE_CAP_VM:
 183         case KVM_CAP_S390_IRQCHIP:
 184         case KVM_CAP_VM_ATTRIBUTES:
 185         case KVM_CAP_MP_STATE:
 186         case KVM_CAP_S390_INJECT_IRQ:
 187         case KVM_CAP_S390_USER_SIGP:
 188         case KVM_CAP_S390_USER_STSI:
 189         case KVM_CAP_S390_SKEYS:
 190         case KVM_CAP_S390_IRQ_STATE:
 191                 r = 1;
 192                 break;
 193         case KVM_CAP_S390_MEM_OP:
 194                 r = MEM_OP_MAX_SIZE;
 195                 break;
 196         case KVM_CAP_NR_VCPUS:
 197         case KVM_CAP_MAX_VCPUS:
 198                 r = KVM_MAX_VCPUS;
 199                 break;
 200         case KVM_CAP_NR_MEMSLOTS:
 201                 r = KVM_USER_MEM_SLOTS;
 202                 break;
 203         case KVM_CAP_S390_COW:
 204                 r = MACHINE_HAS_ESOP;
 205                 break;
 206         case KVM_CAP_S390_VECTOR_REGISTERS:
 207                 r = MACHINE_HAS_VX;
 208                 break;
 209         default:
 210                 r = 0;
 211         }
 212         return r;
 213 }
 214
 215 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 216                                         struct kvm_memory_slot *memslot)
 217 {
 218         gfn_t cur_gfn, last_gfn;
 219         unsigned long address;
 220         struct gmap *gmap = kvm->arch.gmap;
 221
 222         down_read(&gmap->mm->mmap_sem);
 223         /* Loop over all guest pages */
 224         last_gfn = memslot->base_gfn + memslot->npages;
 225         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 226                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 227
 228                 if (gmap_test_and_clear_dirty(address, gmap))
 229                         mark_page_dirty(kvm, cur_gfn);
 230         }
 231         up_read(&gmap->mm->mmap_sem);
 232 }
 233
 234 /* Section: vm related */
 235 /*
 236  * Get (and clear) the dirty memory log for a memory slot.
 237  */
 238 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 239                                struct kvm_dirty_log *log)
 240 {
 241         int r;
 242         unsigned long n;
 243         struct kvm_memslots *slots;
 244         struct kvm_memory_slot *memslot;
 245         int is_dirty = 0;
 246
 247         mutex_lock(&kvm->slots_lock);
 248
 249         r = -EINVAL;
 250         if (log->slot >= KVM_USER_MEM_SLOTS)
 251                 goto out;
 252
 253         slots = kvm_memslots(kvm);
 254         memslot = id_to_memslot(slots, log->slot);
 255         r = -ENOENT;
 256         if (!memslot->dirty_bitmap)
 257                 goto out;
 258
 259         kvm_s390_sync_dirty_log(kvm, memslot);
 260         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 261         if (r)
 262                 goto out;
 263
 264         /* Clear the dirty log */
 265         if (is_dirty) {
 266                 n = kvm_dirty_bitmap_bytes(memslot);
 267                 memset(memslot->dirty_bitmap, 0, n);
 268         }
 269         r = 0;
 270 out:
 271         mutex_unlock(&kvm->slots_lock);
 272         return r;
 273 }
 274
 275 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 276 {
 277         int r;
 278
 279         if (cap->flags)
 280                 return -EINVAL;
 281
 282         switch (cap->cap) {
 283         case KVM_CAP_S390_IRQCHIP:
 284                 kvm->arch.use_irqchip = 1;
 285                 r = 0;
 286                 break;
 287         case KVM_CAP_S390_USER_SIGP:
 288                 kvm->arch.user_sigp = 1;
 289                 r = 0;
 290                 break;
 291         case KVM_CAP_S390_VECTOR_REGISTERS:
 292                 if (MACHINE_HAS_VX) {
 293                         set_kvm_facility(kvm->arch.model.fac->mask, 129);
 294                         set_kvm_facility(kvm->arch.model.fac->list, 129);
 295                         r = 0;
 296                 } else
 297                         r = -EINVAL;
 298                 break;
 299         case KVM_CAP_S390_USER_STSI:
 300                 kvm->arch.user_stsi = 1;
 301                 r = 0;
 302                 break;
 303         default:
 304                 r = -EINVAL;
 305                 break;
 306         }
 307         return r;
 308 }
 309
 310 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 311 {
 312         int ret;
 313
 314         switch (attr->attr) {
 315         case KVM_S390_VM_MEM_LIMIT_SIZE:
 316                 ret = 0;
 317                 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
 318                         ret = -EFAULT;
 319                 break;
 320         default:
 321                 ret = -ENXIO;
 322                 break;
 323         }
 324         return ret;
 325 }
 326
 327 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 328 {
 329         int ret;
 330         unsigned int idx;
 331         switch (attr->attr) {
 332         case KVM_S390_VM_MEM_ENABLE_CMMA:
 333                 ret = -EBUSY;
 334                 mutex_lock(&kvm->lock);
 335                 if (atomic_read(&kvm->online_vcpus) == 0) {
 336                         kvm->arch.use_cmma = 1;
 337                         ret = 0;
 338                 }
 339                 mutex_unlock(&kvm->lock);
 340                 break;
 341         case KVM_S390_VM_MEM_CLR_CMMA:
 342                 mutex_lock(&kvm->lock);
 343                 idx = srcu_read_lock(&kvm->srcu);
 344                 s390_reset_cmma(kvm->arch.gmap->mm);
 345                 srcu_read_unlock(&kvm->srcu, idx);
 346                 mutex_unlock(&kvm->lock);
 347                 ret = 0;
 348                 break;
 349         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 350                 unsigned long new_limit;
 351
 352                 if (kvm_is_ucontrol(kvm))
 353                         return -EINVAL;
 354
 355                 if (get_user(new_limit, (u64 __user *)attr->addr))
 356                         return -EFAULT;
 357
 358                 if (new_limit > kvm->arch.gmap->asce_end)
 359                         return -E2BIG;
 360
 361                 ret = -EBUSY;
 362                 mutex_lock(&kvm->lock);
 363                 if (atomic_read(&kvm->online_vcpus) == 0) {
 364                         /* gmap_alloc will round the limit up */
 365                         struct gmap *new = gmap_alloc(current->mm, new_limit);
 366
 367                         if (!new) {
 368                                 ret = -ENOMEM;
 369                         } else {
 370                                 gmap_free(kvm->arch.gmap);
 371                                 new->private = kvm;
 372                                 kvm->arch.gmap = new;
 373                                 ret = 0;
 374                         }
 375                 }
 376                 mutex_unlock(&kvm->lock);
 377                 break;
 378         }
 379         default:
 380                 ret = -ENXIO;
 381                 break;
 382         }
 383         return ret;
 384 }
 385
 386 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 387
 388 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 389 {
 390         struct kvm_vcpu *vcpu;
 391         int i;
 392
 393         if (!test_kvm_facility(kvm, 76))
 394                 return -EINVAL;
 395
 396         mutex_lock(&kvm->lock);
 397         switch (attr->attr) {
 398         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 399                 get_random_bytes(
 400                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 401                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 402                 kvm->arch.crypto.aes_kw = 1;
 403                 break;
 404         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 405                 get_random_bytes(
 406                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 407                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 408                 kvm->arch.crypto.dea_kw = 1;
 409                 break;
 410         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 411                 kvm->arch.crypto.aes_kw = 0;
 412                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 413                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 414                 break;
 415         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 416                 kvm->arch.crypto.dea_kw = 0;
 417                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 418                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 419                 break;
 420         default:
 421                 mutex_unlock(&kvm->lock);
 422                 return -ENXIO;
 423         }
 424
 425         kvm_for_each_vcpu(i, vcpu, kvm) {
 426                 kvm_s390_vcpu_crypto_setup(vcpu);
 427                 exit_sie(vcpu);
 428         }
 429         mutex_unlock(&kvm->lock);
 430         return 0;
 431 }
 432
 433 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 434 {
 435         u8 gtod_high;
 436
 437         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 438                                            sizeof(gtod_high)))
 439                 return -EFAULT;
 440
 441         if (gtod_high != 0)
 442                 return -EINVAL;
 443
 444         return 0;
 445 }
 446
 447 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 448 {
 449         struct kvm_vcpu *cur_vcpu;
 450         unsigned int vcpu_idx;
 451         u64 host_tod, gtod;
 452         int r;
 453
 454         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 455                 return -EFAULT;
 456
 457         r = store_tod_clock(&host_tod);
 458         if (r)
 459                 return r;
 460
 461         mutex_lock(&kvm->lock);
 462         kvm->arch.epoch = gtod - host_tod;
 463         kvm_s390_vcpu_block_all(kvm);
 464         kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm)
 465                 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
 466         kvm_s390_vcpu_unblock_all(kvm);
 467         mutex_unlock(&kvm->lock);
 468         return 0;
 469 }
 470
 471 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 472 {
 473         int ret;
 474
 475         if (attr->flags)
 476                 return -EINVAL;
 477
 478         switch (attr->attr) {
 479         case KVM_S390_VM_TOD_HIGH:
 480                 ret = kvm_s390_set_tod_high(kvm, attr);
 481                 break;
 482         case KVM_S390_VM_TOD_LOW:
 483                 ret = kvm_s390_set_tod_low(kvm, attr);
 484                 break;
 485         default:
 486                 ret = -ENXIO;
 487                 break;
 488         }
 489         return ret;
 490 }
 491
 492 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 493 {
 494         u8 gtod_high = 0;
 495
 496         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 497                                          sizeof(gtod_high)))
 498                 return -EFAULT;
 499
 500         return 0;
 501 }
 502
 503 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 504 {
 505         u64 host_tod, gtod;
 506         int r;
 507
 508         r = store_tod_clock(&host_tod);
 509         if (r)
 510                 return r;
 511
 512         gtod = host_tod + kvm->arch.epoch;
 513         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 514                 return -EFAULT;
 515
 516         return 0;
 517 }
 518
 519 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 520 {
 521         int ret;
 522
 523         if (attr->flags)
 524                 return -EINVAL;
 525
 526         switch (attr->attr) {
 527         case KVM_S390_VM_TOD_HIGH:
 528                 ret = kvm_s390_get_tod_high(kvm, attr);
 529                 break;
 530         case KVM_S390_VM_TOD_LOW:
 531                 ret = kvm_s390_get_tod_low(kvm, attr);
 532                 break;
 533         default:
 534                 ret = -ENXIO;
 535                 break;
 536         }
 537         return ret;
 538 }
 539
 540 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 541 {
 542         struct kvm_s390_vm_cpu_processor *proc;
 543         int ret = 0;
 544
 545         mutex_lock(&kvm->lock);
 546         if (atomic_read(&kvm->online_vcpus)) {
 547                 ret = -EBUSY;
 548                 goto out;
 549         }
 550         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 551         if (!proc) {
 552                 ret = -ENOMEM;
 553                 goto out;
 554         }
 555         if (!copy_from_user(proc, (void __user *)attr->addr,
 556                             sizeof(*proc))) {
 557                 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
 558                        sizeof(struct cpuid));
 559                 kvm->arch.model.ibc = proc->ibc;
 560                 memcpy(kvm->arch.model.fac->list, proc->fac_list,
 561                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 562         } else
 563                 ret = -EFAULT;
 564         kfree(proc);
 565 out:
 566         mutex_unlock(&kvm->lock);
 567         return ret;
 568 }
 569
 570 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 571 {
 572         int ret = -ENXIO;
 573
 574         switch (attr->attr) {
 575         case KVM_S390_VM_CPU_PROCESSOR:
 576                 ret = kvm_s390_set_processor(kvm, attr);
 577                 break;
 578         }
 579         return ret;
 580 }
 581
 582 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 583 {
 584         struct kvm_s390_vm_cpu_processor *proc;
 585         int ret = 0;
 586
 587         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 588         if (!proc) {
 589                 ret = -ENOMEM;
 590                 goto out;
 591         }
 592         memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
 593         proc->ibc = kvm->arch.model.ibc;
 594         memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
 595         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 596                 ret = -EFAULT;
 597         kfree(proc);
 598 out:
 599         return ret;
 600 }
 601
 602 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 603 {
 604         struct kvm_s390_vm_cpu_machine *mach;
 605         int ret = 0;
 606
 607         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 608         if (!mach) {
 609                 ret = -ENOMEM;
 610                 goto out;
 611         }
 612         get_cpu_id((struct cpuid *) &mach->cpuid);
 613         mach->ibc = sclp.ibc;
 614         memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
 615                S390_ARCH_FAC_LIST_SIZE_BYTE);
 616         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 617                S390_ARCH_FAC_LIST_SIZE_BYTE);
 618         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 619                 ret = -EFAULT;
 620         kfree(mach);
 621 out:
 622         return ret;
 623 }
 624
 625 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 626 {
 627         int ret = -ENXIO;
 628
 629         switch (attr->attr) {
 630         case KVM_S390_VM_CPU_PROCESSOR:
 631                 ret = kvm_s390_get_processor(kvm, attr);
 632                 break;
 633         case KVM_S390_VM_CPU_MACHINE:
 634                 ret = kvm_s390_get_machine(kvm, attr);
 635                 break;
 636         }
 637         return ret;
 638 }
 639
 640 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 641 {
 642         int ret;
 643
 644         switch (attr->group) {
 645         case KVM_S390_VM_MEM_CTRL:
 646                 ret = kvm_s390_set_mem_control(kvm, attr);
 647                 break;
 648         case KVM_S390_VM_TOD:
 649                 ret = kvm_s390_set_tod(kvm, attr);
 650                 break;
 651         case KVM_S390_VM_CPU_MODEL:
 652                 ret = kvm_s390_set_cpu_model(kvm, attr);
 653                 break;
 654         case KVM_S390_VM_CRYPTO:
 655                 ret = kvm_s390_vm_set_crypto(kvm, attr);
 656                 break;
 657         default:
 658                 ret = -ENXIO;
 659                 break;
 660         }
 661
 662         return ret;
 663 }
 664
 665 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 666 {
 667         int ret;
 668
 669         switch (attr->group) {
 670         case KVM_S390_VM_MEM_CTRL:
 671                 ret = kvm_s390_get_mem_control(kvm, attr);
 672                 break;
 673         case KVM_S390_VM_TOD:
 674                 ret = kvm_s390_get_tod(kvm, attr);
 675                 break;
 676         case KVM_S390_VM_CPU_MODEL:
 677                 ret = kvm_s390_get_cpu_model(kvm, attr);
 678                 break;
 679         default:
 680                 ret = -ENXIO;
 681                 break;
 682         }
 683
 684         return ret;
 685 }
 686
 687 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 688 {
 689         int ret;
 690
 691         switch (attr->group) {
 692         case KVM_S390_VM_MEM_CTRL:
 693                 switch (attr->attr) {
 694                 case KVM_S390_VM_MEM_ENABLE_CMMA:
 695                 case KVM_S390_VM_MEM_CLR_CMMA:
 696                 case KVM_S390_VM_MEM_LIMIT_SIZE:
 697                         ret = 0;
 698                         break;
 699                 default:
 700                         ret = -ENXIO;
 701                         break;
 702                 }
 703                 break;
 704         case KVM_S390_VM_TOD:
 705                 switch (attr->attr) {
 706                 case KVM_S390_VM_TOD_LOW:
 707                 case KVM_S390_VM_TOD_HIGH:
 708                         ret = 0;
 709                         break;
 710                 default:
 711                         ret = -ENXIO;
 712                         break;
 713                 }
 714                 break;
 715         case KVM_S390_VM_CPU_MODEL:
 716                 switch (attr->attr) {
 717                 case KVM_S390_VM_CPU_PROCESSOR:
 718                 case KVM_S390_VM_CPU_MACHINE:
 719                         ret = 0;
 720                         break;
 721                 default:
 722                         ret = -ENXIO;
 723                         break;
 724                 }
 725                 break;
 726         case KVM_S390_VM_CRYPTO:
 727                 switch (attr->attr) {
 728                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 729                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 730                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 731                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 732                         ret = 0;
 733                         break;
 734                 default:
 735                         ret = -ENXIO;
 736                         break;
 737                 }
 738                 break;
 739         default:
 740                 ret = -ENXIO;
 741                 break;
 742         }
 743
 744         return ret;
 745 }
 746
 747 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 748 {
 749         uint8_t *keys;
 750         uint64_t hva;
 751         unsigned long curkey;
 752         int i, r = 0;
 753
 754         if (args->flags != 0)
 755                 return -EINVAL;
 756
 757         /* Is this guest using storage keys? */
 758         if (!mm_use_skey(current->mm))
 759                 return KVM_S390_GET_SKEYS_NONE;
 760
 761         /* Enforce sane limit on memory allocation */
 762         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 763                 return -EINVAL;
 764
 765         keys = kmalloc_array(args->count, sizeof(uint8_t),
 766                              GFP_KERNEL | __GFP_NOWARN);
 767         if (!keys)
 768                 keys = vmalloc(sizeof(uint8_t) * args->count);
 769         if (!keys)
 770                 return -ENOMEM;
 771
 772         for (i = 0; i < args->count; i++) {
 773                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 774                 if (kvm_is_error_hva(hva)) {
 775                         r = -EFAULT;
 776                         goto out;
 777                 }
 778
 779                 curkey = get_guest_storage_key(current->mm, hva);
 780                 if (IS_ERR_VALUE(curkey)) {
 781                         r = curkey;
 782                         goto out;
 783                 }
 784                 keys[i] = curkey;
 785         }
 786
 787         r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
 788                          sizeof(uint8_t) * args->count);
 789         if (r)
 790                 r = -EFAULT;
 791 out:
 792         kvfree(keys);
 793         return r;
 794 }
 795
 796 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 797 {
 798         uint8_t *keys;
 799         uint64_t hva;
 800         int i, r = 0;
 801
 802         if (args->flags != 0)
 803                 return -EINVAL;
 804
 805         /* Enforce sane limit on memory allocation */
 806         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 807                 return -EINVAL;
 808
 809         keys = kmalloc_array(args->count, sizeof(uint8_t),
 810                              GFP_KERNEL | __GFP_NOWARN);
 811         if (!keys)
 812                 keys = vmalloc(sizeof(uint8_t) * args->count);
 813         if (!keys)
 814                 return -ENOMEM;
 815
 816         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
 817                            sizeof(uint8_t) * args->count);
 818         if (r) {
 819                 r = -EFAULT;
 820                 goto out;
 821         }
 822
 823         /* Enable storage key handling for the guest */
 824         s390_enable_skey();
 825
 826         for (i = 0; i < args->count; i++) {
 827                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 828                 if (kvm_is_error_hva(hva)) {
 829                         r = -EFAULT;
 830                         goto out;
 831                 }
 832
 833                 /* Lowest order bit is reserved */
 834                 if (keys[i] & 0x01) {
 835                         r = -EINVAL;
 836                         goto out;
 837                 }
 838
 839                 r = set_guest_storage_key(current->mm, hva,
 840                                           (unsigned long)keys[i], 0);
 841                 if (r)
 842                         goto out;
 843         }
 844 out:
 845         kvfree(keys);
 846         return r;
 847 }
 848
 849 long kvm_arch_vm_ioctl(struct file *filp,
 850                        unsigned int ioctl, unsigned long arg)
 851 {
 852         struct kvm *kvm = filp->private_data;
 853         void __user *argp = (void __user *)arg;
 854         struct kvm_device_attr attr;
 855         int r;
 856
 857         switch (ioctl) {
 858         case KVM_S390_INTERRUPT: {
 859                 struct kvm_s390_interrupt s390int;
 860
 861                 r = -EFAULT;
 862                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 863                         break;
 864                 r = kvm_s390_inject_vm(kvm, &s390int);
 865                 break;
 866         }
 867         case KVM_ENABLE_CAP: {
 868                 struct kvm_enable_cap cap;
 869                 r = -EFAULT;
 870                 if (copy_from_user(&cap, argp, sizeof(cap)))
 871                         break;
 872                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 873                 break;
 874         }
 875         case KVM_CREATE_IRQCHIP: {
 876                 struct kvm_irq_routing_entry routing;
 877
 878                 r = -EINVAL;
 879                 if (kvm->arch.use_irqchip) {
 880                         /* Set up dummy routing. */
 881                         memset(&routing, 0, sizeof(routing));
 882                         kvm_set_irq_routing(kvm, &routing, 0, 0);
 883                         r = 0;
 884                 }
 885                 break;
 886         }
 887         case KVM_SET_DEVICE_ATTR: {
 888                 r = -EFAULT;
 889                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 890                         break;
 891                 r = kvm_s390_vm_set_attr(kvm, &attr);
 892                 break;
 893         }
 894         case KVM_GET_DEVICE_ATTR: {
 895                 r = -EFAULT;
 896                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 897                         break;
 898                 r = kvm_s390_vm_get_attr(kvm, &attr);
 899                 break;
 900         }
 901         case KVM_HAS_DEVICE_ATTR: {
 902                 r = -EFAULT;
 903                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 904                         break;
 905                 r = kvm_s390_vm_has_attr(kvm, &attr);
 906                 break;
 907         }
 908         case KVM_S390_GET_SKEYS: {
 909                 struct kvm_s390_skeys args;
 910
 911                 r = -EFAULT;
 912                 if (copy_from_user(&args, argp,
 913                                    sizeof(struct kvm_s390_skeys)))
 914                         break;
 915                 r = kvm_s390_get_skeys(kvm, &args);
 916                 break;
 917         }
 918         case KVM_S390_SET_SKEYS: {
 919                 struct kvm_s390_skeys args;
 920
 921                 r = -EFAULT;
 922                 if (copy_from_user(&args, argp,
 923                                    sizeof(struct kvm_s390_skeys)))
 924                         break;
 925                 r = kvm_s390_set_skeys(kvm, &args);
 926                 break;
 927         }
 928         default:
 929                 r = -ENOTTY;
 930         }
 931
 932         return r;
 933 }
 934
 935 static int kvm_s390_query_ap_config(u8 *config)
 936 {
 937         u32 fcn_code = 0x04000000UL;
 938         u32 cc = 0;
 939
 940         memset(config, 0, 128);
 941         asm volatile(
 942                 "lgr 0,%1\n"
 943                 "lgr 2,%2\n"
 944                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
 945                 "0: ipm %0\n"
 946                 "srl %0,28\n"
 947                 "1:\n"
 948                 EX_TABLE(0b, 1b)
 949                 : "+r" (cc)
 950                 : "r" (fcn_code), "r" (config)
 951                 : "cc", "0", "2", "memory"
 952         );
 953
 954         return cc;
 955 }
 956
 957 static int kvm_s390_apxa_installed(void)
 958 {
 959         u8 config[128];
 960         int cc;
 961
 962         if (test_facility(2) && test_facility(12)) {
 963                 cc = kvm_s390_query_ap_config(config);
 964
 965                 if (cc)
 966                         pr_err("PQAP(QCI) failed with cc=%d", cc);
 967                 else
 968                         return config[0] & 0x40;
 969         }
 970
 971         return 0;
 972 }
 973
 974 static void kvm_s390_set_crycb_format(struct kvm *kvm)
 975 {
 976         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
 977
 978         if (kvm_s390_apxa_installed())
 979                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
 980         else
 981                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
 982 }
 983
 984 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
 985 {
 986         get_cpu_id(cpu_id);
 987         cpu_id->version = 0xff;
 988 }
 989
 990 static int kvm_s390_crypto_init(struct kvm *kvm)
 991 {
 992         if (!test_kvm_facility(kvm, 76))
 993                 return 0;
 994
 995         kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
 996                                          GFP_KERNEL | GFP_DMA);
 997         if (!kvm->arch.crypto.crycb)
 998                 return -ENOMEM;
 999
1000         kvm_s390_set_crycb_format(kvm);
1001
1002         /* Enable AES/DEA protected key functions by default */
1003         kvm->arch.crypto.aes_kw = 1;
1004         kvm->arch.crypto.dea_kw = 1;
1005         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1006                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1007         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1008                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1009
1010         return 0;
1011 }
1012
1013 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1014 {
1015         int i, rc;
1016         char debug_name[16];
1017         static unsigned long sca_offset;
1018
1019         rc = -EINVAL;
1020 #ifdef CONFIG_KVM_S390_UCONTROL
1021         if (type & ~KVM_VM_S390_UCONTROL)
1022                 goto out_err;
1023         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1024                 goto out_err;
1025 #else
1026         if (type)
1027                 goto out_err;
1028 #endif
1029
1030         rc = s390_enable_sie();
1031         if (rc)
1032                 goto out_err;
1033
1034         rc = -ENOMEM;
1035
1036         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1037         if (!kvm->arch.sca)
1038                 goto out_err;
1039         spin_lock(&kvm_lock);
1040         sca_offset = (sca_offset + 16) & 0x7f0;
1041         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1042         spin_unlock(&kvm_lock);
1043
1044         sprintf(debug_name, "kvm-%u", current->pid);
1045
1046         kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
1047         if (!kvm->arch.dbf)
1048                 goto out_err;
1049
1050         /*
1051          * The architectural maximum amount of facilities is 16 kbit. To store
1052          * this amount, 2 kbyte of memory is required. Thus we need a full
1053          * page to hold the guest facility list (arch.model.fac->list) and the
1054          * facility mask (arch.model.fac->mask). Its address size has to be
1055          * 31 bits and word aligned.
1056          */
1057         kvm->arch.model.fac =
1058                 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1059         if (!kvm->arch.model.fac)
1060                 goto out_err;
1061
1062         /* Populate the facility mask initially. */
1063         memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1064                S390_ARCH_FAC_LIST_SIZE_BYTE);
1065         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1066                 if (i < kvm_s390_fac_list_mask_size())
1067                         kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1068                 else
1069                         kvm->arch.model.fac->mask[i] = 0UL;
1070         }
1071
1072         /* Populate the facility list initially. */
1073         memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1074                S390_ARCH_FAC_LIST_SIZE_BYTE);
1075
1076         kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1077         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1078
1079         if (kvm_s390_crypto_init(kvm) < 0)
1080                 goto out_err;
1081
1082         spin_lock_init(&kvm->arch.float_int.lock);
1083         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1084                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1085         init_waitqueue_head(&kvm->arch.ipte_wq);
1086         mutex_init(&kvm->arch.ipte_mutex);
1087
1088         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1089         VM_EVENT(kvm, 3, "%s", "vm created");
1090
1091         if (type & KVM_VM_S390_UCONTROL) {
1092                 kvm->arch.gmap = NULL;
1093         } else {
1094                 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1095                 if (!kvm->arch.gmap)
1096                         goto out_err;
1097                 kvm->arch.gmap->private = kvm;
1098                 kvm->arch.gmap->pfault_enabled = 0;
1099         }
1100
1101         kvm->arch.css_support = 0;
1102         kvm->arch.use_irqchip = 0;
1103         kvm->arch.epoch = 0;
1104
1105         spin_lock_init(&kvm->arch.start_stop_lock);
1106
1107         return 0;
1108 out_err:
1109         kfree(kvm->arch.crypto.crycb);
1110         free_page((unsigned long)kvm->arch.model.fac);
1111         debug_unregister(kvm->arch.dbf);
1112         free_page((unsigned long)(kvm->arch.sca));
1113         return rc;
1114 }
1115
1116 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1117 {
1118         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1119         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1120         kvm_s390_clear_local_irqs(vcpu);
1121         kvm_clear_async_pf_completion_queue(vcpu);
1122         if (!kvm_is_ucontrol(vcpu->kvm)) {
1123                 clear_bit(63 - vcpu->vcpu_id,
1124                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1125                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1126                     (__u64) vcpu->arch.sie_block)
1127                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1128         }
1129         smp_mb();
1130
1131         if (kvm_is_ucontrol(vcpu->kvm))
1132                 gmap_free(vcpu->arch.gmap);
1133
1134         if (kvm_s390_cmma_enabled(vcpu->kvm))
1135                 kvm_s390_vcpu_unsetup_cmma(vcpu);
1136         free_page((unsigned long)(vcpu->arch.sie_block));
1137
1138         kvm_vcpu_uninit(vcpu);
1139         kmem_cache_free(kvm_vcpu_cache, vcpu);
1140 }
1141
1142 static void kvm_free_vcpus(struct kvm *kvm)
1143 {
1144         unsigned int i;
1145         struct kvm_vcpu *vcpu;
1146
1147         kvm_for_each_vcpu(i, vcpu, kvm)
1148                 kvm_arch_vcpu_destroy(vcpu);
1149
1150         mutex_lock(&kvm->lock);
1151         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1152                 kvm->vcpus[i] = NULL;
1153
1154         atomic_set(&kvm->online_vcpus, 0);
1155         mutex_unlock(&kvm->lock);
1156 }
1157
1158 void kvm_arch_destroy_vm(struct kvm *kvm)
1159 {
1160         kvm_free_vcpus(kvm);
1161         free_page((unsigned long)kvm->arch.model.fac);
1162         free_page((unsigned long)(kvm->arch.sca));
1163         debug_unregister(kvm->arch.dbf);
1164         kfree(kvm->arch.crypto.crycb);
1165         if (!kvm_is_ucontrol(kvm))
1166                 gmap_free(kvm->arch.gmap);
1167         kvm_s390_destroy_adapters(kvm);
1168         kvm_s390_clear_float_irqs(kvm);
1169 }
1170
1171 /* Section: vcpu related */
1172 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1173 {
1174         vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1175         if (!vcpu->arch.gmap)
1176                 return -ENOMEM;
1177         vcpu->arch.gmap->private = vcpu->kvm;
1178
1179         return 0;
1180 }
1181
1182 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1183 {
1184         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1185         kvm_clear_async_pf_completion_queue(vcpu);
1186         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1187                                     KVM_SYNC_GPRS |
1188                                     KVM_SYNC_ACRS |
1189                                     KVM_SYNC_CRS |
1190                                     KVM_SYNC_ARCH0 |
1191                                     KVM_SYNC_PFAULT;
1192         if (test_kvm_facility(vcpu->kvm, 129))
1193                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1194
1195         if (kvm_is_ucontrol(vcpu->kvm))
1196                 return __kvm_ucontrol_vcpu_init(vcpu);
1197
1198         return 0;
1199 }
1200
1201 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1202 {
1203         save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1204         if (test_kvm_facility(vcpu->kvm, 129))
1205                 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1206         else
1207                 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1208         save_access_regs(vcpu->arch.host_acrs);
1209         if (test_kvm_facility(vcpu->kvm, 129)) {
1210                 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1211                 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1212         } else {
1213                 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1214                 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1215         }
1216         restore_access_regs(vcpu->run->s.regs.acrs);
1217         gmap_enable(vcpu->arch.gmap);
1218         atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1219 }
1220
1221 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1222 {
1223         atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1224         gmap_disable(vcpu->arch.gmap);
1225         if (test_kvm_facility(vcpu->kvm, 129)) {
1226                 save_fp_ctl(&vcpu->run->s.regs.fpc);
1227                 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1228         } else {
1229                 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1230                 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1231         }
1232         save_access_regs(vcpu->run->s.regs.acrs);
1233         restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1234         if (test_kvm_facility(vcpu->kvm, 129))
1235                 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1236         else
1237                 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1238         restore_access_regs(vcpu->arch.host_acrs);
1239 }
1240
1241 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1242 {
1243         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1244         vcpu->arch.sie_block->gpsw.mask = 0UL;
1245         vcpu->arch.sie_block->gpsw.addr = 0UL;
1246         kvm_s390_set_prefix(vcpu, 0);
1247         vcpu->arch.sie_block->cputm     = 0UL;
1248         vcpu->arch.sie_block->ckc       = 0UL;
1249         vcpu->arch.sie_block->todpr     = 0;
1250         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1251         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1252         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1253         vcpu->arch.guest_fpregs.fpc = 0;
1254         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1255         vcpu->arch.sie_block->gbea = 1;
1256         vcpu->arch.sie_block->pp = 0;
1257         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1258         kvm_clear_async_pf_completion_queue(vcpu);
1259         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1260                 kvm_s390_vcpu_stop(vcpu);
1261         kvm_s390_clear_local_irqs(vcpu);
1262 }
1263
1264 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1265 {
1266         mutex_lock(&vcpu->kvm->lock);
1267         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1268         mutex_unlock(&vcpu->kvm->lock);
1269         if (!kvm_is_ucontrol(vcpu->kvm))
1270                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1271 }
1272
1273 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1274 {
1275         if (!test_kvm_facility(vcpu->kvm, 76))
1276                 return;
1277
1278         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1279
1280         if (vcpu->kvm->arch.crypto.aes_kw)
1281                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1282         if (vcpu->kvm->arch.crypto.dea_kw)
1283                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1284
1285         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1286 }
1287
1288 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1289 {
1290         free_page(vcpu->arch.sie_block->cbrlo);
1291         vcpu->arch.sie_block->cbrlo = 0;
1292 }
1293
1294 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1295 {
1296         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1297         if (!vcpu->arch.sie_block->cbrlo)
1298                 return -ENOMEM;
1299
1300         vcpu->arch.sie_block->ecb2 |= 0x80;
1301         vcpu->arch.sie_block->ecb2 &= ~0x08;
1302         return 0;
1303 }
1304
1305 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1306 {
1307         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1308
1309         vcpu->arch.cpu_id = model->cpu_id;
1310         vcpu->arch.sie_block->ibc = model->ibc;
1311         vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1312 }
1313
1314 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1315 {
1316         int rc = 0;
1317
1318         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1319                                                     CPUSTAT_SM |
1320                                                     CPUSTAT_STOPPED);
1321
1322         if (test_kvm_facility(vcpu->kvm, 78))
1323                 atomic_set_mask(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1324         else if (test_kvm_facility(vcpu->kvm, 8))
1325                 atomic_set_mask(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1326
1327         kvm_s390_vcpu_setup_model(vcpu);
1328
1329         vcpu->arch.sie_block->ecb   = 6;
1330         if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1331                 vcpu->arch.sie_block->ecb |= 0x10;
1332
1333         vcpu->arch.sie_block->ecb2  = 8;
1334         vcpu->arch.sie_block->eca   = 0xC1002000U;
1335         if (sclp.has_siif)
1336                 vcpu->arch.sie_block->eca |= 1;
1337         if (sclp.has_sigpif)
1338                 vcpu->arch.sie_block->eca |= 0x10000000U;
1339         if (test_kvm_facility(vcpu->kvm, 129)) {
1340                 vcpu->arch.sie_block->eca |= 0x00020000;
1341                 vcpu->arch.sie_block->ecd |= 0x20000000;
1342         }
1343         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1344
1345         if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1346                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1347                 if (rc)
1348                         return rc;
1349         }
1350         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1351         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1352
1353         kvm_s390_vcpu_crypto_setup(vcpu);
1354
1355         return rc;
1356 }
1357
1358 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1359                                       unsigned int id)
1360 {
1361         struct kvm_vcpu *vcpu;
1362         struct sie_page *sie_page;
1363         int rc = -EINVAL;
1364
1365         if (id >= KVM_MAX_VCPUS)
1366                 goto out;
1367
1368         rc = -ENOMEM;
1369
1370         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1371         if (!vcpu)
1372                 goto out;
1373
1374         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1375         if (!sie_page)
1376                 goto out_free_cpu;
1377
1378         vcpu->arch.sie_block = &sie_page->sie_block;
1379         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1380         vcpu->arch.host_vregs = &sie_page->vregs;
1381
1382         vcpu->arch.sie_block->icpua = id;
1383         if (!kvm_is_ucontrol(kvm)) {
1384                 if (!kvm->arch.sca) {
1385                         WARN_ON_ONCE(1);
1386                         goto out_free_cpu;
1387                 }
1388                 if (!kvm->arch.sca->cpu[id].sda)
1389                         kvm->arch.sca->cpu[id].sda =
1390                                 (__u64) vcpu->arch.sie_block;
1391                 vcpu->arch.sie_block->scaoh =
1392                         (__u32)(((__u64)kvm->arch.sca) >> 32);
1393                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1394                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1395         }
1396
1397         spin_lock_init(&vcpu->arch.local_int.lock);
1398         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1399         vcpu->arch.local_int.wq = &vcpu->wq;
1400         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1401
1402         rc = kvm_vcpu_init(vcpu, kvm, id);
1403         if (rc)
1404                 goto out_free_sie_block;
1405         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1406                  vcpu->arch.sie_block);
1407         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1408
1409         return vcpu;
1410 out_free_sie_block:
1411         free_page((unsigned long)(vcpu->arch.sie_block));
1412 out_free_cpu:
1413         kmem_cache_free(kvm_vcpu_cache, vcpu);
1414 out:
1415         return ERR_PTR(rc);
1416 }
1417
1418 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1419 {
1420         return kvm_s390_vcpu_has_irq(vcpu, 0);
1421 }
1422
1423 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1424 {
1425         atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1426         exit_sie(vcpu);
1427 }
1428
1429 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1430 {
1431         atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1432 }
1433
1434 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1435 {
1436         atomic_set_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1437         exit_sie(vcpu);
1438 }
1439
1440 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1441 {
1442         atomic_clear_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1443 }
1444
1445 /*
1446  * Kick a guest cpu out of SIE and wait until SIE is not running.
1447  * If the CPU is not running (e.g. waiting as idle) the function will
1448  * return immediately. */
1449 void exit_sie(struct kvm_vcpu *vcpu)
1450 {
1451         atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1452         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1453                 cpu_relax();
1454 }
1455
1456 /* Kick a guest cpu out of SIE to process a request synchronously */
1457 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1458 {
1459         kvm_make_request(req, vcpu);
1460         kvm_s390_vcpu_request(vcpu);
1461 }
1462
1463 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1464 {
1465         int i;
1466         struct kvm *kvm = gmap->private;
1467         struct kvm_vcpu *vcpu;
1468
1469         kvm_for_each_vcpu(i, vcpu, kvm) {
1470                 /* match against both prefix pages */
1471                 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1472                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1473                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1474                 }
1475         }
1476 }
1477
1478 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1479 {
1480         /* kvm common code refers to this, but never calls it */
1481         BUG();
1482         return 0;
1483 }
1484
1485 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1486                                            struct kvm_one_reg *reg)
1487 {
1488         int r = -EINVAL;
1489
1490         switch (reg->id) {
1491         case KVM_REG_S390_TODPR:
1492                 r = put_user(vcpu->arch.sie_block->todpr,
1493                              (u32 __user *)reg->addr);
1494                 break;
1495         case KVM_REG_S390_EPOCHDIFF:
1496                 r = put_user(vcpu->arch.sie_block->epoch,
1497                              (u64 __user *)reg->addr);
1498                 break;
1499         case KVM_REG_S390_CPU_TIMER:
1500                 r = put_user(vcpu->arch.sie_block->cputm,
1501                              (u64 __user *)reg->addr);
1502                 break;
1503         case KVM_REG_S390_CLOCK_COMP:
1504                 r = put_user(vcpu->arch.sie_block->ckc,
1505                              (u64 __user *)reg->addr);
1506                 break;
1507         case KVM_REG_S390_PFTOKEN:
1508                 r = put_user(vcpu->arch.pfault_token,
1509                              (u64 __user *)reg->addr);
1510                 break;
1511         case KVM_REG_S390_PFCOMPARE:
1512                 r = put_user(vcpu->arch.pfault_compare,
1513                              (u64 __user *)reg->addr);
1514                 break;
1515         case KVM_REG_S390_PFSELECT:
1516                 r = put_user(vcpu->arch.pfault_select,
1517                              (u64 __user *)reg->addr);
1518                 break;
1519         case KVM_REG_S390_PP:
1520                 r = put_user(vcpu->arch.sie_block->pp,
1521                              (u64 __user *)reg->addr);
1522                 break;
1523         case KVM_REG_S390_GBEA:
1524                 r = put_user(vcpu->arch.sie_block->gbea,
1525                              (u64 __user *)reg->addr);
1526                 break;
1527         default:
1528                 break;
1529         }
1530
1531         return r;
1532 }
1533
1534 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1535                                            struct kvm_one_reg *reg)
1536 {
1537         int r = -EINVAL;
1538
1539         switch (reg->id) {
1540         case KVM_REG_S390_TODPR:
1541                 r = get_user(vcpu->arch.sie_block->todpr,
1542                              (u32 __user *)reg->addr);
1543                 break;
1544         case KVM_REG_S390_EPOCHDIFF:
1545                 r = get_user(vcpu->arch.sie_block->epoch,
1546                              (u64 __user *)reg->addr);
1547                 break;
1548         case KVM_REG_S390_CPU_TIMER:
1549                 r = get_user(vcpu->arch.sie_block->cputm,
1550                              (u64 __user *)reg->addr);
1551                 break;
1552         case KVM_REG_S390_CLOCK_COMP:
1553                 r = get_user(vcpu->arch.sie_block->ckc,
1554                              (u64 __user *)reg->addr);
1555                 break;
1556         case KVM_REG_S390_PFTOKEN:
1557                 r = get_user(vcpu->arch.pfault_token,
1558                              (u64 __user *)reg->addr);
1559                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1560                         kvm_clear_async_pf_completion_queue(vcpu);
1561                 break;
1562         case KVM_REG_S390_PFCOMPARE:
1563                 r = get_user(vcpu->arch.pfault_compare,
1564                              (u64 __user *)reg->addr);
1565                 break;
1566         case KVM_REG_S390_PFSELECT:
1567                 r = get_user(vcpu->arch.pfault_select,
1568                              (u64 __user *)reg->addr);
1569                 break;
1570         case KVM_REG_S390_PP:
1571                 r = get_user(vcpu->arch.sie_block->pp,
1572                              (u64 __user *)reg->addr);
1573                 break;
1574         case KVM_REG_S390_GBEA:
1575                 r = get_user(vcpu->arch.sie_block->gbea,
1576                              (u64 __user *)reg->addr);
1577                 break;
1578         default:
1579                 break;
1580         }
1581
1582         return r;
1583 }
1584
1585 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1586 {
1587         kvm_s390_vcpu_initial_reset(vcpu);
1588         return 0;
1589 }
1590
1591 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1592 {
1593         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1594         return 0;
1595 }
1596
1597 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1598 {
1599         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1600         return 0;
1601 }
1602
1603 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1604                                   struct kvm_sregs *sregs)
1605 {
1606         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1607         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1608         restore_access_regs(vcpu->run->s.regs.acrs);
1609         return 0;
1610 }
1611
1612 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1613                                   struct kvm_sregs *sregs)
1614 {
1615         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1616         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1617         return 0;
1618 }
1619
1620 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1621 {
1622         if (test_fp_ctl(fpu->fpc))
1623                 return -EINVAL;
1624         memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1625         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1626         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1627         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1628         return 0;
1629 }
1630
1631 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1632 {
1633         memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1634         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1635         return 0;
1636 }
1637
1638 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1639 {
1640         int rc = 0;
1641
1642         if (!is_vcpu_stopped(vcpu))
1643                 rc = -EBUSY;
1644         else {
1645                 vcpu->run->psw_mask = psw.mask;
1646                 vcpu->run->psw_addr = psw.addr;
1647         }
1648         return rc;
1649 }
1650
1651 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1652                                   struct kvm_translation *tr)
1653 {
1654         return -EINVAL; /* not implemented yet */
1655 }
1656
1657 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1658                               KVM_GUESTDBG_USE_HW_BP | \
1659                               KVM_GUESTDBG_ENABLE)
1660
1661 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1662                                         struct kvm_guest_debug *dbg)
1663 {
1664         int rc = 0;
1665
1666         vcpu->guest_debug = 0;
1667         kvm_s390_clear_bp_data(vcpu);
1668
1669         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1670                 return -EINVAL;
1671
1672         if (dbg->control & KVM_GUESTDBG_ENABLE) {
1673                 vcpu->guest_debug = dbg->control;
1674                 /* enforce guest PER */
1675                 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1676
1677                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1678                         rc = kvm_s390_import_bp_data(vcpu, dbg);
1679         } else {
1680                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1681                 vcpu->arch.guestdbg.last_bp = 0;
1682         }
1683
1684         if (rc) {
1685                 vcpu->guest_debug = 0;
1686                 kvm_s390_clear_bp_data(vcpu);
1687                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1688         }
1689
1690         return rc;
1691 }
1692
1693 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1694                                     struct kvm_mp_state *mp_state)
1695 {
1696         /* CHECK_STOP and LOAD are not supported yet */
1697         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1698                                        KVM_MP_STATE_OPERATING;
1699 }
1700
1701 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1702                                     struct kvm_mp_state *mp_state)
1703 {
1704         int rc = 0;
1705
1706         /* user space knows about this interface - let it control the state */
1707         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1708
1709         switch (mp_state->mp_state) {
1710         case KVM_MP_STATE_STOPPED:
1711                 kvm_s390_vcpu_stop(vcpu);
1712                 break;
1713         case KVM_MP_STATE_OPERATING:
1714                 kvm_s390_vcpu_start(vcpu);
1715                 break;
1716         case KVM_MP_STATE_LOAD:
1717         case KVM_MP_STATE_CHECK_STOP:
1718                 /* fall through - CHECK_STOP and LOAD are not supported yet */
1719         default:
1720                 rc = -ENXIO;
1721         }
1722
1723         return rc;
1724 }
1725
1726 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1727 {
1728         if (!MACHINE_IS_LPAR)
1729                 return false;
1730         /* only enable for z10 and later */
1731         if (!MACHINE_HAS_EDAT1)
1732                 return false;
1733         if (!kvm->arch.use_cmma)
1734                 return false;
1735         return true;
1736 }
1737
1738 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1739 {
1740         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1741 }
1742
1743 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1744 {
1745         if (!vcpu->requests)
1746                 return 0;
1747 retry:
1748         kvm_s390_vcpu_request_handled(vcpu);
1749         /*
1750          * We use MMU_RELOAD just to re-arm the ipte notifier for the
1751          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1752          * This ensures that the ipte instruction for this request has
1753          * already finished. We might race against a second unmapper that
1754          * wants to set the blocking bit. Lets just retry the request loop.
1755          */
1756         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1757                 int rc;
1758                 rc = gmap_ipte_notify(vcpu->arch.gmap,
1759                                       kvm_s390_get_prefix(vcpu),
1760                                       PAGE_SIZE * 2);
1761                 if (rc)
1762                         return rc;
1763                 goto retry;
1764         }
1765
1766         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1767                 vcpu->arch.sie_block->ihcpu = 0xffff;
1768                 goto retry;
1769         }
1770
1771         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1772                 if (!ibs_enabled(vcpu)) {
1773                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1774                         atomic_set_mask(CPUSTAT_IBS,
1775                                         &vcpu->arch.sie_block->cpuflags);
1776                 }
1777                 goto retry;
1778         }
1779
1780         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1781                 if (ibs_enabled(vcpu)) {
1782                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1783                         atomic_clear_mask(CPUSTAT_IBS,
1784                                           &vcpu->arch.sie_block->cpuflags);
1785                 }
1786                 goto retry;
1787         }
1788
1789         /* nothing to do, just clear the request */
1790         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1791
1792         return 0;
1793 }
1794
1795 /**
1796  * kvm_arch_fault_in_page - fault-in guest page if necessary
1797  * @vcpu: The corresponding virtual cpu
1798  * @gpa: Guest physical address
1799  * @writable: Whether the page should be writable or not
1800  *
1801  * Make sure that a guest page has been faulted-in on the host.
1802  *
1803  * Return: Zero on success, negative error code otherwise.
1804  */
1805 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1806 {
1807         return gmap_fault(vcpu->arch.gmap, gpa,
1808                           writable ? FAULT_FLAG_WRITE : 0);
1809 }
1810
1811 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1812                                       unsigned long token)
1813 {
1814         struct kvm_s390_interrupt inti;
1815         struct kvm_s390_irq irq;
1816
1817         if (start_token) {
1818                 irq.u.ext.ext_params2 = token;
1819                 irq.type = KVM_S390_INT_PFAULT_INIT;
1820                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1821         } else {
1822                 inti.type = KVM_S390_INT_PFAULT_DONE;
1823                 inti.parm64 = token;
1824                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1825         }
1826 }
1827
1828 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1829                                      struct kvm_async_pf *work)
1830 {
1831         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1832         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1833 }
1834
1835 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1836                                  struct kvm_async_pf *work)
1837 {
1838         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1839         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1840 }
1841
1842 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1843                                struct kvm_async_pf *work)
1844 {
1845         /* s390 will always inject the page directly */
1846 }
1847
1848 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1849 {
1850         /*
1851          * s390 will always inject the page directly,
1852          * but we still want check_async_completion to cleanup
1853          */
1854         return true;
1855 }
1856
1857 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1858 {
1859         hva_t hva;
1860         struct kvm_arch_async_pf arch;
1861         int rc;
1862
1863         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1864                 return 0;
1865         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1866             vcpu->arch.pfault_compare)
1867                 return 0;
1868         if (psw_extint_disabled(vcpu))
1869                 return 0;
1870         if (kvm_s390_vcpu_has_irq(vcpu, 0))
1871                 return 0;
1872         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1873                 return 0;
1874         if (!vcpu->arch.gmap->pfault_enabled)
1875                 return 0;
1876
1877         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1878         hva += current->thread.gmap_addr & ~PAGE_MASK;
1879         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1880                 return 0;
1881
1882         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1883         return rc;
1884 }
1885
1886 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1887 {
1888         int rc, cpuflags;
1889
1890         /*
1891          * On s390 notifications for arriving pages will be delivered directly
1892          * to the guest but the house keeping for completed pfaults is
1893          * handled outside the worker.
1894          */
1895         kvm_check_async_pf_completion(vcpu);
1896
1897         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1898
1899         if (need_resched())
1900                 schedule();
1901
1902         if (test_cpu_flag(CIF_MCCK_PENDING))
1903                 s390_handle_mcck();
1904
1905         if (!kvm_is_ucontrol(vcpu->kvm)) {
1906                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1907                 if (rc)
1908                         return rc;
1909         }
1910
1911         rc = kvm_s390_handle_requests(vcpu);
1912         if (rc)
1913                 return rc;
1914
1915         if (guestdbg_enabled(vcpu)) {
1916                 kvm_s390_backup_guest_per_regs(vcpu);
1917                 kvm_s390_patch_guest_per_regs(vcpu);
1918         }
1919
1920         vcpu->arch.sie_block->icptcode = 0;
1921         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1922         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1923         trace_kvm_s390_sie_enter(vcpu, cpuflags);
1924
1925         return 0;
1926 }
1927
1928 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1929 {
1930         psw_t *psw = &vcpu->arch.sie_block->gpsw;
1931         u8 opcode;
1932         int rc;
1933
1934         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1935         trace_kvm_s390_sie_fault(vcpu);
1936
1937         /*
1938          * We want to inject an addressing exception, which is defined as a
1939          * suppressing or terminating exception. However, since we came here
1940          * by a DAT access exception, the PSW still points to the faulting
1941          * instruction since DAT exceptions are nullifying. So we've got
1942          * to look up the current opcode to get the length of the instruction
1943          * to be able to forward the PSW.
1944          */
1945         rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
1946         if (rc)
1947                 return kvm_s390_inject_prog_cond(vcpu, rc);
1948         psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1949
1950         return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1951 }
1952
1953 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1954 {
1955         int rc = -1;
1956
1957         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1958                    vcpu->arch.sie_block->icptcode);
1959         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1960
1961         if (guestdbg_enabled(vcpu))
1962                 kvm_s390_restore_guest_per_regs(vcpu);
1963
1964         if (exit_reason >= 0) {
1965                 rc = 0;
1966         } else if (kvm_is_ucontrol(vcpu->kvm)) {
1967                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1968                 vcpu->run->s390_ucontrol.trans_exc_code =
1969                                                 current->thread.gmap_addr;
1970                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1971                 rc = -EREMOTE;
1972
1973         } else if (current->thread.gmap_pfault) {
1974                 trace_kvm_s390_major_guest_pfault(vcpu);
1975                 current->thread.gmap_pfault = 0;
1976                 if (kvm_arch_setup_async_pf(vcpu)) {
1977                         rc = 0;
1978                 } else {
1979                         gpa_t gpa = current->thread.gmap_addr;
1980                         rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1981                 }
1982         }
1983
1984         if (rc == -1)
1985                 rc = vcpu_post_run_fault_in_sie(vcpu);
1986
1987         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1988
1989         if (rc == 0) {
1990                 if (kvm_is_ucontrol(vcpu->kvm))
1991                         /* Don't exit for host interrupts. */
1992                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1993                 else
1994                         rc = kvm_handle_sie_intercept(vcpu);
1995         }
1996
1997         return rc;
1998 }
1999
2000 static int __vcpu_run(struct kvm_vcpu *vcpu)
2001 {
2002         int rc, exit_reason;
2003
2004         /*
2005          * We try to hold kvm->srcu during most of vcpu_run (except when run-
2006          * ning the guest), so that memslots (and other stuff) are protected
2007          */
2008         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2009
2010         do {
2011                 rc = vcpu_pre_run(vcpu);
2012                 if (rc)
2013                         break;
2014
2015                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2016                 /*
2017                  * As PF_VCPU will be used in fault handler, between
2018                  * guest_enter and guest_exit should be no uaccess.
2019                  */
2020                 local_irq_disable();
2021                 __kvm_guest_enter();
2022                 local_irq_enable();
2023                 exit_reason = sie64a(vcpu->arch.sie_block,
2024                                      vcpu->run->s.regs.gprs);
2025                 local_irq_disable();
2026                 __kvm_guest_exit();
2027                 local_irq_enable();
2028                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2029
2030                 rc = vcpu_post_run(vcpu, exit_reason);
2031         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2032
2033         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2034         return rc;
2035 }
2036
2037 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2038 {
2039         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2040         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2041         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2042                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2043         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2044                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2045                 /* some control register changes require a tlb flush */
2046                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2047         }
2048         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2049                 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2050                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2051                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2052                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2053                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2054         }
2055         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2056                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2057                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2058                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2059                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2060                         kvm_clear_async_pf_completion_queue(vcpu);
2061         }
2062         kvm_run->kvm_dirty_regs = 0;
2063 }
2064
2065 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2066 {
2067         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2068         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2069         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2070         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2071         kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2072         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2073         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2074         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2075         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2076         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2077         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2078         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2079 }
2080
2081 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2082 {
2083         int rc;
2084         sigset_t sigsaved;
2085
2086         if (guestdbg_exit_pending(vcpu)) {
2087                 kvm_s390_prepare_debug_exit(vcpu);
2088                 return 0;
2089         }
2090
2091         if (vcpu->sigset_active)
2092                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2093
2094         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2095                 kvm_s390_vcpu_start(vcpu);
2096         } else if (is_vcpu_stopped(vcpu)) {
2097                 pr_err_ratelimited("can't run stopped vcpu %d\n",
2098                                    vcpu->vcpu_id);
2099                 return -EINVAL;
2100         }
2101
2102         sync_regs(vcpu, kvm_run);
2103
2104         might_fault();
2105         rc = __vcpu_run(vcpu);
2106
2107         if (signal_pending(current) && !rc) {
2108                 kvm_run->exit_reason = KVM_EXIT_INTR;
2109                 rc = -EINTR;
2110         }
2111
2112         if (guestdbg_exit_pending(vcpu) && !rc)  {
2113                 kvm_s390_prepare_debug_exit(vcpu);
2114                 rc = 0;
2115         }
2116
2117         if (rc == -EOPNOTSUPP) {
2118                 /* intercept cannot be handled in-kernel, prepare kvm-run */
2119                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
2120                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2121                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
2122                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
2123                 rc = 0;
2124         }
2125
2126         if (rc == -EREMOTE) {
2127                 /* intercept was handled, but userspace support is needed
2128                  * kvm_run has been prepared by the handler */
2129                 rc = 0;
2130         }
2131
2132         store_regs(vcpu, kvm_run);
2133
2134         if (vcpu->sigset_active)
2135                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2136
2137         vcpu->stat.exit_userspace++;
2138         return rc;
2139 }
2140
2141 /*
2142  * store status at address
2143  * we use have two special cases:
2144  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2145  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2146  */
2147 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2148 {
2149         unsigned char archmode = 1;
2150         unsigned int px;
2151         u64 clkcomp;
2152         int rc;
2153
2154         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2155                 if (write_guest_abs(vcpu, 163, &archmode, 1))
2156                         return -EFAULT;
2157                 gpa = SAVE_AREA_BASE;
2158         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2159                 if (write_guest_real(vcpu, 163, &archmode, 1))
2160                         return -EFAULT;
2161                 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2162         }
2163         rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2164                              vcpu->arch.guest_fpregs.fprs, 128);
2165         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2166                               vcpu->run->s.regs.gprs, 128);
2167         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2168                               &vcpu->arch.sie_block->gpsw, 16);
2169         px = kvm_s390_get_prefix(vcpu);
2170         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2171                               &px, 4);
2172         rc |= write_guest_abs(vcpu,
2173                               gpa + offsetof(struct save_area, fp_ctrl_reg),
2174                               &vcpu->arch.guest_fpregs.fpc, 4);
2175         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2176                               &vcpu->arch.sie_block->todpr, 4);
2177         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2178                               &vcpu->arch.sie_block->cputm, 8);
2179         clkcomp = vcpu->arch.sie_block->ckc >> 8;
2180         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2181                               &clkcomp, 8);
2182         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2183                               &vcpu->run->s.regs.acrs, 64);
2184         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2185                               &vcpu->arch.sie_block->gcr, 128);
2186         return rc ? -EFAULT : 0;
2187 }
2188
2189 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2190 {
2191         /*
2192          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2193          * copying in vcpu load/put. Lets update our copies before we save
2194          * it into the save area
2195          */
2196         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
2197         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
2198         save_access_regs(vcpu->run->s.regs.acrs);
2199
2200         return kvm_s390_store_status_unloaded(vcpu, addr);
2201 }
2202
2203 /*
2204  * store additional status at address
2205  */
2206 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2207                                         unsigned long gpa)
2208 {
2209         /* Only bits 0-53 are used for address formation */
2210         if (!(gpa & ~0x3ff))
2211                 return 0;
2212
2213         return write_guest_abs(vcpu, gpa & ~0x3ff,
2214                                (void *)&vcpu->run->s.regs.vrs, 512);
2215 }
2216
2217 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2218 {
2219         if (!test_kvm_facility(vcpu->kvm, 129))
2220                 return 0;
2221
2222         /*
2223          * The guest VXRS are in the host VXRs due to the lazy
2224          * copying in vcpu load/put. Let's update our copies before we save
2225          * it into the save area.
2226          */
2227         save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2228
2229         return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2230 }
2231
2232 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2233 {
2234         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2235         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2236 }
2237
2238 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2239 {
2240         unsigned int i;
2241         struct kvm_vcpu *vcpu;
2242
2243         kvm_for_each_vcpu(i, vcpu, kvm) {
2244                 __disable_ibs_on_vcpu(vcpu);
2245         }
2246 }
2247
2248 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2249 {
2250         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2251         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2252 }
2253
2254 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2255 {
2256         int i, online_vcpus, started_vcpus = 0;
2257
2258         if (!is_vcpu_stopped(vcpu))
2259                 return;
2260
2261         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2262         /* Only one cpu at a time may enter/leave the STOPPED state. */
2263         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2264         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2265
2266         for (i = 0; i < online_vcpus; i++) {
2267                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2268                         started_vcpus++;
2269         }
2270
2271         if (started_vcpus == 0) {
2272                 /* we're the only active VCPU -> speed it up */
2273                 __enable_ibs_on_vcpu(vcpu);
2274         } else if (started_vcpus == 1) {
2275                 /*
2276                  * As we are starting a second VCPU, we have to disable
2277                  * the IBS facility on all VCPUs to remove potentially
2278                  * oustanding ENABLE requests.
2279                  */
2280                 __disable_ibs_on_all_vcpus(vcpu->kvm);
2281         }
2282
2283         atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2284         /*
2285          * Another VCPU might have used IBS while we were offline.
2286          * Let's play safe and flush the VCPU at startup.
2287          */
2288         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2289         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2290         return;
2291 }
2292
2293 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2294 {
2295         int i, online_vcpus, started_vcpus = 0;
2296         struct kvm_vcpu *started_vcpu = NULL;
2297
2298         if (is_vcpu_stopped(vcpu))
2299                 return;
2300
2301         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2302         /* Only one cpu at a time may enter/leave the STOPPED state. */
2303         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2304         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2305
2306         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2307         kvm_s390_clear_stop_irq(vcpu);
2308
2309         atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2310         __disable_ibs_on_vcpu(vcpu);
2311
2312         for (i = 0; i < online_vcpus; i++) {
2313                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2314                         started_vcpus++;
2315                         started_vcpu = vcpu->kvm->vcpus[i];
2316                 }
2317         }
2318
2319         if (started_vcpus == 1) {
2320                 /*
2321                  * As we only have one VCPU left, we want to enable the
2322                  * IBS facility for that VCPU to speed it up.
2323                  */
2324                 __enable_ibs_on_vcpu(started_vcpu);
2325         }
2326
2327         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2328         return;
2329 }
2330
2331 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2332                                      struct kvm_enable_cap *cap)
2333 {
2334         int r;
2335
2336         if (cap->flags)
2337                 return -EINVAL;
2338
2339         switch (cap->cap) {
2340         case KVM_CAP_S390_CSS_SUPPORT:
2341                 if (!vcpu->kvm->arch.css_support) {
2342                         vcpu->kvm->arch.css_support = 1;
2343                         trace_kvm_s390_enable_css(vcpu->kvm);
2344                 }
2345                 r = 0;
2346                 break;
2347         default:
2348                 r = -EINVAL;
2349                 break;
2350         }
2351         return r;
2352 }
2353
2354 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2355                                   struct kvm_s390_mem_op *mop)
2356 {
2357         void __user *uaddr = (void __user *)mop->buf;
2358         void *tmpbuf = NULL;
2359         int r, srcu_idx;
2360         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2361                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
2362
2363         if (mop->flags & ~supported_flags)
2364                 return -EINVAL;
2365
2366         if (mop->size > MEM_OP_MAX_SIZE)
2367                 return -E2BIG;
2368
2369         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2370                 tmpbuf = vmalloc(mop->size);
2371                 if (!tmpbuf)
2372                         return -ENOMEM;
2373         }
2374
2375         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2376
2377         switch (mop->op) {
2378         case KVM_S390_MEMOP_LOGICAL_READ:
2379                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2380                         r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2381                         break;
2382                 }
2383                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2384                 if (r == 0) {
2385                         if (copy_to_user(uaddr, tmpbuf, mop->size))
2386                                 r = -EFAULT;
2387                 }
2388                 break;
2389         case KVM_S390_MEMOP_LOGICAL_WRITE:
2390                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2391                         r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2392                         break;
2393                 }
2394                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2395                         r = -EFAULT;
2396                         break;
2397                 }
2398                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2399                 break;
2400         default:
2401                 r = -EINVAL;
2402         }
2403
2404         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2405
2406         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2407                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2408
2409         vfree(tmpbuf);
2410         return r;
2411 }
2412
2413 long kvm_arch_vcpu_ioctl(struct file *filp,
2414                          unsigned int ioctl, unsigned long arg)
2415 {
2416         struct kvm_vcpu *vcpu = filp->private_data;
2417         void __user *argp = (void __user *)arg;
2418         int idx;
2419         long r;
2420
2421         switch (ioctl) {
2422         case KVM_S390_IRQ: {
2423                 struct kvm_s390_irq s390irq;
2424
2425                 r = -EFAULT;
2426                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2427                         break;
2428                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2429                 break;
2430         }
2431         case KVM_S390_INTERRUPT: {
2432                 struct kvm_s390_interrupt s390int;
2433                 struct kvm_s390_irq s390irq;
2434
2435                 r = -EFAULT;
2436                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2437                         break;
2438                 if (s390int_to_s390irq(&s390int, &s390irq))
2439                         return -EINVAL;
2440                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2441                 break;
2442         }
2443         case KVM_S390_STORE_STATUS:
2444                 idx = srcu_read_lock(&vcpu->kvm->srcu);
2445                 r = kvm_s390_vcpu_store_status(vcpu, arg);
2446                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2447                 break;
2448         case KVM_S390_SET_INITIAL_PSW: {
2449                 psw_t psw;
2450
2451                 r = -EFAULT;
2452                 if (copy_from_user(&psw, argp, sizeof(psw)))
2453                         break;
2454                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2455                 break;
2456         }
2457         case KVM_S390_INITIAL_RESET:
2458                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2459                 break;
2460         case KVM_SET_ONE_REG:
2461         case KVM_GET_ONE_REG: {
2462                 struct kvm_one_reg reg;
2463                 r = -EFAULT;
2464                 if (copy_from_user(&reg, argp, sizeof(reg)))
2465                         break;
2466                 if (ioctl == KVM_SET_ONE_REG)
2467                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2468                 else
2469                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2470                 break;
2471         }
2472 #ifdef CONFIG_KVM_S390_UCONTROL
2473         case KVM_S390_UCAS_MAP: {
2474                 struct kvm_s390_ucas_mapping ucasmap;
2475
2476                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2477                         r = -EFAULT;
2478                         break;
2479                 }
2480
2481                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2482                         r = -EINVAL;
2483                         break;
2484                 }
2485
2486                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2487                                      ucasmap.vcpu_addr, ucasmap.length);
2488                 break;
2489         }
2490         case KVM_S390_UCAS_UNMAP: {
2491                 struct kvm_s390_ucas_mapping ucasmap;
2492
2493                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2494                         r = -EFAULT;
2495                         break;
2496                 }
2497
2498                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2499                         r = -EINVAL;
2500                         break;
2501                 }
2502
2503                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2504                         ucasmap.length);
2505                 break;
2506         }
2507 #endif
2508         case KVM_S390_VCPU_FAULT: {
2509                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2510                 break;
2511         }
2512         case KVM_ENABLE_CAP:
2513         {
2514                 struct kvm_enable_cap cap;
2515                 r = -EFAULT;
2516                 if (copy_from_user(&cap, argp, sizeof(cap)))
2517                         break;
2518                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2519                 break;
2520         }
2521         case KVM_S390_MEM_OP: {
2522                 struct kvm_s390_mem_op mem_op;
2523
2524                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2525                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2526                 else
2527                         r = -EFAULT;
2528                 break;
2529         }
2530         case KVM_S390_SET_IRQ_STATE: {
2531                 struct kvm_s390_irq_state irq_state;
2532
2533                 r = -EFAULT;
2534                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2535                         break;
2536                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2537                     irq_state.len == 0 ||
2538                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2539                         r = -EINVAL;
2540                         break;
2541                 }
2542                 r = kvm_s390_set_irq_state(vcpu,
2543                                            (void __user *) irq_state.buf,
2544                                            irq_state.len);
2545                 break;
2546         }
2547         case KVM_S390_GET_IRQ_STATE: {
2548                 struct kvm_s390_irq_state irq_state;
2549
2550                 r = -EFAULT;
2551                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2552                         break;
2553                 if (irq_state.len == 0) {
2554                         r = -EINVAL;
2555                         break;
2556                 }
2557                 r = kvm_s390_get_irq_state(vcpu,
2558                                            (__u8 __user *)  irq_state.buf,
2559                                            irq_state.len);
2560                 break;
2561         }
2562         default:
2563                 r = -ENOTTY;
2564         }
2565         return r;
2566 }
2567
2568 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2569 {
2570 #ifdef CONFIG_KVM_S390_UCONTROL
2571         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2572                  && (kvm_is_ucontrol(vcpu->kvm))) {
2573                 vmf->page = virt_to_page(vcpu->arch.sie_block);
2574                 get_page(vmf->page);
2575                 return 0;
2576         }
2577 #endif
2578         return VM_FAULT_SIGBUS;
2579 }
2580
2581 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2582                             unsigned long npages)
2583 {
2584         return 0;
2585 }
2586
2587 /* Section: memory related */
2588 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2589                                    struct kvm_memory_slot *memslot,
2590                                    const struct kvm_userspace_memory_region *mem,
2591                                    enum kvm_mr_change change)
2592 {
2593         /* A few sanity checks. We can have memory slots which have to be
2594            located/ended at a segment boundary (1MB). The memory in userland is
2595            ok to be fragmented into various different vmas. It is okay to mmap()
2596            and munmap() stuff in this slot after doing this call at any time */
2597
2598         if (mem->userspace_addr & 0xffffful)
2599                 return -EINVAL;
2600
2601         if (mem->memory_size & 0xffffful)
2602                 return -EINVAL;
2603
2604         return 0;
2605 }
2606
2607 void kvm_arch_commit_memory_region(struct kvm *kvm,
2608                                 const struct kvm_userspace_memory_region *mem,
2609                                 const struct kvm_memory_slot *old,
2610                                 const struct kvm_memory_slot *new,
2611                                 enum kvm_mr_change change)
2612 {
2613         int rc;
2614
2615         /* If the basics of the memslot do not change, we do not want
2616          * to update the gmap. Every update causes several unnecessary
2617          * segment translation exceptions. This is usually handled just
2618          * fine by the normal fault handler + gmap, but it will also
2619          * cause faults on the prefix page of running guest CPUs.
2620          */
2621         if (old->userspace_addr == mem->userspace_addr &&
2622             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2623             old->npages * PAGE_SIZE == mem->memory_size)
2624                 return;
2625
2626         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2627                 mem->guest_phys_addr, mem->memory_size);
2628         if (rc)
2629                 pr_warn("failed to commit memory region\n");
2630         return;
2631 }
2632
2633 static int __init kvm_s390_init(void)
2634 {
2635         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2636 }
2637
2638 static void __exit kvm_s390_exit(void)
2639 {
2640         kvm_exit();
2641 }
2642
2643 module_init(kvm_s390_init);
2644 module_exit(kvm_s390_exit);
2645
2646 /*
2647  * Enable autoloading of the kvm module.
2648  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2649  * since x86 takes a different approach.
2650  */
2651 #include <linux/miscdevice.h>
2652 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2653 MODULE_ALIAS("devname:kvm");