arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/random.h>
  26 #include <linux/slab.h>
  27 #include <linux/timer.h>
  28 #include <linux/vmalloc.h>
  29 #include <asm/asm-offsets.h>
  30 #include <asm/lowcore.h>
  31 #include <asm/etr.h>
  32 #include <asm/pgtable.h>
  33 #include <asm/nmi.h>
  34 #include <asm/switch_to.h>
  35 #include <asm/isc.h>
  36 #include <asm/sclp.h>
  37 #include "kvm-s390.h"
  38 #include "gaccess.h"
  39
  40 #define KMSG_COMPONENT "kvm-s390"
  41 #undef pr_fmt
  42 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  43
  44 #define CREATE_TRACE_POINTS
  45 #include "trace.h"
  46 #include "trace-s390.h"
  47
  48 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  49 #define LOCAL_IRQS 32
  50 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  51                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  52
  53 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  54
  55 struct kvm_stats_debugfs_item debugfs_entries[] = {
  56         { "userspace_handled", VCPU_STAT(exit_userspace) },
  57         { "exit_null", VCPU_STAT(exit_null) },
  58         { "exit_validity", VCPU_STAT(exit_validity) },
  59         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  60         { "exit_external_request", VCPU_STAT(exit_external_request) },
  61         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  62         { "exit_instruction", VCPU_STAT(exit_instruction) },
  63         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  64         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  65         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  66         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  67         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  68         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  69         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  70         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  71         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  72         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  73         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  74         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  75         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  76         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  77         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  78         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  79         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  80         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  81         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  82         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  83         { "instruction_spx", VCPU_STAT(instruction_spx) },
  84         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  85         { "instruction_stap", VCPU_STAT(instruction_stap) },
  86         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  87         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  88         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  89         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  90         { "instruction_essa", VCPU_STAT(instruction_essa) },
  91         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  92         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  93         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  94         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  95         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  96         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  97         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  98         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
  99         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 100         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 101         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 102         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 103         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 104         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 105         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 106         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 107         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 108         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 109         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 110         { "diagnose_10", VCPU_STAT(diagnose_10) },
 111         { "diagnose_44", VCPU_STAT(diagnose_44) },
 112         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 113         { "diagnose_258", VCPU_STAT(diagnose_258) },
 114         { "diagnose_308", VCPU_STAT(diagnose_308) },
 115         { "diagnose_500", VCPU_STAT(diagnose_500) },
 116         { NULL }
 117 };
 118
 119 /* upper facilities limit for kvm */
 120 unsigned long kvm_s390_fac_list_mask[] = {
 121         0xffe6fffbfcfdfc40UL,
 122         0x005e800000000000UL,
 123 };
 124
 125 unsigned long kvm_s390_fac_list_mask_size(void)
 126 {
 127         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 128         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 129 }
 130
 131 static struct gmap_notifier gmap_notifier;
 132 debug_info_t *kvm_s390_dbf;
 133
 134 /* Section: not file related */
 135 int kvm_arch_hardware_enable(void)
 136 {
 137         /* every s390 is virtualization enabled ;-) */
 138         return 0;
 139 }
 140
 141 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 142
 143 /*
 144  * This callback is executed during stop_machine(). All CPUs are therefore
 145  * temporarily stopped. In order not to change guest behavior, we have to
 146  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 147  * so a CPU won't be stopped while calculating with the epoch.
 148  */
 149 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 150                           void *v)
 151 {
 152         struct kvm *kvm;
 153         struct kvm_vcpu *vcpu;
 154         int i;
 155         unsigned long long *delta = v;
 156
 157         list_for_each_entry(kvm, &vm_list, vm_list) {
 158                 kvm->arch.epoch -= *delta;
 159                 kvm_for_each_vcpu(i, vcpu, kvm) {
 160                         vcpu->arch.sie_block->epoch -= *delta;
 161                 }
 162         }
 163         return NOTIFY_OK;
 164 }
 165
 166 static struct notifier_block kvm_clock_notifier = {
 167         .notifier_call = kvm_clock_sync,
 168 };
 169
 170 int kvm_arch_hardware_setup(void)
 171 {
 172         gmap_notifier.notifier_call = kvm_gmap_notifier;
 173         gmap_register_ipte_notifier(&gmap_notifier);
 174         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 175                                        &kvm_clock_notifier);
 176         return 0;
 177 }
 178
 179 void kvm_arch_hardware_unsetup(void)
 180 {
 181         gmap_unregister_ipte_notifier(&gmap_notifier);
 182         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 183                                          &kvm_clock_notifier);
 184 }
 185
 186 int kvm_arch_init(void *opaque)
 187 {
 188         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 189         if (!kvm_s390_dbf)
 190                 return -ENOMEM;
 191
 192         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 193                 debug_unregister(kvm_s390_dbf);
 194                 return -ENOMEM;
 195         }
 196
 197         /* Register floating interrupt controller interface. */
 198         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 199 }
 200
 201 void kvm_arch_exit(void)
 202 {
 203         debug_unregister(kvm_s390_dbf);
 204 }
 205
 206 /* Section: device related */
 207 long kvm_arch_dev_ioctl(struct file *filp,
 208                         unsigned int ioctl, unsigned long arg)
 209 {
 210         if (ioctl == KVM_S390_ENABLE_SIE)
 211                 return s390_enable_sie();
 212         return -EINVAL;
 213 }
 214
 215 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 216 {
 217         int r;
 218
 219         switch (ext) {
 220         case KVM_CAP_S390_PSW:
 221         case KVM_CAP_S390_GMAP:
 222         case KVM_CAP_SYNC_MMU:
 223 #ifdef CONFIG_KVM_S390_UCONTROL
 224         case KVM_CAP_S390_UCONTROL:
 225 #endif
 226         case KVM_CAP_ASYNC_PF:
 227         case KVM_CAP_SYNC_REGS:
 228         case KVM_CAP_ONE_REG:
 229         case KVM_CAP_ENABLE_CAP:
 230         case KVM_CAP_S390_CSS_SUPPORT:
 231         case KVM_CAP_IOEVENTFD:
 232         case KVM_CAP_DEVICE_CTRL:
 233         case KVM_CAP_ENABLE_CAP_VM:
 234         case KVM_CAP_S390_IRQCHIP:
 235         case KVM_CAP_VM_ATTRIBUTES:
 236         case KVM_CAP_MP_STATE:
 237         case KVM_CAP_S390_INJECT_IRQ:
 238         case KVM_CAP_S390_USER_SIGP:
 239         case KVM_CAP_S390_USER_STSI:
 240         case KVM_CAP_S390_SKEYS:
 241         case KVM_CAP_S390_IRQ_STATE:
 242                 r = 1;
 243                 break;
 244         case KVM_CAP_S390_MEM_OP:
 245                 r = MEM_OP_MAX_SIZE;
 246                 break;
 247         case KVM_CAP_NR_VCPUS:
 248         case KVM_CAP_MAX_VCPUS:
 249                 r = sclp.has_esca ? KVM_S390_ESCA_CPU_SLOTS
 250                                   : KVM_S390_BSCA_CPU_SLOTS;
 251                 break;
 252         case KVM_CAP_NR_MEMSLOTS:
 253                 r = KVM_USER_MEM_SLOTS;
 254                 break;
 255         case KVM_CAP_S390_COW:
 256                 r = MACHINE_HAS_ESOP;
 257                 break;
 258         case KVM_CAP_S390_VECTOR_REGISTERS:
 259                 r = MACHINE_HAS_VX;
 260                 break;
 261         case KVM_CAP_S390_RI:
 262                 r = test_facility(64);
 263                 break;
 264         default:
 265                 r = 0;
 266         }
 267         return r;
 268 }
 269
 270 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 271                                         struct kvm_memory_slot *memslot)
 272 {
 273         gfn_t cur_gfn, last_gfn;
 274         unsigned long address;
 275         struct gmap *gmap = kvm->arch.gmap;
 276
 277         down_read(&gmap->mm->mmap_sem);
 278         /* Loop over all guest pages */
 279         last_gfn = memslot->base_gfn + memslot->npages;
 280         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 281                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 282
 283                 if (gmap_test_and_clear_dirty(address, gmap))
 284                         mark_page_dirty(kvm, cur_gfn);
 285         }
 286         up_read(&gmap->mm->mmap_sem);
 287 }
 288
 289 /* Section: vm related */
 290 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 291
 292 /*
 293  * Get (and clear) the dirty memory log for a memory slot.
 294  */
 295 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 296                                struct kvm_dirty_log *log)
 297 {
 298         int r;
 299         unsigned long n;
 300         struct kvm_memslots *slots;
 301         struct kvm_memory_slot *memslot;
 302         int is_dirty = 0;
 303
 304         mutex_lock(&kvm->slots_lock);
 305
 306         r = -EINVAL;
 307         if (log->slot >= KVM_USER_MEM_SLOTS)
 308                 goto out;
 309
 310         slots = kvm_memslots(kvm);
 311         memslot = id_to_memslot(slots, log->slot);
 312         r = -ENOENT;
 313         if (!memslot->dirty_bitmap)
 314                 goto out;
 315
 316         kvm_s390_sync_dirty_log(kvm, memslot);
 317         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 318         if (r)
 319                 goto out;
 320
 321         /* Clear the dirty log */
 322         if (is_dirty) {
 323                 n = kvm_dirty_bitmap_bytes(memslot);
 324                 memset(memslot->dirty_bitmap, 0, n);
 325         }
 326         r = 0;
 327 out:
 328         mutex_unlock(&kvm->slots_lock);
 329         return r;
 330 }
 331
 332 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 333 {
 334         int r;
 335
 336         if (cap->flags)
 337                 return -EINVAL;
 338
 339         switch (cap->cap) {
 340         case KVM_CAP_S390_IRQCHIP:
 341                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 342                 kvm->arch.use_irqchip = 1;
 343                 r = 0;
 344                 break;
 345         case KVM_CAP_S390_USER_SIGP:
 346                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 347                 kvm->arch.user_sigp = 1;
 348                 r = 0;
 349                 break;
 350         case KVM_CAP_S390_VECTOR_REGISTERS:
 351                 mutex_lock(&kvm->lock);
 352                 if (atomic_read(&kvm->online_vcpus)) {
 353                         r = -EBUSY;
 354                 } else if (MACHINE_HAS_VX) {
 355                         set_kvm_facility(kvm->arch.model.fac->mask, 129);
 356                         set_kvm_facility(kvm->arch.model.fac->list, 129);
 357                         r = 0;
 358                 } else
 359                         r = -EINVAL;
 360                 mutex_unlock(&kvm->lock);
 361                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 362                          r ? "(not available)" : "(success)");
 363                 break;
 364         case KVM_CAP_S390_RI:
 365                 r = -EINVAL;
 366                 mutex_lock(&kvm->lock);
 367                 if (atomic_read(&kvm->online_vcpus)) {
 368                         r = -EBUSY;
 369                 } else if (test_facility(64)) {
 370                         set_kvm_facility(kvm->arch.model.fac->mask, 64);
 371                         set_kvm_facility(kvm->arch.model.fac->list, 64);
 372                         r = 0;
 373                 }
 374                 mutex_unlock(&kvm->lock);
 375                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 376                          r ? "(not available)" : "(success)");
 377                 break;
 378         case KVM_CAP_S390_USER_STSI:
 379                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 380                 kvm->arch.user_stsi = 1;
 381                 r = 0;
 382                 break;
 383         default:
 384                 r = -EINVAL;
 385                 break;
 386         }
 387         return r;
 388 }
 389
 390 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 391 {
 392         int ret;
 393
 394         switch (attr->attr) {
 395         case KVM_S390_VM_MEM_LIMIT_SIZE:
 396                 ret = 0;
 397                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 398                          kvm->arch.mem_limit);
 399                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 400                         ret = -EFAULT;
 401                 break;
 402         default:
 403                 ret = -ENXIO;
 404                 break;
 405         }
 406         return ret;
 407 }
 408
 409 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 410 {
 411         int ret;
 412         unsigned int idx;
 413         switch (attr->attr) {
 414         case KVM_S390_VM_MEM_ENABLE_CMMA:
 415                 /* enable CMMA only for z10 and later (EDAT_1) */
 416                 ret = -EINVAL;
 417                 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
 418                         break;
 419
 420                 ret = -EBUSY;
 421                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 422                 mutex_lock(&kvm->lock);
 423                 if (atomic_read(&kvm->online_vcpus) == 0) {
 424                         kvm->arch.use_cmma = 1;
 425                         ret = 0;
 426                 }
 427                 mutex_unlock(&kvm->lock);
 428                 break;
 429         case KVM_S390_VM_MEM_CLR_CMMA:
 430                 ret = -EINVAL;
 431                 if (!kvm->arch.use_cmma)
 432                         break;
 433
 434                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 435                 mutex_lock(&kvm->lock);
 436                 idx = srcu_read_lock(&kvm->srcu);
 437                 s390_reset_cmma(kvm->arch.gmap->mm);
 438                 srcu_read_unlock(&kvm->srcu, idx);
 439                 mutex_unlock(&kvm->lock);
 440                 ret = 0;
 441                 break;
 442         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 443                 unsigned long new_limit;
 444
 445                 if (kvm_is_ucontrol(kvm))
 446                         return -EINVAL;
 447
 448                 if (get_user(new_limit, (u64 __user *)attr->addr))
 449                         return -EFAULT;
 450
 451                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 452                     new_limit > kvm->arch.mem_limit)
 453                         return -E2BIG;
 454
 455                 if (!new_limit)
 456                         return -EINVAL;
 457
 458                 /* gmap_alloc takes last usable address */
 459                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 460                         new_limit -= 1;
 461
 462                 ret = -EBUSY;
 463                 mutex_lock(&kvm->lock);
 464                 if (atomic_read(&kvm->online_vcpus) == 0) {
 465                         /* gmap_alloc will round the limit up */
 466                         struct gmap *new = gmap_alloc(current->mm, new_limit);
 467
 468                         if (!new) {
 469                                 ret = -ENOMEM;
 470                         } else {
 471                                 gmap_free(kvm->arch.gmap);
 472                                 new->private = kvm;
 473                                 kvm->arch.gmap = new;
 474                                 ret = 0;
 475                         }
 476                 }
 477                 mutex_unlock(&kvm->lock);
 478                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 479                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 480                          (void *) kvm->arch.gmap->asce);
 481                 break;
 482         }
 483         default:
 484                 ret = -ENXIO;
 485                 break;
 486         }
 487         return ret;
 488 }
 489
 490 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 491
 492 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 493 {
 494         struct kvm_vcpu *vcpu;
 495         int i;
 496
 497         if (!test_kvm_facility(kvm, 76))
 498                 return -EINVAL;
 499
 500         mutex_lock(&kvm->lock);
 501         switch (attr->attr) {
 502         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 503                 get_random_bytes(
 504                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 505                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 506                 kvm->arch.crypto.aes_kw = 1;
 507                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 508                 break;
 509         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 510                 get_random_bytes(
 511                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 512                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 513                 kvm->arch.crypto.dea_kw = 1;
 514                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 515                 break;
 516         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 517                 kvm->arch.crypto.aes_kw = 0;
 518                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 519                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 520                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 521                 break;
 522         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 523                 kvm->arch.crypto.dea_kw = 0;
 524                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 525                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 526                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 527                 break;
 528         default:
 529                 mutex_unlock(&kvm->lock);
 530                 return -ENXIO;
 531         }
 532
 533         kvm_for_each_vcpu(i, vcpu, kvm) {
 534                 kvm_s390_vcpu_crypto_setup(vcpu);
 535                 exit_sie(vcpu);
 536         }
 537         mutex_unlock(&kvm->lock);
 538         return 0;
 539 }
 540
 541 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 542 {
 543         u8 gtod_high;
 544
 545         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 546                                            sizeof(gtod_high)))
 547                 return -EFAULT;
 548
 549         if (gtod_high != 0)
 550                 return -EINVAL;
 551         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
 552
 553         return 0;
 554 }
 555
 556 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 557 {
 558         u64 gtod;
 559
 560         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 561                 return -EFAULT;
 562
 563         kvm_s390_set_tod_clock(kvm, gtod);
 564         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
 565         return 0;
 566 }
 567
 568 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 569 {
 570         int ret;
 571
 572         if (attr->flags)
 573                 return -EINVAL;
 574
 575         switch (attr->attr) {
 576         case KVM_S390_VM_TOD_HIGH:
 577                 ret = kvm_s390_set_tod_high(kvm, attr);
 578                 break;
 579         case KVM_S390_VM_TOD_LOW:
 580                 ret = kvm_s390_set_tod_low(kvm, attr);
 581                 break;
 582         default:
 583                 ret = -ENXIO;
 584                 break;
 585         }
 586         return ret;
 587 }
 588
 589 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 590 {
 591         u8 gtod_high = 0;
 592
 593         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 594                                          sizeof(gtod_high)))
 595                 return -EFAULT;
 596         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
 597
 598         return 0;
 599 }
 600
 601 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 602 {
 603         u64 gtod;
 604
 605         gtod = kvm_s390_get_tod_clock_fast(kvm);
 606         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 607                 return -EFAULT;
 608         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
 609
 610         return 0;
 611 }
 612
 613 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 614 {
 615         int ret;
 616
 617         if (attr->flags)
 618                 return -EINVAL;
 619
 620         switch (attr->attr) {
 621         case KVM_S390_VM_TOD_HIGH:
 622                 ret = kvm_s390_get_tod_high(kvm, attr);
 623                 break;
 624         case KVM_S390_VM_TOD_LOW:
 625                 ret = kvm_s390_get_tod_low(kvm, attr);
 626                 break;
 627         default:
 628                 ret = -ENXIO;
 629                 break;
 630         }
 631         return ret;
 632 }
 633
 634 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 635 {
 636         struct kvm_s390_vm_cpu_processor *proc;
 637         int ret = 0;
 638
 639         mutex_lock(&kvm->lock);
 640         if (atomic_read(&kvm->online_vcpus)) {
 641                 ret = -EBUSY;
 642                 goto out;
 643         }
 644         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 645         if (!proc) {
 646                 ret = -ENOMEM;
 647                 goto out;
 648         }
 649         if (!copy_from_user(proc, (void __user *)attr->addr,
 650                             sizeof(*proc))) {
 651                 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
 652                        sizeof(struct cpuid));
 653                 kvm->arch.model.ibc = proc->ibc;
 654                 memcpy(kvm->arch.model.fac->list, proc->fac_list,
 655                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 656         } else
 657                 ret = -EFAULT;
 658         kfree(proc);
 659 out:
 660         mutex_unlock(&kvm->lock);
 661         return ret;
 662 }
 663
 664 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 665 {
 666         int ret = -ENXIO;
 667
 668         switch (attr->attr) {
 669         case KVM_S390_VM_CPU_PROCESSOR:
 670                 ret = kvm_s390_set_processor(kvm, attr);
 671                 break;
 672         }
 673         return ret;
 674 }
 675
 676 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 677 {
 678         struct kvm_s390_vm_cpu_processor *proc;
 679         int ret = 0;
 680
 681         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 682         if (!proc) {
 683                 ret = -ENOMEM;
 684                 goto out;
 685         }
 686         memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
 687         proc->ibc = kvm->arch.model.ibc;
 688         memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
 689         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 690                 ret = -EFAULT;
 691         kfree(proc);
 692 out:
 693         return ret;
 694 }
 695
 696 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 697 {
 698         struct kvm_s390_vm_cpu_machine *mach;
 699         int ret = 0;
 700
 701         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 702         if (!mach) {
 703                 ret = -ENOMEM;
 704                 goto out;
 705         }
 706         get_cpu_id((struct cpuid *) &mach->cpuid);
 707         mach->ibc = sclp.ibc;
 708         memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
 709                S390_ARCH_FAC_LIST_SIZE_BYTE);
 710         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 711                S390_ARCH_FAC_LIST_SIZE_BYTE);
 712         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 713                 ret = -EFAULT;
 714         kfree(mach);
 715 out:
 716         return ret;
 717 }
 718
 719 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 720 {
 721         int ret = -ENXIO;
 722
 723         switch (attr->attr) {
 724         case KVM_S390_VM_CPU_PROCESSOR:
 725                 ret = kvm_s390_get_processor(kvm, attr);
 726                 break;
 727         case KVM_S390_VM_CPU_MACHINE:
 728                 ret = kvm_s390_get_machine(kvm, attr);
 729                 break;
 730         }
 731         return ret;
 732 }
 733
 734 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 735 {
 736         int ret;
 737
 738         switch (attr->group) {
 739         case KVM_S390_VM_MEM_CTRL:
 740                 ret = kvm_s390_set_mem_control(kvm, attr);
 741                 break;
 742         case KVM_S390_VM_TOD:
 743                 ret = kvm_s390_set_tod(kvm, attr);
 744                 break;
 745         case KVM_S390_VM_CPU_MODEL:
 746                 ret = kvm_s390_set_cpu_model(kvm, attr);
 747                 break;
 748         case KVM_S390_VM_CRYPTO:
 749                 ret = kvm_s390_vm_set_crypto(kvm, attr);
 750                 break;
 751         default:
 752                 ret = -ENXIO;
 753                 break;
 754         }
 755
 756         return ret;
 757 }
 758
 759 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 760 {
 761         int ret;
 762
 763         switch (attr->group) {
 764         case KVM_S390_VM_MEM_CTRL:
 765                 ret = kvm_s390_get_mem_control(kvm, attr);
 766                 break;
 767         case KVM_S390_VM_TOD:
 768                 ret = kvm_s390_get_tod(kvm, attr);
 769                 break;
 770         case KVM_S390_VM_CPU_MODEL:
 771                 ret = kvm_s390_get_cpu_model(kvm, attr);
 772                 break;
 773         default:
 774                 ret = -ENXIO;
 775                 break;
 776         }
 777
 778         return ret;
 779 }
 780
 781 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 782 {
 783         int ret;
 784
 785         switch (attr->group) {
 786         case KVM_S390_VM_MEM_CTRL:
 787                 switch (attr->attr) {
 788                 case KVM_S390_VM_MEM_ENABLE_CMMA:
 789                 case KVM_S390_VM_MEM_CLR_CMMA:
 790                 case KVM_S390_VM_MEM_LIMIT_SIZE:
 791                         ret = 0;
 792                         break;
 793                 default:
 794                         ret = -ENXIO;
 795                         break;
 796                 }
 797                 break;
 798         case KVM_S390_VM_TOD:
 799                 switch (attr->attr) {
 800                 case KVM_S390_VM_TOD_LOW:
 801                 case KVM_S390_VM_TOD_HIGH:
 802                         ret = 0;
 803                         break;
 804                 default:
 805                         ret = -ENXIO;
 806                         break;
 807                 }
 808                 break;
 809         case KVM_S390_VM_CPU_MODEL:
 810                 switch (attr->attr) {
 811                 case KVM_S390_VM_CPU_PROCESSOR:
 812                 case KVM_S390_VM_CPU_MACHINE:
 813                         ret = 0;
 814                         break;
 815                 default:
 816                         ret = -ENXIO;
 817                         break;
 818                 }
 819                 break;
 820         case KVM_S390_VM_CRYPTO:
 821                 switch (attr->attr) {
 822                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 823                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 824                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 825                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 826                         ret = 0;
 827                         break;
 828                 default:
 829                         ret = -ENXIO;
 830                         break;
 831                 }
 832                 break;
 833         default:
 834                 ret = -ENXIO;
 835                 break;
 836         }
 837
 838         return ret;
 839 }
 840
 841 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 842 {
 843         uint8_t *keys;
 844         uint64_t hva;
 845         unsigned long curkey;
 846         int i, r = 0;
 847
 848         if (args->flags != 0)
 849                 return -EINVAL;
 850
 851         /* Is this guest using storage keys? */
 852         if (!mm_use_skey(current->mm))
 853                 return KVM_S390_GET_SKEYS_NONE;
 854
 855         /* Enforce sane limit on memory allocation */
 856         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 857                 return -EINVAL;
 858
 859         keys = kmalloc_array(args->count, sizeof(uint8_t),
 860                              GFP_KERNEL | __GFP_NOWARN);
 861         if (!keys)
 862                 keys = vmalloc(sizeof(uint8_t) * args->count);
 863         if (!keys)
 864                 return -ENOMEM;
 865
 866         for (i = 0; i < args->count; i++) {
 867                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 868                 if (kvm_is_error_hva(hva)) {
 869                         r = -EFAULT;
 870                         goto out;
 871                 }
 872
 873                 curkey = get_guest_storage_key(current->mm, hva);
 874                 if (IS_ERR_VALUE(curkey)) {
 875                         r = curkey;
 876                         goto out;
 877                 }
 878                 keys[i] = curkey;
 879         }
 880
 881         r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
 882                          sizeof(uint8_t) * args->count);
 883         if (r)
 884                 r = -EFAULT;
 885 out:
 886         kvfree(keys);
 887         return r;
 888 }
 889
 890 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
 891 {
 892         uint8_t *keys;
 893         uint64_t hva;
 894         int i, r = 0;
 895
 896         if (args->flags != 0)
 897                 return -EINVAL;
 898
 899         /* Enforce sane limit on memory allocation */
 900         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
 901                 return -EINVAL;
 902
 903         keys = kmalloc_array(args->count, sizeof(uint8_t),
 904                              GFP_KERNEL | __GFP_NOWARN);
 905         if (!keys)
 906                 keys = vmalloc(sizeof(uint8_t) * args->count);
 907         if (!keys)
 908                 return -ENOMEM;
 909
 910         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
 911                            sizeof(uint8_t) * args->count);
 912         if (r) {
 913                 r = -EFAULT;
 914                 goto out;
 915         }
 916
 917         /* Enable storage key handling for the guest */
 918         r = s390_enable_skey();
 919         if (r)
 920                 goto out;
 921
 922         for (i = 0; i < args->count; i++) {
 923                 hva = gfn_to_hva(kvm, args->start_gfn + i);
 924                 if (kvm_is_error_hva(hva)) {
 925                         r = -EFAULT;
 926                         goto out;
 927                 }
 928
 929                 /* Lowest order bit is reserved */
 930                 if (keys[i] & 0x01) {
 931                         r = -EINVAL;
 932                         goto out;
 933                 }
 934
 935                 r = set_guest_storage_key(current->mm, hva,
 936                                           (unsigned long)keys[i], 0);
 937                 if (r)
 938                         goto out;
 939         }
 940 out:
 941         kvfree(keys);
 942         return r;
 943 }
 944
 945 long kvm_arch_vm_ioctl(struct file *filp,
 946                        unsigned int ioctl, unsigned long arg)
 947 {
 948         struct kvm *kvm = filp->private_data;
 949         void __user *argp = (void __user *)arg;
 950         struct kvm_device_attr attr;
 951         int r;
 952
 953         switch (ioctl) {
 954         case KVM_S390_INTERRUPT: {
 955                 struct kvm_s390_interrupt s390int;
 956
 957                 r = -EFAULT;
 958                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 959                         break;
 960                 r = kvm_s390_inject_vm(kvm, &s390int);
 961                 break;
 962         }
 963         case KVM_ENABLE_CAP: {
 964                 struct kvm_enable_cap cap;
 965                 r = -EFAULT;
 966                 if (copy_from_user(&cap, argp, sizeof(cap)))
 967                         break;
 968                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 969                 break;
 970         }
 971         case KVM_CREATE_IRQCHIP: {
 972                 struct kvm_irq_routing_entry routing;
 973
 974                 r = -EINVAL;
 975                 if (kvm->arch.use_irqchip) {
 976                         /* Set up dummy routing. */
 977                         memset(&routing, 0, sizeof(routing));
 978                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
 979                 }
 980                 break;
 981         }
 982         case KVM_SET_DEVICE_ATTR: {
 983                 r = -EFAULT;
 984                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 985                         break;
 986                 r = kvm_s390_vm_set_attr(kvm, &attr);
 987                 break;
 988         }
 989         case KVM_GET_DEVICE_ATTR: {
 990                 r = -EFAULT;
 991                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 992                         break;
 993                 r = kvm_s390_vm_get_attr(kvm, &attr);
 994                 break;
 995         }
 996         case KVM_HAS_DEVICE_ATTR: {
 997                 r = -EFAULT;
 998                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 999                         break;
1000                 r = kvm_s390_vm_has_attr(kvm, &attr);
1001                 break;
1002         }
1003         case KVM_S390_GET_SKEYS: {
1004                 struct kvm_s390_skeys args;
1005
1006                 r = -EFAULT;
1007                 if (copy_from_user(&args, argp,
1008                                    sizeof(struct kvm_s390_skeys)))
1009                         break;
1010                 r = kvm_s390_get_skeys(kvm, &args);
1011                 break;
1012         }
1013         case KVM_S390_SET_SKEYS: {
1014                 struct kvm_s390_skeys args;
1015
1016                 r = -EFAULT;
1017                 if (copy_from_user(&args, argp,
1018                                    sizeof(struct kvm_s390_skeys)))
1019                         break;
1020                 r = kvm_s390_set_skeys(kvm, &args);
1021                 break;
1022         }
1023         default:
1024                 r = -ENOTTY;
1025         }
1026
1027         return r;
1028 }
1029
1030 static int kvm_s390_query_ap_config(u8 *config)
1031 {
1032         u32 fcn_code = 0x04000000UL;
1033         u32 cc = 0;
1034
1035         memset(config, 0, 128);
1036         asm volatile(
1037                 "lgr 0,%1\n"
1038                 "lgr 2,%2\n"
1039                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1040                 "0: ipm %0\n"
1041                 "srl %0,28\n"
1042                 "1:\n"
1043                 EX_TABLE(0b, 1b)
1044                 : "+r" (cc)
1045                 : "r" (fcn_code), "r" (config)
1046                 : "cc", "0", "2", "memory"
1047         );
1048
1049         return cc;
1050 }
1051
1052 static int kvm_s390_apxa_installed(void)
1053 {
1054         u8 config[128];
1055         int cc;
1056
1057         if (test_facility(12)) {
1058                 cc = kvm_s390_query_ap_config(config);
1059
1060                 if (cc)
1061                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1062                 else
1063                         return config[0] & 0x40;
1064         }
1065
1066         return 0;
1067 }
1068
1069 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1070 {
1071         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1072
1073         if (kvm_s390_apxa_installed())
1074                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1075         else
1076                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1077 }
1078
1079 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
1080 {
1081         get_cpu_id(cpu_id);
1082         cpu_id->version = 0xff;
1083 }
1084
1085 static int kvm_s390_crypto_init(struct kvm *kvm)
1086 {
1087         if (!test_kvm_facility(kvm, 76))
1088                 return 0;
1089
1090         kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1091                                          GFP_KERNEL | GFP_DMA);
1092         if (!kvm->arch.crypto.crycb)
1093                 return -ENOMEM;
1094
1095         kvm_s390_set_crycb_format(kvm);
1096
1097         /* Enable AES/DEA protected key functions by default */
1098         kvm->arch.crypto.aes_kw = 1;
1099         kvm->arch.crypto.dea_kw = 1;
1100         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1101                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1102         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1103                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1104
1105         return 0;
1106 }
1107
1108 static void sca_dispose(struct kvm *kvm)
1109 {
1110         if (kvm->arch.use_esca)
1111                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1112         else
1113                 free_page((unsigned long)(kvm->arch.sca));
1114         kvm->arch.sca = NULL;
1115 }
1116
1117 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1118 {
1119         int i, rc;
1120         char debug_name[16];
1121         static unsigned long sca_offset;
1122
1123         rc = -EINVAL;
1124 #ifdef CONFIG_KVM_S390_UCONTROL
1125         if (type & ~KVM_VM_S390_UCONTROL)
1126                 goto out_err;
1127         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1128                 goto out_err;
1129 #else
1130         if (type)
1131                 goto out_err;
1132 #endif
1133
1134         rc = s390_enable_sie();
1135         if (rc)
1136                 goto out_err;
1137
1138         rc = -ENOMEM;
1139
1140         kvm->arch.use_esca = 0; /* start with basic SCA */
1141         rwlock_init(&kvm->arch.sca_lock);
1142         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(GFP_KERNEL);
1143         if (!kvm->arch.sca)
1144                 goto out_err;
1145         spin_lock(&kvm_lock);
1146         sca_offset += 16;
1147         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1148                 sca_offset = 0;
1149         kvm->arch.sca = (struct bsca_block *)
1150                         ((char *) kvm->arch.sca + sca_offset);
1151         spin_unlock(&kvm_lock);
1152
1153         sprintf(debug_name, "kvm-%u", current->pid);
1154
1155         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1156         if (!kvm->arch.dbf)
1157                 goto out_err;
1158
1159         /*
1160          * The architectural maximum amount of facilities is 16 kbit. To store
1161          * this amount, 2 kbyte of memory is required. Thus we need a full
1162          * page to hold the guest facility list (arch.model.fac->list) and the
1163          * facility mask (arch.model.fac->mask). Its address size has to be
1164          * 31 bits and word aligned.
1165          */
1166         kvm->arch.model.fac =
1167                 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1168         if (!kvm->arch.model.fac)
1169                 goto out_err;
1170
1171         /* Populate the facility mask initially. */
1172         memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1173                S390_ARCH_FAC_LIST_SIZE_BYTE);
1174         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1175                 if (i < kvm_s390_fac_list_mask_size())
1176                         kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1177                 else
1178                         kvm->arch.model.fac->mask[i] = 0UL;
1179         }
1180
1181         /* Populate the facility list initially. */
1182         memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1183                S390_ARCH_FAC_LIST_SIZE_BYTE);
1184
1185         kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1186         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1187
1188         if (kvm_s390_crypto_init(kvm) < 0)
1189                 goto out_err;
1190
1191         spin_lock_init(&kvm->arch.float_int.lock);
1192         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1193                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1194         init_waitqueue_head(&kvm->arch.ipte_wq);
1195         mutex_init(&kvm->arch.ipte_mutex);
1196
1197         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1198         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1199
1200         if (type & KVM_VM_S390_UCONTROL) {
1201                 kvm->arch.gmap = NULL;
1202                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1203         } else {
1204                 if (sclp.hamax == U64_MAX)
1205                         kvm->arch.mem_limit = TASK_MAX_SIZE;
1206                 else
1207                         kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1208                                                     sclp.hamax + 1);
1209                 kvm->arch.gmap = gmap_alloc(current->mm, kvm->arch.mem_limit - 1);
1210                 if (!kvm->arch.gmap)
1211                         goto out_err;
1212                 kvm->arch.gmap->private = kvm;
1213                 kvm->arch.gmap->pfault_enabled = 0;
1214         }
1215
1216         kvm->arch.css_support = 0;
1217         kvm->arch.use_irqchip = 0;
1218         kvm->arch.epoch = 0;
1219
1220         spin_lock_init(&kvm->arch.start_stop_lock);
1221         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1222
1223         return 0;
1224 out_err:
1225         kfree(kvm->arch.crypto.crycb);
1226         free_page((unsigned long)kvm->arch.model.fac);
1227         debug_unregister(kvm->arch.dbf);
1228         sca_dispose(kvm);
1229         KVM_EVENT(3, "creation of vm failed: %d", rc);
1230         return rc;
1231 }
1232
1233 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1234 {
1235         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1236         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1237         kvm_s390_clear_local_irqs(vcpu);
1238         kvm_clear_async_pf_completion_queue(vcpu);
1239         if (!kvm_is_ucontrol(vcpu->kvm))
1240                 sca_del_vcpu(vcpu);
1241
1242         if (kvm_is_ucontrol(vcpu->kvm))
1243                 gmap_free(vcpu->arch.gmap);
1244
1245         if (vcpu->kvm->arch.use_cmma)
1246                 kvm_s390_vcpu_unsetup_cmma(vcpu);
1247         kfree(vcpu->arch.guest_fpregs.fprs);
1248         free_page((unsigned long)(vcpu->arch.sie_block));
1249
1250         kvm_vcpu_uninit(vcpu);
1251         kmem_cache_free(kvm_vcpu_cache, vcpu);
1252 }
1253
1254 static void kvm_free_vcpus(struct kvm *kvm)
1255 {
1256         unsigned int i;
1257         struct kvm_vcpu *vcpu;
1258
1259         kvm_for_each_vcpu(i, vcpu, kvm)
1260                 kvm_arch_vcpu_destroy(vcpu);
1261
1262         mutex_lock(&kvm->lock);
1263         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1264                 kvm->vcpus[i] = NULL;
1265
1266         atomic_set(&kvm->online_vcpus, 0);
1267         mutex_unlock(&kvm->lock);
1268 }
1269
1270 void kvm_arch_destroy_vm(struct kvm *kvm)
1271 {
1272         kvm_free_vcpus(kvm);
1273         free_page((unsigned long)kvm->arch.model.fac);
1274         sca_dispose(kvm);
1275         debug_unregister(kvm->arch.dbf);
1276         kfree(kvm->arch.crypto.crycb);
1277         if (!kvm_is_ucontrol(kvm))
1278                 gmap_free(kvm->arch.gmap);
1279         kvm_s390_destroy_adapters(kvm);
1280         kvm_s390_clear_float_irqs(kvm);
1281         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1282 }
1283
1284 /* Section: vcpu related */
1285 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1286 {
1287         vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1288         if (!vcpu->arch.gmap)
1289                 return -ENOMEM;
1290         vcpu->arch.gmap->private = vcpu->kvm;
1291
1292         return 0;
1293 }
1294
1295 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1296 {
1297         read_lock(&vcpu->kvm->arch.sca_lock);
1298         if (vcpu->kvm->arch.use_esca) {
1299                 struct esca_block *sca = vcpu->kvm->arch.sca;
1300
1301                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1302                 sca->cpu[vcpu->vcpu_id].sda = 0;
1303         } else {
1304                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1305
1306                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1307                 sca->cpu[vcpu->vcpu_id].sda = 0;
1308         }
1309         read_unlock(&vcpu->kvm->arch.sca_lock);
1310 }
1311
1312 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1313 {
1314         read_lock(&vcpu->kvm->arch.sca_lock);
1315         if (vcpu->kvm->arch.use_esca) {
1316                 struct esca_block *sca = vcpu->kvm->arch.sca;
1317
1318                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1319                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1320                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1321                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1322                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1323         } else {
1324                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1325
1326                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1327                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1328                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1329                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1330         }
1331         read_unlock(&vcpu->kvm->arch.sca_lock);
1332 }
1333
1334 /* Basic SCA to Extended SCA data copy routines */
1335 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1336 {
1337         d->sda = s->sda;
1338         d->sigp_ctrl.c = s->sigp_ctrl.c;
1339         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1340 }
1341
1342 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1343 {
1344         int i;
1345
1346         d->ipte_control = s->ipte_control;
1347         d->mcn[0] = s->mcn;
1348         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1349                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1350 }
1351
1352 static int sca_switch_to_extended(struct kvm *kvm)
1353 {
1354         struct bsca_block *old_sca = kvm->arch.sca;
1355         struct esca_block *new_sca;
1356         struct kvm_vcpu *vcpu;
1357         unsigned int vcpu_idx;
1358         u32 scaol, scaoh;
1359
1360         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1361         if (!new_sca)
1362                 return -ENOMEM;
1363
1364         scaoh = (u32)((u64)(new_sca) >> 32);
1365         scaol = (u32)(u64)(new_sca) & ~0x3fU;
1366
1367         kvm_s390_vcpu_block_all(kvm);
1368         write_lock(&kvm->arch.sca_lock);
1369
1370         sca_copy_b_to_e(new_sca, old_sca);
1371
1372         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1373                 vcpu->arch.sie_block->scaoh = scaoh;
1374                 vcpu->arch.sie_block->scaol = scaol;
1375                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1376         }
1377         kvm->arch.sca = new_sca;
1378         kvm->arch.use_esca = 1;
1379
1380         write_unlock(&kvm->arch.sca_lock);
1381         kvm_s390_vcpu_unblock_all(kvm);
1382
1383         free_page((unsigned long)old_sca);
1384
1385         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1386                  old_sca, kvm->arch.sca);
1387         return 0;
1388 }
1389
1390 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1391 {
1392         int rc;
1393
1394         if (id < KVM_S390_BSCA_CPU_SLOTS)
1395                 return true;
1396         if (!sclp.has_esca)
1397                 return false;
1398
1399         mutex_lock(&kvm->lock);
1400         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1401         mutex_unlock(&kvm->lock);
1402
1403         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1404 }
1405
1406 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1407 {
1408         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1409         kvm_clear_async_pf_completion_queue(vcpu);
1410         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1411                                     KVM_SYNC_GPRS |
1412                                     KVM_SYNC_ACRS |
1413                                     KVM_SYNC_CRS |
1414                                     KVM_SYNC_ARCH0 |
1415                                     KVM_SYNC_PFAULT;
1416         if (test_kvm_facility(vcpu->kvm, 64))
1417                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1418         if (test_kvm_facility(vcpu->kvm, 129))
1419                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1420
1421         if (kvm_is_ucontrol(vcpu->kvm))
1422                 return __kvm_ucontrol_vcpu_init(vcpu);
1423
1424         return 0;
1425 }
1426
1427 /*
1428  * Backs up the current FP/VX register save area on a particular
1429  * destination.  Used to switch between different register save
1430  * areas.
1431  */
1432 static inline void save_fpu_to(struct fpu *dst)
1433 {
1434         dst->fpc = current->thread.fpu.fpc;
1435         dst->regs = current->thread.fpu.regs;
1436 }
1437
1438 /*
1439  * Switches the FP/VX register save area from which to lazy
1440  * restore register contents.
1441  */
1442 static inline void load_fpu_from(struct fpu *from)
1443 {
1444         current->thread.fpu.fpc = from->fpc;
1445         current->thread.fpu.regs = from->regs;
1446 }
1447
1448 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1449 {
1450         /* Save host register state */
1451         save_fpu_regs();
1452         save_fpu_to(&vcpu->arch.host_fpregs);
1453
1454         if (test_kvm_facility(vcpu->kvm, 129)) {
1455                 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1456                 /*
1457                  * Use the register save area in the SIE-control block
1458                  * for register restore and save in kvm_arch_vcpu_put()
1459                  */
1460                 current->thread.fpu.vxrs =
1461                         (__vector128 *)&vcpu->run->s.regs.vrs;
1462         } else
1463                 load_fpu_from(&vcpu->arch.guest_fpregs);
1464
1465         if (test_fp_ctl(current->thread.fpu.fpc))
1466                 /* User space provided an invalid FPC, let's clear it */
1467                 current->thread.fpu.fpc = 0;
1468
1469         save_access_regs(vcpu->arch.host_acrs);
1470         restore_access_regs(vcpu->run->s.regs.acrs);
1471         gmap_enable(vcpu->arch.gmap);
1472         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1473 }
1474
1475 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1476 {
1477         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1478         gmap_disable(vcpu->arch.gmap);
1479
1480         save_fpu_regs();
1481
1482         if (test_kvm_facility(vcpu->kvm, 129))
1483                 /*
1484                  * kvm_arch_vcpu_load() set up the register save area to
1485                  * the &vcpu->run->s.regs.vrs and, thus, the vector registers
1486                  * are already saved.  Only the floating-point control must be
1487                  * copied.
1488                  */
1489                 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1490         else
1491                 save_fpu_to(&vcpu->arch.guest_fpregs);
1492         load_fpu_from(&vcpu->arch.host_fpregs);
1493
1494         save_access_regs(vcpu->run->s.regs.acrs);
1495         restore_access_regs(vcpu->arch.host_acrs);
1496 }
1497
1498 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1499 {
1500         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1501         vcpu->arch.sie_block->gpsw.mask = 0UL;
1502         vcpu->arch.sie_block->gpsw.addr = 0UL;
1503         kvm_s390_set_prefix(vcpu, 0);
1504         vcpu->arch.sie_block->cputm     = 0UL;
1505         vcpu->arch.sie_block->ckc       = 0UL;
1506         vcpu->arch.sie_block->todpr     = 0;
1507         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1508         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1509         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1510         vcpu->arch.guest_fpregs.fpc = 0;
1511         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1512         vcpu->arch.sie_block->gbea = 1;
1513         vcpu->arch.sie_block->pp = 0;
1514         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1515         kvm_clear_async_pf_completion_queue(vcpu);
1516         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1517                 kvm_s390_vcpu_stop(vcpu);
1518         kvm_s390_clear_local_irqs(vcpu);
1519 }
1520
1521 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1522 {
1523         mutex_lock(&vcpu->kvm->lock);
1524         preempt_disable();
1525         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1526         preempt_enable();
1527         mutex_unlock(&vcpu->kvm->lock);
1528         if (!kvm_is_ucontrol(vcpu->kvm)) {
1529                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1530                 sca_add_vcpu(vcpu);
1531         }
1532
1533 }
1534
1535 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1536 {
1537         if (!test_kvm_facility(vcpu->kvm, 76))
1538                 return;
1539
1540         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1541
1542         if (vcpu->kvm->arch.crypto.aes_kw)
1543                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1544         if (vcpu->kvm->arch.crypto.dea_kw)
1545                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1546
1547         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1548 }
1549
1550 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1551 {
1552         free_page(vcpu->arch.sie_block->cbrlo);
1553         vcpu->arch.sie_block->cbrlo = 0;
1554 }
1555
1556 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1557 {
1558         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1559         if (!vcpu->arch.sie_block->cbrlo)
1560                 return -ENOMEM;
1561
1562         vcpu->arch.sie_block->ecb2 |= 0x80;
1563         vcpu->arch.sie_block->ecb2 &= ~0x08;
1564         return 0;
1565 }
1566
1567 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1568 {
1569         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1570
1571         vcpu->arch.cpu_id = model->cpu_id;
1572         vcpu->arch.sie_block->ibc = model->ibc;
1573         vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1574 }
1575
1576 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1577 {
1578         int rc = 0;
1579
1580         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1581                                                     CPUSTAT_SM |
1582                                                     CPUSTAT_STOPPED);
1583
1584         if (test_kvm_facility(vcpu->kvm, 78))
1585                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1586         else if (test_kvm_facility(vcpu->kvm, 8))
1587                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1588
1589         kvm_s390_vcpu_setup_model(vcpu);
1590
1591         vcpu->arch.sie_block->ecb   = 6;
1592         if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1593                 vcpu->arch.sie_block->ecb |= 0x10;
1594
1595         vcpu->arch.sie_block->ecb2  = 8;
1596         vcpu->arch.sie_block->eca   = 0xC1002000U;
1597         if (sclp.has_siif)
1598                 vcpu->arch.sie_block->eca |= 1;
1599         if (sclp.has_sigpif)
1600                 vcpu->arch.sie_block->eca |= 0x10000000U;
1601         if (test_kvm_facility(vcpu->kvm, 64))
1602                 vcpu->arch.sie_block->ecb3 |= 0x01;
1603         if (test_kvm_facility(vcpu->kvm, 129)) {
1604                 vcpu->arch.sie_block->eca |= 0x00020000;
1605                 vcpu->arch.sie_block->ecd |= 0x20000000;
1606         }
1607         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1608         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1609
1610         if (vcpu->kvm->arch.use_cmma) {
1611                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1612                 if (rc)
1613                         return rc;
1614         }
1615         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1616         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1617
1618         kvm_s390_vcpu_crypto_setup(vcpu);
1619
1620         return rc;
1621 }
1622
1623 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1624                                       unsigned int id)
1625 {
1626         struct kvm_vcpu *vcpu;
1627         struct sie_page *sie_page;
1628         int rc = -EINVAL;
1629
1630         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1631                 goto out;
1632
1633         rc = -ENOMEM;
1634
1635         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1636         if (!vcpu)
1637                 goto out;
1638
1639         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1640         if (!sie_page)
1641                 goto out_free_cpu;
1642
1643         vcpu->arch.sie_block = &sie_page->sie_block;
1644         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1645
1646         vcpu->arch.sie_block->icpua = id;
1647         spin_lock_init(&vcpu->arch.local_int.lock);
1648         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1649         vcpu->arch.local_int.wq = &vcpu->wq;
1650         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1651
1652         /*
1653          * Allocate a save area for floating-point registers.  If the vector
1654          * extension is available, register contents are saved in the SIE
1655          * control block.  The allocated save area is still required in
1656          * particular places, for example, in kvm_s390_vcpu_store_status().
1657          */
1658         vcpu->arch.guest_fpregs.fprs = kzalloc(sizeof(freg_t) * __NUM_FPRS,
1659                                                GFP_KERNEL);
1660         if (!vcpu->arch.guest_fpregs.fprs)
1661                 goto out_free_sie_block;
1662
1663         rc = kvm_vcpu_init(vcpu, kvm, id);
1664         if (rc)
1665                 goto out_free_fprs;
1666         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
1667                  vcpu->arch.sie_block);
1668         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1669
1670         return vcpu;
1671 out_free_fprs:
1672         kfree(vcpu->arch.guest_fpregs.fprs);
1673 out_free_sie_block:
1674         free_page((unsigned long)(vcpu->arch.sie_block));
1675 out_free_cpu:
1676         kmem_cache_free(kvm_vcpu_cache, vcpu);
1677 out:
1678         return ERR_PTR(rc);
1679 }
1680
1681 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1682 {
1683         return kvm_s390_vcpu_has_irq(vcpu, 0);
1684 }
1685
1686 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1687 {
1688         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1689         exit_sie(vcpu);
1690 }
1691
1692 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1693 {
1694         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1695 }
1696
1697 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1698 {
1699         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1700         exit_sie(vcpu);
1701 }
1702
1703 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1704 {
1705         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1706 }
1707
1708 /*
1709  * Kick a guest cpu out of SIE and wait until SIE is not running.
1710  * If the CPU is not running (e.g. waiting as idle) the function will
1711  * return immediately. */
1712 void exit_sie(struct kvm_vcpu *vcpu)
1713 {
1714         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1715         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1716                 cpu_relax();
1717 }
1718
1719 /* Kick a guest cpu out of SIE to process a request synchronously */
1720 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1721 {
1722         kvm_make_request(req, vcpu);
1723         kvm_s390_vcpu_request(vcpu);
1724 }
1725
1726 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1727 {
1728         int i;
1729         struct kvm *kvm = gmap->private;
1730         struct kvm_vcpu *vcpu;
1731
1732         kvm_for_each_vcpu(i, vcpu, kvm) {
1733                 /* match against both prefix pages */
1734                 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1735                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1736                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1737                 }
1738         }
1739 }
1740
1741 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1742 {
1743         /* kvm common code refers to this, but never calls it */
1744         BUG();
1745         return 0;
1746 }
1747
1748 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1749                                            struct kvm_one_reg *reg)
1750 {
1751         int r = -EINVAL;
1752
1753         switch (reg->id) {
1754         case KVM_REG_S390_TODPR:
1755                 r = put_user(vcpu->arch.sie_block->todpr,
1756                              (u32 __user *)reg->addr);
1757                 break;
1758         case KVM_REG_S390_EPOCHDIFF:
1759                 r = put_user(vcpu->arch.sie_block->epoch,
1760                              (u64 __user *)reg->addr);
1761                 break;
1762         case KVM_REG_S390_CPU_TIMER:
1763                 r = put_user(vcpu->arch.sie_block->cputm,
1764                              (u64 __user *)reg->addr);
1765                 break;
1766         case KVM_REG_S390_CLOCK_COMP:
1767                 r = put_user(vcpu->arch.sie_block->ckc,
1768                              (u64 __user *)reg->addr);
1769                 break;
1770         case KVM_REG_S390_PFTOKEN:
1771                 r = put_user(vcpu->arch.pfault_token,
1772                              (u64 __user *)reg->addr);
1773                 break;
1774         case KVM_REG_S390_PFCOMPARE:
1775                 r = put_user(vcpu->arch.pfault_compare,
1776                              (u64 __user *)reg->addr);
1777                 break;
1778         case KVM_REG_S390_PFSELECT:
1779                 r = put_user(vcpu->arch.pfault_select,
1780                              (u64 __user *)reg->addr);
1781                 break;
1782         case KVM_REG_S390_PP:
1783                 r = put_user(vcpu->arch.sie_block->pp,
1784                              (u64 __user *)reg->addr);
1785                 break;
1786         case KVM_REG_S390_GBEA:
1787                 r = put_user(vcpu->arch.sie_block->gbea,
1788                              (u64 __user *)reg->addr);
1789                 break;
1790         default:
1791                 break;
1792         }
1793
1794         return r;
1795 }
1796
1797 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1798                                            struct kvm_one_reg *reg)
1799 {
1800         int r = -EINVAL;
1801
1802         switch (reg->id) {
1803         case KVM_REG_S390_TODPR:
1804                 r = get_user(vcpu->arch.sie_block->todpr,
1805                              (u32 __user *)reg->addr);
1806                 break;
1807         case KVM_REG_S390_EPOCHDIFF:
1808                 r = get_user(vcpu->arch.sie_block->epoch,
1809                              (u64 __user *)reg->addr);
1810                 break;
1811         case KVM_REG_S390_CPU_TIMER:
1812                 r = get_user(vcpu->arch.sie_block->cputm,
1813                              (u64 __user *)reg->addr);
1814                 break;
1815         case KVM_REG_S390_CLOCK_COMP:
1816                 r = get_user(vcpu->arch.sie_block->ckc,
1817                              (u64 __user *)reg->addr);
1818                 break;
1819         case KVM_REG_S390_PFTOKEN:
1820                 r = get_user(vcpu->arch.pfault_token,
1821                              (u64 __user *)reg->addr);
1822                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1823                         kvm_clear_async_pf_completion_queue(vcpu);
1824                 break;
1825         case KVM_REG_S390_PFCOMPARE:
1826                 r = get_user(vcpu->arch.pfault_compare,
1827                              (u64 __user *)reg->addr);
1828                 break;
1829         case KVM_REG_S390_PFSELECT:
1830                 r = get_user(vcpu->arch.pfault_select,
1831                              (u64 __user *)reg->addr);
1832                 break;
1833         case KVM_REG_S390_PP:
1834                 r = get_user(vcpu->arch.sie_block->pp,
1835                              (u64 __user *)reg->addr);
1836                 break;
1837         case KVM_REG_S390_GBEA:
1838                 r = get_user(vcpu->arch.sie_block->gbea,
1839                              (u64 __user *)reg->addr);
1840                 break;
1841         default:
1842                 break;
1843         }
1844
1845         return r;
1846 }
1847
1848 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1849 {
1850         kvm_s390_vcpu_initial_reset(vcpu);
1851         return 0;
1852 }
1853
1854 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1855 {
1856         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1857         return 0;
1858 }
1859
1860 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1861 {
1862         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1863         return 0;
1864 }
1865
1866 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1867                                   struct kvm_sregs *sregs)
1868 {
1869         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1870         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1871         restore_access_regs(vcpu->run->s.regs.acrs);
1872         return 0;
1873 }
1874
1875 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1876                                   struct kvm_sregs *sregs)
1877 {
1878         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1879         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1880         return 0;
1881 }
1882
1883 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1884 {
1885         if (test_fp_ctl(fpu->fpc))
1886                 return -EINVAL;
1887         memcpy(vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1888         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1889         save_fpu_regs();
1890         load_fpu_from(&vcpu->arch.guest_fpregs);
1891         return 0;
1892 }
1893
1894 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1895 {
1896         memcpy(&fpu->fprs, vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1897         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1898         return 0;
1899 }
1900
1901 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1902 {
1903         int rc = 0;
1904
1905         if (!is_vcpu_stopped(vcpu))
1906                 rc = -EBUSY;
1907         else {
1908                 vcpu->run->psw_mask = psw.mask;
1909                 vcpu->run->psw_addr = psw.addr;
1910         }
1911         return rc;
1912 }
1913
1914 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1915                                   struct kvm_translation *tr)
1916 {
1917         return -EINVAL; /* not implemented yet */
1918 }
1919
1920 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1921                               KVM_GUESTDBG_USE_HW_BP | \
1922                               KVM_GUESTDBG_ENABLE)
1923
1924 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1925                                         struct kvm_guest_debug *dbg)
1926 {
1927         int rc = 0;
1928
1929         vcpu->guest_debug = 0;
1930         kvm_s390_clear_bp_data(vcpu);
1931
1932         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1933                 return -EINVAL;
1934
1935         if (dbg->control & KVM_GUESTDBG_ENABLE) {
1936                 vcpu->guest_debug = dbg->control;
1937                 /* enforce guest PER */
1938                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1939
1940                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1941                         rc = kvm_s390_import_bp_data(vcpu, dbg);
1942         } else {
1943                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1944                 vcpu->arch.guestdbg.last_bp = 0;
1945         }
1946
1947         if (rc) {
1948                 vcpu->guest_debug = 0;
1949                 kvm_s390_clear_bp_data(vcpu);
1950                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1951         }
1952
1953         return rc;
1954 }
1955
1956 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1957                                     struct kvm_mp_state *mp_state)
1958 {
1959         /* CHECK_STOP and LOAD are not supported yet */
1960         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1961                                        KVM_MP_STATE_OPERATING;
1962 }
1963
1964 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1965                                     struct kvm_mp_state *mp_state)
1966 {
1967         int rc = 0;
1968
1969         /* user space knows about this interface - let it control the state */
1970         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1971
1972         switch (mp_state->mp_state) {
1973         case KVM_MP_STATE_STOPPED:
1974                 kvm_s390_vcpu_stop(vcpu);
1975                 break;
1976         case KVM_MP_STATE_OPERATING:
1977                 kvm_s390_vcpu_start(vcpu);
1978                 break;
1979         case KVM_MP_STATE_LOAD:
1980         case KVM_MP_STATE_CHECK_STOP:
1981                 /* fall through - CHECK_STOP and LOAD are not supported yet */
1982         default:
1983                 rc = -ENXIO;
1984         }
1985
1986         return rc;
1987 }
1988
1989 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1990 {
1991         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1992 }
1993
1994 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1995 {
1996 retry:
1997         kvm_s390_vcpu_request_handled(vcpu);
1998         if (!vcpu->requests)
1999                 return 0;
2000         /*
2001          * We use MMU_RELOAD just to re-arm the ipte notifier for the
2002          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
2003          * This ensures that the ipte instruction for this request has
2004          * already finished. We might race against a second unmapper that
2005          * wants to set the blocking bit. Lets just retry the request loop.
2006          */
2007         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2008                 int rc;
2009                 rc = gmap_ipte_notify(vcpu->arch.gmap,
2010                                       kvm_s390_get_prefix(vcpu),
2011                                       PAGE_SIZE * 2);
2012                 if (rc)
2013                         return rc;
2014                 goto retry;
2015         }
2016
2017         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2018                 vcpu->arch.sie_block->ihcpu = 0xffff;
2019                 goto retry;
2020         }
2021
2022         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2023                 if (!ibs_enabled(vcpu)) {
2024                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2025                         atomic_or(CPUSTAT_IBS,
2026                                         &vcpu->arch.sie_block->cpuflags);
2027                 }
2028                 goto retry;
2029         }
2030
2031         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2032                 if (ibs_enabled(vcpu)) {
2033                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2034                         atomic_andnot(CPUSTAT_IBS,
2035                                           &vcpu->arch.sie_block->cpuflags);
2036                 }
2037                 goto retry;
2038         }
2039
2040         /* nothing to do, just clear the request */
2041         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2042
2043         return 0;
2044 }
2045
2046 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2047 {
2048         struct kvm_vcpu *vcpu;
2049         int i;
2050
2051         mutex_lock(&kvm->lock);
2052         preempt_disable();
2053         kvm->arch.epoch = tod - get_tod_clock();
2054         kvm_s390_vcpu_block_all(kvm);
2055         kvm_for_each_vcpu(i, vcpu, kvm)
2056                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2057         kvm_s390_vcpu_unblock_all(kvm);
2058         preempt_enable();
2059         mutex_unlock(&kvm->lock);
2060 }
2061
2062 /**
2063  * kvm_arch_fault_in_page - fault-in guest page if necessary
2064  * @vcpu: The corresponding virtual cpu
2065  * @gpa: Guest physical address
2066  * @writable: Whether the page should be writable or not
2067  *
2068  * Make sure that a guest page has been faulted-in on the host.
2069  *
2070  * Return: Zero on success, negative error code otherwise.
2071  */
2072 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2073 {
2074         return gmap_fault(vcpu->arch.gmap, gpa,
2075                           writable ? FAULT_FLAG_WRITE : 0);
2076 }
2077
2078 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2079                                       unsigned long token)
2080 {
2081         struct kvm_s390_interrupt inti;
2082         struct kvm_s390_irq irq;
2083
2084         if (start_token) {
2085                 irq.u.ext.ext_params2 = token;
2086                 irq.type = KVM_S390_INT_PFAULT_INIT;
2087                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2088         } else {
2089                 inti.type = KVM_S390_INT_PFAULT_DONE;
2090                 inti.parm64 = token;
2091                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2092         }
2093 }
2094
2095 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2096                                      struct kvm_async_pf *work)
2097 {
2098         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2099         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2100 }
2101
2102 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2103                                  struct kvm_async_pf *work)
2104 {
2105         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2106         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2107 }
2108
2109 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2110                                struct kvm_async_pf *work)
2111 {
2112         /* s390 will always inject the page directly */
2113 }
2114
2115 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2116 {
2117         /*
2118          * s390 will always inject the page directly,
2119          * but we still want check_async_completion to cleanup
2120          */
2121         return true;
2122 }
2123
2124 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2125 {
2126         hva_t hva;
2127         struct kvm_arch_async_pf arch;
2128         int rc;
2129
2130         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2131                 return 0;
2132         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2133             vcpu->arch.pfault_compare)
2134                 return 0;
2135         if (psw_extint_disabled(vcpu))
2136                 return 0;
2137         if (kvm_s390_vcpu_has_irq(vcpu, 0))
2138                 return 0;
2139         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2140                 return 0;
2141         if (!vcpu->arch.gmap->pfault_enabled)
2142                 return 0;
2143
2144         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2145         hva += current->thread.gmap_addr & ~PAGE_MASK;
2146         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2147                 return 0;
2148
2149         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2150         return rc;
2151 }
2152
2153 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2154 {
2155         int rc, cpuflags;
2156
2157         /*
2158          * On s390 notifications for arriving pages will be delivered directly
2159          * to the guest but the house keeping for completed pfaults is
2160          * handled outside the worker.
2161          */
2162         kvm_check_async_pf_completion(vcpu);
2163
2164         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2165         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2166
2167         if (need_resched())
2168                 schedule();
2169
2170         if (test_cpu_flag(CIF_MCCK_PENDING))
2171                 s390_handle_mcck();
2172
2173         if (!kvm_is_ucontrol(vcpu->kvm)) {
2174                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2175                 if (rc)
2176                         return rc;
2177         }
2178
2179         rc = kvm_s390_handle_requests(vcpu);
2180         if (rc)
2181                 return rc;
2182
2183         if (guestdbg_enabled(vcpu)) {
2184                 kvm_s390_backup_guest_per_regs(vcpu);
2185                 kvm_s390_patch_guest_per_regs(vcpu);
2186         }
2187
2188         vcpu->arch.sie_block->icptcode = 0;
2189         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2190         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2191         trace_kvm_s390_sie_enter(vcpu, cpuflags);
2192
2193         return 0;
2194 }
2195
2196 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2197 {
2198         psw_t *psw = &vcpu->arch.sie_block->gpsw;
2199         u8 opcode;
2200         int rc;
2201
2202         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2203         trace_kvm_s390_sie_fault(vcpu);
2204
2205         /*
2206          * We want to inject an addressing exception, which is defined as a
2207          * suppressing or terminating exception. However, since we came here
2208          * by a DAT access exception, the PSW still points to the faulting
2209          * instruction since DAT exceptions are nullifying. So we've got
2210          * to look up the current opcode to get the length of the instruction
2211          * to be able to forward the PSW.
2212          */
2213         rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
2214         if (rc)
2215                 return kvm_s390_inject_prog_cond(vcpu, rc);
2216         psw->addr = __rewind_psw(*psw, -insn_length(opcode));
2217
2218         return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
2219 }
2220
2221 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2222 {
2223         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2224                    vcpu->arch.sie_block->icptcode);
2225         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2226
2227         if (guestdbg_enabled(vcpu))
2228                 kvm_s390_restore_guest_per_regs(vcpu);
2229
2230         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2231         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2232
2233         if (vcpu->arch.sie_block->icptcode > 0) {
2234                 int rc = kvm_handle_sie_intercept(vcpu);
2235
2236                 if (rc != -EOPNOTSUPP)
2237                         return rc;
2238                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2239                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2240                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2241                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2242                 return -EREMOTE;
2243         } else if (exit_reason != -EFAULT) {
2244                 vcpu->stat.exit_null++;
2245                 return 0;
2246         } else if (kvm_is_ucontrol(vcpu->kvm)) {
2247                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2248                 vcpu->run->s390_ucontrol.trans_exc_code =
2249                                                 current->thread.gmap_addr;
2250                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2251                 return -EREMOTE;
2252         } else if (current->thread.gmap_pfault) {
2253                 trace_kvm_s390_major_guest_pfault(vcpu);
2254                 current->thread.gmap_pfault = 0;
2255                 if (kvm_arch_setup_async_pf(vcpu))
2256                         return 0;
2257                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2258         }
2259         return vcpu_post_run_fault_in_sie(vcpu);
2260 }
2261
2262 static int __vcpu_run(struct kvm_vcpu *vcpu)
2263 {
2264         int rc, exit_reason;
2265
2266         /*
2267          * We try to hold kvm->srcu during most of vcpu_run (except when run-
2268          * ning the guest), so that memslots (and other stuff) are protected
2269          */
2270         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2271
2272         do {
2273                 rc = vcpu_pre_run(vcpu);
2274                 if (rc)
2275                         break;
2276
2277                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2278                 /*
2279                  * As PF_VCPU will be used in fault handler, between
2280                  * guest_enter and guest_exit should be no uaccess.
2281                  */
2282                 local_irq_disable();
2283                 __kvm_guest_enter();
2284                 local_irq_enable();
2285                 exit_reason = sie64a(vcpu->arch.sie_block,
2286                                      vcpu->run->s.regs.gprs);
2287                 local_irq_disable();
2288                 __kvm_guest_exit();
2289                 local_irq_enable();
2290                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2291
2292                 rc = vcpu_post_run(vcpu, exit_reason);
2293         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2294
2295         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2296         return rc;
2297 }
2298
2299 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2300 {
2301         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2302         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2303         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2304                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2305         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2306                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2307                 /* some control register changes require a tlb flush */
2308                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2309         }
2310         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2311                 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2312                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2313                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2314                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2315                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2316         }
2317         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2318                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2319                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2320                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2321                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2322                         kvm_clear_async_pf_completion_queue(vcpu);
2323         }
2324         kvm_run->kvm_dirty_regs = 0;
2325 }
2326
2327 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2328 {
2329         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2330         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2331         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2332         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2333         kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2334         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2335         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2336         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2337         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2338         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2339         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2340         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2341 }
2342
2343 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2344 {
2345         int rc;
2346         sigset_t sigsaved;
2347
2348         if (guestdbg_exit_pending(vcpu)) {
2349                 kvm_s390_prepare_debug_exit(vcpu);
2350                 return 0;
2351         }
2352
2353         if (vcpu->sigset_active)
2354                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2355
2356         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2357                 kvm_s390_vcpu_start(vcpu);
2358         } else if (is_vcpu_stopped(vcpu)) {
2359                 pr_err_ratelimited("can't run stopped vcpu %d\n",
2360                                    vcpu->vcpu_id);
2361                 return -EINVAL;
2362         }
2363
2364         sync_regs(vcpu, kvm_run);
2365
2366         might_fault();
2367         rc = __vcpu_run(vcpu);
2368
2369         if (signal_pending(current) && !rc) {
2370                 kvm_run->exit_reason = KVM_EXIT_INTR;
2371                 rc = -EINTR;
2372         }
2373
2374         if (guestdbg_exit_pending(vcpu) && !rc)  {
2375                 kvm_s390_prepare_debug_exit(vcpu);
2376                 rc = 0;
2377         }
2378
2379         if (rc == -EREMOTE) {
2380                 /* userspace support is needed, kvm_run has been prepared */
2381                 rc = 0;
2382         }
2383
2384         store_regs(vcpu, kvm_run);
2385
2386         if (vcpu->sigset_active)
2387                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2388
2389         vcpu->stat.exit_userspace++;
2390         return rc;
2391 }
2392
2393 /*
2394  * store status at address
2395  * we use have two special cases:
2396  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2397  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2398  */
2399 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2400 {
2401         unsigned char archmode = 1;
2402         unsigned int px;
2403         u64 clkcomp;
2404         int rc;
2405
2406         px = kvm_s390_get_prefix(vcpu);
2407         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2408                 if (write_guest_abs(vcpu, 163, &archmode, 1))
2409                         return -EFAULT;
2410                 gpa = 0;
2411         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2412                 if (write_guest_real(vcpu, 163, &archmode, 1))
2413                         return -EFAULT;
2414                 gpa = px;
2415         } else
2416                 gpa -= __LC_FPREGS_SAVE_AREA;
2417         rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2418                              vcpu->arch.guest_fpregs.fprs, 128);
2419         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2420                               vcpu->run->s.regs.gprs, 128);
2421         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2422                               &vcpu->arch.sie_block->gpsw, 16);
2423         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2424                               &px, 4);
2425         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2426                               &vcpu->arch.guest_fpregs.fpc, 4);
2427         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2428                               &vcpu->arch.sie_block->todpr, 4);
2429         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2430                               &vcpu->arch.sie_block->cputm, 8);
2431         clkcomp = vcpu->arch.sie_block->ckc >> 8;
2432         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2433                               &clkcomp, 8);
2434         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2435                               &vcpu->run->s.regs.acrs, 64);
2436         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2437                               &vcpu->arch.sie_block->gcr, 128);
2438         return rc ? -EFAULT : 0;
2439 }
2440
2441 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2442 {
2443         /*
2444          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2445          * copying in vcpu load/put. Lets update our copies before we save
2446          * it into the save area
2447          */
2448         save_fpu_regs();
2449         if (test_kvm_facility(vcpu->kvm, 129)) {
2450                 /*
2451                  * If the vector extension is available, the vector registers
2452                  * which overlaps with floating-point registers are saved in
2453                  * the SIE-control block.  Hence, extract the floating-point
2454                  * registers and the FPC value and store them in the
2455                  * guest_fpregs structure.
2456                  */
2457                 vcpu->arch.guest_fpregs.fpc = current->thread.fpu.fpc;
2458                 convert_vx_to_fp(vcpu->arch.guest_fpregs.fprs,
2459                                  current->thread.fpu.vxrs);
2460         } else
2461                 save_fpu_to(&vcpu->arch.guest_fpregs);
2462         save_access_regs(vcpu->run->s.regs.acrs);
2463
2464         return kvm_s390_store_status_unloaded(vcpu, addr);
2465 }
2466
2467 /*
2468  * store additional status at address
2469  */
2470 int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2471                                         unsigned long gpa)
2472 {
2473         /* Only bits 0-53 are used for address formation */
2474         if (!(gpa & ~0x3ff))
2475                 return 0;
2476
2477         return write_guest_abs(vcpu, gpa & ~0x3ff,
2478                                (void *)&vcpu->run->s.regs.vrs, 512);
2479 }
2480
2481 int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2482 {
2483         if (!test_kvm_facility(vcpu->kvm, 129))
2484                 return 0;
2485
2486         /*
2487          * The guest VXRS are in the host VXRs due to the lazy
2488          * copying in vcpu load/put. We can simply call save_fpu_regs()
2489          * to save the current register state because we are in the
2490          * middle of a load/put cycle.
2491          *
2492          * Let's update our copies before we save it into the save area.
2493          */
2494         save_fpu_regs();
2495
2496         return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2497 }
2498
2499 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2500 {
2501         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2502         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2503 }
2504
2505 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2506 {
2507         unsigned int i;
2508         struct kvm_vcpu *vcpu;
2509
2510         kvm_for_each_vcpu(i, vcpu, kvm) {
2511                 __disable_ibs_on_vcpu(vcpu);
2512         }
2513 }
2514
2515 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2516 {
2517         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2518         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2519 }
2520
2521 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2522 {
2523         int i, online_vcpus, started_vcpus = 0;
2524
2525         if (!is_vcpu_stopped(vcpu))
2526                 return;
2527
2528         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2529         /* Only one cpu at a time may enter/leave the STOPPED state. */
2530         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2531         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2532
2533         for (i = 0; i < online_vcpus; i++) {
2534                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2535                         started_vcpus++;
2536         }
2537
2538         if (started_vcpus == 0) {
2539                 /* we're the only active VCPU -> speed it up */
2540                 __enable_ibs_on_vcpu(vcpu);
2541         } else if (started_vcpus == 1) {
2542                 /*
2543                  * As we are starting a second VCPU, we have to disable
2544                  * the IBS facility on all VCPUs to remove potentially
2545                  * oustanding ENABLE requests.
2546                  */
2547                 __disable_ibs_on_all_vcpus(vcpu->kvm);
2548         }
2549
2550         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2551         /*
2552          * Another VCPU might have used IBS while we were offline.
2553          * Let's play safe and flush the VCPU at startup.
2554          */
2555         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2556         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2557         return;
2558 }
2559
2560 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2561 {
2562         int i, online_vcpus, started_vcpus = 0;
2563         struct kvm_vcpu *started_vcpu = NULL;
2564
2565         if (is_vcpu_stopped(vcpu))
2566                 return;
2567
2568         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2569         /* Only one cpu at a time may enter/leave the STOPPED state. */
2570         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2571         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2572
2573         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2574         kvm_s390_clear_stop_irq(vcpu);
2575
2576         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2577         __disable_ibs_on_vcpu(vcpu);
2578
2579         for (i = 0; i < online_vcpus; i++) {
2580                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2581                         started_vcpus++;
2582                         started_vcpu = vcpu->kvm->vcpus[i];
2583                 }
2584         }
2585
2586         if (started_vcpus == 1) {
2587                 /*
2588                  * As we only have one VCPU left, we want to enable the
2589                  * IBS facility for that VCPU to speed it up.
2590                  */
2591                 __enable_ibs_on_vcpu(started_vcpu);
2592         }
2593
2594         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2595         return;
2596 }
2597
2598 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2599                                      struct kvm_enable_cap *cap)
2600 {
2601         int r;
2602
2603         if (cap->flags)
2604                 return -EINVAL;
2605
2606         switch (cap->cap) {
2607         case KVM_CAP_S390_CSS_SUPPORT:
2608                 if (!vcpu->kvm->arch.css_support) {
2609                         vcpu->kvm->arch.css_support = 1;
2610                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2611                         trace_kvm_s390_enable_css(vcpu->kvm);
2612                 }
2613                 r = 0;
2614                 break;
2615         default:
2616                 r = -EINVAL;
2617                 break;
2618         }
2619         return r;
2620 }
2621
2622 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2623                                   struct kvm_s390_mem_op *mop)
2624 {
2625         void __user *uaddr = (void __user *)mop->buf;
2626         void *tmpbuf = NULL;
2627         int r, srcu_idx;
2628         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2629                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
2630
2631         if (mop->flags & ~supported_flags)
2632                 return -EINVAL;
2633
2634         if (mop->size > MEM_OP_MAX_SIZE)
2635                 return -E2BIG;
2636
2637         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2638                 tmpbuf = vmalloc(mop->size);
2639                 if (!tmpbuf)
2640                         return -ENOMEM;
2641         }
2642
2643         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2644
2645         switch (mop->op) {
2646         case KVM_S390_MEMOP_LOGICAL_READ:
2647                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2648                         r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2649                         break;
2650                 }
2651                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2652                 if (r == 0) {
2653                         if (copy_to_user(uaddr, tmpbuf, mop->size))
2654                                 r = -EFAULT;
2655                 }
2656                 break;
2657         case KVM_S390_MEMOP_LOGICAL_WRITE:
2658                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2659                         r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2660                         break;
2661                 }
2662                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2663                         r = -EFAULT;
2664                         break;
2665                 }
2666                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2667                 break;
2668         default:
2669                 r = -EINVAL;
2670         }
2671
2672         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2673
2674         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2675                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2676
2677         vfree(tmpbuf);
2678         return r;
2679 }
2680
2681 long kvm_arch_vcpu_ioctl(struct file *filp,
2682                          unsigned int ioctl, unsigned long arg)
2683 {
2684         struct kvm_vcpu *vcpu = filp->private_data;
2685         void __user *argp = (void __user *)arg;
2686         int idx;
2687         long r;
2688
2689         switch (ioctl) {
2690         case KVM_S390_IRQ: {
2691                 struct kvm_s390_irq s390irq;
2692
2693                 r = -EFAULT;
2694                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2695                         break;
2696                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2697                 break;
2698         }
2699         case KVM_S390_INTERRUPT: {
2700                 struct kvm_s390_interrupt s390int;
2701                 struct kvm_s390_irq s390irq;
2702
2703                 r = -EFAULT;
2704                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2705                         break;
2706                 if (s390int_to_s390irq(&s390int, &s390irq))
2707                         return -EINVAL;
2708                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2709                 break;
2710         }
2711         case KVM_S390_STORE_STATUS:
2712                 idx = srcu_read_lock(&vcpu->kvm->srcu);
2713                 r = kvm_s390_vcpu_store_status(vcpu, arg);
2714                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2715                 break;
2716         case KVM_S390_SET_INITIAL_PSW: {
2717                 psw_t psw;
2718
2719                 r = -EFAULT;
2720                 if (copy_from_user(&psw, argp, sizeof(psw)))
2721                         break;
2722                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2723                 break;
2724         }
2725         case KVM_S390_INITIAL_RESET:
2726                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2727                 break;
2728         case KVM_SET_ONE_REG:
2729         case KVM_GET_ONE_REG: {
2730                 struct kvm_one_reg reg;
2731                 r = -EFAULT;
2732                 if (copy_from_user(&reg, argp, sizeof(reg)))
2733                         break;
2734                 if (ioctl == KVM_SET_ONE_REG)
2735                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2736                 else
2737                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2738                 break;
2739         }
2740 #ifdef CONFIG_KVM_S390_UCONTROL
2741         case KVM_S390_UCAS_MAP: {
2742                 struct kvm_s390_ucas_mapping ucasmap;
2743
2744                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2745                         r = -EFAULT;
2746                         break;
2747                 }
2748
2749                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2750                         r = -EINVAL;
2751                         break;
2752                 }
2753
2754                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2755                                      ucasmap.vcpu_addr, ucasmap.length);
2756                 break;
2757         }
2758         case KVM_S390_UCAS_UNMAP: {
2759                 struct kvm_s390_ucas_mapping ucasmap;
2760
2761                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2762                         r = -EFAULT;
2763                         break;
2764                 }
2765
2766                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2767                         r = -EINVAL;
2768                         break;
2769                 }
2770
2771                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2772                         ucasmap.length);
2773                 break;
2774         }
2775 #endif
2776         case KVM_S390_VCPU_FAULT: {
2777                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2778                 break;
2779         }
2780         case KVM_ENABLE_CAP:
2781         {
2782                 struct kvm_enable_cap cap;
2783                 r = -EFAULT;
2784                 if (copy_from_user(&cap, argp, sizeof(cap)))
2785                         break;
2786                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2787                 break;
2788         }
2789         case KVM_S390_MEM_OP: {
2790                 struct kvm_s390_mem_op mem_op;
2791
2792                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2793                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2794                 else
2795                         r = -EFAULT;
2796                 break;
2797         }
2798         case KVM_S390_SET_IRQ_STATE: {
2799                 struct kvm_s390_irq_state irq_state;
2800
2801                 r = -EFAULT;
2802                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2803                         break;
2804                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2805                     irq_state.len == 0 ||
2806                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2807                         r = -EINVAL;
2808                         break;
2809                 }
2810                 r = kvm_s390_set_irq_state(vcpu,
2811                                            (void __user *) irq_state.buf,
2812                                            irq_state.len);
2813                 break;
2814         }
2815         case KVM_S390_GET_IRQ_STATE: {
2816                 struct kvm_s390_irq_state irq_state;
2817
2818                 r = -EFAULT;
2819                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2820                         break;
2821                 if (irq_state.len == 0) {
2822                         r = -EINVAL;
2823                         break;
2824                 }
2825                 r = kvm_s390_get_irq_state(vcpu,
2826                                            (__u8 __user *)  irq_state.buf,
2827                                            irq_state.len);
2828                 break;
2829         }
2830         default:
2831                 r = -ENOTTY;
2832         }
2833         return r;
2834 }
2835
2836 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2837 {
2838 #ifdef CONFIG_KVM_S390_UCONTROL
2839         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2840                  && (kvm_is_ucontrol(vcpu->kvm))) {
2841                 vmf->page = virt_to_page(vcpu->arch.sie_block);
2842                 get_page(vmf->page);
2843                 return 0;
2844         }
2845 #endif
2846         return VM_FAULT_SIGBUS;
2847 }
2848
2849 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2850                             unsigned long npages)
2851 {
2852         return 0;
2853 }
2854
2855 /* Section: memory related */
2856 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2857                                    struct kvm_memory_slot *memslot,
2858                                    const struct kvm_userspace_memory_region *mem,
2859                                    enum kvm_mr_change change)
2860 {
2861         /* A few sanity checks. We can have memory slots which have to be
2862            located/ended at a segment boundary (1MB). The memory in userland is
2863            ok to be fragmented into various different vmas. It is okay to mmap()
2864            and munmap() stuff in this slot after doing this call at any time */
2865
2866         if (mem->userspace_addr & 0xffffful)
2867                 return -EINVAL;
2868
2869         if (mem->memory_size & 0xffffful)
2870                 return -EINVAL;
2871
2872         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
2873                 return -EINVAL;
2874
2875         return 0;
2876 }
2877
2878 void kvm_arch_commit_memory_region(struct kvm *kvm,
2879                                 const struct kvm_userspace_memory_region *mem,
2880                                 const struct kvm_memory_slot *old,
2881                                 const struct kvm_memory_slot *new,
2882                                 enum kvm_mr_change change)
2883 {
2884         int rc;
2885
2886         /* If the basics of the memslot do not change, we do not want
2887          * to update the gmap. Every update causes several unnecessary
2888          * segment translation exceptions. This is usually handled just
2889          * fine by the normal fault handler + gmap, but it will also
2890          * cause faults on the prefix page of running guest CPUs.
2891          */
2892         if (old->userspace_addr == mem->userspace_addr &&
2893             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2894             old->npages * PAGE_SIZE == mem->memory_size)
2895                 return;
2896
2897         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2898                 mem->guest_phys_addr, mem->memory_size);
2899         if (rc)
2900                 pr_warn("failed to commit memory region\n");
2901         return;
2902 }
2903
2904 static int __init kvm_s390_init(void)
2905 {
2906         if (!sclp.has_sief2) {
2907                 pr_info("SIE not available\n");
2908                 return -ENODEV;
2909         }
2910
2911         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2912 }
2913
2914 static void __exit kvm_s390_exit(void)
2915 {
2916         kvm_exit();
2917 }
2918
2919 module_init(kvm_s390_init);
2920 module_exit(kvm_s390_exit);
2921
2922 /*
2923  * Enable autoloading of the kvm module.
2924  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2925  * since x86 takes a different approach.
2926  */
2927 #include <linux/miscdevice.h>
2928 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2929 MODULE_ALIAS("devname:kvm");