arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/slab.h>
  26 #include <linux/timer.h>
  27 #include <asm/asm-offsets.h>
  28 #include <asm/lowcore.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/nmi.h>
  31 #include <asm/switch_to.h>
  32 #include <asm/facility.h>
  33 #include <asm/sclp.h>
  34 #include "kvm-s390.h"
  35 #include "gaccess.h"
  36
  37 #define CREATE_TRACE_POINTS
  38 #include "trace.h"
  39 #include "trace-s390.h"
  40
  41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  42
  43 struct kvm_stats_debugfs_item debugfs_entries[] = {
  44         { "userspace_handled", VCPU_STAT(exit_userspace) },
  45         { "exit_null", VCPU_STAT(exit_null) },
  46         { "exit_validity", VCPU_STAT(exit_validity) },
  47         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  48         { "exit_external_request", VCPU_STAT(exit_external_request) },
  49         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  50         { "exit_instruction", VCPU_STAT(exit_instruction) },
  51         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  52         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  53         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  54         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  55         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  56         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  57         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  58         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  59         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  60         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  61         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  62         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  63         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  64         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  65         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  66         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  67         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  68         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  69         { "instruction_spx", VCPU_STAT(instruction_spx) },
  70         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  71         { "instruction_stap", VCPU_STAT(instruction_stap) },
  72         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  73         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  74         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  75         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  76         { "instruction_essa", VCPU_STAT(instruction_essa) },
  77         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  78         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  79         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  80         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  81         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  82         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  83         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  84         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
  85         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
  86         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
  87         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
  88         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
  89         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
  90         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
  91         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
  92         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
  93         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
  94         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
  95         { "diagnose_10", VCPU_STAT(diagnose_10) },
  96         { "diagnose_44", VCPU_STAT(diagnose_44) },
  97         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
  98         { NULL }
  99 };
 100
 101 unsigned long *vfacilities;
 102 static struct gmap_notifier gmap_notifier;
 103
 104 /* test availability of vfacility */
 105 int test_vfacility(unsigned long nr)
 106 {
 107         return __test_facility(nr, (void *) vfacilities);
 108 }
 109
 110 /* Section: not file related */
 111 int kvm_arch_hardware_enable(void)
 112 {
 113         /* every s390 is virtualization enabled ;-) */
 114         return 0;
 115 }
 116
 117 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 118
 119 int kvm_arch_hardware_setup(void)
 120 {
 121         gmap_notifier.notifier_call = kvm_gmap_notifier;
 122         gmap_register_ipte_notifier(&gmap_notifier);
 123         return 0;
 124 }
 125
 126 void kvm_arch_hardware_unsetup(void)
 127 {
 128         gmap_unregister_ipte_notifier(&gmap_notifier);
 129 }
 130
 131 int kvm_arch_init(void *opaque)
 132 {
 133         /* Register floating interrupt controller interface. */
 134         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 135 }
 136
 137 /* Section: device related */
 138 long kvm_arch_dev_ioctl(struct file *filp,
 139                         unsigned int ioctl, unsigned long arg)
 140 {
 141         if (ioctl == KVM_S390_ENABLE_SIE)
 142                 return s390_enable_sie();
 143         return -EINVAL;
 144 }
 145
 146 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 147 {
 148         int r;
 149
 150         switch (ext) {
 151         case KVM_CAP_S390_PSW:
 152         case KVM_CAP_S390_GMAP:
 153         case KVM_CAP_SYNC_MMU:
 154 #ifdef CONFIG_KVM_S390_UCONTROL
 155         case KVM_CAP_S390_UCONTROL:
 156 #endif
 157         case KVM_CAP_ASYNC_PF:
 158         case KVM_CAP_SYNC_REGS:
 159         case KVM_CAP_ONE_REG:
 160         case KVM_CAP_ENABLE_CAP:
 161         case KVM_CAP_S390_CSS_SUPPORT:
 162         case KVM_CAP_IRQFD:
 163         case KVM_CAP_IOEVENTFD:
 164         case KVM_CAP_DEVICE_CTRL:
 165         case KVM_CAP_ENABLE_CAP_VM:
 166         case KVM_CAP_S390_IRQCHIP:
 167         case KVM_CAP_VM_ATTRIBUTES:
 168         case KVM_CAP_MP_STATE:
 169                 r = 1;
 170                 break;
 171         case KVM_CAP_NR_VCPUS:
 172         case KVM_CAP_MAX_VCPUS:
 173                 r = KVM_MAX_VCPUS;
 174                 break;
 175         case KVM_CAP_NR_MEMSLOTS:
 176                 r = KVM_USER_MEM_SLOTS;
 177                 break;
 178         case KVM_CAP_S390_COW:
 179                 r = MACHINE_HAS_ESOP;
 180                 break;
 181         default:
 182                 r = 0;
 183         }
 184         return r;
 185 }
 186
 187 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 188                                         struct kvm_memory_slot *memslot)
 189 {
 190         gfn_t cur_gfn, last_gfn;
 191         unsigned long address;
 192         struct gmap *gmap = kvm->arch.gmap;
 193
 194         down_read(&gmap->mm->mmap_sem);
 195         /* Loop over all guest pages */
 196         last_gfn = memslot->base_gfn + memslot->npages;
 197         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 198                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 199
 200                 if (gmap_test_and_clear_dirty(address, gmap))
 201                         mark_page_dirty(kvm, cur_gfn);
 202         }
 203         up_read(&gmap->mm->mmap_sem);
 204 }
 205
 206 /* Section: vm related */
 207 /*
 208  * Get (and clear) the dirty memory log for a memory slot.
 209  */
 210 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 211                                struct kvm_dirty_log *log)
 212 {
 213         int r;
 214         unsigned long n;
 215         struct kvm_memory_slot *memslot;
 216         int is_dirty = 0;
 217
 218         mutex_lock(&kvm->slots_lock);
 219
 220         r = -EINVAL;
 221         if (log->slot >= KVM_USER_MEM_SLOTS)
 222                 goto out;
 223
 224         memslot = id_to_memslot(kvm->memslots, log->slot);
 225         r = -ENOENT;
 226         if (!memslot->dirty_bitmap)
 227                 goto out;
 228
 229         kvm_s390_sync_dirty_log(kvm, memslot);
 230         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 231         if (r)
 232                 goto out;
 233
 234         /* Clear the dirty log */
 235         if (is_dirty) {
 236                 n = kvm_dirty_bitmap_bytes(memslot);
 237                 memset(memslot->dirty_bitmap, 0, n);
 238         }
 239         r = 0;
 240 out:
 241         mutex_unlock(&kvm->slots_lock);
 242         return r;
 243 }
 244
 245 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 246 {
 247         int r;
 248
 249         if (cap->flags)
 250                 return -EINVAL;
 251
 252         switch (cap->cap) {
 253         case KVM_CAP_S390_IRQCHIP:
 254                 kvm->arch.use_irqchip = 1;
 255                 r = 0;
 256                 break;
 257         default:
 258                 r = -EINVAL;
 259                 break;
 260         }
 261         return r;
 262 }
 263
 264 static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 265 {
 266         int ret;
 267         unsigned int idx;
 268         switch (attr->attr) {
 269         case KVM_S390_VM_MEM_ENABLE_CMMA:
 270                 ret = -EBUSY;
 271                 mutex_lock(&kvm->lock);
 272                 if (atomic_read(&kvm->online_vcpus) == 0) {
 273                         kvm->arch.use_cmma = 1;
 274                         ret = 0;
 275                 }
 276                 mutex_unlock(&kvm->lock);
 277                 break;
 278         case KVM_S390_VM_MEM_CLR_CMMA:
 279                 mutex_lock(&kvm->lock);
 280                 idx = srcu_read_lock(&kvm->srcu);
 281                 s390_reset_cmma(kvm->arch.gmap->mm);
 282                 srcu_read_unlock(&kvm->srcu, idx);
 283                 mutex_unlock(&kvm->lock);
 284                 ret = 0;
 285                 break;
 286         default:
 287                 ret = -ENXIO;
 288                 break;
 289         }
 290         return ret;
 291 }
 292
 293 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 294 {
 295         int ret;
 296
 297         switch (attr->group) {
 298         case KVM_S390_VM_MEM_CTRL:
 299                 ret = kvm_s390_mem_control(kvm, attr);
 300                 break;
 301         default:
 302                 ret = -ENXIO;
 303                 break;
 304         }
 305
 306         return ret;
 307 }
 308
 309 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 310 {
 311         return -ENXIO;
 312 }
 313
 314 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 315 {
 316         int ret;
 317
 318         switch (attr->group) {
 319         case KVM_S390_VM_MEM_CTRL:
 320                 switch (attr->attr) {
 321                 case KVM_S390_VM_MEM_ENABLE_CMMA:
 322                 case KVM_S390_VM_MEM_CLR_CMMA:
 323                         ret = 0;
 324                         break;
 325                 default:
 326                         ret = -ENXIO;
 327                         break;
 328                 }
 329                 break;
 330         default:
 331                 ret = -ENXIO;
 332                 break;
 333         }
 334
 335         return ret;
 336 }
 337
 338 long kvm_arch_vm_ioctl(struct file *filp,
 339                        unsigned int ioctl, unsigned long arg)
 340 {
 341         struct kvm *kvm = filp->private_data;
 342         void __user *argp = (void __user *)arg;
 343         struct kvm_device_attr attr;
 344         int r;
 345
 346         switch (ioctl) {
 347         case KVM_S390_INTERRUPT: {
 348                 struct kvm_s390_interrupt s390int;
 349
 350                 r = -EFAULT;
 351                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 352                         break;
 353                 r = kvm_s390_inject_vm(kvm, &s390int);
 354                 break;
 355         }
 356         case KVM_ENABLE_CAP: {
 357                 struct kvm_enable_cap cap;
 358                 r = -EFAULT;
 359                 if (copy_from_user(&cap, argp, sizeof(cap)))
 360                         break;
 361                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 362                 break;
 363         }
 364         case KVM_CREATE_IRQCHIP: {
 365                 struct kvm_irq_routing_entry routing;
 366
 367                 r = -EINVAL;
 368                 if (kvm->arch.use_irqchip) {
 369                         /* Set up dummy routing. */
 370                         memset(&routing, 0, sizeof(routing));
 371                         kvm_set_irq_routing(kvm, &routing, 0, 0);
 372                         r = 0;
 373                 }
 374                 break;
 375         }
 376         case KVM_SET_DEVICE_ATTR: {
 377                 r = -EFAULT;
 378                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 379                         break;
 380                 r = kvm_s390_vm_set_attr(kvm, &attr);
 381                 break;
 382         }
 383         case KVM_GET_DEVICE_ATTR: {
 384                 r = -EFAULT;
 385                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 386                         break;
 387                 r = kvm_s390_vm_get_attr(kvm, &attr);
 388                 break;
 389         }
 390         case KVM_HAS_DEVICE_ATTR: {
 391                 r = -EFAULT;
 392                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 393                         break;
 394                 r = kvm_s390_vm_has_attr(kvm, &attr);
 395                 break;
 396         }
 397         default:
 398                 r = -ENOTTY;
 399         }
 400
 401         return r;
 402 }
 403
 404 static int kvm_s390_crypto_init(struct kvm *kvm)
 405 {
 406         if (!test_vfacility(76))
 407                 return 0;
 408
 409         kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
 410                                          GFP_KERNEL | GFP_DMA);
 411         if (!kvm->arch.crypto.crycb)
 412                 return -ENOMEM;
 413
 414         kvm->arch.crypto.crycbd = (__u32) (unsigned long) kvm->arch.crypto.crycb |
 415                                   CRYCB_FORMAT1;
 416
 417         return 0;
 418 }
 419
 420 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 421 {
 422         int rc;
 423         char debug_name[16];
 424         static unsigned long sca_offset;
 425
 426         rc = -EINVAL;
 427 #ifdef CONFIG_KVM_S390_UCONTROL
 428         if (type & ~KVM_VM_S390_UCONTROL)
 429                 goto out_err;
 430         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
 431                 goto out_err;
 432 #else
 433         if (type)
 434                 goto out_err;
 435 #endif
 436
 437         rc = s390_enable_sie();
 438         if (rc)
 439                 goto out_err;
 440
 441         rc = -ENOMEM;
 442
 443         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
 444         if (!kvm->arch.sca)
 445                 goto out_err;
 446         spin_lock(&kvm_lock);
 447         sca_offset = (sca_offset + 16) & 0x7f0;
 448         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
 449         spin_unlock(&kvm_lock);
 450
 451         sprintf(debug_name, "kvm-%u", current->pid);
 452
 453         kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
 454         if (!kvm->arch.dbf)
 455                 goto out_nodbf;
 456
 457         if (kvm_s390_crypto_init(kvm) < 0)
 458                 goto out_crypto;
 459
 460         spin_lock_init(&kvm->arch.float_int.lock);
 461         INIT_LIST_HEAD(&kvm->arch.float_int.list);
 462         init_waitqueue_head(&kvm->arch.ipte_wq);
 463         mutex_init(&kvm->arch.ipte_mutex);
 464
 465         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
 466         VM_EVENT(kvm, 3, "%s", "vm created");
 467
 468         if (type & KVM_VM_S390_UCONTROL) {
 469                 kvm->arch.gmap = NULL;
 470         } else {
 471                 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
 472                 if (!kvm->arch.gmap)
 473                         goto out_nogmap;
 474                 kvm->arch.gmap->private = kvm;
 475                 kvm->arch.gmap->pfault_enabled = 0;
 476         }
 477
 478         kvm->arch.css_support = 0;
 479         kvm->arch.use_irqchip = 0;
 480
 481         spin_lock_init(&kvm->arch.start_stop_lock);
 482
 483         return 0;
 484 out_nogmap:
 485         kfree(kvm->arch.crypto.crycb);
 486 out_crypto:
 487         debug_unregister(kvm->arch.dbf);
 488 out_nodbf:
 489         free_page((unsigned long)(kvm->arch.sca));
 490 out_err:
 491         return rc;
 492 }
 493
 494 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 495 {
 496         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
 497         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
 498         kvm_s390_clear_local_irqs(vcpu);
 499         kvm_clear_async_pf_completion_queue(vcpu);
 500         if (!kvm_is_ucontrol(vcpu->kvm)) {
 501                 clear_bit(63 - vcpu->vcpu_id,
 502                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
 503                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
 504                     (__u64) vcpu->arch.sie_block)
 505                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
 506         }
 507         smp_mb();
 508
 509         if (kvm_is_ucontrol(vcpu->kvm))
 510                 gmap_free(vcpu->arch.gmap);
 511
 512         if (kvm_s390_cmma_enabled(vcpu->kvm))
 513                 kvm_s390_vcpu_unsetup_cmma(vcpu);
 514         free_page((unsigned long)(vcpu->arch.sie_block));
 515
 516         kvm_vcpu_uninit(vcpu);
 517         kmem_cache_free(kvm_vcpu_cache, vcpu);
 518 }
 519
 520 static void kvm_free_vcpus(struct kvm *kvm)
 521 {
 522         unsigned int i;
 523         struct kvm_vcpu *vcpu;
 524
 525         kvm_for_each_vcpu(i, vcpu, kvm)
 526                 kvm_arch_vcpu_destroy(vcpu);
 527
 528         mutex_lock(&kvm->lock);
 529         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 530                 kvm->vcpus[i] = NULL;
 531
 532         atomic_set(&kvm->online_vcpus, 0);
 533         mutex_unlock(&kvm->lock);
 534 }
 535
 536 void kvm_arch_destroy_vm(struct kvm *kvm)
 537 {
 538         kvm_free_vcpus(kvm);
 539         free_page((unsigned long)(kvm->arch.sca));
 540         debug_unregister(kvm->arch.dbf);
 541         kfree(kvm->arch.crypto.crycb);
 542         if (!kvm_is_ucontrol(kvm))
 543                 gmap_free(kvm->arch.gmap);
 544         kvm_s390_destroy_adapters(kvm);
 545         kvm_s390_clear_float_irqs(kvm);
 546 }
 547
 548 /* Section: vcpu related */
 549 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
 550 {
 551         vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
 552         if (!vcpu->arch.gmap)
 553                 return -ENOMEM;
 554         vcpu->arch.gmap->private = vcpu->kvm;
 555
 556         return 0;
 557 }
 558
 559 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 560 {
 561         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 562         kvm_clear_async_pf_completion_queue(vcpu);
 563         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
 564                                     KVM_SYNC_GPRS |
 565                                     KVM_SYNC_ACRS |
 566                                     KVM_SYNC_CRS |
 567                                     KVM_SYNC_ARCH0 |
 568                                     KVM_SYNC_PFAULT;
 569
 570         if (kvm_is_ucontrol(vcpu->kvm))
 571                 return __kvm_ucontrol_vcpu_init(vcpu);
 572
 573         return 0;
 574 }
 575
 576 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 577 {
 578         save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 579         save_fp_regs(vcpu->arch.host_fpregs.fprs);
 580         save_access_regs(vcpu->arch.host_acrs);
 581         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 582         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 583         restore_access_regs(vcpu->run->s.regs.acrs);
 584         gmap_enable(vcpu->arch.gmap);
 585         atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 586 }
 587
 588 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 589 {
 590         atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
 591         gmap_disable(vcpu->arch.gmap);
 592         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 593         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
 594         save_access_regs(vcpu->run->s.regs.acrs);
 595         restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
 596         restore_fp_regs(vcpu->arch.host_fpregs.fprs);
 597         restore_access_regs(vcpu->arch.host_acrs);
 598 }
 599
 600 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
 601 {
 602         /* this equals initial cpu reset in pop, but we don't switch to ESA */
 603         vcpu->arch.sie_block->gpsw.mask = 0UL;
 604         vcpu->arch.sie_block->gpsw.addr = 0UL;
 605         kvm_s390_set_prefix(vcpu, 0);
 606         vcpu->arch.sie_block->cputm     = 0UL;
 607         vcpu->arch.sie_block->ckc       = 0UL;
 608         vcpu->arch.sie_block->todpr     = 0;
 609         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
 610         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
 611         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
 612         vcpu->arch.guest_fpregs.fpc = 0;
 613         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
 614         vcpu->arch.sie_block->gbea = 1;
 615         vcpu->arch.sie_block->pp = 0;
 616         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
 617         kvm_clear_async_pf_completion_queue(vcpu);
 618         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
 619                 kvm_s390_vcpu_stop(vcpu);
 620         kvm_s390_clear_local_irqs(vcpu);
 621 }
 622
 623 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 624 {
 625         if (!kvm_is_ucontrol(vcpu->kvm))
 626                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
 627 }
 628
 629 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
 630 {
 631         if (!test_vfacility(76))
 632                 return;
 633
 634         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
 635 }
 636
 637 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
 638 {
 639         free_page(vcpu->arch.sie_block->cbrlo);
 640         vcpu->arch.sie_block->cbrlo = 0;
 641 }
 642
 643 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
 644 {
 645         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
 646         if (!vcpu->arch.sie_block->cbrlo)
 647                 return -ENOMEM;
 648
 649         vcpu->arch.sie_block->ecb2 |= 0x80;
 650         vcpu->arch.sie_block->ecb2 &= ~0x08;
 651         return 0;
 652 }
 653
 654 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 655 {
 656         int rc = 0;
 657
 658         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
 659                                                     CPUSTAT_SM |
 660                                                     CPUSTAT_STOPPED |
 661                                                     CPUSTAT_GED);
 662         vcpu->arch.sie_block->ecb   = 6;
 663         if (test_vfacility(50) && test_vfacility(73))
 664                 vcpu->arch.sie_block->ecb |= 0x10;
 665
 666         vcpu->arch.sie_block->ecb2  = 8;
 667         vcpu->arch.sie_block->eca   = 0xD1002000U;
 668         if (sclp_has_siif())
 669                 vcpu->arch.sie_block->eca |= 1;
 670         vcpu->arch.sie_block->fac   = (int) (long) vfacilities;
 671         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
 672                                       ICTL_TPROT;
 673
 674         if (kvm_s390_cmma_enabled(vcpu->kvm)) {
 675                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
 676                 if (rc)
 677                         return rc;
 678         }
 679         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 680         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
 681         get_cpu_id(&vcpu->arch.cpu_id);
 682         vcpu->arch.cpu_id.version = 0xff;
 683
 684         kvm_s390_vcpu_crypto_setup(vcpu);
 685
 686         return rc;
 687 }
 688
 689 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 690                                       unsigned int id)
 691 {
 692         struct kvm_vcpu *vcpu;
 693         struct sie_page *sie_page;
 694         int rc = -EINVAL;
 695
 696         if (id >= KVM_MAX_VCPUS)
 697                 goto out;
 698
 699         rc = -ENOMEM;
 700
 701         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
 702         if (!vcpu)
 703                 goto out;
 704
 705         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
 706         if (!sie_page)
 707                 goto out_free_cpu;
 708
 709         vcpu->arch.sie_block = &sie_page->sie_block;
 710         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
 711
 712         vcpu->arch.sie_block->icpua = id;
 713         if (!kvm_is_ucontrol(kvm)) {
 714                 if (!kvm->arch.sca) {
 715                         WARN_ON_ONCE(1);
 716                         goto out_free_cpu;
 717                 }
 718                 if (!kvm->arch.sca->cpu[id].sda)
 719                         kvm->arch.sca->cpu[id].sda =
 720                                 (__u64) vcpu->arch.sie_block;
 721                 vcpu->arch.sie_block->scaoh =
 722                         (__u32)(((__u64)kvm->arch.sca) >> 32);
 723                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
 724                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
 725         }
 726
 727         spin_lock_init(&vcpu->arch.local_int.lock);
 728         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
 729         vcpu->arch.local_int.wq = &vcpu->wq;
 730         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
 731
 732         rc = kvm_vcpu_init(vcpu, kvm, id);
 733         if (rc)
 734                 goto out_free_sie_block;
 735         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
 736                  vcpu->arch.sie_block);
 737         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
 738
 739         return vcpu;
 740 out_free_sie_block:
 741         free_page((unsigned long)(vcpu->arch.sie_block));
 742 out_free_cpu:
 743         kmem_cache_free(kvm_vcpu_cache, vcpu);
 744 out:
 745         return ERR_PTR(rc);
 746 }
 747
 748 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
 749 {
 750         return kvm_cpu_has_interrupt(vcpu);
 751 }
 752
 753 void s390_vcpu_block(struct kvm_vcpu *vcpu)
 754 {
 755         atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 756 }
 757
 758 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
 759 {
 760         atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
 761 }
 762
 763 /*
 764  * Kick a guest cpu out of SIE and wait until SIE is not running.
 765  * If the CPU is not running (e.g. waiting as idle) the function will
 766  * return immediately. */
 767 void exit_sie(struct kvm_vcpu *vcpu)
 768 {
 769         atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
 770         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
 771                 cpu_relax();
 772 }
 773
 774 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
 775 void exit_sie_sync(struct kvm_vcpu *vcpu)
 776 {
 777         s390_vcpu_block(vcpu);
 778         exit_sie(vcpu);
 779 }
 780
 781 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
 782 {
 783         int i;
 784         struct kvm *kvm = gmap->private;
 785         struct kvm_vcpu *vcpu;
 786
 787         kvm_for_each_vcpu(i, vcpu, kvm) {
 788                 /* match against both prefix pages */
 789                 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
 790                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
 791                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
 792                         exit_sie_sync(vcpu);
 793                 }
 794         }
 795 }
 796
 797 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
 798 {
 799         /* kvm common code refers to this, but never calls it */
 800         BUG();
 801         return 0;
 802 }
 803
 804 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
 805                                            struct kvm_one_reg *reg)
 806 {
 807         int r = -EINVAL;
 808
 809         switch (reg->id) {
 810         case KVM_REG_S390_TODPR:
 811                 r = put_user(vcpu->arch.sie_block->todpr,
 812                              (u32 __user *)reg->addr);
 813                 break;
 814         case KVM_REG_S390_EPOCHDIFF:
 815                 r = put_user(vcpu->arch.sie_block->epoch,
 816                              (u64 __user *)reg->addr);
 817                 break;
 818         case KVM_REG_S390_CPU_TIMER:
 819                 r = put_user(vcpu->arch.sie_block->cputm,
 820                              (u64 __user *)reg->addr);
 821                 break;
 822         case KVM_REG_S390_CLOCK_COMP:
 823                 r = put_user(vcpu->arch.sie_block->ckc,
 824                              (u64 __user *)reg->addr);
 825                 break;
 826         case KVM_REG_S390_PFTOKEN:
 827                 r = put_user(vcpu->arch.pfault_token,
 828                              (u64 __user *)reg->addr);
 829                 break;
 830         case KVM_REG_S390_PFCOMPARE:
 831                 r = put_user(vcpu->arch.pfault_compare,
 832                              (u64 __user *)reg->addr);
 833                 break;
 834         case KVM_REG_S390_PFSELECT:
 835                 r = put_user(vcpu->arch.pfault_select,
 836                              (u64 __user *)reg->addr);
 837                 break;
 838         case KVM_REG_S390_PP:
 839                 r = put_user(vcpu->arch.sie_block->pp,
 840                              (u64 __user *)reg->addr);
 841                 break;
 842         case KVM_REG_S390_GBEA:
 843                 r = put_user(vcpu->arch.sie_block->gbea,
 844                              (u64 __user *)reg->addr);
 845                 break;
 846         default:
 847                 break;
 848         }
 849
 850         return r;
 851 }
 852
 853 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
 854                                            struct kvm_one_reg *reg)
 855 {
 856         int r = -EINVAL;
 857
 858         switch (reg->id) {
 859         case KVM_REG_S390_TODPR:
 860                 r = get_user(vcpu->arch.sie_block->todpr,
 861                              (u32 __user *)reg->addr);
 862                 break;
 863         case KVM_REG_S390_EPOCHDIFF:
 864                 r = get_user(vcpu->arch.sie_block->epoch,
 865                              (u64 __user *)reg->addr);
 866                 break;
 867         case KVM_REG_S390_CPU_TIMER:
 868                 r = get_user(vcpu->arch.sie_block->cputm,
 869                              (u64 __user *)reg->addr);
 870                 break;
 871         case KVM_REG_S390_CLOCK_COMP:
 872                 r = get_user(vcpu->arch.sie_block->ckc,
 873                              (u64 __user *)reg->addr);
 874                 break;
 875         case KVM_REG_S390_PFTOKEN:
 876                 r = get_user(vcpu->arch.pfault_token,
 877                              (u64 __user *)reg->addr);
 878                 break;
 879         case KVM_REG_S390_PFCOMPARE:
 880                 r = get_user(vcpu->arch.pfault_compare,
 881                              (u64 __user *)reg->addr);
 882                 break;
 883         case KVM_REG_S390_PFSELECT:
 884                 r = get_user(vcpu->arch.pfault_select,
 885                              (u64 __user *)reg->addr);
 886                 break;
 887         case KVM_REG_S390_PP:
 888                 r = get_user(vcpu->arch.sie_block->pp,
 889                              (u64 __user *)reg->addr);
 890                 break;
 891         case KVM_REG_S390_GBEA:
 892                 r = get_user(vcpu->arch.sie_block->gbea,
 893                              (u64 __user *)reg->addr);
 894                 break;
 895         default:
 896                 break;
 897         }
 898
 899         return r;
 900 }
 901
 902 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 903 {
 904         kvm_s390_vcpu_initial_reset(vcpu);
 905         return 0;
 906 }
 907
 908 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 909 {
 910         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
 911         return 0;
 912 }
 913
 914 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 915 {
 916         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
 917         return 0;
 918 }
 919
 920 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 921                                   struct kvm_sregs *sregs)
 922 {
 923         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
 924         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
 925         restore_access_regs(vcpu->run->s.regs.acrs);
 926         return 0;
 927 }
 928
 929 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 930                                   struct kvm_sregs *sregs)
 931 {
 932         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
 933         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
 934         return 0;
 935 }
 936
 937 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 938 {
 939         if (test_fp_ctl(fpu->fpc))
 940                 return -EINVAL;
 941         memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
 942         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
 943         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
 944         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
 945         return 0;
 946 }
 947
 948 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 949 {
 950         memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
 951         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
 952         return 0;
 953 }
 954
 955 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
 956 {
 957         int rc = 0;
 958
 959         if (!is_vcpu_stopped(vcpu))
 960                 rc = -EBUSY;
 961         else {
 962                 vcpu->run->psw_mask = psw.mask;
 963                 vcpu->run->psw_addr = psw.addr;
 964         }
 965         return rc;
 966 }
 967
 968 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 969                                   struct kvm_translation *tr)
 970 {
 971         return -EINVAL; /* not implemented yet */
 972 }
 973
 974 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
 975                               KVM_GUESTDBG_USE_HW_BP | \
 976                               KVM_GUESTDBG_ENABLE)
 977
 978 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 979                                         struct kvm_guest_debug *dbg)
 980 {
 981         int rc = 0;
 982
 983         vcpu->guest_debug = 0;
 984         kvm_s390_clear_bp_data(vcpu);
 985
 986         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
 987                 return -EINVAL;
 988
 989         if (dbg->control & KVM_GUESTDBG_ENABLE) {
 990                 vcpu->guest_debug = dbg->control;
 991                 /* enforce guest PER */
 992                 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
 993
 994                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
 995                         rc = kvm_s390_import_bp_data(vcpu, dbg);
 996         } else {
 997                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
 998                 vcpu->arch.guestdbg.last_bp = 0;
 999         }
1000
1001         if (rc) {
1002                 vcpu->guest_debug = 0;
1003                 kvm_s390_clear_bp_data(vcpu);
1004                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1005         }
1006
1007         return rc;
1008 }
1009
1010 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1011                                     struct kvm_mp_state *mp_state)
1012 {
1013         /* CHECK_STOP and LOAD are not supported yet */
1014         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1015                                        KVM_MP_STATE_OPERATING;
1016 }
1017
1018 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1019                                     struct kvm_mp_state *mp_state)
1020 {
1021         int rc = 0;
1022
1023         /* user space knows about this interface - let it control the state */
1024         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1025
1026         switch (mp_state->mp_state) {
1027         case KVM_MP_STATE_STOPPED:
1028                 kvm_s390_vcpu_stop(vcpu);
1029                 break;
1030         case KVM_MP_STATE_OPERATING:
1031                 kvm_s390_vcpu_start(vcpu);
1032                 break;
1033         case KVM_MP_STATE_LOAD:
1034         case KVM_MP_STATE_CHECK_STOP:
1035                 /* fall through - CHECK_STOP and LOAD are not supported yet */
1036         default:
1037                 rc = -ENXIO;
1038         }
1039
1040         return rc;
1041 }
1042
1043 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1044 {
1045         if (!MACHINE_IS_LPAR)
1046                 return false;
1047         /* only enable for z10 and later */
1048         if (!MACHINE_HAS_EDAT1)
1049                 return false;
1050         if (!kvm->arch.use_cmma)
1051                 return false;
1052         return true;
1053 }
1054
1055 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1056 {
1057         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1058 }
1059
1060 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1061 {
1062 retry:
1063         s390_vcpu_unblock(vcpu);
1064         /*
1065          * We use MMU_RELOAD just to re-arm the ipte notifier for the
1066          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1067          * This ensures that the ipte instruction for this request has
1068          * already finished. We might race against a second unmapper that
1069          * wants to set the blocking bit. Lets just retry the request loop.
1070          */
1071         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1072                 int rc;
1073                 rc = gmap_ipte_notify(vcpu->arch.gmap,
1074                                       kvm_s390_get_prefix(vcpu),
1075                                       PAGE_SIZE * 2);
1076                 if (rc)
1077                         return rc;
1078                 goto retry;
1079         }
1080
1081         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1082                 vcpu->arch.sie_block->ihcpu = 0xffff;
1083                 goto retry;
1084         }
1085
1086         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1087                 if (!ibs_enabled(vcpu)) {
1088                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1089                         atomic_set_mask(CPUSTAT_IBS,
1090                                         &vcpu->arch.sie_block->cpuflags);
1091                 }
1092                 goto retry;
1093         }
1094
1095         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1096                 if (ibs_enabled(vcpu)) {
1097                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1098                         atomic_clear_mask(CPUSTAT_IBS,
1099                                           &vcpu->arch.sie_block->cpuflags);
1100                 }
1101                 goto retry;
1102         }
1103
1104         /* nothing to do, just clear the request */
1105         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1106
1107         return 0;
1108 }
1109
1110 /**
1111  * kvm_arch_fault_in_page - fault-in guest page if necessary
1112  * @vcpu: The corresponding virtual cpu
1113  * @gpa: Guest physical address
1114  * @writable: Whether the page should be writable or not
1115  *
1116  * Make sure that a guest page has been faulted-in on the host.
1117  *
1118  * Return: Zero on success, negative error code otherwise.
1119  */
1120 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1121 {
1122         return gmap_fault(vcpu->arch.gmap, gpa,
1123                           writable ? FAULT_FLAG_WRITE : 0);
1124 }
1125
1126 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1127                                       unsigned long token)
1128 {
1129         struct kvm_s390_interrupt inti;
1130         struct kvm_s390_irq irq;
1131
1132         if (start_token) {
1133                 irq.u.ext.ext_params2 = token;
1134                 irq.type = KVM_S390_INT_PFAULT_INIT;
1135                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1136         } else {
1137                 inti.type = KVM_S390_INT_PFAULT_DONE;
1138                 inti.parm64 = token;
1139                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1140         }
1141 }
1142
1143 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1144                                      struct kvm_async_pf *work)
1145 {
1146         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1147         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1148 }
1149
1150 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1151                                  struct kvm_async_pf *work)
1152 {
1153         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1154         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1155 }
1156
1157 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1158                                struct kvm_async_pf *work)
1159 {
1160         /* s390 will always inject the page directly */
1161 }
1162
1163 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1164 {
1165         /*
1166          * s390 will always inject the page directly,
1167          * but we still want check_async_completion to cleanup
1168          */
1169         return true;
1170 }
1171
1172 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1173 {
1174         hva_t hva;
1175         struct kvm_arch_async_pf arch;
1176         int rc;
1177
1178         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1179                 return 0;
1180         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1181             vcpu->arch.pfault_compare)
1182                 return 0;
1183         if (psw_extint_disabled(vcpu))
1184                 return 0;
1185         if (kvm_cpu_has_interrupt(vcpu))
1186                 return 0;
1187         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1188                 return 0;
1189         if (!vcpu->arch.gmap->pfault_enabled)
1190                 return 0;
1191
1192         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1193         hva += current->thread.gmap_addr & ~PAGE_MASK;
1194         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1195                 return 0;
1196
1197         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1198         return rc;
1199 }
1200
1201 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1202 {
1203         int rc, cpuflags;
1204
1205         /*
1206          * On s390 notifications for arriving pages will be delivered directly
1207          * to the guest but the house keeping for completed pfaults is
1208          * handled outside the worker.
1209          */
1210         kvm_check_async_pf_completion(vcpu);
1211
1212         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1213
1214         if (need_resched())
1215                 schedule();
1216
1217         if (test_cpu_flag(CIF_MCCK_PENDING))
1218                 s390_handle_mcck();
1219
1220         if (!kvm_is_ucontrol(vcpu->kvm)) {
1221                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1222                 if (rc)
1223                         return rc;
1224         }
1225
1226         rc = kvm_s390_handle_requests(vcpu);
1227         if (rc)
1228                 return rc;
1229
1230         if (guestdbg_enabled(vcpu)) {
1231                 kvm_s390_backup_guest_per_regs(vcpu);
1232                 kvm_s390_patch_guest_per_regs(vcpu);
1233         }
1234
1235         vcpu->arch.sie_block->icptcode = 0;
1236         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1237         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1238         trace_kvm_s390_sie_enter(vcpu, cpuflags);
1239
1240         return 0;
1241 }
1242
1243 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1244 {
1245         int rc = -1;
1246
1247         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1248                    vcpu->arch.sie_block->icptcode);
1249         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1250
1251         if (guestdbg_enabled(vcpu))
1252                 kvm_s390_restore_guest_per_regs(vcpu);
1253
1254         if (exit_reason >= 0) {
1255                 rc = 0;
1256         } else if (kvm_is_ucontrol(vcpu->kvm)) {
1257                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1258                 vcpu->run->s390_ucontrol.trans_exc_code =
1259                                                 current->thread.gmap_addr;
1260                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1261                 rc = -EREMOTE;
1262
1263         } else if (current->thread.gmap_pfault) {
1264                 trace_kvm_s390_major_guest_pfault(vcpu);
1265                 current->thread.gmap_pfault = 0;
1266                 if (kvm_arch_setup_async_pf(vcpu)) {
1267                         rc = 0;
1268                 } else {
1269                         gpa_t gpa = current->thread.gmap_addr;
1270                         rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1271                 }
1272         }
1273
1274         if (rc == -1) {
1275                 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1276                 trace_kvm_s390_sie_fault(vcpu);
1277                 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1278         }
1279
1280         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1281
1282         if (rc == 0) {
1283                 if (kvm_is_ucontrol(vcpu->kvm))
1284                         /* Don't exit for host interrupts. */
1285                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1286                 else
1287                         rc = kvm_handle_sie_intercept(vcpu);
1288         }
1289
1290         return rc;
1291 }
1292
1293 static int __vcpu_run(struct kvm_vcpu *vcpu)
1294 {
1295         int rc, exit_reason;
1296
1297         /*
1298          * We try to hold kvm->srcu during most of vcpu_run (except when run-
1299          * ning the guest), so that memslots (and other stuff) are protected
1300          */
1301         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1302
1303         do {
1304                 rc = vcpu_pre_run(vcpu);
1305                 if (rc)
1306                         break;
1307
1308                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1309                 /*
1310                  * As PF_VCPU will be used in fault handler, between
1311                  * guest_enter and guest_exit should be no uaccess.
1312                  */
1313                 preempt_disable();
1314                 kvm_guest_enter();
1315                 preempt_enable();
1316                 exit_reason = sie64a(vcpu->arch.sie_block,
1317                                      vcpu->run->s.regs.gprs);
1318                 kvm_guest_exit();
1319                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1320
1321                 rc = vcpu_post_run(vcpu, exit_reason);
1322         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1323
1324         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1325         return rc;
1326 }
1327
1328 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1329 {
1330         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1331         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1332         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1333                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1334         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1335                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1336                 /* some control register changes require a tlb flush */
1337                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1338         }
1339         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1340                 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1341                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1342                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1343                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1344                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1345         }
1346         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1347                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1348                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1349                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1350         }
1351         kvm_run->kvm_dirty_regs = 0;
1352 }
1353
1354 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1355 {
1356         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1357         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1358         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1359         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1360         kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1361         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1362         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1363         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1364         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1365         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1366         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1367         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1368 }
1369
1370 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1371 {
1372         int rc;
1373         sigset_t sigsaved;
1374
1375         if (guestdbg_exit_pending(vcpu)) {
1376                 kvm_s390_prepare_debug_exit(vcpu);
1377                 return 0;
1378         }
1379
1380         if (vcpu->sigset_active)
1381                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1382
1383         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1384                 kvm_s390_vcpu_start(vcpu);
1385         } else if (is_vcpu_stopped(vcpu)) {
1386                 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1387                                    vcpu->vcpu_id);
1388                 return -EINVAL;
1389         }
1390
1391         sync_regs(vcpu, kvm_run);
1392
1393         might_fault();
1394         rc = __vcpu_run(vcpu);
1395
1396         if (signal_pending(current) && !rc) {
1397                 kvm_run->exit_reason = KVM_EXIT_INTR;
1398                 rc = -EINTR;
1399         }
1400
1401         if (guestdbg_exit_pending(vcpu) && !rc)  {
1402                 kvm_s390_prepare_debug_exit(vcpu);
1403                 rc = 0;
1404         }
1405
1406         if (rc == -EOPNOTSUPP) {
1407                 /* intercept cannot be handled in-kernel, prepare kvm-run */
1408                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
1409                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1410                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
1411                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
1412                 rc = 0;
1413         }
1414
1415         if (rc == -EREMOTE) {
1416                 /* intercept was handled, but userspace support is needed
1417                  * kvm_run has been prepared by the handler */
1418                 rc = 0;
1419         }
1420
1421         store_regs(vcpu, kvm_run);
1422
1423         if (vcpu->sigset_active)
1424                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1425
1426         vcpu->stat.exit_userspace++;
1427         return rc;
1428 }
1429
1430 /*
1431  * store status at address
1432  * we use have two special cases:
1433  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1434  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1435  */
1436 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1437 {
1438         unsigned char archmode = 1;
1439         unsigned int px;
1440         u64 clkcomp;
1441         int rc;
1442
1443         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1444                 if (write_guest_abs(vcpu, 163, &archmode, 1))
1445                         return -EFAULT;
1446                 gpa = SAVE_AREA_BASE;
1447         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1448                 if (write_guest_real(vcpu, 163, &archmode, 1))
1449                         return -EFAULT;
1450                 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1451         }
1452         rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1453                              vcpu->arch.guest_fpregs.fprs, 128);
1454         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1455                               vcpu->run->s.regs.gprs, 128);
1456         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1457                               &vcpu->arch.sie_block->gpsw, 16);
1458         px = kvm_s390_get_prefix(vcpu);
1459         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1460                               &px, 4);
1461         rc |= write_guest_abs(vcpu,
1462                               gpa + offsetof(struct save_area, fp_ctrl_reg),
1463                               &vcpu->arch.guest_fpregs.fpc, 4);
1464         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1465                               &vcpu->arch.sie_block->todpr, 4);
1466         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1467                               &vcpu->arch.sie_block->cputm, 8);
1468         clkcomp = vcpu->arch.sie_block->ckc >> 8;
1469         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1470                               &clkcomp, 8);
1471         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1472                               &vcpu->run->s.regs.acrs, 64);
1473         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1474                               &vcpu->arch.sie_block->gcr, 128);
1475         return rc ? -EFAULT : 0;
1476 }
1477
1478 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1479 {
1480         /*
1481          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1482          * copying in vcpu load/put. Lets update our copies before we save
1483          * it into the save area
1484          */
1485         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1486         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1487         save_access_regs(vcpu->run->s.regs.acrs);
1488
1489         return kvm_s390_store_status_unloaded(vcpu, addr);
1490 }
1491
1492 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1493 {
1494         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1495         kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1496         exit_sie_sync(vcpu);
1497 }
1498
1499 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1500 {
1501         unsigned int i;
1502         struct kvm_vcpu *vcpu;
1503
1504         kvm_for_each_vcpu(i, vcpu, kvm) {
1505                 __disable_ibs_on_vcpu(vcpu);
1506         }
1507 }
1508
1509 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1510 {
1511         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1512         kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1513         exit_sie_sync(vcpu);
1514 }
1515
1516 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1517 {
1518         int i, online_vcpus, started_vcpus = 0;
1519
1520         if (!is_vcpu_stopped(vcpu))
1521                 return;
1522
1523         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
1524         /* Only one cpu at a time may enter/leave the STOPPED state. */
1525         spin_lock(&vcpu->kvm->arch.start_stop_lock);
1526         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1527
1528         for (i = 0; i < online_vcpus; i++) {
1529                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1530                         started_vcpus++;
1531         }
1532
1533         if (started_vcpus == 0) {
1534                 /* we're the only active VCPU -> speed it up */
1535                 __enable_ibs_on_vcpu(vcpu);
1536         } else if (started_vcpus == 1) {
1537                 /*
1538                  * As we are starting a second VCPU, we have to disable
1539                  * the IBS facility on all VCPUs to remove potentially
1540                  * oustanding ENABLE requests.
1541                  */
1542                 __disable_ibs_on_all_vcpus(vcpu->kvm);
1543         }
1544
1545         atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1546         /*
1547          * Another VCPU might have used IBS while we were offline.
1548          * Let's play safe and flush the VCPU at startup.
1549          */
1550         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1551         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1552         return;
1553 }
1554
1555 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1556 {
1557         int i, online_vcpus, started_vcpus = 0;
1558         struct kvm_vcpu *started_vcpu = NULL;
1559
1560         if (is_vcpu_stopped(vcpu))
1561                 return;
1562
1563         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
1564         /* Only one cpu at a time may enter/leave the STOPPED state. */
1565         spin_lock(&vcpu->kvm->arch.start_stop_lock);
1566         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1567
1568         /* Need to lock access to action_bits to avoid a SIGP race condition */
1569         spin_lock(&vcpu->arch.local_int.lock);
1570         atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
1571
1572         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
1573         vcpu->arch.local_int.action_bits &=
1574                                  ~(ACTION_STOP_ON_STOP | ACTION_STORE_ON_STOP);
1575         spin_unlock(&vcpu->arch.local_int.lock);
1576
1577         __disable_ibs_on_vcpu(vcpu);
1578
1579         for (i = 0; i < online_vcpus; i++) {
1580                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1581                         started_vcpus++;
1582                         started_vcpu = vcpu->kvm->vcpus[i];
1583                 }
1584         }
1585
1586         if (started_vcpus == 1) {
1587                 /*
1588                  * As we only have one VCPU left, we want to enable the
1589                  * IBS facility for that VCPU to speed it up.
1590                  */
1591                 __enable_ibs_on_vcpu(started_vcpu);
1592         }
1593
1594         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
1595         return;
1596 }
1597
1598 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1599                                      struct kvm_enable_cap *cap)
1600 {
1601         int r;
1602
1603         if (cap->flags)
1604                 return -EINVAL;
1605
1606         switch (cap->cap) {
1607         case KVM_CAP_S390_CSS_SUPPORT:
1608                 if (!vcpu->kvm->arch.css_support) {
1609                         vcpu->kvm->arch.css_support = 1;
1610                         trace_kvm_s390_enable_css(vcpu->kvm);
1611                 }
1612                 r = 0;
1613                 break;
1614         default:
1615                 r = -EINVAL;
1616                 break;
1617         }
1618         return r;
1619 }
1620
1621 long kvm_arch_vcpu_ioctl(struct file *filp,
1622                          unsigned int ioctl, unsigned long arg)
1623 {
1624         struct kvm_vcpu *vcpu = filp->private_data;
1625         void __user *argp = (void __user *)arg;
1626         int idx;
1627         long r;
1628
1629         switch (ioctl) {
1630         case KVM_S390_INTERRUPT: {
1631                 struct kvm_s390_interrupt s390int;
1632                 struct kvm_s390_irq s390irq;
1633
1634                 r = -EFAULT;
1635                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1636                         break;
1637                 if (s390int_to_s390irq(&s390int, &s390irq))
1638                         return -EINVAL;
1639                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
1640                 break;
1641         }
1642         case KVM_S390_STORE_STATUS:
1643                 idx = srcu_read_lock(&vcpu->kvm->srcu);
1644                 r = kvm_s390_vcpu_store_status(vcpu, arg);
1645                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
1646                 break;
1647         case KVM_S390_SET_INITIAL_PSW: {
1648                 psw_t psw;
1649
1650                 r = -EFAULT;
1651                 if (copy_from_user(&psw, argp, sizeof(psw)))
1652                         break;
1653                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1654                 break;
1655         }
1656         case KVM_S390_INITIAL_RESET:
1657                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1658                 break;
1659         case KVM_SET_ONE_REG:
1660         case KVM_GET_ONE_REG: {
1661                 struct kvm_one_reg reg;
1662                 r = -EFAULT;
1663                 if (copy_from_user(&reg, argp, sizeof(reg)))
1664                         break;
1665                 if (ioctl == KVM_SET_ONE_REG)
1666                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1667                 else
1668                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1669                 break;
1670         }
1671 #ifdef CONFIG_KVM_S390_UCONTROL
1672         case KVM_S390_UCAS_MAP: {
1673                 struct kvm_s390_ucas_mapping ucasmap;
1674
1675                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1676                         r = -EFAULT;
1677                         break;
1678                 }
1679
1680                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1681                         r = -EINVAL;
1682                         break;
1683                 }
1684
1685                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1686                                      ucasmap.vcpu_addr, ucasmap.length);
1687                 break;
1688         }
1689         case KVM_S390_UCAS_UNMAP: {
1690                 struct kvm_s390_ucas_mapping ucasmap;
1691
1692                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1693                         r = -EFAULT;
1694                         break;
1695                 }
1696
1697                 if (!kvm_is_ucontrol(vcpu->kvm)) {
1698                         r = -EINVAL;
1699                         break;
1700                 }
1701
1702                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1703                         ucasmap.length);
1704                 break;
1705         }
1706 #endif
1707         case KVM_S390_VCPU_FAULT: {
1708                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
1709                 break;
1710         }
1711         case KVM_ENABLE_CAP:
1712         {
1713                 struct kvm_enable_cap cap;
1714                 r = -EFAULT;
1715                 if (copy_from_user(&cap, argp, sizeof(cap)))
1716                         break;
1717                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1718                 break;
1719         }
1720         default:
1721                 r = -ENOTTY;
1722         }
1723         return r;
1724 }
1725
1726 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1727 {
1728 #ifdef CONFIG_KVM_S390_UCONTROL
1729         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1730                  && (kvm_is_ucontrol(vcpu->kvm))) {
1731                 vmf->page = virt_to_page(vcpu->arch.sie_block);
1732                 get_page(vmf->page);
1733                 return 0;
1734         }
1735 #endif
1736         return VM_FAULT_SIGBUS;
1737 }
1738
1739 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1740                             unsigned long npages)
1741 {
1742         return 0;
1743 }
1744
1745 /* Section: memory related */
1746 int kvm_arch_prepare_memory_region(struct kvm *kvm,
1747                                    struct kvm_memory_slot *memslot,
1748                                    struct kvm_userspace_memory_region *mem,
1749                                    enum kvm_mr_change change)
1750 {
1751         /* A few sanity checks. We can have memory slots which have to be
1752            located/ended at a segment boundary (1MB). The memory in userland is
1753            ok to be fragmented into various different vmas. It is okay to mmap()
1754            and munmap() stuff in this slot after doing this call at any time */
1755
1756         if (mem->userspace_addr & 0xffffful)
1757                 return -EINVAL;
1758
1759         if (mem->memory_size & 0xffffful)
1760                 return -EINVAL;
1761
1762         return 0;
1763 }
1764
1765 void kvm_arch_commit_memory_region(struct kvm *kvm,
1766                                 struct kvm_userspace_memory_region *mem,
1767                                 const struct kvm_memory_slot *old,
1768                                 enum kvm_mr_change change)
1769 {
1770         int rc;
1771
1772         /* If the basics of the memslot do not change, we do not want
1773          * to update the gmap. Every update causes several unnecessary
1774          * segment translation exceptions. This is usually handled just
1775          * fine by the normal fault handler + gmap, but it will also
1776          * cause faults on the prefix page of running guest CPUs.
1777          */
1778         if (old->userspace_addr == mem->userspace_addr &&
1779             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1780             old->npages * PAGE_SIZE == mem->memory_size)
1781                 return;
1782
1783         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1784                 mem->guest_phys_addr, mem->memory_size);
1785         if (rc)
1786                 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
1787         return;
1788 }
1789
1790 static int __init kvm_s390_init(void)
1791 {
1792         int ret;
1793         ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1794         if (ret)
1795                 return ret;
1796
1797         /*
1798          * guests can ask for up to 255+1 double words, we need a full page
1799          * to hold the maximum amount of facilities. On the other hand, we
1800          * only set facilities that are known to work in KVM.
1801          */
1802         vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1803         if (!vfacilities) {
1804                 kvm_exit();
1805                 return -ENOMEM;
1806         }
1807         memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
1808         vfacilities[0] &= 0xff82fffbf47c2000UL;
1809         vfacilities[1] &= 0x005c000000000000UL;
1810         return 0;
1811 }
1812
1813 static void __exit kvm_s390_exit(void)
1814 {
1815         free_page((unsigned long) vfacilities);
1816         kvm_exit();
1817 }
1818
1819 module_init(kvm_s390_init);
1820 module_exit(kvm_s390_exit);
1821
1822 /*
1823  * Enable autoloading of the kvm module.
1824  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1825  * since x86 takes a different approach.
1826  */
1827 #include <linux/miscdevice.h>
1828 MODULE_ALIAS_MISCDEV(KVM_MINOR);
1829 MODULE_ALIAS("devname:kvm");