arch/s390/kvm/kvm-s390.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * hosting IBM Z kernel virtual machines (s390x)
   4  *
   5  * Copyright IBM Corp. 2008, 2018
   6  *
   7  *    Author(s): Carsten Otte <cotte@de.ibm.com>
   8  *               Christian Borntraeger <borntraeger@de.ibm.com>
   9  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  10  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  11  *               Jason J. Herne <jjherne@us.ibm.com>
  12  */
  13
  14 #include <linux/compiler.h>
  15 #include <linux/err.h>
  16 #include <linux/fs.h>
  17 #include <linux/hrtimer.h>
  18 #include <linux/init.h>
  19 #include <linux/kvm.h>
  20 #include <linux/kvm_host.h>
  21 #include <linux/mman.h>
  22 #include <linux/module.h>
  23 #include <linux/moduleparam.h>
  24 #include <linux/random.h>
  25 #include <linux/slab.h>
  26 #include <linux/timer.h>
  27 #include <linux/vmalloc.h>
  28 #include <linux/bitmap.h>
  29 #include <linux/sched/signal.h>
  30 #include <linux/string.h>
  31
  32 #include <asm/asm-offsets.h>
  33 #include <asm/lowcore.h>
  34 #include <asm/stp.h>
  35 #include <asm/pgtable.h>
  36 #include <asm/gmap.h>
  37 #include <asm/nmi.h>
  38 #include <asm/switch_to.h>
  39 #include <asm/isc.h>
  40 #include <asm/sclp.h>
  41 #include <asm/cpacf.h>
  42 #include <asm/timex.h>
  43 #include "kvm-s390.h"
  44 #include "gaccess.h"
  45
  46 #define KMSG_COMPONENT "kvm-s390"
  47 #undef pr_fmt
  48 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  49
  50 #define CREATE_TRACE_POINTS
  51 #include "trace.h"
  52 #include "trace-s390.h"
  53
  54 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  55 #define LOCAL_IRQS 32
  56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  57                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  58
  59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  60 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
  61
  62 struct kvm_stats_debugfs_item debugfs_entries[] = {
  63         { "userspace_handled", VCPU_STAT(exit_userspace) },
  64         { "exit_null", VCPU_STAT(exit_null) },
  65         { "exit_validity", VCPU_STAT(exit_validity) },
  66         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  67         { "exit_external_request", VCPU_STAT(exit_external_request) },
  68         { "exit_io_request", VCPU_STAT(exit_io_request) },
  69         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  70         { "exit_instruction", VCPU_STAT(exit_instruction) },
  71         { "exit_pei", VCPU_STAT(exit_pei) },
  72         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  73         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  74         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
  75         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  76         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  77         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
  78         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  79         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  80         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  81         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  82         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  83         { "deliver_ckc", VCPU_STAT(deliver_ckc) },
  84         { "deliver_cputm", VCPU_STAT(deliver_cputm) },
  85         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  86         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  87         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  88         { "deliver_virtio", VCPU_STAT(deliver_virtio) },
  89         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  90         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  91         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  92         { "deliver_program", VCPU_STAT(deliver_program) },
  93         { "deliver_io", VCPU_STAT(deliver_io) },
  94         { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
  95         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  96         { "inject_ckc", VCPU_STAT(inject_ckc) },
  97         { "inject_cputm", VCPU_STAT(inject_cputm) },
  98         { "inject_external_call", VCPU_STAT(inject_external_call) },
  99         { "inject_float_mchk", VM_STAT(inject_float_mchk) },
 100         { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
 101         { "inject_io", VM_STAT(inject_io) },
 102         { "inject_mchk", VCPU_STAT(inject_mchk) },
 103         { "inject_pfault_done", VM_STAT(inject_pfault_done) },
 104         { "inject_program", VCPU_STAT(inject_program) },
 105         { "inject_restart", VCPU_STAT(inject_restart) },
 106         { "inject_service_signal", VM_STAT(inject_service_signal) },
 107         { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
 108         { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
 109         { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
 110         { "inject_virtio", VM_STAT(inject_virtio) },
 111         { "instruction_epsw", VCPU_STAT(instruction_epsw) },
 112         { "instruction_gs", VCPU_STAT(instruction_gs) },
 113         { "instruction_io_other", VCPU_STAT(instruction_io_other) },
 114         { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
 115         { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
 116         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
 117         { "instruction_ptff", VCPU_STAT(instruction_ptff) },
 118         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
 119         { "instruction_sck", VCPU_STAT(instruction_sck) },
 120         { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
 121         { "instruction_spx", VCPU_STAT(instruction_spx) },
 122         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
 123         { "instruction_stap", VCPU_STAT(instruction_stap) },
 124         { "instruction_iske", VCPU_STAT(instruction_iske) },
 125         { "instruction_ri", VCPU_STAT(instruction_ri) },
 126         { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
 127         { "instruction_sske", VCPU_STAT(instruction_sske) },
 128         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
 129         { "instruction_essa", VCPU_STAT(instruction_essa) },
 130         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
 131         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
 132         { "instruction_tb", VCPU_STAT(instruction_tb) },
 133         { "instruction_tpi", VCPU_STAT(instruction_tpi) },
 134         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
 135         { "instruction_tsch", VCPU_STAT(instruction_tsch) },
 136         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
 137         { "instruction_sie", VCPU_STAT(instruction_sie) },
 138         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 139         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
 140         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
 141         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 142         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
 143         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 144         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 145         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 146         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 147         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 148         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 149         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 150         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 151         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 152         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 153         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 154         { "instruction_diag_10", VCPU_STAT(diagnose_10) },
 155         { "instruction_diag_44", VCPU_STAT(diagnose_44) },
 156         { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
 157         { "instruction_diag_258", VCPU_STAT(diagnose_258) },
 158         { "instruction_diag_308", VCPU_STAT(diagnose_308) },
 159         { "instruction_diag_500", VCPU_STAT(diagnose_500) },
 160         { "instruction_diag_other", VCPU_STAT(diagnose_other) },
 161         { NULL }
 162 };
 163
 164 struct kvm_s390_tod_clock_ext {
 165         __u8 epoch_idx;
 166         __u64 tod;
 167         __u8 reserved[7];
 168 } __packed;
 169
 170 /* allow nested virtualization in KVM (if enabled by user space) */
 171 static int nested;
 172 module_param(nested, int, S_IRUGO);
 173 MODULE_PARM_DESC(nested, "Nested virtualization support");
 174
 175
 176 /*
 177  * For now we handle at most 16 double words as this is what the s390 base
 178  * kernel handles and stores in the prefix page. If we ever need to go beyond
 179  * this, this requires changes to code, but the external uapi can stay.
 180  */
 181 #define SIZE_INTERNAL 16
 182
 183 /*
 184  * Base feature mask that defines default mask for facilities. Consists of the
 185  * defines in FACILITIES_KVM and the non-hypervisor managed bits.
 186  */
 187 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
 188 /*
 189  * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
 190  * and defines the facilities that can be enabled via a cpu model.
 191  */
 192 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
 193
 194 static unsigned long kvm_s390_fac_size(void)
 195 {
 196         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
 197         BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
 198         BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
 199                 sizeof(S390_lowcore.stfle_fac_list));
 200
 201         return SIZE_INTERNAL;
 202 }
 203
 204 /* available cpu features supported by kvm */
 205 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 206 /* available subfunctions indicated via query / "test bit" */
 207 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
 208
 209 static struct gmap_notifier gmap_notifier;
 210 static struct gmap_notifier vsie_gmap_notifier;
 211 debug_info_t *kvm_s390_dbf;
 212
 213 /* Section: not file related */
 214 int kvm_arch_hardware_enable(void)
 215 {
 216         /* every s390 is virtualization enabled ;-) */
 217         return 0;
 218 }
 219
 220 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 221                               unsigned long end);
 222
 223 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
 224 {
 225         u8 delta_idx = 0;
 226
 227         /*
 228          * The TOD jumps by delta, we have to compensate this by adding
 229          * -delta to the epoch.
 230          */
 231         delta = -delta;
 232
 233         /* sign-extension - we're adding to signed values below */
 234         if ((s64)delta < 0)
 235                 delta_idx = -1;
 236
 237         scb->epoch += delta;
 238         if (scb->ecd & ECD_MEF) {
 239                 scb->epdx += delta_idx;
 240                 if (scb->epoch < delta)
 241                         scb->epdx += 1;
 242         }
 243 }
 244
 245 /*
 246  * This callback is executed during stop_machine(). All CPUs are therefore
 247  * temporarily stopped. In order not to change guest behavior, we have to
 248  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 249  * so a CPU won't be stopped while calculating with the epoch.
 250  */
 251 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 252                           void *v)
 253 {
 254         struct kvm *kvm;
 255         struct kvm_vcpu *vcpu;
 256         int i;
 257         unsigned long long *delta = v;
 258
 259         list_for_each_entry(kvm, &vm_list, vm_list) {
 260                 kvm_for_each_vcpu(i, vcpu, kvm) {
 261                         kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
 262                         if (i == 0) {
 263                                 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
 264                                 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
 265                         }
 266                         if (vcpu->arch.cputm_enabled)
 267                                 vcpu->arch.cputm_start += *delta;
 268                         if (vcpu->arch.vsie_block)
 269                                 kvm_clock_sync_scb(vcpu->arch.vsie_block,
 270                                                    *delta);
 271                 }
 272         }
 273         return NOTIFY_OK;
 274 }
 275
 276 static struct notifier_block kvm_clock_notifier = {
 277         .notifier_call = kvm_clock_sync,
 278 };
 279
 280 int kvm_arch_hardware_setup(void)
 281 {
 282         gmap_notifier.notifier_call = kvm_gmap_notifier;
 283         gmap_register_pte_notifier(&gmap_notifier);
 284         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
 285         gmap_register_pte_notifier(&vsie_gmap_notifier);
 286         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 287                                        &kvm_clock_notifier);
 288         return 0;
 289 }
 290
 291 void kvm_arch_hardware_unsetup(void)
 292 {
 293         gmap_unregister_pte_notifier(&gmap_notifier);
 294         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
 295         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 296                                          &kvm_clock_notifier);
 297 }
 298
 299 static void allow_cpu_feat(unsigned long nr)
 300 {
 301         set_bit_inv(nr, kvm_s390_available_cpu_feat);
 302 }
 303
 304 static inline int plo_test_bit(unsigned char nr)
 305 {
 306         register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
 307         int cc;
 308
 309         asm volatile(
 310                 /* Parameter registers are ignored for "test bit" */
 311                 "       plo     0,0,0,0(0)\n"
 312                 "       ipm     %0\n"
 313                 "       srl     %0,28\n"
 314                 : "=d" (cc)
 315                 : "d" (r0)
 316                 : "cc");
 317         return cc == 0;
 318 }
 319
 320 static void kvm_s390_cpu_feat_init(void)
 321 {
 322         int i;
 323
 324         for (i = 0; i < 256; ++i) {
 325                 if (plo_test_bit(i))
 326                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
 327         }
 328
 329         if (test_facility(28)) /* TOD-clock steering */
 330                 ptff(kvm_s390_available_subfunc.ptff,
 331                      sizeof(kvm_s390_available_subfunc.ptff),
 332                      PTFF_QAF);
 333
 334         if (test_facility(17)) { /* MSA */
 335                 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
 336                               kvm_s390_available_subfunc.kmac);
 337                 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
 338                               kvm_s390_available_subfunc.kmc);
 339                 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
 340                               kvm_s390_available_subfunc.km);
 341                 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
 342                               kvm_s390_available_subfunc.kimd);
 343                 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
 344                               kvm_s390_available_subfunc.klmd);
 345         }
 346         if (test_facility(76)) /* MSA3 */
 347                 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
 348                               kvm_s390_available_subfunc.pckmo);
 349         if (test_facility(77)) { /* MSA4 */
 350                 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
 351                               kvm_s390_available_subfunc.kmctr);
 352                 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
 353                               kvm_s390_available_subfunc.kmf);
 354                 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
 355                               kvm_s390_available_subfunc.kmo);
 356                 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
 357                               kvm_s390_available_subfunc.pcc);
 358         }
 359         if (test_facility(57)) /* MSA5 */
 360                 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
 361                               kvm_s390_available_subfunc.ppno);
 362
 363         if (test_facility(146)) /* MSA8 */
 364                 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
 365                               kvm_s390_available_subfunc.kma);
 366
 367         if (MACHINE_HAS_ESOP)
 368                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
 369         /*
 370          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
 371          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
 372          */
 373         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
 374             !test_facility(3) || !nested)
 375                 return;
 376         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
 377         if (sclp.has_64bscao)
 378                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
 379         if (sclp.has_siif)
 380                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
 381         if (sclp.has_gpere)
 382                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
 383         if (sclp.has_gsls)
 384                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
 385         if (sclp.has_ib)
 386                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
 387         if (sclp.has_cei)
 388                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
 389         if (sclp.has_ibs)
 390                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
 391         if (sclp.has_kss)
 392                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
 393         /*
 394          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
 395          * all skey handling functions read/set the skey from the PGSTE
 396          * instead of the real storage key.
 397          *
 398          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
 399          * pages being detected as preserved although they are resident.
 400          *
 401          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
 402          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
 403          *
 404          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
 405          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
 406          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
 407          *
 408          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
 409          * cannot easily shadow the SCA because of the ipte lock.
 410          */
 411 }
 412
 413 int kvm_arch_init(void *opaque)
 414 {
 415         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 416         if (!kvm_s390_dbf)
 417                 return -ENOMEM;
 418
 419         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 420                 debug_unregister(kvm_s390_dbf);
 421                 return -ENOMEM;
 422         }
 423
 424         kvm_s390_cpu_feat_init();
 425
 426         /* Register floating interrupt controller interface. */
 427         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 428 }
 429
 430 void kvm_arch_exit(void)
 431 {
 432         debug_unregister(kvm_s390_dbf);
 433 }
 434
 435 /* Section: device related */
 436 long kvm_arch_dev_ioctl(struct file *filp,
 437                         unsigned int ioctl, unsigned long arg)
 438 {
 439         if (ioctl == KVM_S390_ENABLE_SIE)
 440                 return s390_enable_sie();
 441         return -EINVAL;
 442 }
 443
 444 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 445 {
 446         int r;
 447
 448         switch (ext) {
 449         case KVM_CAP_S390_PSW:
 450         case KVM_CAP_S390_GMAP:
 451         case KVM_CAP_SYNC_MMU:
 452 #ifdef CONFIG_KVM_S390_UCONTROL
 453         case KVM_CAP_S390_UCONTROL:
 454 #endif
 455         case KVM_CAP_ASYNC_PF:
 456         case KVM_CAP_SYNC_REGS:
 457         case KVM_CAP_ONE_REG:
 458         case KVM_CAP_ENABLE_CAP:
 459         case KVM_CAP_S390_CSS_SUPPORT:
 460         case KVM_CAP_IOEVENTFD:
 461         case KVM_CAP_DEVICE_CTRL:
 462         case KVM_CAP_ENABLE_CAP_VM:
 463         case KVM_CAP_S390_IRQCHIP:
 464         case KVM_CAP_VM_ATTRIBUTES:
 465         case KVM_CAP_MP_STATE:
 466         case KVM_CAP_IMMEDIATE_EXIT:
 467         case KVM_CAP_S390_INJECT_IRQ:
 468         case KVM_CAP_S390_USER_SIGP:
 469         case KVM_CAP_S390_USER_STSI:
 470         case KVM_CAP_S390_SKEYS:
 471         case KVM_CAP_S390_IRQ_STATE:
 472         case KVM_CAP_S390_USER_INSTR0:
 473         case KVM_CAP_S390_CMMA_MIGRATION:
 474         case KVM_CAP_S390_AIS:
 475         case KVM_CAP_S390_AIS_MIGRATION:
 476                 r = 1;
 477                 break;
 478         case KVM_CAP_S390_MEM_OP:
 479                 r = MEM_OP_MAX_SIZE;
 480                 break;
 481         case KVM_CAP_NR_VCPUS:
 482         case KVM_CAP_MAX_VCPUS:
 483                 r = KVM_S390_BSCA_CPU_SLOTS;
 484                 if (!kvm_s390_use_sca_entries())
 485                         r = KVM_MAX_VCPUS;
 486                 else if (sclp.has_esca && sclp.has_64bscao)
 487                         r = KVM_S390_ESCA_CPU_SLOTS;
 488                 break;
 489         case KVM_CAP_NR_MEMSLOTS:
 490                 r = KVM_USER_MEM_SLOTS;
 491                 break;
 492         case KVM_CAP_S390_COW:
 493                 r = MACHINE_HAS_ESOP;
 494                 break;
 495         case KVM_CAP_S390_VECTOR_REGISTERS:
 496                 r = MACHINE_HAS_VX;
 497                 break;
 498         case KVM_CAP_S390_RI:
 499                 r = test_facility(64);
 500                 break;
 501         case KVM_CAP_S390_GS:
 502                 r = test_facility(133);
 503                 break;
 504         case KVM_CAP_S390_BPB:
 505                 r = test_facility(82);
 506                 break;
 507         default:
 508                 r = 0;
 509         }
 510         return r;
 511 }
 512
 513 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 514                                         struct kvm_memory_slot *memslot)
 515 {
 516         gfn_t cur_gfn, last_gfn;
 517         unsigned long address;
 518         struct gmap *gmap = kvm->arch.gmap;
 519
 520         /* Loop over all guest pages */
 521         last_gfn = memslot->base_gfn + memslot->npages;
 522         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 523                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 524
 525                 if (test_and_clear_guest_dirty(gmap->mm, address))
 526                         mark_page_dirty(kvm, cur_gfn);
 527                 if (fatal_signal_pending(current))
 528                         return;
 529                 cond_resched();
 530         }
 531 }
 532
 533 /* Section: vm related */
 534 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 535
 536 /*
 537  * Get (and clear) the dirty memory log for a memory slot.
 538  */
 539 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 540                                struct kvm_dirty_log *log)
 541 {
 542         int r;
 543         unsigned long n;
 544         struct kvm_memslots *slots;
 545         struct kvm_memory_slot *memslot;
 546         int is_dirty = 0;
 547
 548         if (kvm_is_ucontrol(kvm))
 549                 return -EINVAL;
 550
 551         mutex_lock(&kvm->slots_lock);
 552
 553         r = -EINVAL;
 554         if (log->slot >= KVM_USER_MEM_SLOTS)
 555                 goto out;
 556
 557         slots = kvm_memslots(kvm);
 558         memslot = id_to_memslot(slots, log->slot);
 559         r = -ENOENT;
 560         if (!memslot->dirty_bitmap)
 561                 goto out;
 562
 563         kvm_s390_sync_dirty_log(kvm, memslot);
 564         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 565         if (r)
 566                 goto out;
 567
 568         /* Clear the dirty log */
 569         if (is_dirty) {
 570                 n = kvm_dirty_bitmap_bytes(memslot);
 571                 memset(memslot->dirty_bitmap, 0, n);
 572         }
 573         r = 0;
 574 out:
 575         mutex_unlock(&kvm->slots_lock);
 576         return r;
 577 }
 578
 579 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
 580 {
 581         unsigned int i;
 582         struct kvm_vcpu *vcpu;
 583
 584         kvm_for_each_vcpu(i, vcpu, kvm) {
 585                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
 586         }
 587 }
 588
 589 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 590 {
 591         int r;
 592
 593         if (cap->flags)
 594                 return -EINVAL;
 595
 596         switch (cap->cap) {
 597         case KVM_CAP_S390_IRQCHIP:
 598                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 599                 kvm->arch.use_irqchip = 1;
 600                 r = 0;
 601                 break;
 602         case KVM_CAP_S390_USER_SIGP:
 603                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 604                 kvm->arch.user_sigp = 1;
 605                 r = 0;
 606                 break;
 607         case KVM_CAP_S390_VECTOR_REGISTERS:
 608                 mutex_lock(&kvm->lock);
 609                 if (kvm->created_vcpus) {
 610                         r = -EBUSY;
 611                 } else if (MACHINE_HAS_VX) {
 612                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
 613                         set_kvm_facility(kvm->arch.model.fac_list, 129);
 614                         if (test_facility(134)) {
 615                                 set_kvm_facility(kvm->arch.model.fac_mask, 134);
 616                                 set_kvm_facility(kvm->arch.model.fac_list, 134);
 617                         }
 618                         if (test_facility(135)) {
 619                                 set_kvm_facility(kvm->arch.model.fac_mask, 135);
 620                                 set_kvm_facility(kvm->arch.model.fac_list, 135);
 621                         }
 622                         r = 0;
 623                 } else
 624                         r = -EINVAL;
 625                 mutex_unlock(&kvm->lock);
 626                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 627                          r ? "(not available)" : "(success)");
 628                 break;
 629         case KVM_CAP_S390_RI:
 630                 r = -EINVAL;
 631                 mutex_lock(&kvm->lock);
 632                 if (kvm->created_vcpus) {
 633                         r = -EBUSY;
 634                 } else if (test_facility(64)) {
 635                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
 636                         set_kvm_facility(kvm->arch.model.fac_list, 64);
 637                         r = 0;
 638                 }
 639                 mutex_unlock(&kvm->lock);
 640                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 641                          r ? "(not available)" : "(success)");
 642                 break;
 643         case KVM_CAP_S390_AIS:
 644                 mutex_lock(&kvm->lock);
 645                 if (kvm->created_vcpus) {
 646                         r = -EBUSY;
 647                 } else {
 648                         set_kvm_facility(kvm->arch.model.fac_mask, 72);
 649                         set_kvm_facility(kvm->arch.model.fac_list, 72);
 650                         r = 0;
 651                 }
 652                 mutex_unlock(&kvm->lock);
 653                 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
 654                          r ? "(not available)" : "(success)");
 655                 break;
 656         case KVM_CAP_S390_GS:
 657                 r = -EINVAL;
 658                 mutex_lock(&kvm->lock);
 659                 if (kvm->created_vcpus) {
 660                         r = -EBUSY;
 661                 } else if (test_facility(133)) {
 662                         set_kvm_facility(kvm->arch.model.fac_mask, 133);
 663                         set_kvm_facility(kvm->arch.model.fac_list, 133);
 664                         r = 0;
 665                 }
 666                 mutex_unlock(&kvm->lock);
 667                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
 668                          r ? "(not available)" : "(success)");
 669                 break;
 670         case KVM_CAP_S390_USER_STSI:
 671                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 672                 kvm->arch.user_stsi = 1;
 673                 r = 0;
 674                 break;
 675         case KVM_CAP_S390_USER_INSTR0:
 676                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
 677                 kvm->arch.user_instr0 = 1;
 678                 icpt_operexc_on_all_vcpus(kvm);
 679                 r = 0;
 680                 break;
 681         default:
 682                 r = -EINVAL;
 683                 break;
 684         }
 685         return r;
 686 }
 687
 688 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 689 {
 690         int ret;
 691
 692         switch (attr->attr) {
 693         case KVM_S390_VM_MEM_LIMIT_SIZE:
 694                 ret = 0;
 695                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 696                          kvm->arch.mem_limit);
 697                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 698                         ret = -EFAULT;
 699                 break;
 700         default:
 701                 ret = -ENXIO;
 702                 break;
 703         }
 704         return ret;
 705 }
 706
 707 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 708 {
 709         int ret;
 710         unsigned int idx;
 711         switch (attr->attr) {
 712         case KVM_S390_VM_MEM_ENABLE_CMMA:
 713                 ret = -ENXIO;
 714                 if (!sclp.has_cmma)
 715                         break;
 716
 717                 ret = -EBUSY;
 718                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 719                 mutex_lock(&kvm->lock);
 720                 if (!kvm->created_vcpus) {
 721                         kvm->arch.use_cmma = 1;
 722                         /* Not compatible with cmma. */
 723                         kvm->arch.use_pfmfi = 0;
 724                         ret = 0;
 725                 }
 726                 mutex_unlock(&kvm->lock);
 727                 break;
 728         case KVM_S390_VM_MEM_CLR_CMMA:
 729                 ret = -ENXIO;
 730                 if (!sclp.has_cmma)
 731                         break;
 732                 ret = -EINVAL;
 733                 if (!kvm->arch.use_cmma)
 734                         break;
 735
 736                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 737                 mutex_lock(&kvm->lock);
 738                 idx = srcu_read_lock(&kvm->srcu);
 739                 s390_reset_cmma(kvm->arch.gmap->mm);
 740                 srcu_read_unlock(&kvm->srcu, idx);
 741                 mutex_unlock(&kvm->lock);
 742                 ret = 0;
 743                 break;
 744         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 745                 unsigned long new_limit;
 746
 747                 if (kvm_is_ucontrol(kvm))
 748                         return -EINVAL;
 749
 750                 if (get_user(new_limit, (u64 __user *)attr->addr))
 751                         return -EFAULT;
 752
 753                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 754                     new_limit > kvm->arch.mem_limit)
 755                         return -E2BIG;
 756
 757                 if (!new_limit)
 758                         return -EINVAL;
 759
 760                 /* gmap_create takes last usable address */
 761                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 762                         new_limit -= 1;
 763
 764                 ret = -EBUSY;
 765                 mutex_lock(&kvm->lock);
 766                 if (!kvm->created_vcpus) {
 767                         /* gmap_create will round the limit up */
 768                         struct gmap *new = gmap_create(current->mm, new_limit);
 769
 770                         if (!new) {
 771                                 ret = -ENOMEM;
 772                         } else {
 773                                 gmap_remove(kvm->arch.gmap);
 774                                 new->private = kvm;
 775                                 kvm->arch.gmap = new;
 776                                 ret = 0;
 777                         }
 778                 }
 779                 mutex_unlock(&kvm->lock);
 780                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 781                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 782                          (void *) kvm->arch.gmap->asce);
 783                 break;
 784         }
 785         default:
 786                 ret = -ENXIO;
 787                 break;
 788         }
 789         return ret;
 790 }
 791
 792 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 793
 794 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 795 {
 796         struct kvm_vcpu *vcpu;
 797         int i;
 798
 799         if (!test_kvm_facility(kvm, 76))
 800                 return -EINVAL;
 801
 802         mutex_lock(&kvm->lock);
 803         switch (attr->attr) {
 804         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 805                 get_random_bytes(
 806                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 807                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 808                 kvm->arch.crypto.aes_kw = 1;
 809                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 810                 break;
 811         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 812                 get_random_bytes(
 813                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 814                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 815                 kvm->arch.crypto.dea_kw = 1;
 816                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 817                 break;
 818         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 819                 kvm->arch.crypto.aes_kw = 0;
 820                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 821                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 822                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 823                 break;
 824         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 825                 kvm->arch.crypto.dea_kw = 0;
 826                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 827                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 828                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 829                 break;
 830         default:
 831                 mutex_unlock(&kvm->lock);
 832                 return -ENXIO;
 833         }
 834
 835         kvm_for_each_vcpu(i, vcpu, kvm) {
 836                 kvm_s390_vcpu_crypto_setup(vcpu);
 837                 exit_sie(vcpu);
 838         }
 839         mutex_unlock(&kvm->lock);
 840         return 0;
 841 }
 842
 843 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
 844 {
 845         int cx;
 846         struct kvm_vcpu *vcpu;
 847
 848         kvm_for_each_vcpu(cx, vcpu, kvm)
 849                 kvm_s390_sync_request(req, vcpu);
 850 }
 851
 852 /*
 853  * Must be called with kvm->srcu held to avoid races on memslots, and with
 854  * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
 855  */
 856 static int kvm_s390_vm_start_migration(struct kvm *kvm)
 857 {
 858         struct kvm_s390_migration_state *mgs;
 859         struct kvm_memory_slot *ms;
 860         /* should be the only one */
 861         struct kvm_memslots *slots;
 862         unsigned long ram_pages;
 863         int slotnr;
 864
 865         /* migration mode already enabled */
 866         if (kvm->arch.migration_state)
 867                 return 0;
 868
 869         slots = kvm_memslots(kvm);
 870         if (!slots || !slots->used_slots)
 871                 return -EINVAL;
 872
 873         mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);
 874         if (!mgs)
 875                 return -ENOMEM;
 876         kvm->arch.migration_state = mgs;
 877
 878         if (kvm->arch.use_cmma) {
 879                 /*
 880                  * Get the first slot. They are reverse sorted by base_gfn, so
 881                  * the first slot is also the one at the end of the address
 882                  * space. We have verified above that at least one slot is
 883                  * present.
 884                  */
 885                 ms = slots->memslots;
 886                 /* round up so we only use full longs */
 887                 ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
 888                 /* allocate enough bytes to store all the bits */
 889                 mgs->pgste_bitmap = vmalloc(ram_pages / 8);
 890                 if (!mgs->pgste_bitmap) {
 891                         kfree(mgs);
 892                         kvm->arch.migration_state = NULL;
 893                         return -ENOMEM;
 894                 }
 895
 896                 mgs->bitmap_size = ram_pages;
 897                 atomic64_set(&mgs->dirty_pages, ram_pages);
 898                 /* mark all the pages in active slots as dirty */
 899                 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
 900                         ms = slots->memslots + slotnr;
 901                         bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
 902                 }
 903
 904                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
 905         }
 906         return 0;
 907 }
 908
 909 /*
 910  * Must be called with kvm->slots_lock to avoid races with ourselves and
 911  * kvm_s390_vm_start_migration.
 912  */
 913 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
 914 {
 915         struct kvm_s390_migration_state *mgs;
 916
 917         /* migration mode already disabled */
 918         if (!kvm->arch.migration_state)
 919                 return 0;
 920         mgs = kvm->arch.migration_state;
 921         kvm->arch.migration_state = NULL;
 922
 923         if (kvm->arch.use_cmma) {
 924                 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
 925                 /* We have to wait for the essa emulation to finish */
 926                 synchronize_srcu(&kvm->srcu);
 927                 vfree(mgs->pgste_bitmap);
 928         }
 929         kfree(mgs);
 930         return 0;
 931 }
 932
 933 static int kvm_s390_vm_set_migration(struct kvm *kvm,
 934                                      struct kvm_device_attr *attr)
 935 {
 936         int res = -ENXIO;
 937
 938         mutex_lock(&kvm->slots_lock);
 939         switch (attr->attr) {
 940         case KVM_S390_VM_MIGRATION_START:
 941                 res = kvm_s390_vm_start_migration(kvm);
 942                 break;
 943         case KVM_S390_VM_MIGRATION_STOP:
 944                 res = kvm_s390_vm_stop_migration(kvm);
 945                 break;
 946         default:
 947                 break;
 948         }
 949         mutex_unlock(&kvm->slots_lock);
 950
 951         return res;
 952 }
 953
 954 static int kvm_s390_vm_get_migration(struct kvm *kvm,
 955                                      struct kvm_device_attr *attr)
 956 {
 957         u64 mig = (kvm->arch.migration_state != NULL);
 958
 959         if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
 960                 return -ENXIO;
 961
 962         if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
 963                 return -EFAULT;
 964         return 0;
 965 }
 966
 967 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
 968 {
 969         struct kvm_s390_vm_tod_clock gtod;
 970
 971         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 972                 return -EFAULT;
 973
 974         if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
 975                 return -EINVAL;
 976         kvm_s390_set_tod_clock(kvm, &gtod);
 977
 978         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
 979                 gtod.epoch_idx, gtod.tod);
 980
 981         return 0;
 982 }
 983
 984 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 985 {
 986         u8 gtod_high;
 987
 988         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 989                                            sizeof(gtod_high)))
 990                 return -EFAULT;
 991
 992         if (gtod_high != 0)
 993                 return -EINVAL;
 994         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
 995
 996         return 0;
 997 }
 998
 999 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1000 {
1001         struct kvm_s390_vm_tod_clock gtod = { 0 };
1002
1003         if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
1004                            sizeof(gtod.tod)))
1005                 return -EFAULT;
1006
1007         kvm_s390_set_tod_clock(kvm, &gtod);
1008         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1009         return 0;
1010 }
1011
1012 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1013 {
1014         int ret;
1015
1016         if (attr->flags)
1017                 return -EINVAL;
1018
1019         switch (attr->attr) {
1020         case KVM_S390_VM_TOD_EXT:
1021                 ret = kvm_s390_set_tod_ext(kvm, attr);
1022                 break;
1023         case KVM_S390_VM_TOD_HIGH:
1024                 ret = kvm_s390_set_tod_high(kvm, attr);
1025                 break;
1026         case KVM_S390_VM_TOD_LOW:
1027                 ret = kvm_s390_set_tod_low(kvm, attr);
1028                 break;
1029         default:
1030                 ret = -ENXIO;
1031                 break;
1032         }
1033         return ret;
1034 }
1035
1036 static void kvm_s390_get_tod_clock_ext(struct kvm *kvm,
1037                                         struct kvm_s390_vm_tod_clock *gtod)
1038 {
1039         struct kvm_s390_tod_clock_ext htod;
1040
1041         preempt_disable();
1042
1043         get_tod_clock_ext((char *)&htod);
1044
1045         gtod->tod = htod.tod + kvm->arch.epoch;
1046         gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1047
1048         if (gtod->tod < htod.tod)
1049                 gtod->epoch_idx += 1;
1050
1051         preempt_enable();
1052 }
1053
1054 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1055 {
1056         struct kvm_s390_vm_tod_clock gtod;
1057
1058         memset(&gtod, 0, sizeof(gtod));
1059
1060         if (test_kvm_facility(kvm, 139))
1061                 kvm_s390_get_tod_clock_ext(kvm, &gtod);
1062         else
1063                 gtod.tod = kvm_s390_get_tod_clock_fast(kvm);
1064
1065         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1066                 return -EFAULT;
1067
1068         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1069                 gtod.epoch_idx, gtod.tod);
1070         return 0;
1071 }
1072
1073 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1074 {
1075         u8 gtod_high = 0;
1076
1077         if (copy_to_user((void __user *)attr->addr, &gtod_high,
1078                                          sizeof(gtod_high)))
1079                 return -EFAULT;
1080         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1081
1082         return 0;
1083 }
1084
1085 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1086 {
1087         u64 gtod;
1088
1089         gtod = kvm_s390_get_tod_clock_fast(kvm);
1090         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
1091                 return -EFAULT;
1092         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1093
1094         return 0;
1095 }
1096
1097 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1098 {
1099         int ret;
1100
1101         if (attr->flags)
1102                 return -EINVAL;
1103
1104         switch (attr->attr) {
1105         case KVM_S390_VM_TOD_EXT:
1106                 ret = kvm_s390_get_tod_ext(kvm, attr);
1107                 break;
1108         case KVM_S390_VM_TOD_HIGH:
1109                 ret = kvm_s390_get_tod_high(kvm, attr);
1110                 break;
1111         case KVM_S390_VM_TOD_LOW:
1112                 ret = kvm_s390_get_tod_low(kvm, attr);
1113                 break;
1114         default:
1115                 ret = -ENXIO;
1116                 break;
1117         }
1118         return ret;
1119 }
1120
1121 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1122 {
1123         struct kvm_s390_vm_cpu_processor *proc;
1124         u16 lowest_ibc, unblocked_ibc;
1125         int ret = 0;
1126
1127         mutex_lock(&kvm->lock);
1128         if (kvm->created_vcpus) {
1129                 ret = -EBUSY;
1130                 goto out;
1131         }
1132         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1133         if (!proc) {
1134                 ret = -ENOMEM;
1135                 goto out;
1136         }
1137         if (!copy_from_user(proc, (void __user *)attr->addr,
1138                             sizeof(*proc))) {
1139                 kvm->arch.model.cpuid = proc->cpuid;
1140                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1141                 unblocked_ibc = sclp.ibc & 0xfff;
1142                 if (lowest_ibc && proc->ibc) {
1143                         if (proc->ibc > unblocked_ibc)
1144                                 kvm->arch.model.ibc = unblocked_ibc;
1145                         else if (proc->ibc < lowest_ibc)
1146                                 kvm->arch.model.ibc = lowest_ibc;
1147                         else
1148                                 kvm->arch.model.ibc = proc->ibc;
1149                 }
1150                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1151                        S390_ARCH_FAC_LIST_SIZE_BYTE);
1152                 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1153                          kvm->arch.model.ibc,
1154                          kvm->arch.model.cpuid);
1155                 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1156                          kvm->arch.model.fac_list[0],
1157                          kvm->arch.model.fac_list[1],
1158                          kvm->arch.model.fac_list[2]);
1159         } else
1160                 ret = -EFAULT;
1161         kfree(proc);
1162 out:
1163         mutex_unlock(&kvm->lock);
1164         return ret;
1165 }
1166
1167 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1168                                        struct kvm_device_attr *attr)
1169 {
1170         struct kvm_s390_vm_cpu_feat data;
1171
1172         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1173                 return -EFAULT;
1174         if (!bitmap_subset((unsigned long *) data.feat,
1175                            kvm_s390_available_cpu_feat,
1176                            KVM_S390_VM_CPU_FEAT_NR_BITS))
1177                 return -EINVAL;
1178
1179         mutex_lock(&kvm->lock);
1180         if (kvm->created_vcpus) {
1181                 mutex_unlock(&kvm->lock);
1182                 return -EBUSY;
1183         }
1184         bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1185                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1186         mutex_unlock(&kvm->lock);
1187         VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1188                          data.feat[0],
1189                          data.feat[1],
1190                          data.feat[2]);
1191         return 0;
1192 }
1193
1194 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1195                                           struct kvm_device_attr *attr)
1196 {
1197         /*
1198          * Once supported by kernel + hw, we have to store the subfunctions
1199          * in kvm->arch and remember that user space configured them.
1200          */
1201         return -ENXIO;
1202 }
1203
1204 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1205 {
1206         int ret = -ENXIO;
1207
1208         switch (attr->attr) {
1209         case KVM_S390_VM_CPU_PROCESSOR:
1210                 ret = kvm_s390_set_processor(kvm, attr);
1211                 break;
1212         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1213                 ret = kvm_s390_set_processor_feat(kvm, attr);
1214                 break;
1215         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1216                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1217                 break;
1218         }
1219         return ret;
1220 }
1221
1222 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1223 {
1224         struct kvm_s390_vm_cpu_processor *proc;
1225         int ret = 0;
1226
1227         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1228         if (!proc) {
1229                 ret = -ENOMEM;
1230                 goto out;
1231         }
1232         proc->cpuid = kvm->arch.model.cpuid;
1233         proc->ibc = kvm->arch.model.ibc;
1234         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1235                S390_ARCH_FAC_LIST_SIZE_BYTE);
1236         VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1237                  kvm->arch.model.ibc,
1238                  kvm->arch.model.cpuid);
1239         VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1240                  kvm->arch.model.fac_list[0],
1241                  kvm->arch.model.fac_list[1],
1242                  kvm->arch.model.fac_list[2]);
1243         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1244                 ret = -EFAULT;
1245         kfree(proc);
1246 out:
1247         return ret;
1248 }
1249
1250 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1251 {
1252         struct kvm_s390_vm_cpu_machine *mach;
1253         int ret = 0;
1254
1255         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1256         if (!mach) {
1257                 ret = -ENOMEM;
1258                 goto out;
1259         }
1260         get_cpu_id((struct cpuid *) &mach->cpuid);
1261         mach->ibc = sclp.ibc;
1262         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1263                S390_ARCH_FAC_LIST_SIZE_BYTE);
1264         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1265                sizeof(S390_lowcore.stfle_fac_list));
1266         VM_EVENT(kvm, 3, "GET: host ibc:  0x%4.4x, host cpuid:  0x%16.16llx",
1267                  kvm->arch.model.ibc,
1268                  kvm->arch.model.cpuid);
1269         VM_EVENT(kvm, 3, "GET: host facmask:  0x%16.16llx.%16.16llx.%16.16llx",
1270                  mach->fac_mask[0],
1271                  mach->fac_mask[1],
1272                  mach->fac_mask[2]);
1273         VM_EVENT(kvm, 3, "GET: host faclist:  0x%16.16llx.%16.16llx.%16.16llx",
1274                  mach->fac_list[0],
1275                  mach->fac_list[1],
1276                  mach->fac_list[2]);
1277         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1278                 ret = -EFAULT;
1279         kfree(mach);
1280 out:
1281         return ret;
1282 }
1283
1284 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1285                                        struct kvm_device_attr *attr)
1286 {
1287         struct kvm_s390_vm_cpu_feat data;
1288
1289         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1290                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1291         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1292                 return -EFAULT;
1293         VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1294                          data.feat[0],
1295                          data.feat[1],
1296                          data.feat[2]);
1297         return 0;
1298 }
1299
1300 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1301                                      struct kvm_device_attr *attr)
1302 {
1303         struct kvm_s390_vm_cpu_feat data;
1304
1305         bitmap_copy((unsigned long *) data.feat,
1306                     kvm_s390_available_cpu_feat,
1307                     KVM_S390_VM_CPU_FEAT_NR_BITS);
1308         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1309                 return -EFAULT;
1310         VM_EVENT(kvm, 3, "GET: host feat:  0x%16.16llx.0x%16.16llx.0x%16.16llx",
1311                          data.feat[0],
1312                          data.feat[1],
1313                          data.feat[2]);
1314         return 0;
1315 }
1316
1317 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1318                                           struct kvm_device_attr *attr)
1319 {
1320         /*
1321          * Once we can actually configure subfunctions (kernel + hw support),
1322          * we have to check if they were already set by user space, if so copy
1323          * them from kvm->arch.
1324          */
1325         return -ENXIO;
1326 }
1327
1328 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1329                                         struct kvm_device_attr *attr)
1330 {
1331         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1332             sizeof(struct kvm_s390_vm_cpu_subfunc)))
1333                 return -EFAULT;
1334         return 0;
1335 }
1336 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1337 {
1338         int ret = -ENXIO;
1339
1340         switch (attr->attr) {
1341         case KVM_S390_VM_CPU_PROCESSOR:
1342                 ret = kvm_s390_get_processor(kvm, attr);
1343                 break;
1344         case KVM_S390_VM_CPU_MACHINE:
1345                 ret = kvm_s390_get_machine(kvm, attr);
1346                 break;
1347         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1348                 ret = kvm_s390_get_processor_feat(kvm, attr);
1349                 break;
1350         case KVM_S390_VM_CPU_MACHINE_FEAT:
1351                 ret = kvm_s390_get_machine_feat(kvm, attr);
1352                 break;
1353         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1354                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1355                 break;
1356         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1357                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1358                 break;
1359         }
1360         return ret;
1361 }
1362
1363 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1364 {
1365         int ret;
1366
1367         switch (attr->group) {
1368         case KVM_S390_VM_MEM_CTRL:
1369                 ret = kvm_s390_set_mem_control(kvm, attr);
1370                 break;
1371         case KVM_S390_VM_TOD:
1372                 ret = kvm_s390_set_tod(kvm, attr);
1373                 break;
1374         case KVM_S390_VM_CPU_MODEL:
1375                 ret = kvm_s390_set_cpu_model(kvm, attr);
1376                 break;
1377         case KVM_S390_VM_CRYPTO:
1378                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1379                 break;
1380         case KVM_S390_VM_MIGRATION:
1381                 ret = kvm_s390_vm_set_migration(kvm, attr);
1382                 break;
1383         default:
1384                 ret = -ENXIO;
1385                 break;
1386         }
1387
1388         return ret;
1389 }
1390
1391 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1392 {
1393         int ret;
1394
1395         switch (attr->group) {
1396         case KVM_S390_VM_MEM_CTRL:
1397                 ret = kvm_s390_get_mem_control(kvm, attr);
1398                 break;
1399         case KVM_S390_VM_TOD:
1400                 ret = kvm_s390_get_tod(kvm, attr);
1401                 break;
1402         case KVM_S390_VM_CPU_MODEL:
1403                 ret = kvm_s390_get_cpu_model(kvm, attr);
1404                 break;
1405         case KVM_S390_VM_MIGRATION:
1406                 ret = kvm_s390_vm_get_migration(kvm, attr);
1407                 break;
1408         default:
1409                 ret = -ENXIO;
1410                 break;
1411         }
1412
1413         return ret;
1414 }
1415
1416 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1417 {
1418         int ret;
1419
1420         switch (attr->group) {
1421         case KVM_S390_VM_MEM_CTRL:
1422                 switch (attr->attr) {
1423                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1424                 case KVM_S390_VM_MEM_CLR_CMMA:
1425                         ret = sclp.has_cmma ? 0 : -ENXIO;
1426                         break;
1427                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1428                         ret = 0;
1429                         break;
1430                 default:
1431                         ret = -ENXIO;
1432                         break;
1433                 }
1434                 break;
1435         case KVM_S390_VM_TOD:
1436                 switch (attr->attr) {
1437                 case KVM_S390_VM_TOD_LOW:
1438                 case KVM_S390_VM_TOD_HIGH:
1439                         ret = 0;
1440                         break;
1441                 default:
1442                         ret = -ENXIO;
1443                         break;
1444                 }
1445                 break;
1446         case KVM_S390_VM_CPU_MODEL:
1447                 switch (attr->attr) {
1448                 case KVM_S390_VM_CPU_PROCESSOR:
1449                 case KVM_S390_VM_CPU_MACHINE:
1450                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1451                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1452                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1453                         ret = 0;
1454                         break;
1455                 /* configuring subfunctions is not supported yet */
1456                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1457                 default:
1458                         ret = -ENXIO;
1459                         break;
1460                 }
1461                 break;
1462         case KVM_S390_VM_CRYPTO:
1463                 switch (attr->attr) {
1464                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1465                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1466                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1467                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1468                         ret = 0;
1469                         break;
1470                 default:
1471                         ret = -ENXIO;
1472                         break;
1473                 }
1474                 break;
1475         case KVM_S390_VM_MIGRATION:
1476                 ret = 0;
1477                 break;
1478         default:
1479                 ret = -ENXIO;
1480                 break;
1481         }
1482
1483         return ret;
1484 }
1485
1486 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1487 {
1488         uint8_t *keys;
1489         uint64_t hva;
1490         int srcu_idx, i, r = 0;
1491
1492         if (args->flags != 0)
1493                 return -EINVAL;
1494
1495         /* Is this guest using storage keys? */
1496         if (!mm_use_skey(current->mm))
1497                 return KVM_S390_GET_SKEYS_NONE;
1498
1499         /* Enforce sane limit on memory allocation */
1500         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1501                 return -EINVAL;
1502
1503         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1504         if (!keys)
1505                 return -ENOMEM;
1506
1507         down_read(&current->mm->mmap_sem);
1508         srcu_idx = srcu_read_lock(&kvm->srcu);
1509         for (i = 0; i < args->count; i++) {
1510                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1511                 if (kvm_is_error_hva(hva)) {
1512                         r = -EFAULT;
1513                         break;
1514                 }
1515
1516                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1517                 if (r)
1518                         break;
1519         }
1520         srcu_read_unlock(&kvm->srcu, srcu_idx);
1521         up_read(&current->mm->mmap_sem);
1522
1523         if (!r) {
1524                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1525                                  sizeof(uint8_t) * args->count);
1526                 if (r)
1527                         r = -EFAULT;
1528         }
1529
1530         kvfree(keys);
1531         return r;
1532 }
1533
1534 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1535 {
1536         uint8_t *keys;
1537         uint64_t hva;
1538         int srcu_idx, i, r = 0;
1539
1540         if (args->flags != 0)
1541                 return -EINVAL;
1542
1543         /* Enforce sane limit on memory allocation */
1544         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1545                 return -EINVAL;
1546
1547         keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1548         if (!keys)
1549                 return -ENOMEM;
1550
1551         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1552                            sizeof(uint8_t) * args->count);
1553         if (r) {
1554                 r = -EFAULT;
1555                 goto out;
1556         }
1557
1558         /* Enable storage key handling for the guest */
1559         r = s390_enable_skey();
1560         if (r)
1561                 goto out;
1562
1563         down_read(&current->mm->mmap_sem);
1564         srcu_idx = srcu_read_lock(&kvm->srcu);
1565         for (i = 0; i < args->count; i++) {
1566                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1567                 if (kvm_is_error_hva(hva)) {
1568                         r = -EFAULT;
1569                         break;
1570                 }
1571
1572                 /* Lowest order bit is reserved */
1573                 if (keys[i] & 0x01) {
1574                         r = -EINVAL;
1575                         break;
1576                 }
1577
1578                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1579                 if (r)
1580                         break;
1581         }
1582         srcu_read_unlock(&kvm->srcu, srcu_idx);
1583         up_read(&current->mm->mmap_sem);
1584 out:
1585         kvfree(keys);
1586         return r;
1587 }
1588
1589 /*
1590  * Base address and length must be sent at the start of each block, therefore
1591  * it's cheaper to send some clean data, as long as it's less than the size of
1592  * two longs.
1593  */
1594 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1595 /* for consistency */
1596 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1597
1598 /*
1599  * This function searches for the next page with dirty CMMA attributes, and
1600  * saves the attributes in the buffer up to either the end of the buffer or
1601  * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
1602  * no trailing clean bytes are saved.
1603  * In case no dirty bits were found, or if CMMA was not enabled or used, the
1604  * output buffer will indicate 0 as length.
1605  */
1606 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1607                                   struct kvm_s390_cmma_log *args)
1608 {
1609         struct kvm_s390_migration_state *s = kvm->arch.migration_state;
1610         unsigned long bufsize, hva, pgstev, i, next, cur;
1611         int srcu_idx, peek, r = 0, rr;
1612         u8 *res;
1613
1614         cur = args->start_gfn;
1615         i = next = pgstev = 0;
1616
1617         if (unlikely(!kvm->arch.use_cmma))
1618                 return -ENXIO;
1619         /* Invalid/unsupported flags were specified */
1620         if (args->flags & ~KVM_S390_CMMA_PEEK)
1621                 return -EINVAL;
1622         /* Migration mode query, and we are not doing a migration */
1623         peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1624         if (!peek && !s)
1625                 return -EINVAL;
1626         /* CMMA is disabled or was not used, or the buffer has length zero */
1627         bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
1628         if (!bufsize || !kvm->mm->context.uses_cmm) {
1629                 memset(args, 0, sizeof(*args));
1630                 return 0;
1631         }
1632
1633         if (!peek) {
1634                 /* We are not peeking, and there are no dirty pages */
1635                 if (!atomic64_read(&s->dirty_pages)) {
1636                         memset(args, 0, sizeof(*args));
1637                         return 0;
1638                 }
1639                 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1640                                     args->start_gfn);
1641                 if (cur >= s->bitmap_size)      /* nothing found, loop back */
1642                         cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1643                 if (cur >= s->bitmap_size) {    /* again! (very unlikely) */
1644                         memset(args, 0, sizeof(*args));
1645                         return 0;
1646                 }
1647                 next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
1648         }
1649
1650         res = vmalloc(bufsize);
1651         if (!res)
1652                 return -ENOMEM;
1653
1654         args->start_gfn = cur;
1655
1656         down_read(&kvm->mm->mmap_sem);
1657         srcu_idx = srcu_read_lock(&kvm->srcu);
1658         while (i < bufsize) {
1659                 hva = gfn_to_hva(kvm, cur);
1660                 if (kvm_is_error_hva(hva)) {
1661                         r = -EFAULT;
1662                         break;
1663                 }
1664                 /* decrement only if we actually flipped the bit to 0 */
1665                 if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1666                         atomic64_dec(&s->dirty_pages);
1667                 r = get_pgste(kvm->mm, hva, &pgstev);
1668                 if (r < 0)
1669                         pgstev = 0;
1670                 /* save the value */
1671                 res[i++] = (pgstev >> 24) & 0x43;
1672                 /*
1673                  * if the next bit is too far away, stop.
1674                  * if we reached the previous "next", find the next one
1675                  */
1676                 if (!peek) {
1677                         if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1678                                 break;
1679                         if (cur == next)
1680                                 next = find_next_bit(s->pgste_bitmap,
1681                                                      s->bitmap_size, cur + 1);
1682                 /* reached the end of the bitmap or of the buffer, stop */
1683                         if ((next >= s->bitmap_size) ||
1684                             (next >= args->start_gfn + bufsize))
1685                                 break;
1686                 }
1687                 cur++;
1688         }
1689         srcu_read_unlock(&kvm->srcu, srcu_idx);
1690         up_read(&kvm->mm->mmap_sem);
1691         args->count = i;
1692         args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1693
1694         rr = copy_to_user((void __user *)args->values, res, args->count);
1695         if (rr)
1696                 r = -EFAULT;
1697
1698         vfree(res);
1699         return r;
1700 }
1701
1702 /*
1703  * This function sets the CMMA attributes for the given pages. If the input
1704  * buffer has zero length, no action is taken, otherwise the attributes are
1705  * set and the mm->context.uses_cmm flag is set.
1706  */
1707 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
1708                                   const struct kvm_s390_cmma_log *args)
1709 {
1710         unsigned long hva, mask, pgstev, i;
1711         uint8_t *bits;
1712         int srcu_idx, r = 0;
1713
1714         mask = args->mask;
1715
1716         if (!kvm->arch.use_cmma)
1717                 return -ENXIO;
1718         /* invalid/unsupported flags */
1719         if (args->flags != 0)
1720                 return -EINVAL;
1721         /* Enforce sane limit on memory allocation */
1722         if (args->count > KVM_S390_CMMA_SIZE_MAX)
1723                 return -EINVAL;
1724         /* Nothing to do */
1725         if (args->count == 0)
1726                 return 0;
1727
1728         bits = vmalloc(array_size(sizeof(*bits), args->count));
1729         if (!bits)
1730                 return -ENOMEM;
1731
1732         r = copy_from_user(bits, (void __user *)args->values, args->count);
1733         if (r) {
1734                 r = -EFAULT;
1735                 goto out;
1736         }
1737
1738         down_read(&kvm->mm->mmap_sem);
1739         srcu_idx = srcu_read_lock(&kvm->srcu);
1740         for (i = 0; i < args->count; i++) {
1741                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1742                 if (kvm_is_error_hva(hva)) {
1743                         r = -EFAULT;
1744                         break;
1745                 }
1746
1747                 pgstev = bits[i];
1748                 pgstev = pgstev << 24;
1749                 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
1750                 set_pgste_bits(kvm->mm, hva, mask, pgstev);
1751         }
1752         srcu_read_unlock(&kvm->srcu, srcu_idx);
1753         up_read(&kvm->mm->mmap_sem);
1754
1755         if (!kvm->mm->context.uses_cmm) {
1756                 down_write(&kvm->mm->mmap_sem);
1757                 kvm->mm->context.uses_cmm = 1;
1758                 up_write(&kvm->mm->mmap_sem);
1759         }
1760 out:
1761         vfree(bits);
1762         return r;
1763 }
1764
1765 long kvm_arch_vm_ioctl(struct file *filp,
1766                        unsigned int ioctl, unsigned long arg)
1767 {
1768         struct kvm *kvm = filp->private_data;
1769         void __user *argp = (void __user *)arg;
1770         struct kvm_device_attr attr;
1771         int r;
1772
1773         switch (ioctl) {
1774         case KVM_S390_INTERRUPT: {
1775                 struct kvm_s390_interrupt s390int;
1776
1777                 r = -EFAULT;
1778                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1779                         break;
1780                 r = kvm_s390_inject_vm(kvm, &s390int);
1781                 break;
1782         }
1783         case KVM_ENABLE_CAP: {
1784                 struct kvm_enable_cap cap;
1785                 r = -EFAULT;
1786                 if (copy_from_user(&cap, argp, sizeof(cap)))
1787                         break;
1788                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1789                 break;
1790         }
1791         case KVM_CREATE_IRQCHIP: {
1792                 struct kvm_irq_routing_entry routing;
1793
1794                 r = -EINVAL;
1795                 if (kvm->arch.use_irqchip) {
1796                         /* Set up dummy routing. */
1797                         memset(&routing, 0, sizeof(routing));
1798                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1799                 }
1800                 break;
1801         }
1802         case KVM_SET_DEVICE_ATTR: {
1803                 r = -EFAULT;
1804                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1805                         break;
1806                 r = kvm_s390_vm_set_attr(kvm, &attr);
1807                 break;
1808         }
1809         case KVM_GET_DEVICE_ATTR: {
1810                 r = -EFAULT;
1811                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1812                         break;
1813                 r = kvm_s390_vm_get_attr(kvm, &attr);
1814                 break;
1815         }
1816         case KVM_HAS_DEVICE_ATTR: {
1817                 r = -EFAULT;
1818                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1819                         break;
1820                 r = kvm_s390_vm_has_attr(kvm, &attr);
1821                 break;
1822         }
1823         case KVM_S390_GET_SKEYS: {
1824                 struct kvm_s390_skeys args;
1825
1826                 r = -EFAULT;
1827                 if (copy_from_user(&args, argp,
1828                                    sizeof(struct kvm_s390_skeys)))
1829                         break;
1830                 r = kvm_s390_get_skeys(kvm, &args);
1831                 break;
1832         }
1833         case KVM_S390_SET_SKEYS: {
1834                 struct kvm_s390_skeys args;
1835
1836                 r = -EFAULT;
1837                 if (copy_from_user(&args, argp,
1838                                    sizeof(struct kvm_s390_skeys)))
1839                         break;
1840                 r = kvm_s390_set_skeys(kvm, &args);
1841                 break;
1842         }
1843         case KVM_S390_GET_CMMA_BITS: {
1844                 struct kvm_s390_cmma_log args;
1845
1846                 r = -EFAULT;
1847                 if (copy_from_user(&args, argp, sizeof(args)))
1848                         break;
1849                 mutex_lock(&kvm->slots_lock);
1850                 r = kvm_s390_get_cmma_bits(kvm, &args);
1851                 mutex_unlock(&kvm->slots_lock);
1852                 if (!r) {
1853                         r = copy_to_user(argp, &args, sizeof(args));
1854                         if (r)
1855                                 r = -EFAULT;
1856                 }
1857                 break;
1858         }
1859         case KVM_S390_SET_CMMA_BITS: {
1860                 struct kvm_s390_cmma_log args;
1861
1862                 r = -EFAULT;
1863                 if (copy_from_user(&args, argp, sizeof(args)))
1864                         break;
1865                 mutex_lock(&kvm->slots_lock);
1866                 r = kvm_s390_set_cmma_bits(kvm, &args);
1867                 mutex_unlock(&kvm->slots_lock);
1868                 break;
1869         }
1870         default:
1871                 r = -ENOTTY;
1872         }
1873
1874         return r;
1875 }
1876
1877 static int kvm_s390_query_ap_config(u8 *config)
1878 {
1879         u32 fcn_code = 0x04000000UL;
1880         u32 cc = 0;
1881
1882         memset(config, 0, 128);
1883         asm volatile(
1884                 "lgr 0,%1\n"
1885                 "lgr 2,%2\n"
1886                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1887                 "0: ipm %0\n"
1888                 "srl %0,28\n"
1889                 "1:\n"
1890                 EX_TABLE(0b, 1b)
1891                 : "+r" (cc)
1892                 : "r" (fcn_code), "r" (config)
1893                 : "cc", "0", "2", "memory"
1894         );
1895
1896         return cc;
1897 }
1898
1899 static int kvm_s390_apxa_installed(void)
1900 {
1901         u8 config[128];
1902         int cc;
1903
1904         if (test_facility(12)) {
1905                 cc = kvm_s390_query_ap_config(config);
1906
1907                 if (cc)
1908                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1909                 else
1910                         return config[0] & 0x40;
1911         }
1912
1913         return 0;
1914 }
1915
1916 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1917 {
1918         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1919
1920         if (kvm_s390_apxa_installed())
1921                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1922         else
1923                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1924 }
1925
1926 static u64 kvm_s390_get_initial_cpuid(void)
1927 {
1928         struct cpuid cpuid;
1929
1930         get_cpu_id(&cpuid);
1931         cpuid.version = 0xff;
1932         return *((u64 *) &cpuid);
1933 }
1934
1935 static void kvm_s390_crypto_init(struct kvm *kvm)
1936 {
1937         if (!test_kvm_facility(kvm, 76))
1938                 return;
1939
1940         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1941         kvm_s390_set_crycb_format(kvm);
1942
1943         /* Enable AES/DEA protected key functions by default */
1944         kvm->arch.crypto.aes_kw = 1;
1945         kvm->arch.crypto.dea_kw = 1;
1946         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1947                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1948         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1949                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1950 }
1951
1952 static void sca_dispose(struct kvm *kvm)
1953 {
1954         if (kvm->arch.use_esca)
1955                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1956         else
1957                 free_page((unsigned long)(kvm->arch.sca));
1958         kvm->arch.sca = NULL;
1959 }
1960
1961 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1962 {
1963         gfp_t alloc_flags = GFP_KERNEL;
1964         int i, rc;
1965         char debug_name[16];
1966         static unsigned long sca_offset;
1967
1968         rc = -EINVAL;
1969 #ifdef CONFIG_KVM_S390_UCONTROL
1970         if (type & ~KVM_VM_S390_UCONTROL)
1971                 goto out_err;
1972         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1973                 goto out_err;
1974 #else
1975         if (type)
1976                 goto out_err;
1977 #endif
1978
1979         rc = s390_enable_sie();
1980         if (rc)
1981                 goto out_err;
1982
1983         rc = -ENOMEM;
1984
1985         kvm->arch.use_esca = 0; /* start with basic SCA */
1986         if (!sclp.has_64bscao)
1987                 alloc_flags |= GFP_DMA;
1988         rwlock_init(&kvm->arch.sca_lock);
1989         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1990         if (!kvm->arch.sca)
1991                 goto out_err;
1992         spin_lock(&kvm_lock);
1993         sca_offset += 16;
1994         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1995                 sca_offset = 0;
1996         kvm->arch.sca = (struct bsca_block *)
1997                         ((char *) kvm->arch.sca + sca_offset);
1998         spin_unlock(&kvm_lock);
1999
2000         sprintf(debug_name, "kvm-%u", current->pid);
2001
2002         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2003         if (!kvm->arch.dbf)
2004                 goto out_err;
2005
2006         BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2007         kvm->arch.sie_page2 =
2008              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2009         if (!kvm->arch.sie_page2)
2010                 goto out_err;
2011
2012         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2013
2014         for (i = 0; i < kvm_s390_fac_size(); i++) {
2015                 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2016                                               (kvm_s390_fac_base[i] |
2017                                                kvm_s390_fac_ext[i]);
2018                 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2019                                               kvm_s390_fac_base[i];
2020         }
2021
2022         /* we are always in czam mode - even on pre z14 machines */
2023         set_kvm_facility(kvm->arch.model.fac_mask, 138);
2024         set_kvm_facility(kvm->arch.model.fac_list, 138);
2025         /* we emulate STHYI in kvm */
2026         set_kvm_facility(kvm->arch.model.fac_mask, 74);
2027         set_kvm_facility(kvm->arch.model.fac_list, 74);
2028         if (MACHINE_HAS_TLB_GUEST) {
2029                 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2030                 set_kvm_facility(kvm->arch.model.fac_list, 147);
2031         }
2032
2033         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2034         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2035
2036         kvm_s390_crypto_init(kvm);
2037
2038         mutex_init(&kvm->arch.float_int.ais_lock);
2039         kvm->arch.float_int.simm = 0;
2040         kvm->arch.float_int.nimm = 0;
2041         spin_lock_init(&kvm->arch.float_int.lock);
2042         for (i = 0; i < FIRQ_LIST_COUNT; i++)
2043                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2044         init_waitqueue_head(&kvm->arch.ipte_wq);
2045         mutex_init(&kvm->arch.ipte_mutex);
2046
2047         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2048         VM_EVENT(kvm, 3, "vm created with type %lu", type);
2049
2050         if (type & KVM_VM_S390_UCONTROL) {
2051                 kvm->arch.gmap = NULL;
2052                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2053         } else {
2054                 if (sclp.hamax == U64_MAX)
2055                         kvm->arch.mem_limit = TASK_SIZE_MAX;
2056                 else
2057                         kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2058                                                     sclp.hamax + 1);
2059                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2060                 if (!kvm->arch.gmap)
2061                         goto out_err;
2062                 kvm->arch.gmap->private = kvm;
2063                 kvm->arch.gmap->pfault_enabled = 0;
2064         }
2065
2066         kvm->arch.css_support = 0;
2067         kvm->arch.use_irqchip = 0;
2068         kvm->arch.use_pfmfi = sclp.has_pfmfi;
2069         kvm->arch.epoch = 0;
2070
2071         spin_lock_init(&kvm->arch.start_stop_lock);
2072         kvm_s390_vsie_init(kvm);
2073         kvm_s390_gisa_init(kvm);
2074         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2075
2076         return 0;
2077 out_err:
2078         free_page((unsigned long)kvm->arch.sie_page2);
2079         debug_unregister(kvm->arch.dbf);
2080         sca_dispose(kvm);
2081         KVM_EVENT(3, "creation of vm failed: %d", rc);
2082         return rc;
2083 }
2084
2085 bool kvm_arch_has_vcpu_debugfs(void)
2086 {
2087         return false;
2088 }
2089
2090 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
2091 {
2092         return 0;
2093 }
2094
2095 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2096 {
2097         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2098         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2099         kvm_s390_clear_local_irqs(vcpu);
2100         kvm_clear_async_pf_completion_queue(vcpu);
2101         if (!kvm_is_ucontrol(vcpu->kvm))
2102                 sca_del_vcpu(vcpu);
2103
2104         if (kvm_is_ucontrol(vcpu->kvm))
2105                 gmap_remove(vcpu->arch.gmap);
2106
2107         if (vcpu->kvm->arch.use_cmma)
2108                 kvm_s390_vcpu_unsetup_cmma(vcpu);
2109         free_page((unsigned long)(vcpu->arch.sie_block));
2110
2111         kvm_vcpu_uninit(vcpu);
2112         kmem_cache_free(kvm_vcpu_cache, vcpu);
2113 }
2114
2115 static void kvm_free_vcpus(struct kvm *kvm)
2116 {
2117         unsigned int i;
2118         struct kvm_vcpu *vcpu;
2119
2120         kvm_for_each_vcpu(i, vcpu, kvm)
2121                 kvm_arch_vcpu_destroy(vcpu);
2122
2123         mutex_lock(&kvm->lock);
2124         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2125                 kvm->vcpus[i] = NULL;
2126
2127         atomic_set(&kvm->online_vcpus, 0);
2128         mutex_unlock(&kvm->lock);
2129 }
2130
2131 void kvm_arch_destroy_vm(struct kvm *kvm)
2132 {
2133         kvm_free_vcpus(kvm);
2134         sca_dispose(kvm);
2135         debug_unregister(kvm->arch.dbf);
2136         kvm_s390_gisa_destroy(kvm);
2137         free_page((unsigned long)kvm->arch.sie_page2);
2138         if (!kvm_is_ucontrol(kvm))
2139                 gmap_remove(kvm->arch.gmap);
2140         kvm_s390_destroy_adapters(kvm);
2141         kvm_s390_clear_float_irqs(kvm);
2142         kvm_s390_vsie_destroy(kvm);
2143         if (kvm->arch.migration_state) {
2144                 vfree(kvm->arch.migration_state->pgste_bitmap);
2145                 kfree(kvm->arch.migration_state);
2146         }
2147         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2148 }
2149
2150 /* Section: vcpu related */
2151 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2152 {
2153         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2154         if (!vcpu->arch.gmap)
2155                 return -ENOMEM;
2156         vcpu->arch.gmap->private = vcpu->kvm;
2157
2158         return 0;
2159 }
2160
2161 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2162 {
2163         if (!kvm_s390_use_sca_entries())
2164                 return;
2165         read_lock(&vcpu->kvm->arch.sca_lock);
2166         if (vcpu->kvm->arch.use_esca) {
2167                 struct esca_block *sca = vcpu->kvm->arch.sca;
2168
2169                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2170                 sca->cpu[vcpu->vcpu_id].sda = 0;
2171         } else {
2172                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2173
2174                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2175                 sca->cpu[vcpu->vcpu_id].sda = 0;
2176         }
2177         read_unlock(&vcpu->kvm->arch.sca_lock);
2178 }
2179
2180 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2181 {
2182         if (!kvm_s390_use_sca_entries()) {
2183                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2184
2185                 /* we still need the basic sca for the ipte control */
2186                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2187                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2188                 return;
2189         }
2190         read_lock(&vcpu->kvm->arch.sca_lock);
2191         if (vcpu->kvm->arch.use_esca) {
2192                 struct esca_block *sca = vcpu->kvm->arch.sca;
2193
2194                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2195                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2196                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2197                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2198                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2199         } else {
2200                 struct bsca_block *sca = vcpu->kvm->arch.sca;
2201
2202                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2203                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2204                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2205                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2206         }
2207         read_unlock(&vcpu->kvm->arch.sca_lock);
2208 }
2209
2210 /* Basic SCA to Extended SCA data copy routines */
2211 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2212 {
2213         d->sda = s->sda;
2214         d->sigp_ctrl.c = s->sigp_ctrl.c;
2215         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2216 }
2217
2218 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2219 {
2220         int i;
2221
2222         d->ipte_control = s->ipte_control;
2223         d->mcn[0] = s->mcn;
2224         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2225                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2226 }
2227
2228 static int sca_switch_to_extended(struct kvm *kvm)
2229 {
2230         struct bsca_block *old_sca = kvm->arch.sca;
2231         struct esca_block *new_sca;
2232         struct kvm_vcpu *vcpu;
2233         unsigned int vcpu_idx;
2234         u32 scaol, scaoh;
2235
2236         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2237         if (!new_sca)
2238                 return -ENOMEM;
2239
2240         scaoh = (u32)((u64)(new_sca) >> 32);
2241         scaol = (u32)(u64)(new_sca) & ~0x3fU;
2242
2243         kvm_s390_vcpu_block_all(kvm);
2244         write_lock(&kvm->arch.sca_lock);
2245
2246         sca_copy_b_to_e(new_sca, old_sca);
2247
2248         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2249                 vcpu->arch.sie_block->scaoh = scaoh;
2250                 vcpu->arch.sie_block->scaol = scaol;
2251                 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2252         }
2253         kvm->arch.sca = new_sca;
2254         kvm->arch.use_esca = 1;
2255
2256         write_unlock(&kvm->arch.sca_lock);
2257         kvm_s390_vcpu_unblock_all(kvm);
2258
2259         free_page((unsigned long)old_sca);
2260
2261         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2262                  old_sca, kvm->arch.sca);
2263         return 0;
2264 }
2265
2266 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2267 {
2268         int rc;
2269
2270         if (!kvm_s390_use_sca_entries()) {
2271                 if (id < KVM_MAX_VCPUS)
2272                         return true;
2273                 return false;
2274         }
2275         if (id < KVM_S390_BSCA_CPU_SLOTS)
2276                 return true;
2277         if (!sclp.has_esca || !sclp.has_64bscao)
2278                 return false;
2279
2280         mutex_lock(&kvm->lock);
2281         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2282         mutex_unlock(&kvm->lock);
2283
2284         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2285 }
2286
2287 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2288 {
2289         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2290         kvm_clear_async_pf_completion_queue(vcpu);
2291         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2292                                     KVM_SYNC_GPRS |
2293                                     KVM_SYNC_ACRS |
2294                                     KVM_SYNC_CRS |
2295                                     KVM_SYNC_ARCH0 |
2296                                     KVM_SYNC_PFAULT;
2297         kvm_s390_set_prefix(vcpu, 0);
2298         if (test_kvm_facility(vcpu->kvm, 64))
2299                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
2300         if (test_kvm_facility(vcpu->kvm, 82))
2301                 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
2302         if (test_kvm_facility(vcpu->kvm, 133))
2303                 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
2304         /* fprs can be synchronized via vrs, even if the guest has no vx. With
2305          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2306          */
2307         if (MACHINE_HAS_VX)
2308                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
2309         else
2310                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
2311
2312         if (kvm_is_ucontrol(vcpu->kvm))
2313                 return __kvm_ucontrol_vcpu_init(vcpu);
2314
2315         return 0;
2316 }
2317
2318 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2319 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2320 {
2321         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2322         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2323         vcpu->arch.cputm_start = get_tod_clock_fast();
2324         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2325 }
2326
2327 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2328 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2329 {
2330         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2331         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2332         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2333         vcpu->arch.cputm_start = 0;
2334         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2335 }
2336
2337 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2338 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2339 {
2340         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2341         vcpu->arch.cputm_enabled = true;
2342         __start_cpu_timer_accounting(vcpu);
2343 }
2344
2345 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2346 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2347 {
2348         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2349         __stop_cpu_timer_accounting(vcpu);
2350         vcpu->arch.cputm_enabled = false;
2351 }
2352
2353 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2354 {
2355         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2356         __enable_cpu_timer_accounting(vcpu);
2357         preempt_enable();
2358 }
2359
2360 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2361 {
2362         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2363         __disable_cpu_timer_accounting(vcpu);
2364         preempt_enable();
2365 }
2366
2367 /* set the cpu timer - may only be called from the VCPU thread itself */
2368 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2369 {
2370         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2371         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2372         if (vcpu->arch.cputm_enabled)
2373                 vcpu->arch.cputm_start = get_tod_clock_fast();
2374         vcpu->arch.sie_block->cputm = cputm;
2375         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2376         preempt_enable();
2377 }
2378
2379 /* update and get the cpu timer - can also be called from other VCPU threads */
2380 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2381 {
2382         unsigned int seq;
2383         __u64 value;
2384
2385         if (unlikely(!vcpu->arch.cputm_enabled))
2386                 return vcpu->arch.sie_block->cputm;
2387
2388         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2389         do {
2390                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2391                 /*
2392                  * If the writer would ever execute a read in the critical
2393                  * section, e.g. in irq context, we have a deadlock.
2394                  */
2395                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2396                 value = vcpu->arch.sie_block->cputm;
2397                 /* if cputm_start is 0, accounting is being started/stopped */
2398                 if (likely(vcpu->arch.cputm_start))
2399                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2400         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2401         preempt_enable();
2402         return value;
2403 }
2404
2405 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2406 {
2407
2408         gmap_enable(vcpu->arch.enabled_gmap);
2409         kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
2410         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2411                 __start_cpu_timer_accounting(vcpu);
2412         vcpu->cpu = cpu;
2413 }
2414
2415 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2416 {
2417         vcpu->cpu = -1;
2418         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2419                 __stop_cpu_timer_accounting(vcpu);
2420         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
2421         vcpu->arch.enabled_gmap = gmap_get_enabled();
2422         gmap_disable(vcpu->arch.enabled_gmap);
2423
2424 }
2425
2426 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
2427 {
2428         /* this equals initial cpu reset in pop, but we don't switch to ESA */
2429         vcpu->arch.sie_block->gpsw.mask = 0UL;
2430         vcpu->arch.sie_block->gpsw.addr = 0UL;
2431         kvm_s390_set_prefix(vcpu, 0);
2432         kvm_s390_set_cpu_timer(vcpu, 0);
2433         vcpu->arch.sie_block->ckc       = 0UL;
2434         vcpu->arch.sie_block->todpr     = 0;
2435         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
2436         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
2437         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
2438         /* make sure the new fpc will be lazily loaded */
2439         save_fpu_regs();
2440         current->thread.fpu.fpc = 0;
2441         vcpu->arch.sie_block->gbea = 1;
2442         vcpu->arch.sie_block->pp = 0;
2443         vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
2444         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2445         kvm_clear_async_pf_completion_queue(vcpu);
2446         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
2447                 kvm_s390_vcpu_stop(vcpu);
2448         kvm_s390_clear_local_irqs(vcpu);
2449 }
2450
2451 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2452 {
2453         mutex_lock(&vcpu->kvm->lock);
2454         preempt_disable();
2455         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2456         vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
2457         preempt_enable();
2458         mutex_unlock(&vcpu->kvm->lock);
2459         if (!kvm_is_ucontrol(vcpu->kvm)) {
2460                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2461                 sca_add_vcpu(vcpu);
2462         }
2463         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2464                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2465         /* make vcpu_load load the right gmap on the first trigger */
2466         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2467 }
2468
2469 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2470 {
2471         if (!test_kvm_facility(vcpu->kvm, 76))
2472                 return;
2473
2474         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2475
2476         if (vcpu->kvm->arch.crypto.aes_kw)
2477                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2478         if (vcpu->kvm->arch.crypto.dea_kw)
2479                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2480
2481         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2482 }
2483
2484 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2485 {
2486         free_page(vcpu->arch.sie_block->cbrlo);
2487         vcpu->arch.sie_block->cbrlo = 0;
2488 }
2489
2490 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2491 {
2492         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2493         if (!vcpu->arch.sie_block->cbrlo)
2494                 return -ENOMEM;
2495         return 0;
2496 }
2497
2498 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2499 {
2500         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2501
2502         vcpu->arch.sie_block->ibc = model->ibc;
2503         if (test_kvm_facility(vcpu->kvm, 7))
2504                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2505 }
2506
2507 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2508 {
2509         int rc = 0;
2510
2511         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2512                                                     CPUSTAT_SM |
2513                                                     CPUSTAT_STOPPED);
2514
2515         if (test_kvm_facility(vcpu->kvm, 78))
2516                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
2517         else if (test_kvm_facility(vcpu->kvm, 8))
2518                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
2519
2520         kvm_s390_vcpu_setup_model(vcpu);
2521
2522         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2523         if (MACHINE_HAS_ESOP)
2524                 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2525         if (test_kvm_facility(vcpu->kvm, 9))
2526                 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2527         if (test_kvm_facility(vcpu->kvm, 73))
2528                 vcpu->arch.sie_block->ecb |= ECB_TE;
2529
2530         if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
2531                 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2532         if (test_kvm_facility(vcpu->kvm, 130))
2533                 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2534         vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2535         if (sclp.has_cei)
2536                 vcpu->arch.sie_block->eca |= ECA_CEI;
2537         if (sclp.has_ib)
2538                 vcpu->arch.sie_block->eca |= ECA_IB;
2539         if (sclp.has_siif)
2540                 vcpu->arch.sie_block->eca |= ECA_SII;
2541         if (sclp.has_sigpif)
2542                 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2543         if (test_kvm_facility(vcpu->kvm, 129)) {
2544                 vcpu->arch.sie_block->eca |= ECA_VX;
2545                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2546         }
2547         if (test_kvm_facility(vcpu->kvm, 139))
2548                 vcpu->arch.sie_block->ecd |= ECD_MEF;
2549
2550         if (vcpu->arch.sie_block->gd) {
2551                 vcpu->arch.sie_block->eca |= ECA_AIV;
2552                 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
2553                            vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
2554         }
2555         vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2556                                         | SDNXC;
2557         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2558
2559         if (sclp.has_kss)
2560                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
2561         else
2562                 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2563
2564         if (vcpu->kvm->arch.use_cmma) {
2565                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
2566                 if (rc)
2567                         return rc;
2568         }
2569         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2570         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
2571
2572         kvm_s390_vcpu_crypto_setup(vcpu);
2573
2574         return rc;
2575 }
2576
2577 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
2578                                       unsigned int id)
2579 {
2580         struct kvm_vcpu *vcpu;
2581         struct sie_page *sie_page;
2582         int rc = -EINVAL;
2583
2584         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
2585                 goto out;
2586
2587         rc = -ENOMEM;
2588
2589         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
2590         if (!vcpu)
2591                 goto out;
2592
2593         BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
2594         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
2595         if (!sie_page)
2596                 goto out_free_cpu;
2597
2598         vcpu->arch.sie_block = &sie_page->sie_block;
2599         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
2600
2601         /* the real guest size will always be smaller than msl */
2602         vcpu->arch.sie_block->mso = 0;
2603         vcpu->arch.sie_block->msl = sclp.hamax;
2604
2605         vcpu->arch.sie_block->icpua = id;
2606         spin_lock_init(&vcpu->arch.local_int.lock);
2607         vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa;
2608         if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
2609                 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
2610         seqcount_init(&vcpu->arch.cputm_seqcount);
2611
2612         rc = kvm_vcpu_init(vcpu, kvm, id);
2613         if (rc)
2614                 goto out_free_sie_block;
2615         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
2616                  vcpu->arch.sie_block);
2617         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2618
2619         return vcpu;
2620 out_free_sie_block:
2621         free_page((unsigned long)(vcpu->arch.sie_block));
2622 out_free_cpu:
2623         kmem_cache_free(kvm_vcpu_cache, vcpu);
2624 out:
2625         return ERR_PTR(rc);
2626 }
2627
2628 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2629 {
2630         return kvm_s390_vcpu_has_irq(vcpu, 0);
2631 }
2632
2633 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
2634 {
2635         return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
2636 }
2637
2638 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2639 {
2640         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2641         exit_sie(vcpu);
2642 }
2643
2644 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2645 {
2646         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2647 }
2648
2649 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2650 {
2651         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2652         exit_sie(vcpu);
2653 }
2654
2655 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2656 {
2657         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2658 }
2659
2660 /*
2661  * Kick a guest cpu out of SIE and wait until SIE is not running.
2662  * If the CPU is not running (e.g. waiting as idle) the function will
2663  * return immediately. */
2664 void exit_sie(struct kvm_vcpu *vcpu)
2665 {
2666         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
2667         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2668                 cpu_relax();
2669 }
2670
2671 /* Kick a guest cpu out of SIE to process a request synchronously */
2672 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2673 {
2674         kvm_make_request(req, vcpu);
2675         kvm_s390_vcpu_request(vcpu);
2676 }
2677
2678 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2679                               unsigned long end)
2680 {
2681         struct kvm *kvm = gmap->private;
2682         struct kvm_vcpu *vcpu;
2683         unsigned long prefix;
2684         int i;
2685
2686         if (gmap_is_shadow(gmap))
2687                 return;
2688         if (start >= 1UL << 31)
2689                 /* We are only interested in prefix pages */
2690                 return;
2691         kvm_for_each_vcpu(i, vcpu, kvm) {
2692                 /* match against both prefix pages */
2693                 prefix = kvm_s390_get_prefix(vcpu);
2694                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2695                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2696                                    start, end);
2697                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2698                 }
2699         }
2700 }
2701
2702 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2703 {
2704         /* kvm common code refers to this, but never calls it */
2705         BUG();
2706         return 0;
2707 }
2708
2709 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2710                                            struct kvm_one_reg *reg)
2711 {
2712         int r = -EINVAL;
2713
2714         switch (reg->id) {
2715         case KVM_REG_S390_TODPR:
2716                 r = put_user(vcpu->arch.sie_block->todpr,
2717                              (u32 __user *)reg->addr);
2718                 break;
2719         case KVM_REG_S390_EPOCHDIFF:
2720                 r = put_user(vcpu->arch.sie_block->epoch,
2721                              (u64 __user *)reg->addr);
2722                 break;
2723         case KVM_REG_S390_CPU_TIMER:
2724                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2725                              (u64 __user *)reg->addr);
2726                 break;
2727         case KVM_REG_S390_CLOCK_COMP:
2728                 r = put_user(vcpu->arch.sie_block->ckc,
2729                              (u64 __user *)reg->addr);
2730                 break;
2731         case KVM_REG_S390_PFTOKEN:
2732                 r = put_user(vcpu->arch.pfault_token,
2733                              (u64 __user *)reg->addr);
2734                 break;
2735         case KVM_REG_S390_PFCOMPARE:
2736                 r = put_user(vcpu->arch.pfault_compare,
2737                              (u64 __user *)reg->addr);
2738                 break;
2739         case KVM_REG_S390_PFSELECT:
2740                 r = put_user(vcpu->arch.pfault_select,
2741                              (u64 __user *)reg->addr);
2742                 break;
2743         case KVM_REG_S390_PP:
2744                 r = put_user(vcpu->arch.sie_block->pp,
2745                              (u64 __user *)reg->addr);
2746                 break;
2747         case KVM_REG_S390_GBEA:
2748                 r = put_user(vcpu->arch.sie_block->gbea,
2749                              (u64 __user *)reg->addr);
2750                 break;
2751         default:
2752                 break;
2753         }
2754
2755         return r;
2756 }
2757
2758 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2759                                            struct kvm_one_reg *reg)
2760 {
2761         int r = -EINVAL;
2762         __u64 val;
2763
2764         switch (reg->id) {
2765         case KVM_REG_S390_TODPR:
2766                 r = get_user(vcpu->arch.sie_block->todpr,
2767                              (u32 __user *)reg->addr);
2768                 break;
2769         case KVM_REG_S390_EPOCHDIFF:
2770                 r = get_user(vcpu->arch.sie_block->epoch,
2771                              (u64 __user *)reg->addr);
2772                 break;
2773         case KVM_REG_S390_CPU_TIMER:
2774                 r = get_user(val, (u64 __user *)reg->addr);
2775                 if (!r)
2776                         kvm_s390_set_cpu_timer(vcpu, val);
2777                 break;
2778         case KVM_REG_S390_CLOCK_COMP:
2779                 r = get_user(vcpu->arch.sie_block->ckc,
2780                              (u64 __user *)reg->addr);
2781                 break;
2782         case KVM_REG_S390_PFTOKEN:
2783                 r = get_user(vcpu->arch.pfault_token,
2784                              (u64 __user *)reg->addr);
2785                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2786                         kvm_clear_async_pf_completion_queue(vcpu);
2787                 break;
2788         case KVM_REG_S390_PFCOMPARE:
2789                 r = get_user(vcpu->arch.pfault_compare,
2790                              (u64 __user *)reg->addr);
2791                 break;
2792         case KVM_REG_S390_PFSELECT:
2793                 r = get_user(vcpu->arch.pfault_select,
2794                              (u64 __user *)reg->addr);
2795                 break;
2796         case KVM_REG_S390_PP:
2797                 r = get_user(vcpu->arch.sie_block->pp,
2798                              (u64 __user *)reg->addr);
2799                 break;
2800         case KVM_REG_S390_GBEA:
2801                 r = get_user(vcpu->arch.sie_block->gbea,
2802                              (u64 __user *)reg->addr);
2803                 break;
2804         default:
2805                 break;
2806         }
2807
2808         return r;
2809 }
2810
2811 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2812 {
2813         kvm_s390_vcpu_initial_reset(vcpu);
2814         return 0;
2815 }
2816
2817 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2818 {
2819         vcpu_load(vcpu);
2820         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2821         vcpu_put(vcpu);
2822         return 0;
2823 }
2824
2825 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2826 {
2827         vcpu_load(vcpu);
2828         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2829         vcpu_put(vcpu);
2830         return 0;
2831 }
2832
2833 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2834                                   struct kvm_sregs *sregs)
2835 {
2836         vcpu_load(vcpu);
2837
2838         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2839         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2840
2841         vcpu_put(vcpu);
2842         return 0;
2843 }
2844
2845 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2846                                   struct kvm_sregs *sregs)
2847 {
2848         vcpu_load(vcpu);
2849
2850         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2851         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2852
2853         vcpu_put(vcpu);
2854         return 0;
2855 }
2856
2857 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2858 {
2859         int ret = 0;
2860
2861         vcpu_load(vcpu);
2862
2863         if (test_fp_ctl(fpu->fpc)) {
2864                 ret = -EINVAL;
2865                 goto out;
2866         }
2867         vcpu->run->s.regs.fpc = fpu->fpc;
2868         if (MACHINE_HAS_VX)
2869                 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
2870                                  (freg_t *) fpu->fprs);
2871         else
2872                 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
2873
2874 out:
2875         vcpu_put(vcpu);
2876         return ret;
2877 }
2878
2879 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2880 {
2881         vcpu_load(vcpu);
2882
2883         /* make sure we have the latest values */
2884         save_fpu_regs();
2885         if (MACHINE_HAS_VX)
2886                 convert_vx_to_fp((freg_t *) fpu->fprs,
2887                                  (__vector128 *) vcpu->run->s.regs.vrs);
2888         else
2889                 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
2890         fpu->fpc = vcpu->run->s.regs.fpc;
2891
2892         vcpu_put(vcpu);
2893         return 0;
2894 }
2895
2896 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2897 {
2898         int rc = 0;
2899
2900         if (!is_vcpu_stopped(vcpu))
2901                 rc = -EBUSY;
2902         else {
2903                 vcpu->run->psw_mask = psw.mask;
2904                 vcpu->run->psw_addr = psw.addr;
2905         }
2906         return rc;
2907 }
2908
2909 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2910                                   struct kvm_translation *tr)
2911 {
2912         return -EINVAL; /* not implemented yet */
2913 }
2914
2915 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2916                               KVM_GUESTDBG_USE_HW_BP | \
2917                               KVM_GUESTDBG_ENABLE)
2918
2919 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2920                                         struct kvm_guest_debug *dbg)
2921 {
2922         int rc = 0;
2923
2924         vcpu_load(vcpu);
2925
2926         vcpu->guest_debug = 0;
2927         kvm_s390_clear_bp_data(vcpu);
2928
2929         if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
2930                 rc = -EINVAL;
2931                 goto out;
2932         }
2933         if (!sclp.has_gpere) {
2934                 rc = -EINVAL;
2935                 goto out;
2936         }
2937
2938         if (dbg->control & KVM_GUESTDBG_ENABLE) {
2939                 vcpu->guest_debug = dbg->control;
2940                 /* enforce guest PER */
2941                 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
2942
2943                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2944                         rc = kvm_s390_import_bp_data(vcpu, dbg);
2945         } else {
2946                 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
2947                 vcpu->arch.guestdbg.last_bp = 0;
2948         }
2949
2950         if (rc) {
2951                 vcpu->guest_debug = 0;
2952                 kvm_s390_clear_bp_data(vcpu);
2953                 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
2954         }
2955
2956 out:
2957         vcpu_put(vcpu);
2958         return rc;
2959 }
2960
2961 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2962                                     struct kvm_mp_state *mp_state)
2963 {
2964         int ret;
2965
2966         vcpu_load(vcpu);
2967
2968         /* CHECK_STOP and LOAD are not supported yet */
2969         ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2970                                       KVM_MP_STATE_OPERATING;
2971
2972         vcpu_put(vcpu);
2973         return ret;
2974 }
2975
2976 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2977                                     struct kvm_mp_state *mp_state)
2978 {
2979         int rc = 0;
2980
2981         vcpu_load(vcpu);
2982
2983         /* user space knows about this interface - let it control the state */
2984         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2985
2986         switch (mp_state->mp_state) {
2987         case KVM_MP_STATE_STOPPED:
2988                 kvm_s390_vcpu_stop(vcpu);
2989                 break;
2990         case KVM_MP_STATE_OPERATING:
2991                 kvm_s390_vcpu_start(vcpu);
2992                 break;
2993         case KVM_MP_STATE_LOAD:
2994         case KVM_MP_STATE_CHECK_STOP:
2995                 /* fall through - CHECK_STOP and LOAD are not supported yet */
2996         default:
2997                 rc = -ENXIO;
2998         }
2999
3000         vcpu_put(vcpu);
3001         return rc;
3002 }
3003
3004 static bool ibs_enabled(struct kvm_vcpu *vcpu)
3005 {
3006         return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
3007 }
3008
3009 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3010 {
3011 retry:
3012         kvm_s390_vcpu_request_handled(vcpu);
3013         if (!kvm_request_pending(vcpu))
3014                 return 0;
3015         /*
3016          * We use MMU_RELOAD just to re-arm the ipte notifier for the
3017          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3018          * This ensures that the ipte instruction for this request has
3019          * already finished. We might race against a second unmapper that
3020          * wants to set the blocking bit. Lets just retry the request loop.
3021          */
3022         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3023                 int rc;
3024                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3025                                           kvm_s390_get_prefix(vcpu),
3026                                           PAGE_SIZE * 2, PROT_WRITE);
3027                 if (rc) {
3028                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3029                         return rc;
3030                 }
3031                 goto retry;
3032         }
3033
3034         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3035                 vcpu->arch.sie_block->ihcpu = 0xffff;
3036                 goto retry;
3037         }
3038
3039         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3040                 if (!ibs_enabled(vcpu)) {
3041                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3042                         kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
3043                 }
3044                 goto retry;
3045         }
3046
3047         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3048                 if (ibs_enabled(vcpu)) {
3049                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3050                         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
3051                 }
3052                 goto retry;
3053         }
3054
3055         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3056                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3057                 goto retry;
3058         }
3059
3060         if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3061                 /*
3062                  * Disable CMM virtualization; we will emulate the ESSA
3063                  * instruction manually, in order to provide additional
3064                  * functionalities needed for live migration.
3065                  */
3066                 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3067                 goto retry;
3068         }
3069
3070         if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3071                 /*
3072                  * Re-enable CMM virtualization if CMMA is available and
3073                  * CMM has been used.
3074                  */
3075                 if ((vcpu->kvm->arch.use_cmma) &&
3076                     (vcpu->kvm->mm->context.uses_cmm))
3077                         vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3078                 goto retry;
3079         }
3080
3081         /* nothing to do, just clear the request */
3082         kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3083
3084         return 0;
3085 }
3086
3087 void kvm_s390_set_tod_clock(struct kvm *kvm,
3088                             const struct kvm_s390_vm_tod_clock *gtod)
3089 {
3090         struct kvm_vcpu *vcpu;
3091         struct kvm_s390_tod_clock_ext htod;
3092         int i;
3093
3094         mutex_lock(&kvm->lock);
3095         preempt_disable();
3096
3097         get_tod_clock_ext((char *)&htod);
3098
3099         kvm->arch.epoch = gtod->tod - htod.tod;
3100         kvm->arch.epdx = 0;
3101         if (test_kvm_facility(kvm, 139)) {
3102                 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3103                 if (kvm->arch.epoch > gtod->tod)
3104                         kvm->arch.epdx -= 1;
3105         }
3106
3107         kvm_s390_vcpu_block_all(kvm);
3108         kvm_for_each_vcpu(i, vcpu, kvm) {
3109                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3110                 vcpu->arch.sie_block->epdx  = kvm->arch.epdx;
3111         }
3112
3113         kvm_s390_vcpu_unblock_all(kvm);
3114         preempt_enable();
3115         mutex_unlock(&kvm->lock);
3116 }
3117
3118 /**
3119  * kvm_arch_fault_in_page - fault-in guest page if necessary
3120  * @vcpu: The corresponding virtual cpu
3121  * @gpa: Guest physical address
3122  * @writable: Whether the page should be writable or not
3123  *
3124  * Make sure that a guest page has been faulted-in on the host.
3125  *
3126  * Return: Zero on success, negative error code otherwise.
3127  */
3128 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3129 {
3130         return gmap_fault(vcpu->arch.gmap, gpa,
3131                           writable ? FAULT_FLAG_WRITE : 0);
3132 }
3133
3134 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3135                                       unsigned long token)
3136 {
3137         struct kvm_s390_interrupt inti;
3138         struct kvm_s390_irq irq;
3139
3140         if (start_token) {
3141                 irq.u.ext.ext_params2 = token;
3142                 irq.type = KVM_S390_INT_PFAULT_INIT;
3143                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3144         } else {
3145                 inti.type = KVM_S390_INT_PFAULT_DONE;
3146                 inti.parm64 = token;
3147                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3148         }
3149 }
3150
3151 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3152                                      struct kvm_async_pf *work)
3153 {
3154         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3155         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3156 }
3157
3158 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3159                                  struct kvm_async_pf *work)
3160 {
3161         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3162         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3163 }
3164
3165 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3166                                struct kvm_async_pf *work)
3167 {
3168         /* s390 will always inject the page directly */
3169 }
3170
3171 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3172 {
3173         /*
3174          * s390 will always inject the page directly,
3175          * but we still want check_async_completion to cleanup
3176          */
3177         return true;
3178 }
3179
3180 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3181 {
3182         hva_t hva;
3183         struct kvm_arch_async_pf arch;
3184         int rc;
3185
3186         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3187                 return 0;
3188         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3189             vcpu->arch.pfault_compare)
3190                 return 0;
3191         if (psw_extint_disabled(vcpu))
3192                 return 0;
3193         if (kvm_s390_vcpu_has_irq(vcpu, 0))
3194                 return 0;
3195         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
3196                 return 0;
3197         if (!vcpu->arch.gmap->pfault_enabled)
3198                 return 0;
3199
3200         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3201         hva += current->thread.gmap_addr & ~PAGE_MASK;
3202         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3203                 return 0;
3204
3205         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3206         return rc;
3207 }
3208
3209 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3210 {
3211         int rc, cpuflags;
3212
3213         /*
3214          * On s390 notifications for arriving pages will be delivered directly
3215          * to the guest but the house keeping for completed pfaults is
3216          * handled outside the worker.
3217          */
3218         kvm_check_async_pf_completion(vcpu);
3219
3220         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3221         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3222
3223         if (need_resched())
3224                 schedule();
3225
3226         if (test_cpu_flag(CIF_MCCK_PENDING))
3227                 s390_handle_mcck();
3228
3229         if (!kvm_is_ucontrol(vcpu->kvm)) {
3230                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3231                 if (rc)
3232                         return rc;
3233         }
3234
3235         rc = kvm_s390_handle_requests(vcpu);
3236         if (rc)
3237                 return rc;
3238
3239         if (guestdbg_enabled(vcpu)) {
3240                 kvm_s390_backup_guest_per_regs(vcpu);
3241                 kvm_s390_patch_guest_per_regs(vcpu);
3242         }
3243
3244         vcpu->arch.sie_block->icptcode = 0;
3245         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3246         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3247         trace_kvm_s390_sie_enter(vcpu, cpuflags);
3248
3249         return 0;
3250 }
3251
3252 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3253 {
3254         struct kvm_s390_pgm_info pgm_info = {
3255                 .code = PGM_ADDRESSING,
3256         };
3257         u8 opcode, ilen;
3258         int rc;
3259
3260         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3261         trace_kvm_s390_sie_fault(vcpu);
3262
3263         /*
3264          * We want to inject an addressing exception, which is defined as a
3265          * suppressing or terminating exception. However, since we came here
3266          * by a DAT access exception, the PSW still points to the faulting
3267          * instruction since DAT exceptions are nullifying. So we've got
3268          * to look up the current opcode to get the length of the instruction
3269          * to be able to forward the PSW.
3270          */
3271         rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3272         ilen = insn_length(opcode);
3273         if (rc < 0) {
3274                 return rc;
3275         } else if (rc) {
3276                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3277                  * Forward by arbitrary ilc, injection will take care of
3278                  * nullification if necessary.
3279                  */
3280                 pgm_info = vcpu->arch.pgm;
3281                 ilen = 4;
3282         }
3283         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3284         kvm_s390_forward_psw(vcpu, ilen);
3285         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3286 }
3287
3288 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3289 {
3290         struct mcck_volatile_info *mcck_info;
3291         struct sie_page *sie_page;
3292
3293         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3294                    vcpu->arch.sie_block->icptcode);
3295         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3296
3297         if (guestdbg_enabled(vcpu))
3298                 kvm_s390_restore_guest_per_regs(vcpu);
3299
3300         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3301         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3302
3303         if (exit_reason == -EINTR) {
3304                 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3305                 sie_page = container_of(vcpu->arch.sie_block,
3306                                         struct sie_page, sie_block);
3307                 mcck_info = &sie_page->mcck_info;
3308                 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3309                 return 0;
3310         }
3311
3312         if (vcpu->arch.sie_block->icptcode > 0) {
3313                 int rc = kvm_handle_sie_intercept(vcpu);
3314
3315                 if (rc != -EOPNOTSUPP)
3316                         return rc;
3317                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3318                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3319                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3320                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3321                 return -EREMOTE;
3322         } else if (exit_reason != -EFAULT) {
3323                 vcpu->stat.exit_null++;
3324                 return 0;
3325         } else if (kvm_is_ucontrol(vcpu->kvm)) {
3326                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3327                 vcpu->run->s390_ucontrol.trans_exc_code =
3328                                                 current->thread.gmap_addr;
3329                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
3330                 return -EREMOTE;
3331         } else if (current->thread.gmap_pfault) {
3332                 trace_kvm_s390_major_guest_pfault(vcpu);
3333                 current->thread.gmap_pfault = 0;
3334                 if (kvm_arch_setup_async_pf(vcpu))
3335                         return 0;
3336                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3337         }
3338         return vcpu_post_run_fault_in_sie(vcpu);
3339 }
3340
3341 static int __vcpu_run(struct kvm_vcpu *vcpu)
3342 {
3343         int rc, exit_reason;
3344
3345         /*
3346          * We try to hold kvm->srcu during most of vcpu_run (except when run-
3347          * ning the guest), so that memslots (and other stuff) are protected
3348          */
3349         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3350
3351         do {
3352                 rc = vcpu_pre_run(vcpu);
3353                 if (rc)
3354                         break;
3355
3356                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3357                 /*
3358                  * As PF_VCPU will be used in fault handler, between
3359                  * guest_enter and guest_exit should be no uaccess.
3360                  */
3361                 local_irq_disable();
3362                 guest_enter_irqoff();
3363                 __disable_cpu_timer_accounting(vcpu);
3364                 local_irq_enable();
3365                 exit_reason = sie64a(vcpu->arch.sie_block,
3366                                      vcpu->run->s.regs.gprs);
3367                 local_irq_disable();
3368                 __enable_cpu_timer_accounting(vcpu);
3369                 guest_exit_irqoff();
3370                 local_irq_enable();
3371                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3372
3373                 rc = vcpu_post_run(vcpu, exit_reason);
3374         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3375
3376         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3377         return rc;
3378 }
3379
3380 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3381 {
3382         struct runtime_instr_cb *riccb;
3383         struct gs_cb *gscb;
3384
3385         riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3386         gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3387         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3388         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3389         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3390                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3391         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3392                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3393                 /* some control register changes require a tlb flush */
3394                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3395         }
3396         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3397                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3398                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3399                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3400                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3401                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3402         }
3403         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3404                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3405                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3406                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3407                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3408                         kvm_clear_async_pf_completion_queue(vcpu);
3409         }
3410         /*
3411          * If userspace sets the riccb (e.g. after migration) to a valid state,
3412          * we should enable RI here instead of doing the lazy enablement.
3413          */
3414         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3415             test_kvm_facility(vcpu->kvm, 64) &&
3416             riccb->v &&
3417             !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3418                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3419                 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3420         }
3421         /*
3422          * If userspace sets the gscb (e.g. after migration) to non-zero,
3423          * we should enable GS here instead of doing the lazy enablement.
3424          */
3425         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3426             test_kvm_facility(vcpu->kvm, 133) &&
3427             gscb->gssm &&
3428             !vcpu->arch.gs_enabled) {
3429                 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3430                 vcpu->arch.sie_block->ecb |= ECB_GS;
3431                 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3432                 vcpu->arch.gs_enabled = 1;
3433         }
3434         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3435             test_kvm_facility(vcpu->kvm, 82)) {
3436                 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3437                 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3438         }
3439         save_access_regs(vcpu->arch.host_acrs);
3440         restore_access_regs(vcpu->run->s.regs.acrs);
3441         /* save host (userspace) fprs/vrs */
3442         save_fpu_regs();
3443         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3444         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3445         if (MACHINE_HAS_VX)
3446                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3447         else
3448                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3449         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3450         if (test_fp_ctl(current->thread.fpu.fpc))
3451                 /* User space provided an invalid FPC, let's clear it */
3452                 current->thread.fpu.fpc = 0;
3453         if (MACHINE_HAS_GS) {
3454                 preempt_disable();
3455                 __ctl_set_bit(2, 4);
3456                 if (current->thread.gs_cb) {
3457                         vcpu->arch.host_gscb = current->thread.gs_cb;
3458                         save_gs_cb(vcpu->arch.host_gscb);
3459                 }
3460                 if (vcpu->arch.gs_enabled) {
3461                         current->thread.gs_cb = (struct gs_cb *)
3462                                                 &vcpu->run->s.regs.gscb;
3463                         restore_gs_cb(current->thread.gs_cb);
3464                 }
3465                 preempt_enable();
3466         }
3467
3468         kvm_run->kvm_dirty_regs = 0;
3469 }
3470
3471 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3472 {
3473         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3474         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3475         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3476         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3477         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3478         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3479         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3480         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3481         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3482         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3483         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3484         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3485         kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
3486         save_access_regs(vcpu->run->s.regs.acrs);
3487         restore_access_regs(vcpu->arch.host_acrs);
3488         /* Save guest register state */
3489         save_fpu_regs();
3490         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3491         /* Restore will be done lazily at return */
3492         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3493         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3494         if (MACHINE_HAS_GS) {
3495                 __ctl_set_bit(2, 4);
3496                 if (vcpu->arch.gs_enabled)
3497                         save_gs_cb(current->thread.gs_cb);
3498                 preempt_disable();
3499                 current->thread.gs_cb = vcpu->arch.host_gscb;
3500                 restore_gs_cb(vcpu->arch.host_gscb);
3501                 preempt_enable();
3502                 if (!vcpu->arch.host_gscb)
3503                         __ctl_clear_bit(2, 4);
3504                 vcpu->arch.host_gscb = NULL;
3505         }
3506
3507 }
3508
3509 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3510 {
3511         int rc;
3512
3513         if (kvm_run->immediate_exit)
3514                 return -EINTR;
3515
3516         vcpu_load(vcpu);
3517
3518         if (guestdbg_exit_pending(vcpu)) {
3519                 kvm_s390_prepare_debug_exit(vcpu);
3520                 rc = 0;
3521                 goto out;
3522         }
3523
3524         kvm_sigset_activate(vcpu);
3525
3526         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
3527                 kvm_s390_vcpu_start(vcpu);
3528         } else if (is_vcpu_stopped(vcpu)) {
3529                 pr_err_ratelimited("can't run stopped vcpu %d\n",
3530                                    vcpu->vcpu_id);
3531                 rc = -EINVAL;
3532                 goto out;
3533         }
3534
3535         sync_regs(vcpu, kvm_run);
3536         enable_cpu_timer_accounting(vcpu);
3537
3538         might_fault();
3539         rc = __vcpu_run(vcpu);
3540
3541         if (signal_pending(current) && !rc) {
3542                 kvm_run->exit_reason = KVM_EXIT_INTR;
3543                 rc = -EINTR;
3544         }
3545
3546         if (guestdbg_exit_pending(vcpu) && !rc)  {
3547                 kvm_s390_prepare_debug_exit(vcpu);
3548                 rc = 0;
3549         }
3550
3551         if (rc == -EREMOTE) {
3552                 /* userspace support is needed, kvm_run has been prepared */
3553                 rc = 0;
3554         }
3555
3556         disable_cpu_timer_accounting(vcpu);
3557         store_regs(vcpu, kvm_run);
3558
3559         kvm_sigset_deactivate(vcpu);
3560
3561         vcpu->stat.exit_userspace++;
3562 out:
3563         vcpu_put(vcpu);
3564         return rc;
3565 }
3566
3567 /*
3568  * store status at address
3569  * we use have two special cases:
3570  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
3571  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
3572  */
3573 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
3574 {
3575         unsigned char archmode = 1;
3576         freg_t fprs[NUM_FPRS];
3577         unsigned int px;
3578         u64 clkcomp, cputm;
3579         int rc;
3580
3581         px = kvm_s390_get_prefix(vcpu);
3582         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
3583                 if (write_guest_abs(vcpu, 163, &archmode, 1))
3584                         return -EFAULT;
3585                 gpa = 0;
3586         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
3587                 if (write_guest_real(vcpu, 163, &archmode, 1))
3588                         return -EFAULT;
3589                 gpa = px;
3590         } else
3591                 gpa -= __LC_FPREGS_SAVE_AREA;
3592
3593         /* manually convert vector registers if necessary */
3594         if (MACHINE_HAS_VX) {
3595                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
3596                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3597                                      fprs, 128);
3598         } else {
3599                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
3600                                      vcpu->run->s.regs.fprs, 128);
3601         }
3602         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
3603                               vcpu->run->s.regs.gprs, 128);
3604         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
3605                               &vcpu->arch.sie_block->gpsw, 16);
3606         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
3607                               &px, 4);
3608         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
3609                               &vcpu->run->s.regs.fpc, 4);
3610         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
3611                               &vcpu->arch.sie_block->todpr, 4);
3612         cputm = kvm_s390_get_cpu_timer(vcpu);
3613         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
3614                               &cputm, 8);
3615         clkcomp = vcpu->arch.sie_block->ckc >> 8;
3616         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
3617                               &clkcomp, 8);
3618         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
3619                               &vcpu->run->s.regs.acrs, 64);
3620         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
3621                               &vcpu->arch.sie_block->gcr, 128);
3622         return rc ? -EFAULT : 0;
3623 }
3624
3625 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
3626 {
3627         /*
3628          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
3629          * switch in the run ioctl. Let's update our copies before we save
3630          * it into the save area
3631          */
3632         save_fpu_regs();
3633         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3634         save_access_regs(vcpu->run->s.regs.acrs);
3635
3636         return kvm_s390_store_status_unloaded(vcpu, addr);
3637 }
3638
3639 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3640 {
3641         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
3642         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
3643 }
3644
3645 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
3646 {
3647         unsigned int i;
3648         struct kvm_vcpu *vcpu;
3649
3650         kvm_for_each_vcpu(i, vcpu, kvm) {
3651                 __disable_ibs_on_vcpu(vcpu);
3652         }
3653 }
3654
3655 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
3656 {
3657         if (!sclp.has_ibs)
3658                 return;
3659         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
3660         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
3661 }
3662
3663 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
3664 {
3665         int i, online_vcpus, started_vcpus = 0;
3666
3667         if (!is_vcpu_stopped(vcpu))
3668                 return;
3669
3670         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
3671         /* Only one cpu at a time may enter/leave the STOPPED state. */
3672         spin_lock(&vcpu->kvm->arch.start_stop_lock);
3673         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3674
3675         for (i = 0; i < online_vcpus; i++) {
3676                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
3677                         started_vcpus++;
3678         }
3679
3680         if (started_vcpus == 0) {
3681                 /* we're the only active VCPU -> speed it up */
3682                 __enable_ibs_on_vcpu(vcpu);
3683         } else if (started_vcpus == 1) {
3684                 /*
3685                  * As we are starting a second VCPU, we have to disable
3686                  * the IBS facility on all VCPUs to remove potentially
3687                  * oustanding ENABLE requests.
3688                  */
3689                 __disable_ibs_on_all_vcpus(vcpu->kvm);
3690         }
3691
3692         kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
3693         /*
3694          * Another VCPU might have used IBS while we were offline.
3695          * Let's play safe and flush the VCPU at startup.
3696          */
3697         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3698         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3699         return;
3700 }
3701
3702 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
3703 {
3704         int i, online_vcpus, started_vcpus = 0;
3705         struct kvm_vcpu *started_vcpu = NULL;
3706
3707         if (is_vcpu_stopped(vcpu))
3708                 return;
3709
3710         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
3711         /* Only one cpu at a time may enter/leave the STOPPED state. */
3712         spin_lock(&vcpu->kvm->arch.start_stop_lock);
3713         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
3714
3715         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
3716         kvm_s390_clear_stop_irq(vcpu);
3717
3718         kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
3719         __disable_ibs_on_vcpu(vcpu);
3720
3721         for (i = 0; i < online_vcpus; i++) {
3722                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
3723                         started_vcpus++;
3724                         started_vcpu = vcpu->kvm->vcpus[i];
3725                 }
3726         }
3727
3728         if (started_vcpus == 1) {
3729                 /*
3730                  * As we only have one VCPU left, we want to enable the
3731                  * IBS facility for that VCPU to speed it up.
3732                  */
3733                 __enable_ibs_on_vcpu(started_vcpu);
3734         }
3735
3736         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
3737         return;
3738 }
3739
3740 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
3741                                      struct kvm_enable_cap *cap)
3742 {
3743         int r;
3744
3745         if (cap->flags)
3746                 return -EINVAL;
3747
3748         switch (cap->cap) {
3749         case KVM_CAP_S390_CSS_SUPPORT:
3750                 if (!vcpu->kvm->arch.css_support) {
3751                         vcpu->kvm->arch.css_support = 1;
3752                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
3753                         trace_kvm_s390_enable_css(vcpu->kvm);
3754                 }
3755                 r = 0;
3756                 break;
3757         default:
3758                 r = -EINVAL;
3759                 break;
3760         }
3761         return r;
3762 }
3763
3764 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
3765                                   struct kvm_s390_mem_op *mop)
3766 {
3767         void __user *uaddr = (void __user *)mop->buf;
3768         void *tmpbuf = NULL;
3769         int r, srcu_idx;
3770         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
3771                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
3772
3773         if (mop->flags & ~supported_flags)
3774                 return -EINVAL;
3775
3776         if (mop->size > MEM_OP_MAX_SIZE)
3777                 return -E2BIG;
3778
3779         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
3780                 tmpbuf = vmalloc(mop->size);
3781                 if (!tmpbuf)
3782                         return -ENOMEM;
3783         }
3784
3785         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3786
3787         switch (mop->op) {
3788         case KVM_S390_MEMOP_LOGICAL_READ:
3789                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3790                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3791                                             mop->size, GACC_FETCH);
3792                         break;
3793                 }
3794                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3795                 if (r == 0) {
3796                         if (copy_to_user(uaddr, tmpbuf, mop->size))
3797                                 r = -EFAULT;
3798                 }
3799                 break;
3800         case KVM_S390_MEMOP_LOGICAL_WRITE:
3801                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3802                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3803                                             mop->size, GACC_STORE);
3804                         break;
3805                 }
3806                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3807                         r = -EFAULT;
3808                         break;
3809                 }
3810                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3811                 break;
3812         default:
3813                 r = -EINVAL;
3814         }
3815
3816         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3817
3818         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3819                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3820
3821         vfree(tmpbuf);
3822         return r;
3823 }
3824
3825 long kvm_arch_vcpu_async_ioctl(struct file *filp,
3826                                unsigned int ioctl, unsigned long arg)
3827 {
3828         struct kvm_vcpu *vcpu = filp->private_data;
3829         void __user *argp = (void __user *)arg;
3830
3831         switch (ioctl) {
3832         case KVM_S390_IRQ: {
3833                 struct kvm_s390_irq s390irq;
3834
3835                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3836                         return -EFAULT;
3837                 return kvm_s390_inject_vcpu(vcpu, &s390irq);
3838         }
3839         case KVM_S390_INTERRUPT: {
3840                 struct kvm_s390_interrupt s390int;
3841                 struct kvm_s390_irq s390irq;
3842
3843                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3844                         return -EFAULT;
3845                 if (s390int_to_s390irq(&s390int, &s390irq))
3846                         return -EINVAL;
3847                 return kvm_s390_inject_vcpu(vcpu, &s390irq);
3848         }
3849         }
3850         return -ENOIOCTLCMD;
3851 }
3852
3853 long kvm_arch_vcpu_ioctl(struct file *filp,
3854                          unsigned int ioctl, unsigned long arg)
3855 {
3856         struct kvm_vcpu *vcpu = filp->private_data;
3857         void __user *argp = (void __user *)arg;
3858         int idx;
3859         long r;
3860
3861         vcpu_load(vcpu);
3862
3863         switch (ioctl) {
3864         case KVM_S390_STORE_STATUS:
3865                 idx = srcu_read_lock(&vcpu->kvm->srcu);
3866                 r = kvm_s390_vcpu_store_status(vcpu, arg);
3867                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3868                 break;
3869         case KVM_S390_SET_INITIAL_PSW: {
3870                 psw_t psw;
3871
3872                 r = -EFAULT;
3873                 if (copy_from_user(&psw, argp, sizeof(psw)))
3874                         break;
3875                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3876                 break;
3877         }
3878         case KVM_S390_INITIAL_RESET:
3879                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3880                 break;
3881         case KVM_SET_ONE_REG:
3882         case KVM_GET_ONE_REG: {
3883                 struct kvm_one_reg reg;
3884                 r = -EFAULT;
3885                 if (copy_from_user(&reg, argp, sizeof(reg)))
3886                         break;
3887                 if (ioctl == KVM_SET_ONE_REG)
3888                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3889                 else
3890                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3891                 break;
3892         }
3893 #ifdef CONFIG_KVM_S390_UCONTROL
3894         case KVM_S390_UCAS_MAP: {
3895                 struct kvm_s390_ucas_mapping ucasmap;
3896
3897                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3898                         r = -EFAULT;
3899                         break;
3900                 }
3901
3902                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3903                         r = -EINVAL;
3904                         break;
3905                 }
3906
3907                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3908                                      ucasmap.vcpu_addr, ucasmap.length);
3909                 break;
3910         }
3911         case KVM_S390_UCAS_UNMAP: {
3912                 struct kvm_s390_ucas_mapping ucasmap;
3913
3914                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3915                         r = -EFAULT;
3916                         break;
3917                 }
3918
3919                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3920                         r = -EINVAL;
3921                         break;
3922                 }
3923
3924                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3925                         ucasmap.length);
3926                 break;
3927         }
3928 #endif
3929         case KVM_S390_VCPU_FAULT: {
3930                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3931                 break;
3932         }
3933         case KVM_ENABLE_CAP:
3934         {
3935                 struct kvm_enable_cap cap;
3936                 r = -EFAULT;
3937                 if (copy_from_user(&cap, argp, sizeof(cap)))
3938                         break;
3939                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3940                 break;
3941         }
3942         case KVM_S390_MEM_OP: {
3943                 struct kvm_s390_mem_op mem_op;
3944
3945                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3946                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3947                 else
3948                         r = -EFAULT;
3949                 break;
3950         }
3951         case KVM_S390_SET_IRQ_STATE: {
3952                 struct kvm_s390_irq_state irq_state;
3953
3954                 r = -EFAULT;
3955                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3956                         break;
3957                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3958                     irq_state.len == 0 ||
3959                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3960                         r = -EINVAL;
3961                         break;
3962                 }
3963                 /* do not use irq_state.flags, it will break old QEMUs */
3964                 r = kvm_s390_set_irq_state(vcpu,
3965                                            (void __user *) irq_state.buf,
3966                                            irq_state.len);
3967                 break;
3968         }
3969         case KVM_S390_GET_IRQ_STATE: {
3970                 struct kvm_s390_irq_state irq_state;
3971
3972                 r = -EFAULT;
3973                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3974                         break;
3975                 if (irq_state.len == 0) {
3976                         r = -EINVAL;
3977                         break;
3978                 }
3979                 /* do not use irq_state.flags, it will break old QEMUs */
3980                 r = kvm_s390_get_irq_state(vcpu,
3981                                            (__u8 __user *)  irq_state.buf,
3982                                            irq_state.len);
3983                 break;
3984         }
3985         default:
3986                 r = -ENOTTY;
3987         }
3988
3989         vcpu_put(vcpu);
3990         return r;
3991 }
3992
3993 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3994 {
3995 #ifdef CONFIG_KVM_S390_UCONTROL
3996         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3997                  && (kvm_is_ucontrol(vcpu->kvm))) {
3998                 vmf->page = virt_to_page(vcpu->arch.sie_block);
3999                 get_page(vmf->page);
4000                 return 0;
4001         }
4002 #endif
4003         return VM_FAULT_SIGBUS;
4004 }
4005
4006 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
4007                             unsigned long npages)
4008 {
4009         return 0;
4010 }
4011
4012 /* Section: memory related */
4013 int kvm_arch_prepare_memory_region(struct kvm *kvm,
4014                                    struct kvm_memory_slot *memslot,
4015                                    const struct kvm_userspace_memory_region *mem,
4016                                    enum kvm_mr_change change)
4017 {
4018         /* A few sanity checks. We can have memory slots which have to be
4019            located/ended at a segment boundary (1MB). The memory in userland is
4020            ok to be fragmented into various different vmas. It is okay to mmap()
4021            and munmap() stuff in this slot after doing this call at any time */
4022
4023         if (mem->userspace_addr & 0xffffful)
4024                 return -EINVAL;
4025
4026         if (mem->memory_size & 0xffffful)
4027                 return -EINVAL;
4028
4029         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4030                 return -EINVAL;
4031
4032         return 0;
4033 }
4034
4035 void kvm_arch_commit_memory_region(struct kvm *kvm,
4036                                 const struct kvm_userspace_memory_region *mem,
4037                                 const struct kvm_memory_slot *old,
4038                                 const struct kvm_memory_slot *new,
4039                                 enum kvm_mr_change change)
4040 {
4041         int rc;
4042
4043         /* If the basics of the memslot do not change, we do not want
4044          * to update the gmap. Every update causes several unnecessary
4045          * segment translation exceptions. This is usually handled just
4046          * fine by the normal fault handler + gmap, but it will also
4047          * cause faults on the prefix page of running guest CPUs.
4048          */
4049         if (old->userspace_addr == mem->userspace_addr &&
4050             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
4051             old->npages * PAGE_SIZE == mem->memory_size)
4052                 return;
4053
4054         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4055                 mem->guest_phys_addr, mem->memory_size);
4056         if (rc)
4057                 pr_warn("failed to commit memory region\n");
4058         return;
4059 }
4060
4061 static inline unsigned long nonhyp_mask(int i)
4062 {
4063         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
4064
4065         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
4066 }
4067
4068 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
4069 {
4070         vcpu->valid_wakeup = false;
4071 }
4072
4073 static int __init kvm_s390_init(void)
4074 {
4075         int i;
4076
4077         if (!sclp.has_sief2) {
4078                 pr_info("SIE not available\n");
4079                 return -ENODEV;
4080         }
4081
4082         for (i = 0; i < 16; i++)
4083                 kvm_s390_fac_base[i] |=
4084                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
4085
4086         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
4087 }
4088
4089 static void __exit kvm_s390_exit(void)
4090 {
4091         kvm_exit();
4092 }
4093
4094 module_init(kvm_s390_init);
4095 module_exit(kvm_s390_exit);
4096
4097 /*
4098  * Enable autoloading of the kvm module.
4099  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
4100  * since x86 takes a different approach.
4101  */
4102 #include <linux/miscdevice.h>
4103 MODULE_ALIAS_MISCDEV(KVM_MINOR);
4104 MODULE_ALIAS("devname:kvm");