projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: refactor vcpu injection function
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
41408c28 28#include <linux/vmalloc.h>
cbb870c8 29#include <asm/asm-offsets.h>
b0c632db
HC		 30#include <asm/lowcore.h>
31#include <asm/pgtable.h>
f5daba1d 32#include <asm/nmi.h>
a0616cde 33#include <asm/switch_to.h>
6d3da241 34#include <asm/isc.h>
1526bf9c 35#include <asm/sclp.h>
8f2abe6a 36#include "kvm-s390.h"
b0c632db
HC		 37#include "gaccess.h"
38
5786fffa
CH		 39#define CREATE_TRACE_POINTS
40#include "trace.h"
ade38c31 41#include "trace-s390.h"
5786fffa 42
41408c28
TH		 43#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
44
b0c632db
HC		 45#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
46
47struct kvm_stats_debugfs_item debugfs_entries[] = {
48 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 49 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 50 { "exit_validity", VCPU_STAT(exit_validity) },
51 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
52 { "exit_external_request", VCPU_STAT(exit_external_request) },
53 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 54 { "exit_instruction", VCPU_STAT(exit_instruction) },
55 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
56 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f7819512 57 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
ce2e4f0b 58 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 59 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 60 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 61 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
62 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 63 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 64 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 65 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
66 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
67 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
68 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
69 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
70 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
71 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 72 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 73 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
74 { "instruction_spx", VCPU_STAT(instruction_spx) },
75 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
76 { "instruction_stap", VCPU_STAT(instruction_stap) },
77 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 78 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 79 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
80 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 81 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 82 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
83 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 84 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 85 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 86 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 87 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 88 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 89 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
90 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 91 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 92 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
93 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
cd7b4b61 94 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
5288fbf0
CB		 95 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
96 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
97 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 98 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
99 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
100 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 101 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 102 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 103 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
b0c632db
HC		 104 { NULL }
105};
106
9d8d5786
MM		 107/* upper facilities limit for kvm */
108unsigned long kvm_s390_fac_list_mask[] = {
a3ed8dae
CB		 109 0xffe6fffbfcfdfc40UL,
110 0x205c800000000000UL,
9d8d5786 111};
b0c632db 112
9d8d5786 113unsigned long kvm_s390_fac_list_mask_size(void)
78c4b59f 114{
9d8d5786
MM		 115 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
116 return ARRAY_SIZE(kvm_s390_fac_list_mask);
78c4b59f
MM		 117}
118
9d8d5786
MM		 119static struct gmap_notifier gmap_notifier;
120
b0c632db 121/* Section: not file related */
13a34e06 122int kvm_arch_hardware_enable(void)
b0c632db
HC		 123{
124 /* every s390 is virtualization enabled ;-) */
10474ae8 125 return 0;
b0c632db
HC		 126}
127
2c70fe44
CB		 128static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
129
b0c632db
HC		 130int kvm_arch_hardware_setup(void)
131{
2c70fe44
CB		 132 gmap_notifier.notifier_call = kvm_gmap_notifier;
133 gmap_register_ipte_notifier(&gmap_notifier);
b0c632db
HC		 134 return 0;
135}
136
137void kvm_arch_hardware_unsetup(void)
138{
2c70fe44 139 gmap_unregister_ipte_notifier(&gmap_notifier);
b0c632db
HC		 140}
141
b0c632db
HC		 142int kvm_arch_init(void *opaque)
143{
84877d93
CH		 144 /* Register floating interrupt controller interface. */
145 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 146}
147
b0c632db
HC		 148/* Section: device related */
149long kvm_arch_dev_ioctl(struct file *filp,
150 unsigned int ioctl, unsigned long arg)
151{
152 if (ioctl == KVM_S390_ENABLE_SIE)
153 return s390_enable_sie();
154 return -EINVAL;
155}
156
784aa3d7 157int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 158{
d7b0b5eb
CO		 159 int r;
160
2bd0ac4e 161 switch (ext) {
d7b0b5eb 162 case KVM_CAP_S390_PSW:
b6cf8788 163 case KVM_CAP_S390_GMAP:
52e16b18 164 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 165#ifdef CONFIG_KVM_S390_UCONTROL
166 case KVM_CAP_S390_UCONTROL:
167#endif
3c038e6b 168 case KVM_CAP_ASYNC_PF:
60b413c9 169 case KVM_CAP_SYNC_REGS:
14eebd91 170 case KVM_CAP_ONE_REG:
d6712df9 171 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 172 case KVM_CAP_S390_CSS_SUPPORT:
ebc32262 173 case KVM_CAP_IRQFD:
10ccaa1e 174 case KVM_CAP_IOEVENTFD:
c05c4186 175 case KVM_CAP_DEVICE_CTRL:
d938dc55 176 case KVM_CAP_ENABLE_CAP_VM:
78599d90 177 case KVM_CAP_S390_IRQCHIP:
f2061656 178 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 179 case KVM_CAP_MP_STATE:
47b43c52 180 case KVM_CAP_S390_INJECT_IRQ:
2444b352 181 case KVM_CAP_S390_USER_SIGP:
e44fc8c9 182 case KVM_CAP_S390_USER_STSI:
30ee2a98 183 case KVM_CAP_S390_SKEYS:
d7b0b5eb
CO		 184 r = 1;
185 break;
41408c28
TH		 186 case KVM_CAP_S390_MEM_OP:
187 r = MEM_OP_MAX_SIZE;
188 break;
e726b1bd
CB		 189 case KVM_CAP_NR_VCPUS:
190 case KVM_CAP_MAX_VCPUS:
191 r = KVM_MAX_VCPUS;
192 break;
e1e2e605
NW		 193 case KVM_CAP_NR_MEMSLOTS:
194 r = KVM_USER_MEM_SLOTS;
195 break;
1526bf9c 196 case KVM_CAP_S390_COW:
abf09bed 197 r = MACHINE_HAS_ESOP;
1526bf9c 198 break;
68c55750
EF		 199 case KVM_CAP_S390_VECTOR_REGISTERS:
200 r = MACHINE_HAS_VX;
201 break;
2bd0ac4e 202 default:
d7b0b5eb 203 r = 0;
2bd0ac4e 204 }
d7b0b5eb 205 return r;
b0c632db
HC		 206}
207
15f36ebd
JH		 208static void kvm_s390_sync_dirty_log(struct kvm *kvm,
209 struct kvm_memory_slot *memslot)
210{
211 gfn_t cur_gfn, last_gfn;
212 unsigned long address;
213 struct gmap *gmap = kvm->arch.gmap;
214
215 down_read(&gmap->mm->mmap_sem);
216 /* Loop over all guest pages */
217 last_gfn = memslot->base_gfn + memslot->npages;
218 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
219 address = gfn_to_hva_memslot(memslot, cur_gfn);
220
221 if (gmap_test_and_clear_dirty(address, gmap))
222 mark_page_dirty(kvm, cur_gfn);
223 }
224 up_read(&gmap->mm->mmap_sem);
225}
226
b0c632db
HC		 227/* Section: vm related */
228/*
229 * Get (and clear) the dirty memory log for a memory slot.
230 */
231int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
232 struct kvm_dirty_log *log)
233{
15f36ebd
JH		 234 int r;
235 unsigned long n;
236 struct kvm_memory_slot *memslot;
237 int is_dirty = 0;
238
239 mutex_lock(&kvm->slots_lock);
240
241 r = -EINVAL;
242 if (log->slot >= KVM_USER_MEM_SLOTS)
243 goto out;
244
245 memslot = id_to_memslot(kvm->memslots, log->slot);
246 r = -ENOENT;
247 if (!memslot->dirty_bitmap)
248 goto out;
249
250 kvm_s390_sync_dirty_log(kvm, memslot);
251 r = kvm_get_dirty_log(kvm, log, &is_dirty);
252 if (r)
253 goto out;
254
255 /* Clear the dirty log */
256 if (is_dirty) {
257 n = kvm_dirty_bitmap_bytes(memslot);
258 memset(memslot->dirty_bitmap, 0, n);
259 }
260 r = 0;
261out:
262 mutex_unlock(&kvm->slots_lock);
263 return r;
b0c632db
HC		 264}
265
d938dc55
CH		 266static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
267{
268 int r;
269
270 if (cap->flags)
271 return -EINVAL;
272
273 switch (cap->cap) {
84223598
CH		 274 case KVM_CAP_S390_IRQCHIP:
275 kvm->arch.use_irqchip = 1;
276 r = 0;
277 break;
2444b352
DH		 278 case KVM_CAP_S390_USER_SIGP:
279 kvm->arch.user_sigp = 1;
280 r = 0;
281 break;
68c55750 282 case KVM_CAP_S390_VECTOR_REGISTERS:
18280d8b
MM		 283 if (MACHINE_HAS_VX) {
284 set_kvm_facility(kvm->arch.model.fac->mask, 129);
285 set_kvm_facility(kvm->arch.model.fac->list, 129);
286 r = 0;
287 } else
288 r = -EINVAL;
68c55750 289 break;
e44fc8c9
ET		 290 case KVM_CAP_S390_USER_STSI:
291 kvm->arch.user_stsi = 1;
292 r = 0;
293 break;
d938dc55
CH		 294 default:
295 r = -EINVAL;
296 break;
297 }
298 return r;
299}
300
8c0a7ce6
DD		 301static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
302{
303 int ret;
304
305 switch (attr->attr) {
306 case KVM_S390_VM_MEM_LIMIT_SIZE:
307 ret = 0;
308 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
309 ret = -EFAULT;
310 break;
311 default:
312 ret = -ENXIO;
313 break;
314 }
315 return ret;
316}
317
318static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 319{
320 int ret;
321 unsigned int idx;
322 switch (attr->attr) {
323 case KVM_S390_VM_MEM_ENABLE_CMMA:
324 ret = -EBUSY;
325 mutex_lock(&kvm->lock);
326 if (atomic_read(&kvm->online_vcpus) == 0) {
327 kvm->arch.use_cmma = 1;
328 ret = 0;
329 }
330 mutex_unlock(&kvm->lock);
331 break;
332 case KVM_S390_VM_MEM_CLR_CMMA:
333 mutex_lock(&kvm->lock);
334 idx = srcu_read_lock(&kvm->srcu);
a13cff31 335 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 336 srcu_read_unlock(&kvm->srcu, idx);
337 mutex_unlock(&kvm->lock);
338 ret = 0;
339 break;
8c0a7ce6
DD		 340 case KVM_S390_VM_MEM_LIMIT_SIZE: {
341 unsigned long new_limit;
342
343 if (kvm_is_ucontrol(kvm))
344 return -EINVAL;
345
346 if (get_user(new_limit, (u64 __user *)attr->addr))
347 return -EFAULT;
348
349 if (new_limit > kvm->arch.gmap->asce_end)
350 return -E2BIG;
351
352 ret = -EBUSY;
353 mutex_lock(&kvm->lock);
354 if (atomic_read(&kvm->online_vcpus) == 0) {
355 /* gmap_alloc will round the limit up */
356 struct gmap *new = gmap_alloc(current->mm, new_limit);
357
358 if (!new) {
359 ret = -ENOMEM;
360 } else {
361 gmap_free(kvm->arch.gmap);
362 new->private = kvm;
363 kvm->arch.gmap = new;
364 ret = 0;
365 }
366 }
367 mutex_unlock(&kvm->lock);
368 break;
369 }
4f718eab
DD		 370 default:
371 ret = -ENXIO;
372 break;
373 }
374 return ret;
375}
376
a374e892
TK		 377static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
378
379static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
380{
381 struct kvm_vcpu *vcpu;
382 int i;
383
9d8d5786 384 if (!test_kvm_facility(kvm, 76))
a374e892
TK		 385 return -EINVAL;
386
387 mutex_lock(&kvm->lock);
388 switch (attr->attr) {
389 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
390 get_random_bytes(
391 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
392 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
393 kvm->arch.crypto.aes_kw = 1;
394 break;
395 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
396 get_random_bytes(
397 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
398 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
399 kvm->arch.crypto.dea_kw = 1;
400 break;
401 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
402 kvm->arch.crypto.aes_kw = 0;
403 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
404 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
405 break;
406 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
407 kvm->arch.crypto.dea_kw = 0;
408 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
409 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
410 break;
411 default:
412 mutex_unlock(&kvm->lock);
413 return -ENXIO;
414 }
415
416 kvm_for_each_vcpu(i, vcpu, kvm) {
417 kvm_s390_vcpu_crypto_setup(vcpu);
418 exit_sie(vcpu);
419 }
420 mutex_unlock(&kvm->lock);
421 return 0;
422}
423
72f25020
JH		 424static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
425{
426 u8 gtod_high;
427
428 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
429 sizeof(gtod_high)))
430 return -EFAULT;
431
432 if (gtod_high != 0)
433 return -EINVAL;
434
435 return 0;
436}
437
438static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
439{
440 struct kvm_vcpu *cur_vcpu;
441 unsigned int vcpu_idx;
442 u64 host_tod, gtod;
443 int r;
444
445 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
446 return -EFAULT;
447
448 r = store_tod_clock(&host_tod);
449 if (r)
450 return r;
451
452 mutex_lock(&kvm->lock);
453 kvm->arch.epoch = gtod - host_tod;
454 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
455 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
456 exit_sie(cur_vcpu);
457 }
458 mutex_unlock(&kvm->lock);
459 return 0;
460}
461
462static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
463{
464 int ret;
465
466 if (attr->flags)
467 return -EINVAL;
468
469 switch (attr->attr) {
470 case KVM_S390_VM_TOD_HIGH:
471 ret = kvm_s390_set_tod_high(kvm, attr);
472 break;
473 case KVM_S390_VM_TOD_LOW:
474 ret = kvm_s390_set_tod_low(kvm, attr);
475 break;
476 default:
477 ret = -ENXIO;
478 break;
479 }
480 return ret;
481}
482
483static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
484{
485 u8 gtod_high = 0;
486
487 if (copy_to_user((void __user *)attr->addr, &gtod_high,
488 sizeof(gtod_high)))
489 return -EFAULT;
490
491 return 0;
492}
493
494static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
495{
496 u64 host_tod, gtod;
497 int r;
498
499 r = store_tod_clock(&host_tod);
500 if (r)
501 return r;
502
503 gtod = host_tod + kvm->arch.epoch;
504 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
505 return -EFAULT;
506
507 return 0;
508}
509
510static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
511{
512 int ret;
513
514 if (attr->flags)
515 return -EINVAL;
516
517 switch (attr->attr) {
518 case KVM_S390_VM_TOD_HIGH:
519 ret = kvm_s390_get_tod_high(kvm, attr);
520 break;
521 case KVM_S390_VM_TOD_LOW:
522 ret = kvm_s390_get_tod_low(kvm, attr);
523 break;
524 default:
525 ret = -ENXIO;
526 break;
527 }
528 return ret;
529}
530
658b6eda
MM		 531static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
532{
533 struct kvm_s390_vm_cpu_processor *proc;
534 int ret = 0;
535
536 mutex_lock(&kvm->lock);
537 if (atomic_read(&kvm->online_vcpus)) {
538 ret = -EBUSY;
539 goto out;
540 }
541 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
542 if (!proc) {
543 ret = -ENOMEM;
544 goto out;
545 }
546 if (!copy_from_user(proc, (void __user *)attr->addr,
547 sizeof(*proc))) {
548 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
549 sizeof(struct cpuid));
550 kvm->arch.model.ibc = proc->ibc;
981467c9 551 memcpy(kvm->arch.model.fac->list, proc->fac_list,
658b6eda
MM		 552 S390_ARCH_FAC_LIST_SIZE_BYTE);
553 } else
554 ret = -EFAULT;
555 kfree(proc);
556out:
557 mutex_unlock(&kvm->lock);
558 return ret;
559}
560
561static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
562{
563 int ret = -ENXIO;
564
565 switch (attr->attr) {
566 case KVM_S390_VM_CPU_PROCESSOR:
567 ret = kvm_s390_set_processor(kvm, attr);
568 break;
569 }
570 return ret;
571}
572
573static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
574{
575 struct kvm_s390_vm_cpu_processor *proc;
576 int ret = 0;
577
578 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
579 if (!proc) {
580 ret = -ENOMEM;
581 goto out;
582 }
583 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
584 proc->ibc = kvm->arch.model.ibc;
981467c9 585 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 586 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
587 ret = -EFAULT;
588 kfree(proc);
589out:
590 return ret;
591}
592
593static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
594{
595 struct kvm_s390_vm_cpu_machine *mach;
596 int ret = 0;
597
598 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
599 if (!mach) {
600 ret = -ENOMEM;
601 goto out;
602 }
603 get_cpu_id((struct cpuid *) &mach->cpuid);
604 mach->ibc = sclp_get_ibc();
981467c9
MM		 605 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
606 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda 607 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
94422ee8 608 S390_ARCH_FAC_LIST_SIZE_BYTE);
658b6eda
MM		 609 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
610 ret = -EFAULT;
611 kfree(mach);
612out:
613 return ret;
614}
615
616static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
617{
618 int ret = -ENXIO;
619
620 switch (attr->attr) {
621 case KVM_S390_VM_CPU_PROCESSOR:
622 ret = kvm_s390_get_processor(kvm, attr);
623 break;
624 case KVM_S390_VM_CPU_MACHINE:
625 ret = kvm_s390_get_machine(kvm, attr);
626 break;
627 }
628 return ret;
629}
630
f2061656
DD		 631static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
632{
633 int ret;
634
635 switch (attr->group) {
4f718eab 636 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 637 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 638 break;
72f25020
JH		 639 case KVM_S390_VM_TOD:
640 ret = kvm_s390_set_tod(kvm, attr);
641 break;
658b6eda
MM		 642 case KVM_S390_VM_CPU_MODEL:
643 ret = kvm_s390_set_cpu_model(kvm, attr);
644 break;
a374e892
TK		 645 case KVM_S390_VM_CRYPTO:
646 ret = kvm_s390_vm_set_crypto(kvm, attr);
647 break;
f2061656
DD		 648 default:
649 ret = -ENXIO;
650 break;
651 }
652
653 return ret;
654}
655
656static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
657{
8c0a7ce6
DD		 658 int ret;
659
660 switch (attr->group) {
661 case KVM_S390_VM_MEM_CTRL:
662 ret = kvm_s390_get_mem_control(kvm, attr);
663 break;
72f25020
JH		 664 case KVM_S390_VM_TOD:
665 ret = kvm_s390_get_tod(kvm, attr);
666 break;
658b6eda
MM		 667 case KVM_S390_VM_CPU_MODEL:
668 ret = kvm_s390_get_cpu_model(kvm, attr);
669 break;
8c0a7ce6
DD		 670 default:
671 ret = -ENXIO;
672 break;
673 }
674
675 return ret;
f2061656
DD		 676}
677
678static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
679{
680 int ret;
681
682 switch (attr->group) {
4f718eab
DD		 683 case KVM_S390_VM_MEM_CTRL:
684 switch (attr->attr) {
685 case KVM_S390_VM_MEM_ENABLE_CMMA:
686 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 687 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 688 ret = 0;
689 break;
690 default:
691 ret = -ENXIO;
692 break;
693 }
694 break;
72f25020
JH		 695 case KVM_S390_VM_TOD:
696 switch (attr->attr) {
697 case KVM_S390_VM_TOD_LOW:
698 case KVM_S390_VM_TOD_HIGH:
699 ret = 0;
700 break;
701 default:
702 ret = -ENXIO;
703 break;
704 }
705 break;
658b6eda
MM		 706 case KVM_S390_VM_CPU_MODEL:
707 switch (attr->attr) {
708 case KVM_S390_VM_CPU_PROCESSOR:
709 case KVM_S390_VM_CPU_MACHINE:
710 ret = 0;
711 break;
712 default:
713 ret = -ENXIO;
714 break;
715 }
716 break;
a374e892
TK		 717 case KVM_S390_VM_CRYPTO:
718 switch (attr->attr) {
719 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
720 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
721 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
722 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
723 ret = 0;
724 break;
725 default:
726 ret = -ENXIO;
727 break;
728 }
729 break;
f2061656
DD		 730 default:
731 ret = -ENXIO;
732 break;
733 }
734
735 return ret;
736}
737
30ee2a98
JH		 738static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
739{
740 uint8_t *keys;
741 uint64_t hva;
742 unsigned long curkey;
743 int i, r = 0;
744
745 if (args->flags != 0)
746 return -EINVAL;
747
748 /* Is this guest using storage keys? */
749 if (!mm_use_skey(current->mm))
750 return KVM_S390_GET_SKEYS_NONE;
751
752 /* Enforce sane limit on memory allocation */
753 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
754 return -EINVAL;
755
756 keys = kmalloc_array(args->count, sizeof(uint8_t),
757 GFP_KERNEL | __GFP_NOWARN);
758 if (!keys)
759 keys = vmalloc(sizeof(uint8_t) * args->count);
760 if (!keys)
761 return -ENOMEM;
762
763 for (i = 0; i < args->count; i++) {
764 hva = gfn_to_hva(kvm, args->start_gfn + i);
765 if (kvm_is_error_hva(hva)) {
766 r = -EFAULT;
767 goto out;
768 }
769
770 curkey = get_guest_storage_key(current->mm, hva);
771 if (IS_ERR_VALUE(curkey)) {
772 r = curkey;
773 goto out;
774 }
775 keys[i] = curkey;
776 }
777
778 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
779 sizeof(uint8_t) * args->count);
780 if (r)
781 r = -EFAULT;
782out:
783 kvfree(keys);
784 return r;
785}
786
787static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
788{
789 uint8_t *keys;
790 uint64_t hva;
791 int i, r = 0;
792
793 if (args->flags != 0)
794 return -EINVAL;
795
796 /* Enforce sane limit on memory allocation */
797 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
798 return -EINVAL;
799
800 keys = kmalloc_array(args->count, sizeof(uint8_t),
801 GFP_KERNEL | __GFP_NOWARN);
802 if (!keys)
803 keys = vmalloc(sizeof(uint8_t) * args->count);
804 if (!keys)
805 return -ENOMEM;
806
807 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
808 sizeof(uint8_t) * args->count);
809 if (r) {
810 r = -EFAULT;
811 goto out;
812 }
813
814 /* Enable storage key handling for the guest */
815 s390_enable_skey();
816
817 for (i = 0; i < args->count; i++) {
818 hva = gfn_to_hva(kvm, args->start_gfn + i);
819 if (kvm_is_error_hva(hva)) {
820 r = -EFAULT;
821 goto out;
822 }
823
824 /* Lowest order bit is reserved */
825 if (keys[i] & 0x01) {
826 r = -EINVAL;
827 goto out;
828 }
829
830 r = set_guest_storage_key(current->mm, hva,
831 (unsigned long)keys[i], 0);
832 if (r)
833 goto out;
834 }
835out:
836 kvfree(keys);
837 return r;
838}
839
b0c632db
HC		 840long kvm_arch_vm_ioctl(struct file *filp,
841 unsigned int ioctl, unsigned long arg)
842{
843 struct kvm *kvm = filp->private_data;
844 void __user *argp = (void __user *)arg;
f2061656 845 struct kvm_device_attr attr;
b0c632db
HC		 846 int r;
847
848 switch (ioctl) {
ba5c1e9b
CO		 849 case KVM_S390_INTERRUPT: {
850 struct kvm_s390_interrupt s390int;
851
852 r = -EFAULT;
853 if (copy_from_user(&s390int, argp, sizeof(s390int)))
854 break;
855 r = kvm_s390_inject_vm(kvm, &s390int);
856 break;
857 }
d938dc55
CH		 858 case KVM_ENABLE_CAP: {
859 struct kvm_enable_cap cap;
860 r = -EFAULT;
861 if (copy_from_user(&cap, argp, sizeof(cap)))
862 break;
863 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
864 break;
865 }
84223598
CH		 866 case KVM_CREATE_IRQCHIP: {
867 struct kvm_irq_routing_entry routing;
868
869 r = -EINVAL;
870 if (kvm->arch.use_irqchip) {
871 /* Set up dummy routing. */
872 memset(&routing, 0, sizeof(routing));
873 kvm_set_irq_routing(kvm, &routing, 0, 0);
874 r = 0;
875 }
876 break;
877 }
f2061656
DD		 878 case KVM_SET_DEVICE_ATTR: {
879 r = -EFAULT;
880 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
881 break;
882 r = kvm_s390_vm_set_attr(kvm, &attr);
883 break;
884 }
885 case KVM_GET_DEVICE_ATTR: {
886 r = -EFAULT;
887 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
888 break;
889 r = kvm_s390_vm_get_attr(kvm, &attr);
890 break;
891 }
892 case KVM_HAS_DEVICE_ATTR: {
893 r = -EFAULT;
894 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
895 break;
896 r = kvm_s390_vm_has_attr(kvm, &attr);
897 break;
898 }
30ee2a98
JH		 899 case KVM_S390_GET_SKEYS: {
900 struct kvm_s390_skeys args;
901
902 r = -EFAULT;
903 if (copy_from_user(&args, argp,
904 sizeof(struct kvm_s390_skeys)))
905 break;
906 r = kvm_s390_get_skeys(kvm, &args);
907 break;
908 }
909 case KVM_S390_SET_SKEYS: {
910 struct kvm_s390_skeys args;
911
912 r = -EFAULT;
913 if (copy_from_user(&args, argp,
914 sizeof(struct kvm_s390_skeys)))
915 break;
916 r = kvm_s390_set_skeys(kvm, &args);
917 break;
918 }
b0c632db 919 default:
367e1319 920 r = -ENOTTY;
b0c632db
HC		 921 }
922
923 return r;
924}
925
45c9b47c
TK		 926static int kvm_s390_query_ap_config(u8 *config)
927{
928 u32 fcn_code = 0x04000000UL;
86044c8c 929 u32 cc = 0;
45c9b47c 930
86044c8c 931 memset(config, 0, 128);
45c9b47c
TK		 932 asm volatile(
933 "lgr 0,%1\n"
934 "lgr 2,%2\n"
935 ".long 0xb2af0000\n" /* PQAP(QCI) */
86044c8c 936 "0: ipm %0\n"
45c9b47c 937 "srl %0,28\n"
86044c8c
CB		 938 "1:\n"
939 EX_TABLE(0b, 1b)
940 : "+r" (cc)
45c9b47c
TK		 941 : "r" (fcn_code), "r" (config)
942 : "cc", "0", "2", "memory"
943 );
944
945 return cc;
946}
947
948static int kvm_s390_apxa_installed(void)
949{
950 u8 config[128];
951 int cc;
952
953 if (test_facility(2) && test_facility(12)) {
954 cc = kvm_s390_query_ap_config(config);
955
956 if (cc)
957 pr_err("PQAP(QCI) failed with cc=%d", cc);
958 else
959 return config[0] & 0x40;
960 }
961
962 return 0;
963}
964
965static void kvm_s390_set_crycb_format(struct kvm *kvm)
966{
967 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
968
969 if (kvm_s390_apxa_installed())
970 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
971 else
972 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
973}
974
9d8d5786
MM		 975static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
976{
977 get_cpu_id(cpu_id);
978 cpu_id->version = 0xff;
979}
980
5102ee87
TK		 981static int kvm_s390_crypto_init(struct kvm *kvm)
982{
9d8d5786 983 if (!test_kvm_facility(kvm, 76))
5102ee87
TK		 984 return 0;
985
986 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
987 GFP_KERNEL | GFP_DMA);
988 if (!kvm->arch.crypto.crycb)
989 return -ENOMEM;
990
45c9b47c 991 kvm_s390_set_crycb_format(kvm);
5102ee87 992
ed6f76b4
TK		 993 /* Enable AES/DEA protected key functions by default */
994 kvm->arch.crypto.aes_kw = 1;
995 kvm->arch.crypto.dea_kw = 1;
996 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
997 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
998 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
999 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
a374e892 1000
5102ee87
TK		 1001 return 0;
1002}
1003
e08b9637 1004int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 1005{
9d8d5786 1006 int i, rc;
b0c632db 1007 char debug_name[16];
f6c137ff 1008 static unsigned long sca_offset;
b0c632db 1009
e08b9637
CO		 1010 rc = -EINVAL;
1011#ifdef CONFIG_KVM_S390_UCONTROL
1012 if (type & ~KVM_VM_S390_UCONTROL)
1013 goto out_err;
1014 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1015 goto out_err;
1016#else
1017 if (type)
1018 goto out_err;
1019#endif
1020
b0c632db
HC		 1021 rc = s390_enable_sie();
1022 if (rc)
d89f5eff 1023 goto out_err;
b0c632db 1024
b290411a
CO		 1025 rc = -ENOMEM;
1026
b0c632db
HC		 1027 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1028 if (!kvm->arch.sca)
d89f5eff 1029 goto out_err;
f6c137ff
CB		 1030 spin_lock(&kvm_lock);
1031 sca_offset = (sca_offset + 16) & 0x7f0;
1032 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1033 spin_unlock(&kvm_lock);
b0c632db
HC		 1034
1035 sprintf(debug_name, "kvm-%u", current->pid);
1036
1037 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
1038 if (!kvm->arch.dbf)
40f5b735 1039 goto out_err;
b0c632db 1040
9d8d5786
MM		 1041 /*
1042 * The architectural maximum amount of facilities is 16 kbit. To store
1043 * this amount, 2 kbyte of memory is required. Thus we need a full
981467c9
MM		 1044 * page to hold the guest facility list (arch.model.fac->list) and the
1045 * facility mask (arch.model.fac->mask). Its address size has to be
9d8d5786
MM		 1046 * 31 bits and word aligned.
1047 */
1048 kvm->arch.model.fac =
981467c9 1049 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
9d8d5786 1050 if (!kvm->arch.model.fac)
40f5b735 1051 goto out_err;
9d8d5786 1052
fb5bf93f 1053 /* Populate the facility mask initially. */
981467c9 1054 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
94422ee8 1055 S390_ARCH_FAC_LIST_SIZE_BYTE);
9d8d5786
MM		 1056 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1057 if (i < kvm_s390_fac_list_mask_size())
981467c9 1058 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
9d8d5786 1059 else
981467c9 1060 kvm->arch.model.fac->mask[i] = 0UL;
9d8d5786
MM		 1061 }
1062
981467c9
MM		 1063 /* Populate the facility list initially. */
1064 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1065 S390_ARCH_FAC_LIST_SIZE_BYTE);
1066
9d8d5786 1067 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
658b6eda 1068 kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
9d8d5786 1069
5102ee87 1070 if (kvm_s390_crypto_init(kvm) < 0)
40f5b735 1071 goto out_err;
5102ee87 1072
ba5c1e9b 1073 spin_lock_init(&kvm->arch.float_int.lock);
6d3da241
JF		 1074 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1075 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
8a242234 1076 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 1077 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 1078
b0c632db
HC		 1079 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1080 VM_EVENT(kvm, 3, "%s", "vm created");
1081
e08b9637
CO		 1082 if (type & KVM_VM_S390_UCONTROL) {
1083 kvm->arch.gmap = NULL;
1084 } else {
0349985a 1085 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
e08b9637 1086 if (!kvm->arch.gmap)
40f5b735 1087 goto out_err;
2c70fe44 1088 kvm->arch.gmap->private = kvm;
24eb3a82 1089 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 1090 }
fa6b7fe9
CH		 1091
1092 kvm->arch.css_support = 0;
84223598 1093 kvm->arch.use_irqchip = 0;
72f25020 1094 kvm->arch.epoch = 0;
fa6b7fe9 1095
8ad35755
DH		 1096 spin_lock_init(&kvm->arch.start_stop_lock);
1097
d89f5eff 1098 return 0;
40f5b735 1099out_err:
5102ee87 1100 kfree(kvm->arch.crypto.crycb);
9d8d5786 1101 free_page((unsigned long)kvm->arch.model.fac);
598841ca 1102 debug_unregister(kvm->arch.dbf);
b0c632db 1103 free_page((unsigned long)(kvm->arch.sca));
d89f5eff 1104 return rc;
b0c632db
HC		 1105}
1106
d329c035
CB		 1107void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1108{
1109 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 1110 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 1111 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 1112 kvm_clear_async_pf_completion_queue(vcpu);
58f9460b
CO		 1113 if (!kvm_is_ucontrol(vcpu->kvm)) {
1114 clear_bit(63 - vcpu->vcpu_id,
1115 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1116 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1117 (__u64) vcpu->arch.sie_block)
1118 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1119 }
abf4a71e 1120 smp_mb();
27e0393f
CO		 1121
1122 if (kvm_is_ucontrol(vcpu->kvm))
1123 gmap_free(vcpu->arch.gmap);
1124
b31605c1
DD		 1125 if (kvm_s390_cmma_enabled(vcpu->kvm))
1126 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 1127 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 1128
6692cef3 1129 kvm_vcpu_uninit(vcpu);
b110feaf 1130 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 1131}
1132
1133static void kvm_free_vcpus(struct kvm *kvm)
1134{
1135 unsigned int i;
988a2cae 1136 struct kvm_vcpu *vcpu;
d329c035 1137
988a2cae
GN		 1138 kvm_for_each_vcpu(i, vcpu, kvm)
1139 kvm_arch_vcpu_destroy(vcpu);
1140
1141 mutex_lock(&kvm->lock);
1142 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1143 kvm->vcpus[i] = NULL;
1144
1145 atomic_set(&kvm->online_vcpus, 0);
1146 mutex_unlock(&kvm->lock);
d329c035
CB		 1147}
1148
b0c632db
HC		 1149void kvm_arch_destroy_vm(struct kvm *kvm)
1150{
d329c035 1151 kvm_free_vcpus(kvm);
9d8d5786 1152 free_page((unsigned long)kvm->arch.model.fac);
b0c632db 1153 free_page((unsigned long)(kvm->arch.sca));
d329c035 1154 debug_unregister(kvm->arch.dbf);
5102ee87 1155 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 1156 if (!kvm_is_ucontrol(kvm))
1157 gmap_free(kvm->arch.gmap);
841b91c5 1158 kvm_s390_destroy_adapters(kvm);
67335e63 1159 kvm_s390_clear_float_irqs(kvm);
b0c632db
HC		 1160}
1161
1162/* Section: vcpu related */
dafd032a
DD		 1163static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1164{
1165 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1166 if (!vcpu->arch.gmap)
1167 return -ENOMEM;
1168 vcpu->arch.gmap->private = vcpu->kvm;
1169
1170 return 0;
1171}
1172
b0c632db
HC		 1173int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1174{
3c038e6b
DD		 1175 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1176 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 1177 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1178 KVM_SYNC_GPRS |
9eed0735 1179 KVM_SYNC_ACRS |
b028ee3e
DH		 1180 KVM_SYNC_CRS |
1181 KVM_SYNC_ARCH0 |
1182 KVM_SYNC_PFAULT;
68c55750
EF		 1183 if (test_kvm_facility(vcpu->kvm, 129))
1184 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
dafd032a
DD		 1185
1186 if (kvm_is_ucontrol(vcpu->kvm))
1187 return __kvm_ucontrol_vcpu_init(vcpu);
1188
b0c632db
HC		 1189 return 0;
1190}
1191
b0c632db
HC		 1192void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1193{
4725c860 1194 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
18280d8b 1195 if (test_kvm_facility(vcpu->kvm, 129))
68c55750
EF		 1196 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1197 else
1198 save_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db 1199 save_access_regs(vcpu->arch.host_acrs);
18280d8b 1200 if (test_kvm_facility(vcpu->kvm, 129)) {
68c55750
EF		 1201 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1202 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1203 } else {
1204 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1205 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1206 }
59674c1a 1207 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 1208 gmap_enable(vcpu->arch.gmap);
9e6dabef 1209 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 1210}
1211
1212void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1213{
9e6dabef 1214 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 1215 gmap_disable(vcpu->arch.gmap);
18280d8b 1216 if (test_kvm_facility(vcpu->kvm, 129)) {
68c55750
EF		 1217 save_fp_ctl(&vcpu->run->s.regs.fpc);
1218 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1219 } else {
1220 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1221 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1222 }
59674c1a 1223 save_access_regs(vcpu->run->s.regs.acrs);
4725c860 1224 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
18280d8b 1225 if (test_kvm_facility(vcpu->kvm, 129))
68c55750
EF		 1226 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1227 else
1228 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db
HC		 1229 restore_access_regs(vcpu->arch.host_acrs);
1230}
1231
1232static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1233{
1234 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1235 vcpu->arch.sie_block->gpsw.mask = 0UL;
1236 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 1237 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 1238 vcpu->arch.sie_block->cputm = 0UL;
1239 vcpu->arch.sie_block->ckc = 0UL;
1240 vcpu->arch.sie_block->todpr = 0;
1241 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1242 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1243 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1244 vcpu->arch.guest_fpregs.fpc = 0;
1245 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1246 vcpu->arch.sie_block->gbea = 1;
672550fb 1247 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 1248 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1249 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 1250 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1251 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 1252 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 1253}
1254
31928aa5 1255void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 1256{
72f25020
JH		 1257 mutex_lock(&vcpu->kvm->lock);
1258 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1259 mutex_unlock(&vcpu->kvm->lock);
dafd032a
DD		 1260 if (!kvm_is_ucontrol(vcpu->kvm))
1261 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
42897d86
MT		 1262}
1263
5102ee87
TK		 1264static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1265{
9d8d5786 1266 if (!test_kvm_facility(vcpu->kvm, 76))
5102ee87
TK		 1267 return;
1268
a374e892
TK		 1269 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1270
1271 if (vcpu->kvm->arch.crypto.aes_kw)
1272 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1273 if (vcpu->kvm->arch.crypto.dea_kw)
1274 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1275
5102ee87
TK		 1276 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1277}
1278
b31605c1
DD		 1279void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1280{
1281 free_page(vcpu->arch.sie_block->cbrlo);
1282 vcpu->arch.sie_block->cbrlo = 0;
1283}
1284
1285int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1286{
1287 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1288 if (!vcpu->arch.sie_block->cbrlo)
1289 return -ENOMEM;
1290
1291 vcpu->arch.sie_block->ecb2 |= 0x80;
1292 vcpu->arch.sie_block->ecb2 &= ~0x08;
1293 return 0;
1294}
1295
91520f1a
MM		 1296static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1297{
1298 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1299
1300 vcpu->arch.cpu_id = model->cpu_id;
1301 vcpu->arch.sie_block->ibc = model->ibc;
1302 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1303}
1304
b0c632db
HC		 1305int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1306{
b31605c1 1307 int rc = 0;
b31288fa 1308
9e6dabef
CH		 1309 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1310 CPUSTAT_SM |
69d0d3a3
CB		 1311 CPUSTAT_STOPPED |
1312 CPUSTAT_GED);
91520f1a
MM		 1313 kvm_s390_vcpu_setup_model(vcpu);
1314
fc34531d 1315 vcpu->arch.sie_block->ecb = 6;
9d8d5786 1316 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
7feb6bb8
MM		 1317 vcpu->arch.sie_block->ecb |= 0x10;
1318
69d0d3a3 1319 vcpu->arch.sie_block->ecb2 = 8;
ea5f4969 1320 vcpu->arch.sie_block->eca = 0xC1002000U;
217a4406
HC		 1321 if (sclp_has_siif())
1322 vcpu->arch.sie_block->eca |= 1;
ea5f4969
DH		 1323 if (sclp_has_sigpif())
1324 vcpu->arch.sie_block->eca |= 0x10000000U;
18280d8b 1325 if (test_kvm_facility(vcpu->kvm, 129)) {
13211ea7
EF		 1326 vcpu->arch.sie_block->eca |= 0x00020000;
1327 vcpu->arch.sie_block->ecd |= 0x20000000;
1328 }
492d8642 1329 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
5a5e6536 1330
b31605c1
DD		 1331 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1332 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1333 if (rc)
1334 return rc;
b31288fa 1335 }
0ac96caf 1336 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 1337 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
9d8d5786 1338
5102ee87
TK		 1339 kvm_s390_vcpu_crypto_setup(vcpu);
1340
b31605c1 1341 return rc;
b0c632db
HC		 1342}
1343
1344struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1345 unsigned int id)
1346{
4d47555a 1347 struct kvm_vcpu *vcpu;
7feb6bb8 1348 struct sie_page *sie_page;
4d47555a
CO		 1349 int rc = -EINVAL;
1350
1351 if (id >= KVM_MAX_VCPUS)
1352 goto out;
1353
1354 rc = -ENOMEM;
b0c632db 1355
b110feaf 1356 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 1357 if (!vcpu)
4d47555a 1358 goto out;
b0c632db 1359
7feb6bb8
MM		 1360 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1361 if (!sie_page)
b0c632db
HC		 1362 goto out_free_cpu;
1363
7feb6bb8
MM		 1364 vcpu->arch.sie_block = &sie_page->sie_block;
1365 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
68c55750 1366 vcpu->arch.host_vregs = &sie_page->vregs;
7feb6bb8 1367
b0c632db 1368 vcpu->arch.sie_block->icpua = id;
58f9460b
CO		 1369 if (!kvm_is_ucontrol(kvm)) {
1370 if (!kvm->arch.sca) {
1371 WARN_ON_ONCE(1);
1372 goto out_free_cpu;
1373 }
1374 if (!kvm->arch.sca->cpu[id].sda)
1375 kvm->arch.sca->cpu[id].sda =
1376 (__u64) vcpu->arch.sie_block;
1377 vcpu->arch.sie_block->scaoh =
1378 (__u32)(((__u64)kvm->arch.sca) >> 32);
1379 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1380 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1381 }
b0c632db 1382
ba5c1e9b 1383 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 1384 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 1385 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1386 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 1387
b0c632db
HC		 1388 rc = kvm_vcpu_init(vcpu, kvm, id);
1389 if (rc)
7b06bf2f 1390 goto out_free_sie_block;
b0c632db
HC		 1391 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1392 vcpu->arch.sie_block);
ade38c31 1393 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1394
b0c632db 1395 return vcpu;
7b06bf2f
WY		 1396out_free_sie_block:
1397 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1398out_free_cpu:
b110feaf 1399 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1400out:
b0c632db
HC		 1401 return ERR_PTR(rc);
1402}
1403
b0c632db
HC		 1404int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1405{
9a022067 1406 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1407}
1408
49b99e1e
CB		 1409void s390_vcpu_block(struct kvm_vcpu *vcpu)
1410{
1411 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1412}
1413
1414void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1415{
1416 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1417}
1418
1419/*
1420 * Kick a guest cpu out of SIE and wait until SIE is not running.
1421 * If the CPU is not running (e.g. waiting as idle) the function will
1422 * return immediately. */
1423void exit_sie(struct kvm_vcpu *vcpu)
1424{
1425 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1426 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1427 cpu_relax();
1428}
1429
1430/* Kick a guest cpu out of SIE and prevent SIE-reentry */
1431void exit_sie_sync(struct kvm_vcpu *vcpu)
1432{
1433 s390_vcpu_block(vcpu);
1434 exit_sie(vcpu);
1435}
1436
2c70fe44
CB		 1437static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1438{
1439 int i;
1440 struct kvm *kvm = gmap->private;
1441 struct kvm_vcpu *vcpu;
1442
1443 kvm_for_each_vcpu(i, vcpu, kvm) {
1444 /* match against both prefix pages */
fda902cb 1445 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44
CB		 1446 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1447 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1448 exit_sie_sync(vcpu);
1449 }
1450 }
1451}
1452
b6d33834
CD		 1453int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1454{
1455 /* kvm common code refers to this, but never calls it */
1456 BUG();
1457 return 0;
1458}
1459
14eebd91
CO		 1460static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1461 struct kvm_one_reg *reg)
1462{
1463 int r = -EINVAL;
1464
1465 switch (reg->id) {
29b7c71b
CO		 1466 case KVM_REG_S390_TODPR:
1467 r = put_user(vcpu->arch.sie_block->todpr,
1468 (u32 __user *)reg->addr);
1469 break;
1470 case KVM_REG_S390_EPOCHDIFF:
1471 r = put_user(vcpu->arch.sie_block->epoch,
1472 (u64 __user *)reg->addr);
1473 break;
46a6dd1c
J		 1474 case KVM_REG_S390_CPU_TIMER:
1475 r = put_user(vcpu->arch.sie_block->cputm,
1476 (u64 __user *)reg->addr);
1477 break;
1478 case KVM_REG_S390_CLOCK_COMP:
1479 r = put_user(vcpu->arch.sie_block->ckc,
1480 (u64 __user *)reg->addr);
1481 break;
536336c2
DD		 1482 case KVM_REG_S390_PFTOKEN:
1483 r = put_user(vcpu->arch.pfault_token,
1484 (u64 __user *)reg->addr);
1485 break;
1486 case KVM_REG_S390_PFCOMPARE:
1487 r = put_user(vcpu->arch.pfault_compare,
1488 (u64 __user *)reg->addr);
1489 break;
1490 case KVM_REG_S390_PFSELECT:
1491 r = put_user(vcpu->arch.pfault_select,
1492 (u64 __user *)reg->addr);
1493 break;
672550fb
CB		 1494 case KVM_REG_S390_PP:
1495 r = put_user(vcpu->arch.sie_block->pp,
1496 (u64 __user *)reg->addr);
1497 break;
afa45ff5
CB		 1498 case KVM_REG_S390_GBEA:
1499 r = put_user(vcpu->arch.sie_block->gbea,
1500 (u64 __user *)reg->addr);
1501 break;
14eebd91
CO		 1502 default:
1503 break;
1504 }
1505
1506 return r;
1507}
1508
1509static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1510 struct kvm_one_reg *reg)
1511{
1512 int r = -EINVAL;
1513
1514 switch (reg->id) {
29b7c71b
CO		 1515 case KVM_REG_S390_TODPR:
1516 r = get_user(vcpu->arch.sie_block->todpr,
1517 (u32 __user *)reg->addr);
1518 break;
1519 case KVM_REG_S390_EPOCHDIFF:
1520 r = get_user(vcpu->arch.sie_block->epoch,
1521 (u64 __user *)reg->addr);
1522 break;
46a6dd1c
J		 1523 case KVM_REG_S390_CPU_TIMER:
1524 r = get_user(vcpu->arch.sie_block->cputm,
1525 (u64 __user *)reg->addr);
1526 break;
1527 case KVM_REG_S390_CLOCK_COMP:
1528 r = get_user(vcpu->arch.sie_block->ckc,
1529 (u64 __user *)reg->addr);
1530 break;
536336c2
DD		 1531 case KVM_REG_S390_PFTOKEN:
1532 r = get_user(vcpu->arch.pfault_token,
1533 (u64 __user *)reg->addr);
9fbd8082
DH		 1534 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1535 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 1536 break;
1537 case KVM_REG_S390_PFCOMPARE:
1538 r = get_user(vcpu->arch.pfault_compare,
1539 (u64 __user *)reg->addr);
1540 break;
1541 case KVM_REG_S390_PFSELECT:
1542 r = get_user(vcpu->arch.pfault_select,
1543 (u64 __user *)reg->addr);
1544 break;
672550fb
CB		 1545 case KVM_REG_S390_PP:
1546 r = get_user(vcpu->arch.sie_block->pp,
1547 (u64 __user *)reg->addr);
1548 break;
afa45ff5
CB		 1549 case KVM_REG_S390_GBEA:
1550 r = get_user(vcpu->arch.sie_block->gbea,
1551 (u64 __user *)reg->addr);
1552 break;
14eebd91
CO		 1553 default:
1554 break;
1555 }
1556
1557 return r;
1558}
b6d33834 1559
b0c632db
HC		 1560static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1561{
b0c632db 1562 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 1563 return 0;
1564}
1565
1566int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1567{
5a32c1af 1568 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 1569 return 0;
1570}
1571
1572int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1573{
5a32c1af 1574 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 1575 return 0;
1576}
1577
1578int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1579 struct kvm_sregs *sregs)
1580{
59674c1a 1581 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 1582 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 1583 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1584 return 0;
1585}
1586
1587int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1588 struct kvm_sregs *sregs)
1589{
59674c1a 1590 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 1591 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 1592 return 0;
1593}
1594
1595int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1596{
4725c860
MS		 1597 if (test_fp_ctl(fpu->fpc))
1598 return -EINVAL;
b0c632db 1599 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860
MS		 1600 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1601 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1602 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
b0c632db
HC		 1603 return 0;
1604}
1605
1606int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1607{
b0c632db
HC		 1608 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1609 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 1610 return 0;
1611}
1612
1613static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1614{
1615 int rc = 0;
1616
7a42fdc2 1617 if (!is_vcpu_stopped(vcpu))
b0c632db 1618 rc = -EBUSY;
d7b0b5eb
CO		 1619 else {
1620 vcpu->run->psw_mask = psw.mask;
1621 vcpu->run->psw_addr = psw.addr;
1622 }
b0c632db
HC		 1623 return rc;
1624}
1625
1626int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1627 struct kvm_translation *tr)
1628{
1629 return -EINVAL; /* not implemented yet */
1630}
1631
27291e21
DH		 1632#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1633 KVM_GUESTDBG_USE_HW_BP | \
1634 KVM_GUESTDBG_ENABLE)
1635
d0bfb940
JK		 1636int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1637 struct kvm_guest_debug *dbg)
b0c632db 1638{
27291e21
DH		 1639 int rc = 0;
1640
1641 vcpu->guest_debug = 0;
1642 kvm_s390_clear_bp_data(vcpu);
1643
2de3bfc2 1644 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 1645 return -EINVAL;
1646
1647 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1648 vcpu->guest_debug = dbg->control;
1649 /* enforce guest PER */
1650 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1651
1652 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1653 rc = kvm_s390_import_bp_data(vcpu, dbg);
1654 } else {
1655 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1656 vcpu->arch.guestdbg.last_bp = 0;
1657 }
1658
1659 if (rc) {
1660 vcpu->guest_debug = 0;
1661 kvm_s390_clear_bp_data(vcpu);
1662 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1663 }
1664
1665 return rc;
b0c632db
HC		 1666}
1667
62d9f0db
MT		 1668int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1669 struct kvm_mp_state *mp_state)
1670{
6352e4d2
DH		 1671 /* CHECK_STOP and LOAD are not supported yet */
1672 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1673 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 1674}
1675
1676int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1677 struct kvm_mp_state *mp_state)
1678{
6352e4d2
DH		 1679 int rc = 0;
1680
1681 /* user space knows about this interface - let it control the state */
1682 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1683
1684 switch (mp_state->mp_state) {
1685 case KVM_MP_STATE_STOPPED:
1686 kvm_s390_vcpu_stop(vcpu);
1687 break;
1688 case KVM_MP_STATE_OPERATING:
1689 kvm_s390_vcpu_start(vcpu);
1690 break;
1691 case KVM_MP_STATE_LOAD:
1692 case KVM_MP_STATE_CHECK_STOP:
1693 /* fall through - CHECK_STOP and LOAD are not supported yet */
1694 default:
1695 rc = -ENXIO;
1696 }
1697
1698 return rc;
62d9f0db
MT		 1699}
1700
b31605c1
DD		 1701bool kvm_s390_cmma_enabled(struct kvm *kvm)
1702{
1703 if (!MACHINE_IS_LPAR)
1704 return false;
1705 /* only enable for z10 and later */
1706 if (!MACHINE_HAS_EDAT1)
1707 return false;
1708 if (!kvm->arch.use_cmma)
1709 return false;
1710 return true;
1711}
1712
8ad35755
DH		 1713static bool ibs_enabled(struct kvm_vcpu *vcpu)
1714{
1715 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1716}
1717
2c70fe44
CB		 1718static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1719{
8ad35755
DH		 1720retry:
1721 s390_vcpu_unblock(vcpu);
2c70fe44
CB		 1722 /*
1723 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1724 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1725 * This ensures that the ipte instruction for this request has
1726 * already finished. We might race against a second unmapper that
1727 * wants to set the blocking bit. Lets just retry the request loop.
1728 */
8ad35755 1729 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 1730 int rc;
1731 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 1732 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 1733 PAGE_SIZE * 2);
1734 if (rc)
1735 return rc;
8ad35755 1736 goto retry;
2c70fe44 1737 }
8ad35755 1738
d3d692c8
DH		 1739 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1740 vcpu->arch.sie_block->ihcpu = 0xffff;
1741 goto retry;
1742 }
1743
8ad35755
DH		 1744 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1745 if (!ibs_enabled(vcpu)) {
1746 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1747 atomic_set_mask(CPUSTAT_IBS,
1748 &vcpu->arch.sie_block->cpuflags);
1749 }
1750 goto retry;
2c70fe44 1751 }
8ad35755
DH		 1752
1753 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1754 if (ibs_enabled(vcpu)) {
1755 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1756 atomic_clear_mask(CPUSTAT_IBS,
1757 &vcpu->arch.sie_block->cpuflags);
1758 }
1759 goto retry;
1760 }
1761
0759d068
DH		 1762 /* nothing to do, just clear the request */
1763 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1764
2c70fe44
CB		 1765 return 0;
1766}
1767
fa576c58
TH		 1768/**
1769 * kvm_arch_fault_in_page - fault-in guest page if necessary
1770 * @vcpu: The corresponding virtual cpu
1771 * @gpa: Guest physical address
1772 * @writable: Whether the page should be writable or not
1773 *
1774 * Make sure that a guest page has been faulted-in on the host.
1775 *
1776 * Return: Zero on success, negative error code otherwise.
1777 */
1778long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 1779{
527e30b4
MS		 1780 return gmap_fault(vcpu->arch.gmap, gpa,
1781 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 1782}
1783
3c038e6b
DD		 1784static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1785 unsigned long token)
1786{
1787 struct kvm_s390_interrupt inti;
383d0b05 1788 struct kvm_s390_irq irq;
3c038e6b
DD		 1789
1790 if (start_token) {
383d0b05
JF		 1791 irq.u.ext.ext_params2 = token;
1792 irq.type = KVM_S390_INT_PFAULT_INIT;
1793 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 1794 } else {
1795 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 1796 inti.parm64 = token;
3c038e6b
DD		 1797 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1798 }
1799}
1800
1801void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1802 struct kvm_async_pf *work)
1803{
1804 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1805 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1806}
1807
1808void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1809 struct kvm_async_pf *work)
1810{
1811 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1812 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1813}
1814
1815void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1816 struct kvm_async_pf *work)
1817{
1818 /* s390 will always inject the page directly */
1819}
1820
1821bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1822{
1823 /*
1824 * s390 will always inject the page directly,
1825 * but we still want check_async_completion to cleanup
1826 */
1827 return true;
1828}
1829
1830static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1831{
1832 hva_t hva;
1833 struct kvm_arch_async_pf arch;
1834 int rc;
1835
1836 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1837 return 0;
1838 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1839 vcpu->arch.pfault_compare)
1840 return 0;
1841 if (psw_extint_disabled(vcpu))
1842 return 0;
9a022067 1843 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 1844 return 0;
1845 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1846 return 0;
1847 if (!vcpu->arch.gmap->pfault_enabled)
1848 return 0;
1849
81480cc1
HC		 1850 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1851 hva += current->thread.gmap_addr & ~PAGE_MASK;
1852 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 1853 return 0;
1854
1855 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1856 return rc;
1857}
1858
3fb4c40f 1859static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 1860{
3fb4c40f 1861 int rc, cpuflags;
e168bf8d 1862
3c038e6b
DD		 1863 /*
1864 * On s390 notifications for arriving pages will be delivered directly
1865 * to the guest but the house keeping for completed pfaults is
1866 * handled outside the worker.
1867 */
1868 kvm_check_async_pf_completion(vcpu);
1869
5a32c1af 1870 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
b0c632db
HC		 1871
1872 if (need_resched())
1873 schedule();
1874
d3a73acb 1875 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 1876 s390_handle_mcck();
1877
79395031
JF		 1878 if (!kvm_is_ucontrol(vcpu->kvm)) {
1879 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1880 if (rc)
1881 return rc;
1882 }
0ff31867 1883
2c70fe44
CB		 1884 rc = kvm_s390_handle_requests(vcpu);
1885 if (rc)
1886 return rc;
1887
27291e21
DH		 1888 if (guestdbg_enabled(vcpu)) {
1889 kvm_s390_backup_guest_per_regs(vcpu);
1890 kvm_s390_patch_guest_per_regs(vcpu);
1891 }
1892
b0c632db 1893 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 1894 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1895 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1896 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 1897
3fb4c40f
TH		 1898 return 0;
1899}
1900
492d8642
TH		 1901static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1902{
1903 psw_t *psw = &vcpu->arch.sie_block->gpsw;
1904 u8 opcode;
1905 int rc;
1906
1907 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1908 trace_kvm_s390_sie_fault(vcpu);
1909
1910 /*
1911 * We want to inject an addressing exception, which is defined as a
1912 * suppressing or terminating exception. However, since we came here
1913 * by a DAT access exception, the PSW still points to the faulting
1914 * instruction since DAT exceptions are nullifying. So we've got
1915 * to look up the current opcode to get the length of the instruction
1916 * to be able to forward the PSW.
1917 */
8ae04b8f 1918 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
492d8642
TH		 1919 if (rc)
1920 return kvm_s390_inject_prog_cond(vcpu, rc);
1921 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1922
1923 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1924}
1925
3fb4c40f
TH		 1926static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1927{
24eb3a82 1928 int rc = -1;
2b29a9fd
DD		 1929
1930 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1931 vcpu->arch.sie_block->icptcode);
1932 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1933
27291e21
DH		 1934 if (guestdbg_enabled(vcpu))
1935 kvm_s390_restore_guest_per_regs(vcpu);
1936
3fb4c40f 1937 if (exit_reason >= 0) {
7c470539 1938 rc = 0;
210b1607
TH		 1939 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1940 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1941 vcpu->run->s390_ucontrol.trans_exc_code =
1942 current->thread.gmap_addr;
1943 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1944 rc = -EREMOTE;
24eb3a82
DD		 1945
1946 } else if (current->thread.gmap_pfault) {
3c038e6b 1947 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 1948 current->thread.gmap_pfault = 0;
fa576c58 1949 if (kvm_arch_setup_async_pf(vcpu)) {
24eb3a82 1950 rc = 0;
fa576c58
TH		 1951 } else {
1952 gpa_t gpa = current->thread.gmap_addr;
1953 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1954 }
24eb3a82
DD		 1955 }
1956
492d8642
TH		 1957 if (rc == -1)
1958 rc = vcpu_post_run_fault_in_sie(vcpu);
b0c632db 1959
5a32c1af 1960 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
3fb4c40f 1961
a76ccff6
TH		 1962 if (rc == 0) {
1963 if (kvm_is_ucontrol(vcpu->kvm))
2955c83f
CB		 1964 /* Don't exit for host interrupts. */
1965 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
a76ccff6
TH		 1966 else
1967 rc = kvm_handle_sie_intercept(vcpu);
1968 }
1969
3fb4c40f
TH		 1970 return rc;
1971}
1972
1973static int __vcpu_run(struct kvm_vcpu *vcpu)
1974{
1975 int rc, exit_reason;
1976
800c1065
TH		 1977 /*
1978 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1979 * ning the guest), so that memslots (and other stuff) are protected
1980 */
1981 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1982
a76ccff6
TH		 1983 do {
1984 rc = vcpu_pre_run(vcpu);
1985 if (rc)
1986 break;
3fb4c40f 1987
800c1065 1988 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 1989 /*
1990 * As PF_VCPU will be used in fault handler, between
1991 * guest_enter and guest_exit should be no uaccess.
1992 */
1993 preempt_disable();
1994 kvm_guest_enter();
1995 preempt_enable();
1996 exit_reason = sie64a(vcpu->arch.sie_block,
1997 vcpu->run->s.regs.gprs);
1998 kvm_guest_exit();
800c1065 1999 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 2000
2001 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 2002 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 2003
800c1065 2004 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 2005 return rc;
b0c632db
HC		 2006}
2007
b028ee3e
DH		 2008static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2009{
2010 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2011 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2012 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2013 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2014 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2015 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 2016 /* some control register changes require a tlb flush */
2017 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 2018 }
2019 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2020 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2021 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2022 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2023 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2024 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2025 }
2026 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2027 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2028 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2029 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 2030 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2031 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 2032 }
2033 kvm_run->kvm_dirty_regs = 0;
2034}
2035
2036static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2037{
2038 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2039 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2040 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2041 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2042 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2043 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2044 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2045 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2046 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2047 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2048 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2049 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2050}
2051
b0c632db
HC		 2052int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2053{
8f2abe6a 2054 int rc;
b0c632db
HC		 2055 sigset_t sigsaved;
2056
27291e21
DH		 2057 if (guestdbg_exit_pending(vcpu)) {
2058 kvm_s390_prepare_debug_exit(vcpu);
2059 return 0;
2060 }
2061
b0c632db
HC		 2062 if (vcpu->sigset_active)
2063 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2064
6352e4d2
DH		 2065 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2066 kvm_s390_vcpu_start(vcpu);
2067 } else if (is_vcpu_stopped(vcpu)) {
2068 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
2069 vcpu->vcpu_id);
2070 return -EINVAL;
2071 }
b0c632db 2072
b028ee3e 2073 sync_regs(vcpu, kvm_run);
d7b0b5eb 2074
dab4079d 2075 might_fault();
a76ccff6 2076 rc = __vcpu_run(vcpu);
9ace903d 2077
b1d16c49
CE		 2078 if (signal_pending(current) && !rc) {
2079 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 2080 rc = -EINTR;
b1d16c49 2081 }
8f2abe6a 2082
27291e21
DH		 2083 if (guestdbg_exit_pending(vcpu) && !rc) {
2084 kvm_s390_prepare_debug_exit(vcpu);
2085 rc = 0;
2086 }
2087
b8e660b8 2088 if (rc == -EOPNOTSUPP) {
8f2abe6a
CB		 2089 /* intercept cannot be handled in-kernel, prepare kvm-run */
2090 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
2091 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
8f2abe6a
CB		 2092 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2093 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2094 rc = 0;
2095 }
2096
2097 if (rc == -EREMOTE) {
2098 /* intercept was handled, but userspace support is needed
2099 * kvm_run has been prepared by the handler */
2100 rc = 0;
2101 }
b0c632db 2102
b028ee3e 2103 store_regs(vcpu, kvm_run);
d7b0b5eb 2104
b0c632db
HC		 2105 if (vcpu->sigset_active)
2106 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2107
b0c632db 2108 vcpu->stat.exit_userspace++;
7e8e6ab4 2109 return rc;
b0c632db
HC		 2110}
2111
b0c632db
HC		 2112/*
2113 * store status at address
2114 * we use have two special cases:
2115 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2116 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2117 */
d0bce605 2118int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 2119{
092670cd 2120 unsigned char archmode = 1;
fda902cb 2121 unsigned int px;
178bd789 2122 u64 clkcomp;
d0bce605 2123 int rc;
b0c632db 2124
d0bce605
HC		 2125 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2126 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 2127 return -EFAULT;
d0bce605
HC		 2128 gpa = SAVE_AREA_BASE;
2129 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2130 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 2131 return -EFAULT;
d0bce605
HC		 2132 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2133 }
2134 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2135 vcpu->arch.guest_fpregs.fprs, 128);
2136 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2137 vcpu->run->s.regs.gprs, 128);
2138 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2139 &vcpu->arch.sie_block->gpsw, 16);
fda902cb 2140 px = kvm_s390_get_prefix(vcpu);
d0bce605 2141 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
fda902cb 2142 &px, 4);
d0bce605
HC		 2143 rc |= write_guest_abs(vcpu,
2144 gpa + offsetof(struct save_area, fp_ctrl_reg),
2145 &vcpu->arch.guest_fpregs.fpc, 4);
2146 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2147 &vcpu->arch.sie_block->todpr, 4);
2148 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2149 &vcpu->arch.sie_block->cputm, 8);
178bd789 2150 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 2151 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2152 &clkcomp, 8);
2153 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2154 &vcpu->run->s.regs.acrs, 64);
2155 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2156 &vcpu->arch.sie_block->gcr, 128);
2157 return rc ? -EFAULT : 0;
b0c632db
HC		 2158}
2159
e879892c
TH		 2160int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2161{
2162 /*
2163 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2164 * copying in vcpu load/put. Lets update our copies before we save
2165 * it into the save area
2166 */
2167 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
2168 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
2169 save_access_regs(vcpu->run->s.regs.acrs);
2170
2171 return kvm_s390_store_status_unloaded(vcpu, addr);
2172}
2173
bc17de7c
EF		 2174/*
2175 * store additional status at address
2176 */
2177int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2178 unsigned long gpa)
2179{
2180 /* Only bits 0-53 are used for address formation */
2181 if (!(gpa & ~0x3ff))
2182 return 0;
2183
2184 return write_guest_abs(vcpu, gpa & ~0x3ff,
2185 (void *)&vcpu->run->s.regs.vrs, 512);
2186}
2187
2188int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2189{
2190 if (!test_kvm_facility(vcpu->kvm, 129))
2191 return 0;
2192
2193 /*
2194 * The guest VXRS are in the host VXRs due to the lazy
2195 * copying in vcpu load/put. Let's update our copies before we save
2196 * it into the save area.
2197 */
2198 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2199
2200 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2201}
2202
8ad35755
DH		 2203static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2204{
2205 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2206 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
2207 exit_sie_sync(vcpu);
2208}
2209
2210static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2211{
2212 unsigned int i;
2213 struct kvm_vcpu *vcpu;
2214
2215 kvm_for_each_vcpu(i, vcpu, kvm) {
2216 __disable_ibs_on_vcpu(vcpu);
2217 }
2218}
2219
2220static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2221{
2222 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2223 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
2224 exit_sie_sync(vcpu);
2225}
2226
6852d7b6
DH		 2227void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2228{
8ad35755
DH		 2229 int i, online_vcpus, started_vcpus = 0;
2230
2231 if (!is_vcpu_stopped(vcpu))
2232 return;
2233
6852d7b6 2234 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 2235 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2236 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2237 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2238
2239 for (i = 0; i < online_vcpus; i++) {
2240 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2241 started_vcpus++;
2242 }
2243
2244 if (started_vcpus == 0) {
2245 /* we're the only active VCPU -> speed it up */
2246 __enable_ibs_on_vcpu(vcpu);
2247 } else if (started_vcpus == 1) {
2248 /*
2249 * As we are starting a second VCPU, we have to disable
2250 * the IBS facility on all VCPUs to remove potentially
2251 * oustanding ENABLE requests.
2252 */
2253 __disable_ibs_on_all_vcpus(vcpu->kvm);
2254 }
2255
6852d7b6 2256 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2257 /*
2258 * Another VCPU might have used IBS while we were offline.
2259 * Let's play safe and flush the VCPU at startup.
2260 */
d3d692c8 2261 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 2262 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2263 return;
6852d7b6
DH		 2264}
2265
2266void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2267{
8ad35755
DH		 2268 int i, online_vcpus, started_vcpus = 0;
2269 struct kvm_vcpu *started_vcpu = NULL;
2270
2271 if (is_vcpu_stopped(vcpu))
2272 return;
2273
6852d7b6 2274 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 2275 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 2276 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 2277 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2278
32f5ff63 2279 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 2280 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 2281
6cddd432 2282 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 2283 __disable_ibs_on_vcpu(vcpu);
2284
2285 for (i = 0; i < online_vcpus; i++) {
2286 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2287 started_vcpus++;
2288 started_vcpu = vcpu->kvm->vcpus[i];
2289 }
2290 }
2291
2292 if (started_vcpus == 1) {
2293 /*
2294 * As we only have one VCPU left, we want to enable the
2295 * IBS facility for that VCPU to speed it up.
2296 */
2297 __enable_ibs_on_vcpu(started_vcpu);
2298 }
2299
433b9ee4 2300 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 2301 return;
6852d7b6
DH		 2302}
2303
d6712df9
CH		 2304static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2305 struct kvm_enable_cap *cap)
2306{
2307 int r;
2308
2309 if (cap->flags)
2310 return -EINVAL;
2311
2312 switch (cap->cap) {
fa6b7fe9
CH		 2313 case KVM_CAP_S390_CSS_SUPPORT:
2314 if (!vcpu->kvm->arch.css_support) {
2315 vcpu->kvm->arch.css_support = 1;
2316 trace_kvm_s390_enable_css(vcpu->kvm);
2317 }
2318 r = 0;
2319 break;
d6712df9
CH		 2320 default:
2321 r = -EINVAL;
2322 break;
2323 }
2324 return r;
2325}
2326
41408c28
TH		 2327static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2328 struct kvm_s390_mem_op *mop)
2329{
2330 void __user *uaddr = (void __user *)mop->buf;
2331 void *tmpbuf = NULL;
2332 int r, srcu_idx;
2333 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2334 | KVM_S390_MEMOP_F_CHECK_ONLY;
2335
2336 if (mop->flags & ~supported_flags)
2337 return -EINVAL;
2338
2339 if (mop->size > MEM_OP_MAX_SIZE)
2340 return -E2BIG;
2341
2342 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2343 tmpbuf = vmalloc(mop->size);
2344 if (!tmpbuf)
2345 return -ENOMEM;
2346 }
2347
2348 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2349
2350 switch (mop->op) {
2351 case KVM_S390_MEMOP_LOGICAL_READ:
2352 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2353 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2354 break;
2355 }
2356 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2357 if (r == 0) {
2358 if (copy_to_user(uaddr, tmpbuf, mop->size))
2359 r = -EFAULT;
2360 }
2361 break;
2362 case KVM_S390_MEMOP_LOGICAL_WRITE:
2363 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2364 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2365 break;
2366 }
2367 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2368 r = -EFAULT;
2369 break;
2370 }
2371 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2372 break;
2373 default:
2374 r = -EINVAL;
2375 }
2376
2377 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2378
2379 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2380 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2381
2382 vfree(tmpbuf);
2383 return r;
2384}
2385
b0c632db
HC		 2386long kvm_arch_vcpu_ioctl(struct file *filp,
2387 unsigned int ioctl, unsigned long arg)
2388{
2389 struct kvm_vcpu *vcpu = filp->private_data;
2390 void __user *argp = (void __user *)arg;
800c1065 2391 int idx;
bc923cc9 2392 long r;
b0c632db 2393
93736624 2394 switch (ioctl) {
47b43c52
JF		 2395 case KVM_S390_IRQ: {
2396 struct kvm_s390_irq s390irq;
2397
2398 r = -EFAULT;
2399 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2400 break;
2401 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2402 break;
2403 }
93736624 2404 case KVM_S390_INTERRUPT: {
ba5c1e9b 2405 struct kvm_s390_interrupt s390int;
383d0b05 2406 struct kvm_s390_irq s390irq;
ba5c1e9b 2407
93736624 2408 r = -EFAULT;
ba5c1e9b 2409 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 2410 break;
383d0b05
JF		 2411 if (s390int_to_s390irq(&s390int, &s390irq))
2412 return -EINVAL;
2413 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 2414 break;
ba5c1e9b 2415 }
b0c632db 2416 case KVM_S390_STORE_STATUS:
800c1065 2417 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 2418 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 2419 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 2420 break;
b0c632db
HC		 2421 case KVM_S390_SET_INITIAL_PSW: {
2422 psw_t psw;
2423
bc923cc9 2424 r = -EFAULT;
b0c632db 2425 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 2426 break;
2427 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2428 break;
b0c632db
HC		 2429 }
2430 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 2431 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2432 break;
14eebd91
CO		 2433 case KVM_SET_ONE_REG:
2434 case KVM_GET_ONE_REG: {
2435 struct kvm_one_reg reg;
2436 r = -EFAULT;
2437 if (copy_from_user(&reg, argp, sizeof(reg)))
2438 break;
2439 if (ioctl == KVM_SET_ONE_REG)
2440 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2441 else
2442 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2443 break;
2444 }
27e0393f
CO		 2445#ifdef CONFIG_KVM_S390_UCONTROL
2446 case KVM_S390_UCAS_MAP: {
2447 struct kvm_s390_ucas_mapping ucasmap;
2448
2449 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2450 r = -EFAULT;
2451 break;
2452 }
2453
2454 if (!kvm_is_ucontrol(vcpu->kvm)) {
2455 r = -EINVAL;
2456 break;
2457 }
2458
2459 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2460 ucasmap.vcpu_addr, ucasmap.length);
2461 break;
2462 }
2463 case KVM_S390_UCAS_UNMAP: {
2464 struct kvm_s390_ucas_mapping ucasmap;
2465
2466 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2467 r = -EFAULT;
2468 break;
2469 }
2470
2471 if (!kvm_is_ucontrol(vcpu->kvm)) {
2472 r = -EINVAL;
2473 break;
2474 }
2475
2476 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2477 ucasmap.length);
2478 break;
2479 }
2480#endif
ccc7910f 2481 case KVM_S390_VCPU_FAULT: {
527e30b4 2482 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 2483 break;
2484 }
d6712df9
CH		 2485 case KVM_ENABLE_CAP:
2486 {
2487 struct kvm_enable_cap cap;
2488 r = -EFAULT;
2489 if (copy_from_user(&cap, argp, sizeof(cap)))
2490 break;
2491 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2492 break;
2493 }
41408c28
TH		 2494 case KVM_S390_MEM_OP: {
2495 struct kvm_s390_mem_op mem_op;
2496
2497 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2498 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2499 else
2500 r = -EFAULT;
2501 break;
2502 }
b0c632db 2503 default:
3e6afcf1 2504 r = -ENOTTY;
b0c632db 2505 }
bc923cc9 2506 return r;
b0c632db
HC		 2507}
2508
5b1c1493
CO		 2509int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2510{
2511#ifdef CONFIG_KVM_S390_UCONTROL
2512 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2513 && (kvm_is_ucontrol(vcpu->kvm))) {
2514 vmf->page = virt_to_page(vcpu->arch.sie_block);
2515 get_page(vmf->page);
2516 return 0;
2517 }
2518#endif
2519 return VM_FAULT_SIGBUS;
2520}
2521
5587027c
AK		 2522int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2523 unsigned long npages)
db3fe4eb
TY		 2524{
2525 return 0;
2526}
2527
b0c632db 2528/* Section: memory related */
f7784b8e
MT		 2529int kvm_arch_prepare_memory_region(struct kvm *kvm,
2530 struct kvm_memory_slot *memslot,
7b6195a9
TY		 2531 struct kvm_userspace_memory_region *mem,
2532 enum kvm_mr_change change)
b0c632db 2533{
dd2887e7
NW		 2534 /* A few sanity checks. We can have memory slots which have to be
2535 located/ended at a segment boundary (1MB). The memory in userland is
2536 ok to be fragmented into various different vmas. It is okay to mmap()
2537 and munmap() stuff in this slot after doing this call at any time */
b0c632db 2538
598841ca 2539 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 2540 return -EINVAL;
2541
598841ca 2542 if (mem->memory_size & 0xffffful)
b0c632db
HC		 2543 return -EINVAL;
2544
f7784b8e
MT		 2545 return 0;
2546}
2547
2548void kvm_arch_commit_memory_region(struct kvm *kvm,
2549 struct kvm_userspace_memory_region *mem,
8482644a
TY		 2550 const struct kvm_memory_slot *old,
2551 enum kvm_mr_change change)
f7784b8e 2552{
f7850c92 2553 int rc;
f7784b8e 2554
2cef4deb
CB		 2555 /* If the basics of the memslot do not change, we do not want
2556 * to update the gmap. Every update causes several unnecessary
2557 * segment translation exceptions. This is usually handled just
2558 * fine by the normal fault handler + gmap, but it will also
2559 * cause faults on the prefix page of running guest CPUs.
2560 */
2561 if (old->userspace_addr == mem->userspace_addr &&
2562 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2563 old->npages * PAGE_SIZE == mem->memory_size)
2564 return;
598841ca
CO		 2565
2566 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2567 mem->guest_phys_addr, mem->memory_size);
2568 if (rc)
f7850c92 2569 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
598841ca 2570 return;
b0c632db
HC		 2571}
2572
b0c632db
HC		 2573static int __init kvm_s390_init(void)
2574{
9d8d5786 2575 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
b0c632db
HC		 2576}
2577
2578static void __exit kvm_s390_exit(void)
2579{
2580 kvm_exit();
2581}
2582
2583module_init(kvm_s390_init);
2584module_exit(kvm_s390_exit);
566af940
CH		 2585
2586/*
2587 * Enable autoloading of the kvm module.
2588 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2589 * since x86 takes a different approach.
2590 */
2591#include <linux/miscdevice.h>
2592MODULE_ALIAS_MISCDEV(KVM_MINOR);
2593MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)