projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
s390/kernel: Update /proc/sysinfo file with Extended Name and UUID
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
a374e892 25#include <linux/random.h>
b0c632db 26#include <linux/slab.h>
ba5c1e9b 27#include <linux/timer.h>
cbb870c8 28#include <asm/asm-offsets.h>
b0c632db
HC		 29#include <asm/lowcore.h>
30#include <asm/pgtable.h>
f5daba1d 31#include <asm/nmi.h>
a0616cde 32#include <asm/switch_to.h>
78c4b59f 33#include <asm/facility.h>
1526bf9c 34#include <asm/sclp.h>
8f2abe6a 35#include "kvm-s390.h"
b0c632db
HC		 36#include "gaccess.h"
37
5786fffa
CH		 38#define CREATE_TRACE_POINTS
39#include "trace.h"
ade38c31 40#include "trace-s390.h"
5786fffa 41
b0c632db
HC		 42#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
43
44struct kvm_stats_debugfs_item debugfs_entries[] = {
45 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 46 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 47 { "exit_validity", VCPU_STAT(exit_validity) },
48 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
49 { "exit_external_request", VCPU_STAT(exit_external_request) },
50 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 51 { "exit_instruction", VCPU_STAT(exit_instruction) },
52 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
53 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f7819512 54 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
ce2e4f0b 55 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 56 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 57 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 58 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
59 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 60 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 61 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 62 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
63 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
64 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
65 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
66 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
67 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
68 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 69 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 70 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
71 { "instruction_spx", VCPU_STAT(instruction_spx) },
72 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
73 { "instruction_stap", VCPU_STAT(instruction_stap) },
74 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 75 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 76 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
77 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 78 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 79 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
80 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 81 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 82 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 83 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 84 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 85 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 86 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
87 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 88 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 89 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
90 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
5288fbf0
CB		 91 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
92 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
93 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 94 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
95 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
96 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 97 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 98 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 99 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
b0c632db
HC		 100 { NULL }
101};
102
78c4b59f 103unsigned long *vfacilities;
2c70fe44 104static struct gmap_notifier gmap_notifier;
b0c632db 105
78c4b59f 106/* test availability of vfacility */
280ef0f1 107int test_vfacility(unsigned long nr)
78c4b59f
MM		 108{
109 return __test_facility(nr, (void *) vfacilities);
110}
111
b0c632db 112/* Section: not file related */
13a34e06 113int kvm_arch_hardware_enable(void)
b0c632db
HC		 114{
115 /* every s390 is virtualization enabled ;-) */
10474ae8 116 return 0;
b0c632db
HC		 117}
118
2c70fe44
CB		 119static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
120
b0c632db
HC		 121int kvm_arch_hardware_setup(void)
122{
2c70fe44
CB		 123 gmap_notifier.notifier_call = kvm_gmap_notifier;
124 gmap_register_ipte_notifier(&gmap_notifier);
b0c632db
HC		 125 return 0;
126}
127
128void kvm_arch_hardware_unsetup(void)
129{
2c70fe44 130 gmap_unregister_ipte_notifier(&gmap_notifier);
b0c632db
HC		 131}
132
b0c632db
HC		 133int kvm_arch_init(void *opaque)
134{
84877d93
CH		 135 /* Register floating interrupt controller interface. */
136 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 137}
138
b0c632db
HC		 139/* Section: device related */
140long kvm_arch_dev_ioctl(struct file *filp,
141 unsigned int ioctl, unsigned long arg)
142{
143 if (ioctl == KVM_S390_ENABLE_SIE)
144 return s390_enable_sie();
145 return -EINVAL;
146}
147
784aa3d7 148int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 149{
d7b0b5eb
CO		 150 int r;
151
2bd0ac4e 152 switch (ext) {
d7b0b5eb 153 case KVM_CAP_S390_PSW:
b6cf8788 154 case KVM_CAP_S390_GMAP:
52e16b18 155 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 156#ifdef CONFIG_KVM_S390_UCONTROL
157 case KVM_CAP_S390_UCONTROL:
158#endif
3c038e6b 159 case KVM_CAP_ASYNC_PF:
60b413c9 160 case KVM_CAP_SYNC_REGS:
14eebd91 161 case KVM_CAP_ONE_REG:
d6712df9 162 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 163 case KVM_CAP_S390_CSS_SUPPORT:
ebc32262 164 case KVM_CAP_IRQFD:
10ccaa1e 165 case KVM_CAP_IOEVENTFD:
c05c4186 166 case KVM_CAP_DEVICE_CTRL:
d938dc55 167 case KVM_CAP_ENABLE_CAP_VM:
78599d90 168 case KVM_CAP_S390_IRQCHIP:
f2061656 169 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 170 case KVM_CAP_MP_STATE:
2444b352 171 case KVM_CAP_S390_USER_SIGP:
d7b0b5eb
CO		 172 r = 1;
173 break;
e726b1bd
CB		 174 case KVM_CAP_NR_VCPUS:
175 case KVM_CAP_MAX_VCPUS:
176 r = KVM_MAX_VCPUS;
177 break;
e1e2e605
NW		 178 case KVM_CAP_NR_MEMSLOTS:
179 r = KVM_USER_MEM_SLOTS;
180 break;
1526bf9c 181 case KVM_CAP_S390_COW:
abf09bed 182 r = MACHINE_HAS_ESOP;
1526bf9c 183 break;
2bd0ac4e 184 default:
d7b0b5eb 185 r = 0;
2bd0ac4e 186 }
d7b0b5eb 187 return r;
b0c632db
HC		 188}
189
15f36ebd
JH		 190static void kvm_s390_sync_dirty_log(struct kvm *kvm,
191 struct kvm_memory_slot *memslot)
192{
193 gfn_t cur_gfn, last_gfn;
194 unsigned long address;
195 struct gmap *gmap = kvm->arch.gmap;
196
197 down_read(&gmap->mm->mmap_sem);
198 /* Loop over all guest pages */
199 last_gfn = memslot->base_gfn + memslot->npages;
200 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
201 address = gfn_to_hva_memslot(memslot, cur_gfn);
202
203 if (gmap_test_and_clear_dirty(address, gmap))
204 mark_page_dirty(kvm, cur_gfn);
205 }
206 up_read(&gmap->mm->mmap_sem);
207}
208
b0c632db
HC		 209/* Section: vm related */
210/*
211 * Get (and clear) the dirty memory log for a memory slot.
212 */
213int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
214 struct kvm_dirty_log *log)
215{
15f36ebd
JH		 216 int r;
217 unsigned long n;
218 struct kvm_memory_slot *memslot;
219 int is_dirty = 0;
220
221 mutex_lock(&kvm->slots_lock);
222
223 r = -EINVAL;
224 if (log->slot >= KVM_USER_MEM_SLOTS)
225 goto out;
226
227 memslot = id_to_memslot(kvm->memslots, log->slot);
228 r = -ENOENT;
229 if (!memslot->dirty_bitmap)
230 goto out;
231
232 kvm_s390_sync_dirty_log(kvm, memslot);
233 r = kvm_get_dirty_log(kvm, log, &is_dirty);
234 if (r)
235 goto out;
236
237 /* Clear the dirty log */
238 if (is_dirty) {
239 n = kvm_dirty_bitmap_bytes(memslot);
240 memset(memslot->dirty_bitmap, 0, n);
241 }
242 r = 0;
243out:
244 mutex_unlock(&kvm->slots_lock);
245 return r;
b0c632db
HC		 246}
247
d938dc55
CH		 248static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
249{
250 int r;
251
252 if (cap->flags)
253 return -EINVAL;
254
255 switch (cap->cap) {
84223598
CH		 256 case KVM_CAP_S390_IRQCHIP:
257 kvm->arch.use_irqchip = 1;
258 r = 0;
259 break;
2444b352
DH		 260 case KVM_CAP_S390_USER_SIGP:
261 kvm->arch.user_sigp = 1;
262 r = 0;
263 break;
d938dc55
CH		 264 default:
265 r = -EINVAL;
266 break;
267 }
268 return r;
269}
270
8c0a7ce6
DD		 271static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
272{
273 int ret;
274
275 switch (attr->attr) {
276 case KVM_S390_VM_MEM_LIMIT_SIZE:
277 ret = 0;
278 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
279 ret = -EFAULT;
280 break;
281 default:
282 ret = -ENXIO;
283 break;
284 }
285 return ret;
286}
287
288static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 289{
290 int ret;
291 unsigned int idx;
292 switch (attr->attr) {
293 case KVM_S390_VM_MEM_ENABLE_CMMA:
294 ret = -EBUSY;
295 mutex_lock(&kvm->lock);
296 if (atomic_read(&kvm->online_vcpus) == 0) {
297 kvm->arch.use_cmma = 1;
298 ret = 0;
299 }
300 mutex_unlock(&kvm->lock);
301 break;
302 case KVM_S390_VM_MEM_CLR_CMMA:
303 mutex_lock(&kvm->lock);
304 idx = srcu_read_lock(&kvm->srcu);
a13cff31 305 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 306 srcu_read_unlock(&kvm->srcu, idx);
307 mutex_unlock(&kvm->lock);
308 ret = 0;
309 break;
8c0a7ce6
DD		 310 case KVM_S390_VM_MEM_LIMIT_SIZE: {
311 unsigned long new_limit;
312
313 if (kvm_is_ucontrol(kvm))
314 return -EINVAL;
315
316 if (get_user(new_limit, (u64 __user *)attr->addr))
317 return -EFAULT;
318
319 if (new_limit > kvm->arch.gmap->asce_end)
320 return -E2BIG;
321
322 ret = -EBUSY;
323 mutex_lock(&kvm->lock);
324 if (atomic_read(&kvm->online_vcpus) == 0) {
325 /* gmap_alloc will round the limit up */
326 struct gmap *new = gmap_alloc(current->mm, new_limit);
327
328 if (!new) {
329 ret = -ENOMEM;
330 } else {
331 gmap_free(kvm->arch.gmap);
332 new->private = kvm;
333 kvm->arch.gmap = new;
334 ret = 0;
335 }
336 }
337 mutex_unlock(&kvm->lock);
338 break;
339 }
4f718eab
DD		 340 default:
341 ret = -ENXIO;
342 break;
343 }
344 return ret;
345}
346
a374e892
TK		 347static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
348
349static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
350{
351 struct kvm_vcpu *vcpu;
352 int i;
353
354 if (!test_vfacility(76))
355 return -EINVAL;
356
357 mutex_lock(&kvm->lock);
358 switch (attr->attr) {
359 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
360 get_random_bytes(
361 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
362 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
363 kvm->arch.crypto.aes_kw = 1;
364 break;
365 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
366 get_random_bytes(
367 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
368 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
369 kvm->arch.crypto.dea_kw = 1;
370 break;
371 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
372 kvm->arch.crypto.aes_kw = 0;
373 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
374 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
375 break;
376 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
377 kvm->arch.crypto.dea_kw = 0;
378 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
379 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
380 break;
381 default:
382 mutex_unlock(&kvm->lock);
383 return -ENXIO;
384 }
385
386 kvm_for_each_vcpu(i, vcpu, kvm) {
387 kvm_s390_vcpu_crypto_setup(vcpu);
388 exit_sie(vcpu);
389 }
390 mutex_unlock(&kvm->lock);
391 return 0;
392}
393
72f25020
JH		 394static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
395{
396 u8 gtod_high;
397
398 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
399 sizeof(gtod_high)))
400 return -EFAULT;
401
402 if (gtod_high != 0)
403 return -EINVAL;
404
405 return 0;
406}
407
408static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
409{
410 struct kvm_vcpu *cur_vcpu;
411 unsigned int vcpu_idx;
412 u64 host_tod, gtod;
413 int r;
414
415 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
416 return -EFAULT;
417
418 r = store_tod_clock(&host_tod);
419 if (r)
420 return r;
421
422 mutex_lock(&kvm->lock);
423 kvm->arch.epoch = gtod - host_tod;
424 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
425 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
426 exit_sie(cur_vcpu);
427 }
428 mutex_unlock(&kvm->lock);
429 return 0;
430}
431
432static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
433{
434 int ret;
435
436 if (attr->flags)
437 return -EINVAL;
438
439 switch (attr->attr) {
440 case KVM_S390_VM_TOD_HIGH:
441 ret = kvm_s390_set_tod_high(kvm, attr);
442 break;
443 case KVM_S390_VM_TOD_LOW:
444 ret = kvm_s390_set_tod_low(kvm, attr);
445 break;
446 default:
447 ret = -ENXIO;
448 break;
449 }
450 return ret;
451}
452
453static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
454{
455 u8 gtod_high = 0;
456
457 if (copy_to_user((void __user *)attr->addr, &gtod_high,
458 sizeof(gtod_high)))
459 return -EFAULT;
460
461 return 0;
462}
463
464static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
465{
466 u64 host_tod, gtod;
467 int r;
468
469 r = store_tod_clock(&host_tod);
470 if (r)
471 return r;
472
473 gtod = host_tod + kvm->arch.epoch;
474 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
475 return -EFAULT;
476
477 return 0;
478}
479
480static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
481{
482 int ret;
483
484 if (attr->flags)
485 return -EINVAL;
486
487 switch (attr->attr) {
488 case KVM_S390_VM_TOD_HIGH:
489 ret = kvm_s390_get_tod_high(kvm, attr);
490 break;
491 case KVM_S390_VM_TOD_LOW:
492 ret = kvm_s390_get_tod_low(kvm, attr);
493 break;
494 default:
495 ret = -ENXIO;
496 break;
497 }
498 return ret;
499}
500
f2061656
DD		 501static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
502{
503 int ret;
504
505 switch (attr->group) {
4f718eab 506 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 507 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 508 break;
72f25020
JH		 509 case KVM_S390_VM_TOD:
510 ret = kvm_s390_set_tod(kvm, attr);
511 break;
a374e892
TK		 512 case KVM_S390_VM_CRYPTO:
513 ret = kvm_s390_vm_set_crypto(kvm, attr);
514 break;
f2061656
DD		 515 default:
516 ret = -ENXIO;
517 break;
518 }
519
520 return ret;
521}
522
523static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
524{
8c0a7ce6
DD		 525 int ret;
526
527 switch (attr->group) {
528 case KVM_S390_VM_MEM_CTRL:
529 ret = kvm_s390_get_mem_control(kvm, attr);
530 break;
72f25020
JH		 531 case KVM_S390_VM_TOD:
532 ret = kvm_s390_get_tod(kvm, attr);
533 break;
8c0a7ce6
DD		 534 default:
535 ret = -ENXIO;
536 break;
537 }
538
539 return ret;
f2061656
DD		 540}
541
542static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
543{
544 int ret;
545
546 switch (attr->group) {
4f718eab
DD		 547 case KVM_S390_VM_MEM_CTRL:
548 switch (attr->attr) {
549 case KVM_S390_VM_MEM_ENABLE_CMMA:
550 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 551 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 552 ret = 0;
553 break;
554 default:
555 ret = -ENXIO;
556 break;
557 }
558 break;
72f25020
JH		 559 case KVM_S390_VM_TOD:
560 switch (attr->attr) {
561 case KVM_S390_VM_TOD_LOW:
562 case KVM_S390_VM_TOD_HIGH:
563 ret = 0;
564 break;
565 default:
566 ret = -ENXIO;
567 break;
568 }
569 break;
a374e892
TK		 570 case KVM_S390_VM_CRYPTO:
571 switch (attr->attr) {
572 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
573 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
574 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
575 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
576 ret = 0;
577 break;
578 default:
579 ret = -ENXIO;
580 break;
581 }
582 break;
f2061656
DD		 583 default:
584 ret = -ENXIO;
585 break;
586 }
587
588 return ret;
589}
590
b0c632db
HC		 591long kvm_arch_vm_ioctl(struct file *filp,
592 unsigned int ioctl, unsigned long arg)
593{
594 struct kvm *kvm = filp->private_data;
595 void __user *argp = (void __user *)arg;
f2061656 596 struct kvm_device_attr attr;
b0c632db
HC		 597 int r;
598
599 switch (ioctl) {
ba5c1e9b
CO		 600 case KVM_S390_INTERRUPT: {
601 struct kvm_s390_interrupt s390int;
602
603 r = -EFAULT;
604 if (copy_from_user(&s390int, argp, sizeof(s390int)))
605 break;
606 r = kvm_s390_inject_vm(kvm, &s390int);
607 break;
608 }
d938dc55
CH		 609 case KVM_ENABLE_CAP: {
610 struct kvm_enable_cap cap;
611 r = -EFAULT;
612 if (copy_from_user(&cap, argp, sizeof(cap)))
613 break;
614 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
615 break;
616 }
84223598
CH		 617 case KVM_CREATE_IRQCHIP: {
618 struct kvm_irq_routing_entry routing;
619
620 r = -EINVAL;
621 if (kvm->arch.use_irqchip) {
622 /* Set up dummy routing. */
623 memset(&routing, 0, sizeof(routing));
624 kvm_set_irq_routing(kvm, &routing, 0, 0);
625 r = 0;
626 }
627 break;
628 }
f2061656
DD		 629 case KVM_SET_DEVICE_ATTR: {
630 r = -EFAULT;
631 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
632 break;
633 r = kvm_s390_vm_set_attr(kvm, &attr);
634 break;
635 }
636 case KVM_GET_DEVICE_ATTR: {
637 r = -EFAULT;
638 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
639 break;
640 r = kvm_s390_vm_get_attr(kvm, &attr);
641 break;
642 }
643 case KVM_HAS_DEVICE_ATTR: {
644 r = -EFAULT;
645 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
646 break;
647 r = kvm_s390_vm_has_attr(kvm, &attr);
648 break;
649 }
b0c632db 650 default:
367e1319 651 r = -ENOTTY;
b0c632db
HC		 652 }
653
654 return r;
655}
656
5102ee87
TK		 657static int kvm_s390_crypto_init(struct kvm *kvm)
658{
659 if (!test_vfacility(76))
660 return 0;
661
662 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
663 GFP_KERNEL | GFP_DMA);
664 if (!kvm->arch.crypto.crycb)
665 return -ENOMEM;
666
667 kvm->arch.crypto.crycbd = (__u32) (unsigned long) kvm->arch.crypto.crycb |
668 CRYCB_FORMAT1;
669
a374e892
TK		 670 /* Disable AES/DEA protected key functions by default */
671 kvm->arch.crypto.aes_kw = 0;
672 kvm->arch.crypto.dea_kw = 0;
673
5102ee87
TK		 674 return 0;
675}
676
e08b9637 677int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 678{
b0c632db
HC		 679 int rc;
680 char debug_name[16];
f6c137ff 681 static unsigned long sca_offset;
b0c632db 682
e08b9637
CO		 683 rc = -EINVAL;
684#ifdef CONFIG_KVM_S390_UCONTROL
685 if (type & ~KVM_VM_S390_UCONTROL)
686 goto out_err;
687 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
688 goto out_err;
689#else
690 if (type)
691 goto out_err;
692#endif
693
b0c632db
HC		 694 rc = s390_enable_sie();
695 if (rc)
d89f5eff 696 goto out_err;
b0c632db 697
b290411a
CO		 698 rc = -ENOMEM;
699
b0c632db
HC		 700 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
701 if (!kvm->arch.sca)
d89f5eff 702 goto out_err;
f6c137ff
CB		 703 spin_lock(&kvm_lock);
704 sca_offset = (sca_offset + 16) & 0x7f0;
705 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
706 spin_unlock(&kvm_lock);
b0c632db
HC		 707
708 sprintf(debug_name, "kvm-%u", current->pid);
709
710 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
711 if (!kvm->arch.dbf)
712 goto out_nodbf;
713
5102ee87
TK		 714 if (kvm_s390_crypto_init(kvm) < 0)
715 goto out_crypto;
716
ba5c1e9b
CO		 717 spin_lock_init(&kvm->arch.float_int.lock);
718 INIT_LIST_HEAD(&kvm->arch.float_int.list);
8a242234 719 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 720 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 721
b0c632db
HC		 722 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
723 VM_EVENT(kvm, 3, "%s", "vm created");
724
e08b9637
CO		 725 if (type & KVM_VM_S390_UCONTROL) {
726 kvm->arch.gmap = NULL;
727 } else {
0349985a 728 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
e08b9637
CO		 729 if (!kvm->arch.gmap)
730 goto out_nogmap;
2c70fe44 731 kvm->arch.gmap->private = kvm;
24eb3a82 732 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 733 }
fa6b7fe9
CH		 734
735 kvm->arch.css_support = 0;
84223598 736 kvm->arch.use_irqchip = 0;
72f25020 737 kvm->arch.epoch = 0;
fa6b7fe9 738
8ad35755
DH		 739 spin_lock_init(&kvm->arch.start_stop_lock);
740
d89f5eff 741 return 0;
598841ca 742out_nogmap:
5102ee87
TK		 743 kfree(kvm->arch.crypto.crycb);
744out_crypto:
598841ca 745 debug_unregister(kvm->arch.dbf);
b0c632db
HC		 746out_nodbf:
747 free_page((unsigned long)(kvm->arch.sca));
d89f5eff
JK		 748out_err:
749 return rc;
b0c632db
HC		 750}
751
d329c035
CB		 752void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
753{
754 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 755 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 756 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 757 kvm_clear_async_pf_completion_queue(vcpu);
58f9460b
CO		 758 if (!kvm_is_ucontrol(vcpu->kvm)) {
759 clear_bit(63 - vcpu->vcpu_id,
760 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
761 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
762 (__u64) vcpu->arch.sie_block)
763 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
764 }
abf4a71e 765 smp_mb();
27e0393f
CO		 766
767 if (kvm_is_ucontrol(vcpu->kvm))
768 gmap_free(vcpu->arch.gmap);
769
b31605c1
DD		 770 if (kvm_s390_cmma_enabled(vcpu->kvm))
771 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 772 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 773
6692cef3 774 kvm_vcpu_uninit(vcpu);
b110feaf 775 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 776}
777
778static void kvm_free_vcpus(struct kvm *kvm)
779{
780 unsigned int i;
988a2cae 781 struct kvm_vcpu *vcpu;
d329c035 782
988a2cae
GN		 783 kvm_for_each_vcpu(i, vcpu, kvm)
784 kvm_arch_vcpu_destroy(vcpu);
785
786 mutex_lock(&kvm->lock);
787 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
788 kvm->vcpus[i] = NULL;
789
790 atomic_set(&kvm->online_vcpus, 0);
791 mutex_unlock(&kvm->lock);
d329c035
CB		 792}
793
b0c632db
HC		 794void kvm_arch_destroy_vm(struct kvm *kvm)
795{
d329c035 796 kvm_free_vcpus(kvm);
b0c632db 797 free_page((unsigned long)(kvm->arch.sca));
d329c035 798 debug_unregister(kvm->arch.dbf);
5102ee87 799 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 800 if (!kvm_is_ucontrol(kvm))
801 gmap_free(kvm->arch.gmap);
841b91c5 802 kvm_s390_destroy_adapters(kvm);
67335e63 803 kvm_s390_clear_float_irqs(kvm);
b0c632db
HC		 804}
805
806/* Section: vcpu related */
dafd032a
DD		 807static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
808{
809 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
810 if (!vcpu->arch.gmap)
811 return -ENOMEM;
812 vcpu->arch.gmap->private = vcpu->kvm;
813
814 return 0;
815}
816
b0c632db
HC		 817int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
818{
3c038e6b
DD		 819 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
820 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 821 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
822 KVM_SYNC_GPRS |
9eed0735 823 KVM_SYNC_ACRS |
b028ee3e
DH		 824 KVM_SYNC_CRS |
825 KVM_SYNC_ARCH0 |
826 KVM_SYNC_PFAULT;
dafd032a
DD		 827
828 if (kvm_is_ucontrol(vcpu->kvm))
829 return __kvm_ucontrol_vcpu_init(vcpu);
830
b0c632db
HC		 831 return 0;
832}
833
b0c632db
HC		 834void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
835{
4725c860
MS		 836 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
837 save_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db 838 save_access_regs(vcpu->arch.host_acrs);
4725c860
MS		 839 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
840 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 841 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 842 gmap_enable(vcpu->arch.gmap);
9e6dabef 843 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 844}
845
846void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
847{
9e6dabef 848 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 849 gmap_disable(vcpu->arch.gmap);
4725c860
MS		 850 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
851 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 852 save_access_regs(vcpu->run->s.regs.acrs);
4725c860
MS		 853 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
854 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db
HC		 855 restore_access_regs(vcpu->arch.host_acrs);
856}
857
858static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
859{
860 /* this equals initial cpu reset in pop, but we don't switch to ESA */
861 vcpu->arch.sie_block->gpsw.mask = 0UL;
862 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 863 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 864 vcpu->arch.sie_block->cputm = 0UL;
865 vcpu->arch.sie_block->ckc = 0UL;
866 vcpu->arch.sie_block->todpr = 0;
867 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
868 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
869 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
870 vcpu->arch.guest_fpregs.fpc = 0;
871 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
872 vcpu->arch.sie_block->gbea = 1;
672550fb 873 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 874 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
875 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 876 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
877 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 878 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 879}
880
31928aa5 881void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 882{
72f25020
JH		 883 mutex_lock(&vcpu->kvm->lock);
884 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
885 mutex_unlock(&vcpu->kvm->lock);
dafd032a
DD		 886 if (!kvm_is_ucontrol(vcpu->kvm))
887 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
42897d86
MT		 888}
889
5102ee87
TK		 890static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
891{
892 if (!test_vfacility(76))
893 return;
894
a374e892
TK		 895 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
896
897 if (vcpu->kvm->arch.crypto.aes_kw)
898 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
899 if (vcpu->kvm->arch.crypto.dea_kw)
900 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
901
5102ee87
TK		 902 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
903}
904
b31605c1
DD		 905void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
906{
907 free_page(vcpu->arch.sie_block->cbrlo);
908 vcpu->arch.sie_block->cbrlo = 0;
909}
910
911int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
912{
913 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
914 if (!vcpu->arch.sie_block->cbrlo)
915 return -ENOMEM;
916
917 vcpu->arch.sie_block->ecb2 |= 0x80;
918 vcpu->arch.sie_block->ecb2 &= ~0x08;
919 return 0;
920}
921
b0c632db
HC		 922int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
923{
b31605c1 924 int rc = 0;
b31288fa 925
9e6dabef
CH		 926 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
927 CPUSTAT_SM |
69d0d3a3
CB		 928 CPUSTAT_STOPPED |
929 CPUSTAT_GED);
fc34531d 930 vcpu->arch.sie_block->ecb = 6;
7feb6bb8
MM		 931 if (test_vfacility(50) && test_vfacility(73))
932 vcpu->arch.sie_block->ecb |= 0x10;
933
69d0d3a3 934 vcpu->arch.sie_block->ecb2 = 8;
ea5f4969 935 vcpu->arch.sie_block->eca = 0xC1002000U;
217a4406
HC		 936 if (sclp_has_siif())
937 vcpu->arch.sie_block->eca |= 1;
ea5f4969
DH		 938 if (sclp_has_sigpif())
939 vcpu->arch.sie_block->eca |= 0x10000000U;
78c4b59f 940 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
5a5e6536
MR		 941 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
942 ICTL_TPROT;
943
b31605c1
DD		 944 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
945 rc = kvm_s390_vcpu_setup_cmma(vcpu);
946 if (rc)
947 return rc;
b31288fa 948 }
0ac96caf 949 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 950 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
453423dc 951 get_cpu_id(&vcpu->arch.cpu_id);
92e6ecf3 952 vcpu->arch.cpu_id.version = 0xff;
5102ee87
TK		 953
954 kvm_s390_vcpu_crypto_setup(vcpu);
955
b31605c1 956 return rc;
b0c632db
HC		 957}
958
959struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
960 unsigned int id)
961{
4d47555a 962 struct kvm_vcpu *vcpu;
7feb6bb8 963 struct sie_page *sie_page;
4d47555a
CO		 964 int rc = -EINVAL;
965
966 if (id >= KVM_MAX_VCPUS)
967 goto out;
968
969 rc = -ENOMEM;
b0c632db 970
b110feaf 971 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 972 if (!vcpu)
4d47555a 973 goto out;
b0c632db 974
7feb6bb8
MM		 975 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
976 if (!sie_page)
b0c632db
HC		 977 goto out_free_cpu;
978
7feb6bb8
MM		 979 vcpu->arch.sie_block = &sie_page->sie_block;
980 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
981
b0c632db 982 vcpu->arch.sie_block->icpua = id;
58f9460b
CO		 983 if (!kvm_is_ucontrol(kvm)) {
984 if (!kvm->arch.sca) {
985 WARN_ON_ONCE(1);
986 goto out_free_cpu;
987 }
988 if (!kvm->arch.sca->cpu[id].sda)
989 kvm->arch.sca->cpu[id].sda =
990 (__u64) vcpu->arch.sie_block;
991 vcpu->arch.sie_block->scaoh =
992 (__u32)(((__u64)kvm->arch.sca) >> 32);
993 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
994 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
995 }
b0c632db 996
ba5c1e9b 997 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 998 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 999 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 1000 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 1001
b0c632db
HC		 1002 rc = kvm_vcpu_init(vcpu, kvm, id);
1003 if (rc)
7b06bf2f 1004 goto out_free_sie_block;
b0c632db
HC		 1005 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1006 vcpu->arch.sie_block);
ade38c31 1007 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 1008
b0c632db 1009 return vcpu;
7b06bf2f
WY		 1010out_free_sie_block:
1011 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 1012out_free_cpu:
b110feaf 1013 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 1014out:
b0c632db
HC		 1015 return ERR_PTR(rc);
1016}
1017
b0c632db
HC		 1018int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1019{
9a022067 1020 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 1021}
1022
49b99e1e
CB		 1023void s390_vcpu_block(struct kvm_vcpu *vcpu)
1024{
1025 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1026}
1027
1028void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1029{
1030 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1031}
1032
1033/*
1034 * Kick a guest cpu out of SIE and wait until SIE is not running.
1035 * If the CPU is not running (e.g. waiting as idle) the function will
1036 * return immediately. */
1037void exit_sie(struct kvm_vcpu *vcpu)
1038{
1039 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1040 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1041 cpu_relax();
1042}
1043
1044/* Kick a guest cpu out of SIE and prevent SIE-reentry */
1045void exit_sie_sync(struct kvm_vcpu *vcpu)
1046{
1047 s390_vcpu_block(vcpu);
1048 exit_sie(vcpu);
1049}
1050
2c70fe44
CB		 1051static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1052{
1053 int i;
1054 struct kvm *kvm = gmap->private;
1055 struct kvm_vcpu *vcpu;
1056
1057 kvm_for_each_vcpu(i, vcpu, kvm) {
1058 /* match against both prefix pages */
fda902cb 1059 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44
CB		 1060 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1061 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1062 exit_sie_sync(vcpu);
1063 }
1064 }
1065}
1066
b6d33834
CD		 1067int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1068{
1069 /* kvm common code refers to this, but never calls it */
1070 BUG();
1071 return 0;
1072}
1073
14eebd91
CO		 1074static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1075 struct kvm_one_reg *reg)
1076{
1077 int r = -EINVAL;
1078
1079 switch (reg->id) {
29b7c71b
CO		 1080 case KVM_REG_S390_TODPR:
1081 r = put_user(vcpu->arch.sie_block->todpr,
1082 (u32 __user *)reg->addr);
1083 break;
1084 case KVM_REG_S390_EPOCHDIFF:
1085 r = put_user(vcpu->arch.sie_block->epoch,
1086 (u64 __user *)reg->addr);
1087 break;
46a6dd1c
J		 1088 case KVM_REG_S390_CPU_TIMER:
1089 r = put_user(vcpu->arch.sie_block->cputm,
1090 (u64 __user *)reg->addr);
1091 break;
1092 case KVM_REG_S390_CLOCK_COMP:
1093 r = put_user(vcpu->arch.sie_block->ckc,
1094 (u64 __user *)reg->addr);
1095 break;
536336c2
DD		 1096 case KVM_REG_S390_PFTOKEN:
1097 r = put_user(vcpu->arch.pfault_token,
1098 (u64 __user *)reg->addr);
1099 break;
1100 case KVM_REG_S390_PFCOMPARE:
1101 r = put_user(vcpu->arch.pfault_compare,
1102 (u64 __user *)reg->addr);
1103 break;
1104 case KVM_REG_S390_PFSELECT:
1105 r = put_user(vcpu->arch.pfault_select,
1106 (u64 __user *)reg->addr);
1107 break;
672550fb
CB		 1108 case KVM_REG_S390_PP:
1109 r = put_user(vcpu->arch.sie_block->pp,
1110 (u64 __user *)reg->addr);
1111 break;
afa45ff5
CB		 1112 case KVM_REG_S390_GBEA:
1113 r = put_user(vcpu->arch.sie_block->gbea,
1114 (u64 __user *)reg->addr);
1115 break;
14eebd91
CO		 1116 default:
1117 break;
1118 }
1119
1120 return r;
1121}
1122
1123static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1124 struct kvm_one_reg *reg)
1125{
1126 int r = -EINVAL;
1127
1128 switch (reg->id) {
29b7c71b
CO		 1129 case KVM_REG_S390_TODPR:
1130 r = get_user(vcpu->arch.sie_block->todpr,
1131 (u32 __user *)reg->addr);
1132 break;
1133 case KVM_REG_S390_EPOCHDIFF:
1134 r = get_user(vcpu->arch.sie_block->epoch,
1135 (u64 __user *)reg->addr);
1136 break;
46a6dd1c
J		 1137 case KVM_REG_S390_CPU_TIMER:
1138 r = get_user(vcpu->arch.sie_block->cputm,
1139 (u64 __user *)reg->addr);
1140 break;
1141 case KVM_REG_S390_CLOCK_COMP:
1142 r = get_user(vcpu->arch.sie_block->ckc,
1143 (u64 __user *)reg->addr);
1144 break;
536336c2
DD		 1145 case KVM_REG_S390_PFTOKEN:
1146 r = get_user(vcpu->arch.pfault_token,
1147 (u64 __user *)reg->addr);
9fbd8082
DH		 1148 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1149 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 1150 break;
1151 case KVM_REG_S390_PFCOMPARE:
1152 r = get_user(vcpu->arch.pfault_compare,
1153 (u64 __user *)reg->addr);
1154 break;
1155 case KVM_REG_S390_PFSELECT:
1156 r = get_user(vcpu->arch.pfault_select,
1157 (u64 __user *)reg->addr);
1158 break;
672550fb
CB		 1159 case KVM_REG_S390_PP:
1160 r = get_user(vcpu->arch.sie_block->pp,
1161 (u64 __user *)reg->addr);
1162 break;
afa45ff5
CB		 1163 case KVM_REG_S390_GBEA:
1164 r = get_user(vcpu->arch.sie_block->gbea,
1165 (u64 __user *)reg->addr);
1166 break;
14eebd91
CO		 1167 default:
1168 break;
1169 }
1170
1171 return r;
1172}
b6d33834 1173
b0c632db
HC		 1174static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1175{
b0c632db 1176 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 1177 return 0;
1178}
1179
1180int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1181{
5a32c1af 1182 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 1183 return 0;
1184}
1185
1186int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1187{
5a32c1af 1188 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 1189 return 0;
1190}
1191
1192int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1193 struct kvm_sregs *sregs)
1194{
59674c1a 1195 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 1196 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 1197 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 1198 return 0;
1199}
1200
1201int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1202 struct kvm_sregs *sregs)
1203{
59674c1a 1204 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 1205 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 1206 return 0;
1207}
1208
1209int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1210{
4725c860
MS		 1211 if (test_fp_ctl(fpu->fpc))
1212 return -EINVAL;
b0c632db 1213 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860
MS		 1214 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1215 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1216 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
b0c632db
HC		 1217 return 0;
1218}
1219
1220int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1221{
b0c632db
HC		 1222 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1223 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 1224 return 0;
1225}
1226
1227static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1228{
1229 int rc = 0;
1230
7a42fdc2 1231 if (!is_vcpu_stopped(vcpu))
b0c632db 1232 rc = -EBUSY;
d7b0b5eb
CO		 1233 else {
1234 vcpu->run->psw_mask = psw.mask;
1235 vcpu->run->psw_addr = psw.addr;
1236 }
b0c632db
HC		 1237 return rc;
1238}
1239
1240int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1241 struct kvm_translation *tr)
1242{
1243 return -EINVAL; /* not implemented yet */
1244}
1245
27291e21
DH		 1246#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1247 KVM_GUESTDBG_USE_HW_BP | \
1248 KVM_GUESTDBG_ENABLE)
1249
d0bfb940
JK		 1250int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1251 struct kvm_guest_debug *dbg)
b0c632db 1252{
27291e21
DH		 1253 int rc = 0;
1254
1255 vcpu->guest_debug = 0;
1256 kvm_s390_clear_bp_data(vcpu);
1257
2de3bfc2 1258 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 1259 return -EINVAL;
1260
1261 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1262 vcpu->guest_debug = dbg->control;
1263 /* enforce guest PER */
1264 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1265
1266 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1267 rc = kvm_s390_import_bp_data(vcpu, dbg);
1268 } else {
1269 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1270 vcpu->arch.guestdbg.last_bp = 0;
1271 }
1272
1273 if (rc) {
1274 vcpu->guest_debug = 0;
1275 kvm_s390_clear_bp_data(vcpu);
1276 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1277 }
1278
1279 return rc;
b0c632db
HC		 1280}
1281
62d9f0db
MT		 1282int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1283 struct kvm_mp_state *mp_state)
1284{
6352e4d2
DH		 1285 /* CHECK_STOP and LOAD are not supported yet */
1286 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1287 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 1288}
1289
1290int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1291 struct kvm_mp_state *mp_state)
1292{
6352e4d2
DH		 1293 int rc = 0;
1294
1295 /* user space knows about this interface - let it control the state */
1296 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1297
1298 switch (mp_state->mp_state) {
1299 case KVM_MP_STATE_STOPPED:
1300 kvm_s390_vcpu_stop(vcpu);
1301 break;
1302 case KVM_MP_STATE_OPERATING:
1303 kvm_s390_vcpu_start(vcpu);
1304 break;
1305 case KVM_MP_STATE_LOAD:
1306 case KVM_MP_STATE_CHECK_STOP:
1307 /* fall through - CHECK_STOP and LOAD are not supported yet */
1308 default:
1309 rc = -ENXIO;
1310 }
1311
1312 return rc;
62d9f0db
MT		 1313}
1314
b31605c1
DD		 1315bool kvm_s390_cmma_enabled(struct kvm *kvm)
1316{
1317 if (!MACHINE_IS_LPAR)
1318 return false;
1319 /* only enable for z10 and later */
1320 if (!MACHINE_HAS_EDAT1)
1321 return false;
1322 if (!kvm->arch.use_cmma)
1323 return false;
1324 return true;
1325}
1326
8ad35755
DH		 1327static bool ibs_enabled(struct kvm_vcpu *vcpu)
1328{
1329 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1330}
1331
2c70fe44
CB		 1332static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1333{
8ad35755
DH		 1334retry:
1335 s390_vcpu_unblock(vcpu);
2c70fe44
CB		 1336 /*
1337 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1338 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1339 * This ensures that the ipte instruction for this request has
1340 * already finished. We might race against a second unmapper that
1341 * wants to set the blocking bit. Lets just retry the request loop.
1342 */
8ad35755 1343 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 1344 int rc;
1345 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 1346 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 1347 PAGE_SIZE * 2);
1348 if (rc)
1349 return rc;
8ad35755 1350 goto retry;
2c70fe44 1351 }
8ad35755 1352
d3d692c8
DH		 1353 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1354 vcpu->arch.sie_block->ihcpu = 0xffff;
1355 goto retry;
1356 }
1357
8ad35755
DH		 1358 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1359 if (!ibs_enabled(vcpu)) {
1360 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1361 atomic_set_mask(CPUSTAT_IBS,
1362 &vcpu->arch.sie_block->cpuflags);
1363 }
1364 goto retry;
2c70fe44 1365 }
8ad35755
DH		 1366
1367 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1368 if (ibs_enabled(vcpu)) {
1369 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1370 atomic_clear_mask(CPUSTAT_IBS,
1371 &vcpu->arch.sie_block->cpuflags);
1372 }
1373 goto retry;
1374 }
1375
0759d068
DH		 1376 /* nothing to do, just clear the request */
1377 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1378
2c70fe44
CB		 1379 return 0;
1380}
1381
fa576c58
TH		 1382/**
1383 * kvm_arch_fault_in_page - fault-in guest page if necessary
1384 * @vcpu: The corresponding virtual cpu
1385 * @gpa: Guest physical address
1386 * @writable: Whether the page should be writable or not
1387 *
1388 * Make sure that a guest page has been faulted-in on the host.
1389 *
1390 * Return: Zero on success, negative error code otherwise.
1391 */
1392long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 1393{
527e30b4
MS		 1394 return gmap_fault(vcpu->arch.gmap, gpa,
1395 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 1396}
1397
3c038e6b
DD		 1398static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1399 unsigned long token)
1400{
1401 struct kvm_s390_interrupt inti;
383d0b05 1402 struct kvm_s390_irq irq;
3c038e6b
DD		 1403
1404 if (start_token) {
383d0b05
JF		 1405 irq.u.ext.ext_params2 = token;
1406 irq.type = KVM_S390_INT_PFAULT_INIT;
1407 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 1408 } else {
1409 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 1410 inti.parm64 = token;
3c038e6b
DD		 1411 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1412 }
1413}
1414
1415void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1416 struct kvm_async_pf *work)
1417{
1418 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1419 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1420}
1421
1422void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1423 struct kvm_async_pf *work)
1424{
1425 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1426 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1427}
1428
1429void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1430 struct kvm_async_pf *work)
1431{
1432 /* s390 will always inject the page directly */
1433}
1434
1435bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1436{
1437 /*
1438 * s390 will always inject the page directly,
1439 * but we still want check_async_completion to cleanup
1440 */
1441 return true;
1442}
1443
1444static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1445{
1446 hva_t hva;
1447 struct kvm_arch_async_pf arch;
1448 int rc;
1449
1450 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1451 return 0;
1452 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1453 vcpu->arch.pfault_compare)
1454 return 0;
1455 if (psw_extint_disabled(vcpu))
1456 return 0;
9a022067 1457 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 1458 return 0;
1459 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1460 return 0;
1461 if (!vcpu->arch.gmap->pfault_enabled)
1462 return 0;
1463
81480cc1
HC		 1464 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1465 hva += current->thread.gmap_addr & ~PAGE_MASK;
1466 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 1467 return 0;
1468
1469 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1470 return rc;
1471}
1472
3fb4c40f 1473static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 1474{
3fb4c40f 1475 int rc, cpuflags;
e168bf8d 1476
3c038e6b
DD		 1477 /*
1478 * On s390 notifications for arriving pages will be delivered directly
1479 * to the guest but the house keeping for completed pfaults is
1480 * handled outside the worker.
1481 */
1482 kvm_check_async_pf_completion(vcpu);
1483
5a32c1af 1484 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
b0c632db
HC		 1485
1486 if (need_resched())
1487 schedule();
1488
d3a73acb 1489 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 1490 s390_handle_mcck();
1491
79395031
JF		 1492 if (!kvm_is_ucontrol(vcpu->kvm)) {
1493 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1494 if (rc)
1495 return rc;
1496 }
0ff31867 1497
2c70fe44
CB		 1498 rc = kvm_s390_handle_requests(vcpu);
1499 if (rc)
1500 return rc;
1501
27291e21
DH		 1502 if (guestdbg_enabled(vcpu)) {
1503 kvm_s390_backup_guest_per_regs(vcpu);
1504 kvm_s390_patch_guest_per_regs(vcpu);
1505 }
1506
b0c632db 1507 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 1508 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1509 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1510 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 1511
3fb4c40f
TH		 1512 return 0;
1513}
1514
1515static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1516{
24eb3a82 1517 int rc = -1;
2b29a9fd
DD		 1518
1519 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1520 vcpu->arch.sie_block->icptcode);
1521 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1522
27291e21
DH		 1523 if (guestdbg_enabled(vcpu))
1524 kvm_s390_restore_guest_per_regs(vcpu);
1525
3fb4c40f 1526 if (exit_reason >= 0) {
7c470539 1527 rc = 0;
210b1607
TH		 1528 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1529 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1530 vcpu->run->s390_ucontrol.trans_exc_code =
1531 current->thread.gmap_addr;
1532 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1533 rc = -EREMOTE;
24eb3a82
DD		 1534
1535 } else if (current->thread.gmap_pfault) {
3c038e6b 1536 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 1537 current->thread.gmap_pfault = 0;
fa576c58 1538 if (kvm_arch_setup_async_pf(vcpu)) {
24eb3a82 1539 rc = 0;
fa576c58
TH		 1540 } else {
1541 gpa_t gpa = current->thread.gmap_addr;
1542 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1543 }
24eb3a82
DD		 1544 }
1545
1546 if (rc == -1) {
699bde3b
CB		 1547 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1548 trace_kvm_s390_sie_fault(vcpu);
1549 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1f0d0f09 1550 }
b0c632db 1551
5a32c1af 1552 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
3fb4c40f 1553
a76ccff6
TH		 1554 if (rc == 0) {
1555 if (kvm_is_ucontrol(vcpu->kvm))
2955c83f
CB		 1556 /* Don't exit for host interrupts. */
1557 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
a76ccff6
TH		 1558 else
1559 rc = kvm_handle_sie_intercept(vcpu);
1560 }
1561
3fb4c40f
TH		 1562 return rc;
1563}
1564
1565static int __vcpu_run(struct kvm_vcpu *vcpu)
1566{
1567 int rc, exit_reason;
1568
800c1065
TH		 1569 /*
1570 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1571 * ning the guest), so that memslots (and other stuff) are protected
1572 */
1573 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1574
a76ccff6
TH		 1575 do {
1576 rc = vcpu_pre_run(vcpu);
1577 if (rc)
1578 break;
3fb4c40f 1579
800c1065 1580 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 1581 /*
1582 * As PF_VCPU will be used in fault handler, between
1583 * guest_enter and guest_exit should be no uaccess.
1584 */
1585 preempt_disable();
1586 kvm_guest_enter();
1587 preempt_enable();
1588 exit_reason = sie64a(vcpu->arch.sie_block,
1589 vcpu->run->s.regs.gprs);
1590 kvm_guest_exit();
800c1065 1591 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 1592
1593 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 1594 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 1595
800c1065 1596 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 1597 return rc;
b0c632db
HC		 1598}
1599
b028ee3e
DH		 1600static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1601{
1602 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1603 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1604 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1605 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1606 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1607 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 1608 /* some control register changes require a tlb flush */
1609 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 1610 }
1611 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1612 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1613 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1614 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1615 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1616 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1617 }
1618 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1619 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1620 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1621 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 1622 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1623 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 1624 }
1625 kvm_run->kvm_dirty_regs = 0;
1626}
1627
1628static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1629{
1630 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1631 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1632 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1633 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1634 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1635 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1636 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1637 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1638 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1639 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1640 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1641 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1642}
1643
b0c632db
HC		 1644int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1645{
8f2abe6a 1646 int rc;
b0c632db
HC		 1647 sigset_t sigsaved;
1648
27291e21
DH		 1649 if (guestdbg_exit_pending(vcpu)) {
1650 kvm_s390_prepare_debug_exit(vcpu);
1651 return 0;
1652 }
1653
b0c632db
HC		 1654 if (vcpu->sigset_active)
1655 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1656
6352e4d2
DH		 1657 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1658 kvm_s390_vcpu_start(vcpu);
1659 } else if (is_vcpu_stopped(vcpu)) {
1660 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1661 vcpu->vcpu_id);
1662 return -EINVAL;
1663 }
b0c632db 1664
b028ee3e 1665 sync_regs(vcpu, kvm_run);
d7b0b5eb 1666
dab4079d 1667 might_fault();
a76ccff6 1668 rc = __vcpu_run(vcpu);
9ace903d 1669
b1d16c49
CE		 1670 if (signal_pending(current) && !rc) {
1671 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 1672 rc = -EINTR;
b1d16c49 1673 }
8f2abe6a 1674
27291e21
DH		 1675 if (guestdbg_exit_pending(vcpu) && !rc) {
1676 kvm_s390_prepare_debug_exit(vcpu);
1677 rc = 0;
1678 }
1679
b8e660b8 1680 if (rc == -EOPNOTSUPP) {
8f2abe6a
CB		 1681 /* intercept cannot be handled in-kernel, prepare kvm-run */
1682 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1683 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
8f2abe6a
CB		 1684 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1685 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1686 rc = 0;
1687 }
1688
1689 if (rc == -EREMOTE) {
1690 /* intercept was handled, but userspace support is needed
1691 * kvm_run has been prepared by the handler */
1692 rc = 0;
1693 }
b0c632db 1694
b028ee3e 1695 store_regs(vcpu, kvm_run);
d7b0b5eb 1696
b0c632db
HC		 1697 if (vcpu->sigset_active)
1698 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1699
b0c632db 1700 vcpu->stat.exit_userspace++;
7e8e6ab4 1701 return rc;
b0c632db
HC		 1702}
1703
b0c632db
HC		 1704/*
1705 * store status at address
1706 * we use have two special cases:
1707 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1708 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1709 */
d0bce605 1710int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 1711{
092670cd 1712 unsigned char archmode = 1;
fda902cb 1713 unsigned int px;
178bd789 1714 u64 clkcomp;
d0bce605 1715 int rc;
b0c632db 1716
d0bce605
HC		 1717 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1718 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 1719 return -EFAULT;
d0bce605
HC		 1720 gpa = SAVE_AREA_BASE;
1721 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1722 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 1723 return -EFAULT;
d0bce605
HC		 1724 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1725 }
1726 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1727 vcpu->arch.guest_fpregs.fprs, 128);
1728 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1729 vcpu->run->s.regs.gprs, 128);
1730 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1731 &vcpu->arch.sie_block->gpsw, 16);
fda902cb 1732 px = kvm_s390_get_prefix(vcpu);
d0bce605 1733 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
fda902cb 1734 &px, 4);
d0bce605
HC		 1735 rc |= write_guest_abs(vcpu,
1736 gpa + offsetof(struct save_area, fp_ctrl_reg),
1737 &vcpu->arch.guest_fpregs.fpc, 4);
1738 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1739 &vcpu->arch.sie_block->todpr, 4);
1740 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1741 &vcpu->arch.sie_block->cputm, 8);
178bd789 1742 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 1743 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1744 &clkcomp, 8);
1745 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1746 &vcpu->run->s.regs.acrs, 64);
1747 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1748 &vcpu->arch.sie_block->gcr, 128);
1749 return rc ? -EFAULT : 0;
b0c632db
HC		 1750}
1751
e879892c
TH		 1752int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1753{
1754 /*
1755 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1756 * copying in vcpu load/put. Lets update our copies before we save
1757 * it into the save area
1758 */
1759 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1760 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1761 save_access_regs(vcpu->run->s.regs.acrs);
1762
1763 return kvm_s390_store_status_unloaded(vcpu, addr);
1764}
1765
8ad35755
DH		 1766static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1767{
1768 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1769 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1770 exit_sie_sync(vcpu);
1771}
1772
1773static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1774{
1775 unsigned int i;
1776 struct kvm_vcpu *vcpu;
1777
1778 kvm_for_each_vcpu(i, vcpu, kvm) {
1779 __disable_ibs_on_vcpu(vcpu);
1780 }
1781}
1782
1783static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1784{
1785 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1786 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1787 exit_sie_sync(vcpu);
1788}
1789
6852d7b6
DH		 1790void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1791{
8ad35755
DH		 1792 int i, online_vcpus, started_vcpus = 0;
1793
1794 if (!is_vcpu_stopped(vcpu))
1795 return;
1796
6852d7b6 1797 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 1798 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1799 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1800 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1801
1802 for (i = 0; i < online_vcpus; i++) {
1803 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1804 started_vcpus++;
1805 }
1806
1807 if (started_vcpus == 0) {
1808 /* we're the only active VCPU -> speed it up */
1809 __enable_ibs_on_vcpu(vcpu);
1810 } else if (started_vcpus == 1) {
1811 /*
1812 * As we are starting a second VCPU, we have to disable
1813 * the IBS facility on all VCPUs to remove potentially
1814 * oustanding ENABLE requests.
1815 */
1816 __disable_ibs_on_all_vcpus(vcpu->kvm);
1817 }
1818
6852d7b6 1819 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 1820 /*
1821 * Another VCPU might have used IBS while we were offline.
1822 * Let's play safe and flush the VCPU at startup.
1823 */
d3d692c8 1824 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 1825 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1826 return;
6852d7b6
DH		 1827}
1828
1829void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1830{
8ad35755
DH		 1831 int i, online_vcpus, started_vcpus = 0;
1832 struct kvm_vcpu *started_vcpu = NULL;
1833
1834 if (is_vcpu_stopped(vcpu))
1835 return;
1836
6852d7b6 1837 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 1838 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1839 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1840 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1841
32f5ff63 1842 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 1843 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 1844
6cddd432 1845 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 1846 __disable_ibs_on_vcpu(vcpu);
1847
1848 for (i = 0; i < online_vcpus; i++) {
1849 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1850 started_vcpus++;
1851 started_vcpu = vcpu->kvm->vcpus[i];
1852 }
1853 }
1854
1855 if (started_vcpus == 1) {
1856 /*
1857 * As we only have one VCPU left, we want to enable the
1858 * IBS facility for that VCPU to speed it up.
1859 */
1860 __enable_ibs_on_vcpu(started_vcpu);
1861 }
1862
433b9ee4 1863 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1864 return;
6852d7b6
DH		 1865}
1866
d6712df9
CH		 1867static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1868 struct kvm_enable_cap *cap)
1869{
1870 int r;
1871
1872 if (cap->flags)
1873 return -EINVAL;
1874
1875 switch (cap->cap) {
fa6b7fe9
CH		 1876 case KVM_CAP_S390_CSS_SUPPORT:
1877 if (!vcpu->kvm->arch.css_support) {
1878 vcpu->kvm->arch.css_support = 1;
1879 trace_kvm_s390_enable_css(vcpu->kvm);
1880 }
1881 r = 0;
1882 break;
d6712df9
CH		 1883 default:
1884 r = -EINVAL;
1885 break;
1886 }
1887 return r;
1888}
1889
b0c632db
HC		 1890long kvm_arch_vcpu_ioctl(struct file *filp,
1891 unsigned int ioctl, unsigned long arg)
1892{
1893 struct kvm_vcpu *vcpu = filp->private_data;
1894 void __user *argp = (void __user *)arg;
800c1065 1895 int idx;
bc923cc9 1896 long r;
b0c632db 1897
93736624
AK		 1898 switch (ioctl) {
1899 case KVM_S390_INTERRUPT: {
ba5c1e9b 1900 struct kvm_s390_interrupt s390int;
383d0b05 1901 struct kvm_s390_irq s390irq;
ba5c1e9b 1902
93736624 1903 r = -EFAULT;
ba5c1e9b 1904 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 1905 break;
383d0b05
JF		 1906 if (s390int_to_s390irq(&s390int, &s390irq))
1907 return -EINVAL;
1908 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 1909 break;
ba5c1e9b 1910 }
b0c632db 1911 case KVM_S390_STORE_STATUS:
800c1065 1912 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 1913 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 1914 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 1915 break;
b0c632db
HC		 1916 case KVM_S390_SET_INITIAL_PSW: {
1917 psw_t psw;
1918
bc923cc9 1919 r = -EFAULT;
b0c632db 1920 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 1921 break;
1922 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1923 break;
b0c632db
HC		 1924 }
1925 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 1926 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1927 break;
14eebd91
CO		 1928 case KVM_SET_ONE_REG:
1929 case KVM_GET_ONE_REG: {
1930 struct kvm_one_reg reg;
1931 r = -EFAULT;
1932 if (copy_from_user(&reg, argp, sizeof(reg)))
1933 break;
1934 if (ioctl == KVM_SET_ONE_REG)
1935 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1936 else
1937 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1938 break;
1939 }
27e0393f
CO		 1940#ifdef CONFIG_KVM_S390_UCONTROL
1941 case KVM_S390_UCAS_MAP: {
1942 struct kvm_s390_ucas_mapping ucasmap;
1943
1944 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1945 r = -EFAULT;
1946 break;
1947 }
1948
1949 if (!kvm_is_ucontrol(vcpu->kvm)) {
1950 r = -EINVAL;
1951 break;
1952 }
1953
1954 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1955 ucasmap.vcpu_addr, ucasmap.length);
1956 break;
1957 }
1958 case KVM_S390_UCAS_UNMAP: {
1959 struct kvm_s390_ucas_mapping ucasmap;
1960
1961 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1962 r = -EFAULT;
1963 break;
1964 }
1965
1966 if (!kvm_is_ucontrol(vcpu->kvm)) {
1967 r = -EINVAL;
1968 break;
1969 }
1970
1971 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1972 ucasmap.length);
1973 break;
1974 }
1975#endif
ccc7910f 1976 case KVM_S390_VCPU_FAULT: {
527e30b4 1977 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 1978 break;
1979 }
d6712df9
CH		 1980 case KVM_ENABLE_CAP:
1981 {
1982 struct kvm_enable_cap cap;
1983 r = -EFAULT;
1984 if (copy_from_user(&cap, argp, sizeof(cap)))
1985 break;
1986 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1987 break;
1988 }
b0c632db 1989 default:
3e6afcf1 1990 r = -ENOTTY;
b0c632db 1991 }
bc923cc9 1992 return r;
b0c632db
HC		 1993}
1994
5b1c1493
CO		 1995int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1996{
1997#ifdef CONFIG_KVM_S390_UCONTROL
1998 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1999 && (kvm_is_ucontrol(vcpu->kvm))) {
2000 vmf->page = virt_to_page(vcpu->arch.sie_block);
2001 get_page(vmf->page);
2002 return 0;
2003 }
2004#endif
2005 return VM_FAULT_SIGBUS;
2006}
2007
5587027c
AK		 2008int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2009 unsigned long npages)
db3fe4eb
TY		 2010{
2011 return 0;
2012}
2013
b0c632db 2014/* Section: memory related */
f7784b8e
MT		 2015int kvm_arch_prepare_memory_region(struct kvm *kvm,
2016 struct kvm_memory_slot *memslot,
7b6195a9
TY		 2017 struct kvm_userspace_memory_region *mem,
2018 enum kvm_mr_change change)
b0c632db 2019{
dd2887e7
NW		 2020 /* A few sanity checks. We can have memory slots which have to be
2021 located/ended at a segment boundary (1MB). The memory in userland is
2022 ok to be fragmented into various different vmas. It is okay to mmap()
2023 and munmap() stuff in this slot after doing this call at any time */
b0c632db 2024
598841ca 2025 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 2026 return -EINVAL;
2027
598841ca 2028 if (mem->memory_size & 0xffffful)
b0c632db
HC		 2029 return -EINVAL;
2030
f7784b8e
MT		 2031 return 0;
2032}
2033
2034void kvm_arch_commit_memory_region(struct kvm *kvm,
2035 struct kvm_userspace_memory_region *mem,
8482644a
TY		 2036 const struct kvm_memory_slot *old,
2037 enum kvm_mr_change change)
f7784b8e 2038{
f7850c92 2039 int rc;
f7784b8e 2040
2cef4deb
CB		 2041 /* If the basics of the memslot do not change, we do not want
2042 * to update the gmap. Every update causes several unnecessary
2043 * segment translation exceptions. This is usually handled just
2044 * fine by the normal fault handler + gmap, but it will also
2045 * cause faults on the prefix page of running guest CPUs.
2046 */
2047 if (old->userspace_addr == mem->userspace_addr &&
2048 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2049 old->npages * PAGE_SIZE == mem->memory_size)
2050 return;
598841ca
CO		 2051
2052 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2053 mem->guest_phys_addr, mem->memory_size);
2054 if (rc)
f7850c92 2055 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
598841ca 2056 return;
b0c632db
HC		 2057}
2058
b0c632db
HC		 2059static int __init kvm_s390_init(void)
2060{
ef50f7ac 2061 int ret;
0ee75bea 2062 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
ef50f7ac
CB		 2063 if (ret)
2064 return ret;
2065
2066 /*
2067 * guests can ask for up to 255+1 double words, we need a full page
25985edc 2068 * to hold the maximum amount of facilities. On the other hand, we
ef50f7ac
CB		 2069 * only set facilities that are known to work in KVM.
2070 */
78c4b59f
MM		 2071 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
2072 if (!vfacilities) {
ef50f7ac
CB		 2073 kvm_exit();
2074 return -ENOMEM;
2075 }
78c4b59f 2076 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
c23f397c 2077 vfacilities[0] &= 0xff82fffbf4fc2000UL;
7feb6bb8 2078 vfacilities[1] &= 0x005c000000000000UL;
ef50f7ac 2079 return 0;
b0c632db
HC		 2080}
2081
2082static void __exit kvm_s390_exit(void)
2083{
78c4b59f 2084 free_page((unsigned long) vfacilities);
b0c632db
HC		 2085 kvm_exit();
2086}
2087
2088module_init(kvm_s390_init);
2089module_exit(kvm_s390_exit);
566af940
CH		 2090
2091/*
2092 * Enable autoloading of the kvm module.
2093 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2094 * since x86 takes a different approach.
2095 */
2096#include <linux/miscdevice.h>
2097MODULE_ALIAS_MISCDEV(KVM_MINOR);
2098MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)