projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: trace correct values for set prefix and machine checks
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
25#include <linux/slab.h>
ba5c1e9b 26#include <linux/timer.h>
cbb870c8 27#include <asm/asm-offsets.h>
b0c632db
HC		 28#include <asm/lowcore.h>
29#include <asm/pgtable.h>
f5daba1d 30#include <asm/nmi.h>
a0616cde 31#include <asm/switch_to.h>
78c4b59f 32#include <asm/facility.h>
1526bf9c 33#include <asm/sclp.h>
8f2abe6a 34#include "kvm-s390.h"
b0c632db
HC		 35#include "gaccess.h"
36
5786fffa
CH		 37#define CREATE_TRACE_POINTS
38#include "trace.h"
ade38c31 39#include "trace-s390.h"
5786fffa 40
b0c632db
HC		 41#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 45 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
ce2e4f0b 53 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
f5e10b09 54 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 55 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 56 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
57 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 58 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 59 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 60 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
61 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
62 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
63 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
64 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
65 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
66 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 67 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 68 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
69 { "instruction_spx", VCPU_STAT(instruction_spx) },
70 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
71 { "instruction_stap", VCPU_STAT(instruction_stap) },
72 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 73 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 74 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
75 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 76 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 77 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
78 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 79 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 80 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 81 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 82 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0 83 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
42cb0c9f
DH		 84 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
85 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
5288fbf0 86 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
42cb0c9f
DH		 87 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
88 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
5288fbf0
CB		 89 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
90 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
91 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
42cb0c9f
DH		 92 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
93 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
94 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
388186bc 95 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 96 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 97 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
b0c632db
HC		 98 { NULL }
99};
100
78c4b59f 101unsigned long *vfacilities;
2c70fe44 102static struct gmap_notifier gmap_notifier;
b0c632db 103
78c4b59f 104/* test availability of vfacility */
280ef0f1 105int test_vfacility(unsigned long nr)
78c4b59f
MM		 106{
107 return __test_facility(nr, (void *) vfacilities);
108}
109
b0c632db 110/* Section: not file related */
13a34e06 111int kvm_arch_hardware_enable(void)
b0c632db
HC		 112{
113 /* every s390 is virtualization enabled ;-) */
10474ae8 114 return 0;
b0c632db
HC		 115}
116
2c70fe44
CB		 117static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
118
b0c632db
HC		 119int kvm_arch_hardware_setup(void)
120{
2c70fe44
CB		 121 gmap_notifier.notifier_call = kvm_gmap_notifier;
122 gmap_register_ipte_notifier(&gmap_notifier);
b0c632db
HC		 123 return 0;
124}
125
126void kvm_arch_hardware_unsetup(void)
127{
2c70fe44 128 gmap_unregister_ipte_notifier(&gmap_notifier);
b0c632db
HC		 129}
130
b0c632db
HC		 131int kvm_arch_init(void *opaque)
132{
84877d93
CH		 133 /* Register floating interrupt controller interface. */
134 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
b0c632db
HC		 135}
136
b0c632db
HC		 137/* Section: device related */
138long kvm_arch_dev_ioctl(struct file *filp,
139 unsigned int ioctl, unsigned long arg)
140{
141 if (ioctl == KVM_S390_ENABLE_SIE)
142 return s390_enable_sie();
143 return -EINVAL;
144}
145
784aa3d7 146int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
b0c632db 147{
d7b0b5eb
CO		 148 int r;
149
2bd0ac4e 150 switch (ext) {
d7b0b5eb 151 case KVM_CAP_S390_PSW:
b6cf8788 152 case KVM_CAP_S390_GMAP:
52e16b18 153 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 154#ifdef CONFIG_KVM_S390_UCONTROL
155 case KVM_CAP_S390_UCONTROL:
156#endif
3c038e6b 157 case KVM_CAP_ASYNC_PF:
60b413c9 158 case KVM_CAP_SYNC_REGS:
14eebd91 159 case KVM_CAP_ONE_REG:
d6712df9 160 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 161 case KVM_CAP_S390_CSS_SUPPORT:
ebc32262 162 case KVM_CAP_IRQFD:
10ccaa1e 163 case KVM_CAP_IOEVENTFD:
c05c4186 164 case KVM_CAP_DEVICE_CTRL:
d938dc55 165 case KVM_CAP_ENABLE_CAP_VM:
78599d90 166 case KVM_CAP_S390_IRQCHIP:
f2061656 167 case KVM_CAP_VM_ATTRIBUTES:
6352e4d2 168 case KVM_CAP_MP_STATE:
2444b352 169 case KVM_CAP_S390_USER_SIGP:
d7b0b5eb
CO		 170 r = 1;
171 break;
e726b1bd
CB		 172 case KVM_CAP_NR_VCPUS:
173 case KVM_CAP_MAX_VCPUS:
174 r = KVM_MAX_VCPUS;
175 break;
e1e2e605
NW		 176 case KVM_CAP_NR_MEMSLOTS:
177 r = KVM_USER_MEM_SLOTS;
178 break;
1526bf9c 179 case KVM_CAP_S390_COW:
abf09bed 180 r = MACHINE_HAS_ESOP;
1526bf9c 181 break;
2bd0ac4e 182 default:
d7b0b5eb 183 r = 0;
2bd0ac4e 184 }
d7b0b5eb 185 return r;
b0c632db
HC		 186}
187
15f36ebd
JH		 188static void kvm_s390_sync_dirty_log(struct kvm *kvm,
189 struct kvm_memory_slot *memslot)
190{
191 gfn_t cur_gfn, last_gfn;
192 unsigned long address;
193 struct gmap *gmap = kvm->arch.gmap;
194
195 down_read(&gmap->mm->mmap_sem);
196 /* Loop over all guest pages */
197 last_gfn = memslot->base_gfn + memslot->npages;
198 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
199 address = gfn_to_hva_memslot(memslot, cur_gfn);
200
201 if (gmap_test_and_clear_dirty(address, gmap))
202 mark_page_dirty(kvm, cur_gfn);
203 }
204 up_read(&gmap->mm->mmap_sem);
205}
206
b0c632db
HC		 207/* Section: vm related */
208/*
209 * Get (and clear) the dirty memory log for a memory slot.
210 */
211int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
212 struct kvm_dirty_log *log)
213{
15f36ebd
JH		 214 int r;
215 unsigned long n;
216 struct kvm_memory_slot *memslot;
217 int is_dirty = 0;
218
219 mutex_lock(&kvm->slots_lock);
220
221 r = -EINVAL;
222 if (log->slot >= KVM_USER_MEM_SLOTS)
223 goto out;
224
225 memslot = id_to_memslot(kvm->memslots, log->slot);
226 r = -ENOENT;
227 if (!memslot->dirty_bitmap)
228 goto out;
229
230 kvm_s390_sync_dirty_log(kvm, memslot);
231 r = kvm_get_dirty_log(kvm, log, &is_dirty);
232 if (r)
233 goto out;
234
235 /* Clear the dirty log */
236 if (is_dirty) {
237 n = kvm_dirty_bitmap_bytes(memslot);
238 memset(memslot->dirty_bitmap, 0, n);
239 }
240 r = 0;
241out:
242 mutex_unlock(&kvm->slots_lock);
243 return r;
b0c632db
HC		 244}
245
d938dc55
CH		 246static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
247{
248 int r;
249
250 if (cap->flags)
251 return -EINVAL;
252
253 switch (cap->cap) {
84223598
CH		 254 case KVM_CAP_S390_IRQCHIP:
255 kvm->arch.use_irqchip = 1;
256 r = 0;
257 break;
2444b352
DH		 258 case KVM_CAP_S390_USER_SIGP:
259 kvm->arch.user_sigp = 1;
260 r = 0;
261 break;
d938dc55
CH		 262 default:
263 r = -EINVAL;
264 break;
265 }
266 return r;
267}
268
8c0a7ce6
DD		 269static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
270{
271 int ret;
272
273 switch (attr->attr) {
274 case KVM_S390_VM_MEM_LIMIT_SIZE:
275 ret = 0;
276 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
277 ret = -EFAULT;
278 break;
279 default:
280 ret = -ENXIO;
281 break;
282 }
283 return ret;
284}
285
286static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
4f718eab
DD		 287{
288 int ret;
289 unsigned int idx;
290 switch (attr->attr) {
291 case KVM_S390_VM_MEM_ENABLE_CMMA:
292 ret = -EBUSY;
293 mutex_lock(&kvm->lock);
294 if (atomic_read(&kvm->online_vcpus) == 0) {
295 kvm->arch.use_cmma = 1;
296 ret = 0;
297 }
298 mutex_unlock(&kvm->lock);
299 break;
300 case KVM_S390_VM_MEM_CLR_CMMA:
301 mutex_lock(&kvm->lock);
302 idx = srcu_read_lock(&kvm->srcu);
a13cff31 303 s390_reset_cmma(kvm->arch.gmap->mm);
4f718eab
DD		 304 srcu_read_unlock(&kvm->srcu, idx);
305 mutex_unlock(&kvm->lock);
306 ret = 0;
307 break;
8c0a7ce6
DD		 308 case KVM_S390_VM_MEM_LIMIT_SIZE: {
309 unsigned long new_limit;
310
311 if (kvm_is_ucontrol(kvm))
312 return -EINVAL;
313
314 if (get_user(new_limit, (u64 __user *)attr->addr))
315 return -EFAULT;
316
317 if (new_limit > kvm->arch.gmap->asce_end)
318 return -E2BIG;
319
320 ret = -EBUSY;
321 mutex_lock(&kvm->lock);
322 if (atomic_read(&kvm->online_vcpus) == 0) {
323 /* gmap_alloc will round the limit up */
324 struct gmap *new = gmap_alloc(current->mm, new_limit);
325
326 if (!new) {
327 ret = -ENOMEM;
328 } else {
329 gmap_free(kvm->arch.gmap);
330 new->private = kvm;
331 kvm->arch.gmap = new;
332 ret = 0;
333 }
334 }
335 mutex_unlock(&kvm->lock);
336 break;
337 }
4f718eab
DD		 338 default:
339 ret = -ENXIO;
340 break;
341 }
342 return ret;
343}
344
f2061656
DD		 345static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
346{
347 int ret;
348
349 switch (attr->group) {
4f718eab 350 case KVM_S390_VM_MEM_CTRL:
8c0a7ce6 351 ret = kvm_s390_set_mem_control(kvm, attr);
4f718eab 352 break;
f2061656
DD		 353 default:
354 ret = -ENXIO;
355 break;
356 }
357
358 return ret;
359}
360
361static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
362{
8c0a7ce6
DD		 363 int ret;
364
365 switch (attr->group) {
366 case KVM_S390_VM_MEM_CTRL:
367 ret = kvm_s390_get_mem_control(kvm, attr);
368 break;
369 default:
370 ret = -ENXIO;
371 break;
372 }
373
374 return ret;
f2061656
DD		 375}
376
377static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
378{
379 int ret;
380
381 switch (attr->group) {
4f718eab
DD		 382 case KVM_S390_VM_MEM_CTRL:
383 switch (attr->attr) {
384 case KVM_S390_VM_MEM_ENABLE_CMMA:
385 case KVM_S390_VM_MEM_CLR_CMMA:
8c0a7ce6 386 case KVM_S390_VM_MEM_LIMIT_SIZE:
4f718eab
DD		 387 ret = 0;
388 break;
389 default:
390 ret = -ENXIO;
391 break;
392 }
393 break;
f2061656
DD		 394 default:
395 ret = -ENXIO;
396 break;
397 }
398
399 return ret;
400}
401
b0c632db
HC		 402long kvm_arch_vm_ioctl(struct file *filp,
403 unsigned int ioctl, unsigned long arg)
404{
405 struct kvm *kvm = filp->private_data;
406 void __user *argp = (void __user *)arg;
f2061656 407 struct kvm_device_attr attr;
b0c632db
HC		 408 int r;
409
410 switch (ioctl) {
ba5c1e9b
CO		 411 case KVM_S390_INTERRUPT: {
412 struct kvm_s390_interrupt s390int;
413
414 r = -EFAULT;
415 if (copy_from_user(&s390int, argp, sizeof(s390int)))
416 break;
417 r = kvm_s390_inject_vm(kvm, &s390int);
418 break;
419 }
d938dc55
CH		 420 case KVM_ENABLE_CAP: {
421 struct kvm_enable_cap cap;
422 r = -EFAULT;
423 if (copy_from_user(&cap, argp, sizeof(cap)))
424 break;
425 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
426 break;
427 }
84223598
CH		 428 case KVM_CREATE_IRQCHIP: {
429 struct kvm_irq_routing_entry routing;
430
431 r = -EINVAL;
432 if (kvm->arch.use_irqchip) {
433 /* Set up dummy routing. */
434 memset(&routing, 0, sizeof(routing));
435 kvm_set_irq_routing(kvm, &routing, 0, 0);
436 r = 0;
437 }
438 break;
439 }
f2061656
DD		 440 case KVM_SET_DEVICE_ATTR: {
441 r = -EFAULT;
442 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
443 break;
444 r = kvm_s390_vm_set_attr(kvm, &attr);
445 break;
446 }
447 case KVM_GET_DEVICE_ATTR: {
448 r = -EFAULT;
449 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
450 break;
451 r = kvm_s390_vm_get_attr(kvm, &attr);
452 break;
453 }
454 case KVM_HAS_DEVICE_ATTR: {
455 r = -EFAULT;
456 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
457 break;
458 r = kvm_s390_vm_has_attr(kvm, &attr);
459 break;
460 }
b0c632db 461 default:
367e1319 462 r = -ENOTTY;
b0c632db
HC		 463 }
464
465 return r;
466}
467
5102ee87
TK		 468static int kvm_s390_crypto_init(struct kvm *kvm)
469{
470 if (!test_vfacility(76))
471 return 0;
472
473 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
474 GFP_KERNEL | GFP_DMA);
475 if (!kvm->arch.crypto.crycb)
476 return -ENOMEM;
477
478 kvm->arch.crypto.crycbd = (__u32) (unsigned long) kvm->arch.crypto.crycb |
479 CRYCB_FORMAT1;
480
481 return 0;
482}
483
e08b9637 484int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 485{
b0c632db
HC		 486 int rc;
487 char debug_name[16];
f6c137ff 488 static unsigned long sca_offset;
b0c632db 489
e08b9637
CO		 490 rc = -EINVAL;
491#ifdef CONFIG_KVM_S390_UCONTROL
492 if (type & ~KVM_VM_S390_UCONTROL)
493 goto out_err;
494 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
495 goto out_err;
496#else
497 if (type)
498 goto out_err;
499#endif
500
b0c632db
HC		 501 rc = s390_enable_sie();
502 if (rc)
d89f5eff 503 goto out_err;
b0c632db 504
b290411a
CO		 505 rc = -ENOMEM;
506
b0c632db
HC		 507 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
508 if (!kvm->arch.sca)
d89f5eff 509 goto out_err;
f6c137ff
CB		 510 spin_lock(&kvm_lock);
511 sca_offset = (sca_offset + 16) & 0x7f0;
512 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
513 spin_unlock(&kvm_lock);
b0c632db
HC		 514
515 sprintf(debug_name, "kvm-%u", current->pid);
516
517 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
518 if (!kvm->arch.dbf)
519 goto out_nodbf;
520
5102ee87
TK		 521 if (kvm_s390_crypto_init(kvm) < 0)
522 goto out_crypto;
523
ba5c1e9b
CO		 524 spin_lock_init(&kvm->arch.float_int.lock);
525 INIT_LIST_HEAD(&kvm->arch.float_int.list);
8a242234 526 init_waitqueue_head(&kvm->arch.ipte_wq);
a6b7e459 527 mutex_init(&kvm->arch.ipte_mutex);
ba5c1e9b 528
b0c632db
HC		 529 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
530 VM_EVENT(kvm, 3, "%s", "vm created");
531
e08b9637
CO		 532 if (type & KVM_VM_S390_UCONTROL) {
533 kvm->arch.gmap = NULL;
534 } else {
0349985a 535 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
e08b9637
CO		 536 if (!kvm->arch.gmap)
537 goto out_nogmap;
2c70fe44 538 kvm->arch.gmap->private = kvm;
24eb3a82 539 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 540 }
fa6b7fe9
CH		 541
542 kvm->arch.css_support = 0;
84223598 543 kvm->arch.use_irqchip = 0;
fa6b7fe9 544
8ad35755
DH		 545 spin_lock_init(&kvm->arch.start_stop_lock);
546
d89f5eff 547 return 0;
598841ca 548out_nogmap:
5102ee87
TK		 549 kfree(kvm->arch.crypto.crycb);
550out_crypto:
598841ca 551 debug_unregister(kvm->arch.dbf);
b0c632db
HC		 552out_nodbf:
553 free_page((unsigned long)(kvm->arch.sca));
d89f5eff
JK		 554out_err:
555 return rc;
b0c632db
HC		 556}
557
d329c035
CB		 558void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
559{
560 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 561 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
67335e63 562 kvm_s390_clear_local_irqs(vcpu);
3c038e6b 563 kvm_clear_async_pf_completion_queue(vcpu);
58f9460b
CO		 564 if (!kvm_is_ucontrol(vcpu->kvm)) {
565 clear_bit(63 - vcpu->vcpu_id,
566 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
567 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
568 (__u64) vcpu->arch.sie_block)
569 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
570 }
abf4a71e 571 smp_mb();
27e0393f
CO		 572
573 if (kvm_is_ucontrol(vcpu->kvm))
574 gmap_free(vcpu->arch.gmap);
575
b31605c1
DD		 576 if (kvm_s390_cmma_enabled(vcpu->kvm))
577 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 578 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 579
6692cef3 580 kvm_vcpu_uninit(vcpu);
b110feaf 581 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 582}
583
584static void kvm_free_vcpus(struct kvm *kvm)
585{
586 unsigned int i;
988a2cae 587 struct kvm_vcpu *vcpu;
d329c035 588
988a2cae
GN		 589 kvm_for_each_vcpu(i, vcpu, kvm)
590 kvm_arch_vcpu_destroy(vcpu);
591
592 mutex_lock(&kvm->lock);
593 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
594 kvm->vcpus[i] = NULL;
595
596 atomic_set(&kvm->online_vcpus, 0);
597 mutex_unlock(&kvm->lock);
d329c035
CB		 598}
599
b0c632db
HC		 600void kvm_arch_destroy_vm(struct kvm *kvm)
601{
d329c035 602 kvm_free_vcpus(kvm);
b0c632db 603 free_page((unsigned long)(kvm->arch.sca));
d329c035 604 debug_unregister(kvm->arch.dbf);
5102ee87 605 kfree(kvm->arch.crypto.crycb);
27e0393f
CO		 606 if (!kvm_is_ucontrol(kvm))
607 gmap_free(kvm->arch.gmap);
841b91c5 608 kvm_s390_destroy_adapters(kvm);
67335e63 609 kvm_s390_clear_float_irqs(kvm);
b0c632db
HC		 610}
611
612/* Section: vcpu related */
dafd032a
DD		 613static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
614{
615 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
616 if (!vcpu->arch.gmap)
617 return -ENOMEM;
618 vcpu->arch.gmap->private = vcpu->kvm;
619
620 return 0;
621}
622
b0c632db
HC		 623int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
624{
3c038e6b
DD		 625 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
626 kvm_clear_async_pf_completion_queue(vcpu);
59674c1a
CB		 627 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
628 KVM_SYNC_GPRS |
9eed0735 629 KVM_SYNC_ACRS |
b028ee3e
DH		 630 KVM_SYNC_CRS |
631 KVM_SYNC_ARCH0 |
632 KVM_SYNC_PFAULT;
dafd032a
DD		 633
634 if (kvm_is_ucontrol(vcpu->kvm))
635 return __kvm_ucontrol_vcpu_init(vcpu);
636
b0c632db
HC		 637 return 0;
638}
639
b0c632db
HC		 640void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
641{
4725c860
MS		 642 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
643 save_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db 644 save_access_regs(vcpu->arch.host_acrs);
4725c860
MS		 645 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
646 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 647 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 648 gmap_enable(vcpu->arch.gmap);
9e6dabef 649 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 650}
651
652void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
653{
9e6dabef 654 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 655 gmap_disable(vcpu->arch.gmap);
4725c860
MS		 656 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
657 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 658 save_access_regs(vcpu->run->s.regs.acrs);
4725c860
MS		 659 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
660 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db
HC		 661 restore_access_regs(vcpu->arch.host_acrs);
662}
663
664static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
665{
666 /* this equals initial cpu reset in pop, but we don't switch to ESA */
667 vcpu->arch.sie_block->gpsw.mask = 0UL;
668 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 669 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 670 vcpu->arch.sie_block->cputm = 0UL;
671 vcpu->arch.sie_block->ckc = 0UL;
672 vcpu->arch.sie_block->todpr = 0;
673 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
674 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
675 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
676 vcpu->arch.guest_fpregs.fpc = 0;
677 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
678 vcpu->arch.sie_block->gbea = 1;
672550fb 679 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 680 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
681 kvm_clear_async_pf_completion_queue(vcpu);
6352e4d2
DH		 682 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
683 kvm_s390_vcpu_stop(vcpu);
2ed10cc1 684 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 685}
686
31928aa5 687void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
42897d86 688{
dafd032a
DD		 689 if (!kvm_is_ucontrol(vcpu->kvm))
690 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
42897d86
MT		 691}
692
5102ee87
TK		 693static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
694{
695 if (!test_vfacility(76))
696 return;
697
698 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
699}
700
b31605c1
DD		 701void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
702{
703 free_page(vcpu->arch.sie_block->cbrlo);
704 vcpu->arch.sie_block->cbrlo = 0;
705}
706
707int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
708{
709 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
710 if (!vcpu->arch.sie_block->cbrlo)
711 return -ENOMEM;
712
713 vcpu->arch.sie_block->ecb2 |= 0x80;
714 vcpu->arch.sie_block->ecb2 &= ~0x08;
715 return 0;
716}
717
b0c632db
HC		 718int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
719{
b31605c1 720 int rc = 0;
b31288fa 721
9e6dabef
CH		 722 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
723 CPUSTAT_SM |
69d0d3a3
CB		 724 CPUSTAT_STOPPED |
725 CPUSTAT_GED);
fc34531d 726 vcpu->arch.sie_block->ecb = 6;
7feb6bb8
MM		 727 if (test_vfacility(50) && test_vfacility(73))
728 vcpu->arch.sie_block->ecb |= 0x10;
729
69d0d3a3 730 vcpu->arch.sie_block->ecb2 = 8;
ea5f4969 731 vcpu->arch.sie_block->eca = 0xC1002000U;
217a4406
HC		 732 if (sclp_has_siif())
733 vcpu->arch.sie_block->eca |= 1;
ea5f4969
DH		 734 if (sclp_has_sigpif())
735 vcpu->arch.sie_block->eca |= 0x10000000U;
78c4b59f 736 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
5a5e6536
MR		 737 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
738 ICTL_TPROT;
739
b31605c1
DD		 740 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
741 rc = kvm_s390_vcpu_setup_cmma(vcpu);
742 if (rc)
743 return rc;
b31288fa 744 }
0ac96caf 745 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
ca872302 746 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
453423dc 747 get_cpu_id(&vcpu->arch.cpu_id);
92e6ecf3 748 vcpu->arch.cpu_id.version = 0xff;
5102ee87
TK		 749
750 kvm_s390_vcpu_crypto_setup(vcpu);
751
b31605c1 752 return rc;
b0c632db
HC		 753}
754
755struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
756 unsigned int id)
757{
4d47555a 758 struct kvm_vcpu *vcpu;
7feb6bb8 759 struct sie_page *sie_page;
4d47555a
CO		 760 int rc = -EINVAL;
761
762 if (id >= KVM_MAX_VCPUS)
763 goto out;
764
765 rc = -ENOMEM;
b0c632db 766
b110feaf 767 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 768 if (!vcpu)
4d47555a 769 goto out;
b0c632db 770
7feb6bb8
MM		 771 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
772 if (!sie_page)
b0c632db
HC		 773 goto out_free_cpu;
774
7feb6bb8
MM		 775 vcpu->arch.sie_block = &sie_page->sie_block;
776 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
777
b0c632db 778 vcpu->arch.sie_block->icpua = id;
58f9460b
CO		 779 if (!kvm_is_ucontrol(kvm)) {
780 if (!kvm->arch.sca) {
781 WARN_ON_ONCE(1);
782 goto out_free_cpu;
783 }
784 if (!kvm->arch.sca->cpu[id].sda)
785 kvm->arch.sca->cpu[id].sda =
786 (__u64) vcpu->arch.sie_block;
787 vcpu->arch.sie_block->scaoh =
788 (__u32)(((__u64)kvm->arch.sca) >> 32);
789 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
790 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
791 }
b0c632db 792
ba5c1e9b 793 spin_lock_init(&vcpu->arch.local_int.lock);
ba5c1e9b 794 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 795 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 796 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 797
b0c632db
HC		 798 rc = kvm_vcpu_init(vcpu, kvm, id);
799 if (rc)
7b06bf2f 800 goto out_free_sie_block;
b0c632db
HC		 801 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
802 vcpu->arch.sie_block);
ade38c31 803 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 804
b0c632db 805 return vcpu;
7b06bf2f
WY		 806out_free_sie_block:
807 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 808out_free_cpu:
b110feaf 809 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 810out:
b0c632db
HC		 811 return ERR_PTR(rc);
812}
813
b0c632db
HC		 814int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
815{
9a022067 816 return kvm_s390_vcpu_has_irq(vcpu, 0);
b0c632db
HC		 817}
818
49b99e1e
CB		 819void s390_vcpu_block(struct kvm_vcpu *vcpu)
820{
821 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
822}
823
824void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
825{
826 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
827}
828
829/*
830 * Kick a guest cpu out of SIE and wait until SIE is not running.
831 * If the CPU is not running (e.g. waiting as idle) the function will
832 * return immediately. */
833void exit_sie(struct kvm_vcpu *vcpu)
834{
835 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
836 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
837 cpu_relax();
838}
839
840/* Kick a guest cpu out of SIE and prevent SIE-reentry */
841void exit_sie_sync(struct kvm_vcpu *vcpu)
842{
843 s390_vcpu_block(vcpu);
844 exit_sie(vcpu);
845}
846
2c70fe44
CB		 847static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
848{
849 int i;
850 struct kvm *kvm = gmap->private;
851 struct kvm_vcpu *vcpu;
852
853 kvm_for_each_vcpu(i, vcpu, kvm) {
854 /* match against both prefix pages */
fda902cb 855 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
2c70fe44
CB		 856 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
857 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
858 exit_sie_sync(vcpu);
859 }
860 }
861}
862
b6d33834
CD		 863int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
864{
865 /* kvm common code refers to this, but never calls it */
866 BUG();
867 return 0;
868}
869
14eebd91
CO		 870static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
871 struct kvm_one_reg *reg)
872{
873 int r = -EINVAL;
874
875 switch (reg->id) {
29b7c71b
CO		 876 case KVM_REG_S390_TODPR:
877 r = put_user(vcpu->arch.sie_block->todpr,
878 (u32 __user *)reg->addr);
879 break;
880 case KVM_REG_S390_EPOCHDIFF:
881 r = put_user(vcpu->arch.sie_block->epoch,
882 (u64 __user *)reg->addr);
883 break;
46a6dd1c
J		 884 case KVM_REG_S390_CPU_TIMER:
885 r = put_user(vcpu->arch.sie_block->cputm,
886 (u64 __user *)reg->addr);
887 break;
888 case KVM_REG_S390_CLOCK_COMP:
889 r = put_user(vcpu->arch.sie_block->ckc,
890 (u64 __user *)reg->addr);
891 break;
536336c2
DD		 892 case KVM_REG_S390_PFTOKEN:
893 r = put_user(vcpu->arch.pfault_token,
894 (u64 __user *)reg->addr);
895 break;
896 case KVM_REG_S390_PFCOMPARE:
897 r = put_user(vcpu->arch.pfault_compare,
898 (u64 __user *)reg->addr);
899 break;
900 case KVM_REG_S390_PFSELECT:
901 r = put_user(vcpu->arch.pfault_select,
902 (u64 __user *)reg->addr);
903 break;
672550fb
CB		 904 case KVM_REG_S390_PP:
905 r = put_user(vcpu->arch.sie_block->pp,
906 (u64 __user *)reg->addr);
907 break;
afa45ff5
CB		 908 case KVM_REG_S390_GBEA:
909 r = put_user(vcpu->arch.sie_block->gbea,
910 (u64 __user *)reg->addr);
911 break;
14eebd91
CO		 912 default:
913 break;
914 }
915
916 return r;
917}
918
919static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
920 struct kvm_one_reg *reg)
921{
922 int r = -EINVAL;
923
924 switch (reg->id) {
29b7c71b
CO		 925 case KVM_REG_S390_TODPR:
926 r = get_user(vcpu->arch.sie_block->todpr,
927 (u32 __user *)reg->addr);
928 break;
929 case KVM_REG_S390_EPOCHDIFF:
930 r = get_user(vcpu->arch.sie_block->epoch,
931 (u64 __user *)reg->addr);
932 break;
46a6dd1c
J		 933 case KVM_REG_S390_CPU_TIMER:
934 r = get_user(vcpu->arch.sie_block->cputm,
935 (u64 __user *)reg->addr);
936 break;
937 case KVM_REG_S390_CLOCK_COMP:
938 r = get_user(vcpu->arch.sie_block->ckc,
939 (u64 __user *)reg->addr);
940 break;
536336c2
DD		 941 case KVM_REG_S390_PFTOKEN:
942 r = get_user(vcpu->arch.pfault_token,
943 (u64 __user *)reg->addr);
9fbd8082
DH		 944 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
945 kvm_clear_async_pf_completion_queue(vcpu);
536336c2
DD		 946 break;
947 case KVM_REG_S390_PFCOMPARE:
948 r = get_user(vcpu->arch.pfault_compare,
949 (u64 __user *)reg->addr);
950 break;
951 case KVM_REG_S390_PFSELECT:
952 r = get_user(vcpu->arch.pfault_select,
953 (u64 __user *)reg->addr);
954 break;
672550fb
CB		 955 case KVM_REG_S390_PP:
956 r = get_user(vcpu->arch.sie_block->pp,
957 (u64 __user *)reg->addr);
958 break;
afa45ff5
CB		 959 case KVM_REG_S390_GBEA:
960 r = get_user(vcpu->arch.sie_block->gbea,
961 (u64 __user *)reg->addr);
962 break;
14eebd91
CO		 963 default:
964 break;
965 }
966
967 return r;
968}
b6d33834 969
b0c632db
HC		 970static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
971{
b0c632db 972 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 973 return 0;
974}
975
976int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
977{
5a32c1af 978 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 979 return 0;
980}
981
982int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
983{
5a32c1af 984 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 985 return 0;
986}
987
988int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
989 struct kvm_sregs *sregs)
990{
59674c1a 991 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 992 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 993 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 994 return 0;
995}
996
997int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
998 struct kvm_sregs *sregs)
999{
59674c1a 1000 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 1001 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 1002 return 0;
1003}
1004
1005int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1006{
4725c860
MS		 1007 if (test_fp_ctl(fpu->fpc))
1008 return -EINVAL;
b0c632db 1009 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860
MS		 1010 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1011 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1012 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
b0c632db
HC		 1013 return 0;
1014}
1015
1016int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1017{
b0c632db
HC		 1018 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1019 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 1020 return 0;
1021}
1022
1023static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1024{
1025 int rc = 0;
1026
7a42fdc2 1027 if (!is_vcpu_stopped(vcpu))
b0c632db 1028 rc = -EBUSY;
d7b0b5eb
CO		 1029 else {
1030 vcpu->run->psw_mask = psw.mask;
1031 vcpu->run->psw_addr = psw.addr;
1032 }
b0c632db
HC		 1033 return rc;
1034}
1035
1036int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1037 struct kvm_translation *tr)
1038{
1039 return -EINVAL; /* not implemented yet */
1040}
1041
27291e21
DH		 1042#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1043 KVM_GUESTDBG_USE_HW_BP | \
1044 KVM_GUESTDBG_ENABLE)
1045
d0bfb940
JK		 1046int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1047 struct kvm_guest_debug *dbg)
b0c632db 1048{
27291e21
DH		 1049 int rc = 0;
1050
1051 vcpu->guest_debug = 0;
1052 kvm_s390_clear_bp_data(vcpu);
1053
2de3bfc2 1054 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
27291e21
DH		 1055 return -EINVAL;
1056
1057 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1058 vcpu->guest_debug = dbg->control;
1059 /* enforce guest PER */
1060 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1061
1062 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1063 rc = kvm_s390_import_bp_data(vcpu, dbg);
1064 } else {
1065 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1066 vcpu->arch.guestdbg.last_bp = 0;
1067 }
1068
1069 if (rc) {
1070 vcpu->guest_debug = 0;
1071 kvm_s390_clear_bp_data(vcpu);
1072 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1073 }
1074
1075 return rc;
b0c632db
HC		 1076}
1077
62d9f0db
MT		 1078int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1079 struct kvm_mp_state *mp_state)
1080{
6352e4d2
DH		 1081 /* CHECK_STOP and LOAD are not supported yet */
1082 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1083 KVM_MP_STATE_OPERATING;
62d9f0db
MT		 1084}
1085
1086int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1087 struct kvm_mp_state *mp_state)
1088{
6352e4d2
DH		 1089 int rc = 0;
1090
1091 /* user space knows about this interface - let it control the state */
1092 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1093
1094 switch (mp_state->mp_state) {
1095 case KVM_MP_STATE_STOPPED:
1096 kvm_s390_vcpu_stop(vcpu);
1097 break;
1098 case KVM_MP_STATE_OPERATING:
1099 kvm_s390_vcpu_start(vcpu);
1100 break;
1101 case KVM_MP_STATE_LOAD:
1102 case KVM_MP_STATE_CHECK_STOP:
1103 /* fall through - CHECK_STOP and LOAD are not supported yet */
1104 default:
1105 rc = -ENXIO;
1106 }
1107
1108 return rc;
62d9f0db
MT		 1109}
1110
b31605c1
DD		 1111bool kvm_s390_cmma_enabled(struct kvm *kvm)
1112{
1113 if (!MACHINE_IS_LPAR)
1114 return false;
1115 /* only enable for z10 and later */
1116 if (!MACHINE_HAS_EDAT1)
1117 return false;
1118 if (!kvm->arch.use_cmma)
1119 return false;
1120 return true;
1121}
1122
8ad35755
DH		 1123static bool ibs_enabled(struct kvm_vcpu *vcpu)
1124{
1125 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1126}
1127
2c70fe44
CB		 1128static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1129{
8ad35755
DH		 1130retry:
1131 s390_vcpu_unblock(vcpu);
2c70fe44
CB		 1132 /*
1133 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1134 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1135 * This ensures that the ipte instruction for this request has
1136 * already finished. We might race against a second unmapper that
1137 * wants to set the blocking bit. Lets just retry the request loop.
1138 */
8ad35755 1139 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2c70fe44
CB		 1140 int rc;
1141 rc = gmap_ipte_notify(vcpu->arch.gmap,
fda902cb 1142 kvm_s390_get_prefix(vcpu),
2c70fe44
CB		 1143 PAGE_SIZE * 2);
1144 if (rc)
1145 return rc;
8ad35755 1146 goto retry;
2c70fe44 1147 }
8ad35755 1148
d3d692c8
DH		 1149 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1150 vcpu->arch.sie_block->ihcpu = 0xffff;
1151 goto retry;
1152 }
1153
8ad35755
DH		 1154 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1155 if (!ibs_enabled(vcpu)) {
1156 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1157 atomic_set_mask(CPUSTAT_IBS,
1158 &vcpu->arch.sie_block->cpuflags);
1159 }
1160 goto retry;
2c70fe44 1161 }
8ad35755
DH		 1162
1163 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1164 if (ibs_enabled(vcpu)) {
1165 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1166 atomic_clear_mask(CPUSTAT_IBS,
1167 &vcpu->arch.sie_block->cpuflags);
1168 }
1169 goto retry;
1170 }
1171
0759d068
DH		 1172 /* nothing to do, just clear the request */
1173 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1174
2c70fe44
CB		 1175 return 0;
1176}
1177
fa576c58
TH		 1178/**
1179 * kvm_arch_fault_in_page - fault-in guest page if necessary
1180 * @vcpu: The corresponding virtual cpu
1181 * @gpa: Guest physical address
1182 * @writable: Whether the page should be writable or not
1183 *
1184 * Make sure that a guest page has been faulted-in on the host.
1185 *
1186 * Return: Zero on success, negative error code otherwise.
1187 */
1188long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
24eb3a82 1189{
527e30b4
MS		 1190 return gmap_fault(vcpu->arch.gmap, gpa,
1191 writable ? FAULT_FLAG_WRITE : 0);
24eb3a82
DD		 1192}
1193
3c038e6b
DD		 1194static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1195 unsigned long token)
1196{
1197 struct kvm_s390_interrupt inti;
383d0b05 1198 struct kvm_s390_irq irq;
3c038e6b
DD		 1199
1200 if (start_token) {
383d0b05
JF		 1201 irq.u.ext.ext_params2 = token;
1202 irq.type = KVM_S390_INT_PFAULT_INIT;
1203 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3c038e6b
DD		 1204 } else {
1205 inti.type = KVM_S390_INT_PFAULT_DONE;
383d0b05 1206 inti.parm64 = token;
3c038e6b
DD		 1207 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1208 }
1209}
1210
1211void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1212 struct kvm_async_pf *work)
1213{
1214 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1215 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1216}
1217
1218void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1219 struct kvm_async_pf *work)
1220{
1221 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1222 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1223}
1224
1225void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1226 struct kvm_async_pf *work)
1227{
1228 /* s390 will always inject the page directly */
1229}
1230
1231bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1232{
1233 /*
1234 * s390 will always inject the page directly,
1235 * but we still want check_async_completion to cleanup
1236 */
1237 return true;
1238}
1239
1240static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1241{
1242 hva_t hva;
1243 struct kvm_arch_async_pf arch;
1244 int rc;
1245
1246 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1247 return 0;
1248 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1249 vcpu->arch.pfault_compare)
1250 return 0;
1251 if (psw_extint_disabled(vcpu))
1252 return 0;
9a022067 1253 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3c038e6b
DD		 1254 return 0;
1255 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1256 return 0;
1257 if (!vcpu->arch.gmap->pfault_enabled)
1258 return 0;
1259
81480cc1
HC		 1260 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1261 hva += current->thread.gmap_addr & ~PAGE_MASK;
1262 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 1263 return 0;
1264
1265 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1266 return rc;
1267}
1268
3fb4c40f 1269static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 1270{
3fb4c40f 1271 int rc, cpuflags;
e168bf8d 1272
3c038e6b
DD		 1273 /*
1274 * On s390 notifications for arriving pages will be delivered directly
1275 * to the guest but the house keeping for completed pfaults is
1276 * handled outside the worker.
1277 */
1278 kvm_check_async_pf_completion(vcpu);
1279
5a32c1af 1280 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
b0c632db
HC		 1281
1282 if (need_resched())
1283 schedule();
1284
d3a73acb 1285 if (test_cpu_flag(CIF_MCCK_PENDING))
71cde587
CB		 1286 s390_handle_mcck();
1287
79395031
JF		 1288 if (!kvm_is_ucontrol(vcpu->kvm)) {
1289 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1290 if (rc)
1291 return rc;
1292 }
0ff31867 1293
2c70fe44
CB		 1294 rc = kvm_s390_handle_requests(vcpu);
1295 if (rc)
1296 return rc;
1297
27291e21
DH		 1298 if (guestdbg_enabled(vcpu)) {
1299 kvm_s390_backup_guest_per_regs(vcpu);
1300 kvm_s390_patch_guest_per_regs(vcpu);
1301 }
1302
b0c632db 1303 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 1304 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1305 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1306 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 1307
3fb4c40f
TH		 1308 return 0;
1309}
1310
1311static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1312{
24eb3a82 1313 int rc = -1;
2b29a9fd
DD		 1314
1315 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1316 vcpu->arch.sie_block->icptcode);
1317 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1318
27291e21
DH		 1319 if (guestdbg_enabled(vcpu))
1320 kvm_s390_restore_guest_per_regs(vcpu);
1321
3fb4c40f 1322 if (exit_reason >= 0) {
7c470539 1323 rc = 0;
210b1607
TH		 1324 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1325 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1326 vcpu->run->s390_ucontrol.trans_exc_code =
1327 current->thread.gmap_addr;
1328 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1329 rc = -EREMOTE;
24eb3a82
DD		 1330
1331 } else if (current->thread.gmap_pfault) {
3c038e6b 1332 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 1333 current->thread.gmap_pfault = 0;
fa576c58 1334 if (kvm_arch_setup_async_pf(vcpu)) {
24eb3a82 1335 rc = 0;
fa576c58
TH		 1336 } else {
1337 gpa_t gpa = current->thread.gmap_addr;
1338 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1339 }
24eb3a82
DD		 1340 }
1341
1342 if (rc == -1) {
699bde3b
CB		 1343 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1344 trace_kvm_s390_sie_fault(vcpu);
1345 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1f0d0f09 1346 }
b0c632db 1347
5a32c1af 1348 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
3fb4c40f 1349
a76ccff6
TH		 1350 if (rc == 0) {
1351 if (kvm_is_ucontrol(vcpu->kvm))
2955c83f
CB		 1352 /* Don't exit for host interrupts. */
1353 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
a76ccff6
TH		 1354 else
1355 rc = kvm_handle_sie_intercept(vcpu);
1356 }
1357
3fb4c40f
TH		 1358 return rc;
1359}
1360
1361static int __vcpu_run(struct kvm_vcpu *vcpu)
1362{
1363 int rc, exit_reason;
1364
800c1065
TH		 1365 /*
1366 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1367 * ning the guest), so that memslots (and other stuff) are protected
1368 */
1369 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1370
a76ccff6
TH		 1371 do {
1372 rc = vcpu_pre_run(vcpu);
1373 if (rc)
1374 break;
3fb4c40f 1375
800c1065 1376 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 1377 /*
1378 * As PF_VCPU will be used in fault handler, between
1379 * guest_enter and guest_exit should be no uaccess.
1380 */
1381 preempt_disable();
1382 kvm_guest_enter();
1383 preempt_enable();
1384 exit_reason = sie64a(vcpu->arch.sie_block,
1385 vcpu->run->s.regs.gprs);
1386 kvm_guest_exit();
800c1065 1387 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 1388
1389 rc = vcpu_post_run(vcpu, exit_reason);
27291e21 1390 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3fb4c40f 1391
800c1065 1392 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 1393 return rc;
b0c632db
HC		 1394}
1395
b028ee3e
DH		 1396static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1397{
1398 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1399 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1400 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1401 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1402 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1403 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
d3d692c8
DH		 1404 /* some control register changes require a tlb flush */
1405 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
b028ee3e
DH		 1406 }
1407 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1408 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1409 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1410 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1411 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1412 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1413 }
1414 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1415 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1416 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1417 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
9fbd8082
DH		 1418 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1419 kvm_clear_async_pf_completion_queue(vcpu);
b028ee3e
DH		 1420 }
1421 kvm_run->kvm_dirty_regs = 0;
1422}
1423
1424static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1425{
1426 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1427 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1428 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1429 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1430 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1431 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1432 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1433 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1434 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1435 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1436 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1437 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1438}
1439
b0c632db
HC		 1440int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1441{
8f2abe6a 1442 int rc;
b0c632db
HC		 1443 sigset_t sigsaved;
1444
27291e21
DH		 1445 if (guestdbg_exit_pending(vcpu)) {
1446 kvm_s390_prepare_debug_exit(vcpu);
1447 return 0;
1448 }
1449
b0c632db
HC		 1450 if (vcpu->sigset_active)
1451 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1452
6352e4d2
DH		 1453 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1454 kvm_s390_vcpu_start(vcpu);
1455 } else if (is_vcpu_stopped(vcpu)) {
1456 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1457 vcpu->vcpu_id);
1458 return -EINVAL;
1459 }
b0c632db 1460
b028ee3e 1461 sync_regs(vcpu, kvm_run);
d7b0b5eb 1462
dab4079d 1463 might_fault();
a76ccff6 1464 rc = __vcpu_run(vcpu);
9ace903d 1465
b1d16c49
CE		 1466 if (signal_pending(current) && !rc) {
1467 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 1468 rc = -EINTR;
b1d16c49 1469 }
8f2abe6a 1470
27291e21
DH		 1471 if (guestdbg_exit_pending(vcpu) && !rc) {
1472 kvm_s390_prepare_debug_exit(vcpu);
1473 rc = 0;
1474 }
1475
b8e660b8 1476 if (rc == -EOPNOTSUPP) {
8f2abe6a
CB		 1477 /* intercept cannot be handled in-kernel, prepare kvm-run */
1478 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1479 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
8f2abe6a
CB		 1480 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1481 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1482 rc = 0;
1483 }
1484
1485 if (rc == -EREMOTE) {
1486 /* intercept was handled, but userspace support is needed
1487 * kvm_run has been prepared by the handler */
1488 rc = 0;
1489 }
b0c632db 1490
b028ee3e 1491 store_regs(vcpu, kvm_run);
d7b0b5eb 1492
b0c632db
HC		 1493 if (vcpu->sigset_active)
1494 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1495
b0c632db 1496 vcpu->stat.exit_userspace++;
7e8e6ab4 1497 return rc;
b0c632db
HC		 1498}
1499
b0c632db
HC		 1500/*
1501 * store status at address
1502 * we use have two special cases:
1503 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1504 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1505 */
d0bce605 1506int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 1507{
092670cd 1508 unsigned char archmode = 1;
fda902cb 1509 unsigned int px;
178bd789 1510 u64 clkcomp;
d0bce605 1511 int rc;
b0c632db 1512
d0bce605
HC		 1513 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1514 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 1515 return -EFAULT;
d0bce605
HC		 1516 gpa = SAVE_AREA_BASE;
1517 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1518 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 1519 return -EFAULT;
d0bce605
HC		 1520 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1521 }
1522 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1523 vcpu->arch.guest_fpregs.fprs, 128);
1524 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1525 vcpu->run->s.regs.gprs, 128);
1526 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1527 &vcpu->arch.sie_block->gpsw, 16);
fda902cb 1528 px = kvm_s390_get_prefix(vcpu);
d0bce605 1529 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
fda902cb 1530 &px, 4);
d0bce605
HC		 1531 rc |= write_guest_abs(vcpu,
1532 gpa + offsetof(struct save_area, fp_ctrl_reg),
1533 &vcpu->arch.guest_fpregs.fpc, 4);
1534 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1535 &vcpu->arch.sie_block->todpr, 4);
1536 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1537 &vcpu->arch.sie_block->cputm, 8);
178bd789 1538 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 1539 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1540 &clkcomp, 8);
1541 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1542 &vcpu->run->s.regs.acrs, 64);
1543 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1544 &vcpu->arch.sie_block->gcr, 128);
1545 return rc ? -EFAULT : 0;
b0c632db
HC		 1546}
1547
e879892c
TH		 1548int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1549{
1550 /*
1551 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1552 * copying in vcpu load/put. Lets update our copies before we save
1553 * it into the save area
1554 */
1555 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1556 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1557 save_access_regs(vcpu->run->s.regs.acrs);
1558
1559 return kvm_s390_store_status_unloaded(vcpu, addr);
1560}
1561
8ad35755
DH		 1562static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1563{
1564 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1565 kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1566 exit_sie_sync(vcpu);
1567}
1568
1569static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1570{
1571 unsigned int i;
1572 struct kvm_vcpu *vcpu;
1573
1574 kvm_for_each_vcpu(i, vcpu, kvm) {
1575 __disable_ibs_on_vcpu(vcpu);
1576 }
1577}
1578
1579static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1580{
1581 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1582 kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
1583 exit_sie_sync(vcpu);
1584}
1585
6852d7b6
DH		 1586void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
1587{
8ad35755
DH		 1588 int i, online_vcpus, started_vcpus = 0;
1589
1590 if (!is_vcpu_stopped(vcpu))
1591 return;
1592
6852d7b6 1593 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
8ad35755 1594 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1595 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1596 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1597
1598 for (i = 0; i < online_vcpus; i++) {
1599 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
1600 started_vcpus++;
1601 }
1602
1603 if (started_vcpus == 0) {
1604 /* we're the only active VCPU -> speed it up */
1605 __enable_ibs_on_vcpu(vcpu);
1606 } else if (started_vcpus == 1) {
1607 /*
1608 * As we are starting a second VCPU, we have to disable
1609 * the IBS facility on all VCPUs to remove potentially
1610 * oustanding ENABLE requests.
1611 */
1612 __disable_ibs_on_all_vcpus(vcpu->kvm);
1613 }
1614
6852d7b6 1615 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 1616 /*
1617 * Another VCPU might have used IBS while we were offline.
1618 * Let's play safe and flush the VCPU at startup.
1619 */
d3d692c8 1620 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
433b9ee4 1621 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1622 return;
6852d7b6
DH		 1623}
1624
1625void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
1626{
8ad35755
DH		 1627 int i, online_vcpus, started_vcpus = 0;
1628 struct kvm_vcpu *started_vcpu = NULL;
1629
1630 if (is_vcpu_stopped(vcpu))
1631 return;
1632
6852d7b6 1633 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
8ad35755 1634 /* Only one cpu at a time may enter/leave the STOPPED state. */
433b9ee4 1635 spin_lock(&vcpu->kvm->arch.start_stop_lock);
8ad35755
DH		 1636 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
1637
32f5ff63 1638 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
6cddd432 1639 kvm_s390_clear_stop_irq(vcpu);
32f5ff63 1640
6cddd432 1641 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
8ad35755
DH		 1642 __disable_ibs_on_vcpu(vcpu);
1643
1644 for (i = 0; i < online_vcpus; i++) {
1645 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
1646 started_vcpus++;
1647 started_vcpu = vcpu->kvm->vcpus[i];
1648 }
1649 }
1650
1651 if (started_vcpus == 1) {
1652 /*
1653 * As we only have one VCPU left, we want to enable the
1654 * IBS facility for that VCPU to speed it up.
1655 */
1656 __enable_ibs_on_vcpu(started_vcpu);
1657 }
1658
433b9ee4 1659 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
8ad35755 1660 return;
6852d7b6
DH		 1661}
1662
d6712df9
CH		 1663static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1664 struct kvm_enable_cap *cap)
1665{
1666 int r;
1667
1668 if (cap->flags)
1669 return -EINVAL;
1670
1671 switch (cap->cap) {
fa6b7fe9
CH		 1672 case KVM_CAP_S390_CSS_SUPPORT:
1673 if (!vcpu->kvm->arch.css_support) {
1674 vcpu->kvm->arch.css_support = 1;
1675 trace_kvm_s390_enable_css(vcpu->kvm);
1676 }
1677 r = 0;
1678 break;
d6712df9
CH		 1679 default:
1680 r = -EINVAL;
1681 break;
1682 }
1683 return r;
1684}
1685
b0c632db
HC		 1686long kvm_arch_vcpu_ioctl(struct file *filp,
1687 unsigned int ioctl, unsigned long arg)
1688{
1689 struct kvm_vcpu *vcpu = filp->private_data;
1690 void __user *argp = (void __user *)arg;
800c1065 1691 int idx;
bc923cc9 1692 long r;
b0c632db 1693
93736624
AK		 1694 switch (ioctl) {
1695 case KVM_S390_INTERRUPT: {
ba5c1e9b 1696 struct kvm_s390_interrupt s390int;
383d0b05 1697 struct kvm_s390_irq s390irq;
ba5c1e9b 1698
93736624 1699 r = -EFAULT;
ba5c1e9b 1700 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624 1701 break;
383d0b05
JF		 1702 if (s390int_to_s390irq(&s390int, &s390irq))
1703 return -EINVAL;
1704 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
93736624 1705 break;
ba5c1e9b 1706 }
b0c632db 1707 case KVM_S390_STORE_STATUS:
800c1065 1708 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 1709 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 1710 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 1711 break;
b0c632db
HC		 1712 case KVM_S390_SET_INITIAL_PSW: {
1713 psw_t psw;
1714
bc923cc9 1715 r = -EFAULT;
b0c632db 1716 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 1717 break;
1718 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1719 break;
b0c632db
HC		 1720 }
1721 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 1722 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1723 break;
14eebd91
CO		 1724 case KVM_SET_ONE_REG:
1725 case KVM_GET_ONE_REG: {
1726 struct kvm_one_reg reg;
1727 r = -EFAULT;
1728 if (copy_from_user(&reg, argp, sizeof(reg)))
1729 break;
1730 if (ioctl == KVM_SET_ONE_REG)
1731 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1732 else
1733 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1734 break;
1735 }
27e0393f
CO		 1736#ifdef CONFIG_KVM_S390_UCONTROL
1737 case KVM_S390_UCAS_MAP: {
1738 struct kvm_s390_ucas_mapping ucasmap;
1739
1740 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1741 r = -EFAULT;
1742 break;
1743 }
1744
1745 if (!kvm_is_ucontrol(vcpu->kvm)) {
1746 r = -EINVAL;
1747 break;
1748 }
1749
1750 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1751 ucasmap.vcpu_addr, ucasmap.length);
1752 break;
1753 }
1754 case KVM_S390_UCAS_UNMAP: {
1755 struct kvm_s390_ucas_mapping ucasmap;
1756
1757 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1758 r = -EFAULT;
1759 break;
1760 }
1761
1762 if (!kvm_is_ucontrol(vcpu->kvm)) {
1763 r = -EINVAL;
1764 break;
1765 }
1766
1767 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1768 ucasmap.length);
1769 break;
1770 }
1771#endif
ccc7910f 1772 case KVM_S390_VCPU_FAULT: {
527e30b4 1773 r = gmap_fault(vcpu->arch.gmap, arg, 0);
ccc7910f
CO		 1774 break;
1775 }
d6712df9
CH		 1776 case KVM_ENABLE_CAP:
1777 {
1778 struct kvm_enable_cap cap;
1779 r = -EFAULT;
1780 if (copy_from_user(&cap, argp, sizeof(cap)))
1781 break;
1782 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1783 break;
1784 }
b0c632db 1785 default:
3e6afcf1 1786 r = -ENOTTY;
b0c632db 1787 }
bc923cc9 1788 return r;
b0c632db
HC		 1789}
1790
5b1c1493
CO		 1791int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1792{
1793#ifdef CONFIG_KVM_S390_UCONTROL
1794 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1795 && (kvm_is_ucontrol(vcpu->kvm))) {
1796 vmf->page = virt_to_page(vcpu->arch.sie_block);
1797 get_page(vmf->page);
1798 return 0;
1799 }
1800#endif
1801 return VM_FAULT_SIGBUS;
1802}
1803
5587027c
AK		 1804int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1805 unsigned long npages)
db3fe4eb
TY		 1806{
1807 return 0;
1808}
1809
b0c632db 1810/* Section: memory related */
f7784b8e
MT		 1811int kvm_arch_prepare_memory_region(struct kvm *kvm,
1812 struct kvm_memory_slot *memslot,
7b6195a9
TY		 1813 struct kvm_userspace_memory_region *mem,
1814 enum kvm_mr_change change)
b0c632db 1815{
dd2887e7
NW		 1816 /* A few sanity checks. We can have memory slots which have to be
1817 located/ended at a segment boundary (1MB). The memory in userland is
1818 ok to be fragmented into various different vmas. It is okay to mmap()
1819 and munmap() stuff in this slot after doing this call at any time */
b0c632db 1820
598841ca 1821 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 1822 return -EINVAL;
1823
598841ca 1824 if (mem->memory_size & 0xffffful)
b0c632db
HC		 1825 return -EINVAL;
1826
f7784b8e
MT		 1827 return 0;
1828}
1829
1830void kvm_arch_commit_memory_region(struct kvm *kvm,
1831 struct kvm_userspace_memory_region *mem,
8482644a
TY		 1832 const struct kvm_memory_slot *old,
1833 enum kvm_mr_change change)
f7784b8e 1834{
f7850c92 1835 int rc;
f7784b8e 1836
2cef4deb
CB		 1837 /* If the basics of the memslot do not change, we do not want
1838 * to update the gmap. Every update causes several unnecessary
1839 * segment translation exceptions. This is usually handled just
1840 * fine by the normal fault handler + gmap, but it will also
1841 * cause faults on the prefix page of running guest CPUs.
1842 */
1843 if (old->userspace_addr == mem->userspace_addr &&
1844 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1845 old->npages * PAGE_SIZE == mem->memory_size)
1846 return;
598841ca
CO		 1847
1848 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1849 mem->guest_phys_addr, mem->memory_size);
1850 if (rc)
f7850c92 1851 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
598841ca 1852 return;
b0c632db
HC		 1853}
1854
b0c632db
HC		 1855static int __init kvm_s390_init(void)
1856{
ef50f7ac 1857 int ret;
0ee75bea 1858 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
ef50f7ac
CB		 1859 if (ret)
1860 return ret;
1861
1862 /*
1863 * guests can ask for up to 255+1 double words, we need a full page
25985edc 1864 * to hold the maximum amount of facilities. On the other hand, we
ef50f7ac
CB		 1865 * only set facilities that are known to work in KVM.
1866 */
78c4b59f
MM		 1867 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1868 if (!vfacilities) {
ef50f7ac
CB		 1869 kvm_exit();
1870 return -ENOMEM;
1871 }
78c4b59f 1872 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
7be81a46 1873 vfacilities[0] &= 0xff82fffbf47c2000UL;
7feb6bb8 1874 vfacilities[1] &= 0x005c000000000000UL;
ef50f7ac 1875 return 0;
b0c632db
HC		 1876}
1877
1878static void __exit kvm_s390_exit(void)
1879{
78c4b59f 1880 free_page((unsigned long) vfacilities);
b0c632db
HC		 1881 kvm_exit();
1882}
1883
1884module_init(kvm_s390_init);
1885module_exit(kvm_s390_exit);
566af940
CH		 1886
1887/*
1888 * Enable autoloading of the kvm module.
1889 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1890 * since x86 takes a different approach.
1891 */
1892#include <linux/miscdevice.h>
1893MODULE_ALIAS_MISCDEV(KVM_MINOR);
1894MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)