projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: deliver program irq parameters and use correct ilc
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
25#include <linux/slab.h>
ba5c1e9b 26#include <linux/timer.h>
cbb870c8 27#include <asm/asm-offsets.h>
b0c632db
HC		 28#include <asm/lowcore.h>
29#include <asm/pgtable.h>
f5daba1d 30#include <asm/nmi.h>
a0616cde 31#include <asm/switch_to.h>
78c4b59f 32#include <asm/facility.h>
1526bf9c 33#include <asm/sclp.h>
8f2abe6a 34#include "kvm-s390.h"
b0c632db
HC		 35#include "gaccess.h"
36
5786fffa
CH		 37#define CREATE_TRACE_POINTS
38#include "trace.h"
ade38c31 39#include "trace-s390.h"
5786fffa 40
b0c632db
HC		 41#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 45 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f5e10b09 53 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b
CO		 54 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
55 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 56 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 57 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
58 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
59 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
60 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
61 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
62 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
63 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 64 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 65 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
66 { "instruction_spx", VCPU_STAT(instruction_spx) },
67 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
68 { "instruction_stap", VCPU_STAT(instruction_stap) },
69 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 70 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 71 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
72 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 73 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 74 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
75 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 76 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 77 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 78 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 79 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0
CB		 80 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
81 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
82 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
83 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
84 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
388186bc 85 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 86 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 87 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
b0c632db
HC		 88 { NULL }
89};
90
78c4b59f 91unsigned long *vfacilities;
2c70fe44 92static struct gmap_notifier gmap_notifier;
b0c632db 93
78c4b59f 94/* test availability of vfacility */
280ef0f1 95int test_vfacility(unsigned long nr)
78c4b59f
MM		 96{
97 return __test_facility(nr, (void *) vfacilities);
98}
99
b0c632db 100/* Section: not file related */
10474ae8 101int kvm_arch_hardware_enable(void *garbage)
b0c632db
HC		 102{
103 /* every s390 is virtualization enabled ;-) */
10474ae8 104 return 0;
b0c632db
HC		 105}
106
107void kvm_arch_hardware_disable(void *garbage)
108{
109}
110
2c70fe44
CB		 111static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
112
b0c632db
HC		 113int kvm_arch_hardware_setup(void)
114{
2c70fe44
CB		 115 gmap_notifier.notifier_call = kvm_gmap_notifier;
116 gmap_register_ipte_notifier(&gmap_notifier);
b0c632db
HC		 117 return 0;
118}
119
120void kvm_arch_hardware_unsetup(void)
121{
2c70fe44 122 gmap_unregister_ipte_notifier(&gmap_notifier);
b0c632db
HC		 123}
124
125void kvm_arch_check_processor_compat(void *rtn)
126{
127}
128
129int kvm_arch_init(void *opaque)
130{
131 return 0;
132}
133
134void kvm_arch_exit(void)
135{
136}
137
138/* Section: device related */
139long kvm_arch_dev_ioctl(struct file *filp,
140 unsigned int ioctl, unsigned long arg)
141{
142 if (ioctl == KVM_S390_ENABLE_SIE)
143 return s390_enable_sie();
144 return -EINVAL;
145}
146
147int kvm_dev_ioctl_check_extension(long ext)
148{
d7b0b5eb
CO		 149 int r;
150
2bd0ac4e 151 switch (ext) {
d7b0b5eb 152 case KVM_CAP_S390_PSW:
b6cf8788 153 case KVM_CAP_S390_GMAP:
52e16b18 154 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 155#ifdef CONFIG_KVM_S390_UCONTROL
156 case KVM_CAP_S390_UCONTROL:
157#endif
3c038e6b 158 case KVM_CAP_ASYNC_PF:
60b413c9 159 case KVM_CAP_SYNC_REGS:
14eebd91 160 case KVM_CAP_ONE_REG:
d6712df9 161 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 162 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 163 case KVM_CAP_IOEVENTFD:
c05c4186 164 case KVM_CAP_DEVICE_CTRL:
d938dc55 165 case KVM_CAP_ENABLE_CAP_VM:
f2061656 166 case KVM_CAP_VM_ATTRIBUTES:
d7b0b5eb
CO		 167 r = 1;
168 break;
e726b1bd
CB		 169 case KVM_CAP_NR_VCPUS:
170 case KVM_CAP_MAX_VCPUS:
171 r = KVM_MAX_VCPUS;
172 break;
e1e2e605
NW		 173 case KVM_CAP_NR_MEMSLOTS:
174 r = KVM_USER_MEM_SLOTS;
175 break;
1526bf9c 176 case KVM_CAP_S390_COW:
abf09bed 177 r = MACHINE_HAS_ESOP;
1526bf9c 178 break;
2bd0ac4e 179 default:
d7b0b5eb 180 r = 0;
2bd0ac4e 181 }
d7b0b5eb 182 return r;
b0c632db
HC		 183}
184
15f36ebd
JH		 185static void kvm_s390_sync_dirty_log(struct kvm *kvm,
186 struct kvm_memory_slot *memslot)
187{
188 gfn_t cur_gfn, last_gfn;
189 unsigned long address;
190 struct gmap *gmap = kvm->arch.gmap;
191
192 down_read(&gmap->mm->mmap_sem);
193 /* Loop over all guest pages */
194 last_gfn = memslot->base_gfn + memslot->npages;
195 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
196 address = gfn_to_hva_memslot(memslot, cur_gfn);
197
198 if (gmap_test_and_clear_dirty(address, gmap))
199 mark_page_dirty(kvm, cur_gfn);
200 }
201 up_read(&gmap->mm->mmap_sem);
202}
203
b0c632db
HC		 204/* Section: vm related */
205/*
206 * Get (and clear) the dirty memory log for a memory slot.
207 */
208int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
209 struct kvm_dirty_log *log)
210{
15f36ebd
JH		 211 int r;
212 unsigned long n;
213 struct kvm_memory_slot *memslot;
214 int is_dirty = 0;
215
216 mutex_lock(&kvm->slots_lock);
217
218 r = -EINVAL;
219 if (log->slot >= KVM_USER_MEM_SLOTS)
220 goto out;
221
222 memslot = id_to_memslot(kvm->memslots, log->slot);
223 r = -ENOENT;
224 if (!memslot->dirty_bitmap)
225 goto out;
226
227 kvm_s390_sync_dirty_log(kvm, memslot);
228 r = kvm_get_dirty_log(kvm, log, &is_dirty);
229 if (r)
230 goto out;
231
232 /* Clear the dirty log */
233 if (is_dirty) {
234 n = kvm_dirty_bitmap_bytes(memslot);
235 memset(memslot->dirty_bitmap, 0, n);
236 }
237 r = 0;
238out:
239 mutex_unlock(&kvm->slots_lock);
240 return r;
b0c632db
HC		 241}
242
d938dc55
CH		 243static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
244{
245 int r;
246
247 if (cap->flags)
248 return -EINVAL;
249
250 switch (cap->cap) {
84223598
CH		 251 case KVM_CAP_S390_IRQCHIP:
252 kvm->arch.use_irqchip = 1;
253 r = 0;
254 break;
d938dc55
CH		 255 default:
256 r = -EINVAL;
257 break;
258 }
259 return r;
260}
261
4f718eab
DD		 262static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
263{
264 int ret;
265 unsigned int idx;
266 switch (attr->attr) {
267 case KVM_S390_VM_MEM_ENABLE_CMMA:
268 ret = -EBUSY;
269 mutex_lock(&kvm->lock);
270 if (atomic_read(&kvm->online_vcpus) == 0) {
271 kvm->arch.use_cmma = 1;
272 ret = 0;
273 }
274 mutex_unlock(&kvm->lock);
275 break;
276 case KVM_S390_VM_MEM_CLR_CMMA:
277 mutex_lock(&kvm->lock);
278 idx = srcu_read_lock(&kvm->srcu);
279 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
280 srcu_read_unlock(&kvm->srcu, idx);
281 mutex_unlock(&kvm->lock);
282 ret = 0;
283 break;
284 default:
285 ret = -ENXIO;
286 break;
287 }
288 return ret;
289}
290
f2061656
DD		 291static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
292{
293 int ret;
294
295 switch (attr->group) {
4f718eab
DD		 296 case KVM_S390_VM_MEM_CTRL:
297 ret = kvm_s390_mem_control(kvm, attr);
298 break;
f2061656
DD		 299 default:
300 ret = -ENXIO;
301 break;
302 }
303
304 return ret;
305}
306
307static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
308{
309 return -ENXIO;
310}
311
312static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
313{
314 int ret;
315
316 switch (attr->group) {
4f718eab
DD		 317 case KVM_S390_VM_MEM_CTRL:
318 switch (attr->attr) {
319 case KVM_S390_VM_MEM_ENABLE_CMMA:
320 case KVM_S390_VM_MEM_CLR_CMMA:
321 ret = 0;
322 break;
323 default:
324 ret = -ENXIO;
325 break;
326 }
327 break;
f2061656
DD		 328 default:
329 ret = -ENXIO;
330 break;
331 }
332
333 return ret;
334}
335
b0c632db
HC		 336long kvm_arch_vm_ioctl(struct file *filp,
337 unsigned int ioctl, unsigned long arg)
338{
339 struct kvm *kvm = filp->private_data;
340 void __user *argp = (void __user *)arg;
f2061656 341 struct kvm_device_attr attr;
b0c632db
HC		 342 int r;
343
344 switch (ioctl) {
ba5c1e9b
CO		 345 case KVM_S390_INTERRUPT: {
346 struct kvm_s390_interrupt s390int;
347
348 r = -EFAULT;
349 if (copy_from_user(&s390int, argp, sizeof(s390int)))
350 break;
351 r = kvm_s390_inject_vm(kvm, &s390int);
352 break;
353 }
d938dc55
CH		 354 case KVM_ENABLE_CAP: {
355 struct kvm_enable_cap cap;
356 r = -EFAULT;
357 if (copy_from_user(&cap, argp, sizeof(cap)))
358 break;
359 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
360 break;
361 }
84223598
CH		 362 case KVM_CREATE_IRQCHIP: {
363 struct kvm_irq_routing_entry routing;
364
365 r = -EINVAL;
366 if (kvm->arch.use_irqchip) {
367 /* Set up dummy routing. */
368 memset(&routing, 0, sizeof(routing));
369 kvm_set_irq_routing(kvm, &routing, 0, 0);
370 r = 0;
371 }
372 break;
373 }
f2061656
DD		 374 case KVM_SET_DEVICE_ATTR: {
375 r = -EFAULT;
376 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
377 break;
378 r = kvm_s390_vm_set_attr(kvm, &attr);
379 break;
380 }
381 case KVM_GET_DEVICE_ATTR: {
382 r = -EFAULT;
383 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
384 break;
385 r = kvm_s390_vm_get_attr(kvm, &attr);
386 break;
387 }
388 case KVM_HAS_DEVICE_ATTR: {
389 r = -EFAULT;
390 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
391 break;
392 r = kvm_s390_vm_has_attr(kvm, &attr);
393 break;
394 }
b0c632db 395 default:
367e1319 396 r = -ENOTTY;
b0c632db
HC		 397 }
398
399 return r;
400}
401
e08b9637 402int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 403{
b0c632db
HC		 404 int rc;
405 char debug_name[16];
f6c137ff 406 static unsigned long sca_offset;
b0c632db 407
e08b9637
CO		 408 rc = -EINVAL;
409#ifdef CONFIG_KVM_S390_UCONTROL
410 if (type & ~KVM_VM_S390_UCONTROL)
411 goto out_err;
412 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
413 goto out_err;
414#else
415 if (type)
416 goto out_err;
417#endif
418
b0c632db
HC		 419 rc = s390_enable_sie();
420 if (rc)
d89f5eff 421 goto out_err;
b0c632db 422
b290411a
CO		 423 rc = -ENOMEM;
424
b0c632db
HC		 425 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
426 if (!kvm->arch.sca)
d89f5eff 427 goto out_err;
f6c137ff
CB		 428 spin_lock(&kvm_lock);
429 sca_offset = (sca_offset + 16) & 0x7f0;
430 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
431 spin_unlock(&kvm_lock);
b0c632db
HC		 432
433 sprintf(debug_name, "kvm-%u", current->pid);
434
435 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
436 if (!kvm->arch.dbf)
437 goto out_nodbf;
438
ba5c1e9b
CO		 439 spin_lock_init(&kvm->arch.float_int.lock);
440 INIT_LIST_HEAD(&kvm->arch.float_int.list);
8a242234 441 init_waitqueue_head(&kvm->arch.ipte_wq);
ba5c1e9b 442
b0c632db
HC		 443 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
444 VM_EVENT(kvm, 3, "%s", "vm created");
445
e08b9637
CO		 446 if (type & KVM_VM_S390_UCONTROL) {
447 kvm->arch.gmap = NULL;
448 } else {
449 kvm->arch.gmap = gmap_alloc(current->mm);
450 if (!kvm->arch.gmap)
451 goto out_nogmap;
2c70fe44 452 kvm->arch.gmap->private = kvm;
24eb3a82 453 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 454 }
fa6b7fe9
CH		 455
456 kvm->arch.css_support = 0;
84223598 457 kvm->arch.use_irqchip = 0;
fa6b7fe9 458
d89f5eff 459 return 0;
598841ca
CO		 460out_nogmap:
461 debug_unregister(kvm->arch.dbf);
b0c632db
HC		 462out_nodbf:
463 free_page((unsigned long)(kvm->arch.sca));
d89f5eff
JK		 464out_err:
465 return rc;
b0c632db
HC		 466}
467
d329c035
CB		 468void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
469{
470 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 471 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
3c038e6b 472 kvm_clear_async_pf_completion_queue(vcpu);
58f9460b
CO		 473 if (!kvm_is_ucontrol(vcpu->kvm)) {
474 clear_bit(63 - vcpu->vcpu_id,
475 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
476 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
477 (__u64) vcpu->arch.sie_block)
478 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
479 }
abf4a71e 480 smp_mb();
27e0393f
CO		 481
482 if (kvm_is_ucontrol(vcpu->kvm))
483 gmap_free(vcpu->arch.gmap);
484
b31605c1
DD		 485 if (kvm_s390_cmma_enabled(vcpu->kvm))
486 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 487 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 488
6692cef3 489 kvm_vcpu_uninit(vcpu);
b110feaf 490 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 491}
492
493static void kvm_free_vcpus(struct kvm *kvm)
494{
495 unsigned int i;
988a2cae 496 struct kvm_vcpu *vcpu;
d329c035 497
988a2cae
GN		 498 kvm_for_each_vcpu(i, vcpu, kvm)
499 kvm_arch_vcpu_destroy(vcpu);
500
501 mutex_lock(&kvm->lock);
502 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
503 kvm->vcpus[i] = NULL;
504
505 atomic_set(&kvm->online_vcpus, 0);
506 mutex_unlock(&kvm->lock);
d329c035
CB		 507}
508
ad8ba2cd
SY		 509void kvm_arch_sync_events(struct kvm *kvm)
510{
511}
512
b0c632db
HC		 513void kvm_arch_destroy_vm(struct kvm *kvm)
514{
d329c035 515 kvm_free_vcpus(kvm);
b0c632db 516 free_page((unsigned long)(kvm->arch.sca));
d329c035 517 debug_unregister(kvm->arch.dbf);
27e0393f
CO		 518 if (!kvm_is_ucontrol(kvm))
519 gmap_free(kvm->arch.gmap);
841b91c5 520 kvm_s390_destroy_adapters(kvm);
b0c632db
HC		 521}
522
523/* Section: vcpu related */
524int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
525{
3c038e6b
DD		 526 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
527 kvm_clear_async_pf_completion_queue(vcpu);
27e0393f
CO		 528 if (kvm_is_ucontrol(vcpu->kvm)) {
529 vcpu->arch.gmap = gmap_alloc(current->mm);
530 if (!vcpu->arch.gmap)
531 return -ENOMEM;
2c70fe44 532 vcpu->arch.gmap->private = vcpu->kvm;
27e0393f
CO		 533 return 0;
534 }
535
598841ca 536 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
59674c1a
CB		 537 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
538 KVM_SYNC_GPRS |
9eed0735
CB		 539 KVM_SYNC_ACRS |
540 KVM_SYNC_CRS;
b0c632db
HC		 541 return 0;
542}
543
544void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
545{
6692cef3 546 /* Nothing todo */
b0c632db
HC		 547}
548
549void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
550{
4725c860
MS		 551 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
552 save_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db 553 save_access_regs(vcpu->arch.host_acrs);
4725c860
MS		 554 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
555 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 556 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 557 gmap_enable(vcpu->arch.gmap);
9e6dabef 558 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 559}
560
561void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
562{
9e6dabef 563 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 564 gmap_disable(vcpu->arch.gmap);
4725c860
MS		 565 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
566 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 567 save_access_regs(vcpu->run->s.regs.acrs);
4725c860
MS		 568 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
569 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db
HC		 570 restore_access_regs(vcpu->arch.host_acrs);
571}
572
573static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
574{
575 /* this equals initial cpu reset in pop, but we don't switch to ESA */
576 vcpu->arch.sie_block->gpsw.mask = 0UL;
577 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 578 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 579 vcpu->arch.sie_block->cputm = 0UL;
580 vcpu->arch.sie_block->ckc = 0UL;
581 vcpu->arch.sie_block->todpr = 0;
582 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
583 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
584 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
585 vcpu->arch.guest_fpregs.fpc = 0;
586 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
587 vcpu->arch.sie_block->gbea = 1;
672550fb 588 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 589 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
590 kvm_clear_async_pf_completion_queue(vcpu);
61bde82c 591 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2ed10cc1 592 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 593}
594
42897d86
MT		 595int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
596{
597 return 0;
598}
599
b31605c1
DD		 600void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
601{
602 free_page(vcpu->arch.sie_block->cbrlo);
603 vcpu->arch.sie_block->cbrlo = 0;
604}
605
606int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
607{
608 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
609 if (!vcpu->arch.sie_block->cbrlo)
610 return -ENOMEM;
611
612 vcpu->arch.sie_block->ecb2 |= 0x80;
613 vcpu->arch.sie_block->ecb2 &= ~0x08;
614 return 0;
615}
616
b0c632db
HC		 617int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
618{
b31605c1 619 int rc = 0;
b31288fa 620
9e6dabef
CH		 621 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
622 CPUSTAT_SM |
69d0d3a3
CB		 623 CPUSTAT_STOPPED |
624 CPUSTAT_GED);
fc34531d 625 vcpu->arch.sie_block->ecb = 6;
7feb6bb8
MM		 626 if (test_vfacility(50) && test_vfacility(73))
627 vcpu->arch.sie_block->ecb |= 0x10;
628
69d0d3a3 629 vcpu->arch.sie_block->ecb2 = 8;
217a4406
HC		 630 vcpu->arch.sie_block->eca = 0xC1002000U;
631 if (sclp_has_siif())
632 vcpu->arch.sie_block->eca |= 1;
78c4b59f 633 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
693ffc08 634 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
b31605c1
DD		 635 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
636 rc = kvm_s390_vcpu_setup_cmma(vcpu);
637 if (rc)
638 return rc;
b31288fa 639 }
ca872302
CB		 640 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
641 tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
642 (unsigned long) vcpu);
643 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
453423dc 644 get_cpu_id(&vcpu->arch.cpu_id);
92e6ecf3 645 vcpu->arch.cpu_id.version = 0xff;
b31605c1 646 return rc;
b0c632db
HC		 647}
648
649struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
650 unsigned int id)
651{
4d47555a 652 struct kvm_vcpu *vcpu;
7feb6bb8 653 struct sie_page *sie_page;
4d47555a
CO		 654 int rc = -EINVAL;
655
656 if (id >= KVM_MAX_VCPUS)
657 goto out;
658
659 rc = -ENOMEM;
b0c632db 660
b110feaf 661 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 662 if (!vcpu)
4d47555a 663 goto out;
b0c632db 664
7feb6bb8
MM		 665 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
666 if (!sie_page)
b0c632db
HC		 667 goto out_free_cpu;
668
7feb6bb8
MM		 669 vcpu->arch.sie_block = &sie_page->sie_block;
670 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
671
b0c632db 672 vcpu->arch.sie_block->icpua = id;
58f9460b
CO		 673 if (!kvm_is_ucontrol(kvm)) {
674 if (!kvm->arch.sca) {
675 WARN_ON_ONCE(1);
676 goto out_free_cpu;
677 }
678 if (!kvm->arch.sca->cpu[id].sda)
679 kvm->arch.sca->cpu[id].sda =
680 (__u64) vcpu->arch.sie_block;
681 vcpu->arch.sie_block->scaoh =
682 (__u32)(((__u64)kvm->arch.sca) >> 32);
683 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
684 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
685 }
b0c632db 686
ba5c1e9b
CO		 687 spin_lock_init(&vcpu->arch.local_int.lock);
688 INIT_LIST_HEAD(&vcpu->arch.local_int.list);
689 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 690 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 691 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 692
b0c632db
HC		 693 rc = kvm_vcpu_init(vcpu, kvm, id);
694 if (rc)
7b06bf2f 695 goto out_free_sie_block;
b0c632db
HC		 696 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
697 vcpu->arch.sie_block);
ade38c31 698 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 699
b0c632db 700 return vcpu;
7b06bf2f
WY		 701out_free_sie_block:
702 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 703out_free_cpu:
b110feaf 704 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 705out:
b0c632db
HC		 706 return ERR_PTR(rc);
707}
708
b0c632db
HC		 709int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
710{
f87618e8 711 return kvm_cpu_has_interrupt(vcpu);
b0c632db
HC		 712}
713
49b99e1e
CB		 714void s390_vcpu_block(struct kvm_vcpu *vcpu)
715{
716 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
717}
718
719void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
720{
721 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
722}
723
724/*
725 * Kick a guest cpu out of SIE and wait until SIE is not running.
726 * If the CPU is not running (e.g. waiting as idle) the function will
727 * return immediately. */
728void exit_sie(struct kvm_vcpu *vcpu)
729{
730 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
731 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
732 cpu_relax();
733}
734
735/* Kick a guest cpu out of SIE and prevent SIE-reentry */
736void exit_sie_sync(struct kvm_vcpu *vcpu)
737{
738 s390_vcpu_block(vcpu);
739 exit_sie(vcpu);
740}
741
2c70fe44
CB		 742static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
743{
744 int i;
745 struct kvm *kvm = gmap->private;
746 struct kvm_vcpu *vcpu;
747
748 kvm_for_each_vcpu(i, vcpu, kvm) {
749 /* match against both prefix pages */
750 if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) {
751 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
752 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
753 exit_sie_sync(vcpu);
754 }
755 }
756}
757
b6d33834
CD		 758int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
759{
760 /* kvm common code refers to this, but never calls it */
761 BUG();
762 return 0;
763}
764
14eebd91
CO		 765static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
766 struct kvm_one_reg *reg)
767{
768 int r = -EINVAL;
769
770 switch (reg->id) {
29b7c71b
CO		 771 case KVM_REG_S390_TODPR:
772 r = put_user(vcpu->arch.sie_block->todpr,
773 (u32 __user *)reg->addr);
774 break;
775 case KVM_REG_S390_EPOCHDIFF:
776 r = put_user(vcpu->arch.sie_block->epoch,
777 (u64 __user *)reg->addr);
778 break;
46a6dd1c
J		 779 case KVM_REG_S390_CPU_TIMER:
780 r = put_user(vcpu->arch.sie_block->cputm,
781 (u64 __user *)reg->addr);
782 break;
783 case KVM_REG_S390_CLOCK_COMP:
784 r = put_user(vcpu->arch.sie_block->ckc,
785 (u64 __user *)reg->addr);
786 break;
536336c2
DD		 787 case KVM_REG_S390_PFTOKEN:
788 r = put_user(vcpu->arch.pfault_token,
789 (u64 __user *)reg->addr);
790 break;
791 case KVM_REG_S390_PFCOMPARE:
792 r = put_user(vcpu->arch.pfault_compare,
793 (u64 __user *)reg->addr);
794 break;
795 case KVM_REG_S390_PFSELECT:
796 r = put_user(vcpu->arch.pfault_select,
797 (u64 __user *)reg->addr);
798 break;
672550fb
CB		 799 case KVM_REG_S390_PP:
800 r = put_user(vcpu->arch.sie_block->pp,
801 (u64 __user *)reg->addr);
802 break;
afa45ff5
CB		 803 case KVM_REG_S390_GBEA:
804 r = put_user(vcpu->arch.sie_block->gbea,
805 (u64 __user *)reg->addr);
806 break;
14eebd91
CO		 807 default:
808 break;
809 }
810
811 return r;
812}
813
814static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
815 struct kvm_one_reg *reg)
816{
817 int r = -EINVAL;
818
819 switch (reg->id) {
29b7c71b
CO		 820 case KVM_REG_S390_TODPR:
821 r = get_user(vcpu->arch.sie_block->todpr,
822 (u32 __user *)reg->addr);
823 break;
824 case KVM_REG_S390_EPOCHDIFF:
825 r = get_user(vcpu->arch.sie_block->epoch,
826 (u64 __user *)reg->addr);
827 break;
46a6dd1c
J		 828 case KVM_REG_S390_CPU_TIMER:
829 r = get_user(vcpu->arch.sie_block->cputm,
830 (u64 __user *)reg->addr);
831 break;
832 case KVM_REG_S390_CLOCK_COMP:
833 r = get_user(vcpu->arch.sie_block->ckc,
834 (u64 __user *)reg->addr);
835 break;
536336c2
DD		 836 case KVM_REG_S390_PFTOKEN:
837 r = get_user(vcpu->arch.pfault_token,
838 (u64 __user *)reg->addr);
839 break;
840 case KVM_REG_S390_PFCOMPARE:
841 r = get_user(vcpu->arch.pfault_compare,
842 (u64 __user *)reg->addr);
843 break;
844 case KVM_REG_S390_PFSELECT:
845 r = get_user(vcpu->arch.pfault_select,
846 (u64 __user *)reg->addr);
847 break;
672550fb
CB		 848 case KVM_REG_S390_PP:
849 r = get_user(vcpu->arch.sie_block->pp,
850 (u64 __user *)reg->addr);
851 break;
afa45ff5
CB		 852 case KVM_REG_S390_GBEA:
853 r = get_user(vcpu->arch.sie_block->gbea,
854 (u64 __user *)reg->addr);
855 break;
14eebd91
CO		 856 default:
857 break;
858 }
859
860 return r;
861}
b6d33834 862
b0c632db
HC		 863static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
864{
b0c632db 865 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 866 return 0;
867}
868
869int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
870{
5a32c1af 871 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 872 return 0;
873}
874
875int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
876{
5a32c1af 877 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 878 return 0;
879}
880
881int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
882 struct kvm_sregs *sregs)
883{
59674c1a 884 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 885 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 886 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 887 return 0;
888}
889
890int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
891 struct kvm_sregs *sregs)
892{
59674c1a 893 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 894 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 895 return 0;
896}
897
898int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
899{
4725c860
MS		 900 if (test_fp_ctl(fpu->fpc))
901 return -EINVAL;
b0c632db 902 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860
MS		 903 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
904 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
905 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
b0c632db
HC		 906 return 0;
907}
908
909int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
910{
b0c632db
HC		 911 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
912 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 913 return 0;
914}
915
916static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
917{
918 int rc = 0;
919
9e6dabef 920 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED))
b0c632db 921 rc = -EBUSY;
d7b0b5eb
CO		 922 else {
923 vcpu->run->psw_mask = psw.mask;
924 vcpu->run->psw_addr = psw.addr;
925 }
b0c632db
HC		 926 return rc;
927}
928
929int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
930 struct kvm_translation *tr)
931{
932 return -EINVAL; /* not implemented yet */
933}
934
d0bfb940
JK		 935int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
936 struct kvm_guest_debug *dbg)
b0c632db
HC		 937{
938 return -EINVAL; /* not implemented yet */
939}
940
62d9f0db
MT		 941int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
942 struct kvm_mp_state *mp_state)
943{
944 return -EINVAL; /* not implemented yet */
945}
946
947int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
948 struct kvm_mp_state *mp_state)
949{
950 return -EINVAL; /* not implemented yet */
951}
952
b31605c1
DD		 953bool kvm_s390_cmma_enabled(struct kvm *kvm)
954{
955 if (!MACHINE_IS_LPAR)
956 return false;
957 /* only enable for z10 and later */
958 if (!MACHINE_HAS_EDAT1)
959 return false;
960 if (!kvm->arch.use_cmma)
961 return false;
962 return true;
963}
964
2c70fe44
CB		 965static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
966{
967 /*
968 * We use MMU_RELOAD just to re-arm the ipte notifier for the
969 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
970 * This ensures that the ipte instruction for this request has
971 * already finished. We might race against a second unmapper that
972 * wants to set the blocking bit. Lets just retry the request loop.
973 */
974 while (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
975 int rc;
976 rc = gmap_ipte_notify(vcpu->arch.gmap,
977 vcpu->arch.sie_block->prefix,
978 PAGE_SIZE * 2);
979 if (rc)
980 return rc;
981 s390_vcpu_unblock(vcpu);
982 }
983 return 0;
984}
985
24eb3a82
DD		 986static long kvm_arch_fault_in_sync(struct kvm_vcpu *vcpu)
987{
988 long rc;
989 hva_t fault = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
990 struct mm_struct *mm = current->mm;
991 down_read(&mm->mmap_sem);
992 rc = get_user_pages(current, mm, fault, 1, 1, 0, NULL, NULL);
993 up_read(&mm->mmap_sem);
994 return rc;
995}
996
3c038e6b
DD		 997static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
998 unsigned long token)
999{
1000 struct kvm_s390_interrupt inti;
1001 inti.parm64 = token;
1002
1003 if (start_token) {
1004 inti.type = KVM_S390_INT_PFAULT_INIT;
1005 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1006 } else {
1007 inti.type = KVM_S390_INT_PFAULT_DONE;
1008 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1009 }
1010}
1011
1012void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1013 struct kvm_async_pf *work)
1014{
1015 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1016 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1017}
1018
1019void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1020 struct kvm_async_pf *work)
1021{
1022 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1023 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1024}
1025
1026void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1027 struct kvm_async_pf *work)
1028{
1029 /* s390 will always inject the page directly */
1030}
1031
1032bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1033{
1034 /*
1035 * s390 will always inject the page directly,
1036 * but we still want check_async_completion to cleanup
1037 */
1038 return true;
1039}
1040
1041static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1042{
1043 hva_t hva;
1044 struct kvm_arch_async_pf arch;
1045 int rc;
1046
1047 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1048 return 0;
1049 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1050 vcpu->arch.pfault_compare)
1051 return 0;
1052 if (psw_extint_disabled(vcpu))
1053 return 0;
1054 if (kvm_cpu_has_interrupt(vcpu))
1055 return 0;
1056 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1057 return 0;
1058 if (!vcpu->arch.gmap->pfault_enabled)
1059 return 0;
1060
81480cc1
HC		 1061 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1062 hva += current->thread.gmap_addr & ~PAGE_MASK;
1063 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 1064 return 0;
1065
1066 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1067 return rc;
1068}
1069
3fb4c40f 1070static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 1071{
3fb4c40f 1072 int rc, cpuflags;
e168bf8d 1073
3c038e6b
DD		 1074 /*
1075 * On s390 notifications for arriving pages will be delivered directly
1076 * to the guest but the house keeping for completed pfaults is
1077 * handled outside the worker.
1078 */
1079 kvm_check_async_pf_completion(vcpu);
1080
5a32c1af 1081 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
b0c632db
HC		 1082
1083 if (need_resched())
1084 schedule();
1085
71cde587
CB		 1086 if (test_thread_flag(TIF_MCCK_PENDING))
1087 s390_handle_mcck();
1088
d6b6d166
CO		 1089 if (!kvm_is_ucontrol(vcpu->kvm))
1090 kvm_s390_deliver_pending_interrupts(vcpu);
0ff31867 1091
2c70fe44
CB		 1092 rc = kvm_s390_handle_requests(vcpu);
1093 if (rc)
1094 return rc;
1095
b0c632db 1096 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 1097 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1098 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1099 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 1100
3fb4c40f
TH		 1101 return 0;
1102}
1103
1104static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1105{
24eb3a82 1106 int rc = -1;
2b29a9fd
DD		 1107
1108 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1109 vcpu->arch.sie_block->icptcode);
1110 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1111
3fb4c40f 1112 if (exit_reason >= 0) {
7c470539 1113 rc = 0;
210b1607
TH		 1114 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1115 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1116 vcpu->run->s390_ucontrol.trans_exc_code =
1117 current->thread.gmap_addr;
1118 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1119 rc = -EREMOTE;
24eb3a82
DD		 1120
1121 } else if (current->thread.gmap_pfault) {
3c038e6b 1122 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 1123 current->thread.gmap_pfault = 0;
3c038e6b
DD		 1124 if (kvm_arch_setup_async_pf(vcpu) ||
1125 (kvm_arch_fault_in_sync(vcpu) >= 0))
24eb3a82
DD		 1126 rc = 0;
1127 }
1128
1129 if (rc == -1) {
699bde3b
CB		 1130 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1131 trace_kvm_s390_sie_fault(vcpu);
1132 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1f0d0f09 1133 }
b0c632db 1134
5a32c1af 1135 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
3fb4c40f 1136
a76ccff6
TH		 1137 if (rc == 0) {
1138 if (kvm_is_ucontrol(vcpu->kvm))
2955c83f
CB		 1139 /* Don't exit for host interrupts. */
1140 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
a76ccff6
TH		 1141 else
1142 rc = kvm_handle_sie_intercept(vcpu);
1143 }
1144
3fb4c40f
TH		 1145 return rc;
1146}
1147
1148static int __vcpu_run(struct kvm_vcpu *vcpu)
1149{
1150 int rc, exit_reason;
1151
800c1065
TH		 1152 /*
1153 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1154 * ning the guest), so that memslots (and other stuff) are protected
1155 */
1156 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1157
a76ccff6
TH		 1158 do {
1159 rc = vcpu_pre_run(vcpu);
1160 if (rc)
1161 break;
3fb4c40f 1162
800c1065 1163 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 1164 /*
1165 * As PF_VCPU will be used in fault handler, between
1166 * guest_enter and guest_exit should be no uaccess.
1167 */
1168 preempt_disable();
1169 kvm_guest_enter();
1170 preempt_enable();
1171 exit_reason = sie64a(vcpu->arch.sie_block,
1172 vcpu->run->s.regs.gprs);
1173 kvm_guest_exit();
800c1065 1174 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 1175
1176 rc = vcpu_post_run(vcpu, exit_reason);
1177 } while (!signal_pending(current) && !rc);
3fb4c40f 1178
800c1065 1179 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 1180 return rc;
b0c632db
HC		 1181}
1182
1183int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1184{
8f2abe6a 1185 int rc;
b0c632db
HC		 1186 sigset_t sigsaved;
1187
b0c632db
HC		 1188 if (vcpu->sigset_active)
1189 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1190
9e6dabef 1191 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
b0c632db 1192
8f2abe6a
CB		 1193 switch (kvm_run->exit_reason) {
1194 case KVM_EXIT_S390_SIEIC:
8f2abe6a 1195 case KVM_EXIT_UNKNOWN:
9ace903d 1196 case KVM_EXIT_INTR:
8f2abe6a 1197 case KVM_EXIT_S390_RESET:
e168bf8d 1198 case KVM_EXIT_S390_UCONTROL:
fa6b7fe9 1199 case KVM_EXIT_S390_TSCH:
8f2abe6a
CB		 1200 break;
1201 default:
1202 BUG();
1203 }
1204
d7b0b5eb
CO		 1205 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1206 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
60b413c9
CB		 1207 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) {
1208 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX;
1209 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1210 }
9eed0735
CB		 1211 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1212 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS;
1213 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1214 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1215 }
d7b0b5eb 1216
dab4079d 1217 might_fault();
a76ccff6 1218 rc = __vcpu_run(vcpu);
9ace903d 1219
b1d16c49
CE		 1220 if (signal_pending(current) && !rc) {
1221 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 1222 rc = -EINTR;
b1d16c49 1223 }
8f2abe6a 1224
b8e660b8 1225 if (rc == -EOPNOTSUPP) {
8f2abe6a
CB		 1226 /* intercept cannot be handled in-kernel, prepare kvm-run */
1227 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1228 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
8f2abe6a
CB		 1229 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1230 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1231 rc = 0;
1232 }
1233
1234 if (rc == -EREMOTE) {
1235 /* intercept was handled, but userspace support is needed
1236 * kvm_run has been prepared by the handler */
1237 rc = 0;
1238 }
b0c632db 1239
d7b0b5eb
CO		 1240 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1241 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
60b413c9 1242 kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix;
9eed0735 1243 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
d7b0b5eb 1244
b0c632db
HC		 1245 if (vcpu->sigset_active)
1246 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1247
b0c632db 1248 vcpu->stat.exit_userspace++;
7e8e6ab4 1249 return rc;
b0c632db
HC		 1250}
1251
b0c632db
HC		 1252/*
1253 * store status at address
1254 * we use have two special cases:
1255 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1256 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1257 */
d0bce605 1258int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 1259{
092670cd 1260 unsigned char archmode = 1;
178bd789 1261 u64 clkcomp;
d0bce605 1262 int rc;
b0c632db 1263
d0bce605
HC		 1264 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1265 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 1266 return -EFAULT;
d0bce605
HC		 1267 gpa = SAVE_AREA_BASE;
1268 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1269 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 1270 return -EFAULT;
d0bce605
HC		 1271 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1272 }
1273 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1274 vcpu->arch.guest_fpregs.fprs, 128);
1275 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1276 vcpu->run->s.regs.gprs, 128);
1277 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1278 &vcpu->arch.sie_block->gpsw, 16);
1279 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1280 &vcpu->arch.sie_block->prefix, 4);
1281 rc |= write_guest_abs(vcpu,
1282 gpa + offsetof(struct save_area, fp_ctrl_reg),
1283 &vcpu->arch.guest_fpregs.fpc, 4);
1284 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1285 &vcpu->arch.sie_block->todpr, 4);
1286 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1287 &vcpu->arch.sie_block->cputm, 8);
178bd789 1288 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 1289 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1290 &clkcomp, 8);
1291 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1292 &vcpu->run->s.regs.acrs, 64);
1293 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1294 &vcpu->arch.sie_block->gcr, 128);
1295 return rc ? -EFAULT : 0;
b0c632db
HC		 1296}
1297
e879892c
TH		 1298int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1299{
1300 /*
1301 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1302 * copying in vcpu load/put. Lets update our copies before we save
1303 * it into the save area
1304 */
1305 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1306 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1307 save_access_regs(vcpu->run->s.regs.acrs);
1308
1309 return kvm_s390_store_status_unloaded(vcpu, addr);
1310}
1311
d6712df9
CH		 1312static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1313 struct kvm_enable_cap *cap)
1314{
1315 int r;
1316
1317 if (cap->flags)
1318 return -EINVAL;
1319
1320 switch (cap->cap) {
fa6b7fe9
CH		 1321 case KVM_CAP_S390_CSS_SUPPORT:
1322 if (!vcpu->kvm->arch.css_support) {
1323 vcpu->kvm->arch.css_support = 1;
1324 trace_kvm_s390_enable_css(vcpu->kvm);
1325 }
1326 r = 0;
1327 break;
d6712df9
CH		 1328 default:
1329 r = -EINVAL;
1330 break;
1331 }
1332 return r;
1333}
1334
b0c632db
HC		 1335long kvm_arch_vcpu_ioctl(struct file *filp,
1336 unsigned int ioctl, unsigned long arg)
1337{
1338 struct kvm_vcpu *vcpu = filp->private_data;
1339 void __user *argp = (void __user *)arg;
800c1065 1340 int idx;
bc923cc9 1341 long r;
b0c632db 1342
93736624
AK		 1343 switch (ioctl) {
1344 case KVM_S390_INTERRUPT: {
ba5c1e9b
CO		 1345 struct kvm_s390_interrupt s390int;
1346
93736624 1347 r = -EFAULT;
ba5c1e9b 1348 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624
AK		 1349 break;
1350 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1351 break;
ba5c1e9b 1352 }
b0c632db 1353 case KVM_S390_STORE_STATUS:
800c1065 1354 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 1355 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 1356 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 1357 break;
b0c632db
HC		 1358 case KVM_S390_SET_INITIAL_PSW: {
1359 psw_t psw;
1360
bc923cc9 1361 r = -EFAULT;
b0c632db 1362 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 1363 break;
1364 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1365 break;
b0c632db
HC		 1366 }
1367 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 1368 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1369 break;
14eebd91
CO		 1370 case KVM_SET_ONE_REG:
1371 case KVM_GET_ONE_REG: {
1372 struct kvm_one_reg reg;
1373 r = -EFAULT;
1374 if (copy_from_user(&reg, argp, sizeof(reg)))
1375 break;
1376 if (ioctl == KVM_SET_ONE_REG)
1377 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1378 else
1379 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1380 break;
1381 }
27e0393f
CO		 1382#ifdef CONFIG_KVM_S390_UCONTROL
1383 case KVM_S390_UCAS_MAP: {
1384 struct kvm_s390_ucas_mapping ucasmap;
1385
1386 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1387 r = -EFAULT;
1388 break;
1389 }
1390
1391 if (!kvm_is_ucontrol(vcpu->kvm)) {
1392 r = -EINVAL;
1393 break;
1394 }
1395
1396 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1397 ucasmap.vcpu_addr, ucasmap.length);
1398 break;
1399 }
1400 case KVM_S390_UCAS_UNMAP: {
1401 struct kvm_s390_ucas_mapping ucasmap;
1402
1403 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1404 r = -EFAULT;
1405 break;
1406 }
1407
1408 if (!kvm_is_ucontrol(vcpu->kvm)) {
1409 r = -EINVAL;
1410 break;
1411 }
1412
1413 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1414 ucasmap.length);
1415 break;
1416 }
1417#endif
ccc7910f
CO		 1418 case KVM_S390_VCPU_FAULT: {
1419 r = gmap_fault(arg, vcpu->arch.gmap);
1420 if (!IS_ERR_VALUE(r))
1421 r = 0;
1422 break;
1423 }
d6712df9
CH		 1424 case KVM_ENABLE_CAP:
1425 {
1426 struct kvm_enable_cap cap;
1427 r = -EFAULT;
1428 if (copy_from_user(&cap, argp, sizeof(cap)))
1429 break;
1430 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1431 break;
1432 }
b0c632db 1433 default:
3e6afcf1 1434 r = -ENOTTY;
b0c632db 1435 }
bc923cc9 1436 return r;
b0c632db
HC		 1437}
1438
5b1c1493
CO		 1439int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1440{
1441#ifdef CONFIG_KVM_S390_UCONTROL
1442 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1443 && (kvm_is_ucontrol(vcpu->kvm))) {
1444 vmf->page = virt_to_page(vcpu->arch.sie_block);
1445 get_page(vmf->page);
1446 return 0;
1447 }
1448#endif
1449 return VM_FAULT_SIGBUS;
1450}
1451
5587027c 1452void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
db3fe4eb
TY		 1453 struct kvm_memory_slot *dont)
1454{
1455}
1456
5587027c
AK		 1457int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1458 unsigned long npages)
db3fe4eb
TY		 1459{
1460 return 0;
1461}
1462
e59dbe09
TY		 1463void kvm_arch_memslots_updated(struct kvm *kvm)
1464{
1465}
1466
b0c632db 1467/* Section: memory related */
f7784b8e
MT		 1468int kvm_arch_prepare_memory_region(struct kvm *kvm,
1469 struct kvm_memory_slot *memslot,
7b6195a9
TY		 1470 struct kvm_userspace_memory_region *mem,
1471 enum kvm_mr_change change)
b0c632db 1472{
dd2887e7
NW		 1473 /* A few sanity checks. We can have memory slots which have to be
1474 located/ended at a segment boundary (1MB). The memory in userland is
1475 ok to be fragmented into various different vmas. It is okay to mmap()
1476 and munmap() stuff in this slot after doing this call at any time */
b0c632db 1477
598841ca 1478 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 1479 return -EINVAL;
1480
598841ca 1481 if (mem->memory_size & 0xffffful)
b0c632db
HC		 1482 return -EINVAL;
1483
f7784b8e
MT		 1484 return 0;
1485}
1486
1487void kvm_arch_commit_memory_region(struct kvm *kvm,
1488 struct kvm_userspace_memory_region *mem,
8482644a
TY		 1489 const struct kvm_memory_slot *old,
1490 enum kvm_mr_change change)
f7784b8e 1491{
f7850c92 1492 int rc;
f7784b8e 1493
2cef4deb
CB		 1494 /* If the basics of the memslot do not change, we do not want
1495 * to update the gmap. Every update causes several unnecessary
1496 * segment translation exceptions. This is usually handled just
1497 * fine by the normal fault handler + gmap, but it will also
1498 * cause faults on the prefix page of running guest CPUs.
1499 */
1500 if (old->userspace_addr == mem->userspace_addr &&
1501 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1502 old->npages * PAGE_SIZE == mem->memory_size)
1503 return;
598841ca
CO		 1504
1505 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1506 mem->guest_phys_addr, mem->memory_size);
1507 if (rc)
f7850c92 1508 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
598841ca 1509 return;
b0c632db
HC		 1510}
1511
2df72e9b
MT		 1512void kvm_arch_flush_shadow_all(struct kvm *kvm)
1513{
1514}
1515
1516void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1517 struct kvm_memory_slot *slot)
34d4cb8f
MT		 1518{
1519}
1520
b0c632db
HC		 1521static int __init kvm_s390_init(void)
1522{
ef50f7ac 1523 int ret;
0ee75bea 1524 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
ef50f7ac
CB		 1525 if (ret)
1526 return ret;
1527
1528 /*
1529 * guests can ask for up to 255+1 double words, we need a full page
25985edc 1530 * to hold the maximum amount of facilities. On the other hand, we
ef50f7ac
CB		 1531 * only set facilities that are known to work in KVM.
1532 */
78c4b59f
MM		 1533 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1534 if (!vfacilities) {
ef50f7ac
CB		 1535 kvm_exit();
1536 return -ENOMEM;
1537 }
78c4b59f 1538 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
d208c79d 1539 vfacilities[0] &= 0xff82fff3f4fc2000UL;
7feb6bb8 1540 vfacilities[1] &= 0x005c000000000000UL;
ef50f7ac 1541 return 0;
b0c632db
HC		 1542}
1543
1544static void __exit kvm_s390_exit(void)
1545{
78c4b59f 1546 free_page((unsigned long) vfacilities);
b0c632db
HC		 1547 kvm_exit();
1548}
1549
1550module_init(kvm_s390_init);
1551module_exit(kvm_s390_exit);
566af940
CH		 1552
1553/*
1554 * Enable autoloading of the kvm module.
1555 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1556 * since x86 takes a different approach.
1557 */
1558#include <linux/miscdevice.h>
1559MODULE_ALIAS_MISCDEV(KVM_MINOR);
1560MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)