projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
KVM: s390: kernel header addition for guest debugging
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
CommitLineData
b0c632db 1/*
a53c8fab 2 * hosting zSeries kernel virtual machines
b0c632db 3 *
a53c8fab 4 * Copyright IBM Corp. 2008, 2009
b0c632db
HC		 5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
628eb9b8 13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
15f36ebd 14 * Jason J. Herne <jjherne@us.ibm.com>
b0c632db
HC		 15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
ca872302 20#include <linux/hrtimer.h>
b0c632db
HC		 21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
25#include <linux/slab.h>
ba5c1e9b 26#include <linux/timer.h>
cbb870c8 27#include <asm/asm-offsets.h>
b0c632db
HC		 28#include <asm/lowcore.h>
29#include <asm/pgtable.h>
f5daba1d 30#include <asm/nmi.h>
a0616cde 31#include <asm/switch_to.h>
78c4b59f 32#include <asm/facility.h>
1526bf9c 33#include <asm/sclp.h>
8f2abe6a 34#include "kvm-s390.h"
b0c632db
HC		 35#include "gaccess.h"
36
5786fffa
CH		 37#define CREATE_TRACE_POINTS
38#include "trace.h"
ade38c31 39#include "trace-s390.h"
5786fffa 40
b0c632db
HC		 41#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
42
43struct kvm_stats_debugfs_item debugfs_entries[] = {
44 { "userspace_handled", VCPU_STAT(exit_userspace) },
0eaeafa1 45 { "exit_null", VCPU_STAT(exit_null) },
8f2abe6a
CB		 46 { "exit_validity", VCPU_STAT(exit_validity) },
47 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
48 { "exit_external_request", VCPU_STAT(exit_external_request) },
49 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
ba5c1e9b
CO		 50 { "exit_instruction", VCPU_STAT(exit_instruction) },
51 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
52 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
f5e10b09 53 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
ba5c1e9b 54 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
aba07508
DH		 55 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
56 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
ba5c1e9b 57 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
7697e71f 58 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
ba5c1e9b
CO		 59 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
60 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
61 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
62 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
63 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
64 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
65 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
69d0d3a3 66 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
453423dc
CB		 67 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
68 { "instruction_spx", VCPU_STAT(instruction_spx) },
69 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
70 { "instruction_stap", VCPU_STAT(instruction_stap) },
71 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
8a242234 72 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
453423dc
CB		 73 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
74 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
b31288fa 75 { "instruction_essa", VCPU_STAT(instruction_essa) },
453423dc
CB		 76 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
77 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
bb25b9ba 78 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
5288fbf0 79 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
bd59d3a4 80 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
7697e71f 81 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
5288fbf0
CB		 82 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
83 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
84 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
85 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
86 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
388186bc 87 { "diagnose_10", VCPU_STAT(diagnose_10) },
e28acfea 88 { "diagnose_44", VCPU_STAT(diagnose_44) },
41628d33 89 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
b0c632db
HC		 90 { NULL }
91};
92
78c4b59f 93unsigned long *vfacilities;
2c70fe44 94static struct gmap_notifier gmap_notifier;
b0c632db 95
78c4b59f 96/* test availability of vfacility */
280ef0f1 97int test_vfacility(unsigned long nr)
78c4b59f
MM		 98{
99 return __test_facility(nr, (void *) vfacilities);
100}
101
b0c632db 102/* Section: not file related */
10474ae8 103int kvm_arch_hardware_enable(void *garbage)
b0c632db
HC		 104{
105 /* every s390 is virtualization enabled ;-) */
10474ae8 106 return 0;
b0c632db
HC		 107}
108
109void kvm_arch_hardware_disable(void *garbage)
110{
111}
112
2c70fe44
CB		 113static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
114
b0c632db
HC		 115int kvm_arch_hardware_setup(void)
116{
2c70fe44
CB		 117 gmap_notifier.notifier_call = kvm_gmap_notifier;
118 gmap_register_ipte_notifier(&gmap_notifier);
b0c632db
HC		 119 return 0;
120}
121
122void kvm_arch_hardware_unsetup(void)
123{
2c70fe44 124 gmap_unregister_ipte_notifier(&gmap_notifier);
b0c632db
HC		 125}
126
127void kvm_arch_check_processor_compat(void *rtn)
128{
129}
130
131int kvm_arch_init(void *opaque)
132{
133 return 0;
134}
135
136void kvm_arch_exit(void)
137{
138}
139
140/* Section: device related */
141long kvm_arch_dev_ioctl(struct file *filp,
142 unsigned int ioctl, unsigned long arg)
143{
144 if (ioctl == KVM_S390_ENABLE_SIE)
145 return s390_enable_sie();
146 return -EINVAL;
147}
148
149int kvm_dev_ioctl_check_extension(long ext)
150{
d7b0b5eb
CO		 151 int r;
152
2bd0ac4e 153 switch (ext) {
d7b0b5eb 154 case KVM_CAP_S390_PSW:
b6cf8788 155 case KVM_CAP_S390_GMAP:
52e16b18 156 case KVM_CAP_SYNC_MMU:
1efd0f59
CO		 157#ifdef CONFIG_KVM_S390_UCONTROL
158 case KVM_CAP_S390_UCONTROL:
159#endif
3c038e6b 160 case KVM_CAP_ASYNC_PF:
60b413c9 161 case KVM_CAP_SYNC_REGS:
14eebd91 162 case KVM_CAP_ONE_REG:
d6712df9 163 case KVM_CAP_ENABLE_CAP:
fa6b7fe9 164 case KVM_CAP_S390_CSS_SUPPORT:
10ccaa1e 165 case KVM_CAP_IOEVENTFD:
c05c4186 166 case KVM_CAP_DEVICE_CTRL:
d938dc55 167 case KVM_CAP_ENABLE_CAP_VM:
f2061656 168 case KVM_CAP_VM_ATTRIBUTES:
d7b0b5eb
CO		 169 r = 1;
170 break;
e726b1bd
CB		 171 case KVM_CAP_NR_VCPUS:
172 case KVM_CAP_MAX_VCPUS:
173 r = KVM_MAX_VCPUS;
174 break;
e1e2e605
NW		 175 case KVM_CAP_NR_MEMSLOTS:
176 r = KVM_USER_MEM_SLOTS;
177 break;
1526bf9c 178 case KVM_CAP_S390_COW:
abf09bed 179 r = MACHINE_HAS_ESOP;
1526bf9c 180 break;
2bd0ac4e 181 default:
d7b0b5eb 182 r = 0;
2bd0ac4e 183 }
d7b0b5eb 184 return r;
b0c632db
HC		 185}
186
15f36ebd
JH		 187static void kvm_s390_sync_dirty_log(struct kvm *kvm,
188 struct kvm_memory_slot *memslot)
189{
190 gfn_t cur_gfn, last_gfn;
191 unsigned long address;
192 struct gmap *gmap = kvm->arch.gmap;
193
194 down_read(&gmap->mm->mmap_sem);
195 /* Loop over all guest pages */
196 last_gfn = memslot->base_gfn + memslot->npages;
197 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
198 address = gfn_to_hva_memslot(memslot, cur_gfn);
199
200 if (gmap_test_and_clear_dirty(address, gmap))
201 mark_page_dirty(kvm, cur_gfn);
202 }
203 up_read(&gmap->mm->mmap_sem);
204}
205
b0c632db
HC		 206/* Section: vm related */
207/*
208 * Get (and clear) the dirty memory log for a memory slot.
209 */
210int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
211 struct kvm_dirty_log *log)
212{
15f36ebd
JH		 213 int r;
214 unsigned long n;
215 struct kvm_memory_slot *memslot;
216 int is_dirty = 0;
217
218 mutex_lock(&kvm->slots_lock);
219
220 r = -EINVAL;
221 if (log->slot >= KVM_USER_MEM_SLOTS)
222 goto out;
223
224 memslot = id_to_memslot(kvm->memslots, log->slot);
225 r = -ENOENT;
226 if (!memslot->dirty_bitmap)
227 goto out;
228
229 kvm_s390_sync_dirty_log(kvm, memslot);
230 r = kvm_get_dirty_log(kvm, log, &is_dirty);
231 if (r)
232 goto out;
233
234 /* Clear the dirty log */
235 if (is_dirty) {
236 n = kvm_dirty_bitmap_bytes(memslot);
237 memset(memslot->dirty_bitmap, 0, n);
238 }
239 r = 0;
240out:
241 mutex_unlock(&kvm->slots_lock);
242 return r;
b0c632db
HC		 243}
244
d938dc55
CH		 245static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
246{
247 int r;
248
249 if (cap->flags)
250 return -EINVAL;
251
252 switch (cap->cap) {
84223598
CH		 253 case KVM_CAP_S390_IRQCHIP:
254 kvm->arch.use_irqchip = 1;
255 r = 0;
256 break;
d938dc55
CH		 257 default:
258 r = -EINVAL;
259 break;
260 }
261 return r;
262}
263
4f718eab
DD		 264static int kvm_s390_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
265{
266 int ret;
267 unsigned int idx;
268 switch (attr->attr) {
269 case KVM_S390_VM_MEM_ENABLE_CMMA:
270 ret = -EBUSY;
271 mutex_lock(&kvm->lock);
272 if (atomic_read(&kvm->online_vcpus) == 0) {
273 kvm->arch.use_cmma = 1;
274 ret = 0;
275 }
276 mutex_unlock(&kvm->lock);
277 break;
278 case KVM_S390_VM_MEM_CLR_CMMA:
279 mutex_lock(&kvm->lock);
280 idx = srcu_read_lock(&kvm->srcu);
281 page_table_reset_pgste(kvm->arch.gmap->mm, 0, TASK_SIZE, false);
282 srcu_read_unlock(&kvm->srcu, idx);
283 mutex_unlock(&kvm->lock);
284 ret = 0;
285 break;
286 default:
287 ret = -ENXIO;
288 break;
289 }
290 return ret;
291}
292
f2061656
DD		 293static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
294{
295 int ret;
296
297 switch (attr->group) {
4f718eab
DD		 298 case KVM_S390_VM_MEM_CTRL:
299 ret = kvm_s390_mem_control(kvm, attr);
300 break;
f2061656
DD		 301 default:
302 ret = -ENXIO;
303 break;
304 }
305
306 return ret;
307}
308
309static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
310{
311 return -ENXIO;
312}
313
314static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
315{
316 int ret;
317
318 switch (attr->group) {
4f718eab
DD		 319 case KVM_S390_VM_MEM_CTRL:
320 switch (attr->attr) {
321 case KVM_S390_VM_MEM_ENABLE_CMMA:
322 case KVM_S390_VM_MEM_CLR_CMMA:
323 ret = 0;
324 break;
325 default:
326 ret = -ENXIO;
327 break;
328 }
329 break;
f2061656
DD		 330 default:
331 ret = -ENXIO;
332 break;
333 }
334
335 return ret;
336}
337
b0c632db
HC		 338long kvm_arch_vm_ioctl(struct file *filp,
339 unsigned int ioctl, unsigned long arg)
340{
341 struct kvm *kvm = filp->private_data;
342 void __user *argp = (void __user *)arg;
f2061656 343 struct kvm_device_attr attr;
b0c632db
HC		 344 int r;
345
346 switch (ioctl) {
ba5c1e9b
CO		 347 case KVM_S390_INTERRUPT: {
348 struct kvm_s390_interrupt s390int;
349
350 r = -EFAULT;
351 if (copy_from_user(&s390int, argp, sizeof(s390int)))
352 break;
353 r = kvm_s390_inject_vm(kvm, &s390int);
354 break;
355 }
d938dc55
CH		 356 case KVM_ENABLE_CAP: {
357 struct kvm_enable_cap cap;
358 r = -EFAULT;
359 if (copy_from_user(&cap, argp, sizeof(cap)))
360 break;
361 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
362 break;
363 }
84223598
CH		 364 case KVM_CREATE_IRQCHIP: {
365 struct kvm_irq_routing_entry routing;
366
367 r = -EINVAL;
368 if (kvm->arch.use_irqchip) {
369 /* Set up dummy routing. */
370 memset(&routing, 0, sizeof(routing));
371 kvm_set_irq_routing(kvm, &routing, 0, 0);
372 r = 0;
373 }
374 break;
375 }
f2061656
DD		 376 case KVM_SET_DEVICE_ATTR: {
377 r = -EFAULT;
378 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
379 break;
380 r = kvm_s390_vm_set_attr(kvm, &attr);
381 break;
382 }
383 case KVM_GET_DEVICE_ATTR: {
384 r = -EFAULT;
385 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
386 break;
387 r = kvm_s390_vm_get_attr(kvm, &attr);
388 break;
389 }
390 case KVM_HAS_DEVICE_ATTR: {
391 r = -EFAULT;
392 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
393 break;
394 r = kvm_s390_vm_has_attr(kvm, &attr);
395 break;
396 }
b0c632db 397 default:
367e1319 398 r = -ENOTTY;
b0c632db
HC		 399 }
400
401 return r;
402}
403
e08b9637 404int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
b0c632db 405{
b0c632db
HC		 406 int rc;
407 char debug_name[16];
f6c137ff 408 static unsigned long sca_offset;
b0c632db 409
e08b9637
CO		 410 rc = -EINVAL;
411#ifdef CONFIG_KVM_S390_UCONTROL
412 if (type & ~KVM_VM_S390_UCONTROL)
413 goto out_err;
414 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
415 goto out_err;
416#else
417 if (type)
418 goto out_err;
419#endif
420
b0c632db
HC		 421 rc = s390_enable_sie();
422 if (rc)
d89f5eff 423 goto out_err;
b0c632db 424
b290411a
CO		 425 rc = -ENOMEM;
426
b0c632db
HC		 427 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
428 if (!kvm->arch.sca)
d89f5eff 429 goto out_err;
f6c137ff
CB		 430 spin_lock(&kvm_lock);
431 sca_offset = (sca_offset + 16) & 0x7f0;
432 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
433 spin_unlock(&kvm_lock);
b0c632db
HC		 434
435 sprintf(debug_name, "kvm-%u", current->pid);
436
437 kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
438 if (!kvm->arch.dbf)
439 goto out_nodbf;
440
ba5c1e9b
CO		 441 spin_lock_init(&kvm->arch.float_int.lock);
442 INIT_LIST_HEAD(&kvm->arch.float_int.list);
8a242234 443 init_waitqueue_head(&kvm->arch.ipte_wq);
ba5c1e9b 444
b0c632db
HC		 445 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
446 VM_EVENT(kvm, 3, "%s", "vm created");
447
e08b9637
CO		 448 if (type & KVM_VM_S390_UCONTROL) {
449 kvm->arch.gmap = NULL;
450 } else {
451 kvm->arch.gmap = gmap_alloc(current->mm);
452 if (!kvm->arch.gmap)
453 goto out_nogmap;
2c70fe44 454 kvm->arch.gmap->private = kvm;
24eb3a82 455 kvm->arch.gmap->pfault_enabled = 0;
e08b9637 456 }
fa6b7fe9
CH		 457
458 kvm->arch.css_support = 0;
84223598 459 kvm->arch.use_irqchip = 0;
fa6b7fe9 460
d89f5eff 461 return 0;
598841ca
CO		 462out_nogmap:
463 debug_unregister(kvm->arch.dbf);
b0c632db
HC		 464out_nodbf:
465 free_page((unsigned long)(kvm->arch.sca));
d89f5eff
JK		 466out_err:
467 return rc;
b0c632db
HC		 468}
469
d329c035
CB		 470void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
471{
472 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
ade38c31 473 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
3c038e6b 474 kvm_clear_async_pf_completion_queue(vcpu);
58f9460b
CO		 475 if (!kvm_is_ucontrol(vcpu->kvm)) {
476 clear_bit(63 - vcpu->vcpu_id,
477 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
478 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
479 (__u64) vcpu->arch.sie_block)
480 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
481 }
abf4a71e 482 smp_mb();
27e0393f
CO		 483
484 if (kvm_is_ucontrol(vcpu->kvm))
485 gmap_free(vcpu->arch.gmap);
486
b31605c1
DD		 487 if (kvm_s390_cmma_enabled(vcpu->kvm))
488 kvm_s390_vcpu_unsetup_cmma(vcpu);
d329c035 489 free_page((unsigned long)(vcpu->arch.sie_block));
b31288fa 490
6692cef3 491 kvm_vcpu_uninit(vcpu);
b110feaf 492 kmem_cache_free(kvm_vcpu_cache, vcpu);
d329c035
CB		 493}
494
495static void kvm_free_vcpus(struct kvm *kvm)
496{
497 unsigned int i;
988a2cae 498 struct kvm_vcpu *vcpu;
d329c035 499
988a2cae
GN		 500 kvm_for_each_vcpu(i, vcpu, kvm)
501 kvm_arch_vcpu_destroy(vcpu);
502
503 mutex_lock(&kvm->lock);
504 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
505 kvm->vcpus[i] = NULL;
506
507 atomic_set(&kvm->online_vcpus, 0);
508 mutex_unlock(&kvm->lock);
d329c035
CB		 509}
510
ad8ba2cd
SY		 511void kvm_arch_sync_events(struct kvm *kvm)
512{
513}
514
b0c632db
HC		 515void kvm_arch_destroy_vm(struct kvm *kvm)
516{
d329c035 517 kvm_free_vcpus(kvm);
b0c632db 518 free_page((unsigned long)(kvm->arch.sca));
d329c035 519 debug_unregister(kvm->arch.dbf);
27e0393f
CO		 520 if (!kvm_is_ucontrol(kvm))
521 gmap_free(kvm->arch.gmap);
841b91c5 522 kvm_s390_destroy_adapters(kvm);
b0c632db
HC		 523}
524
525/* Section: vcpu related */
526int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
527{
3c038e6b
DD		 528 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
529 kvm_clear_async_pf_completion_queue(vcpu);
27e0393f
CO		 530 if (kvm_is_ucontrol(vcpu->kvm)) {
531 vcpu->arch.gmap = gmap_alloc(current->mm);
532 if (!vcpu->arch.gmap)
533 return -ENOMEM;
2c70fe44 534 vcpu->arch.gmap->private = vcpu->kvm;
27e0393f
CO		 535 return 0;
536 }
537
598841ca 538 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
59674c1a
CB		 539 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
540 KVM_SYNC_GPRS |
9eed0735
CB		 541 KVM_SYNC_ACRS |
542 KVM_SYNC_CRS;
b0c632db
HC		 543 return 0;
544}
545
546void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
547{
6692cef3 548 /* Nothing todo */
b0c632db
HC		 549}
550
551void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
552{
4725c860
MS		 553 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
554 save_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db 555 save_access_regs(vcpu->arch.host_acrs);
4725c860
MS		 556 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
557 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 558 restore_access_regs(vcpu->run->s.regs.acrs);
480e5926 559 gmap_enable(vcpu->arch.gmap);
9e6dabef 560 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
b0c632db
HC		 561}
562
563void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
564{
9e6dabef 565 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
480e5926 566 gmap_disable(vcpu->arch.gmap);
4725c860
MS		 567 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
568 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
59674c1a 569 save_access_regs(vcpu->run->s.regs.acrs);
4725c860
MS		 570 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
571 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
b0c632db
HC		 572 restore_access_regs(vcpu->arch.host_acrs);
573}
574
575static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
576{
577 /* this equals initial cpu reset in pop, but we don't switch to ESA */
578 vcpu->arch.sie_block->gpsw.mask = 0UL;
579 vcpu->arch.sie_block->gpsw.addr = 0UL;
8d26cf7b 580 kvm_s390_set_prefix(vcpu, 0);
b0c632db
HC		 581 vcpu->arch.sie_block->cputm = 0UL;
582 vcpu->arch.sie_block->ckc = 0UL;
583 vcpu->arch.sie_block->todpr = 0;
584 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
585 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
586 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
587 vcpu->arch.guest_fpregs.fpc = 0;
588 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
589 vcpu->arch.sie_block->gbea = 1;
672550fb 590 vcpu->arch.sie_block->pp = 0;
3c038e6b
DD		 591 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
592 kvm_clear_async_pf_completion_queue(vcpu);
61bde82c 593 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2ed10cc1 594 kvm_s390_clear_local_irqs(vcpu);
b0c632db
HC		 595}
596
42897d86
MT		 597int kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
598{
599 return 0;
600}
601
b31605c1
DD		 602void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
603{
604 free_page(vcpu->arch.sie_block->cbrlo);
605 vcpu->arch.sie_block->cbrlo = 0;
606}
607
608int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
609{
610 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
611 if (!vcpu->arch.sie_block->cbrlo)
612 return -ENOMEM;
613
614 vcpu->arch.sie_block->ecb2 |= 0x80;
615 vcpu->arch.sie_block->ecb2 &= ~0x08;
616 return 0;
617}
618
b0c632db
HC		 619int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
620{
b31605c1 621 int rc = 0;
b31288fa 622
9e6dabef
CH		 623 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
624 CPUSTAT_SM |
69d0d3a3
CB		 625 CPUSTAT_STOPPED |
626 CPUSTAT_GED);
fc34531d 627 vcpu->arch.sie_block->ecb = 6;
7feb6bb8
MM		 628 if (test_vfacility(50) && test_vfacility(73))
629 vcpu->arch.sie_block->ecb |= 0x10;
630
69d0d3a3 631 vcpu->arch.sie_block->ecb2 = 8;
217a4406
HC		 632 vcpu->arch.sie_block->eca = 0xC1002000U;
633 if (sclp_has_siif())
634 vcpu->arch.sie_block->eca |= 1;
78c4b59f 635 vcpu->arch.sie_block->fac = (int) (long) vfacilities;
693ffc08 636 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
b31605c1
DD		 637 if (kvm_s390_cmma_enabled(vcpu->kvm)) {
638 rc = kvm_s390_vcpu_setup_cmma(vcpu);
639 if (rc)
640 return rc;
b31288fa 641 }
ca872302
CB		 642 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
643 tasklet_init(&vcpu->arch.tasklet, kvm_s390_tasklet,
644 (unsigned long) vcpu);
645 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
453423dc 646 get_cpu_id(&vcpu->arch.cpu_id);
92e6ecf3 647 vcpu->arch.cpu_id.version = 0xff;
b31605c1 648 return rc;
b0c632db
HC		 649}
650
651struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
652 unsigned int id)
653{
4d47555a 654 struct kvm_vcpu *vcpu;
7feb6bb8 655 struct sie_page *sie_page;
4d47555a
CO		 656 int rc = -EINVAL;
657
658 if (id >= KVM_MAX_VCPUS)
659 goto out;
660
661 rc = -ENOMEM;
b0c632db 662
b110feaf 663 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
b0c632db 664 if (!vcpu)
4d47555a 665 goto out;
b0c632db 666
7feb6bb8
MM		 667 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
668 if (!sie_page)
b0c632db
HC		 669 goto out_free_cpu;
670
7feb6bb8
MM		 671 vcpu->arch.sie_block = &sie_page->sie_block;
672 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
673
b0c632db 674 vcpu->arch.sie_block->icpua = id;
58f9460b
CO		 675 if (!kvm_is_ucontrol(kvm)) {
676 if (!kvm->arch.sca) {
677 WARN_ON_ONCE(1);
678 goto out_free_cpu;
679 }
680 if (!kvm->arch.sca->cpu[id].sda)
681 kvm->arch.sca->cpu[id].sda =
682 (__u64) vcpu->arch.sie_block;
683 vcpu->arch.sie_block->scaoh =
684 (__u32)(((__u64)kvm->arch.sca) >> 32);
685 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
686 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
687 }
b0c632db 688
ba5c1e9b
CO		 689 spin_lock_init(&vcpu->arch.local_int.lock);
690 INIT_LIST_HEAD(&vcpu->arch.local_int.list);
691 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
d0321a24 692 vcpu->arch.local_int.wq = &vcpu->wq;
5288fbf0 693 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
ba5c1e9b 694
b0c632db
HC		 695 rc = kvm_vcpu_init(vcpu, kvm, id);
696 if (rc)
7b06bf2f 697 goto out_free_sie_block;
b0c632db
HC		 698 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
699 vcpu->arch.sie_block);
ade38c31 700 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
b0c632db 701
b0c632db 702 return vcpu;
7b06bf2f
WY		 703out_free_sie_block:
704 free_page((unsigned long)(vcpu->arch.sie_block));
b0c632db 705out_free_cpu:
b110feaf 706 kmem_cache_free(kvm_vcpu_cache, vcpu);
4d47555a 707out:
b0c632db
HC		 708 return ERR_PTR(rc);
709}
710
b0c632db
HC		 711int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
712{
f87618e8 713 return kvm_cpu_has_interrupt(vcpu);
b0c632db
HC		 714}
715
49b99e1e
CB		 716void s390_vcpu_block(struct kvm_vcpu *vcpu)
717{
718 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
719}
720
721void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
722{
723 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
724}
725
726/*
727 * Kick a guest cpu out of SIE and wait until SIE is not running.
728 * If the CPU is not running (e.g. waiting as idle) the function will
729 * return immediately. */
730void exit_sie(struct kvm_vcpu *vcpu)
731{
732 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
733 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
734 cpu_relax();
735}
736
737/* Kick a guest cpu out of SIE and prevent SIE-reentry */
738void exit_sie_sync(struct kvm_vcpu *vcpu)
739{
740 s390_vcpu_block(vcpu);
741 exit_sie(vcpu);
742}
743
2c70fe44
CB		 744static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
745{
746 int i;
747 struct kvm *kvm = gmap->private;
748 struct kvm_vcpu *vcpu;
749
750 kvm_for_each_vcpu(i, vcpu, kvm) {
751 /* match against both prefix pages */
752 if (vcpu->arch.sie_block->prefix == (address & ~0x1000UL)) {
753 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
754 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
755 exit_sie_sync(vcpu);
756 }
757 }
758}
759
b6d33834
CD		 760int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
761{
762 /* kvm common code refers to this, but never calls it */
763 BUG();
764 return 0;
765}
766
14eebd91
CO		 767static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
768 struct kvm_one_reg *reg)
769{
770 int r = -EINVAL;
771
772 switch (reg->id) {
29b7c71b
CO		 773 case KVM_REG_S390_TODPR:
774 r = put_user(vcpu->arch.sie_block->todpr,
775 (u32 __user *)reg->addr);
776 break;
777 case KVM_REG_S390_EPOCHDIFF:
778 r = put_user(vcpu->arch.sie_block->epoch,
779 (u64 __user *)reg->addr);
780 break;
46a6dd1c
J		 781 case KVM_REG_S390_CPU_TIMER:
782 r = put_user(vcpu->arch.sie_block->cputm,
783 (u64 __user *)reg->addr);
784 break;
785 case KVM_REG_S390_CLOCK_COMP:
786 r = put_user(vcpu->arch.sie_block->ckc,
787 (u64 __user *)reg->addr);
788 break;
536336c2
DD		 789 case KVM_REG_S390_PFTOKEN:
790 r = put_user(vcpu->arch.pfault_token,
791 (u64 __user *)reg->addr);
792 break;
793 case KVM_REG_S390_PFCOMPARE:
794 r = put_user(vcpu->arch.pfault_compare,
795 (u64 __user *)reg->addr);
796 break;
797 case KVM_REG_S390_PFSELECT:
798 r = put_user(vcpu->arch.pfault_select,
799 (u64 __user *)reg->addr);
800 break;
672550fb
CB		 801 case KVM_REG_S390_PP:
802 r = put_user(vcpu->arch.sie_block->pp,
803 (u64 __user *)reg->addr);
804 break;
afa45ff5
CB		 805 case KVM_REG_S390_GBEA:
806 r = put_user(vcpu->arch.sie_block->gbea,
807 (u64 __user *)reg->addr);
808 break;
14eebd91
CO		 809 default:
810 break;
811 }
812
813 return r;
814}
815
816static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
817 struct kvm_one_reg *reg)
818{
819 int r = -EINVAL;
820
821 switch (reg->id) {
29b7c71b
CO		 822 case KVM_REG_S390_TODPR:
823 r = get_user(vcpu->arch.sie_block->todpr,
824 (u32 __user *)reg->addr);
825 break;
826 case KVM_REG_S390_EPOCHDIFF:
827 r = get_user(vcpu->arch.sie_block->epoch,
828 (u64 __user *)reg->addr);
829 break;
46a6dd1c
J		 830 case KVM_REG_S390_CPU_TIMER:
831 r = get_user(vcpu->arch.sie_block->cputm,
832 (u64 __user *)reg->addr);
833 break;
834 case KVM_REG_S390_CLOCK_COMP:
835 r = get_user(vcpu->arch.sie_block->ckc,
836 (u64 __user *)reg->addr);
837 break;
536336c2
DD		 838 case KVM_REG_S390_PFTOKEN:
839 r = get_user(vcpu->arch.pfault_token,
840 (u64 __user *)reg->addr);
841 break;
842 case KVM_REG_S390_PFCOMPARE:
843 r = get_user(vcpu->arch.pfault_compare,
844 (u64 __user *)reg->addr);
845 break;
846 case KVM_REG_S390_PFSELECT:
847 r = get_user(vcpu->arch.pfault_select,
848 (u64 __user *)reg->addr);
849 break;
672550fb
CB		 850 case KVM_REG_S390_PP:
851 r = get_user(vcpu->arch.sie_block->pp,
852 (u64 __user *)reg->addr);
853 break;
afa45ff5
CB		 854 case KVM_REG_S390_GBEA:
855 r = get_user(vcpu->arch.sie_block->gbea,
856 (u64 __user *)reg->addr);
857 break;
14eebd91
CO		 858 default:
859 break;
860 }
861
862 return r;
863}
b6d33834 864
b0c632db
HC		 865static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
866{
b0c632db 867 kvm_s390_vcpu_initial_reset(vcpu);
b0c632db
HC		 868 return 0;
869}
870
871int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
872{
5a32c1af 873 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
b0c632db
HC		 874 return 0;
875}
876
877int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
878{
5a32c1af 879 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
b0c632db
HC		 880 return 0;
881}
882
883int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
884 struct kvm_sregs *sregs)
885{
59674c1a 886 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
b0c632db 887 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
59674c1a 888 restore_access_regs(vcpu->run->s.regs.acrs);
b0c632db
HC		 889 return 0;
890}
891
892int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
893 struct kvm_sregs *sregs)
894{
59674c1a 895 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
b0c632db 896 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
b0c632db
HC		 897 return 0;
898}
899
900int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
901{
4725c860
MS		 902 if (test_fp_ctl(fpu->fpc))
903 return -EINVAL;
b0c632db 904 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4725c860
MS		 905 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
906 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
907 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
b0c632db
HC		 908 return 0;
909}
910
911int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
912{
b0c632db
HC		 913 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
914 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
b0c632db
HC		 915 return 0;
916}
917
918static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
919{
920 int rc = 0;
921
9e6dabef 922 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED))
b0c632db 923 rc = -EBUSY;
d7b0b5eb
CO		 924 else {
925 vcpu->run->psw_mask = psw.mask;
926 vcpu->run->psw_addr = psw.addr;
927 }
b0c632db
HC		 928 return rc;
929}
930
931int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
932 struct kvm_translation *tr)
933{
934 return -EINVAL; /* not implemented yet */
935}
936
d0bfb940
JK		 937int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
938 struct kvm_guest_debug *dbg)
b0c632db
HC		 939{
940 return -EINVAL; /* not implemented yet */
941}
942
62d9f0db
MT		 943int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
944 struct kvm_mp_state *mp_state)
945{
946 return -EINVAL; /* not implemented yet */
947}
948
949int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
950 struct kvm_mp_state *mp_state)
951{
952 return -EINVAL; /* not implemented yet */
953}
954
b31605c1
DD		 955bool kvm_s390_cmma_enabled(struct kvm *kvm)
956{
957 if (!MACHINE_IS_LPAR)
958 return false;
959 /* only enable for z10 and later */
960 if (!MACHINE_HAS_EDAT1)
961 return false;
962 if (!kvm->arch.use_cmma)
963 return false;
964 return true;
965}
966
2c70fe44
CB		 967static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
968{
969 /*
970 * We use MMU_RELOAD just to re-arm the ipte notifier for the
971 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
972 * This ensures that the ipte instruction for this request has
973 * already finished. We might race against a second unmapper that
974 * wants to set the blocking bit. Lets just retry the request loop.
975 */
976 while (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
977 int rc;
978 rc = gmap_ipte_notify(vcpu->arch.gmap,
979 vcpu->arch.sie_block->prefix,
980 PAGE_SIZE * 2);
981 if (rc)
982 return rc;
983 s390_vcpu_unblock(vcpu);
984 }
985 return 0;
986}
987
24eb3a82
DD		 988static long kvm_arch_fault_in_sync(struct kvm_vcpu *vcpu)
989{
990 long rc;
991 hva_t fault = gmap_fault(current->thread.gmap_addr, vcpu->arch.gmap);
992 struct mm_struct *mm = current->mm;
993 down_read(&mm->mmap_sem);
994 rc = get_user_pages(current, mm, fault, 1, 1, 0, NULL, NULL);
995 up_read(&mm->mmap_sem);
996 return rc;
997}
998
3c038e6b
DD		 999static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1000 unsigned long token)
1001{
1002 struct kvm_s390_interrupt inti;
1003 inti.parm64 = token;
1004
1005 if (start_token) {
1006 inti.type = KVM_S390_INT_PFAULT_INIT;
1007 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &inti));
1008 } else {
1009 inti.type = KVM_S390_INT_PFAULT_DONE;
1010 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1011 }
1012}
1013
1014void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1015 struct kvm_async_pf *work)
1016{
1017 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1018 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1019}
1020
1021void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1022 struct kvm_async_pf *work)
1023{
1024 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1025 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1026}
1027
1028void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1029 struct kvm_async_pf *work)
1030{
1031 /* s390 will always inject the page directly */
1032}
1033
1034bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1035{
1036 /*
1037 * s390 will always inject the page directly,
1038 * but we still want check_async_completion to cleanup
1039 */
1040 return true;
1041}
1042
1043static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1044{
1045 hva_t hva;
1046 struct kvm_arch_async_pf arch;
1047 int rc;
1048
1049 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1050 return 0;
1051 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1052 vcpu->arch.pfault_compare)
1053 return 0;
1054 if (psw_extint_disabled(vcpu))
1055 return 0;
1056 if (kvm_cpu_has_interrupt(vcpu))
1057 return 0;
1058 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1059 return 0;
1060 if (!vcpu->arch.gmap->pfault_enabled)
1061 return 0;
1062
81480cc1
HC		 1063 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1064 hva += current->thread.gmap_addr & ~PAGE_MASK;
1065 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3c038e6b
DD		 1066 return 0;
1067
1068 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1069 return rc;
1070}
1071
3fb4c40f 1072static int vcpu_pre_run(struct kvm_vcpu *vcpu)
b0c632db 1073{
3fb4c40f 1074 int rc, cpuflags;
e168bf8d 1075
3c038e6b
DD		 1076 /*
1077 * On s390 notifications for arriving pages will be delivered directly
1078 * to the guest but the house keeping for completed pfaults is
1079 * handled outside the worker.
1080 */
1081 kvm_check_async_pf_completion(vcpu);
1082
5a32c1af 1083 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
b0c632db
HC		 1084
1085 if (need_resched())
1086 schedule();
1087
71cde587
CB		 1088 if (test_thread_flag(TIF_MCCK_PENDING))
1089 s390_handle_mcck();
1090
d6b6d166
CO		 1091 if (!kvm_is_ucontrol(vcpu->kvm))
1092 kvm_s390_deliver_pending_interrupts(vcpu);
0ff31867 1093
2c70fe44
CB		 1094 rc = kvm_s390_handle_requests(vcpu);
1095 if (rc)
1096 return rc;
1097
b0c632db 1098 vcpu->arch.sie_block->icptcode = 0;
3fb4c40f
TH		 1099 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1100 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1101 trace_kvm_s390_sie_enter(vcpu, cpuflags);
2b29a9fd 1102
3fb4c40f
TH		 1103 return 0;
1104}
1105
1106static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1107{
24eb3a82 1108 int rc = -1;
2b29a9fd
DD		 1109
1110 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1111 vcpu->arch.sie_block->icptcode);
1112 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1113
3fb4c40f 1114 if (exit_reason >= 0) {
7c470539 1115 rc = 0;
210b1607
TH		 1116 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1117 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1118 vcpu->run->s390_ucontrol.trans_exc_code =
1119 current->thread.gmap_addr;
1120 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1121 rc = -EREMOTE;
24eb3a82
DD		 1122
1123 } else if (current->thread.gmap_pfault) {
3c038e6b 1124 trace_kvm_s390_major_guest_pfault(vcpu);
24eb3a82 1125 current->thread.gmap_pfault = 0;
3c038e6b
DD		 1126 if (kvm_arch_setup_async_pf(vcpu) ||
1127 (kvm_arch_fault_in_sync(vcpu) >= 0))
24eb3a82
DD		 1128 rc = 0;
1129 }
1130
1131 if (rc == -1) {
699bde3b
CB		 1132 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1133 trace_kvm_s390_sie_fault(vcpu);
1134 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1f0d0f09 1135 }
b0c632db 1136
5a32c1af 1137 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
3fb4c40f 1138
a76ccff6
TH		 1139 if (rc == 0) {
1140 if (kvm_is_ucontrol(vcpu->kvm))
2955c83f
CB		 1141 /* Don't exit for host interrupts. */
1142 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
a76ccff6
TH		 1143 else
1144 rc = kvm_handle_sie_intercept(vcpu);
1145 }
1146
3fb4c40f
TH		 1147 return rc;
1148}
1149
1150static int __vcpu_run(struct kvm_vcpu *vcpu)
1151{
1152 int rc, exit_reason;
1153
800c1065
TH		 1154 /*
1155 * We try to hold kvm->srcu during most of vcpu_run (except when run-
1156 * ning the guest), so that memslots (and other stuff) are protected
1157 */
1158 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1159
a76ccff6
TH		 1160 do {
1161 rc = vcpu_pre_run(vcpu);
1162 if (rc)
1163 break;
3fb4c40f 1164
800c1065 1165 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
a76ccff6
TH		 1166 /*
1167 * As PF_VCPU will be used in fault handler, between
1168 * guest_enter and guest_exit should be no uaccess.
1169 */
1170 preempt_disable();
1171 kvm_guest_enter();
1172 preempt_enable();
1173 exit_reason = sie64a(vcpu->arch.sie_block,
1174 vcpu->run->s.regs.gprs);
1175 kvm_guest_exit();
800c1065 1176 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
a76ccff6
TH		 1177
1178 rc = vcpu_post_run(vcpu, exit_reason);
1179 } while (!signal_pending(current) && !rc);
3fb4c40f 1180
800c1065 1181 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
e168bf8d 1182 return rc;
b0c632db
HC		 1183}
1184
1185int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1186{
8f2abe6a 1187 int rc;
b0c632db
HC		 1188 sigset_t sigsaved;
1189
b0c632db
HC		 1190 if (vcpu->sigset_active)
1191 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1192
9e6dabef 1193 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
b0c632db 1194
8f2abe6a
CB		 1195 switch (kvm_run->exit_reason) {
1196 case KVM_EXIT_S390_SIEIC:
8f2abe6a 1197 case KVM_EXIT_UNKNOWN:
9ace903d 1198 case KVM_EXIT_INTR:
8f2abe6a 1199 case KVM_EXIT_S390_RESET:
e168bf8d 1200 case KVM_EXIT_S390_UCONTROL:
fa6b7fe9 1201 case KVM_EXIT_S390_TSCH:
8f2abe6a
CB		 1202 break;
1203 default:
1204 BUG();
1205 }
1206
d7b0b5eb
CO		 1207 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1208 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
60b413c9
CB		 1209 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) {
1210 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_PREFIX;
1211 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1212 }
9eed0735
CB		 1213 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1214 kvm_run->kvm_dirty_regs &= ~KVM_SYNC_CRS;
1215 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1216 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1217 }
d7b0b5eb 1218
dab4079d 1219 might_fault();
a76ccff6 1220 rc = __vcpu_run(vcpu);
9ace903d 1221
b1d16c49
CE		 1222 if (signal_pending(current) && !rc) {
1223 kvm_run->exit_reason = KVM_EXIT_INTR;
8f2abe6a 1224 rc = -EINTR;
b1d16c49 1225 }
8f2abe6a 1226
b8e660b8 1227 if (rc == -EOPNOTSUPP) {
8f2abe6a
CB		 1228 /* intercept cannot be handled in-kernel, prepare kvm-run */
1229 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
1230 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
8f2abe6a
CB		 1231 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
1232 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
1233 rc = 0;
1234 }
1235
1236 if (rc == -EREMOTE) {
1237 /* intercept was handled, but userspace support is needed
1238 * kvm_run has been prepared by the handler */
1239 rc = 0;
1240 }
b0c632db 1241
d7b0b5eb
CO		 1242 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1243 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
60b413c9 1244 kvm_run->s.regs.prefix = vcpu->arch.sie_block->prefix;
9eed0735 1245 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
d7b0b5eb 1246
b0c632db
HC		 1247 if (vcpu->sigset_active)
1248 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1249
b0c632db 1250 vcpu->stat.exit_userspace++;
7e8e6ab4 1251 return rc;
b0c632db
HC		 1252}
1253
b0c632db
HC		 1254/*
1255 * store status at address
1256 * we use have two special cases:
1257 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1258 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1259 */
d0bce605 1260int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
b0c632db 1261{
092670cd 1262 unsigned char archmode = 1;
178bd789 1263 u64 clkcomp;
d0bce605 1264 int rc;
b0c632db 1265
d0bce605
HC		 1266 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1267 if (write_guest_abs(vcpu, 163, &archmode, 1))
b0c632db 1268 return -EFAULT;
d0bce605
HC		 1269 gpa = SAVE_AREA_BASE;
1270 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1271 if (write_guest_real(vcpu, 163, &archmode, 1))
b0c632db 1272 return -EFAULT;
d0bce605
HC		 1273 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1274 }
1275 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1276 vcpu->arch.guest_fpregs.fprs, 128);
1277 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1278 vcpu->run->s.regs.gprs, 128);
1279 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1280 &vcpu->arch.sie_block->gpsw, 16);
1281 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1282 &vcpu->arch.sie_block->prefix, 4);
1283 rc |= write_guest_abs(vcpu,
1284 gpa + offsetof(struct save_area, fp_ctrl_reg),
1285 &vcpu->arch.guest_fpregs.fpc, 4);
1286 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1287 &vcpu->arch.sie_block->todpr, 4);
1288 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1289 &vcpu->arch.sie_block->cputm, 8);
178bd789 1290 clkcomp = vcpu->arch.sie_block->ckc >> 8;
d0bce605
HC		 1291 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1292 &clkcomp, 8);
1293 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1294 &vcpu->run->s.regs.acrs, 64);
1295 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1296 &vcpu->arch.sie_block->gcr, 128);
1297 return rc ? -EFAULT : 0;
b0c632db
HC		 1298}
1299
e879892c
TH		 1300int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1301{
1302 /*
1303 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1304 * copying in vcpu load/put. Lets update our copies before we save
1305 * it into the save area
1306 */
1307 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1308 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1309 save_access_regs(vcpu->run->s.regs.acrs);
1310
1311 return kvm_s390_store_status_unloaded(vcpu, addr);
1312}
1313
d6712df9
CH		 1314static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1315 struct kvm_enable_cap *cap)
1316{
1317 int r;
1318
1319 if (cap->flags)
1320 return -EINVAL;
1321
1322 switch (cap->cap) {
fa6b7fe9
CH		 1323 case KVM_CAP_S390_CSS_SUPPORT:
1324 if (!vcpu->kvm->arch.css_support) {
1325 vcpu->kvm->arch.css_support = 1;
1326 trace_kvm_s390_enable_css(vcpu->kvm);
1327 }
1328 r = 0;
1329 break;
d6712df9
CH		 1330 default:
1331 r = -EINVAL;
1332 break;
1333 }
1334 return r;
1335}
1336
b0c632db
HC		 1337long kvm_arch_vcpu_ioctl(struct file *filp,
1338 unsigned int ioctl, unsigned long arg)
1339{
1340 struct kvm_vcpu *vcpu = filp->private_data;
1341 void __user *argp = (void __user *)arg;
800c1065 1342 int idx;
bc923cc9 1343 long r;
b0c632db 1344
93736624
AK		 1345 switch (ioctl) {
1346 case KVM_S390_INTERRUPT: {
ba5c1e9b
CO		 1347 struct kvm_s390_interrupt s390int;
1348
93736624 1349 r = -EFAULT;
ba5c1e9b 1350 if (copy_from_user(&s390int, argp, sizeof(s390int)))
93736624
AK		 1351 break;
1352 r = kvm_s390_inject_vcpu(vcpu, &s390int);
1353 break;
ba5c1e9b 1354 }
b0c632db 1355 case KVM_S390_STORE_STATUS:
800c1065 1356 idx = srcu_read_lock(&vcpu->kvm->srcu);
bc923cc9 1357 r = kvm_s390_vcpu_store_status(vcpu, arg);
800c1065 1358 srcu_read_unlock(&vcpu->kvm->srcu, idx);
bc923cc9 1359 break;
b0c632db
HC		 1360 case KVM_S390_SET_INITIAL_PSW: {
1361 psw_t psw;
1362
bc923cc9 1363 r = -EFAULT;
b0c632db 1364 if (copy_from_user(&psw, argp, sizeof(psw)))
bc923cc9
AK		 1365 break;
1366 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
1367 break;
b0c632db
HC		 1368 }
1369 case KVM_S390_INITIAL_RESET:
bc923cc9
AK		 1370 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
1371 break;
14eebd91
CO		 1372 case KVM_SET_ONE_REG:
1373 case KVM_GET_ONE_REG: {
1374 struct kvm_one_reg reg;
1375 r = -EFAULT;
1376 if (copy_from_user(&reg, argp, sizeof(reg)))
1377 break;
1378 if (ioctl == KVM_SET_ONE_REG)
1379 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
1380 else
1381 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
1382 break;
1383 }
27e0393f
CO		 1384#ifdef CONFIG_KVM_S390_UCONTROL
1385 case KVM_S390_UCAS_MAP: {
1386 struct kvm_s390_ucas_mapping ucasmap;
1387
1388 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1389 r = -EFAULT;
1390 break;
1391 }
1392
1393 if (!kvm_is_ucontrol(vcpu->kvm)) {
1394 r = -EINVAL;
1395 break;
1396 }
1397
1398 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
1399 ucasmap.vcpu_addr, ucasmap.length);
1400 break;
1401 }
1402 case KVM_S390_UCAS_UNMAP: {
1403 struct kvm_s390_ucas_mapping ucasmap;
1404
1405 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
1406 r = -EFAULT;
1407 break;
1408 }
1409
1410 if (!kvm_is_ucontrol(vcpu->kvm)) {
1411 r = -EINVAL;
1412 break;
1413 }
1414
1415 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
1416 ucasmap.length);
1417 break;
1418 }
1419#endif
ccc7910f
CO		 1420 case KVM_S390_VCPU_FAULT: {
1421 r = gmap_fault(arg, vcpu->arch.gmap);
1422 if (!IS_ERR_VALUE(r))
1423 r = 0;
1424 break;
1425 }
d6712df9
CH		 1426 case KVM_ENABLE_CAP:
1427 {
1428 struct kvm_enable_cap cap;
1429 r = -EFAULT;
1430 if (copy_from_user(&cap, argp, sizeof(cap)))
1431 break;
1432 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1433 break;
1434 }
b0c632db 1435 default:
3e6afcf1 1436 r = -ENOTTY;
b0c632db 1437 }
bc923cc9 1438 return r;
b0c632db
HC		 1439}
1440
5b1c1493
CO		 1441int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
1442{
1443#ifdef CONFIG_KVM_S390_UCONTROL
1444 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
1445 && (kvm_is_ucontrol(vcpu->kvm))) {
1446 vmf->page = virt_to_page(vcpu->arch.sie_block);
1447 get_page(vmf->page);
1448 return 0;
1449 }
1450#endif
1451 return VM_FAULT_SIGBUS;
1452}
1453
5587027c 1454void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
db3fe4eb
TY		 1455 struct kvm_memory_slot *dont)
1456{
1457}
1458
5587027c
AK		 1459int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
1460 unsigned long npages)
db3fe4eb
TY		 1461{
1462 return 0;
1463}
1464
e59dbe09
TY		 1465void kvm_arch_memslots_updated(struct kvm *kvm)
1466{
1467}
1468
b0c632db 1469/* Section: memory related */
f7784b8e
MT		 1470int kvm_arch_prepare_memory_region(struct kvm *kvm,
1471 struct kvm_memory_slot *memslot,
7b6195a9
TY		 1472 struct kvm_userspace_memory_region *mem,
1473 enum kvm_mr_change change)
b0c632db 1474{
dd2887e7
NW		 1475 /* A few sanity checks. We can have memory slots which have to be
1476 located/ended at a segment boundary (1MB). The memory in userland is
1477 ok to be fragmented into various different vmas. It is okay to mmap()
1478 and munmap() stuff in this slot after doing this call at any time */
b0c632db 1479
598841ca 1480 if (mem->userspace_addr & 0xffffful)
b0c632db
HC		 1481 return -EINVAL;
1482
598841ca 1483 if (mem->memory_size & 0xffffful)
b0c632db
HC		 1484 return -EINVAL;
1485
f7784b8e
MT		 1486 return 0;
1487}
1488
1489void kvm_arch_commit_memory_region(struct kvm *kvm,
1490 struct kvm_userspace_memory_region *mem,
8482644a
TY		 1491 const struct kvm_memory_slot *old,
1492 enum kvm_mr_change change)
f7784b8e 1493{
f7850c92 1494 int rc;
f7784b8e 1495
2cef4deb
CB		 1496 /* If the basics of the memslot do not change, we do not want
1497 * to update the gmap. Every update causes several unnecessary
1498 * segment translation exceptions. This is usually handled just
1499 * fine by the normal fault handler + gmap, but it will also
1500 * cause faults on the prefix page of running guest CPUs.
1501 */
1502 if (old->userspace_addr == mem->userspace_addr &&
1503 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
1504 old->npages * PAGE_SIZE == mem->memory_size)
1505 return;
598841ca
CO		 1506
1507 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
1508 mem->guest_phys_addr, mem->memory_size);
1509 if (rc)
f7850c92 1510 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
598841ca 1511 return;
b0c632db
HC		 1512}
1513
2df72e9b
MT		 1514void kvm_arch_flush_shadow_all(struct kvm *kvm)
1515{
1516}
1517
1518void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
1519 struct kvm_memory_slot *slot)
34d4cb8f
MT		 1520{
1521}
1522
b0c632db
HC		 1523static int __init kvm_s390_init(void)
1524{
ef50f7ac 1525 int ret;
0ee75bea 1526 ret = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
ef50f7ac
CB		 1527 if (ret)
1528 return ret;
1529
1530 /*
1531 * guests can ask for up to 255+1 double words, we need a full page
25985edc 1532 * to hold the maximum amount of facilities. On the other hand, we
ef50f7ac
CB		 1533 * only set facilities that are known to work in KVM.
1534 */
78c4b59f
MM		 1535 vfacilities = (unsigned long *) get_zeroed_page(GFP_KERNEL|GFP_DMA);
1536 if (!vfacilities) {
ef50f7ac
CB		 1537 kvm_exit();
1538 return -ENOMEM;
1539 }
78c4b59f 1540 memcpy(vfacilities, S390_lowcore.stfle_fac_list, 16);
d208c79d 1541 vfacilities[0] &= 0xff82fff3f4fc2000UL;
7feb6bb8 1542 vfacilities[1] &= 0x005c000000000000UL;
ef50f7ac 1543 return 0;
b0c632db
HC		 1544}
1545
1546static void __exit kvm_s390_exit(void)
1547{
78c4b59f 1548 free_page((unsigned long) vfacilities);
b0c632db
HC		 1549 kvm_exit();
1550}
1551
1552module_init(kvm_s390_init);
1553module_exit(kvm_s390_exit);
566af940
CH		 1554
1555/*
1556 * Enable autoloading of the kvm module.
1557 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
1558 * since x86 takes a different approach.
1559 */
1560#include <linux/miscdevice.h>
1561MODULE_ALIAS_MISCDEV(KVM_MINOR);
1562MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)