projects / linux-2.6-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
KVM: s390: adapt debug entries for instruction handling
[linux-2.6-block.git] / arch / s390 / kvm / kvm-s390.c
... / ...
CommitLineData
1/*
2 * hosting zSeries kernel virtual machines
3 *
4 * Copyright IBM Corp. 2008, 2009
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License (version 2 only)
8 * as published by the Free Software Foundation.
9 *
10 * Author(s): Carsten Otte <cotte@de.ibm.com>
11 * Christian Borntraeger <borntraeger@de.ibm.com>
12 * Heiko Carstens <heiko.carstens@de.ibm.com>
13 * Christian Ehrhardt <ehrhardt@de.ibm.com>
14 * Jason J. Herne <jjherne@us.ibm.com>
15 */
16
17#include <linux/compiler.h>
18#include <linux/err.h>
19#include <linux/fs.h>
20#include <linux/hrtimer.h>
21#include <linux/init.h>
22#include <linux/kvm.h>
23#include <linux/kvm_host.h>
24#include <linux/module.h>
25#include <linux/random.h>
26#include <linux/slab.h>
27#include <linux/timer.h>
28#include <linux/vmalloc.h>
29#include <asm/asm-offsets.h>
30#include <asm/lowcore.h>
31#include <asm/pgtable.h>
32#include <asm/nmi.h>
33#include <asm/switch_to.h>
34#include <asm/isc.h>
35#include <asm/sclp.h>
36#include "kvm-s390.h"
37#include "gaccess.h"
38
39#define KMSG_COMPONENT "kvm-s390"
40#undef pr_fmt
41#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
42
43#define CREATE_TRACE_POINTS
44#include "trace.h"
45#include "trace-s390.h"
46
47#define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
48#define LOCAL_IRQS 32
49#define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
50 (KVM_MAX_VCPUS + LOCAL_IRQS))
51
52#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
53
54struct kvm_stats_debugfs_item debugfs_entries[] = {
55 { "userspace_handled", VCPU_STAT(exit_userspace) },
56 { "exit_null", VCPU_STAT(exit_null) },
57 { "exit_validity", VCPU_STAT(exit_validity) },
58 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
59 { "exit_external_request", VCPU_STAT(exit_external_request) },
60 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
61 { "exit_instruction", VCPU_STAT(exit_instruction) },
62 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
63 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
64 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
65 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
66 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
67 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
68 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
69 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
70 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
71 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
72 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
73 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
74 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
75 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
76 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
77 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
78 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
79 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
80 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
81 { "instruction_spx", VCPU_STAT(instruction_spx) },
82 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
83 { "instruction_stap", VCPU_STAT(instruction_stap) },
84 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
85 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
86 { "instruction_stsch", VCPU_STAT(instruction_stsch) },
87 { "instruction_chsc", VCPU_STAT(instruction_chsc) },
88 { "instruction_essa", VCPU_STAT(instruction_essa) },
89 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
90 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
91 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
92 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
93 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
94 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
95 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
96 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
97 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
98 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
99 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
100 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
101 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
102 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
103 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
104 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
105 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
106 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
107 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
108 { "diagnose_10", VCPU_STAT(diagnose_10) },
109 { "diagnose_44", VCPU_STAT(diagnose_44) },
110 { "diagnose_9c", VCPU_STAT(diagnose_9c) },
111 { "diagnose_258", VCPU_STAT(diagnose_258) },
112 { "diagnose_308", VCPU_STAT(diagnose_308) },
113 { "diagnose_500", VCPU_STAT(diagnose_500) },
114 { NULL }
115};
116
117/* upper facilities limit for kvm */
118unsigned long kvm_s390_fac_list_mask[] = {
119 0xffe6fffbfcfdfc40UL,
120 0x005e800000000000UL,
121};
122
123unsigned long kvm_s390_fac_list_mask_size(void)
124{
125 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
126 return ARRAY_SIZE(kvm_s390_fac_list_mask);
127}
128
129static struct gmap_notifier gmap_notifier;
130
131/* Section: not file related */
132int kvm_arch_hardware_enable(void)
133{
134 /* every s390 is virtualization enabled ;-) */
135 return 0;
136}
137
138static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
139
140int kvm_arch_hardware_setup(void)
141{
142 gmap_notifier.notifier_call = kvm_gmap_notifier;
143 gmap_register_ipte_notifier(&gmap_notifier);
144 return 0;
145}
146
147void kvm_arch_hardware_unsetup(void)
148{
149 gmap_unregister_ipte_notifier(&gmap_notifier);
150}
151
152int kvm_arch_init(void *opaque)
153{
154 /* Register floating interrupt controller interface. */
155 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
156}
157
158/* Section: device related */
159long kvm_arch_dev_ioctl(struct file *filp,
160 unsigned int ioctl, unsigned long arg)
161{
162 if (ioctl == KVM_S390_ENABLE_SIE)
163 return s390_enable_sie();
164 return -EINVAL;
165}
166
167int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
168{
169 int r;
170
171 switch (ext) {
172 case KVM_CAP_S390_PSW:
173 case KVM_CAP_S390_GMAP:
174 case KVM_CAP_SYNC_MMU:
175#ifdef CONFIG_KVM_S390_UCONTROL
176 case KVM_CAP_S390_UCONTROL:
177#endif
178 case KVM_CAP_ASYNC_PF:
179 case KVM_CAP_SYNC_REGS:
180 case KVM_CAP_ONE_REG:
181 case KVM_CAP_ENABLE_CAP:
182 case KVM_CAP_S390_CSS_SUPPORT:
183 case KVM_CAP_IOEVENTFD:
184 case KVM_CAP_DEVICE_CTRL:
185 case KVM_CAP_ENABLE_CAP_VM:
186 case KVM_CAP_S390_IRQCHIP:
187 case KVM_CAP_VM_ATTRIBUTES:
188 case KVM_CAP_MP_STATE:
189 case KVM_CAP_S390_INJECT_IRQ:
190 case KVM_CAP_S390_USER_SIGP:
191 case KVM_CAP_S390_USER_STSI:
192 case KVM_CAP_S390_SKEYS:
193 case KVM_CAP_S390_IRQ_STATE:
194 r = 1;
195 break;
196 case KVM_CAP_S390_MEM_OP:
197 r = MEM_OP_MAX_SIZE;
198 break;
199 case KVM_CAP_NR_VCPUS:
200 case KVM_CAP_MAX_VCPUS:
201 r = KVM_MAX_VCPUS;
202 break;
203 case KVM_CAP_NR_MEMSLOTS:
204 r = KVM_USER_MEM_SLOTS;
205 break;
206 case KVM_CAP_S390_COW:
207 r = MACHINE_HAS_ESOP;
208 break;
209 case KVM_CAP_S390_VECTOR_REGISTERS:
210 r = MACHINE_HAS_VX;
211 break;
212 default:
213 r = 0;
214 }
215 return r;
216}
217
218static void kvm_s390_sync_dirty_log(struct kvm *kvm,
219 struct kvm_memory_slot *memslot)
220{
221 gfn_t cur_gfn, last_gfn;
222 unsigned long address;
223 struct gmap *gmap = kvm->arch.gmap;
224
225 down_read(&gmap->mm->mmap_sem);
226 /* Loop over all guest pages */
227 last_gfn = memslot->base_gfn + memslot->npages;
228 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
229 address = gfn_to_hva_memslot(memslot, cur_gfn);
230
231 if (gmap_test_and_clear_dirty(address, gmap))
232 mark_page_dirty(kvm, cur_gfn);
233 }
234 up_read(&gmap->mm->mmap_sem);
235}
236
237/* Section: vm related */
238/*
239 * Get (and clear) the dirty memory log for a memory slot.
240 */
241int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
242 struct kvm_dirty_log *log)
243{
244 int r;
245 unsigned long n;
246 struct kvm_memslots *slots;
247 struct kvm_memory_slot *memslot;
248 int is_dirty = 0;
249
250 mutex_lock(&kvm->slots_lock);
251
252 r = -EINVAL;
253 if (log->slot >= KVM_USER_MEM_SLOTS)
254 goto out;
255
256 slots = kvm_memslots(kvm);
257 memslot = id_to_memslot(slots, log->slot);
258 r = -ENOENT;
259 if (!memslot->dirty_bitmap)
260 goto out;
261
262 kvm_s390_sync_dirty_log(kvm, memslot);
263 r = kvm_get_dirty_log(kvm, log, &is_dirty);
264 if (r)
265 goto out;
266
267 /* Clear the dirty log */
268 if (is_dirty) {
269 n = kvm_dirty_bitmap_bytes(memslot);
270 memset(memslot->dirty_bitmap, 0, n);
271 }
272 r = 0;
273out:
274 mutex_unlock(&kvm->slots_lock);
275 return r;
276}
277
278static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
279{
280 int r;
281
282 if (cap->flags)
283 return -EINVAL;
284
285 switch (cap->cap) {
286 case KVM_CAP_S390_IRQCHIP:
287 kvm->arch.use_irqchip = 1;
288 r = 0;
289 break;
290 case KVM_CAP_S390_USER_SIGP:
291 kvm->arch.user_sigp = 1;
292 r = 0;
293 break;
294 case KVM_CAP_S390_VECTOR_REGISTERS:
295 if (MACHINE_HAS_VX) {
296 set_kvm_facility(kvm->arch.model.fac->mask, 129);
297 set_kvm_facility(kvm->arch.model.fac->list, 129);
298 r = 0;
299 } else
300 r = -EINVAL;
301 break;
302 case KVM_CAP_S390_USER_STSI:
303 kvm->arch.user_stsi = 1;
304 r = 0;
305 break;
306 default:
307 r = -EINVAL;
308 break;
309 }
310 return r;
311}
312
313static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
314{
315 int ret;
316
317 switch (attr->attr) {
318 case KVM_S390_VM_MEM_LIMIT_SIZE:
319 ret = 0;
320 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
321 ret = -EFAULT;
322 break;
323 default:
324 ret = -ENXIO;
325 break;
326 }
327 return ret;
328}
329
330static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
331{
332 int ret;
333 unsigned int idx;
334 switch (attr->attr) {
335 case KVM_S390_VM_MEM_ENABLE_CMMA:
336 /* enable CMMA only for z10 and later (EDAT_1) */
337 ret = -EINVAL;
338 if (!MACHINE_IS_LPAR || !MACHINE_HAS_EDAT1)
339 break;
340
341 ret = -EBUSY;
342 mutex_lock(&kvm->lock);
343 if (atomic_read(&kvm->online_vcpus) == 0) {
344 kvm->arch.use_cmma = 1;
345 ret = 0;
346 }
347 mutex_unlock(&kvm->lock);
348 break;
349 case KVM_S390_VM_MEM_CLR_CMMA:
350 ret = -EINVAL;
351 if (!kvm->arch.use_cmma)
352 break;
353
354 mutex_lock(&kvm->lock);
355 idx = srcu_read_lock(&kvm->srcu);
356 s390_reset_cmma(kvm->arch.gmap->mm);
357 srcu_read_unlock(&kvm->srcu, idx);
358 mutex_unlock(&kvm->lock);
359 ret = 0;
360 break;
361 case KVM_S390_VM_MEM_LIMIT_SIZE: {
362 unsigned long new_limit;
363
364 if (kvm_is_ucontrol(kvm))
365 return -EINVAL;
366
367 if (get_user(new_limit, (u64 __user *)attr->addr))
368 return -EFAULT;
369
370 if (new_limit > kvm->arch.gmap->asce_end)
371 return -E2BIG;
372
373 ret = -EBUSY;
374 mutex_lock(&kvm->lock);
375 if (atomic_read(&kvm->online_vcpus) == 0) {
376 /* gmap_alloc will round the limit up */
377 struct gmap *new = gmap_alloc(current->mm, new_limit);
378
379 if (!new) {
380 ret = -ENOMEM;
381 } else {
382 gmap_free(kvm->arch.gmap);
383 new->private = kvm;
384 kvm->arch.gmap = new;
385 ret = 0;
386 }
387 }
388 mutex_unlock(&kvm->lock);
389 break;
390 }
391 default:
392 ret = -ENXIO;
393 break;
394 }
395 return ret;
396}
397
398static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
399
400static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
401{
402 struct kvm_vcpu *vcpu;
403 int i;
404
405 if (!test_kvm_facility(kvm, 76))
406 return -EINVAL;
407
408 mutex_lock(&kvm->lock);
409 switch (attr->attr) {
410 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
411 get_random_bytes(
412 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
413 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
414 kvm->arch.crypto.aes_kw = 1;
415 break;
416 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
417 get_random_bytes(
418 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
419 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
420 kvm->arch.crypto.dea_kw = 1;
421 break;
422 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
423 kvm->arch.crypto.aes_kw = 0;
424 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
425 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
426 break;
427 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
428 kvm->arch.crypto.dea_kw = 0;
429 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
430 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
431 break;
432 default:
433 mutex_unlock(&kvm->lock);
434 return -ENXIO;
435 }
436
437 kvm_for_each_vcpu(i, vcpu, kvm) {
438 kvm_s390_vcpu_crypto_setup(vcpu);
439 exit_sie(vcpu);
440 }
441 mutex_unlock(&kvm->lock);
442 return 0;
443}
444
445static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
446{
447 u8 gtod_high;
448
449 if (copy_from_user(&gtod_high, (void __user *)attr->addr,
450 sizeof(gtod_high)))
451 return -EFAULT;
452
453 if (gtod_high != 0)
454 return -EINVAL;
455
456 return 0;
457}
458
459static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
460{
461 struct kvm_vcpu *cur_vcpu;
462 unsigned int vcpu_idx;
463 u64 host_tod, gtod;
464 int r;
465
466 if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
467 return -EFAULT;
468
469 r = store_tod_clock(&host_tod);
470 if (r)
471 return r;
472
473 mutex_lock(&kvm->lock);
474 kvm->arch.epoch = gtod - host_tod;
475 kvm_s390_vcpu_block_all(kvm);
476 kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm)
477 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
478 kvm_s390_vcpu_unblock_all(kvm);
479 mutex_unlock(&kvm->lock);
480 return 0;
481}
482
483static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
484{
485 int ret;
486
487 if (attr->flags)
488 return -EINVAL;
489
490 switch (attr->attr) {
491 case KVM_S390_VM_TOD_HIGH:
492 ret = kvm_s390_set_tod_high(kvm, attr);
493 break;
494 case KVM_S390_VM_TOD_LOW:
495 ret = kvm_s390_set_tod_low(kvm, attr);
496 break;
497 default:
498 ret = -ENXIO;
499 break;
500 }
501 return ret;
502}
503
504static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
505{
506 u8 gtod_high = 0;
507
508 if (copy_to_user((void __user *)attr->addr, &gtod_high,
509 sizeof(gtod_high)))
510 return -EFAULT;
511
512 return 0;
513}
514
515static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
516{
517 u64 host_tod, gtod;
518 int r;
519
520 r = store_tod_clock(&host_tod);
521 if (r)
522 return r;
523
524 gtod = host_tod + kvm->arch.epoch;
525 if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
526 return -EFAULT;
527
528 return 0;
529}
530
531static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
532{
533 int ret;
534
535 if (attr->flags)
536 return -EINVAL;
537
538 switch (attr->attr) {
539 case KVM_S390_VM_TOD_HIGH:
540 ret = kvm_s390_get_tod_high(kvm, attr);
541 break;
542 case KVM_S390_VM_TOD_LOW:
543 ret = kvm_s390_get_tod_low(kvm, attr);
544 break;
545 default:
546 ret = -ENXIO;
547 break;
548 }
549 return ret;
550}
551
552static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
553{
554 struct kvm_s390_vm_cpu_processor *proc;
555 int ret = 0;
556
557 mutex_lock(&kvm->lock);
558 if (atomic_read(&kvm->online_vcpus)) {
559 ret = -EBUSY;
560 goto out;
561 }
562 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
563 if (!proc) {
564 ret = -ENOMEM;
565 goto out;
566 }
567 if (!copy_from_user(proc, (void __user *)attr->addr,
568 sizeof(*proc))) {
569 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
570 sizeof(struct cpuid));
571 kvm->arch.model.ibc = proc->ibc;
572 memcpy(kvm->arch.model.fac->list, proc->fac_list,
573 S390_ARCH_FAC_LIST_SIZE_BYTE);
574 } else
575 ret = -EFAULT;
576 kfree(proc);
577out:
578 mutex_unlock(&kvm->lock);
579 return ret;
580}
581
582static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
583{
584 int ret = -ENXIO;
585
586 switch (attr->attr) {
587 case KVM_S390_VM_CPU_PROCESSOR:
588 ret = kvm_s390_set_processor(kvm, attr);
589 break;
590 }
591 return ret;
592}
593
594static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
595{
596 struct kvm_s390_vm_cpu_processor *proc;
597 int ret = 0;
598
599 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
600 if (!proc) {
601 ret = -ENOMEM;
602 goto out;
603 }
604 memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
605 proc->ibc = kvm->arch.model.ibc;
606 memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
607 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
608 ret = -EFAULT;
609 kfree(proc);
610out:
611 return ret;
612}
613
614static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
615{
616 struct kvm_s390_vm_cpu_machine *mach;
617 int ret = 0;
618
619 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
620 if (!mach) {
621 ret = -ENOMEM;
622 goto out;
623 }
624 get_cpu_id((struct cpuid *) &mach->cpuid);
625 mach->ibc = sclp.ibc;
626 memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
627 S390_ARCH_FAC_LIST_SIZE_BYTE);
628 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
629 S390_ARCH_FAC_LIST_SIZE_BYTE);
630 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
631 ret = -EFAULT;
632 kfree(mach);
633out:
634 return ret;
635}
636
637static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
638{
639 int ret = -ENXIO;
640
641 switch (attr->attr) {
642 case KVM_S390_VM_CPU_PROCESSOR:
643 ret = kvm_s390_get_processor(kvm, attr);
644 break;
645 case KVM_S390_VM_CPU_MACHINE:
646 ret = kvm_s390_get_machine(kvm, attr);
647 break;
648 }
649 return ret;
650}
651
652static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
653{
654 int ret;
655
656 switch (attr->group) {
657 case KVM_S390_VM_MEM_CTRL:
658 ret = kvm_s390_set_mem_control(kvm, attr);
659 break;
660 case KVM_S390_VM_TOD:
661 ret = kvm_s390_set_tod(kvm, attr);
662 break;
663 case KVM_S390_VM_CPU_MODEL:
664 ret = kvm_s390_set_cpu_model(kvm, attr);
665 break;
666 case KVM_S390_VM_CRYPTO:
667 ret = kvm_s390_vm_set_crypto(kvm, attr);
668 break;
669 default:
670 ret = -ENXIO;
671 break;
672 }
673
674 return ret;
675}
676
677static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
678{
679 int ret;
680
681 switch (attr->group) {
682 case KVM_S390_VM_MEM_CTRL:
683 ret = kvm_s390_get_mem_control(kvm, attr);
684 break;
685 case KVM_S390_VM_TOD:
686 ret = kvm_s390_get_tod(kvm, attr);
687 break;
688 case KVM_S390_VM_CPU_MODEL:
689 ret = kvm_s390_get_cpu_model(kvm, attr);
690 break;
691 default:
692 ret = -ENXIO;
693 break;
694 }
695
696 return ret;
697}
698
699static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
700{
701 int ret;
702
703 switch (attr->group) {
704 case KVM_S390_VM_MEM_CTRL:
705 switch (attr->attr) {
706 case KVM_S390_VM_MEM_ENABLE_CMMA:
707 case KVM_S390_VM_MEM_CLR_CMMA:
708 case KVM_S390_VM_MEM_LIMIT_SIZE:
709 ret = 0;
710 break;
711 default:
712 ret = -ENXIO;
713 break;
714 }
715 break;
716 case KVM_S390_VM_TOD:
717 switch (attr->attr) {
718 case KVM_S390_VM_TOD_LOW:
719 case KVM_S390_VM_TOD_HIGH:
720 ret = 0;
721 break;
722 default:
723 ret = -ENXIO;
724 break;
725 }
726 break;
727 case KVM_S390_VM_CPU_MODEL:
728 switch (attr->attr) {
729 case KVM_S390_VM_CPU_PROCESSOR:
730 case KVM_S390_VM_CPU_MACHINE:
731 ret = 0;
732 break;
733 default:
734 ret = -ENXIO;
735 break;
736 }
737 break;
738 case KVM_S390_VM_CRYPTO:
739 switch (attr->attr) {
740 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
741 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
742 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
743 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
744 ret = 0;
745 break;
746 default:
747 ret = -ENXIO;
748 break;
749 }
750 break;
751 default:
752 ret = -ENXIO;
753 break;
754 }
755
756 return ret;
757}
758
759static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
760{
761 uint8_t *keys;
762 uint64_t hva;
763 unsigned long curkey;
764 int i, r = 0;
765
766 if (args->flags != 0)
767 return -EINVAL;
768
769 /* Is this guest using storage keys? */
770 if (!mm_use_skey(current->mm))
771 return KVM_S390_GET_SKEYS_NONE;
772
773 /* Enforce sane limit on memory allocation */
774 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
775 return -EINVAL;
776
777 keys = kmalloc_array(args->count, sizeof(uint8_t),
778 GFP_KERNEL | __GFP_NOWARN);
779 if (!keys)
780 keys = vmalloc(sizeof(uint8_t) * args->count);
781 if (!keys)
782 return -ENOMEM;
783
784 for (i = 0; i < args->count; i++) {
785 hva = gfn_to_hva(kvm, args->start_gfn + i);
786 if (kvm_is_error_hva(hva)) {
787 r = -EFAULT;
788 goto out;
789 }
790
791 curkey = get_guest_storage_key(current->mm, hva);
792 if (IS_ERR_VALUE(curkey)) {
793 r = curkey;
794 goto out;
795 }
796 keys[i] = curkey;
797 }
798
799 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
800 sizeof(uint8_t) * args->count);
801 if (r)
802 r = -EFAULT;
803out:
804 kvfree(keys);
805 return r;
806}
807
808static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
809{
810 uint8_t *keys;
811 uint64_t hva;
812 int i, r = 0;
813
814 if (args->flags != 0)
815 return -EINVAL;
816
817 /* Enforce sane limit on memory allocation */
818 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
819 return -EINVAL;
820
821 keys = kmalloc_array(args->count, sizeof(uint8_t),
822 GFP_KERNEL | __GFP_NOWARN);
823 if (!keys)
824 keys = vmalloc(sizeof(uint8_t) * args->count);
825 if (!keys)
826 return -ENOMEM;
827
828 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
829 sizeof(uint8_t) * args->count);
830 if (r) {
831 r = -EFAULT;
832 goto out;
833 }
834
835 /* Enable storage key handling for the guest */
836 r = s390_enable_skey();
837 if (r)
838 goto out;
839
840 for (i = 0; i < args->count; i++) {
841 hva = gfn_to_hva(kvm, args->start_gfn + i);
842 if (kvm_is_error_hva(hva)) {
843 r = -EFAULT;
844 goto out;
845 }
846
847 /* Lowest order bit is reserved */
848 if (keys[i] & 0x01) {
849 r = -EINVAL;
850 goto out;
851 }
852
853 r = set_guest_storage_key(current->mm, hva,
854 (unsigned long)keys[i], 0);
855 if (r)
856 goto out;
857 }
858out:
859 kvfree(keys);
860 return r;
861}
862
863long kvm_arch_vm_ioctl(struct file *filp,
864 unsigned int ioctl, unsigned long arg)
865{
866 struct kvm *kvm = filp->private_data;
867 void __user *argp = (void __user *)arg;
868 struct kvm_device_attr attr;
869 int r;
870
871 switch (ioctl) {
872 case KVM_S390_INTERRUPT: {
873 struct kvm_s390_interrupt s390int;
874
875 r = -EFAULT;
876 if (copy_from_user(&s390int, argp, sizeof(s390int)))
877 break;
878 r = kvm_s390_inject_vm(kvm, &s390int);
879 break;
880 }
881 case KVM_ENABLE_CAP: {
882 struct kvm_enable_cap cap;
883 r = -EFAULT;
884 if (copy_from_user(&cap, argp, sizeof(cap)))
885 break;
886 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
887 break;
888 }
889 case KVM_CREATE_IRQCHIP: {
890 struct kvm_irq_routing_entry routing;
891
892 r = -EINVAL;
893 if (kvm->arch.use_irqchip) {
894 /* Set up dummy routing. */
895 memset(&routing, 0, sizeof(routing));
896 kvm_set_irq_routing(kvm, &routing, 0, 0);
897 r = 0;
898 }
899 break;
900 }
901 case KVM_SET_DEVICE_ATTR: {
902 r = -EFAULT;
903 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
904 break;
905 r = kvm_s390_vm_set_attr(kvm, &attr);
906 break;
907 }
908 case KVM_GET_DEVICE_ATTR: {
909 r = -EFAULT;
910 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
911 break;
912 r = kvm_s390_vm_get_attr(kvm, &attr);
913 break;
914 }
915 case KVM_HAS_DEVICE_ATTR: {
916 r = -EFAULT;
917 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
918 break;
919 r = kvm_s390_vm_has_attr(kvm, &attr);
920 break;
921 }
922 case KVM_S390_GET_SKEYS: {
923 struct kvm_s390_skeys args;
924
925 r = -EFAULT;
926 if (copy_from_user(&args, argp,
927 sizeof(struct kvm_s390_skeys)))
928 break;
929 r = kvm_s390_get_skeys(kvm, &args);
930 break;
931 }
932 case KVM_S390_SET_SKEYS: {
933 struct kvm_s390_skeys args;
934
935 r = -EFAULT;
936 if (copy_from_user(&args, argp,
937 sizeof(struct kvm_s390_skeys)))
938 break;
939 r = kvm_s390_set_skeys(kvm, &args);
940 break;
941 }
942 default:
943 r = -ENOTTY;
944 }
945
946 return r;
947}
948
949static int kvm_s390_query_ap_config(u8 *config)
950{
951 u32 fcn_code = 0x04000000UL;
952 u32 cc = 0;
953
954 memset(config, 0, 128);
955 asm volatile(
956 "lgr 0,%1\n"
957 "lgr 2,%2\n"
958 ".long 0xb2af0000\n" /* PQAP(QCI) */
959 "0: ipm %0\n"
960 "srl %0,28\n"
961 "1:\n"
962 EX_TABLE(0b, 1b)
963 : "+r" (cc)
964 : "r" (fcn_code), "r" (config)
965 : "cc", "0", "2", "memory"
966 );
967
968 return cc;
969}
970
971static int kvm_s390_apxa_installed(void)
972{
973 u8 config[128];
974 int cc;
975
976 if (test_facility(2) && test_facility(12)) {
977 cc = kvm_s390_query_ap_config(config);
978
979 if (cc)
980 pr_err("PQAP(QCI) failed with cc=%d", cc);
981 else
982 return config[0] & 0x40;
983 }
984
985 return 0;
986}
987
988static void kvm_s390_set_crycb_format(struct kvm *kvm)
989{
990 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
991
992 if (kvm_s390_apxa_installed())
993 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
994 else
995 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
996}
997
998static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
999{
1000 get_cpu_id(cpu_id);
1001 cpu_id->version = 0xff;
1002}
1003
1004static int kvm_s390_crypto_init(struct kvm *kvm)
1005{
1006 if (!test_kvm_facility(kvm, 76))
1007 return 0;
1008
1009 kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
1010 GFP_KERNEL | GFP_DMA);
1011 if (!kvm->arch.crypto.crycb)
1012 return -ENOMEM;
1013
1014 kvm_s390_set_crycb_format(kvm);
1015
1016 /* Enable AES/DEA protected key functions by default */
1017 kvm->arch.crypto.aes_kw = 1;
1018 kvm->arch.crypto.dea_kw = 1;
1019 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1020 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1021 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1022 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1023
1024 return 0;
1025}
1026
1027int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1028{
1029 int i, rc;
1030 char debug_name[16];
1031 static unsigned long sca_offset;
1032
1033 rc = -EINVAL;
1034#ifdef CONFIG_KVM_S390_UCONTROL
1035 if (type & ~KVM_VM_S390_UCONTROL)
1036 goto out_err;
1037 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1038 goto out_err;
1039#else
1040 if (type)
1041 goto out_err;
1042#endif
1043
1044 rc = s390_enable_sie();
1045 if (rc)
1046 goto out_err;
1047
1048 rc = -ENOMEM;
1049
1050 kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
1051 if (!kvm->arch.sca)
1052 goto out_err;
1053 spin_lock(&kvm_lock);
1054 sca_offset = (sca_offset + 16) & 0x7f0;
1055 kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
1056 spin_unlock(&kvm_lock);
1057
1058 sprintf(debug_name, "kvm-%u", current->pid);
1059
1060 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1061 if (!kvm->arch.dbf)
1062 goto out_err;
1063
1064 /*
1065 * The architectural maximum amount of facilities is 16 kbit. To store
1066 * this amount, 2 kbyte of memory is required. Thus we need a full
1067 * page to hold the guest facility list (arch.model.fac->list) and the
1068 * facility mask (arch.model.fac->mask). Its address size has to be
1069 * 31 bits and word aligned.
1070 */
1071 kvm->arch.model.fac =
1072 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1073 if (!kvm->arch.model.fac)
1074 goto out_err;
1075
1076 /* Populate the facility mask initially. */
1077 memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
1078 S390_ARCH_FAC_LIST_SIZE_BYTE);
1079 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1080 if (i < kvm_s390_fac_list_mask_size())
1081 kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
1082 else
1083 kvm->arch.model.fac->mask[i] = 0UL;
1084 }
1085
1086 /* Populate the facility list initially. */
1087 memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
1088 S390_ARCH_FAC_LIST_SIZE_BYTE);
1089
1090 kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
1091 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1092
1093 if (kvm_s390_crypto_init(kvm) < 0)
1094 goto out_err;
1095
1096 spin_lock_init(&kvm->arch.float_int.lock);
1097 for (i = 0; i < FIRQ_LIST_COUNT; i++)
1098 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1099 init_waitqueue_head(&kvm->arch.ipte_wq);
1100 mutex_init(&kvm->arch.ipte_mutex);
1101
1102 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1103 VM_EVENT(kvm, 3, "%s", "vm created");
1104
1105 if (type & KVM_VM_S390_UCONTROL) {
1106 kvm->arch.gmap = NULL;
1107 } else {
1108 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
1109 if (!kvm->arch.gmap)
1110 goto out_err;
1111 kvm->arch.gmap->private = kvm;
1112 kvm->arch.gmap->pfault_enabled = 0;
1113 }
1114
1115 kvm->arch.css_support = 0;
1116 kvm->arch.use_irqchip = 0;
1117 kvm->arch.epoch = 0;
1118
1119 spin_lock_init(&kvm->arch.start_stop_lock);
1120
1121 return 0;
1122out_err:
1123 kfree(kvm->arch.crypto.crycb);
1124 free_page((unsigned long)kvm->arch.model.fac);
1125 debug_unregister(kvm->arch.dbf);
1126 free_page((unsigned long)(kvm->arch.sca));
1127 return rc;
1128}
1129
1130void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1131{
1132 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1133 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1134 kvm_s390_clear_local_irqs(vcpu);
1135 kvm_clear_async_pf_completion_queue(vcpu);
1136 if (!kvm_is_ucontrol(vcpu->kvm)) {
1137 clear_bit(63 - vcpu->vcpu_id,
1138 (unsigned long *) &vcpu->kvm->arch.sca->mcn);
1139 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
1140 (__u64) vcpu->arch.sie_block)
1141 vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
1142 }
1143 smp_mb();
1144
1145 if (kvm_is_ucontrol(vcpu->kvm))
1146 gmap_free(vcpu->arch.gmap);
1147
1148 if (vcpu->kvm->arch.use_cmma)
1149 kvm_s390_vcpu_unsetup_cmma(vcpu);
1150 free_page((unsigned long)(vcpu->arch.sie_block));
1151
1152 kvm_vcpu_uninit(vcpu);
1153 kmem_cache_free(kvm_vcpu_cache, vcpu);
1154}
1155
1156static void kvm_free_vcpus(struct kvm *kvm)
1157{
1158 unsigned int i;
1159 struct kvm_vcpu *vcpu;
1160
1161 kvm_for_each_vcpu(i, vcpu, kvm)
1162 kvm_arch_vcpu_destroy(vcpu);
1163
1164 mutex_lock(&kvm->lock);
1165 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1166 kvm->vcpus[i] = NULL;
1167
1168 atomic_set(&kvm->online_vcpus, 0);
1169 mutex_unlock(&kvm->lock);
1170}
1171
1172void kvm_arch_destroy_vm(struct kvm *kvm)
1173{
1174 kvm_free_vcpus(kvm);
1175 free_page((unsigned long)kvm->arch.model.fac);
1176 free_page((unsigned long)(kvm->arch.sca));
1177 debug_unregister(kvm->arch.dbf);
1178 kfree(kvm->arch.crypto.crycb);
1179 if (!kvm_is_ucontrol(kvm))
1180 gmap_free(kvm->arch.gmap);
1181 kvm_s390_destroy_adapters(kvm);
1182 kvm_s390_clear_float_irqs(kvm);
1183}
1184
1185/* Section: vcpu related */
1186static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1187{
1188 vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1189 if (!vcpu->arch.gmap)
1190 return -ENOMEM;
1191 vcpu->arch.gmap->private = vcpu->kvm;
1192
1193 return 0;
1194}
1195
1196int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1197{
1198 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1199 kvm_clear_async_pf_completion_queue(vcpu);
1200 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1201 KVM_SYNC_GPRS |
1202 KVM_SYNC_ACRS |
1203 KVM_SYNC_CRS |
1204 KVM_SYNC_ARCH0 |
1205 KVM_SYNC_PFAULT;
1206 if (test_kvm_facility(vcpu->kvm, 129))
1207 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1208
1209 if (kvm_is_ucontrol(vcpu->kvm))
1210 return __kvm_ucontrol_vcpu_init(vcpu);
1211
1212 return 0;
1213}
1214
1215void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1216{
1217 save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1218 if (test_kvm_facility(vcpu->kvm, 129))
1219 save_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1220 else
1221 save_fp_regs(vcpu->arch.host_fpregs.fprs);
1222 save_access_regs(vcpu->arch.host_acrs);
1223 if (test_kvm_facility(vcpu->kvm, 129)) {
1224 restore_fp_ctl(&vcpu->run->s.regs.fpc);
1225 restore_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1226 } else {
1227 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1228 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1229 }
1230 restore_access_regs(vcpu->run->s.regs.acrs);
1231 gmap_enable(vcpu->arch.gmap);
1232 atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1233}
1234
1235void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1236{
1237 atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1238 gmap_disable(vcpu->arch.gmap);
1239 if (test_kvm_facility(vcpu->kvm, 129)) {
1240 save_fp_ctl(&vcpu->run->s.regs.fpc);
1241 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
1242 } else {
1243 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1244 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1245 }
1246 save_access_regs(vcpu->run->s.regs.acrs);
1247 restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1248 if (test_kvm_facility(vcpu->kvm, 129))
1249 restore_vx_regs((__vector128 *)&vcpu->arch.host_vregs->vrs);
1250 else
1251 restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1252 restore_access_regs(vcpu->arch.host_acrs);
1253}
1254
1255static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1256{
1257 /* this equals initial cpu reset in pop, but we don't switch to ESA */
1258 vcpu->arch.sie_block->gpsw.mask = 0UL;
1259 vcpu->arch.sie_block->gpsw.addr = 0UL;
1260 kvm_s390_set_prefix(vcpu, 0);
1261 vcpu->arch.sie_block->cputm = 0UL;
1262 vcpu->arch.sie_block->ckc = 0UL;
1263 vcpu->arch.sie_block->todpr = 0;
1264 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1265 vcpu->arch.sie_block->gcr[0] = 0xE0UL;
1266 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1267 vcpu->arch.guest_fpregs.fpc = 0;
1268 asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1269 vcpu->arch.sie_block->gbea = 1;
1270 vcpu->arch.sie_block->pp = 0;
1271 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1272 kvm_clear_async_pf_completion_queue(vcpu);
1273 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1274 kvm_s390_vcpu_stop(vcpu);
1275 kvm_s390_clear_local_irqs(vcpu);
1276}
1277
1278void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1279{
1280 mutex_lock(&vcpu->kvm->lock);
1281 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1282 mutex_unlock(&vcpu->kvm->lock);
1283 if (!kvm_is_ucontrol(vcpu->kvm))
1284 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1285}
1286
1287static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1288{
1289 if (!test_kvm_facility(vcpu->kvm, 76))
1290 return;
1291
1292 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1293
1294 if (vcpu->kvm->arch.crypto.aes_kw)
1295 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1296 if (vcpu->kvm->arch.crypto.dea_kw)
1297 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1298
1299 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1300}
1301
1302void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1303{
1304 free_page(vcpu->arch.sie_block->cbrlo);
1305 vcpu->arch.sie_block->cbrlo = 0;
1306}
1307
1308int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1309{
1310 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1311 if (!vcpu->arch.sie_block->cbrlo)
1312 return -ENOMEM;
1313
1314 vcpu->arch.sie_block->ecb2 |= 0x80;
1315 vcpu->arch.sie_block->ecb2 &= ~0x08;
1316 return 0;
1317}
1318
1319static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1320{
1321 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1322
1323 vcpu->arch.cpu_id = model->cpu_id;
1324 vcpu->arch.sie_block->ibc = model->ibc;
1325 vcpu->arch.sie_block->fac = (int) (long) model->fac->list;
1326}
1327
1328int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1329{
1330 int rc = 0;
1331
1332 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1333 CPUSTAT_SM |
1334 CPUSTAT_STOPPED);
1335
1336 if (test_kvm_facility(vcpu->kvm, 78))
1337 atomic_set_mask(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1338 else if (test_kvm_facility(vcpu->kvm, 8))
1339 atomic_set_mask(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1340
1341 kvm_s390_vcpu_setup_model(vcpu);
1342
1343 vcpu->arch.sie_block->ecb = 6;
1344 if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1345 vcpu->arch.sie_block->ecb |= 0x10;
1346
1347 vcpu->arch.sie_block->ecb2 = 8;
1348 vcpu->arch.sie_block->eca = 0xC1002000U;
1349 if (sclp.has_siif)
1350 vcpu->arch.sie_block->eca |= 1;
1351 if (sclp.has_sigpif)
1352 vcpu->arch.sie_block->eca |= 0x10000000U;
1353 if (test_kvm_facility(vcpu->kvm, 129)) {
1354 vcpu->arch.sie_block->eca |= 0x00020000;
1355 vcpu->arch.sie_block->ecd |= 0x20000000;
1356 }
1357 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1358
1359 if (vcpu->kvm->arch.use_cmma) {
1360 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1361 if (rc)
1362 return rc;
1363 }
1364 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1365 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1366
1367 kvm_s390_vcpu_crypto_setup(vcpu);
1368
1369 return rc;
1370}
1371
1372struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1373 unsigned int id)
1374{
1375 struct kvm_vcpu *vcpu;
1376 struct sie_page *sie_page;
1377 int rc = -EINVAL;
1378
1379 if (id >= KVM_MAX_VCPUS)
1380 goto out;
1381
1382 rc = -ENOMEM;
1383
1384 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1385 if (!vcpu)
1386 goto out;
1387
1388 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1389 if (!sie_page)
1390 goto out_free_cpu;
1391
1392 vcpu->arch.sie_block = &sie_page->sie_block;
1393 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1394 vcpu->arch.host_vregs = &sie_page->vregs;
1395
1396 vcpu->arch.sie_block->icpua = id;
1397 if (!kvm_is_ucontrol(kvm)) {
1398 if (!kvm->arch.sca) {
1399 WARN_ON_ONCE(1);
1400 goto out_free_cpu;
1401 }
1402 if (!kvm->arch.sca->cpu[id].sda)
1403 kvm->arch.sca->cpu[id].sda =
1404 (__u64) vcpu->arch.sie_block;
1405 vcpu->arch.sie_block->scaoh =
1406 (__u32)(((__u64)kvm->arch.sca) >> 32);
1407 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1408 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1409 }
1410
1411 spin_lock_init(&vcpu->arch.local_int.lock);
1412 vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1413 vcpu->arch.local_int.wq = &vcpu->wq;
1414 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1415
1416 rc = kvm_vcpu_init(vcpu, kvm, id);
1417 if (rc)
1418 goto out_free_sie_block;
1419 VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1420 vcpu->arch.sie_block);
1421 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1422
1423 return vcpu;
1424out_free_sie_block:
1425 free_page((unsigned long)(vcpu->arch.sie_block));
1426out_free_cpu:
1427 kmem_cache_free(kvm_vcpu_cache, vcpu);
1428out:
1429 return ERR_PTR(rc);
1430}
1431
1432int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1433{
1434 return kvm_s390_vcpu_has_irq(vcpu, 0);
1435}
1436
1437void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
1438{
1439 atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1440 exit_sie(vcpu);
1441}
1442
1443void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1444{
1445 atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1446}
1447
1448static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
1449{
1450 atomic_set_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1451 exit_sie(vcpu);
1452}
1453
1454static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
1455{
1456 atomic_clear_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
1457}
1458
1459/*
1460 * Kick a guest cpu out of SIE and wait until SIE is not running.
1461 * If the CPU is not running (e.g. waiting as idle) the function will
1462 * return immediately. */
1463void exit_sie(struct kvm_vcpu *vcpu)
1464{
1465 atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1466 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1467 cpu_relax();
1468}
1469
1470/* Kick a guest cpu out of SIE to process a request synchronously */
1471void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
1472{
1473 kvm_make_request(req, vcpu);
1474 kvm_s390_vcpu_request(vcpu);
1475}
1476
1477static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1478{
1479 int i;
1480 struct kvm *kvm = gmap->private;
1481 struct kvm_vcpu *vcpu;
1482
1483 kvm_for_each_vcpu(i, vcpu, kvm) {
1484 /* match against both prefix pages */
1485 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1486 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1487 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
1488 }
1489 }
1490}
1491
1492int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1493{
1494 /* kvm common code refers to this, but never calls it */
1495 BUG();
1496 return 0;
1497}
1498
1499static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1500 struct kvm_one_reg *reg)
1501{
1502 int r = -EINVAL;
1503
1504 switch (reg->id) {
1505 case KVM_REG_S390_TODPR:
1506 r = put_user(vcpu->arch.sie_block->todpr,
1507 (u32 __user *)reg->addr);
1508 break;
1509 case KVM_REG_S390_EPOCHDIFF:
1510 r = put_user(vcpu->arch.sie_block->epoch,
1511 (u64 __user *)reg->addr);
1512 break;
1513 case KVM_REG_S390_CPU_TIMER:
1514 r = put_user(vcpu->arch.sie_block->cputm,
1515 (u64 __user *)reg->addr);
1516 break;
1517 case KVM_REG_S390_CLOCK_COMP:
1518 r = put_user(vcpu->arch.sie_block->ckc,
1519 (u64 __user *)reg->addr);
1520 break;
1521 case KVM_REG_S390_PFTOKEN:
1522 r = put_user(vcpu->arch.pfault_token,
1523 (u64 __user *)reg->addr);
1524 break;
1525 case KVM_REG_S390_PFCOMPARE:
1526 r = put_user(vcpu->arch.pfault_compare,
1527 (u64 __user *)reg->addr);
1528 break;
1529 case KVM_REG_S390_PFSELECT:
1530 r = put_user(vcpu->arch.pfault_select,
1531 (u64 __user *)reg->addr);
1532 break;
1533 case KVM_REG_S390_PP:
1534 r = put_user(vcpu->arch.sie_block->pp,
1535 (u64 __user *)reg->addr);
1536 break;
1537 case KVM_REG_S390_GBEA:
1538 r = put_user(vcpu->arch.sie_block->gbea,
1539 (u64 __user *)reg->addr);
1540 break;
1541 default:
1542 break;
1543 }
1544
1545 return r;
1546}
1547
1548static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1549 struct kvm_one_reg *reg)
1550{
1551 int r = -EINVAL;
1552
1553 switch (reg->id) {
1554 case KVM_REG_S390_TODPR:
1555 r = get_user(vcpu->arch.sie_block->todpr,
1556 (u32 __user *)reg->addr);
1557 break;
1558 case KVM_REG_S390_EPOCHDIFF:
1559 r = get_user(vcpu->arch.sie_block->epoch,
1560 (u64 __user *)reg->addr);
1561 break;
1562 case KVM_REG_S390_CPU_TIMER:
1563 r = get_user(vcpu->arch.sie_block->cputm,
1564 (u64 __user *)reg->addr);
1565 break;
1566 case KVM_REG_S390_CLOCK_COMP:
1567 r = get_user(vcpu->arch.sie_block->ckc,
1568 (u64 __user *)reg->addr);
1569 break;
1570 case KVM_REG_S390_PFTOKEN:
1571 r = get_user(vcpu->arch.pfault_token,
1572 (u64 __user *)reg->addr);
1573 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1574 kvm_clear_async_pf_completion_queue(vcpu);
1575 break;
1576 case KVM_REG_S390_PFCOMPARE:
1577 r = get_user(vcpu->arch.pfault_compare,
1578 (u64 __user *)reg->addr);
1579 break;
1580 case KVM_REG_S390_PFSELECT:
1581 r = get_user(vcpu->arch.pfault_select,
1582 (u64 __user *)reg->addr);
1583 break;
1584 case KVM_REG_S390_PP:
1585 r = get_user(vcpu->arch.sie_block->pp,
1586 (u64 __user *)reg->addr);
1587 break;
1588 case KVM_REG_S390_GBEA:
1589 r = get_user(vcpu->arch.sie_block->gbea,
1590 (u64 __user *)reg->addr);
1591 break;
1592 default:
1593 break;
1594 }
1595
1596 return r;
1597}
1598
1599static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1600{
1601 kvm_s390_vcpu_initial_reset(vcpu);
1602 return 0;
1603}
1604
1605int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1606{
1607 memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1608 return 0;
1609}
1610
1611int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1612{
1613 memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1614 return 0;
1615}
1616
1617int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1618 struct kvm_sregs *sregs)
1619{
1620 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1621 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1622 restore_access_regs(vcpu->run->s.regs.acrs);
1623 return 0;
1624}
1625
1626int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1627 struct kvm_sregs *sregs)
1628{
1629 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1630 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1631 return 0;
1632}
1633
1634int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1635{
1636 if (test_fp_ctl(fpu->fpc))
1637 return -EINVAL;
1638 memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1639 vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1640 restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1641 restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1642 return 0;
1643}
1644
1645int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1646{
1647 memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1648 fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1649 return 0;
1650}
1651
1652static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1653{
1654 int rc = 0;
1655
1656 if (!is_vcpu_stopped(vcpu))
1657 rc = -EBUSY;
1658 else {
1659 vcpu->run->psw_mask = psw.mask;
1660 vcpu->run->psw_addr = psw.addr;
1661 }
1662 return rc;
1663}
1664
1665int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1666 struct kvm_translation *tr)
1667{
1668 return -EINVAL; /* not implemented yet */
1669}
1670
1671#define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1672 KVM_GUESTDBG_USE_HW_BP | \
1673 KVM_GUESTDBG_ENABLE)
1674
1675int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1676 struct kvm_guest_debug *dbg)
1677{
1678 int rc = 0;
1679
1680 vcpu->guest_debug = 0;
1681 kvm_s390_clear_bp_data(vcpu);
1682
1683 if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1684 return -EINVAL;
1685
1686 if (dbg->control & KVM_GUESTDBG_ENABLE) {
1687 vcpu->guest_debug = dbg->control;
1688 /* enforce guest PER */
1689 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1690
1691 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1692 rc = kvm_s390_import_bp_data(vcpu, dbg);
1693 } else {
1694 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1695 vcpu->arch.guestdbg.last_bp = 0;
1696 }
1697
1698 if (rc) {
1699 vcpu->guest_debug = 0;
1700 kvm_s390_clear_bp_data(vcpu);
1701 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1702 }
1703
1704 return rc;
1705}
1706
1707int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1708 struct kvm_mp_state *mp_state)
1709{
1710 /* CHECK_STOP and LOAD are not supported yet */
1711 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1712 KVM_MP_STATE_OPERATING;
1713}
1714
1715int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1716 struct kvm_mp_state *mp_state)
1717{
1718 int rc = 0;
1719
1720 /* user space knows about this interface - let it control the state */
1721 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1722
1723 switch (mp_state->mp_state) {
1724 case KVM_MP_STATE_STOPPED:
1725 kvm_s390_vcpu_stop(vcpu);
1726 break;
1727 case KVM_MP_STATE_OPERATING:
1728 kvm_s390_vcpu_start(vcpu);
1729 break;
1730 case KVM_MP_STATE_LOAD:
1731 case KVM_MP_STATE_CHECK_STOP:
1732 /* fall through - CHECK_STOP and LOAD are not supported yet */
1733 default:
1734 rc = -ENXIO;
1735 }
1736
1737 return rc;
1738}
1739
1740static bool ibs_enabled(struct kvm_vcpu *vcpu)
1741{
1742 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1743}
1744
1745static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1746{
1747 if (!vcpu->requests)
1748 return 0;
1749retry:
1750 kvm_s390_vcpu_request_handled(vcpu);
1751 /*
1752 * We use MMU_RELOAD just to re-arm the ipte notifier for the
1753 * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1754 * This ensures that the ipte instruction for this request has
1755 * already finished. We might race against a second unmapper that
1756 * wants to set the blocking bit. Lets just retry the request loop.
1757 */
1758 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1759 int rc;
1760 rc = gmap_ipte_notify(vcpu->arch.gmap,
1761 kvm_s390_get_prefix(vcpu),
1762 PAGE_SIZE * 2);
1763 if (rc)
1764 return rc;
1765 goto retry;
1766 }
1767
1768 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1769 vcpu->arch.sie_block->ihcpu = 0xffff;
1770 goto retry;
1771 }
1772
1773 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1774 if (!ibs_enabled(vcpu)) {
1775 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1776 atomic_set_mask(CPUSTAT_IBS,
1777 &vcpu->arch.sie_block->cpuflags);
1778 }
1779 goto retry;
1780 }
1781
1782 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1783 if (ibs_enabled(vcpu)) {
1784 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1785 atomic_clear_mask(CPUSTAT_IBS,
1786 &vcpu->arch.sie_block->cpuflags);
1787 }
1788 goto retry;
1789 }
1790
1791 /* nothing to do, just clear the request */
1792 clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1793
1794 return 0;
1795}
1796
1797/**
1798 * kvm_arch_fault_in_page - fault-in guest page if necessary
1799 * @vcpu: The corresponding virtual cpu
1800 * @gpa: Guest physical address
1801 * @writable: Whether the page should be writable or not
1802 *
1803 * Make sure that a guest page has been faulted-in on the host.
1804 *
1805 * Return: Zero on success, negative error code otherwise.
1806 */
1807long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1808{
1809 return gmap_fault(vcpu->arch.gmap, gpa,
1810 writable ? FAULT_FLAG_WRITE : 0);
1811}
1812
1813static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1814 unsigned long token)
1815{
1816 struct kvm_s390_interrupt inti;
1817 struct kvm_s390_irq irq;
1818
1819 if (start_token) {
1820 irq.u.ext.ext_params2 = token;
1821 irq.type = KVM_S390_INT_PFAULT_INIT;
1822 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1823 } else {
1824 inti.type = KVM_S390_INT_PFAULT_DONE;
1825 inti.parm64 = token;
1826 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1827 }
1828}
1829
1830void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1831 struct kvm_async_pf *work)
1832{
1833 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1834 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1835}
1836
1837void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1838 struct kvm_async_pf *work)
1839{
1840 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1841 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1842}
1843
1844void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1845 struct kvm_async_pf *work)
1846{
1847 /* s390 will always inject the page directly */
1848}
1849
1850bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1851{
1852 /*
1853 * s390 will always inject the page directly,
1854 * but we still want check_async_completion to cleanup
1855 */
1856 return true;
1857}
1858
1859static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1860{
1861 hva_t hva;
1862 struct kvm_arch_async_pf arch;
1863 int rc;
1864
1865 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1866 return 0;
1867 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1868 vcpu->arch.pfault_compare)
1869 return 0;
1870 if (psw_extint_disabled(vcpu))
1871 return 0;
1872 if (kvm_s390_vcpu_has_irq(vcpu, 0))
1873 return 0;
1874 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1875 return 0;
1876 if (!vcpu->arch.gmap->pfault_enabled)
1877 return 0;
1878
1879 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1880 hva += current->thread.gmap_addr & ~PAGE_MASK;
1881 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1882 return 0;
1883
1884 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1885 return rc;
1886}
1887
1888static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1889{
1890 int rc, cpuflags;
1891
1892 /*
1893 * On s390 notifications for arriving pages will be delivered directly
1894 * to the guest but the house keeping for completed pfaults is
1895 * handled outside the worker.
1896 */
1897 kvm_check_async_pf_completion(vcpu);
1898
1899 memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1900
1901 if (need_resched())
1902 schedule();
1903
1904 if (test_cpu_flag(CIF_MCCK_PENDING))
1905 s390_handle_mcck();
1906
1907 if (!kvm_is_ucontrol(vcpu->kvm)) {
1908 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1909 if (rc)
1910 return rc;
1911 }
1912
1913 rc = kvm_s390_handle_requests(vcpu);
1914 if (rc)
1915 return rc;
1916
1917 if (guestdbg_enabled(vcpu)) {
1918 kvm_s390_backup_guest_per_regs(vcpu);
1919 kvm_s390_patch_guest_per_regs(vcpu);
1920 }
1921
1922 vcpu->arch.sie_block->icptcode = 0;
1923 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1924 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1925 trace_kvm_s390_sie_enter(vcpu, cpuflags);
1926
1927 return 0;
1928}
1929
1930static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
1931{
1932 psw_t *psw = &vcpu->arch.sie_block->gpsw;
1933 u8 opcode;
1934 int rc;
1935
1936 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1937 trace_kvm_s390_sie_fault(vcpu);
1938
1939 /*
1940 * We want to inject an addressing exception, which is defined as a
1941 * suppressing or terminating exception. However, since we came here
1942 * by a DAT access exception, the PSW still points to the faulting
1943 * instruction since DAT exceptions are nullifying. So we've got
1944 * to look up the current opcode to get the length of the instruction
1945 * to be able to forward the PSW.
1946 */
1947 rc = read_guest(vcpu, psw->addr, 0, &opcode, 1);
1948 if (rc)
1949 return kvm_s390_inject_prog_cond(vcpu, rc);
1950 psw->addr = __rewind_psw(*psw, -insn_length(opcode));
1951
1952 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1953}
1954
1955static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1956{
1957 int rc = -1;
1958
1959 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1960 vcpu->arch.sie_block->icptcode);
1961 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1962
1963 if (guestdbg_enabled(vcpu))
1964 kvm_s390_restore_guest_per_regs(vcpu);
1965
1966 if (exit_reason >= 0) {
1967 rc = 0;
1968 } else if (kvm_is_ucontrol(vcpu->kvm)) {
1969 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1970 vcpu->run->s390_ucontrol.trans_exc_code =
1971 current->thread.gmap_addr;
1972 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1973 rc = -EREMOTE;
1974
1975 } else if (current->thread.gmap_pfault) {
1976 trace_kvm_s390_major_guest_pfault(vcpu);
1977 current->thread.gmap_pfault = 0;
1978 if (kvm_arch_setup_async_pf(vcpu)) {
1979 rc = 0;
1980 } else {
1981 gpa_t gpa = current->thread.gmap_addr;
1982 rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1983 }
1984 }
1985
1986 if (rc == -1)
1987 rc = vcpu_post_run_fault_in_sie(vcpu);
1988
1989 memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1990
1991 if (rc == 0) {
1992 if (kvm_is_ucontrol(vcpu->kvm))
1993 /* Don't exit for host interrupts. */
1994 rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1995 else
1996 rc = kvm_handle_sie_intercept(vcpu);
1997 }
1998
1999 return rc;
2000}
2001
2002static int __vcpu_run(struct kvm_vcpu *vcpu)
2003{
2004 int rc, exit_reason;
2005
2006 /*
2007 * We try to hold kvm->srcu during most of vcpu_run (except when run-
2008 * ning the guest), so that memslots (and other stuff) are protected
2009 */
2010 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2011
2012 do {
2013 rc = vcpu_pre_run(vcpu);
2014 if (rc)
2015 break;
2016
2017 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2018 /*
2019 * As PF_VCPU will be used in fault handler, between
2020 * guest_enter and guest_exit should be no uaccess.
2021 */
2022 local_irq_disable();
2023 __kvm_guest_enter();
2024 local_irq_enable();
2025 exit_reason = sie64a(vcpu->arch.sie_block,
2026 vcpu->run->s.regs.gprs);
2027 local_irq_disable();
2028 __kvm_guest_exit();
2029 local_irq_enable();
2030 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2031
2032 rc = vcpu_post_run(vcpu, exit_reason);
2033 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2034
2035 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2036 return rc;
2037}
2038
2039static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2040{
2041 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2042 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2043 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2044 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2045 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2046 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2047 /* some control register changes require a tlb flush */
2048 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2049 }
2050 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2051 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
2052 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2053 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2054 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2055 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2056 }
2057 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2058 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2059 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2060 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2061 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2062 kvm_clear_async_pf_completion_queue(vcpu);
2063 }
2064 kvm_run->kvm_dirty_regs = 0;
2065}
2066
2067static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2068{
2069 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2070 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2071 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2072 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2073 kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
2074 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2075 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2076 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2077 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2078 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2079 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2080 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2081}
2082
2083int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2084{
2085 int rc;
2086 sigset_t sigsaved;
2087
2088 if (guestdbg_exit_pending(vcpu)) {
2089 kvm_s390_prepare_debug_exit(vcpu);
2090 return 0;
2091 }
2092
2093 if (vcpu->sigset_active)
2094 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2095
2096 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2097 kvm_s390_vcpu_start(vcpu);
2098 } else if (is_vcpu_stopped(vcpu)) {
2099 pr_err_ratelimited("can't run stopped vcpu %d\n",
2100 vcpu->vcpu_id);
2101 return -EINVAL;
2102 }
2103
2104 sync_regs(vcpu, kvm_run);
2105
2106 might_fault();
2107 rc = __vcpu_run(vcpu);
2108
2109 if (signal_pending(current) && !rc) {
2110 kvm_run->exit_reason = KVM_EXIT_INTR;
2111 rc = -EINTR;
2112 }
2113
2114 if (guestdbg_exit_pending(vcpu) && !rc) {
2115 kvm_s390_prepare_debug_exit(vcpu);
2116 rc = 0;
2117 }
2118
2119 if (rc == -EOPNOTSUPP) {
2120 /* intercept cannot be handled in-kernel, prepare kvm-run */
2121 kvm_run->exit_reason = KVM_EXIT_S390_SIEIC;
2122 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2123 kvm_run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2124 kvm_run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2125 rc = 0;
2126 }
2127
2128 if (rc == -EREMOTE) {
2129 /* intercept was handled, but userspace support is needed
2130 * kvm_run has been prepared by the handler */
2131 rc = 0;
2132 }
2133
2134 store_regs(vcpu, kvm_run);
2135
2136 if (vcpu->sigset_active)
2137 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2138
2139 vcpu->stat.exit_userspace++;
2140 return rc;
2141}
2142
2143/*
2144 * store status at address
2145 * we use have two special cases:
2146 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2147 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2148 */
2149int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2150{
2151 unsigned char archmode = 1;
2152 unsigned int px;
2153 u64 clkcomp;
2154 int rc;
2155
2156 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2157 if (write_guest_abs(vcpu, 163, &archmode, 1))
2158 return -EFAULT;
2159 gpa = SAVE_AREA_BASE;
2160 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2161 if (write_guest_real(vcpu, 163, &archmode, 1))
2162 return -EFAULT;
2163 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
2164 }
2165 rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
2166 vcpu->arch.guest_fpregs.fprs, 128);
2167 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
2168 vcpu->run->s.regs.gprs, 128);
2169 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
2170 &vcpu->arch.sie_block->gpsw, 16);
2171 px = kvm_s390_get_prefix(vcpu);
2172 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
2173 &px, 4);
2174 rc |= write_guest_abs(vcpu,
2175 gpa + offsetof(struct save_area, fp_ctrl_reg),
2176 &vcpu->arch.guest_fpregs.fpc, 4);
2177 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
2178 &vcpu->arch.sie_block->todpr, 4);
2179 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
2180 &vcpu->arch.sie_block->cputm, 8);
2181 clkcomp = vcpu->arch.sie_block->ckc >> 8;
2182 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
2183 &clkcomp, 8);
2184 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
2185 &vcpu->run->s.regs.acrs, 64);
2186 rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
2187 &vcpu->arch.sie_block->gcr, 128);
2188 return rc ? -EFAULT : 0;
2189}
2190
2191int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2192{
2193 /*
2194 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2195 * copying in vcpu load/put. Lets update our copies before we save
2196 * it into the save area
2197 */
2198 save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
2199 save_fp_regs(vcpu->arch.guest_fpregs.fprs);
2200 save_access_regs(vcpu->run->s.regs.acrs);
2201
2202 return kvm_s390_store_status_unloaded(vcpu, addr);
2203}
2204
2205/*
2206 * store additional status at address
2207 */
2208int kvm_s390_store_adtl_status_unloaded(struct kvm_vcpu *vcpu,
2209 unsigned long gpa)
2210{
2211 /* Only bits 0-53 are used for address formation */
2212 if (!(gpa & ~0x3ff))
2213 return 0;
2214
2215 return write_guest_abs(vcpu, gpa & ~0x3ff,
2216 (void *)&vcpu->run->s.regs.vrs, 512);
2217}
2218
2219int kvm_s390_vcpu_store_adtl_status(struct kvm_vcpu *vcpu, unsigned long addr)
2220{
2221 if (!test_kvm_facility(vcpu->kvm, 129))
2222 return 0;
2223
2224 /*
2225 * The guest VXRS are in the host VXRs due to the lazy
2226 * copying in vcpu load/put. Let's update our copies before we save
2227 * it into the save area.
2228 */
2229 save_vx_regs((__vector128 *)&vcpu->run->s.regs.vrs);
2230
2231 return kvm_s390_store_adtl_status_unloaded(vcpu, addr);
2232}
2233
2234static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2235{
2236 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2237 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2238}
2239
2240static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2241{
2242 unsigned int i;
2243 struct kvm_vcpu *vcpu;
2244
2245 kvm_for_each_vcpu(i, vcpu, kvm) {
2246 __disable_ibs_on_vcpu(vcpu);
2247 }
2248}
2249
2250static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2251{
2252 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2253 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2254}
2255
2256void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2257{
2258 int i, online_vcpus, started_vcpus = 0;
2259
2260 if (!is_vcpu_stopped(vcpu))
2261 return;
2262
2263 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2264 /* Only one cpu at a time may enter/leave the STOPPED state. */
2265 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2266 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2267
2268 for (i = 0; i < online_vcpus; i++) {
2269 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2270 started_vcpus++;
2271 }
2272
2273 if (started_vcpus == 0) {
2274 /* we're the only active VCPU -> speed it up */
2275 __enable_ibs_on_vcpu(vcpu);
2276 } else if (started_vcpus == 1) {
2277 /*
2278 * As we are starting a second VCPU, we have to disable
2279 * the IBS facility on all VCPUs to remove potentially
2280 * oustanding ENABLE requests.
2281 */
2282 __disable_ibs_on_all_vcpus(vcpu->kvm);
2283 }
2284
2285 atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2286 /*
2287 * Another VCPU might have used IBS while we were offline.
2288 * Let's play safe and flush the VCPU at startup.
2289 */
2290 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2291 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2292 return;
2293}
2294
2295void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2296{
2297 int i, online_vcpus, started_vcpus = 0;
2298 struct kvm_vcpu *started_vcpu = NULL;
2299
2300 if (is_vcpu_stopped(vcpu))
2301 return;
2302
2303 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2304 /* Only one cpu at a time may enter/leave the STOPPED state. */
2305 spin_lock(&vcpu->kvm->arch.start_stop_lock);
2306 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2307
2308 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2309 kvm_s390_clear_stop_irq(vcpu);
2310
2311 atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2312 __disable_ibs_on_vcpu(vcpu);
2313
2314 for (i = 0; i < online_vcpus; i++) {
2315 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2316 started_vcpus++;
2317 started_vcpu = vcpu->kvm->vcpus[i];
2318 }
2319 }
2320
2321 if (started_vcpus == 1) {
2322 /*
2323 * As we only have one VCPU left, we want to enable the
2324 * IBS facility for that VCPU to speed it up.
2325 */
2326 __enable_ibs_on_vcpu(started_vcpu);
2327 }
2328
2329 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2330 return;
2331}
2332
2333static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2334 struct kvm_enable_cap *cap)
2335{
2336 int r;
2337
2338 if (cap->flags)
2339 return -EINVAL;
2340
2341 switch (cap->cap) {
2342 case KVM_CAP_S390_CSS_SUPPORT:
2343 if (!vcpu->kvm->arch.css_support) {
2344 vcpu->kvm->arch.css_support = 1;
2345 trace_kvm_s390_enable_css(vcpu->kvm);
2346 }
2347 r = 0;
2348 break;
2349 default:
2350 r = -EINVAL;
2351 break;
2352 }
2353 return r;
2354}
2355
2356static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2357 struct kvm_s390_mem_op *mop)
2358{
2359 void __user *uaddr = (void __user *)mop->buf;
2360 void *tmpbuf = NULL;
2361 int r, srcu_idx;
2362 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2363 | KVM_S390_MEMOP_F_CHECK_ONLY;
2364
2365 if (mop->flags & ~supported_flags)
2366 return -EINVAL;
2367
2368 if (mop->size > MEM_OP_MAX_SIZE)
2369 return -E2BIG;
2370
2371 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2372 tmpbuf = vmalloc(mop->size);
2373 if (!tmpbuf)
2374 return -ENOMEM;
2375 }
2376
2377 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2378
2379 switch (mop->op) {
2380 case KVM_S390_MEMOP_LOGICAL_READ:
2381 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2382 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, false);
2383 break;
2384 }
2385 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2386 if (r == 0) {
2387 if (copy_to_user(uaddr, tmpbuf, mop->size))
2388 r = -EFAULT;
2389 }
2390 break;
2391 case KVM_S390_MEMOP_LOGICAL_WRITE:
2392 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2393 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size, true);
2394 break;
2395 }
2396 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2397 r = -EFAULT;
2398 break;
2399 }
2400 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
2401 break;
2402 default:
2403 r = -EINVAL;
2404 }
2405
2406 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
2407
2408 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
2409 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
2410
2411 vfree(tmpbuf);
2412 return r;
2413}
2414
2415long kvm_arch_vcpu_ioctl(struct file *filp,
2416 unsigned int ioctl, unsigned long arg)
2417{
2418 struct kvm_vcpu *vcpu = filp->private_data;
2419 void __user *argp = (void __user *)arg;
2420 int idx;
2421 long r;
2422
2423 switch (ioctl) {
2424 case KVM_S390_IRQ: {
2425 struct kvm_s390_irq s390irq;
2426
2427 r = -EFAULT;
2428 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
2429 break;
2430 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2431 break;
2432 }
2433 case KVM_S390_INTERRUPT: {
2434 struct kvm_s390_interrupt s390int;
2435 struct kvm_s390_irq s390irq;
2436
2437 r = -EFAULT;
2438 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2439 break;
2440 if (s390int_to_s390irq(&s390int, &s390irq))
2441 return -EINVAL;
2442 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2443 break;
2444 }
2445 case KVM_S390_STORE_STATUS:
2446 idx = srcu_read_lock(&vcpu->kvm->srcu);
2447 r = kvm_s390_vcpu_store_status(vcpu, arg);
2448 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2449 break;
2450 case KVM_S390_SET_INITIAL_PSW: {
2451 psw_t psw;
2452
2453 r = -EFAULT;
2454 if (copy_from_user(&psw, argp, sizeof(psw)))
2455 break;
2456 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2457 break;
2458 }
2459 case KVM_S390_INITIAL_RESET:
2460 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2461 break;
2462 case KVM_SET_ONE_REG:
2463 case KVM_GET_ONE_REG: {
2464 struct kvm_one_reg reg;
2465 r = -EFAULT;
2466 if (copy_from_user(&reg, argp, sizeof(reg)))
2467 break;
2468 if (ioctl == KVM_SET_ONE_REG)
2469 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2470 else
2471 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2472 break;
2473 }
2474#ifdef CONFIG_KVM_S390_UCONTROL
2475 case KVM_S390_UCAS_MAP: {
2476 struct kvm_s390_ucas_mapping ucasmap;
2477
2478 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2479 r = -EFAULT;
2480 break;
2481 }
2482
2483 if (!kvm_is_ucontrol(vcpu->kvm)) {
2484 r = -EINVAL;
2485 break;
2486 }
2487
2488 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2489 ucasmap.vcpu_addr, ucasmap.length);
2490 break;
2491 }
2492 case KVM_S390_UCAS_UNMAP: {
2493 struct kvm_s390_ucas_mapping ucasmap;
2494
2495 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2496 r = -EFAULT;
2497 break;
2498 }
2499
2500 if (!kvm_is_ucontrol(vcpu->kvm)) {
2501 r = -EINVAL;
2502 break;
2503 }
2504
2505 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2506 ucasmap.length);
2507 break;
2508 }
2509#endif
2510 case KVM_S390_VCPU_FAULT: {
2511 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2512 break;
2513 }
2514 case KVM_ENABLE_CAP:
2515 {
2516 struct kvm_enable_cap cap;
2517 r = -EFAULT;
2518 if (copy_from_user(&cap, argp, sizeof(cap)))
2519 break;
2520 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2521 break;
2522 }
2523 case KVM_S390_MEM_OP: {
2524 struct kvm_s390_mem_op mem_op;
2525
2526 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
2527 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
2528 else
2529 r = -EFAULT;
2530 break;
2531 }
2532 case KVM_S390_SET_IRQ_STATE: {
2533 struct kvm_s390_irq_state irq_state;
2534
2535 r = -EFAULT;
2536 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2537 break;
2538 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
2539 irq_state.len == 0 ||
2540 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
2541 r = -EINVAL;
2542 break;
2543 }
2544 r = kvm_s390_set_irq_state(vcpu,
2545 (void __user *) irq_state.buf,
2546 irq_state.len);
2547 break;
2548 }
2549 case KVM_S390_GET_IRQ_STATE: {
2550 struct kvm_s390_irq_state irq_state;
2551
2552 r = -EFAULT;
2553 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
2554 break;
2555 if (irq_state.len == 0) {
2556 r = -EINVAL;
2557 break;
2558 }
2559 r = kvm_s390_get_irq_state(vcpu,
2560 (__u8 __user *) irq_state.buf,
2561 irq_state.len);
2562 break;
2563 }
2564 default:
2565 r = -ENOTTY;
2566 }
2567 return r;
2568}
2569
2570int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2571{
2572#ifdef CONFIG_KVM_S390_UCONTROL
2573 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2574 && (kvm_is_ucontrol(vcpu->kvm))) {
2575 vmf->page = virt_to_page(vcpu->arch.sie_block);
2576 get_page(vmf->page);
2577 return 0;
2578 }
2579#endif
2580 return VM_FAULT_SIGBUS;
2581}
2582
2583int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2584 unsigned long npages)
2585{
2586 return 0;
2587}
2588
2589/* Section: memory related */
2590int kvm_arch_prepare_memory_region(struct kvm *kvm,
2591 struct kvm_memory_slot *memslot,
2592 const struct kvm_userspace_memory_region *mem,
2593 enum kvm_mr_change change)
2594{
2595 /* A few sanity checks. We can have memory slots which have to be
2596 located/ended at a segment boundary (1MB). The memory in userland is
2597 ok to be fragmented into various different vmas. It is okay to mmap()
2598 and munmap() stuff in this slot after doing this call at any time */
2599
2600 if (mem->userspace_addr & 0xffffful)
2601 return -EINVAL;
2602
2603 if (mem->memory_size & 0xffffful)
2604 return -EINVAL;
2605
2606 return 0;
2607}
2608
2609void kvm_arch_commit_memory_region(struct kvm *kvm,
2610 const struct kvm_userspace_memory_region *mem,
2611 const struct kvm_memory_slot *old,
2612 const struct kvm_memory_slot *new,
2613 enum kvm_mr_change change)
2614{
2615 int rc;
2616
2617 /* If the basics of the memslot do not change, we do not want
2618 * to update the gmap. Every update causes several unnecessary
2619 * segment translation exceptions. This is usually handled just
2620 * fine by the normal fault handler + gmap, but it will also
2621 * cause faults on the prefix page of running guest CPUs.
2622 */
2623 if (old->userspace_addr == mem->userspace_addr &&
2624 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2625 old->npages * PAGE_SIZE == mem->memory_size)
2626 return;
2627
2628 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2629 mem->guest_phys_addr, mem->memory_size);
2630 if (rc)
2631 pr_warn("failed to commit memory region\n");
2632 return;
2633}
2634
2635static int __init kvm_s390_init(void)
2636{
2637 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2638}
2639
2640static void __exit kvm_s390_exit(void)
2641{
2642 kvm_exit();
2643}
2644
2645module_init(kvm_s390_init);
2646module_exit(kvm_s390_exit);
2647
2648/*
2649 * Enable autoloading of the kvm module.
2650 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2651 * since x86 takes a different approach.
2652 */
2653#include <linux/miscdevice.h>
2654MODULE_ALIAS_MISCDEV(KVM_MINOR);
2655MODULE_ALIAS("devname:kvm");
Linux 6.x block layer and io_uring tree(s)